cernora 0.1.0__py3-none-any.whl

cernora/__init__.py

@@ -0,0 +1,94 @@
+"""Cernora: evidence-bound evaluation for tool-using agents."""
+
+from cernora.adapter import AdaptedBundle, Adapter, CompletedExport
+from cernora.conformance import (
+    AdapterConformance,
+    ConformanceError,
+    ProfileConformance,
+    check_adapter_conformance,
+    check_profile_conformance,
+)
+from cernora.core.case import (
+    Case,
+    CaseInput,
+    CaseProfile,
+    FixtureReference,
+    GatePolicy,
+    ScorerPolicy,
+)
+from cernora.core.evidence import (
+    AnswerClaim,
+    Artifact,
+    Evidence,
+    EvidenceReference,
+    Failure,
+    ProcessResult,
+    StructuredAnswer,
+    ToolAction,
+)
+from cernora.core.evidence_bundle_v2 import EvidenceBundleV2
+from cernora.core.gate import GateDecision
+from cernora.core.identity import (
+    ComponentIdentity,
+    ExternalProducerIdentity,
+    component_identity,
+    external_producer_identity,
+)
+from cernora.core.score import Score, ScoreObservation
+from cernora.evaluation.package import evaluate_imported_case, read_imported_evaluation
+from cernora.ingestion.contracts_v2 import AuthorityBoundImportPackageV2
+from cernora.ingestion.package_v2 import import_evidence_bundle_v2
+from cernora.profile import Profile, ProfileAssessment, ProfileEvaluationContext
+from cernora.profile_loader import ProfileLoadError, load_local_profile
+from cernora.profile_workspace import ProfileInitResult, ProfileWorkspaceError, init_profile
+from cernora.resources import PUBLIC_SCHEMAS, read_public_schema
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "PUBLIC_SCHEMAS",
+    "AdaptedBundle",
+    "Adapter",
+    "AdapterConformance",
+    "AnswerClaim",
+    "Artifact",
+    "AuthorityBoundImportPackageV2",
+    "Case",
+    "CaseInput",
+    "CaseProfile",
+    "CompletedExport",
+    "ComponentIdentity",
+    "ConformanceError",
+    "Evidence",
+    "EvidenceBundleV2",
+    "EvidenceReference",
+    "ExternalProducerIdentity",
+    "Failure",
+    "FixtureReference",
+    "GateDecision",
+    "GatePolicy",
+    "ProcessResult",
+    "Profile",
+    "ProfileAssessment",
+    "ProfileConformance",
+    "ProfileEvaluationContext",
+    "ProfileInitResult",
+    "ProfileLoadError",
+    "ProfileWorkspaceError",
+    "Score",
+    "ScoreObservation",
+    "ScorerPolicy",
+    "StructuredAnswer",
+    "ToolAction",
+    "__version__",
+    "check_adapter_conformance",
+    "check_profile_conformance",
+    "component_identity",
+    "evaluate_imported_case",
+    "external_producer_identity",
+    "import_evidence_bundle_v2",
+    "init_profile",
+    "load_local_profile",
+    "read_imported_evaluation",
+    "read_public_schema",
+]

cernora/__main__.py

@@ -0,0 +1,5 @@
+"""Run Cernora as `python -m cernora`."""
+
+from cernora.cli.main import main
+
+raise SystemExit(main())

cernora/adapter.py

@@ -0,0 +1,31 @@
+"""Preview completed-export Adapter interface."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+
+@dataclass(frozen=True)
+class CompletedExport:
+    """One already terminal ordinary-file export supplied by a third party."""
+
+    root: Path
+
+
+@dataclass(frozen=True)
+class AdaptedBundle:
+    """Paths to one normalized EvidenceBundle v2 and its sibling artifacts."""
+
+    bundle_path: Path
+
+
+@runtime_checkable
+class Adapter(Protocol):
+    """Normalize completed local files without execution, credentials or network."""
+
+    def adapt(self, completed_export: CompletedExport, output: Path) -> AdaptedBundle: ...
+
+
+__all__ = ["AdaptedBundle", "Adapter", "CompletedExport"]

cernora/cli/__init__.py

@@ -0,0 +1 @@
+"""Cernora command-line interface."""

cernora/cli/main.py

@@ -0,0 +1,121 @@
+"""Public completed-evidence CLI with stable fail-closed exit classes."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+from typing import Any, NoReturn
+
+from cernora import __version__
+from cernora.cli.profiles import BUILTIN_PROFILE_SELECTORS, load_builtin_profile
+from cernora.conformance import ConformanceError, check_profile_conformance
+from cernora.core.canonical import canonical_json
+from cernora.core.errors import ContractError
+from cernora.evaluation.package import evaluate_imported_case
+from cernora.ingestion.errors import IngestionConfigurationError, IngestionIntegrityError
+from cernora.ingestion.package_v2 import import_evidence_bundle_v2
+from cernora.profile import Profile
+from cernora.profile_loader import ProfileLoadError, load_local_profile
+from cernora.profile_workspace import ProfileWorkspaceError, init_profile
+
+
+class UsageParser(argparse.ArgumentParser):
+    def error(self, message: str) -> NoReturn:
+        self.print_usage(sys.stderr)
+        print(f"cernora: error: {message}", file=sys.stderr)
+        raise SystemExit(2)
+
+
+def parser() -> argparse.ArgumentParser:
+    root = UsageParser(prog="cernora")
+    root.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    commands = root.add_subparsers(dest="command", required=True)
+
+    profile = commands.add_parser("profile")
+    profile_commands = profile.add_subparsers(dest="profile_command", required=True)
+    init = profile_commands.add_parser("init")
+    init.add_argument("name")
+    init.add_argument("--output", type=Path)
+    validate = profile_commands.add_parser("validate")
+    _add_profile_selector(validate)
+
+    evidence = commands.add_parser("evidence")
+    evidence_commands = evidence.add_subparsers(dest="evidence_command", required=True)
+    evidence_import = evidence_commands.add_parser("import")
+    _add_profile_selector(evidence_import)
+    evidence_import.add_argument("--bundle", type=Path, required=True)
+    evidence_import.add_argument("--output", type=Path, required=True)
+    evidence_evaluate = evidence_commands.add_parser("evaluate")
+    _add_profile_selector(evidence_evaluate)
+    evidence_evaluate.add_argument("--import-root", type=Path, required=True)
+    evidence_evaluate.add_argument("--output", type=Path, required=True)
+    return root
+
+
+def _add_profile_selector(command: argparse.ArgumentParser) -> None:
+    selection = command.add_mutually_exclusive_group(required=True)
+    selection.add_argument("--profile", choices=BUILTIN_PROFILE_SELECTORS)
+    selection.add_argument("--profile-path", type=Path)
+
+
+def _load_selected_profile(args: argparse.Namespace) -> Profile:
+    if args.profile_path is not None:
+        return load_local_profile(args.profile_path)
+    return load_builtin_profile(args.profile)
+
+
+def _emit(value: Any) -> None:
+    sys.stdout.buffer.write(canonical_json(value) + b"\n")
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parser().parse_args(argv)
+    result: Any
+    try:
+        if args.command == "profile" and args.profile_command == "init":
+            result = init_profile(args.name, output=args.output)
+            code = 0
+        else:
+            profile = _load_selected_profile(args)
+        if args.command == "profile" and args.profile_command == "validate":
+            check_profile_conformance(profile)
+            result = profile.authority
+            code = 0
+        elif args.command == "evidence" and args.evidence_command == "import":
+            result = import_evidence_bundle_v2(
+                profile=profile,
+                bundle_path=args.bundle,
+                output=args.output,
+            )
+            code = 0
+        elif args.command == "evidence":
+            result = evaluate_imported_case(
+                profile=profile,
+                import_root=args.import_root,
+                output=args.output,
+            )
+            code = {"pass": 0, "fail": 1, "inconclusive": 3}[result.case_outcome]
+    except (
+        ConformanceError,
+        IngestionConfigurationError,
+        ProfileLoadError,
+        ProfileWorkspaceError,
+    ) as exc:
+        print(str(exc), file=sys.stderr)
+        return 2
+    except IngestionIntegrityError as exc:
+        print(str(exc), file=sys.stderr)
+        return 3
+    except (ContractError, OSError, ValueError, KeyError) as exc:
+        print(str(exc), file=sys.stderr)
+        return 2 if args.command == "profile" else 3
+    except Exception as exc:
+        print(f"evaluation failed closed: {type(exc).__name__}", file=sys.stderr)
+        return 3
+    _emit(result)
+    return code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

cernora/cli/profiles.py

@@ -0,0 +1,25 @@
+"""Explicit CLI wiring for wheel-bundled public Profiles."""
+
+from cernora.profile import Profile
+
+BUILTIN_PROFILE_SELECTORS = ("builtin:coding-task", "builtin:offline-workflow")
+
+
+def load_builtin_profile(selector: str) -> Profile:
+    """Instantiate one explicitly selected built-in Profile.
+
+    This closed CLI switch is not an SDK registry or discovery mechanism.
+    """
+
+    if selector == "builtin:offline-workflow":
+        from cernora.profiles.offline_workflow import OfflineWorkflowProfile
+
+        return OfflineWorkflowProfile()
+    if selector == "builtin:coding-task":
+        from cernora.profiles.coding_task import CodingTaskProfile
+
+        return CodingTaskProfile()
+    raise ValueError(f"unknown built-in Profile selector: {selector}")
+
+
+__all__ = ["BUILTIN_PROFILE_SELECTORS", "load_builtin_profile"]

cernora/composition/__init__.py

@@ -0,0 +1 @@
+"""Evaluator-owned Score and Gate composition."""

cernora/composition/gating.py

@@ -0,0 +1,126 @@
+"""Deterministic fail-closed GateDecision composition."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+
+from cernora.composition.scoring import index_scores
+from cernora.core.evidence import (
+    EvidenceReference,
+    Failure,
+    evidence_reference_sort_key,
+)
+from cernora.core.gate import GateDecision
+from cernora.core.identity import component_identity
+from cernora.core.score import Score
+
+
+def compose_gate(
+    *,
+    decision_id: str,
+    policy_version: str,
+    required_score_ids: Sequence[str],
+    required_observations: Sequence[str],
+    scores: Iterable[Score],
+) -> GateDecision:
+    """Compose a decision without converting invalid/missing observations into pass."""
+
+    indexed = index_scores(scores)
+    blocking: list[str] = []
+    inconclusive: list[str] = []
+    evidence: dict[tuple[str, str, str | None], EvidenceReference] = {}
+
+    for score_id in required_score_ids:
+        score = indexed.get(score_id)
+        if score is None:
+            inconclusive.append(f"missing required score: {score_id}")
+            continue
+        observations = {item.observation_id: item for item in score.observations}
+        for observation in score.observations:
+            for reference in observation.evidence_references:
+                evidence[(reference.evidence_id, reference.locator, reference.sha256)] = reference
+        for observation_id in required_observations:
+            required = observations.get(observation_id)
+            if required is None:
+                inconclusive.append(f"missing required observation {score_id}/{observation_id}")
+            elif required.applicability == "invalid":
+                inconclusive.append(
+                    f"invalid observation {score_id}/{required.observation_id}: {required.reason}"
+                )
+            elif required.applicability == "not_applicable":
+                inconclusive.append(
+                    f"required observation not applicable {score_id}/{required.observation_id}: "
+                    f"{required.reason}"
+                )
+            elif required.applicability == "observed" and required.value is not True:
+                blocking.append(f"failed observation {score_id}/{required.observation_id}")
+
+    present_score_ids = tuple(score_id for score_id in required_score_ids if score_id in indexed)
+    references = tuple(sorted(evidence.values(), key=evidence_reference_sort_key))
+    scorer_identities = tuple(
+        component_identity("scorer", indexed[score_id].scorer_version)
+        for score_id in present_score_ids
+    )
+    input_digests = tuple(
+        sorted({reference.sha256 for reference in references if reference.sha256 is not None})
+    )
+    if inconclusive:
+        message = "; ".join(inconclusive)
+        return GateDecision(
+            schema_version="agent.evaluator.gate-decision/v1",
+            decision_id=decision_id,
+            decision="inconclusive",
+            policy_version=policy_version,
+            policy_identity=component_identity("gate_policy", policy_version),
+            blocking_reasons=tuple(inconclusive),
+            score_ids=present_score_ids,
+            evidence_references=references,
+            infrastructure_failure="required_score_invalid_or_missing",
+            eligible=False,
+            failure=Failure(
+                domain="scorer",
+                code="required_score_invalid_or_missing",
+                message=message,
+                evidence_references=references,
+            ),
+            scorer_identities=scorer_identities,
+            input_digests=input_digests,
+            harness_contribution="blocks_harness",
+        )
+    if blocking:
+        message = "; ".join(blocking)
+        return GateDecision(
+            schema_version="agent.evaluator.gate-decision/v1",
+            decision_id=decision_id,
+            decision="fail",
+            policy_version=policy_version,
+            policy_identity=component_identity("gate_policy", policy_version),
+            blocking_reasons=tuple(blocking),
+            score_ids=present_score_ids,
+            evidence_references=references,
+            eligible=True,
+            failure=Failure(
+                domain="agent",
+                code="required_observation_failed",
+                message=message,
+                evidence_references=references,
+            ),
+            scorer_identities=scorer_identities,
+            input_digests=input_digests,
+            harness_contribution="eligible_evaluation",
+        )
+    return GateDecision(
+        schema_version="agent.evaluator.gate-decision/v1",
+        decision_id=decision_id,
+        decision="pass",
+        policy_version=policy_version,
+        policy_identity=component_identity("gate_policy", policy_version),
+        blocking_reasons=(),
+        score_ids=present_score_ids,
+        evidence_references=references,
+        eligible=True,
+        failure=None,
+        scorer_identities=scorer_identities,
+        input_digests=input_digests,
+        harness_contribution="eligible_evaluation",
+    )

cernora/composition/scoring.py

@@ -0,0 +1,18 @@
+"""Generic score helpers shared by profile adapters."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from cernora.core.score import Score
+
+
+def index_scores(scores: Iterable[Score]) -> dict[str, Score]:
+    """Index scores by ID and reject ambiguous duplicate identities."""
+
+    indexed: dict[str, Score] = {}
+    for score in scores:
+        if score.score_id in indexed:
+            raise ValueError(f"duplicate score ID: {score.score_id}")
+        indexed[score.score_id] = score
+    return indexed

cernora/conformance.py

@@ -0,0 +1,195 @@
+"""Preview conformance helpers for public Profile and Adapter authors."""
+
+from __future__ import annotations
+
+import os
+import stat
+from dataclasses import dataclass
+from pathlib import Path
+
+from cernora.adapter import AdaptedBundle, Adapter, CompletedExport
+from cernora.core.canonical import canonical_json
+from cernora.core.case import CaseProfile
+from cernora.core.evidence_bundle_v2 import (
+    decode_evidence_bundle_v2,
+    verify_artifact_payloads_v2,
+)
+from cernora.profile import Profile
+
+_MAX_CONFORMANCE_FILE_BYTES = 16_000_000
+
+
+class ConformanceError(ValueError):
+    """A public Profile or Adapter does not satisfy the Preview contract."""
+
+
+@dataclass(frozen=True)
+class ProfileConformance:
+    """Validated public identity summary for one Profile."""
+
+    profile_id: str
+    profile_version: str
+    projection_version: str
+    case_ids: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class AdapterConformance:
+    """Validated identity summary for one canonical Adapter output."""
+
+    bundle_id: str
+    bundle_sha256: str
+    artifact_ids: tuple[str, ...]
+    bundle_path: Path
+
+
+def check_profile_conformance(candidate: object) -> ProfileConformance:
+    """Validate the static public Profile contract without evaluating evidence.
+
+    Local Profile Python is trusted code. This helper validates its exposed authority
+    and identity; a real import/evaluation remains the behavioral acceptance test.
+    """
+
+    if not isinstance(candidate, Profile):
+        raise ConformanceError("object does not implement the Cernora Profile contract")
+    authority = candidate.authority
+    if not isinstance(authority, CaseProfile):
+        raise ConformanceError("Profile authority must be a CaseProfile")
+    projection_version = candidate.projection_version
+    if not isinstance(projection_version, str) or not projection_version:
+        raise ConformanceError("Profile projection_version must be a non-empty string")
+    case_ids = tuple(case.case_id for case in authority.cases)
+    if not case_ids or len(case_ids) != len(set(case_ids)):
+        raise ConformanceError("Profile authority must contain uniquely identified Cases")
+    try:
+        canonical_json(authority)
+    except (TypeError, ValueError) as exc:
+        raise ConformanceError("Profile authority must have canonical JSON bytes") from exc
+    return ProfileConformance(
+        profile_id=authority.profile_id,
+        profile_version=authority.profile_version,
+        projection_version=projection_version,
+        case_ids=case_ids,
+    )
+
+
+def _read_ordinary_file(path: Path) -> bytes:
+    try:
+        before = path.lstat()
+    except OSError as exc:
+        raise ConformanceError("Adapter output contains an unreadable path") from exc
+    if not stat.S_ISREG(before.st_mode):
+        raise ConformanceError("Adapter output files must be ordinary files")
+    descriptor = -1
+    try:
+        descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+        opened = os.fstat(descriptor)
+        if (before.st_dev, before.st_ino) != (opened.st_dev, opened.st_ino):
+            raise ConformanceError("Adapter output changed while it was inspected")
+        if opened.st_size > _MAX_CONFORMANCE_FILE_BYTES:
+            raise ConformanceError("Adapter output file exceeds the conformance size limit")
+        with os.fdopen(descriptor, "rb", closefd=True) as stream:
+            descriptor = -1
+            payload = stream.read(_MAX_CONFORMANCE_FILE_BYTES + 1)
+        if len(payload) > _MAX_CONFORMANCE_FILE_BYTES:
+            raise ConformanceError("Adapter output file exceeds the conformance size limit")
+        return payload
+    except OSError as exc:
+        raise ConformanceError("Adapter output contains an unreadable path") from exc
+    finally:
+        if descriptor >= 0:
+            os.close(descriptor)
+
+
+def _closed_output_files(root: Path) -> dict[str, bytes]:
+    try:
+        root_info = root.lstat()
+    except OSError as exc:
+        raise ConformanceError("Adapter output root is missing or unreadable") from exc
+    if stat.S_ISLNK(root_info.st_mode) or not stat.S_ISDIR(root_info.st_mode):
+        raise ConformanceError("Adapter output root must be an ordinary directory")
+    files: dict[str, bytes] = {}
+    for path in sorted(root.rglob("*")):
+        try:
+            info = path.lstat()
+        except OSError as exc:
+            raise ConformanceError("Adapter output contains an unreadable path") from exc
+        if stat.S_ISLNK(info.st_mode):
+            raise ConformanceError("Adapter output must not contain symbolic links")
+        if stat.S_ISDIR(info.st_mode):
+            continue
+        if not stat.S_ISREG(info.st_mode):
+            raise ConformanceError("Adapter output must contain only ordinary files")
+        relative = path.relative_to(root).as_posix()
+        files[relative] = _read_ordinary_file(path)
+    return files
+
+
+def check_adapter_conformance(
+    candidate: object,
+    completed_export: CompletedExport,
+    output: Path,
+) -> AdapterConformance:
+    """Run one Adapter and verify its closed canonical EvidenceBundle v2 tree.
+
+    The caller supplies a disposable, non-existing output path. Determinism should be
+    tested by invoking this helper repeatedly with equivalent completed exports.
+    """
+
+    if not isinstance(candidate, Adapter):
+        raise ConformanceError("object does not implement the Cernora Adapter contract")
+    if output.exists() or output.is_symlink():
+        raise ConformanceError("Adapter conformance output must not already exist")
+    try:
+        result = candidate.adapt(completed_export, output)
+    except Exception as exc:
+        raise ConformanceError("Adapter rejected the supplied completed export") from exc
+    if not isinstance(result, AdaptedBundle):
+        raise ConformanceError("Adapter must return a Cernora AdaptedBundle")
+    try:
+        expected_bundle_path = (output / "bundle.json").resolve(strict=True)
+        actual_bundle_path = result.bundle_path.resolve(strict=True)
+    except OSError as exc:
+        raise ConformanceError("Adapter did not return a readable bundle path") from exc
+    if actual_bundle_path != expected_bundle_path:
+        raise ConformanceError("Adapter bundle path must be <output>/bundle.json")
+
+    files = _closed_output_files(output)
+    bundle_bytes = files.get("bundle.json")
+    if bundle_bytes is None:
+        raise ConformanceError("Adapter output is missing bundle.json")
+    try:
+        bundle = decode_evidence_bundle_v2(bundle_bytes)
+    except (TypeError, ValueError) as exc:
+        raise ConformanceError("Adapter bundle.json is not strict EvidenceBundle v2") from exc
+    if bundle_bytes != canonical_json(bundle):
+        raise ConformanceError("Adapter bundle.json is not canonical JSON")
+
+    expected_files = {"bundle.json", *(artifact.path for artifact in bundle.artifacts)}
+    if set(files) != expected_files:
+        raise ConformanceError("Adapter output does not match the Bundle artifact set")
+    payloads = {artifact.artifact_id: files[artifact.path] for artifact in bundle.artifacts}
+    try:
+        verify_artifact_payloads_v2(bundle, payloads)
+    except (TypeError, ValueError) as exc:
+        raise ConformanceError("Adapter artifact payloads do not match the Bundle") from exc
+    if bundle.terminal.answer is not None:
+        answer_id = bundle.terminal.answer.artifact.artifact_id
+        if payloads[answer_id] != bundle.terminal.answer.content.encode("utf-8"):
+            raise ConformanceError("Adapter terminal answer bytes do not match its artifact")
+
+    return AdapterConformance(
+        bundle_id=bundle.bundle_id,
+        bundle_sha256=bundle.bundle_sha256,
+        artifact_ids=tuple(artifact.artifact_id for artifact in bundle.artifacts),
+        bundle_path=result.bundle_path,
+    )
+
+
+__all__ = [
+    "AdapterConformance",
+    "ConformanceError",
+    "ProfileConformance",
+    "check_adapter_conformance",
+    "check_profile_conformance",
+]

cernora/core/__init__.py

@@ -0,0 +1 @@
+"""Stable Cernora contracts and deterministic primitives."""