agentic-security-harness 1.0.0__py3-none-any.whl

agentic_security_harness/__init__.py

@@ -0,0 +1,220 @@
+"""Agentic Security Harness - defensive benchmark core.
+
+Pipeline: ``pattern -> target -> trace -> scorecard``. Built-in/local targets are
+deterministic and offline (no network, no provider calls). The experimental
+``run-external --execute`` makes OpenAI-compatible calls only on explicit opt-in; native
+provider and agent-host adapters are future. A ``DataEnvelope`` is a policy label, not
+encryption.
+"""
+
+from agentic_security_harness.adapter_base import TargetAdapterBase
+from agentic_security_harness.corpus import (
+    CORPUS_MANIFEST_SCHEMA_VERSION,
+    DEPRECATED_PATTERN_IDS,
+    PATTERN_REPLACEMENTS,
+    V1_PATTERN_IDS,
+    CorpusEntry,
+    CorpusManifestV1,
+    CorpusPatternReplacement,
+    corpus_contract,
+    corpus_manifest,
+    corpus_manifest_sha256,
+    parse_corpus_contract_json,
+    parse_corpus_contract_payload,
+)
+from agentic_security_harness.demo_adapter import DemoAgentTarget, run_scenarios
+from agentic_security_harness.demo_agent import DemoAgent
+from agentic_security_harness.doctor import DoctorReport, run_doctor
+from agentic_security_harness.html_report import render_report, write_html_report
+from agentic_security_harness.local_swarm import (
+    LocalSwarmSummary,
+    SwarmMetrics,
+    SwarmScenarioResult,
+    estimate_request_count,
+    run_local_swarm,
+)
+from agentic_security_harness.mock_target import MockTarget
+from agentic_security_harness.models import (
+    AuditEntry,
+    CapabilityCheckResult,
+    CapabilityToken,
+    DataEnvelope,
+    DefensivePattern,
+    ExploitTrace,
+    Finding,
+    HealthStatus,
+    Observation,
+    Target,
+    TargetDescriptor,
+    TargetMetadata,
+    ToolSchemaRecord,
+    TraceStep,
+)
+from agentic_security_harness.patterns import seed_patterns
+from agentic_security_harness.presets import Preset, apply_preset, list_presets
+from agentic_security_harness.protected_demo_agent import (
+    ProtectedDemoAgent,
+    ProtectedDemoAgentTarget,
+)
+from agentic_security_harness.remediation import (
+    ControlRecommendation,
+    RemediationReport,
+    build_recommendations,
+    build_remediation_md,
+    remediation_to_json,
+    write_remediation,
+)
+from agentic_security_harness.reporting import (
+    build_comparison_md,
+    build_executive_md,
+    build_summary_md,
+    scorecard_to_json,
+    traces_to_json,
+    write_comparison,
+    write_reports,
+)
+from agentic_security_harness.run_diff import RunDiff, diff_runs, write_run_diff
+from agentic_security_harness.run_manifest import (
+    RunManifest,
+    build_manifest,
+    load_run_manifests,
+    write_run_manifest,
+)
+from agentic_security_harness.rundb import index_runs, list_db_runs
+from agentic_security_harness.runner import HarnessRunner
+from agentic_security_harness.scenario_timeline import (
+    ScenarioTimeline,
+    TimelineReplay,
+    TimelineReplayStep,
+    TimelineValidatorResult,
+    replay_timeline,
+    validate_timeline,
+)
+from agentic_security_harness.schema_versions import (
+    SCHEMA_VERSIONS,
+    check_schema_version,
+    schema_version,
+)
+from agentic_security_harness.scorecard import ScorecardSummary, build_scorecard
+from agentic_security_harness.standards_mapping import (
+    CategoryStandards,
+    standards_mapping,
+    validate_standards_mapping,
+)
+from agentic_security_harness.stats import (
+    RetentionCandidate,
+    RetentionPlan,
+    RunStats,
+    apply_retention_plan,
+    build_retention_plan,
+    build_run_stats,
+    write_run_stats,
+)
+from agentic_security_harness.trace_schema import (
+    TRACE_SCHEMA_LEGACY_VERSIONS,
+    TRACE_SCHEMA_V1,
+    migrate_trace_payload_to_v1,
+    parse_trace_payload,
+)
+from agentic_security_harness.validation import ValidationResult, validate_path
+from agentic_security_harness.version import __version__
+
+__all__ = [
+    "__version__",
+    "DataEnvelope",
+    "CapabilityToken",
+    "CapabilityCheckResult",
+    "ToolSchemaRecord",
+    "AuditEntry",
+    "DefensivePattern",
+    "ExploitTrace",
+    "Finding",
+    "HealthStatus",
+    "Observation",
+    "Target",
+    "TargetAdapterBase",
+    "TargetDescriptor",
+    "TargetMetadata",
+    "TraceStep",
+    "MockTarget",
+    "DemoAgent",
+    "DemoAgentTarget",
+    "ProtectedDemoAgent",
+    "ProtectedDemoAgentTarget",
+    "run_scenarios",
+    "HarnessRunner",
+    "ScorecardSummary",
+    "SCHEMA_VERSIONS",
+    "schema_version",
+    "check_schema_version",
+    "TRACE_SCHEMA_V1",
+    "TRACE_SCHEMA_LEGACY_VERSIONS",
+    "parse_trace_payload",
+    "migrate_trace_payload_to_v1",
+    "RunDiff",
+    "diff_runs",
+    "write_run_diff",
+    "ScenarioTimeline",
+    "TimelineReplay",
+    "TimelineReplayStep",
+    "TimelineValidatorResult",
+    "replay_timeline",
+    "validate_timeline",
+    "Preset",
+    "list_presets",
+    "apply_preset",
+    "index_runs",
+    "list_db_runs",
+    "build_scorecard",
+    "seed_patterns",
+    "CorpusEntry",
+    "CorpusManifestV1",
+    "CorpusPatternReplacement",
+    "CORPUS_MANIFEST_SCHEMA_VERSION",
+    "V1_PATTERN_IDS",
+    "DEPRECATED_PATTERN_IDS",
+    "PATTERN_REPLACEMENTS",
+    "corpus_contract",
+    "corpus_manifest",
+    "corpus_manifest_sha256",
+    "parse_corpus_contract_json",
+    "parse_corpus_contract_payload",
+    "write_reports",
+    "write_comparison",
+    "build_summary_md",
+    "build_executive_md",
+    "build_comparison_md",
+    "traces_to_json",
+    "scorecard_to_json",
+    "ValidationResult",
+    "validate_path",
+    "RunManifest",
+    "build_manifest",
+    "write_run_manifest",
+    "load_run_manifests",
+    "render_report",
+    "write_html_report",
+    "LocalSwarmSummary",
+    "SwarmMetrics",
+    "SwarmScenarioResult",
+    "run_local_swarm",
+    "estimate_request_count",
+    "run_doctor",
+    "DoctorReport",
+    "CategoryStandards",
+    "standards_mapping",
+    "validate_standards_mapping",
+    "RunStats",
+    "RetentionCandidate",
+    "RetentionPlan",
+    "build_run_stats",
+    "write_run_stats",
+    "build_retention_plan",
+    "apply_retention_plan",
+    "ControlRecommendation",
+    "RemediationReport",
+    "build_recommendations",
+    "build_remediation_md",
+    "remediation_to_json",
+    "write_remediation",
+]

agentic_security_harness/adapter_base.py

@@ -0,0 +1,61 @@
+"""Base lifecycle hooks for target adapters.
+
+Adapters can still satisfy the small structural ``Target`` protocol directly. This base
+class exists for adapters that want explicit readiness, reproducibility metadata, and
+per-pattern capability checks without re-implementing boilerplate.
+"""
+
+from __future__ import annotations
+
+from agentic_security_harness.models import (
+    CapabilityCheckResult,
+    DefensivePattern,
+    HealthStatus,
+    TargetMetadata,
+)
+
+
+class TargetAdapterBase:
+    """Convenience base class for target adapters with lifecycle hooks."""
+
+    target_type = "custom"
+    adapter_name = "custom"
+    adapter_version = "0.1"
+    deterministic = True
+    network_mode = "off"
+    memory_mode = "off"
+    permission_model = "none"
+    provider_calls = False
+
+    def __init__(self, name: str | None = None) -> None:
+        self.name = name or self.adapter_name
+
+    def descriptor_fields(self) -> tuple[str, str, str]:
+        return (self.target_type, self.name, self.adapter_name)
+
+    def health(self) -> HealthStatus:
+        return HealthStatus(
+            ok=True,
+            status="ready",
+            message=f"{self.adapter_name} adapter ready",
+            checks={"descriptor": True, "observe": True},
+        )
+
+    def metadata(self, run_id: str) -> TargetMetadata:
+        return TargetMetadata(
+            adapter_name=self.adapter_name,
+            adapter_version=self.adapter_version,
+            memory_mode=self.memory_mode,  # type: ignore[arg-type]
+            permission_model=self.permission_model,  # type: ignore[arg-type]
+            network_mode=self.network_mode,  # type: ignore[arg-type]
+            deterministic=self.deterministic,
+            provider_calls=self.provider_calls,
+            run_id=run_id,
+        )
+
+    def capability_check(self, pattern: DefensivePattern) -> CapabilityCheckResult:
+        return CapabilityCheckResult(
+            pattern_id=pattern.pattern_id,
+            supported=True,
+            safety_gates_passed=True,
+        )

agentic_security_harness/adapters.py

@@ -0,0 +1,197 @@
+"""Adapter registry: maps target ids to constructors and metadata.
+
+Built-in targets are local, deterministic, and make no network or provider calls.
+This module provides ``list_targets()`` and ``make_target()`` for CLI discovery.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from agentic_security_harness.models import Target
+
+
+class TargetInfo:
+    """Metadata for a registered built-in target."""
+
+    __slots__ = ("target_id", "name", "type", "deterministic", "description", "_factory")
+
+    def __init__(
+        self,
+        target_id: str,
+        name: str,
+        type: str,
+        deterministic: bool,
+        description: str,
+        factory: Any,
+    ) -> None:
+        self.target_id = target_id
+        self.name = name
+        self.type = type
+        self.deterministic = deterministic
+        self.description = description
+        self._factory = factory
+
+    def make(self) -> Target:
+        return self._factory()
+
+
+_REGISTRY: list[TargetInfo] = []
+
+
+def _register(info: TargetInfo) -> None:
+    _REGISTRY.append(info)
+
+
+def _init_registry() -> None:
+    if _REGISTRY:
+        return
+
+    from agentic_security_harness.demo_adapter import DemoAgentTarget
+    from agentic_security_harness.mock_target import MockTarget
+    from agentic_security_harness.protected_demo_agent import ProtectedDemoAgentTarget
+
+    _register(
+        TargetInfo(
+            target_id="mock",
+            name="demo-mock-agent",
+            type="mock_agent",
+            deterministic=True,
+            description="Deterministic minimal target for fast benchmark checks",
+            factory=MockTarget,
+        )
+    )
+    _register(
+        TargetInfo(
+            target_id="demo-agent",
+            name="demo-local-agent",
+            type="demo_agent",
+            deterministic=True,
+            description="Local vulnerable-by-design synthetic agent",
+            factory=DemoAgentTarget,
+        )
+    )
+    _register(
+        TargetInfo(
+            target_id="protected-demo-agent",
+            name="protected-demo-agent",
+            type="protected_demo_agent",
+            deterministic=True,
+            description="Local controlled synthetic agent; demonstrates risk reduction",
+            factory=ProtectedDemoAgentTarget,
+        )
+    )
+    _register(
+        TargetInfo(
+            target_id="toy-local-function",
+            name="toy-local-function",
+            type="toy_local",
+            deterministic=True,
+            description="Toy adapter wrapping a plain Python function as a target",
+            factory=_ToyLocalFunctionTarget,
+        )
+    )
+
+    from agentic_security_harness.toy_adapters import (
+        ProtectedToyMultiAgentHandoffTarget,
+        ToyMultiAgentHandoffTarget,
+        ToyRagTarget,
+        ToyToolsTarget,
+    )
+
+    _register(
+        TargetInfo(
+            target_id="toy-rag",
+            name="toy-rag",
+            type="toy_rag",
+            deterministic=True,
+            description="Toy retrieval agent; exercises data/memory/injection surfaces",
+            factory=ToyRagTarget,
+        )
+    )
+    _register(
+        TargetInfo(
+            target_id="toy-tools",
+            name="toy-tools",
+            type="toy_tools",
+            deterministic=True,
+            description="Toy tool-using agent; exercises tool/authority surfaces",
+            factory=ToyToolsTarget,
+        )
+    )
+    _register(
+        TargetInfo(
+            target_id="toy-multi-agent",
+            name="toy-multi-agent",
+            type="toy_multi_agent",
+            deterministic=True,
+            description=(
+                "Toy coordinator/worker handoff adapter; exercises data-label "
+                "handoff and capability delegation surfaces"
+            ),
+            factory=ToyMultiAgentHandoffTarget,
+        )
+    )
+    _register(
+        TargetInfo(
+            target_id="protected-toy-multi-agent",
+            name="protected-toy-multi-agent",
+            type="protected_toy_multi_agent",
+            deterministic=True,
+            description=(
+                "Protected toy coordinator/worker handoff adapter; blocks malformed "
+                "handoffs through deterministic verifier decisions"
+            ),
+            factory=ProtectedToyMultiAgentHandoffTarget,
+        )
+    )
+
+
+class _ToyLocalFunctionTarget:
+    """Toy adapter: wraps a deterministic Python function as a target.
+
+    Returns PASS (no findings) for every pattern, useful for docs and tests.
+    """
+
+    def __init__(self, name: str = "toy-local-function") -> None:
+        self.name = name
+
+    def descriptor_fields(self) -> tuple[str, str, str]:
+        return ("toy_local", self.name, "toy")
+
+    def observe(self, pattern: Any) -> Any:
+        from agentic_security_harness.models import Observation, TraceStep
+
+        steps = [
+            TraceStep(index=i, actor="toy", action=node)
+            for i, node in enumerate(pattern.graph_path)
+        ]
+        return Observation(
+            steps=steps,
+            observed_behavior="toy adapter completed with no observed violation",
+            findings=[],
+        )
+
+
+def list_targets() -> list[TargetInfo]:
+    """Return metadata for all registered built-in targets."""
+    _init_registry()
+    return list(_REGISTRY)
+
+
+def make_target(target_id: str) -> Target:
+    """Create a target instance by its registered id.
+
+    Raises ``KeyError`` with a helpful message if the id is unknown.
+    """
+    _init_registry()
+    for info in _REGISTRY:
+        if info.target_id == target_id:
+            return info.make()
+    known = ", ".join(info.target_id for info in _REGISTRY)
+    raise KeyError(f"unknown target id '{target_id}'. Known targets: {known}")
+
+
+def target_ids() -> list[str]:
+    """Return just the registered target ids."""
+    return [info.target_id for info in list_targets()]

agentic_security_harness/attestation_policy.py

@@ -0,0 +1,203 @@
+"""Fail-closed policy checks for GitHub artifact-attestation verification output.
+
+Cryptographic verification remains the responsibility of ``gh attestation verify``.
+This module checks the authenticated result against the release policy that the CLI's
+generic signature verification cannot infer for this repository.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+SLSA_PROVENANCE_V1 = "https://slsa.dev/provenance/v1"
+GITHUB_WORKFLOW_BUILD_TYPE = "https://actions.github.io/buildtypes/workflow/v1"
+IN_TOTO_STATEMENT_V1 = "https://in-toto.io/Statement/v1"
+
+
+def _mapping(value: object) -> Mapping[str, Any] | None:
+    return value if isinstance(value, Mapping) else None
+
+
+def _sequence(value: object) -> Sequence[object] | None:
+    if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+        return value
+    return None
+
+
+def _entry_issues(
+    entry: object,
+    *,
+    repository: str,
+    workflow_path: str,
+    source_ref: str,
+    source_digest: str,
+    subject_sha256: str,
+) -> list[str]:
+    issues: list[str] = []
+    row = _mapping(entry)
+    if row is None:
+        return ["verification entry is not an object"]
+
+    result = _mapping(row.get("verificationResult"))
+    if result is None:
+        return ["verificationResult is missing"]
+    timestamps = _sequence(result.get("verifiedTimestamps"))
+    if not timestamps:
+        issues.append("verified transparency/timestamp evidence is missing")
+
+    statement = _mapping(result.get("statement"))
+    if statement is None:
+        return [*issues, "verified statement is missing"]
+    if statement.get("_type") != IN_TOTO_STATEMENT_V1:
+        issues.append("statement type is not in-toto Statement v1")
+    if statement.get("predicateType") != SLSA_PROVENANCE_V1:
+        issues.append("predicate type is not SLSA provenance v1")
+
+    subjects = _sequence(statement.get("subject"))
+    subject_match = False
+    if subjects is not None:
+        for subject in subjects:
+            subject_row = _mapping(subject)
+            digest = _mapping(subject_row.get("digest")) if subject_row else None
+            if digest and str(digest.get("sha256", "")).lower() == subject_sha256:
+                subject_match = True
+                break
+    if not subject_match:
+        issues.append("attested subject digest does not match the artifact")
+
+    predicate = _mapping(statement.get("predicate"))
+    build_definition = _mapping(predicate.get("buildDefinition")) if predicate else None
+    run_details = _mapping(predicate.get("runDetails")) if predicate else None
+    if build_definition is None or run_details is None:
+        return [*issues, "SLSA buildDefinition or runDetails is missing"]
+    if build_definition.get("buildType") != GITHUB_WORKFLOW_BUILD_TYPE:
+        issues.append("unexpected SLSA build type")
+
+    repo_url = f"https://github.com/{repository}"
+    external_parameters = _mapping(build_definition.get("externalParameters"))
+    if external_parameters is None or set(external_parameters) != {"workflow"}:
+        issues.append("unexpected SLSA external parameters")
+    workflow = _mapping(external_parameters.get("workflow")) if external_parameters else None
+    expected_workflow = {
+        "repository": repo_url,
+        "path": workflow_path,
+        "ref": source_ref,
+    }
+    if (
+        workflow is None
+        or set(workflow) != set(expected_workflow)
+        or any(workflow.get(key) != value for key, value in expected_workflow.items())
+    ):
+        issues.append("workflow repository/path/ref does not match release policy")
+
+    internal_parameters = _mapping(build_definition.get("internalParameters"))
+    github_parameters = _mapping(internal_parameters.get("github")) if internal_parameters else None
+    if github_parameters is None or github_parameters.get("event_name") != "push":
+        issues.append("attestation was not produced by a push event")
+    if github_parameters is None or github_parameters.get("runner_environment") != "github-hosted":
+        issues.append("attestation was not produced on a GitHub-hosted runner")
+
+    expected_uri = f"git+{repo_url}@{source_ref}"
+    dependency_match = False
+    dependencies = _sequence(build_definition.get("resolvedDependencies"))
+    if dependencies is not None:
+        for dependency in dependencies:
+            dependency_row = _mapping(dependency)
+            digest = _mapping(dependency_row.get("digest")) if dependency_row else None
+            if (
+                dependency_row
+                and dependency_row.get("uri") == expected_uri
+                and digest
+                and digest.get("gitCommit") == source_digest
+            ):
+                dependency_match = True
+                break
+    if not dependency_match:
+        issues.append("resolved source ref/digest does not match the release source")
+
+    builder = _mapping(run_details.get("builder"))
+    expected_builder = f"{repo_url}/{workflow_path}@{source_ref}"
+    if builder is None or builder.get("id") != expected_builder:
+        issues.append("builder identity does not match the release workflow")
+    return issues
+
+
+def validate_verification_results(
+    payload: object,
+    *,
+    repository: str,
+    workflow_path: str,
+    source_ref: str,
+    source_digest: str,
+    subject_sha256: str,
+) -> list[str]:
+    """Return policy failures; one fully matching verified attestation is sufficient."""
+    rows = _sequence(payload)
+    if not rows:
+        return ["verification output contains no attestations"]
+    failures = [
+        _entry_issues(
+            entry,
+            repository=repository,
+            workflow_path=workflow_path,
+            source_ref=source_ref,
+            source_digest=source_digest,
+            subject_sha256=subject_sha256,
+        )
+        for entry in rows
+    ]
+    if any(not issues for issues in failures):
+        return []
+    return [
+        f"attestation[{index}]: {issue}"
+        for index, issues in enumerate(failures)
+        for issue in issues
+    ]
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="enforce the ASH release-attestation policy")
+    parser.add_argument("--artifact", type=Path, required=True)
+    parser.add_argument("--verification-json", type=Path, required=True)
+    parser.add_argument("--repository", required=True)
+    parser.add_argument("--workflow-path", required=True)
+    parser.add_argument("--source-ref", required=True)
+    parser.add_argument("--source-digest", required=True)
+    args = parser.parse_args(argv)
+
+    try:
+        payload = json.loads(args.verification_json.read_text(encoding="utf-8"))
+        issues = validate_verification_results(
+            payload,
+            repository=args.repository,
+            workflow_path=args.workflow_path,
+            source_ref=args.source_ref,
+            source_digest=args.source_digest,
+            subject_sha256=_sha256(args.artifact),
+        )
+    except (OSError, json.JSONDecodeError, ValueError) as exc:
+        print(f"release attestation policy error: {type(exc).__name__}")
+        return 2
+    if issues:
+        for issue in issues:
+            print(f"release attestation policy failure: {issue}")
+        return 1
+    print(f"release attestation policy accepted: {args.artifact.name}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())