iris-security-cli 0.1.0__py3-none-any.whl

tests/test_evidence.py

@@ -0,0 +1,296 @@
+"""Tests for Evidence Vault read path and iris evidence CLI."""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import pytest
+from click.testing import CliRunner
+
+from iris_cli.evidence import (
+    aggregate_stats,
+    build_report_data,
+    format_report_json,
+    format_report_markdown,
+)
+from iris_cli.main import cli
+from iris_core.evidence.vault import EvidenceVault
+
+
+PASSPORT_YAML = """
+apiVersion: iris.io/v1alpha1
+kind: AgentPassport
+metadata:
+  name: payment-agent
+  agent_id: 6684638e-582c-4be5-ad94-f6029738305f
+spec:
+  version: 0.1.0
+  owner: gilbert.martin@gmail.com
+  team: iris-platform
+  data_classification: pii
+  compliance_tags:
+  - colorado-ai-act
+  environments:
+  - dev
+  - staging
+  intent_ref: governance/agents/payment-agent/policy-intent.md
+  is_high_risk_ai: true
+  evidence_vault_id: IA-payment-agent-A3F2B1C4
+  last_reviewed_at: '2026-05-20T07:28:24.684454'
+"""
+
+
+def _write_vault(vault_root: Path, agent: str, events: list, assessments: list | None = None):
+    agent_dir = vault_root / agent
+    agent_dir.mkdir(parents=True)
+    with open(agent_dir / "events.jsonl", "w") as f:
+        for event in events:
+            f.write(json.dumps(event) + "\n")
+    if assessments is not None:
+        with open(agent_dir / "assessments.jsonl", "w") as f:
+            for assessment in assessments:
+                f.write(json.dumps(assessment) + "\n")
+
+
+def _sample_events(agent: str) -> list:
+    return [
+        {
+            "event_id": "e1",
+            "timestamp": "2026-05-28T10:00:00",
+            "agent_id": agent,
+            "action": "call",
+            "resource": "payments-api",
+            "environment": "dev",
+            "decision": "PERMIT",
+            "violations": [],
+        },
+        {
+            "event_id": "e2",
+            "timestamp": "2026-05-28T11:00:00",
+            "agent_id": agent,
+            "action": "call",
+            "resource": "unknown-tool",
+            "environment": "dev",
+            "decision": "DENY",
+            "violations": [
+                {
+                    "rule_id": "IRIS-TOOL-001",
+                    "severity": "HIGH",
+                    "message": "Tool not in declared permissions",
+                    "compliance_refs": [],
+                }
+            ],
+        },
+        {
+            "event_id": "e3",
+            "timestamp": "2026-05-28T12:00:00",
+            "agent_id": agent,
+            "action": "write",
+            "resource": "storage-api",
+            "environment": "staging",
+            "decision": "DENY",
+            "violations": [
+                {
+                    "rule_id": "IRIS-XR-001",
+                    "severity": "CRITICAL",
+                    "message": "Cross-region transfer attempted",
+                    "compliance_refs": ["china-pipl"],
+                }
+            ],
+        },
+        {
+            "event_id": "e4",
+            "timestamp": "2026-05-28T13:00:00",
+            "agent_id": agent,
+            "action": "approve",
+            "resource": "loan-decision",
+            "environment": "staging",
+            "decision": "HITL",
+            "violations": [],
+        },
+    ]
+
+
+@pytest.fixture
+def gov_dir(tmp_path):
+    agent_dir = tmp_path / "governance" / "agents" / "payment-agent"
+    agent_dir.mkdir(parents=True)
+    (agent_dir / "passport.yaml").write_text(PASSPORT_YAML)
+    return tmp_path / "governance" / "agents"
+
+
+@pytest.fixture
+def vault_root(tmp_path):
+    return tmp_path / "evidence"
+
+
+@pytest.fixture
+def populated_vault(vault_root):
+    assessments = [
+        {
+            "assessment_id": "IA-payment-agent-A3F2B1C4",
+            "agent": "payment-agent",
+            "risk_level": "MEDIUM",
+            "assessed_by": "iris-platform",
+            "timestamp": "2026-05-20T07:28:24.684454",
+            "findings_count": 2,
+            "framework": "colorado-ai-act",
+        }
+    ]
+    _write_vault(
+        vault_root,
+        "payment-agent",
+        _sample_events("payment-agent"),
+        assessments=assessments,
+    )
+    return vault_root
+
+
+def test_vault_summary_calculates_correctly(populated_vault):
+    vault = EvidenceVault(agent_id="payment-agent", vault_dir=populated_vault)
+    summary = vault.get_summary(last_reviewed_at="2026-05-20T07:28:24.684454")
+
+    assert summary.agent_id == "payment-agent"
+    assert summary.total_evaluations == 4
+    assert summary.total_violations == 2
+    assert summary.violations_by_severity["HIGH"] == 1
+    assert summary.violations_by_severity["CRITICAL"] == 1
+    assert summary.violations_by_rule["IRIS-TOOL-001"] == 1
+    assert summary.violations_by_rule["IRIS-XR-001"] == 1
+    assert summary.most_violated_rule in ("IRIS-TOOL-001", "IRIS-XR-001")
+    assert summary.compliance_pass_rate == pytest.approx(0.5)
+    assert set(summary.environments_active) == {"dev", "staging"}
+    assert summary.cross_region_blocks == 1
+    assert summary.hitl_gates_triggered == 1
+    assert summary.last_assessment_date.startswith("2026-05-20")
+
+
+def test_report_includes_all_sections(gov_dir, populated_vault):
+    from iris_core.models.passport import AgentPassport
+
+    passport = AgentPassport.from_yaml((gov_dir / "payment-agent" / "passport.yaml").read_text())
+    vault = EvidenceVault(agent_id="payment-agent", vault_dir=populated_vault)
+    data = build_report_data("payment-agent", passport, vault)
+
+    markdown = format_report_markdown(data)
+    required_sections = [
+        "## Agent Identity",
+        "## Compliance Status",
+        "## Evaluation Statistics",
+        "## Top Violations",
+        "## Impact Assessment History",
+        "## Cross-Region Detection",
+        "## HITL Gate Events",
+        "## Annual Review",
+        "## Evidence Vault Integrity",
+    ]
+    for section in required_sections:
+        assert section in markdown
+
+    json_report = json.loads(format_report_json(data))
+    assert json_report["schema_version"] == "1.0"
+    assert json_report["integrity"]["valid"] is True
+
+
+def test_export_json_is_valid(gov_dir, populated_vault, tmp_path):
+    runner = CliRunner()
+    output = tmp_path / "audit.json"
+    result = runner.invoke(
+        cli,
+        [
+            "evidence",
+            "export",
+            "--agent",
+            "payment-agent",
+            "--output",
+            str(output),
+            "--dir",
+            str(gov_dir),
+            "--vault-dir",
+            str(populated_vault),
+        ],
+    )
+    assert result.exit_code == 0, result.output
+
+    exported = json.loads(output.read_text())
+    assert exported["agent_id"] == "payment-agent"
+    assert len(exported["events"]) == 4
+    assert len(exported["assessments"]) == 1
+    assert exported["integrity"]["valid"] is True
+
+
+def test_stats_aggregates_across_agents(gov_dir, vault_root):
+    loan_dir = gov_dir / "loan-agent"
+    loan_dir.mkdir()
+    (loan_dir / "passport.yaml").write_text(
+        PASSPORT_YAML.replace("payment-agent", "loan-agent").replace(
+            "IA-payment-agent-A3F2B1C4", "IA-loan-agent-B1C2D3E4"
+        )
+    )
+
+    _write_vault(
+        vault_root,
+        "payment-agent",
+        _sample_events("payment-agent"),
+        assessments=[
+            {
+                "assessment_id": "IA-payment-agent-A3F2B1C4",
+                "timestamp": "2026-05-20T07:28:24.684454",
+            }
+        ],
+    )
+    _write_vault(
+        vault_root,
+        "loan-agent",
+        [
+            {
+                "event_id": "l1",
+                "timestamp": datetime.utcnow().isoformat(),
+                "agent_id": "loan-agent",
+                "action": "read",
+                "resource": "credit-score",
+                "environment": "dev",
+                "decision": "DENY",
+                "violations": [
+                    {
+                        "rule_id": "IRIS-TOOL-001",
+                        "severity": "CRITICAL",
+                        "message": "blocked",
+                        "compliance_refs": [],
+                    }
+                ],
+            }
+        ],
+    )
+
+    stats = aggregate_stats(gov_dir, vault_root=vault_root)
+    assert stats["total_agents"] == 2
+    assert stats["total_evaluations_this_week"] >= 1
+    assert any(r["rule_id"] == "IRIS-TOOL-001" for r in stats["top_violated_rules"])
+    assert any(a["agent"] == "loan-agent" for a in stats["agents_with_critical_violations"])
+
+
+def test_annual_review_deadline_calculated(gov_dir, populated_vault):
+    from iris_core.models.passport import AgentPassport
+
+    passport = AgentPassport.from_yaml((gov_dir / "payment-agent" / "passport.yaml").read_text())
+    vault = EvidenceVault(agent_id="payment-agent", vault_dir=populated_vault)
+
+    reviewed = datetime.utcnow() - timedelta(days=10)
+    passport.last_reviewed_at = reviewed
+    summary = vault.get_summary(last_reviewed_at=reviewed.isoformat())
+
+    assert summary.days_until_annual_review == 355
+
+    data = build_report_data("payment-agent", passport, vault)
+    assert data["annual_review"]["status"] == "CURRENT"
+    assert data["annual_review"]["days_until"] == 355
+
+    passport.last_reviewed_at = None
+    summary_no_review = vault.get_summary(last_reviewed_at=None)
+    assert summary_no_review.days_until_annual_review is None
+
+    data_overdue = build_report_data("payment-agent", passport, vault)
+    assert data_overdue["annual_review"]["status"] == "OVERDUE"

tests/test_policy_diff.py

@@ -0,0 +1,250 @@
+"""Tests for iris policy diff and Cedar parser."""
+
+import json
+
+import pytest
+
+from iris_cli.cedar_parser import CedarRule, diff_cedar, parse_cedar, summarize_diffs
+from iris_cli.policy_cache import check_draft_cache, save_policy_draft
+from iris_cli.policy_diff import format_diff_json, run_policy_diff
+
+
+PERMIT_READ_PAYMENTS = """
+// Satisfies: CO-004
+permit (
+  principal == iris::AgentPassport::"payment-agent",
+  action    == iris::Action::"read",
+  resource  == iris::API::"payments"
+)
+when {
+  context.environment in ["dev", "test", "staging", "production"] &&
+  context.user_consent_logged == true
+};
+"""
+
+PERMIT_CALL_SENDGRID = """
+// Satisfies: CO-004 (consent gate)
+permit (
+  principal == iris::AgentPassport::"payment-agent",
+  action    == iris::Action::"call",
+  resource  == iris::API::"sendgrid-email-api"
+)
+when {
+  context.environment in ["dev", "test", "staging", "production"] &&
+  context.user_consent_logged == true
+};
+"""
+
+FORBID_PII = """
+// Satisfies: CO-004, GDPR
+forbid (
+  principal == iris::AgentPassport::"payment-agent",
+  action    == iris::Action::"write",
+  resource  == iris::DataClass::"pii"
+)
+unless {
+  context.user_consent_logged == true
+};
+"""
+
+PERMIT_READ_PAYMENTS_PROD_ONLY = """
+// Satisfies: CO-004
+permit (
+  principal == iris::AgentPassport::"payment-agent",
+  action    == iris::Action::"read",
+  resource  == iris::API::"payments"
+)
+when {
+  context.environment == "production" &&
+  context.user_consent_logged == true
+};
+"""
+
+
+@pytest.fixture
+def gov_dir(tmp_path):
+    agent_dir = tmp_path / "governance" / "agents" / "payment-agent"
+    agent_dir.mkdir(parents=True)
+    (agent_dir / "passport.yaml").write_text(
+        """
+name: payment-agent
+owner: test@example.com
+team: platform
+data_classification: pii
+compliance_tags:
+  - colorado-ai-act
+environments:
+  - dev
+is_high_risk_ai: true
+"""
+    )
+    (agent_dir / "policy-intent.md").write_text("# Intent\nAllow payments access.")
+    return agent_dir
+
+
+@pytest.fixture
+def sample_old_cedar():
+    return PERMIT_READ_PAYMENTS + FORBID_PII
+
+
+@pytest.fixture
+def sample_new_cedar():
+    return PERMIT_READ_PAYMENTS + PERMIT_CALL_SENDGRID + FORBID_PII
+
+
+def test_diff_detects_added_rule(sample_old_cedar, sample_new_cedar):
+    old_rules = parse_cedar(sample_old_cedar)
+    new_rules = parse_cedar(sample_new_cedar)
+    diffs = diff_cedar(old_rules, new_rules)
+
+    added = [d for d in diffs if d.status == "ADDED"]
+    assert len(added) == 1
+    assert added[0].new_rule.action == 'iris::Action::"call"'
+    assert "sendgrid" in added[0].new_rule.resource.lower()
+    assert added[0].risk_delta == "NEUTRAL"
+    assert "CO-004" in added[0].compliance_affected
+
+
+def test_diff_detects_removed_permit_as_increased_risk():
+    old = parse_cedar(PERMIT_READ_PAYMENTS + PERMIT_CALL_SENDGRID)
+    new = parse_cedar(PERMIT_READ_PAYMENTS)
+    diffs = diff_cedar(old, new)
+
+    removed = [d for d in diffs if d.status == "REMOVED"]
+    assert len(removed) == 1
+    assert removed[0].old_rule.type == "permit"
+    assert removed[0].risk_delta == "INCREASED"
+    assert "capability removed" in removed[0].risk_reason.lower()
+
+
+def test_diff_detects_added_forbid_as_decreased_risk(sample_old_cedar):
+    old = parse_cedar(PERMIT_READ_PAYMENTS)
+    new = parse_cedar(PERMIT_READ_PAYMENTS + FORBID_PII)
+    diffs = diff_cedar(old, new)
+
+    added = [d for d in diffs if d.status == "ADDED"]
+    assert len(added) == 1
+    assert added[0].new_rule.type == "forbid"
+    assert added[0].risk_delta == "DECREASED"
+    assert "restriction" in added[0].risk_reason.lower()
+
+
+def test_diff_unchanged_not_shown_by_default(sample_old_cedar, sample_new_cedar):
+    old_rules = parse_cedar(sample_old_cedar)
+    new_rules = parse_cedar(sample_new_cedar)
+    all_diffs = diff_cedar(old_rules, new_rules)
+    visible = [d for d in all_diffs if d.status != "UNCHANGED"]
+
+    unchanged = [d for d in all_diffs if d.status == "UNCHANGED"]
+    assert len(unchanged) >= 1
+    assert all(d.status != "UNCHANGED" for d in visible)
+
+
+def test_diff_compliance_impact_shown():
+    old = parse_cedar(PERMIT_READ_PAYMENTS)
+    new = parse_cedar(PERMIT_READ_PAYMENTS_PROD_ONLY)
+    diffs = diff_cedar(old, new)
+    summary = summarize_diffs(diffs)
+
+    modified = [d for d in diffs if d.status == "MODIFIED"]
+    assert len(modified) == 1
+    assert "CO-004" in modified[0].compliance_affected
+    assert modified[0].risk_delta == "DECREASED"
+    assert summary["violations_opened"] == 0
+    assert summary["coverage_strengthened"].get("CO-004", 0) >= 1
+
+
+def test_diff_json_output(sample_old_cedar, sample_new_cedar, gov_dir):
+    (gov_dir / "policy.cedar").write_text(sample_old_cedar)
+    intent = (gov_dir / "policy-intent.md").read_text()
+    save_policy_draft(gov_dir, intent, sample_new_cedar, "anthropic", "claude-sonnet-4-6")
+
+    result = run_policy_diff(
+        agent="payment-agent",
+        governance_dir=gov_dir,
+    )
+    payload = json.loads(format_diff_json(result))
+
+    assert payload["agent"] == "payment-agent"
+    assert payload["draft_stale"] is False
+    assert "summary" in payload
+    assert payload["summary"]["counts"]["ADDED"] == 1
+    assert len(payload["diffs"]) == 1
+    assert payload["diffs"][0]["status"] == "ADDED"
+    assert payload["diffs"][0]["risk_delta"] == "NEUTRAL"
+
+
+def test_diff_uses_cached_draft_offline(sample_old_cedar, sample_new_cedar, gov_dir):
+    (gov_dir / "policy.cedar").write_text(sample_old_cedar)
+    intent = (gov_dir / "policy-intent.md").read_text()
+    save_policy_draft(gov_dir, intent, sample_new_cedar, "openai", "gpt-4o")
+
+    result = run_policy_diff(agent="payment-agent", governance_dir=gov_dir)
+    assert result.summary["counts"]["ADDED"] == 1
+    assert result.draft_stale is False
+    assert result.draft_status.meta.compiler_backend == "openai"
+
+
+def test_diff_detects_stale_draft(sample_old_cedar, sample_new_cedar, gov_dir):
+    (gov_dir / "policy.cedar").write_text(sample_old_cedar)
+    save_policy_draft(
+        gov_dir,
+        "old intent text",
+        sample_new_cedar,
+        "anthropic",
+        "claude-sonnet-4-6",
+    )
+
+    result = run_policy_diff(agent="payment-agent", governance_dir=gov_dir)
+    assert result.draft_stale is True
+
+
+def test_diff_missing_draft_raises_helpful_error(gov_dir):
+    (gov_dir / "policy.cedar").write_text(PERMIT_READ_PAYMENTS)
+
+    with pytest.raises(FileNotFoundError, match="No cached policy draft"):
+        run_policy_diff(agent="payment-agent", governance_dir=gov_dir)
+
+
+def test_save_and_check_draft_cache(gov_dir):
+    intent = "# Intent\nAllow payments."
+    save_policy_draft(gov_dir, intent, PERMIT_READ_PAYMENTS, "anthropic", "test-model")
+    status = check_draft_cache(gov_dir, intent)
+    assert status.draft_exists
+    assert status.is_stale is False
+
+    status_after_edit = check_draft_cache(gov_dir, intent + "\nMore text.")
+    assert status_after_edit.is_stale is True
+
+
+def test_cedar_parser_extracts_permit():
+    rules = parse_cedar(PERMIT_READ_PAYMENTS)
+    assert len(rules) == 1
+    assert rules[0].type == "permit"
+    assert "read" in rules[0].plain_english.lower()
+    assert "payments" in rules[0].plain_english.lower()
+    assert rules[0].compliance_refs == ["CO-004"]
+
+
+def test_cedar_parser_extracts_forbid_with_unless():
+    rules = parse_cedar(FORBID_PII)
+    assert len(rules) == 1
+    assert "forbidden" in rules[0].plain_english.lower()
+    assert "PII" in rules[0].plain_english
+    assert "CO-004" in rules[0].compliance_refs
+    assert "GDPR" in rules[0].compliance_refs
+
+
+def test_plain_english_generation():
+    permit = parse_cedar(PERMIT_READ_PAYMENTS)[0]
+    assert permit.plain_english == "Agent may read from payments API with consent"
+
+    forbid = parse_cedar(FORBID_PII)[0]
+    assert forbid.plain_english == (
+        "Agent is forbidden from write to PII data unless conditions are met"
+    )
+
+    sendgrid = parse_cedar(PERMIT_CALL_SENDGRID)[0]
+    assert "call" in sendgrid.plain_english.lower()
+    assert "sendgrid" in sendgrid.plain_english.lower()
+    assert "consent" in sendgrid.plain_english.lower()