ai-kit-python 1.4.2__py3-none-any.whl

aikit/tests/test_benchmark.py

@@ -0,0 +1,241 @@
+"""Synthetic workers exercise isolation, execution evidence, semantic review and private export."""
+import copy
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+import benchmark
+import install
+import metrics
+
+WORKER = '''import json, os, pathlib, shutil, subprocess, sys
+r = json.load(sys.stdin)
+w = pathlib.Path.cwd()
+assert (w / "value.txt").read_text() == "initial\\n"
+assert "DATABASE_URL" not in os.environ
+assert not (w / ".git").exists()
+git = shutil.which("git")
+if git:
+    assert subprocess.run([git, "rev-parse", "--show-toplevel"], capture_output=True).returncode != 0
+assert (w / "ai-kit/CORE.md").exists() == (r["arm"] != "baseline")
+(w / "value.txt").write_text("changed\\n")
+a = dict(r["requested"], disposition="succeeded", input_tokens=100, output_tokens=10,
+         cached_input_tokens=0, reasoning_tokens=0,
+         cost={"amount": "1.00" if r["arm"] == "baseline" else "0.50", "currency": "USD", "source": "provider-usage"})
+print(json.dumps({"schema": 1, "attempts": [a], "rework": 0, "escalations": 0}))
+'''
+
+
+class BenchmarkTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.base = Path(self.temp.name).resolve()
+        self.root = self.base / "project"
+        self.root.mkdir()
+        self.git = shutil.which("git")
+        if not self.git:
+            self.skipTest("Git unavailable for immutable fixture snapshots")
+        self.git_run("init", "-q")
+        (self.root / "value.txt").write_text("initial\n", encoding="utf-8")
+        (self.root / "pyproject.toml").write_text('[project]\nname = "fixture"\nrequires-python = ">=3.10"\n', encoding="utf-8")
+        self.commit()
+        self.revision = self.git_run("rev-parse", "HEAD").strip()
+        # Current installed config is outside the fixture's pre-kit Git revision.
+        shutil.copytree(install.ROOT / "template/ai-kit", self.root / "ai-kit")
+        self.worker = self.base / "worker.py"
+        self.worker.write_text(WORKER, encoding="utf-8")
+        (self.base / "prompt.txt").write_text("Implement the requested fixture change.", encoding="utf-8")
+        (self.base / "rubric.txt").write_text("Manually verify the intended change and absence of unrelated edits.", encoding="utf-8")
+        self.pack = {"schema": 1, "experiment": "fixture-only", "revision": self.revision, "repeats": 2,
+                     "fixed": {"provider": "fixture", "model": "fixture-model", "effort": None},
+                     "worker": {"protocol": "aikit-worker-v1", "argv": [sys.executable, str(self.worker)]},
+                     "checks": [[sys.executable, "-c", "from pathlib import Path; assert Path('value.txt').read_text() == 'changed\\n'"]],
+                     "timeout_seconds": 15, "tasks": [{"task_id": "change", "prompt_file": "prompt.txt",
+                         "rubric_file": "rubric.txt", "path": ".", "stack": "PYTHON", "operation": "implement",
+                         "risks": [], "capabilities": []}]}
+        self.pack_file = self.base / "pack.json"
+
+    def git_run(self, *args):
+        return subprocess.run([self.git, "-C", str(self.root), *args], capture_output=True, text=True, check=True).stdout
+
+    def commit(self):
+        self.git_run("add", "--", "value.txt", "pyproject.toml")
+        self.git_run("-c", "user.name=Fixture", "-c", "user.email=fixture@example.test", "commit", "-qm", "Fixture")
+
+    def prepare(self):
+        self.pack_file.write_text(json.dumps(self.pack), encoding="utf-8")
+        return benchmark.prepare(self.root, self.pack_file, git=self.git)
+
+    def reviews(self, run, *, outcome="correct"):
+        value = benchmark.report(run)
+        path = self.base / "reviews.json"
+        path.write_text(json.dumps({"schema": 1, "reviews": [
+            {"task_id": r["record"]["task_id"], "arm": r["record"]["arm"], "review": "manual", "outcome": outcome}
+            for r in value["results"]]}), encoding="utf-8")
+        return path
+
+    def test_isolated_repeats_fixed_models_semantic_gate_reports_and_export(self):
+        plan = self.prepare()
+        self.assertFalse((self.root / plan["relative"]).exists())
+        self.assertEqual(len(plan["units"]), 6)
+        self.assertEqual(plan["units"][0]["route"], plan["units"][1]["route"])
+        (self.root / "value.txt").write_text("uncommitted owner change\n", encoding="utf-8")
+        with mock.patch.dict(os.environ, {"DATABASE_URL": "fixture-sensitive-value"}):
+            run = benchmark.execute(plan)
+        value = benchmark.report(run)
+        self.assertTrue(value["complete"])
+        self.assertEqual(value["observed_units"], 6)
+        self.assertTrue(all(r["record"]["checks"] == "passed" for r in value["results"]))
+        self.assertTrue(all(r["record"]["outcome"] == "incomplete" for r in value["results"]))
+        self.assertTrue(all(c["savings_percent"] is None for c in value["comparisons"]))
+        self.assertEqual((self.root / "value.txt").read_text(), "uncommitted owner change\n")
+        with self.assertRaises(ValueError):
+            benchmark.export_records(run, self.root)
+        reviews = self.reviews(run)
+        benchmark.review(run, reviews)
+        self.assertEqual(benchmark.report(run)["results"][0]["record"]["outcome"], "incomplete")
+        benchmark.review(run, reviews, apply=True)
+        report = benchmark.report(run)
+        self.assertEqual(report["comparisons"][0]["savings_percent"], "50.0")
+        self.assertTrue((run / "results.json").is_file())
+        self.assertIn("50.0", (run / "report.md").read_text())
+        self.assertEqual(benchmark.export_records(run, self.root, apply=True)["new_records"], 6)
+        self.assertEqual(benchmark.export_records(run, self.root, apply=True)["new_records"], 0)
+        self.assertEqual(len(metrics.read_records(self.root)), 6)
+        again = self.prepare()
+        self.assertEqual(benchmark.execute(again), run)
+        self.assertEqual(benchmark.report(run)["observed_units"], 6)
+        self.pack["repeats"] = 3
+        self.pack_file.write_text(json.dumps(self.pack), encoding="utf-8")
+        with self.assertRaisesRegex(ValueError, "different plan"):
+            benchmark.execute(benchmark.prepare(self.root, self.pack_file, git=self.git))
+
+    def test_failed_checks_cannot_be_reviewed_as_correct(self):
+        self.pack["repeats"] = 1
+        self.pack["checks"] = [[sys.executable, "-c", "raise SystemExit(1)"]]
+        run = benchmark.execute(self.prepare())
+        with self.assertRaises(ValueError):
+            benchmark.review(run, self.reviews(run), apply=True)
+        benchmark.review(run, self.reviews(run, outcome="incorrect"), apply=True)
+        self.assertTrue(all(c["savings_percent"] is None for c in benchmark.report(run)["comparisons"]))
+
+    def test_unknown_telemetry_and_unconfirmed_execution_block_savings(self):
+        self.worker.write_text('import json; print(json.dumps({"schema": 1, "attempts": [{"disposition": "inconclusive"}], "rework": 0, "escalations": 0}))', encoding="utf-8")
+        self.pack["repeats"] = 1
+        self.pack["checks"] = []
+        run = benchmark.execute(self.prepare())
+        benchmark.review(run, self.reviews(run), apply=True)
+        value = benchmark.report(run)
+        self.assertFalse(value["results"][0]["observed_matches_request"])
+        self.assertIsNone(value["summary"]["groups"][0]["cost_per_correct_result"])
+        self.assertTrue(all(c["savings_percent"] is None for c in value["comparisons"]))
+
+    def test_invalid_packs_prompt_traversal_secret_revision_and_links_refused(self):
+        self.pack["tasks"][0]["prompt_file"] = "../outside.txt"
+        with self.assertRaises(ValueError):
+            self.prepare()
+        self.pack["tasks"][0]["prompt_file"] = "prompt.txt"
+        original = install.is_link
+        with mock.patch.object(install, "is_link", side_effect=lambda p: p.name == "benchmarks" or original(p)):
+            with self.assertRaises(ValueError):
+                self.prepare()
+        (self.root / ".env").write_text("FIXTURE_SECRET=not-real\n", encoding="utf-8")
+        self.git_run("add", "-f", ".env")
+        self.git_run("-c", "user.name=Fixture", "-c", "user.email=fixture@example.test", "commit", "-qm", "Secret fixture")
+        self.pack["revision"] = self.git_run("rev-parse", "HEAD").strip()
+        with self.assertRaisesRegex(ValueError, "secret/runtime"):
+            self.prepare()
+
+    def test_worker_timeout_produces_unknown_attempt_not_a_free_success(self):
+        self.pack["repeats"] = 1
+        self.pack["checks"] = []
+        plan = self.prepare()
+        with mock.patch.object(benchmark, "invoke", return_value=(None, "")):
+            run = benchmark.execute(plan)
+        value = benchmark.report(run)
+        self.assertTrue(all(r["worker_exit_code"] is None for r in value["results"]))
+        self.assertTrue(all(r["record"]["rework"] is None and r["record"]["escalations"] is None for r in value["results"]))
+        self.assertTrue(all(g["rework"] is None and g["escalations"] is None for g in value["summary"]["groups"]))
+        with self.assertRaises(ValueError):
+            benchmark.review(run, self.reviews(run), apply=True)
+
+    def test_command_validation_secret_exceptions_and_shell_refusal(self):
+        bad = copy.deepcopy(self.pack)
+        bad["worker"]["argv"] = "python worker.py"
+        with self.assertRaises(ValueError):
+            benchmark.validate_pack(bad, self.root)
+        self.assertFalse(benchmark.secret_path(".env.example", [".env.*", "!.env.example"]))
+        self.assertTrue(benchmark.secret_path("secrets/test.txt", ["/secrets/"]))
+        routing = self.root / "ai-kit/router/routing.json"
+        config = json.loads(routing.read_text())
+        config["project_pin"] = {"provider": "local", "model": "fixture-local", "effort": None}
+        routing.write_text(json.dumps(config), encoding="utf-8")
+        plan = self.prepare()
+        adaptive_unit = next(unit for unit in plan["units"] if unit["arm"] == "kit-adaptive")
+        self.assertEqual(adaptive_unit["route"], config["project_pin"])
+        duplicate = copy.deepcopy(self.pack["tasks"][0])
+        duplicate["task_id"] = "renamed-same-case"
+        self.pack["tasks"].append(duplicate)
+        with self.assertRaisesRegex(ValueError, "Duplicate task material"):
+            self.prepare()
+        listing = b"100644 blob " + b"a" * 40 + b"\tCase.txt\x00" + b"100644 blob " + b"b" * 40 + b"\tcase.txt\x00"
+        with mock.patch.object(benchmark, "git_command", side_effect=[str(self.root).encode() + b"\n", self.revision.encode() + b"\n", listing]):
+            with self.assertRaisesRegex(ValueError, "Case-colliding"):
+                benchmark.snapshot_files(self.root, self.revision)
+
+    def test_task_path_is_probed_against_the_project(self):
+        bad = copy.deepcopy(self.pack)
+        bad["tasks"][0]["path"] = "../outside"
+        with self.assertRaises(ValueError):
+            benchmark.validate_pack(bad, self.root)
+        benchmark.validate_pack(copy.deepcopy(self.pack), self.root)
+
+    def test_kit_arm_agents_and_preset_are_configurable_and_validated(self):
+        self.pack["repeats"] = 1
+        self.pack["kit_agents"] = ["not-a-client"]
+        with self.assertRaises(ValueError):
+            self.prepare()
+        self.pack["kit_agents"] = ["claude"]
+        self.pack["kit_preset"] = "unknown-preset"
+        with self.assertRaises(ValueError):
+            self.prepare()
+        self.pack["kit_preset"] = "strict"
+        run = benchmark.execute(self.prepare())
+        workspaces = sorted(run.glob("unit-*/workspace"))
+        kit_workspaces = [w for w in workspaces if (w / "ai-kit/CORE.md").exists()]
+        baseline_workspaces = [w for w in workspaces if not (w / "ai-kit/CORE.md").exists()]
+        self.assertEqual(len(kit_workspaces), 2)
+        self.assertEqual(len(baseline_workspaces), 1)
+        self.assertTrue(all((w / "CLAUDE.md").is_file() for w in kit_workspaces))
+        self.assertFalse((baseline_workspaces[0] / "CLAUDE.md").exists())
+
+    def test_unit_failures_are_recorded_and_runs_resume_without_retry(self):
+        self.pack["repeats"] = 1
+        # The fixture evidences PYTHON only, so both kit arms fail before any paid worker call.
+        self.pack["tasks"][0]["stack"] = "GO"
+        plan = self.prepare()
+        run = benchmark.execute(plan)
+        value = benchmark.report(run)
+        self.assertTrue(value["complete"])
+        self.assertEqual(value["observed_units"], 3)
+        failed = [r for r in value["results"] if not r["telemetry_valid"]]
+        self.assertEqual(len(failed), 2)
+        self.assertTrue(all(r["record"]["arm"] in {"kit-fixed", "kit-adaptive"} for r in failed))
+        self.assertTrue(all("error" in r for r in failed))
+        baseline = next(r for r in value["results"] if r["record"]["arm"] == "baseline")
+        self.assertTrue(baseline["telemetry_valid"])
+        self.assertTrue(all(c["savings_percent"] is None for c in value["comparisons"]))
+        self.assertEqual(benchmark.execute(plan), run)
+        self.assertEqual(benchmark.report(run)["observed_units"], 3)
+        self.pack["repeats"] = 2
+        self.pack_file.write_text(json.dumps(self.pack), encoding="utf-8")
+        with self.assertRaisesRegex(ValueError, "different plan"):
+            benchmark.execute(benchmark.prepare(self.root, self.pack_file, git=self.git))

aikit/tests/test_check_kit.py

@@ -0,0 +1,203 @@
+"""Catch concrete metadata, link, template, and checkout regressions."""
+from pathlib import Path
+import json
+import shutil
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+import check_kit
+
+
+class CheckerTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory(prefix="ai-kit-check-")
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name) / "reference"
+        shutil.copytree(check_kit.ROOT, self.root, ignore=shutil.ignore_patterns("__pycache__", ".git"))
+
+    def check(self):
+        # Real Git visibility is exercised by the top-level checker/CI invocation.
+        with patch.object(check_kit, "visibility", return_value=[]):
+            return check_kit.check(self.root, git="not-invoked")[0]
+
+    def test_clean_distribution(self):
+        self.assertEqual(self.check(), [])
+
+    def test_distribution_does_not_require_local_maintainer_files(self):
+        for name in check_kit.LOCAL_MAINTAINER_FILES:
+            (self.root / name).unlink(missing_ok=True)
+        self.assertEqual(self.check(), [])
+
+    def test_private_root_files_are_excluded_without_hiding_project_templates(self):
+        for name in check_kit.LOCAL_MAINTAINER_FILES:
+            (self.root / name).write_bytes(b"\xff\x00LOCAL_MAINTAINER_DATA")
+        self.assertEqual(self.check(), [])
+        sources = {p.relative_to(self.root).as_posix() for p in check_kit.source_files(self.root)}
+        self.assertTrue(check_kit.LOCAL_MAINTAINER_FILES.isdisjoint(sources))
+        for name in check_kit.LOCAL_MAINTAINER_FILES:
+            self.assertIn("template/" + name, sources)
+
+    def test_public_document_cannot_depend_on_optional_private_file(self):
+        with (self.root / "README.md").open("a", encoding="utf-8") as stream:
+            stream.write("\n[Private map](WIKI.md)\n")
+        self.assertTrue(any("public link depends on local maintainer file" in error for error in self.check()))
+
+    def test_modified_pinned_package_is_reported(self):
+        path = self.root / "template/ai-kit/vendor/security-audit/SKILL.md"
+        path.write_bytes(path.read_bytes() + b"\nUnexpected edit.\n")
+        self.assertTrue(any("integrity mismatch" in error for error in self.check()))
+
+    def test_upstream_fixture_language_exception_does_not_cover_authored_files(self):
+        (self.root / "scripts/unreviewed.txt").write_text("\u043f\u0440\u0438\u0432\u0435\u0442\n", encoding="utf-8")
+        self.assertTrue(any("Unexpected Cyrillic" in error for error in self.check()))
+
+    def test_missing_pinned_manifest_is_reported(self):
+        (self.root / "integrations/security-audit.json").unlink()
+        self.assertTrue(any("pinned security-audit package" in error for error in self.check()))
+
+    def test_git_checkout_binary_metadata_is_excluded(self):
+        path = self.root / ".git/objects/binary"
+        path.parent.mkdir(parents=True)
+        path.write_bytes(b"\xff\x00")
+        self.assertEqual(self.check(), [])
+
+    def test_missing_heading_is_reported(self):
+        with (self.root / "README.md").open("a", encoding="utf-8") as stream:
+            stream.write("\n[Broken](template/PROJECT_CONTEXT.md#nonexistent-heading)\n")
+        self.assertTrue(any("missing heading" in error for error in self.check()))
+
+    def test_user_documentation_has_no_size_quota_while_conditional_guides_keep_theirs(self):
+        documents = [self.root / "README.md", self.root / "docs/DOCKER_HUB_OVERVIEW.md"]
+        for document in documents:
+            document.write_bytes(document.read_bytes() + b"\n" + b"Reader documentation.\n" * 6000)
+        guide = self.root / "template/ai-kit/WEB-DESIGN.md"
+        guide.write_bytes(guide.read_bytes() + b"\n" + b"Conditional instructions.\n" * 600)
+        scoped_overview = self.root / "template/ai-kit/DOCKER_HUB_OVERVIEW.md"
+        scoped_overview.write_bytes(b"# Scoped instructions\n\n" + b"Conditional instructions.\n" * 600)
+        with patch.object(check_kit, "visibility", return_value=[]):
+            errors, warnings = check_kit.check(self.root, git="not-invoked")
+        self.assertEqual(errors, [])
+        for document in documents:
+            self.assertFalse(any(warning.endswith(str(document)) for warning in warnings))
+        self.assertTrue(any(warning.endswith(str(guide)) for warning in warnings))
+        self.assertTrue(any(warning.endswith(str(scoped_overview)) for warning in warnings))
+
+    def test_link_cannot_escape_reference(self):
+        with (self.root / "README.md").open("a", encoding="utf-8") as stream:
+            stream.write("\n[Escape](../../outside.md)\n")
+        self.assertTrue(any("escapes reference" in error for error in self.check()))
+
+    def test_skill_folder_name_mismatch(self):
+        path = self.root / "template/.agents/skills/go-work/SKILL.md"
+        path.write_text(path.read_text().replace("name: go-work", "name: wrong-name"), encoding="utf-8")
+        self.assertTrue(any("name/folder mismatch" in error for error in self.check()))
+
+    def test_registry_duplicate_is_reported(self):
+        path = self.root / "template/ai-kit/SKILLS.md"
+        with path.open("a", encoding="utf-8") as stream:
+            stream.write("\n[Repeated](../.agents/skills/go-work/SKILL.md)\n")
+        self.assertTrue(any("registry differs" in error for error in self.check()))
+
+    def test_corrupted_strix_reference_is_reported(self):
+        path = self.root / "template/ai-kit/vendor/strix/penetration-testing-with-strix/SKILL.md"
+        path.write_bytes(path.read_bytes() + b"\nChanged reference.\n")
+        self.assertTrue(any("Invalid pinned strix package" in error for error in self.check()))
+
+    def test_missing_strix_manifest_is_reported(self):
+        (self.root / "integrations/strix.json").unlink()
+        self.assertTrue(any("Missing: integrations/strix.json" in error for error in self.check()))
+
+    def test_history_is_not_installable_project_fact(self):
+        path = self.root / "template/PROJECT_CONTEXT.md"
+        with path.open("a", encoding="utf-8") as stream:
+            stream.write("\nKit review date: 2026-10-08.\n")
+        self.assertTrue(any("history leaked" in error for error in self.check()))
+
+    def test_invalid_provider_json_is_failure_not_crash(self):
+        path = self.root / "template/ai-kit/router/providers/openai.json"
+        path.write_text("{", encoding="utf-8")
+        self.assertTrue(any("invalid JSON" in error for error in self.check()))
+
+    def test_provider_effort_outside_supported_list_is_reported(self):
+        path = self.root / "template/ai-kit/router/providers/openai.json"
+        config = json.loads(path.read_text(encoding="utf-8"))
+        config["roles"]["work"]["effort"] = "ultra"
+        path.write_text(json.dumps(config), encoding="utf-8")
+        self.assertTrue(any("Unsupported configured effort" in error for error in self.check()))
+
+    def test_active_model_id_in_markdown_is_reported(self):
+        with (self.root / "template/ai-kit/ENGINEERING.md").open("a", encoding="utf-8") as stream:
+            stream.write("\nRoute this step to claude-opus-5-5.\n")
+        self.assertTrue(any("duplicated outside provider configuration" in error
+                            for error in self.check()))
+
+    def test_provider_without_verified_date_is_reported(self):
+        path = self.root / "template/ai-kit/router/providers/anthropic.json"
+        config = json.loads(path.read_text(encoding="utf-8"))
+        del config["verified_documentation_date"]
+        path.write_text(json.dumps(config), encoding="utf-8")
+        self.assertTrue(any("verified_documentation_date" in error for error in self.check()))
+
+    def test_source_enumeration_skips_git_but_keeps_hidden_instructions(self):
+        measurement = self.root / "ai-kit/.metrics/private-notes.md"
+        measurement.parent.mkdir(parents=True)
+        measurement.write_bytes(b"Private runtime data\n")
+        sources = {p.relative_to(self.root).as_posix() for p in check_kit.source_files(self.root)}
+        self.assertIn(".agents/skills/project-continuity/SKILL.md", sources)
+        self.assertIn("template/.agents/skills/go-work/SKILL.md", sources)
+        self.assertFalse(any(p.startswith(".git/") for p in sources))
+        self.assertFalse(any(p.startswith("ai-kit/.metrics/") for p in sources))
+        self.assertIn("template/ai-kit/METRICS.md", sources)
+
+    def test_agent_registry_missing_optional_source_is_reported(self):
+        path = self.root / "integrations/agents.json"
+        data = json.loads(path.read_text(encoding="utf-8"))
+        data["agents"]["aider"]["optional"] = [{"source": "missing.md", "destination": "CONVENTIONS.md"}]
+        path.write_text(json.dumps(data) + "\n", encoding="utf-8")
+        self.assertTrue(any("optional source missing" in error for error in self.check()))
+
+    def test_agent_registry_entry_missing_from_template_is_reported(self):
+        path = self.root / "integrations/agents.json"
+        data = json.loads(path.read_text(encoding="utf-8"))
+        data["agents"]["gemini"]["entries"] = ["NOPE.md"]
+        path.write_text(json.dumps(data) + "\n", encoding="utf-8")
+        self.assertTrue(any("entry missing from template" in error for error in self.check()))
+
+    def test_agent_registry_invalid_documentation_metadata_is_reported(self):
+        path = self.root / "integrations/agents.json"
+        data = json.loads(path.read_text(encoding="utf-8"))
+        data["agents"]["cursor"]["documentation"]["sources"] = []
+        path.write_text(json.dumps(data) + "\n", encoding="utf-8")
+        self.assertTrue(any("Invalid agent registry" in error for error in self.check()))
+
+    def test_agent_registry_missing_documentation_metadata_is_reported(self):
+        path = self.root / "integrations/agents.json"
+        data = json.loads(path.read_text(encoding="utf-8"))
+        del data["agents"]["cursor"]["documentation"]
+        path.write_text(json.dumps(data) + "\n", encoding="utf-8")
+        self.assertTrue(any("Invalid agent registry" in error for error in self.check()))
+
+    def test_project_template_requires_an_empty_evidence_snapshot(self):
+        path = self.root / "template/ai-kit/project.json"
+        data = json.loads(path.read_text(encoding="utf-8"))
+        data["evidence"]["modules"] = [{"path": "/", "manifests": {"go.mod": "hash"}}]
+        path.write_text(json.dumps(data) + "\n", encoding="utf-8")
+        self.assertTrue(any("must start empty" in error for error in self.check()))
+
+    def test_pending_provider_is_warning_not_error(self):
+        path = self.root / "template/ai-kit/router/providers/gemini.json"
+        config = json.loads(path.read_text(encoding="utf-8"))
+        config.pop("verified_documentation_date", None)
+        config["verification_status"] = "pending"
+        path.write_text(json.dumps(config) + "\n", encoding="utf-8")
+        with patch.object(check_kit, "visibility", return_value=[]):
+            errors, warnings = check_kit.check(self.root, git="not-invoked")
+        self.assertEqual(errors, [])
+        self.assertTrue(any("pending" in warning for warning in warnings))
+
+
+if __name__ == "__main__":
+    unittest.main()

aikit/tests/test_context.py

@@ -0,0 +1,76 @@
+"""Exercise manifest-evidence snapshots and doctor freshness warnings."""
+import contextlib
+import io
+import json
+from pathlib import Path
+import shutil
+import sys
+import tempfile
+import unittest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+import context
+import doctor
+import install
+
+
+class ContextFreshnessTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory(prefix="ai-kit-context-")
+        self.addCleanup(self.temp.cleanup)
+        self.base = Path(self.temp.name).resolve()
+        self.target = self.base / "project"
+        self.source = self.base / "reference"
+        shutil.copytree(install.ROOT, self.source, ignore=shutil.ignore_patterns("__pycache__", ".git"))
+
+    def write(self, relative, data):
+        path = self.target / relative
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(data, encoding="utf-8")
+
+    def apply(self):
+        plan = install.build_plan(self.target, root=self.source)
+        self.assertTrue(install.apply_plan(plan))
+
+    def test_fresh_install_captures_manifest_evidence(self):
+        self.write("go.mod", "module example.test/project\n\ngo 1.23\n")
+        self.apply()
+        project = json.loads((self.target / "ai-kit/project.json").read_text(encoding="utf-8"))
+        self.assertEqual(project["evidence"]["schema"], 1)
+        self.assertIn("go.mod", project["evidence"]["modules"][0]["manifests"])
+        report = doctor.examine(self.target, root=self.source)
+        self.assertFalse(any("evidence" in warning for warning in report["warnings"]))
+
+    def test_changed_manifest_and_new_module_are_reported(self):
+        self.write("go.mod", "module example.test/project\n\ngo 1.23\n")
+        self.apply()
+        self.write("go.mod", "module example.test/project\n\ngo 1.24\n")
+        self.write("api/pyproject.toml", '[project]\nname = "api"\nrequires-python = ">=3.10"\n')
+        warnings = doctor.examine(self.target, root=self.source)["warnings"]
+        self.assertTrue(any("Manifest evidence changed at /" in warning for warning in warnings))
+        self.assertTrue(any("Detected module missing from project facts: /api/" in warning for warning in warnings))
+
+    def test_changed_detected_command_is_reported(self):
+        self.write("package.json", '{"scripts": {"test": "vitest"}}\n')
+        self.apply()
+        self.write("package.json", '{"scripts": {"lint": "eslint ."}}\n')
+        warnings = doctor.examine(self.target, root=self.source)["warnings"]
+        self.assertTrue(any("Detected commands changed at /" in warning for warning in warnings))
+
+    def test_snapshot_preserves_project_facts_and_refreshes_evidence_only_on_apply(self):
+        self.write("go.mod", "module example.test/project\n\ngo 1.23\n")
+        self.apply()
+        self.write("go.mod", "module example.test/project\n\ngo 1.24\n")
+        before = json.loads((self.target / "ai-kit/project.json").read_text(encoding="utf-8"))
+        plan = context.snapshot_plan(self.target)
+        self.assertTrue(plan["changed"])
+        self.assertEqual(json.loads(plan["data"])["modules"], before["modules"])
+        self.assertTrue(context.apply_snapshot(plan))
+        report = doctor.examine(self.target, root=self.source)
+        self.assertFalse(any("Manifest evidence changed" in warning for warning in report["warnings"]))
+        with contextlib.redirect_stdout(io.StringIO()):
+            self.assertEqual(context.main(["snapshot", str(self.target)]), 0)
+
+
+if __name__ == "__main__":
+    unittest.main()