ai-kit-python 1.4.2__py3-none-any.whl

aikit/tests/test_doctor.py

@@ -0,0 +1,148 @@
+"""Exercise the installed-project health check."""
+import contextlib
+import io
+import json
+from pathlib import Path
+import shutil
+import sys
+import tempfile
+import unittest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+import doctor
+import install
+
+
+class DoctorTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory(prefix="ai-kit-doctor-")
+        self.addCleanup(self.temp.cleanup)
+        self.base = Path(self.temp.name).resolve()
+        self.target = self.base / "project with spaces"
+        self.source = self.base / "reference"
+        shutil.copytree(install.ROOT, self.source, ignore=shutil.ignore_patterns("__pycache__", ".git"))
+        self.initial_version = (self.source / "VERSION").read_text(encoding="utf-8").strip()
+
+    def apply(self, **options):
+        plan = install.build_plan(self.target, root=self.source, **options)
+        self.assertTrue(install.apply_plan(plan))
+        return plan
+
+    def write(self, path, data):
+        destination = self.target / path
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_bytes(data)
+
+    def examine(self):
+        return doctor.examine(self.target, root=self.source)
+
+    def test_healthy_installation_reports_no_errors_or_warnings(self):
+        self.apply()
+        # Runtime experiment notes are private data, not managed instruction/link sources.
+        self.write("ai-kit/.metrics/private-notes.md", b"[Runtime reference](missing.md)\n\xff")
+        report = self.examine()
+        self.assertEqual(report["errors"], [])
+        self.assertEqual(report["warnings"], [])
+        self.assertEqual(report["installed_version"], self.initial_version)
+
+    def test_module_map_drift_is_reported(self):
+        self.write("go.mod", b"module example.test/app\n\ngo 1.21\n")
+        self.apply()
+        report = self.examine()
+        self.assertFalse(any("module maps differ" in warning for warning in report["warnings"]))
+        project = json.loads((self.target / "ai-kit/project.json").read_text(encoding="utf-8"))
+        project["modules"][0]["path"] = "/api/"
+        (self.target / "ai-kit/project.json").write_text(json.dumps(project) + "\n", encoding="utf-8")
+        report = self.examine()
+        self.assertTrue(any("module maps differ" in warning for warning in report["warnings"]))
+
+    def test_user_edits_to_client_settings_are_not_reported(self):
+        self.apply(agents=["claude"], extras=["guards", "session-start"])
+        path = self.target / ".claude/settings.local.json"
+        settings = json.loads(path.read_text(encoding="utf-8"))
+        settings["model"] = "user-choice"
+        path.write_text(json.dumps(settings) + "\n", encoding="utf-8")
+        self.assertEqual(self.examine()["warnings"], [])
+
+    def test_removed_managed_client_entry_is_reported(self):
+        self.apply(agents=["claude"], extras=["guards"])
+        path = self.target / ".claude/settings.local.json"
+        settings = json.loads(path.read_text(encoding="utf-8"))
+        settings["permissions"]["ask"].remove("Bash(git push *)")
+        path.write_text(json.dumps(settings) + "\n", encoding="utf-8")
+        self.assertIn("AI-KIT entry missing from .claude/settings.local.json: permissions.ask Bash(git push *)",
+                      self.examine()["warnings"])
+        path.write_text("[]\n", encoding="utf-8")
+        self.assertTrue(any(w.startswith("Cannot read AI-KIT entries") for w in self.examine()["warnings"]))
+
+    def test_missing_installation_is_reported(self):
+        self.target.mkdir(parents=True)
+        report = self.examine()
+        self.assertTrue(any("No installer state" in error for error in report["errors"]))
+
+    def test_modified_managed_file_is_reported(self):
+        self.apply()
+        self.write("ai-kit/CORE.md", b"Locally edited core\n")
+        report = self.examine()
+        self.assertTrue(any("Modified after installation" in warning and "ai-kit/CORE.md" in warning
+                            for warning in report["warnings"]))
+
+    def test_accepted_local_adaptation_is_not_a_modification_warning(self):
+        self.apply()
+        self.write("ai-kit/CORE.md", b"Reviewed semantic merge\n")
+        self.apply(accept_local=["ai-kit/CORE.md"])
+        report = self.examine()
+        self.assertEqual(report["errors"], [])
+        self.assertFalse(any("Modified after installation" in warning for warning in report["warnings"]))
+
+    def test_missing_owned_document_is_reported(self):
+        self.apply()
+        (self.target / "WIKI.md").unlink()
+        report = self.examine()
+        self.assertTrue(any("Owned project document missing: WIKI.md" in warning
+                            for warning in report["warnings"]))
+
+    def test_broken_kit_link_is_reported(self):
+        self.apply()
+        self.write("WIKI.md", b"[Missing](no-such-file.md)\n")
+        report = self.examine()
+        self.assertTrue(any("missing link" in warning and "WIKI.md" in warning
+                            for warning in report["warnings"]))
+
+    def test_stale_provider_verification_is_reported(self):
+        self.apply()
+        provider = self.target / "ai-kit" / "router" / "providers" / "openai.json"
+        config = json.loads(provider.read_text(encoding="utf-8"))
+        config["verified_documentation_date"] = "2025-01-01"
+        provider.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
+        report = self.examine()
+        self.assertTrue(any("older than" in warning for warning in report["warnings"]))
+
+    def test_main_returns_one_when_errors_exist(self):
+        with contextlib.redirect_stdout(io.StringIO()):
+            self.assertEqual(doctor.main([str(self.target), "--root", str(self.source)]), 1)
+
+    def test_main_returns_zero_for_healthy_installation(self):
+        self.apply()
+        with contextlib.redirect_stdout(io.StringIO()):
+            self.assertEqual(doctor.main([str(self.target), "--root", str(self.source)]), 0)
+
+    def test_fix_restores_missing_managed_file(self):
+        self.apply()
+        (self.target / "ai-kit/CORE.md").unlink()
+        self.assertTrue(any("Missing managed file: ai-kit/CORE.md" in error
+                            for error in self.examine()["errors"]))
+        with contextlib.redirect_stdout(io.StringIO()):
+            code = doctor.main([str(self.target), "--root", str(self.source), "--fix"])
+        self.assertEqual(code, 0)
+        self.assertTrue((self.target / "ai-kit/CORE.md").is_file())
+        self.assertEqual(self.examine()["errors"], [])
+
+    def test_fix_is_a_noop_for_a_healthy_installation(self):
+        self.apply()
+        with contextlib.redirect_stdout(io.StringIO()):
+            self.assertEqual(doctor.main([str(self.target), "--root", str(self.source), "--fix"]), 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

aikit/tests/test_evaluate.py

@@ -0,0 +1,66 @@
+"""Exercise private task packs and measurement-pair coverage reports."""
+import contextlib
+import io
+import json
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+import evaluate
+import metrics
+
+
+def record(task_id, arm, outcome="correct"):
+    return {"schema": 2, "task_id": task_id, "experiment": "trial", "arm": arm,
+            "kit_version": "test", "outcome": outcome, "checks": "passed" if outcome == "correct" else "failed",
+            "rework": 0, "escalations": 0,
+            "attempts": [{"provider": "provider", "model": "model", "effort": None,
+                          "disposition": "succeeded" if outcome == "correct" else "failed",
+                          "duration_seconds": None, "input_tokens": None, "cached_input_tokens": None,
+                          "output_tokens": None, "reasoning_tokens": None, "cost": None}]}
+
+
+class EvaluationTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory(prefix="ai-kit-evaluate-")
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name).resolve()
+        self.project = self.root / "project"
+        self.project.mkdir()
+        self.pack_path = self.root / "tasks.json"
+
+    def test_scaffold_is_preview_first_and_refuses_overwrite(self):
+        plan = evaluate.scaffold_plan(self.pack_path, "trial", ["task-01", "task-02"])
+        self.assertTrue(plan["changed"])
+        self.assertFalse(self.pack_path.exists())
+        self.assertTrue(evaluate.apply_scaffold(plan))
+        self.assertTrue(self.pack_path.is_file())
+        with self.assertRaises(ValueError):
+            evaluate.scaffold_plan(self.pack_path, "other", ["task-01"])
+
+    def test_coverage_reports_missing_baseline_kit_pairs_and_unexpected_records(self):
+        plan = evaluate.scaffold_plan(self.pack_path, "trial", ["task-01", "task-02"])
+        evaluate.apply_scaffold(plan)
+        metrics.record_task(self.project, record("task-01", "baseline"), apply=True)
+        metrics.record_task(self.project, record("task-01", "kit"), apply=True)
+        metrics.record_task(self.project, record("task-02", "baseline", "incorrect"), apply=True)
+        metrics.record_task(self.project, record("other", "kit"), apply=True)
+        report = evaluate.coverage(self.project, self.pack_path)
+        self.assertEqual(report["complete_pairs"], 1)
+        self.assertEqual(report["missing"], [{"task_id": "task-02", "arms": ["kit"]}])
+        self.assertEqual(report["unexpected_records"], [{"task_id": "other", "arm": "kit"}])
+        self.assertEqual(report["reviewed_outcomes"], {"correct": 2, "incorrect": 1, "incomplete": 0})
+
+    def test_validation_and_cli_reject_nonopaque_task_content(self):
+        with self.assertRaises(ValueError):
+            evaluate.validate_pack({"schema": 1, "experiment": "trial", "arms": ["baseline", "kit"],
+                                    "tasks": [{"task_id": "please fix payment bug"}]})
+        with contextlib.redirect_stderr(io.StringIO()):
+            self.assertEqual(evaluate.main(["scaffold", "--output", str(self.pack_path), "--experiment", "trial",
+                                             "--task", "task one"]), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()