akernel-runtime 0.1.0__py3-none-any.whl

context_kernel/context.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+from typing import Any
+
+from .budget import allocate_budget
+from .memory import MemoryStore
+from .policy import summarize_command_policy
+from .project import compact_project_profile, load_project_profile
+from .skills import SkillRegistry
+from .storage import Workspace
+from .tasks import TaskStore
+from .tokenizer import estimate_tokens
+
+
+class ContextBuilder:
+    def __init__(self, workspace: Workspace):
+        self.workspace = workspace
+        self.memory = MemoryStore(workspace)
+        self.skills = SkillRegistry(workspace)
+
+    def build(
+        self,
+        request: str,
+        total_budget: int | None,
+        profile: str = "balanced",
+        task_id: str | None = None,
+        resume: bool = False,
+    ) -> dict[str, Any]:
+        budget = allocate_budget(request, total_budget, profile)
+        config = self.workspace.load_config()
+        runtime_instructions = config.get("runtime_instructions", [])
+        task_brief = TaskStore(self.workspace).brief(task_id) if task_id and resume else None
+
+        selected_memory = self.memory.search(request, limit=6, budget_tokens=budget.memory)
+        selected_skills = self.skills.select(request, budget_tokens=budget.skills)
+
+        packet = {
+            "request": request,
+            "runtime": {
+                "instructions": runtime_instructions,
+                "budget_policy": "Load the smallest useful context packet. Escalate skill levels only when needed.",
+                "command_policy": summarize_command_policy(self.workspace),
+                "project": compact_project_profile(load_project_profile(self.workspace)),
+            },
+            "task": {
+                "resume": bool(task_brief),
+                "brief": task_brief,
+            },
+            "memory": [
+                {
+                    "record": item.record.to_dict(),
+                    "score": item.score,
+                    "reason": item.reason,
+                    "matched_terms": item.matched_terms,
+                    "estimated_tokens": estimate_tokens(item.record.to_dict()),
+                }
+                for item in selected_memory
+            ],
+            "skills": [
+                {
+                    "level": item.level,
+                    "score": item.score,
+                    "reason": item.reason,
+                    "matched_terms": item.matched_terms,
+                    "estimated_tokens": estimate_tokens(item.skill.render_level(item.level)),
+                    "contract": item.skill.render_level(item.level),
+                }
+                for item in selected_skills
+            ],
+        }
+
+        used = estimate_tokens(packet)
+        packet["budget"] = {
+            "profile": budget.profile,
+            "total": budget.total,
+            "allocated": {
+                "request": budget.request,
+                "runtime": budget.runtime,
+                "memory": budget.memory,
+                "skills": budget.skills,
+                "reserve": budget.reserve,
+            },
+            "estimated_used": used,
+            "estimated_remaining": max(0, budget.total - used),
+            "over_budget": used > budget.total,
+        }
+        packet["omissions"] = self._omissions(packet)
+        return packet
+
+    def build_baseline(self, request: str) -> dict[str, Any]:
+        config = self.workspace.load_config()
+        packet = {
+            "request": request,
+            "runtime": {
+                "instructions": config.get("runtime_instructions", []),
+                "budget_policy": "Naive baseline loads all memory and full skill procedures.",
+                "command_policy": summarize_command_policy(self.workspace),
+                "project": compact_project_profile(load_project_profile(self.workspace)),
+            },
+            "memory": [record.to_dict() for record in self.memory.all()],
+            "skills": [skill.render_level("l3") for skill in self.skills.all()],
+        }
+        packet["budget"] = {
+            "estimated_used": estimate_tokens(packet),
+            "memory_count": len(packet["memory"]),
+            "skill_count": len(packet["skills"]),
+            "skill_level": "l3",
+        }
+        return packet
+
+    def compare(
+        self,
+        request: str,
+        total_budget: int | None,
+        profile: str = "balanced",
+        task_id: str | None = None,
+        resume: bool = False,
+    ) -> dict[str, Any]:
+        kernel = self.build(request, total_budget, profile, task_id=task_id, resume=resume)
+        baseline = self.build_baseline(request)
+        kernel_tokens = kernel["budget"]["estimated_used"]
+        baseline_tokens = baseline["budget"]["estimated_used"]
+        savings = max(0, baseline_tokens - kernel_tokens)
+        savings_ratio = savings / baseline_tokens if baseline_tokens else 0.0
+        return {
+            "request": request,
+            "budget": kernel["budget"]["total"],
+            "profile": kernel["budget"]["profile"],
+            "kernel": {
+                "estimated_tokens": kernel_tokens,
+                "selected_memory": len(kernel["memory"]),
+                "selected_skills": len(kernel["skills"]),
+                "over_budget": kernel["budget"]["over_budget"],
+                "packet": kernel,
+            },
+            "baseline": {
+                "estimated_tokens": baseline_tokens,
+                "loaded_memory": baseline["budget"]["memory_count"],
+                "loaded_skills": baseline["budget"]["skill_count"],
+                "skill_level": baseline["budget"]["skill_level"],
+                "packet": baseline,
+            },
+            "savings": {
+                "estimated_tokens": savings,
+                "ratio": round(savings_ratio, 4),
+                "percent": round(savings_ratio * 100, 2),
+            },
+        }
+
+    @staticmethod
+    def _omissions(packet: dict[str, Any]) -> list[str]:
+        omissions: list[str] = []
+        if packet.get("task", {}).get("resume") and not packet.get("task", {}).get("brief"):
+            omissions.append("Task resume was requested but no task brief was loaded.")
+        if not packet["memory"]:
+            omissions.append("No relevant memory matched the request.")
+        if not packet["skills"]:
+            omissions.append("No skill contract matched the request.")
+        if packet["budget"]["over_budget"]:
+            omissions.append("Context packet exceeded the requested budget estimate.")
+        return omissions

context_kernel/evals.py

@@ -0,0 +1,347 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+from .budget import DEFAULT_PROFILE
+from .context import ContextBuilder
+from .models import utc_now
+from .providers import get_provider
+from .report_costs import build_eval_cost_report, diff_cost_reports
+from .storage import Workspace
+from .verifier import combine_verifications, verify_preflight, verify_response
+
+
+REGRESSION_TOKEN_TOLERANCE = 10
+
+
+class EvalRunner:
+    def __init__(self, workspace: Workspace):
+        self.workspace = workspace
+        self.context = ContextBuilder(workspace)
+
+    def run_fixture(
+        self,
+        path: Path,
+        default_budget: int | None = None,
+        default_profile: str = DEFAULT_PROFILE,
+        save: bool = True,
+        execute_provider: str | None = None,
+        execute_model: str | None = None,
+        execute_base_url: str | None = None,
+    ) -> dict[str, Any]:
+        fixture = Workspace.read_json(path)
+        tasks = fixture.get("tasks", [])
+        if not isinstance(tasks, list) or not tasks:
+            raise ValueError("Eval fixture must contain a non-empty `tasks` list.")
+
+        reports = [
+            self._run_task(
+                task,
+                default_budget=default_budget,
+                default_profile=default_profile,
+                execute_provider=execute_provider,
+                execute_model=execute_model,
+                execute_base_url=execute_base_url,
+            )
+            for task in tasks
+        ]
+        report = {
+            "id": uuid4().hex[:12],
+            "created_at": utc_now(),
+            "fixture": str(path),
+            "name": fixture.get("name", path.stem),
+            "execution": {
+                "enabled": bool(execute_provider),
+                "provider": execute_provider,
+                "model": execute_model,
+            },
+            "tasks": reports,
+            "summary": summarize_reports(reports),
+        }
+        if save:
+            self.save_report(report)
+        return report
+
+    def save_report(self, report: dict[str, Any]) -> Path:
+        self.workspace.evals_dir.mkdir(parents=True, exist_ok=True)
+        path = self.workspace.evals_dir / f"{report['id']}.json"
+        Workspace.write_json(path, report)
+        return path
+
+    def list_reports(self) -> list[dict[str, Any]]:
+        self.workspace.evals_dir.mkdir(parents=True, exist_ok=True)
+        reports: list[dict[str, Any]] = []
+        for path in sorted(self.workspace.evals_dir.glob("*.json")):
+            report = Workspace.read_json(path)
+            summary = report.get("summary", {})
+            reports.append(
+                {
+                    "id": report.get("id", path.stem),
+                    "created_at": report.get("created_at", ""),
+                    "name": report.get("name", ""),
+                    "fixture": report.get("fixture", ""),
+                    "task_count": summary.get("task_count", 0),
+                    "average_savings_percent": summary.get("average_savings_percent", 0),
+                    "checks": f"{summary.get('passed_checks', 0)}/{summary.get('total_checks', 0)}",
+                    "ok": summary.get("ok", False),
+                }
+            )
+        return sorted(reports, key=lambda item: item["created_at"], reverse=True)
+
+    def get_report(self, report_id: str) -> dict[str, Any]:
+        path = self.workspace.evals_dir / f"{report_id}.json"
+        if not path.exists():
+            raise KeyError(f"Unknown eval report: {report_id}")
+        return Workspace.read_json(path)
+
+    def diff_reports(self, before_id: str, after_id: str) -> dict[str, Any]:
+        before = self.get_report(before_id)
+        after = self.get_report(after_id)
+        return diff_reports(before, after)
+
+    def _run_task(
+        self,
+        task: dict[str, Any],
+        default_budget: int | None,
+        default_profile: str,
+        execute_provider: str | None,
+        execute_model: str | None,
+        execute_base_url: str | None,
+    ) -> dict[str, Any]:
+        request = task.get("request")
+        if not request:
+            raise ValueError("Eval task is missing required field: request")
+
+        comparison = self.context.compare(
+            str(request),
+            total_budget=task.get("budget", default_budget),
+            profile=str(task.get("profile", default_profile)),
+        )
+        execution = execute_task(comparison, execute_provider, execute_model, execute_base_url)
+        checks = evaluate_checks(comparison, task, execution)
+        report = {
+            "id": str(task.get("id", request)),
+            "request": str(request),
+            "profile": comparison["profile"],
+            "budget": comparison["budget"],
+            "kernel": {
+                "estimated_tokens": comparison["kernel"]["estimated_tokens"],
+                "selected_memory": comparison["kernel"]["selected_memory"],
+                "selected_skills": comparison["kernel"]["selected_skills"],
+            },
+            "baseline": {
+                "estimated_tokens": comparison["baseline"]["estimated_tokens"],
+                "loaded_memory": comparison["baseline"]["loaded_memory"],
+                "loaded_skills": comparison["baseline"]["loaded_skills"],
+            },
+            "savings": comparison["savings"],
+            "checks": checks,
+        }
+        if execution:
+            report["execution"] = execution
+        return report
+
+
+def execute_task(
+    comparison: dict[str, Any],
+    provider_name: str | None,
+    model: str | None,
+    base_url: str | None,
+) -> dict[str, Any] | None:
+    if not provider_name:
+        return None
+    packet = comparison["kernel"]["packet"]
+    preflight = verify_preflight(packet)
+    if not preflight["ok"]:
+        return {
+            "provider": provider_name,
+            "model": model,
+            "blocked": True,
+            "block_reason": "preflight_failed",
+            "response": "",
+            "input_tokens": 0,
+            "output_tokens": 0,
+            "total_tokens": 0,
+            "verifier": preflight,
+        }
+    provider = get_provider(provider_name, model=model, base_url=base_url)
+    response = provider.run(packet)
+    response_verifier = verify_response(response.text)
+    return {
+        "provider": provider.name,
+        "model": getattr(provider, "model", model),
+        "blocked": False,
+        "response": response.text,
+        "input_tokens": response.input_tokens,
+        "output_tokens": response.output_tokens,
+        "total_tokens": response.input_tokens + response.output_tokens,
+        "verifier": combine_verifications("eval_execution", preflight, response_verifier),
+    }
+
+
+def evaluate_checks(
+    comparison: dict[str, Any],
+    task: dict[str, Any],
+    execution: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    packet = comparison["kernel"]["packet"]
+    selected_skill_ids = [item["contract"]["id"] for item in packet["skills"]]
+    selected_memory_text = " ".join(item["record"]["text"].lower() for item in packet["memory"])
+
+    checks: list[dict[str, Any]] = []
+    for skill_id in task.get("expected_skills", []):
+        checks.append(
+            {
+                "name": f"expected_skill:{skill_id}",
+                "passed": skill_id in selected_skill_ids,
+            }
+        )
+    for term in task.get("expected_memory_terms", []):
+        checks.append(
+            {
+                "name": f"expected_memory_term:{term}",
+                "passed": str(term).lower() in selected_memory_text,
+            }
+        )
+    if execution:
+        response_text = execution.get("response", "").lower()
+        for term in task.get("expected_response_terms", []):
+            checks.append(
+                {
+                    "name": f"expected_response_term:{term}",
+                    "passed": str(term).lower() in response_text,
+                }
+            )
+    minimum_savings = task.get("minimum_savings_percent")
+    if minimum_savings is not None:
+        checks.append(
+            {
+                "name": f"minimum_savings_percent:{minimum_savings}",
+                "passed": comparison["savings"]["percent"] >= float(minimum_savings),
+            }
+        )
+
+    passed = sum(1 for check in checks if check["passed"])
+    return {
+        "passed": passed,
+        "total": len(checks),
+        "ok": passed == len(checks),
+        "items": checks,
+    }
+
+
+def summarize_reports(reports: list[dict[str, Any]]) -> dict[str, Any]:
+    total_kernel = sum(report["kernel"]["estimated_tokens"] for report in reports)
+    total_baseline = sum(report["baseline"]["estimated_tokens"] for report in reports)
+    total_savings = max(0, total_baseline - total_kernel)
+    total_checks = sum(report["checks"]["total"] for report in reports)
+    passed_checks = sum(report["checks"]["passed"] for report in reports)
+    total_execution_tokens = sum(report.get("execution", {}).get("total_tokens", 0) for report in reports)
+    executed_tasks = sum(1 for report in reports if report.get("execution") and not report["execution"].get("blocked"))
+    blocked_tasks = sum(1 for report in reports if report.get("execution", {}).get("blocked"))
+    average_savings = (
+        sum(report["savings"]["percent"] for report in reports) / len(reports)
+        if reports
+        else 0.0
+    )
+    return {
+        "task_count": len(reports),
+        "total_kernel_tokens": total_kernel,
+        "total_baseline_tokens": total_baseline,
+        "total_savings_tokens": total_savings,
+        "total_savings_percent": round((total_savings / total_baseline) * 100, 2) if total_baseline else 0.0,
+        "average_savings_percent": round(average_savings, 2),
+        "passed_checks": passed_checks,
+        "total_checks": total_checks,
+        "executed_tasks": executed_tasks,
+        "blocked_tasks": blocked_tasks,
+        "total_execution_tokens": total_execution_tokens,
+        "ok": passed_checks == total_checks,
+    }
+
+
+def diff_reports(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]:
+    before_summary = before.get("summary", {})
+    after_summary = after.get("summary", {})
+    task_diffs = diff_tasks(before.get("tasks", []), after.get("tasks", []))
+    cost_diff = diff_cost_reports(build_eval_cost_report(before), build_eval_cost_report(after))
+    regressions = [
+        item
+        for item in task_diffs
+        if item["status"] == "changed"
+        and (
+            item["kernel_token_delta"] > REGRESSION_TOKEN_TOLERANCE
+            or item["savings_percent_delta"] < 0
+            or item["passed_check_delta"] < 0
+        )
+    ]
+    return {
+        "before": {
+            "id": before.get("id"),
+            "created_at": before.get("created_at"),
+            "name": before.get("name"),
+        },
+        "after": {
+            "id": after.get("id"),
+            "created_at": after.get("created_at"),
+            "name": after.get("name"),
+        },
+        "summary_delta": {
+            "kernel_tokens": delta(before_summary, after_summary, "total_kernel_tokens"),
+            "baseline_tokens": delta(before_summary, after_summary, "total_baseline_tokens"),
+            "savings_tokens": delta(before_summary, after_summary, "total_savings_tokens"),
+            "savings_percent": round(delta(before_summary, after_summary, "total_savings_percent"), 2),
+            "passed_checks": delta(before_summary, after_summary, "passed_checks"),
+            "total_checks": delta(before_summary, after_summary, "total_checks"),
+        },
+        "cost_diff": cost_diff,
+        "cost_regressions": cost_diff["regressions"],
+        "tasks": task_diffs,
+        "regressions": regressions,
+        "ok": not regressions and cost_diff["ok"],
+    }
+
+
+def diff_tasks(before_tasks: list[dict[str, Any]], after_tasks: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    before_by_id = {task["id"]: task for task in before_tasks}
+    after_by_id = {task["id"]: task for task in after_tasks}
+    task_ids = sorted(set(before_by_id).union(after_by_id))
+    diffs: list[dict[str, Any]] = []
+    for task_id in task_ids:
+        before = before_by_id.get(task_id)
+        after = after_by_id.get(task_id)
+        if before is None:
+            diffs.append({"id": task_id, "status": "added"})
+            continue
+        if after is None:
+            diffs.append({"id": task_id, "status": "removed"})
+            continue
+        diffs.append(
+            {
+                "id": task_id,
+                "status": "changed",
+                "kernel_token_delta": after["kernel"]["estimated_tokens"] - before["kernel"]["estimated_tokens"],
+                "baseline_token_delta": after["baseline"]["estimated_tokens"] - before["baseline"]["estimated_tokens"],
+                "savings_percent_delta": round(after["savings"]["percent"] - before["savings"]["percent"], 2),
+                "passed_check_delta": after["checks"]["passed"] - before["checks"]["passed"],
+                "total_check_delta": after["checks"]["total"] - before["checks"]["total"],
+                "before": task_snapshot(before),
+                "after": task_snapshot(after),
+            }
+        )
+    return diffs
+
+
+def task_snapshot(task: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "kernel_tokens": task["kernel"]["estimated_tokens"],
+        "baseline_tokens": task["baseline"]["estimated_tokens"],
+        "savings_percent": task["savings"]["percent"],
+        "checks": f"{task['checks']['passed']}/{task['checks']['total']}",
+    }
+
+
+def delta(before: dict[str, Any], after: dict[str, Any], key: str) -> int | float:
+    return after.get(key, 0) - before.get(key, 0)

context_kernel/global_memory.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from .memory import MemoryStore
+from .storage import Workspace
+
+
+def global_workspace(root: Path | None = None) -> Workspace:
+    workspace = Workspace(root or Path.home() / ".context-kernel" / "global")
+    workspace.init()
+    return workspace
+
+
+def push_global_memories(
+    workspace: Workspace,
+    *,
+    kind: str | None = None,
+    namespace: str | None = None,
+    tag: str | None = None,
+    dry_run: bool = False,
+    global_root: Path | None = None,
+) -> dict[str, Any]:
+    source = MemoryStore(workspace)
+    global_ws = global_workspace(global_root)
+    target = MemoryStore(global_ws)
+    pushed = []
+    project_name = workspace.root.name
+    sync_namespace = normalize_namespace(namespace or project_name)
+    for record in filter_records(source.all(kind=kind), tag=tag):
+        tags = sorted(
+            set(record.tags).union(
+                {
+                    "global",
+                    f"namespace:{sync_namespace}",
+                    f"source:{project_name}",
+                    f"source_project:{project_name}",
+                    f"source_root:{workspace.root.as_posix()}",
+                }
+            )
+        )
+        if dry_run:
+            copied = record.to_dict()
+            copied["tags"] = tags
+        else:
+            copied = target.add(record.kind, record.text, tags=tags).to_dict()
+        pushed.append(copied)
+    return {
+        "direction": "push",
+        "dry_run": dry_run,
+        "namespace": sync_namespace,
+        "source": str(workspace.root),
+        "target": str(global_ws.root),
+        "count": 0 if dry_run else len(pushed),
+        "candidate_count": len(pushed),
+        "records": pushed,
+    }
+
+
+def pull_global_memories(
+    workspace: Workspace,
+    *,
+    kind: str | None = None,
+    namespace: str | None = None,
+    source_project: str | None = None,
+    tag: str | None = None,
+    limit: int | None = None,
+    dry_run: bool = False,
+    global_root: Path | None = None,
+) -> dict[str, Any]:
+    global_ws = global_workspace(global_root)
+    source = MemoryStore(global_ws)
+    target = MemoryStore(workspace)
+    records = filter_records(
+        source.all(kind=kind),
+        namespace=namespace,
+        source_project=source_project,
+        tag=tag,
+    )
+    if limit is not None:
+        records = records[: max(0, limit)]
+    pulled = []
+    for record in records:
+        tags = sorted(set(record.tags).union({"global", "imported_global"}))
+        if dry_run:
+            copied = record.to_dict()
+            copied["tags"] = tags
+        else:
+            copied = target.add(record.kind, record.text, tags=tags).to_dict()
+        pulled.append(copied)
+    return {
+        "direction": "pull",
+        "dry_run": dry_run,
+        "namespace": normalize_namespace(namespace) if namespace else None,
+        "source_project": source_project,
+        "source": str(global_ws.root),
+        "target": str(workspace.root),
+        "count": 0 if dry_run else len(pulled),
+        "candidate_count": len(pulled),
+        "records": pulled,
+    }
+
+
+def filter_records(
+    records: list[Any],
+    *,
+    namespace: str | None = None,
+    source_project: str | None = None,
+    tag: str | None = None,
+) -> list[Any]:
+    filtered = records
+    if namespace:
+        namespace_tag = f"namespace:{normalize_namespace(namespace)}"
+        filtered = [record for record in filtered if namespace_tag in record.tags]
+    if source_project:
+        source_tags = {f"source_project:{source_project}", f"source:{source_project}"}
+        filtered = [record for record in filtered if source_tags.intersection(record.tags)]
+    if tag:
+        filtered = [record for record in filtered if tag in record.tags]
+    return filtered
+
+
+def normalize_namespace(value: str) -> str:
+    normalized = "-".join(str(value).strip().split()).lower()
+    return normalized or "default"