dokimi 0.1.0__py3-none-any.whl

dokimi/__init__.py

@@ -0,0 +1,41 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""Dokimi: measure whether context improves a coding agent's answers and task outcomes.
+
+The context under test is documentation, a skill, an MCP doc source or an agent
+persona prompt.
+
+The command line, `dokimi` (`python cli.py` in a source checkout), has its own
+CLI reference; this reference covers the modules behind it. They group by
+evaluation track, as `docs/architecture.md` lays out:
+
+- Static documentation track: [`ingest`][dokimi.ingest] loads Markdown,
+  [`metrics`][dokimi.metrics] scores it, [`runner`][dokimi.runner] runs a
+  benchmark spec and compares snapshots, [`gate`][dokimi.gate] applies the
+  gates, [`report`][dokimi.report] writes the result.
+- LLM evaluation track: [`personas`][dokimi.personas],
+  [`questions`][dokimi.questions], [`mcp`][dokimi.mcp] (doc-source clients),
+  [`eval`][dokimi.eval] (answering and judging),
+  [`dashboard`][dokimi.dashboard] and [`orchestrator`][dokimi.orchestrator].
+- Treatment arms: [`treatments`][dokimi.treatments], with the content they
+  inject from [`skills`][dokimi.skills] and
+  [`agent_profiles`][dokimi.agent_profiles].
+- Executable tasks: [`harnesses`][dokimi.harnesses] runs coding agents on task
+  directories; [`context`][dokimi.context] reads what their context window
+  held.
+- Content novelty: [`novelty`][dokimi.novelty]. Repositories as subjects:
+  [`repos`][dokimi.repos]. Scorecards over time: [`subjects`][dokimi.subjects].
+  Exploration quality: [`explore`][dokimi.explore]. External catalogs:
+  [`external`][dokimi.external].
+
+Shared plumbing sits in the top-level modules: [`artifacts`][dokimi.artifacts]
+(versioned artifact schemas), [`llm`][dokimi.llm] (model calls),
+[`config`][dokimi.config] and [`registry`][dokimi.registry].
+
+The installed version is `dokimi.__version__`.
+"""
+
+from dokimi._version import __version__
+
+__all__ = ["__version__"]

dokimi/__main__.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from dokimi.cli import main
+
+if __name__ == "__main__":
+    main()

dokimi/_version.py

@@ -0,0 +1,32 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""The dokimi version, single-sourced from ``project.version`` in pyproject.toml.
+
+A source checkout reads pyproject.toml directly, so the version is right even
+when the tree is used without installing it (``python cli.py``) or after a bump
+that an editable install's metadata has not caught up with. An installed wheel
+has no pyproject.toml next to the package and reads the distribution metadata.
+"""
+
+from __future__ import annotations
+
+import tomllib
+from importlib import metadata
+from pathlib import Path
+
+_PYPROJECT = Path(__file__).resolve().parent.parent / "pyproject.toml"
+
+
+def _read_version() -> str:
+    if _PYPROJECT.is_file():
+        project = tomllib.loads(_PYPROJECT.read_text(encoding="utf-8")).get("project", {})
+        if project.get("name") == "dokimi" and project.get("version"):
+            return str(project["version"])
+    try:
+        return metadata.version("dokimi")
+    except metadata.PackageNotFoundError:
+        return "0+unknown"
+
+
+__version__ = _read_version()

dokimi/agent_profiles/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""Agent persona prompts: load system prompts that shape the answering agent.
+
+These are deliberately *not* called "personas" — in this codebase ``persona``
+means a synthetic user who asks questions (``dokimi/personas``). An
+agent profile is a property of the *answering* agent (its system prompt).
+"""
+
+from .loader import AgentProfile, load_agent_profile, discover_agent_profiles
+
+__all__ = ["AgentProfile", "load_agent_profile", "discover_agent_profiles"]

dokimi/agent_profiles/loader.py

@@ -0,0 +1,101 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""Load agent persona prompts (answering-agent system prompts).
+
+An agent profile is a Markdown file with optional YAML frontmatter::
+
+    ---
+    id: concise_expert
+    name: Concise Expert
+    description: Terse, code-first senior engineer.
+    ---
+    You are a senior systems engineer. Answer tersely, lead with code, …
+
+The body (frontmatter stripped) is the system prompt. When no frontmatter is
+present the whole file is the system prompt and the id is derived from the
+filename.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List
+
+from dokimi.utils import parse_frontmatter
+
+
+@dataclass
+class AgentProfile:
+    """A loaded agent persona prompt.
+
+    Attributes:
+        id: Stable identifier (frontmatter ``id`` or the filename stem).
+        name: Human-readable name.
+        description: One-line summary of the profile.
+        system_prompt: The system prompt sent to the answering agent.
+        path: Source file path.
+    """
+
+    id: str
+    name: str
+    description: str
+    system_prompt: str
+    path: Path
+
+    def as_prompt(self, max_chars: int = 12_000) -> str:
+        """Render the prompt as it will be sent, capped at *max_chars*.
+
+        The cap mirrors :meth:`dokimi.skills.Skill.as_context` — see that
+        docstring for where the number came from (an inherited convention, not a tuned
+        value). The parity, not the figure, is what matters here: comparing the
+        system-prompt channel against the context channel is only meaningful if one
+        side cannot carry an unbounded payload the other would have had truncated.
+        Truncation is recorded by the caller rather than hidden — see
+        ``AgentProfileTreatment``.
+        """
+        prompt = self.system_prompt.strip()
+        if len(prompt) > max_chars:
+            return prompt[:max_chars].rstrip() + "\n…"
+        return prompt
+
+
+def load_agent_profile(path) -> AgentProfile:
+    """Load a single agent profile from a Markdown file.
+
+    Args:
+        path: Path to the profile ``.md`` file.
+
+    Returns:
+        An :class:`AgentProfile`.
+
+    Raises:
+        FileNotFoundError: If the file does not exist.
+        ValueError: If the profile has no system-prompt body.
+    """
+    p = Path(path)
+    if not p.is_file():
+        raise FileNotFoundError(f"Agent profile not found: {p}")
+
+    meta, body = parse_frontmatter(p.read_text(encoding="utf-8"))
+    system_prompt = body.strip()
+    if not system_prompt:
+        raise ValueError(
+            f"Agent profile '{p}' has an empty body; the body is the system prompt."
+        )
+
+    pid = str(meta.get("id") or p.stem).strip()
+    if not pid:
+        raise ValueError(f"Agent profile '{p}' has an empty 'id' after normalization.")
+    return AgentProfile(
+        id=pid,
+        name=str(meta.get("name") or pid).strip(),
+        description=str(meta.get("description") or "").strip(),
+        system_prompt=system_prompt,
+        path=p,
+    )
+
+
+def discover_agent_profiles(root) -> List[AgentProfile]:
+    """Load every ``.md`` agent profile under *root*."""
+    root = Path(root)
+    return [load_agent_profile(p) for p in sorted(root.glob("*.md"))]

dokimi/artifacts.py

@@ -0,0 +1,201 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""Versioned, schema-validated I/O for produced artifacts.
+
+The benchmark produces long-lived JSON artifact kinds: ``questions``,
+``answers``, ``eval`` (LLM-judge scores), ``arms`` (treatment-arm comparisons),
+``task_runs`` (executable task results), ``explore_runs`` (ExploreBench
+localization results), ``novelty`` (what an artifact's content adds to one
+model), ``guidance_reach`` (L1 reach probes) and ``repo_readiness`` (L0 claim
+checks) and ``tool_preference`` (which of two equal tools a model picks, and
+what moves it). This module stamps each with a ``schema_version`` and
+validates it against the JSON Schema in ``dokimi/schemas/`` on save
+and load, so fixtures and baselines stay forward-compatible and self-describing.
+
+Every artifact of a registered kind is written through :func:`save_artifact`, partial
+(incremental) saves included: a run that would write an invalid artifact fails on its first
+save, not after the last paid model call. :func:`schema_errors` is the one place a JSON Schema
+is applied, for artifacts and for validated *inputs* (manifests, profiles, specs) alike.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+SCHEMA_DIR = Path(__file__).resolve().parent / "schemas"
+
+# kind -> current schema_version string (also the schema filename stem).
+SCHEMA_VERSIONS: dict[str, str] = {
+    "questions": "questions.v2",
+    "answers": "answers.v1",
+    "eval": "eval.v1",
+    "arms": "arms.v1",
+    "matrix_rollup": "matrix_rollup.v1",
+    "plugin_delta": "plugin_delta.v1",
+    "subject_scorecard": "subject_scorecard.v2",
+    "novelty": "novelty.v1",
+    "task_runs": "task_runs.v1",
+    "explore_runs": "explore_runs.v1",
+    "guidance_reach": "guidance_reach.v1",
+    "repo_readiness": "repo_readiness.v1",
+    "tool_preference": "tool_preference.v1",
+}
+
+# kind -> superseded versions still accepted on *load*, newest first. An artifact written under an
+# older version is still a valid artifact and reading one must not fail; `stamp` always writes the
+# current version, so nothing is written back at an old version. A reader that needs the current
+# shape upgrades on read (questions.v1 `text` -> v2 `question`: dokimi.questions.artifact).
+SUPERSEDED_VERSIONS: dict[str, tuple[str, ...]] = {
+    "questions": ("questions.v1",),
+    "subject_scorecard": ("subject_scorecard.v1",),
+}
+
+
+class ArtifactValidationError(ValueError):
+    """Raised when an artifact does not conform to its schema."""
+
+
+def _schema_path(kind: str, version: str | None = None) -> Path:
+    if kind not in SCHEMA_VERSIONS:
+        raise KeyError(f"Unknown artifact kind: {kind!r}. Known: {sorted(SCHEMA_VERSIONS)}")
+    return SCHEMA_DIR / f"{version or SCHEMA_VERSIONS[kind]}.json"
+
+
+def schema_version_for(kind: str, data: dict) -> str:
+    """Which schema version ``data`` should be validated against.
+
+    The current version, unless the payload declares a superseded one for this kind — an artifact
+    written before a bump is still valid, so reading it must not fail. An unknown version string is
+    *not* honoured: it falls through to the current schema, whose ``const`` then rejects it, which
+    is how a typo or a future version stays an error instead of silently skipping validation.
+
+    An *unstamped* payload (no ``schema_version`` key: hand-written and older sets, e.g. curated
+    question files) is matched against the current version, then each superseded one, newest
+    first, and resolves to the first it satisfies; when it satisfies none, to the one it violates
+    least (the newest on a tie), so a set with one real fault reports that fault, not every
+    record as a wrong shape.
+    """
+    if "schema_version" not in data and SUPERSEDED_VERSIONS.get(kind):
+        counts = {v: len(schema_errors(v, data))
+                  for v in (SCHEMA_VERSIONS[kind], *SUPERSEDED_VERSIONS[kind])}
+        return min(counts, key=counts.__getitem__)
+    declared = data.get("schema_version")
+    if declared in SUPERSEDED_VERSIONS.get(kind, ()):
+        return str(declared)
+    return SCHEMA_VERSIONS[kind]
+
+
+def schema_errors(schema: str | Path, instance: Any) -> list[str]:
+    """Every way ``instance`` violates a JSON Schema (draft 2020-12); empty when it conforms.
+
+    ``schema`` is a schema name in :data:`SCHEMA_DIR` (``"repo_static.v1"``) or the path of a
+    schema file elsewhere. Each problem reads ``<path>: <message>``, ``<path>`` being the
+    ``/``-joined location in ``instance`` (``<root>`` for the top level), sorted by location.
+    An unreadable or malformed schema file raises ``OSError`` / ``json.JSONDecodeError``: that
+    is a packaging fault, not a problem with ``instance``.
+    """
+    from jsonschema import Draft202012Validator
+
+    path = SCHEMA_DIR / f"{schema}.json" if isinstance(schema, str) else Path(schema)
+    document = json.loads(path.read_text(encoding="utf-8"))
+    return [
+        f"{'/'.join(map(str, e.path)) or '<root>'}: {e.message}"
+        for e in sorted(Draft202012Validator(document).iter_errors(instance),
+                        key=lambda e: list(e.path))
+    ]
+
+
+def validate_artifact(kind: str, data: dict) -> None:
+    """Validate ``data`` against the schema for ``kind`` (raises on failure)."""
+    errors = schema_errors(_schema_path(kind, schema_version_for(kind, data)), data)
+    if errors:
+        details = "\n".join(f"  - {e}" for e in errors)
+        raise ArtifactValidationError(f"{kind} artifact validation failed:\n{details}")
+
+
+def stamp(kind: str, data: dict) -> dict:
+    """Return ``data`` with its ``schema_version`` set for ``kind``."""
+    if kind not in SCHEMA_VERSIONS:
+        raise KeyError(f"Unknown artifact kind: {kind!r}")
+    data["schema_version"] = SCHEMA_VERSIONS[kind]
+    return data
+
+
+def save_artifact(kind: str, data: dict, path: Path | str) -> dict:
+    """Write ``data`` to ``path`` as a ``kind`` artifact; return exactly what was written.
+
+    - ``schema_version`` is set to the current version for ``kind`` and leads the document
+      (a value already in ``data`` is replaced); ``data`` itself is not mutated.
+    - The payload is validated first. :class:`ArtifactValidationError` means nothing was
+      written at ``path`` (a file already there is untouched); the rejected payload is kept
+      at ``<name>.rejected`` so a paid run is not lost to a schema problem. A later valid
+      save to ``path`` removes that ``<name>.rejected`` (only that sibling), so a leftover
+      always belongs to the latest attempt; if it cannot be removed the save still stands and
+      a warning is logged. ``KeyError`` means ``kind`` is not registered in
+      :data:`SCHEMA_VERSIONS`.
+    - The write is atomic (a ``<name>.<pid>.<thread>.tmp`` sibling, then ``os.replace``), so a
+      reader or a crash never sees half a file, and concurrent saves to one path (threads or
+      processes) each publish one whole payload; the last one wins. A failed write removes its
+      tmp file. Missing parent directories of ``path`` are created.
+    - A symlink at ``path`` is kept: the payload replaces the file it resolves to (the tmp file
+      sits beside that target, so the rename stays on one filesystem). A dangling link creates
+      its target when the target's directory exists; otherwise ``FileNotFoundError`` is raised
+      and nothing is written (the target's directories are *not* created). A link loop raises
+      ``OSError``. ``<name>.rejected`` stays beside ``path``, not the target.
+    - JSON with ``indent=2``, ASCII-escaped, UTF-8.
+    """
+    if kind not in SCHEMA_VERSIONS:
+        raise KeyError(f"Unknown artifact kind: {kind!r}")
+    payload = {"schema_version": SCHEMA_VERSIONS[kind],
+               **{k: v for k, v in data.items() if k != "schema_version"}}
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    text = json.dumps(payload, indent=2)
+    try:
+        validate_artifact(kind, payload)
+    except ArtifactValidationError as exc:
+        rejected = path.with_name(path.name + ".rejected")
+        rejected.write_text(text, encoding="utf-8")
+        raise ArtifactValidationError(f"{exc}\n(not written to {path}; payload kept at {rejected})") from exc
+    target = _write_target(path)
+    tmp = target.with_name(f"{target.name}.{os.getpid()}.{threading.get_ident()}.tmp")
+    try:
+        tmp.write_text(text, encoding="utf-8")
+        os.replace(tmp, target)
+    except BaseException:
+        tmp.unlink(missing_ok=True)
+        raise
+    stale = path.with_name(path.name + ".rejected")
+    try:
+        stale.unlink(missing_ok=True)
+    except OSError as exc:
+        logger.warning("saved %s, but could not remove the stale rejected payload %s: %s", path, stale, exc)
+    return payload
+
+
+def _write_target(path: Path) -> Path:
+    """The file a save to ``path`` replaces: ``path`` itself, or what a symlink there resolves to."""
+    if not path.is_symlink():
+        return path
+    target = Path(os.path.realpath(path))
+    if target.is_symlink():  # realpath gives up on a loop and hands back a link
+        raise OSError(f"symlink {path} does not resolve to a file (link loop at {target})")
+    if not target.parent.is_dir():
+        raise FileNotFoundError(f"symlink {path} points into a missing directory: {target}")
+    return target
+
+
+def load_artifact(kind: str, path: Path, *, validate: bool = True) -> dict:
+    """Read and (optionally) validate an artifact from ``path``."""
+    data = json.loads(Path(path).read_text(encoding="utf-8"))
+    if validate:
+        validate_artifact(kind, data)
+    return data