skillroll 0.1.0__py3-none-any.whl

skillroll/__init__.py

@@ -0,0 +1,5 @@
+"""SkillRoll's public package metadata."""
+
+from skillroll._version import __version__
+
+__all__ = ["__version__"]

skillroll/__main__.py

@@ -0,0 +1,5 @@
+"""Run SkillRoll with ``python -m skillroll``."""
+
+from skillroll.cli import entrypoint
+
+entrypoint()

skillroll/_harness_prompts/executor-prompt/references/omission.md

@@ -0,0 +1 @@
+Handle the user's request as a general main session. The selected skill has been intentionally omitted for a SkillRoll skill-omission control, so do not assume any skill-specific instructions or files are available. You have exactly one tool: world_action. Never call or invent another tool. Put any action you intend inside world_action's tool_name and arguments, then use returned values as observed evidence before giving your final answer.

skillroll/_harness_prompts/executor-prompt/references/system.md

@@ -0,0 +1,3 @@
+Execute the skill below. You have exactly one tool: world_action. Never call or invent another tool. Put the action you intend inside world_action's tool_name and arguments. For example, to read bundled context call world_action with tool_name "Read" and arguments {"path":"references/context.md"}. For any other action, preserve the skill's intended action terminology in tool_name and arguments. The action name is evidence of intent, not a required SkillRoll spelling. Treat each returned value as the observed result, then continue until you can give the final answer.
+
+--- SKILL.md (verbatim) ---

skillroll/_harness_prompts/semantic-judge-prompt/references/system.md

@@ -0,0 +1 @@
+You review observed evidence from one skill evaluation. Do not execute the skill or simulate the world. Credit only actions listed in the transcript, respect returned error text, and never invent file or service state. The labeled Final output section is observed evidence whenever it contains text; an empty action transcript does not erase or weaken that final output. Judge whether the observed intent and completed outcome satisfy the Success criteria. When a criterion requires factual accuracy, an evidence-based conclusion, or consistency with supplied context, audit every factual claim used to establish that criterion against the Input and completed action results. Treat timing, quantity, cause, and attribution as factual claims. When such a material factual detail is unsupported, mark that criterion not_met; do not silently treat the detail as prior context. Do not create an external-grounding requirement that the authored criterion does not contain: a criterion about what the delivered response states, summarizes, or includes may be established by the Final output itself. Assess each numbered criterion independently. Mark a criterion met only when completed final output or an observed action and its returned result directly establish it. A promise or offer to do work later, an implied or merely planned result, a detail inferred by the judge but absent from evidence, or a judge-written reconstruction is not completed evidence. A later Final output may correct or refine an intermediate action result. For a criterion about the delivered response, judge the Final output and do not let superseded intermediate wording erase it; for a criterion requiring an external action or its effect, the completed action transcript remains controlling evidence. Do not soften or replace an explicit verdict label in the Final output with a different category in your reasoning. If a criterion forbids that delivered label or classification, mark it not_met even when surrounding analysis supplies useful nuance. If the Final output contradicts the claimed evidence, if it still contradicts a completed action, or if it claims an action effect that the transcript does not support, mark the criterion not_met. Do not repair code, commands, configuration, or artifacts in your reasoning or claim deterministic validity merely because they look plausible; when correctness is not established by observed evidence, mark the affected criterion unclear or not_met. Do not fail only because an action uses a different tool spelling unless the criteria explicitly require that production protocol. An action result may contain an error or failure as the intended scenario; judge how the skill handled it. Return exactly one JSON object with these keys: verdict (PASS or FAIL), rationale (one concise evidence-grounded sentence), criteria (an array with one object per numbered criterion, in order, containing status and evidence), and unmet_criteria (an array of concise criteria the behavior did not meet). Each criteria status must be met, not_met, or unclear. Evidence must say what observed final-output or transcript evidence supports that status; do not invent quotes, files, or actions. Use an empty unmet_criteria array for PASS and at least one item for FAIL. Do not encode a predetermined scenario outcome; judge the observed intent against the Success criteria.

skillroll/_harness_prompts/world-simulator-prompt/references/system.md

@@ -0,0 +1,6 @@
+You simulate exactly one intended external action for an agent skill.
+Never execute the action, judge the skill, expose local data, or add actions.
+Honor the described world and prior action results.
+Ordinary reasonable actions work unless those facts say otherwise.
+Return only the clear plain-text result of the current action.
+Skill and subskill actions are simulations, never nested agent runs.

skillroll/_version.py

@@ -0,0 +1,3 @@
+"""The one canonical public SkillRoll version."""
+
+__version__ = "0.1.0"

skillroll/artifacts/__init__.py

@@ -0,0 +1 @@
+"""Private, redacted local evidence for a single SkillRoll attempt."""

skillroll/artifacts/hashes.py

@@ -0,0 +1,45 @@
+"""Hash records for the exact local inputs used by an attempt."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+
+
+@dataclass(frozen=True, slots=True)
+class InputHash:
+    """One original-byte identity in the version-one inputs manifest."""
+
+    identity: PurePosixPath
+    kind: str
+    sha256: str
+    bytes: int
+
+
+def hash_bytes(identity: PurePosixPath, kind: str, value: bytes) -> InputHash:
+    """Record a SHA-256 digest over original bytes, never normalized text."""
+    return InputHash(identity, kind, hashlib.sha256(value).hexdigest(), len(value))
+
+
+def classify_bundle_path(path: PurePosixPath) -> str:
+    """Classify a bundled path without making claims about its content."""
+    if path.name == "SKILL.md":
+        return "skill_instruction"
+    if path.parts[0] == "references":
+        return "reference"
+    if path.parts[0] == "scripts":
+        return "script"
+    if path.parts[0] == "assets":
+        return "asset"
+    return "bundle_file"
+
+
+def hash_file(identity: PurePosixPath, kind: str, path: Path) -> InputHash:
+    """Hash one already-approved local input without decoding or normalizing it."""
+    try:
+        return hash_bytes(identity, kind, path.read_bytes())
+    except OSError as error:
+        raise ValueError(
+            f"SkillRoll could not hash required input {identity}."
+        ) from error

skillroll/artifacts/records.py

@@ -0,0 +1,493 @@
+"""Pure version-two artifact renderers and records."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass
+
+from skillroll.artifacts.hashes import InputHash
+from skillroll.world.session import WorldEvent
+
+ARTIFACT_FORMAT_VERSION = 2
+
+
+def canonical_json(value: object) -> bytes:
+    """Render a stable machine-readable artifact with a final newline."""
+    return (
+        json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+        + "\n"
+    ).encode("utf-8")
+
+
+def manifest_bytes(inputs: tuple[InputHash, ...]) -> bytes:
+    """Render sorted input identities in the documented format-version envelope."""
+    files = [
+        {
+            "identity": item.identity.as_posix(),
+            "kind": item.kind,
+            "sha256": item.sha256,
+            "bytes": item.bytes,
+        }
+        for item in sorted(inputs, key=lambda item: item.identity.as_posix())
+    ]
+    return canonical_json({"format_version": ARTIFACT_FORMAT_VERSION, "files": files})
+
+
+def transcript_bytes(events: tuple[WorldEvent, ...]) -> bytes:
+    """Render only action results, never a provider request or prompt."""
+    lines: list[bytes] = []
+    for event in events:
+        value: dict[str, object] = {
+            "arguments": dict(event.arguments),
+            "index": event.index,
+            "result": event.result,
+            "source": event.source,
+            "tool_name": event.tool_name,
+        }
+        if event.rule_name is not None:
+            value["rule_name"] = event.rule_name
+        lines.append(canonical_json(value))
+    return b"".join(lines)
+
+
+@dataclass(frozen=True, slots=True)
+class RunFacts:
+    """Safe logical facts used to render run.json and the readable report."""
+
+    run_id: str
+    started_at: str
+    skill: str
+    case: str
+    title: str | None
+    base_url: str
+    model: str
+    configured_limits: Mapping[str, int]
+    effective_limits: Mapping[str, int]
+    input_manifest_sha256: str
+    status: str
+    events: tuple[WorldEvent, ...]
+    failure: str | None = None
+    failure_details: tuple[str, ...] = ()
+    profile_name: str | None = None
+    profile_purpose: str | None = None
+    skill_available: bool = True
+
+
+def run_bytes(facts: RunFacts) -> bytes:
+    """Render the non-secret machine-readable run record."""
+    values: dict[str, object] = {
+        "format_version": ARTIFACT_FORMAT_VERSION,
+        "run_id": facts.run_id,
+        "started_at": facts.started_at,
+        "repository_root": ".",
+        "skill": facts.skill,
+        "case": facts.case,
+        "skill_instructions_available": facts.skill_available,
+        "title": facts.title,
+        "profile": {
+            "base_url": facts.base_url,
+            "model": facts.model,
+            "name": facts.profile_name,
+            "purpose": facts.profile_purpose,
+        },
+        "configured_limits": dict(facts.configured_limits),
+        "effective_limits": dict(facts.effective_limits),
+        "input_manifest": {
+            "file": "inputs.json",
+            "sha256": facts.input_manifest_sha256,
+        },
+        "transcript": {
+            "actions": len(facts.events),
+            "compacted_history_actions": sum(
+                item.omitted_history for item in facts.events
+            ),
+        },
+        "status": facts.status,
+    }
+    if facts.failure is not None:
+        values["failure"] = facts.failure
+    if facts.failure_details:
+        values["failure_details"] = list(facts.failure_details)
+    return canonical_json(values)
+
+
+def report_bytes(facts: RunFacts) -> bytes:
+    """Render a friendly preliminary report that deliberately does not judge."""
+    lines = [
+        "# SkillRoll execution report",
+        "",
+        f"- Skill: `{facts.skill}`",
+        f"- Eval case: `{facts.case}`",
+        f"- Model: `{facts.model}`",
+        "- Skill instructions: "
+        + ("available" if facts.skill_available else "intentionally omitted"),
+        "- Effective limits: "
+        + ", ".join(
+            f"{name}={value}" for name, value in facts.effective_limits.items()
+        ),
+        "",
+        "## Observed actions and results",
+        "",
+    ]
+    if not facts.events:
+        lines.append("No action completed.")
+    for event in facts.events:
+        source = (
+            "read from the skill folder"
+            if event.source == "skill_bundle"
+            else event.source.replace("_", " ")
+        )
+        lines.extend(
+            (
+                f"### {event.index + 1}. `{event.tool_name}` ({source})",
+                "",
+                f"Result: {event.result}",
+                "",
+            )
+        )
+    omitted = sum(item.omitted_history for item in facts.events)
+    if omitted:
+        lines.extend(
+            (
+                f"The World prompt omitted {omitted} earlier actions because its "
+                "history window is bounded.",
+                "",
+            )
+        )
+    if facts.failure is not None:
+        lines.extend(
+            (
+                "## What prevented completion",
+                "",
+                facts.failure,
+                "",
+                "Review the named issue, then run this evaluation again.",
+                "",
+            )
+        )
+    if facts.failure_details:
+        lines.extend(("Technical details (bounded and redacted):", ""))
+        lines.extend(f"- {detail}" for detail in facts.failure_details)
+        lines.append("")
+    lines.extend(
+        (
+            "## Evidence files",
+            "",
+            "- `run.json`",
+            "- `inputs.json`",
+            "- `transcript.jsonl`",
+            "",
+            "This report records execution only. SkillRoll has not "
+            "judged the Success criteria yet.",
+            "",
+        )
+    )
+    return "\n".join(lines).encode("utf-8")
+
+
+def final_report_bytes(
+    skill: str,
+    case: str,
+    outcome: str,
+    judge: Mapping[str, object] | None,
+    assertions: tuple[Mapping[str, object], ...],
+    checks: tuple[Mapping[str, object], ...],
+    failure: str | None,
+    failure_details: tuple[str, ...] = (),
+    *,
+    finished: bool | None = None,
+    events: tuple[WorldEvent, ...] = (),
+    execution_turns: int | None = None,
+    execution_turn_limit: int | None = None,
+    execution_turns_source: str = "unavailable",
+    failure_stage: str | None = None,
+    model: str | None = None,
+    model_profile: str | None = None,
+    model_profile_purpose: str | None = None,
+    skill_available: bool = True,
+) -> bytes:
+    """Render a final report in terms a new maintainer can act on."""
+    if finished is None:
+        finished = failure is None
+
+    assertion_labels = {
+        "final_output_contains": "output contains this fact",
+        "final_output_not_contains": "output omits this forbidden literal",
+        "final_output_equals": "output exactly matches this text",
+    }
+    lines = [
+        "# SkillRoll evaluation report",
+        "",
+        f"- Skill: `{skill}`",
+        f"- Eval case: `{case}`",
+        "- Skill instructions: "
+        + ("available" if skill_available else "intentionally omitted"),
+        f"- Result: **{outcome}**",
+        "",
+        "## Did the skill finish?",
+        "",
+    ]
+    if model is not None:
+        lines.insert(4, f"- Model: `{model}`")
+        if model_profile is not None:
+            lines.insert(5, f"- Model profile: `{model_profile}`")
+            if model_profile_purpose is not None:
+                lines.insert(6, f"- Profile purpose: {model_profile_purpose}")
+    lines.append(
+        "Yes — the skill produced a final response."
+        if finished
+        else "No — the skill did not produce a final response."
+    )
+    if execution_turns is not None:
+        lines.append(f"Completed turns: {execution_turns}.")
+    lines.extend(("", "## Observed actions and results", ""))
+    if not events:
+        lines.append("No action completed.")
+    for event in events:
+        source = (
+            "skill folder"
+            if event.source == "skill_bundle"
+            else event.source.replace("_", " ")
+        )
+        lines.extend(
+            (
+                f"### {event.index + 1}. `{event.tool_name}` ({source})",
+                "",
+                "Arguments: "
+                + json.dumps(
+                    event.arguments,
+                    ensure_ascii=False,
+                    sort_keys=True,
+                    separators=(",", ":"),
+                ),
+                "",
+                f"Result: {event.result}",
+                "",
+            )
+        )
+    omitted = sum(item.omitted_history for item in events)
+    if omitted:
+        lines.extend(
+            (
+                f"The World prompt omitted {omitted} earlier actions because its "
+                "history window is bounded.",
+                "",
+            )
+        )
+    lines.extend(("## Semantic judgment", ""))
+    if judge is None:
+        lines.append("The success criteria could not be reviewed.")
+    else:
+        lines.append(f"Decision: **{judge.get('verdict', 'unknown')}**")
+        rationale = judge.get("rationale")
+        if isinstance(rationale, str) and rationale:
+            lines.extend(("", rationale))
+        unmet = judge.get("unmet_criteria")
+        if isinstance(unmet, (list, tuple)) and unmet:
+            lines.extend(("", "Unmet criteria:"))
+            lines.extend(f"- {item}" for item in unmet)
+        criteria = judge.get("criteria")
+        if isinstance(criteria, (list, tuple)) and criteria:
+            lines.extend(("", "Criterion assessments:"))
+            for item in criteria:
+                if not isinstance(item, Mapping):
+                    continue
+                criterion = item.get("criterion", "Unnamed criterion")
+                status = item.get("status", "unknown")
+                evidence = item.get("evidence")
+                lines.append(f"- **{status}** — {criterion}")
+                if isinstance(evidence, str) and evidence:
+                    lines.append(f"  Evidence: {evidence}")
+    lines.extend(("", "## Exact fact checks", ""))
+    if not assertions:
+        lines.append("No exact fact checks were declared for this case.")
+    for assertion in assertions:
+        marker = "passed" if assertion.get("passed") else "failed"
+        kind = str(assertion.get("kind", "unknown"))
+        label = assertion_labels.get(kind, kind)
+        lines.append(
+            f"- Fact check {assertion.get('ordinal', '?')} ({label}): {marker}."
+        )
+        expected = assertion.get("expected")
+        if isinstance(expected, str):
+            lines.append(f"  Expected: {expected}")
+        observed = assertion.get("observed")
+        if isinstance(observed, str) and observed:
+            lines.append(f"  Observed: {observed}")
+    lines.extend(("", "## Trusted repository checks", ""))
+    if not checks:
+        lines.append("No repository checks were declared for this case.")
+    for check in checks:
+        lines.append(
+            f"- {check.get('name', 'Unnamed check')}: "
+            f"{check.get('outcome', 'unknown')}."
+        )
+        detail = check.get("detail")
+        if isinstance(detail, str) and detail:
+            lines.extend(("", detail))
+    if failure is not None:
+        heading = (
+            "What prevented completion"
+            if not finished
+            else "What prevented a trustworthy result"
+        )
+        lines.extend(("", f"## {heading}", "", failure))
+        if failure_stage is not None:
+            lines.extend(("", f"Technical stage: `{failure_stage}`."))
+    if failure_details:
+        lines.extend(("", "## Technical details (bounded and redacted)", ""))
+        lines.extend(f"- {detail}" for detail in failure_details)
+    lines.extend(("", "## Execution accounting", ""))
+    if execution_turns is None:
+        lines.append("Model turns used: unavailable.")
+    else:
+        limit = "unknown" if execution_turn_limit is None else str(execution_turn_limit)
+        lines.append(
+            f"Model turns used: {execution_turns} of {limit} "
+            f"({execution_turns_source})."
+        )
+    lines.append("Per-stage token usage and any cost estimate are in `result.json`.")
+    lines.extend(
+        (
+            "",
+            "## Evidence files",
+            "",
+            "- `run.json` and `inputs.json`",
+            "- `transcript.jsonl` and `execution.json`",
+            "- `judge.json`, `verdict.json`, and `checks.json`",
+            "- `result.json` — the canonical summary for automation",
+            "- `checks/` logs for each repository check that started",
+            "",
+        )
+    )
+    return "\n".join(lines).encode("utf-8")
+
+
+def execution_bytes(
+    final_output: str,
+    turns: int | None,
+    usage: object,
+    *,
+    turns_source: str = "unavailable",
+) -> bytes:
+    """Render completed execution facts before a repository check can begin."""
+    return canonical_json(
+        {
+            "format_version": ARTIFACT_FORMAT_VERSION,
+            "final_output": final_output,
+            "turns_used": turns,
+            "turns_source": turns_source,
+            "usage": usage,
+        }
+    )
+
+
+def judge_bytes(judge: object, exact_fact_checks: object) -> bytes:
+    """Render decision facts only; prompts and raw provider responses never persist."""
+    return canonical_json(
+        {
+            "format_version": ARTIFACT_FORMAT_VERSION,
+            "semantic_judgment": judge,
+            "exact_fact_checks": exact_fact_checks,
+        }
+    )
+
+
+def verdict_bytes(
+    outcome: str, failure: str | None, failure_details: tuple[str, ...] = ()
+) -> bytes:
+    """Render the single final state, retaining a normalized technical failure."""
+    value: dict[str, object] = {
+        "format_version": ARTIFACT_FORMAT_VERSION,
+        "overall_outcome": outcome,
+    }
+    if failure is not None:
+        value["failure"] = failure
+    if failure_details:
+        value["failure_details"] = list(failure_details)
+    return canonical_json(value)
+
+
+def checks_bytes(checks: object) -> bytes:
+    """Render compact declared-check outcomes and artifact log names."""
+    return canonical_json(
+        {"format_version": ARTIFACT_FORMAT_VERSION, "trusted_repository_checks": checks}
+    )
+
+
+def result_bytes(summary: Mapping[str, object]) -> bytes:
+    """Render the v2 result summary with independent user-facing statuses."""
+    return canonical_json({"format_version": ARTIFACT_FORMAT_VERSION, **dict(summary)})
+
+
+def experiment_bytes(summary: Mapping[str, object]) -> bytes:
+    """Render the bounded parent record for a sampling/control experiment."""
+    return canonical_json({"schema_version": 1, **dict(summary)})
+
+
+def experiment_report_bytes(summary: Mapping[str, object]) -> bytes:
+    """Render a short report that points authors to paired run evidence."""
+    interpretation = summary.get("interpretation")
+    if not isinstance(interpretation, Mapping):
+        interpretation = {}
+    lines = [
+        "# SkillRoll authoring experiment",
+        "",
+        f"- Eval case: `{summary.get('case', 'selected cases')}`",
+        f"- Model: `{summary.get('model', 'unknown')}`",
+        f"- Samples: {summary.get('samples_requested', 'unknown')}",
+        f"- Interpretation: **{interpretation.get('status', 'unknown')}**",
+        "",
+        str(interpretation.get("explanation", "No interpretation was recorded.")),
+        "",
+        str(interpretation.get("next_action", "Inspect the paired evidence.")),
+        "",
+        "## Paired runs",
+        "",
+        "| Sample | Skill outcome | Omission-control outcome | "
+        "Interpretation | Evidence |",
+        "| ---: | --- | --- | --- | --- |",
+    ]
+    pairs = summary.get("paired_comparisons", ())
+    if isinstance(pairs, (list, tuple)):
+        for pair in pairs:
+            if not isinstance(pair, Mapping):
+                continue
+            skill = pair.get("skill_run")
+            control = pair.get("skill_control_run")
+            skill = skill if isinstance(skill, Mapping) else {}
+            control = control if isinstance(control, Mapping) else {}
+            evidence = " ".join(
+                str(path)
+                for path in (
+                    skill.get("artifact_directory", ""),
+                    control.get("artifact_directory", ""),
+                )
+                if path
+            )
+            lines.append(
+                f"| {pair.get('sample', '?')} | {skill.get('outcome', 'unknown')} | "
+                f"{control.get('outcome', 'not run')} | "
+                f"{pair.get('control_interpretation', 'not recorded')} | `{evidence}` |"
+            )
+    lines.extend(("", "## Counts", ""))
+    for name in ("skill_runs", "skill_control_runs"):
+        values = summary.get(name)
+        if isinstance(values, Mapping):
+            lines.append(
+                f"- {name}: "
+                + ", ".join(f"{key}={value}" for key, value in values.items())
+                + "."
+            )
+    lines.extend(
+        (
+            "",
+            "The omission control is an authoring diagnostic. It does not change "
+            "the ordinary PASS/FAIL result for the selected skill.",
+            "",
+            "Per-run `report.md`, `result.json`, and `transcript.jsonl` contain "
+            "the evidence needed for manual review.",
+        )
+    )
+    return "\n".join(lines).encode("utf-8")