procesio-cli 0.1.1__py3-none-any.whl

procesio_cli/skills/tests/test_skill_portfolio_review.py

@@ -0,0 +1,70 @@
+"""Static instruction contracts, not evidence of model behavioral improvement."""
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def text(skill, resource="SKILL.md"):
+    return " ".join((ROOT / "skills" / skill / resource).read_text(encoding="utf-8").split())
+
+
+@pytest.mark.parametrize("skill", ["procesio-cli", "sql-server-optimizer"])
+def test_skill_authoring_handoff_has_one_owner(skill):
+    body = text(skill).split("## Boundary", 1)[1].split("## ", 1)[0]
+    assert "`agent-skill-engineer`" in body
+    assert "`procesio-cli-maintainer`" in body
+
+
+def test_operational_verification_does_not_grant_execution_permission():
+    body = text("procesio-cli")
+    contract = text("procesio-cli", "references/operation-contract.md")
+    assert "Verification is not additional execution permission" in body
+    assert "CLI and MCP" in contract
+    assert "explicit operator approval" in contract
+    assert "Run, test, submit, enable, and trigger" in contract
+    assert "Unknown outcome" in contract
+
+
+def test_credential_recovery_does_not_restore_exposed_keys_or_probe_forbidden_targets():
+    contract = text("procesio-cli", "references/credentials-admin.md")
+    assert "Never restore a disclosed or revoked credential" in contract
+    assert "preserve the incident" in contract
+    assert "Never probe a forbidden workspace" in contract
+    assert "separately approved" in contract
+
+
+def test_sql_actual_plans_and_deployment_require_bounded_approval():
+    body = text("sql-server-optimizer")
+    context = text("sql-server-optimizer", "references/db-context.md")
+    assert "Tuning permission is not deployment permission" in body
+    assert "An actual execution plan executes the statement" in context
+    assert "ROLLBACK" in context
+    assert "unknown outcome" in context
+    assert "explicit approval" in context
+
+
+def test_advisor_distinguishes_occupied_capacity_from_ee_count():
+    sizing = text("procesio-platform-advisor", "references/sizing-method.md")
+    assert "mean_occupied_seconds" in sizing
+    assert "verified_slots_per_EE" in sizing
+    assert "steady-state" in sizing
+    assert "not a peak or SLA guarantee" in sizing
+    assert "do not invent an EE count" in sizing
+
+
+def test_advisor_missing_sources_stays_provisional_and_non_mutating():
+    body = text("procesio-platform-advisor")
+    assert "If current sources are unavailable" in body
+    assert "provisional" in body
+    assert "separately approved operational handoff" in body
+
+
+def test_maintainer_covers_secret_bearing_reads_and_scoped_release_evidence():
+    contract = text("procesio-cli-maintainer", "references/change-contract.md")
+    integration = text("procesio-cli-maintainer", "references/skill-authoring.md")
+    assert "secret-bearing reads" in contract
+    assert "redacted projection" in contract
+    assert "reference or helper" in integration
+    assert "evaluated package fingerprint" in integration

procesio_cli/skills/tests/test_skill_release_binding.py

@@ -0,0 +1,204 @@
+"""Release claims must bind to bytes, not merely green ledger labels."""
+from __future__ import annotations
+
+import hashlib
+import subprocess
+import sys
+from pathlib import Path, PurePosixPath
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from tools._lib.skill_release import check_release_binding
+
+
+def git(root, *args):
+    return subprocess.check_output(["git", "-C", str(root), *args])
+
+
+@pytest.fixture
+def snapshot(tmp_path):
+    git(tmp_path, "init", "-q")
+    package = tmp_path / "skills" / "demo"
+    (package / "references").mkdir(parents=True)
+    (tmp_path / "skills" / "evals").mkdir()
+    for name, value in {
+        "demo/SKILL.md": "---\nname: demo\n---\n# Demo\n",
+        "demo/references/rule.md": "Original rule\n",
+        "evals/behavioral.json": "{}\n",
+        "evals/gate5-thresholds.json": "{}\n",
+        "evals/gates.json": "{}\n",
+    }.items():
+        # write_bytes, not write_text: on Windows write_text translates \n to
+        # \r\n, but git (autocrlf) stores an LF blob, so the working tree would
+        # then diverge from the committed bytes skill_release digests. The Linux
+        # author's write_text already yields LF; this keeps parity cross-platform.
+        (tmp_path / "skills" / name).write_bytes(value.encode("utf-8"))
+    git(tmp_path, "add", "skills")
+    git(tmp_path, "-c", "user.name=Test", "-c", "user.email=test@example.com",
+        "commit", "-qm", "fixture")
+    commit = git(tmp_path, "rev-parse", "HEAD").decode().strip()
+    # Order by PurePosixPath, matching skill_release._digest (and the Linux
+    # series launcher that produced the committed fingerprints). sorted(Path.rglob)
+    # is platform-dependent: WindowsPath sorts case-insensitively, so a mixed-case
+    # name like SKILL.md lands in a different slot and the digest diverges from the
+    # canonical one on Windows.
+    contents = {p.relative_to(tmp_path / "skills").as_posix(): p.read_bytes()
+                for p in (tmp_path / "skills").rglob("*") if p.is_file()}
+    digest = hashlib.sha256()
+    for name in sorted(contents, key=PurePosixPath):
+        digest.update(name.encode("utf-8") + b"\0" + contents[name] + b"\0")
+    return tmp_path, commit, digest.hexdigest()
+
+
+def check(snapshot):
+    root, commit, fingerprint = snapshot
+    return check_release_binding(root, root / "skills", commit, fingerprint)
+
+
+def test_crlf_working_tree_matches_an_lf_fingerprint(snapshot):
+    # SVN materialises CRLF where the fingerprint was recorded over LF bytes; the
+    # gate normalises newlines, so an SVN checkout is not read as tampering.
+    root, commit, fingerprint = snapshot
+    for p in (root / "skills").rglob("*"):
+        if p.is_file() and p.suffix in {".md", ".json"}:
+            p.write_bytes(p.read_bytes().replace(b"\n", b"\r\n"))
+    assert check_release_binding(root, root / "skills", commit, fingerprint) == []
+
+
+def test_matching_package_and_full_snapshot_pass(snapshot):
+    assert check(snapshot) == []
+
+
+@pytest.mark.parametrize("relative", ["demo/SKILL.md", "demo/references/rule.md",
+                                      "evals/behavioral.json", "evals/gate5-thresholds.json"])
+def test_dirty_loaded_or_experiment_bytes_fail(snapshot, relative):
+    (snapshot[0] / "skills" / relative).write_text("changed", encoding="utf-8")
+    assert "release-content-drift" in {row["code"] for row in check(snapshot)}
+
+
+def test_added_and_deleted_package_resources_fail(snapshot):
+    root = snapshot[0] / "skills" / "demo"
+    (root / "references" / "rule.md").unlink()
+    (root / "references" / "new.md").write_text("new")
+    assert "release-content-drift" in {row["code"] for row in check(snapshot)}
+
+
+def test_new_package_cannot_hide_outside_evaluated_inventory(snapshot):
+    package = snapshot[0] / "skills" / "another"
+    package.mkdir()
+    (package / "SKILL.md").write_text("another")
+    assert "release-content-drift" in {row["code"] for row in check(snapshot)}
+
+
+def test_ledger_update_does_not_poison_loaded_package_binding(snapshot):
+    root = snapshot[0] / "skills"
+    (root / "evals" / "gates.json").write_text('{"new_evidence": true}')
+    assert check(snapshot) == []
+
+
+@pytest.mark.parametrize("relative", ["scripts/payload.pyc", "__pycache__/generated.pyc"])
+def test_release_requires_a_clean_source_package_without_bytecode(snapshot, relative):
+    path = snapshot[0] / "skills" / "demo" / relative
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(b"potentially executable bytes")
+    assert "unsafe-release-content" in {row["code"] for row in check(snapshot)}
+
+
+def test_tree_object_is_not_commit_lineage(snapshot):
+    root, _, fingerprint = snapshot
+    tree = git(root, "rev-parse", "HEAD^{tree}").decode().strip()
+    result = check_release_binding(root, root / "skills", tree, fingerprint)
+    assert "unavailable-release-snapshot" in {row["code"] for row in result}
+
+
+def test_wrong_historical_full_tree_hash_fails(snapshot):
+    root, commit, _ = snapshot
+    result = check_release_binding(root, root / "skills", commit, "0" * 64)
+    assert "release-snapshot-mismatch" in {row["code"] for row in result}
+
+
+@pytest.mark.parametrize("commit,fingerprint", [(None, None), ("HEAD", "a" * 64),
+                                               ("a" * 40, "bad"), (True, False)])
+def test_malformed_binding_fails_closed(snapshot, commit, fingerprint):
+    root = snapshot[0]
+    result = check_release_binding(root, root / "skills", commit, fingerprint)
+    assert "invalid-release-binding" in {row["code"] for row in result}
+
+
+def test_symlink_resource_is_rejected_without_reading_target(snapshot):
+    root = snapshot[0]
+    outside = root / "outside.txt"
+    outside.write_text("must not load")
+    link = root / "skills" / "demo" / "references" / "escape.md"
+    try:
+        link.symlink_to(outside)
+    except OSError:
+        pytest.skip("symlink creation unavailable")
+    assert "unsafe-release-content" in {row["code"] for row in check(snapshot)}
+
+
+def _ledger_check(snapshot, mutate=lambda ledger: None):
+    import json
+    import runpy
+
+    root, commit, fingerprint = snapshot
+    ledger = {"release_eligible": True, "release_blockers": [],
+              "gates": [{"id": value, "status": "passed"} for value in range(7)]}
+    ledger["gates"][5].update(evaluated_candidate_commit=commit,
+                             evaluated_candidate_fingerprint=fingerprint)
+    mutate(ledger)
+    path = root / "ledger.json"
+    path.write_text(json.dumps(ledger), encoding="utf-8")
+    check_status = runpy.run_path(str(ROOT / "scripts" / "check-skill-governance.py"))["check_status"]
+    return check_status(path, repo_root=root, skills_root=root / "skills")
+
+
+def test_green_ledger_automatically_checks_real_content(snapshot):
+    assert _ledger_check(snapshot) == []
+    (snapshot[0] / "skills" / "demo" / "SKILL.md").write_text("changed")
+    assert "release-content-drift" in {row["code"] for row in _ledger_check(snapshot)}
+
+
+@pytest.mark.parametrize("value", ["false", 0, [], None])
+def test_ledger_requires_real_boolean(snapshot, value):
+    findings = _ledger_check(snapshot, lambda doc: doc.update(release_eligible=value))
+    assert "invalid-gate-status" in {row["code"] for row in findings}
+
+
+@pytest.mark.parametrize("value", [False, [], "0"])
+def test_ledger_rejects_non_integer_gate_ids_without_crashing(snapshot, value):
+    findings = _ledger_check(snapshot, lambda doc: doc["gates"][0].update(id=value))
+    assert "incomplete-gate-set" in {row["code"] for row in findings}
+
+
+@pytest.mark.parametrize("field,value,code", [
+    ("evidence", "not-a-list", "invalid-gate-evidence"),
+    ("evidence", ["../outside"], "unsafe-gate-evidence"),
+    ("status", [], "invalid-gate-state"),
+])
+def test_malformed_evidence_and_status_fail_closed(snapshot, field, value, code):
+    findings = _ledger_check(snapshot, lambda doc: doc["gates"][0].update({field: value}))
+    assert code in {row["code"] for row in findings}
+
+
+@pytest.mark.parametrize("evidence", ["nul\x00path", "x" * 5000], ids=["nul", "overlong"])
+def test_filesystem_metadata_errors_stay_structured(snapshot, evidence):
+    findings = _ledger_check(snapshot, lambda doc: doc["gates"][0].update(evidence=[evidence]))
+    assert "unsafe-gate-evidence" in {row["code"] for row in findings}
+
+
+def test_symlink_loop_resolution_error_stays_structured(snapshot, monkeypatch):
+    original = Path.resolve
+
+    def resolve(path, *args, **kwargs):
+        if path.name == "loop":
+            raise RuntimeError("symlink loop")
+        return original(path, *args, **kwargs)
+
+    monkeypatch.setattr(Path, "resolve", resolve)
+    findings = _ledger_check(snapshot, lambda doc: doc["gates"][0].update(evidence=["loop"]))
+    assert "unsafe-gate-evidence" in {row["code"] for row in findings}

procesio_cli/skills/tests/test_skill_routing.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+SCRIPT = ROOT / "scripts" / "evaluate-skill-routing.py"
+SPEC = importlib.util.spec_from_file_location("evaluate_skill_routing", SCRIPT)
+assert SPEC and SPEC.loader
+module = importlib.util.module_from_spec(SPEC)
+sys.modules[SPEC.name] = module
+SPEC.loader.exec_module(module)
+
+from tools._lib.manifest import load_skill
+
+
+def _governed_names(skills_root: Path) -> set[str]:
+    """Skills that opt into the governance/eval discipline, by the `source_policy`
+    marker. The routing corpus is authored for this portfolio universe; AAT also
+    hosts imported/portable skills whose prompts legitimately fall outside it, so
+    the live routing gate is scored over the governed set, not the whole tree
+    (procesio-cli's `skills/` IS the portfolio, so there the two coincide)."""
+    names: set[str] = set()
+    for skill_md in sorted(skills_root.glob("*/SKILL.md")):
+        try:
+            manifest = load_skill(skill_md)
+        except Exception:  # noqa: BLE001 - a broken skill is validate-skills' job
+            continue
+        if manifest.source_policy:
+            names.add(manifest.name)
+    return names
+
+
+def test_frozen_v2_baseline_is_reproducible():
+    report = module.evaluate(
+        module.load_catalog(ROOT / "skills" / "evals" / "baseline-catalog.json"),
+        module.load_cases(ROOT / "skills" / "evals" / "routing.json"),
+    )
+    expected = json.loads((ROOT / "skills" / "evals" / "baseline-routing-v2.json").read_text(encoding="utf-8"))
+    assert module.projection(report) == expected
+
+
+def test_live_skill_descriptions_clear_gate_three():
+    governed = _governed_names(ROOT / "skills")
+    assert governed, "no governed skills found (source_policy marker missing?)"
+    live = [s for s in module.load_skills(ROOT / "skills") if s.name in governed]
+    report = module.evaluate(
+        live,
+        module.load_cases(ROOT / "skills" / "evals" / "routing.json"),
+    )
+    assert report["routing_accuracy"] >= 0.95
+    assert report["collision_rate"] == 0
+
+
+def test_explicit_exclusion_prevents_postgresql_false_positive():
+    skills = [module.Skill(
+        "sql-server-optimizer",
+        "Analyze SQL Server T-SQL; do not use for PostgreSQL or MySQL.",
+    )]
+    assert module.select("Tune this PostgreSQL query", skills)[0] is None
+
+
+def test_hyphenated_tsql_matches():
+    skills = [module.Skill(
+        "sql-server-optimizer",
+        "Analyze SQL Server T-SQL predicates and sargability.",
+    )]
+    assert module.select(
+        "Why is this T-SQL predicate not sargable?", skills
+    )[0] == "sql-server-optimizer"

procesio_cli/skills/tests/test_skill_scaffold_v2.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import pytest
+import yaml
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = ROOT / "skills" / "agent-skill-engineer" / "scripts" / "scaffold_skill.py"
+SPEC = importlib.util.spec_from_file_location("agent_skill_scaffold_v2", SCRIPT)
+assert SPEC and SPEC.loader
+SCAFFOLD = importlib.util.module_from_spec(SPEC)
+sys.modules[SPEC.name] = SCAFFOLD
+SPEC.loader.exec_module(SCAFFOLD)
+
+
+def _frontmatter(path: Path) -> dict:
+    lines = path.read_text(encoding="utf-8").splitlines()
+    end = lines[1:].index("---") + 1
+    value = yaml.safe_load("\n".join(lines[1:end]))
+    assert isinstance(value, dict)
+    return value
+
+
+def test_governed_scaffold_preserves_v1_contract_and_adds_causal_evidence_fields(tmp_path):
+    args = SCAFFOLD._parser().parse_args(
+        [
+            "incident-triage",
+            "--root",
+            str(tmp_path),
+            "--description",
+            "Triage recurring incidents. Use when an incident report needs a bounded diagnostic workflow.",
+            "--profile",
+            "governed",
+            "--owner",
+            "platform team",
+            "--baseline-version",
+            "abc123",
+            "--trigger",
+            "triage a recurring production incident",
+            "--non-trigger",
+            "implement the incident-management product itself",
+            "--target-client",
+            "Agent Skills compatible coding agents",
+            "--evidence-tier",
+            "3",
+        ]
+    )
+    result = SCAFFOLD.scaffold(args)
+    target = tmp_path / "incident-triage"
+    frontmatter = _frontmatter(target / "SKILL.md")
+    body = (target / "SKILL.md").read_text(encoding="utf-8")
+    suite = json.loads((target / "evals" / "evals.json").read_text(encoding="utf-8"))
+
+    assert result["schema_version"] == 2
+    assert result["evidence_tier"] == 3
+    assert frontmatter["name"] == "incident-triage"
+    assert frontmatter["owner"] == "platform team"
+    assert frontmatter["eval_suite"] == "evals/evals.json"
+    assert frontmatter["metadata"]["status"] == "draft"
+    assert "## Causal contract" in body
+    assert "Protected successes" in body
+    assert "repairs and regressions" in body
+    assert "implement the incident-management product itself" in body
+    assert suite["rubric_contract"] == "fixed-jury-rubric-v2"
+    assert suite["evidence_tier"] == 3
+    assert suite["experiment_contract"]["test"].startswith("untouched")
+    assert {case["kind"] for case in suite["cases"]} == {
+        "positive",
+        "negative",
+        "overlap",
+        "pressure",
+    }
+
+
+def test_scaffold_still_refuses_overwrite_and_escape(tmp_path):
+    parser = SCAFFOLD._parser()
+    args = parser.parse_args(
+        [
+            "demo-skill",
+            "--root",
+            str(tmp_path),
+            "--description",
+            "Create a demo. Use when a reusable demo workflow is requested.",
+        ]
+    )
+    SCAFFOLD.scaffold(args)
+    with pytest.raises(FileExistsError, match="refusing to overwrite"):
+        SCAFFOLD.scaffold(args)
+    with pytest.raises(ValueError, match="lowercase"):
+        SCAFFOLD.scaffold(argparse.Namespace(**{**vars(args), "name": "../escape"}))
+
+
+def test_failed_governed_scaffold_leaves_no_partial_directory(tmp_path):
+    args = SCAFFOLD._parser().parse_args(
+        [
+            "governed-skill",
+            "--root",
+            str(tmp_path),
+            "--description",
+            "Govern work. Use when the governed workflow is required.",
+            "--profile",
+            "governed",
+            "--owner",
+            "team",
+            "--baseline-version",
+            "abc",
+        ]
+    )
+    with pytest.raises(ValueError, match="--trigger"):
+        SCAFFOLD.scaffold(args)
+    assert not (tmp_path / "governed-skill").exists()

procesio_cli/skills/tests/test_validate_skills.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = ROOT / "scripts" / "validate-skills.py"
+SPEC = importlib.util.spec_from_file_location("validate_skills", SCRIPT)
+assert SPEC and SPEC.loader
+module = importlib.util.module_from_spec(SPEC)
+sys.modules[SPEC.name] = module
+SPEC.loader.exec_module(module)
+
+
+def _write(path: Path, content: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content, encoding="utf-8")
+
+
+def _skill(root: Path, folder: str = "demo", name: str = "demo", body: str = "# Demo") -> Path:
+    path = root / "skills" / folder / "SKILL.md"
+    _write(path, f"---\nname: {name}\ndescription: Use when testing the demo skill.\n---\n\n{body}\n")
+    return path
+
+
+def test_valid_skill_has_no_blocking_findings(tmp_path):
+    _skill(tmp_path)
+    assert module.validate_repo(tmp_path / "skills", tmp_path) == []
+
+
+def test_missing_reference_and_folder_mismatch_are_detected(tmp_path):
+    _skill(tmp_path, folder="wrong", name="demo", body="Read `references/missing.md`.")
+    codes = {finding.code for finding in module.validate_repo(tmp_path / "skills", tmp_path)}
+    assert {"folder-name-mismatch", "missing-reference"} <= codes
+
+
+def test_nested_scripts_are_rejected(tmp_path):
+    _skill(tmp_path)
+    _write(tmp_path / "skills" / "demo" / "references" / "scripts" / "dump.sql", "select 1")
+    codes = {finding.code for finding in module.validate_repo(tmp_path / "skills", tmp_path)}
+    assert {"nested-resource", "script-in-references"} <= codes
+
+
+def test_python_bytecode_caches_are_ignored(tmp_path):
+    _skill(tmp_path)
+    _write(
+        tmp_path
+        / "skills"
+        / "demo"
+        / "scripts"
+        / "__pycache__"
+        / "helper.cpython-312.pyc",
+        "transient bytecode fixture",
+    )
+
+    assert module.validate_repo(tmp_path / "skills", tmp_path) == []
+
+
+def test_command_examples_must_name_real_actions(tmp_path):
+    _write(tmp_path / "tools" / "widget" / "tool.yaml", """
+name: widget
+description: widgets
+actions:
+  - name: list-things
+    args:
+      - {name: limit, type: integer}
+""".lstrip())
+    _skill(tmp_path, body=(
+        "`python scripts/run-tool.py widget list-things --limit 1`\n\n"
+        "`python scripts/run-tool.py widget delete-world`"
+    ))
+    findings = module.validate_repo(tmp_path / "skills", tmp_path)
+    assert any(finding.code == "unknown-action" and "delete-world" in finding.message
+               for finding in findings)
+    assert not any(finding.code == "unknown-action" and "list-things" in finding.message
+                   for finding in findings)
+
+
+def test_waiver_matches_only_the_declared_finding():
+    finding = module.Finding("error", "missing-reference", "demo", "SKILL.md", "x")
+    assert module._waived(finding, [{"skill": "demo", "code": "missing-*", "path": "*"}])
+    assert not module._waived(finding, [{"skill": "other", "code": "missing-*"}])
+
+
+def test_committed_waiver_file_is_valid_json():
+    data = json.loads((ROOT / "skills" / "evals" / "validation-baseline.json").read_text(encoding="utf-8"))
+    assert isinstance(data["allow"], list)
+
+
+def test_committed_skills_tree_passes_the_validator():
+    """The read-back the hand-written skills never had.
+
+    Every other test in this file builds its own fixture under tmp_path, so they
+    prove the validator's logic and nothing about the tree that ships. The
+    mirror guard for the GENERATED manuals does scan for real
+    (tests/test_tool_skills.py::test_skill_frontmatter_parses) - which is why a
+    description carrying a colon-space is caught under tools/ and was not caught
+    under skills/.
+
+    Why one bad file is not one file's problem: registry.list_skills() parses
+    every manifest in a single pass, so one unloadable SKILL.md fails get-skill
+    for EVERY skill, not just its own.
+
+    This calls main() rather than validate_repo() on purpose. validate_repo()
+    applies the full rubric to every folder, ignoring the `source_policy`
+    governance opt-in, so it fails imported skills that were never written to
+    that rubric. main() is what the CLI and both pre-commit hooks run, so this
+    test and the command a developer runs cannot drift apart.
+    """
+    assert module.main([]) == 0, (
+        "the committed skills tree does not pass the validator - "
+        "run `python scripts/validate-skills.py` for the findings"
+    )