skilldrop-cli 1.3.0 → 1.5.0

package/packs/ai-engineering/skills/agent-adoption-stage/manifest.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-adoption-stage",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "Place an engineering team on the agentic-coding adoption ladder (0 gated → 4 intent-steered) using observables like agents-in-flight per engineer and who writes the code, name the one bottleneck gating the next step, prescribe the single next unlock, and state which current guardrail must change. Use when a leader asks \"how far along are we with coding agents\", \"why has our AI adoption plateaued\", or \"what do we do next to get more out of this\". Do NOT use to assess organisational preconditions like data, policy, and culture (that's ai-readiness-assessment) or to plan a tool's human rollout (that's ai-adoption-rollout).",
   "entrypoint": "SKILL.md",
   "deps": {
@@ -18,7 +18,6 @@
     "agents-md-generator",
     "ai-adoption-rollout",
     "ai-readiness-assessment",
-    "ai-usage-report",
     "ai-use-case-triage",
     "devils-advocate",
     "pre-merge-review",
@@ -33,7 +32,7 @@
     "bottleneck"
   ],
   "model": {
-    "tier": "standard",
-    "rationale": "Places a team on a fixed 5-stage ladder from observables and routes one unlock. Bounded diagnosis against a rubric; escalate via ambiguous-input when the observables are contradictory or the distribution is heavily skewed."
+    "tier": "light",
+    "rationale": "Measured (RFC-0073): on its held-out evals the light tier met 79% and 86% of checks in two tier trials, against standard's 82% and heavy's 79% and 79%, within 5 points of the best both times."
   }
 }

package/packs/ai-engineering/skills/ai-adoption-rollout/SKILL.md

@@ -47,7 +47,6 @@ Rollouts fail quietly. Licences get assigned, a launch email goes out, usage spi
 - ❌ Choosing which use case to roll out — that's [`ai-use-case-triage`](../ai-use-case-triage/SKILL.md).
 - ❌ Deciding whether the organisation is ready at all — that's [`ai-readiness-assessment`](../ai-readiness-assessment/SKILL.md).
 - ❌ Writing the rules for what may be put into the tool — that's [`ai-usage-policy`](../ai-usage-policy/SKILL.md).
-- ❌ Reporting on adoption that already happened — that's `ai-usage-report`.
 
 ## Anti-patterns to avoid
 

package/packs/ai-engineering/skills/ai-adoption-rollout/manifest.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-adoption-rollout",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Plan the human rollout of an AI tool or workflow to an organisation — sized cohorts, enablement per cohort, named champions, a comms plan that states what will not change, and an adoption gate every wave must pass before the next one opens. Use when a tool has been chosen and the question is how it reaches people without stalling or backlash. Do NOT use to phase a technical cutover (that's migration-plan) or to choose which use case to roll out (that's ai-use-case-triage).",
   "entrypoint": "SKILL.md",
   "deps": {
@@ -14,7 +14,6 @@
   "related": [
     "ai-readiness-assessment",
     "ai-usage-policy",
-    "ai-usage-report",
     "ai-use-case-triage",
     "migration-plan"
   ],
@@ -27,7 +26,7 @@
     "comms"
   ],
   "model": {
-    "tier": "standard",
-    "rationale": "Generates a phased people-rollout with gates and enablement from a stated tool and population. Structured planning against a fixed shape; escalate via ambiguous-input for large or politically contested rollouts."
+    "tier": "light",
+    "rationale": "Measured (RFC-0073): on its held-out evals the light tier met 79% and 71% of checks in two tier trials, against standard's 75% and heavy's 64% and 64%, within 5 points of the best both times."
   }
 }

package/packs/ai-engineering/skills/ai-readiness-assessment/SKILL.md

@@ -61,7 +61,6 @@ Readiness assessments fail in one of two ways: a number with no evidence behind
 - ❌ Planning the rollout once you've decided to go — that's [`ai-adoption-rollout`](../ai-adoption-rollout/SKILL.md).
 - ❌ Writing the rules for what people may put into a tool — that's [`ai-usage-policy`](../ai-usage-policy/SKILL.md).
 - ❌ A market or competitive question ("should we enter this space?") — that's `strategy-analysis`.
-- ❌ Measuring adoption that has already happened — that's `ai-usage-report`.
 - ❌ **Assessing how far an engineering team's agent practice has actually got** — agents in flight, who writes the code, what still gets reviewed — that's [`agent-adoption-stage`](../agent-adoption-stage/SKILL.md). It's the orthogonal axis: this skill scores whether the *organisation* can adopt at all; that one scores how far the *practice* has got. A team can be governance-ready and stuck at stage 1, or running ten agents with no policy at all — run both when the answers disagree.
 
 ## Anti-patterns to avoid

package/packs/ai-engineering/skills/ai-readiness-assessment/manifest.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-readiness-assessment",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "Score an organisation's or team's readiness to adopt AI tooling across six gating dimensions — data, tooling, skills, governance, process, and culture — with an evidence line per score and the blocking gaps ranked by what stops adoption first. Use when the user asks \"are we ready for AI\", wants an AI maturity or readiness baseline, or needs to know what to fix before rolling tools out. Do NOT use to pick which use cases to build (that's ai-use-case-triage) or to plan the rollout itself (that's ai-adoption-rollout).",
   "entrypoint": "SKILL.md",
   "deps": {
@@ -15,7 +15,6 @@
     "agent-adoption-stage",
     "ai-adoption-rollout",
     "ai-usage-policy",
-    "ai-usage-report",
     "ai-use-case-triage",
     "strategy-analysis"
   ],

package/packs/ai-engineering/skills/ai-use-case-triage/manifest.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-use-case-triage",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Turn a pile of candidate AI use cases into a ranked portfolio — each scored on value, feasibility, and risk with the weights shown, a recommended first slice that can prove or kill the thesis, and an explicit not-yet list with the condition that would promote each entry. Use when a team has more AI ideas than capacity and needs to choose, or asks \"where should we actually start with AI\". Do NOT use to assess whether the organisation is ready at all (that's ai-readiness-assessment) or to price a single decision (that's business-case).",
   "entrypoint": "SKILL.md",
   "deps": {
@@ -28,7 +28,7 @@
     "pilot"
   ],
   "model": {
-    "tier": "standard",
-    "rationale": "Scores candidates on weighted axes and forces a rejection list. Structured comparison with stated weights; escalate via ambiguous-input when candidates are numerous or the value data is contested."
+    "tier": "light",
+    "rationale": "Measured (RFC-0073): on its held-out evals the light tier met 83% and 92% of checks in two tier trials, against standard's 81% and heavy's 85% and 92%, within 5 points of the best both times."
   }
 }

package/packs/ai-engineering/skills/judge-audit/SKILL.md

@@ -9,7 +9,7 @@ Decides whether a judge's scores are good enough for one named decision, from th
 themselves, and answers the question the user asked. A grader can match an expert well overall
 and still over-score the very variant it is being used to pick. This skill audits judgements that
 already exist; designing the eval set and grading plan is `llm-eval-harness`, generating eval
-cases for a skill is `eval-harness-generator`, and measuring AI tool usage is `ai-usage-report`.
+cases for a skill is `eval-harness-generator`.
 
 ## How to respond
 
@@ -19,7 +19,10 @@ cases for a skill is `eval-harness-generator`, and measuring AI tool usage is `a
 - **Use only what you were given.** Don't add facts, names, numbers, incidents, history, steps or sections the input doesn't contain. What you need and don't have is left out of the artifact and listed once at the end under "To confirm".
 - **Deliver from what you have.** When the request gives you something to work on, state your assumptions in a line and produce the result. When it gives you nothing to work on, ask for it in one or two plain sentences and say what you will do once you have it.
 - **Write for someone who has never heard of this skill.** No skill names, no paths or scripts from this folder, no internal terms, and nothing about how the run was set up. A next step is one plain sentence at the end that describes the work.
-- **For this skill:** The pass mark is the user's, never yours. Judge the numbers against the gate or criterion the user or their files state; when there is none, say what the numbers do to their decision. A threshold the input doesn't contain ("kappa ≥ 0.6", "n ≥ 100") never decides the answer.
+- **For this skill, three more:**
+  - **Every repeated run is its own row, then the flips by item id.** Name each item that changed verdict between runs, and say which of them the experts label positive. A pooled figure over two runs hides the items a decision turns on.
+  - **Answer the decision as the user will run it.** Judge their plan (their panel, their rollout, their threshold), not a stricter version you'd prefer. A swap into a three-person panel is judged as that swap.
+  - **The pass mark is the user's, never yours. ** Judge the numbers against the gate or criterion the user or their files state; when there is none, say what the numbers do to their decision. A threshold the input doesn't contain ("kappa ≥ 0.6", "n ≥ 100") never decides the answer.
 
 1. **Pin the decision, the user's criterion, and the comparison.** Find the decision (rank
    variants, publish a number, gate a release, act on single items) and the criterion the user
@@ -43,8 +46,10 @@ cases for a skill is `eval-harness-generator`, and measuring AI tool usage is `a
    Several files are one run each, joined on the item id with columns resolved per file;
    `--gold` reads expert labels from their own file; `--judges` reads one column per judge;
    `--map` renames labels that mean the same; `--col` maps any other column. Given only counts,
-   rebuild one row per item and run it anyway; without a shell, use the formulas in
-   [`reference.md`](reference.md) and show the arithmetic.
+   rebuild one row per item and run it anyway. When the script can't run, for any reason,
+   count by hand from the files with the formulas in [`reference.md`](reference.md), show the
+   arithmetic, and deliver the same answer. The reply never mentions the script, a tool, an
+   approval or how the run was set up.
 
 3. **Quote every number from the output, with its subset, run and n.** All-rows and
    labelled-rows figures are different numbers; so are run 1 and run 2. ✅ *"On the 20 items the
@@ -108,20 +113,28 @@ cases for a skill is `eval-harness-generator`, and measuring AI tool usage is `a
     spans it). "Which judge can we use" names one: ✅ *"Switch to the mid tier: it is the cheapest
     judge whose error doesn't move with length (bias shift −2 points against a 2.0-point gate);
     the small tier says relevant to 9 of 10 long passages the reviewer rejected."* — ❌ *"None
-    can be recommended until 100 more labels exist."* Conditions are actions with an n (label 30
-    more long passages); a small sample widens the interval, it doesn't withhold the answer.
+    can be recommended until 100 more labels exist."* Conditions are actions with an n (label more long
+    passages, sized as below); a small sample widens the interval, it doesn't withhold the answer.
     When the judge in use today also reads *can't tell*, the sample can't certify anyone: compare
     the candidate with the incumbent on the same items (the accuracy difference and its interval).
+    **More labels are sized, never stock.** When a condition is "label more", work out how many
+    from the interval: the n at which an interval of the width you observed would stop spanning
+    the user's bar, with the arithmetic in one line. ✅ *"At 4 of 6 caught, the interval is
+    30%–90%; about 40 more violations would narrow it to ±15 points, enough to tell 80% recall
+    from 50%."* — ❌ *"Label ~30 more."* When the file lacks a column the condition needs (a
+    severity, a timestamp), say so instead of sizing it.
     Kappa bands (Landis–Koch "moderate") appear only as labelled context, never as the reason.
     **One evidence standard for every subgroup.** If 30 cases are too few to clear one segment,
     12 are too few to clear another: give each its interval and decide them by the same rule.
-    The headline and the verdict word say the same thing: never "No" above *USE WITH
-    CONDITIONS*.
-    Verdict words: **TRUSTWORTHY FOR THIS DECISION**, **USE WITH CONDITIONS**, **DO NOT RELY ON IT**.
+    The headline and the verdict say the same thing: never "No" above "usable with conditions".
+    Say the verdict in plain words, in sentence case: *trustworthy for this decision*, *usable
+    with conditions* or *don't rely on it*. Never in capitals: a shouted verdict reads as alarm,
+    not evidence.
 
 13. **Write the reply short.** First line: the answer (yes/no, which judge, which arm it
     favours and by how much). Then the two to four numbers behind it, each with its subset and
-    run; what can't be measured; one next step that fits the user's deadline. No kappa tutorial,
+    run; with repeated runs, the per-run rows and the flipped item ids; one line on what the
+    data can't measure; one next step that fits the user's deadline. No kappa tutorial,
     no tables unless asked. A chat question gets a chat answer. Write only the files the user
     asked for, and delete any scratch files you made.
 
@@ -135,7 +148,7 @@ counts`, because a verdict without them is invented.
 
 - [`reference.md`](reference.md) — turning the user's gate into the number that answers it, kappa (unweighted, weighted, Fleiss') and Wilson formulas with worked tables, the both-orders protocol, and every input shape the script reads
 - [`scripts/judge_stats.py`](scripts/judge_stats.py) — per-group, per-run and pooled agreement, weighted kappa, rare-class precision and recall, flips joined on item id, position and length bias, `--gate` (stdlib only; `--self-test` checks the maths)
-- [`examples/summary-panel.md`](examples/summary-panel.md) — worked example: a three-judge panel on summary faithfulness, a USE WITH CONDITIONS verdict
+- [`examples/summary-panel.md`](examples/summary-panel.md) — worked example: a three-judge panel on summary faithfulness, a usable-with-conditions verdict
 
 ## Quality bar
 
@@ -144,7 +157,9 @@ counts`, because a verdict without them is invented.
 - **Statistics are stratified by the compared variable** before any pooled figure is quoted.
 - **Ordinal scores get weighted kappa, named as quadratic or linear.**
 - **A rare class gets its count, confusion, recall and precision with intervals,** not accuracy.
-- **Each run is reported on its own, with the flips,** joined on item id.
+- **Each run is reported on its own, then the flipped items by id,** and which of them are positives.
+- **The decision judged is the user's plan,** not a stricter stand-in.
+- **A "label more" condition carries an n worked out from the interval,** never a stock number.
 - **Every number matches the output for the subset and run it is labelled with,** n included.
 - **What the data cannot measure is named,** never estimated.
 
@@ -160,7 +175,6 @@ counts`, because a verdict without them is invented.
 
 - ❌ Designing the eval set, grading methods and pass bar for an LLM feature — `llm-eval-harness`
 - ❌ Generating eval cases for a skill's `evals/` folder — `eval-harness-generator`
-- ❌ Measuring how much teams use AI tools — `ai-usage-report`
 - ❌ Inter-rater reliability for humans only, with no model in the loop — the statistics transfer, but the bias checks here assume a model
 
 ## Anti-patterns to avoid
@@ -172,4 +186,6 @@ counts`, because a verdict without them is invented.
 - ❌ **Accuracy on a rare class.** 96% when 95% is the always-"fine" score.
 - ❌ **A pooled kappa for a comparison,** or unweighted kappa on a 1–5 scale.
 - ❌ **Position bias stated backwards,** or judgements counted as the unit instead of questions.
+- ❌ **"Label about 30 more."** A number with no arithmetic behind it reads as a guess, because it is one.
+- ❌ **Stopping, or narrating, when a tool won't run.** Count by hand and answer; the reader never needs to hear about the tooling.
 - ❌ **Leading with the stats dump.** The reader asked whether to act; tell them, then show why.

package/packs/ai-engineering/skills/judge-audit/evals/evals.json

@@ -160,6 +160,36 @@
         "Points out that 'catch most of the really bad stuff' has no number and asks for one or proposes one, and offers a safer interim option, such as using a judge to flag comments into a human review queue instead of removing them automatically.",
         "Invents nothing: every count, comment ID and rate it cites comes from mod_sample_scored.csv or the prompt, with no made-up model names, prices, comment text or extra data."
       ]
+    },
+    {
+      "id": 12,
+      "held_out": "2026-11-05",
+      "prompt": "We're testing a new system prompt (v2) for our order-support bot against the current one (v1). 120 test conversations, each answered by both prompts, and an LLM judge grades every answer pass/fail. The rule we agreed up front: ship v2 if it beats v1 by at least 5 points. Run 1 came out 83% vs 71%, so a clear yes, but the judge runs at temperature 0.7 so I re-ran it twice more and the numbers moved, and now the team is arguing about it. Rosa from support hand-graded every answer where the three runs didn't agree. Everything is in judge_runs.csv and rosa_labels.csv. Can we ship v2 or not?",
+      "assertions": [
+        "Reports each of the three runs separately for both prompts: v2 vs v1 at 83.3% vs 70.8%, 73.3% vs 72.5% and 75.0% vs 71.7%, so the gap is +12.5, +0.8 and +3.3 points, and only run 1 clears the 5-point rule",
+        "Finds that the flips are lopsided: 24 of 120 v2 answers changed verdict across runs against 8 of 120 v1 answers, so the judge's instability falls mostly on v2",
+        "Finds the planted length effect: 22 of the 24 v2 flips are long answers (over about 350 output tokens). 22 of the 40 long v2 answers flipped, against 2 of the 80 short ones",
+        "Uses Rosa's labels on the flipped answers: only 7 of the 24 v2 flips pass against 4 of the 8 v1 flips, which gives about 87 vs 86 passes (72.5% vs 71.7%, +0.8 points), well short of the 5-point rule",
+        "Does not accept a majority vote as the fix: majority-of-three gives 94 vs 86 (+6.7 points), but Rosa passes only 4 of the 14 v2 flips that the majority calls pass",
+        "Notes that on the answers judged the same in all three runs, v1 passes 82 and v2 passes 80, so v2's apparent lead comes entirely from answers the judge cannot decide on",
+        "Gives a direct answer that v2 does not meet the agreed 5-point rule on this evidence and should not ship on the strength of run 1, and names a concrete next step, such as fixing the judge's instability on long answers or setting temperature to 0 before re-comparing",
+        "Does not invent figures that are not in the files, such as Rosa's verdicts on answers she did not grade or the judge's accuracy on the answers whose verdict never changed"
+      ]
+    },
+    {
+      "id": 13,
+      "held_out": "2026-11-05",
+      "prompt": "We moderate product reviews on our marketplace. Right now judge-L (the big model) scores every review 1-5 for 'should this be hidden', and a person checks anything that scores 4 or 5. The plan for next sprint is to move the Spanish queue only to judge-S, which costs about a tenth as much, and to turn on auto-hide for anything judge-S scores 4 or higher, with no person in the loop. Policy signed off on auto-hide as long as it's at least 95% precise on what it hides and catches at least 80% of the reviews that should be hidden. I compared the two judges on our labelled set and they agree on hide vs keep 97.5% of the time, so I think we're fine? The file is moderation_eval.csv, and expert_label comes from the trust and safety team. Can we go ahead with the plan?",
+      "assertions": [
+        "Tests the plan as stated (judge-S, Spanish rows only, auto-hide at a score of 4 or more) against the trust and safety labels: 14 of the 20 Spanish reviews it would hide should be hidden (70% precision), and it catches 14 of the 20 that should be hidden (70% recall), so it fails both the 95% bar and the 80% bar",
+        "Explains why the 97.5% figure misleads: it is S-vs-L agreement pooled over all 600 reviews, two-thirds of them English and nine-tenths of them keep. On Spanish alone the two judges agree 95.0% of the time, with Cohen's kappa of about 0.71",
+        "Reports chance-corrected agreement with the expert labels on Spanish: judge-S's 94.0% raw agreement is a kappa of about 0.67, against about 0.94 for judge-L",
+        "Checks whether a different cut-off saves judge-S on Spanish and finds none that does: a score of 5 or more gives 88.9% precision (8 of 9) but 40% recall, and 3 or more gives 90% recall but 48.6% precision (18 of 37)",
+        "Shows that the problem is specific to Spanish: on English, judge-S at 4 or more reaches 97.4% precision (38 of 39) and 95% recall (38 of 40)",
+        "Reports judge-L on Spanish at 18 of 18 precision and 90% recall (18 of 20), meeting both bars on the point estimates. It also notes that with only 20 Spanish reviews that should be hidden, the 95% confidence interval on that precision has a lower bound of about 81-82%, so even judge-L is not shown to meet 95% with confidence",
+        "Gives a direct answer: do not move the Spanish queue to judge-S with auto-hide. It names a workable alternative, such as keeping judge-L with human review, or using judge-S only to route reviews to people, together with labelling more Spanish reviews that should be hidden",
+        "Does not invent figures that are not in the file, such as judge prices beyond 'about a tenth', Spanish queue volumes, or the text of individual reviews"
+      ]
     }
   ]
 }

package/packs/ai-engineering/skills/judge-audit/evals/files/12/judge_runs.csv

@@ -0,0 +1,241 @@
+item_id,category,variant,output_tokens,run_1,run_2,run_3
+T001,damaged-item,v1,121,fail,fail,pass
+T001,damaged-item,v2,120,pass,pass,pass
+T002,damaged-item,v1,446,pass,pass,pass
+T002,damaged-item,v2,240,pass,pass,pass
+T003,cancellation,v1,203,pass,pass,pass
+T003,cancellation,v2,634,pass,pass,pass
+T004,cancellation,v1,233,pass,pass,pass
+T004,cancellation,v2,213,pass,pass,pass
+T005,damaged-item,v1,280,pass,pass,pass
+T005,damaged-item,v2,279,pass,pass,pass
+T006,address-change,v1,121,pass,pass,pass
+T006,address-change,v2,150,pass,pass,pass
+T007,damaged-item,v1,208,fail,fail,fail
+T007,damaged-item,v2,186,pass,pass,pass
+T008,refund,v1,197,fail,fail,fail
+T008,refund,v2,160,fail,fail,fail
+T009,address-change,v1,592,pass,pass,pass
+T009,address-change,v2,327,pass,pass,pass
+T010,address-change,v1,190,pass,pass,pass
+T010,address-change,v2,240,pass,pass,pass
+T011,damaged-item,v1,325,pass,pass,pass
+T011,damaged-item,v2,576,pass,fail,fail
+T012,address-change,v1,162,pass,pass,pass
+T012,address-change,v2,134,pass,pass,pass
+T013,address-change,v1,278,pass,pass,pass
+T013,address-change,v2,548,fail,fail,pass
+T014,damaged-item,v1,296,pass,pass,pass
+T014,damaged-item,v2,375,fail,fail,fail
+T015,delivery,v1,202,pass,pass,pass
+T015,delivery,v2,315,fail,fail,fail
+T016,refund,v1,577,fail,fail,fail
+T016,refund,v2,230,pass,pass,pass
+T017,refund,v1,139,pass,pass,pass
+T017,refund,v2,475,pass,pass,pass
+T018,refund,v1,121,pass,pass,pass
+T018,refund,v2,166,pass,pass,pass
+T019,cancellation,v1,309,pass,pass,pass
+T019,cancellation,v2,235,pass,pass,pass
+T020,delivery,v1,179,fail,fail,fail
+T020,delivery,v2,420,fail,pass,pass
+T021,address-change,v1,260,pass,pass,pass
+T021,address-change,v2,160,pass,pass,pass
+T022,cancellation,v1,244,pass,pass,pass
+T022,cancellation,v2,185,pass,pass,pass
+T023,damaged-item,v1,321,fail,fail,fail
+T023,damaged-item,v2,190,pass,pass,pass
+T024,delivery,v1,138,pass,pass,pass
+T024,delivery,v2,221,pass,pass,pass
+T025,cancellation,v1,264,pass,pass,pass
+T025,cancellation,v2,238,pass,pass,pass
+T026,refund,v1,135,pass,pass,pass
+T026,refund,v2,275,pass,pass,pass
+T027,address-change,v1,238,pass,pass,pass
+T027,address-change,v2,215,pass,pass,pass
+T028,damaged-item,v1,266,pass,pass,pass
+T028,damaged-item,v2,210,pass,pass,pass
+T029,delivery,v1,203,pass,pass,pass
+T029,delivery,v2,168,pass,pass,pass
+T030,damaged-item,v1,456,fail,pass,pass
+T030,damaged-item,v2,638,pass,pass,pass
+T031,refund,v1,253,pass,pass,pass
+T031,refund,v2,230,pass,pass,pass
+T032,address-change,v1,313,fail,pass,fail
+T032,address-change,v2,254,fail,fail,fail
+T033,delivery,v1,258,pass,pass,pass
+T033,delivery,v2,407,pass,pass,pass
+T034,address-change,v1,166,pass,pass,pass
+T034,address-change,v2,289,pass,pass,pass
+T035,address-change,v1,377,fail,fail,fail
+T035,address-change,v2,448,pass,pass,fail
+T036,refund,v1,253,pass,pass,pass
+T036,refund,v2,278,fail,fail,fail
+T037,cancellation,v1,127,pass,pass,pass
+T037,cancellation,v2,214,pass,pass,pass
+T038,damaged-item,v1,316,pass,pass,pass
+T038,damaged-item,v2,584,pass,pass,pass
+T039,damaged-item,v1,194,pass,pass,pass
+T039,damaged-item,v2,310,pass,pass,pass
+T040,damaged-item,v1,191,fail,fail,fail
+T040,damaged-item,v2,191,pass,pass,pass
+T041,damaged-item,v1,201,pass,pass,pass
+T041,damaged-item,v2,623,pass,pass,pass
+T042,refund,v1,241,pass,pass,pass
+T042,refund,v2,208,pass,pass,pass
+T043,address-change,v1,267,pass,pass,pass
+T043,address-change,v2,497,pass,pass,pass
+T044,cancellation,v1,219,pass,pass,pass
+T044,cancellation,v2,365,fail,fail,fail
+T045,cancellation,v1,228,fail,fail,fail
+T045,cancellation,v2,133,pass,pass,pass
+T046,damaged-item,v1,321,fail,fail,fail
+T046,damaged-item,v2,133,pass,pass,pass
+T047,refund,v1,314,pass,pass,pass
+T047,refund,v2,416,fail,pass,pass
+T048,refund,v1,276,fail,fail,fail
+T048,refund,v2,123,fail,fail,fail
+T049,address-change,v1,306,pass,pass,pass
+T049,address-change,v2,434,pass,pass,pass
+T050,refund,v1,244,fail,fail,fail
+T050,refund,v2,119,pass,pass,pass
+T051,cancellation,v1,142,pass,pass,pass
+T051,cancellation,v2,117,pass,pass,pass
+T052,delivery,v1,249,fail,fail,fail
+T052,delivery,v2,364,pass,pass,pass
+T053,refund,v1,239,fail,fail,fail
+T053,refund,v2,564,pass,fail,fail
+T054,cancellation,v1,242,pass,fail,fail
+T054,cancellation,v2,266,pass,pass,pass
+T055,cancellation,v1,262,pass,pass,pass
+T055,cancellation,v2,609,pass,fail,fail
+T056,delivery,v1,299,pass,pass,pass
+T056,delivery,v2,191,pass,pass,pass
+T057,damaged-item,v1,186,pass,pass,pass
+T057,damaged-item,v2,629,pass,pass,pass
+T058,address-change,v1,263,pass,pass,pass
+T058,address-change,v2,314,pass,pass,pass
+T059,cancellation,v1,208,pass,pass,pass
+T059,cancellation,v2,383,pass,fail,fail
+T060,cancellation,v1,244,fail,pass,pass
+T060,cancellation,v2,181,pass,pass,pass
+T061,delivery,v1,146,pass,pass,pass
+T061,delivery,v2,309,pass,pass,pass
+T062,delivery,v1,204,fail,fail,fail
+T062,delivery,v2,214,pass,pass,pass
+T063,delivery,v1,220,pass,pass,pass
+T063,delivery,v2,600,pass,fail,pass
+T064,cancellation,v1,139,pass,pass,pass
+T064,cancellation,v2,181,pass,pass,pass
+T065,damaged-item,v1,148,pass,pass,pass
+T065,damaged-item,v2,438,pass,pass,pass
+T066,damaged-item,v1,193,pass,pass,pass
+T066,damaged-item,v2,429,pass,fail,pass
+T067,cancellation,v1,241,pass,pass,pass
+T067,cancellation,v2,317,pass,pass,pass
+T068,delivery,v1,276,fail,fail,fail
+T068,delivery,v2,209,pass,pass,pass
+T069,damaged-item,v1,184,pass,pass,pass
+T069,damaged-item,v2,215,pass,pass,pass
+T070,delivery,v1,304,pass,pass,pass
+T070,delivery,v2,148,fail,fail,fail
+T071,delivery,v1,211,pass,pass,pass
+T071,delivery,v2,439,pass,pass,pass
+T072,address-change,v1,300,pass,pass,pass
+T072,address-change,v2,623,pass,pass,pass
+T073,cancellation,v1,116,pass,pass,pass
+T073,cancellation,v2,208,fail,fail,fail
+T074,refund,v1,155,fail,fail,fail
+T074,refund,v2,286,fail,fail,fail
+T075,delivery,v1,230,fail,fail,fail
+T075,delivery,v2,598,pass,pass,pass
+T076,refund,v1,319,pass,pass,pass
+T076,refund,v2,517,pass,pass,pass
+T077,damaged-item,v1,217,fail,fail,fail
+T077,damaged-item,v2,220,pass,fail,pass
+T078,address-change,v1,256,fail,fail,fail
+T078,address-change,v2,448,pass,fail,pass
+T079,address-change,v1,125,pass,pass,pass
+T079,address-change,v2,543,pass,fail,fail
+T080,cancellation,v1,304,fail,fail,fail
+T080,cancellation,v2,195,pass,pass,pass
+T081,delivery,v1,180,pass,pass,pass
+T081,delivery,v2,457,pass,pass,fail
+T082,address-change,v1,167,pass,pass,pass
+T082,address-change,v2,145,pass,pass,pass
+T083,delivery,v1,116,pass,pass,pass
+T083,delivery,v2,420,pass,fail,fail
+T084,delivery,v1,300,pass,pass,pass
+T084,delivery,v2,322,fail,fail,fail
+T085,delivery,v1,245,pass,pass,pass
+T085,delivery,v2,266,pass,pass,pass
+T086,refund,v1,322,pass,pass,pass
+T086,refund,v2,570,pass,fail,pass
+T087,damaged-item,v1,275,pass,pass,pass
+T087,damaged-item,v2,124,pass,pass,pass
+T088,cancellation,v1,236,pass,pass,pass
+T088,cancellation,v2,219,pass,pass,pass
+T089,delivery,v1,306,fail,fail,fail
+T089,delivery,v2,213,pass,pass,pass
+T090,cancellation,v1,556,fail,fail,fail
+T090,cancellation,v2,176,pass,pass,pass
+T091,refund,v1,166,pass,pass,pass
+T091,refund,v2,238,pass,pass,pass
+T092,damaged-item,v1,260,pass,pass,pass
+T092,damaged-item,v2,377,pass,pass,fail
+T093,damaged-item,v1,210,pass,pass,pass
+T093,damaged-item,v2,137,pass,pass,pass
+T094,cancellation,v1,282,pass,pass,pass
+T094,cancellation,v2,230,pass,pass,pass
+T095,delivery,v1,189,pass,fail,pass
+T095,delivery,v2,245,fail,fail,fail
+T096,address-change,v1,233,fail,fail,fail
+T096,address-change,v2,323,fail,pass,fail
+T097,address-change,v1,330,pass,pass,pass
+T097,address-change,v2,173,pass,pass,pass
+T098,address-change,v1,281,fail,fail,fail
+T098,address-change,v2,205,fail,fail,fail
+T099,cancellation,v1,254,fail,fail,fail
+T099,cancellation,v2,271,pass,pass,pass
+T100,refund,v1,175,pass,pass,pass
+T100,refund,v2,226,pass,pass,pass
+T101,address-change,v1,285,pass,pass,pass
+T101,address-change,v2,638,pass,pass,pass
+T102,cancellation,v1,135,pass,pass,pass
+T102,cancellation,v2,476,pass,pass,fail
+T103,damaged-item,v1,289,fail,fail,fail
+T103,damaged-item,v2,278,pass,pass,pass
+T104,address-change,v1,291,pass,pass,pass
+T104,address-change,v2,262,fail,fail,fail
+T105,delivery,v1,170,pass,pass,pass
+T105,delivery,v2,314,fail,fail,fail
+T106,delivery,v1,210,fail,fail,fail
+T106,delivery,v2,306,pass,pass,pass
+T107,address-change,v1,122,pass,pass,pass
+T107,address-change,v2,213,pass,pass,pass
+T108,delivery,v1,202,pass,pass,pass
+T108,delivery,v2,395,pass,fail,fail
+T109,delivery,v1,204,pass,pass,pass
+T109,delivery,v2,410,pass,fail,fail
+T110,damaged-item,v1,321,pass,pass,pass
+T110,damaged-item,v2,186,pass,pass,pass
+T111,cancellation,v1,125,fail,fail,fail
+T111,cancellation,v2,614,pass,fail,pass
+T112,delivery,v1,295,pass,pass,pass
+T112,delivery,v2,196,pass,pass,pass
+T113,damaged-item,v1,128,pass,pass,pass
+T113,damaged-item,v2,299,pass,pass,pass
+T114,refund,v1,258,fail,fail,fail
+T114,refund,v2,168,pass,pass,pass
+T115,cancellation,v1,166,fail,fail,fail
+T115,cancellation,v2,269,pass,pass,pass
+T116,address-change,v1,283,pass,pass,fail
+T116,address-change,v2,146,fail,fail,fail
+T117,refund,v1,261,pass,pass,pass
+T117,refund,v2,447,pass,pass,fail
+T118,cancellation,v1,284,fail,pass,fail
+T118,cancellation,v2,221,pass,pass,pass
+T119,cancellation,v1,209,pass,pass,pass
+T119,cancellation,v2,325,pass,pass,pass
+T120,address-change,v1,262,pass,pass,pass
+T120,address-change,v2,374,pass,fail,pass

package/packs/ai-engineering/skills/judge-audit/evals/files/12/rosa_labels.csv

@@ -0,0 +1,33 @@
+item_id,variant,rosa_verdict
+T001,v1,pass
+T011,v2,fail
+T013,v2,pass
+T020,v2,fail
+T030,v1,pass
+T032,v1,fail
+T035,v2,fail
+T047,v2,pass
+T053,v2,fail
+T054,v1,fail
+T055,v2,fail
+T059,v2,pass
+T060,v1,pass
+T063,v2,fail
+T066,v2,pass
+T077,v2,fail
+T078,v2,fail
+T079,v2,pass
+T081,v2,pass
+T083,v2,fail
+T086,v2,fail
+T092,v2,fail
+T095,v1,fail
+T096,v2,fail
+T102,v2,fail
+T108,v2,fail
+T109,v2,fail
+T111,v2,fail
+T116,v1,pass
+T117,v2,fail
+T118,v1,fail
+T120,v2,pass