skilldrop-cli 0.16.36 → 0.16.37

package/packs/ai-engineering/skills/judge-audit/evals/files/5/pairwise_judgements.jsonl

@@ -0,0 +1,120 @@
+{"item_id": "q101", "judge": "orca-judge-large", "trial": 1, "first_shown": "A", "verdict": "tie"}
+{"item_id": "q101", "judge": "orca-judge-large", "trial": 2, "first_shown": "A", "verdict": "A"}
+{"item_id": "q101", "judge": "wren-judge-small", "trial": 1, "first_shown": "A", "verdict": "B"}
+{"item_id": "q101", "judge": "wren-judge-small", "trial": 2, "first_shown": "A", "verdict": "A"}
+{"item_id": "q102", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q102", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q102", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q102", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q103", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q103", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q103", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q103", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "A"}
+{"item_id": "q104", "judge": "orca-judge-large", "trial": 1, "first_shown": "A", "verdict": "A"}
+{"item_id": "q104", "judge": "orca-judge-large", "trial": 2, "first_shown": "A", "verdict": "A"}
+{"item_id": "q104", "judge": "wren-judge-small", "trial": 1, "first_shown": "A", "verdict": "A"}
+{"item_id": "q104", "judge": "wren-judge-small", "trial": 2, "first_shown": "A", "verdict": "A"}
+{"item_id": "q105", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q105", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q105", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q105", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q106", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q106", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q106", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "tie"}
+{"item_id": "q106", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "tie"}
+{"item_id": "q107", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q107", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q107", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q107", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q108", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q108", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q108", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q108", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q109", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q109", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "A"}
+{"item_id": "q109", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q109", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "A"}
+{"item_id": "q110", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q110", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q110", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q110", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q111", "judge": "orca-judge-large", "trial": 1, "first_shown": "A", "verdict": "A"}
+{"item_id": "q111", "judge": "orca-judge-large", "trial": 2, "first_shown": "A", "verdict": "B"}
+{"item_id": "q111", "judge": "wren-judge-small", "trial": 1, "first_shown": "A", "verdict": "A"}
+{"item_id": "q111", "judge": "wren-judge-small", "trial": 2, "first_shown": "A", "verdict": "A"}
+{"item_id": "q112", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q112", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q112", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q112", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q113", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q113", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "A"}
+{"item_id": "q113", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q113", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q114", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q114", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q114", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q114", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "A"}
+{"item_id": "q115", "judge": "orca-judge-large", "trial": 1, "first_shown": "A", "verdict": "A"}
+{"item_id": "q115", "judge": "orca-judge-large", "trial": 2, "first_shown": "A", "verdict": "A"}
+{"item_id": "q115", "judge": "wren-judge-small", "trial": 1, "first_shown": "A", "verdict": "B"}
+{"item_id": "q115", "judge": "wren-judge-small", "trial": 2, "first_shown": "A", "verdict": "A"}
+{"item_id": "q116", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q116", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q116", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q116", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q117", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "A"}
+{"item_id": "q117", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q117", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "A"}
+{"item_id": "q117", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q118", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q118", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q118", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q118", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q119", "judge": "orca-judge-large", "trial": 1, "first_shown": "A", "verdict": "A"}
+{"item_id": "q119", "judge": "orca-judge-large", "trial": 2, "first_shown": "A", "verdict": "A"}
+{"item_id": "q119", "judge": "wren-judge-small", "trial": 1, "first_shown": "A", "verdict": "A"}
+{"item_id": "q119", "judge": "wren-judge-small", "trial": 2, "first_shown": "A", "verdict": "A"}
+{"item_id": "q120", "judge": "orca-judge-large", "trial": 1, "first_shown": "A", "verdict": "A"}
+{"item_id": "q120", "judge": "orca-judge-large", "trial": 2, "first_shown": "A", "verdict": "A"}
+{"item_id": "q120", "judge": "wren-judge-small", "trial": 1, "first_shown": "A", "verdict": "A"}
+{"item_id": "q120", "judge": "wren-judge-small", "trial": 2, "first_shown": "A", "verdict": "tie"}
+{"item_id": "q121", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "A"}
+{"item_id": "q121", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "A"}
+{"item_id": "q121", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "A"}
+{"item_id": "q121", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "tie"}
+{"item_id": "q122", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q122", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q122", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "A"}
+{"item_id": "q122", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q123", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q123", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q123", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q123", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q124", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q124", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "tie"}
+{"item_id": "q124", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "A"}
+{"item_id": "q124", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "A"}
+{"item_id": "q125", "judge": "orca-judge-large", "trial": 1, "first_shown": "A", "verdict": "B"}
+{"item_id": "q125", "judge": "orca-judge-large", "trial": 2, "first_shown": "A", "verdict": "A"}
+{"item_id": "q125", "judge": "wren-judge-small", "trial": 1, "first_shown": "A", "verdict": "A"}
+{"item_id": "q125", "judge": "wren-judge-small", "trial": 2, "first_shown": "A", "verdict": "tie"}
+{"item_id": "q126", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q126", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q126", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q126", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q127", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q127", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q127", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "A"}
+{"item_id": "q127", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "A"}
+{"item_id": "q128", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q128", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q128", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "A"}
+{"item_id": "q128", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "A"}
+{"item_id": "q129", "judge": "orca-judge-large", "trial": 1, "first_shown": "A", "verdict": "A"}
+{"item_id": "q129", "judge": "orca-judge-large", "trial": 2, "first_shown": "A", "verdict": "B"}
+{"item_id": "q129", "judge": "wren-judge-small", "trial": 1, "first_shown": "A", "verdict": "B"}
+{"item_id": "q129", "judge": "wren-judge-small", "trial": 2, "first_shown": "A", "verdict": "A"}
+{"item_id": "q130", "judge": "orca-judge-large", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q130", "judge": "orca-judge-large", "trial": 2, "first_shown": "B", "verdict": "B"}
+{"item_id": "q130", "judge": "wren-judge-small", "trial": 1, "first_shown": "B", "verdict": "B"}
+{"item_id": "q130", "judge": "wren-judge-small", "trial": 2, "first_shown": "B", "verdict": "B"}

package/packs/ai-engineering/skills/judge-audit/evals/files/6/rag-judge/README.md

@@ -0,0 +1,36 @@
+# Relevance judge bake-off — Wayfinder search (Kestrel Docs)
+
+## What the judge is for
+
+Our nightly retrieval eval asks an LLM judge whether each retrieved passage is `relevant` or
+`irrelevant` to its query. From those labels we compute precision@5 per retriever build.
+
+**The gate:** a retriever change ships only if its precision@5 is no more than 1.0 point below
+the current production retriever's. So the judge's labels directly decide what ships.
+
+## Volume and price
+
+- Nightly run: ~20,000 query–passage judgments.
+- Price per 1,000 judgments (from the vendor's October price sheet):
+
+| Judge       | $ / 1k judgments |
+|-------------|------------------|
+| atlas-xl    | 4.80             |
+| atlas-m     | 1.10             |
+| atlas-nano  | 0.18             |
+
+atlas-xl is the current production judge.
+
+## This bake-off (`judgments.csv`)
+
+- 300 query–passage pairs: 30 queries (`q01`–`q30`) x their top-10 retrieved passages.
+- Each of the three judges labelled every pair once, same prompt, temperature 0.
+- `passage_tokens` is the passage length. The index currently mixes legacy ~900-token chunks
+  with newer short chunks (under ~400 tokens).
+- `expert_label`: Priya Halvorsen (search lead) hand-labelled 60 pairs drawn uniformly at random
+  from the 300. Blank means not expert-labelled. She was the only annotator.
+
+## What's next
+
+Q1 experiment: re-chunk the whole index to large chunks (~900 tokens) and compare the new
+retriever against production through this same gate.

package/packs/ai-engineering/skills/judge-audit/evals/files/6/rag-judge/judgments.csv

@@ -0,0 +1,301 @@
+passage_id,query_id,passage_tokens,atlas_xl,atlas_m,atlas_nano,expert_label
+p001,q01,247,relevant,relevant,relevant,
+p002,q01,214,relevant,relevant,relevant,
+p003,q01,717,relevant,relevant,relevant,
+p004,q01,145,relevant,relevant,relevant,relevant
+p005,q01,330,relevant,relevant,relevant,
+p006,q01,156,relevant,relevant,relevant,
+p007,q01,721,irrelevant,irrelevant,relevant,
+p008,q01,164,relevant,relevant,relevant,
+p009,q01,335,irrelevant,irrelevant,relevant,
+p010,q01,455,relevant,relevant,relevant,
+p011,q02,109,irrelevant,irrelevant,relevant,
+p012,q02,795,irrelevant,relevant,relevant,
+p013,q02,128,relevant,relevant,relevant,
+p014,q02,265,relevant,irrelevant,relevant,
+p015,q02,958,relevant,relevant,relevant,
+p016,q02,801,relevant,relevant,relevant,
+p017,q02,459,irrelevant,irrelevant,relevant,irrelevant
+p018,q02,346,relevant,relevant,relevant,
+p019,q02,339,relevant,relevant,relevant,
+p020,q02,239,relevant,irrelevant,relevant,relevant
+p021,q03,266,relevant,relevant,relevant,
+p022,q03,164,relevant,relevant,relevant,
+p023,q03,804,irrelevant,irrelevant,relevant,
+p024,q03,180,relevant,relevant,relevant,relevant
+p025,q03,366,relevant,relevant,relevant,
+p026,q03,964,irrelevant,irrelevant,relevant,irrelevant
+p027,q03,528,irrelevant,irrelevant,relevant,
+p028,q03,917,relevant,relevant,relevant,
+p029,q03,154,irrelevant,relevant,relevant,
+p030,q03,610,relevant,relevant,relevant,
+p031,q04,918,relevant,relevant,relevant,
+p032,q04,127,relevant,relevant,relevant,relevant
+p033,q04,922,irrelevant,irrelevant,relevant,
+p034,q04,972,irrelevant,relevant,relevant,relevant
+p035,q04,498,relevant,relevant,relevant,
+p036,q04,692,relevant,relevant,relevant,
+p037,q04,258,relevant,relevant,relevant,
+p038,q04,528,relevant,relevant,relevant,
+p039,q04,886,relevant,relevant,relevant,
+p040,q04,174,relevant,relevant,relevant,
+p041,q05,583,relevant,relevant,relevant,
+p042,q05,258,relevant,relevant,relevant,relevant
+p043,q05,437,irrelevant,relevant,relevant,
+p044,q05,469,relevant,relevant,relevant,
+p045,q05,920,relevant,relevant,relevant,
+p046,q05,500,irrelevant,irrelevant,relevant,
+p047,q05,449,relevant,relevant,relevant,relevant
+p048,q05,243,relevant,relevant,relevant,relevant
+p049,q05,668,relevant,relevant,relevant,relevant
+p050,q05,508,relevant,relevant,relevant,
+p051,q06,705,irrelevant,irrelevant,relevant,
+p052,q06,143,relevant,relevant,relevant,
+p053,q06,320,irrelevant,relevant,relevant,
+p054,q06,276,relevant,irrelevant,irrelevant,
+p055,q06,280,irrelevant,relevant,irrelevant,
+p056,q06,969,irrelevant,irrelevant,relevant,
+p057,q06,187,relevant,irrelevant,relevant,
+p058,q06,219,relevant,relevant,relevant,
+p059,q06,915,relevant,relevant,relevant,relevant
+p060,q06,349,relevant,relevant,relevant,
+p061,q07,100,relevant,relevant,relevant,
+p062,q07,525,relevant,relevant,relevant,
+p063,q07,302,irrelevant,relevant,relevant,
+p064,q07,191,relevant,relevant,relevant,
+p065,q07,222,relevant,relevant,relevant,
+p066,q07,691,irrelevant,irrelevant,relevant,
+p067,q07,569,irrelevant,irrelevant,relevant,irrelevant
+p068,q07,593,relevant,relevant,relevant,
+p069,q07,781,irrelevant,relevant,relevant,
+p070,q07,236,relevant,relevant,relevant,
+p071,q08,787,irrelevant,irrelevant,relevant,irrelevant
+p072,q08,893,irrelevant,irrelevant,relevant,
+p073,q08,784,irrelevant,relevant,irrelevant,
+p074,q08,128,relevant,relevant,relevant,
+p075,q08,753,relevant,relevant,relevant,
+p076,q08,247,relevant,relevant,relevant,relevant
+p077,q08,466,irrelevant,irrelevant,relevant,
+p078,q08,147,relevant,irrelevant,relevant,
+p079,q08,671,irrelevant,irrelevant,relevant,irrelevant
+p080,q08,291,irrelevant,irrelevant,irrelevant,
+p081,q09,557,relevant,relevant,relevant,relevant
+p082,q09,839,relevant,relevant,relevant,relevant
+p083,q09,130,relevant,relevant,relevant,
+p084,q09,359,irrelevant,relevant,irrelevant,
+p085,q09,157,relevant,irrelevant,relevant,
+p086,q09,614,relevant,relevant,relevant,
+p087,q09,299,relevant,relevant,relevant,
+p088,q09,149,irrelevant,irrelevant,irrelevant,irrelevant
+p089,q09,256,irrelevant,irrelevant,irrelevant,
+p090,q09,230,relevant,relevant,relevant,
+p091,q10,336,relevant,relevant,relevant,
+p092,q10,655,relevant,relevant,relevant,
+p093,q10,275,relevant,irrelevant,relevant,relevant
+p094,q10,179,relevant,relevant,relevant,
+p095,q10,228,relevant,irrelevant,relevant,
+p096,q10,194,relevant,relevant,relevant,
+p097,q10,483,relevant,relevant,relevant,
+p098,q10,686,relevant,relevant,relevant,relevant
+p099,q10,160,relevant,relevant,relevant,relevant
+p100,q10,315,relevant,relevant,relevant,
+p101,q11,717,irrelevant,irrelevant,relevant,
+p102,q11,522,relevant,relevant,irrelevant,
+p103,q11,436,relevant,relevant,relevant,
+p104,q11,358,relevant,relevant,relevant,
+p105,q11,372,irrelevant,irrelevant,irrelevant,irrelevant
+p106,q11,262,irrelevant,irrelevant,relevant,relevant
+p107,q11,891,irrelevant,irrelevant,irrelevant,
+p108,q11,123,relevant,relevant,relevant,relevant
+p109,q11,742,relevant,relevant,relevant,relevant
+p110,q11,317,relevant,irrelevant,relevant,irrelevant
+p111,q12,738,relevant,relevant,relevant,relevant
+p112,q12,172,irrelevant,irrelevant,relevant,
+p113,q12,739,relevant,relevant,relevant,
+p114,q12,674,irrelevant,irrelevant,irrelevant,irrelevant
+p115,q12,330,irrelevant,irrelevant,relevant,
+p116,q12,704,irrelevant,irrelevant,relevant,
+p117,q12,438,relevant,relevant,relevant,
+p118,q12,912,irrelevant,irrelevant,relevant,
+p119,q12,280,irrelevant,irrelevant,irrelevant,
+p120,q12,781,relevant,irrelevant,irrelevant,
+p121,q13,539,relevant,relevant,relevant,
+p122,q13,252,irrelevant,irrelevant,irrelevant,
+p123,q13,264,relevant,relevant,relevant,
+p124,q13,973,relevant,relevant,relevant,
+p125,q13,755,relevant,irrelevant,relevant,
+p126,q13,540,relevant,relevant,relevant,
+p127,q13,359,irrelevant,relevant,irrelevant,irrelevant
+p128,q13,186,relevant,relevant,relevant,
+p129,q13,599,relevant,relevant,relevant,
+p130,q13,626,relevant,irrelevant,relevant,
+p131,q14,543,relevant,relevant,irrelevant,
+p132,q14,320,relevant,relevant,relevant,
+p133,q14,272,irrelevant,relevant,relevant,
+p134,q14,857,relevant,relevant,relevant,
+p135,q14,150,relevant,relevant,relevant,
+p136,q14,189,relevant,relevant,relevant,
+p137,q14,141,relevant,relevant,relevant,
+p138,q14,154,irrelevant,irrelevant,irrelevant,irrelevant
+p139,q14,109,irrelevant,relevant,relevant,
+p140,q14,372,relevant,relevant,relevant,
+p141,q15,269,relevant,relevant,relevant,relevant
+p142,q15,607,relevant,relevant,relevant,
+p143,q15,729,relevant,relevant,relevant,
+p144,q15,100,relevant,relevant,irrelevant,
+p145,q15,160,relevant,relevant,relevant,
+p146,q15,149,relevant,relevant,relevant,
+p147,q15,171,relevant,relevant,relevant,
+p148,q15,574,relevant,relevant,relevant,
+p149,q15,314,relevant,irrelevant,relevant,
+p150,q15,271,relevant,relevant,relevant,
+p151,q16,108,relevant,relevant,relevant,
+p152,q16,183,relevant,relevant,relevant,
+p153,q16,115,irrelevant,irrelevant,irrelevant,
+p154,q16,106,relevant,relevant,relevant,
+p155,q16,179,relevant,irrelevant,relevant,
+p156,q16,473,relevant,relevant,relevant,
+p157,q16,320,relevant,relevant,relevant,
+p158,q16,357,relevant,relevant,relevant,
+p159,q16,747,relevant,irrelevant,relevant,
+p160,q16,368,relevant,relevant,relevant,relevant
+p161,q17,561,irrelevant,irrelevant,relevant,irrelevant
+p162,q17,216,relevant,relevant,relevant,
+p163,q17,719,relevant,relevant,relevant,
+p164,q17,512,relevant,relevant,relevant,irrelevant
+p165,q17,773,relevant,relevant,relevant,relevant
+p166,q17,362,relevant,relevant,relevant,relevant
+p167,q17,739,relevant,relevant,relevant,
+p168,q17,960,irrelevant,irrelevant,relevant,
+p169,q17,253,relevant,relevant,relevant,
+p170,q17,220,relevant,relevant,relevant,
+p171,q18,352,relevant,relevant,relevant,
+p172,q18,219,irrelevant,irrelevant,irrelevant,
+p173,q18,131,relevant,relevant,relevant,
+p174,q18,338,relevant,relevant,relevant,
+p175,q18,94,relevant,relevant,relevant,
+p176,q18,659,irrelevant,irrelevant,relevant,
+p177,q18,719,relevant,relevant,relevant,
+p178,q18,245,relevant,relevant,relevant,
+p179,q18,215,relevant,irrelevant,relevant,
+p180,q18,531,relevant,relevant,relevant,
+p181,q19,351,relevant,irrelevant,relevant,
+p182,q19,723,relevant,relevant,relevant,
+p183,q19,298,irrelevant,relevant,irrelevant,
+p184,q19,455,relevant,relevant,relevant,
+p185,q19,205,irrelevant,irrelevant,irrelevant,
+p186,q19,243,relevant,irrelevant,irrelevant,
+p187,q19,254,relevant,relevant,relevant,
+p188,q19,366,relevant,relevant,relevant,relevant
+p189,q19,914,relevant,irrelevant,relevant,
+p190,q19,749,relevant,relevant,relevant,
+p191,q20,813,relevant,relevant,relevant,
+p192,q20,312,relevant,irrelevant,relevant,
+p193,q20,501,irrelevant,irrelevant,relevant,
+p194,q20,324,irrelevant,irrelevant,irrelevant,
+p195,q20,220,relevant,relevant,relevant,
+p196,q20,852,relevant,relevant,relevant,
+p197,q20,767,irrelevant,relevant,relevant,
+p198,q20,715,relevant,relevant,relevant,
+p199,q20,452,relevant,relevant,relevant,
+p200,q20,615,relevant,relevant,relevant,
+p201,q21,281,irrelevant,relevant,irrelevant,
+p202,q21,252,relevant,relevant,relevant,relevant
+p203,q21,172,relevant,relevant,relevant,
+p204,q21,272,relevant,relevant,relevant,relevant
+p205,q21,941,irrelevant,irrelevant,relevant,irrelevant
+p206,q21,173,relevant,relevant,relevant,relevant
+p207,q21,962,relevant,relevant,relevant,
+p208,q21,101,relevant,irrelevant,relevant,
+p209,q21,269,relevant,relevant,irrelevant,relevant
+p210,q21,594,relevant,relevant,relevant,
+p211,q22,594,relevant,relevant,relevant,
+p212,q22,105,relevant,relevant,relevant,
+p213,q22,160,irrelevant,irrelevant,relevant,
+p214,q22,104,irrelevant,irrelevant,irrelevant,
+p215,q22,183,relevant,relevant,relevant,
+p216,q22,177,irrelevant,irrelevant,irrelevant,
+p217,q22,289,relevant,relevant,irrelevant,
+p218,q22,292,relevant,relevant,relevant,
+p219,q22,809,irrelevant,relevant,relevant,irrelevant
+p220,q22,349,irrelevant,irrelevant,irrelevant,
+p221,q23,95,relevant,relevant,relevant,
+p222,q23,169,relevant,relevant,relevant,
+p223,q23,180,irrelevant,irrelevant,irrelevant,
+p224,q23,128,relevant,relevant,relevant,
+p225,q23,283,relevant,relevant,relevant,
+p226,q23,360,relevant,relevant,relevant,
+p227,q23,347,irrelevant,irrelevant,irrelevant,
+p228,q23,279,relevant,relevant,relevant,relevant
+p229,q23,192,irrelevant,relevant,irrelevant,
+p230,q23,361,relevant,relevant,relevant,
+p231,q24,339,relevant,relevant,relevant,
+p232,q24,372,relevant,relevant,relevant,
+p233,q24,362,irrelevant,irrelevant,irrelevant,
+p234,q24,762,irrelevant,irrelevant,relevant,
+p235,q24,588,relevant,relevant,relevant,relevant
+p236,q24,796,relevant,relevant,relevant,relevant
+p237,q24,332,relevant,relevant,relevant,
+p238,q24,268,relevant,relevant,relevant,
+p239,q24,806,relevant,relevant,relevant,relevant
+p240,q24,708,irrelevant,irrelevant,relevant,
+p241,q25,698,irrelevant,relevant,relevant,
+p242,q25,578,relevant,relevant,relevant,
+p243,q25,805,relevant,irrelevant,relevant,
+p244,q25,814,relevant,irrelevant,irrelevant,
+p245,q25,100,irrelevant,relevant,irrelevant,
+p246,q25,860,relevant,relevant,relevant,
+p247,q25,189,relevant,relevant,relevant,relevant
+p248,q25,342,relevant,relevant,relevant,
+p249,q25,541,relevant,relevant,relevant,
+p250,q25,178,irrelevant,relevant,relevant,
+p251,q26,239,relevant,relevant,relevant,
+p252,q26,247,relevant,relevant,relevant,
+p253,q26,317,relevant,irrelevant,relevant,
+p254,q26,575,irrelevant,irrelevant,relevant,irrelevant
+p255,q26,91,relevant,relevant,relevant,
+p256,q26,189,relevant,relevant,relevant,
+p257,q26,183,relevant,relevant,relevant,
+p258,q26,807,relevant,relevant,relevant,
+p259,q26,730,relevant,relevant,relevant,relevant
+p260,q26,379,irrelevant,irrelevant,irrelevant,irrelevant
+p261,q27,576,relevant,relevant,relevant,relevant
+p262,q27,184,relevant,relevant,relevant,
+p263,q27,556,relevant,irrelevant,relevant,
+p264,q27,270,irrelevant,irrelevant,irrelevant,
+p265,q27,777,relevant,relevant,relevant,
+p266,q27,781,relevant,relevant,relevant,
+p267,q27,96,irrelevant,irrelevant,irrelevant,
+p268,q27,117,relevant,relevant,relevant,
+p269,q27,640,irrelevant,irrelevant,relevant,
+p270,q27,126,relevant,irrelevant,relevant,
+p271,q28,522,relevant,relevant,relevant,relevant
+p272,q28,455,irrelevant,irrelevant,relevant,
+p273,q28,234,relevant,irrelevant,relevant,
+p274,q28,301,irrelevant,irrelevant,irrelevant,
+p275,q28,847,irrelevant,relevant,relevant,
+p276,q28,321,irrelevant,irrelevant,irrelevant,
+p277,q28,685,relevant,relevant,relevant,relevant
+p278,q28,220,relevant,relevant,relevant,
+p279,q28,105,irrelevant,irrelevant,irrelevant,
+p280,q28,235,irrelevant,irrelevant,irrelevant,
+p281,q29,757,irrelevant,irrelevant,relevant,
+p282,q29,890,relevant,relevant,relevant,
+p283,q29,306,relevant,relevant,relevant,irrelevant
+p284,q29,551,irrelevant,irrelevant,relevant,
+p285,q29,738,relevant,relevant,relevant,relevant
+p286,q29,735,relevant,relevant,relevant,
+p287,q29,199,relevant,relevant,relevant,
+p288,q29,374,relevant,relevant,irrelevant,
+p289,q29,481,relevant,relevant,relevant,
+p290,q29,645,relevant,relevant,relevant,
+p291,q30,124,relevant,relevant,relevant,
+p292,q30,369,relevant,relevant,relevant,relevant
+p293,q30,548,relevant,relevant,relevant,
+p294,q30,208,irrelevant,irrelevant,irrelevant,
+p295,q30,663,relevant,relevant,relevant,
+p296,q30,128,relevant,relevant,relevant,
+p297,q30,187,relevant,relevant,relevant,
+p298,q30,282,relevant,relevant,relevant,relevant
+p299,q30,215,relevant,relevant,relevant,relevant
+p300,q30,118,relevant,relevant,irrelevant,

package/packs/ai-engineering/skills/judge-audit/evals/files/7/moderation/gold_labels.csv

@@ -0,0 +1,101 @@
+item_id,gold_label,policy_category
+fp-30011,ok,
+fp-30039,ok,
+fp-30134,ok,
+fp-30209,ok,
+fp-30281,ok,
+fp-30437,ok,
+fp-30464,ok,
+fp-30575,ok,
+fp-30646,ok,
+fp-30855,ok,
+fp-30916,ok,
+fp-30923,violation,harassment
+fp-31019,ok,
+fp-31177,ok,
+fp-31250,violation,spam
+fp-31299,ok,
+fp-31326,ok,
+fp-31428,ok,
+fp-31437,ok,
+fp-31504,ok,
+fp-31555,ok,
+fp-31559,ok,
+fp-31635,ok,
+fp-31744,ok,
+fp-31885,ok,
+fp-31921,ok,
+fp-31931,ok,
+fp-31990,ok,
+fp-32266,ok,
+fp-32383,ok,
+fp-32399,ok,
+fp-32415,ok,
+fp-32829,ok,
+fp-32996,ok,
+fp-33133,ok,
+fp-33140,ok,
+fp-33164,ok,
+fp-33232,ok,
+fp-33278,ok,
+fp-33358,ok,
+fp-33474,ok,
+fp-33590,ok,
+fp-33696,ok,
+fp-33896,ok,
+fp-33930,ok,
+fp-33940,ok,
+fp-34060,ok,
+fp-34140,ok,
+fp-34194,ok,
+fp-34265,violation,self_harm
+fp-34455,ok,
+fp-34586,ok,
+fp-34789,ok,
+fp-34886,ok,
+fp-35267,ok,
+fp-35339,ok,
+fp-35388,ok,
+fp-35431,ok,
+fp-35757,ok,
+fp-35782,violation,harassment
+fp-35866,ok,
+fp-36296,ok,
+fp-36366,ok,
+fp-36596,ok,
+fp-36620,ok,
+fp-36807,ok,
+fp-37091,ok,
+fp-37170,ok,
+fp-37286,ok,
+fp-37556,ok,
+fp-37694,ok,
+fp-37725,ok,
+fp-37737,ok,
+fp-37802,ok,
+fp-37887,ok,
+fp-37954,ok,
+fp-38009,ok,
+fp-38123,ok,
+fp-38145,ok,
+fp-38202,ok,
+fp-38257,ok,
+fp-38311,ok,
+fp-38678,ok,
+fp-38699,ok,
+fp-38702,ok,
+fp-38748,violation,spam
+fp-38814,ok,
+fp-38826,ok,
+fp-39027,ok,
+fp-39044,ok,
+fp-39107,ok,
+fp-39140,ok,
+fp-39176,ok,
+fp-39221,ok,
+fp-39382,ok,
+fp-39430,ok,
+fp-39451,ok,
+fp-39543,ok,
+fp-39753,ok,
+fp-39922,ok,