aspera-grader-kit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,11 @@
1
+ """aspera-grader-kit: a small pattern for grading the files knowledge-work agents produce.
2
+
3
+ Inspect the file into facts, run checks that each test one thing and explain themselves, then score with a rubric
4
+ whose weights sit in plain JSON next to the task.
5
+ """
6
+ from .checks import Check, Context, Result, check
7
+ from .grade import grade, render
8
+ from .inspect import inspect
9
+
10
+ __all__ = ["Check", "Context", "Result", "check", "grade", "render", "inspect"]
11
+ __version__ = "0.1.0"
@@ -0,0 +1,18 @@
1
+ """python -m aspera_grader_kit <task_dir> <output_file> [--json]"""
2
+ import argparse, json, sys
3
+
4
+ from .grade import grade, render
5
+
6
+
7
+ def main(argv=None):
8
+ ap = argparse.ArgumentParser(prog="aspera_grader_kit", description="Grade one output file against one task folder.")
9
+ ap.add_argument("task_dir"); ap.add_argument("output")
10
+ ap.add_argument("--json", action="store_true", help="print the full report as JSON")
11
+ a = ap.parse_args(argv)
12
+ rep = grade(a.output, a.task_dir)
13
+ print(json.dumps(rep, indent=2, default=str) if a.json else render(rep))
14
+ return 0
15
+
16
+
17
+ if __name__ == "__main__":
18
+ sys.exit(main())
@@ -0,0 +1,106 @@
1
+ """Checks: small functions that each test one thing about an output and say why.
2
+
3
+ A check gets a Context (the inspected facts, the task and its answer key) and returns a Result. A check that tests a
4
+ list of things (five key values, three required sections) returns one element per item, so the rubric can count every
5
+ item instead of treating the whole list as one pass or fail.
6
+
7
+ from aspera_grader_kit import check, Result
8
+
9
+ @check("R1", "requirements", "Every key value appears in the memo")
10
+ def key_values(ctx):
11
+ return Result.items([(v["item"], str(v["value"]) in ctx.text, v["value"]) for v in ctx.key["values"]])
12
+ """
13
+ from dataclasses import dataclass, field
14
+ from typing import Callable, Dict, List, Optional
15
+
16
+
17
+ @dataclass
18
+ class Result:
19
+ status: str # pass | fail | na | error
20
+ evidence: str = ""
21
+ elements: Optional[List[dict]] = None
22
+
23
+ @classmethod
24
+ def ok(cls, evidence=""): return cls("pass", evidence)
25
+
26
+ @classmethod
27
+ def fail(cls, evidence=""): return cls("fail", evidence)
28
+
29
+ @classmethod
30
+ def na(cls, evidence="does not apply"): return cls("na", evidence)
31
+
32
+ @classmethod
33
+ def when(cls, condition, evidence=""):
34
+ return cls("pass" if condition else "fail", evidence)
35
+
36
+ @classmethod
37
+ def items(cls, triples):
38
+ """One element per (item, passed, evidence). Passes only when every item passes."""
39
+ els = [{"item": str(i), "status": "pass" if ok else "fail", "evidence": str(e)} for i, ok, e in triples]
40
+ if not els: return cls.na("no items to test")
41
+ n = sum(e["status"] == "pass" for e in els)
42
+ return cls("pass" if n == len(els) else "fail", "%d of %d items met" % (n, len(els)), els)
43
+
44
+ def units(self):
45
+ """(passed, total) units for scoring, or None when the check does not count."""
46
+ if self.status not in ("pass", "fail"): return None
47
+ if self.elements: return sum(e["status"] == "pass" for e in self.elements), len(self.elements)
48
+ return (1, 1) if self.status == "pass" else (0, 1)
49
+
50
+ def to_dict(self):
51
+ d = {"status": self.status, "evidence": self.evidence}
52
+ if self.elements is not None: d["elements"] = self.elements
53
+ return d
54
+
55
+
56
+ @dataclass
57
+ class Check:
58
+ id: str
59
+ category: str
60
+ description: str
61
+ fn: Callable
62
+ weight: float = 1.0
63
+
64
+
65
+ @dataclass
66
+ class Context:
67
+ path: str
68
+ facts: dict
69
+ task: dict = field(default_factory=dict)
70
+ key: dict = field(default_factory=dict)
71
+
72
+ @property
73
+ def text(self): return self.facts.get("text", "")
74
+
75
+
76
+ _REGISTRY: Dict[str, List[Check]] = {}
77
+
78
+
79
+ def check(id, category, description, weight=1.0, registry="default"):
80
+ """Decorator that registers a check function under a registry name (one per task, usually)."""
81
+ def wrap(fn):
82
+ _REGISTRY.setdefault(registry, []).append(Check(id, category, description, fn, weight))
83
+ return fn
84
+ return wrap
85
+
86
+
87
+ def registered(registry="default"):
88
+ return list(_REGISTRY.get(registry, []))
89
+
90
+
91
+ def clear(registry="default"):
92
+ _REGISTRY.pop(registry, None)
93
+
94
+
95
+ def run(checks, ctx):
96
+ """Run every check; an exception inside a check becomes status 'error' with the message as evidence."""
97
+ out = {}
98
+ for c in checks:
99
+ try:
100
+ r = c.fn(ctx)
101
+ if not isinstance(r, Result):
102
+ raise TypeError("check %s returned %r, not a Result" % (c.id, type(r).__name__))
103
+ except Exception as e: # a broken check must never crash the grade
104
+ r = Result("error", "%s: %s" % (type(e).__name__, e))
105
+ out[c.id] = r
106
+ return out
@@ -0,0 +1,63 @@
1
+ """Grade one output file against one task folder.
2
+
3
+ A task folder holds:
4
+ task.md the brief the agent was given
5
+ task.json {"id", "format", "output"} (format: docx | pptx | xlsx)
6
+ key.json the answer key (values, required items); never shown to the agent
7
+ rubric.json categories, points and gates
8
+ checks.py the task's checks, registered with @check(..., registry=<task id>)
9
+ """
10
+ import importlib.util, json, os
11
+
12
+ from . import checks as C
13
+ from . import rubric as R
14
+ from .inspect import inspect
15
+
16
+
17
+ def _load_json(task_dir, name, default=None):
18
+ p = os.path.join(task_dir, name)
19
+ if not os.path.exists(p): return default
20
+ with open(p) as f: return json.load(f)
21
+
22
+
23
+ def load_task_checks(task_dir, task_id):
24
+ C.clear(task_id)
25
+ spec = importlib.util.spec_from_file_location("task_checks_%s" % task_id.replace("-", "_"), os.path.join(task_dir, "checks.py"))
26
+ mod = importlib.util.module_from_spec(spec); spec.loader.exec_module(mod)
27
+ return C.registered(task_id)
28
+
29
+
30
+ def grade(output_path, task_dir):
31
+ task = _load_json(task_dir, "task.json"); key = _load_json(task_dir, "key.json", {})
32
+ rubric = R.load(os.path.join(task_dir, "rubric.json"))
33
+ checks = load_task_checks(task_dir, task["id"])
34
+ report = {"task": task["id"], "output": os.path.basename(output_path) if output_path else None, "checks": {}}
35
+ gates = []
36
+ if not output_path or not os.path.exists(output_path):
37
+ gates.append({"id": "missing", "evidence": "no output file"})
38
+ else:
39
+ try:
40
+ facts = inspect(output_path)
41
+ except Exception as e:
42
+ gates.append({"id": "unreadable", "evidence": "%s: %s" % (type(e).__name__, e)})
43
+ else:
44
+ results = C.run(checks, C.Context(output_path, facts, task, key))
45
+ report["checks"] = {c.id: dict(results[c.id].to_dict(), category=c.category, description=c.description) for c in checks}
46
+ report.update(R.score(results, checks, rubric, gates))
47
+ return report
48
+ report.update(R.score({}, checks, rubric, gates)) # nothing to check: every category is empty, so the total is 0
49
+ report["gates"] = gates
50
+ return report
51
+
52
+
53
+ def render(report):
54
+ """Plain-text summary of a report for the terminal."""
55
+ lines = ["%s %s -> %.1f / 100" % (report["task"], report.get("output"), report["total"])]
56
+ for g in report.get("gates", []): lines.append(" GATE %s: %s" % (g["id"], g["evidence"]))
57
+ for k, c in report.get("categories", {}).items():
58
+ if c["possible"]: lines.append(" %-20s %5.1f / %5.1f" % (c["label"], c["earned"], c["possible"]))
59
+ for cid, r in report.get("checks", {}).items():
60
+ lines.append(" [%s] %-4s %s (%s)" % (r["status"].upper()[:4].ljust(4), cid, r["description"], r["evidence"]))
61
+ for e in r.get("elements") or []:
62
+ if e["status"] != "pass": lines.append(" - missed: %s (%s)" % (e["item"], e["evidence"]))
63
+ return "\n".join(lines)
@@ -0,0 +1,14 @@
1
+ """File inspectors: turn an Office file into a plain dict of facts. Checks test facts, never raw XML."""
2
+ import os
3
+
4
+ from . import docx, pptx, xlsx
5
+
6
+ INSPECTORS = {".docx": docx.inspect, ".pptx": pptx.inspect, ".xlsx": xlsx.inspect}
7
+
8
+
9
+ def inspect(path, **kw):
10
+ """Facts for one file, picked by extension. Raises ValueError for a format with no inspector."""
11
+ ext = os.path.splitext(path)[1].lower()
12
+ if ext not in INSPECTORS:
13
+ raise ValueError("no inspector for %s files" % ext)
14
+ return INSPECTORS[ext](path, **kw)
@@ -0,0 +1,162 @@
1
+ """Read a .docx into plain facts a check can test: body blocks in order (paragraphs with style,
2
+ numbering and run formatting; tables with header-repeat, shading, cell margins, alignment), tracked changes, comments,
3
+ content controls, protection, fields (PAGE, NUMPAGES, TOC, SEQ), headers/footers, footnotes, images, hyperlinks."""
4
+ import re, zipfile
5
+ from lxml import etree
6
+
7
+ W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
8
+ NS = {"w": W, "w14": "http://schemas.microsoft.com/office/word/2010/wordml",
9
+ "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships",
10
+ "a": "http://schemas.openxmlformats.org/drawingml/2006/main", "wp": "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"}
11
+ REL = "{http://schemas.openxmlformats.org/package/2006/relationships}Relationship"
12
+ q = lambda t: "{%s}%s" % (NS[t.split(":")[0]], t.split(":")[1])
13
+ wv = lambda el, path: (el.find(path, NS).get(q("w:val")) if el is not None and el.find(path, NS) is not None else None)
14
+
15
+
16
+ def text_of(el, deleted=False):
17
+ out = []
18
+ for t in el.iter(q("w:t"), q("w:delText"), q("w:tab"), q("w:br")):
19
+ if t.tag == q("w:t"): out.append(t.text or "")
20
+ elif t.tag == q("w:delText") and deleted: out.append(t.text or "")
21
+ elif t.tag == q("w:tab"): out.append("\t")
22
+ return "".join(out)
23
+
24
+
25
+ def styles(z):
26
+ if "word/styles.xml" not in z.namelist(): return {}
27
+ root = etree.fromstring(z.read("word/styles.xml"))
28
+ return {s.get(q("w:styleId")): (s.find("w:name", NS).get(q("w:val")) if s.find("w:name", NS) is not None else s.get(q("w:styleId")))
29
+ for s in root.iter(q("w:style"))}
30
+
31
+
32
+ def fields_in(el):
33
+ """Field codes (simple and complex) in an element, upper-cased first word: PAGE, NUMPAGES, TOC, SEQ, REF ..."""
34
+ codes = [f.get(q("w:instr")) or "" for f in el.iter(q("w:fldSimple"))]
35
+ codes += [(t.text or "") for t in el.iter(q("w:instrText"))]
36
+ return [c.strip() for c in codes if c and c.strip()]
37
+
38
+
39
+ def para_facts(p, S):
40
+ ppr = p.find("w:pPr", NS)
41
+ sid = wv(ppr, "w:pStyle") if ppr is not None else None
42
+ num = ppr.find("w:numPr", NS) if ppr is not None else None
43
+ runs = []
44
+ for r in p.iter(q("w:r")):
45
+ rpr = r.find("w:rPr", NS); t = text_of(r)
46
+ if not t.strip(): continue
47
+ b = rpr is not None and rpr.find("w:b", NS) is not None and (rpr.find("w:b", NS).get(q("w:val")) not in ("0", "false"))
48
+ sz = wv(rpr, "w:sz"); col = wv(rpr, "w:color"); font = rpr.find("w:rFonts", NS).get(q("w:ascii")) if rpr is not None and rpr.find("w:rFonts", NS) is not None else None
49
+ sup = wv(rpr, "w:vertAlign") == "superscript" if rpr is not None else False
50
+ runs.append({"t": t[:200], "b": b, "sz": int(sz) / 2 if sz and sz.isdigit() else None, "color": col, "font": font, "sup": sup})
51
+ return {"type": "p", "style": S.get(sid, sid) if sid else "Normal", "text": text_of(p)[:1200],
52
+ "num": {"numId": wv(num, "w:numId"), "ilvl": wv(num, "w:ilvl")} if num is not None else None,
53
+ "align": wv(ppr, "w:jc") if ppr is not None else None, "runs": runs[:40],
54
+ "fields": fields_in(p), "has_drawing": p.find(".//w:drawing", NS) is not None or p.find(".//w:pict", NS) is not None,
55
+ "in_del": text_of(p, deleted=True) != text_of(p)}
56
+
57
+
58
+ def table_facts(t, S):
59
+ rows = []
60
+ for tr in t.findall("w:tr", NS):
61
+ trpr = tr.find("w:trPr", NS)
62
+ cells = []
63
+ for tc in tr.findall("w:tc", NS):
64
+ tcpr = tc.find("w:tcPr", NS)
65
+ shd = tcpr.find("w:shd", NS) if tcpr is not None else None
66
+ mar = tcpr.find("w:tcMar", NS) if tcpr is not None else None
67
+ ps = tc.findall("w:p", NS)
68
+ bold = any(r.find("w:rPr/w:b", NS) is not None for p in ps for r in p.iter(q("w:r")) if text_of(r).strip())
69
+ colors = sorted({wv(r.find("w:rPr", NS), "w:color") for p in ps for r in p.iter(q("w:r")) if text_of(r).strip() and r.find("w:rPr", NS) is not None} - {None})
70
+ cells.append({"text": " ".join(text_of(p) for p in ps).strip()[:200], "fill": shd.get(q("w:fill")) if shd is not None else None,
71
+ "align": next((wv(p.find("w:pPr", NS), "w:jc") for p in ps if p.find("w:pPr/w:jc", NS) is not None), None),
72
+ "margins": {e.tag.split("}")[1]: e.get(q("w:w")) for e in mar} if mar is not None else None, "bold": bold, "colors": colors})
73
+ spans = sum(int(wv(tc.find("w:tcPr", NS), "w:gridSpan") or 1) if tc.find("w:tcPr", NS) is not None else 1 for tc in tr.findall("w:tc", NS))
74
+ sizes = [int(sz.get(q("w:val"))) / 2 for sz in tr.iter(q("w:sz")) if sz.get(q("w:val"), "").isdigit() and sz.getparent().tag == q("w:rPr")]
75
+ rows.append({"header": trpr is not None and trpr.find("w:tblHeader", NS) is not None, "cells": cells, "span": spans, "min_pt": min(sizes) if sizes else None})
76
+ tpr = t.find("w:tblPr", NS)
77
+ tmar = tpr.find("w:tblCellMar", NS) if tpr is not None else None
78
+ grid = [int(g.get(q("w:w")) or 0) for g in t.findall("w:tblGrid/w:gridCol", NS)]
79
+ return {"type": "table", "grid_w_twips": sum(grid), "style": S.get(wv(tpr, "w:tblStyle"), wv(tpr, "w:tblStyle")) if tpr is not None else None,
80
+ "n_rows": len(rows), "n_cols": max((len(r["cells"]) for r in rows), default=0),
81
+ "table_cell_margins": {e.tag.split("}")[1]: e.get(q("w:w")) for e in tmar} if tmar is not None else None,
82
+ "rows": rows[:60]}
83
+
84
+
85
+ def inspect(path):
86
+ z = zipfile.ZipFile(path); names = z.namelist(); S = styles(z)
87
+ doc = etree.fromstring(z.read("word/document.xml")); body = doc.find("w:body", NS)
88
+ blocks = []
89
+ for el in body:
90
+ if el.tag == q("w:p"): blocks.append(para_facts(el, S))
91
+ elif el.tag == q("w:tbl"): blocks.append(table_facts(el, S))
92
+ elif el.tag == q("w:sdt"):
93
+ for p in el.iter(q("w:p")): blocks.append(dict(para_facts(p, S), sdt_block=True))
94
+ ins = [{"author": e.get(q("w:author")), "text": text_of(e)[:300]} for e in doc.iter(q("w:ins"))]
95
+ dels = [{"author": e.get(q("w:author")), "text": "".join((t.text or "") for t in e.iter(q("w:delText")))[:300]} for e in doc.iter(q("w:del"))]
96
+ comments = []
97
+ if "word/comments.xml" in names:
98
+ for c in etree.fromstring(z.read("word/comments.xml")).iter(q("w:comment")):
99
+ comments.append({"id": c.get(q("w:id")), "author": c.get(q("w:author")), "text": text_of(c)[:500]})
100
+ anchors = {}
101
+ for p in doc.iter(q("w:p")):
102
+ for s in p.iter(q("w:commentRangeStart"), q("w:commentReference")):
103
+ anchors.setdefault(s.get(q("w:id")), text_of(p)[:160])
104
+ for c in comments: c["anchored_in"] = anchors.get(c["id"])
105
+ sdts = []
106
+ for s in doc.iter(q("w:sdt")):
107
+ pr = s.find("w:sdtPr", NS)
108
+ if pr is None: continue
109
+ kind = ("dropdown" if pr.find("w:dropDownList", NS) is not None else "combobox" if pr.find("w:comboBox", NS) is not None
110
+ else "date" if pr.find("w:date", NS) is not None else "checkbox" if pr.find("w14:checkbox", NS) is not None
111
+ else "richtext" if pr.find("w:richText", NS) is not None else "picture" if pr.find("w:picture", NS) is not None
112
+ else "text" if pr.find("w:text", NS) is not None else "docpart" if pr.find("w:docPartObj", NS) is not None else "other")
113
+ items = [li.get(q("w:displayText")) or li.get(q("w:value")) for li in pr.iter(q("w:listItem"))]
114
+ df = pr.find("w:date/w:dateFormat", NS)
115
+ sdts.append({"kind": kind, "tag": wv(pr, "w:tag"), "alias": wv(pr, "w:alias"), "placeholder": pr.find("w:showingPlcHdr", NS) is not None or pr.find("w:placeholder", NS) is not None,
116
+ "items": items, "date_format": df.get(q("w:val")) if df is not None else None,
117
+ "text": text_of(s.find("w:sdtContent", NS))[:120] if s.find("w:sdtContent", NS) is not None else ""})
118
+ prot = None
119
+ if "word/settings.xml" in names:
120
+ dp = etree.fromstring(z.read("word/settings.xml")).find("w:documentProtection", NS)
121
+ if dp is not None: prot = {"edit": dp.get(q("w:edit")), "enforcement": dp.get(q("w:enforcement")), "password": any("hash" in k.lower() or "salt" in k.lower() for k in dp.attrib)}
122
+ hf = {}
123
+ for n in names:
124
+ m = re.match(r"word/(header|footer)\d+\.xml$", n)
125
+ if m:
126
+ root = etree.fromstring(z.read(n)); hf.setdefault(m.group(1), []).append({"text": text_of(root)[:300], "fields": fields_in(root)})
127
+ sect = body.find("w:sectPr", NS)
128
+ pg = {}
129
+ if sect is not None:
130
+ ps, pm = sect.find("w:pgSz", NS), sect.find("w:pgMar", NS)
131
+ if ps is not None: pg = {"w_twips": int(ps.get(q("w:w")) or 0), "h_twips": int(ps.get(q("w:h")) or 0)}
132
+ if pm is not None: pg.update({k: int(pm.get(q("w:" + k)) or 0) for k in ("left", "right", "top", "bottom")})
133
+ foot = []
134
+ if "word/footnotes.xml" in names:
135
+ for f in etree.fromstring(z.read("word/footnotes.xml")).iter(q("w:footnote")):
136
+ if f.get(q("w:type")) in (None, "normal"): foot.append(text_of(f)[:200])
137
+ links = []
138
+ rels = {}
139
+ if "word/_rels/document.xml.rels" in names:
140
+ for r in etree.fromstring(z.read("word/_rels/document.xml.rels")).iter(REL):
141
+ rels[r.get("Id")] = (r.get("Target"), r.get("TargetMode"))
142
+ for h in doc.iter(q("w:hyperlink")):
143
+ rid = h.get(q("r:id")); links.append({"text": text_of(h)[:120], "target": rels.get(rid, (None,))[0] if rid else None, "anchor": h.get(q("w:anchor"))})
144
+ fields = fields_in(doc)
145
+ images = sum(1 for _ in doc.iter(q("w:drawing"))) + sum(1 for _ in doc.iter(q("w:pict")))
146
+ dd = {}
147
+ if "word/styles.xml" in names:
148
+ sr = etree.fromstring(z.read("word/styles.xml"))
149
+ rp = sr.find("w:docDefaults/w:rPrDefault/w:rPr", NS)
150
+ if rp is not None:
151
+ dd["sz"] = int(wv(rp, "w:sz")) / 2 if wv(rp, "w:sz") and wv(rp, "w:sz").isdigit() else None
152
+ f = rp.find("w:rFonts", NS); dd["font"] = f.get(q("w:ascii")) if f is not None else None
153
+ for st in sr.iter(q("w:style")):
154
+ if st.get(q("w:styleId")) == "Normal":
155
+ rp = st.find("w:rPr", NS)
156
+ if rp is not None and wv(rp, "w:sz"): dd["normal_sz"] = int(wv(rp, "w:sz")) / 2
157
+ if rp is not None and rp.find("w:rFonts", NS) is not None: dd["normal_font"] = rp.find("w:rFonts", NS).get(q("w:ascii"))
158
+ num_xml = z.read("word/numbering.xml").decode("utf-8", "ignore") if "word/numbering.xml" in names else ""
159
+ return {"blocks": blocks, "ins": ins, "del": dels, "comments": comments, "sdts": sdts, "protection": prot, "headers_footers": hf,
160
+ "page": pg, "footnotes": foot, "hyperlinks": links, "fields": fields, "images": images,
161
+ "has_numbering_part": bool(num_xml), "defaults": dd, "styles_used": sorted({b["style"] for b in blocks if b["type"] == "p"}),
162
+ "text": "\n".join(b["text"] if b["type"] == "p" else "\n".join(" | ".join(c["text"] for c in r["cells"]) for r in b["rows"]) for b in blocks)}
@@ -0,0 +1,132 @@
1
+ """Read a .pptx into plain facts a check can test: slide size, slides in presentation order, titles, text, fonts and
2
+ sizes (body vs footer zone), charts (and whether each keeps an embedded workbook), tables, pictures, off-canvas shapes,
3
+ source lines and speaker notes. Reads the XML directly, so it needs no PowerPoint and never renders anything."""
4
+ import hashlib, os, re, zipfile
5
+ from lxml import etree
6
+
7
+ NS = {"p": "http://schemas.openxmlformats.org/presentationml/2006/main",
8
+ "a": "http://schemas.openxmlformats.org/drawingml/2006/main",
9
+ "c": "http://schemas.openxmlformats.org/drawingml/2006/chart",
10
+ "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships"}
11
+ REL = "{http://schemas.openxmlformats.org/package/2006/relationships}Relationship"
12
+ EMU_IN = 914400
13
+ NUM = re.compile(r"-?\(?\$?\d[\d,]*\.?\d*\)?%?")
14
+ FOOTER_TEXT = re.compile(r"^\s*(sources?|notes?|footnotes?|confidential|©|\*|\d+\s*(/|of)\s*\d+\s*$)", re.I)
15
+
16
+
17
+ def _top_y(el):
18
+ """y offset of the outermost group holding el (child offsets inside a group are in group space)."""
19
+ y, cur = None, el
20
+ while cur is not None:
21
+ if cur.tag == "{%s}grpSp" % NS["p"]:
22
+ off = cur.find("./p:grpSpPr/a:xfrm/a:off", NS)
23
+ elif cur.tag == "{%s}graphicFrame" % NS["p"]:
24
+ off = cur.find("./p:xfrm/a:off", NS)
25
+ elif cur.tag == "{%s}sp" % NS["p"]:
26
+ off = cur.find("./p:spPr/a:xfrm/a:off", NS)
27
+ else:
28
+ off = None
29
+ if off is not None: y = int(off.get("y"))
30
+ cur = cur.getparent()
31
+ return y
32
+
33
+
34
+ def font_roles(root, slide_h):
35
+ """Split run sizes into body and footer. Footer = footer/date/slide-number placeholder, a shape in the bottom 15% of
36
+ the slide, or text that starts like a source line, a note or a page number."""
37
+ body, foot = [], []
38
+ for shp in list(root.iter("{%s}sp" % NS["p"])) + list(root.iter("{%s}graphicFrame" % NS["p"])):
39
+ sizes = [int(r.get("sz")) / 100 for r in shp.iter("{%s}rPr" % NS["a"]) if r.get("sz")]
40
+ if not sizes: continue
41
+ ph = shp.find(".//p:nvPr/p:ph", NS)
42
+ y = _top_y(shp)
43
+ is_foot = (ph is not None and ph.get("type") in ("ftr", "sldNum", "dt")) or (y is not None and y >= 0.85 * slide_h) \
44
+ or bool(FOOTER_TEXT.match("".join(shp.itertext()).strip()))
45
+ (foot if is_foot else body).extend(sizes)
46
+ return body, foot
47
+
48
+
49
+ def slide_parts(z):
50
+ """Slide parts in presentation order (sldIdLst). File names keep their numbers when slides are reordered, so name
51
+ order is only a fallback."""
52
+ s = [n for n in z.namelist() if re.match(r"ppt/slides/slide\d+\.xml$", n)]
53
+ by_name = sorted(s, key=lambda n: int(re.findall(r"\d+", n)[-1]))
54
+ try:
55
+ pres = etree.fromstring(z.read("ppt/presentation.xml"))
56
+ rl = {r.get("Id"): r.get("Target") for r in etree.fromstring(z.read("ppt/_rels/presentation.xml.rels")).iter(REL)}
57
+ order = []
58
+ for sid in pres.iter("{%s}sldId" % NS["p"]):
59
+ t = rl.get(sid.get("{%s}id" % NS["r"]))
60
+ if t: order.append(os.path.normpath(os.path.join("ppt", t)).replace("\\", "/"))
61
+ order = [o for o in order if o in s]
62
+ return order if len(order) == len(s) else by_name
63
+ except Exception:
64
+ return by_name
65
+
66
+
67
+ def rels(z, part):
68
+ d, f = os.path.split(part); rp = d + "/_rels/" + f + ".rels"
69
+ if rp not in z.namelist(): return {}
70
+ out = {}
71
+ for r in etree.fromstring(z.read(rp)).iter(REL):
72
+ if r.get("TargetMode") == "External": continue
73
+ out[r.get("Id")] = (os.path.normpath(os.path.join(d, r.get("Target"))).replace("\\", "/"), r.get("Type", "").rsplit("/", 1)[-1])
74
+ return out
75
+
76
+
77
+ def numbers_in(text):
78
+ """Numbers written in text: 1,250 / (3.5) / -4% / $12.0 -> floats; brackets and minus signs mean negative."""
79
+ vals = []
80
+ for m in NUM.findall(text):
81
+ neg = m.startswith("-") or (m.startswith("(") and m.endswith(")"))
82
+ s = re.sub(r"[^\d.]", "", m)
83
+ if not s or s == ".": continue
84
+ try: v = float(s)
85
+ except ValueError: continue
86
+ vals.append(-v if neg else v)
87
+ return vals
88
+
89
+
90
+ def inspect(path):
91
+ z = zipfile.ZipFile(path)
92
+ pres = etree.fromstring(z.read("ppt/presentation.xml"))
93
+ sz = pres.find("p:sldSz", NS); W, H = int(sz.get("cx")), int(sz.get("cy"))
94
+ slides, all_text, all_nums, hashes = [], [], [], []
95
+ for i, sp in enumerate(slide_parts(z), 1):
96
+ xml = z.read(sp); root = etree.fromstring(xml); rl = rels(z, sp)
97
+ hashes.append(hashlib.sha1(xml).hexdigest())
98
+ texts = ["".join(t.itertext()) for t in root.iter("{%s}p" % NS["a"])]
99
+ text = "\n".join(t for t in texts if t.strip())
100
+ charts = embedded = pics = big_pics = tables = offcanvas = 0
101
+ chart_nums = []
102
+ for gf in root.iter("{%s}graphicFrame" % NS["p"]):
103
+ ch = gf.find(".//c:chart", NS)
104
+ if ch is not None:
105
+ charts += 1
106
+ tgt = rl.get(ch.get("{%s}id" % NS["r"]), (None,))[0]
107
+ if tgt and tgt in z.namelist():
108
+ if any(t in ("package", "oleObject") for _, t in rels(z, tgt).values()): embedded += 1
109
+ cx = etree.fromstring(z.read(tgt))
110
+ chart_nums += [float(v.text) for v in cx.iter("{%s}v" % NS["c"]) if re.match(r"^-?\d+\.?\d*(E-?\d+)?$", v.text or "")]
111
+ if gf.find(".//a:tbl", NS) is not None: tables += 1
112
+ for pic in root.iter("{%s}pic" % NS["p"]):
113
+ pics += 1
114
+ ext = pic.find(".//a:xfrm/a:ext", NS)
115
+ if ext is not None and int(ext.get("cx")) * int(ext.get("cy")) > 0.08 * W * H: big_pics += 1
116
+ for off in root.iter("{%s}off" % NS["a"]):
117
+ x, y = int(off.get("x")), int(off.get("y"))
118
+ if x < -EMU_IN * 0.05 or y < -EMU_IN * 0.05 or x > W or y > H: offcanvas += 1
119
+ sizes = [int(r.get("sz")) / 100 for r in root.iter("{%s}rPr" % NS["a"]) if r.get("sz")]
120
+ body_sz, foot_sz = font_roles(root, H)
121
+ slides.append({"slide": i, "title": texts[0][:120] if texts else "", "text": text[:2000],
122
+ "charts": charts, "charts_with_workbook": embedded, "tables": tables,
123
+ "pictures": pics, "large_pictures": big_pics, "offcanvas_shapes": offcanvas,
124
+ "fonts": sorted({e.get("typeface") for e in root.iter("{%s}latin" % NS["a"]) if e.get("typeface")}),
125
+ "min_font_pt": min(sizes) if sizes else None,
126
+ "min_body_font_pt": min(body_sz) if body_sz else None,
127
+ "min_footer_font_pt": min(foot_sz) if foot_sz else None,
128
+ "has_source_line": bool(re.search(r"\bsources?\b|\bsource:", text, re.I)),
129
+ "has_notes": any(t == "notesSlide" for _, t in rl.values())})
130
+ all_text.append(text); all_nums += numbers_in(text) + chart_nums
131
+ return {"slide_w_in": round(W / EMU_IN, 2), "slide_h_in": round(H / EMU_IN, 2), "slides": slides,
132
+ "text": "\n".join(all_text), "numbers": all_nums, "slide_hashes": hashes}
@@ -0,0 +1,110 @@
1
+ """Read an .xlsx into plain facts a check can test: sheets, cells (formula, cached value,
2
+ number format, input styling), errors, hard-coded constants in formulas, charts and their ranges, names, validation,
3
+ comments, freeze panes, merged cells, external links; plus a LibreOffice recalculation to compare cached values."""
4
+ import io, os, re, shutil, subprocess, tempfile, zipfile
5
+ import openpyxl
6
+ from openpyxl.utils import get_column_letter
7
+
8
+ ERR = re.compile(r"^#(REF!|DIV/0!|VALUE!|NAME\?|N/A|NUM!|NULL!|SPILL!|CALC!)")
9
+ CONST = re.compile(r"(?<![A-Za-z_$:\d.])(-?\d+(?:\.\d+)?)(?![\d.]*[:A-Za-z(])")
10
+ BENIGN = {"0", "1", "2", "3", "4", "5", "6", "7", "12", "100", "1000", "-1", "0.5", "365", "4"}
11
+
12
+
13
+ def recalc(path):
14
+ """LibreOffice opens the workbook, recalculates and saves a copy; returns {sheet: {cell: value}} or None."""
15
+ tmp = tempfile.mkdtemp(prefix="xr-")
16
+ try:
17
+ src = os.path.join(tmp, "in.xlsx"); shutil.copy(path, src); out = os.path.join(tmp, "out")
18
+ os.makedirs(out)
19
+ subprocess.run(["soffice", "--headless", "--norestore", "-env:UserInstallation=file://" + os.path.join(tmp, "p"),
20
+ "--convert-to", "xlsx:Calc MS Excel 2007 XML", "--outdir", out, src], capture_output=True, timeout=600)
21
+ f = os.path.join(out, "in.xlsx")
22
+ if not os.path.exists(f): return None
23
+ wb = openpyxl.load_workbook(f, data_only=True)
24
+ return {ws.title: {c.coordinate: c.value for row in ws.iter_rows() for c in row if c.value is not None} for ws in wb.worksheets}
25
+ except Exception:
26
+ return None
27
+ finally:
28
+ shutil.rmtree(tmp, ignore_errors=True)
29
+
30
+
31
+ def inspect(path, with_recalc=True):
32
+ wbf = openpyxl.load_workbook(path, data_only=False)
33
+ wbv = openpyxl.load_workbook(path, data_only=True)
34
+ z = zipfile.ZipFile(path)
35
+ sheets = []
36
+ for ws in wbf.worksheets:
37
+ wv = wbv[ws.title]
38
+ cells, n_formula, n_num, errs, consts, inputs_styled = [], 0, 0, [], [], 0
39
+ for row in ws.iter_rows():
40
+ for c in row:
41
+ if c.value is None: continue
42
+ v = wv[c.coordinate].value
43
+ isf = isinstance(c.value, str) and c.value.startswith("=")
44
+ if isf:
45
+ n_formula += 1
46
+ k = [x for x in CONST.findall(re.sub(r"\"[^\"]*\"", "", c.value[1:])) if x not in BENIGN]
47
+ if k: consts.append({"cell": c.coordinate, "formula": c.value[:120], "constants": k[:4]})
48
+ elif isinstance(c.value, (int, float)) and not isinstance(c.value, bool):
49
+ n_num += 1
50
+ fill = c.fill.fgColor.rgb if c.fill and c.fill.fill_type == "solid" and c.fill.fgColor is not None else None
51
+ fc = c.font.color.rgb if c.font and c.font.color is not None and c.font.color.type == "rgb" else None
52
+ if (fill and fill not in ("00000000", "FFFFFFFF")) or (fc and fc not in ("FF000000", None)): inputs_styled += 1
53
+ if isinstance(v, str) and ERR.match(v): errs.append(c.coordinate)
54
+ if len(cells) < 400:
55
+ cells.append({"c": c.coordinate, "f": c.value[:160] if isf else None, "v": v if not isinstance(v, str) else v[:120],
56
+ "fmt": c.number_format if c.number_format != "General" else None})
57
+ charts = []
58
+ for ch in getattr(ws, "_charts", []):
59
+ refs = []
60
+ try:
61
+ for s in ch.series:
62
+ if s.val and s.val.numRef: refs.append(s.val.numRef.f)
63
+ except Exception: pass
64
+ def _axt(ax):
65
+ try: return bool(ax is not None and ax.title is not None)
66
+ except Exception: return False
67
+ pie = "Pie" in type(ch).__name__ or "Doughnut" in type(ch).__name__
68
+ charts.append({"type": type(ch).__name__, "title": _title(ch), "refs": refs,
69
+ "axis_titles": None if pie else (_axt(getattr(ch, "x_axis", None)) and _axt(getattr(ch, "y_axis", None)))})
70
+ dv = [{"sqref": str(d.sqref), "type": d.type, "formula": d.formula1} for d in ws.data_validations.dataValidation] if ws.data_validations else []
71
+ comments = [{"cell": c.coordinate, "text": c.comment.text[:200]} for row in ws.iter_rows() for c in row if c.comment]
72
+ sheets.append({"name": ws.title, "dims": ws.dimensions, "max_row": ws.max_row, "max_col": ws.max_column, "formulas": n_formula,
73
+ "numbers": n_num, "inputs_styled": inputs_styled, "errors": errs[:50], "n_errors": len(errs),
74
+ "constants_in_formulas": consts[:40], "n_constants_in_formulas": len(consts), "charts": charts,
75
+ "freeze": ws.freeze_panes, "merged": [str(m) for m in ws.merged_cells.ranges][:30], "validation": dv[:20],
76
+ "comments": comments[:30], "cells": cells, "col_widths": {k: v.width for k, v in list(ws.column_dimensions.items())[:40] if v.width}})
77
+ ext = [n for n in z.namelist() if n.startswith("xl/externalLinks/")]
78
+ names = []
79
+ try: names = [{"name": n, "ref": str(d.attr_text)[:120]} for n, d in wbf.defined_names.items()]
80
+ except Exception: pass
81
+ out = {"sheets": sheets, "sheet_names": [s["name"] for s in sheets], "external_links": len(ext), "defined_names": names}
82
+ rc = recalc(path) if with_recalc else None
83
+ # _grid: every non-empty value (recalculated when available), for value checks against an answer key
84
+ out["_grid"] = {ws.title: {c.coordinate: (rc.get(ws.title, {}).get(c.coordinate) if rc is not None and isinstance(c.value, str) and c.value.startswith("=") else wbv[ws.title][c.coordinate].value)
85
+ for row in ws.iter_rows() for c in row if c.value is not None} for ws in wbf.worksheets}
86
+ if with_recalc:
87
+ out["recalc_ok"] = rc is not None
88
+ if rc is not None:
89
+ diffs, rc_err = [], []
90
+ for ws in wbf.worksheets:
91
+ wv = wbv[ws.title]; R = rc.get(ws.title, {})
92
+ for row in ws.iter_rows():
93
+ for c in row:
94
+ if not (isinstance(c.value, str) and c.value.startswith("=")): continue
95
+ a, b = wv[c.coordinate].value, R.get(c.coordinate)
96
+ if isinstance(b, str) and ERR.match(b): rc_err.append("%s!%s" % (ws.title, c.coordinate))
97
+ if isinstance(a, (int, float)) and isinstance(b, (int, float)) and abs(a - b) > max(1e-6, abs(b) * 1e-6):
98
+ diffs.append({"cell": "%s!%s" % (ws.title, c.coordinate), "cached": a, "recalc": b})
99
+ elif a is None and b is not None: diffs.append({"cell": "%s!%s" % (ws.title, c.coordinate), "cached": None, "recalc": b if not isinstance(b, str) else b[:40]})
100
+ out["recalc_diffs"] = diffs[:40]; out["n_recalc_diffs"] = len(diffs); out["recalc_errors"] = rc_err[:40]
101
+ return out
102
+
103
+
104
+ def _title(ch):
105
+ try:
106
+ t = ch.title
107
+ if t is None: return None
108
+ return " ".join(r.t for p in t.tx.rich.p for r in (p.r or []))[:100]
109
+ except Exception:
110
+ return None
@@ -0,0 +1,61 @@
1
+ """Rubric: turn check results into a score out of 100.
2
+
3
+ A rubric is plain JSON, kept next to the task so the weights are visible and versioned:
4
+
5
+ {
6
+ "version": "example-1",
7
+ "categories": {
8
+ "requirements": {"label": "Task requirements", "points": 60, "per_item": true},
9
+ "craft": {"label": "Craft", "points": 40}
10
+ },
11
+ "gates": {"missing": "zero", "unreadable": "zero"}
12
+ }
13
+
14
+ Rules, deliberately simple so a reader can predict a score by hand:
15
+ - Each category earns points x (share of its units passed).
16
+ - per_item: every element of a check counts as one unit (ten required values weigh ten times, not once).
17
+ Otherwise each check counts once, scaled by its weight.
18
+ - A category with no applicable checks hands its points to the others, so a task is never scored on what it
19
+ did not ask for.
20
+ - Gates override the sum: "zero" sets the total to 0; {"cap": N} caps it at N.
21
+ These are example mechanics for the kit; tune them for your own tasks.
22
+ """
23
+ import json
24
+
25
+
26
+ def load(path):
27
+ with open(path) as f:
28
+ return json.load(f)
29
+
30
+
31
+ def score(results, checks, rubric, gates=()):
32
+ cats = {k: {"label": v.get("label", k), "points": float(v["points"]), "per_item": bool(v.get("per_item")),
33
+ "num": 0.0, "den": 0.0, "checks": 0}
34
+ for k, v in rubric["categories"].items()}
35
+ for c in checks:
36
+ r = results.get(c.id)
37
+ u = r.units() if r is not None else None
38
+ if u is None or c.category not in cats: continue
39
+ p, n = u; x = cats[c.category]; x["checks"] += 1
40
+ if x["per_item"]:
41
+ x["num"] += p; x["den"] += n
42
+ else:
43
+ x["num"] += c.weight * p / n; x["den"] += c.weight
44
+ live = {k: x for k, x in cats.items() if x["den"] > 0}
45
+ total_points = sum(x["points"] for x in live.values()) or 1.0
46
+ out_cats, total = {}, 0.0
47
+ for k, x in cats.items():
48
+ possible = x["points"] * 100.0 / total_points if k in live else 0.0
49
+ share = x["num"] / x["den"] if k in live else None
50
+ earned = possible * share if share is not None else 0.0
51
+ total += earned
52
+ out_cats[k] = {"label": x["label"], "possible": round(possible, 2), "earned": round(earned, 2),
53
+ "share": None if share is None else round(share, 3), "checks": x["checks"]}
54
+ applied = []
55
+ for g in gates:
56
+ rule = rubric.get("gates", {}).get(g["id"])
57
+ if rule == "zero":
58
+ total = 0.0; applied.append(g)
59
+ elif isinstance(rule, dict) and "cap" in rule:
60
+ total = min(total, float(rule["cap"])); applied.append(g)
61
+ return {"total": round(total, 1), "categories": out_cats, "gates": applied, "rubric_version": rubric.get("version")}
@@ -0,0 +1,129 @@
1
+ Metadata-Version: 2.4
2
+ Name: aspera-grader-kit
3
+ Version: 0.1.0
4
+ Summary: Grade the Word, PowerPoint and Excel files AI agents produce: inspectors, explainable checks and a JSON rubric.
5
+ Author-email: Motif Platforms <dsaran@motifplatforms.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/motifplatforms/aspera-grader-kit
8
+ Project-URL: Repository, https://github.com/motifplatforms/aspera-grader-kit
9
+ Project-URL: Issues, https://github.com/motifplatforms/aspera-grader-kit/issues
10
+ Project-URL: Writing, https://www.motifplatforms.com/insights
11
+ Project-URL: Company, https://www.motifplatforms.com
12
+ Keywords: llm-evaluation,ai-agents,graders,rubrics,reward-modeling,docx,pptx,xlsx
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Intended Audience :: Science/Research
16
+ Classifier: License :: OSI Approved :: MIT License
17
+ Classifier: Operating System :: OS Independent
18
+ Classifier: Programming Language :: Python :: 3
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
24
+ Classifier: Topic :: Software Development :: Testing
25
+ Requires-Python: >=3.10
26
+ Description-Content-Type: text/markdown
27
+ License-File: LICENSE
28
+ Requires-Dist: lxml>=4.9
29
+ Requires-Dist: openpyxl>=3.1
30
+ Provides-Extra: examples
31
+ Requires-Dist: python-docx>=1.1; extra == "examples"
32
+ Provides-Extra: test
33
+ Requires-Dist: pytest>=7; extra == "test"
34
+ Requires-Dist: python-docx>=1.1; extra == "test"
35
+ Requires-Dist: python-pptx>=0.6; extra == "test"
36
+ Dynamic: license-file
37
+
38
+ # aspera-grader-kit
39
+
40
+ [![tests](https://github.com/motifplatforms/aspera-grader-kit/actions/workflows/tests.yml/badge.svg)](https://github.com/motifplatforms/aspera-grader-kit/actions/workflows/tests.yml) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) [![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](pyproject.toml)
41
+
42
+ A small, readable pattern for grading the files that knowledge-work agents produce: Word memos, PowerPoint decks and
43
+ Excel models.
44
+
45
+ Most agent evals grade the chat transcript. The client never sees the transcript. They open the file. A memo with
46
+ its headings faked in bold, a deck whose chart is a pasted picture, or a model with hard-coded numbers inside its
47
+ formulas all *look* correct. They are still wrong, and a grader that only reads text will reward them.
48
+
49
+ This kit grades the file itself:
50
+
51
+ ```
52
+ output file ──► inspect ──► facts ──► checks ──► results ──► rubric ──► score + reasons
53
+ (docx/pptx/xlsx) (one thing each, (weights in plain JSON,
54
+ says why) next to the task)
55
+ ```
56
+
57
+ ## Quick start
58
+
59
+ ```bash
60
+ pip install -e ".[test]"
61
+ python examples/board-memo/make_samples.py
62
+ python -m aspera_grader_kit examples/board-memo examples/board-memo/samples/board_memo_weak.docx
63
+ ```
64
+
65
+ ```
66
+ board-memo board_memo_weak.docx -> 26.4 / 100
67
+ Task requirements 16.4 / 60.0
68
+ Craft 10.0 / 40.0
69
+ [FAIL] R2 Required sections exist as real headings (0 of 2 items met)
70
+ - missed: Summary (no heading called Summary)
71
+ [FAIL] R4 Metrics table has every metric with its value and change (2 of 4 items met)
72
+ - missed: Operating expenses (no row)
73
+ - missed: EBITDA (no row)
74
+ [FAIL] C2 No faked headings (bold Normal text standing in for a heading) (fake headings: ['Summary', 'Risks'])
75
+ ...
76
+ ```
77
+
78
+ The strong sample scores 100. Every point lost comes with the check that took it and the evidence, so a disagreement
79
+ with the grader is a conversation about one line, not about a number.
80
+
81
+ ## What's inside
82
+
83
+ | Part | What it does |
84
+ |---|---|
85
+ | `inspect/docx.py` | Body blocks in order (style, numbering, run formatting), tables (header rows, shading, margins), tracked changes, comments, content controls, fields, headers/footers, footnotes, hyperlinks |
86
+ | `inspect/pptx.py` | Slides in presentation order, titles, text and numbers, chart count and whether each keeps its workbook, tables, pictures, off-canvas shapes, body vs footer font floors, source lines, notes |
87
+ | `inspect/xlsx.py` | Formulas vs typed numbers, hard-coded constants inside formulas, error cells, charts and their ranges, validation, names, external links, and a LibreOffice recalculation to catch stale cached values |
88
+ | `checks.py` | `@check` decorator, `Result.items()` for per-item checks, runner that turns a crashing check into an `error` result instead of a crashed grade |
89
+ | `rubric.py` | Scores from a JSON rubric: categories, per-item counting, empty categories hand their points on, gates (`zero`, `cap`) |
90
+ | `examples/board-memo` | A complete task: brief, input CSV, answer key, rubric, nine checks and a strong/weak pair of outputs |
91
+
92
+ ## Design rules
93
+
94
+ 1. **Grade the artifact, not the transcript.** Inspect the file the way the client will open it.
95
+ 2. **One check, one variable.** A check that tests three things can't tell you which one failed.
96
+ 3. **Every check explains itself.** Evidence is part of the result, not a log line.
97
+ 4. **Count requirements item by item.** A task with ten required values should weigh them ten times, not once.
98
+ 5. **The answer key never reaches the agent.** Keep `key.json` out of anything the agent can read.
99
+ 6. **Gates for the unforgivable.** A missing or unopenable file scores zero, however good the reasoning was.
100
+ 7. **Weights live in data.** The rubric is JSON in the task folder, versioned with it, readable by a reviewer.
101
+
102
+ ## Writing your own task
103
+
104
+ Copy `examples/board-memo`, then edit:
105
+
106
+ - `task.md`: the brief the agent sees.
107
+ - `key.json`: what a correct answer contains.
108
+ - `rubric.json`: categories, points, gates.
109
+ - `checks.py`: checks registered with `@check("ID", "category", "description", registry="<task id>")`.
110
+
111
+ ## Where this comes from
112
+
113
+ This is the open, generic core of the harness we use at [Motif Platforms](https://www.motifplatforms.com) to grade AI
114
+ agents on real business deliverables. Our production tasks, answer keys, weights and judge prompts stay private. The
115
+ reasoning behind the pattern is in the series
116
+ [Teaching machines to do real work](https://www.motifplatforms.com/insights):
117
+
118
+ - [Reward hacking explained: how one bad grader teaches AI a thousand bad lessons](https://www.motifplatforms.com/insights/reward-hacking-explained-one-bad-grader)
119
+ - [How to design training exercises for AI agents](https://www.motifplatforms.com/insights/how-to-design-training-exercises-for-ai-agents)
120
+ - [Ten ways a grader lies in AI post-training](https://www.motifplatforms.com/insights/ten-ways-an-ai-grader-lies)
121
+ - [The correct-looking file: why AI agents fail and what experts offer](https://www.motifplatforms.com/insights/why-ai-agents-fail-at-spreadsheets-decks-documents)
122
+
123
+ Benchmark results from the full harness: [motifplatforms/mosaic-benchmarks](https://github.com/motifplatforms/mosaic-benchmarks).
124
+
125
+ Need graders or exercises built for your own agents? [dsaran@motifplatforms.com](mailto:dsaran@motifplatforms.com)
126
+
127
+ ## License
128
+
129
+ MIT
@@ -0,0 +1,14 @@
1
+ aspera_grader_kit/__init__.py,sha256=knolxvMvIcozdPq-0YuCgAiLj4iVI5LytwaWfXC17zE,475
2
+ aspera_grader_kit/__main__.py,sha256=Nc9bJWsvG35Yank6DNN3lUZRbjZXOygCN9fwDs3-d3I,627
3
+ aspera_grader_kit/checks.py,sha256=QfYALrh6MheqR1VC7RNP-xi1Auyo_J5cs2RpiRZPKxY,3583
4
+ aspera_grader_kit/grade.py,sha256=C56SF6icHBTS3loS7krno4_2Je8nmbZT8IKKgDLg8os,2964
5
+ aspera_grader_kit/rubric.py,sha256=WHLonOiqBLi9Bx6_3eQgpEltzCgdkLS-AbP8oIvp58c,2729
6
+ aspera_grader_kit/inspect/__init__.py,sha256=QRQMmC9kQEvR3X5ZwXKrptmZVZQZJDzOQZ6kfzdUPYc,532
7
+ aspera_grader_kit/inspect/docx.py,sha256=390iUrzhxzRGOOYYFuoh7mHF42R-CmFb4OdwQChgRGk,11005
8
+ aspera_grader_kit/inspect/pptx.py,sha256=IH4sq22HhnS7jC1LZ2laOusbWhvDbNT3Uuvm9h0xPd4,7065
9
+ aspera_grader_kit/inspect/xlsx.py,sha256=YvzoY_FSdrEFdY5KQ6C_YdfMBb0Eig-X8Lhw5pkhL60,7133
10
+ aspera_grader_kit-0.1.0.dist-info/licenses/LICENSE,sha256=L91093P0vyBVPft5_iYp05RWBuuVBBsUPAxE17RJpvM,1072
11
+ aspera_grader_kit-0.1.0.dist-info/METADATA,sha256=14QSFpTVY4x9wpm6DDkMMlx1jWmWCpLRMfbphXy4BZ4,7197
12
+ aspera_grader_kit-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
13
+ aspera_grader_kit-0.1.0.dist-info/top_level.txt,sha256=ikuU6Q6js789emYjXSQCEEk_HgLqfXId69pU_29f8Gw,18
14
+ aspera_grader_kit-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Motif Platforms
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ aspera_grader_kit