aspera-grader-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aspera_grader_kit/__init__.py +11 -0
- aspera_grader_kit/__main__.py +18 -0
- aspera_grader_kit/checks.py +106 -0
- aspera_grader_kit/grade.py +63 -0
- aspera_grader_kit/inspect/__init__.py +14 -0
- aspera_grader_kit/inspect/docx.py +162 -0
- aspera_grader_kit/inspect/pptx.py +132 -0
- aspera_grader_kit/inspect/xlsx.py +110 -0
- aspera_grader_kit/rubric.py +61 -0
- aspera_grader_kit-0.1.0.dist-info/METADATA +129 -0
- aspera_grader_kit-0.1.0.dist-info/RECORD +14 -0
- aspera_grader_kit-0.1.0.dist-info/WHEEL +5 -0
- aspera_grader_kit-0.1.0.dist-info/licenses/LICENSE +21 -0
- aspera_grader_kit-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""aspera-grader-kit: a small pattern for grading the files knowledge-work agents produce.
|
|
2
|
+
|
|
3
|
+
Inspect the file into facts, run checks that each test one thing and explain themselves, then score with a rubric
|
|
4
|
+
whose weights sit in plain JSON next to the task.
|
|
5
|
+
"""
|
|
6
|
+
from .checks import Check, Context, Result, check
|
|
7
|
+
from .grade import grade, render
|
|
8
|
+
from .inspect import inspect
|
|
9
|
+
|
|
10
|
+
__all__ = ["Check", "Context", "Result", "check", "grade", "render", "inspect"]
|
|
11
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""python -m aspera_grader_kit <task_dir> <output_file> [--json]"""
|
|
2
|
+
import argparse, json, sys
|
|
3
|
+
|
|
4
|
+
from .grade import grade, render
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def main(argv=None):
|
|
8
|
+
ap = argparse.ArgumentParser(prog="aspera_grader_kit", description="Grade one output file against one task folder.")
|
|
9
|
+
ap.add_argument("task_dir"); ap.add_argument("output")
|
|
10
|
+
ap.add_argument("--json", action="store_true", help="print the full report as JSON")
|
|
11
|
+
a = ap.parse_args(argv)
|
|
12
|
+
rep = grade(a.output, a.task_dir)
|
|
13
|
+
print(json.dumps(rep, indent=2, default=str) if a.json else render(rep))
|
|
14
|
+
return 0
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
if __name__ == "__main__":
|
|
18
|
+
sys.exit(main())
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Checks: small functions that each test one thing about an output and say why.
|
|
2
|
+
|
|
3
|
+
A check gets a Context (the inspected facts, the task and its answer key) and returns a Result. A check that tests a
|
|
4
|
+
list of things (five key values, three required sections) returns one element per item, so the rubric can count every
|
|
5
|
+
item instead of treating the whole list as one pass or fail.
|
|
6
|
+
|
|
7
|
+
from aspera_grader_kit import check, Result
|
|
8
|
+
|
|
9
|
+
@check("R1", "requirements", "Every key value appears in the memo")
|
|
10
|
+
def key_values(ctx):
|
|
11
|
+
return Result.items([(v["item"], str(v["value"]) in ctx.text, v["value"]) for v in ctx.key["values"]])
|
|
12
|
+
"""
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from typing import Callable, Dict, List, Optional
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class Result:
|
|
19
|
+
status: str # pass | fail | na | error
|
|
20
|
+
evidence: str = ""
|
|
21
|
+
elements: Optional[List[dict]] = None
|
|
22
|
+
|
|
23
|
+
@classmethod
|
|
24
|
+
def ok(cls, evidence=""): return cls("pass", evidence)
|
|
25
|
+
|
|
26
|
+
@classmethod
|
|
27
|
+
def fail(cls, evidence=""): return cls("fail", evidence)
|
|
28
|
+
|
|
29
|
+
@classmethod
|
|
30
|
+
def na(cls, evidence="does not apply"): return cls("na", evidence)
|
|
31
|
+
|
|
32
|
+
@classmethod
|
|
33
|
+
def when(cls, condition, evidence=""):
|
|
34
|
+
return cls("pass" if condition else "fail", evidence)
|
|
35
|
+
|
|
36
|
+
@classmethod
|
|
37
|
+
def items(cls, triples):
|
|
38
|
+
"""One element per (item, passed, evidence). Passes only when every item passes."""
|
|
39
|
+
els = [{"item": str(i), "status": "pass" if ok else "fail", "evidence": str(e)} for i, ok, e in triples]
|
|
40
|
+
if not els: return cls.na("no items to test")
|
|
41
|
+
n = sum(e["status"] == "pass" for e in els)
|
|
42
|
+
return cls("pass" if n == len(els) else "fail", "%d of %d items met" % (n, len(els)), els)
|
|
43
|
+
|
|
44
|
+
def units(self):
|
|
45
|
+
"""(passed, total) units for scoring, or None when the check does not count."""
|
|
46
|
+
if self.status not in ("pass", "fail"): return None
|
|
47
|
+
if self.elements: return sum(e["status"] == "pass" for e in self.elements), len(self.elements)
|
|
48
|
+
return (1, 1) if self.status == "pass" else (0, 1)
|
|
49
|
+
|
|
50
|
+
def to_dict(self):
|
|
51
|
+
d = {"status": self.status, "evidence": self.evidence}
|
|
52
|
+
if self.elements is not None: d["elements"] = self.elements
|
|
53
|
+
return d
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass
|
|
57
|
+
class Check:
|
|
58
|
+
id: str
|
|
59
|
+
category: str
|
|
60
|
+
description: str
|
|
61
|
+
fn: Callable
|
|
62
|
+
weight: float = 1.0
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass
|
|
66
|
+
class Context:
|
|
67
|
+
path: str
|
|
68
|
+
facts: dict
|
|
69
|
+
task: dict = field(default_factory=dict)
|
|
70
|
+
key: dict = field(default_factory=dict)
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def text(self): return self.facts.get("text", "")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
_REGISTRY: Dict[str, List[Check]] = {}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def check(id, category, description, weight=1.0, registry="default"):
|
|
80
|
+
"""Decorator that registers a check function under a registry name (one per task, usually)."""
|
|
81
|
+
def wrap(fn):
|
|
82
|
+
_REGISTRY.setdefault(registry, []).append(Check(id, category, description, fn, weight))
|
|
83
|
+
return fn
|
|
84
|
+
return wrap
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def registered(registry="default"):
|
|
88
|
+
return list(_REGISTRY.get(registry, []))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def clear(registry="default"):
|
|
92
|
+
_REGISTRY.pop(registry, None)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def run(checks, ctx):
|
|
96
|
+
"""Run every check; an exception inside a check becomes status 'error' with the message as evidence."""
|
|
97
|
+
out = {}
|
|
98
|
+
for c in checks:
|
|
99
|
+
try:
|
|
100
|
+
r = c.fn(ctx)
|
|
101
|
+
if not isinstance(r, Result):
|
|
102
|
+
raise TypeError("check %s returned %r, not a Result" % (c.id, type(r).__name__))
|
|
103
|
+
except Exception as e: # a broken check must never crash the grade
|
|
104
|
+
r = Result("error", "%s: %s" % (type(e).__name__, e))
|
|
105
|
+
out[c.id] = r
|
|
106
|
+
return out
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Grade one output file against one task folder.
|
|
2
|
+
|
|
3
|
+
A task folder holds:
|
|
4
|
+
task.md the brief the agent was given
|
|
5
|
+
task.json {"id", "format", "output"} (format: docx | pptx | xlsx)
|
|
6
|
+
key.json the answer key (values, required items); never shown to the agent
|
|
7
|
+
rubric.json categories, points and gates
|
|
8
|
+
checks.py the task's checks, registered with @check(..., registry=<task id>)
|
|
9
|
+
"""
|
|
10
|
+
import importlib.util, json, os
|
|
11
|
+
|
|
12
|
+
from . import checks as C
|
|
13
|
+
from . import rubric as R
|
|
14
|
+
from .inspect import inspect
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _load_json(task_dir, name, default=None):
|
|
18
|
+
p = os.path.join(task_dir, name)
|
|
19
|
+
if not os.path.exists(p): return default
|
|
20
|
+
with open(p) as f: return json.load(f)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def load_task_checks(task_dir, task_id):
|
|
24
|
+
C.clear(task_id)
|
|
25
|
+
spec = importlib.util.spec_from_file_location("task_checks_%s" % task_id.replace("-", "_"), os.path.join(task_dir, "checks.py"))
|
|
26
|
+
mod = importlib.util.module_from_spec(spec); spec.loader.exec_module(mod)
|
|
27
|
+
return C.registered(task_id)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def grade(output_path, task_dir):
|
|
31
|
+
task = _load_json(task_dir, "task.json"); key = _load_json(task_dir, "key.json", {})
|
|
32
|
+
rubric = R.load(os.path.join(task_dir, "rubric.json"))
|
|
33
|
+
checks = load_task_checks(task_dir, task["id"])
|
|
34
|
+
report = {"task": task["id"], "output": os.path.basename(output_path) if output_path else None, "checks": {}}
|
|
35
|
+
gates = []
|
|
36
|
+
if not output_path or not os.path.exists(output_path):
|
|
37
|
+
gates.append({"id": "missing", "evidence": "no output file"})
|
|
38
|
+
else:
|
|
39
|
+
try:
|
|
40
|
+
facts = inspect(output_path)
|
|
41
|
+
except Exception as e:
|
|
42
|
+
gates.append({"id": "unreadable", "evidence": "%s: %s" % (type(e).__name__, e)})
|
|
43
|
+
else:
|
|
44
|
+
results = C.run(checks, C.Context(output_path, facts, task, key))
|
|
45
|
+
report["checks"] = {c.id: dict(results[c.id].to_dict(), category=c.category, description=c.description) for c in checks}
|
|
46
|
+
report.update(R.score(results, checks, rubric, gates))
|
|
47
|
+
return report
|
|
48
|
+
report.update(R.score({}, checks, rubric, gates)) # nothing to check: every category is empty, so the total is 0
|
|
49
|
+
report["gates"] = gates
|
|
50
|
+
return report
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def render(report):
|
|
54
|
+
"""Plain-text summary of a report for the terminal."""
|
|
55
|
+
lines = ["%s %s -> %.1f / 100" % (report["task"], report.get("output"), report["total"])]
|
|
56
|
+
for g in report.get("gates", []): lines.append(" GATE %s: %s" % (g["id"], g["evidence"]))
|
|
57
|
+
for k, c in report.get("categories", {}).items():
|
|
58
|
+
if c["possible"]: lines.append(" %-20s %5.1f / %5.1f" % (c["label"], c["earned"], c["possible"]))
|
|
59
|
+
for cid, r in report.get("checks", {}).items():
|
|
60
|
+
lines.append(" [%s] %-4s %s (%s)" % (r["status"].upper()[:4].ljust(4), cid, r["description"], r["evidence"]))
|
|
61
|
+
for e in r.get("elements") or []:
|
|
62
|
+
if e["status"] != "pass": lines.append(" - missed: %s (%s)" % (e["item"], e["evidence"]))
|
|
63
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""File inspectors: turn an Office file into a plain dict of facts. Checks test facts, never raw XML."""
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
from . import docx, pptx, xlsx
|
|
5
|
+
|
|
6
|
+
INSPECTORS = {".docx": docx.inspect, ".pptx": pptx.inspect, ".xlsx": xlsx.inspect}
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def inspect(path, **kw):
|
|
10
|
+
"""Facts for one file, picked by extension. Raises ValueError for a format with no inspector."""
|
|
11
|
+
ext = os.path.splitext(path)[1].lower()
|
|
12
|
+
if ext not in INSPECTORS:
|
|
13
|
+
raise ValueError("no inspector for %s files" % ext)
|
|
14
|
+
return INSPECTORS[ext](path, **kw)
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""Read a .docx into plain facts a check can test: body blocks in order (paragraphs with style,
|
|
2
|
+
numbering and run formatting; tables with header-repeat, shading, cell margins, alignment), tracked changes, comments,
|
|
3
|
+
content controls, protection, fields (PAGE, NUMPAGES, TOC, SEQ), headers/footers, footnotes, images, hyperlinks."""
|
|
4
|
+
import re, zipfile
|
|
5
|
+
from lxml import etree
|
|
6
|
+
|
|
7
|
+
W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
|
|
8
|
+
NS = {"w": W, "w14": "http://schemas.microsoft.com/office/word/2010/wordml",
|
|
9
|
+
"r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships",
|
|
10
|
+
"a": "http://schemas.openxmlformats.org/drawingml/2006/main", "wp": "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"}
|
|
11
|
+
REL = "{http://schemas.openxmlformats.org/package/2006/relationships}Relationship"
|
|
12
|
+
q = lambda t: "{%s}%s" % (NS[t.split(":")[0]], t.split(":")[1])
|
|
13
|
+
wv = lambda el, path: (el.find(path, NS).get(q("w:val")) if el is not None and el.find(path, NS) is not None else None)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def text_of(el, deleted=False):
|
|
17
|
+
out = []
|
|
18
|
+
for t in el.iter(q("w:t"), q("w:delText"), q("w:tab"), q("w:br")):
|
|
19
|
+
if t.tag == q("w:t"): out.append(t.text or "")
|
|
20
|
+
elif t.tag == q("w:delText") and deleted: out.append(t.text or "")
|
|
21
|
+
elif t.tag == q("w:tab"): out.append("\t")
|
|
22
|
+
return "".join(out)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def styles(z):
|
|
26
|
+
if "word/styles.xml" not in z.namelist(): return {}
|
|
27
|
+
root = etree.fromstring(z.read("word/styles.xml"))
|
|
28
|
+
return {s.get(q("w:styleId")): (s.find("w:name", NS).get(q("w:val")) if s.find("w:name", NS) is not None else s.get(q("w:styleId")))
|
|
29
|
+
for s in root.iter(q("w:style"))}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def fields_in(el):
|
|
33
|
+
"""Field codes (simple and complex) in an element, upper-cased first word: PAGE, NUMPAGES, TOC, SEQ, REF ..."""
|
|
34
|
+
codes = [f.get(q("w:instr")) or "" for f in el.iter(q("w:fldSimple"))]
|
|
35
|
+
codes += [(t.text or "") for t in el.iter(q("w:instrText"))]
|
|
36
|
+
return [c.strip() for c in codes if c and c.strip()]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def para_facts(p, S):
|
|
40
|
+
ppr = p.find("w:pPr", NS)
|
|
41
|
+
sid = wv(ppr, "w:pStyle") if ppr is not None else None
|
|
42
|
+
num = ppr.find("w:numPr", NS) if ppr is not None else None
|
|
43
|
+
runs = []
|
|
44
|
+
for r in p.iter(q("w:r")):
|
|
45
|
+
rpr = r.find("w:rPr", NS); t = text_of(r)
|
|
46
|
+
if not t.strip(): continue
|
|
47
|
+
b = rpr is not None and rpr.find("w:b", NS) is not None and (rpr.find("w:b", NS).get(q("w:val")) not in ("0", "false"))
|
|
48
|
+
sz = wv(rpr, "w:sz"); col = wv(rpr, "w:color"); font = rpr.find("w:rFonts", NS).get(q("w:ascii")) if rpr is not None and rpr.find("w:rFonts", NS) is not None else None
|
|
49
|
+
sup = wv(rpr, "w:vertAlign") == "superscript" if rpr is not None else False
|
|
50
|
+
runs.append({"t": t[:200], "b": b, "sz": int(sz) / 2 if sz and sz.isdigit() else None, "color": col, "font": font, "sup": sup})
|
|
51
|
+
return {"type": "p", "style": S.get(sid, sid) if sid else "Normal", "text": text_of(p)[:1200],
|
|
52
|
+
"num": {"numId": wv(num, "w:numId"), "ilvl": wv(num, "w:ilvl")} if num is not None else None,
|
|
53
|
+
"align": wv(ppr, "w:jc") if ppr is not None else None, "runs": runs[:40],
|
|
54
|
+
"fields": fields_in(p), "has_drawing": p.find(".//w:drawing", NS) is not None or p.find(".//w:pict", NS) is not None,
|
|
55
|
+
"in_del": text_of(p, deleted=True) != text_of(p)}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def table_facts(t, S):
|
|
59
|
+
rows = []
|
|
60
|
+
for tr in t.findall("w:tr", NS):
|
|
61
|
+
trpr = tr.find("w:trPr", NS)
|
|
62
|
+
cells = []
|
|
63
|
+
for tc in tr.findall("w:tc", NS):
|
|
64
|
+
tcpr = tc.find("w:tcPr", NS)
|
|
65
|
+
shd = tcpr.find("w:shd", NS) if tcpr is not None else None
|
|
66
|
+
mar = tcpr.find("w:tcMar", NS) if tcpr is not None else None
|
|
67
|
+
ps = tc.findall("w:p", NS)
|
|
68
|
+
bold = any(r.find("w:rPr/w:b", NS) is not None for p in ps for r in p.iter(q("w:r")) if text_of(r).strip())
|
|
69
|
+
colors = sorted({wv(r.find("w:rPr", NS), "w:color") for p in ps for r in p.iter(q("w:r")) if text_of(r).strip() and r.find("w:rPr", NS) is not None} - {None})
|
|
70
|
+
cells.append({"text": " ".join(text_of(p) for p in ps).strip()[:200], "fill": shd.get(q("w:fill")) if shd is not None else None,
|
|
71
|
+
"align": next((wv(p.find("w:pPr", NS), "w:jc") for p in ps if p.find("w:pPr/w:jc", NS) is not None), None),
|
|
72
|
+
"margins": {e.tag.split("}")[1]: e.get(q("w:w")) for e in mar} if mar is not None else None, "bold": bold, "colors": colors})
|
|
73
|
+
spans = sum(int(wv(tc.find("w:tcPr", NS), "w:gridSpan") or 1) if tc.find("w:tcPr", NS) is not None else 1 for tc in tr.findall("w:tc", NS))
|
|
74
|
+
sizes = [int(sz.get(q("w:val"))) / 2 for sz in tr.iter(q("w:sz")) if sz.get(q("w:val"), "").isdigit() and sz.getparent().tag == q("w:rPr")]
|
|
75
|
+
rows.append({"header": trpr is not None and trpr.find("w:tblHeader", NS) is not None, "cells": cells, "span": spans, "min_pt": min(sizes) if sizes else None})
|
|
76
|
+
tpr = t.find("w:tblPr", NS)
|
|
77
|
+
tmar = tpr.find("w:tblCellMar", NS) if tpr is not None else None
|
|
78
|
+
grid = [int(g.get(q("w:w")) or 0) for g in t.findall("w:tblGrid/w:gridCol", NS)]
|
|
79
|
+
return {"type": "table", "grid_w_twips": sum(grid), "style": S.get(wv(tpr, "w:tblStyle"), wv(tpr, "w:tblStyle")) if tpr is not None else None,
|
|
80
|
+
"n_rows": len(rows), "n_cols": max((len(r["cells"]) for r in rows), default=0),
|
|
81
|
+
"table_cell_margins": {e.tag.split("}")[1]: e.get(q("w:w")) for e in tmar} if tmar is not None else None,
|
|
82
|
+
"rows": rows[:60]}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def inspect(path):
|
|
86
|
+
z = zipfile.ZipFile(path); names = z.namelist(); S = styles(z)
|
|
87
|
+
doc = etree.fromstring(z.read("word/document.xml")); body = doc.find("w:body", NS)
|
|
88
|
+
blocks = []
|
|
89
|
+
for el in body:
|
|
90
|
+
if el.tag == q("w:p"): blocks.append(para_facts(el, S))
|
|
91
|
+
elif el.tag == q("w:tbl"): blocks.append(table_facts(el, S))
|
|
92
|
+
elif el.tag == q("w:sdt"):
|
|
93
|
+
for p in el.iter(q("w:p")): blocks.append(dict(para_facts(p, S), sdt_block=True))
|
|
94
|
+
ins = [{"author": e.get(q("w:author")), "text": text_of(e)[:300]} for e in doc.iter(q("w:ins"))]
|
|
95
|
+
dels = [{"author": e.get(q("w:author")), "text": "".join((t.text or "") for t in e.iter(q("w:delText")))[:300]} for e in doc.iter(q("w:del"))]
|
|
96
|
+
comments = []
|
|
97
|
+
if "word/comments.xml" in names:
|
|
98
|
+
for c in etree.fromstring(z.read("word/comments.xml")).iter(q("w:comment")):
|
|
99
|
+
comments.append({"id": c.get(q("w:id")), "author": c.get(q("w:author")), "text": text_of(c)[:500]})
|
|
100
|
+
anchors = {}
|
|
101
|
+
for p in doc.iter(q("w:p")):
|
|
102
|
+
for s in p.iter(q("w:commentRangeStart"), q("w:commentReference")):
|
|
103
|
+
anchors.setdefault(s.get(q("w:id")), text_of(p)[:160])
|
|
104
|
+
for c in comments: c["anchored_in"] = anchors.get(c["id"])
|
|
105
|
+
sdts = []
|
|
106
|
+
for s in doc.iter(q("w:sdt")):
|
|
107
|
+
pr = s.find("w:sdtPr", NS)
|
|
108
|
+
if pr is None: continue
|
|
109
|
+
kind = ("dropdown" if pr.find("w:dropDownList", NS) is not None else "combobox" if pr.find("w:comboBox", NS) is not None
|
|
110
|
+
else "date" if pr.find("w:date", NS) is not None else "checkbox" if pr.find("w14:checkbox", NS) is not None
|
|
111
|
+
else "richtext" if pr.find("w:richText", NS) is not None else "picture" if pr.find("w:picture", NS) is not None
|
|
112
|
+
else "text" if pr.find("w:text", NS) is not None else "docpart" if pr.find("w:docPartObj", NS) is not None else "other")
|
|
113
|
+
items = [li.get(q("w:displayText")) or li.get(q("w:value")) for li in pr.iter(q("w:listItem"))]
|
|
114
|
+
df = pr.find("w:date/w:dateFormat", NS)
|
|
115
|
+
sdts.append({"kind": kind, "tag": wv(pr, "w:tag"), "alias": wv(pr, "w:alias"), "placeholder": pr.find("w:showingPlcHdr", NS) is not None or pr.find("w:placeholder", NS) is not None,
|
|
116
|
+
"items": items, "date_format": df.get(q("w:val")) if df is not None else None,
|
|
117
|
+
"text": text_of(s.find("w:sdtContent", NS))[:120] if s.find("w:sdtContent", NS) is not None else ""})
|
|
118
|
+
prot = None
|
|
119
|
+
if "word/settings.xml" in names:
|
|
120
|
+
dp = etree.fromstring(z.read("word/settings.xml")).find("w:documentProtection", NS)
|
|
121
|
+
if dp is not None: prot = {"edit": dp.get(q("w:edit")), "enforcement": dp.get(q("w:enforcement")), "password": any("hash" in k.lower() or "salt" in k.lower() for k in dp.attrib)}
|
|
122
|
+
hf = {}
|
|
123
|
+
for n in names:
|
|
124
|
+
m = re.match(r"word/(header|footer)\d+\.xml$", n)
|
|
125
|
+
if m:
|
|
126
|
+
root = etree.fromstring(z.read(n)); hf.setdefault(m.group(1), []).append({"text": text_of(root)[:300], "fields": fields_in(root)})
|
|
127
|
+
sect = body.find("w:sectPr", NS)
|
|
128
|
+
pg = {}
|
|
129
|
+
if sect is not None:
|
|
130
|
+
ps, pm = sect.find("w:pgSz", NS), sect.find("w:pgMar", NS)
|
|
131
|
+
if ps is not None: pg = {"w_twips": int(ps.get(q("w:w")) or 0), "h_twips": int(ps.get(q("w:h")) or 0)}
|
|
132
|
+
if pm is not None: pg.update({k: int(pm.get(q("w:" + k)) or 0) for k in ("left", "right", "top", "bottom")})
|
|
133
|
+
foot = []
|
|
134
|
+
if "word/footnotes.xml" in names:
|
|
135
|
+
for f in etree.fromstring(z.read("word/footnotes.xml")).iter(q("w:footnote")):
|
|
136
|
+
if f.get(q("w:type")) in (None, "normal"): foot.append(text_of(f)[:200])
|
|
137
|
+
links = []
|
|
138
|
+
rels = {}
|
|
139
|
+
if "word/_rels/document.xml.rels" in names:
|
|
140
|
+
for r in etree.fromstring(z.read("word/_rels/document.xml.rels")).iter(REL):
|
|
141
|
+
rels[r.get("Id")] = (r.get("Target"), r.get("TargetMode"))
|
|
142
|
+
for h in doc.iter(q("w:hyperlink")):
|
|
143
|
+
rid = h.get(q("r:id")); links.append({"text": text_of(h)[:120], "target": rels.get(rid, (None,))[0] if rid else None, "anchor": h.get(q("w:anchor"))})
|
|
144
|
+
fields = fields_in(doc)
|
|
145
|
+
images = sum(1 for _ in doc.iter(q("w:drawing"))) + sum(1 for _ in doc.iter(q("w:pict")))
|
|
146
|
+
dd = {}
|
|
147
|
+
if "word/styles.xml" in names:
|
|
148
|
+
sr = etree.fromstring(z.read("word/styles.xml"))
|
|
149
|
+
rp = sr.find("w:docDefaults/w:rPrDefault/w:rPr", NS)
|
|
150
|
+
if rp is not None:
|
|
151
|
+
dd["sz"] = int(wv(rp, "w:sz")) / 2 if wv(rp, "w:sz") and wv(rp, "w:sz").isdigit() else None
|
|
152
|
+
f = rp.find("w:rFonts", NS); dd["font"] = f.get(q("w:ascii")) if f is not None else None
|
|
153
|
+
for st in sr.iter(q("w:style")):
|
|
154
|
+
if st.get(q("w:styleId")) == "Normal":
|
|
155
|
+
rp = st.find("w:rPr", NS)
|
|
156
|
+
if rp is not None and wv(rp, "w:sz"): dd["normal_sz"] = int(wv(rp, "w:sz")) / 2
|
|
157
|
+
if rp is not None and rp.find("w:rFonts", NS) is not None: dd["normal_font"] = rp.find("w:rFonts", NS).get(q("w:ascii"))
|
|
158
|
+
num_xml = z.read("word/numbering.xml").decode("utf-8", "ignore") if "word/numbering.xml" in names else ""
|
|
159
|
+
return {"blocks": blocks, "ins": ins, "del": dels, "comments": comments, "sdts": sdts, "protection": prot, "headers_footers": hf,
|
|
160
|
+
"page": pg, "footnotes": foot, "hyperlinks": links, "fields": fields, "images": images,
|
|
161
|
+
"has_numbering_part": bool(num_xml), "defaults": dd, "styles_used": sorted({b["style"] for b in blocks if b["type"] == "p"}),
|
|
162
|
+
"text": "\n".join(b["text"] if b["type"] == "p" else "\n".join(" | ".join(c["text"] for c in r["cells"]) for r in b["rows"]) for b in blocks)}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Read a .pptx into plain facts a check can test: slide size, slides in presentation order, titles, text, fonts and
|
|
2
|
+
sizes (body vs footer zone), charts (and whether each keeps an embedded workbook), tables, pictures, off-canvas shapes,
|
|
3
|
+
source lines and speaker notes. Reads the XML directly, so it needs no PowerPoint and never renders anything."""
|
|
4
|
+
import hashlib, os, re, zipfile
|
|
5
|
+
from lxml import etree
|
|
6
|
+
|
|
7
|
+
NS = {"p": "http://schemas.openxmlformats.org/presentationml/2006/main",
|
|
8
|
+
"a": "http://schemas.openxmlformats.org/drawingml/2006/main",
|
|
9
|
+
"c": "http://schemas.openxmlformats.org/drawingml/2006/chart",
|
|
10
|
+
"r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships"}
|
|
11
|
+
REL = "{http://schemas.openxmlformats.org/package/2006/relationships}Relationship"
|
|
12
|
+
EMU_IN = 914400
|
|
13
|
+
NUM = re.compile(r"-?\(?\$?\d[\d,]*\.?\d*\)?%?")
|
|
14
|
+
FOOTER_TEXT = re.compile(r"^\s*(sources?|notes?|footnotes?|confidential|©|\*|\d+\s*(/|of)\s*\d+\s*$)", re.I)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _top_y(el):
|
|
18
|
+
"""y offset of the outermost group holding el (child offsets inside a group are in group space)."""
|
|
19
|
+
y, cur = None, el
|
|
20
|
+
while cur is not None:
|
|
21
|
+
if cur.tag == "{%s}grpSp" % NS["p"]:
|
|
22
|
+
off = cur.find("./p:grpSpPr/a:xfrm/a:off", NS)
|
|
23
|
+
elif cur.tag == "{%s}graphicFrame" % NS["p"]:
|
|
24
|
+
off = cur.find("./p:xfrm/a:off", NS)
|
|
25
|
+
elif cur.tag == "{%s}sp" % NS["p"]:
|
|
26
|
+
off = cur.find("./p:spPr/a:xfrm/a:off", NS)
|
|
27
|
+
else:
|
|
28
|
+
off = None
|
|
29
|
+
if off is not None: y = int(off.get("y"))
|
|
30
|
+
cur = cur.getparent()
|
|
31
|
+
return y
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def font_roles(root, slide_h):
|
|
35
|
+
"""Split run sizes into body and footer. Footer = footer/date/slide-number placeholder, a shape in the bottom 15% of
|
|
36
|
+
the slide, or text that starts like a source line, a note or a page number."""
|
|
37
|
+
body, foot = [], []
|
|
38
|
+
for shp in list(root.iter("{%s}sp" % NS["p"])) + list(root.iter("{%s}graphicFrame" % NS["p"])):
|
|
39
|
+
sizes = [int(r.get("sz")) / 100 for r in shp.iter("{%s}rPr" % NS["a"]) if r.get("sz")]
|
|
40
|
+
if not sizes: continue
|
|
41
|
+
ph = shp.find(".//p:nvPr/p:ph", NS)
|
|
42
|
+
y = _top_y(shp)
|
|
43
|
+
is_foot = (ph is not None and ph.get("type") in ("ftr", "sldNum", "dt")) or (y is not None and y >= 0.85 * slide_h) \
|
|
44
|
+
or bool(FOOTER_TEXT.match("".join(shp.itertext()).strip()))
|
|
45
|
+
(foot if is_foot else body).extend(sizes)
|
|
46
|
+
return body, foot
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def slide_parts(z):
|
|
50
|
+
"""Slide parts in presentation order (sldIdLst). File names keep their numbers when slides are reordered, so name
|
|
51
|
+
order is only a fallback."""
|
|
52
|
+
s = [n for n in z.namelist() if re.match(r"ppt/slides/slide\d+\.xml$", n)]
|
|
53
|
+
by_name = sorted(s, key=lambda n: int(re.findall(r"\d+", n)[-1]))
|
|
54
|
+
try:
|
|
55
|
+
pres = etree.fromstring(z.read("ppt/presentation.xml"))
|
|
56
|
+
rl = {r.get("Id"): r.get("Target") for r in etree.fromstring(z.read("ppt/_rels/presentation.xml.rels")).iter(REL)}
|
|
57
|
+
order = []
|
|
58
|
+
for sid in pres.iter("{%s}sldId" % NS["p"]):
|
|
59
|
+
t = rl.get(sid.get("{%s}id" % NS["r"]))
|
|
60
|
+
if t: order.append(os.path.normpath(os.path.join("ppt", t)).replace("\\", "/"))
|
|
61
|
+
order = [o for o in order if o in s]
|
|
62
|
+
return order if len(order) == len(s) else by_name
|
|
63
|
+
except Exception:
|
|
64
|
+
return by_name
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def rels(z, part):
|
|
68
|
+
d, f = os.path.split(part); rp = d + "/_rels/" + f + ".rels"
|
|
69
|
+
if rp not in z.namelist(): return {}
|
|
70
|
+
out = {}
|
|
71
|
+
for r in etree.fromstring(z.read(rp)).iter(REL):
|
|
72
|
+
if r.get("TargetMode") == "External": continue
|
|
73
|
+
out[r.get("Id")] = (os.path.normpath(os.path.join(d, r.get("Target"))).replace("\\", "/"), r.get("Type", "").rsplit("/", 1)[-1])
|
|
74
|
+
return out
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def numbers_in(text):
|
|
78
|
+
"""Numbers written in text: 1,250 / (3.5) / -4% / $12.0 -> floats; brackets and minus signs mean negative."""
|
|
79
|
+
vals = []
|
|
80
|
+
for m in NUM.findall(text):
|
|
81
|
+
neg = m.startswith("-") or (m.startswith("(") and m.endswith(")"))
|
|
82
|
+
s = re.sub(r"[^\d.]", "", m)
|
|
83
|
+
if not s or s == ".": continue
|
|
84
|
+
try: v = float(s)
|
|
85
|
+
except ValueError: continue
|
|
86
|
+
vals.append(-v if neg else v)
|
|
87
|
+
return vals
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def inspect(path):
|
|
91
|
+
z = zipfile.ZipFile(path)
|
|
92
|
+
pres = etree.fromstring(z.read("ppt/presentation.xml"))
|
|
93
|
+
sz = pres.find("p:sldSz", NS); W, H = int(sz.get("cx")), int(sz.get("cy"))
|
|
94
|
+
slides, all_text, all_nums, hashes = [], [], [], []
|
|
95
|
+
for i, sp in enumerate(slide_parts(z), 1):
|
|
96
|
+
xml = z.read(sp); root = etree.fromstring(xml); rl = rels(z, sp)
|
|
97
|
+
hashes.append(hashlib.sha1(xml).hexdigest())
|
|
98
|
+
texts = ["".join(t.itertext()) for t in root.iter("{%s}p" % NS["a"])]
|
|
99
|
+
text = "\n".join(t for t in texts if t.strip())
|
|
100
|
+
charts = embedded = pics = big_pics = tables = offcanvas = 0
|
|
101
|
+
chart_nums = []
|
|
102
|
+
for gf in root.iter("{%s}graphicFrame" % NS["p"]):
|
|
103
|
+
ch = gf.find(".//c:chart", NS)
|
|
104
|
+
if ch is not None:
|
|
105
|
+
charts += 1
|
|
106
|
+
tgt = rl.get(ch.get("{%s}id" % NS["r"]), (None,))[0]
|
|
107
|
+
if tgt and tgt in z.namelist():
|
|
108
|
+
if any(t in ("package", "oleObject") for _, t in rels(z, tgt).values()): embedded += 1
|
|
109
|
+
cx = etree.fromstring(z.read(tgt))
|
|
110
|
+
chart_nums += [float(v.text) for v in cx.iter("{%s}v" % NS["c"]) if re.match(r"^-?\d+\.?\d*(E-?\d+)?$", v.text or "")]
|
|
111
|
+
if gf.find(".//a:tbl", NS) is not None: tables += 1
|
|
112
|
+
for pic in root.iter("{%s}pic" % NS["p"]):
|
|
113
|
+
pics += 1
|
|
114
|
+
ext = pic.find(".//a:xfrm/a:ext", NS)
|
|
115
|
+
if ext is not None and int(ext.get("cx")) * int(ext.get("cy")) > 0.08 * W * H: big_pics += 1
|
|
116
|
+
for off in root.iter("{%s}off" % NS["a"]):
|
|
117
|
+
x, y = int(off.get("x")), int(off.get("y"))
|
|
118
|
+
if x < -EMU_IN * 0.05 or y < -EMU_IN * 0.05 or x > W or y > H: offcanvas += 1
|
|
119
|
+
sizes = [int(r.get("sz")) / 100 for r in root.iter("{%s}rPr" % NS["a"]) if r.get("sz")]
|
|
120
|
+
body_sz, foot_sz = font_roles(root, H)
|
|
121
|
+
slides.append({"slide": i, "title": texts[0][:120] if texts else "", "text": text[:2000],
|
|
122
|
+
"charts": charts, "charts_with_workbook": embedded, "tables": tables,
|
|
123
|
+
"pictures": pics, "large_pictures": big_pics, "offcanvas_shapes": offcanvas,
|
|
124
|
+
"fonts": sorted({e.get("typeface") for e in root.iter("{%s}latin" % NS["a"]) if e.get("typeface")}),
|
|
125
|
+
"min_font_pt": min(sizes) if sizes else None,
|
|
126
|
+
"min_body_font_pt": min(body_sz) if body_sz else None,
|
|
127
|
+
"min_footer_font_pt": min(foot_sz) if foot_sz else None,
|
|
128
|
+
"has_source_line": bool(re.search(r"\bsources?\b|\bsource:", text, re.I)),
|
|
129
|
+
"has_notes": any(t == "notesSlide" for _, t in rl.values())})
|
|
130
|
+
all_text.append(text); all_nums += numbers_in(text) + chart_nums
|
|
131
|
+
return {"slide_w_in": round(W / EMU_IN, 2), "slide_h_in": round(H / EMU_IN, 2), "slides": slides,
|
|
132
|
+
"text": "\n".join(all_text), "numbers": all_nums, "slide_hashes": hashes}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Read an .xlsx into plain facts a check can test: sheets, cells (formula, cached value,
|
|
2
|
+
number format, input styling), errors, hard-coded constants in formulas, charts and their ranges, names, validation,
|
|
3
|
+
comments, freeze panes, merged cells, external links; plus a LibreOffice recalculation to compare cached values."""
|
|
4
|
+
import io, os, re, shutil, subprocess, tempfile, zipfile
|
|
5
|
+
import openpyxl
|
|
6
|
+
from openpyxl.utils import get_column_letter
|
|
7
|
+
|
|
8
|
+
ERR = re.compile(r"^#(REF!|DIV/0!|VALUE!|NAME\?|N/A|NUM!|NULL!|SPILL!|CALC!)")
|
|
9
|
+
CONST = re.compile(r"(?<![A-Za-z_$:\d.])(-?\d+(?:\.\d+)?)(?![\d.]*[:A-Za-z(])")
|
|
10
|
+
BENIGN = {"0", "1", "2", "3", "4", "5", "6", "7", "12", "100", "1000", "-1", "0.5", "365", "4"}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def recalc(path):
|
|
14
|
+
"""LibreOffice opens the workbook, recalculates and saves a copy; returns {sheet: {cell: value}} or None."""
|
|
15
|
+
tmp = tempfile.mkdtemp(prefix="xr-")
|
|
16
|
+
try:
|
|
17
|
+
src = os.path.join(tmp, "in.xlsx"); shutil.copy(path, src); out = os.path.join(tmp, "out")
|
|
18
|
+
os.makedirs(out)
|
|
19
|
+
subprocess.run(["soffice", "--headless", "--norestore", "-env:UserInstallation=file://" + os.path.join(tmp, "p"),
|
|
20
|
+
"--convert-to", "xlsx:Calc MS Excel 2007 XML", "--outdir", out, src], capture_output=True, timeout=600)
|
|
21
|
+
f = os.path.join(out, "in.xlsx")
|
|
22
|
+
if not os.path.exists(f): return None
|
|
23
|
+
wb = openpyxl.load_workbook(f, data_only=True)
|
|
24
|
+
return {ws.title: {c.coordinate: c.value for row in ws.iter_rows() for c in row if c.value is not None} for ws in wb.worksheets}
|
|
25
|
+
except Exception:
|
|
26
|
+
return None
|
|
27
|
+
finally:
|
|
28
|
+
shutil.rmtree(tmp, ignore_errors=True)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def inspect(path, with_recalc=True):
|
|
32
|
+
wbf = openpyxl.load_workbook(path, data_only=False)
|
|
33
|
+
wbv = openpyxl.load_workbook(path, data_only=True)
|
|
34
|
+
z = zipfile.ZipFile(path)
|
|
35
|
+
sheets = []
|
|
36
|
+
for ws in wbf.worksheets:
|
|
37
|
+
wv = wbv[ws.title]
|
|
38
|
+
cells, n_formula, n_num, errs, consts, inputs_styled = [], 0, 0, [], [], 0
|
|
39
|
+
for row in ws.iter_rows():
|
|
40
|
+
for c in row:
|
|
41
|
+
if c.value is None: continue
|
|
42
|
+
v = wv[c.coordinate].value
|
|
43
|
+
isf = isinstance(c.value, str) and c.value.startswith("=")
|
|
44
|
+
if isf:
|
|
45
|
+
n_formula += 1
|
|
46
|
+
k = [x for x in CONST.findall(re.sub(r"\"[^\"]*\"", "", c.value[1:])) if x not in BENIGN]
|
|
47
|
+
if k: consts.append({"cell": c.coordinate, "formula": c.value[:120], "constants": k[:4]})
|
|
48
|
+
elif isinstance(c.value, (int, float)) and not isinstance(c.value, bool):
|
|
49
|
+
n_num += 1
|
|
50
|
+
fill = c.fill.fgColor.rgb if c.fill and c.fill.fill_type == "solid" and c.fill.fgColor is not None else None
|
|
51
|
+
fc = c.font.color.rgb if c.font and c.font.color is not None and c.font.color.type == "rgb" else None
|
|
52
|
+
if (fill and fill not in ("00000000", "FFFFFFFF")) or (fc and fc not in ("FF000000", None)): inputs_styled += 1
|
|
53
|
+
if isinstance(v, str) and ERR.match(v): errs.append(c.coordinate)
|
|
54
|
+
if len(cells) < 400:
|
|
55
|
+
cells.append({"c": c.coordinate, "f": c.value[:160] if isf else None, "v": v if not isinstance(v, str) else v[:120],
|
|
56
|
+
"fmt": c.number_format if c.number_format != "General" else None})
|
|
57
|
+
charts = []
|
|
58
|
+
for ch in getattr(ws, "_charts", []):
|
|
59
|
+
refs = []
|
|
60
|
+
try:
|
|
61
|
+
for s in ch.series:
|
|
62
|
+
if s.val and s.val.numRef: refs.append(s.val.numRef.f)
|
|
63
|
+
except Exception: pass
|
|
64
|
+
def _axt(ax):
|
|
65
|
+
try: return bool(ax is not None and ax.title is not None)
|
|
66
|
+
except Exception: return False
|
|
67
|
+
pie = "Pie" in type(ch).__name__ or "Doughnut" in type(ch).__name__
|
|
68
|
+
charts.append({"type": type(ch).__name__, "title": _title(ch), "refs": refs,
|
|
69
|
+
"axis_titles": None if pie else (_axt(getattr(ch, "x_axis", None)) and _axt(getattr(ch, "y_axis", None)))})
|
|
70
|
+
dv = [{"sqref": str(d.sqref), "type": d.type, "formula": d.formula1} for d in ws.data_validations.dataValidation] if ws.data_validations else []
|
|
71
|
+
comments = [{"cell": c.coordinate, "text": c.comment.text[:200]} for row in ws.iter_rows() for c in row if c.comment]
|
|
72
|
+
sheets.append({"name": ws.title, "dims": ws.dimensions, "max_row": ws.max_row, "max_col": ws.max_column, "formulas": n_formula,
|
|
73
|
+
"numbers": n_num, "inputs_styled": inputs_styled, "errors": errs[:50], "n_errors": len(errs),
|
|
74
|
+
"constants_in_formulas": consts[:40], "n_constants_in_formulas": len(consts), "charts": charts,
|
|
75
|
+
"freeze": ws.freeze_panes, "merged": [str(m) for m in ws.merged_cells.ranges][:30], "validation": dv[:20],
|
|
76
|
+
"comments": comments[:30], "cells": cells, "col_widths": {k: v.width for k, v in list(ws.column_dimensions.items())[:40] if v.width}})
|
|
77
|
+
ext = [n for n in z.namelist() if n.startswith("xl/externalLinks/")]
|
|
78
|
+
names = []
|
|
79
|
+
try: names = [{"name": n, "ref": str(d.attr_text)[:120]} for n, d in wbf.defined_names.items()]
|
|
80
|
+
except Exception: pass
|
|
81
|
+
out = {"sheets": sheets, "sheet_names": [s["name"] for s in sheets], "external_links": len(ext), "defined_names": names}
|
|
82
|
+
rc = recalc(path) if with_recalc else None
|
|
83
|
+
# _grid: every non-empty value (recalculated when available), for value checks against an answer key
|
|
84
|
+
out["_grid"] = {ws.title: {c.coordinate: (rc.get(ws.title, {}).get(c.coordinate) if rc is not None and isinstance(c.value, str) and c.value.startswith("=") else wbv[ws.title][c.coordinate].value)
|
|
85
|
+
for row in ws.iter_rows() for c in row if c.value is not None} for ws in wbf.worksheets}
|
|
86
|
+
if with_recalc:
|
|
87
|
+
out["recalc_ok"] = rc is not None
|
|
88
|
+
if rc is not None:
|
|
89
|
+
diffs, rc_err = [], []
|
|
90
|
+
for ws in wbf.worksheets:
|
|
91
|
+
wv = wbv[ws.title]; R = rc.get(ws.title, {})
|
|
92
|
+
for row in ws.iter_rows():
|
|
93
|
+
for c in row:
|
|
94
|
+
if not (isinstance(c.value, str) and c.value.startswith("=")): continue
|
|
95
|
+
a, b = wv[c.coordinate].value, R.get(c.coordinate)
|
|
96
|
+
if isinstance(b, str) and ERR.match(b): rc_err.append("%s!%s" % (ws.title, c.coordinate))
|
|
97
|
+
if isinstance(a, (int, float)) and isinstance(b, (int, float)) and abs(a - b) > max(1e-6, abs(b) * 1e-6):
|
|
98
|
+
diffs.append({"cell": "%s!%s" % (ws.title, c.coordinate), "cached": a, "recalc": b})
|
|
99
|
+
elif a is None and b is not None: diffs.append({"cell": "%s!%s" % (ws.title, c.coordinate), "cached": None, "recalc": b if not isinstance(b, str) else b[:40]})
|
|
100
|
+
out["recalc_diffs"] = diffs[:40]; out["n_recalc_diffs"] = len(diffs); out["recalc_errors"] = rc_err[:40]
|
|
101
|
+
return out
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _title(ch):
|
|
105
|
+
try:
|
|
106
|
+
t = ch.title
|
|
107
|
+
if t is None: return None
|
|
108
|
+
return " ".join(r.t for p in t.tx.rich.p for r in (p.r or []))[:100]
|
|
109
|
+
except Exception:
|
|
110
|
+
return None
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Rubric: turn check results into a score out of 100.
|
|
2
|
+
|
|
3
|
+
A rubric is plain JSON, kept next to the task so the weights are visible and versioned:
|
|
4
|
+
|
|
5
|
+
{
|
|
6
|
+
"version": "example-1",
|
|
7
|
+
"categories": {
|
|
8
|
+
"requirements": {"label": "Task requirements", "points": 60, "per_item": true},
|
|
9
|
+
"craft": {"label": "Craft", "points": 40}
|
|
10
|
+
},
|
|
11
|
+
"gates": {"missing": "zero", "unreadable": "zero"}
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
Rules, deliberately simple so a reader can predict a score by hand:
|
|
15
|
+
- Each category earns points x (share of its units passed).
|
|
16
|
+
- per_item: every element of a check counts as one unit (ten required values weigh ten times, not once).
|
|
17
|
+
Otherwise each check counts once, scaled by its weight.
|
|
18
|
+
- A category with no applicable checks hands its points to the others, so a task is never scored on what it
|
|
19
|
+
did not ask for.
|
|
20
|
+
- Gates override the sum: "zero" sets the total to 0; {"cap": N} caps it at N.
|
|
21
|
+
These are example mechanics for the kit; tune them for your own tasks.
|
|
22
|
+
"""
|
|
23
|
+
import json
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def load(path):
|
|
27
|
+
with open(path) as f:
|
|
28
|
+
return json.load(f)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def score(results, checks, rubric, gates=()):
|
|
32
|
+
cats = {k: {"label": v.get("label", k), "points": float(v["points"]), "per_item": bool(v.get("per_item")),
|
|
33
|
+
"num": 0.0, "den": 0.0, "checks": 0}
|
|
34
|
+
for k, v in rubric["categories"].items()}
|
|
35
|
+
for c in checks:
|
|
36
|
+
r = results.get(c.id)
|
|
37
|
+
u = r.units() if r is not None else None
|
|
38
|
+
if u is None or c.category not in cats: continue
|
|
39
|
+
p, n = u; x = cats[c.category]; x["checks"] += 1
|
|
40
|
+
if x["per_item"]:
|
|
41
|
+
x["num"] += p; x["den"] += n
|
|
42
|
+
else:
|
|
43
|
+
x["num"] += c.weight * p / n; x["den"] += c.weight
|
|
44
|
+
live = {k: x for k, x in cats.items() if x["den"] > 0}
|
|
45
|
+
total_points = sum(x["points"] for x in live.values()) or 1.0
|
|
46
|
+
out_cats, total = {}, 0.0
|
|
47
|
+
for k, x in cats.items():
|
|
48
|
+
possible = x["points"] * 100.0 / total_points if k in live else 0.0
|
|
49
|
+
share = x["num"] / x["den"] if k in live else None
|
|
50
|
+
earned = possible * share if share is not None else 0.0
|
|
51
|
+
total += earned
|
|
52
|
+
out_cats[k] = {"label": x["label"], "possible": round(possible, 2), "earned": round(earned, 2),
|
|
53
|
+
"share": None if share is None else round(share, 3), "checks": x["checks"]}
|
|
54
|
+
applied = []
|
|
55
|
+
for g in gates:
|
|
56
|
+
rule = rubric.get("gates", {}).get(g["id"])
|
|
57
|
+
if rule == "zero":
|
|
58
|
+
total = 0.0; applied.append(g)
|
|
59
|
+
elif isinstance(rule, dict) and "cap" in rule:
|
|
60
|
+
total = min(total, float(rule["cap"])); applied.append(g)
|
|
61
|
+
return {"total": round(total, 1), "categories": out_cats, "gates": applied, "rubric_version": rubric.get("version")}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: aspera-grader-kit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Grade the Word, PowerPoint and Excel files AI agents produce: inspectors, explainable checks and a JSON rubric.
|
|
5
|
+
Author-email: Motif Platforms <dsaran@motifplatforms.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/motifplatforms/aspera-grader-kit
|
|
8
|
+
Project-URL: Repository, https://github.com/motifplatforms/aspera-grader-kit
|
|
9
|
+
Project-URL: Issues, https://github.com/motifplatforms/aspera-grader-kit/issues
|
|
10
|
+
Project-URL: Writing, https://www.motifplatforms.com/insights
|
|
11
|
+
Project-URL: Company, https://www.motifplatforms.com
|
|
12
|
+
Keywords: llm-evaluation,ai-agents,graders,rubrics,reward-modeling,docx,pptx,xlsx
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
+
Classifier: Topic :: Software Development :: Testing
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Requires-Dist: lxml>=4.9
|
|
29
|
+
Requires-Dist: openpyxl>=3.1
|
|
30
|
+
Provides-Extra: examples
|
|
31
|
+
Requires-Dist: python-docx>=1.1; extra == "examples"
|
|
32
|
+
Provides-Extra: test
|
|
33
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
34
|
+
Requires-Dist: python-docx>=1.1; extra == "test"
|
|
35
|
+
Requires-Dist: python-pptx>=0.6; extra == "test"
|
|
36
|
+
Dynamic: license-file
|
|
37
|
+
|
|
38
|
+
# aspera-grader-kit
|
|
39
|
+
|
|
40
|
+
[](https://github.com/motifplatforms/aspera-grader-kit/actions/workflows/tests.yml) [](LICENSE) [](pyproject.toml)
|
|
41
|
+
|
|
42
|
+
A small, readable pattern for grading the files that knowledge-work agents produce: Word memos, PowerPoint decks and
|
|
43
|
+
Excel models.
|
|
44
|
+
|
|
45
|
+
Most agent evals grade the chat transcript. The client never sees the transcript. They open the file. A memo with
|
|
46
|
+
its headings faked in bold, a deck whose chart is a pasted picture, or a model with hard-coded numbers inside its
|
|
47
|
+
formulas all *look* correct. They are still wrong, and a grader that only reads text will reward them.
|
|
48
|
+
|
|
49
|
+
This kit grades the file itself:
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
output file ──► inspect ──► facts ──► checks ──► results ──► rubric ──► score + reasons
|
|
53
|
+
(docx/pptx/xlsx) (one thing each, (weights in plain JSON,
|
|
54
|
+
says why) next to the task)
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Quick start
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
pip install -e ".[test]"
|
|
61
|
+
python examples/board-memo/make_samples.py
|
|
62
|
+
python -m aspera_grader_kit examples/board-memo examples/board-memo/samples/board_memo_weak.docx
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
board-memo board_memo_weak.docx -> 26.4 / 100
|
|
67
|
+
Task requirements 16.4 / 60.0
|
|
68
|
+
Craft 10.0 / 40.0
|
|
69
|
+
[FAIL] R2 Required sections exist as real headings (0 of 2 items met)
|
|
70
|
+
- missed: Summary (no heading called Summary)
|
|
71
|
+
[FAIL] R4 Metrics table has every metric with its value and change (2 of 4 items met)
|
|
72
|
+
- missed: Operating expenses (no row)
|
|
73
|
+
- missed: EBITDA (no row)
|
|
74
|
+
[FAIL] C2 No faked headings (bold Normal text standing in for a heading) (fake headings: ['Summary', 'Risks'])
|
|
75
|
+
...
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
The strong sample scores 100. Every point lost comes with the check that took it and the evidence, so a disagreement
|
|
79
|
+
with the grader is a conversation about one line, not about a number.
|
|
80
|
+
|
|
81
|
+
## What's inside
|
|
82
|
+
|
|
83
|
+
| Part | What it does |
|
|
84
|
+
|---|---|
|
|
85
|
+
| `inspect/docx.py` | Body blocks in order (style, numbering, run formatting), tables (header rows, shading, margins), tracked changes, comments, content controls, fields, headers/footers, footnotes, hyperlinks |
|
|
86
|
+
| `inspect/pptx.py` | Slides in presentation order, titles, text and numbers, chart count and whether each keeps its workbook, tables, pictures, off-canvas shapes, body vs footer font floors, source lines, notes |
|
|
87
|
+
| `inspect/xlsx.py` | Formulas vs typed numbers, hard-coded constants inside formulas, error cells, charts and their ranges, validation, names, external links, and a LibreOffice recalculation to catch stale cached values |
|
|
88
|
+
| `checks.py` | `@check` decorator, `Result.items()` for per-item checks, runner that turns a crashing check into an `error` result instead of a crashed grade |
|
|
89
|
+
| `rubric.py` | Scores from a JSON rubric: categories, per-item counting, empty categories hand their points on, gates (`zero`, `cap`) |
|
|
90
|
+
| `examples/board-memo` | A complete task: brief, input CSV, answer key, rubric, nine checks and a strong/weak pair of outputs |
|
|
91
|
+
|
|
92
|
+
## Design rules
|
|
93
|
+
|
|
94
|
+
1. **Grade the artifact, not the transcript.** Inspect the file the way the client will open it.
|
|
95
|
+
2. **One check, one variable.** A check that tests three things can't tell you which one failed.
|
|
96
|
+
3. **Every check explains itself.** Evidence is part of the result, not a log line.
|
|
97
|
+
4. **Count requirements item by item.** A task with ten required values should weigh them ten times, not once.
|
|
98
|
+
5. **The answer key never reaches the agent.** Keep `key.json` out of anything the agent can read.
|
|
99
|
+
6. **Gates for the unforgivable.** A missing or unopenable file scores zero, however good the reasoning was.
|
|
100
|
+
7. **Weights live in data.** The rubric is JSON in the task folder, versioned with it, readable by a reviewer.
|
|
101
|
+
|
|
102
|
+
## Writing your own task
|
|
103
|
+
|
|
104
|
+
Copy `examples/board-memo`, then edit:
|
|
105
|
+
|
|
106
|
+
- `task.md`: the brief the agent sees.
|
|
107
|
+
- `key.json`: what a correct answer contains.
|
|
108
|
+
- `rubric.json`: categories, points, gates.
|
|
109
|
+
- `checks.py`: checks registered with `@check("ID", "category", "description", registry="<task id>")`.
|
|
110
|
+
|
|
111
|
+
## Where this comes from
|
|
112
|
+
|
|
113
|
+
This is the open, generic core of the harness we use at [Motif Platforms](https://www.motifplatforms.com) to grade AI
|
|
114
|
+
agents on real business deliverables. Our production tasks, answer keys, weights and judge prompts stay private. The
|
|
115
|
+
reasoning behind the pattern is in the series
|
|
116
|
+
[Teaching machines to do real work](https://www.motifplatforms.com/insights):
|
|
117
|
+
|
|
118
|
+
- [Reward hacking explained: how one bad grader teaches AI a thousand bad lessons](https://www.motifplatforms.com/insights/reward-hacking-explained-one-bad-grader)
|
|
119
|
+
- [How to design training exercises for AI agents](https://www.motifplatforms.com/insights/how-to-design-training-exercises-for-ai-agents)
|
|
120
|
+
- [Ten ways a grader lies in AI post-training](https://www.motifplatforms.com/insights/ten-ways-an-ai-grader-lies)
|
|
121
|
+
- [The correct-looking file: why AI agents fail and what experts offer](https://www.motifplatforms.com/insights/why-ai-agents-fail-at-spreadsheets-decks-documents)
|
|
122
|
+
|
|
123
|
+
Benchmark results from the full harness: [motifplatforms/mosaic-benchmarks](https://github.com/motifplatforms/mosaic-benchmarks).
|
|
124
|
+
|
|
125
|
+
Need graders or exercises built for your own agents? [dsaran@motifplatforms.com](mailto:dsaran@motifplatforms.com)
|
|
126
|
+
|
|
127
|
+
## License
|
|
128
|
+
|
|
129
|
+
MIT
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
aspera_grader_kit/__init__.py,sha256=knolxvMvIcozdPq-0YuCgAiLj4iVI5LytwaWfXC17zE,475
|
|
2
|
+
aspera_grader_kit/__main__.py,sha256=Nc9bJWsvG35Yank6DNN3lUZRbjZXOygCN9fwDs3-d3I,627
|
|
3
|
+
aspera_grader_kit/checks.py,sha256=QfYALrh6MheqR1VC7RNP-xi1Auyo_J5cs2RpiRZPKxY,3583
|
|
4
|
+
aspera_grader_kit/grade.py,sha256=C56SF6icHBTS3loS7krno4_2Je8nmbZT8IKKgDLg8os,2964
|
|
5
|
+
aspera_grader_kit/rubric.py,sha256=WHLonOiqBLi9Bx6_3eQgpEltzCgdkLS-AbP8oIvp58c,2729
|
|
6
|
+
aspera_grader_kit/inspect/__init__.py,sha256=QRQMmC9kQEvR3X5ZwXKrptmZVZQZJDzOQZ6kfzdUPYc,532
|
|
7
|
+
aspera_grader_kit/inspect/docx.py,sha256=390iUrzhxzRGOOYYFuoh7mHF42R-CmFb4OdwQChgRGk,11005
|
|
8
|
+
aspera_grader_kit/inspect/pptx.py,sha256=IH4sq22HhnS7jC1LZ2laOusbWhvDbNT3Uuvm9h0xPd4,7065
|
|
9
|
+
aspera_grader_kit/inspect/xlsx.py,sha256=YvzoY_FSdrEFdY5KQ6C_YdfMBb0Eig-X8Lhw5pkhL60,7133
|
|
10
|
+
aspera_grader_kit-0.1.0.dist-info/licenses/LICENSE,sha256=L91093P0vyBVPft5_iYp05RWBuuVBBsUPAxE17RJpvM,1072
|
|
11
|
+
aspera_grader_kit-0.1.0.dist-info/METADATA,sha256=14QSFpTVY4x9wpm6DDkMMlx1jWmWCpLRMfbphXy4BZ4,7197
|
|
12
|
+
aspera_grader_kit-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
13
|
+
aspera_grader_kit-0.1.0.dist-info/top_level.txt,sha256=ikuU6Q6js789emYjXSQCEEk_HgLqfXId69pU_29f8Gw,18
|
|
14
|
+
aspera_grader_kit-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Motif Platforms
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
aspera_grader_kit
|