analyticsreceipts 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- analyticsreceipts/__init__.py +3 -0
- analyticsreceipts/broken.py +25 -0
- analyticsreceipts/compare.py +67 -0
- analyticsreceipts/compiler.py +216 -0
- analyticsreceipts/contract.py +50 -0
- analyticsreceipts/control_server.py +26 -0
- analyticsreceipts/data/OSSIE-NOTICE.md +7 -0
- analyticsreceipts/data/online_retail.ossie.yaml +128 -0
- analyticsreceipts/data/ossie-schema.json +374 -0
- analyticsreceipts/execution.py +185 -0
- analyticsreceipts/fixtures.py +181 -0
- analyticsreceipts/grader.py +348 -0
- analyticsreceipts/live.py +423 -0
- analyticsreceipts/load.py +74 -0
- analyticsreceipts/materialise.py +86 -0
- analyticsreceipts/model.py +122 -0
- analyticsreceipts/numbers.py +93 -0
- analyticsreceipts/paths.py +31 -0
- analyticsreceipts/rawtools.py +105 -0
- analyticsreceipts/receipt.schema.json +56 -0
- analyticsreceipts/receipts.py +17 -0
- analyticsreceipts/report.py +162 -0
- analyticsreceipts/server.py +165 -0
- analyticsreceipts-0.0.2.dist-info/METADATA +313 -0
- analyticsreceipts-0.0.2.dist-info/RECORD +29 -0
- analyticsreceipts-0.0.2.dist-info/WHEEL +4 -0
- analyticsreceipts-0.0.2.dist-info/licenses/LICENSE +21 -0
- analyticsreceipts-0.0.2.dist-info/licenses/vendor/ossie/LICENSE +201 -0
- analyticsreceipts-0.0.2.dist-info/licenses/vendor/ossie/NOTICE +5 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""PLANTED BROKEN COMPILER: a control, never a product path.
|
|
2
|
+
|
|
3
|
+
It compiles exactly like the real compiler except that it drops convention C1: the dataset source is
|
|
4
|
+
`raw_lines` as loaded, so the duplicated 2010-12-01..09 window is counted twice. The canary and the
|
|
5
|
+
December 2010 fixture must FAIL when the server runs with it. If they pass, the eval cannot see a
|
|
6
|
+
broken semantic layer, and its other results mean nothing.
|
|
7
|
+
|
|
8
|
+
Used only through `python -m analyticsreceipts.control_server --bug drop_c1` (the fixture harness's control
|
|
9
|
+
run) or by passing compile_fn explicitly. The product server has no switch for it. Receipts produced with it
|
|
10
|
+
carry compiler="planted:drop_c1" and a model hash that is not the governed model's.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import dataclasses
|
|
14
|
+
|
|
15
|
+
from .compiler import compile_request
|
|
16
|
+
|
|
17
|
+
BUG_NAME = "drop_c1"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def compile_request_broken(model, req):
|
|
21
|
+
broken = dataclasses.replace(model, dataset_source="SELECT * FROM raw_lines")
|
|
22
|
+
return compile_request(broken, req)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
compile_request_broken.compiler_id = f"planted:{BUG_NAME}"
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Side-by-side table of two LIVE-LITE replays graded with the same grader. Descriptive only.
|
|
2
|
+
|
|
3
|
+
python -m analyticsreceipts.compare replays/live-lite-v1.jsonl replays/live-lite-v2.jsonl --names v1,v2
|
|
4
|
+
|
|
5
|
+
Prints a markdown table: per arm, answers, axis passes, cells, tool calls, query calls, and the count of
|
|
6
|
+
answerable MCP runs ending in a well-formed query_metric call plus a final numeric answer. Both replays are
|
|
7
|
+
graded by the grader installed now, so a difference comes from the runs, not from grading.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import collections
|
|
12
|
+
import json
|
|
13
|
+
|
|
14
|
+
from .live import read_jsonl
|
|
15
|
+
from .report import CELLS, grade_runs, load_refs
|
|
16
|
+
|
|
17
|
+
AXES = ("a", "b", "c", "d")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def summarize(path):
|
|
21
|
+
golden, _ = load_refs()
|
|
22
|
+
runs = [r for r in read_jsonl(path) if r.get("kind", "run") == "run" and not r.get("incomplete")]
|
|
23
|
+
for r in runs:
|
|
24
|
+
r["id"] = f"{r['item']}/{r['arm']}"
|
|
25
|
+
grades = {g["run_id"]: g for g in grade_runs(runs)}
|
|
26
|
+
out = {}
|
|
27
|
+
for arm in ("mcp", "raw"):
|
|
28
|
+
rs = [r for r in runs if r["arm"] == arm]
|
|
29
|
+
gs = [grades[r["id"]] for r in rs]
|
|
30
|
+
answerable = [r for r in rs if golden[r["item"]]["expected_behavior"] in ("answer", "answer_with_caveat")]
|
|
31
|
+
tool = "query_metric" if arm == "mcp" else "run_sql"
|
|
32
|
+
wf = sum(1 for r in answerable if any(c["name"] == tool and not c["tool_error"] and c["result_type"] == "ok"
|
|
33
|
+
for c in r["tool_calls"]) and grades[r["id"]]["c"] is True)
|
|
34
|
+
row = {"answers": len(gs), "answerable": len(answerable), "well_formed_call_plus_numeric": wf,
|
|
35
|
+
"tool_calls": sum(len(r["tool_calls"]) for r in rs),
|
|
36
|
+
"primary_tool_calls (query_metric or run_sql)": sum(1 for r in rs for c in r["tool_calls"] if c["name"] == tool),
|
|
37
|
+
"median_seconds": sorted(r["seconds"] for r in rs)[len(rs) // 2] if rs else None}
|
|
38
|
+
for ax in AXES:
|
|
39
|
+
app = [g[ax] for g in gs if g[ax] != "N/A"]
|
|
40
|
+
row[ax] = f"{sum(1 for v in app if v is True)}/{len(app)}"
|
|
41
|
+
cnt = collections.Counter(g["cell"] for g in gs)
|
|
42
|
+
for c in CELLS:
|
|
43
|
+
row[c] = cnt[c]
|
|
44
|
+
out[arm] = row
|
|
45
|
+
return out
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def table(a, b, names):
|
|
49
|
+
keys = list(next(iter(a.values())).keys())
|
|
50
|
+
lines = [f"| measure | {names[0]} mcp | {names[1]} mcp | {names[0]} raw | {names[1]} raw |", "|---|---|---|---|---|"]
|
|
51
|
+
for k in keys:
|
|
52
|
+
lines.append(f"| {k} | {a['mcp'][k]} | {b['mcp'][k]} | {a['raw'][k]} | {b['raw'][k]} |")
|
|
53
|
+
return "\n".join(lines)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def main():
|
|
57
|
+
ap = argparse.ArgumentParser()
|
|
58
|
+
ap.add_argument("a")
|
|
59
|
+
ap.add_argument("b")
|
|
60
|
+
ap.add_argument("--names", default="v1,v2")
|
|
61
|
+
x = ap.parse_args()
|
|
62
|
+
names = x.names.split(",")
|
|
63
|
+
print(table(summarize(x.a), summarize(x.b), names))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
if __name__ == "__main__":
|
|
67
|
+
main()
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""Deterministic compiler: a structured metric request -> one DuckDB SQL statement + bound parameters.
|
|
2
|
+
|
|
3
|
+
Rules (each one has a test):
|
|
4
|
+
* only governed metrics and declared dimensions; anything else is refused with a typed result,
|
|
5
|
+
never guessed (no raw SQL path exists);
|
|
6
|
+
* one dataset, so no joins and no fan-out;
|
|
7
|
+
* filter values are bound parameters, never interpolated into the SQL text;
|
|
8
|
+
* ratio and count-distinct metrics are computed from their components inside every group; there is
|
|
9
|
+
no rollup path that could sum or average them across groups;
|
|
10
|
+
* the same request (after canonical ordering) always yields byte-identical SQL and the same hash.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import datetime as dt
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
|
|
18
|
+
from .model import TIME_GRAINS
|
|
19
|
+
|
|
20
|
+
OPS = ("eq", "ne", "in", "is_null", "not_null")
|
|
21
|
+
ROW_CAP = 1000
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class RequestError(Exception):
|
|
25
|
+
"""A request the compiler refuses. `result_type` is what the server returns instead of rows."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, result_type, reason, candidates=()):
|
|
28
|
+
super().__init__(reason)
|
|
29
|
+
self.result_type = result_type
|
|
30
|
+
self.reason = reason
|
|
31
|
+
self.candidates = list(candidates)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class Compiled:
|
|
36
|
+
sql: str
|
|
37
|
+
params: tuple
|
|
38
|
+
sha256: str
|
|
39
|
+
request: dict # canonical form
|
|
40
|
+
columns: tuple
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _date(s, what):
|
|
44
|
+
try:
|
|
45
|
+
return dt.date.fromisoformat(str(s)).isoformat()
|
|
46
|
+
except ValueError:
|
|
47
|
+
raise RequestError("invalid_request", f"{what} must be YYYY-MM-DD, got {s!r}")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _range(r, what):
|
|
51
|
+
if not isinstance(r, dict) or "start" not in r or "end" not in r:
|
|
52
|
+
raise RequestError("needs_clarification", f"{what} needs a start and an end date")
|
|
53
|
+
a, b = _date(r["start"], f"{what}.start"), _date(r["end"], f"{what}.end")
|
|
54
|
+
if a > b:
|
|
55
|
+
raise RequestError("invalid_request", f"{what} starts after it ends")
|
|
56
|
+
return {"start": a, "end": b}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def candidates_for(model, text):
|
|
60
|
+
t = (text or "").lower().replace("_", " ")
|
|
61
|
+
out = []
|
|
62
|
+
for m in model.metrics.values():
|
|
63
|
+
names = [m.name.replace("_", " ")] + [s.lower() for s in m.synonyms]
|
|
64
|
+
if any(n in t or t in n for n in names if n):
|
|
65
|
+
out.append(m.name)
|
|
66
|
+
return sorted(set(out))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def canonical(model, req):
|
|
70
|
+
"""Validate and canonicalise. Raises RequestError for anything the model cannot answer."""
|
|
71
|
+
if not isinstance(req, dict):
|
|
72
|
+
raise RequestError("invalid_request", "request must be an object")
|
|
73
|
+
metric = req.get("metric")
|
|
74
|
+
if not metric:
|
|
75
|
+
raise RequestError("needs_clarification", "no metric was named", sorted(model.metrics))
|
|
76
|
+
if metric not in model.metrics:
|
|
77
|
+
raise RequestError("no_governed_metric", f"no governed metric named {metric!r}",
|
|
78
|
+
candidates_for(model, metric))
|
|
79
|
+
if not req.get("time_range"):
|
|
80
|
+
raise RequestError("needs_clarification", "a period (time_range) is required")
|
|
81
|
+
out = {"metric": metric, "time_range": _range(req["time_range"], "time_range")}
|
|
82
|
+
comp = req.get("comparison_range")
|
|
83
|
+
out["comparison_range"] = _range(comp, "comparison_range") if comp else None
|
|
84
|
+
grain = req.get("grain")
|
|
85
|
+
if grain is not None and grain not in TIME_GRAINS:
|
|
86
|
+
raise RequestError("invalid_request", f"grain must be one of {TIME_GRAINS}")
|
|
87
|
+
out["grain"] = grain
|
|
88
|
+
dims = list(req.get("dimensions") or [])
|
|
89
|
+
for d in dims:
|
|
90
|
+
if d not in model.dimensions or model.dimensions[d].is_time:
|
|
91
|
+
raise RequestError("invalid_request", f"{d!r} is not a declared dimension; use grain for time")
|
|
92
|
+
if len(set(dims)) != len(dims):
|
|
93
|
+
raise RequestError("invalid_request", "duplicate dimension")
|
|
94
|
+
out["dimensions"] = dims
|
|
95
|
+
filters = []
|
|
96
|
+
for f in req.get("filters") or []:
|
|
97
|
+
if not isinstance(f, dict):
|
|
98
|
+
raise RequestError("invalid_request", "each filter is {dim, op, value}")
|
|
99
|
+
d, op, v = f.get("dim"), f.get("op"), f.get("value")
|
|
100
|
+
if d not in model.dimensions or model.dimensions[d].is_time:
|
|
101
|
+
raise RequestError("invalid_request", f"filter dimension {d!r} is not declared; use time_range for dates")
|
|
102
|
+
if op not in OPS:
|
|
103
|
+
raise RequestError("invalid_request", f"filter op must be one of {OPS}")
|
|
104
|
+
if op in ("is_null", "not_null"):
|
|
105
|
+
v = None
|
|
106
|
+
elif op == "in":
|
|
107
|
+
if not isinstance(v, list) or not v or not all(isinstance(x, str) for x in v):
|
|
108
|
+
raise RequestError("invalid_request", "op 'in' needs a non-empty list of strings")
|
|
109
|
+
v = sorted(v)
|
|
110
|
+
elif not isinstance(v, str):
|
|
111
|
+
raise RequestError("invalid_request", f"op {op!r} needs a string value")
|
|
112
|
+
filters.append({"dim": d, "op": op, "value": v})
|
|
113
|
+
out["filters"] = sorted(filters, key=lambda x: json.dumps(x, sort_keys=True))
|
|
114
|
+
rk = req.get("ranking")
|
|
115
|
+
if rk:
|
|
116
|
+
if rk.get("by", metric) != metric:
|
|
117
|
+
raise RequestError("invalid_request", "ranking.by must be the requested metric")
|
|
118
|
+
direction = rk.get("direction", "desc")
|
|
119
|
+
if direction not in ("asc", "desc"):
|
|
120
|
+
raise RequestError("invalid_request", "ranking.direction is asc or desc")
|
|
121
|
+
limit = rk.get("limit")
|
|
122
|
+
if not isinstance(limit, int) or not 1 <= limit <= ROW_CAP:
|
|
123
|
+
raise RequestError("invalid_request", f"ranking.limit is an integer 1..{ROW_CAP}")
|
|
124
|
+
if not dims:
|
|
125
|
+
raise RequestError("invalid_request", "ranking needs at least one dimension to rank")
|
|
126
|
+
out["ranking"] = {"by": metric, "direction": direction, "limit": limit}
|
|
127
|
+
else:
|
|
128
|
+
out["ranking"] = None
|
|
129
|
+
return out
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _bucket(model, grain):
|
|
133
|
+
t = model.fields[model.time_dimension]
|
|
134
|
+
return f"CAST(date_trunc('{grain}', {t}) AS DATE)"
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _select(model, req, start, end, label):
|
|
138
|
+
params = []
|
|
139
|
+
t = model.fields[model.time_dimension]
|
|
140
|
+
cols, group, group_exprs = [], [], []
|
|
141
|
+
if label is not None:
|
|
142
|
+
cols.append(f"'{label}' AS period") # label is one of two fixed literals, never user text
|
|
143
|
+
if req["grain"]:
|
|
144
|
+
cols.append(f"{_bucket(model, req['grain'])} AS period_start")
|
|
145
|
+
group.append("period_start")
|
|
146
|
+
group_exprs.append(_bucket(model, req["grain"]))
|
|
147
|
+
for d in req["dimensions"]:
|
|
148
|
+
cols.append(f"{model.fields[d]} AS {d}")
|
|
149
|
+
group.append(d)
|
|
150
|
+
# group by the field EXPRESSION: an alias that shadows a raw column (stock_code) would bind to
|
|
151
|
+
# the raw column and split one trimmed code into several groups
|
|
152
|
+
group_exprs.append(model.fields[d])
|
|
153
|
+
cols.append(f"{model.metric_sql(req['metric'])} AS {req['metric']}")
|
|
154
|
+
where = [f"{t} BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)"]
|
|
155
|
+
params += [start, end]
|
|
156
|
+
for f in req["filters"]:
|
|
157
|
+
e = model.fields[f["dim"]]
|
|
158
|
+
if f["op"] == "eq":
|
|
159
|
+
where.append(f"{e} = ?")
|
|
160
|
+
params.append(f["value"])
|
|
161
|
+
elif f["op"] == "ne":
|
|
162
|
+
# "not X" keeps rows where the value is missing; SQL's <> alone would silently drop them
|
|
163
|
+
where.append(f"({e} <> ? OR {e} IS NULL)")
|
|
164
|
+
params.append(f["value"])
|
|
165
|
+
elif f["op"] == "in":
|
|
166
|
+
where.append(f"{e} IN ({', '.join('?' for _ in f['value'])})")
|
|
167
|
+
params += f["value"]
|
|
168
|
+
elif f["op"] == "is_null":
|
|
169
|
+
where.append(f"{e} IS NULL")
|
|
170
|
+
else:
|
|
171
|
+
where.append(f"{e} IS NOT NULL")
|
|
172
|
+
sql = f"SELECT {', '.join(cols)} FROM {model.dataset_name} WHERE {' AND '.join(where)}"
|
|
173
|
+
if group_exprs:
|
|
174
|
+
sql += f" GROUP BY {', '.join(group_exprs)}"
|
|
175
|
+
return sql, params, group
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def compile_request(model, req):
|
|
179
|
+
req = canonical(model, req)
|
|
180
|
+
with_cte = f"WITH {model.dataset_name} AS ({' '.join(model.dataset_source.split())}) "
|
|
181
|
+
tr, cr = req["time_range"], req["comparison_range"]
|
|
182
|
+
if cr:
|
|
183
|
+
s1, p1, group = _select(model, req, tr["start"], tr["end"], "current")
|
|
184
|
+
s2, p2, _ = _select(model, req, cr["start"], cr["end"], "comparison")
|
|
185
|
+
body = f"{s1} UNION ALL {s2}"
|
|
186
|
+
params = p1 + p2
|
|
187
|
+
order = ["period DESC"] + group # 'current' before 'comparison'
|
|
188
|
+
cols = ("period",)
|
|
189
|
+
else:
|
|
190
|
+
body, params, group = _select(model, req, tr["start"], tr["end"], None)
|
|
191
|
+
order = list(group)
|
|
192
|
+
cols = ()
|
|
193
|
+
cols += (("period_start",) if req["grain"] else ()) + tuple(req["dimensions"]) + (req["metric"],)
|
|
194
|
+
sql = with_cte + f"SELECT * FROM ({body}) AS q"
|
|
195
|
+
if req["ranking"] and cr:
|
|
196
|
+
# rank WITHIN each period: top-N of the current period, then top-N of the comparison period
|
|
197
|
+
rk = req["ranking"]
|
|
198
|
+
by = f"{req['metric']} {rk['direction'].upper()} NULLS LAST"
|
|
199
|
+
tiebreak = ", ".join(group)
|
|
200
|
+
within = by + (f", {tiebreak}" if tiebreak else "")
|
|
201
|
+
sql += f" QUALIFY row_number() OVER (PARTITION BY period ORDER BY {within}) <= {int(rk['limit'])}"
|
|
202
|
+
sql += f" ORDER BY period DESC, {within}"
|
|
203
|
+
elif req["ranking"]:
|
|
204
|
+
rk = req["ranking"]
|
|
205
|
+
tiebreak = ", ".join(order)
|
|
206
|
+
sql += f" ORDER BY {req['metric']} {rk['direction'].upper()} NULLS LAST" + (f", {tiebreak}" if tiebreak else "")
|
|
207
|
+
sql += f" LIMIT {int(rk['limit'])}"
|
|
208
|
+
else:
|
|
209
|
+
if order:
|
|
210
|
+
sql += f" ORDER BY {', '.join(order)}"
|
|
211
|
+
sql += f" LIMIT {ROW_CAP + 1}"
|
|
212
|
+
payload = sql + "\n" + json.dumps(params)
|
|
213
|
+
return Compiled(sql, tuple(params), hashlib.sha256(payload.encode()).hexdigest(), req, cols)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
compile_request.compiler_id = "governed" # recorded in every receipt
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""The output contract both arms must follow, and its parser.
|
|
2
|
+
|
|
3
|
+
The agent's last message is one JSON object:
|
|
4
|
+
{"behavior": "answer" | "clarify" | "abstain",
|
|
5
|
+
"resolved_spec": {metric, filters, period{start, end, incomplete}, comparison, grain, dimensions, ranking} | null,
|
|
6
|
+
"answer": "<one or two sentences stating the number>" | null,
|
|
7
|
+
"clarification_question": "<question>" | null,
|
|
8
|
+
"abstain_reason": "<reason>" | null}
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
|
|
14
|
+
BEHAVIORS = ("answer", "clarify", "abstain")
|
|
15
|
+
|
|
16
|
+
CONTRACT_TEXT = (
|
|
17
|
+
"When you are done, reply with ONLY one JSON object and nothing else:\n"
|
|
18
|
+
'{"behavior": "answer" | "clarify" | "abstain",\n'
|
|
19
|
+
' "resolved_spec": {"metric": str, "filters": [{"dim": str, "op": "eq"|"ne"|"in"|"is_null", "value": ...}],\n'
|
|
20
|
+
' "period": {"start": "YYYY-MM-DD", "end": "YYYY-MM-DD", "incomplete": bool},\n'
|
|
21
|
+
' "comparison": {"period": {"start": "YYYY-MM-DD", "end": "YYYY-MM-DD"}} | null,\n'
|
|
22
|
+
' "grain": "day"|"month"|"quarter"|"year"|null, "dimensions": [str],\n'
|
|
23
|
+
' "ranking": {"by": str, "direction": "asc"|"desc", "limit": int} | null} | null,\n'
|
|
24
|
+
' "answer": str | null, "clarification_question": str | null, "abstain_reason": str | null}\n'
|
|
25
|
+
"Use behavior \"clarify\" when the question is ambiguous and \"abstain\" when the data cannot answer it; "
|
|
26
|
+
"then resolved_spec is null. Set period.incomplete to true when the period extends past the end of the data."
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def parse_final(text):
|
|
31
|
+
"""Returns (dict, None) or (None, reason). Accepts a fenced ```json block or the first JSON object."""
|
|
32
|
+
if not text or not text.strip():
|
|
33
|
+
return None, "empty final message"
|
|
34
|
+
m = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.S)
|
|
35
|
+
candidates = [m.group(1)] if m else []
|
|
36
|
+
start = text.find("{")
|
|
37
|
+
if start >= 0: # the first JSON object, decoded properly (braces inside strings do not count)
|
|
38
|
+
try:
|
|
39
|
+
candidates.append(json.JSONDecoder().raw_decode(text, start)[0])
|
|
40
|
+
except json.JSONDecodeError:
|
|
41
|
+
pass
|
|
42
|
+
for c in candidates:
|
|
43
|
+
if isinstance(c, str):
|
|
44
|
+
try:
|
|
45
|
+
c = json.loads(c)
|
|
46
|
+
except json.JSONDecodeError:
|
|
47
|
+
continue
|
|
48
|
+
if isinstance(c, dict) and c.get("behavior") in BEHAVIORS:
|
|
49
|
+
return c, None
|
|
50
|
+
return None, "final message is not a contract JSON object"
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""CONTROL entry point: the MCP server with a planted broken compiler. Never a product path.
|
|
2
|
+
|
|
3
|
+
Used by the fixture harness to prove the eval can see a broken semantic layer (the canary and the
|
|
4
|
+
December 2010 fixture must FAIL). Receipts say compiler="planted:<bug>".
|
|
5
|
+
|
|
6
|
+
Usage: python -m analyticsreceipts.control_server --bug drop_c1 [--db ...] [--model ...]
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import os
|
|
11
|
+
|
|
12
|
+
from .broken import BUG_NAME, compile_request_broken
|
|
13
|
+
from .server import build_server
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def main(argv=None):
|
|
17
|
+
ap = argparse.ArgumentParser(prog="analyticsreceipts.control_server")
|
|
18
|
+
ap.add_argument("--bug", required=True, choices=[BUG_NAME])
|
|
19
|
+
ap.add_argument("--db", default=os.environ.get("ANALYTICSRECEIPTS_DB"))
|
|
20
|
+
ap.add_argument("--model", default=os.environ.get("ANALYTICSRECEIPTS_MODEL"))
|
|
21
|
+
a = ap.parse_args(argv)
|
|
22
|
+
build_server(db_path=a.db, model_path=a.model, compile_fn=compile_request_broken).run("stdio")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
if __name__ == "__main__":
|
|
26
|
+
main()
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Vendored: Apache Ossie core spec
|
|
2
|
+
|
|
3
|
+
`ossie-schema.json` and `spec.md` are copied unmodified from https://github.com/apache/ossie
|
|
4
|
+
(`core-spec/`), commit `6c9386ee95e31c63d06cec88812c3a4969fa9fd9` (2026-09-26).
|
|
5
|
+
Licensed under the Apache License, Version 2.0: `LICENSE` and `NOTICE` here are copied unmodified from the
|
|
6
|
+
same commit (https://www.apache.org/licenses/LICENSE-2.0).
|
|
7
|
+
They are used only to validate `model/online_retail.ossie.yaml` against the specification.
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
version: 0.2.0.dev0
|
|
2
|
+
name: online_retail
|
|
3
|
+
description: >-
|
|
4
|
+
UK online retailer, invoice lines 2009-12-01 to 2011-12-09 (UCI Online Retail II, dataset 502, CC BY 4.0).
|
|
5
|
+
Governed metrics for the analyticsreceipts eval. Expressions are ANSI SQL as executed by DuckDB.
|
|
6
|
+
ai_context:
|
|
7
|
+
instructions: >-
|
|
8
|
+
Business conventions. (C1) The two yearly sheets overlap on 2010-12-01 to 2010-12-09; that window is
|
|
9
|
+
counted once. (C2) A product line has a stock code (spaces trimmed) of five digits optionally followed by
|
|
10
|
+
up to two letters; postage, fees, manual entries, discounts, bad-debt adjustments, samples, charity,
|
|
11
|
+
tests, gift vouchers and other short codes are non-product. (C3) Revenue means net product revenue:
|
|
12
|
+
quantity times price over product lines, including cancellation lines (negative quantities), excluding
|
|
13
|
+
non-product lines. (C4) An order is an invoice that does not start with C (cancellation) or A (bad-debt
|
|
14
|
+
adjustment) and has at least one product line. (C5) A customer is a distinct customer ID; lines without
|
|
15
|
+
one are never a customer, and revenue per customer uses only revenue on lines with a customer ID.
|
|
16
|
+
(C6) A cancellation is an invoice starting with C; cancelled units are minus the summed quantity on its
|
|
17
|
+
product lines. (C7) Periods are inclusive calendar dates; the data ends 2011-12-09 12:50, so a period
|
|
18
|
+
after that date is incomplete and must be said to be. (C8) Identical lines within one sheet are kept.
|
|
19
|
+
datasets:
|
|
20
|
+
- name: lines
|
|
21
|
+
source: >-
|
|
22
|
+
SELECT * FROM raw_lines
|
|
23
|
+
WHERE NOT (source_sheet = 'Year 2010-2011' AND invoice_ts < TIMESTAMP '2010-12-10 00:00:00')
|
|
24
|
+
primary_key: [source_sheet, row_in_sheet]
|
|
25
|
+
description: One row per invoice line, with the duplicated 2010-12-01..09 window counted once (C1).
|
|
26
|
+
fields:
|
|
27
|
+
- name: invoice
|
|
28
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: invoice}]}
|
|
29
|
+
datatype: String
|
|
30
|
+
- name: stock_code
|
|
31
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: trim(stock_code)}]}
|
|
32
|
+
datatype: String
|
|
33
|
+
dimension: {is_time: false}
|
|
34
|
+
ai_context: {synonyms: [product code, SKU, item]}
|
|
35
|
+
- name: quantity
|
|
36
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: quantity}]}
|
|
37
|
+
datatype: Integer
|
|
38
|
+
- name: price
|
|
39
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: price}]}
|
|
40
|
+
datatype: Decimal
|
|
41
|
+
- name: line_value
|
|
42
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: quantity * price}]}
|
|
43
|
+
datatype: Decimal
|
|
44
|
+
- name: customer_id
|
|
45
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: customer_id}]}
|
|
46
|
+
datatype: String
|
|
47
|
+
dimension: {is_time: false}
|
|
48
|
+
description: Missing on about 22.5 % of lines; filter with is_null to select those lines.
|
|
49
|
+
- name: country
|
|
50
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: country}]}
|
|
51
|
+
datatype: String
|
|
52
|
+
dimension: {is_time: false}
|
|
53
|
+
ai_context: {synonyms: [market, region]}
|
|
54
|
+
- name: invoice_date
|
|
55
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: CAST(invoice_ts AS DATE)}]}
|
|
56
|
+
datatype: Date
|
|
57
|
+
dimension: {is_time: true}
|
|
58
|
+
- name: is_cancellation
|
|
59
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: "invoice LIKE 'C%'"}]}
|
|
60
|
+
datatype: Boolean
|
|
61
|
+
- name: is_bad_debt
|
|
62
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: "invoice LIKE 'A%'"}]}
|
|
63
|
+
datatype: Boolean
|
|
64
|
+
- name: is_product
|
|
65
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: "trim(stock_code) SIMILAR TO '[0-9]{5}[A-Za-z]{0,2}'"}]}
|
|
66
|
+
datatype: Boolean
|
|
67
|
+
- name: is_order_line
|
|
68
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: "invoice NOT LIKE 'C%' AND invoice NOT LIKE 'A%' AND trim(stock_code) SIMILAR TO '[0-9]{5}[A-Za-z]{0,2}'"}]}
|
|
69
|
+
datatype: Boolean
|
|
70
|
+
metrics:
|
|
71
|
+
- name: net_product_revenue
|
|
72
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: "SUM(CASE WHEN lines.is_product THEN lines.line_value END)"}]}
|
|
73
|
+
description: Net product revenue in GBP (C2, C3). The default meaning of revenue, sales or turnover.
|
|
74
|
+
datatype: Decimal
|
|
75
|
+
ai_context:
|
|
76
|
+
synonyms: [revenue, sales, turnover, net revenue]
|
|
77
|
+
instructions: Use for revenue unless the question says otherwise. Excludes postage, fees and adjustments.
|
|
78
|
+
custom_extensions:
|
|
79
|
+
- vendor_name: analyticsreceipts
|
|
80
|
+
data: '{"type":"simple","version":"1.0.0","unit":"GBP","non_additive":false,"conventions":["C1","C2","C3"],"status":"approved"}'
|
|
81
|
+
- name: units_sold
|
|
82
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: "SUM(CASE WHEN lines.is_order_line THEN lines.quantity END)"}]}
|
|
83
|
+
description: Units sold on product lines of orders (C4).
|
|
84
|
+
datatype: Integer
|
|
85
|
+
ai_context: {synonyms: [units, quantity sold, items sold]}
|
|
86
|
+
custom_extensions:
|
|
87
|
+
- vendor_name: analyticsreceipts
|
|
88
|
+
data: '{"type":"simple","version":"1.0.0","unit":"units","non_additive":false,"conventions":["C1","C2","C4"],"status":"approved"}'
|
|
89
|
+
- name: cancelled_units
|
|
90
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: "-SUM(CASE WHEN lines.is_cancellation AND lines.is_product THEN lines.quantity END)"}]}
|
|
91
|
+
description: Units on product lines of cancellation invoices, reported as a positive number (C6).
|
|
92
|
+
datatype: Integer
|
|
93
|
+
ai_context: {synonyms: [returned units, cancelled quantity]}
|
|
94
|
+
custom_extensions:
|
|
95
|
+
- vendor_name: analyticsreceipts
|
|
96
|
+
data: '{"type":"simple","version":"1.0.0","unit":"units","non_additive":false,"conventions":["C1","C2","C6"],"status":"approved"}'
|
|
97
|
+
- name: orders
|
|
98
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: "COUNT(DISTINCT CASE WHEN lines.is_order_line THEN lines.invoice END)"}]}
|
|
99
|
+
description: Count of orders (C4). Cancellation and bad-debt invoices are not orders.
|
|
100
|
+
datatype: Integer
|
|
101
|
+
ai_context: {synonyms: [order count, number of orders, purchases]}
|
|
102
|
+
custom_extensions:
|
|
103
|
+
- vendor_name: analyticsreceipts
|
|
104
|
+
data: '{"type":"count_distinct","version":"1.0.0","unit":"orders","non_additive":true,"conventions":["C1","C2","C4"],"status":"approved"}'
|
|
105
|
+
- name: active_customers
|
|
106
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: "COUNT(DISTINCT CASE WHEN lines.is_order_line AND lines.customer_id IS NOT NULL THEN lines.customer_id END)"}]}
|
|
107
|
+
description: Distinct customer IDs with at least one order (C4, C5).
|
|
108
|
+
datatype: Integer
|
|
109
|
+
ai_context: {synonyms: [customers, buyers, number of customers]}
|
|
110
|
+
custom_extensions:
|
|
111
|
+
- vendor_name: analyticsreceipts
|
|
112
|
+
data: '{"type":"count_distinct","version":"1.0.0","unit":"customers","non_additive":true,"conventions":["C1","C2","C4","C5"],"status":"approved"}'
|
|
113
|
+
- name: revenue_per_customer
|
|
114
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: "SUM(CASE WHEN lines.is_product AND lines.customer_id IS NOT NULL THEN lines.line_value END) / NULLIF(COUNT(DISTINCT CASE WHEN lines.is_order_line AND lines.customer_id IS NOT NULL THEN lines.customer_id END), 0)"}]}
|
|
115
|
+
description: Net product revenue on lines with a customer ID divided by active customers (C5). A ratio.
|
|
116
|
+
datatype: Decimal
|
|
117
|
+
ai_context: {synonyms: [average revenue per customer, revenue per buyer, ARPC]}
|
|
118
|
+
custom_extensions:
|
|
119
|
+
- vendor_name: analyticsreceipts
|
|
120
|
+
data: '{"type":"ratio","version":"1.0.0","unit":"GBP per customer","non_additive":true,"conventions":["C1","C2","C3","C5"],"status":"approved"}'
|
|
121
|
+
- name: cancellation_rate
|
|
122
|
+
expression: {dialects: [{dialect: ANSI_SQL, expression: "COUNT(DISTINCT CASE WHEN lines.is_cancellation THEN lines.invoice END) * 1.0 / NULLIF(COUNT(DISTINCT CASE WHEN NOT lines.is_bad_debt THEN lines.invoice END), 0)"}]}
|
|
123
|
+
description: Cancellation invoices divided by invoices starting with a digit or C (A excluded). A ratio between 0 and 1.
|
|
124
|
+
datatype: Decimal
|
|
125
|
+
ai_context: {synonyms: [cancel rate, share of invoices cancelled]}
|
|
126
|
+
custom_extensions:
|
|
127
|
+
- vendor_name: analyticsreceipts
|
|
128
|
+
data: '{"type":"ratio","version":"1.0.0","unit":"ratio","non_additive":true,"conventions":["C1","C6"],"status":"approved"}'
|