overscore-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- overscore_cli/__init__.py +4 -0
- overscore_cli/ovs.py +1766 -0
- overscore_cli/skills/overscore/SKILL.md +52 -0
- overscore_cli/skills/overscore-model/SKILL.md +45 -0
- overscore_cli/skills/overscore-model/model.py +29 -0
- overscore_cli/skills/overscore-model/model.toml +15 -0
- overscore_cli/skills/overscore-sync/SKILL.md +37 -0
- overscore_cli-0.1.0.dist-info/METADATA +39 -0
- overscore_cli-0.1.0.dist-info/RECORD +12 -0
- overscore_cli-0.1.0.dist-info/WHEEL +4 -0
- overscore_cli-0.1.0.dist-info/entry_points.txt +2 -0
- overscore_model.py +75 -0
overscore_cli/ovs.py
ADDED
|
@@ -0,0 +1,1766 @@
|
|
|
1
|
+
"""ovs: the customer-side command line. Talks only HTTP, so it can ship on its own.
|
|
2
|
+
|
|
3
|
+
Built for agents as much as people:
|
|
4
|
+
- every command takes `-o json`; when stdout is not a terminal, json is the default
|
|
5
|
+
- failures print {"success": false, "error": "...", "status": N} and exit 1
|
|
6
|
+
- the key comes from OVERSCORE_KEY (or --key), the address from OVERSCORE_URL (or --url)
|
|
7
|
+
- `ovs guide` prints the short reference an agent should read before doing anything
|
|
8
|
+
|
|
9
|
+
ovs whoami
|
|
10
|
+
ovs tables ovs describe sales orders
|
|
11
|
+
ovs query "select 1" echo "select 1" | ovs query
|
|
12
|
+
ovs insert sales orders rows.json
|
|
13
|
+
ovs load sales prices prices.csv [--tab Sheet1 --skip 2]
|
|
14
|
+
ovs notes sales ovs note sales refunds "A refund counts when issued."
|
|
15
|
+
ovs search refund
|
|
16
|
+
ovs models sales ovs run sales [--only m1] ovs test sales
|
|
17
|
+
ovs runs [-n 20] ovs run-log <id>
|
|
18
|
+
ovs signoff sales orders_clean "One row per order"
|
|
19
|
+
ovs export sales paid "select * from sales.orders_clean where status='paid'"
|
|
20
|
+
ovs connectors ovs connect stripe key=rk_live_... ovs connect shopify shop=acme token=shpat_...
|
|
21
|
+
ovs link mydb "host=... user=... password=... dbname=..." [--copy] [--tables a b] [--schema public]
|
|
22
|
+
ovs links ovs copy mydb ovs unlink mydb
|
|
23
|
+
ovs guide
|
|
24
|
+
|
|
25
|
+
The laptop side (CLAUDE_CODE_PLAN.md): a project folder an agent can read and write, run-it-here before push-it-there.
|
|
26
|
+
ovs init [dir] --dish sales [--for claude|codex|cursor|gemini|agents|all]
|
|
27
|
+
ovs pull sales.leads [--rows 10000] [--where "..."] a sample as Parquet into .overscore/, for local runs
|
|
28
|
+
ovs model run lead_scores --local models/lead_scores/model.py here, on the pulled samples
|
|
29
|
+
ovs model push lead_scores ovs model run lead_scores ovs model log lead_scores
|
|
30
|
+
ovs uploads ovs upload put lead_model files/lead_model.pkl ovs upload live lead_model 1
|
|
31
|
+
ovs destinations ovs destination add webhook "Their CRM" https://... ovs destination add hubspot HubSpot --source sales.hubspot
|
|
32
|
+
ovs syncs ovs sync add scores_out --from lead_scores --key lead_id --to "Their CRM" ovs sync run scores_out
|
|
33
|
+
|
|
34
|
+
Shipped on its own as overscore-cli (cli/pyproject.toml): this file, model_helper.py (the overscore_model a script imports)
|
|
35
|
+
and the skills ovs init writes. Nothing here imports the server.
|
|
36
|
+
"""
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
import argparse
|
|
40
|
+
import hashlib
|
|
41
|
+
import json
|
|
42
|
+
import os
|
|
43
|
+
import pathlib
|
|
44
|
+
import re
|
|
45
|
+
import shutil
|
|
46
|
+
import subprocess
|
|
47
|
+
import sys
|
|
48
|
+
import tempfile
|
|
49
|
+
import threading
|
|
50
|
+
import time
|
|
51
|
+
|
|
52
|
+
import requests
|
|
53
|
+
|
|
54
|
+
GUIDE = """ovs: talk to Overscore.
|
|
55
|
+
|
|
56
|
+
Setup: ovs login (approve this computer in the browser; the key goes in the keychain)
|
|
57
|
+
or export OVERSCORE_KEY=ovs_... (a key from the Agents screen) and OVERSCORE_URL=https://<you>.overscore.tech
|
|
58
|
+
Check: ovs whoami (your dishes and levels: pour=add rows, read=read only, hot=read+write, masked=stand-ins)
|
|
59
|
+
|
|
60
|
+
Find things: ovs tables | ovs index | ovs search <word> | ovs describe <dish> <table> | ovs notes <dish>
|
|
61
|
+
Ask: ovs query "select ..." (DuckDB SQL; tables are dish.table; webhook rows keep JSON in payload)
|
|
62
|
+
Load: ovs insert <dish> <table> rows.json | ovs load <dish> <table> file.csv|xlsx|parquet
|
|
63
|
+
Remember: ovs note <dish> <name> "what you learned" (confirmed if your key is hot, else heard)
|
|
64
|
+
Model: ovs models <dish> | ovs run <dish> | ovs test <dish> | ovs signoff <dish> <table> "description"
|
|
65
|
+
Watch: ovs runs | ovs run-log <id>
|
|
66
|
+
Share: ovs export <dish> <name> "select ..." -> a live link, no key needed by the reader
|
|
67
|
+
Sources: ovs connectors | ovs how <name> | ovs connect <name> k=v ... (Stripe, Shopify, QuickBooks; hourly)
|
|
68
|
+
Work: ovs fields <dish> <table> | ovs field <dish> <table> status one_of "what it means" --allowed new done
|
|
69
|
+
ovs mark <dish> <table> <id,id> done=true note="shipped" | ovs marks <dish> <table> [id] (the <table>_work view has it all)
|
|
70
|
+
Plan: ovs plan today's meters and ceilings; ovs claim <code> pull a trial conversation into your Overscore
|
|
71
|
+
Watch: ovs issues [--sweep] what failed or went quiet, what looks wrong, the proposed fix; ovs fix <id> = "yeah, go for it"
|
|
72
|
+
Research: ovs research acme.com read the website first: systems seen, what it sells, who is named
|
|
73
|
+
Interview: ovs interview [area] [--person Dana] a consultant's first conversation; 'done' saves what was heard as confirmed
|
|
74
|
+
ovs map how much is known per area
|
|
75
|
+
Inbox: open <your Overscore address>/inbox in a browser with a hot key: the same cards, with buttons
|
|
76
|
+
Context: ovs context "<question>" one search over facts, notes and tables; then SQL on the table it names
|
|
77
|
+
ovs propose "Dana owns Shopify now" | ovs cards | ovs approve <id> | ovs ignore <id> | ovs facts people
|
|
78
|
+
Webhooks: ovs hook <dish> <table> [--provider stripe --secret whsec_...] -> an address to paste into the other app
|
|
79
|
+
ovs hooks | ovs hook-log <token> | ovs unhook <token>
|
|
80
|
+
Your Postgres: ovs link <dish> "<dsn>" live: queries hit your database in place (use a read replica)
|
|
81
|
+
ovs link <dish> "<dsn>" --copy copy: snapshot into the lake every hour
|
|
82
|
+
ovs links | ovs copy <dish> | ovs unlink <dish>
|
|
83
|
+
Project: ovs init --dish <dish> [--for claude|codex|cursor|gemini|agents|all] overscore.toml, models/, files/, syncs.toml, the rules and skills
|
|
84
|
+
Python models: ovs pull <dish.table> a sample (10,000 rows at most) into .overscore/ for local runs
|
|
85
|
+
ovs model run <name> --local models/<name>/model.py here, on the samples and files/; nothing leaves the laptop
|
|
86
|
+
ovs model push <name> save model.py, its reads, uploads and requirements (model.toml), and run it in Overscore
|
|
87
|
+
ovs model run <name> | ovs model log <name>
|
|
88
|
+
Files: ovs uploads | ovs upload put <name> <file> [--note ...] | ovs upload live <name> <version> | ovs upload rm <name>
|
|
89
|
+
Destinations: ovs destinations | ovs destination add webhook "<title>" <https-url> | ovs destination add hubspot "<title>" --source <dish.source>
|
|
90
|
+
ovs destination check <dest> | ovs destination rm <dest>
|
|
91
|
+
Syncs: ovs sync add <name> --from <table> --key <id column> --to <dest> [--columns a b=field] [--after-change | --every 5m | --manual]
|
|
92
|
+
ovs syncs | ovs sync run <name> | ovs sync show <name> | ovs sync pause|resume|rm <name>
|
|
93
|
+
|
|
94
|
+
Rules: read notes before writing SQL; prefer signed_off tables, then models, then raw, and say which you used.
|
|
95
|
+
Every command: -o json for machine output. Failures exit 1 with {"success": false, "error": ...}.
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class Fail(Exception):
|
|
100
|
+
"""A failure the person (or agent) can act on. `data` rides along in the JSON (a run's log, say)."""
|
|
101
|
+
def __init__(self, msg: str, status: int | None = None, data: dict | None = None):
|
|
102
|
+
super().__init__(msg); self.status = status; self.data = data or {}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class Client:
|
|
106
|
+
def __init__(self, url: str, key: str | None):
|
|
107
|
+
self.url = url.rstrip("/"); self.h = {"Authorization": f"Bearer {key}"} if key else {}
|
|
108
|
+
|
|
109
|
+
def call(self, method: str, path: str, **kw):
|
|
110
|
+
headers = {**self.h, **(kw.pop("headers", None) or {})}
|
|
111
|
+
try:
|
|
112
|
+
r = requests.request(method, self.url + path, headers=headers, timeout=kw.pop("timeout", 120), **kw)
|
|
113
|
+
except requests.RequestException as e:
|
|
114
|
+
raise Fail(f"cannot reach {self.url}: {e}")
|
|
115
|
+
if r.status_code >= 400:
|
|
116
|
+
try:
|
|
117
|
+
j = r.json()
|
|
118
|
+
detail = (j.get("error") or j.get("detail") or r.text) if isinstance(j, dict) else r.text
|
|
119
|
+
except ValueError:
|
|
120
|
+
detail = r.text
|
|
121
|
+
raise Fail(str(detail)[:500], r.status_code)
|
|
122
|
+
return r.json() if r.headers.get("content-type", "").startswith("application/json") else r.text
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _table(rows: list[dict] | list[list], columns: list[str] | None = None) -> str:
|
|
126
|
+
if not rows:
|
|
127
|
+
return "(no rows)"
|
|
128
|
+
if isinstance(rows[0], dict):
|
|
129
|
+
columns = columns or list(rows[0].keys())
|
|
130
|
+
rows = [[r.get(c) for c in columns] for r in rows]
|
|
131
|
+
cells = [[("" if v is None else str(v))[:60] for v in r] for r in rows]
|
|
132
|
+
widths = [max(len(c), *(len(r[i]) for r in cells)) for i, c in enumerate(columns)]
|
|
133
|
+
line = " ".join(c.ljust(w) for c, w in zip(columns, widths))
|
|
134
|
+
out = [line, " ".join("-" * w for w in widths)]
|
|
135
|
+
out += [" ".join(v.ljust(w) for v, w in zip(r, widths)) for r in cells]
|
|
136
|
+
return "\n".join(out)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def run(a, c: Client):
|
|
140
|
+
"""Returns (json_value, human_text)."""
|
|
141
|
+
if a.cmd in LAPTOP:
|
|
142
|
+
return LAPTOP[a.cmd](a, c)
|
|
143
|
+
if a.cmd == "guide":
|
|
144
|
+
return {"guide": GUIDE}, GUIDE
|
|
145
|
+
if a.cmd == "whoami":
|
|
146
|
+
d = c.call("GET", "/whoami"); return d, json.dumps(d, indent=1)
|
|
147
|
+
if a.cmd == "tables":
|
|
148
|
+
d = c.call("GET", "/tables")
|
|
149
|
+
rows = d if isinstance(d, list) else d.get("tables", d)
|
|
150
|
+
return d, _table(rows) if isinstance(rows, list) else json.dumps(d, indent=1)
|
|
151
|
+
if a.cmd == "index":
|
|
152
|
+
d = c.call("GET", "/index")
|
|
153
|
+
lines = []
|
|
154
|
+
for dish, ts in d.get("dishes", {}).items():
|
|
155
|
+
for t in ts:
|
|
156
|
+
lines.append({"table": f"{dish}.{t['table']}", "tier": t["tier"], "rows": t["rows"],
|
|
157
|
+
"description": t["description"]})
|
|
158
|
+
for dish, ns in d.get("notes", {}).items():
|
|
159
|
+
for n in ns:
|
|
160
|
+
lines.append({"table": f"{dish} note: {n['name']}", "tier": n["trust"], "rows": "", "description": n["title"]})
|
|
161
|
+
return d, _table(lines, ["table", "tier", "rows", "description"])
|
|
162
|
+
if a.cmd == "describe":
|
|
163
|
+
d = c.call("GET", f"/index/{a.dish}/{a.table}")
|
|
164
|
+
cols = [{"column": x["name"], "type": x["type"], "null_rate": x.get("null_rate"),
|
|
165
|
+
"sample": ", ".join(str(v) for v in (x.get("samples") or [v["value"] for v in x.get("values", [])])[:3])}
|
|
166
|
+
for x in d.get("columns", [])]
|
|
167
|
+
head = f"{a.dish}.{a.table} tier={d.get('tier')} rows={d.get('rows')} {d.get('description') or ''}\n"
|
|
168
|
+
return d, head + _table(cols, ["column", "type", "null_rate", "sample"])
|
|
169
|
+
if a.cmd == "search":
|
|
170
|
+
d = c.call("GET", "/search", params={"q": a.text})
|
|
171
|
+
t = [{"kind": "table", "name": f"{x['dish']}.{x['table']}", "tier": x["tier"], "hit": ",".join(x["matched"])} for x in d["tables"]]
|
|
172
|
+
t += [{"kind": "note", "name": f"{x['dish']}/{x['note']}", "tier": x["trust"], "hit": x["snippet"][:60]} for x in d["notes"]]
|
|
173
|
+
return d, _table(t, ["kind", "name", "tier", "hit"])
|
|
174
|
+
if a.cmd == "query":
|
|
175
|
+
sql = a.sql if a.sql and a.sql != "-" else sys.stdin.read()
|
|
176
|
+
d = c.call("POST", "/query", json={"sql": sql, "max_rows": a.max_rows})
|
|
177
|
+
note = f"\n({d['row_count']} rows, {d['ms']:.0f} ms{', truncated' if d.get('truncated') else ''})"
|
|
178
|
+
return d, _table(d["rows"], d["columns"]) + note
|
|
179
|
+
if a.cmd == "insert":
|
|
180
|
+
rows = json.load(open(a.file)) if a.file != "-" else json.load(sys.stdin)
|
|
181
|
+
d = c.call("POST", f"/in/{a.dish}/{a.table}", json=rows); return d, f"inserted {d.get('rows')} rows into {a.dish}.{a.table}"
|
|
182
|
+
if a.cmd == "load":
|
|
183
|
+
params = {"name": os.path.basename(a.file)}
|
|
184
|
+
if a.tab: params["tab"] = a.tab
|
|
185
|
+
if a.skip: params["skip"] = a.skip
|
|
186
|
+
with open(a.file, "rb") as f:
|
|
187
|
+
d = c.call("POST", f"/files/{a.dish}/{a.table}", params=params, data=f, timeout=600)
|
|
188
|
+
return d, f"loaded {a.dish}.{a.table}: {json.dumps(d)}"
|
|
189
|
+
if a.cmd == "notes":
|
|
190
|
+
d = c.call("GET", f"/notes/{a.dish}", params={"full": "true"})
|
|
191
|
+
text = "\n\n".join(f"## {n['title']} [{n['trust']}, {n['name']}, by {n['written_by']}]\n{n['body']}" for n in d["notes"]) or "(no notes)"
|
|
192
|
+
return d, text
|
|
193
|
+
if a.cmd == "note":
|
|
194
|
+
body = a.body if a.body and a.body != "-" else sys.stdin.read()
|
|
195
|
+
d = c.call("PUT", f"/notes/{a.dish}/{a.name}", json={"body": body, "title": a.title}); return d, f"saved note {a.dish}/{a.name} as {d['trust']} v{d['version']}"
|
|
196
|
+
if a.cmd == "models":
|
|
197
|
+
d = c.call("GET", f"/models/{a.dish}")
|
|
198
|
+
rows = [{"model": m["name"], "depends_on": ",".join(m.get("depends_on", [])), "description": m.get("description", "")} for m in d.get("detail", [])]
|
|
199
|
+
return d, _table(rows, ["model", "depends_on", "description"])
|
|
200
|
+
if a.cmd == "run":
|
|
201
|
+
d = c.call("POST", f"/run/{a.dish}", json={"only": a.only} if a.only else None, headers={"x-overscore-trigger": "cli"}, timeout=3600)
|
|
202
|
+
rows = [{"model": r.get("model") or r.get("name"), "ok": r["ok"], "rows": r.get("rows"), "error": (r.get("error") or "")[:60]} for r in d["results"]]
|
|
203
|
+
return d, _table(rows, ["model", "ok", "rows", "error"]) + ("\nall ok" if d["ok"] else "\nSOME FAILED")
|
|
204
|
+
if a.cmd == "test":
|
|
205
|
+
d = c.call("POST", f"/tests/{a.dish}", timeout=3600)
|
|
206
|
+
rows = [{"test": r.get("test") or r.get("name"), "ok": r["ok"], "failing_rows": r.get("rows"), "error": (r.get("error") or "")[:60]} for r in d["results"]]
|
|
207
|
+
return d, _table(rows, ["test", "ok", "failing_rows", "error"]) + ("\nall ok" if d["ok"] else "\nSOME FAILED")
|
|
208
|
+
if a.cmd == "signoff":
|
|
209
|
+
d = c.call("POST", f"/index/{a.dish}/{a.table}/signoff", json={"on": not a.off, "description": a.description}); return d, json.dumps(d)
|
|
210
|
+
if a.cmd == "runs":
|
|
211
|
+
d = c.call("GET", "/runs", params={"limit": a.n})
|
|
212
|
+
runs = d if isinstance(d, list) else d.get("runs", [])
|
|
213
|
+
rows = [{k: r.get(k) for k in ("id", "kind", "dish", "name", "status", "ms", "started_at")} for r in runs]
|
|
214
|
+
return d, _table(rows, ["id", "kind", "dish", "name", "status", "ms", "started_at"])
|
|
215
|
+
if a.cmd == "run-log":
|
|
216
|
+
d = c.call("GET", f"/runs/{a.id}"); return d, json.dumps(d, indent=1, default=str)
|
|
217
|
+
if a.cmd == "export":
|
|
218
|
+
d = c.call("POST", "/exports", json={"dish": a.dish, "name": a.name, "sql": a.sql, "format": a.format})
|
|
219
|
+
return d, f"live link: {c.url}{d.get('path') or '/x/' + str(d.get('token'))}"
|
|
220
|
+
if a.cmd == "connectors":
|
|
221
|
+
d = c.call("GET", "/connectors")
|
|
222
|
+
rows = [{"name": x["name"], "needs": ", ".join(x["needs"]), "dish": x["dish"], "every": x["every"], "how": x["how"][:70]} for x in d["connectors"]]
|
|
223
|
+
return d, _table(rows, ["name", "needs", "dish", "every", "how"])
|
|
224
|
+
if a.cmd == "how":
|
|
225
|
+
d = c.call("GET", f"/connectors/{a.name}/guide"); return {"guide": d}, d
|
|
226
|
+
if a.cmd == "connect":
|
|
227
|
+
secrets = {}
|
|
228
|
+
for kv in a.secrets:
|
|
229
|
+
if "=" not in kv:
|
|
230
|
+
raise Fail(f"expected key=value, got {kv}")
|
|
231
|
+
k, v = kv.split("=", 1); secrets[k] = v
|
|
232
|
+
body = {"secrets": secrets, "dish": a.dish, "resources": a.only or None, "run_now": not a.no_run}
|
|
233
|
+
d = c.call("POST", f"/connectors/{a.name}", json=body, headers={"x-overscore-trigger": "cli"}, timeout=3600)
|
|
234
|
+
fr = d.get("first_run") or {}
|
|
235
|
+
text = f"connected {a.name} into {d['dish']}, {d['schedule']}"
|
|
236
|
+
if fr:
|
|
237
|
+
text += "\nlanded: " + ", ".join(f"{k} {v}" for k, v in fr.get("landed", {}).items())
|
|
238
|
+
text += "\nviews: " + ", ".join(d.get("views") or []) if d.get("views") else ""
|
|
239
|
+
return d, text
|
|
240
|
+
if a.cmd == "link":
|
|
241
|
+
body = {"dsn": a.dsn, "mode": "copy" if a.copy else "live", "tables": a.tables or None, "source_schema": a.schema, "every": a.every}
|
|
242
|
+
d = c.call("PUT", f"/links/{a.dish}", json=body, timeout=3600)
|
|
243
|
+
text = f"linked {a.dish} ({d['mode']}, schema {d['source_schema']}, tables {d['tables']})"
|
|
244
|
+
if d.get("first_copy"):
|
|
245
|
+
text += "\ncopied: " + ", ".join(f"{k} {v}" for k, v in d["first_copy"]["copied"].items())
|
|
246
|
+
return d, text
|
|
247
|
+
if a.cmd == "links":
|
|
248
|
+
d = c.call("GET", "/links")
|
|
249
|
+
rows = [{k: x.get(k) for k in ("dish", "mode", "host", "source_schema", "tables", "last_copied_at")} for x in d["links"]]
|
|
250
|
+
return d, _table(rows, ["dish", "mode", "host", "source_schema", "tables", "last_copied_at"])
|
|
251
|
+
if a.cmd == "context":
|
|
252
|
+
d = c.call("GET", "/context", params={"q": a.q})
|
|
253
|
+
rows = [{"kind": h["kind"], "ref": h["ref"], "trust": h.get("trust") or "", "sentence": (h.get("sentence") or "")[:90], "look_in": h.get("table") or ""} for h in d["hits"]]
|
|
254
|
+
return d, _table(rows, ["kind", "ref", "trust", "sentence", "look_in"])
|
|
255
|
+
if a.cmd == "propose":
|
|
256
|
+
d = c.call("POST", "/context/cards", json={"text": a.text, "source": a.source})
|
|
257
|
+
return d, f"card {d['id']}: {d.get('bin') or 'unsorted'} -> {d.get('target') or 'note'} {d.get('claim') or ''}\nstatus {d['status']}" + (", sorted by the model" if d.get("sorted") else ", waiting for a person to sort")
|
|
258
|
+
if a.cmd == "cards":
|
|
259
|
+
d = c.call("GET", "/context/cards", params={"status": a.status})
|
|
260
|
+
rows = [{"id": x["id"], "bin": x.get("bin") or "", "target": x.get("target") or "", "claim": (x.get("claim") or x["text"])[:80], "source": x["source"]} for x in d["cards"]]
|
|
261
|
+
return d, _table(rows, ["id", "bin", "target", "claim", "source"])
|
|
262
|
+
if a.cmd in ("approve", "ignore"):
|
|
263
|
+
body = {}
|
|
264
|
+
if a.cmd == "approve":
|
|
265
|
+
body = {"target": a.target, "key": a.card_key, "values": dict(kv.split("=", 1) for kv in (a.set or [])) or None}
|
|
266
|
+
d = c.call("POST", f"/context/cards/{a.id}/{a.cmd}", json=body)
|
|
267
|
+
return d, f"{d['status']}" + (f": wrote {d['written']}" if d.get("written") else "")
|
|
268
|
+
if a.cmd == "issues":
|
|
269
|
+
if a.sweep:
|
|
270
|
+
c.call("POST", "/watch/sweep", timeout=300)
|
|
271
|
+
d = c.call("GET", "/watch")
|
|
272
|
+
if not d["issues"]:
|
|
273
|
+
return d, "nothing wrong that I can see"
|
|
274
|
+
text = "\n\n".join(f"[{i['id']}] {i['what']}\n{i['text']}\n-> ovs fix {i['id']} to {i['fix']}" for i in d["issues"])
|
|
275
|
+
return d, text
|
|
276
|
+
if a.cmd == "fix":
|
|
277
|
+
d = c.call("POST", f"/context/cards/{a.id}/approve", json={}, timeout=3600)
|
|
278
|
+
return d, f"{d['status']}: {json.dumps(d.get('written'))}"
|
|
279
|
+
if a.cmd == "research":
|
|
280
|
+
d = c.call("POST", "/research", json={"site": a.site}, timeout=600)
|
|
281
|
+
f = d.get("found") or {}
|
|
282
|
+
lines = [f"read {len(d['pages'])} pages of {d['site']} in {d['ms']/1000:.0f}s",
|
|
283
|
+
"systems seen: " + (", ".join(s["name"] for s in d["systems"]) or "none"),
|
|
284
|
+
f"sells: {f.get('sells')}", f"to: {f.get('to_whom')}", f"money: {f.get('makes_money')}",
|
|
285
|
+
"people: " + (", ".join(f"{p['name']} ({p.get('title')})" for p in f.get("people", [])) or "none named"),
|
|
286
|
+
"products: " + (", ".join(p["name"] for p in f.get("products", [])) or "none"),
|
|
287
|
+
f"wrote {len(d['written'])} rows (heard/seen); web search " + ("done" if d["searched_web"] else "skipped (no TAVILY_KEY)")]
|
|
288
|
+
return d, "\n".join(lines)
|
|
289
|
+
if a.cmd == "fields":
|
|
290
|
+
d = c.call("GET", f"/work/{a.dish}/{a.table}/fields")
|
|
291
|
+
rows = [{"field": f["field"], "type": f["type"] + (" of " + ", ".join(map(str, f["allowed"])) if f.get("allowed") else ""), "meaning": f["description"]} for f in d["fields"]]
|
|
292
|
+
return d, (f"row id: {d.get('key_column') or '(none; table-level marks only)'}\n" + (_table(rows, ["field", "type", "meaning"]) if rows else "(no fields yet; ovs field <dish> <table> <name> <type> \"meaning\")"))
|
|
293
|
+
if a.cmd == "field":
|
|
294
|
+
d = c.call("PUT", f"/work/{a.dish}/{a.table}/fields", json={"field": a.name, "type": a.type, "description": a.description, "allowed": a.allowed})
|
|
295
|
+
return d, f"{'exists' if d.get('existing') else 'defined'}: {d['field']} ({d['type']}): {d['description']}"
|
|
296
|
+
if a.cmd == "mark":
|
|
297
|
+
vals = {}
|
|
298
|
+
for kv in a.values:
|
|
299
|
+
k, _, v = kv.partition("="); vals[k] = v
|
|
300
|
+
keys = None if a.keys == "-" else [k for k in a.keys.split(",") if k]
|
|
301
|
+
d = c.call("POST", f"/work/{a.dish}/{a.table}/mark", json={"keys": keys, "values": vals})
|
|
302
|
+
return d, f"marked {d['rows']} row(s) with {', '.join(d['fields'])}"
|
|
303
|
+
if a.cmd == "marks":
|
|
304
|
+
d = c.call("GET", f"/work/{a.dish}/{a.table}/history", params={"key": a.row_key} if a.row_key else None)
|
|
305
|
+
return d, _table([{"at": h["at"], "field": h["field"], "value": h["value"], "by": h.get("person") or h["who"]} for h in d["history"]], ["at", "field", "value", "by"])
|
|
306
|
+
if a.cmd == "plan":
|
|
307
|
+
d = c.call("GET", "/plan")
|
|
308
|
+
gb = lambda v: f"{(v or 0) / 2**30:.2f} GB"
|
|
309
|
+
t = d["today"]
|
|
310
|
+
lines = [f"plan: {d['plan']}",
|
|
311
|
+
f"storage: {gb(d['storage']['used'])} of {gb(d['storage']['cap'])}",
|
|
312
|
+
f"today: uploads {gb(t['upload_bytes']['used'])}/{gb(t['upload_bytes']['cap'])}, egress {gb(t['egress_bytes']['used'])}/{gb(t['egress_bytes']['cap'])}, "
|
|
313
|
+
f"pipeline rows {int(t['pipeline_rows']['used']):,}/{int(t['pipeline_rows']['cap']):,}, compute {t['run_ms']['used']/60000:.0f}/{t['run_ms']['cap']/60000:.0f} min",
|
|
314
|
+
f"AI today: ${d['ai']['customer_usd']:.2f} of ${d['ai_usd_day']:.2f} (this key); schedules at most every {d['schedule_min_minutes']} min"]
|
|
315
|
+
return d, "\n".join(lines)
|
|
316
|
+
if a.cmd == "claim":
|
|
317
|
+
d = c.call("POST", f"/try/{a.code}/claim", timeout=300)
|
|
318
|
+
return d, f"claimed: {len(d['written'])} facts landed as confirmed"
|
|
319
|
+
if a.cmd == "map":
|
|
320
|
+
d = c.call("GET", "/interview/coverage")
|
|
321
|
+
return d, _table([{k: x[k] for k in ("area", "state", "known", "confirmed", "weight")} for x in d["areas"]], ["area", "state", "known", "confirmed", "weight"])
|
|
322
|
+
if a.cmd == "interview":
|
|
323
|
+
return _interview(a, c)
|
|
324
|
+
if a.cmd == "facts":
|
|
325
|
+
d = c.call("GET", f"/context/facts/{a.table}")
|
|
326
|
+
rows = d["rows"]
|
|
327
|
+
cols = [k for k in (rows[0].keys() if rows else []) if k not in ("detail", "superseded_at")]
|
|
328
|
+
return d, _table(rows, cols) if rows else "(no rows)"
|
|
329
|
+
if a.cmd == "hook":
|
|
330
|
+
d = c.call("POST", "/hooks", json={"dish": a.dish, "table": a.table, "provider": a.provider, "secret": a.secret})
|
|
331
|
+
return d, f"hook address: {d['url']}\nlands in {d['dish']}.{d['table']}" + (f", signed by {d['provider']}" if d.get("provider") else ", unsigned")
|
|
332
|
+
if a.cmd == "hooks":
|
|
333
|
+
d = c.call("GET", "/hooks")
|
|
334
|
+
rows = [{"token": x["token"], "lands_in": f"{x['dish']}.{x['table']}", "provider": x.get("provider") or "", "received": x["received"],
|
|
335
|
+
"rejected": x["rejected"], "url": x["url"]} for x in d["hooks"]]
|
|
336
|
+
return d, _table(rows, ["token", "lands_in", "provider", "received", "rejected", "url"])
|
|
337
|
+
if a.cmd == "hook-log":
|
|
338
|
+
d = c.call("GET", f"/hooks/{a.token}", params={"n": a.n})
|
|
339
|
+
rows = [{"when": x["received_at"], "ok": "landed", "ip": x.get("ip"), "detail": x["payload"][:60]} for x in d["deliveries"]]
|
|
340
|
+
rows += [{"when": x["at"], "ok": "REJECTED", "ip": x.get("ip"), "detail": x["reason"]} for x in d["rejections"]]
|
|
341
|
+
rows.sort(key=lambda r: str(r["when"]), reverse=True)
|
|
342
|
+
head = f"{d['dish']}.{d['table']} received={d['received']} rejected={d['rejected']}\n"
|
|
343
|
+
return d, head + _table(rows, ["when", "ok", "ip", "detail"])
|
|
344
|
+
if a.cmd == "unhook":
|
|
345
|
+
d = c.call("DELETE", f"/hooks/{a.token}"); return d, f"removed ({d['deleted']})"
|
|
346
|
+
if a.cmd == "copy":
|
|
347
|
+
d = c.call("POST", f"/links/{a.dish}/copy", headers={"x-overscore-trigger": "cli"}, timeout=3600)
|
|
348
|
+
return d, "copied: " + ", ".join(f"{k} {v}" for k, v in d["copied"].items())
|
|
349
|
+
if a.cmd == "unlink":
|
|
350
|
+
d = c.call("DELETE", f"/links/{a.dish}"); return d, f"unlinked {a.dish}"
|
|
351
|
+
raise Fail(f"unknown command {a.cmd}")
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _interview(a, c: Client):
|
|
355
|
+
"""A back-and-forth in the terminal. Type 'done' to wrap up, 'skip' to move on, 'quit' to leave it open."""
|
|
356
|
+
if a.resume:
|
|
357
|
+
conv = a.resume
|
|
358
|
+
t = c.call("GET", f"/interview/{conv}")
|
|
359
|
+
q = next((m["text"] for m in reversed(t["messages"]) if m["role"] == "assistant"), "Where were we?")
|
|
360
|
+
else:
|
|
361
|
+
d = c.call("POST", "/interview", json={"area": a.area, "person": a.person})
|
|
362
|
+
conv, q = d["conversation"], d["question"]
|
|
363
|
+
print(f"(interview {conv}, area {d['area']}; say 'done' when it looks right, 'quit' to leave it open)\n")
|
|
364
|
+
while True:
|
|
365
|
+
print(f"> {q}\n")
|
|
366
|
+
try:
|
|
367
|
+
text = input("you: ").strip()
|
|
368
|
+
except (EOFError, KeyboardInterrupt):
|
|
369
|
+
print(f"\nleft open; resume with: ovs interview --resume {conv}"); return {"conversation": conv}, ""
|
|
370
|
+
if not text:
|
|
371
|
+
continue
|
|
372
|
+
if text.lower() == "quit":
|
|
373
|
+
print(f"left open; resume with: ovs interview --resume {conv}"); return {"conversation": conv}, ""
|
|
374
|
+
if text.lower() == "done":
|
|
375
|
+
w = c.call("POST", f"/interview/{conv}/wrap", json={}, timeout=300)
|
|
376
|
+
lines = [f"saved {len(w['written'])} facts" + (f", set aside {w['ignored']}" if w["ignored"] else "")]
|
|
377
|
+
if w.get("first_read"):
|
|
378
|
+
lines += ["", "First read:", w["first_read"]]
|
|
379
|
+
lines += ["", "The map now: " + ", ".join(f"{x['area']} {x['state']}" for x in w["coverage"])]
|
|
380
|
+
return w, "\n".join(lines)
|
|
381
|
+
r = c.call("POST", f"/interview/{conv}", json={"text": text}, timeout=300)
|
|
382
|
+
for h in r["heard"]:
|
|
383
|
+
where = f" -> {h['target']}" if h.get("target") else ""
|
|
384
|
+
flag = " [disagrees with what is written]" if h.get("conflict") else ""
|
|
385
|
+
print(f" heard: {h['claim']}{where}{flag}")
|
|
386
|
+
if r["heard"]:
|
|
387
|
+
print()
|
|
388
|
+
q = r["question"]
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
# --- the laptop side: a project folder, files, Python models, destinations, syncs (CLAUDE_CODE_PLAN.md) --------------
|
|
392
|
+
#
|
|
393
|
+
# Every command here calls a door the server already has; nothing is decided here that the server would decide.
|
|
394
|
+
# The project folder:
|
|
395
|
+
# overscore.toml url, dish (never the key)
|
|
396
|
+
# models/<name>/model.py a Python model; model.toml beside it: reads, uploads, requirements, description, refresh
|
|
397
|
+
# files/<name>.<ext> local copies of uploads
|
|
398
|
+
# syncs.toml what goes where, written by ovs sync add
|
|
399
|
+
# .overscore/ pulled samples (data/), local outputs (out/), work dirs, environments; never committed
|
|
400
|
+
|
|
401
|
+
PROJECT_FILE = "overscore.toml"
|
|
402
|
+
LOCAL = ".overscore"
|
|
403
|
+
DEFAULT_URL = "https://app.overscore.tech"
|
|
404
|
+
PULL_MAX = 10_000 # what the query door answers at most
|
|
405
|
+
SYNC_EVERY = ("5m", "15m", "60m", "6h", "24h")
|
|
406
|
+
SHIPPED_SKILLS = ("overscore", "overscore-model", "overscore-sync")
|
|
407
|
+
AGENTS = ("claude", "codex", "cursor", "gemini", "agents", "all")
|
|
408
|
+
BLOCK_OPEN = "<!-- overscore:start" # matched on this much, so a reworded start line is still found
|
|
409
|
+
BLOCK_START = BLOCK_OPEN + " (written by ovs init, which rewrites this block; edit outside it) -->"
|
|
410
|
+
BLOCK_END = "<!-- overscore:end -->"
|
|
411
|
+
REWRITTEN = "<!-- Written by ovs init, which rewrites this file; put your own steps in another file. -->"
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _find_project(start: str | os.PathLike | None = None) -> pathlib.Path | None:
|
|
415
|
+
"""The nearest folder at or above `start` with an overscore.toml, like git finds .git."""
|
|
416
|
+
d = pathlib.Path(start or os.getcwd()).resolve()
|
|
417
|
+
for p in (d, *d.parents):
|
|
418
|
+
if (p / PROJECT_FILE).is_file():
|
|
419
|
+
return p
|
|
420
|
+
return None
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def _read_toml(path: pathlib.Path) -> dict:
|
|
424
|
+
import tomllib
|
|
425
|
+
try:
|
|
426
|
+
with open(path, "rb") as f:
|
|
427
|
+
return tomllib.load(f)
|
|
428
|
+
except tomllib.TOMLDecodeError as e:
|
|
429
|
+
raise Fail(f"{path.name} does not parse: {e}")
|
|
430
|
+
except UnicodeDecodeError:
|
|
431
|
+
raise Fail(f"{path.name} is not UTF-8 text; save it as UTF-8") from None
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def _project(a, required: bool = True) -> pathlib.Path | None:
|
|
435
|
+
root = getattr(a, "project", None) or _find_project()
|
|
436
|
+
if root is None and required:
|
|
437
|
+
raise Fail("this folder is not an Overscore project: run ovs init --dish <dish> here first")
|
|
438
|
+
return root
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def _config(root: pathlib.Path | None) -> dict:
|
|
442
|
+
return _read_toml(root / PROJECT_FILE) if root and (root / PROJECT_FILE).is_file() else {}
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
NAME_RE = re.compile(r"^[a-z][a-z0-9_]{0,62}$") # the server's rule for dishes, tables, models, uploads and syncs
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _name(n: str, what: str) -> str:
|
|
449
|
+
"""Names go into paths on this machine and into door URLs: hold them to the server's rule first."""
|
|
450
|
+
if not NAME_RE.match(n or ""):
|
|
451
|
+
raise Fail(f"a {what} name is lowercase letters, digits and underscores, starting with a letter: {n!r}")
|
|
452
|
+
return n
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def _dish(a, *more: str | None) -> str:
|
|
456
|
+
for d in (getattr(a, "dish", None), *more, _config(_project(a, False)).get("dish")):
|
|
457
|
+
if d:
|
|
458
|
+
return _name(d, "dish")
|
|
459
|
+
raise Fail("which dish? pass --dish, or write it into overscore.toml with ovs init --dish <dish>")
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _split(ref: str, dish: str | None) -> tuple[str, str]:
|
|
463
|
+
d, _, t = ref.strip().rpartition(".")
|
|
464
|
+
if not (d or dish):
|
|
465
|
+
raise Fail(f"say which dish {ref} is in: dish.{ref}")
|
|
466
|
+
return _name(d or dish, "dish"), _name(t, "table")
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _sq(s: str | os.PathLike) -> str:
|
|
470
|
+
return str(s).replace("'", "''")
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _plain(v):
|
|
474
|
+
"""A value from DuckDB as something JSON can carry, the way the server's query door does it (NaN and infinity as text)."""
|
|
475
|
+
if isinstance(v, float) and (v != v or v in (float("inf"), float("-inf"))):
|
|
476
|
+
return str(v)
|
|
477
|
+
if isinstance(v, (bytes, bytearray)):
|
|
478
|
+
return v.hex()
|
|
479
|
+
if hasattr(v, "isoformat"):
|
|
480
|
+
return v.isoformat()
|
|
481
|
+
if isinstance(v, (list, tuple)):
|
|
482
|
+
return [_plain(x) for x in v]
|
|
483
|
+
if isinstance(v, dict):
|
|
484
|
+
return {k: _plain(x) for k, x in v.items()}
|
|
485
|
+
try:
|
|
486
|
+
json.dumps(v)
|
|
487
|
+
return v
|
|
488
|
+
except TypeError:
|
|
489
|
+
return str(v)
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def _when(ts) -> str:
|
|
493
|
+
try:
|
|
494
|
+
return time.strftime("%Y-%m-%d %H:%M", time.localtime(float(ts)))
|
|
495
|
+
except (TypeError, ValueError):
|
|
496
|
+
return str(ts or "")
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def _sha256(path: pathlib.Path) -> str:
|
|
500
|
+
h = hashlib.sha256()
|
|
501
|
+
with open(path, "rb") as f:
|
|
502
|
+
for chunk in iter(lambda: f.read(1 << 20), b""):
|
|
503
|
+
h.update(chunk)
|
|
504
|
+
return h.hexdigest()
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
# --- files (uploads) ----------------------------------------------------------------------------------------------
|
|
508
|
+
|
|
509
|
+
def _uploads_list(c: Client) -> list[dict]:
|
|
510
|
+
return c.call("GET", "/uploads")["uploads"]
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _uploads(a, c: Client):
|
|
514
|
+
d = {"uploads": _uploads_list(c)}
|
|
515
|
+
rows = []
|
|
516
|
+
for u in d["uploads"]:
|
|
517
|
+
live = next((v for v in u["versions"] if v["version"] == u["live"]), None) or {}
|
|
518
|
+
rows.append({"name": u["name"], "live": f"v{u['live']}" if u["live"] else "none", "versions": len(u["versions"]),
|
|
519
|
+
"file": live.get("filename", ""), "bytes": live.get("size", ""), "by": live.get("uploaded_by", ""),
|
|
520
|
+
"when": _when(live.get("uploaded_at")), "note": live.get("note", "")})
|
|
521
|
+
return d, _table(rows, ["name", "live", "versions", "file", "bytes", "by", "when", "note"]) if rows else "(no files yet: ovs upload put <name> <file>)"
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def _upload(a, c: Client):
|
|
525
|
+
if a.sub == "list":
|
|
526
|
+
return _uploads(a, c)
|
|
527
|
+
_name(a.name, "file")
|
|
528
|
+
if a.sub == "put":
|
|
529
|
+
path = pathlib.Path(a.file)
|
|
530
|
+
if not path.is_file():
|
|
531
|
+
raise Fail(f"no file at {a.file}")
|
|
532
|
+
with open(path, "rb") as f:
|
|
533
|
+
d = c.call("POST", f"/uploads/{a.name}", params={"filename": path.name, "note": a.note or ""}, data=f,
|
|
534
|
+
headers={"Content-Type": "application/octet-stream"}, timeout=900)
|
|
535
|
+
return d, f"uploaded {a.name} v{d['version']} ({d['filename']}, {d['size']:,} bytes), live now"
|
|
536
|
+
if a.sub == "live":
|
|
537
|
+
d = c.call("POST", f"/uploads/{a.name}/live", json={"version": a.version})
|
|
538
|
+
return d, f"{a.name}: v{d['live']} is live; models that read it use it from their next run"
|
|
539
|
+
if a.sub == "rm":
|
|
540
|
+
d = c.call("DELETE", f"/uploads/{a.name}")
|
|
541
|
+
return d, f"removed {a.name} and every version of it" if d["removed"] else f"there was no file called {a.name}"
|
|
542
|
+
raise Fail(f"unknown upload command {a.sub}")
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
def _local_file(root: pathlib.Path, name: str) -> pathlib.Path | None:
|
|
546
|
+
"""files/<name>.<ext> (or files/<name>): the local copy of an upload."""
|
|
547
|
+
folder = root / "files"
|
|
548
|
+
if not folder.is_dir():
|
|
549
|
+
return None
|
|
550
|
+
hits = sorted(p for p in folder.iterdir() if p.is_file() and (p.name == name or p.name.startswith(name + ".")))
|
|
551
|
+
if len(hits) > 1:
|
|
552
|
+
raise Fail(f"more than one local file for {name} in files/: {', '.join(p.name for p in hits)}; keep one")
|
|
553
|
+
return hits[0] if hits else None
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
# --- python models --------------------------------------------------------------------------------------------------
|
|
557
|
+
|
|
558
|
+
def _model(a, name: str) -> tuple[pathlib.Path, pathlib.Path, dict, str]:
|
|
559
|
+
"""(project root, the model's folder, model.toml, model.py) for models/<name>."""
|
|
560
|
+
root = _project(a)
|
|
561
|
+
folder = root / "models" / _name(name, "model")
|
|
562
|
+
if not (folder / "model.py").is_file():
|
|
563
|
+
raise Fail(f"no models/{name}/model.py in {root}")
|
|
564
|
+
cfg = _read_toml(folder / "model.toml") if (folder / "model.toml").is_file() else {}
|
|
565
|
+
for k in ("reads", "uploads"):
|
|
566
|
+
v = cfg.get(k, [])
|
|
567
|
+
if isinstance(v, str):
|
|
568
|
+
v = [v]
|
|
569
|
+
if not isinstance(v, list) or not all(isinstance(x, str) for x in v):
|
|
570
|
+
raise Fail(f"models/{name}/model.toml: {k} is a list of names, like {k} = [\"a\", \"b\"]")
|
|
571
|
+
cfg[k] = [x.strip() for x in v if x.strip()]
|
|
572
|
+
req = cfg.get("requirements", "")
|
|
573
|
+
cfg["requirements"] = "\n".join(req) if isinstance(req, list) else str(req or "")
|
|
574
|
+
if not cfg["reads"]:
|
|
575
|
+
raise Fail(f"models/{name}/model.toml: say which tables the model reads, like reads = [\"dish.table\"]")
|
|
576
|
+
return root, folder, cfg, (folder / "model.py").read_text(encoding="utf-8")
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def _check_script(script: str) -> None:
|
|
580
|
+
"""What the server checks when the script is saved, checked here first."""
|
|
581
|
+
if "overscore_model" not in script:
|
|
582
|
+
raise Fail("model.py should import from overscore_model (inputs, read, rows, file, output, log)")
|
|
583
|
+
if "output(" not in script:
|
|
584
|
+
raise Fail("model.py never calls output(rows); nothing would become the table")
|
|
585
|
+
try:
|
|
586
|
+
compile(script, "model.py", "exec")
|
|
587
|
+
except SyntaxError as e:
|
|
588
|
+
raise Fail(f"model.py does not parse: line {e.lineno}: {e.msg}") from None
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def _last_run(c: Client, dish: str, name: str, run_id: int | None = None) -> dict | None:
|
|
592
|
+
if run_id is None:
|
|
593
|
+
rs = c.call("GET", "/runs", params={"kind": "model", "dish": dish, "limit": 500})["runs"] # the door's most; it has no name filter
|
|
594
|
+
mine = next((r for r in rs if r["name"] == name), None)
|
|
595
|
+
if not mine:
|
|
596
|
+
return None
|
|
597
|
+
run_id = mine["id"]
|
|
598
|
+
return c.call("GET", f"/runs/{run_id}")
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def _run_there(c: Client, dish: str, name: str) -> tuple[dict, str]:
|
|
602
|
+
"""Run the model in Overscore and read its log: the receipt."""
|
|
603
|
+
d = c.call("POST", f"/models/{dish}/{name}/run", timeout=3600)
|
|
604
|
+
res = next((r for r in reversed(d.get("results") or []) if (r.get("model") or r.get("name")) == name), None) or {}
|
|
605
|
+
run = _last_run(c, dish, name, res.get("run_id")) if res.get("run_id") else None
|
|
606
|
+
log = (run or {}).get("log") or ""
|
|
607
|
+
out = {"model": f"{dish}.{name}", "ok": bool(res.get("ok")), "rows": res.get("rows"), "ms": res.get("ms"),
|
|
608
|
+
"run_id": res.get("run_id"), "error": res.get("error"), "log": log, "results": d.get("results")}
|
|
609
|
+
head = (f"ran {dish}.{name} in Overscore: {'ok' if out['ok'] else 'FAILED'}"
|
|
610
|
+
+ (f", {out['rows']:,} rows" if out["rows"] is not None else "") + (f", {out['ms'] / 1000:.1f}s" if out["ms"] else "")
|
|
611
|
+
+ (f" (run {out['run_id']})" if out["run_id"] else ""))
|
|
612
|
+
text = head + ("\n" + log.rstrip() if log else "") + (f"\nerror: {out['error']}" if out["error"] else "")
|
|
613
|
+
if not out["ok"]:
|
|
614
|
+
raise Fail(out["error"] or f"{dish}.{name} did not build", data={k: v for k, v in out.items() if k != "results"})
|
|
615
|
+
return out, text
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
def _model_cmd(a, c: Client):
|
|
619
|
+
if a.sub == "run" and a.local:
|
|
620
|
+
return _run_local(a)
|
|
621
|
+
if a.sub == "run":
|
|
622
|
+
dish = _dish(a, _model_dish_hint(a, a.name))
|
|
623
|
+
return _run_there(c, dish, a.name)
|
|
624
|
+
if a.sub == "push":
|
|
625
|
+
return _push(a, c)
|
|
626
|
+
if a.sub == "log":
|
|
627
|
+
dish = _dish(a, _model_dish_hint(a, a.name))
|
|
628
|
+
run = _last_run(c, dish, a.name)
|
|
629
|
+
if not run:
|
|
630
|
+
raise Fail(f"no run of {dish}.{a.name} among the last 500 model runs in {dish}; ovs runs lists them")
|
|
631
|
+
text = (f"run {run['id']}: {run['status']}, {run.get('trigger') or ''}, {_when(run.get('started_at'))}"
|
|
632
|
+
+ (f", {run['rows']:,} rows" if run.get("rows") is not None else "") + "\n" + (run.get("log") or "(no log)").rstrip())
|
|
633
|
+
if run.get("error"):
|
|
634
|
+
text += f"\nerror: {run['error']}"
|
|
635
|
+
return run, text
|
|
636
|
+
raise Fail(f"unknown model command {a.sub}")
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def _model_dish_hint(a, name: str) -> str | None:
|
|
640
|
+
"""model.toml may name the dish, when the folder exists."""
|
|
641
|
+
root = _project(a, False)
|
|
642
|
+
p = root / "models" / _name(name, "model") / "model.toml" if root else None
|
|
643
|
+
return _read_toml(p).get("dish") if p and p.is_file() else None
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
def _push(a, c: Client):
|
|
647
|
+
root, folder, cfg, script = _model(a, a.name)
|
|
648
|
+
dish = _dish(a, cfg.get("dish"))
|
|
649
|
+
_check_script(script)
|
|
650
|
+
notes = []
|
|
651
|
+
if cfg["uploads"]:
|
|
652
|
+
have = {u["name"]: u for u in _uploads_list(c)}
|
|
653
|
+
for u in cfg["uploads"]:
|
|
654
|
+
local = _local_file(root, u)
|
|
655
|
+
there = have.get(u)
|
|
656
|
+
if not there or there["live"] is None:
|
|
657
|
+
hint = f"ovs upload put {u} {os.path.relpath(local)}" if local else f"ovs upload put {u} <file>"
|
|
658
|
+
raise Fail(f"the file {u} is not in Overscore yet: {hint}")
|
|
659
|
+
live = next(v for v in there["versions"] if v["version"] == there["live"])
|
|
660
|
+
if local and _sha256(local) != live["sha256"]:
|
|
661
|
+
notes.append(f"{local.relative_to(root)} is not the live {u} in Overscore (v{there['live']}, {live['filename']}); "
|
|
662
|
+
f"the run there uses v{there['live']}. To send yours: ovs upload put {u} {os.path.relpath(local)}")
|
|
663
|
+
body = {"script": script, "reads": cfg["reads"], "uploads": cfg["uploads"], "requirements": cfg["requirements"],
|
|
664
|
+
"description": cfg.get("description", ""), "refresh": cfg.get("refresh")}
|
|
665
|
+
saved = c.call("PUT", f"/models/{dish}/{a.name}/python", json=body)
|
|
666
|
+
out = {"saved": saved, "model": f"{dish}.{a.name}", "notes": notes}
|
|
667
|
+
text = (f"saved {dish}.{a.name}: reads {', '.join(saved.get('reads') or cfg['reads'])}"
|
|
668
|
+
+ (f"; files {', '.join(cfg['uploads'])}" if cfg["uploads"] else "") + f"; refresh {saved.get('refresh')}")
|
|
669
|
+
text += "".join(f"\nnote: {n}" for n in notes)
|
|
670
|
+
if a.no_run:
|
|
671
|
+
return out, text + f"\nnot run; ovs model run {a.name} runs it"
|
|
672
|
+
try:
|
|
673
|
+
ran, ran_text = _run_there(c, dish, a.name)
|
|
674
|
+
except Fail as e:
|
|
675
|
+
e.data = {**out, **e.data}
|
|
676
|
+
raise
|
|
677
|
+
return {**out, "run": ran}, text + "\n" + ran_text
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
# --- pull: a sample of a table as Parquet, for local runs -----------------------------------------------------------
|
|
681
|
+
|
|
682
|
+
def _data_path(root: pathlib.Path, dish: str, table: str) -> pathlib.Path:
|
|
683
|
+
return root / LOCAL / "data" / dish / f"{table}.parquet"
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def _ident(col: str) -> str:
|
|
687
|
+
return '"' + col.replace('"', '""') + '"'
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def _pull_plan(types: dict) -> tuple[list[str], dict, set]:
|
|
691
|
+
"""How each column crosses the query door and comes back typed.
|
|
692
|
+
|
|
693
|
+
The door hands JSON as text, bytes as hex, and cannot carry NaN or infinity at all (a 500); an interval comes as
|
|
694
|
+
Python's text, which DuckDB will not read back. So those are asked for as DuckDB's own text and cast back here,
|
|
695
|
+
and JSON nested in a list or struct is asked for as one JSON text and parsed back before reading.
|
|
696
|
+
Returns (the select's replace list, {column: (read as, pick expression)}, columns to parse from JSON text)."""
|
|
697
|
+
reps, how, nested = [], {}, set()
|
|
698
|
+
for col, t in types.items():
|
|
699
|
+
i, T = _ident(col), t.upper()
|
|
700
|
+
if T in ("DOUBLE", "FLOAT", "REAL") or T.startswith("INTERVAL"):
|
|
701
|
+
reps.append(f"{i}::VARCHAR as {i}")
|
|
702
|
+
how[col] = ("VARCHAR", f"{i}::{'INTERVAL' if T.startswith('INTERVAL') else T} as {i}") # never the server's text in SQL
|
|
703
|
+
elif T == "JSON":
|
|
704
|
+
how[col] = ("VARCHAR", f"{i}::JSON as {i}")
|
|
705
|
+
elif T == "BLOB":
|
|
706
|
+
how[col] = ("VARCHAR", f"unhex({i}) as {i}")
|
|
707
|
+
elif "JSON" in T or any(x in T for x in ("DOUBLE", "FLOAT", "REAL")):
|
|
708
|
+
reps.append(f"to_json({i}) as {i}")
|
|
709
|
+
nested.add(col)
|
|
710
|
+
how[col] = (t, i)
|
|
711
|
+
else:
|
|
712
|
+
how[col] = (t, i)
|
|
713
|
+
return reps, how, nested
|
|
714
|
+
|
|
715
|
+
|
|
716
|
+
def _pull(a, c: Client):
|
|
717
|
+
import duckdb
|
|
718
|
+
root = _project(a)
|
|
719
|
+
n = max(1, min(a.rows or PULL_MAX, PULL_MAX))
|
|
720
|
+
out, lines = {"pulled": []}, []
|
|
721
|
+
for ref in a.tables:
|
|
722
|
+
dish, table = _split(ref, _dish(a) if "." not in ref else None)
|
|
723
|
+
types = {x["column"]: x["type"] for x in c.call("GET", f"/tables/{dish}/{table}")["columns"]}
|
|
724
|
+
where = f" where {a.where}" if a.where else ""
|
|
725
|
+
reps, how, nested = _pull_plan(types)
|
|
726
|
+
total = c.call("POST", "/query", json={"sql": f"select count(*) from {dish}.{table}{where}", "max_rows": 1})["rows"][0][0]
|
|
727
|
+
star = "select * replace (" + ", ".join(reps) + ")" if reps else "select *"
|
|
728
|
+
q = c.call("POST", "/query", json={"sql": f"{star} from {dish}.{table}{where} limit {n}", "max_rows": n}, timeout=900)
|
|
729
|
+
cols = q["columns"]
|
|
730
|
+
dest = _data_path(root, dish, table)
|
|
731
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
732
|
+
raw = dest.with_name(f".{table}.rows.jsonl")
|
|
733
|
+
with open(raw, "w", encoding="utf-8") as f:
|
|
734
|
+
for r in q["rows"]:
|
|
735
|
+
row = dict(zip(cols, r))
|
|
736
|
+
for col in nested:
|
|
737
|
+
if isinstance(row.get(col), str):
|
|
738
|
+
row[col] = json.loads(row[col])
|
|
739
|
+
f.write(json.dumps(row) + "\n")
|
|
740
|
+
spec = lambda cs: ", ".join(f"'{_sq(col)}': '{_sq(how.get(col, ('VARCHAR', ''))[0])}'" for col in cs)
|
|
741
|
+
src = lambda cs: f"read_json('{_sq(raw)}', format='newline_delimited', columns={{{spec(cs)}}})"
|
|
742
|
+
pick = lambda col: how.get(col, ("VARCHAR", _ident(col)))[1] or _ident(col)
|
|
743
|
+
guessed: list[str] = []
|
|
744
|
+
con = duckdb.connect()
|
|
745
|
+
try:
|
|
746
|
+
con.execute("set TimeZone='UTC'") # Overscore's machines run in UTC; moments are written the way they write them
|
|
747
|
+
try:
|
|
748
|
+
con.execute(f"create table t as select {', '.join(pick(col) for col in cols)} from {src(cols)}")
|
|
749
|
+
except duckdb.Error:
|
|
750
|
+
# one column that will not come back typed keeps its text; the others keep their types
|
|
751
|
+
for col in cols:
|
|
752
|
+
try:
|
|
753
|
+
con.execute(f"select {pick(col)} from {src([col])}").fetchall()
|
|
754
|
+
except duckdb.Error:
|
|
755
|
+
how[col] = ("VARCHAR", _ident(col))
|
|
756
|
+
guessed.append(col)
|
|
757
|
+
con.execute(f"create table t as select {', '.join(pick(col) for col in cols)} from {src(cols)}")
|
|
758
|
+
con.execute(f"copy t to '{_sq(dest)}' (format parquet)")
|
|
759
|
+
con.execute(f"copy t to '{_sq(dest.with_suffix('.jsonl'))}' (format json)") # what rows() reads, written as the server writes it
|
|
760
|
+
kept = [{"column": r[0], "type": r[1]} for r in con.execute("describe t").fetchall()]
|
|
761
|
+
finally:
|
|
762
|
+
con.close()
|
|
763
|
+
raw.unlink(missing_ok=True)
|
|
764
|
+
meta = {"table": f"{dish}.{table}", "rows": len(q["rows"]), "of": total, "where": a.where or None,
|
|
765
|
+
"pulled_at": time.time(), "columns": kept, "kept_as_text": guessed}
|
|
766
|
+
dest.with_suffix(".json").write_text(json.dumps(meta, indent=1), encoding="utf-8")
|
|
767
|
+
out["pulled"].append({**meta, "path": str(dest.relative_to(root))})
|
|
768
|
+
lines.append(f"pulled {dish}.{table}: {len(q['rows']):,} of {total:,} rows{' (' + a.where + ')' if a.where else ''} -> {dest.relative_to(root)}"
|
|
769
|
+
+ (f"\n kept as text, not as their types in Overscore: {', '.join(guessed)}" if guessed else ""))
|
|
770
|
+
return out, "\n".join(lines)
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
# --- local run: model.py on this machine, the way Overscore runs it ----------------------------------------------
|
|
774
|
+
|
|
775
|
+
def _helper_source() -> pathlib.Path:
|
|
776
|
+
here = pathlib.Path(__file__).with_name("model_helper.py")
|
|
777
|
+
if here.is_file():
|
|
778
|
+
return here
|
|
779
|
+
import overscore_model # the installed package carries it at the top level
|
|
780
|
+
return pathlib.Path(overscore_model.__file__)
|
|
781
|
+
|
|
782
|
+
|
|
783
|
+
SERVER_PYTHON = "3.12" # what Overscore runs scripts on
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
def _real_python(py: str) -> str:
|
|
787
|
+
"""The interpreter a shim (pyenv, asdf, mise) stands for: the script runs with its own HOME, where no shim works."""
|
|
788
|
+
try:
|
|
789
|
+
r = subprocess.run([py, "-c", "import sys; print(sys.executable)"], capture_output=True, text=True, timeout=60)
|
|
790
|
+
except (OSError, subprocess.TimeoutExpired) as e:
|
|
791
|
+
raise Fail(f"could not start {py}: {e}") from None
|
|
792
|
+
return r.stdout.strip() if r.returncode == 0 and r.stdout.strip() else py
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
def _local_python(a, requirements: str, root: pathlib.Path, log) -> str:
|
|
796
|
+
"""--python; else a Python 3.12 environment built with uv from the requirements, empty when there are none, so a
|
|
797
|
+
library the script forgets to list fails here and not only there; else python3 from PATH, said so in the log."""
|
|
798
|
+
if getattr(a, "python", None):
|
|
799
|
+
py = os.path.abspath(a.python) if (os.sep in a.python or a.python.startswith(".")) else shutil.which(a.python)
|
|
800
|
+
if not py or not os.path.isfile(py) or not os.access(py, os.X_OK):
|
|
801
|
+
raise Fail(f"no interpreter at {a.python}")
|
|
802
|
+
return _real_python(py)
|
|
803
|
+
req = "\n".join(l.strip() for l in requirements.splitlines() if l.strip() and not l.strip().startswith("#"))
|
|
804
|
+
uv = shutil.which("uv")
|
|
805
|
+
if not uv:
|
|
806
|
+
if req:
|
|
807
|
+
raise Fail("model.toml lists requirements; install uv (https://docs.astral.sh/uv/) so ovs can build an environment from them, or pass --python")
|
|
808
|
+
real = _real_python(shutil.which("python3") or sys.executable)
|
|
809
|
+
log(f"no uv here: ran with {real}, whose libraries Overscore does not have; install uv to run as Overscore does")
|
|
810
|
+
return real
|
|
811
|
+
d = root / LOCAL / "venvs" / hashlib.sha256(f"{SERVER_PYTHON}\n{req}".encode()).hexdigest()[:16]
|
|
812
|
+
py = d / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
|
|
813
|
+
d.parent.mkdir(parents=True, exist_ok=True)
|
|
814
|
+
with open(d.parent / f"{d.name}.lock", "w") as lock:
|
|
815
|
+
try:
|
|
816
|
+
import fcntl
|
|
817
|
+
fcntl.flock(lock, fcntl.LOCK_EX) # released when the file closes, or the process ends
|
|
818
|
+
except ImportError:
|
|
819
|
+
pass
|
|
820
|
+
if (d / ".ready").is_file() and py.exists():
|
|
821
|
+
return str(py)
|
|
822
|
+
shutil.rmtree(d, ignore_errors=True) # a half-built one from a run that was stopped
|
|
823
|
+
t0 = time.time()
|
|
824
|
+
rf = d.parent / f"{d.name}.requirements.txt"
|
|
825
|
+
try:
|
|
826
|
+
r = subprocess.run([uv, "venv", "--quiet", "--python", SERVER_PYTHON, str(d)], capture_output=True, text=True, timeout=600)
|
|
827
|
+
if r.returncode != 0: # no 3.12 to be had (offline, downloads off): the nearest Python, said so
|
|
828
|
+
shutil.rmtree(d, ignore_errors=True)
|
|
829
|
+
r = subprocess.run([uv, "venv", "--quiet", str(d)], capture_output=True, text=True, timeout=600)
|
|
830
|
+
if r.returncode == 0:
|
|
831
|
+
log(f"Overscore runs Python {SERVER_PYTHON}; none here, so this ran on {_real_python(str(py))}")
|
|
832
|
+
if r.returncode == 0 and req:
|
|
833
|
+
rf.write_text(req + "\n", encoding="utf-8")
|
|
834
|
+
r = subprocess.run([uv, "pip", "install", "--quiet", "--python", str(py), "-r", str(rf)], capture_output=True, text=True, timeout=1800)
|
|
835
|
+
if r.returncode != 0:
|
|
836
|
+
raise Fail(("requirements did not install: " if req else "could not make a Python environment: ")
|
|
837
|
+
+ ((r.stderr or r.stdout).strip().splitlines() or ["uv failed"])[-1][:300])
|
|
838
|
+
(d / ".ready").write_text(req + "\n", encoding="utf-8")
|
|
839
|
+
except subprocess.TimeoutExpired:
|
|
840
|
+
shutil.rmtree(d, ignore_errors=True)
|
|
841
|
+
raise Fail("installing the requirements took too long (30 minutes); try again, or pass --python") from None
|
|
842
|
+
except BaseException:
|
|
843
|
+
shutil.rmtree(d, ignore_errors=True)
|
|
844
|
+
raise
|
|
845
|
+
finally:
|
|
846
|
+
rf.unlink(missing_ok=True)
|
|
847
|
+
if req:
|
|
848
|
+
log(f"installed requirements in {time.time() - t0:.1f}s")
|
|
849
|
+
return str(py)
|
|
850
|
+
|
|
851
|
+
|
|
852
|
+
def _run_local(a):
|
|
853
|
+
import duckdb
|
|
854
|
+
root, folder, cfg, script = _model(a, a.name)
|
|
855
|
+
dish = _dish(a, cfg.get("dish"))
|
|
856
|
+
_check_script(script)
|
|
857
|
+
echo = getattr(a, "echo", False)
|
|
858
|
+
logs: list[str] = []
|
|
859
|
+
|
|
860
|
+
def log(line: str) -> None:
|
|
861
|
+
logs.append(line)
|
|
862
|
+
if echo:
|
|
863
|
+
print(" " + line, flush=True)
|
|
864
|
+
|
|
865
|
+
inputs, samples, missing = {}, {}, []
|
|
866
|
+
for rd in cfg["reads"]:
|
|
867
|
+
d, t = _split(rd, dish)
|
|
868
|
+
p = _data_path(root, d, t)
|
|
869
|
+
if not p.is_file():
|
|
870
|
+
missing.append(f"{d}.{t}")
|
|
871
|
+
continue
|
|
872
|
+
inputs[f"{d}.{t}"] = str(p)
|
|
873
|
+
meta = json.loads(p.with_suffix(".json").read_text(encoding="utf-8")) if p.with_suffix(".json").is_file() else {}
|
|
874
|
+
samples[f"{d}.{t}"] = {"rows": meta.get("rows"), "of": meta.get("of"), "pulled_at": meta.get("pulled_at")}
|
|
875
|
+
if missing:
|
|
876
|
+
raise Fail("pull the tables it reads first: " + " && ".join(f"ovs pull {m}" for m in missing))
|
|
877
|
+
files = {}
|
|
878
|
+
for u in cfg["uploads"]:
|
|
879
|
+
lf = _local_file(root, u)
|
|
880
|
+
if not lf:
|
|
881
|
+
raise Fail(f"model.toml needs the file {u}; put a copy at files/{u}.<ext> (Overscore never hands uploads back)")
|
|
882
|
+
files[u] = str(lf)
|
|
883
|
+
python = _local_python(a, cfg["requirements"], root, log)
|
|
884
|
+
(root / LOCAL / "work").mkdir(parents=True, exist_ok=True)
|
|
885
|
+
work = pathlib.Path(tempfile.mkdtemp(prefix=f"{a.name}-", dir=root / LOCAL / "work"))
|
|
886
|
+
try:
|
|
887
|
+
# only model.py and the helper sit beside each other, as in Overscore: a second file imported locally would not be there
|
|
888
|
+
shutil.copy(folder / "model.py", work / "model.py")
|
|
889
|
+
shutil.copy(_helper_source(), work / "overscore_model.py")
|
|
890
|
+
out_path = work / "output.jsonl"
|
|
891
|
+
# the environment Overscore gives a script, built from nothing: no key, no PYTHONPATH, no caches in a home folder,
|
|
892
|
+
# and every proxy pointed at a closed port, so a download that works on a laptop fails here as it would there
|
|
893
|
+
closed = "http://127.0.0.1:9"
|
|
894
|
+
env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "HOME": str(work), "LANG": "C.UTF-8", "TZ": "UTC",
|
|
895
|
+
"PYTHONUNBUFFERED": "1", "PYTHONDONTWRITEBYTECODE": "1", "OVERSCORE_DISH": dish, "OVERSCORE_MODEL": a.name,
|
|
896
|
+
"OVERSCORE_INPUTS": json.dumps(inputs), "OVERSCORE_FILES": json.dumps(files), "OVERSCORE_OUTPUT": str(out_path),
|
|
897
|
+
**{k: closed for k in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy")}, "NO_PROXY": "", "no_proxy": ""}
|
|
898
|
+
if os.name == "nt":
|
|
899
|
+
env["SYSTEMROOT"] = os.environ.get("SYSTEMROOT", "")
|
|
900
|
+
shown = f"models/{a.name}/model.py"
|
|
901
|
+
fix = lambda s: s.replace(f'File "{work / "model.py"}"', f'File "{shown}"').replace('File "model.py"', f'File "{shown}"')
|
|
902
|
+
for full, s in samples.items():
|
|
903
|
+
log(f"input {full}" + (f": {s['rows']:,} rows" if s.get("rows") is not None else "")
|
|
904
|
+
+ (f" (a sample of {s['of']:,})" if s.get("of") is not None and s.get("rows") is not None and s["of"] != s["rows"] else ""))
|
|
905
|
+
rows_out: list[int | None] = [None]
|
|
906
|
+
err: list[str] = []
|
|
907
|
+
t0 = time.time()
|
|
908
|
+
try:
|
|
909
|
+
proc = subprocess.Popen([python, "model.py"], cwd=work, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1)
|
|
910
|
+
except OSError as e:
|
|
911
|
+
raise Fail(f"could not start {python}: {e}") from None
|
|
912
|
+
|
|
913
|
+
def drain() -> None:
|
|
914
|
+
for line in proc.stderr:
|
|
915
|
+
err.append(fix(line.rstrip("\n")))
|
|
916
|
+
t_err = threading.Thread(target=drain, daemon=True)
|
|
917
|
+
t_err.start()
|
|
918
|
+
for line in proc.stdout:
|
|
919
|
+
line = line.rstrip("\n")
|
|
920
|
+
msg = {"log": line}
|
|
921
|
+
if line.startswith("@overscore "):
|
|
922
|
+
try:
|
|
923
|
+
msg = json.loads(line[len("@overscore "):])
|
|
924
|
+
except ValueError:
|
|
925
|
+
pass
|
|
926
|
+
if "output" in msg:
|
|
927
|
+
rows_out[0] = int((msg["output"] or {}).get("rows") or 0)
|
|
928
|
+
elif "log" in msg:
|
|
929
|
+
log(str(msg["log"]))
|
|
930
|
+
proc.wait()
|
|
931
|
+
t_err.join(timeout=5)
|
|
932
|
+
ms = round((time.time() - t0) * 1000)
|
|
933
|
+
base = {"model": f"{dish}.{a.name}", "where": "local", "log": logs, "stderr": err[-30:], "inputs": samples, "files": files, "ms": ms}
|
|
934
|
+
if proc.returncode != 0:
|
|
935
|
+
last = next((l for l in reversed(err) if l.strip()), f"exit {proc.returncode}")
|
|
936
|
+
raise Fail(f"model.py failed: {last[:300]}", data=base)
|
|
937
|
+
if rows_out[0] is None or not out_path.exists():
|
|
938
|
+
raise Fail("model.py finished without calling output(rows)", data=base)
|
|
939
|
+
if rows_out[0] == 0:
|
|
940
|
+
raise Fail("model.py returned no rows; in Overscore the table would be left as it was", data=base)
|
|
941
|
+
# land it the way the server lands it, so the types shown are the types the table gets
|
|
942
|
+
result = root / LOCAL / "out" / f"{a.name}.parquet"
|
|
943
|
+
result.parent.mkdir(parents=True, exist_ok=True)
|
|
944
|
+
con = duckdb.connect()
|
|
945
|
+
try:
|
|
946
|
+
con.execute("set TimeZone='UTC'")
|
|
947
|
+
con.execute(f"create table t as select * from read_json_auto('{_sq(out_path)}', maximum_object_size=67108864)")
|
|
948
|
+
n = con.execute("select count(*) from t").fetchone()[0]
|
|
949
|
+
types = [{"column": r[0], "type": r[1]} for r in con.execute("describe t").fetchall()]
|
|
950
|
+
cur = con.execute("select * from t limit 10")
|
|
951
|
+
names = [x[0] for x in cur.description]
|
|
952
|
+
preview = [{k: _plain(v) for k, v in zip(names, r)} for r in cur.fetchall()]
|
|
953
|
+
con.execute(f"copy t to '{_sq(result)}' (format parquet)")
|
|
954
|
+
finally:
|
|
955
|
+
con.close()
|
|
956
|
+
out = {**base, "ok": True, "rows": n, "columns": types, "preview": preview, "output": str(result.relative_to(root))}
|
|
957
|
+
sampled = [f"{k} {v['rows']:,} of {v['of']:,}" for k, v in samples.items() if v.get("of") is not None and v.get("rows") is not None and v["of"] != v["rows"]]
|
|
958
|
+
text = "\n".join([
|
|
959
|
+
*([] if echo else ["log:", *(" " + l for l in logs)]),
|
|
960
|
+
*(["stderr (Overscore keeps these only when the script fails):", *(" " + l for l in err[-30:])] if err else []),
|
|
961
|
+
f"{n:,} rows in {ms / 1000:.1f}s -> {result.relative_to(root)}" + (f" (ran on samples: {', '.join(sampled)})" if sampled else ""),
|
|
962
|
+
"columns: " + ", ".join(f"{t['column']} {t['type']}" for t in types),
|
|
963
|
+
_table(preview, names),
|
|
964
|
+
f"next: ovs model push {a.name}"])
|
|
965
|
+
return out, text
|
|
966
|
+
finally:
|
|
967
|
+
shutil.rmtree(work, ignore_errors=True)
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
# --- destinations and syncs -----------------------------------------------------------------------------------------
|
|
971
|
+
|
|
972
|
+
def _destinations_list(c: Client) -> list[dict]:
|
|
973
|
+
return c.call("GET", "/destinations")["destinations"]
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
def _destination(c: Client, ref: str) -> dict:
|
|
977
|
+
ds = _destinations_list(c)
|
|
978
|
+
hit = [d for d in ds if d["id"] == ref] or [d for d in ds if d["title"].strip().lower() == ref.strip().lower()]
|
|
979
|
+
if not hit:
|
|
980
|
+
raise Fail(f"no destination {ref}; ovs destinations lists them")
|
|
981
|
+
if len(hit) > 1:
|
|
982
|
+
raise Fail(f"{len(hit)} destinations are called {ref}; use the id: {', '.join(d['id'] for d in hit)}")
|
|
983
|
+
return hit[0]
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
def _target(d: dict) -> str:
|
|
987
|
+
cfg = d.get("config") or {}
|
|
988
|
+
if d["kind"] == "webhook":
|
|
989
|
+
return cfg.get("url", "")
|
|
990
|
+
return f"{cfg.get('object')}" + (f" by {cfg['id_property']}" if cfg.get("id_property") else " by record id") + (f", key of {cfg['source']}" if cfg.get("source") else "")
|
|
991
|
+
|
|
992
|
+
|
|
993
|
+
def _destinations(a, c: Client):
|
|
994
|
+
ds = _destinations_list(c)
|
|
995
|
+
rows = [{"id": d["id"], "kind": d["kind"], "title": d["title"], "goes_to": _target(d), "syncs": len(d.get("syncs") or [])} for d in ds]
|
|
996
|
+
return {"destinations": ds}, _table(rows, ["id", "kind", "title", "goes_to", "syncs"]) if rows else "(no destinations yet)"
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
def _destination_cmd(a, c: Client):
|
|
1000
|
+
if a.sub == "list":
|
|
1001
|
+
return _destinations(a, c)
|
|
1002
|
+
if a.sub == "add":
|
|
1003
|
+
if a.kind == "webhook":
|
|
1004
|
+
if not a.webhook_url:
|
|
1005
|
+
raise Fail('a webhook destination needs its URL: ovs destination add webhook "<title>" https://...')
|
|
1006
|
+
d = c.call("POST", "/destinations", json={"kind": "webhook", "title": a.title, "config": {"url": a.webhook_url}})
|
|
1007
|
+
return d, (f"added {d['title']} ({d['id']}): every batch of changed rows is POSTed to {a.webhook_url} as JSON\n"
|
|
1008
|
+
f"signing secret (shown once, keep it where their side can check X-Overscore-Signature): {d['signing_secret']}")
|
|
1009
|
+
if a.kind == "hubspot":
|
|
1010
|
+
token = sys.stdin.readline().strip() if a.token == "-" else (a.token or "")
|
|
1011
|
+
if not token and not a.source:
|
|
1012
|
+
raise Fail("say whose HubSpot key to use: --source <dish.source> (the HubSpot source's own key), or --token - to read one from stdin")
|
|
1013
|
+
d = c.call("POST", "/destinations", json={"kind": "hubspot", "title": a.title,
|
|
1014
|
+
"config": {"object": a.object, "id_property": a.id_property, "source": a.source},
|
|
1015
|
+
"secrets": {"hubspot_token": token} if token else {}})
|
|
1016
|
+
return d, f"added {d['title']} ({d['id']}): rows become properties on HubSpot {_target(d)}\ncheck the key can write: ovs destination check {d['id']}"
|
|
1017
|
+
raise Fail("a destination is webhook or hubspot")
|
|
1018
|
+
dest = _destination(c, a.dest)
|
|
1019
|
+
if a.sub == "check":
|
|
1020
|
+
d = c.call("POST", f"/destinations/{dest['id']}/check", timeout=120)
|
|
1021
|
+
return d, f"{dest['title']}: ok in {d.get('ms')} ms" + (f", {d['properties']} properties on {d['object']}" if d.get("object") else f", answered {d.get('status')}")
|
|
1022
|
+
if a.sub == "rm":
|
|
1023
|
+
# the destination door leaves a scheduled sync's clock behind, failing every interval; the sync door clears it
|
|
1024
|
+
root = _project(a, False)
|
|
1025
|
+
syncs = _read_syncs(root)
|
|
1026
|
+
gone = []
|
|
1027
|
+
for sy in dest.get("syncs") or []:
|
|
1028
|
+
c.call("DELETE", f"/syncs/{sy['id']}")
|
|
1029
|
+
gone.append(sy["name"])
|
|
1030
|
+
for k in [k for k, e in syncs.items() if e.get("id") == sy["id"]]:
|
|
1031
|
+
del syncs[k]
|
|
1032
|
+
d = c.call("DELETE", f"/destinations/{dest['id']}")
|
|
1033
|
+
if root and (root / "syncs.toml").is_file():
|
|
1034
|
+
_write_syncs(root, syncs)
|
|
1035
|
+
return {**d, "syncs_removed": gone}, f"removed {dest['title']}" + (f" and its syncs: {', '.join(gone)}" if gone else "")
|
|
1036
|
+
raise Fail(f"unknown destination command {a.sub}")
|
|
1037
|
+
|
|
1038
|
+
|
|
1039
|
+
def _all_syncs(c: Client) -> list[dict]:
|
|
1040
|
+
return [{**s, "to": d["title"], "to_kind": d["kind"]} for d in _destinations_list(c) for s in (d.get("syncs") or [])]
|
|
1041
|
+
|
|
1042
|
+
|
|
1043
|
+
def _sync_ref(a, c: Client, ref: str) -> dict:
|
|
1044
|
+
ss = _all_syncs(c)
|
|
1045
|
+
hit = [s for s in ss if s["id"] == ref] or [s for s in ss if s["name"] == ref]
|
|
1046
|
+
if len(hit) > 1:
|
|
1047
|
+
dish = getattr(a, "dish", None) or _config(_project(a, False)).get("dish")
|
|
1048
|
+
hit = [s for s in hit if s["dish"] == dish] or hit
|
|
1049
|
+
if not hit:
|
|
1050
|
+
raise Fail(f"no sync {ref}; ovs syncs lists them")
|
|
1051
|
+
if len(hit) > 1:
|
|
1052
|
+
raise Fail(f"{len(hit)} syncs are called {ref}; use the id: {', '.join(s['id'] + ' (' + s['dish'] + ')' for s in hit)}")
|
|
1053
|
+
return hit[0]
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
def _when_text(s: dict) -> str:
|
|
1057
|
+
return {"after_change": "after the table changes", "manual": "by hand"}.get(s["trigger"], f"every {s.get('every')}")
|
|
1058
|
+
|
|
1059
|
+
|
|
1060
|
+
def _syncs(a, c: Client):
|
|
1061
|
+
ss = _all_syncs(c)
|
|
1062
|
+
rows = [{"name": s["name"], "dish": s["dish"], "from": s["source"][:40], "key": s["key"], "to": s["to"], "when": _when_text(s),
|
|
1063
|
+
"on": "yes" if s["enabled"] else "paused", "last": f"{s.get('last_status') or '-'} {_when(s['last_run_at']) if s.get('last_run_at') else ''}".strip(),
|
|
1064
|
+
"sent": s.get("sent_total") or 0, "refused": s.get("failures") or 0} for s in ss]
|
|
1065
|
+
return {"syncs": ss}, _table(rows, ["name", "dish", "from", "key", "to", "when", "on", "last", "sent", "refused"]) if rows else "(no syncs yet)"
|
|
1066
|
+
|
|
1067
|
+
|
|
1068
|
+
SYNCS_HEADER = """# syncs.toml: what goes where, one [name] per sync.
|
|
1069
|
+
# `ovs sync add <name> --from <table> --key <column> --to <destination>` sets one up in Overscore and writes it here.
|
|
1070
|
+
# Edit an entry, then run `ovs sync add <name>` to apply it; editing this file alone changes nothing in Overscore.
|
|
1071
|
+
# ovs rewrites this file when it adds or removes a sync: comments below this header are not kept.
|
|
1072
|
+
# from the table in the dish (or sql = "select ..." in its place)
|
|
1073
|
+
# key the column that identifies the record at the destination
|
|
1074
|
+
# to the destination's name or id (ovs destinations)
|
|
1075
|
+
# columns column = "field at the destination"; leave it out to send every column but the key, as the table
|
|
1076
|
+
# has them when `ovs sync add` runs (run it again to pick up a new column)
|
|
1077
|
+
# when after_change | manual | every 5m | every 15m | every 60m | every 6h | every 24h
|
|
1078
|
+
# id filled in by ovs
|
|
1079
|
+
"""
|
|
1080
|
+
SYNC_KEYS = ("dish", "from", "sql", "key", "to", "columns", "when", "id")
|
|
1081
|
+
|
|
1082
|
+
|
|
1083
|
+
def _toml_key(k: str) -> str:
|
|
1084
|
+
return k if re.fullmatch(r"[A-Za-z0-9_-]+", k) else _toml_str(k)
|
|
1085
|
+
|
|
1086
|
+
|
|
1087
|
+
def _toml_value(v) -> str:
|
|
1088
|
+
if isinstance(v, bool):
|
|
1089
|
+
return "true" if v else "false"
|
|
1090
|
+
if isinstance(v, (int, float)):
|
|
1091
|
+
return str(v)
|
|
1092
|
+
if isinstance(v, list):
|
|
1093
|
+
return "[" + ", ".join(_toml_value(x) for x in v) + "]"
|
|
1094
|
+
if isinstance(v, dict):
|
|
1095
|
+
return "{ " + ", ".join(f"{_toml_key(k)} = {_toml_value(x)}" for k, x in v.items()) + " }"
|
|
1096
|
+
return _toml_str(str(v))
|
|
1097
|
+
|
|
1098
|
+
|
|
1099
|
+
def _toml_str(v: str) -> str:
|
|
1100
|
+
"""A TOML basic string: JSON's escapes, but characters beyond the first plane kept as themselves (TOML refuses
|
|
1101
|
+
JSON's surrogate pairs) and DEL escaped (TOML refuses it bare)."""
|
|
1102
|
+
return json.dumps(v, ensure_ascii=False).replace("\x7f", "\\u007f")
|
|
1103
|
+
|
|
1104
|
+
|
|
1105
|
+
def _read_syncs(root: pathlib.Path | None) -> dict:
|
|
1106
|
+
p = root / "syncs.toml" if root else None
|
|
1107
|
+
return _read_toml(p) if p and p.is_file() else {}
|
|
1108
|
+
|
|
1109
|
+
|
|
1110
|
+
def _write_syncs(root: pathlib.Path, syncs: dict) -> None:
|
|
1111
|
+
parts = [SYNCS_HEADER.rstrip("\n")]
|
|
1112
|
+
for name, e in syncs.items():
|
|
1113
|
+
lines = [f"[{_toml_key(name)}]"]
|
|
1114
|
+
lines += [f"{k} = {_toml_value(e[k])}" for k in (*SYNC_KEYS, *(k for k in e if k not in SYNC_KEYS)) if e.get(k) not in (None, "", [], {})]
|
|
1115
|
+
parts.append("\n".join(lines))
|
|
1116
|
+
text = "\n\n".join(parts) + "\n"
|
|
1117
|
+
import tomllib
|
|
1118
|
+
tomllib.loads(text) # never leave a file that will not read back
|
|
1119
|
+
(root / "syncs.toml").write_text(text, encoding="utf-8")
|
|
1120
|
+
|
|
1121
|
+
|
|
1122
|
+
def _columns(items: list[str]) -> dict:
|
|
1123
|
+
out = {}
|
|
1124
|
+
for it in items:
|
|
1125
|
+
col, _, field = it.partition("=")
|
|
1126
|
+
out[col.strip()] = field.strip() or col.strip()
|
|
1127
|
+
return out
|
|
1128
|
+
|
|
1129
|
+
|
|
1130
|
+
def _sync_add(a, c: Client):
|
|
1131
|
+
_name(a.name, "sync")
|
|
1132
|
+
root = _project(a, False)
|
|
1133
|
+
syncs = _read_syncs(root)
|
|
1134
|
+
e = dict(syncs.get(a.name) or {})
|
|
1135
|
+
if a.source:
|
|
1136
|
+
e["from"] = a.source; e.pop("sql", None)
|
|
1137
|
+
if a.sql:
|
|
1138
|
+
e["sql"] = a.sql; e.pop("from", None)
|
|
1139
|
+
if a.id_column:
|
|
1140
|
+
e["key"] = a.id_column
|
|
1141
|
+
if a.to:
|
|
1142
|
+
e["to"] = a.to
|
|
1143
|
+
if a.columns:
|
|
1144
|
+
e["columns"] = _columns(a.columns)
|
|
1145
|
+
if a.after_change:
|
|
1146
|
+
e["when"] = "after_change"
|
|
1147
|
+
elif a.every:
|
|
1148
|
+
e["when"] = f"every {a.every}"
|
|
1149
|
+
elif a.manual:
|
|
1150
|
+
e["when"] = "manual"
|
|
1151
|
+
if a.dish:
|
|
1152
|
+
e["dish"] = a.dish
|
|
1153
|
+
lacking = [f for f, k in (("--from <table> (or --sql)", "from" if "sql" not in e else "sql"), ("--key <id column>", "key"), ("--to <destination>", "to")) if not e.get(k)]
|
|
1154
|
+
if lacking:
|
|
1155
|
+
raise Fail(f"sync {a.name} needs " + ", ".join(lacking) + (" (nor is it in syncs.toml)" if root else ""))
|
|
1156
|
+
dish = e.get("dish")
|
|
1157
|
+
if e.get("from"):
|
|
1158
|
+
d, _, source = e["from"].rpartition(".")
|
|
1159
|
+
if d and dish and d != dish:
|
|
1160
|
+
raise Fail(f"{e['from']} is not in the dish {dish}; a sync sends a table from its own dish")
|
|
1161
|
+
dish = d or dish
|
|
1162
|
+
else:
|
|
1163
|
+
source = e["sql"]
|
|
1164
|
+
dish = _dish(a, dish)
|
|
1165
|
+
when = e.get("when") or "after_change"
|
|
1166
|
+
if when in ("after_change", "manual"):
|
|
1167
|
+
trigger, every = when, None
|
|
1168
|
+
elif when.startswith("every ") and when.split(" ", 1)[1] in SYNC_EVERY:
|
|
1169
|
+
trigger, every = "schedule", when.split(" ", 1)[1]
|
|
1170
|
+
else:
|
|
1171
|
+
raise Fail(f"when is after_change, manual, or every {'/'.join(SYNC_EVERY)}; syncs.toml has {when!r}")
|
|
1172
|
+
cols = e.get("columns")
|
|
1173
|
+
if isinstance(cols, list):
|
|
1174
|
+
cols = {x: x for x in cols}
|
|
1175
|
+
if not cols: # every column but the key, as the source has them now
|
|
1176
|
+
src = f"select * from {dish}.{source}" if e.get("from") else source
|
|
1177
|
+
found = c.call("POST", "/query", json={"sql": f"select * from ({src}) limit 0", "dish": dish, "max_rows": 1})["columns"]
|
|
1178
|
+
if e["key"] not in found:
|
|
1179
|
+
raise Fail(f"the key column {e['key']} is not in {e.get('from') or 'the query'}; it has {', '.join(found)}")
|
|
1180
|
+
cols = {x: x for x in found if x != e["key"]}
|
|
1181
|
+
dest = _destination(c, e["to"])
|
|
1182
|
+
body = {"dish": dish, "name": a.name, "source": source, "key": e["key"], "columns": cols, "trigger": trigger, "every": every}
|
|
1183
|
+
ss = _all_syncs(c)
|
|
1184
|
+
cur = next((s for s in ss if e.get("id") and s["id"] == e["id"]), None) or next((s for s in ss if s["name"] == a.name and s["dish"] == dish), None)
|
|
1185
|
+
# What went is remembered by the rows' values, per sync. Another destination, or a column going to a field of another
|
|
1186
|
+
# name, needs a sync that remembers nothing, or unchanged rows would never reach the new place: make the new one,
|
|
1187
|
+
# then remove the old (in that order, so a refusal leaves the old one as it was).
|
|
1188
|
+
fresh = bool(cur) and (cur["destination_id"] != dest["id"] or any(cur["columns"].get(k) not in (None, v) for k, v in cols.items()))
|
|
1189
|
+
left = None
|
|
1190
|
+
if cur and not fresh:
|
|
1191
|
+
s = c.call("PUT", f"/syncs/{cur['id']}", json=body)
|
|
1192
|
+
else:
|
|
1193
|
+
s = c.call("POST", f"/destinations/{dest['id']}/syncs", json=body)
|
|
1194
|
+
if fresh and not cur["enabled"]:
|
|
1195
|
+
s = c.call("POST", f"/syncs/{s['id']}/enable", json={"on": False})
|
|
1196
|
+
if root:
|
|
1197
|
+
e["id"] = s["id"]
|
|
1198
|
+
syncs[a.name] = e
|
|
1199
|
+
_write_syncs(root, syncs) # before the old one goes, so a retry finds the new one and makes no third
|
|
1200
|
+
if fresh:
|
|
1201
|
+
try:
|
|
1202
|
+
c.call("DELETE", f"/syncs/{cur['id']}")
|
|
1203
|
+
except Fail as x:
|
|
1204
|
+
left = f"the old sync {cur['id']} is still there ({x}); remove it: ovs sync rm {cur['id']}"
|
|
1205
|
+
what = f"{dish}.{source}" if e.get("from") else "the query"
|
|
1206
|
+
fields = ", ".join(c_ if f == c_ else f"{c_} as {f}" for c_, f in s["columns"].items())
|
|
1207
|
+
verb = ("made again (a new place or field: every row goes on the next send" + ("; still paused" if fresh and not cur["enabled"] else "") + ")"
|
|
1208
|
+
if fresh else "updated" if cur else "added")
|
|
1209
|
+
text = (f"{verb} sync {a.name} ({s['id']}): {what} to {dest['title']}, keyed by {s['key']}, "
|
|
1210
|
+
f"columns {fields}, {_when_text(s)}" + ("; written to syncs.toml" if root else "")
|
|
1211
|
+
+ f"\nonly rows that changed since they last went are sent. To send now: ovs sync run {a.name}"
|
|
1212
|
+
+ (f"\nwarning: {left}" if left else ""))
|
|
1213
|
+
return {"sync": s, "updated": bool(cur) and not fresh, "made_again": fresh, **({"warning": left} if left else {})}, text
|
|
1214
|
+
|
|
1215
|
+
|
|
1216
|
+
def _sync_cmd(a, c: Client):
|
|
1217
|
+
if a.sub == "list":
|
|
1218
|
+
return _syncs(a, c)
|
|
1219
|
+
if a.sub == "add":
|
|
1220
|
+
return _sync_add(a, c)
|
|
1221
|
+
s = _sync_ref(a, c, a.name)
|
|
1222
|
+
if a.sub == "run":
|
|
1223
|
+
try:
|
|
1224
|
+
d = c.call("POST", f"/syncs/{s['id']}/run", headers={"x-overscore-trigger": "cli"}, timeout=3600)
|
|
1225
|
+
except Fail as e:
|
|
1226
|
+
if e.status != 400 or "rows were refused" not in str(e):
|
|
1227
|
+
raise
|
|
1228
|
+
shown = c.call("GET", f"/syncs/{s['id']}")
|
|
1229
|
+
raise Fail(f"{e} (ovs sync show {s['name']} lists what was refused; they go again next run)", 400,
|
|
1230
|
+
data={"sync": s["id"], "refused": shown["failures"][:20]}) from None
|
|
1231
|
+
text = f"{s['name']} to {s['to']}: sent {d['sent']:,} rows" + (f", {d['failed']:,} refused (ovs sync show {s['name']} says why)" if d.get("failed") else "")
|
|
1232
|
+
return d, text + f" (run {d.get('run_id')})"
|
|
1233
|
+
if a.sub == "show":
|
|
1234
|
+
d = c.call("GET", f"/syncs/{s['id']}")
|
|
1235
|
+
me = d["sync"]
|
|
1236
|
+
every = c.call("GET", "/runs", params={"kind": "sync", "dish": me["dish"], "limit": 500})["runs"]
|
|
1237
|
+
runs = [r for r in every if r.get("name") == f"{me['name']} to {s['to']}"][:10]
|
|
1238
|
+
text = (f"{me['name']} ({me['id']}): {me['dish']}.{me['source'] if len(me['source']) < 60 else 'query'} to {s['to']}, keyed by {me['key']}, "
|
|
1239
|
+
f"{_when_text(me)}, {'on' if me['enabled'] else 'paused'}; {me.get('sent_total') or 0:,} rows sent in all\n")
|
|
1240
|
+
text += "runs:\n" + (_table([{"id": r["id"], "status": r["status"], "when": _when(r.get("started_at")), "rows": r.get("rows"), "error": (r.get("error") or "")[:70]} for r in runs],
|
|
1241
|
+
["id", "status", "when", "rows", "error"]) if runs else "(none yet)")
|
|
1242
|
+
if d["failures"]:
|
|
1243
|
+
text += "\nrefused (they go again next run):\n" + _table([{"key": f["key"], "when": _when(f["at"]), "reason": f["reason"][:90]} for f in d["failures"][:20]], ["key", "when", "reason"])
|
|
1244
|
+
return {**d, "runs": runs}, text
|
|
1245
|
+
if a.sub in ("pause", "resume"):
|
|
1246
|
+
d = c.call("POST", f"/syncs/{s['id']}/enable", json={"on": a.sub == "resume"})
|
|
1247
|
+
return d, f"{s['name']}: {'paused' if a.sub == 'pause' else 'on again'}"
|
|
1248
|
+
if a.sub == "rm":
|
|
1249
|
+
root = _project(a, False)
|
|
1250
|
+
syncs = _read_syncs(root)
|
|
1251
|
+
d = c.call("DELETE", f"/syncs/{s['id']}")
|
|
1252
|
+
if root and s["name"] in syncs and syncs[s["name"]].get("id") in (None, s["id"]):
|
|
1253
|
+
del syncs[s["name"]]
|
|
1254
|
+
_write_syncs(root, syncs)
|
|
1255
|
+
return d, f"removed {s['name']}" + (" (and its entry in syncs.toml)" if root else "")
|
|
1256
|
+
raise Fail(f"unknown sync command {a.sub}")
|
|
1257
|
+
|
|
1258
|
+
|
|
1259
|
+
# --- ovs login: the key in this computer's keychain, the address beside it (CLI_LOGIN_PLAN.md) --------------
|
|
1260
|
+
#
|
|
1261
|
+
# The laptop asks the front desk for a code, the person approves it in the browser, and the laptop collects a key of
|
|
1262
|
+
# its own and its Overscore's address (https://<customer>.overscore.tech). The key goes in the OS keychain (the
|
|
1263
|
+
# keyring library: macOS Keychain, Windows Credential Locker, the Secret Service on Linux); the address, who and
|
|
1264
|
+
# which laptop go in ~/.config/overscore/login.json. No key is ever written to a file.
|
|
1265
|
+
|
|
1266
|
+
KEYCHAIN = "overscore"
|
|
1267
|
+
|
|
1268
|
+
|
|
1269
|
+
def _login_file() -> pathlib.Path:
|
|
1270
|
+
base = os.environ.get("XDG_CONFIG_HOME") or os.path.join(os.path.expanduser("~"), ".config")
|
|
1271
|
+
return pathlib.Path(base) / "overscore" / "login.json"
|
|
1272
|
+
|
|
1273
|
+
|
|
1274
|
+
def _saved_login() -> dict:
|
|
1275
|
+
try:
|
|
1276
|
+
return json.loads(_login_file().read_text(encoding="utf-8"))
|
|
1277
|
+
except (OSError, ValueError):
|
|
1278
|
+
return {}
|
|
1279
|
+
|
|
1280
|
+
|
|
1281
|
+
def _keychain():
|
|
1282
|
+
try:
|
|
1283
|
+
import keyring
|
|
1284
|
+
except ImportError:
|
|
1285
|
+
raise Fail("this ovs cannot reach a keychain (the keyring library is missing); reinstall overscore-cli, or export OVERSCORE_KEY")
|
|
1286
|
+
return keyring
|
|
1287
|
+
|
|
1288
|
+
|
|
1289
|
+
def _saved_key(address: str) -> str | None:
|
|
1290
|
+
try:
|
|
1291
|
+
return _keychain().get_password(KEYCHAIN, address.rstrip("/"))
|
|
1292
|
+
except Exception: # noqa: BLE001 (no keychain on this machine: then there is no saved key)
|
|
1293
|
+
return None
|
|
1294
|
+
|
|
1295
|
+
|
|
1296
|
+
def _device_id() -> str:
|
|
1297
|
+
"""This computer's own random id (not a secret), kept beside the login so logging in again replaces its key."""
|
|
1298
|
+
f = _login_file().with_name("device")
|
|
1299
|
+
try:
|
|
1300
|
+
d = f.read_text(encoding="utf-8").strip()
|
|
1301
|
+
if d:
|
|
1302
|
+
return d
|
|
1303
|
+
except OSError:
|
|
1304
|
+
pass
|
|
1305
|
+
import secrets as _s
|
|
1306
|
+
d = _s.token_hex(8)
|
|
1307
|
+
f.parent.mkdir(parents=True, exist_ok=True)
|
|
1308
|
+
f.write_text(d, encoding="utf-8")
|
|
1309
|
+
return d
|
|
1310
|
+
|
|
1311
|
+
|
|
1312
|
+
def _open_browser(url: str) -> None:
|
|
1313
|
+
try:
|
|
1314
|
+
import webbrowser
|
|
1315
|
+
webbrowser.open(url)
|
|
1316
|
+
except Exception: # noqa: BLE001
|
|
1317
|
+
pass
|
|
1318
|
+
|
|
1319
|
+
|
|
1320
|
+
def _login(a, c):
|
|
1321
|
+
import socket
|
|
1322
|
+
front = _check_address((getattr(a, "url_given", None) or DEFAULT_URL).rstrip("/"), given=bool(getattr(a, "url_given", None)))
|
|
1323
|
+
kc = _keychain()
|
|
1324
|
+
try: # the keychain must take a key before a key is asked for
|
|
1325
|
+
kc.set_password(KEYCHAIN, "overscore-probe", "probe")
|
|
1326
|
+
kc.delete_password(KEYCHAIN, "overscore-probe")
|
|
1327
|
+
except Exception as e: # noqa: BLE001
|
|
1328
|
+
raise Fail(f"this computer's keychain would not keep a key ({type(e).__name__}), so ovs login cannot work here. "
|
|
1329
|
+
"Make a key on the Agents screen instead and export OVERSCORE_KEY=ovs_... in the shell") from None
|
|
1330
|
+
desk = Client(front, None)
|
|
1331
|
+
st = desk.call("POST", "/cli/login/start", json={"machine": socket.gethostname(), "device_id": _device_id()})
|
|
1332
|
+
say = f"Open {st['url']} and type the code {st['code']}, then approve. Waiting..."
|
|
1333
|
+
print(say, file=sys.stderr, flush=True)
|
|
1334
|
+
if not getattr(a, "echo", True): # an agent reading JSON sees it too, before the wait
|
|
1335
|
+
print(json.dumps({"status": "waiting", "url": st["url"], "code": st["code"], "say": "ask the person to open the page and type the code"}), flush=True)
|
|
1336
|
+
if not getattr(a, "no_browser", False):
|
|
1337
|
+
_open_browser(st["url"])
|
|
1338
|
+
deadline = time.time() + st.get("expires_in", 600) + 30
|
|
1339
|
+
wait = st.get("interval", 2)
|
|
1340
|
+
while time.time() < deadline:
|
|
1341
|
+
time.sleep(wait)
|
|
1342
|
+
try:
|
|
1343
|
+
r = desk.call("POST", "/cli/login/poll", json={"device": st["device"]})
|
|
1344
|
+
except Fail as e: # a blip (a redeploy, a busy office network): keep asking, a little slower
|
|
1345
|
+
if e.status is not None and e.status < 500 and e.status != 429:
|
|
1346
|
+
raise
|
|
1347
|
+
wait = min(wait * 2, 15)
|
|
1348
|
+
continue
|
|
1349
|
+
wait = st.get("interval", 2)
|
|
1350
|
+
if r["status"] == "approved":
|
|
1351
|
+
address = r["address"].rstrip("/")
|
|
1352
|
+
try:
|
|
1353
|
+
kc.set_password(KEYCHAIN, address, r["key"])
|
|
1354
|
+
except Exception as e: # noqa: BLE001 (the key was handed over once: turn it off rather than lose it)
|
|
1355
|
+
try:
|
|
1356
|
+
Client(address, r["key"]).call("POST", "/cli/logout")
|
|
1357
|
+
except Fail:
|
|
1358
|
+
pass
|
|
1359
|
+
raise Fail(f"the keychain refused the key ({type(e).__name__}); it has been turned off. Unlock the keychain and run ovs login again") from None
|
|
1360
|
+
f = _login_file()
|
|
1361
|
+
f.parent.mkdir(parents=True, exist_ok=True)
|
|
1362
|
+
f.write_text(json.dumps({"address": address, "customer": r.get("customer"), "email": r.get("email"),
|
|
1363
|
+
"machine": r.get("machine"), "at": time.time()}, indent=1), encoding="utf-8")
|
|
1364
|
+
out = {"logged_in": True, "address": address, "customer": r.get("customer"), "email": r.get("email")}
|
|
1365
|
+
return out, (f"logged in as {r.get('email')} to {r.get('customer')} ({address}); the key is in this computer's keychain.\n"
|
|
1366
|
+
f"next: ovs init --dish <dish> in a project folder (ovs tables lists what you can reach)")
|
|
1367
|
+
if r["status"] in ("cancelled", "expired"):
|
|
1368
|
+
raise Fail(f"the login was {r['status']}; run ovs login again")
|
|
1369
|
+
raise Fail("nobody approved the login in ten minutes; run ovs login again")
|
|
1370
|
+
|
|
1371
|
+
|
|
1372
|
+
def _logout(a, c):
|
|
1373
|
+
saved = _saved_login()
|
|
1374
|
+
toml = _config(_project(a, False)).get("url")
|
|
1375
|
+
address = (getattr(a, "url_given", None) or (toml if toml and toml != DEFAULT_URL else None) or saved.get("address") or DEFAULT_URL).rstrip("/")
|
|
1376
|
+
key = _saved_key(address)
|
|
1377
|
+
revoked = 0
|
|
1378
|
+
if key:
|
|
1379
|
+
try:
|
|
1380
|
+
revoked = Client(address, key).call("POST", "/cli/logout").get("revoked", 0)
|
|
1381
|
+
except Fail as e:
|
|
1382
|
+
print(f"(could not turn the key off in Overscore: {e}; it ends by itself after 90 days unused, or on the Agents screen)", file=sys.stderr)
|
|
1383
|
+
try:
|
|
1384
|
+
_keychain().delete_password(KEYCHAIN, address)
|
|
1385
|
+
except Exception: # noqa: BLE001
|
|
1386
|
+
pass
|
|
1387
|
+
if (saved.get("address") or "").rstrip("/") == address:
|
|
1388
|
+
_login_file().unlink(missing_ok=True)
|
|
1389
|
+
return {"logged_out": True, "address": address, "revoked": revoked}, (
|
|
1390
|
+
f"logged out of {address}" + ("; the key is off" if revoked else "") if key or saved else "not logged in here")
|
|
1391
|
+
|
|
1392
|
+
|
|
1393
|
+
# --- init: the project folder, and the rules and skills rendered for the agent that will work in it (SKILLS_PLAN.md) ---
|
|
1394
|
+
|
|
1395
|
+
def _skills_root() -> pathlib.Path:
|
|
1396
|
+
here = pathlib.Path(__file__).resolve().parent
|
|
1397
|
+
for d in (here / "skills", here.parent / "skills"): # inside the package, or the repo's skills/ beside overscore/
|
|
1398
|
+
if all((d / n / "SKILL.md").is_file() for n in SHIPPED_SKILLS):
|
|
1399
|
+
return d
|
|
1400
|
+
raise Fail("the skills are missing from this install of ovs; reinstall overscore-cli")
|
|
1401
|
+
|
|
1402
|
+
|
|
1403
|
+
def _frontmatter(text: str) -> tuple[dict, str]:
|
|
1404
|
+
if text.startswith("---\n"):
|
|
1405
|
+
end = text.find("\n---", 4)
|
|
1406
|
+
if end != -1:
|
|
1407
|
+
meta = {}
|
|
1408
|
+
for line in text[4:end].splitlines():
|
|
1409
|
+
k, sep, v = line.partition(":")
|
|
1410
|
+
if sep:
|
|
1411
|
+
v = v.strip()
|
|
1412
|
+
meta[k.strip()] = json.loads(v) if v.startswith('"') else v
|
|
1413
|
+
return meta, text[end + 4:].lstrip("\n")
|
|
1414
|
+
return {}, text
|
|
1415
|
+
|
|
1416
|
+
|
|
1417
|
+
def _skill(name: str) -> dict:
|
|
1418
|
+
d = _skills_root() / name
|
|
1419
|
+
meta, body = _frontmatter((d / "SKILL.md").read_text(encoding="utf-8"))
|
|
1420
|
+
extras = {p.name: p.read_text(encoding="utf-8") for p in sorted(d.iterdir()) if p.is_file() and p.name != "SKILL.md" and not p.name.startswith(".")}
|
|
1421
|
+
return {"name": name, "description": meta.get("description", ""), "body": body, "extras": extras, "dir": d}
|
|
1422
|
+
|
|
1423
|
+
|
|
1424
|
+
def _demote(md: str, by: int) -> str:
|
|
1425
|
+
"""Push every heading down `by` levels, leaving code blocks alone."""
|
|
1426
|
+
out, fenced = [], False
|
|
1427
|
+
for line in md.splitlines():
|
|
1428
|
+
if line.startswith("```"):
|
|
1429
|
+
fenced = not fenced
|
|
1430
|
+
out.append("#" * by + line if not fenced and re.match(r"#{1,6} ", line) else line)
|
|
1431
|
+
return "\n".join(out)
|
|
1432
|
+
|
|
1433
|
+
|
|
1434
|
+
def _with_extras(s: dict) -> str:
|
|
1435
|
+
"""The skill's body with its templates written out, for layouts that hold one file per skill."""
|
|
1436
|
+
parts = [s["body"].rstrip()]
|
|
1437
|
+
for fname, content in s["extras"].items():
|
|
1438
|
+
lang = "python" if fname.endswith(".py") else "toml" if fname.endswith(".toml") else ""
|
|
1439
|
+
parts.append(f"### {fname} (template)\n\n```{lang}\n{content.rstrip()}\n```")
|
|
1440
|
+
return "\n\n".join(parts)
|
|
1441
|
+
|
|
1442
|
+
|
|
1443
|
+
RULES = """# Overscore
|
|
1444
|
+
|
|
1445
|
+
This folder builds things that run on Overscore: Python models that score or reshape a company's data, and syncs that send the results to the systems people work in. The `ovs` command does all of it (`ovs guide` lists every command). {where}
|
|
1446
|
+
|
|
1447
|
+
## The folder
|
|
1448
|
+
- `overscore.toml`: the Overscore address and the dish (the dataset) this project works in. Never the key.
|
|
1449
|
+
- `models/<name>/model.py` and `model.toml`: one Python model per folder. The folder's name is the model's name and the name of the table it builds.
|
|
1450
|
+
- `files/<name>.<ext>`: local copies of the files a model needs (a fitted model, a lookup). `ovs upload put <name> files/<name>.<ext>` sends one to Overscore.
|
|
1451
|
+
- `syncs.toml`: what goes where, written by `ovs sync add`.
|
|
1452
|
+
- `.overscore/`: pulled samples, local outputs, environments. It is the company's data: never commit it, never paste it into a message.
|
|
1453
|
+
|
|
1454
|
+
## Rules
|
|
1455
|
+
- Tables are `dish.table`; a model's output is `dish.model`. Names are lowercase letters, digits and underscores.
|
|
1456
|
+
- `ovs login` keeps this computer's key in its keychain (`OVERSCORE_KEY` in the environment works too). A key never goes in a file, a commit or a message.
|
|
1457
|
+
- Run a model here (`ovs model run <name> --local`) before pushing it. Call `output()` once. The script gets no network in Overscore, and only model.py goes up: keep the model in that one file.
|
|
1458
|
+
- List every library the script imports in model.toml `requirements`; both this machine and Overscore build the environment from it.
|
|
1459
|
+
- After `ovs model push`, read the run's result and log before saying it worked. A run is the receipt.
|
|
1460
|
+
- Don't write to `sent`, `control` or any schema that is Overscore's own.
|
|
1461
|
+
- Say what changed in the words of the business, not the column names.
|
|
1462
|
+
- Every `ovs` command prints JSON when its output is not a terminal: `{"success": true, ...}`, or `{"success": false, "error": "..."}` with exit code 1. The error says what to do next."""
|
|
1463
|
+
|
|
1464
|
+
WHERE = {
|
|
1465
|
+
"claude": "The steps for each job are skills in `.claude/skills/`: `overscore` (start here), `overscore-model` (build and deploy a Python model), `overscore-sync` (send a table to another system).",
|
|
1466
|
+
"cursor": "The steps for each job are rules in `.cursor/rules/`: `overscore` (start here), `overscore-model` (build and deploy a Python model), `overscore-sync` (send a table to another system).",
|
|
1467
|
+
"inline": "The steps for each job follow below: Overscore itself (start here), building and deploying a Python model, and sending a table to another system.",
|
|
1468
|
+
}
|
|
1469
|
+
|
|
1470
|
+
|
|
1471
|
+
def _block(path: pathlib.Path, body: str) -> None:
|
|
1472
|
+
"""Write our block into a rules file: replace it if there, append it if not, and leave the rest alone."""
|
|
1473
|
+
block = f"{BLOCK_START}\n{body.rstrip()}\n{BLOCK_END}\n"
|
|
1474
|
+
text = path.read_text(encoding="utf-8") if path.is_file() else ""
|
|
1475
|
+
i = text.find(BLOCK_OPEN)
|
|
1476
|
+
if i != -1:
|
|
1477
|
+
j = text.find(BLOCK_END, i)
|
|
1478
|
+
if j == -1:
|
|
1479
|
+
raise Fail(f"{path.name} has the start of the Overscore block but not its end ({BLOCK_END}); put the end back, or delete the start line")
|
|
1480
|
+
text = text[:i] + block + text[j + len(BLOCK_END):].lstrip("\n")
|
|
1481
|
+
else:
|
|
1482
|
+
text = (text.rstrip() + "\n\n" if text.strip() else "") + block
|
|
1483
|
+
path.write_text(text, encoding="utf-8")
|
|
1484
|
+
|
|
1485
|
+
|
|
1486
|
+
def _render(root: pathlib.Path, target: str) -> list[str]:
|
|
1487
|
+
"""The one source (skills/) in one agent's layout. Returns what was written, relative to root."""
|
|
1488
|
+
skills = [_skill(n) for n in SHIPPED_SKILLS]
|
|
1489
|
+
written = []
|
|
1490
|
+
if target == "claude":
|
|
1491
|
+
_block(root / "CLAUDE.md", RULES.replace("{where}", WHERE["claude"]))
|
|
1492
|
+
written.append("CLAUDE.md")
|
|
1493
|
+
for s in skills:
|
|
1494
|
+
dest = root / ".claude" / "skills" / s["name"]
|
|
1495
|
+
# the shipped files are refreshed; anything else a team put in the folder stays
|
|
1496
|
+
shutil.copytree(s["dir"], dest, dirs_exist_ok=True, ignore=shutil.ignore_patterns("__pycache__", ".*", "*.pyc"))
|
|
1497
|
+
text = (dest / "SKILL.md").read_text(encoding="utf-8")
|
|
1498
|
+
head, sep, body = text.partition("\n---\n") # the notice goes under the frontmatter
|
|
1499
|
+
(dest / "SKILL.md").write_text(head + sep + REWRITTEN + "\n\n" + body.lstrip("\n"), encoding="utf-8")
|
|
1500
|
+
written.append(f".claude/skills/{s['name']}/")
|
|
1501
|
+
elif target in ("codex", "agents", "gemini"):
|
|
1502
|
+
name = "GEMINI.md" if target == "gemini" else "AGENTS.md"
|
|
1503
|
+
inline = "\n\n".join(f"## {s['name']}\n\n_{s['description']}_\n\n" + _demote(_with_extras(s), 2) for s in skills)
|
|
1504
|
+
_block(root / name, RULES.replace("{where}", WHERE["inline"]) + "\n\n" + inline)
|
|
1505
|
+
written.append(name)
|
|
1506
|
+
elif target == "cursor":
|
|
1507
|
+
rules = root / ".cursor" / "rules"
|
|
1508
|
+
rules.mkdir(parents=True, exist_ok=True)
|
|
1509
|
+
(rules / "overscore-rules.mdc").write_text("---\ndescription: How this project works with Overscore; always on.\nglobs:\nalwaysApply: true\n---\n\n"
|
|
1510
|
+
+ REWRITTEN + "\n\n" + RULES.replace("{where}", WHERE["cursor"]) + "\n", encoding="utf-8")
|
|
1511
|
+
written.append(".cursor/rules/overscore-rules.mdc")
|
|
1512
|
+
for s in skills:
|
|
1513
|
+
(rules / f"{s['name']}.mdc").write_text(f"---\ndescription: {json.dumps(s['description'], ensure_ascii=False)}\nglobs:\nalwaysApply: false\n---\n\n"
|
|
1514
|
+
+ REWRITTEN + "\n\n" + _with_extras(s) + "\n", encoding="utf-8")
|
|
1515
|
+
written.append(f".cursor/rules/{s['name']}.mdc")
|
|
1516
|
+
return written
|
|
1517
|
+
|
|
1518
|
+
|
|
1519
|
+
def _init(a, c: Client | None):
|
|
1520
|
+
root = pathlib.Path(a.path or ".").resolve()
|
|
1521
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
1522
|
+
cur = _config(root)
|
|
1523
|
+
url = getattr(a, "url_given", None) or (cur.get("url") if cur.get("url") != DEFAULT_URL else None) or _saved_login().get("address") or DEFAULT_URL
|
|
1524
|
+
dish, seen, check = a.dish or cur.get("dish"), None, None
|
|
1525
|
+
if c is not None:
|
|
1526
|
+
c.url = url.rstrip("/")
|
|
1527
|
+
try:
|
|
1528
|
+
c.url = _check_address(c.url, given=bool(getattr(a, "url_given", None)) or c.url == (_saved_login().get("address") or "").rstrip("/"))
|
|
1529
|
+
seen = c.call("GET", "/whoami")
|
|
1530
|
+
except Fail as e:
|
|
1531
|
+
check = f"did not check the key against {url}: {e}"
|
|
1532
|
+
if not dish:
|
|
1533
|
+
scopes = (seen or {}).get("scopes") or {}
|
|
1534
|
+
hot = sorted(d for d, v in scopes.items() if d != "*" and (v == "hot" or (isinstance(v, dict) and v.get("level") == "hot")))
|
|
1535
|
+
if len(hot) != 1:
|
|
1536
|
+
reach = "the key reaches every dish" if "*" in scopes else f"the key writes to: {', '.join(hot)}" if hot else ""
|
|
1537
|
+
raise Fail("which dish does this project work in? ovs init --dish <dish>" + (f" ({reach}; ovs tables lists them)" if reach else ""))
|
|
1538
|
+
dish = hot[0]
|
|
1539
|
+
_name(dish, "dish")
|
|
1540
|
+
targets = ["claude", "agents", "cursor", "gemini"] if a.agent == "all" else [a.agent or "claude"]
|
|
1541
|
+
(root / PROJECT_FILE).write_text("# This folder is an Overscore project; ovs reads the address and the dish from here.\n"
|
|
1542
|
+
"# The key is never written here: ovs login keeps it in the keychain.\n"
|
|
1543
|
+
+ (f"url = {_toml_value(url)}\n" if url != DEFAULT_URL else "# url: the address ovs login gives, when there is one\n")
|
|
1544
|
+
+ f"dish = {_toml_value(dish)}\n", encoding="utf-8")
|
|
1545
|
+
for d in ("models", "files"):
|
|
1546
|
+
(root / d).mkdir(exist_ok=True)
|
|
1547
|
+
if not (root / "syncs.toml").exists():
|
|
1548
|
+
_write_syncs(root, {})
|
|
1549
|
+
gi = root / ".gitignore"
|
|
1550
|
+
lines = gi.read_text(encoding="utf-8").splitlines() if gi.is_file() else []
|
|
1551
|
+
if not any(l.strip().rstrip("/") in (LOCAL, "/" + LOCAL) for l in lines):
|
|
1552
|
+
gi.write_text("\n".join([*lines, "# pulled samples and local outputs: the company's data", LOCAL + "/"]) + "\n", encoding="utf-8")
|
|
1553
|
+
written = [PROJECT_FILE, "models/", "files/", "syncs.toml", ".gitignore"]
|
|
1554
|
+
for t in dict.fromkeys("agents" if t == "codex" else t for t in targets):
|
|
1555
|
+
written += _render(root, t)
|
|
1556
|
+
if seen is not None:
|
|
1557
|
+
level = (seen.get("scopes") or {}).get(dish) or (seen.get("scopes") or {}).get("*")
|
|
1558
|
+
level = level.get("level") if isinstance(level, dict) else level
|
|
1559
|
+
check = f"the key ({seen.get('hand')}) reaches {dish} at level {level}" if level else f"the key ({seen.get('hand')}) does not reach {dish}; ovs whoami lists what it does reach"
|
|
1560
|
+
elif check is None:
|
|
1561
|
+
check = "not logged in yet: run ovs login (or export OVERSCORE_KEY=ovs_...), then ovs whoami"
|
|
1562
|
+
out = {"project": str(root), "url": url, "dish": dish, "for": targets, "written": written, "key": check}
|
|
1563
|
+
return out, (f"{root}: an Overscore project on {url}, dish {dish}\nwrote {', '.join(written)}\n{check}\n"
|
|
1564
|
+
f"next: ovs pull {dish}.<table>, then write models/<name>/model.py and model.toml (the overscore-model skill has the steps)")
|
|
1565
|
+
|
|
1566
|
+
|
|
1567
|
+
LAPTOP = {"login": _login, "logout": _logout, "init": _init, "pull": _pull, "model": _model_cmd, "uploads": _uploads, "upload": _upload,
|
|
1568
|
+
"destinations": _destinations, "destination": _destination_cmd, "syncs": _syncs, "sync": _sync_cmd}
|
|
1569
|
+
|
|
1570
|
+
|
|
1571
|
+
class _Parser(argparse.ArgumentParser):
|
|
1572
|
+
"""A usage mistake is a failure like any other: JSON for an agent, exit 1."""
|
|
1573
|
+
def error(self, message):
|
|
1574
|
+
raise Fail(f"{self.prog}: {message}", data={"usage": self.format_usage().strip()})
|
|
1575
|
+
|
|
1576
|
+
|
|
1577
|
+
def _version_number(v: str) -> int:
|
|
1578
|
+
try:
|
|
1579
|
+
return int(v.strip().lstrip("vV"))
|
|
1580
|
+
except ValueError:
|
|
1581
|
+
raise argparse.ArgumentTypeError(f"a version is a number, like 3 or v3: {v!r}") from None
|
|
1582
|
+
|
|
1583
|
+
|
|
1584
|
+
def _check_address(url: str, *, given: bool) -> str:
|
|
1585
|
+
"""Where the key may go. Only over https (localhost aside), and to an address a project file names only when it
|
|
1586
|
+
is Overscore's own: a cloned repo's overscore.toml must not be able to send the key somewhere else.
|
|
1587
|
+
Returns the address rebuilt from what was checked, so the request goes where the check looked."""
|
|
1588
|
+
from urllib.parse import urlparse
|
|
1589
|
+
u = urlparse(url.strip())
|
|
1590
|
+
if not u.netloc or any(ch in u.netloc for ch in "\\@ \t") or u.query or u.fragment or "\\" in u.path:
|
|
1591
|
+
raise Fail(f"{url!r} is not a plain address like https://app.overscore.tech")
|
|
1592
|
+
try:
|
|
1593
|
+
port = u.port
|
|
1594
|
+
except ValueError:
|
|
1595
|
+
raise Fail(f"{url!r} has a port that is not a number") from None
|
|
1596
|
+
host = (u.hostname or "").lower()
|
|
1597
|
+
local = host in ("localhost", "127.0.0.1", "::1")
|
|
1598
|
+
if u.scheme != "https" and not (u.scheme == "http" and local):
|
|
1599
|
+
raise Fail(f"the key only goes over https (or to localhost); {url} is not")
|
|
1600
|
+
if not given and not local and host != "overscore.tech" and not host.endswith(".overscore.tech"):
|
|
1601
|
+
raise Fail(f"overscore.toml says to send the key to {url}, which is not an Overscore address. "
|
|
1602
|
+
f"If it is yours, say so for this shell: export OVERSCORE_URL={url}")
|
|
1603
|
+
return f"{u.scheme}://{host if ':' not in host else '[' + host + ']'}{f':{port}' if port else ''}{u.path.rstrip('/')}"
|
|
1604
|
+
|
|
1605
|
+
|
|
1606
|
+
def _version() -> str:
|
|
1607
|
+
try:
|
|
1608
|
+
from importlib.metadata import version
|
|
1609
|
+
return version("overscore-cli")
|
|
1610
|
+
except Exception: # noqa: BLE001
|
|
1611
|
+
return "dev"
|
|
1612
|
+
|
|
1613
|
+
|
|
1614
|
+
def main(argv: list[str] | None = None) -> int:
|
|
1615
|
+
p = _Parser(prog="ovs", description="Overscore command line")
|
|
1616
|
+
p.add_argument("--url", default=None, help=f"default: OVERSCORE_URL, else the url in overscore.toml, else {DEFAULT_URL}")
|
|
1617
|
+
p.add_argument("--key", default=os.environ.get("OVERSCORE_KEY"))
|
|
1618
|
+
p.add_argument("-o", "--output", choices=["json", "text"], default=None, help="default: text on a terminal, json otherwise")
|
|
1619
|
+
p.add_argument("--version", action="version", version=f"ovs {_version()}")
|
|
1620
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
1621
|
+
sub.add_parser("guide"); sub.add_parser("whoami"); sub.add_parser("tables"); sub.add_parser("index")
|
|
1622
|
+
x = sub.add_parser("describe"); x.add_argument("dish"); x.add_argument("table")
|
|
1623
|
+
x = sub.add_parser("search"); x.add_argument("text")
|
|
1624
|
+
x = sub.add_parser("query"); x.add_argument("sql", nargs="?"); x.add_argument("--max-rows", type=int, default=200)
|
|
1625
|
+
x = sub.add_parser("insert"); x.add_argument("dish"); x.add_argument("table"); x.add_argument("file", help="json file, or - for stdin")
|
|
1626
|
+
x = sub.add_parser("load"); x.add_argument("dish"); x.add_argument("table"); x.add_argument("file"); x.add_argument("--tab"); x.add_argument("--skip", type=int)
|
|
1627
|
+
x = sub.add_parser("notes"); x.add_argument("dish")
|
|
1628
|
+
x = sub.add_parser("note"); x.add_argument("dish"); x.add_argument("name"); x.add_argument("body", nargs="?"); x.add_argument("--title")
|
|
1629
|
+
x = sub.add_parser("models"); x.add_argument("dish")
|
|
1630
|
+
x = sub.add_parser("run"); x.add_argument("dish"); x.add_argument("--only", nargs="*")
|
|
1631
|
+
x = sub.add_parser("test"); x.add_argument("dish")
|
|
1632
|
+
x = sub.add_parser("signoff"); x.add_argument("dish"); x.add_argument("table"); x.add_argument("description", nargs="?"); x.add_argument("--off", action="store_true")
|
|
1633
|
+
x = sub.add_parser("runs"); x.add_argument("-n", type=int, default=20)
|
|
1634
|
+
x = sub.add_parser("run-log"); x.add_argument("id", type=int)
|
|
1635
|
+
x = sub.add_parser("export"); x.add_argument("dish"); x.add_argument("name"); x.add_argument("sql"); x.add_argument("--format", default="csv")
|
|
1636
|
+
sub.add_parser("connectors")
|
|
1637
|
+
x = sub.add_parser("how"); x.add_argument("name", help="stripe, shopify, quickbooks, postgres")
|
|
1638
|
+
x = sub.add_parser("connect"); x.add_argument("name"); x.add_argument("secrets", nargs="*", help="key=value, see ovs connectors")
|
|
1639
|
+
x.add_argument("--dish"); x.add_argument("--only", nargs="*"); x.add_argument("--no-run", action="store_true")
|
|
1640
|
+
x = sub.add_parser("link"); x.add_argument("dish"); x.add_argument("dsn"); x.add_argument("--copy", action="store_true")
|
|
1641
|
+
x.add_argument("--tables", nargs="*"); x.add_argument("--schema", default="public"); x.add_argument("--every", default="every 60m")
|
|
1642
|
+
sub.add_parser("links")
|
|
1643
|
+
x = sub.add_parser("hook"); x.add_argument("dish"); x.add_argument("table"); x.add_argument("--provider"); x.add_argument("--secret")
|
|
1644
|
+
sub.add_parser("hooks")
|
|
1645
|
+
x = sub.add_parser("context"); x.add_argument("q")
|
|
1646
|
+
x = sub.add_parser("propose"); x.add_argument("text"); x.add_argument("--source", default="cli")
|
|
1647
|
+
x = sub.add_parser("cards"); x.add_argument("--status", default="waiting")
|
|
1648
|
+
x = sub.add_parser("approve"); x.add_argument("id"); x.add_argument("--target"); x.add_argument("--key", dest="card_key"); x.add_argument("--set", nargs="*", help="column=value")
|
|
1649
|
+
x = sub.add_parser("ignore"); x.add_argument("id")
|
|
1650
|
+
x = sub.add_parser("facts"); x.add_argument("table")
|
|
1651
|
+
x = sub.add_parser("interview"); x.add_argument("area", nargs="?"); x.add_argument("--person"); x.add_argument("--resume")
|
|
1652
|
+
sub.add_parser("map")
|
|
1653
|
+
sub.add_parser("plan")
|
|
1654
|
+
x = sub.add_parser("fields"); x.add_argument("dish"); x.add_argument("table")
|
|
1655
|
+
x = sub.add_parser("field"); x.add_argument("dish"); x.add_argument("table"); x.add_argument("name"); x.add_argument("type"); x.add_argument("description"); x.add_argument("--allowed", nargs="*")
|
|
1656
|
+
x = sub.add_parser("mark"); x.add_argument("dish"); x.add_argument("table"); x.add_argument("keys", help="id, or id,id,id, or - for the table itself"); x.add_argument("values", nargs="+", help="field=value")
|
|
1657
|
+
x = sub.add_parser("marks"); x.add_argument("dish"); x.add_argument("table"); x.add_argument("row_key", nargs="?", metavar="key")
|
|
1658
|
+
x = sub.add_parser("claim"); x.add_argument("code", help="the claim code from the trial conversation")
|
|
1659
|
+
x = sub.add_parser("research"); x.add_argument("site")
|
|
1660
|
+
x = sub.add_parser("issues"); x.add_argument("--sweep", action="store_true")
|
|
1661
|
+
x = sub.add_parser("fix"); x.add_argument("id")
|
|
1662
|
+
x = sub.add_parser("hook-log"); x.add_argument("token"); x.add_argument("-n", type=int, default=20)
|
|
1663
|
+
x = sub.add_parser("unhook"); x.add_argument("token")
|
|
1664
|
+
x = sub.add_parser("copy"); x.add_argument("dish")
|
|
1665
|
+
x = sub.add_parser("unlink"); x.add_argument("dish")
|
|
1666
|
+
# the laptop side (CLAUDE_CODE_PLAN.md)
|
|
1667
|
+
x = sub.add_parser("login", help="approve this computer in the browser; the key goes in the keychain"); x.add_argument("--no-browser", action="store_true")
|
|
1668
|
+
sub.add_parser("logout", help="turn this computer's key off and forget it")
|
|
1669
|
+
x = sub.add_parser("init", help="make this folder an Overscore project"); x.add_argument("path", nargs="?", default=".")
|
|
1670
|
+
x.add_argument("--dish"); x.add_argument("--for", dest="agent", choices=AGENTS, default="claude", help="which agent's layout the rules and skills are written in")
|
|
1671
|
+
x = sub.add_parser("pull", help="a sample of a table as Parquet into .overscore/data/"); x.add_argument("tables", nargs="+", help="dish.table")
|
|
1672
|
+
x.add_argument("--rows", type=int, default=PULL_MAX); x.add_argument("--where", help="a SQL condition, to pull the rows that matter"); x.add_argument("--dish")
|
|
1673
|
+
m = sub.add_parser("model", help="a Python model in models/<name>/").add_subparsers(dest="sub", required=True)
|
|
1674
|
+
x = m.add_parser("push"); x.add_argument("name"); x.add_argument("--dish"); x.add_argument("--no-run", action="store_true")
|
|
1675
|
+
x = m.add_parser("run"); x.add_argument("name"); x.add_argument("--dish"); x.add_argument("--local", action="store_true", help="here, on the pulled samples")
|
|
1676
|
+
x.add_argument("--python", help="the interpreter for a local run (default: one built from requirements, else python3)")
|
|
1677
|
+
x = m.add_parser("log"); x.add_argument("name"); x.add_argument("--dish")
|
|
1678
|
+
sub.add_parser("uploads")
|
|
1679
|
+
u = sub.add_parser("upload", help="files a model needs, versioned").add_subparsers(dest="sub", required=True)
|
|
1680
|
+
u.add_parser("list")
|
|
1681
|
+
x = u.add_parser("put"); x.add_argument("name"); x.add_argument("file"); x.add_argument("--note")
|
|
1682
|
+
x = u.add_parser("live"); x.add_argument("name"); x.add_argument("version", type=_version_number, help="3, or v3")
|
|
1683
|
+
x = u.add_parser("rm"); x.add_argument("name")
|
|
1684
|
+
sub.add_parser("destinations")
|
|
1685
|
+
d = sub.add_parser("destination", help="where data goes").add_subparsers(dest="sub", required=True)
|
|
1686
|
+
d.add_parser("list")
|
|
1687
|
+
x = d.add_parser("add"); x.add_argument("kind", choices=["webhook", "hubspot"]); x.add_argument("title"); x.add_argument("webhook_url", nargs="?", metavar="url", help="webhook: their https:// URL")
|
|
1688
|
+
x.add_argument("--object", default="contacts", choices=["contacts", "companies", "deals", "tickets"]); x.add_argument("--id-property", help="hubspot: a unique property to match on (default: HubSpot's record id)")
|
|
1689
|
+
x.add_argument("--source", help="hubspot: the HubSpot source whose key to use, dish.name"); x.add_argument("--token", help="hubspot: - to read a key from stdin")
|
|
1690
|
+
for verb in ("check", "rm"):
|
|
1691
|
+
x = d.add_parser(verb); x.add_argument("dest", help="its name or id")
|
|
1692
|
+
sub.add_parser("syncs")
|
|
1693
|
+
s = sub.add_parser("sync", help="a table sent to a destination").add_subparsers(dest="sub", required=True)
|
|
1694
|
+
s.add_parser("list")
|
|
1695
|
+
x = s.add_parser("add"); x.add_argument("name"); x.add_argument("--from", dest="source", help="a table in the dish"); x.add_argument("--sql", help="a query, in place of --from")
|
|
1696
|
+
x.add_argument("--key", dest="id_column", help="the column that identifies the record"); x.add_argument("--to", help="the destination's name or id")
|
|
1697
|
+
x.add_argument("--columns", nargs="+", help="column or column=field; default every column but the key"); x.add_argument("--dish")
|
|
1698
|
+
g = x.add_mutually_exclusive_group(); g.add_argument("--after-change", action="store_true"); g.add_argument("--every", choices=SYNC_EVERY); g.add_argument("--manual", action="store_true")
|
|
1699
|
+
for verb in ("run", "show", "pause", "resume", "rm"):
|
|
1700
|
+
x = s.add_parser(verb); x.add_argument("name"); x.add_argument("--dish")
|
|
1701
|
+
# Accept -o / --url / --key anywhere on the line, not only before the command. Two commands have a --key of their
|
|
1702
|
+
# own (the id column of `sync add`, the row of `approve`): there it is theirs.
|
|
1703
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
1704
|
+
front, rest, i, words = [], [], 0, []
|
|
1705
|
+
while i < len(argv):
|
|
1706
|
+
t = argv[i]
|
|
1707
|
+
flag = t.split("=", 1)[0]
|
|
1708
|
+
theirs = flag == "--key" and (words[:2] == ["sync", "add"] or words[:1] == ["approve"])
|
|
1709
|
+
if flag in ("-o", "--output", "--url", "--key") and not theirs and ("=" in t or i + 1 < len(argv)):
|
|
1710
|
+
n = 1 if "=" in t else 2
|
|
1711
|
+
front += argv[i:i + n]; i += n
|
|
1712
|
+
else:
|
|
1713
|
+
if not t.startswith("-"):
|
|
1714
|
+
words.append(t)
|
|
1715
|
+
rest.append(t); i += 1
|
|
1716
|
+
wants = next((front[k + 1] for k in range(len(front) - 1) if front[k] in ("-o", "--output")), None) \
|
|
1717
|
+
or next((f.split("=", 1)[1] for f in front if f.startswith("--output=")), None)
|
|
1718
|
+
as_json = wants == "json" or (wants != "text" and not sys.stdout.isatty())
|
|
1719
|
+
try:
|
|
1720
|
+
a = p.parse_args(front + rest)
|
|
1721
|
+
as_json = a.output == "json" or (a.output is None and not sys.stdout.isatty())
|
|
1722
|
+
a.echo = not as_json
|
|
1723
|
+
a.url_given = a.url or os.environ.get("OVERSCORE_URL")
|
|
1724
|
+
a.project = _find_project()
|
|
1725
|
+
saved = _saved_login().get("address", "").rstrip("/")
|
|
1726
|
+
toml = _config(a.project).get("url")
|
|
1727
|
+
url = (a.url_given or (toml if toml and toml.rstrip("/") != DEFAULT_URL else None) or saved or DEFAULT_URL).rstrip("/")
|
|
1728
|
+
if not a.key and a.cmd not in ("login", "logout"):
|
|
1729
|
+
a.key = _saved_key(url) # the key ovs login kept for this address, if any
|
|
1730
|
+
keyless = a.cmd in ("guide", "init", "login", "logout") or (a.cmd == "model" and a.sub == "run" and a.local)
|
|
1731
|
+
if not a.key and not keyless:
|
|
1732
|
+
raise Fail("not logged in: run ovs login (or export OVERSCORE_KEY=ovs_..., never written into a file)")
|
|
1733
|
+
talks = a.key and a.cmd not in ("guide", "login", "logout") and not (a.cmd == "model" and a.sub == "run" and a.local)
|
|
1734
|
+
if talks and a.cmd != "init": # init checks the address it will write
|
|
1735
|
+
url = _check_address(url, given=bool(a.url_given) or (bool(saved) and url == saved))
|
|
1736
|
+
data, text = run(a, Client(url, a.key) if talks else None)
|
|
1737
|
+
except Fail as e:
|
|
1738
|
+
err = {"success": False, "error": str(e), "status": e.status, **e.data}
|
|
1739
|
+
if as_json:
|
|
1740
|
+
print(json.dumps(err, default=str))
|
|
1741
|
+
else:
|
|
1742
|
+
if e.data.get("saved"):
|
|
1743
|
+
print(f"saved {e.data.get('model')}")
|
|
1744
|
+
for n in e.data.get("notes") or []:
|
|
1745
|
+
print(f"note: {n}")
|
|
1746
|
+
log = e.data.get("log")
|
|
1747
|
+
if isinstance(log, str) and log:
|
|
1748
|
+
print((f"run {e.data['run_id']}:\n" if e.data.get("run_id") else "") + log.rstrip())
|
|
1749
|
+
elif log and e.data.get("where") != "local": # a local run's log was shown as it came
|
|
1750
|
+
print("\n".join(" " + l for l in log))
|
|
1751
|
+
for l in e.data.get("stderr") or []:
|
|
1752
|
+
print(" " + l)
|
|
1753
|
+
if e.data.get("usage"):
|
|
1754
|
+
print(e.data["usage"])
|
|
1755
|
+
print(f"error: {e}" + (f" (HTTP {e.status})" if e.status else ""))
|
|
1756
|
+
return 1
|
|
1757
|
+
if as_json:
|
|
1758
|
+
out = data if isinstance(data, dict) else {"result": data}
|
|
1759
|
+
print(json.dumps({"success": True, **out}, default=str))
|
|
1760
|
+
else:
|
|
1761
|
+
print(text)
|
|
1762
|
+
return 0
|
|
1763
|
+
|
|
1764
|
+
|
|
1765
|
+
if __name__ == "__main__":
|
|
1766
|
+
sys.exit(main())
|