passiveworkers 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- council/__init__.py +1 -0
- council/artifacts.py +161 -0
- council/batch.py +84 -0
- council/cli.py +54 -0
- council/coordinator.py +133 -0
- council/crypto.py +133 -0
- council/fidelity.py +197 -0
- council/judge.py +393 -0
- council/ledger.py +230 -0
- council/library.py +431 -0
- council/local.py +228 -0
- council/mcp_server.py +87 -0
- council/net/__init__.py +1 -0
- council/net/agent.py +231 -0
- council/net/app.py +390 -0
- council/net/baseline.py +86 -0
- council/net/config.py +79 -0
- council/net/coordinator_app.py +370 -0
- council/net/dashboard.py +111 -0
- council/net/store.py +964 -0
- council/net/submit.py +102 -0
- council/operator.py +412 -0
- council/research.py +520 -0
- council/researcher.py +300 -0
- council/retrieval.py +80 -0
- council/run_demo.py +175 -0
- council/sanitize.py +78 -0
- council/serve.py +183 -0
- council/trust.py +168 -0
- council/worker.py +123 -0
- passiveworkers-0.1.0.dist-info/METADATA +269 -0
- passiveworkers-0.1.0.dist-info/RECORD +36 -0
- passiveworkers-0.1.0.dist-info/WHEEL +5 -0
- passiveworkers-0.1.0.dist-info/entry_points.txt +2 -0
- passiveworkers-0.1.0.dist-info/licenses/LICENSE +21 -0
- passiveworkers-0.1.0.dist-info/top_level.txt +1 -0
council/net/store.py
ADDED
|
@@ -0,0 +1,964 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
council/net/store.py — SQLite persistence + orchestration state (hardened)
|
|
4
|
+
=========================================================================
|
|
5
|
+
Holds nodes, jobs, tasks, and the (reused, tested) credit Ledger. All mutations AND
|
|
6
|
+
reads go through one re-entrant lock so the FastAPI thread pool can't race.
|
|
7
|
+
|
|
8
|
+
Security/correctness invariants (see docs/DECISIONS + the M4 hardening review):
|
|
9
|
+
• Per-node SECRET: register mints a secret (returned once, only its hash stored); node
|
|
10
|
+
operations are authenticated by that secret, and a node can only complete its OWN tasks.
|
|
11
|
+
• Settle is FAIL-CLOSED: the ledger is settled FIRST; only on success do we write scores,
|
|
12
|
+
reputation, and 'done'. An over-budget job fails cleanly instead of stranding/corrupting.
|
|
13
|
+
• Scores are sanitized: non-finite (inf/NaN) or out-of-range judge scores → 0; an empty or
|
|
14
|
+
errored answer scores 0 and earns no reputation. min(10, NaN)==10, so isfinite comes first.
|
|
15
|
+
• A REAPER thread fails jobs whose assigned node went stale or that exceed the run deadline,
|
|
16
|
+
so a dead worker/judge can never wedge a job forever.
|
|
17
|
+
|
|
18
|
+
Job lifecycle:
|
|
19
|
+
submit → N `answer` tasks → all answers done → one `judge` task → settle → done.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import hashlib
|
|
25
|
+
import json
|
|
26
|
+
import math
|
|
27
|
+
import os
|
|
28
|
+
import secrets as _secrets
|
|
29
|
+
import sqlite3
|
|
30
|
+
import threading
|
|
31
|
+
import time
|
|
32
|
+
import uuid
|
|
33
|
+
from typing import Any, Optional
|
|
34
|
+
|
|
35
|
+
from council.ledger import Account, InsufficientCredit, Ledger
|
|
36
|
+
from council.net.config import CONFIG, JOB_TYPES
|
|
37
|
+
from council.sanitize import sanitize_brief
|
|
38
|
+
|
|
39
|
+
_now = time.time # server runtime (not a workflow script)
|
|
40
|
+
_FIELD_MAX = 80 # cap node/owner string lengths (defense-in-depth vs. abuse)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _hash(secret: str) -> str:
|
|
44
|
+
return hashlib.sha256(secret.encode()).hexdigest()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _clip(s: Any) -> str:
|
|
48
|
+
return str(s if s is not None else "")[:_FIELD_MAX]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class Store:
|
|
52
|
+
def __init__(self, path: str = None):
|
|
53
|
+
self.lock = threading.RLock()
|
|
54
|
+
self.conn = sqlite3.connect(path or CONFIG.db_path, check_same_thread=False)
|
|
55
|
+
self.conn.row_factory = sqlite3.Row
|
|
56
|
+
self._init_schema()
|
|
57
|
+
self.ledger = self._load_ledger()
|
|
58
|
+
# Reaper: fail stuck jobs so a dead node can't wedge the queue forever.
|
|
59
|
+
self._stop = threading.Event()
|
|
60
|
+
self._reaper = threading.Thread(target=self._reap_loop, daemon=True, name="pw-reaper")
|
|
61
|
+
self._reaper.start()
|
|
62
|
+
|
|
63
|
+
# ------------------------------------------------------------------ schema
|
|
64
|
+
def _init_schema(self) -> None:
|
|
65
|
+
self.conn.executescript(
|
|
66
|
+
"""
|
|
67
|
+
CREATE TABLE IF NOT EXISTS nodes(
|
|
68
|
+
node_id TEXT PRIMARY KEY, name TEXT, country TEXT, owner TEXT,
|
|
69
|
+
answer_model TEXT, lens TEXT, can_judge INT, judge_model TEXT,
|
|
70
|
+
profile TEXT, last_seen REAL, load REAL, status TEXT, ip TEXT, secret_hash TEXT,
|
|
71
|
+
machine_id TEXT);
|
|
72
|
+
CREATE TABLE IF NOT EXISTS jobs(
|
|
73
|
+
job_id TEXT PRIMARY KEY, asker TEXT, question TEXT, status TEXT,
|
|
74
|
+
created REAL, merged TEXT, receipt TEXT, error TEXT, council TEXT);
|
|
75
|
+
CREATE TABLE IF NOT EXISTS users(
|
|
76
|
+
handle TEXT PRIMARY KEY, secret_hash TEXT, created REAL);
|
|
77
|
+
CREATE TABLE IF NOT EXISTS feedback(
|
|
78
|
+
job_id TEXT PRIMARY KEY, verdict TEXT, who TEXT, created REAL);
|
|
79
|
+
CREATE TABLE IF NOT EXISTS tasks(
|
|
80
|
+
task_id TEXT PRIMARY KEY, job_id TEXT, type TEXT, node_id TEXT,
|
|
81
|
+
status TEXT, payload TEXT, result TEXT, worker_id TEXT, owner TEXT,
|
|
82
|
+
lens TEXT, country TEXT, model TEXT, created REAL, score REAL, claimed_at REAL);
|
|
83
|
+
CREATE TABLE IF NOT EXISTS ledger(id INTEGER PRIMARY KEY, data TEXT);
|
|
84
|
+
"""
|
|
85
|
+
)
|
|
86
|
+
# one row per (hash, job_id): content-addressed within a job, but each job keeps its
|
|
87
|
+
# own copy so cross-job content collisions never strand a second asker (D22 review).
|
|
88
|
+
self.conn.execute(
|
|
89
|
+
"CREATE TABLE IF NOT EXISTS blobs("
|
|
90
|
+
"hash TEXT, job_id TEXT, data BLOB, created REAL, PRIMARY KEY(hash, job_id))")
|
|
91
|
+
# which (asker, operator) pairs have ALREADY moved reputation — anti-farming (D24 review):
|
|
92
|
+
# one rater can lift a given operator's gate-average at most once.
|
|
93
|
+
self.conn.execute(
|
|
94
|
+
"CREATE TABLE IF NOT EXISTS rater_pairs(asker TEXT, operator TEXT, PRIMARY KEY(asker, operator))")
|
|
95
|
+
# migrations (ALTER TABLE on boot — re-installs must never wipe the DB)
|
|
96
|
+
cols = {r["name"] for r in self.conn.execute("PRAGMA table_info(jobs)")}
|
|
97
|
+
if "baseline" not in cols: # independent single-model baseline (council/net/baseline.py)
|
|
98
|
+
self.conn.execute("ALTER TABLE jobs ADD COLUMN baseline TEXT")
|
|
99
|
+
if "pool" not in cols: # per-job worker pool (responder dial: cost scales with minds)
|
|
100
|
+
self.conn.execute("ALTER TABLE jobs ADD COLUMN pool REAL")
|
|
101
|
+
if "type" not in cols: # job type — async work marketplace (D13); null/legacy = chat
|
|
102
|
+
self.conn.execute("ALTER TABLE jobs ADD COLUMN type TEXT")
|
|
103
|
+
self.conn.commit()
|
|
104
|
+
|
|
105
|
+
# ------------------------------------------------------------------ ledger persistence
|
|
106
|
+
def _load_ledger(self) -> Ledger:
|
|
107
|
+
row = self.conn.execute("SELECT data FROM ledger WHERE id=1").fetchone()
|
|
108
|
+
led = Ledger()
|
|
109
|
+
if row:
|
|
110
|
+
d = json.loads(row["data"])
|
|
111
|
+
led._granted_total = d.get("granted", 0.0)
|
|
112
|
+
led._job_count = d.get("jobs", 0)
|
|
113
|
+
for a in d.get("accounts", []):
|
|
114
|
+
led.accounts[a["user_id"]] = Account(**a)
|
|
115
|
+
return led
|
|
116
|
+
|
|
117
|
+
def _save_ledger(self) -> None:
|
|
118
|
+
d = {
|
|
119
|
+
"granted": self.ledger._granted_total,
|
|
120
|
+
"jobs": self.ledger._job_count,
|
|
121
|
+
"accounts": [vars(a) for a in self.ledger.accounts.values()],
|
|
122
|
+
}
|
|
123
|
+
self.conn.execute(
|
|
124
|
+
"INSERT INTO ledger(id, data) VALUES(1, ?) "
|
|
125
|
+
"ON CONFLICT(id) DO UPDATE SET data=excluded.data",
|
|
126
|
+
(json.dumps(d),),
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
# ------------------------------------------------------------------ nodes
|
|
130
|
+
def register_node(self, body: dict, ip: str = "") -> dict:
|
|
131
|
+
"""Returns {node_id, node_secret}. The secret is shown ONCE; only its hash is stored."""
|
|
132
|
+
with self.lock:
|
|
133
|
+
node_id = str(uuid.uuid4())
|
|
134
|
+
secret = _secrets.token_urlsafe(24)
|
|
135
|
+
self.ledger.open_account(_clip(body["owner"]))
|
|
136
|
+
self.conn.execute(
|
|
137
|
+
"INSERT INTO nodes VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
138
|
+
(
|
|
139
|
+
node_id, _clip(body.get("name", "node")), _clip(body.get("country", "?")),
|
|
140
|
+
_clip(body["owner"]), _clip(body.get("answer_model", "")),
|
|
141
|
+
_clip(body.get("lens", "neutral")), int(bool(body.get("can_judge", False))),
|
|
142
|
+
_clip(body.get("judge_model", "")), json.dumps(body.get("profile", {})),
|
|
143
|
+
_now(), 0.0, "online", ip, _hash(secret), _clip(body.get("machine_id", "?")),
|
|
144
|
+
),
|
|
145
|
+
)
|
|
146
|
+
self._save_ledger()
|
|
147
|
+
self.conn.commit()
|
|
148
|
+
return {"node_id": node_id, "node_secret": secret}
|
|
149
|
+
|
|
150
|
+
def node_for_secret(self, secret: str) -> Optional[str]:
|
|
151
|
+
"""Resolve the authenticated node_id from its secret (None if unknown)."""
|
|
152
|
+
if not secret:
|
|
153
|
+
return None
|
|
154
|
+
with self.lock:
|
|
155
|
+
row = self.conn.execute(
|
|
156
|
+
"SELECT node_id FROM nodes WHERE secret_hash=?", (_hash(secret),)).fetchone()
|
|
157
|
+
return row["node_id"] if row else None
|
|
158
|
+
|
|
159
|
+
def heartbeat(self, node_id: str, load: float = 0.0) -> bool:
|
|
160
|
+
with self.lock:
|
|
161
|
+
cur = self.conn.execute(
|
|
162
|
+
"UPDATE nodes SET last_seen=?, load=?, status='online' WHERE node_id=?",
|
|
163
|
+
(_now(), load, node_id))
|
|
164
|
+
self.conn.commit()
|
|
165
|
+
return cur.rowcount > 0
|
|
166
|
+
|
|
167
|
+
def online_nodes(self, judge_only: bool = False) -> list[sqlite3.Row]:
|
|
168
|
+
with self.lock:
|
|
169
|
+
cutoff = _now() - CONFIG.node_ttl_s
|
|
170
|
+
q = "SELECT * FROM nodes WHERE last_seen >= ?"
|
|
171
|
+
if judge_only:
|
|
172
|
+
q += " AND can_judge = 1"
|
|
173
|
+
return list(self.conn.execute(q + " ORDER BY last_seen DESC", (cutoff,)))
|
|
174
|
+
|
|
175
|
+
def get_node(self, node_id: str) -> Optional[sqlite3.Row]:
|
|
176
|
+
with self.lock:
|
|
177
|
+
return self.conn.execute("SELECT * FROM nodes WHERE node_id=?", (node_id,)).fetchone()
|
|
178
|
+
|
|
179
|
+
# ------------------------------------------------------------------ users (askers)
|
|
180
|
+
def register_user(self, handle: str) -> dict:
|
|
181
|
+
handle = _clip(handle).strip() or "anon"
|
|
182
|
+
with self.lock:
|
|
183
|
+
if self.conn.execute("SELECT 1 FROM users WHERE handle=?", (handle,)).fetchone():
|
|
184
|
+
return {"error": "handle taken"}
|
|
185
|
+
secret = _secrets.token_urlsafe(24)
|
|
186
|
+
self.conn.execute("INSERT INTO users(handle, secret_hash, created) VALUES(?,?,?)",
|
|
187
|
+
(handle, _hash(secret), _now()))
|
|
188
|
+
self.ledger.open_account(handle)
|
|
189
|
+
self._save_ledger()
|
|
190
|
+
self.conn.commit()
|
|
191
|
+
return {"handle": handle, "user_secret": secret, **self.user_balance(handle)}
|
|
192
|
+
|
|
193
|
+
def user_for_secret(self, secret: str) -> Optional[str]:
|
|
194
|
+
if not secret:
|
|
195
|
+
return None
|
|
196
|
+
with self.lock:
|
|
197
|
+
row = self.conn.execute("SELECT handle FROM users WHERE secret_hash=?",
|
|
198
|
+
(_hash(secret),)).fetchone()
|
|
199
|
+
return row["handle"] if row else None
|
|
200
|
+
|
|
201
|
+
def user_balance(self, handle: str) -> dict:
|
|
202
|
+
a = self.ledger.accounts.get(handle)
|
|
203
|
+
if not a:
|
|
204
|
+
return {"handle": handle, "balance": 0.0, "reputation": 0.0, "helped": 0, "asked": 0}
|
|
205
|
+
return {"handle": handle, "balance": round(a.balance, 1), "reputation": a.avg_quality,
|
|
206
|
+
"helped": a.jobs_helped, "asked": a.jobs_asked}
|
|
207
|
+
|
|
208
|
+
def record_feedback(self, job_id: str, verdict: str, who: str = "") -> bool:
|
|
209
|
+
if verdict not in ("council", "single", "tie"):
|
|
210
|
+
return False
|
|
211
|
+
with self.lock:
|
|
212
|
+
if not self.conn.execute("SELECT 1 FROM jobs WHERE job_id=?", (job_id,)).fetchone():
|
|
213
|
+
return False
|
|
214
|
+
self.conn.execute(
|
|
215
|
+
"INSERT INTO feedback(job_id, verdict, who, created) VALUES(?,?,?,?) "
|
|
216
|
+
"ON CONFLICT(job_id) DO UPDATE SET verdict=excluded.verdict, who=excluded.who",
|
|
217
|
+
(job_id, verdict, _clip(who), _now()))
|
|
218
|
+
self.conn.commit()
|
|
219
|
+
return True
|
|
220
|
+
|
|
221
|
+
def metrics(self) -> dict:
|
|
222
|
+
with self.lock:
|
|
223
|
+
by = {r["verdict"]: r["c"] for r in
|
|
224
|
+
self.conn.execute("SELECT verdict, COUNT(*) c FROM feedback GROUP BY verdict")}
|
|
225
|
+
council, single, tie = by.get("council", 0), by.get("single", 0), by.get("tie", 0)
|
|
226
|
+
decisive = council + single
|
|
227
|
+
return {"council": council, "single": single, "tie": tie,
|
|
228
|
+
"total": council + single + tie,
|
|
229
|
+
"council_win_rate": round(council / decisive, 3) if decisive else None}
|
|
230
|
+
|
|
231
|
+
# ------------------------------------------------------------------ jobs / tasks
|
|
232
|
+
@staticmethod
|
|
233
|
+
def _meets(n: Any, requires: Optional[dict]) -> bool:
|
|
234
|
+
"""Capability match (D15 v1): required model installed, minimum RAM. Nodes report
|
|
235
|
+
their profile at register; jobs may declare `requires` — not all tasks are open
|
|
236
|
+
to all nodes."""
|
|
237
|
+
if not requires:
|
|
238
|
+
return True
|
|
239
|
+
try:
|
|
240
|
+
prof = json.loads(n["profile"]) if isinstance(n["profile"], str) else (n["profile"] or {})
|
|
241
|
+
except Exception:
|
|
242
|
+
prof = {}
|
|
243
|
+
want = requires.get("model")
|
|
244
|
+
if want and want != n["answer_model"] and want not in (prof.get("models") or []):
|
|
245
|
+
return False
|
|
246
|
+
min_ram = requires.get("min_ram_gb")
|
|
247
|
+
if min_ram:
|
|
248
|
+
try:
|
|
249
|
+
if float(prof.get("ram_gb") or 0) < float(min_ram):
|
|
250
|
+
return False
|
|
251
|
+
except (TypeError, ValueError):
|
|
252
|
+
return False
|
|
253
|
+
return True
|
|
254
|
+
|
|
255
|
+
def create_job(self, asker: str, question: str, minds: int | None = None,
|
|
256
|
+
job_type: str = "chat", items: Optional[list] = None,
|
|
257
|
+
requires: Optional[dict] = None, fetch: bool = False,
|
|
258
|
+
context: str = "", encrypt_to: str = "") -> dict:
|
|
259
|
+
with self.lock:
|
|
260
|
+
asker = _clip(asker)
|
|
261
|
+
# The single networked choke point for the brief/instruction (D26): scrub invisible/bidi
|
|
262
|
+
# injection vectors + length-bound here so EVERY downstream prompt (worker, researcher,
|
|
263
|
+
# judge.score/merge/deliberate/compile_report/spot_check) and the report get a clean value,
|
|
264
|
+
# regardless of which API endpoint or caller created the job.
|
|
265
|
+
question = sanitize_brief(question)
|
|
266
|
+
context = sanitize_brief(context) if context else context
|
|
267
|
+
if job_type not in JOB_TYPES:
|
|
268
|
+
job_type = "chat"
|
|
269
|
+
# assisted (D21): human-in-the-loop work. NOT pre-assigned — it's an OPEN offer
|
|
270
|
+
# any consenting, capable operator may claim, do (with their own AI or by hand),
|
|
271
|
+
# and deliver. No autonomous computer-use by us; the human is the agent.
|
|
272
|
+
if job_type == "assisted":
|
|
273
|
+
return self._create_assisted(asker, question, context, requires, encrypt_to)
|
|
274
|
+
# Answer-workers = online nodes that declare a model AND meet the job's
|
|
275
|
+
# capability requirements; prefer higher reputation.
|
|
276
|
+
candidates = [n for n in self.online_nodes()
|
|
277
|
+
if n["answer_model"] and self._meets(n, requires)]
|
|
278
|
+
|
|
279
|
+
def _rep(n):
|
|
280
|
+
acct = self.ledger.accounts.get(n["owner"])
|
|
281
|
+
return acct.avg_quality if acct else 0.0
|
|
282
|
+
|
|
283
|
+
candidates.sort(key=lambda n: (_rep(n), n["last_seen"]), reverse=True)
|
|
284
|
+
# Responder dial: the asker picks how many minds; cost scales with the count
|
|
285
|
+
# (per-mind price = worker_pool / fleet_size, so the default stays unchanged).
|
|
286
|
+
n_minds = max(1, min(int(minds), len(candidates))) if minds else CONFIG.fleet_size
|
|
287
|
+
workers = candidates[:n_minds]
|
|
288
|
+
pool = round(CONFIG.worker_pool / CONFIG.fleet_size * len(workers)
|
|
289
|
+
* JOB_TYPES[job_type]["pool_mult"], 4)
|
|
290
|
+
job_id = str(uuid.uuid4())
|
|
291
|
+
|
|
292
|
+
if not workers:
|
|
293
|
+
why = ("no online node meets the job's requirements"
|
|
294
|
+
if requires else "no worker nodes online")
|
|
295
|
+
self.conn.execute(
|
|
296
|
+
"INSERT INTO jobs(job_id,asker,question,status,created,merged,receipt,error,council)"
|
|
297
|
+
" VALUES(?,?,?,?,?,?,?,?,?)",
|
|
298
|
+
(job_id, asker, question, "failed", _now(), None, None, why, None))
|
|
299
|
+
self.conn.commit()
|
|
300
|
+
return {"job_id": job_id, "status": "failed", "error": why}
|
|
301
|
+
|
|
302
|
+
# shard_map: split the items across the selected workers (round-robin, keeping
|
|
303
|
+
# each item's global index so the merged output preserves input order).
|
|
304
|
+
shards: dict = {}
|
|
305
|
+
if job_type == "shard_map":
|
|
306
|
+
clean = [str(x).strip()[:2000] for x in (items or []) if str(x).strip()][:200]
|
|
307
|
+
if not clean:
|
|
308
|
+
self.conn.execute(
|
|
309
|
+
"INSERT INTO jobs(job_id,asker,question,status,created,merged,receipt,error,council)"
|
|
310
|
+
" VALUES(?,?,?,?,?,?,?,?,?)",
|
|
311
|
+
(job_id, asker, question, "failed", _now(), None, None,
|
|
312
|
+
"batch job needs a non-empty items list", None))
|
|
313
|
+
self.conn.commit()
|
|
314
|
+
return {"job_id": job_id, "status": "failed",
|
|
315
|
+
"error": "batch job needs a non-empty items list"}
|
|
316
|
+
shards = {w["node_id"]: [] for w in workers}
|
|
317
|
+
for idx, it in enumerate(clean):
|
|
318
|
+
shards[workers[idx % len(workers)]["node_id"]].append({"i": idx, "item": it})
|
|
319
|
+
|
|
320
|
+
self.ledger.open_account(asker)
|
|
321
|
+
cost = self.ledger.quote(pool, CONFIG.judge_fee)
|
|
322
|
+
if not self.ledger.can_afford(asker, cost):
|
|
323
|
+
self.conn.execute(
|
|
324
|
+
"INSERT INTO jobs(job_id,asker,question,status,created,merged,receipt,error,council)"
|
|
325
|
+
" VALUES(?,?,?,?,?,?,?,?,?)",
|
|
326
|
+
(job_id, asker, question, "failed", _now(), None, None,
|
|
327
|
+
"insufficient credit — help on a job first", None))
|
|
328
|
+
self._save_ledger()
|
|
329
|
+
self.conn.commit()
|
|
330
|
+
return {"job_id": job_id, "status": "failed",
|
|
331
|
+
"error": "insufficient credit — help on a job first"}
|
|
332
|
+
|
|
333
|
+
self.conn.execute(
|
|
334
|
+
"INSERT INTO jobs(job_id,asker,question,status,created,merged,receipt,error,council,pool,type)"
|
|
335
|
+
" VALUES(?,?,?,?,?,?,?,?,?,?,?)",
|
|
336
|
+
(job_id, asker, question, "pending_answers", _now(), None, None, None, None, pool, job_type))
|
|
337
|
+
for n in workers:
|
|
338
|
+
payload = {"question": question, "job_type": job_type}
|
|
339
|
+
if job_type == "shard_map":
|
|
340
|
+
payload["shard"] = shards.get(n["node_id"], [])
|
|
341
|
+
payload["fetch"] = bool(fetch)
|
|
342
|
+
self.conn.execute(
|
|
343
|
+
"INSERT INTO tasks VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
344
|
+
(str(uuid.uuid4()), job_id, "answer", n["node_id"], "queued",
|
|
345
|
+
json.dumps(payload), None,
|
|
346
|
+
n["node_id"], n["owner"],
|
|
347
|
+
n["lens"], n["country"], n["answer_model"], _now(), None, None))
|
|
348
|
+
self._save_ledger() # persist the new asker account before acknowledging
|
|
349
|
+
self.conn.commit()
|
|
350
|
+
return {"job_id": job_id, "status": "pending_answers",
|
|
351
|
+
"assigned": [n["node_id"] for n in workers]}
|
|
352
|
+
|
|
353
|
+
# ------------------------------------------------------------------ assisted (D21)
|
|
354
|
+
def _meets_reputation(self, owner: str, requires: Optional[dict]) -> bool:
|
|
355
|
+
"""Reputation gate (D24): when an offer sets `min_reputation`, only operators whose
|
|
356
|
+
rating average meets it AND who have at least one rating qualify (newcomers take the
|
|
357
|
+
ungated offers — cold-start isn't blocked). FAIL CLOSED on a malformed threshold
|
|
358
|
+
(matches the capability gate _meets), so a fat-fingered gate never silently admits
|
|
359
|
+
unqualified operators."""
|
|
360
|
+
if not requires or "min_reputation" not in requires:
|
|
361
|
+
return True # genuinely ungated → open to everyone (incl. newcomers)
|
|
362
|
+
try:
|
|
363
|
+
need = float(requires["min_reputation"])
|
|
364
|
+
except (TypeError, ValueError):
|
|
365
|
+
return False # malformed gate → admit no one (fail closed)
|
|
366
|
+
if not math.isfinite(need):
|
|
367
|
+
return False
|
|
368
|
+
a = self.ledger.accounts.get(_clip(owner))
|
|
369
|
+
return bool(a and a.quality_n > 0 and a.avg_quality >= need)
|
|
370
|
+
|
|
371
|
+
def _create_assisted(self, asker: str, question: str, context: str,
|
|
372
|
+
requires: Optional[dict], encrypt_to: str = "") -> dict:
|
|
373
|
+
"""Create an OPEN assisted offer (called under self.lock from create_job)."""
|
|
374
|
+
job_id = str(uuid.uuid4())
|
|
375
|
+
# validate a reputation gate up front so a fat-fingered value surfaces to the asker
|
|
376
|
+
if requires and "min_reputation" in requires:
|
|
377
|
+
try:
|
|
378
|
+
mr = float(requires["min_reputation"])
|
|
379
|
+
bad = not math.isfinite(mr) or not (0 <= mr <= 10)
|
|
380
|
+
except (TypeError, ValueError):
|
|
381
|
+
bad = True
|
|
382
|
+
if bad:
|
|
383
|
+
return {"job_id": job_id, "status": "failed",
|
|
384
|
+
"error": "min_reputation must be a number 0-10"}
|
|
385
|
+
pool = round(CONFIG.worker_pool / CONFIG.fleet_size * JOB_TYPES["assisted"]["pool_mult"], 4)
|
|
386
|
+
self.ledger.open_account(asker)
|
|
387
|
+
# HOLD the reward in escrow now so it can't be spent before the operator delivers.
|
|
388
|
+
try:
|
|
389
|
+
self.ledger.hold(asker, pool)
|
|
390
|
+
except InsufficientCredit:
|
|
391
|
+
self.conn.execute(
|
|
392
|
+
"INSERT INTO jobs(job_id,asker,question,status,created,merged,receipt,error,council)"
|
|
393
|
+
" VALUES(?,?,?,?,?,?,?,?,?)",
|
|
394
|
+
(job_id, asker, question, "failed", _now(), None, None,
|
|
395
|
+
"insufficient credit — help on a job first", None))
|
|
396
|
+
self._save_ledger(); self.conn.commit()
|
|
397
|
+
return {"job_id": job_id, "status": "failed",
|
|
398
|
+
"error": "insufficient credit — help on a job first"}
|
|
399
|
+
self.conn.execute(
|
|
400
|
+
"INSERT INTO jobs(job_id,asker,question,status,created,merged,receipt,error,council,pool,type)"
|
|
401
|
+
" VALUES(?,?,?,?,?,?,?,?,?,?,?)",
|
|
402
|
+
(job_id, asker, question, "pending_assist", _now(), None, None, None, None, pool, "assisted"))
|
|
403
|
+
# ONE open task, no node_id — any consenting capable operator may claim it.
|
|
404
|
+
payload = {"question": question, "job_type": "assisted",
|
|
405
|
+
"context": (context or "")[:4000], "requires": requires or {}, "price": pool,
|
|
406
|
+
"encrypt_to": (encrypt_to or "")[:100]}
|
|
407
|
+
self.conn.execute(
|
|
408
|
+
"INSERT INTO tasks VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
409
|
+
(str(uuid.uuid4()), job_id, "assisted", None, "open",
|
|
410
|
+
json.dumps(payload), None, None, None, "", "", "", _now(), None, None))
|
|
411
|
+
self._save_ledger(); self.conn.commit()
|
|
412
|
+
return {"job_id": job_id, "status": "pending_assist", "price": pool}
|
|
413
|
+
|
|
414
|
+
def assisted_offers(self, node: dict) -> list:
|
|
415
|
+
"""Open assisted offers this operator's node is eligible for (capability-matched).
|
|
416
|
+
Returns the brief + BOUNDED context + price so the operator can give informed consent."""
|
|
417
|
+
with self.lock:
|
|
418
|
+
out = []
|
|
419
|
+
for t in self.conn.execute(
|
|
420
|
+
"SELECT * FROM tasks WHERE type='assisted' AND status='open' ORDER BY created"):
|
|
421
|
+
payload = json.loads(t["payload"]) if t["payload"] else {}
|
|
422
|
+
req = payload.get("requires") or None
|
|
423
|
+
if not self._meets(node, req) or not self._meets_reputation(node["owner"], req):
|
|
424
|
+
continue
|
|
425
|
+
out.append({"task_id": t["task_id"], "job_id": t["job_id"],
|
|
426
|
+
"brief": payload.get("question", ""),
|
|
427
|
+
"context": payload.get("context", ""),
|
|
428
|
+
"requires": payload.get("requires") or {},
|
|
429
|
+
"price": payload.get("price"),
|
|
430
|
+
"encrypt_to": payload.get("encrypt_to", ""),
|
|
431
|
+
"age_s": round(_now() - t["created"], 1)})
|
|
432
|
+
return out
|
|
433
|
+
|
|
434
|
+
def accept_assisted(self, task_id: str, node_id: str, owner: str) -> dict:
|
|
435
|
+
"""Operator consents to + claims an open offer (atomic, under the lock). Returns the
|
|
436
|
+
full brief+context. Guards: not your own offer (no self-deal), capability still met."""
|
|
437
|
+
with self.lock:
|
|
438
|
+
t = self.conn.execute(
|
|
439
|
+
"SELECT * FROM tasks WHERE task_id=? AND type='assisted'", (task_id,)).fetchone()
|
|
440
|
+
if not t:
|
|
441
|
+
return {"ok": False, "error": "no such assisted task"}
|
|
442
|
+
if t["status"] != "open":
|
|
443
|
+
return {"ok": False, "error": f"already {t['status']}"}
|
|
444
|
+
owner = _clip(owner)
|
|
445
|
+
if not owner:
|
|
446
|
+
# every assisted claim must bind to a non-empty operator identity — it's the
|
|
447
|
+
# handle an asker pins for out-of-band key trust (D25); an empty one has no anchor.
|
|
448
|
+
return {"ok": False, "error": "operator identity (owner) required to accept"}
|
|
449
|
+
job = self.conn.execute("SELECT asker FROM jobs WHERE job_id=?", (t["job_id"],)).fetchone()
|
|
450
|
+
if job and owner == job["asker"]:
|
|
451
|
+
return {"ok": False, "error": "cannot accept your own assisted offer"}
|
|
452
|
+
payload = json.loads(t["payload"]) if t["payload"] else {}
|
|
453
|
+
req = payload.get("requires") or None
|
|
454
|
+
node = self.get_node(node_id)
|
|
455
|
+
# Capability gate: only enforced when the offer actually sets requirements. An
|
|
456
|
+
# unregistered node can take a no-requirement task, but it can NEVER bypass a
|
|
457
|
+
# capability requirement (unknown node → cannot prove capability → ineligible).
|
|
458
|
+
if req and (node is None or not self._meets(dict(node), req)):
|
|
459
|
+
return {"ok": False, "error": "your node does not meet this offer's requirements"}
|
|
460
|
+
if not self._meets_reputation(owner, req):
|
|
461
|
+
return {"ok": False, "error": "this offer requires a higher operator reputation"}
|
|
462
|
+
self.ledger.open_account(owner) # operator must have an account to be paid
|
|
463
|
+
self.conn.execute(
|
|
464
|
+
"UPDATE tasks SET status='claimed', node_id=?, owner=?, claimed_at=? WHERE task_id=?",
|
|
465
|
+
(node_id, owner, _now(), task_id))
|
|
466
|
+
self.conn.execute("UPDATE jobs SET status='assisting' WHERE job_id=?", (t["job_id"],))
|
|
467
|
+
self._save_ledger()
|
|
468
|
+
self.conn.commit()
|
|
469
|
+
return {"ok": True, "task_id": task_id, "job_id": t["job_id"],
|
|
470
|
+
"brief": payload.get("question", ""), "context": payload.get("context", ""),
|
|
471
|
+
"encrypt_to": payload.get("encrypt_to", "")}
|
|
472
|
+
|
|
473
|
+
def deliver_assisted(self, task_id: str, node_id: str, deliverable: str,
|
|
474
|
+
signature: str = "", signer_pub: str = "") -> dict:
|
|
475
|
+
"""Operator returns the owned deliverable; settle (operator paid the pool, conserved)."""
|
|
476
|
+
with self.lock:
|
|
477
|
+
t = self.conn.execute(
|
|
478
|
+
"SELECT * FROM tasks WHERE task_id=? AND type='assisted'", (task_id,)).fetchone()
|
|
479
|
+
if not t:
|
|
480
|
+
return {"ok": False, "error": "no such assisted task"}
|
|
481
|
+
if t["node_id"] != node_id:
|
|
482
|
+
return {"ok": False, "error": "not your task"}
|
|
483
|
+
if t["status"] == "done":
|
|
484
|
+
return {"ok": False, "error": "already delivered"}
|
|
485
|
+
job = self.conn.execute("SELECT * FROM jobs WHERE job_id=?", (t["job_id"],)).fetchone()
|
|
486
|
+
# guard against settling a job the reaper already expired/failed (no double-pay,
|
|
487
|
+
# no charging an asker for an offer they were told had lapsed).
|
|
488
|
+
if job is None or job["status"] != "assisting":
|
|
489
|
+
return {"ok": False,
|
|
490
|
+
"error": f"offer no longer open for delivery ({job and job['status']})"}
|
|
491
|
+
# if it's a file artifact, refuse to pay unless every chunk was actually uploaded
|
|
492
|
+
from council.artifacts import read_artifact, verify_manifest
|
|
493
|
+
manifest = read_artifact(deliverable)
|
|
494
|
+
if manifest is not None:
|
|
495
|
+
if not verify_manifest(manifest) or not self.blobs_present(t["job_id"], manifest["chunks"]):
|
|
496
|
+
return {"ok": False, "error": "file incomplete — upload all chunks before delivering"}
|
|
497
|
+
result = {"text": deliverable}
|
|
498
|
+
if signature and signer_pub: # D23: operator's signature over the deliverable
|
|
499
|
+
result["signature"] = signature[:200]
|
|
500
|
+
result["signer_pub"] = signer_pub[:100]
|
|
501
|
+
result["_digest"] = self.result_digest(result)
|
|
502
|
+
pool = job["pool"] or 0.0
|
|
503
|
+
# release the escrow hold to the operator (asker was already debited at offer
|
|
504
|
+
# creation, so this can't fail on asker balance and pays exactly once).
|
|
505
|
+
self.ledger.release(t["owner"], pool)
|
|
506
|
+
acct = self.ledger.accounts.get(t["owner"])
|
|
507
|
+
if acct:
|
|
508
|
+
acct.jobs_helped += 1
|
|
509
|
+
receipt = {"job_id": t["job_id"], "asker_id": job["asker"], "total_cost": pool,
|
|
510
|
+
"payouts": {t["owner"]: pool}, "judge_fee": 0.0}
|
|
511
|
+
# score stays NULL — it's the asker's rating slot (set by rate_assisted, D24)
|
|
512
|
+
self.conn.execute("UPDATE tasks SET status='done', result=? WHERE task_id=?",
|
|
513
|
+
(json.dumps(result), task_id))
|
|
514
|
+
self.conn.execute("UPDATE jobs SET status='done', merged=?, receipt=? WHERE job_id=?",
|
|
515
|
+
(deliverable, json.dumps(receipt), t["job_id"]))
|
|
516
|
+
self._save_ledger(); self.conn.commit()
|
|
517
|
+
return {"ok": True, "job_id": t["job_id"]}
|
|
518
|
+
|
|
519
|
+
# ------------------------------------------------------------------ blobs (D22)
|
|
520
|
+
def put_blob(self, job_id: str, blob_hash: str, data: bytes) -> dict:
|
|
521
|
+
"""Store a content-addressed chunk for a job. Verifies the hash (content IS the
|
|
522
|
+
address), enforces per-chunk + per-job-total caps. Operators upload here before
|
|
523
|
+
delivering a manifest that references these blobs."""
|
|
524
|
+
import hashlib as _hl
|
|
525
|
+
if len(data) > 512 * 1024: # chunk cap (codec uses 256 KiB; allow headroom)
|
|
526
|
+
return {"ok": False, "error": "chunk too large"}
|
|
527
|
+
if _hl.sha256(data).hexdigest() != blob_hash:
|
|
528
|
+
return {"ok": False, "error": "hash does not match content"}
|
|
529
|
+
with self.lock:
|
|
530
|
+
# per-job total-bytes cap bounds a hostile operator filling the store (aligned to
|
|
531
|
+
# the codec's per-file cap, with headroom for a few files per job).
|
|
532
|
+
total = self.conn.execute(
|
|
533
|
+
"SELECT COALESCE(SUM(LENGTH(data)),0) s FROM blobs WHERE job_id=?",
|
|
534
|
+
(job_id,)).fetchone()["s"]
|
|
535
|
+
if total + len(data) > 200 * 1024 * 1024:
|
|
536
|
+
return {"ok": False, "error": "per-job storage cap reached"}
|
|
537
|
+
self.conn.execute(
|
|
538
|
+
"INSERT OR IGNORE INTO blobs(hash,job_id,data,created) VALUES(?,?,?,?)",
|
|
539
|
+
(blob_hash, job_id, data, _now()))
|
|
540
|
+
self.conn.commit()
|
|
541
|
+
# confirm it's actually stored for THIS job (don't report false success)
|
|
542
|
+
ok = self.conn.execute("SELECT 1 FROM blobs WHERE hash=? AND job_id=?",
|
|
543
|
+
(blob_hash, job_id)).fetchone() is not None
|
|
544
|
+
return {"ok": ok, "hash": blob_hash} if ok else {"ok": False, "error": "not stored"}
|
|
545
|
+
|
|
546
|
+
def blobs_present(self, job_id: str, hashes: list) -> bool:
|
|
547
|
+
"""True iff every hash is stored for this job (used to gate payment on full upload)."""
|
|
548
|
+
with self.lock:
|
|
549
|
+
for h in hashes:
|
|
550
|
+
if not self.conn.execute("SELECT 1 FROM blobs WHERE hash=? AND job_id=?",
|
|
551
|
+
(h, job_id)).fetchone():
|
|
552
|
+
return False
|
|
553
|
+
return True
|
|
554
|
+
|
|
555
|
+
def get_blob(self, job_id: str, blob_hash: str) -> Optional[bytes]:
|
|
556
|
+
"""Fetch a chunk that belongs to this job (job-scoped so one asker can't read
|
|
557
|
+
another job's blobs)."""
|
|
558
|
+
with self.lock:
|
|
559
|
+
row = self.conn.execute("SELECT data FROM blobs WHERE hash=? AND job_id=?",
|
|
560
|
+
(blob_hash, job_id)).fetchone()
|
|
561
|
+
return bytes(row["data"]) if row else None
|
|
562
|
+
|
|
563
|
+
def job_asker(self, job_id: str) -> Optional[str]:
|
|
564
|
+
with self.lock:
|
|
565
|
+
row = self.conn.execute("SELECT asker FROM jobs WHERE job_id=?", (job_id,)).fetchone()
|
|
566
|
+
return row["asker"] if row else None
|
|
567
|
+
|
|
568
|
+
def assisted_claimant(self, job_id: str) -> Optional[str]:
|
|
569
|
+
"""node_id of the operator who claimed this job's assisted task (for blob-upload auth)."""
|
|
570
|
+
with self.lock:
|
|
571
|
+
row = self.conn.execute(
|
|
572
|
+
"SELECT node_id FROM tasks WHERE job_id=? AND type='assisted'", (job_id,)).fetchone()
|
|
573
|
+
return row["node_id"] if row else None
|
|
574
|
+
|
|
575
|
+
def rate_assisted(self, job_id: str, asker: str, score: float) -> dict:
|
|
576
|
+
"""The asker rates a completed assisted deliverable (0-10). Feeds the operator's
|
|
577
|
+
reputation (the same quality signal as council judge scores). One rating per job."""
|
|
578
|
+
with self.lock:
|
|
579
|
+
job = self.conn.execute("SELECT * FROM jobs WHERE job_id=?", (job_id,)).fetchone()
|
|
580
|
+
if not job or (job["type"] or "") != "assisted":
|
|
581
|
+
return {"ok": False, "error": "not an assisted job"}
|
|
582
|
+
if _clip(asker) != job["asker"]:
|
|
583
|
+
return {"ok": False, "error": "only the asker can rate this job"}
|
|
584
|
+
if job["status"] != "done":
|
|
585
|
+
return {"ok": False, "error": "job not delivered yet"}
|
|
586
|
+
t = self.conn.execute(
|
|
587
|
+
"SELECT task_id, owner, score FROM tasks WHERE job_id=? AND type='assisted' LIMIT 1",
|
|
588
|
+
(job_id,)).fetchone()
|
|
589
|
+
if not t or not t["owner"]:
|
|
590
|
+
return {"ok": False, "error": "no operator to rate"}
|
|
591
|
+
if t["score"] is not None: # score stays NULL until rated → idempotent
|
|
592
|
+
return {"ok": False, "error": "already rated"}
|
|
593
|
+
s = self._sane_score(score)
|
|
594
|
+
self.conn.execute("UPDATE tasks SET score=? WHERE task_id=?", (s, t["task_id"]))
|
|
595
|
+
# Anti-farming (D24 review): a rating moves the operator's REPUTATION only if the
|
|
596
|
+
# rater has independent earned standing (give/take — not a throwaway starter handle),
|
|
597
|
+
# and at most once per (asker, operator) pair. The rating is always recorded above;
|
|
598
|
+
# this only governs whether it counts toward the gate metric.
|
|
599
|
+
acct = self.ledger.accounts.get(t["owner"])
|
|
600
|
+
asker_acct = self.ledger.accounts.get(_clip(asker))
|
|
601
|
+
counted = False
|
|
602
|
+
if acct and asker_acct and asker_acct.lifetime_earned > 0:
|
|
603
|
+
dup = self.conn.execute(
|
|
604
|
+
"SELECT 1 FROM rater_pairs WHERE asker=? AND operator=?",
|
|
605
|
+
(_clip(asker), t["owner"])).fetchone()
|
|
606
|
+
if not dup:
|
|
607
|
+
acct.quality_sum = round(acct.quality_sum + s, 4)
|
|
608
|
+
acct.quality_n += 1
|
|
609
|
+
self.conn.execute("INSERT OR IGNORE INTO rater_pairs(asker,operator) VALUES(?,?)",
|
|
610
|
+
(_clip(asker), t["owner"]))
|
|
611
|
+
counted = True
|
|
612
|
+
self._save_ledger(); self.conn.commit()
|
|
613
|
+
return {"ok": True, "counted_toward_reputation": counted,
|
|
614
|
+
"operator_reputation": acct.avg_quality if acct else 0.0}
|
|
615
|
+
|
|
616
|
+
def operator_reputation(self, owner: str):
|
|
617
|
+
"""(avg_quality, num_ratings) for an owner — the marketplace trust signal."""
|
|
618
|
+
a = self.ledger.accounts.get(_clip(owner))
|
|
619
|
+
return (a.avg_quality, a.quality_n) if a else (0.0, 0)
|
|
620
|
+
|
|
621
|
+
def job_status(self, job_id: str) -> Optional[str]:
|
|
622
|
+
with self.lock:
|
|
623
|
+
row = self.conn.execute("SELECT status FROM jobs WHERE job_id=?", (job_id,)).fetchone()
|
|
624
|
+
return row["status"] if row else None
|
|
625
|
+
|
|
626
|
+
def jobs_for_asker(self, asker: str, limit: int = 25) -> list:
|
|
627
|
+
"""The asker's own question history (newest first) — powers the app's history list."""
|
|
628
|
+
with self.lock:
|
|
629
|
+
return [dict(r) for r in self.conn.execute(
|
|
630
|
+
"SELECT job_id, question, status, created, type FROM jobs WHERE asker=?"
|
|
631
|
+
" ORDER BY created DESC LIMIT ?", (asker, limit))]
|
|
632
|
+
|
|
633
|
+
def set_baseline(self, job_id: str, data: dict) -> None:
|
|
634
|
+
"""Attach the independent single-model baseline (generated off-thread)."""
|
|
635
|
+
with self.lock:
|
|
636
|
+
self.conn.execute("UPDATE jobs SET baseline=? WHERE job_id=?",
|
|
637
|
+
(json.dumps(data), job_id))
|
|
638
|
+
self.conn.commit()
|
|
639
|
+
|
|
640
|
+
def next_task(self, node_id: str) -> Optional[dict]:
|
|
641
|
+
with self.lock:
|
|
642
|
+
row = self.conn.execute(
|
|
643
|
+
"SELECT * FROM tasks WHERE node_id=? AND status='queued' ORDER BY created LIMIT 1",
|
|
644
|
+
(node_id,)).fetchone()
|
|
645
|
+
if not row:
|
|
646
|
+
return None
|
|
647
|
+
self.conn.execute("UPDATE tasks SET status='claimed', claimed_at=? WHERE task_id=?",
|
|
648
|
+
(_now(), row["task_id"]))
|
|
649
|
+
self.conn.commit()
|
|
650
|
+
return {"task_id": row["task_id"], "job_id": row["job_id"], "type": row["type"],
|
|
651
|
+
"payload": json.loads(row["payload"]), "lens": row["lens"],
|
|
652
|
+
"country": row["country"], "model": row["model"]}
|
|
653
|
+
|
|
654
|
+
@staticmethod
|
|
655
|
+
def result_digest(result: dict) -> str:
|
|
656
|
+
"""Canonical SHA-256 of a task result — tamper-evidence (FEDERATION_V2 trust step 1).
|
|
657
|
+
Any later alteration of a stored deliverable becomes detectable. Canonical = sorted
|
|
658
|
+
keys, compact separators, so the hash is stable across serializations."""
|
|
659
|
+
canonical = json.dumps(result, sort_keys=True, separators=(",", ":"),
|
|
660
|
+
ensure_ascii=False)
|
|
661
|
+
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
662
|
+
|
|
663
|
+
def complete_task(self, task_id: str, result: dict, node_id: Optional[str] = None) -> bool:
|
|
664
|
+
"""node_id (when provided) must own the task — blocks task hijacking."""
|
|
665
|
+
with self.lock:
|
|
666
|
+
t = self.conn.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
|
667
|
+
if not t or t["status"] == "done":
|
|
668
|
+
return False
|
|
669
|
+
if node_id is not None and t["node_id"] != node_id:
|
|
670
|
+
return False # not your task
|
|
671
|
+
# stamp a tamper-evident digest into the stored result before persisting
|
|
672
|
+
result = dict(result)
|
|
673
|
+
result["_digest"] = self.result_digest({k: v for k, v in result.items()
|
|
674
|
+
if k != "_digest"})
|
|
675
|
+
self.conn.execute("UPDATE tasks SET status='done', result=? WHERE task_id=?",
|
|
676
|
+
(json.dumps(result), task_id))
|
|
677
|
+
self.conn.commit()
|
|
678
|
+
if t["type"] == "answer":
|
|
679
|
+
self._maybe_start_judging(t["job_id"])
|
|
680
|
+
elif t["type"] == "judge":
|
|
681
|
+
self._settle(t["job_id"], t["owner"], result)
|
|
682
|
+
return True
|
|
683
|
+
|
|
684
|
+
def _maybe_start_judging(self, job_id: str) -> None:
|
|
685
|
+
answers = list(self.conn.execute(
|
|
686
|
+
"SELECT * FROM tasks WHERE job_id=? AND type='answer'", (job_id,)))
|
|
687
|
+
if any(a["status"] != "done" for a in answers):
|
|
688
|
+
return
|
|
689
|
+
if self.conn.execute("SELECT 1 FROM tasks WHERE job_id=? AND type='judge'",
|
|
690
|
+
(job_id,)).fetchone():
|
|
691
|
+
return
|
|
692
|
+
job = self.conn.execute("SELECT * FROM jobs WHERE job_id=?", (job_id,)).fetchone()
|
|
693
|
+
judges = self.online_nodes(judge_only=True)
|
|
694
|
+
if not judges:
|
|
695
|
+
self.conn.execute("UPDATE jobs SET status='failed', error=? WHERE job_id=?",
|
|
696
|
+
("no judge node online", job_id))
|
|
697
|
+
self.conn.commit()
|
|
698
|
+
return
|
|
699
|
+
# Self-dealing guard: prefer a judge that did NOT answer this job.
|
|
700
|
+
answer_owners = {a["owner"] for a in answers}
|
|
701
|
+
external = [j for j in judges if j["owner"] not in answer_owners]
|
|
702
|
+
judge = external[0] if external else judges[0]
|
|
703
|
+
payload_answers = []
|
|
704
|
+
for a in answers:
|
|
705
|
+
res = json.loads(a["result"]) if a["result"] else {}
|
|
706
|
+
entry = {"worker_id": a["worker_id"], "text": res.get("text", ""),
|
|
707
|
+
"model": a["model"], "lens": a["lens"], "country": a["country"],
|
|
708
|
+
# structured research contribution (findings + sources) for the editor pass
|
|
709
|
+
"research": res.get("research")}
|
|
710
|
+
if (job["type"] or "") == "shard_map":
|
|
711
|
+
# deterministic per-node sample for the judge's QA spot-check
|
|
712
|
+
rows = res.get("results") or []
|
|
713
|
+
seed = int(hashlib.sha256(f"{job_id}:{a['worker_id']}".encode()).hexdigest(), 16)
|
|
714
|
+
idx = sorted({seed % max(1, len(rows)), (seed // 7) % max(1, len(rows)),
|
|
715
|
+
(seed // 131) % max(1, len(rows))})
|
|
716
|
+
entry["sample"] = [rows[i] for i in idx][:3] if rows else []
|
|
717
|
+
payload_answers.append(entry)
|
|
718
|
+
self.conn.execute(
|
|
719
|
+
"INSERT INTO tasks VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
720
|
+
(str(uuid.uuid4()), job_id, "judge", judge["node_id"], "queued",
|
|
721
|
+
json.dumps({"question": job["question"], "answers": payload_answers,
|
|
722
|
+
"job_type": job["type"] or "chat"}), None,
|
|
723
|
+
judge["node_id"], judge["owner"], "", judge["country"], judge["judge_model"],
|
|
724
|
+
_now(), None, None))
|
|
725
|
+
self.conn.execute("UPDATE jobs SET status='judging' WHERE job_id=?", (job_id,))
|
|
726
|
+
self.conn.commit()
|
|
727
|
+
|
|
728
|
+
@staticmethod
|
|
729
|
+
def _sane_score(raw: Any) -> float:
|
|
730
|
+
"""Clamp to [0,10]; non-finite / non-numeric → 0.0. isfinite BEFORE min/max."""
|
|
731
|
+
try:
|
|
732
|
+
v = float(raw)
|
|
733
|
+
except (TypeError, ValueError):
|
|
734
|
+
return 0.0
|
|
735
|
+
if not math.isfinite(v):
|
|
736
|
+
return 0.0
|
|
737
|
+
return max(0.0, min(10.0, v))
|
|
738
|
+
|
|
739
|
+
def _settle(self, job_id: str, judge_owner: str, result: dict) -> None:
|
|
740
|
+
job = self.conn.execute("SELECT * FROM jobs WHERE job_id=?", (job_id,)).fetchone()
|
|
741
|
+
if not job or job["status"] not in ("judging", "pending_answers"):
|
|
742
|
+
return
|
|
743
|
+
answers = list(self.conn.execute(
|
|
744
|
+
"SELECT * FROM tasks WHERE job_id=? AND type='answer'", (job_id,)))
|
|
745
|
+
raw_scores = result.get("scores", {}) if isinstance(result.get("scores"), dict) else {}
|
|
746
|
+
self_judged = any(a["owner"] == judge_owner for a in answers)
|
|
747
|
+
|
|
748
|
+
# Compute sanitized per-answer scores (empty/errored answer → 0, no reputation).
|
|
749
|
+
per_answer = [] # (task_id, owner, score, count_reputation)
|
|
750
|
+
for a in answers:
|
|
751
|
+
res = json.loads(a["result"]) if a["result"] else {}
|
|
752
|
+
text = (res.get("text") or "").strip()
|
|
753
|
+
errored = bool(res.get("error")) or not text
|
|
754
|
+
if errored:
|
|
755
|
+
s, rep = 0.0, False
|
|
756
|
+
else:
|
|
757
|
+
s, rep = self._sane_score(raw_scores.get(a["worker_id"], 5.0)), True
|
|
758
|
+
# Self-dealing fallback: a judge that judged its own answer can't score itself high.
|
|
759
|
+
if self_judged and a["owner"] == judge_owner:
|
|
760
|
+
s = 5.0
|
|
761
|
+
per_answer.append((a["task_id"], a["owner"], s, rep))
|
|
762
|
+
|
|
763
|
+
score_by_owner: dict[str, float] = {}
|
|
764
|
+
for _, owner, s, _rep in per_answer:
|
|
765
|
+
score_by_owner[owner] = score_by_owner.get(owner, 0.0) + s
|
|
766
|
+
|
|
767
|
+
# FAIL-CLOSED: settle the ledger FIRST. Only on success do we write scores/rep/done.
|
|
768
|
+
try:
|
|
769
|
+
receipt = self.ledger.settle_job(
|
|
770
|
+
job_id=job_id, asker_id=job["asker"], score_by_worker=score_by_owner,
|
|
771
|
+
worker_pool=job["pool"] if job["pool"] else CONFIG.worker_pool,
|
|
772
|
+
judge_id=judge_owner, judge_fee=CONFIG.judge_fee)
|
|
773
|
+
except InsufficientCredit as exc:
|
|
774
|
+
self.conn.execute("UPDATE jobs SET status='failed', error=? WHERE job_id=?",
|
|
775
|
+
(f"settlement failed: {exc}", job_id))
|
|
776
|
+
self._save_ledger()
|
|
777
|
+
self.conn.commit()
|
|
778
|
+
return
|
|
779
|
+
|
|
780
|
+
# shard_map: the deliverable is the ASSEMBLED shards in input order (the judge only
|
|
781
|
+
# spot-checks quality); overwrite merged with the full results array (JSON).
|
|
782
|
+
if (job["type"] or "") == "shard_map":
|
|
783
|
+
allr = []
|
|
784
|
+
for a in answers:
|
|
785
|
+
res = json.loads(a["result"]) if a["result"] else {}
|
|
786
|
+
allr.extend(res.get("results") or [])
|
|
787
|
+
allr.sort(key=lambda r: r.get("i", 0))
|
|
788
|
+
result = dict(result)
|
|
789
|
+
result["merged"] = json.dumps(allr, ensure_ascii=False)
|
|
790
|
+
|
|
791
|
+
for task_id, owner, s, rep in per_answer:
|
|
792
|
+
self.conn.execute("UPDATE tasks SET score=? WHERE task_id=?", (s, task_id))
|
|
793
|
+
if rep:
|
|
794
|
+
acct = self.ledger.accounts.get(owner)
|
|
795
|
+
if acct:
|
|
796
|
+
acct.quality_sum = round(acct.quality_sum + s, 4)
|
|
797
|
+
acct.quality_n += 1
|
|
798
|
+
council = result.get("council") if isinstance(result.get("council"), dict) else None
|
|
799
|
+
self.conn.execute(
|
|
800
|
+
"UPDATE jobs SET status='done', merged=?, receipt=?, council=? WHERE job_id=?",
|
|
801
|
+
(result.get("merged", ""), json.dumps(receipt.__dict__),
|
|
802
|
+
json.dumps(council) if council else None, job_id))
|
|
803
|
+
self._save_ledger()
|
|
804
|
+
self.conn.commit()
|
|
805
|
+
|
|
806
|
+
# ------------------------------------------------------------------ reaper
|
|
807
|
+
def _reap_loop(self) -> None:
|
|
808
|
+
interval = max(10.0, CONFIG.node_ttl_s / 2)
|
|
809
|
+
while not self._stop.wait(interval):
|
|
810
|
+
try:
|
|
811
|
+
self._reap_once()
|
|
812
|
+
except Exception:
|
|
813
|
+
pass # never let the reaper die
|
|
814
|
+
|
|
815
|
+
def _reap_once(self) -> None:
|
|
816
|
+
with self.lock:
|
|
817
|
+
now = _now()
|
|
818
|
+
# reclaim blobs of long-finished jobs (retention window gives the asker time to
|
|
819
|
+
# fetch); bounds unbounded SQLite growth from delivered files.
|
|
820
|
+
retain = float(os.environ.get("PW_BLOB_RETAIN_S", str(7 * 86400)))
|
|
821
|
+
self.conn.execute(
|
|
822
|
+
"DELETE FROM blobs WHERE job_id IN ("
|
|
823
|
+
" SELECT job_id FROM jobs WHERE status IN ('done','failed') AND created < ?)",
|
|
824
|
+
(now - retain,))
|
|
825
|
+
# assisted jobs are human-paced: only expire them on their (long) deadline,
|
|
826
|
+
# never on node-liveness (no node is assigned until a human accepts).
|
|
827
|
+
for job in list(self.conn.execute(
|
|
828
|
+
"SELECT * FROM jobs WHERE status IN ('pending_assist','assisting')")):
|
|
829
|
+
deadline = JOB_TYPES["assisted"]["deadline_s"]
|
|
830
|
+
if now - job["created"] > deadline:
|
|
831
|
+
# refund the held reward to the asker before failing (escrow → asker)
|
|
832
|
+
try:
|
|
833
|
+
if job["pool"]:
|
|
834
|
+
self.ledger.refund(job["asker"], job["pool"])
|
|
835
|
+
except Exception:
|
|
836
|
+
pass
|
|
837
|
+
self.conn.execute("UPDATE jobs SET status='failed', error=? WHERE job_id=?",
|
|
838
|
+
(f"assisted offer expired ({int(deadline)}s)", job["job_id"]))
|
|
839
|
+
self._save_ledger()
|
|
840
|
+
stuck = list(self.conn.execute(
|
|
841
|
+
"SELECT * FROM jobs WHERE status IN ('pending_answers','judging')"))
|
|
842
|
+
for job in stuck:
|
|
843
|
+
fail = None
|
|
844
|
+
deadline = JOB_TYPES.get(job["type"] or "chat", JOB_TYPES["chat"])["deadline_s"]
|
|
845
|
+
if now - job["created"] > deadline:
|
|
846
|
+
fail = f"deadline exceeded ({int(deadline)}s)"
|
|
847
|
+
else:
|
|
848
|
+
# If any not-done task's assigned node has gone stale, the job can't progress.
|
|
849
|
+
tasks = self.conn.execute(
|
|
850
|
+
"SELECT node_id, status FROM tasks WHERE job_id=? AND status!='done'",
|
|
851
|
+
(job["job_id"],)).fetchall()
|
|
852
|
+
for t in tasks:
|
|
853
|
+
nd = self.conn.execute("SELECT last_seen FROM nodes WHERE node_id=?",
|
|
854
|
+
(t["node_id"],)).fetchone()
|
|
855
|
+
if not nd or now - nd["last_seen"] > CONFIG.node_ttl_s:
|
|
856
|
+
fail = "assigned node went offline"
|
|
857
|
+
break
|
|
858
|
+
if fail:
|
|
859
|
+
self.conn.execute("UPDATE jobs SET status='failed', error=? WHERE job_id=?",
|
|
860
|
+
(fail, job["job_id"]))
|
|
861
|
+
self.conn.commit()
|
|
862
|
+
|
|
863
|
+
# ------------------------------------------------------------------ views
|
|
864
|
+
def job_view(self, job_id: str) -> Optional[dict]:
|
|
865
|
+
with self.lock:
|
|
866
|
+
job = self.conn.execute("SELECT * FROM jobs WHERE job_id=?", (job_id,)).fetchone()
|
|
867
|
+
if not job:
|
|
868
|
+
return None
|
|
869
|
+
answers = list(self.conn.execute(
|
|
870
|
+
"SELECT * FROM tasks WHERE job_id=? AND type='answer'", (job_id,)))
|
|
871
|
+
nm = {r["node_id"]: (r["machine_id"] or r["node_id"])
|
|
872
|
+
for r in self.conn.execute("SELECT node_id, machine_id FROM nodes")}
|
|
873
|
+
mkey = lambda nid: hashlib.sha256((nm.get(nid) or nid or "").encode()).hexdigest()[:12]
|
|
874
|
+
label = {"queued": "assigned", "claimed": "thinking", "done": "answered"}
|
|
875
|
+
ans = []
|
|
876
|
+
for a in answers:
|
|
877
|
+
res = json.loads(a["result"]) if a["result"] else {}
|
|
878
|
+
ans.append({"worker_id": a["worker_id"], "owner": a["owner"], "model": a["model"],
|
|
879
|
+
"lens": a["lens"], "country": a["country"],
|
|
880
|
+
"node_key": hashlib.sha256((a["node_id"] or "").encode()).hexdigest()[:12],
|
|
881
|
+
"machine_key": mkey(a["node_id"]),
|
|
882
|
+
"status": a["status"], "status_label": label.get(a["status"], a["status"]),
|
|
883
|
+
"score": a["score"], "text": res.get("text", ""),
|
|
884
|
+
"tokens": res.get("tokens"), "elapsed_s": res.get("elapsed_s"),
|
|
885
|
+
"digest": res.get("_digest"), # tamper-evidence (FEDERATION_V2)
|
|
886
|
+
"is_baseline": False})
|
|
887
|
+
baseline = json.loads(job["baseline"]) if job["baseline"] else None
|
|
888
|
+
done = [x for x in ans if x["status"] == "done" and x["score"] is not None]
|
|
889
|
+
if done and not baseline: # fallback only: best single COUNCIL answer as baseline
|
|
890
|
+
max(done, key=lambda x: x["score"])["is_baseline"] = True
|
|
891
|
+
jt = self.conn.execute(
|
|
892
|
+
"SELECT node_id, country, status FROM tasks WHERE job_id=? AND type='judge' LIMIT 1",
|
|
893
|
+
(job_id,)).fetchone()
|
|
894
|
+
# assisted: surface the operator's signature over the deliverable (D23). The asker
|
|
895
|
+
# verifies it against an OUT-OF-BAND PINNED key (D25), not a coordinator-reported one,
|
|
896
|
+
# so the coordinator's view of the key is no longer part of the trust decision.
|
|
897
|
+
sig = signer = None
|
|
898
|
+
encrypt_to = ""
|
|
899
|
+
operator = op_rep = op_ratings = None
|
|
900
|
+
rated = False
|
|
901
|
+
at = self.conn.execute(
|
|
902
|
+
"SELECT result, payload, node_id, owner, score FROM tasks"
|
|
903
|
+
" WHERE job_id=? AND type='assisted' LIMIT 1", (job_id,)).fetchone()
|
|
904
|
+
if at:
|
|
905
|
+
operator = at["owner"]
|
|
906
|
+
rated = at["score"] is not None
|
|
907
|
+
if operator:
|
|
908
|
+
op_rep, op_ratings = self.operator_reputation(operator)
|
|
909
|
+
if at["result"]:
|
|
910
|
+
ares = json.loads(at["result"])
|
|
911
|
+
sig, signer = ares.get("signature"), ares.get("signer_pub")
|
|
912
|
+
if at["payload"]:
|
|
913
|
+
encrypt_to = json.loads(at["payload"]).get("encrypt_to", "")
|
|
914
|
+
return {
|
|
915
|
+
"job_id": job["job_id"], "asker": job["asker"], "question": job["question"],
|
|
916
|
+
"type": job["type"] or "chat",
|
|
917
|
+
"status": job["status"], "error": job["error"], "merged": job["merged"],
|
|
918
|
+
"council": json.loads(job["council"]) if job["council"] else None,
|
|
919
|
+
"judge_country": jt["country"] if jt else None,
|
|
920
|
+
"judge_status": jt["status"] if jt else None,
|
|
921
|
+
"judge_machine_key": mkey(jt["node_id"]) if jt else None,
|
|
922
|
+
"receipt": json.loads(job["receipt"]) if job["receipt"] else None,
|
|
923
|
+
"baseline": baseline,
|
|
924
|
+
"signature": sig, "signer_pub": signer, "encrypt_to": encrypt_to,
|
|
925
|
+
"operator": operator, "operator_reputation": op_rep,
|
|
926
|
+
"operator_ratings": op_ratings, "rated": rated,
|
|
927
|
+
"answers": ans,
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
def status(self) -> dict:
|
|
931
|
+
with self.lock:
|
|
932
|
+
nodes = []
|
|
933
|
+
machines = set()
|
|
934
|
+
for n in self.online_nodes():
|
|
935
|
+
acct = self.ledger.accounts.get(n["owner"])
|
|
936
|
+
machines.add(n["machine_id"] or n["node_id"])
|
|
937
|
+
nodes.append({
|
|
938
|
+
# opaque key (not the raw node_id, which authenticates nothing now but
|
|
939
|
+
# shouldn't be freely enumerable) — stable for the map's jitter.
|
|
940
|
+
"node_key": hashlib.sha256(n["node_id"].encode()).hexdigest()[:12],
|
|
941
|
+
"machine_key": hashlib.sha256((n["machine_id"] or n["node_id"]).encode()).hexdigest()[:12],
|
|
942
|
+
"name": n["name"], "country": n["country"], "owner": n["owner"],
|
|
943
|
+
"answer_model": n["answer_model"], "lens": n["lens"],
|
|
944
|
+
"can_judge": n["can_judge"], "load": n["load"],
|
|
945
|
+
"age_s": round(_now() - n["last_seen"], 1),
|
|
946
|
+
"reputation": acct.avg_quality if acct else 0.0,
|
|
947
|
+
"jobs_helped": acct.jobs_helped if acct else 0,
|
|
948
|
+
})
|
|
949
|
+
jobs = [dict(j) for j in self.conn.execute(
|
|
950
|
+
"SELECT job_id, asker, status, created FROM jobs ORDER BY created DESC LIMIT 10")]
|
|
951
|
+
from council.ledger import ESCROW_ID
|
|
952
|
+
accounts = [(u, a) for u, a in self.ledger.accounts.items()
|
|
953
|
+
if u != ESCROW_ID] # hide the internal escrow holding account
|
|
954
|
+
return {
|
|
955
|
+
"online_nodes": nodes,
|
|
956
|
+
"machines": len(machines), # distinct physical computers online
|
|
957
|
+
"minds": len(nodes), # worker processes (a computer can run several)
|
|
958
|
+
"recent_jobs": jobs,
|
|
959
|
+
"ledger_total": self.ledger.total_credit(),
|
|
960
|
+
"ledger_conserved": self.ledger.conservation_ok(),
|
|
961
|
+
"accounts": {u: {"balance": round(a.balance, 1), "reputation": a.avg_quality,
|
|
962
|
+
"helped": a.jobs_helped, "asked": a.jobs_asked}
|
|
963
|
+
for u, a in accounts},
|
|
964
|
+
}
|