puremethod 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/__init__.py +2 -0
- app/__main__.py +31 -0
- app/_web/404/index.html +1 -0
- app/_web/404.html +1 -0
- app/_web/__next.!KGNvbnNvbGUp.__PAGE__.txt +42 -0
- app/_web/__next._full.txt +36 -0
- app/_web/__next._tree.txt +4 -0
- app/_web/_next/static/6az3RZxafySyERvJdZAgV/_buildManifest.js +11 -0
- app/_web/_next/static/6az3RZxafySyERvJdZAgV/_clientMiddlewareManifest.js +1 -0
- app/_web/_next/static/6az3RZxafySyERvJdZAgV/_ssgManifest.js +1 -0
- app/_web/_next/static/chunks/00fylloju-1x2.js +1 -0
- app/_web/_next/static/chunks/03hxwfej4sm9g.js +1 -0
- app/_web/_next/static/chunks/0cz1d0mv5g_q7.js +1 -0
- app/_web/_next/static/chunks/0gus9jro9n3j3.js +1 -0
- app/_web/_next/static/chunks/0n12a63ieedt9.js +1 -0
- app/_web/_next/static/chunks/0ns7paypbayry.js +31 -0
- app/_web/_next/static/chunks/0s1d7y-_wn8wn.js +1 -0
- app/_web/_next/static/chunks/12s40azz56b3b.js +1 -0
- app/_web/_next/static/chunks/1b_14wnek5geq.js +1 -0
- app/_web/_next/static/chunks/1oderptqhsaox.css +1 -0
- app/_web/_next/static/chunks/1rcnm-dleh7p6.js +1 -0
- app/_web/_next/static/chunks/1u4klctqx_w0w.js +1 -0
- app/_web/_next/static/chunks/1vahkxwiu_bqp.js +22 -0
- app/_web/_next/static/chunks/26mp4u1wew2xu.js +1 -0
- app/_web/_next/static/chunks/2aq880yo-2z88.js +1 -0
- app/_web/_next/static/chunks/2f5ovhdilzkal.js +1 -0
- app/_web/_next/static/chunks/2qds0qd18u74f.js +1 -0
- app/_web/_next/static/chunks/2qgmxvbiean_r.js +1 -0
- app/_web/_next/static/chunks/3b5v1mle2gmac.js +1 -0
- app/_web/_next/static/chunks/3hw8x3h98ysh4.js +6 -0
- app/_web/_next/static/chunks/3op5__grw4752.js +1 -0
- app/_web/_next/static/chunks/3scuzzds0vtp7.js +1 -0
- app/_web/_next/static/chunks/3v_5wdpvvcxz8.js +1 -0
- app/_web/_next/static/chunks/3wevuf_oyh97l.js +1 -0
- app/_web/_next/static/chunks/turbopack-3-ry682tkyiqk.js +1 -0
- app/_web/_not-found/__next._full.txt +24 -0
- app/_web/_not-found/__next._not-found.__PAGE__.txt +30 -0
- app/_web/_not-found/__next._tree.txt +4 -0
- app/_web/_not-found/index.html +1 -0
- app/_web/_not-found/index.txt +24 -0
- app/_web/credentials/__next.!KGNvbnNvbGUp.credentials.__PAGE__.txt +46 -0
- app/_web/credentials/__next._full.txt +39 -0
- app/_web/credentials/__next._tree.txt +5 -0
- app/_web/credentials/index.html +1 -0
- app/_web/credentials/index.txt +39 -0
- app/_web/executors/__next.!KGNvbnNvbGUp.executors.__PAGE__.txt +46 -0
- app/_web/executors/__next._full.txt +39 -0
- app/_web/executors/__next._tree.txt +5 -0
- app/_web/executors/index.html +1 -0
- app/_web/executors/index.txt +39 -0
- app/_web/index.html +1 -0
- app/_web/index.txt +36 -0
- app/_web/new/__next.!KGNvbnNvbGUp.new.__PAGE__.txt +46 -0
- app/_web/new/__next._full.txt +39 -0
- app/_web/new/__next._tree.txt +5 -0
- app/_web/new/index.html +1 -0
- app/_web/new/index.txt +39 -0
- app/_web/profiles/__next.!KGNvbnNvbGUp.profiles.__PAGE__.txt +46 -0
- app/_web/profiles/__next._full.txt +39 -0
- app/_web/profiles/__next._tree.txt +5 -0
- app/_web/profiles/index.html +1 -0
- app/_web/profiles/index.txt +39 -0
- app/_web/run/__next.!KGNvbnNvbGUp.run.__PAGE__.txt +47 -0
- app/_web/run/__next._full.txt +40 -0
- app/_web/run/__next._tree.txt +5 -0
- app/_web/run/index.html +1 -0
- app/_web/run/index.txt +40 -0
- app/_web/services/__next.!KGNvbnNvbGUp.services.__PAGE__.txt +46 -0
- app/_web/services/__next._full.txt +39 -0
- app/_web/services/__next._tree.txt +5 -0
- app/_web/services/index.html +1 -0
- app/_web/services/index.txt +39 -0
- app/_web/setup/__next.!KGNvbnNvbGUp.setup.__PAGE__.txt +46 -0
- app/_web/setup/__next._full.txt +39 -0
- app/_web/setup/__next._tree.txt +5 -0
- app/_web/setup/index.html +1 -0
- app/_web/setup/index.txt +39 -0
- app/_web/signin/__next._full.txt +30 -0
- app/_web/signin/__next._tree.txt +4 -0
- app/_web/signin/__next.signin.__PAGE__.txt +36 -0
- app/_web/signin/index.html +1 -0
- app/_web/signin/index.txt +30 -0
- app/accounts.py +137 -0
- app/addresses.py +30 -0
- app/api/__init__.py +61 -0
- app/api/auth_routes.py +54 -0
- app/api/common.py +102 -0
- app/api/config_routes.py +562 -0
- app/api/links.py +22 -0
- app/api/run_routes.py +311 -0
- app/api/setup_routes.py +267 -0
- app/bundled.py +521 -0
- app/compose.py +229 -0
- app/db.py +186 -0
- app/executor_config.py +95 -0
- app/keys.py +29 -0
- app/launcher.py +389 -0
- app/leases.py +40 -0
- app/literature_sources.py +131 -0
- app/model_endpoints.py +104 -0
- app/model_search.py +252 -0
- app/outputs.py +77 -0
- app/paths.py +93 -0
- app/readonly.py +70 -0
- app/repo.py +746 -0
- app/scan.py +65 -0
- app/server.py +24 -0
- app/settings.py +125 -0
- app/setup.py +117 -0
- app/ssh_chain.py +102 -0
- app/ssh_hosts.py +244 -0
- app/ssh_machines.py +101 -0
- app/statuses.py +4 -0
- app/supervisor.py +87 -0
- app/tunnels.py +222 -0
- app/unseal.py +27 -0
- app/worker.py +657 -0
- executor/__init__.py +47 -0
- executor/base.py +161 -0
- executor/common.py +139 -0
- executor/local_docker.py +239 -0
- executor/local_process.py +247 -0
- executor/ssh.py +481 -0
- executor/types.py +128 -0
- executor/watchdog.py +108 -0
- ledger/__init__.py +46 -0
- ledger/artifacts.py +62 -0
- ledger/canonical.py +48 -0
- ledger/core.py +337 -0
- ledger/errors.py +36 -0
- ledger/model.py +107 -0
- ledger/payloads.py +228 -0
- ledger/rules.py +204 -0
- ledger/storage.py +226 -0
- llm/__init__.py +45 -0
- llm/backends/__init__.py +0 -0
- llm/backends/anthropic_backend.py +145 -0
- llm/backends/base.py +87 -0
- llm/backends/openai_backend.py +211 -0
- llm/core.py +416 -0
- llm/prices.py +98 -0
- llm/tokens.py +18 -0
- llm/types.py +106 -0
- loop/__init__.py +19 -0
- loop/config.py +108 -0
- loop/core.py +1160 -0
- loop/decide.py +61 -0
- loop/harness.py +132 -0
- loop/prompts.py +88 -0
- loop/view.py +202 -0
- puremethod-0.1.0.dist-info/METADATA +79 -0
- puremethod-0.1.0.dist-info/RECORD +171 -0
- puremethod-0.1.0.dist-info/WHEEL +4 -0
- puremethod-0.1.0.dist-info/entry_points.txt +2 -0
- puremethod-0.1.0.dist-info/licenses/LICENSE +96 -0
- puremethod-0.1.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +173 -0
- purescholar/__init__.py +43 -0
- purescholar/client.py +380 -0
- purescholar/types.py +164 -0
- puretrace/__init__.py +49 -0
- puretrace/client.py +320 -0
- puretrace/types.py +166 -0
- writer/__init__.py +8 -0
- writer/core.py +430 -0
- writer/facts.py +162 -0
- writer/latex.py +62 -0
- writer/lint.py +85 -0
- writer/macros.py +114 -0
- writer/render.py +120 -0
- writer/tectonic.py +79 -0
- writer/types.py +61 -0
app/compose.py
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
"""The composition root: workers build the clients a `Loop` is made from (08-app.md §8,
|
|
2
|
+
KB-021). Worker side only; `app.api` may not import it (import-linter)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
from collections.abc import Callable
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from app import bundled, executor_config, model_endpoints, tunnels
|
|
12
|
+
from app.repo import ExecutorRow, RunRow, TenantRepo
|
|
13
|
+
from app.settings import Settings
|
|
14
|
+
from app.setup import LOCAL, PURESCHOLAR_CREDENTIAL, PURETRACE_CREDENTIAL, service_url
|
|
15
|
+
from app.unseal import unseal
|
|
16
|
+
from executor import Executor, LocalDockerExecutor, LocalProcessExecutor, SSHExecutor
|
|
17
|
+
from ledger import Ledger
|
|
18
|
+
from llm import DEFAULT_PRICES, LLM, ModelPrice, PriceTable
|
|
19
|
+
from loop import Loop, RunConfig
|
|
20
|
+
from purescholar import PureScholar
|
|
21
|
+
from puretrace import PureTrace
|
|
22
|
+
from writer import Writer
|
|
23
|
+
|
|
24
|
+
Lookup = Callable[[str], str]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class ServiceUrls:
|
|
28
|
+
"""Where a tenant's runs reach PureScholar and PureTrace (08 §4, KB-029): the tenant's
|
|
29
|
+
setting, else `PM_PURESCHOLAR_URL` / `PM_PURETRACE_URL`."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, purescholar: str | None, puretrace: str | None, *,
|
|
32
|
+
purescholar_local: bool = False) -> None:
|
|
33
|
+
self.purescholar, self.puretrace = purescholar, puretrace
|
|
34
|
+
self.purescholar_local = purescholar_local
|
|
35
|
+
|
|
36
|
+
def purescholar_options(self) -> dict[str, Any]:
|
|
37
|
+
"""The request timeout `app` supplies for the bundled instance (01 §3)."""
|
|
38
|
+
return {"timeout_s": bundled.REQUEST_TIMEOUT_S} if self.purescholar_local else {}
|
|
39
|
+
|
|
40
|
+
@classmethod
|
|
41
|
+
def for_tenant(cls, repo: TenantRepo, settings: Settings) -> ServiceUrls:
|
|
42
|
+
lit = repo.service_setting("literature") or {}
|
|
43
|
+
local = lit.get("mode") == "local"
|
|
44
|
+
ps = bundled.url(repo) if local else \
|
|
45
|
+
service_url(repo, "literature", settings.purescholar_url)
|
|
46
|
+
return cls(ps, service_url(repo, "puretrace", settings.puretrace_url),
|
|
47
|
+
purescholar_local=local)
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def from_settings(cls, settings: Settings) -> ServiceUrls:
|
|
51
|
+
return cls(settings.purescholar_url, settings.puretrace_url)
|
|
52
|
+
|
|
53
|
+
@classmethod
|
|
54
|
+
def for_run(cls, run: RunRow, repo: TenantRepo, settings: Settings) -> ServiceUrls:
|
|
55
|
+
"""The run's snapshot (made at creation); the tenant's current setting for runs made
|
|
56
|
+
before KB-029. A bundled PureScholar ("local") is reached at its current address."""
|
|
57
|
+
if run.services is None:
|
|
58
|
+
return cls.for_tenant(repo, settings)
|
|
59
|
+
ps = run.services.get("purescholar")
|
|
60
|
+
local = ps == LOCAL
|
|
61
|
+
if local:
|
|
62
|
+
ps = bundled.url(repo) # its current address; None while it is not ready
|
|
63
|
+
return cls(ps, run.services.get("puretrace"), purescholar_local=local)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class CredentialMissing(KeyError):
|
|
67
|
+
"""A credential the run needs is not stored, or is not one this worker may read."""
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class Credentials:
|
|
71
|
+
"""Decrypts only names on the allow list, for one tenant (§5 step 3), and remembers the
|
|
72
|
+
values it handed out so worker logs can be scrubbed."""
|
|
73
|
+
|
|
74
|
+
def __init__(self, repo: TenantRepo, master_key: bytes, allowed: set[str]) -> None:
|
|
75
|
+
self._repo = repo
|
|
76
|
+
self._key = master_key
|
|
77
|
+
self._allowed = frozenset(allowed)
|
|
78
|
+
self._cache: dict[str, str] = {}
|
|
79
|
+
|
|
80
|
+
def __call__(self, name: str) -> str:
|
|
81
|
+
if name not in self._allowed:
|
|
82
|
+
raise CredentialMissing(name)
|
|
83
|
+
if name not in self._cache:
|
|
84
|
+
sealed = self._repo.sealed(name)
|
|
85
|
+
if sealed is None:
|
|
86
|
+
raise CredentialMissing(name)
|
|
87
|
+
self._cache[name] = unseal(self._key, self._repo.tenant_id, sealed.id,
|
|
88
|
+
sealed.nonce, sealed.ciphertext)
|
|
89
|
+
return self._cache[name]
|
|
90
|
+
|
|
91
|
+
def scrub(self, text: str) -> str:
|
|
92
|
+
for value in sorted(self._cache.values(), key=len, reverse=True):
|
|
93
|
+
if value:
|
|
94
|
+
text = text.replace(value, "[secret]")
|
|
95
|
+
return text
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def run_config(run: RunRow) -> RunConfig:
|
|
99
|
+
return RunConfig.model_validate(run.config)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def run_executor(run: RunRow) -> ExecutorRow:
|
|
103
|
+
"""The executor as it was when the run was created: later edits to the executor never
|
|
104
|
+
reach a run that passed preflight consent checks with the old one (review finding 1)."""
|
|
105
|
+
x = run.executor
|
|
106
|
+
return ExecutorRow(str(x["id"]), str(x["name"]), str(x["kind"]), dict(x["config"]), "")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def allowed_for_run(run: RunRow, config: RunConfig, settings: Settings,
|
|
110
|
+
urls: ServiceUrls | None = None) -> set[str]:
|
|
111
|
+
"""The run's LLM and job credentials, its executor's key, and the PureScholar and PureTrace
|
|
112
|
+
keys only when those services are configured and (PureTrace) used."""
|
|
113
|
+
urls = urls or ServiceUrls.from_settings(settings)
|
|
114
|
+
names = set(config.credential_names())
|
|
115
|
+
x = run_executor(run)
|
|
116
|
+
names |= set(executor_config.credentials_named(x.kind, x.config))
|
|
117
|
+
names |= set(model_endpoints.ssh_credentials(config.llm)) # model tunnels (KB-036)
|
|
118
|
+
if urls.purescholar:
|
|
119
|
+
names.add(PURESCHOLAR_CREDENTIAL)
|
|
120
|
+
if urls.puretrace and (config.puretrace.screening or config.puretrace.selfcheck):
|
|
121
|
+
names.add(PURETRACE_CREDENTIAL)
|
|
122
|
+
return names
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def client_credentials(run: RunRow, settings: Settings,
|
|
126
|
+
urls: ServiceUrls | None = None) -> Callable[[RunConfig], list[str]]:
|
|
127
|
+
"""The names `loop` registers as run secrets beyond the config's own (03-loop.md §3 Run
|
|
128
|
+
secrets, 08 §5 step 3, KB-022): the executor's key and the service keys in use."""
|
|
129
|
+
return lambda config: sorted(allowed_for_run(run, config, settings, urls)
|
|
130
|
+
- set(config.credential_names()))
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class GuardedLedger(Ledger):
|
|
134
|
+
"""A `Ledger` whose writes first check the worker still holds the run's lease, so two
|
|
135
|
+
workers never both write one run (KB-021; review finding 3)."""
|
|
136
|
+
|
|
137
|
+
def __init__(self, storage_url: str, artifact_root: str | Path, guard: Callable[[], None],
|
|
138
|
+
**kw: Any) -> None:
|
|
139
|
+
super().__init__(storage_url, artifact_root, **kw)
|
|
140
|
+
self._guard = guard
|
|
141
|
+
|
|
142
|
+
def open_run(self, config: dict[str, Any], secrets: list[str]) -> str:
|
|
143
|
+
self._guard()
|
|
144
|
+
return super().open_run(config, secrets)
|
|
145
|
+
|
|
146
|
+
def append(self, *args: Any, **kwargs: Any) -> Any:
|
|
147
|
+
self._guard()
|
|
148
|
+
return super().append(*args, **kwargs)
|
|
149
|
+
|
|
150
|
+
def close_dangling(self, run_id: str) -> Any:
|
|
151
|
+
self._guard()
|
|
152
|
+
return super().close_dangling(run_id)
|
|
153
|
+
|
|
154
|
+
def put_artifact(self, *args: Any, **kwargs: Any) -> Any:
|
|
155
|
+
self._guard()
|
|
156
|
+
return super().put_artifact(*args, **kwargs)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def prices(overrides: dict[str, dict[str, float]]) -> PriceTable:
|
|
160
|
+
if not overrides:
|
|
161
|
+
return DEFAULT_PRICES
|
|
162
|
+
table = dict(DEFAULT_PRICES.prices)
|
|
163
|
+
for key, p in overrides.items():
|
|
164
|
+
provider, _, model = key.partition("/")
|
|
165
|
+
table[(provider, model)] = (ModelPrice.unmetered_price() if p.get("unmetered") else
|
|
166
|
+
ModelPrice(float(p["input"]), float(p["output"])))
|
|
167
|
+
return PriceTable(version=f"{DEFAULT_PRICES.version}+profile",
|
|
168
|
+
source=f"{DEFAULT_PRICES.source}; profile overrides", prices=table)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def build_executor(row: ExecutorRow, lookup: Lookup, settings: Settings) -> Executor:
|
|
172
|
+
cfg = executor_config.validate(row.kind, row.config, settings.mode) # INV-5 again here
|
|
173
|
+
root = settings.executor_root / row.id
|
|
174
|
+
common: dict[str, Any] = {"executor_id": row.id, "concurrency": cfg.concurrency,
|
|
175
|
+
"rate_usd_per_hour": cfg.rate_usd_per_hour}
|
|
176
|
+
if isinstance(cfg, executor_config.LocalDockerConfig):
|
|
177
|
+
return LocalDockerExecutor(lookup, root=root, network=cfg.network, **common)
|
|
178
|
+
if isinstance(cfg, executor_config.SSHConfig):
|
|
179
|
+
return SSHExecutor(lookup, host=cfg.host, user=cfg.user,
|
|
180
|
+
key_credential=cfg.key_credential, port=cfg.port,
|
|
181
|
+
remote_root=cfg.remote_root, use_docker=cfg.use_docker,
|
|
182
|
+
network=cfg.network, known_hosts=cfg.known_hosts,
|
|
183
|
+
accept_unknown_host=cfg.accept_unknown_host,
|
|
184
|
+
host_key=cfg.host_key,
|
|
185
|
+
jumps=[h.model_dump() for h in cfg.jumps], **common)
|
|
186
|
+
return LocalProcessExecutor(lookup, root=root, **common)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def make_loop(settings: Settings, run: RunRow, lookup: Lookup,
|
|
190
|
+
guard: Callable[[], None], urls: ServiceUrls | None = None) -> Loop:
|
|
191
|
+
"""A new `Loop` on its own guarded `Ledger` connection. Workers build one per thread."""
|
|
192
|
+
urls = urls or ServiceUrls.from_settings(settings)
|
|
193
|
+
ledger = GuardedLedger(settings.ledger_url, settings.artifact_root, guard)
|
|
194
|
+
table = prices(run.prices)
|
|
195
|
+
image = {"tectonic_image": settings.tectonic_image} if settings.tectonic_image else {}
|
|
196
|
+
snapshot = run_executor(run)
|
|
197
|
+
|
|
198
|
+
def executor(executor_id: str, look: Lookup) -> Executor:
|
|
199
|
+
if executor_id != snapshot.id:
|
|
200
|
+
raise ValueError(f"executor {executor_id} is not this run's")
|
|
201
|
+
return build_executor(snapshot, look, settings)
|
|
202
|
+
|
|
203
|
+
def purescholar() -> PureScholar:
|
|
204
|
+
if not urls.purescholar:
|
|
205
|
+
raise ValueError("PureScholar is not configured: finish setup")
|
|
206
|
+
return PureScholar(urls.purescholar, lookup(PURESCHOLAR_CREDENTIAL),
|
|
207
|
+
**urls.purescholar_options())
|
|
208
|
+
|
|
209
|
+
def puretrace(ledger_run_id: str, uses: list[str]) -> PureTrace:
|
|
210
|
+
if not urls.puretrace:
|
|
211
|
+
raise ValueError("PureTrace is not configured: set its URL in setup")
|
|
212
|
+
return PureTrace(urls.puretrace, lookup(PURETRACE_CREDENTIAL), ledger_run_id, uses)
|
|
213
|
+
|
|
214
|
+
opened: dict[str, dict[str, dict[str, Any]]] = {}
|
|
215
|
+
|
|
216
|
+
def llm(profile: dict[str, dict[str, Any]], creds: dict[str, str]) -> LLM:
|
|
217
|
+
"""`via_ssh` roles reach their model through this worker's tunnels (07 §5 step 9),
|
|
218
|
+
opened once per profile and closed when the worker process ends."""
|
|
219
|
+
if model_endpoints.ssh_credentials(profile):
|
|
220
|
+
key = json.dumps(profile, sort_keys=True)
|
|
221
|
+
if key not in opened:
|
|
222
|
+
opened[key], _ = tunnels.open_tunnels(profile, lookup)
|
|
223
|
+
profile = opened[key]
|
|
224
|
+
return LLM(profile, table, creds)
|
|
225
|
+
|
|
226
|
+
return Loop(ledger, lambda llm, ps: Writer(ledger, llm, ps, **image), llm=llm,
|
|
227
|
+
executor=executor, purescholar=purescholar, puretrace=puretrace,
|
|
228
|
+
credentials=lookup, output_root=settings.output_root,
|
|
229
|
+
client_credentials=client_credentials(run, settings, urls))
|
app/db.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""The app's own database: tenants, sign-in, credentials, configuration, the run index,
|
|
2
|
+
commands and leases (08-app.md §4). Not the ledger: `app` never writes ledger storage (INV-3).
|
|
3
|
+
|
|
4
|
+
`Database` is the storage seam: v0.1 ships SQLite (self-hosted); hosted mode adds a
|
|
5
|
+
PostgreSQL class with the same `tx` contract. SQL in `app` uses `?` placeholders and only
|
|
6
|
+
syntax both engines accept."""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import secrets
|
|
12
|
+
import sqlite3
|
|
13
|
+
from collections.abc import Iterator, Sequence
|
|
14
|
+
from contextlib import contextmanager
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, Protocol
|
|
17
|
+
|
|
18
|
+
SCHEMA_VERSION = 1
|
|
19
|
+
|
|
20
|
+
SCHEMA = """
|
|
21
|
+
CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL);
|
|
22
|
+
CREATE TABLE IF NOT EXISTS tenants (
|
|
23
|
+
id TEXT PRIMARY KEY, created_at TEXT NOT NULL);
|
|
24
|
+
CREATE TABLE IF NOT EXISTS tokens (
|
|
25
|
+
hash TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id),
|
|
26
|
+
created_at TEXT NOT NULL);
|
|
27
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
28
|
+
hash TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id),
|
|
29
|
+
expires_at REAL NOT NULL);
|
|
30
|
+
CREATE TABLE IF NOT EXISTS credentials (
|
|
31
|
+
id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id),
|
|
32
|
+
name TEXT NOT NULL, type TEXT NOT NULL, nonce BLOB NOT NULL, ciphertext BLOB NOT NULL,
|
|
33
|
+
created_at TEXT NOT NULL, UNIQUE (tenant_id, name));
|
|
34
|
+
CREATE TABLE IF NOT EXISTS executors (
|
|
35
|
+
id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id),
|
|
36
|
+
name TEXT NOT NULL, kind TEXT NOT NULL, config TEXT NOT NULL, created_at TEXT NOT NULL,
|
|
37
|
+
UNIQUE (tenant_id, name));
|
|
38
|
+
CREATE TABLE IF NOT EXISTS ssh_machines (
|
|
39
|
+
id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id),
|
|
40
|
+
name TEXT NOT NULL, config TEXT NOT NULL, created_at TEXT NOT NULL,
|
|
41
|
+
UNIQUE (tenant_id, name));
|
|
42
|
+
CREATE TABLE IF NOT EXISTS profile_checks (
|
|
43
|
+
id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id),
|
|
44
|
+
profile_id TEXT NOT NULL REFERENCES llm_profiles(id), state TEXT NOT NULL, report TEXT,
|
|
45
|
+
created_at TEXT NOT NULL, done_at TEXT, version INTEGER NOT NULL DEFAULT 1,
|
|
46
|
+
proposal TEXT, applied INTEGER NOT NULL DEFAULT 0);
|
|
47
|
+
CREATE TABLE IF NOT EXISTS llm_searches (
|
|
48
|
+
id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id),
|
|
49
|
+
target TEXT NOT NULL, state TEXT NOT NULL, report TEXT, created_at TEXT NOT NULL,
|
|
50
|
+
done_at TEXT);
|
|
51
|
+
CREATE TABLE IF NOT EXISTS rate_events (
|
|
52
|
+
tenant_id TEXT NOT NULL REFERENCES tenants(id), kind TEXT NOT NULL, at REAL NOT NULL);
|
|
53
|
+
CREATE TABLE IF NOT EXISTS llm_profiles (
|
|
54
|
+
id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id),
|
|
55
|
+
name TEXT NOT NULL, roles TEXT NOT NULL, prices TEXT NOT NULL, created_at TEXT NOT NULL,
|
|
56
|
+
version INTEGER NOT NULL DEFAULT 1, deleted_at TEXT, UNIQUE (tenant_id, name));
|
|
57
|
+
CREATE TABLE IF NOT EXISTS runs (
|
|
58
|
+
id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id),
|
|
59
|
+
title TEXT NOT NULL, ledger_run_id TEXT, state TEXT NOT NULL, start_error TEXT,
|
|
60
|
+
worker_error TEXT, config TEXT NOT NULL, prices TEXT NOT NULL, executor TEXT NOT NULL,
|
|
61
|
+
created_at TEXT NOT NULL, services TEXT, llm_profile_id TEXT);
|
|
62
|
+
CREATE TABLE IF NOT EXISTS commands (
|
|
63
|
+
id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id),
|
|
64
|
+
run_id TEXT NOT NULL REFERENCES runs(id), seq INTEGER NOT NULL, kind TEXT NOT NULL,
|
|
65
|
+
payload TEXT NOT NULL, state TEXT NOT NULL, reason TEXT, created_at TEXT NOT NULL,
|
|
66
|
+
done_at TEXT, UNIQUE (run_id, seq));
|
|
67
|
+
CREATE TABLE IF NOT EXISTS preflight_checks (
|
|
68
|
+
id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id),
|
|
69
|
+
executor_id TEXT NOT NULL REFERENCES executors(id), state TEXT NOT NULL,
|
|
70
|
+
report TEXT, created_at TEXT NOT NULL, done_at TEXT);
|
|
71
|
+
CREATE TABLE IF NOT EXISTS leases (
|
|
72
|
+
key TEXT PRIMARY KEY, holder TEXT NOT NULL, expires_at REAL NOT NULL);
|
|
73
|
+
CREATE TABLE IF NOT EXISTS signin_codes (
|
|
74
|
+
hash TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id),
|
|
75
|
+
expires_at REAL NOT NULL);
|
|
76
|
+
CREATE TABLE IF NOT EXISTS credential_checks (
|
|
77
|
+
id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id),
|
|
78
|
+
credential_id TEXT NOT NULL REFERENCES credentials(id), kind TEXT NOT NULL,
|
|
79
|
+
request TEXT NOT NULL, state TEXT NOT NULL, report TEXT, created_at TEXT NOT NULL,
|
|
80
|
+
done_at TEXT);
|
|
81
|
+
CREATE TABLE IF NOT EXISTS bundled_services (
|
|
82
|
+
tenant_id TEXT PRIMARY KEY REFERENCES tenants(id), state TEXT NOT NULL, reason TEXT,
|
|
83
|
+
port INTEGER, key_sha256 TEXT, key_credential_id TEXT, updated_at TEXT NOT NULL);
|
|
84
|
+
CREATE TABLE IF NOT EXISTS service_settings (
|
|
85
|
+
tenant_id TEXT NOT NULL REFERENCES tenants(id), service TEXT NOT NULL,
|
|
86
|
+
config TEXT NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY (tenant_id, service));
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
_ADDED_COLUMNS = (
|
|
91
|
+
("runs", "services", "TEXT"),
|
|
92
|
+
("runs", "llm_profile_id", "TEXT"),
|
|
93
|
+
("llm_profiles", "version", "INTEGER NOT NULL DEFAULT 1"),
|
|
94
|
+
("llm_profiles", "deleted_at", "TEXT"),
|
|
95
|
+
("profile_checks", "version", "INTEGER NOT NULL DEFAULT 1"),
|
|
96
|
+
("profile_checks", "proposal", "TEXT"),
|
|
97
|
+
("profile_checks", "applied", "INTEGER NOT NULL DEFAULT 0"),
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
_CONNECTION = ("host", "user", "key_credential", "port", "host_key", "known_hosts")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _machines_for_ssh_executors(conn: sqlite3.Connection) -> None:
|
|
104
|
+
"""KB-036: every `ssh` executor names an SSH machine. Executors from before get one each,
|
|
105
|
+
made from their own connection fields, named after the executor."""
|
|
106
|
+
ids = [r[0] for r in conn.execute("SELECT id FROM executors WHERE kind = 'ssh'").fetchall()]
|
|
107
|
+
for xid in ids:
|
|
108
|
+
conn.execute("BEGIN IMMEDIATE") # API and workers open the database concurrently
|
|
109
|
+
try:
|
|
110
|
+
row = conn.execute("SELECT tenant_id, name, config, created_at FROM executors "
|
|
111
|
+
"WHERE id = ?", (xid,)).fetchone()
|
|
112
|
+
cfg = json.loads(row[2]) if row else {"machine": None}
|
|
113
|
+
if row is None or cfg.get("machine"):
|
|
114
|
+
conn.execute("COMMIT")
|
|
115
|
+
continue
|
|
116
|
+
tid, name, at = row[0], row[1], row[3]
|
|
117
|
+
machine = {k: cfg[k] for k in _CONNECTION if cfg.get(k) is not None}
|
|
118
|
+
machine.setdefault("port", 22)
|
|
119
|
+
taken = {r[0] for r in conn.execute(
|
|
120
|
+
"SELECT name FROM ssh_machines WHERE tenant_id = ?", (tid,)).fetchall()}
|
|
121
|
+
mname, n = name, 2
|
|
122
|
+
while mname in taken:
|
|
123
|
+
mname, n = f"{name}-{n}", n + 1
|
|
124
|
+
mid = "m_" + secrets.token_hex(8)
|
|
125
|
+
conn.execute("INSERT INTO ssh_machines VALUES (?, ?, ?, ?, ?)",
|
|
126
|
+
(mid, tid, mname, json.dumps(machine), at))
|
|
127
|
+
conn.execute("UPDATE executors SET config = ? WHERE id = ?",
|
|
128
|
+
(json.dumps({**cfg, "machine": mid, "jumps": []}), xid))
|
|
129
|
+
conn.execute("COMMIT")
|
|
130
|
+
except BaseException:
|
|
131
|
+
conn.execute("ROLLBACK")
|
|
132
|
+
raise
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class Conn(Protocol):
|
|
136
|
+
def execute(self, sql: str, params: Sequence[Any] = ()) -> list[tuple[Any, ...]]: ...
|
|
137
|
+
def rowcount(self, sql: str, params: Sequence[Any] = ()) -> int: ...
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class _SQLiteConn:
|
|
141
|
+
def __init__(self, conn: sqlite3.Connection) -> None:
|
|
142
|
+
self._c = conn
|
|
143
|
+
|
|
144
|
+
def execute(self, sql: str, params: Sequence[Any] = ()) -> list[tuple[Any, ...]]:
|
|
145
|
+
return list(self._c.execute(sql, tuple(params)).fetchall())
|
|
146
|
+
|
|
147
|
+
def rowcount(self, sql: str, params: Sequence[Any] = ()) -> int:
|
|
148
|
+
return self._c.execute(sql, tuple(params)).rowcount
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class Database:
|
|
152
|
+
"""SQLite, one connection per transaction, so threads and processes share it safely."""
|
|
153
|
+
|
|
154
|
+
def __init__(self, path: str | Path) -> None:
|
|
155
|
+
self._path = str(path)
|
|
156
|
+
Path(self._path).parent.mkdir(parents=True, exist_ok=True)
|
|
157
|
+
conn = sqlite3.connect(self._path, isolation_level=None)
|
|
158
|
+
try:
|
|
159
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
160
|
+
conn.executescript(SCHEMA)
|
|
161
|
+
# Columns added after a table was first created: KB-029 (runs' service snapshot)
|
|
162
|
+
# and KB-041 (profile versions, edit proposals, deletion, a run's profile).
|
|
163
|
+
for table, column, decl in _ADDED_COLUMNS:
|
|
164
|
+
cols = {r[1] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()}
|
|
165
|
+
if column not in cols:
|
|
166
|
+
conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {decl}")
|
|
167
|
+
_machines_for_ssh_executors(conn)
|
|
168
|
+
if not conn.execute("SELECT version FROM schema_version").fetchall():
|
|
169
|
+
conn.execute("INSERT INTO schema_version VALUES (?)", (SCHEMA_VERSION,))
|
|
170
|
+
finally:
|
|
171
|
+
conn.close()
|
|
172
|
+
|
|
173
|
+
@contextmanager
|
|
174
|
+
def tx(self, write: bool = False) -> Iterator[Conn]:
|
|
175
|
+
conn = sqlite3.connect(self._path, timeout=30, isolation_level=None)
|
|
176
|
+
try:
|
|
177
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
178
|
+
conn.execute("BEGIN IMMEDIATE" if write else "BEGIN")
|
|
179
|
+
try:
|
|
180
|
+
yield _SQLiteConn(conn)
|
|
181
|
+
except BaseException:
|
|
182
|
+
conn.execute("ROLLBACK")
|
|
183
|
+
raise
|
|
184
|
+
conn.execute("COMMIT")
|
|
185
|
+
finally:
|
|
186
|
+
conn.close()
|
app/executor_config.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Executor configurations a tenant can store (08-app.md §3 `/executors`). The fields mirror
|
|
2
|
+
the backend constructors in `05-executor.md`; job roots are always under the data directory,
|
|
3
|
+
never user-chosen. Hosted mode accepts `ssh` only (INV-5)."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from typing import Annotated, Any, Literal
|
|
8
|
+
|
|
9
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
|
10
|
+
|
|
11
|
+
from app.settings import Mode
|
|
12
|
+
from app.ssh_hosts import parse_host_key
|
|
13
|
+
|
|
14
|
+
KINDS = ("local-docker", "local-process", "ssh")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class _Strict(BaseModel):
|
|
18
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class _Common(_Strict):
|
|
22
|
+
concurrency: Annotated[int, Field(ge=1, le=64)] = 1
|
|
23
|
+
rate_usd_per_hour: Annotated[float, Field(ge=0)] | None = None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class LocalDockerConfig(_Common):
|
|
27
|
+
network: Literal["on", "off"] = "on"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class LocalProcessConfig(_Common):
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class JumpHop(_Strict):
|
|
35
|
+
"""One jump host, resolved from its SSH machine (KB-036). Its host key is required."""
|
|
36
|
+
host: Annotated[str, Field(min_length=1, max_length=253)]
|
|
37
|
+
user: Annotated[str, Field(min_length=1, max_length=64)]
|
|
38
|
+
key_credential: Annotated[str, Field(min_length=1)]
|
|
39
|
+
port: Annotated[int, Field(ge=1, le=65535)] = 22
|
|
40
|
+
host_key: str
|
|
41
|
+
|
|
42
|
+
@field_validator("host_key")
|
|
43
|
+
@classmethod
|
|
44
|
+
def _valid_host_key(cls, v: str) -> str:
|
|
45
|
+
parse_host_key(v)
|
|
46
|
+
return v
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class SSHConfig(_Common):
|
|
50
|
+
host: Annotated[str, Field(min_length=1, max_length=253)]
|
|
51
|
+
user: Annotated[str, Field(min_length=1, max_length=64)]
|
|
52
|
+
key_credential: Annotated[str, Field(min_length=1)]
|
|
53
|
+
port: Annotated[int, Field(ge=1, le=65535)] = 22
|
|
54
|
+
remote_root: Annotated[str, Field(min_length=1)] = "puremethod"
|
|
55
|
+
use_docker: bool = False
|
|
56
|
+
network: Literal["on", "off"] = "on"
|
|
57
|
+
known_hosts: str | None = None # self-hosted only; kept for configurations made before KB-035
|
|
58
|
+
accept_unknown_host: Literal[False] = False # INV-10: never accept any host key
|
|
59
|
+
host_key: str | None = None # the confirmed "<type> <base64>" (INV-10)
|
|
60
|
+
machine: str | None = None # the SSH machine these fields are resolved from (KB-036)
|
|
61
|
+
jumps: Annotated[list[JumpHop], Field(max_length=4)] = []
|
|
62
|
+
|
|
63
|
+
@field_validator("host_key")
|
|
64
|
+
@classmethod
|
|
65
|
+
def _valid_host_key(cls, v: str | None) -> str | None:
|
|
66
|
+
if v is not None:
|
|
67
|
+
parse_host_key(v) # raises SSHSetupError, a ValueError
|
|
68
|
+
return v
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
MODELS: dict[str, type[_Common]] = {"local-docker": LocalDockerConfig,
|
|
72
|
+
"local-process": LocalProcessConfig, "ssh": SSHConfig}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class ExecutorKindRefused(ValueError):
|
|
76
|
+
pass
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def validate(kind: str, config: dict[str, Any], mode: Mode) -> _Common:
|
|
80
|
+
if kind not in MODELS:
|
|
81
|
+
raise ValueError(f"unknown executor kind {kind!r}")
|
|
82
|
+
if mode == "hosted" and kind != "ssh":
|
|
83
|
+
raise ExecutorKindRefused("hosted mode allows only ssh executors (INV-5)")
|
|
84
|
+
cfg = MODELS[kind].model_validate(config)
|
|
85
|
+
if mode == "hosted" and isinstance(cfg, SSHConfig) and cfg.known_hosts is not None:
|
|
86
|
+
# A path names a file on the platform's server (KB-035).
|
|
87
|
+
raise ValueError("hosted mode does not accept known_hosts paths; confirm the host key")
|
|
88
|
+
return cfg
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def credentials_named(kind: str, config: dict[str, Any]) -> list[str]:
|
|
92
|
+
if kind != "ssh":
|
|
93
|
+
return []
|
|
94
|
+
return [str(config["key_credential"]),
|
|
95
|
+
*(str(h["key_credential"]) for h in config.get("jumps") or [])]
|
app/keys.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Sealing credentials and deriving the link key (08-app.md §5 steps 3, 10).
|
|
2
|
+
|
|
3
|
+
There is no decrypt here: that lives in `app.unseal`, which `app.api` may not import
|
|
4
|
+
(INV-2, enforced by import-linter)."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
|
|
10
|
+
from cryptography.hazmat.primitives import hashes
|
|
11
|
+
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
|
12
|
+
from cryptography.hazmat.primitives.kdf.hkdf import HKDF
|
|
13
|
+
|
|
14
|
+
NONCE_BYTES = 12 # 96-bit nonce (KB-021)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def aad(tenant_id: str, credential_id: str) -> bytes:
|
|
18
|
+
"""Binds a ciphertext to its tenant and row, so a moved ciphertext does not decrypt."""
|
|
19
|
+
return f"puremethod-credential\x00{tenant_id}\x00{credential_id}".encode()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def seal(master_key: bytes, tenant_id: str, credential_id: str, value: str) -> tuple[bytes, bytes]:
|
|
23
|
+
nonce = os.urandom(NONCE_BYTES)
|
|
24
|
+
return nonce, AESGCM(master_key).encrypt(nonce, value.encode(), aad(tenant_id, credential_id))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def link_key(master_key: bytes) -> bytes:
|
|
28
|
+
return HKDF(algorithm=hashes.SHA256(), length=32, salt=None,
|
|
29
|
+
info=b"download-links").derive(master_key)
|