seren-probe 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seren_probe/ProbeConfig.yml +52 -0
- seren_probe/README.md +65 -0
- seren_probe/__init__.py +48 -0
- seren_probe/__main__.py +65 -0
- seren_probe/_version.py +24 -0
- seren_probe/app.py +167 -0
- seren_probe/config.py +134 -0
- seren_probe/core/__init__.py +15 -0
- seren_probe/core/docket.py +208 -0
- seren_probe/core/knob_caps.py +51 -0
- seren_probe/core/lint_cli.py +105 -0
- seren_probe/core/linters/__init__.py +18 -0
- seren_probe/core/linters/adapters.py +70 -0
- seren_probe/core/linters/checks.py +211 -0
- seren_probe/core/linters/model.py +134 -0
- seren_probe/core/linters/plan.py +230 -0
- seren_probe/core/linters/quiet.py +116 -0
- seren_probe/core/linters/text.py +39 -0
- seren_probe/core/metrics.py +268 -0
- seren_probe/core/resolve.py +264 -0
- seren_probe/core/seed_dataset.py +892 -0
- seren_probe/core/topology.py +872 -0
- seren_probe/core/topology_emit.py +331 -0
- seren_probe/dockerfiles/corpus.Dockerfile +27 -0
- seren_probe/dockerfiles/loci.Dockerfile +112 -0
- seren_probe/dockerfiles/memory.Dockerfile +20 -0
- seren_probe/mcp/__init__.py +14 -0
- seren_probe/mcp/server.py +147 -0
- seren_probe/mcp/tools.py +75 -0
- seren_probe/routes/__init__.py +13 -0
- seren_probe/routes/config.py +35 -0
- seren_probe/routes/docker.py +604 -0
- seren_probe/routes/eval.py +873 -0
- seren_probe/runtime/__init__.py +11 -0
- seren_probe/runtime/docker_env.py +1149 -0
- seren_probe/runtime/live_eval.py +942 -0
- seren_probe/runtime/live_import.py +119 -0
- seren_probe/runtime/progress.py +127 -0
- seren_probe/runtime/regrade.py +889 -0
- seren_probe/runtime/regrade_live.py +621 -0
- seren_probe/runtime/write_guard.py +148 -0
- seren_probe/viewer/ui/body.html +112 -0
- seren_probe/viewer/ui/header_aside.html +2 -0
- seren_probe/viewer/ui/scripts.js +1803 -0
- seren_probe/viewer/ui/styles.css +1327 -0
- seren_probe/viewer/ui/tabs.html +3 -0
- seren_probe-1.0.0.dist-info/METADATA +337 -0
- seren_probe-1.0.0.dist-info/RECORD +51 -0
- seren_probe-1.0.0.dist-info/WHEEL +5 -0
- seren_probe-1.0.0.dist-info/entry_points.txt +2 -0
- seren_probe-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# ═══════════════════════════════════════════════════════════════════════
|
|
2
|
+
# ProbeConfig.yml - the shipped example topology
|
|
3
|
+
# ═══════════════════════════════════════════════════════════════════════
|
|
4
|
+
# A fuller, multi-project topology: two "projects" (X and Y), each with a
|
|
5
|
+
# vector + a plain Loci and its own Memory and SCC, plus a pool of shared
|
|
6
|
+
# Memory stores and one auto-generated node. Everything not referenced by an
|
|
7
|
+
# explicit SCC falls into the auto-generated catch-all Corpus.
|
|
8
|
+
#
|
|
9
|
+
# It exercises the whole compiler: flags, explicit + auto ports, node
|
|
10
|
+
# auto-generation, and the catch-all sweep - and compiles with zero warnings.
|
|
11
|
+
# For a minimal starting point to copy from, see ProbeConfig.template.yml.
|
|
12
|
+
# ═══════════════════════════════════════════════════════════════════════
|
|
13
|
+
|
|
14
|
+
ProbeConfig:
|
|
15
|
+
StartingPort: 7440
|
|
16
|
+
|
|
17
|
+
Loci:
|
|
18
|
+
LociCount: 5
|
|
19
|
+
LociConfigs:
|
|
20
|
+
- { Name: loci-vector-projectX, Port: 7441, Flags: [vector] }
|
|
21
|
+
- { Name: loci-plain-projectX, Port: 7442 }
|
|
22
|
+
- { Name: loci-vector-projectY, Port: 7443, Flags: [vector] }
|
|
23
|
+
- { Name: loci-plain-projectY, Port: 7444 }
|
|
24
|
+
# + 1 auto-generated Loci (no flags, auto port) to reach LociCount 5
|
|
25
|
+
|
|
26
|
+
Memory:
|
|
27
|
+
MemoryCount: 7
|
|
28
|
+
MemoryConfigs:
|
|
29
|
+
- { Name: memory-projectX, Port: 7445 }
|
|
30
|
+
- { Name: memory-projectY, Port: 7446 }
|
|
31
|
+
- { Name: memory-shared-a, Port: 7450 }
|
|
32
|
+
- { Name: memory-shared-b, Port: 7451 }
|
|
33
|
+
- { Name: memory-shared-c, Port: 7452 }
|
|
34
|
+
- { Name: memory-shared-d, Port: 7453 }
|
|
35
|
+
# + 1 auto-generated Memory to reach MemoryCount 7
|
|
36
|
+
|
|
37
|
+
Corpus:
|
|
38
|
+
CorpusCount: 3
|
|
39
|
+
CorpusConfigs:
|
|
40
|
+
- Name: corpus-projectX
|
|
41
|
+
Port: 7460
|
|
42
|
+
Stores:
|
|
43
|
+
- { Store: loci-vector-projectX }
|
|
44
|
+
- { Store: loci-plain-projectX }
|
|
45
|
+
- { Store: memory-projectX }
|
|
46
|
+
- Name: corpus-projectY
|
|
47
|
+
Port: 7461
|
|
48
|
+
Stores:
|
|
49
|
+
- { Store: loci-vector-projectY }
|
|
50
|
+
- { Store: memory-projectY }
|
|
51
|
+
# + 1 auto-generated catch-all Corpus fanning the 7 unreferenced stores:
|
|
52
|
+
# loci-plain-projectY, the auto Loci, memory-shared-a..d, the auto Memory
|
seren_probe/README.md
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# `seren_probe/` - what lives where
|
|
2
|
+
|
|
3
|
+
Nineteen files in a flat root is a lot to land in cold. This is the map. It's the
|
|
4
|
+
same grouping a `core/` + `runtime/` split would give you - drawn, not moved, so
|
|
5
|
+
nothing's import path changes while you find your feet.
|
|
6
|
+
|
|
7
|
+
The boundary below isn't a suggestion: **`tests/test_layering.py` enforces it.**
|
|
8
|
+
Nothing in the PURE layer may import `httpx`, and no module anywhere may name a
|
|
9
|
+
live Seren port (7420–7424) in executable code. Both fail the build.
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## 🧠 PURE - config, parsing, maths. Touches no network, no Docker, no store.
|
|
14
|
+
|
|
15
|
+
| module | what it is |
|
|
16
|
+
|---|---|
|
|
17
|
+
| `topology.py` | **the compiler.** ProbeConfig → validated `CompiledTopology`. Owns `REGRADE_KNOBS` + `KNOBS_NEEDING_CAPABILITY`. It's the landmine guard: two corpora that secretly fan the same stores, a decoy wired into a measured SCC - all caught here, before Docker breathes. |
|
|
18
|
+
| `topology_emit.py` | `CompiledTopology` → docker-compose + one `seren-corpus-callosum.yaml` per corpus, with correct-by-construction store wiring. |
|
|
19
|
+
| `seed_dataset.py` | seed/question **loading + validation**, the seeder (`seed_from_plan`), and `rehydrate_ref_map` (rebuilds the answer key from a live store when a pod was adopted). |
|
|
20
|
+
| `resolve.py` | decides which store gets which seed, and which questions get scored. |
|
|
21
|
+
| `question_lint.py` | **can this question be answered at all?** Three tiers: expectation-in-corpus, lexical rail (`multihop`), no rail at all (`unbridged`). An unanswerable question looks *exactly* like a retrieval failure on the dashboard. |
|
|
22
|
+
| `metrics.py` | HR / MRR / P@k / R@k / nDCG / IoU / P-Ω + docket coverage & density. |
|
|
23
|
+
| `docket.py` | the with-edges vs without-edges SCC comparison. |
|
|
24
|
+
| `knob_caps.py` | refuses to sweep a knob the SCC doesn't advertise. An ignored knob produces a flatline that reads as a ceiling. |
|
|
25
|
+
| `lint_cli.py` | `python -m seren_probe.lint_cli` - non-zero exit gate for CI. |
|
|
26
|
+
|
|
27
|
+
## 🌐 RUNTIME - everything that speaks to a running thing. `httpx` lives only here.
|
|
28
|
+
|
|
29
|
+
| module | what it is |
|
|
30
|
+
|---|---|
|
|
31
|
+
| **`write_guard.py`** | **THE INTERLOCK. Read this first.** SerenProbe cannot write to a store it did not spin up. Fail-closed: no topology → nothing writable, anywhere. |
|
|
32
|
+
| `live_eval.py` | `run_topology_evaluation` - evals every store in a topology as its own column. The main event. |
|
|
33
|
+
| `live_import.py` | read-only copy of a real store's data into a container. |
|
|
34
|
+
| `regrade_live.py` | the **live** SCC knob sweep - reconfigures the running container per combo. This is what the ⚙ Regrades button drives. |
|
|
35
|
+
| `regrade.py` | the **capture/replay** sweep - freeze each store's candidate pool once, replay the real Federation in-process over the frozen trace. Read-only. Owns `DEFAULT_GRID`. |
|
|
36
|
+
| `docker_env.py` | container lifecycle: compose up/down/ps, health-gating, pod adoption, topology-state persistence. |
|
|
37
|
+
|
|
38
|
+
## ⚙️ SERVICE
|
|
39
|
+
|
|
40
|
+
| module | what it is |
|
|
41
|
+
|---|---|
|
|
42
|
+
| `app.py` | the FastAPI app. Auth, request logging, viewer, route mounts, optional MCP. |
|
|
43
|
+
| `config.py` | `seren-probe.yaml` → typed config. Defaults → yaml → env. |
|
|
44
|
+
| `__main__.py` | `python -m seren_probe`. |
|
|
45
|
+
| `routes/` | `eval.py` (run + regrade) · `docker.py` (start/stop/status/adopt) · `config.py` |
|
|
46
|
+
| `viewer/ui/` | the dashboard: `body.html` · `tabs.html` · `scripts.js` · `styles.css` |
|
|
47
|
+
| `mcp/` | optional MCP surface (`pip install seren-probe[mcp]`) |
|
|
48
|
+
| `dockerfiles/` | the three store images the emitter builds from |
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## The four rules this package learned the hard way
|
|
53
|
+
|
|
54
|
+
1. **The summary lies; the rows tell the truth.** Every confident read of an
|
|
55
|
+
aggregate on 2026-07-13 was wrong, and wrong in the direction that made the
|
|
56
|
+
story tidier. Dump the tier. Read every row.
|
|
57
|
+
2. **A knob swept in one direction is half a knob.** `loci_weight` was only ever
|
|
58
|
+
swept *down*, toward the other store. The entire win was above 1.0.
|
|
59
|
+
3. **Ground truth must be re-resolvable from the live system.** A minted UUID held
|
|
60
|
+
in the seeding process's RAM is not ground truth, it's a receipt. `expect_key`
|
|
61
|
+
survived an adopted pod; `expect_ref` didn't, and reported a perfectly healthy
|
|
62
|
+
store as dead for a full day.
|
|
63
|
+
4. **Interlocks catch what discipline doesn't.** Six live-store hazards and eleven
|
|
64
|
+
silent measurement bugs in one day. Discipline caught none of them. Every guard
|
|
65
|
+
we built caught one - including two the guards found *after* we'd stopped looking.
|
seren_probe/__init__.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""
|
|
2
|
+
seren_probe - RAG evaluation toolkit for Seren's memory architecture.
|
|
3
|
+
|
|
4
|
+
Evaluates retrieval quality across:
|
|
5
|
+
- SerenLoci (SQLite factstore: exact/lexical/vector)
|
|
6
|
+
- SerenMemory (3-tier Chroma DB: short/near/long)
|
|
7
|
+
- SerenCorpusCallosum (federated RRF merge of both)
|
|
8
|
+
|
|
9
|
+
Standard metrics: Hit Rate, MRR, Precision@k, Recall@k, NDCG@k.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
# Version flows from the git tag via setuptools-scm (written to _version.py at
|
|
14
|
+
# build time, read here). Fallback only fires in a bare source checkout that was
|
|
15
|
+
# never built. Mirrors the family so every seren_* exposes __version__ alike.
|
|
16
|
+
try:
|
|
17
|
+
from ._version import version as __version__
|
|
18
|
+
except Exception: # noqa: BLE001 - source checkout without a build
|
|
19
|
+
__version__ = "0.0.0+unknown"
|
|
20
|
+
|
|
21
|
+
# NOT exported anymore: EvalDataset / EvalQuery / seed_synthetic_dataset (dataset.py)
|
|
22
|
+
# and LociEvaluator / MemoryEvaluator / CorpusCallosumEvaluator (evaluators.py).
|
|
23
|
+
#
|
|
24
|
+
# dataset.py is the SYNTHETIC corpus generator. Importing it here meant it loaded on
|
|
25
|
+
# every `import seren_probe` -- so the fake data was in the import graph of every
|
|
26
|
+
# code path that so much as touched an eval, sitting next to a module whose defaults
|
|
27
|
+
# named the operator's real stores. That combination is how synthetic content ends up
|
|
28
|
+
# inside a live SerenMemory, and it did.
|
|
29
|
+
#
|
|
30
|
+
# The in-process legacy suite (dataset / evaluators / runner) is in _attic/. Nothing
|
|
31
|
+
# in the topology path needs it: seeding is config-driven via seed_dataset.py, and
|
|
32
|
+
# evaluation goes through live_eval.run_topology_evaluation, which only ever addresses
|
|
33
|
+
# containers SerenProbe spun up itself.
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"__version__",
|
|
37
|
+
"EvalMetrics",
|
|
38
|
+
"SerenProbeConfig",
|
|
39
|
+
"load_config",
|
|
40
|
+
"DockerEnv",
|
|
41
|
+
"DockerEnvState",
|
|
42
|
+
"launch_and_eval",
|
|
43
|
+
"build_image",
|
|
44
|
+
"start_container",
|
|
45
|
+
"wait_for_healthy",
|
|
46
|
+
"stop_container",
|
|
47
|
+
"container_status",
|
|
48
|
+
]
|
seren_probe/__main__.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Entry point: python -m seren_probe [--config path] (also the `seren-probe` script)
|
|
3
|
+
|
|
4
|
+
Boots the FastAPI app with uvicorn using the resolved config.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
import uvicorn
|
|
12
|
+
|
|
13
|
+
from .app import create_app
|
|
14
|
+
from .config import load_config
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _force_utf8_stdio() -> None:
|
|
18
|
+
"""Make stdout/stderr UTF-8 regardless of OS locale."""
|
|
19
|
+
for stream in (sys.stdout, sys.stderr):
|
|
20
|
+
try:
|
|
21
|
+
stream.reconfigure(encoding="utf-8")
|
|
22
|
+
except (AttributeError, ValueError):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _maybe_inject_truststore(cfg, log=print) -> None:
|
|
27
|
+
"""If tls.trust_system_store is on, route Python TLS through the OS trust
|
|
28
|
+
store via `truststore`. MUST run before any SSLContext is created."""
|
|
29
|
+
if not cfg.tls.trust_system_store:
|
|
30
|
+
return
|
|
31
|
+
try:
|
|
32
|
+
import truststore
|
|
33
|
+
except ImportError:
|
|
34
|
+
log("[seren-probe] tls.trust_system_store is ON but 'truststore' isn't "
|
|
35
|
+
"installed. Install the corp extra: pip install 'seren-probe[corp]' "
|
|
36
|
+
"(continuing with certifi defaults).")
|
|
37
|
+
return
|
|
38
|
+
truststore.inject_into_ssl()
|
|
39
|
+
log("[seren-probe] TLS: using OS trust store (truststore injected)")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def main() -> None:
|
|
43
|
+
_force_utf8_stdio()
|
|
44
|
+
parser = argparse.ArgumentParser(
|
|
45
|
+
prog="seren-probe",
|
|
46
|
+
description="SerenProbe - RAG evaluation toolkit. Operator dashboard.")
|
|
47
|
+
parser.add_argument(
|
|
48
|
+
"--config", "-c", default=None,
|
|
49
|
+
help="Path to seren-probe.yaml (default: ./seren-probe.yaml or "
|
|
50
|
+
"$SEREN_PROBE_CONFIG, falling back to built-in defaults).")
|
|
51
|
+
args = parser.parse_args()
|
|
52
|
+
|
|
53
|
+
cfg = load_config(args.config)
|
|
54
|
+
_maybe_inject_truststore(cfg)
|
|
55
|
+
app = create_app(cfg)
|
|
56
|
+
|
|
57
|
+
print(f"[seren-probe] listening on {cfg.server.host}:{cfg.server.port}")
|
|
58
|
+
print(f"[seren-probe] auth: "
|
|
59
|
+
f"{'enabled' if cfg.server.bearer_token else 'DISABLED (no token)'}")
|
|
60
|
+
|
|
61
|
+
uvicorn.run(app, host=cfg.server.host, port=cfg.server.port, log_level="info")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
if __name__ == "__main__":
|
|
65
|
+
main()
|
seren_probe/_version.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '1.0.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 0, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = 'gf7f16db10'
|
seren_probe/app.py
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""
|
|
2
|
+
seren_probe.app
|
|
3
|
+
════════════════════════════════════════════════════════════════════════
|
|
4
|
+
|
|
5
|
+
The FastAPI application for the SerenProbe RAG evaluation suite. Wires the
|
|
6
|
+
operator dashboard with endpoints for running evaluations and live configuration.
|
|
7
|
+
|
|
8
|
+
ENDPOINTS:
|
|
9
|
+
GET / - service info + version
|
|
10
|
+
GET /health - liveness
|
|
11
|
+
GET /viewer - the SerenProbe operator dashboard
|
|
12
|
+
GET /eval/results - latest full eval results
|
|
13
|
+
POST /eval/run - run full evaluation against live stores
|
|
14
|
+
GET /eval/config - current viewer config
|
|
15
|
+
POST /eval/config - update viewer configuration
|
|
16
|
+
POST /docker/start - build + start Docker test environment
|
|
17
|
+
POST /docker/stop - stop + remove Docker environment
|
|
18
|
+
GET /docker/status - current Docker environment status
|
|
19
|
+
POST /docker/run-eval - one-shot: spin up, eval, tear down
|
|
20
|
+
GET /docker/config - list deployment configs + Docker availability
|
|
21
|
+
POST /docker/config/save - save a new Dockerfile/compose config
|
|
22
|
+
POST /docker/config/activate - set the active deployment config
|
|
23
|
+
POST /docker/validate - validate Docker install + config syntax
|
|
24
|
+
MCP /mcp - MCP tools (when [mcp] extras installed)
|
|
25
|
+
|
|
26
|
+
Deliberately parallel to SerenLoci / SerenMemory: same auth posture, same
|
|
27
|
+
conditional-MCP-mount shape, same public-paths set.
|
|
28
|
+
"""
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import logging
|
|
32
|
+
import time
|
|
33
|
+
from contextlib import asynccontextmanager, AsyncExitStack
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
|
|
36
|
+
from fastapi import FastAPI, Request
|
|
37
|
+
from fastapi.responses import HTMLResponse
|
|
38
|
+
|
|
39
|
+
from .config import SerenProbeConfig, load_config
|
|
40
|
+
from .routes import eval as eval_routes
|
|
41
|
+
from .routes import config as config_routes
|
|
42
|
+
from .routes import docker as docker_routes
|
|
43
|
+
|
|
44
|
+
from seren_meninges import get_version
|
|
45
|
+
from seren_meninges.auth import bearer_auth_middleware
|
|
46
|
+
from seren_meninges.viewer import render_from_dir
|
|
47
|
+
from seren_sinew.request_log import RequestLoggingMiddleware
|
|
48
|
+
|
|
49
|
+
from . import __version__ as _fallback_version
|
|
50
|
+
APP_VERSION = get_version("seren-probe", fallback=_fallback_version)
|
|
51
|
+
|
|
52
|
+
logger = logging.getLogger(__name__)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def create_app(config: SerenProbeConfig | None = None) -> FastAPI:
|
|
56
|
+
cfg = config or load_config()
|
|
57
|
+
bearer = cfg.server.resolve_bearer()
|
|
58
|
+
|
|
59
|
+
# Shared mutable state that MCP tools also read/write.
|
|
60
|
+
_mcp_state_ref: dict = {}
|
|
61
|
+
|
|
62
|
+
@asynccontextmanager
|
|
63
|
+
async def lifespan(app: FastAPI):
|
|
64
|
+
# -- Startup --
|
|
65
|
+
app.state.config = cfg
|
|
66
|
+
app.state.eval_results = {}
|
|
67
|
+
app.state.docker_state = {}
|
|
68
|
+
app.state.docker_config_active = "default"
|
|
69
|
+
app.state.docker_config_path = None
|
|
70
|
+
app.state.store_config = {
|
|
71
|
+
"memory_url": cfg.stores.memory_url,
|
|
72
|
+
"loci_nv_url": cfg.stores.loci_nv_url,
|
|
73
|
+
"loci_v_url": cfg.stores.loci_v_url,
|
|
74
|
+
"scc_nv_url": cfg.stores.scc_nv_url,
|
|
75
|
+
"scc_v_url": cfg.stores.scc_v_url,
|
|
76
|
+
"capture_path": cfg.stores.capture_path,
|
|
77
|
+
}
|
|
78
|
+
# MCP state ref - a dict the MCP tools mutate, same references as the
|
|
79
|
+
# route handlers read. Both see the same values.
|
|
80
|
+
app.state._mcp_state_ref = _mcp_state_ref
|
|
81
|
+
_mcp_state_ref["eval_results"] = app.state.eval_results
|
|
82
|
+
|
|
83
|
+
print(f"[seren-probe] viewer ready on {cfg.server.host}:{cfg.server.port}")
|
|
84
|
+
print(f"[seren-probe] stores: memory={cfg.stores.memory_url} "
|
|
85
|
+
f"loci-nv={cfg.stores.loci_nv_url} loci-v={cfg.stores.loci_v_url} "
|
|
86
|
+
f"scc-nv={cfg.stores.scc_nv_url} scc-v={cfg.stores.scc_v_url}")
|
|
87
|
+
|
|
88
|
+
# -- Optional MCP server --
|
|
89
|
+
try:
|
|
90
|
+
from .mcp.server import mount_mcp_routes
|
|
91
|
+
mcp_server = mount_mcp_routes(app)
|
|
92
|
+
except ImportError as exc:
|
|
93
|
+
mcp_server = None
|
|
94
|
+
print(f"[seren-probe] MCP surface not available; HTTP-only mode ({exc})")
|
|
95
|
+
except Exception as exc: # noqa: BLE001
|
|
96
|
+
mcp_server = None
|
|
97
|
+
print(f"[seren-probe] MCP mount failed: {exc!r} - continuing without MCP")
|
|
98
|
+
|
|
99
|
+
# Enter the MCP session manager's task group if mounted.
|
|
100
|
+
async with AsyncExitStack() as _mcp_stack:
|
|
101
|
+
session_manager = getattr(mcp_server, "session_manager", None)
|
|
102
|
+
if session_manager is not None:
|
|
103
|
+
await _mcp_stack.enter_async_context(session_manager.run())
|
|
104
|
+
print("[seren-probe] MCP session manager running")
|
|
105
|
+
yield
|
|
106
|
+
|
|
107
|
+
# -- Shutdown: tear down any running Docker container --
|
|
108
|
+
docker_state = app.state.docker_state
|
|
109
|
+
if docker_state and docker_state.get("container_id"):
|
|
110
|
+
try:
|
|
111
|
+
from .runtime.docker_env import stop_container as _stop
|
|
112
|
+
_stop(docker_state)
|
|
113
|
+
except Exception as e:
|
|
114
|
+
logger.warning("Docker teardown on shutdown: %s", e)
|
|
115
|
+
|
|
116
|
+
print("[seren-probe] shut down")
|
|
117
|
+
|
|
118
|
+
app = FastAPI(
|
|
119
|
+
title="SerenProbe",
|
|
120
|
+
description="RAG evaluation toolkit for Seren's memory architecture.",
|
|
121
|
+
version=APP_VERSION,
|
|
122
|
+
lifespan=lifespan,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
# -- Bearer auth --
|
|
126
|
+
app.add_middleware(bearer_auth_middleware(bearer))
|
|
127
|
+
|
|
128
|
+
# -- Request logging --
|
|
129
|
+
app.add_middleware(
|
|
130
|
+
RequestLoggingMiddleware,
|
|
131
|
+
service_name="seren-probe",
|
|
132
|
+
env_prefix="SEREN_PROBE",
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
viewer_dir = Path(__file__).resolve().parent / "viewer" / "ui"
|
|
136
|
+
|
|
137
|
+
# -- Info routes --
|
|
138
|
+
@app.get("/")
|
|
139
|
+
async def root():
|
|
140
|
+
return {
|
|
141
|
+
"service": "SerenProbe",
|
|
142
|
+
"version": APP_VERSION,
|
|
143
|
+
"stores": 5,
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
@app.get("/health")
|
|
147
|
+
async def health():
|
|
148
|
+
return {"ok": True, "ts": time.time(), "version": APP_VERSION}
|
|
149
|
+
|
|
150
|
+
# -- The operator dashboard viewer --
|
|
151
|
+
@app.get("/viewer")
|
|
152
|
+
async def viewer():
|
|
153
|
+
html = render_from_dir(
|
|
154
|
+
viewer_dir,
|
|
155
|
+
title="SerenProbe",
|
|
156
|
+
brand="Seren<b>Probe</b> · RAG Evaluation Suite",
|
|
157
|
+
subtitle=f"v{APP_VERSION} · probe retrieval quality",
|
|
158
|
+
accent="#8fffb4",
|
|
159
|
+
)
|
|
160
|
+
return HTMLResponse(html)
|
|
161
|
+
|
|
162
|
+
# -- Route subpackage mounts --
|
|
163
|
+
app.include_router(eval_routes.router)
|
|
164
|
+
app.include_router(config_routes.router)
|
|
165
|
+
app.include_router(docker_routes.router)
|
|
166
|
+
|
|
167
|
+
return app
|
seren_probe/config.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""
|
|
2
|
+
seren_probe.config
|
|
3
|
+
════════════════════════════════════════════════════════════════════════
|
|
4
|
+
|
|
5
|
+
Typed configuration for the SerenProbe evaluation viewer. Same pattern as
|
|
6
|
+
the family: dataclass-based (like SCC), adopting the shared ServerConfig
|
|
7
|
+
and TlsConfig from SerenMeninges so the whole family shares one definition.
|
|
8
|
+
|
|
9
|
+
Resolution order (later wins):
|
|
10
|
+
1. Defaults (this file)
|
|
11
|
+
2. seren-probe.yaml (path from --config or ./seren-probe.yaml)
|
|
12
|
+
3. Environment variables (SEREN_PROBE_*)
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import logging
|
|
17
|
+
import os
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any, Optional
|
|
21
|
+
|
|
22
|
+
import yaml
|
|
23
|
+
|
|
24
|
+
# Shared server/tls config blocks from Meninges - ONE definition for the family.
|
|
25
|
+
from seren_meninges import ServerConfig, TlsConfig
|
|
26
|
+
|
|
27
|
+
log = logging.getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class StoreUrlsConfig:
|
|
32
|
+
"""URLs for live stores the operator may point the viewer at.
|
|
33
|
+
|
|
34
|
+
THE DEFAULTS ARE DELIBERATELY EMPTY. They used to be:
|
|
35
|
+
|
|
36
|
+
memory_url = "http://127.0.0.1:7420" <- the operator's REAL SerenMemory
|
|
37
|
+
loci_v_url = "http://127.0.0.1:7421" <- the operator's REAL SerenLoci
|
|
38
|
+
loci_nv_url = "http://127.0.0.1:7422"
|
|
39
|
+
scc_nv_url = "http://127.0.0.1:7423"
|
|
40
|
+
scc_v_url = "http://127.0.0.1:7424"
|
|
41
|
+
|
|
42
|
+
A tool whose job is to MANUFACTURE SYNTHETIC DATA shipped with the addresses of
|
|
43
|
+
the operator's live brain preloaded into its config. Those defaults fed
|
|
44
|
+
app.state.store_config, which fed the legacy /eval/run fallback, which SEEDED
|
|
45
|
+
those stores if it found them empty. Exactly one thing stood between that and a
|
|
46
|
+
contaminated memory store: the store happening to be non-empty.
|
|
47
|
+
|
|
48
|
+
An address you have to TYPE is a decision. An address that arrives as a default
|
|
49
|
+
is an accident waiting for a tired Tuesday.
|
|
50
|
+
|
|
51
|
+
Nothing in the topology path reads these anymore (eval and regrade only ever
|
|
52
|
+
address containers SerenProbe spun up itself, and write_guard refuses anything
|
|
53
|
+
else at the transport). They survive only as an operator-settable field on
|
|
54
|
+
/eval/config. tests/test_layering.py enforces that no module in this package
|
|
55
|
+
contains a live-store port literal -- do not put them back.
|
|
56
|
+
"""
|
|
57
|
+
memory_url: str = ""
|
|
58
|
+
loci_nv_url: str = ""
|
|
59
|
+
loci_v_url: str = ""
|
|
60
|
+
scc_nv_url: str = ""
|
|
61
|
+
scc_v_url: str = ""
|
|
62
|
+
capture_path: str = ""
|
|
63
|
+
|
|
64
|
+
@classmethod
|
|
65
|
+
def from_dict(cls, d: Optional[dict[str, Any]]) -> "StoreUrlsConfig":
|
|
66
|
+
d = d or {}
|
|
67
|
+
return cls(
|
|
68
|
+
memory_url=str(d.get("memory_url", "")),
|
|
69
|
+
loci_nv_url=str(d.get("loci_nv_url", "")),
|
|
70
|
+
loci_v_url=str(d.get("loci_v_url", "")),
|
|
71
|
+
scc_nv_url=str(d.get("scc_nv_url", "")),
|
|
72
|
+
scc_v_url=str(d.get("scc_v_url", "")),
|
|
73
|
+
capture_path=str(d.get("capture_path", "")),
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass
|
|
78
|
+
class SerenProbeConfig:
|
|
79
|
+
"""The whole service: server + tls + store URLs."""
|
|
80
|
+
server: ServerConfig = field(default_factory=lambda: ServerConfig(port=7430))
|
|
81
|
+
tls: TlsConfig = field(default_factory=TlsConfig)
|
|
82
|
+
stores: StoreUrlsConfig = field(default_factory=StoreUrlsConfig)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _apply_env_overrides(cfg: SerenProbeConfig) -> SerenProbeConfig:
|
|
86
|
+
"""SEREN_PROBE_* env wins last, same precedence as the family's SEREN_<X>_*."""
|
|
87
|
+
env = os.environ
|
|
88
|
+
if v := env.get("SEREN_PROBE_HOST"):
|
|
89
|
+
cfg.server.host = v
|
|
90
|
+
if v := env.get("SEREN_PROBE_PORT"):
|
|
91
|
+
cfg.server.port = int(v)
|
|
92
|
+
if v := env.get("SEREN_PROBE_BEARER_TOKEN"):
|
|
93
|
+
cfg.server.bearer_token = v
|
|
94
|
+
if v := env.get("SEREN_PROBE_BEARER_TOKEN_ENV"):
|
|
95
|
+
cfg.server.bearer_token_env = v
|
|
96
|
+
if v := env.get("SEREN_PROBE_BEARER_TOKEN_KEYRING"):
|
|
97
|
+
cfg.server.bearer_token_keyring = v
|
|
98
|
+
if v := env.get("SEREN_PROBE_TRUST_SYSTEM_STORE"):
|
|
99
|
+
cfg.tls.trust_system_store = v.lower() in ("1", "true", "yes", "on")
|
|
100
|
+
if v := env.get("SEREN_PROBE_MEMORY_URL"):
|
|
101
|
+
cfg.stores.memory_url = v
|
|
102
|
+
if v := env.get("SEREN_PROBE_LOCI_NV_URL"):
|
|
103
|
+
cfg.stores.loci_nv_url = v
|
|
104
|
+
if v := env.get("SEREN_PROBE_LOCI_V_URL"):
|
|
105
|
+
cfg.stores.loci_v_url = v
|
|
106
|
+
if v := env.get("SEREN_PROBE_SCC_NV_URL"):
|
|
107
|
+
cfg.stores.scc_nv_url = v
|
|
108
|
+
if v := env.get("SEREN_PROBE_SCC_V_URL"):
|
|
109
|
+
cfg.stores.scc_v_url = v
|
|
110
|
+
if v := env.get("SEREN_PROBE_CAPTURE_PATH"):
|
|
111
|
+
cfg.stores.capture_path = v
|
|
112
|
+
return cfg
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def load_config(path: Optional[str] = None) -> SerenProbeConfig:
|
|
116
|
+
"""Defaults -> yaml -> env (later wins). A missing file is fine - defaults
|
|
117
|
+
+ env is a valid zero-config run. YAML is imported lazily so the core
|
|
118
|
+
import path stays light."""
|
|
119
|
+
data: dict[str, Any] = {}
|
|
120
|
+
candidate = path or os.environ.get("SEREN_PROBE_CONFIG") or "seren-probe.yaml"
|
|
121
|
+
cfg_path = Path(os.path.expanduser(candidate))
|
|
122
|
+
if cfg_path.is_file():
|
|
123
|
+
try:
|
|
124
|
+
with open(cfg_path) as f:
|
|
125
|
+
data = yaml.safe_load(f) or {}
|
|
126
|
+
except Exception: # noqa: BLE001 - unreadable degrades to defaults
|
|
127
|
+
data = {}
|
|
128
|
+
|
|
129
|
+
server = ServerConfig.from_dict(data.get("server"), default_port=7430)
|
|
130
|
+
tls = TlsConfig.from_dict(data.get("tls"))
|
|
131
|
+
stores = StoreUrlsConfig.from_dict(data.get("stores"))
|
|
132
|
+
|
|
133
|
+
cfg = SerenProbeConfig(server=server, tls=tls, stores=stores)
|
|
134
|
+
return _apply_env_overrides(cfg)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""
|
|
2
|
+
seren_probe.core -- the pure layer.
|
|
3
|
+
|
|
4
|
+
Everything in here is enforced by tests/test_layering.py to be free of httpx,
|
|
5
|
+
Docker, and any live-store port literal. It compiles config, parses seeds,
|
|
6
|
+
computes metrics, and emits compose files. It never touches the network.
|
|
7
|
+
|
|
8
|
+
The folder exists for navigation. The boundary exists because a test says so.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
# Re-export every module so `from seren_probe.core.foo import bar` works and,
|
|
13
|
+
# via the package __init__ shim, `from seren_probe.foo import bar` keeps
|
|
14
|
+
# working unchanged.
|
|
15
|
+
from . import topology, topology_emit, seed_dataset, resolve, metrics, docket, knob_caps, lint_cli
|