prometheusnn 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/CHANGELOG.md +39 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/PKG-INFO +2 -2
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/pyproject.toml +1 -1
- prometheusnn-0.2.2/src/prometheus/core/config.py +106 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/inference/novelty.py +17 -8
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/server/tools/evaluation.py +1 -1
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/server/tools/management.py +9 -4
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/ui_server.py +120 -5
- prometheusnn-0.2.2/tests/test_novelty.py +100 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_server.py +85 -4
- prometheusnn-0.2.2/tests/test_ui_server.py +101 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/uv.lock +1 -1
- prometheusnn-0.2.0/src/prometheus/core/config.py +0 -51
- prometheusnn-0.2.0/tests/test_novelty.py +0 -50
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/.github/workflows/ci.yml +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/.github/workflows/publish.yml +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/.gitignore +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/LICENSE +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/README.md +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/docs/beam-search-strategy.md +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/docs/roadmap-self-aware-server.md +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/scripts/parse_hts.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/scripts/parse_mesh.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/__init__.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/core/__init__.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/core/embedding.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/core/logging.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/core/lru.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/core/mcp_safety.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/core/responses.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/data/__init__.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/data/loader.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/data/splits.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/data/taxonomy.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/inference/__init__.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/inference/beam_search.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/inference/predictor.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/inference/router.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/inference/summary.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/insights.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/intake/__init__.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/intake/detector.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/intake/models.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/intake/profiler.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/intake/recommender.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/intake/scoper.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/py.typed +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/server/INSTRUCTIONS.md +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/server/__init__.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/server/tools/__init__.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/server/tools/inference.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/server/tools/intake.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/server/tools/training.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/storage/__init__.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/storage/model_store.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/training/__init__.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/training/calibration.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/training/cascade.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/training/classifier.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/training/noise.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/src/prometheus/training/trainer.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/__init__.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/conftest.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/fixtures/naics_test.csv +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_beam_search.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_calibration.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_classifier.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_insights.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_intake.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_integration.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_noise.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_predictor.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_router.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_splits.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_storage.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_summary.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_taxonomy.py +0 -0
- {prometheusnn-0.2.0 → prometheusnn-0.2.2}/tests/test_trainer.py +0 -0
|
@@ -5,6 +5,45 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.2.2] - 2026-10-30
|
|
9
|
+
|
|
10
|
+
### Security
|
|
11
|
+
- **Novelty data is loaded without pickle.** `novelty_centroids.npz` was loaded
|
|
12
|
+
with `allow_pickle=True`, so a crafted model directory could run code in the
|
|
13
|
+
server when any tool loaded that model. In 0.2.0 and earlier this was
|
|
14
|
+
reachable through the `model_name` path traversal fixed in 0.2.1; in 0.2.1 it
|
|
15
|
+
still applied to a malicious model placed in the models directory. Both
|
|
16
|
+
novelty files now load with `allow_pickle=False`. Models saved by
|
|
17
|
+
PrometheusNN contain only numeric and string arrays and load unchanged; a
|
|
18
|
+
file holding pickled objects is refused with an error asking for a retrain.
|
|
19
|
+
|
|
20
|
+
## [0.2.1] - 2026-10-30
|
|
21
|
+
|
|
22
|
+
### Security
|
|
23
|
+
- **Model-name path traversal.** `delete_model`, `describe_model`,
|
|
24
|
+
`submit_feedback` and every other tool that takes a `model_name` now accept
|
|
25
|
+
only a single path segment (`[A-Za-z0-9._-]`, no `..`), and every resolved
|
|
26
|
+
path must stay under its base directory. In 0.2.0, `delete_model("..")`
|
|
27
|
+
deleted all of `PROMETHEUS_HOME`, an absolute name could delete any
|
|
28
|
+
directory the process could write, and `submit_feedback` could append to a
|
|
29
|
+
file outside the feedback directory. Upgrade if the MCP server or the UI
|
|
30
|
+
server is reachable by anything you do not fully trust.
|
|
31
|
+
- **Unauthenticated UI WebSocket.** `prometheusnn-ui` now requires a token
|
|
32
|
+
(`?token=` on the WebSocket URL; set with `--token` or
|
|
33
|
+
`PROMETHEUS_UI_TOKEN`, otherwise generated and printed at startup) and
|
|
34
|
+
rejects browser handshakes from origins outside an allowlist (the hosted
|
|
35
|
+
demo and `localhost:3000` by default; add more with `--allowed-origin`).
|
|
36
|
+
In 0.2.0 any web page open in the same browser could drive every tool.
|
|
37
|
+
- `delete_model` and `export_model` are withheld from the UI unless the
|
|
38
|
+
server is started with `--allow-destructive`.
|
|
39
|
+
|
|
40
|
+
### Changed
|
|
41
|
+
- `delete_model` refuses a directory that has no `manifest.json`.
|
|
42
|
+
- `export_model` writes only under the exports directory
|
|
43
|
+
(`PROMETHEUS_EXPORT_DIR`, default `<PROMETHEUS_HOME>/exports`). A relative
|
|
44
|
+
`output_path` is placed there; an absolute path outside it is refused.
|
|
45
|
+
**Breaking** for callers that exported to arbitrary absolute paths.
|
|
46
|
+
|
|
8
47
|
## [0.2.0] - 2026-04-30
|
|
9
48
|
|
|
10
49
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: prometheusnn
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Frictionless hierarchical text classification — an MCP server that guides you from CSV to trained classifier
|
|
5
5
|
Project-URL: Homepage, https://github.com/mfbaig35r/prometheusnn
|
|
6
6
|
Project-URL: Repository, https://github.com/mfbaig35r/prometheusnn
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "prometheusnn"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "Frictionless hierarchical text classification — an MCP server that guides you from CSV to trained classifier"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Configuration and directory management."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from prometheus.core.mcp_safety import ToolValidationError
|
|
9
|
+
|
|
10
|
+
# A model name is one path segment: letters, digits, dot, underscore, hyphen; no leading dot.
|
|
11
|
+
_MODEL_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def validate_model_name(model_name: str) -> str:
|
|
15
|
+
"""Reject anything that is not a single, plain directory name (no '/', '..', absolute paths)."""
|
|
16
|
+
if not isinstance(model_name, str) or not _MODEL_NAME.match(model_name) or ".." in model_name:
|
|
17
|
+
raise ToolValidationError(
|
|
18
|
+
f"Invalid model name {model_name!r}: use letters, digits, '.', '_' or '-' "
|
|
19
|
+
"(max 128 characters, no path separators)",
|
|
20
|
+
field="model_name",
|
|
21
|
+
)
|
|
22
|
+
return model_name
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _contained(base: Path, child: Path) -> Path:
|
|
26
|
+
"""Resolve child and require it to stay inside base (defence in depth after validation)."""
|
|
27
|
+
base_r, child_r = base.resolve(), child.resolve()
|
|
28
|
+
if child_r != base_r and base_r not in child_r.parents:
|
|
29
|
+
raise ToolValidationError(f"Path escapes {base}", field="model_name")
|
|
30
|
+
return child_r
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class PrometheusConfig:
|
|
35
|
+
"""Global configuration for Prometheus."""
|
|
36
|
+
|
|
37
|
+
base_dir: Path = field(
|
|
38
|
+
default_factory=lambda: Path(os.environ.get("PROMETHEUS_HOME", Path.home() / ".prometheus"))
|
|
39
|
+
)
|
|
40
|
+
default_embedding_model: str = field(
|
|
41
|
+
default_factory=lambda: os.environ.get("PROMETHEUS_EMBEDDING_MODEL", "all-MiniLM-L6-v2")
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
def __post_init__(self):
|
|
45
|
+
self.base_dir = Path(self.base_dir)
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def models_dir(self) -> Path:
|
|
49
|
+
return self.base_dir / "models"
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def feedback_dir(self) -> Path:
|
|
53
|
+
return self.base_dir / "feedback"
|
|
54
|
+
|
|
55
|
+
@property
|
|
56
|
+
def logs_dir(self) -> Path:
|
|
57
|
+
return self.base_dir / "logs"
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def exports_dir(self) -> Path:
|
|
61
|
+
override = os.environ.get("PROMETHEUS_EXPORT_DIR")
|
|
62
|
+
return Path(override) if override else self.base_dir / "exports"
|
|
63
|
+
|
|
64
|
+
def ensure_dirs(self):
|
|
65
|
+
"""Create all required directories."""
|
|
66
|
+
self.models_dir.mkdir(parents=True, exist_ok=True)
|
|
67
|
+
self.feedback_dir.mkdir(parents=True, exist_ok=True)
|
|
68
|
+
self.logs_dir.mkdir(parents=True, exist_ok=True)
|
|
69
|
+
|
|
70
|
+
def model_path(self, model_name: str) -> Path:
|
|
71
|
+
"""Directory for a model; always inside models_dir."""
|
|
72
|
+
return _contained(self.models_dir, self.models_dir / validate_model_name(model_name))
|
|
73
|
+
|
|
74
|
+
def feedback_path(self, model_name: str) -> Path:
|
|
75
|
+
"""Feedback directory for a model; always inside feedback_dir."""
|
|
76
|
+
return _contained(self.feedback_dir, self.feedback_dir / validate_model_name(model_name))
|
|
77
|
+
|
|
78
|
+
def export_path(self, output_path: str) -> Path:
|
|
79
|
+
"""Destination for export_model; always inside exports_dir.
|
|
80
|
+
|
|
81
|
+
A relative path is taken under exports_dir; an absolute path must already point inside
|
|
82
|
+
it. Set PROMETHEUS_EXPORT_DIR to export somewhere else.
|
|
83
|
+
"""
|
|
84
|
+
if not isinstance(output_path, str) or not output_path.strip():
|
|
85
|
+
raise ToolValidationError("output_path is required", field="output_path")
|
|
86
|
+
candidate = Path(output_path).expanduser()
|
|
87
|
+
if not candidate.is_absolute():
|
|
88
|
+
candidate = self.exports_dir / candidate
|
|
89
|
+
try:
|
|
90
|
+
return _contained(self.exports_dir, candidate)
|
|
91
|
+
except ToolValidationError as exc:
|
|
92
|
+
raise ToolValidationError(
|
|
93
|
+
f"output_path must be inside the export directory {self.exports_dir} "
|
|
94
|
+
"(set PROMETHEUS_EXPORT_DIR to change it)",
|
|
95
|
+
field="output_path",
|
|
96
|
+
) from exc
|
|
97
|
+
|
|
98
|
+
def list_models(self) -> list[str]:
|
|
99
|
+
"""List all saved model names."""
|
|
100
|
+
if not self.models_dir.exists():
|
|
101
|
+
return []
|
|
102
|
+
return sorted(
|
|
103
|
+
d.name
|
|
104
|
+
for d in self.models_dir.iterdir()
|
|
105
|
+
if d.is_dir() and (d / "manifest.json").exists()
|
|
106
|
+
)
|
|
@@ -116,17 +116,26 @@ class NoveltyDetector:
|
|
|
116
116
|
|
|
117
117
|
def load(self, path: Path):
|
|
118
118
|
path = Path(path)
|
|
119
|
-
data
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
119
|
+
# Never unpickle: a model directory is data, and a crafted .npz with object arrays would
|
|
120
|
+
# run code on load. save() writes only numeric and string arrays, which need no pickle.
|
|
121
|
+
data = np.load(path / "novelty_centroids.npz", allow_pickle=False)
|
|
122
|
+
try:
|
|
123
|
+
class_names = data["class_names"]
|
|
124
|
+
mean_dists = data["mean_dists"]
|
|
125
|
+
std_dists = data["std_dists"]
|
|
126
|
+
p95_dists = data["p95_dists"]
|
|
127
|
+
counts = data["counts"]
|
|
128
|
+
centroids = [data[f"centroid_{i}"] for i in range(len(class_names))]
|
|
129
|
+
except ValueError as exc:
|
|
130
|
+
raise ValueError(
|
|
131
|
+
f"{path / 'novelty_centroids.npz'} contains pickled objects; refusing to load it. "
|
|
132
|
+
"Retrain the model to regenerate it."
|
|
133
|
+
) from exc
|
|
125
134
|
|
|
126
135
|
self.centroids = {}
|
|
127
136
|
for i, name in enumerate(class_names):
|
|
128
137
|
self.centroids[str(name)] = ClassCentroid(
|
|
129
|
-
centroid=
|
|
138
|
+
centroid=centroids[i],
|
|
130
139
|
mean_dist=float(mean_dists[i]),
|
|
131
140
|
std_dist=max(float(std_dists[i]), 0.1),
|
|
132
141
|
p95_dist=float(p95_dists[i]),
|
|
@@ -135,7 +144,7 @@ class NoveltyDetector:
|
|
|
135
144
|
|
|
136
145
|
emb_path = path / "novelty_embeddings.npy"
|
|
137
146
|
if emb_path.exists():
|
|
138
|
-
self.training_embeddings = np.load(emb_path)
|
|
147
|
+
self.training_embeddings = np.load(emb_path, allow_pickle=False)
|
|
139
148
|
|
|
140
149
|
def get_centroids_data(self) -> dict:
|
|
141
150
|
"""Return data dict suitable for np.savez."""
|
|
@@ -242,7 +242,7 @@ def submit_feedback(
|
|
|
242
242
|
prediction_id: Optional prediction ID to link feedback to
|
|
243
243
|
"""
|
|
244
244
|
logger.info("tool=submit_feedback model=%s prediction_id=%s", model_name, prediction_id)
|
|
245
|
-
feedback_dir = server._config.
|
|
245
|
+
feedback_dir = server._config.feedback_path(model_name)
|
|
246
246
|
feedback_dir.mkdir(parents=True, exist_ok=True)
|
|
247
247
|
|
|
248
248
|
entry = {
|
|
@@ -4,7 +4,6 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import json
|
|
6
6
|
import shutil
|
|
7
|
-
from pathlib import Path
|
|
8
7
|
|
|
9
8
|
from prometheus import server
|
|
10
9
|
from prometheus.core.mcp_safety import ToolValidationError, mcp_tool_safe
|
|
@@ -86,6 +85,9 @@ def delete_model(model_name: str) -> dict:
|
|
|
86
85
|
model_path = server._config.model_path(model_name)
|
|
87
86
|
if not model_path.exists():
|
|
88
87
|
raise ToolValidationError(f"Model '{model_name}' not found")
|
|
88
|
+
# Only ever delete a directory that is a saved model.
|
|
89
|
+
if not (model_path / "manifest.json").is_file():
|
|
90
|
+
raise ToolValidationError(f"'{model_name}' is not a saved model (no manifest.json)")
|
|
89
91
|
|
|
90
92
|
delete_model_from_disk(model_path)
|
|
91
93
|
server._predictor_cache.pop(model_name, None)
|
|
@@ -99,21 +101,24 @@ def export_model(
|
|
|
99
101
|
model_name: str,
|
|
100
102
|
output_path: str,
|
|
101
103
|
) -> dict:
|
|
102
|
-
"""Copy model directory
|
|
104
|
+
"""Copy model directory into the export directory.
|
|
103
105
|
|
|
104
106
|
Args:
|
|
105
107
|
model_name: Name of the model to export
|
|
106
|
-
output_path: Destination directory
|
|
108
|
+
output_path: Destination directory, relative to the export directory
|
|
109
|
+
(PROMETHEUS_HOME/exports by default, or PROMETHEUS_EXPORT_DIR). An absolute path
|
|
110
|
+
must point inside it.
|
|
107
111
|
"""
|
|
108
112
|
logger.info("tool=export_model model=%s dest=%s", model_name, output_path)
|
|
109
113
|
model_path = server._config.model_path(model_name)
|
|
110
114
|
if not model_path.exists():
|
|
111
115
|
raise ToolValidationError(f"Model '{model_name}' not found")
|
|
112
116
|
|
|
113
|
-
dest =
|
|
117
|
+
dest = server._config.export_path(output_path)
|
|
114
118
|
if dest.exists():
|
|
115
119
|
raise ToolValidationError(f"Destination already exists: {output_path}")
|
|
116
120
|
|
|
121
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
117
122
|
shutil.copytree(model_path, dest)
|
|
118
123
|
return success_response({"exported_to": str(dest)})
|
|
119
124
|
|
|
@@ -6,6 +6,14 @@ https://prometheusnn.cognitionkernel.agency — toggle to "connected mode"
|
|
|
6
6
|
and point it at this server (default: ws://localhost:8000/ws/prometheus)
|
|
7
7
|
to drive the same UI against your own data, on your own hardware.
|
|
8
8
|
|
|
9
|
+
Security. Browsers let any web page open a WebSocket to localhost, so the server:
|
|
10
|
+
|
|
11
|
+
- requires a per-launch token, passed as ``?token=...`` in the URL (printed at startup, or set
|
|
12
|
+
with ``--token`` / ``PROMETHEUS_UI_TOKEN``); paste the printed URL into connected mode;
|
|
13
|
+
- accepts browser connections only from allowed origins (the companion site and local dev by
|
|
14
|
+
default; ``--allowed-origin`` to change);
|
|
15
|
+
- does not offer ``delete_model`` or ``export_model`` unless started with ``--allow-destructive``.
|
|
16
|
+
|
|
9
17
|
Protocol — JSON over WebSocket. Atomic request/response per tool call.
|
|
10
18
|
|
|
11
19
|
Client → Server:
|
|
@@ -25,9 +33,12 @@ from __future__ import annotations
|
|
|
25
33
|
|
|
26
34
|
import argparse
|
|
27
35
|
import asyncio
|
|
36
|
+
import hmac
|
|
28
37
|
import inspect
|
|
29
38
|
import json
|
|
30
39
|
import logging
|
|
40
|
+
import os
|
|
41
|
+
import secrets
|
|
31
42
|
from collections.abc import Callable
|
|
32
43
|
from typing import Any
|
|
33
44
|
|
|
@@ -49,6 +60,58 @@ def _build_tool_registry() -> dict[str, Callable[..., dict]]:
|
|
|
49
60
|
return {name: tool.fn for name, tool in tool_manager._tools.items()}
|
|
50
61
|
|
|
51
62
|
|
|
63
|
+
# ─── Connection security ─────────────────────────────────────────────────────
|
|
64
|
+
|
|
65
|
+
DEFAULT_ALLOWED_ORIGINS = (
|
|
66
|
+
"https://prometheusnn.cognitionkernel.agency",
|
|
67
|
+
"http://localhost:3000",
|
|
68
|
+
"http://127.0.0.1:3000",
|
|
69
|
+
)
|
|
70
|
+
# Tools that delete or write outside the model store; off over the UI unless --allow-destructive.
|
|
71
|
+
DESTRUCTIVE_TOOLS = frozenset({"delete_model", "export_model"})
|
|
72
|
+
|
|
73
|
+
_SECURITY: dict[str, Any] = {
|
|
74
|
+
"token": None,
|
|
75
|
+
"allowed_origins": frozenset(DEFAULT_ALLOWED_ORIGINS),
|
|
76
|
+
"allow_destructive": False,
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def configure(
|
|
81
|
+
*,
|
|
82
|
+
token: str | None,
|
|
83
|
+
allowed_origins: tuple[str, ...] | list[str] = DEFAULT_ALLOWED_ORIGINS,
|
|
84
|
+
allow_destructive: bool = False,
|
|
85
|
+
) -> None:
|
|
86
|
+
"""Set connection security. main() calls this; tests call it directly."""
|
|
87
|
+
_SECURITY["token"] = token
|
|
88
|
+
_SECURITY["allowed_origins"] = frozenset(o.rstrip("/") for o in allowed_origins)
|
|
89
|
+
_SECURITY["allow_destructive"] = allow_destructive
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def check_handshake(origin: str | None, token: str | None) -> str | None:
|
|
93
|
+
"""None if the connection may proceed, otherwise the reason it is refused.
|
|
94
|
+
|
|
95
|
+
Browsers always send Origin on a WebSocket handshake, so a present Origin must be allowed;
|
|
96
|
+
a missing Origin means a non-browser client, which still needs the token.
|
|
97
|
+
"""
|
|
98
|
+
expected = _SECURITY["token"]
|
|
99
|
+
if not expected:
|
|
100
|
+
return "server has no token configured"
|
|
101
|
+
if origin is not None and origin.rstrip("/") not in _SECURITY["allowed_origins"]:
|
|
102
|
+
return f"origin not allowed: {origin}"
|
|
103
|
+
if not token or not hmac.compare_digest(token, expected):
|
|
104
|
+
return "missing or invalid token"
|
|
105
|
+
return None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _ui_tools() -> dict[str, Callable[..., dict]]:
|
|
109
|
+
tools = _get_tools()
|
|
110
|
+
if _SECURITY["allow_destructive"]:
|
|
111
|
+
return tools
|
|
112
|
+
return {n: f for n, f in tools.items() if n not in DESTRUCTIVE_TOOLS}
|
|
113
|
+
|
|
114
|
+
|
|
52
115
|
_TOOLS_CACHE: dict[str, Callable[..., dict]] | None = None
|
|
53
116
|
|
|
54
117
|
|
|
@@ -90,7 +153,7 @@ def _serialize_tool_metadata(tool_fn: Callable[..., dict]) -> dict[str, Any]:
|
|
|
90
153
|
|
|
91
154
|
async def _handle_message(ws, msg: dict[str, Any]) -> None:
|
|
92
155
|
msg_type = msg.get("type")
|
|
93
|
-
tools =
|
|
156
|
+
tools = _ui_tools()
|
|
94
157
|
|
|
95
158
|
if msg_type == "ping":
|
|
96
159
|
await ws.send_json({"type": "pong"})
|
|
@@ -111,12 +174,22 @@ async def _handle_message(ws, msg: dict[str, Any]) -> None:
|
|
|
111
174
|
args = msg.get("args", {}) or {}
|
|
112
175
|
|
|
113
176
|
if tool_name not in tools:
|
|
177
|
+
error = (
|
|
178
|
+
f"{tool_name} is not available over the UI server (start it with --allow-destructive)"
|
|
179
|
+
if tool_name in DESTRUCTIVE_TOOLS
|
|
180
|
+
else f"unknown tool: {tool_name}"
|
|
181
|
+
)
|
|
182
|
+
await ws.send_json(
|
|
183
|
+
{"type": "tool_error", "id": call_id, "tool": tool_name, "error": error}
|
|
184
|
+
)
|
|
185
|
+
return
|
|
186
|
+
if not isinstance(args, dict):
|
|
114
187
|
await ws.send_json(
|
|
115
188
|
{
|
|
116
189
|
"type": "tool_error",
|
|
117
190
|
"id": call_id,
|
|
118
191
|
"tool": tool_name,
|
|
119
|
-
"error":
|
|
192
|
+
"error": "args must be an object",
|
|
120
193
|
}
|
|
121
194
|
)
|
|
122
195
|
return
|
|
@@ -163,7 +236,7 @@ async def _handle_message(ws, msg: dict[str, Any]) -> None:
|
|
|
163
236
|
try:
|
|
164
237
|
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
|
165
238
|
|
|
166
|
-
app: FastAPI | None = FastAPI(title="prometheusnn-ui", version="0.2.
|
|
239
|
+
app: FastAPI | None = FastAPI(title="prometheusnn-ui", version="0.2.2")
|
|
167
240
|
|
|
168
241
|
@app.get("/health")
|
|
169
242
|
def _health() -> dict:
|
|
@@ -171,6 +244,13 @@ try:
|
|
|
171
244
|
|
|
172
245
|
@app.websocket("/ws/prometheus")
|
|
173
246
|
async def _ws_endpoint(websocket: WebSocket) -> None:
|
|
247
|
+
refused = check_handshake(
|
|
248
|
+
websocket.headers.get("origin"), websocket.query_params.get("token")
|
|
249
|
+
)
|
|
250
|
+
if refused is not None:
|
|
251
|
+
logger.warning("refused WebSocket connection: %s", refused)
|
|
252
|
+
await websocket.close(code=1008) # before accept(): the handshake fails
|
|
253
|
+
return
|
|
174
254
|
await websocket.accept()
|
|
175
255
|
peer = f"{websocket.client.host}:{websocket.client.port}" if websocket.client else "unknown"
|
|
176
256
|
logger.info("client connected: %s", peer)
|
|
@@ -191,12 +271,21 @@ except ImportError:
|
|
|
191
271
|
|
|
192
272
|
|
|
193
273
|
def create_app():
|
|
194
|
-
"""Factory used by uvicorn --factory. Returns the module-level app.
|
|
274
|
+
"""Factory used by uvicorn --factory. Returns the module-level app.
|
|
275
|
+
|
|
276
|
+
Outside main() (e.g. ``uvicorn --factory``), the token comes from PROMETHEUS_UI_TOKEN or is
|
|
277
|
+
generated and printed, so the server never runs without one.
|
|
278
|
+
"""
|
|
195
279
|
if app is None:
|
|
196
280
|
raise SystemExit(
|
|
197
281
|
"prometheusnn-ui requires the [ui] extras. Install with:\n"
|
|
198
282
|
" pip install prometheusnn[ui]"
|
|
199
283
|
)
|
|
284
|
+
if not _SECURITY["token"]:
|
|
285
|
+
token = os.environ.get("PROMETHEUS_UI_TOKEN") or secrets.token_urlsafe(24)
|
|
286
|
+
configure(token=token)
|
|
287
|
+
if "PROMETHEUS_UI_TOKEN" not in os.environ:
|
|
288
|
+
print(f"\n prometheusnn-ui token: {token} (append ?token=... to the URL)\n")
|
|
200
289
|
return app
|
|
201
290
|
|
|
202
291
|
|
|
@@ -216,6 +305,23 @@ def main() -> None:
|
|
|
216
305
|
default="info",
|
|
217
306
|
choices=["debug", "info", "warning", "error"],
|
|
218
307
|
)
|
|
308
|
+
parser.add_argument(
|
|
309
|
+
"--token",
|
|
310
|
+
default=os.environ.get("PROMETHEUS_UI_TOKEN"),
|
|
311
|
+
help="Connection token (default: PROMETHEUS_UI_TOKEN, or a new random one per launch)",
|
|
312
|
+
)
|
|
313
|
+
parser.add_argument(
|
|
314
|
+
"--allowed-origin",
|
|
315
|
+
action="append",
|
|
316
|
+
dest="allowed_origins",
|
|
317
|
+
help="Browser origin allowed to connect; repeatable (default: the companion site and "
|
|
318
|
+
"localhost:3000)",
|
|
319
|
+
)
|
|
320
|
+
parser.add_argument(
|
|
321
|
+
"--allow-destructive",
|
|
322
|
+
action="store_true",
|
|
323
|
+
help="Also offer delete_model and export_model over the UI",
|
|
324
|
+
)
|
|
219
325
|
args = parser.parse_args()
|
|
220
326
|
|
|
221
327
|
try:
|
|
@@ -227,12 +333,21 @@ def main() -> None:
|
|
|
227
333
|
) from e
|
|
228
334
|
|
|
229
335
|
logging.basicConfig(level=args.log_level.upper())
|
|
336
|
+
token = args.token or secrets.token_urlsafe(24)
|
|
337
|
+
configure(
|
|
338
|
+
token=token,
|
|
339
|
+
allowed_origins=args.allowed_origins or DEFAULT_ALLOWED_ORIGINS,
|
|
340
|
+
allow_destructive=args.allow_destructive,
|
|
341
|
+
)
|
|
342
|
+
if args.host not in ("127.0.0.1", "localhost", "::1"):
|
|
343
|
+
logger.warning("binding to %s exposes the server beyond this machine", args.host)
|
|
230
344
|
# Warm the tool registry so the count is logged before serving
|
|
231
345
|
tools = _get_tools()
|
|
232
346
|
logger.info("Registered %d tools", len(tools))
|
|
233
347
|
print(
|
|
234
348
|
f"\n prometheusnn-ui listening on ws://{args.host}:{args.port}/ws/prometheus\n"
|
|
235
|
-
f" Open https://prometheusnn.cognitionkernel.agency
|
|
349
|
+
f" Open https://prometheusnn.cognitionkernel.agency, toggle 'Connected mode', and use:\n"
|
|
350
|
+
f" ws://{args.host}:{args.port}/ws/prometheus?token={token}\n"
|
|
236
351
|
)
|
|
237
352
|
uvicorn.run(
|
|
238
353
|
create_app(),
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Tests for novelty detection."""
|
|
2
|
+
|
|
3
|
+
import tempfile
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
from prometheus.inference.novelty import NoveltyDetector
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_fit_and_score():
|
|
13
|
+
rng = np.random.default_rng(42)
|
|
14
|
+
n = 50
|
|
15
|
+
embs = rng.standard_normal((n, 16)).astype(np.float32)
|
|
16
|
+
labels = np.array([0] * 25 + [1] * 25)
|
|
17
|
+
classes = ["classA", "classB"]
|
|
18
|
+
|
|
19
|
+
det = NoveltyDetector(z_score_threshold=2.5)
|
|
20
|
+
det.fit(embs, labels, classes)
|
|
21
|
+
|
|
22
|
+
assert len(det.centroids) == 2
|
|
23
|
+
|
|
24
|
+
# Score an in-distribution sample
|
|
25
|
+
score = det.score(embs[0], "classA", confidence=0.9)
|
|
26
|
+
assert not score.is_novel
|
|
27
|
+
|
|
28
|
+
# Score a far-out sample
|
|
29
|
+
outlier = np.ones(16, dtype=np.float32) * 100
|
|
30
|
+
score = det.score(outlier, "classA", confidence=0.3)
|
|
31
|
+
assert score.centroid_z_score > 2.5
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_save_load_roundtrip():
|
|
35
|
+
rng = np.random.default_rng(42)
|
|
36
|
+
embs = rng.standard_normal((30, 8)).astype(np.float32)
|
|
37
|
+
labels = np.array([0] * 15 + [1] * 15)
|
|
38
|
+
classes = ["X", "Y"]
|
|
39
|
+
|
|
40
|
+
det = NoveltyDetector()
|
|
41
|
+
det.fit(embs, labels, classes)
|
|
42
|
+
|
|
43
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
44
|
+
det.save(Path(tmpdir))
|
|
45
|
+
|
|
46
|
+
det2 = NoveltyDetector()
|
|
47
|
+
det2.load(Path(tmpdir))
|
|
48
|
+
|
|
49
|
+
assert set(det2.centroids.keys()) == {"X", "Y"}
|
|
50
|
+
assert det2.training_embeddings is not None
|
|
51
|
+
assert det2.training_embeddings.shape == (30, 8)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class _Payload:
|
|
55
|
+
"""Unpickling this creates a file: proof that loading ran attacker code."""
|
|
56
|
+
|
|
57
|
+
def __init__(self, marker: Path):
|
|
58
|
+
self.marker = str(marker)
|
|
59
|
+
|
|
60
|
+
def __reduce__(self):
|
|
61
|
+
return (open, (self.marker, "w"))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def test_load_never_unpickles_centroids(tmp_path):
|
|
65
|
+
marker = tmp_path / "executed"
|
|
66
|
+
np.savez(
|
|
67
|
+
tmp_path / "novelty_centroids.npz",
|
|
68
|
+
centroid_0=np.zeros(4),
|
|
69
|
+
class_names=np.array([_Payload(marker)], dtype=object),
|
|
70
|
+
mean_dists=[1.0],
|
|
71
|
+
std_dists=[1.0],
|
|
72
|
+
p95_dists=[1.0],
|
|
73
|
+
counts=[1],
|
|
74
|
+
)
|
|
75
|
+
with pytest.raises(ValueError, match="pickled objects"):
|
|
76
|
+
NoveltyDetector().load(tmp_path)
|
|
77
|
+
assert not marker.exists()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_load_never_unpickles_embeddings(tmp_path):
|
|
81
|
+
marker = tmp_path / "executed"
|
|
82
|
+
det = NoveltyDetector()
|
|
83
|
+
det.fit(np.eye(4, dtype=np.float32), np.array([0, 0, 1, 1]), ["a", "b"])
|
|
84
|
+
det.save(tmp_path)
|
|
85
|
+
np.save(tmp_path / "novelty_embeddings.npy", np.array([_Payload(marker)], dtype=object))
|
|
86
|
+
with pytest.raises(ValueError):
|
|
87
|
+
NoveltyDetector().load(tmp_path)
|
|
88
|
+
assert not marker.exists()
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def test_saved_files_need_no_pickle(tmp_path):
|
|
92
|
+
rng = np.random.default_rng(0)
|
|
93
|
+
det = NoveltyDetector()
|
|
94
|
+
det.fit(rng.standard_normal((20, 4)), np.array([0] * 10 + [1] * 10), ["Café", "naïve/class"])
|
|
95
|
+
det.save(tmp_path)
|
|
96
|
+
with np.load(tmp_path / "novelty_centroids.npz", allow_pickle=False) as data:
|
|
97
|
+
assert all(data[k].dtype != object for k in data.files)
|
|
98
|
+
det2 = NoveltyDetector()
|
|
99
|
+
det2.load(tmp_path)
|
|
100
|
+
assert set(det2.centroids) == {"Café", "naïve/class"}
|
|
@@ -267,11 +267,92 @@ class TestDeleteModel:
|
|
|
267
267
|
|
|
268
268
|
|
|
269
269
|
class TestExportModel:
|
|
270
|
-
def test_export(self, trained_model,
|
|
271
|
-
|
|
272
|
-
result = server.export_model(trained_model,
|
|
270
|
+
def test_export(self, trained_model, tmp_config, monkeypatch):
|
|
271
|
+
monkeypatch.delenv("PROMETHEUS_EXPORT_DIR", raising=False)
|
|
272
|
+
result = server.export_model(trained_model, "exported_model")
|
|
273
273
|
assert result["status"] == "success"
|
|
274
|
-
assert
|
|
274
|
+
assert (tmp_config.exports_dir / "exported_model" / "manifest.json").exists()
|
|
275
|
+
|
|
276
|
+
def test_export_outside_export_dir_is_refused(self, trained_model, tmp_path, monkeypatch):
|
|
277
|
+
monkeypatch.delenv("PROMETHEUS_EXPORT_DIR", raising=False)
|
|
278
|
+
for dest in (str(tmp_path / "elsewhere"), "../../escape", "/tmp/prometheus-escape"):
|
|
279
|
+
result = server.export_model(trained_model, dest)
|
|
280
|
+
assert result["status"] == "validation_error", dest
|
|
281
|
+
assert not (tmp_path / "elsewhere").exists()
|
|
282
|
+
|
|
283
|
+
def test_export_dir_override(self, trained_model, tmp_path, monkeypatch):
|
|
284
|
+
target = tmp_path / "my-exports"
|
|
285
|
+
monkeypatch.setenv("PROMETHEUS_EXPORT_DIR", str(target))
|
|
286
|
+
result = server.export_model(trained_model, str(target / "copy"))
|
|
287
|
+
assert result["status"] == "success"
|
|
288
|
+
assert (target / "copy" / "manifest.json").exists()
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
# ─── Model-name path traversal (security) ───
|
|
292
|
+
|
|
293
|
+
BAD_NAMES = [
|
|
294
|
+
# Never list a real system path here: against vulnerable code these names reach rmtree.
|
|
295
|
+
"../escape",
|
|
296
|
+
"../../nonexistent-parent",
|
|
297
|
+
"/nonexistent-prometheus-test-path",
|
|
298
|
+
"/tmp/nonexistent-prometheus-test-dir",
|
|
299
|
+
"a/b",
|
|
300
|
+
"..",
|
|
301
|
+
".hidden",
|
|
302
|
+
"",
|
|
303
|
+
"x" * 200,
|
|
304
|
+
"name\\with\\backslash",
|
|
305
|
+
"name\x00null",
|
|
306
|
+
]
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
class TestModelNameTraversal:
|
|
310
|
+
@pytest.mark.parametrize("name", BAD_NAMES)
|
|
311
|
+
def test_delete_refuses_bad_names(self, name, tmp_path):
|
|
312
|
+
victim = tmp_path / "victim"
|
|
313
|
+
victim.mkdir()
|
|
314
|
+
(victim / "keep.txt").write_text("keep")
|
|
315
|
+
result = server.delete_model(name)
|
|
316
|
+
assert result["status"] == "validation_error"
|
|
317
|
+
assert (victim / "keep.txt").exists()
|
|
318
|
+
|
|
319
|
+
def test_delete_absolute_path_to_real_dir_is_refused(self, tmp_path):
|
|
320
|
+
victim = tmp_path / "important"
|
|
321
|
+
victim.mkdir()
|
|
322
|
+
(victim / "data.txt").write_text("precious")
|
|
323
|
+
result = server.delete_model(str(victim))
|
|
324
|
+
assert result["status"] == "validation_error"
|
|
325
|
+
assert (victim / "data.txt").read_text() == "precious"
|
|
326
|
+
|
|
327
|
+
def test_delete_refuses_a_directory_without_manifest(self, tmp_config):
|
|
328
|
+
stray = tmp_config.models_dir / "not_a_model"
|
|
329
|
+
stray.mkdir()
|
|
330
|
+
(stray / "file.txt").write_text("x")
|
|
331
|
+
result = server.delete_model("not_a_model")
|
|
332
|
+
assert result["status"] == "validation_error"
|
|
333
|
+
assert stray.exists()
|
|
334
|
+
|
|
335
|
+
@pytest.mark.parametrize("name", ["../outside", "/tmp/fb-outside"])
|
|
336
|
+
def test_feedback_cannot_write_outside_feedback_dir(self, name, tmp_config, tmp_path):
|
|
337
|
+
result = server.submit_feedback(name, "some text", {"level_1": "x"})
|
|
338
|
+
assert result["status"] == "validation_error"
|
|
339
|
+
assert not (tmp_config.feedback_dir.parent / "outside").exists()
|
|
340
|
+
|
|
341
|
+
@pytest.mark.parametrize("tool", ["describe_model", "predict", "evaluate_model"])
|
|
342
|
+
def test_other_tools_validate_names(self, tool):
|
|
343
|
+
fn = getattr(server, tool)
|
|
344
|
+
args = {
|
|
345
|
+
"describe_model": ("../x",),
|
|
346
|
+
"predict": ("../x", "text"),
|
|
347
|
+
"evaluate_model": ("../x",),
|
|
348
|
+
}[tool]
|
|
349
|
+
result = fn(*args)
|
|
350
|
+
assert result["status"] in ("validation_error", "error"), result
|
|
351
|
+
|
|
352
|
+
@pytest.mark.parametrize("name", ["model-1", "my_model.v2", "A1", "x" * 128])
|
|
353
|
+
def test_good_names_resolve_inside_models_dir(self, name, tmp_config):
|
|
354
|
+
path = tmp_config.model_path(name)
|
|
355
|
+
assert path.parent == tmp_config.models_dir.resolve()
|
|
275
356
|
|
|
276
357
|
|
|
277
358
|
# ─── Code Mapping ───
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""WebSocket UI server: origin allowlist, per-launch token, destructive tools withheld."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
fastapi = pytest.importorskip("fastapi")
|
|
8
|
+
from fastapi.testclient import TestClient # noqa: E402
|
|
9
|
+
from starlette.websockets import WebSocketDisconnect # noqa: E402
|
|
10
|
+
|
|
11
|
+
from prometheus import ui_server # noqa: E402
|
|
12
|
+
|
|
13
|
+
TOKEN = "test-token-123"
|
|
14
|
+
GOOD = "https://prometheusnn.cognitionkernel.agency"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.fixture(autouse=True)
|
|
18
|
+
def secured():
|
|
19
|
+
ui_server.configure(token=TOKEN)
|
|
20
|
+
yield
|
|
21
|
+
ui_server.configure(token=None)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@pytest.fixture
|
|
25
|
+
def client():
|
|
26
|
+
return TestClient(ui_server.app)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _connect(client, origin=GOOD, token=TOKEN):
|
|
30
|
+
url = "/ws/prometheus" + (f"?token={token}" if token is not None else "")
|
|
31
|
+
headers = {"origin": origin} if origin is not None else {}
|
|
32
|
+
return client.websocket_connect(url, headers=headers)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_allowed_origin_with_token_connects(client):
|
|
36
|
+
with _connect(client) as ws:
|
|
37
|
+
ws.send_json({"type": "ping"})
|
|
38
|
+
assert ws.receive_json() == {"type": "pong"}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@pytest.mark.parametrize("origin", ["https://evil.example", "http://localhost:9999", "null"])
|
|
42
|
+
def test_foreign_origin_is_refused(client, origin):
|
|
43
|
+
with pytest.raises(WebSocketDisconnect) as exc, _connect(client, origin=origin):
|
|
44
|
+
pass
|
|
45
|
+
assert exc.value.code == 1008
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@pytest.mark.parametrize("token", [None, "", "wrong"])
|
|
49
|
+
def test_missing_or_wrong_token_is_refused(client, token):
|
|
50
|
+
with pytest.raises(WebSocketDisconnect), _connect(client, token=token):
|
|
51
|
+
pass
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_non_browser_client_still_needs_the_token(client):
|
|
55
|
+
with _connect(client, origin=None) as ws:
|
|
56
|
+
ws.send_json({"type": "ping"})
|
|
57
|
+
assert ws.receive_json()["type"] == "pong"
|
|
58
|
+
with pytest.raises(WebSocketDisconnect), _connect(client, origin=None, token="nope"):
|
|
59
|
+
pass
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def test_no_token_configured_refuses_everything(client):
|
|
63
|
+
ui_server.configure(token=None)
|
|
64
|
+
with pytest.raises(WebSocketDisconnect), _connect(client):
|
|
65
|
+
pass
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_destructive_tools_are_withheld_by_default(client):
|
|
69
|
+
with _connect(client) as ws:
|
|
70
|
+
ws.send_json({"type": "list_tools"})
|
|
71
|
+
names = {t["name"] for t in ws.receive_json()["tools"]}
|
|
72
|
+
assert not names & ui_server.DESTRUCTIVE_TOOLS
|
|
73
|
+
assert "start_here" in names
|
|
74
|
+
ws.send_json(
|
|
75
|
+
{"type": "tool_call", "id": "1", "tool": "delete_model", "args": {"model_name": "x"}}
|
|
76
|
+
)
|
|
77
|
+
msg = ws.receive_json()
|
|
78
|
+
assert msg["type"] == "tool_error" and "--allow-destructive" in msg["error"]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_allow_destructive_opt_in(client):
|
|
82
|
+
ui_server.configure(token=TOKEN, allow_destructive=True)
|
|
83
|
+
with _connect(client) as ws:
|
|
84
|
+
ws.send_json({"type": "list_tools"})
|
|
85
|
+
names = {t["name"] for t in ws.receive_json()["tools"]}
|
|
86
|
+
assert names >= ui_server.DESTRUCTIVE_TOOLS
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_non_object_args_are_rejected(client):
|
|
90
|
+
with _connect(client) as ws:
|
|
91
|
+
ws.send_json({"type": "tool_call", "id": "2", "tool": "list_models", "args": ["a"]})
|
|
92
|
+
assert ws.receive_json()["error"] == "args must be an object"
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def test_custom_origin_list(client):
|
|
96
|
+
ui_server.configure(token=TOKEN, allowed_origins=["http://localhost:5173"])
|
|
97
|
+
with _connect(client, origin="http://localhost:5173") as ws:
|
|
98
|
+
ws.send_json({"type": "ping"})
|
|
99
|
+
assert ws.receive_json()["type"] == "pong"
|
|
100
|
+
with pytest.raises(WebSocketDisconnect), _connect(client, origin=GOOD):
|
|
101
|
+
pass
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
"""Configuration and directory management."""
|
|
2
|
-
|
|
3
|
-
import os
|
|
4
|
-
from dataclasses import dataclass, field
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
@dataclass
|
|
9
|
-
class PrometheusConfig:
|
|
10
|
-
"""Global configuration for Prometheus."""
|
|
11
|
-
|
|
12
|
-
base_dir: Path = field(
|
|
13
|
-
default_factory=lambda: Path(os.environ.get("PROMETHEUS_HOME", Path.home() / ".prometheus"))
|
|
14
|
-
)
|
|
15
|
-
default_embedding_model: str = field(
|
|
16
|
-
default_factory=lambda: os.environ.get("PROMETHEUS_EMBEDDING_MODEL", "all-MiniLM-L6-v2")
|
|
17
|
-
)
|
|
18
|
-
|
|
19
|
-
def __post_init__(self):
|
|
20
|
-
self.base_dir = Path(self.base_dir)
|
|
21
|
-
|
|
22
|
-
@property
|
|
23
|
-
def models_dir(self) -> Path:
|
|
24
|
-
return self.base_dir / "models"
|
|
25
|
-
|
|
26
|
-
@property
|
|
27
|
-
def feedback_dir(self) -> Path:
|
|
28
|
-
return self.base_dir / "feedback"
|
|
29
|
-
|
|
30
|
-
@property
|
|
31
|
-
def logs_dir(self) -> Path:
|
|
32
|
-
return self.base_dir / "logs"
|
|
33
|
-
|
|
34
|
-
def ensure_dirs(self):
|
|
35
|
-
"""Create all required directories."""
|
|
36
|
-
self.models_dir.mkdir(parents=True, exist_ok=True)
|
|
37
|
-
self.feedback_dir.mkdir(parents=True, exist_ok=True)
|
|
38
|
-
self.logs_dir.mkdir(parents=True, exist_ok=True)
|
|
39
|
-
|
|
40
|
-
def model_path(self, model_name: str) -> Path:
|
|
41
|
-
return self.models_dir / model_name
|
|
42
|
-
|
|
43
|
-
def list_models(self) -> list[str]:
|
|
44
|
-
"""List all saved model names."""
|
|
45
|
-
if not self.models_dir.exists():
|
|
46
|
-
return []
|
|
47
|
-
return sorted(
|
|
48
|
-
d.name
|
|
49
|
-
for d in self.models_dir.iterdir()
|
|
50
|
-
if d.is_dir() and (d / "manifest.json").exists()
|
|
51
|
-
)
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
"""Tests for novelty detection."""
|
|
2
|
-
|
|
3
|
-
import tempfile
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
|
|
6
|
-
import numpy as np
|
|
7
|
-
|
|
8
|
-
from prometheus.inference.novelty import NoveltyDetector
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
def test_fit_and_score():
|
|
12
|
-
rng = np.random.default_rng(42)
|
|
13
|
-
n = 50
|
|
14
|
-
embs = rng.standard_normal((n, 16)).astype(np.float32)
|
|
15
|
-
labels = np.array([0] * 25 + [1] * 25)
|
|
16
|
-
classes = ["classA", "classB"]
|
|
17
|
-
|
|
18
|
-
det = NoveltyDetector(z_score_threshold=2.5)
|
|
19
|
-
det.fit(embs, labels, classes)
|
|
20
|
-
|
|
21
|
-
assert len(det.centroids) == 2
|
|
22
|
-
|
|
23
|
-
# Score an in-distribution sample
|
|
24
|
-
score = det.score(embs[0], "classA", confidence=0.9)
|
|
25
|
-
assert not score.is_novel
|
|
26
|
-
|
|
27
|
-
# Score a far-out sample
|
|
28
|
-
outlier = np.ones(16, dtype=np.float32) * 100
|
|
29
|
-
score = det.score(outlier, "classA", confidence=0.3)
|
|
30
|
-
assert score.centroid_z_score > 2.5
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def test_save_load_roundtrip():
|
|
34
|
-
rng = np.random.default_rng(42)
|
|
35
|
-
embs = rng.standard_normal((30, 8)).astype(np.float32)
|
|
36
|
-
labels = np.array([0] * 15 + [1] * 15)
|
|
37
|
-
classes = ["X", "Y"]
|
|
38
|
-
|
|
39
|
-
det = NoveltyDetector()
|
|
40
|
-
det.fit(embs, labels, classes)
|
|
41
|
-
|
|
42
|
-
with tempfile.TemporaryDirectory() as tmpdir:
|
|
43
|
-
det.save(Path(tmpdir))
|
|
44
|
-
|
|
45
|
-
det2 = NoveltyDetector()
|
|
46
|
-
det2.load(Path(tmpdir))
|
|
47
|
-
|
|
48
|
-
assert set(det2.centroids.keys()) == {"X", "Y"}
|
|
49
|
-
assert det2.training_embeddings is not None
|
|
50
|
-
assert det2.training_embeddings.shape == (30, 8)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|