prometheusnn 0.2.1__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/CHANGELOG.md +52 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/PKG-INFO +4 -4
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/README.md +3 -3
- prometheusnn-0.2.3/experiments/llm_coupling/.gitignore +1 -0
- prometheusnn-0.2.3/experiments/llm_coupling/README.md +111 -0
- prometheusnn-0.2.3/experiments/llm_coupling/data/plain_items.json +12402 -0
- prometheusnn-0.2.3/experiments/llm_coupling/data/plain_texts.json +102 -0
- prometheusnn-0.2.3/experiments/llm_coupling/data/tariff_items.json +11882 -0
- prometheusnn-0.2.3/experiments/llm_coupling/prompts/judge.md +36 -0
- prometheusnn-0.2.3/experiments/llm_coupling/prompts/paraphrase.md +26 -0
- prometheusnn-0.2.3/experiments/llm_coupling/runs/haiku-4.5/plain/answers_1.json +52 -0
- prometheusnn-0.2.3/experiments/llm_coupling/runs/haiku-4.5/plain/answers_2.json +52 -0
- prometheusnn-0.2.3/experiments/llm_coupling/runs/haiku-4.5/tariff/answers_1.json +52 -0
- prometheusnn-0.2.3/experiments/llm_coupling/runs/haiku-4.5/tariff/answers_2.json +52 -0
- prometheusnn-0.2.3/experiments/llm_coupling/runs/opus-5.5/plain/answers_1.json +52 -0
- prometheusnn-0.2.3/experiments/llm_coupling/runs/opus-5.5/plain/answers_2.json +52 -0
- prometheusnn-0.2.3/experiments/llm_coupling/runs/opus-5.5/tariff/answers_1.json +27 -0
- prometheusnn-0.2.3/experiments/llm_coupling/runs/opus-5.5/tariff/answers_2.json +27 -0
- prometheusnn-0.2.3/experiments/llm_coupling/runs/opus-5.5/tariff/answers_3.json +27 -0
- prometheusnn-0.2.3/experiments/llm_coupling/runs/opus-5.5/tariff/answers_4.json +27 -0
- prometheusnn-0.2.3/experiments/llm_coupling/runs/sonnet-5/plain/answers_1.json +52 -0
- prometheusnn-0.2.3/experiments/llm_coupling/runs/sonnet-5/plain/answers_2.json +52 -0
- prometheusnn-0.2.3/experiments/llm_coupling/runs/sonnet-5/tariff/answers_1.json +52 -0
- prometheusnn-0.2.3/experiments/llm_coupling/runs/sonnet-5/tariff/answers_2.json +52 -0
- prometheusnn-0.2.3/experiments/llm_coupling/scripts/01_candidate_recall.py +103 -0
- prometheusnn-0.2.3/experiments/llm_coupling/scripts/02_sample_items.py +42 -0
- prometheusnn-0.2.3/experiments/llm_coupling/scripts/03_plain_pipeline.py +60 -0
- prometheusnn-0.2.3/experiments/llm_coupling/scripts/common.py +44 -0
- prometheusnn-0.2.3/experiments/llm_coupling/scripts/score.py +54 -0
- prometheusnn-0.2.3/experiments/llm_coupling/scripts/to_text.py +26 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/pyproject.toml +1 -1
- prometheusnn-0.2.3/src/prometheus/inference/novelty.py +215 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/inference/predictor.py +3 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/inference/router.py +8 -1
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/server/INSTRUCTIONS.md +23 -10
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/server/tools/inference.py +3 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/server/tools/training.py +18 -0
- prometheusnn-0.2.3/src/prometheus/training/routing.py +148 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/training/trainer.py +3 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/ui_server.py +1 -1
- prometheusnn-0.2.3/tests/test_novelty.py +189 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_router.py +9 -0
- prometheusnn-0.2.3/tests/test_routing_fit.py +91 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/uv.lock +1 -1
- prometheusnn-0.2.1/src/prometheus/inference/novelty.py +0 -150
- prometheusnn-0.2.1/tests/test_novelty.py +0 -50
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/.github/workflows/ci.yml +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/.github/workflows/publish.yml +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/.gitignore +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/LICENSE +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/docs/beam-search-strategy.md +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/docs/roadmap-self-aware-server.md +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/scripts/parse_hts.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/scripts/parse_mesh.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/__init__.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/core/__init__.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/core/config.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/core/embedding.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/core/logging.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/core/lru.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/core/mcp_safety.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/core/responses.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/data/__init__.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/data/loader.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/data/splits.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/data/taxonomy.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/inference/__init__.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/inference/beam_search.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/inference/summary.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/insights.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/intake/__init__.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/intake/detector.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/intake/models.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/intake/profiler.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/intake/recommender.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/intake/scoper.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/py.typed +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/server/__init__.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/server/tools/__init__.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/server/tools/evaluation.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/server/tools/intake.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/server/tools/management.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/storage/__init__.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/storage/model_store.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/training/__init__.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/training/calibration.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/training/cascade.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/training/classifier.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/src/prometheus/training/noise.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/__init__.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/conftest.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/fixtures/naics_test.csv +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_beam_search.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_calibration.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_classifier.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_insights.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_intake.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_integration.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_noise.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_predictor.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_server.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_splits.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_storage.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_summary.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_taxonomy.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_trainer.py +0 -0
- {prometheusnn-0.2.1 → prometheusnn-0.2.3}/tests/test_ui_server.py +0 -0
|
@@ -5,6 +5,58 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.2.3] - 2026-11-20
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
- **OUT_OF_TAXONOMY no longer fires on ordinary in-taxonomy items.** Novelty was
|
|
12
|
+
judged by the distance to the *predicted leaf's* centroid, in units of that
|
|
13
|
+
leaf's spread. Leaves with a few near-identical training rows had their spread
|
|
14
|
+
clamped to 0.1, so an input belonging to a sibling leaf scored as extreme, and
|
|
15
|
+
the leaf prediction itself was usually the wrong one. On the HTS sample, 29% of
|
|
16
|
+
held-out tariff items and 35% of plain-English rewrites were routed
|
|
17
|
+
OUT_OF_TAXONOMY. Novelty now uses the distance to the k-th nearest training
|
|
18
|
+
embedding, against the 99th percentile of the training set's own leave-one-out
|
|
19
|
+
kNN distances: 1% of held-out items and 0% of the rewrites are flagged, while
|
|
20
|
+
40 of 40 non-product sentences still are. The centroid z-score is still
|
|
21
|
+
reported, and decides only for models saved without training embeddings.
|
|
22
|
+
- **AUTO_ACCEPT is fit to a target precision instead of fixed thresholds.**
|
|
23
|
+
Training now raises each level's threshold, top-down, until auto-accepted
|
|
24
|
+
predictions on the validation split reach `auto_accept_precision` (new
|
|
25
|
+
`TrainingConfig` field, default 0.95). Thresholds never go below the old
|
|
26
|
+
defaults; a level that cannot reach the target is never auto-accepted (its
|
|
27
|
+
threshold is 1.01) and those items go to HUMAN_REVIEW with the confident levels
|
|
28
|
+
above it. On the HTS sample the old thresholds auto-accepted 7.5% of held-out
|
|
29
|
+
items at 62.8% precision; the fitted ones auto-accept none, because leaf
|
|
30
|
+
predictions cannot reach 95% there, while the confident chapter and heading
|
|
31
|
+
are right 98.6% and 95.3% of the time. `train_model` and `resume_training`
|
|
32
|
+
return a `routing` report. With fewer than 40 validation items the defaults are
|
|
33
|
+
kept.
|
|
34
|
+
- `resume_training` now refits novelty data for the new weights and training data.
|
|
35
|
+
It previously saved new training embeddings next to stale centroids.
|
|
36
|
+
|
|
37
|
+
### Changed
|
|
38
|
+
- Predictions report `knn_threshold` in their novelty details, and the
|
|
39
|
+
OUT_OF_TAXONOMY reason names the kNN distance and threshold.
|
|
40
|
+
|
|
41
|
+
### Upgrade notes
|
|
42
|
+
- Models saved by 0.2.2 or earlier get the new novelty rule automatically: the
|
|
43
|
+
kNN threshold is fit from their stored training embeddings when they load. Their
|
|
44
|
+
thresholds are unchanged; retrain to fit thresholds to a target precision.
|
|
45
|
+
- Expect far fewer AUTO_ACCEPT decisions on large or sparse taxonomies. That is
|
|
46
|
+
the fix: those auto-accepts were often wrong.
|
|
47
|
+
|
|
48
|
+
## [0.2.2] - 2026-10-30
|
|
49
|
+
|
|
50
|
+
### Security
|
|
51
|
+
- **Novelty data is loaded without pickle.** `novelty_centroids.npz` was loaded
|
|
52
|
+
with `allow_pickle=True`, so a crafted model directory could run code in the
|
|
53
|
+
server when any tool loaded that model. In 0.2.0 and earlier this was
|
|
54
|
+
reachable through the `model_name` path traversal fixed in 0.2.1; in 0.2.1 it
|
|
55
|
+
still applied to a malicious model placed in the models directory. Both
|
|
56
|
+
novelty files now load with `allow_pickle=False`. Models saved by
|
|
57
|
+
PrometheusNN contain only numeric and string arrays and load unchanged; a
|
|
58
|
+
file holding pickled objects is refused with an error asking for a retrain.
|
|
59
|
+
|
|
8
60
|
## [0.2.1] - 2026-10-30
|
|
9
61
|
|
|
10
62
|
### Security
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: prometheusnn
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: Frictionless hierarchical text classification — an MCP server that guides you from CSV to trained classifier
|
|
5
5
|
Project-URL: Homepage, https://github.com/mfbaig35r/prometheusnn
|
|
6
6
|
Project-URL: Repository, https://github.com/mfbaig35r/prometheusnn
|
|
@@ -96,7 +96,7 @@ From there, the LLM routes through the existing tool surface (training, evaluati
|
|
|
96
96
|
|
|
97
97
|
## What's actually being built
|
|
98
98
|
|
|
99
|
-
Under the conversational surface, PrometheusNN trains a cascade of neural network classifiers — one per level of your taxonomy — with parent-noise injection so deeper levels stay robust when upper levels are uncertain. Beam search explores alternative paths when confidence drops. Temperature calibration produces probabilities you can trust for routing
|
|
99
|
+
Under the conversational surface, PrometheusNN trains a cascade of neural network classifiers — one per level of your taxonomy — with parent-noise injection so deeper levels stay robust when upper levels are uncertain. Beam search explores alternative paths when confidence drops. Temperature calibration produces probabilities you can trust for routing, and routing thresholds are fit on validation data so auto-accepted predictions meet a target precision. Novelty detection flags items far from everything the model was trained on.
|
|
100
100
|
|
|
101
101
|
```
|
|
102
102
|
Text descriptions + taxonomy labels
|
|
@@ -116,8 +116,8 @@ Text descriptions + taxonomy labels
|
|
|
116
116
|
└──────┬──────┘
|
|
117
117
|
▼
|
|
118
118
|
┌─────────────┐
|
|
119
|
-
│ Router │
|
|
120
|
-
│ + Novelty │
|
|
119
|
+
│ Router │ thresholds fit to a target precision → accept / review / reject
|
|
120
|
+
│ + Novelty │ kNN distance to the training data
|
|
121
121
|
└─────────────┘
|
|
122
122
|
```
|
|
123
123
|
|
|
@@ -60,7 +60,7 @@ From there, the LLM routes through the existing tool surface (training, evaluati
|
|
|
60
60
|
|
|
61
61
|
## What's actually being built
|
|
62
62
|
|
|
63
|
-
Under the conversational surface, PrometheusNN trains a cascade of neural network classifiers — one per level of your taxonomy — with parent-noise injection so deeper levels stay robust when upper levels are uncertain. Beam search explores alternative paths when confidence drops. Temperature calibration produces probabilities you can trust for routing
|
|
63
|
+
Under the conversational surface, PrometheusNN trains a cascade of neural network classifiers — one per level of your taxonomy — with parent-noise injection so deeper levels stay robust when upper levels are uncertain. Beam search explores alternative paths when confidence drops. Temperature calibration produces probabilities you can trust for routing, and routing thresholds are fit on validation data so auto-accepted predictions meet a target precision. Novelty detection flags items far from everything the model was trained on.
|
|
64
64
|
|
|
65
65
|
```
|
|
66
66
|
Text descriptions + taxonomy labels
|
|
@@ -80,8 +80,8 @@ Text descriptions + taxonomy labels
|
|
|
80
80
|
└──────┬──────┘
|
|
81
81
|
▼
|
|
82
82
|
┌─────────────┐
|
|
83
|
-
│ Router │
|
|
84
|
-
│ + Novelty │
|
|
83
|
+
│ Router │ thresholds fit to a target precision → accept / review / reject
|
|
84
|
+
│ + Novelty │ kNN distance to the training data
|
|
85
85
|
└─────────────┘
|
|
86
86
|
```
|
|
87
87
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
data/candidates.json
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# Experiment: coupling the cascade with an LLM judge
|
|
2
|
+
|
|
3
|
+
**Question.** PrometheusNN's cascade is fast, calibrated and cheap, but at the leaf level of a
|
|
4
|
+
large taxonomy it is often wrong. LLMs know a great deal but are slow and expensive per call.
|
|
5
|
+
Can the cascade do the cheap structured work and hand an LLM only the hard part, the way DLSS
|
|
6
|
+
renders cheaply and lets a network resolve only what the renderer cannot?
|
|
7
|
+
|
|
8
|
+
**Setup.** The `hts_viz` model: the U.S. Harmonized Tariff Schedule sample in
|
|
9
|
+
`data/hts_training.csv` (11,414 rows; 96 chapters, 1,228 headings, 4,724 subheadings), trained
|
|
10
|
+
with default settings. Everything below runs on its held-out test split (1,141 rows, rebuilt
|
|
11
|
+
exactly with `prepare_data(random_state=42)`).
|
|
12
|
+
|
|
13
|
+
## Results
|
|
14
|
+
|
|
15
|
+
### 1. Picking from the beam's own candidates does not work
|
|
16
|
+
|
|
17
|
+
| HUMAN_REVIEW items (720) | |
|
|
18
|
+
|---|---|
|
|
19
|
+
| Cascade top-1 leaf correct | 25.6% |
|
|
20
|
+
| True leaf among the beam's surviving paths (width 3 / 5 / 10) | 44.0% / 48.8% / 51.0% |
|
|
21
|
+
| Held-out items whose true subheading has **no training rows** (all 1,141) | 27.6% |
|
|
22
|
+
|
|
23
|
+
Over a quarter of held-out items belong to subheadings the model never saw, so no beam can
|
|
24
|
+
offer them.
|
|
25
|
+
|
|
26
|
+
### 2. Let the taxonomy supply the candidates
|
|
27
|
+
|
|
28
|
+
| Candidate set for review items | Contains the answer | Size (median / p90 / max) |
|
|
29
|
+
|---|---|---|
|
|
30
|
+
| A: the beam's leaves | 44.0% | 6 / 6 / 6 |
|
|
31
|
+
| B: every subheading under the confident prefix | 98.2% | 169 / 4,724 / 4,724 |
|
|
32
|
+
| **C: every subheading under the beam's headings** | **83.2%** | **20 / 48 / 136** |
|
|
33
|
+
|
|
34
|
+
Set C uses the cascade to narrow to a few headings and the taxonomy to list every subheading
|
|
35
|
+
under them, including ones with no training data. That is the design tested below.
|
|
36
|
+
|
|
37
|
+
### 3. LLM judges on 100 review items
|
|
38
|
+
|
|
39
|
+
100 HUMAN_REVIEW items sampled from the test split (`random.seed(7)`), candidates shuffled, the
|
|
40
|
+
judge blind to the answer and to the cascade's pick. "Plain English" is the same 100 items
|
|
41
|
+
rewritten by an LLM into invoice or catalog language and run through the whole pipeline again.
|
|
42
|
+
A pick of "none of these" counts as correct when the answer was not offered.
|
|
43
|
+
|
|
44
|
+
| Correct out of 100 | Tariff wording | Plain English |
|
|
45
|
+
|---|---|---|
|
|
46
|
+
| Cascade top-1 alone | 25 | 23 |
|
|
47
|
+
| Answer is in candidate set C (ceiling) | 89 | 85 |
|
|
48
|
+
| + Claude Haiku 4.5 | 72 | 68 |
|
|
49
|
+
| + Claude Sonnet 5 | 86 | 87 |
|
|
50
|
+
| + Claude Opus 5.5 | 98 | 95 |
|
|
51
|
+
|
|
52
|
+
- **Model tier dominates; wording barely matters.** Rewriting costs 4 points of candidate
|
|
53
|
+
recall; Haiku loses 4 points, Opus 3, Sonnet none (86 to 87).
|
|
54
|
+
- **Opus 5.5:** never wrong when it answered "none of these" (9/9, 13/13), overturned a correct
|
|
55
|
+
cascade answer once in 200 items, and its "high" confidence was right 128 of 129 times.
|
|
56
|
+
- **Haiku 4.5:** overturned 7 correct cascade answers per run; "high" confidence was right only
|
|
57
|
+
75 to 83% of the time, and agreeing with the cascade did not make it more reliable (72%).
|
|
58
|
+
- **Sonnet 5:** in between, 86 / 87. "None of these" was right 19 of 20 times, it overturned
|
|
59
|
+
8 correct cascade answers over 200 items, and "high" confidence was right 114 of 121 times
|
|
60
|
+
(94%). On plain English it classified 28 of the 35 items routed to OUT_OF_TAXONOMY.
|
|
61
|
+
- **The novelty gate discards classifiable items.** On plain English the cascade routed 35 items
|
|
62
|
+
to OUT_OF_TAXONOMY; Opus classified 32 of them correctly.
|
|
63
|
+
- **AUTO_ACCEPT is miscalibrated at the leaf:** 62.8% correct on the 86 held-out items it
|
|
64
|
+
auto-accepted.
|
|
65
|
+
|
|
66
|
+
### Caveats
|
|
67
|
+
|
|
68
|
+
- 100 items, one run per setting: read the numbers as direction, not as precise rates.
|
|
69
|
+
- The "plain English" rewrites are only moderately plain; technical goods stay technical and 11
|
|
70
|
+
of 100 are near-copies of the original.
|
|
71
|
+
- The judges likely know the public tariff text, which flatters them. No real invoice data yet.
|
|
72
|
+
- Judges ran as Claude Code subagents, not through the API; cost was not measured. The first
|
|
73
|
+
Opus tariff run used 25-item JSON batches, every other run the 50-item text format in
|
|
74
|
+
`scripts/to_text.py`.
|
|
75
|
+
|
|
76
|
+
## Layout
|
|
77
|
+
|
|
78
|
+
```
|
|
79
|
+
scripts/common.py paths, the held-out split, candidate-set helpers
|
|
80
|
+
scripts/01_candidate_recall.py sections 1 and 2; writes data/candidates.json (gitignored)
|
|
81
|
+
scripts/02_sample_items.py data/tariff_items.json: the 100 items + answer key
|
|
82
|
+
scripts/03_plain_pipeline.py data/plain_items.json from data/plain_texts.json
|
|
83
|
+
scripts/to_text.py blind judge batches (answer-key fields stripped)
|
|
84
|
+
scripts/score.py score a run against its key
|
|
85
|
+
prompts/judge.md judge instructions
|
|
86
|
+
prompts/paraphrase.md instructions that produced data/plain_texts.json
|
|
87
|
+
runs/<model>/<tariff|plain>/ each judge's raw answers
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Fields starting with `_` in `data/*_items.json` are the answer key. Never point a judge at
|
|
91
|
+
`data/`; generate batches into a separate folder with `to_text.py`.
|
|
92
|
+
|
|
93
|
+
## Reproduce
|
|
94
|
+
|
|
95
|
+
```sh
|
|
96
|
+
# Train the model (about 30 s). Scripts 01 to 03 assume the name hts_viz.
|
|
97
|
+
uv run python -c "
|
|
98
|
+
from prometheus.server.tools.training import train_model
|
|
99
|
+
train_model('data/hts_training.csv', 'text', ['Chapter', 'Heading', 'Subheading'], 'hts_viz')"
|
|
100
|
+
|
|
101
|
+
uv run python experiments/llm_coupling/scripts/01_candidate_recall.py
|
|
102
|
+
uv run python experiments/llm_coupling/scripts/02_sample_items.py
|
|
103
|
+
uv run python experiments/llm_coupling/scripts/03_plain_pipeline.py
|
|
104
|
+
|
|
105
|
+
# Judge: build blind batches outside data/, run prompts/judge.md on each, save answers_N.json
|
|
106
|
+
python3 experiments/llm_coupling/scripts/to_text.py data/plain_items.json /tmp/judge
|
|
107
|
+
python3 experiments/llm_coupling/scripts/score.py runs/opus-5.5/plain data/plain_items.json
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Training is not seeded, so a retrained model will differ slightly from the committed results.
|
|
111
|
+
Steps 1 to 3 reproduced this run's numbers and item sets exactly against the original model.
|