pi-research 1.4.0 → 1.4.1

package/ml/models/conflict-structured/feature-names.json

@@ -0,0 +1,22 @@
+[
+  "authoritative_source_count",
+  "blocked_source_count",
+  "blog_count",
+  "candidate_conflict",
+  "file_count",
+  "forum_count",
+  "github_readme_count",
+  "github_repo_count",
+  "has_authority_resolution_path",
+  "negative_signal_sources",
+  "official_doc_count",
+  "other_count",
+  "paper_count",
+  "positive_signal_sources",
+  "query_academic",
+  "query_comparison",
+  "query_procedural",
+  "query_temporal",
+  "query_versioned",
+  "source_count"
+]

package/ml/models/conflict-structured/meta.json

@@ -0,0 +1,5 @@
+{
+  "task": "conflict",
+  "bestModel": "lr",
+  "rows": 80
+}

package/ml/models/domain/metrics.json

@@ -0,0 +1,16 @@
+{
+  "macro_f1": 0.5773809523809524,
+  "train_size": 122,
+  "val_size": 34,
+  "high_risk_downgrades": 1,
+  "classes": [
+    "changelog",
+    "github",
+    "package-registry",
+    "papers",
+    "security",
+    "specs",
+    "vendor-status",
+    "web"
+  ]
+}

package/ml/models/domain-lr/metrics.json

@@ -0,0 +1,16 @@
+{
+  "macro_f1": 0.41485507246376807,
+  "train_size": 122,
+  "val_size": 34,
+  "high_risk_downgrades": 1,
+  "classes": [
+    "changelog",
+    "github",
+    "package-registry",
+    "papers",
+    "security",
+    "specs",
+    "vendor-status",
+    "web"
+  ]
+}

package/ml/models/followup/meta.json

@@ -0,0 +1,3 @@
+{
+  "confidenceThreshold": 0.75
+}

package/ml/models/sufficiency-structured/feature-names.json

@@ -0,0 +1,22 @@
+[
+  "authoritative_source_count",
+  "blocked_source_count",
+  "blog_count",
+  "file_count",
+  "forum_count",
+  "github_readme_count",
+  "github_repo_count",
+  "has_authority",
+  "has_only_one_good_source",
+  "negative_signal_sources",
+  "official_doc_count",
+  "other_count",
+  "paper_count",
+  "positive_signal_sources",
+  "query_academic",
+  "query_comparison",
+  "query_procedural",
+  "query_temporal",
+  "query_versioned",
+  "source_count"
+]

package/ml/models/sufficiency-structured/meta.json

@@ -0,0 +1,5 @@
+{
+  "task": "sufficiency",
+  "bestModel": "lr",
+  "rows": 78
+}

package/ml/router/README.md

@@ -0,0 +1,106 @@
+# Tiny Router Training Runbook
+
+Target budget:
+
+- GPU RAM: 2 GB
+- CPU RAM: 20 GB
+- Default path: CPU-first, frozen embeddings, small models
+
+## Environment
+
+```bash
+python3 -m venv .venv-router
+. .venv-router/bin/activate
+pip install -r ml/router/requirements.txt
+```
+
+## Phase 1 — domain router
+
+```bash
+node scripts/router/audit-cache.mjs
+node scripts/router/export-examples.mjs
+node scripts/router/split-examples.mjs
+
+python ml/router/embed_model2vec.py \
+  --input data/router/examples.jsonl \
+  --gold data/router/gold-domain.jsonl \
+  --synthetic data/router/synthetic-train.jsonl
+
+python ml/router/train_domain_classifier.py \
+  --embeddings data/router/domain-model2vec.npz data/router/synthetic-model2vec.npz \
+  --gold-embeddings data/router/gold-model2vec.npz \
+  --out .cache/models/pi-research-router/domain \
+  --model-type auto
+
+python ml/router/evaluate_domain.py \
+  --model .cache/models/pi-research-router/domain/model.joblib \
+  --embeddings data/router/gold-model2vec.npz \
+  --out metrics/router/domain-model2vec-lr.json
+
+python ml/router/benchmark_latency.py \
+  --model-dir .cache/models/pi-research-router/domain \
+  --examples data/router/gold-domain.jsonl \
+  --out metrics/router/latency.json
+
+python scripts/router/eval_domain_unknown.py \
+  --model-dir .cache/models/pi-research-router/domain \
+  --input data/router/unknown-domain-smoke.jsonl
+```
+
+## Phase 2 — structured baselines
+
+Build provisional structured rows:
+
+```bash
+node scripts/router/export_structured_provisional.mjs
+node scripts/router/eval_structured_baselines.mjs
+```
+
+Train conservative structured classifiers:
+
+```bash
+python ml/router/train_structured_baseline.py --task conflict
+python ml/router/train_structured_baseline.py --task sufficiency
+```
+
+Outputs:
+
+- `.cache/models/pi-research-router/conflict-structured/`
+- `.cache/models/pi-research-router/sufficiency-structured/`
+- `metrics/router/conflict-structured-models.json`
+- `metrics/router/sufficiency-structured-models.json`
+
+## Runtime flags
+
+```bash
+PI_RESEARCH_TINY_ROUTER=1
+PI_RESEARCH_TINY_ROUTER_MODEL=.cache/models/pi-research-router
+PI_RESEARCH_TINY_ROUTER_TIMEOUT_MS=50
+PI_RESEARCH_TINY_ROUTER_DOMAIN=1
+PI_RESEARCH_TINY_ROUTER_FOLLOWUP=1
+PI_RESEARCH_TINY_ROUTER_CONFLICT=0
+PI_RESEARCH_TINY_ROUTER_SUFFICIENCY=0
+```
+
+Keep conflict/sufficiency off until metrics are reviewed.
+
+## Server deploy
+
+Safe MCP runtime deploy:
+
+```bash
+scripts/router/deploy-server-runtime.sh \
+  blackknight@100.98.190.19 \
+  ~/work/pi-research-runtime
+```
+
+This syncs the repo, installs user-local Node if needed, copies trained router models, runs `npm install`, and writes:
+
+- `start-mcp-tiny-router-safe.sh`
+- `start-mcp-tiny-router-experimental.sh`
+
+Recommended start command:
+
+```bash
+ssh blackknight@100.98.190.19 'cd ~/work/pi-research-runtime && ./start-mcp-tiny-router-safe.sh'
+```

package/ml/router/benchmark_latency.py

@@ -0,0 +1,81 @@
+import json
+import argparse
+import time
+import numpy as np
+import joblib
+import os
+
+sys_path_added = False
+if not sys_path_added:
+    import sys
+    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+    sys_path_added = True
+    
+from features import load_embedding_model, extract_domain_features
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-dir", required=True)
+    parser.add_argument("--examples", required=True)
+    parser.add_argument("--out", required=True)
+    args = parser.parse_args()
+
+    print(f"Loading Model2Vec...")
+    emb_model = load_embedding_model()
+    
+    print(f"Loading Classifier...")
+    clf = joblib.load(f"{args.model_dir}/model.joblib")
+
+    # Load a few queries to test
+    queries = []
+    with open(args.examples, "r") as f:
+        for line in f:
+            if not line.strip(): continue
+            ex = json.loads(line)
+            queries.append(ex["query"])
+            
+    # Warmup
+    print("Warming up...")
+    for q in queries[:10]:
+        feats = extract_domain_features([q], ["fast"], emb_model=emb_model, show_progress_bar=False)
+        clf.predict(feats)
+        
+    # Benchmark
+    print(f"Benchmarking {len(queries)} queries sequentially...")
+    latencies = []
+    
+    for q in queries:
+        t0 = time.perf_counter()
+        
+        feats = extract_domain_features([q], ["fast"], emb_model=emb_model, show_progress_bar=False)
+        pred = clf.predict(feats)[0]
+        
+        t1 = time.perf_counter()
+        latencies.append((t1 - t0) * 1000) # ms
+
+    latencies = np.array(latencies)
+    p50 = np.percentile(latencies, 50)
+    p95 = np.percentile(latencies, 95)
+    mean = np.mean(latencies)
+    
+    print(f"p50: {p50:.2f} ms")
+    print(f"p95: {p95:.2f} ms")
+    print(f"Mean: {mean:.2f} ms")
+    
+    os.makedirs(os.path.dirname(args.out), exist_ok=True)
+    
+    metrics = {
+        "task": "domain",
+        "latency_ms": {
+            "p50": p50,
+            "p95": p95,
+            "mean": mean,
+            "samples": len(latencies)
+        }
+    }
+    
+    with open(args.out, "w") as f:
+        json.dump(metrics, f, indent=2)
+
+if __name__ == "__main__":
+    main()

package/ml/router/daemon.py

@@ -0,0 +1,140 @@
+import sys
+import json
+import logging
+import joblib
+import numpy as np
+import traceback
+import os
+
+# Add the directory containing features.py to sys.path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from features import load_embedding_model, extract_domain_features, extract_followup_features
+
+logging.basicConfig(level=logging.ERROR)
+
+
+def load_model(path):
+    return joblib.load(path) if os.path.exists(path) else None
+
+
+def load_feature_names(path):
+    if not os.path.exists(path):
+        return None
+    with open(path, "r") as f:
+        return json.load(f)
+
+
+def predict_proba_like(clf, features):
+    if hasattr(clf, "predict_proba"):
+        proba = clf.predict_proba(features)[0]
+        max_idx = int(np.argmax(proba))
+        return clf.classes_[max_idx], float(proba[max_idx])
+
+    pred = clf.predict(features)[0]
+    return pred, 1.0
+
+
+def vectorize_structured_features(feature_names, features):
+    row = [float(features.get(name, 0.0)) for name in feature_names]
+    return np.array([row], dtype=np.float32)
+
+
+def main():
+    if len(sys.argv) < 2:
+        print(json.dumps({"error": "Missing model path"}))
+        sys.exit(1)
+
+    model_dir = sys.argv[1]
+
+    try:
+        emb_model = load_embedding_model()
+        domain_clf = load_model(os.path.join(model_dir, "domain", "model.joblib"))
+        followup_clf = load_model(os.path.join(model_dir, "followup", "model.joblib"))
+        conflict_clf = load_model(os.path.join(model_dir, "conflict-structured", "model.joblib"))
+        sufficiency_clf = load_model(os.path.join(model_dir, "sufficiency-structured", "model.joblib"))
+        conflict_feature_names = load_feature_names(os.path.join(model_dir, "conflict-structured", "feature-names.json"))
+        sufficiency_feature_names = load_feature_names(os.path.join(model_dir, "sufficiency-structured", "feature-names.json"))
+    except Exception as e:
+        print(json.dumps({"error": f"Failed to load models: {str(e)}"}))
+        sys.exit(1)
+
+    print("READY", flush=True)
+
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+
+        try:
+            req = json.loads(line)
+            req_id = req.get("id")
+            task = req.get("task", "domain")
+            query = req.get("query", "")
+            mode = req.get("mode", "fast")
+
+            if task == "domain":
+                if not domain_clf:
+                    print(json.dumps({"id": req_id, "error": "Domain model not loaded"}), flush=True)
+                    continue
+
+                feats = extract_domain_features([query], [mode], emb_model=emb_model, show_progress_bar=False)
+                pred, confidence = predict_proba_like(domain_clf, feats)
+
+                print(json.dumps({
+                    "id": req_id,
+                    "domain": str(pred),
+                    "confidence": confidence
+                }), flush=True)
+
+            elif task == "followup":
+                if not followup_clf:
+                    print(json.dumps({"id": req_id, "error": "Followup model not loaded"}), flush=True)
+                    continue
+
+                conflict = req.get("conflict", "none")
+                sources = req.get("sources", {})
+
+                feats = extract_followup_features([query], [mode], [conflict], [sources], emb_model=emb_model, show_progress_bar=False)
+                pred, confidence = predict_proba_like(followup_clf, feats)
+
+                print(json.dumps({
+                    "id": req_id,
+                    "action": str(pred),
+                    "confidence": confidence
+                }), flush=True)
+
+            elif task == "conflict":
+                if not conflict_clf or not conflict_feature_names:
+                    print(json.dumps({"id": req_id, "error": "Conflict model not loaded"}), flush=True)
+                    continue
+
+                feats = vectorize_structured_features(conflict_feature_names, req.get("features", {}))
+                pred, confidence = predict_proba_like(conflict_clf, feats)
+                print(json.dumps({
+                    "id": req_id,
+                    "decision": str(pred),
+                    "confidence": confidence
+                }), flush=True)
+
+            elif task == "sufficiency":
+                if not sufficiency_clf or not sufficiency_feature_names:
+                    print(json.dumps({"id": req_id, "error": "Sufficiency model not loaded"}), flush=True)
+                    continue
+
+                feats = vectorize_structured_features(sufficiency_feature_names, req.get("features", {}))
+                pred, confidence = predict_proba_like(sufficiency_clf, feats)
+                print(json.dumps({
+                    "id": req_id,
+                    "decision": str(pred),
+                    "confidence": confidence
+                }), flush=True)
+
+            else:
+                print(json.dumps({"id": req_id, "error": f"Unknown task: {task}"}), flush=True)
+
+        except Exception as e:
+            print(json.dumps({"error": str(e), "trace": traceback.format_exc()}), flush=True)
+
+
+if __name__ == "__main__":
+    main()

package/ml/router/embed_model2vec.py

@@ -0,0 +1,48 @@
+import json
+import argparse
+import numpy as np
+from features import load_embedding_model, extract_domain_features, extract_followup_features
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input", required=True)
+    parser.add_argument("--out", required=True)
+    parser.add_argument("--model", default="minishlab/potion-base-8M")
+    parser.add_argument("--task", default="domain")
+    args = parser.parse_args()
+
+    examples = []
+    with open(args.input, "r") as f:
+        for line in f:
+            if not line.strip(): continue
+            ex = json.loads(line)
+            if "task" not in ex or ex["task"] == args.task:
+                examples.append(ex)
+    
+    print(f"Loaded {len(examples)} examples for task '{args.task}'")
+    
+    print(f"Loading StaticModel: {args.model}")
+    model = load_embedding_model()
+    
+    queries = [ex["query"] for ex in examples]
+    modes = [ex.get("mode", ex.get("meta", {}).get("mode", "fast")) for ex in examples]
+    
+    if args.task == "domain":
+        print(f"Encoding {len(queries)} queries for domain routing...")
+        features = extract_domain_features(queries, modes, emb_model=model, show_progress_bar=True)
+    elif args.task == "followup":
+        print(f"Encoding {len(queries)} queries for followup action...")
+        conflicts = [ex.get("conflict", "none") for ex in examples]
+        sources_list = [ex.get("sources", {}) for ex in examples]
+        features = extract_followup_features(queries, modes, conflicts, sources_list, emb_model=model, show_progress_bar=True)
+    else:
+        raise ValueError(f"Unknown task: {args.task}")
+    
+    ids = np.array([ex.get("id", str(i)) for i, ex in enumerate(examples)])
+    labels = np.array([ex["label"] for ex in examples])
+    
+    print(f"Saving features shape {features.shape} to {args.out}")
+    np.savez(args.out, features=features, ids=ids, labels=labels)
+
+if __name__ == "__main__":
+    main()

package/ml/router/evaluate_domain.py

@@ -0,0 +1,67 @@
+import json
+import argparse
+import numpy as np
+import joblib
+import os
+from sklearn.metrics import classification_report, f1_score, confusion_matrix, accuracy_score
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--embeddings", required=True)
+    parser.add_argument("--out", required=True)
+    args = parser.parse_args()
+
+    # Load model
+    print(f"Loading model from {args.model}")
+    clf = joblib.load(args.model)
+    
+    # Load data
+    data = np.load(args.embeddings)
+    X = data["features"]
+    y_true = data["labels"]
+    
+    # Evaluate
+    y_pred = clf.predict(X)
+    
+    macro_f1 = f1_score(y_true, y_pred, average="macro", zero_division=0)
+    accuracy = accuracy_score(y_true, y_pred)
+    
+    print("\nGold Validation Report:")
+    print(classification_report(y_true, y_pred, zero_division=0))
+    
+    print(f"\nMacro-F1: {macro_f1:.4f}")
+    
+    # Extract high-risk misclassifications
+    classes = clf.classes_
+    high_risk_classes = ["security", "papers", "specs"]
+    cm = confusion_matrix(y_true, y_pred, labels=classes)
+    
+    high_risk_errors = 0
+    web_idx = np.where(classes == "web")[0]
+    if len(web_idx) > 0:
+        web_idx = web_idx[0]
+        for hr_class in high_risk_classes:
+            hr_idx = np.where(classes == hr_class)[0]
+            if len(hr_idx) > 0:
+                errors = cm[hr_idx[0], web_idx]
+                if errors > 0:
+                    print(f"HIGH RISK WARNING: {errors} '{hr_class}' queries routed to 'web'")
+                    high_risk_errors += errors
+
+    # Save artifacts
+    os.makedirs(os.path.dirname(args.out), exist_ok=True)
+    
+    metrics = {
+        "task": "domain",
+        "eval_set_size": len(X),
+        "macro_f1": macro_f1,
+        "accuracy": accuracy,
+        "high_risk_downgrades": int(high_risk_errors),
+        "classes": classes.tolist()
+    }
+    with open(args.out, "w") as f:
+        json.dump(metrics, f, indent=2)
+
+if __name__ == "__main__":
+    main()

package/ml/router/features.py

@@ -0,0 +1,60 @@
+import numpy as np
+from model2vec import StaticModel
+
+EMBEDDING_MODEL_NAME = "minishlab/potion-base-8M"
+
+def load_embedding_model() -> StaticModel:
+    """Loads the base static model for feature extraction."""
+    return StaticModel.from_pretrained(EMBEDDING_MODEL_NAME)
+
+def encode_modes(modes: list) -> np.ndarray:
+    """Encodes a list of mode strings into a one-hot float32 numpy array."""
+    encoded = []
+    for mode in modes:
+        encoded.append([
+            1.0 if mode == "fast" else 0.0,
+            1.0 if mode == "deep" else 0.0,
+            1.0 if mode == "academic" else 0.0,
+            1.0 if mode == "code" else 0.0,
+        ])
+    return np.array(encoded, dtype=np.float32)
+
+def extract_domain_features(queries: list, modes: list, emb_model: StaticModel = None, show_progress_bar: bool = False) -> np.ndarray:
+    """Extracts the combined feature vector (text embeddings + one-hot mode) for domain routing."""
+    if emb_model is None:
+        emb_model = load_embedding_model()
+        
+    emb = emb_model.encode(queries, show_progress_bar=show_progress_bar)
+    modes_np = encode_modes(modes)
+    return np.hstack([emb, modes_np])
+
+def encode_followup_meta(conflicts: list, sources_list: list) -> np.ndarray:
+    """Encodes conflict and source metadata into a feature array for followup classification."""
+    encoded = []
+    for conflict, sources in zip(conflicts, sources_list):
+        row = [
+            1.0 if conflict == "severe" else 0.0,
+            1.0 if conflict == "minor" else 0.0,
+            1.0 if conflict == "none" else 0.0,
+            
+            1.0 if sources.get("has_authority", False) else 0.0,
+            1.0 if sources.get("has_forum", False) else 0.0,
+            1.0 if sources.get("has_news", False) else 0.0,
+            1.0 if sources.get("has_recent", False) else 0.0,
+            
+            # Normalize source count (cap at 10)
+            min(float(sources.get("source_count", 3)) / 10.0, 1.0)
+        ]
+        encoded.append(row)
+    return np.array(encoded, dtype=np.float32)
+
+def extract_followup_features(queries: list, modes: list, conflicts: list, sources_list: list, emb_model: StaticModel = None, show_progress_bar: bool = False) -> np.ndarray:
+    """Extracts features for the followup action classifier."""
+    if emb_model is None:
+        emb_model = load_embedding_model()
+        
+    emb = emb_model.encode(queries, show_progress_bar=show_progress_bar)
+    modes_np = encode_modes(modes)
+    meta_np = encode_followup_meta(conflicts, sources_list)
+    
+    return np.hstack([emb, modes_np, meta_np])

package/ml/router/requirements.txt

@@ -0,0 +1,5 @@
+model2vec>=0.2.0
+scikit-learn
+numpy
+joblib
+imbalanced-learn

package/ml/router/train_classifier.py

@@ -0,0 +1,57 @@
+import json
+import argparse
+import numpy as np
+import joblib
+from sklearn.svm import LinearSVC
+from sklearn.linear_model import LogisticRegression
+from sklearn.calibration import CalibratedClassifierCV
+from sklearn.metrics import classification_report, f1_score, confusion_matrix
+import os
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--embeddings", required=True, nargs="+")
+    parser.add_argument("--out", required=True)
+    parser.add_argument("--model-type", choices=["svc", "lr"], default="svc")
+    args = parser.parse_args()
+
+    # Load all data files and combine
+    X_list, y_list = [], []
+    for emb_file in args.embeddings:
+        data = np.load(emb_file)
+        X_list.append(data["features"])
+        y_list.append(data["labels"])
+        
+    X_train = np.vstack(X_list)
+    y_train = np.hstack(y_list)
+
+    print(f"Combined Train size: {len(X_train)}")
+    
+    from imblearn.over_sampling import RandomOverSampler
+    ros = RandomOverSampler(random_state=42)
+    X_res, y_res = ros.fit_resample(X_train, y_train)
+    print(f"Size after OverSampling: {len(X_res)}")
+
+    # Train model
+    print(f"Training {args.model_type} with class_weight='balanced'...")
+    if args.model_type == "svc":
+        base_clf = LinearSVC(class_weight="balanced", dual=False, max_iter=5000, C=0.5)
+        clf = CalibratedClassifierCV(base_clf, method="sigmoid", cv=5)
+    else:
+        clf = LogisticRegression(class_weight="balanced", max_iter=5000)
+        
+    clf.fit(X_res, y_res)
+
+    # Save artifacts
+    os.makedirs(args.out, exist_ok=True)
+    model_path = os.path.join(args.out, "model.joblib")
+    joblib.dump(clf, model_path)
+    print(f"Model saved to {model_path}")
+    
+    # Save a small report to standard out
+    preds = clf.predict(X_res)
+    print("Train Report:")
+    print(classification_report(y_res, preds))
+
+if __name__ == "__main__":
+    main()

package/ml/router/train_domain_classifier.py

@@ -0,0 +1,209 @@
+import argparse
+import json
+import os
+from collections import Counter
+
+import joblib
+import numpy as np
+from imblearn.over_sampling import RandomOverSampler
+from sklearn.calibration import CalibratedClassifierCV
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import accuracy_score, f1_score
+from sklearn.svm import LinearSVC
+
+HIGH_RISK_CLASSES = {"security", "papers", "specs"}
+PRECISION_TARGETS = (0.95, 0.90, 0.85)
+MIN_DEFAULT_THRESHOLD = 0.35
+MIN_HIGH_RISK_THRESHOLD = 0.55
+
+
+def load_embeddings(paths):
+    features, labels = [], []
+    for path in paths:
+        data = np.load(path)
+        features.append(data["features"])
+        labels.append(data["labels"])
+    return np.vstack(features), np.hstack(labels)
+
+
+def build_classifier(model_type):
+    if model_type == "svc":
+        base = LinearSVC(class_weight="balanced", dual=False, max_iter=5000, C=0.5)
+        return CalibratedClassifierCV(base, method="sigmoid", cv=5)
+    if model_type == "lr":
+        return LogisticRegression(class_weight="balanced", max_iter=5000)
+    raise ValueError(f"Unsupported model_type: {model_type}")
+
+
+def train_classifier(model_type, X_train, y_train):
+    ros = RandomOverSampler(random_state=42)
+    X_resampled, y_resampled = ros.fit_resample(X_train, y_train)
+    clf = build_classifier(model_type)
+    clf.fit(X_resampled, y_resampled)
+    return clf, len(X_resampled)
+
+
+def evaluate_classifier(clf, X_eval, y_eval):
+    probs = clf.predict_proba(X_eval)
+    pred_idx = np.argmax(probs, axis=1)
+    preds = clf.classes_[pred_idx]
+    confs = np.max(probs, axis=1)
+    accuracy = accuracy_score(y_eval, preds)
+    macro_f1 = f1_score(y_eval, preds, average="macro", zero_division=0)
+
+    high_risk_downgrades = 0
+    for gold, pred in zip(y_eval, preds):
+        if gold in HIGH_RISK_CLASSES and pred == "web":
+            high_risk_downgrades += 1
+
+    return {
+        "accuracy": float(accuracy),
+        "macro_f1": float(macro_f1),
+        "high_risk_downgrades": int(high_risk_downgrades),
+        "preds": preds,
+        "confs": confs,
+    }
+
+
+def derive_threshold_for_label(label, y_true, preds, confs):
+    candidate_thresholds = sorted({0.0, *[float(conf) for pred, conf in zip(preds, confs) if pred == label]})
+    best = None
+    floor = MIN_HIGH_RISK_THRESHOLD if label in HIGH_RISK_CLASSES else MIN_DEFAULT_THRESHOLD
+
+    gold_support = sum(1 for gold in y_true if gold == label)
+    for target_precision in PRECISION_TARGETS:
+        for threshold in candidate_thresholds:
+            accepted = [i for i, (pred, conf) in enumerate(zip(preds, confs)) if pred == label and conf >= threshold]
+            if not accepted:
+                continue
+            tp = sum(1 for i in accepted if y_true[i] == label)
+            fp = len(accepted) - tp
+            precision = tp / len(accepted)
+            recall = tp / gold_support if gold_support else 0.0
+            score = (precision >= target_precision, recall, precision, -threshold)
+            if best is None or score > best[0]:
+                best = (score, {
+                    "threshold": float(threshold),
+                    "precision": float(precision),
+                    "recall": float(recall),
+                    "accepted": len(accepted),
+                    "tp": int(tp),
+                    "fp": int(fp),
+                    "target_precision": float(target_precision),
+                })
+        if best and best[0][0]:
+            best[1]["threshold"] = max(float(best[1]["threshold"]), floor)
+            return best[1]
+
+    if best:
+        best[1]["threshold"] = max(float(best[1]["threshold"]), floor)
+        return best[1]
+    return {
+        "threshold": 0.75 if label in HIGH_RISK_CLASSES else 0.80,
+        "precision": 0.0,
+        "recall": 0.0,
+        "accepted": 0,
+        "tp": 0,
+        "fp": 0,
+        "target_precision": PRECISION_TARGETS[-1],
+    }
+
+
+def derive_calibration(clf, X_eval, y_eval):
+    evaluation = evaluate_classifier(clf, X_eval, y_eval)
+    preds = evaluation["preds"]
+    confs = evaluation["confs"]
+    thresholds = {}
+    diagnostics = {}
+
+    for label in clf.classes_:
+        diag = derive_threshold_for_label(label, y_eval, preds, confs)
+        thresholds[str(label)] = float(diag["threshold"])
+        diagnostics[str(label)] = diag
+
+    return {
+        "defaultThreshold": 0.80,
+        "highRiskThreshold": 0.75,
+        "domainThresholds": thresholds,
+        "diagnostics": diagnostics,
+    }
+
+
+def choose_best_report(reports):
+    return max(
+        reports,
+        key=lambda item: (-item["metrics"]["high_risk_downgrades"], item["metrics"]["accuracy"], item["metrics"]["macro_f1"], item["model_type"]),
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--embeddings", required=True, nargs="+")
+    parser.add_argument("--gold-embeddings")
+    parser.add_argument("--out", required=True)
+    parser.add_argument("--model-type", choices=["svc", "lr", "auto"], default="auto")
+    args = parser.parse_args()
+
+    X_train, y_train = load_embeddings(args.embeddings)
+    print(f"Combined Train size: {len(X_train)}")
+    print(f"Train label distribution: {dict(Counter(y_train))}")
+
+    candidate_model_types = [args.model_type] if args.model_type != "auto" else ["svc", "lr"]
+    gold_data = load_embeddings([args.gold_embeddings]) if args.gold_embeddings else None
+
+    reports = []
+    for model_type in candidate_model_types:
+        print(f"Training {model_type} with class_weight='balanced'...")
+        clf, resampled_size = train_classifier(model_type, X_train, y_train)
+        report = {
+            "model_type": model_type,
+            "clf": clf,
+            "resampled_size": resampled_size,
+        }
+        if gold_data:
+            X_gold, y_gold = gold_data
+            report["metrics"] = evaluate_classifier(clf, X_gold, y_gold)
+            report["calibration"] = derive_calibration(clf, X_gold, y_gold)
+            print(json.dumps({
+                "model_type": model_type,
+                "accuracy": report["metrics"]["accuracy"],
+                "macro_f1": report["metrics"]["macro_f1"],
+                "high_risk_downgrades": report["metrics"]["high_risk_downgrades"],
+            }, indent=2))
+        reports.append(report)
+
+    best = reports[0] if len(reports) == 1 or not gold_data else choose_best_report(reports)
+    print(f"Selected model: {best['model_type']}")
+
+    os.makedirs(args.out, exist_ok=True)
+    model_path = os.path.join(args.out, "model.joblib")
+    joblib.dump(best["clf"], model_path)
+    print(f"Model saved to {model_path}")
+
+    meta = {
+        "modelType": best["model_type"],
+        "trainSize": int(len(X_train)),
+        "resampledTrainSize": int(best["resampled_size"]),
+    }
+    with open(os.path.join(args.out, "meta.json"), "w") as f:
+        json.dump(meta, f, indent=2)
+
+    if best.get("calibration"):
+        with open(os.path.join(args.out, "calibration.json"), "w") as f:
+            json.dump(best["calibration"], f, indent=2)
+
+    if best.get("metrics"):
+        metrics = {
+            "task": "domain",
+            "modelType": best["model_type"],
+            "accuracy": best["metrics"]["accuracy"],
+            "macro_f1": best["metrics"]["macro_f1"],
+            "high_risk_downgrades": best["metrics"]["high_risk_downgrades"],
+            "classes": [str(label) for label in best["clf"].classes_],
+        }
+        with open(os.path.join(args.out, "metrics.json"), "w") as f:
+            json.dump(metrics, f, indent=2)
+
+
+if __name__ == "__main__":
+    main()

package/ml/router/train_structured_baseline.py

@@ -0,0 +1,174 @@
+import argparse
+import json
+import os
+import re
+from collections import Counter
+
+import joblib
+import numpy as np
+from imblearn.over_sampling import RandomOverSampler
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import accuracy_score, classification_report, f1_score
+from sklearn.model_selection import GroupKFold
+from sklearn.neural_network import MLPClassifier
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+
+TASK_INPUTS = {
+    "conflict": os.path.join("data", "router", "gold-conflict-structured.jsonl"),
+    "sufficiency": os.path.join("data", "router", "gold-sufficiency-structured.jsonl"),
+}
+
+TASK_BASELINES = {
+    "conflict": os.path.join("metrics", "router", "conflict-baseline-provisional.json"),
+    "sufficiency": os.path.join("metrics", "router", "sufficiency-baseline-provisional.json"),
+}
+
+MODEL_BUILDERS = {
+    "lr": lambda: Pipeline([
+        ("scaler", StandardScaler()),
+        ("clf", LogisticRegression(max_iter=5000, class_weight="balanced")),
+    ]),
+    "mlp": lambda: Pipeline([
+        ("scaler", StandardScaler()),
+        ("clf", MLPClassifier(hidden_layer_sizes=(32, 16), max_iter=2000, random_state=42, early_stopping=False)),
+    ]),
+}
+
+
+def normalize_query_group(query: str) -> str:
+    return re.sub(r"\s+", " ", re.sub(r"[^a-z0-9\s]+", " ", (query or "").lower())).strip()
+
+
+def load_jsonl(path: str):
+    with open(path, "r") as f:
+        return [json.loads(line) for line in f if line.strip()]
+
+
+def build_xy(rows):
+    feature_names = sorted(rows[0]["features"].keys())
+    X = np.array([[row["features"][name] for name in feature_names] for row in rows], dtype=np.float32)
+    y = np.array([row["label"] for row in rows])
+    groups = np.array([normalize_query_group(row["query"]) for row in rows])
+    return X, y, groups, feature_names
+
+
+def choose_n_splits(y, groups):
+    min_class = min(Counter(y).values())
+    return max(2, min(5, len(set(groups)), min_class))
+
+
+def evaluate_model(model_name, rows):
+    X, y, groups, feature_names = build_xy(rows)
+    splitter = GroupKFold(n_splits=choose_n_splits(y, groups))
+    gold, pred = [], []
+    fold_rows = []
+
+    for fold, (train_idx, test_idx) in enumerate(splitter.split(X, y, groups), start=1):
+        ros = RandomOverSampler(random_state=42)
+        X_train, y_train = ros.fit_resample(X[train_idx], y[train_idx])
+        clf = MODEL_BUILDERS[model_name]()
+        clf.fit(X_train, y_train)
+        probs = clf.predict_proba(X[test_idx]) if hasattr(clf, "predict_proba") else None
+        preds = clf.predict(X[test_idx])
+
+        for local_idx, pred_label in enumerate(preds):
+            idx = test_idx[local_idx]
+            confidence = None
+            if probs is not None:
+                confidence = float(np.max(probs[local_idx]))
+            gold.append(str(y[idx]))
+            pred.append(str(pred_label))
+            fold_rows.append({
+                "fold": fold,
+                "query": rows[idx]["query"],
+                "gold": str(y[idx]),
+                "pred": str(pred_label),
+                "confidence": confidence,
+            })
+
+    return {
+        "model": model_name,
+        "accuracy": accuracy_score(gold, pred),
+        "macro_f1": f1_score(gold, pred, average="macro"),
+        "classification_report": classification_report(gold, pred, output_dict=True),
+        "rows": fold_rows,
+        "feature_names": feature_names,
+    }
+
+
+def train_full_model(model_name, rows):
+    X, y, _, feature_names = build_xy(rows)
+    ros = RandomOverSampler(random_state=42)
+    X_train, y_train = ros.fit_resample(X, y)
+    clf = MODEL_BUILDERS[model_name]()
+    clf.fit(X_train, y_train)
+    return clf, feature_names
+
+
+def load_baseline_metrics(task: str):
+    path = TASK_BASELINES[task]
+    if not os.path.exists(path):
+        return None
+    with open(path, "r") as f:
+        data = json.load(f)
+    return {
+        "accuracy": data.get("accuracy"),
+        "macroF1": data.get("macroF1"),
+        "falseSufficient": data.get("falseSufficient"),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--task", choices=["conflict", "sufficiency"], required=True)
+    parser.add_argument("--input")
+    parser.add_argument("--out-dir")
+    args = parser.parse_args()
+
+    input_path = args.input or TASK_INPUTS[args.task]
+    out_dir = args.out_dir or os.path.join(".cache", "models", "pi-research-router", f"{args.task}-structured")
+    metrics_path = os.path.join("metrics", "router", f"{args.task}-structured-models.json")
+
+    rows = load_jsonl(input_path)
+    baseline = load_baseline_metrics(args.task)
+
+    reports = {
+        model_name: evaluate_model(model_name, rows)
+        for model_name in ["lr", "mlp"]
+    }
+    best_name = max(reports.keys(), key=lambda name: (reports[name]["macro_f1"], reports[name]["accuracy"]))
+    best_model, feature_names = train_full_model(best_name, rows)
+
+    os.makedirs(out_dir, exist_ok=True)
+    os.makedirs(os.path.dirname(metrics_path), exist_ok=True)
+    joblib.dump(best_model, os.path.join(out_dir, "model.joblib"))
+    with open(os.path.join(out_dir, "feature-names.json"), "w") as f:
+        json.dump(feature_names, f, indent=2)
+    with open(os.path.join(out_dir, "meta.json"), "w") as f:
+        json.dump({"task": args.task, "bestModel": best_name, "rows": len(rows)}, f, indent=2)
+
+    summary = {
+        "task": args.task,
+        "rows": len(rows),
+        "baseline": baseline,
+        "best_model": best_name,
+        "models": reports,
+    }
+    with open(metrics_path, "w") as f:
+        json.dump(summary, f, indent=2)
+
+    print(json.dumps({
+        "task": args.task,
+        "rows": len(rows),
+        "baseline": baseline,
+        "best_model": best_name,
+        "best_accuracy": reports[best_name]["accuracy"],
+        "best_macro_f1": reports[best_name]["macro_f1"],
+        "lr_macro_f1": reports["lr"]["macro_f1"],
+        "mlp_macro_f1": reports["mlp"]["macro_f1"],
+    }, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/package.json

@@ -1,20 +1,21 @@
 {
   "name": "pi-research",
-  "version": "1.4.0",
+  "version": "1.4.1",
   "private": false,
   "type": "module",
   "description": "Zero-setup grounded web research for AI coding agents.",
   "license": "MIT",
   "main": "./index.js",
   "bin": {
-    "pi-research": "./bin/pi-research.js",
-    "unblind-mcp": "./bin/unblind-mcp.js"
+    "pi-research": "bin/pi-research.js",
+    "unblind-mcp": "bin/unblind-mcp.js"
   },
   "files": [
     "bin",
     "extensions",
     "index.js",
     "lib",
+    "ml",
     "mcp",
     "mcp-server.js",
     "pi-research.js",