soup-cli 0.71.4__tar.gz → 0.71.6__tar.gz

soup_cli-0.71.6/.mailmap

@@ -0,0 +1,8 @@
+# .mailmap — consolidate contributor identities for `git shortlog` and the
+# GitHub contributors graph. See: https://git-scm.com/docs/gitmailmap
+#
+# Salil Mhatre committed under four name/email combinations
+# (two emails, three display names). Map them all to one canonical identity.
+Salil Mhatre <32305505+Deadpool2000@users.noreply.github.com> <d2kyt@protonmail.com>
+Salil Mhatre <32305505+Deadpool2000@users.noreply.github.com> Salil <32305505+Deadpool2000@users.noreply.github.com>
+Salil Mhatre <32305505+Deadpool2000@users.noreply.github.com> Salil M <32305505+Deadpool2000@users.noreply.github.com>

{soup_cli-0.71.4 → soup_cli-0.71.6}/CHANGELOG.md

@@ -12,6 +12,98 @@ reproducing 70+ versions of notes.
 
 ## [Unreleased]
 
+## [0.71.6] - 2026-06-02
+
+### Added
+- **`soup build` live runner** — the dbt-for-SFT DAG (`soup build <manifest>`) now
+  *materialises* datasets instead of only dry-running the plan. Five built-in
+  transforms ship live (`identity`, `drop_empty`, `lowercase`, `strip`,
+  `dedup_exact`); `table` rebuilds from scratch, `view` re-derives on every run,
+  and `incremental` re-transforms only the rows whose content hash changed
+  (tracked in a SQLite state store, keyed by row hash **and** the model's
+  transform+config fingerprint so a transform change re-runs everything). Custom
+  transforms are passed per-run via the Python API's `transforms=` map. Outputs
+  are written atomically; the `--output-dir` is symlink-checked before any
+  directory is created.
+- **`soup data gen-magpie` live generator** — the Magpie synthetic generator
+  (Xu et al. 2024) now actually generates. It feeds an aligned model its
+  chat-template prefix (chatml / llama3 / gemma / mistral families auto-detected)
+  and harvests the self-generated user instruction + assistant response via raw
+  completion. Live providers: `ollama` (`/api/generate` raw) and `vllm`
+  (`/v1/completions`) — both SSRF-hardened (loopback-only HTTP); `anthropic` is
+  rejected (no raw-completion endpoint). Optional `--quality-filter` drops
+  low-quality rows via the v0.47 toxicity/educational scorers; exact-duplicate
+  instructions are de-duplicated.
+- **`soup eval irt-subset --model {1pl,2pl,3pl}`** — the IRT eval-cost optimiser
+  gained 2PL (per-item discrimination) and 3PL (+guessing floor) joint
+  coordinate-ascent MLE fits alongside the existing 1PL Rasch. `1pl` keeps the
+  closed-form path for back-compat; `2pl`/`3pl` route through the new `fit_irt`.
+- **Tokenizer-aware memorization probe** — `score_memorization(..., tokenizer=...)`
+  and `split_prefix(..., tokenizer=...)` (used by `soup diagnose`) now split the
+  prefix/suffix on real token-id boundaries and measure echo-overlap over
+  sub-word tokens when a tokenizer (HF id / path / duck-typed object) is supplied,
+  catching BPE-level memorization that whitespace tokenisation misses. Default
+  (no tokenizer) keeps the whitespace behaviour.
+
+### Fixed
+- **`soup data augment --provider ollama|vllm` no longer crashes** — the command
+  imported a non-existent `OllamaProvider` symbol and raised `ImportError` on
+  every non-OpenAI provider. It now routes through the shared, SSRF-hardened
+  provider factory; `--model` / `--base-url` are honoured, the output path is
+  containment- and symlink-checked, and the write is atomic.
+
+### Security
+- **Ollama / vLLM provider URLs reject `0.0.0.0`** — `validate_ollama_url` /
+  `validate_vllm_url` dropped the bind-any wildcard from their loopback allow-set
+  (now `localhost` / `127.0.0.1` / `::1` only), matching the newer
+  `validate_hub_endpoint` / `validate_webhook_url` SSRF validators. Reachable now
+  that Magpie threads a user-supplied `--base-url` through these providers.
+
+## [0.71.5] - 2026-06-02
+
+### Added
+- **`soup eval against` now reads eval metrics** — `ExperimentTracker.get_metric_series`
+  falls back to the `eval_results` table when the metric is not a per-step
+  training column (`loss` / `lr` / `grad_norm` / `speed` / `gpu_mem`). So
+  `soup eval against <base> --candidate <run> --metric task_accuracy` returns a
+  real score series (benchmark scores live in `eval_results`, not `metrics`)
+  instead of "Empty series". Per-step columns still read from `metrics` — no
+  regression for existing callers.
+- **`soup advise` learns from past project outcomes** — `soup advise` now reads
+  this project's accepted-verdict history (`~/.soup/advise_history.jsonl`) and
+  biases the rubric: 3+ successful SFT precedents flip a marginal RAG call to
+  SFT; 3+ negative GRPO outcomes suppress GRPO in favour of SFT-on-traces; an
+  encouraged choice gets a small confidence nudge. Scoped per-project (one
+  project's record never biases another). No history → identical to before.
+- **Slack/Discord webhooks on four more commands** — `--slack-url` / `--discord-url`
+  (SSRF-hardened, loopback-only HTTP, RFC1918 rejected, never crashes the
+  command) now ship on `soup ingest`, `soup prune-prompt`, `soup ab` (fires only
+  on a `reject_h0` / `accept_h0` decision, not `continue`), and
+  `soup data active-sample` — not just `soup drift-alarm`. The validator + sender
+  moved to a shared `soup_cli/utils/webhooks.py`.
+- **Tokenizer-aware `soup prune-prompt`** — `--tokenizer <model_or_path>` detects
+  and strips the shared system-prompt prefix on **token** boundaries instead of
+  characters, so a multi-byte UTF-8 prefix can never be split mid-code-point.
+  Default (no `--tokenizer`) keeps the whitespace-character behaviour.
+- **Curriculum bucketing by loss percentile** — `DynamicCurriculumCallback` now
+  buckets samples by the percentile rank of the live loss (or perplexity)
+  signal within a rolling window when `data.curriculum_metric` is `loss` /
+  `perplexity`, so a consistently-hard sample is routed to the same difficulty
+  bucket across recomputes. `length` and warm-up still use round-robin.
+- **`--hub` on `soup data push` and `soup data forge`** — `soup data push
+  --hub modelscope|modelers` uploads a dataset via the matching SDK
+  (`repo_type=dataset`, commit message sanitised); `soup data forge --hub
+  <non-hf> --teacher owner/name` pre-fetches the teacher model from that hub
+  (and warns when the teacher is not a repo id so `--hub` is never silently
+  ignored). HF stays the default.
+
+### Notes
+- Live SaaS *pull* adapters for `soup ingest` (Langfuse / LangSmith / Helicone /
+  OpenPipe / OpenAI SDKs, issue #204) remain deferred: they need credentialed
+  vendor accounts with populated trace data to validate honestly. Tracked as an
+  open, `infra-blocked` (external-account) item. `soup ingest` continues to parse
+  the JSONL export you pull from your dashboard.
+
 ## [0.71.4] - 2026-06-02
 
 ### Added

{soup_cli-0.71.4 → soup_cli-0.71.6}/CONTRIBUTING.md

@@ -120,7 +120,7 @@ src/soup_cli/
   templates/          - 17 built-in soup.yaml templates (YAML + manifest.json) with load_template loader (v0.39.0, +bco v0.40.0)
   ui/                 - Web UI (FastAPI + HTML/JS SPA)
 
-tests/                - Test suite (274 files, 12474 tests)
+tests/                - Test suite (276 files, 12703 tests)
 examples/             - Real-world config examples and datasets
 ```
 
@@ -150,7 +150,11 @@ pytest tests/test_data.py::test_detect_alpaca_format -v
 pytest tests/ --cov=soup_cli --cov-report=html
 ```
 
-### Test Files (86 files)
+### Test Files (275 files)
+
+> A representative sample of the suite below. The full table lives in
+> [`.claude/CLAUDE.md`](.claude/CLAUDE.md); run `pytest tests/ -v` for the complete list.
+
 
 | File | Covers |
 |------|--------|
@@ -441,6 +445,17 @@ The project follows semantic versioning: `MAJOR.MINOR.PATCH`
 5. Tag: `git tag v0.X.0 && git push --tags`
 6. GitHub Actions auto-publishes to PyPI on the `v*` tag (Trusted Publisher / OIDC)
 
+## Recognition
+
+We credit every contributor — recognition is free and you earned it.
+
+- **[CONTRIBUTORS.md](CONTRIBUTORS.md):** every merged (or in-tree-adopted) PR adds you here, with your work and PR number.
+- **CHANGELOG.md:** from the release that ships your change, the entry is tagged `(#NNN by @you)`.
+- **Commit trailers:** PRs land with a `Co-Authored-By:` line for you, so the GitHub
+  contributors graph reflects your work even after a squash merge.
+
+If you contributed and aren't credited somewhere, that's a bug — open a PR or an issue and we'll fix it.
+
 ## Community
 
 - **Issues:** Report bugs and request features on [GitHub Issues](https://github.com/MakazhanAlpamys/Soup/issues)

soup_cli-0.71.6/CONTRIBUTORS.md

@@ -0,0 +1,43 @@
+# Contributors
+
+Soup is built by its community. Thank you to everyone who has contributed code,
+tests, docs, and ideas. ❤️
+
+This list is maintained by hand alongside the GitHub
+[contributors graph](https://github.com/MakazhanAlpamys/Soup/graphs/contributors).
+Merged a PR and don't see yourself here? Open a PR adding your line — that counts too.
+
+## Maintainer
+
+- **Alpamys** ([@MakazhanAlpamys](https://github.com/MakazhanAlpamys)) — creator & lead maintainer
+
+## Contributors
+
+Listed by first contribution. PR numbers link the work.
+
+- **Salil Mhatre** ([@Deadpool2000](https://github.com/Deadpool2000))
+  - `soup version --json` for machine-readable CI output ([#6](https://github.com/MakazhanAlpamys/Soup/pull/6))
+  - RAM + disk-space checks in `soup doctor` ([#7](https://github.com/MakazhanAlpamys/Soup/pull/7))
+  - `soup runs clean` for smart checkpoint space management ([#9](https://github.com/MakazhanAlpamys/Soup/pull/9))
+  - Official Docker support for easier onboarding ([#20](https://github.com/MakazhanAlpamys/Soup/pull/20))
+  - `soup bench` — model speed + VRAM measurement ([#25](https://github.com/MakazhanAlpamys/Soup/pull/25))
+  - `--prompts-file` option for `soup bench` ([#30](https://github.com/MakazhanAlpamys/Soup/pull/30))
+  - Happy-path + CPU-warning tests for `soup bench` ([#31](https://github.com/MakazhanAlpamys/Soup/pull/31))
+  - `soup cost` — cloud GPU training cost estimation ([#42](https://github.com/MakazhanAlpamys/Soup/pull/42))
+  - `--nccl` flag for `soup doctor` multi-GPU bandwidth checks ([#178](https://github.com/MakazhanAlpamys/Soup/pull/178))
+- **Chinmaya Sahu** ([@csking101](https://github.com/csking101))
+  - DPO example config, sample data, and tests ([#48](https://github.com/MakazhanAlpamys/Soup/pull/48))
+  - FP8 `rowwise` + `rowwise_with_gw_hp` scaling recipes ([#62](https://github.com/MakazhanAlpamys/Soup/pull/62))
+- **Yixuan Xu** ([@mzl2233](https://github.com/mzl2233))
+  - Guard diagnose-gate on distributed worker ranks ([#169](https://github.com/MakazhanAlpamys/Soup/pull/169))
+- **dreamer0129** ([@dreamer0129](https://github.com/dreamer0129))
+  - Rich-markup escape fix in legacy `soup adapters` commands ([#175](https://github.com/MakazhanAlpamys/Soup/pull/175), adopted in-tree as [#174](https://github.com/MakazhanAlpamys/Soup/issues/174))
+- **Vivaan Dhawan** ([@VIVAAN-DHAWAN](https://github.com/VIVAAN-DHAWAN))
+  - Reject pickle/zip streams renamed to `.safetensors` via magic-byte check ([#198](https://github.com/MakazhanAlpamys/Soup/pull/198))
+- **Shivam** ([@shivam2931120](https://github.com/shivam2931120))
+  - Tokenizer-aware repetition scoring for the echo-trap detector ([#242](https://github.com/MakazhanAlpamys/Soup/pull/242))
+
+---
+
+Want to join this list? See [CONTRIBUTING.md](CONTRIBUTING.md) — good first issues are
+labelled in the [issue tracker](https://github.com/MakazhanAlpamys/Soup/labels/good%20first%20issue).

{soup_cli-0.71.4 → soup_cli-0.71.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: soup-cli
-Version: 0.71.4
+Version: 0.71.6
 Summary: Fine-tune LLMs in one command. No SSH, no config hell.
 Project-URL: Homepage, https://github.com/MakazhanAlpamys/Soup
 Project-URL: Repository, https://github.com/MakazhanAlpamys/Soup
@@ -183,21 +183,20 @@ infrastructure instead of improving models. Soup fixes that.
 
 ## What's New
 
-**v0.71.4 — Adapter lifecycle + loop wiring.** The merge, PR, and continuous-loop surfaces go live:
-
-- **Canary verdict on merge** — `soup adapters merge … --canary suite.json` scores the merged
-  adapter and reports **OK / MINOR / MAJOR**; `--strict-verdict` exits non-zero on a MAJOR
-  regression. Works with no model load using a pre-scored canary suite.
-- **Evolutionary merge for real** — `soup adapters merge --strategy cmaes --eval suite --budget 1h`
-  now runs the full CMA-ES search (merge → score → optimise) and writes the best blend, instead of
-  just printing a plan.
-- **Publish an adapter PR** — `soup adapters pr <title> --base-sha <hex> --adapter <path> --push
-  owner/repo#42` posts the rendered PR straight to a GitHub PR comment.
-- **Continuous fine-tuning loop, wired up** — `soup loop watch --pre-wired` runs the real
-  traces → DPO → eval-gate → canary pipeline; `--pack-cans` snapshots every iteration as a
-  shareable Soup Can with Registry lineage (`soup loop replay <id> --extract dir`).
-- **Branches ↔ Registry** — `soup adapters branch <name> --attach-to-registry <id>` /
-  `--from-registry <id>` links training-env snapshots into the Registry lineage DAG.
+**v0.71.6 — Synth data & build pipeline go live.** Three deferred stubs are now real:
+
+- **`soup build` materialises** — the dbt-for-SFT DAG no longer just dry-runs. `soup build
+  manifest.yaml --output-dir out/` runs the transforms (`drop_empty` / `lowercase` / `strip` /
+  `dedup_exact` / `identity` built in), rebuilds `table` / `view` models from scratch, and
+  re-transforms only the changed rows for `incremental` models (SQLite-tracked).
+- **`soup data gen-magpie` generates** — feed an aligned model its chat-template prefix and
+  harvest the self-generated instruction + response. Live for `--provider ollama` and `vllm`
+  (loopback-only), with an optional `--quality-filter`.
+- **2PL / 3PL eval-cost models** — `soup eval irt-subset --model 2pl|3pl` adds per-item
+  discrimination (and a 3PL guessing floor) on top of the existing 1PL Rasch fit.
+- **Bug fix:** `soup data augment --provider ollama|vllm` no longer crashes with an `ImportError`
+  — it now routes through the shared, SSRF-hardened provider factory and honours `--model` /
+  `--base-url`.
 
 Full history: [CHANGELOG.md](CHANGELOG.md) &middot; [GitHub Releases](https://github.com/MakazhanAlpamys/Soup/releases).
 

{soup_cli-0.71.4 → soup_cli-0.71.6}/README.md

@@ -49,21 +49,20 @@ infrastructure instead of improving models. Soup fixes that.
 
 ## What's New
 
-**v0.71.4 — Adapter lifecycle + loop wiring.** The merge, PR, and continuous-loop surfaces go live:
-
-- **Canary verdict on merge** — `soup adapters merge … --canary suite.json` scores the merged
-  adapter and reports **OK / MINOR / MAJOR**; `--strict-verdict` exits non-zero on a MAJOR
-  regression. Works with no model load using a pre-scored canary suite.
-- **Evolutionary merge for real** — `soup adapters merge --strategy cmaes --eval suite --budget 1h`
-  now runs the full CMA-ES search (merge → score → optimise) and writes the best blend, instead of
-  just printing a plan.
-- **Publish an adapter PR** — `soup adapters pr <title> --base-sha <hex> --adapter <path> --push
-  owner/repo#42` posts the rendered PR straight to a GitHub PR comment.
-- **Continuous fine-tuning loop, wired up** — `soup loop watch --pre-wired` runs the real
-  traces → DPO → eval-gate → canary pipeline; `--pack-cans` snapshots every iteration as a
-  shareable Soup Can with Registry lineage (`soup loop replay <id> --extract dir`).
-- **Branches ↔ Registry** — `soup adapters branch <name> --attach-to-registry <id>` /
-  `--from-registry <id>` links training-env snapshots into the Registry lineage DAG.
+**v0.71.6 — Synth data & build pipeline go live.** Three deferred stubs are now real:
+
+- **`soup build` materialises** — the dbt-for-SFT DAG no longer just dry-runs. `soup build
+  manifest.yaml --output-dir out/` runs the transforms (`drop_empty` / `lowercase` / `strip` /
+  `dedup_exact` / `identity` built in), rebuilds `table` / `view` models from scratch, and
+  re-transforms only the changed rows for `incremental` models (SQLite-tracked).
+- **`soup data gen-magpie` generates** — feed an aligned model its chat-template prefix and
+  harvest the self-generated instruction + response. Live for `--provider ollama` and `vllm`
+  (loopback-only), with an optional `--quality-filter`.
+- **2PL / 3PL eval-cost models** — `soup eval irt-subset --model 2pl|3pl` adds per-item
+  discrimination (and a 3PL guessing floor) on top of the existing 1PL Rasch fit.
+- **Bug fix:** `soup data augment --provider ollama|vllm` no longer crashes with an `ImportError`
+  — it now routes through the shared, SSRF-hardened provider factory and honours `--model` /
+  `--base-url`.
 
 Full history: [CHANGELOG.md](CHANGELOG.md) &middot; [GitHub Releases](https://github.com/MakazhanAlpamys/Soup/releases).
 

{soup_cli-0.71.4 → soup_cli-0.71.6}/docs/backends-and-ops.md

@@ -459,6 +459,11 @@ Every completed run also stores an estimated cost (`$` per run) computed from th
 captured GPU device name and duration. `soup runs show` renders `—` for CPU /
 MPS / unknown GPUs (no fabricated zeros).
 
+As of v0.71.5, the metric-series lookup that powers replay (`ExperimentTracker.get_metric_series`)
+transparently falls back to the `eval_results` table when a metric has no per-step
+rows — so you can plot a benchmark-score curve (e.g. `mmlu`, `gsm8k`) the same way
+you plot `loss`, without caring which table holds the series.
+
 ### Tracker integrations (--tracker mlflow / swanlab / trackio)
 
 ```bash

{soup_cli-0.71.4 → soup_cli-0.71.6}/docs/commands.md

@@ -72,7 +72,8 @@ soup data generate ... --provider anthropic   Use Claude API
 soup data generate ... --provider vllm        Use local vLLM server
 soup data generate ... --template code        Domain templates (code/conversation/qa/preference/reasoning)
 soup data generate ... --quality-pipeline     Auto validate + filter + dedup
-soup data augment <path> --strategy rephrase|translate|style  LLM-driven augmentationsoup data from-traces --logs l.jsonl --format langchain --signal thumbs_up --output p.jsonl  Preference pairs from traces
+soup data augment <path> --strategy rephrase|translate|style [--provider ollama|vllm --model <m> --base-url <url>]  LLM-driven augmentation
+soup data from-traces --logs l.jsonl --format langchain --signal thumbs_up --output p.jsonl  Preference pairs from traces
 soup data from-traces ... --judge --min-confidence 0.7  LLM-judge confidence filter
 soup data review prefs.jsonl --sample 10      Preview preference pairssoup data filter <path> --coherence 0.3       Quality filter (perplexity/coherence)
 soup data sample <path> --n 1000             Random sample subset
@@ -90,10 +91,12 @@ soup data download user/ds --samples 1000    Stream first 1000 samples
 soup data register --name my-ds --path d.jsonl --format alpaca  Register dataset
 soup data unregister --name my-ds            Remove from registry
 soup data push --input d.jsonl --hf-dataset user/name  Upload local JSONL as HF dataset
+soup data push --input d.jsonl --hf-dataset u/n --hub modelscope|modelers  Upload to an alternative hub
 soup data registry                           List all registered datasets
 soup data demo                                List bundled demo JSONL fixtures
 soup data demo alpaca_demo --output ./d.jsonl Copy a bundled demo JSONL fixture
 soup data forge --docs ./docs --task sft --target-rows 1000  Synthetic data pipeline + provenance
+soup data forge --docs ./docs --hub modelscope --teacher owner/name  Pre-fetch the teacher from an alternative hub
 soup data score --input rows.jsonl            Composite quality scorecard (PII + toxicity + lang + edu)
 soup data decontaminate --input rows.jsonl --benchmarks mmlu,gsm8k  Drop benchmark-overlap rows
 soup data toxicity --input rows.jsonl -o tox.jsonl  Flag toxic rows (keyword baseline)
@@ -137,7 +140,7 @@ soup can publish r.can --hf-hub user/name    Publish .can to HF Hub as dataset
 soup runs                                     List training runs
 soup runs show <run_id>                       Run details + loss graph + cost
 soup runs compare <run_1> <run_2>             Compare two runs
-soup runs replay <run_id>                     Replay summary + loss curve from history
+soup runs replay <run_id>                     Replay summary + loss curve from history (also plots a benchmark-score curve when the metric lives in eval_results)
 soup why [run_id]                             Explain training anomalies (heuristic)
 soup tui                                      Full-screen Textual dashboard (requires [tui] extra)
 soup train --config soup.yaml --profile       Record torch.profiler trace to <output>/profiles/
@@ -169,8 +172,10 @@ soup edit set --base <m> --method rome|memit|alphaedit --subject "..." --target
 soup edit diff <before-run> <after-run> --probes p.jsonl  Knowledge-injection diff visualizer
 soup ingest --source langfuse|langsmith|helicone|openpipe|otel|openai-stored --logs <jsonl>  Universal trace importer (6 SaaS adapters → normalised JSONL)
 soup prune-prompt --input <jsonl> --output <jsonl> --min-frequency 0.95  Detect + strip shared system-prompt prefix
+soup prune-prompt ... --tokenizer <id-or-path>  Tokenizer-aware prefix detection (decodes remaining ids, boundary-safe)
 soup data active-sample --input <jsonl> --output <jsonl> --budget N  Top-N uncertain prod traces for human review
 soup ab --input <jsonl> --metric latency|judge_score|retry_rate  mSPRT sequential A/B (decision: continue / reject_h0 / accept_h0)
+soup ingest|prune-prompt|ab|data active-sample ... --slack-url <https> | --discord-url <https>  Shared SSRF-validated webhook on completion
 soup drift-alarm --reference <jsonl> --live <jsonl> --threshold 0.2  Rolling-KL drift alarm (exit 3 on drift)
 soup drift-alarm ... --slack-url <https> | --discord-url <https>  Optional SSRF-validated webhook on drift detected
 soup tunability --list                                   List built-in candidate-base catalogue
@@ -206,9 +211,9 @@ soup local-rl status --db <path>              Print interactions / thumbs-up / t
 soup local-rl record --db <path> --prompt <q> --response <r> --thumb up|down  Append thumbs record
 soup local-rl harvest --db <path> -o <pairs.jsonl>  Harvest DPO pairs from thumbs into JSONL
 soup local-rl train --db <path> --backend ollama|mlx --model <id>  Nightly DPO/KTO/ORPO train (v0.68.1)
-soup build <manifest.yaml> [--dry-run]        dbt-for-SFT DAG: validate + plan dataset transforms (v0.69.0)
+soup build <manifest.yaml> [--dry-run] [--output-dir <dir>]  dbt-for-SFT DAG: validate + plan + live materialise (v0.69.0; live v0.71.6)
 soup expect <data.jsonl> <suite.yaml>         Expectations suite: PII / token-length / refusal / judge (v0.69.0)
-soup data gen-magpie --base <m> --provider ollama|anthropic|vllm --target N [--plan-only]  Magpie synthetic generator (v0.69.0)
+soup data gen-magpie --base <m> --provider ollama|vllm --target N --output <jsonl> [--base-url <url>] [--quality-filter]  Magpie synthetic generator — live (v0.69.0; live v0.71.6)
 soup data persona-mix --prompts <jsonl> --n N --output <jsonl>  Persona-Hub diversity sampler (v0.69.0)
 soup data brain-rot <data.jsonl> [--strict]   Brain-rot detector — arXiv 2510.13928 (v0.69.0)
 soup iterative-dpo --base-model <m> --reward-model <rm> --prompts <p.jsonl> --output-dir <out> --rounds N --pairs-per-round N [--plan-only]  Iterative DPO loop driver (v0.70.0; live runner v0.70.1)

{soup_cli-0.71.4 → soup_cli-0.71.6}/docs/data.md

@@ -41,8 +41,8 @@ models:
   - {name: filtered, kind: incremental, refs: [raw], transform: filter_low_quality}
   - {name: tokenized, kind: incremental, refs: [filtered], transform: tokenize}
 EOF
-soup build build.yaml --dry-run   # validate topology + plan
-# soup build build.yaml           # live materialise (v0.69.1)
+soup build build.yaml --dry-run                 # validate topology + plan
+soup build build.yaml --output-dir built/       # live materialise (v0.71.6)
 
 # Expectations suite — Great Expectations for chat data
 cat > suite.yaml << 'EOF'
@@ -53,9 +53,9 @@ expectations:
 EOF
 soup expect data.jsonl suite.yaml   # exit 3 on suite failure
 
-# Magpie synthetic data — chat-template-prefix harvest (plan, runner v0.69.1)
+# Magpie synthetic data — chat-template-prefix harvest (live, v0.71.6)
 soup data gen-magpie --base meta-llama/Llama-3.1-8B-Instruct \
-    --provider ollama --target 1000 --plan-only
+    --provider ollama --target 1000 --output magpie.jsonl --quality-filter
 
 # Persona-Hub diversity — prompt × persona × style matrix sampling
 soup data persona-mix --prompts prompts.jsonl --n 500 --output mixed.jsonl
@@ -64,7 +64,7 @@ soup data persona-mix --prompts prompts.jsonl --n 500 --output mixed.jsonl
 soup data brain-rot data.jsonl --strict --max-major-fraction 0.10
 ```
 
-Every command applies the project-wide TOCTOU policy (`os.lstat + S_ISLNK` symlink rejection before any open) and cwd containment via the shared `paths.enforce_under_cwd_and_no_symlink` helper. Live runners for `soup build` and `soup data gen-magpie` land in v0.69.1; the other three are LIVE today.
+Every command applies the project-wide TOCTOU policy (`os.lstat + S_ISLNK` symlink rejection before any open) and cwd containment via the shared `paths.enforce_under_cwd_and_no_symlink` helper. All five are LIVE: `soup build` materialises with five built-in transforms (`identity` / `drop_empty` / `lowercase` / `strip` / `dedup_exact`) and SQLite-tracked incremental re-transform (v0.71.6); `soup data gen-magpie` harvests via raw completion against `--provider ollama|vllm` (loopback-only; `anthropic` rejected — no raw-completion endpoint, v0.71.6).
 
 
 ## Production Trace Ecosystem (`soup ingest`)
@@ -94,6 +94,14 @@ soup prune-prompt --input traces.jsonl --output pruned.jsonl --min-frequency 0.9
 
 Binary-search over up-to-32 candidate templates finds the longest qualifying prefix (a longer threshold-meeting prefix may exist beyond the universal one — Soup does not early-exit on the 100% match). Two-pass file read with a 100 000-row DoS cap.
 
+**Tokenizer-aware mode (v0.71.5).** Pass `--tokenizer <id-or-path>` (a HuggingFace repo id, a local path, or anything `AutoTokenizer.from_pretrained` accepts) to detect the shared prefix in *token* space and decode only the remaining ids:
+
+```bash
+soup prune-prompt --input traces.jsonl --output pruned.jsonl --tokenizer Qwen/Qwen2.5-0.5B
+```
+
+Char-level stripping can cut a BPE multi-byte sequence in half when the shared prefix ends mid-token; token-aware pruning finds the longest shared *token-id* prefix and decodes the remainder, so the boundary always lands on a real token. Per-row encoding is capped at 50 000 tokens. Omit `--tokenizer` to keep the original character-level behaviour.
+
 
 ## Active-Learning Sampler (`soup data active-sample`)
 
@@ -108,6 +116,8 @@ soup data active-sample --input traces.jsonl --output for-review.jsonl --budget
 
 The output JSONL is a drop-in prompt set for `soup eval human` (v0.19). Budget is bounded `[1, 100 000]`.
 
+**Webhooks (v0.71.5).** `soup ingest`, `soup prune-prompt`, `soup ab`, and `soup data active-sample` all accept `--slack-url` / `--discord-url` and POST a one-line summary on completion through the same SSRF-hardened validator as `soup drift-alarm` (scheme allowlist, loopback-only HTTP, RFC1918 / link-local / reserved / multicast rejected; the post never raises, so a flaky webhook can't fail the command). `soup ab` only fires when the sequential test actually decides (`reject_h0` / `accept_h0`), not while it's still `continue`-ing.
+
 
 ## Synthetic Data Generation
 
@@ -196,9 +206,13 @@ soup data augment ./data/train.jsonl --strategy translate --lang es,fr,de \
 # Style transfer (formal / casual / technical / etc.)
 soup data augment ./data/train.jsonl --strategy style --styles formal,casual \
   --output ./data/train_styled.jsonl
+
+# Local provider (Ollama / vLLM) — loopback-only, pick the model + base URL
+soup data augment ./data/train.jsonl --strategy rephrase --count 2 \
+  --provider ollama --model qwen2.5:0.5b --output ./data/train_local.jsonl
 ```
 
-Works with any provider supported by `soup data generate` (OpenAI, Ollama, Anthropic, vLLM, local server). `--count` is capped at 10; `--lang` and `--styles` each capped at 10 entries × 32 chars.
+Works with any provider supported by `soup data generate` (OpenAI, Ollama, vLLM, local server). `--model` and `--base-url` select a specific local model/endpoint; the Ollama/vLLM paths are loopback-only (SSRF-hardened). `--count` is capped at 10; `--lang` and `--styles` each capped at 10 entries × 32 chars.
 
 
 ## Trace-to-Preference
@@ -479,6 +493,8 @@ Three tasks supported: `sft` (Q&A pairs), `preference` (chosen/rejected), `tool`
 
 Document discovery is one level deep over `.txt` / `.md` / `.json` / `.jsonl`; dotfiles + symlinked directories are skipped. All paths are cwd-contained, all writes are atomic via staged-tempfile + `os.replace`, and write targets are rejected if they're symlinks. **Judge providers are live**: `--judge-provider ollama` (localhost-only), `--judge-provider anthropic` (env-only API key), `--judge-provider vllm` (scheme-validated). Per-call judge exceptions logged at DEBUG.
 
+**Alternative teacher hubs (v0.71.5).** `--hub modelscope|modelers` pre-fetches the `--teacher` from that hub when the teacher is a routable repo id (`owner/name`); `--hub hf` (default) is a no-op and leaves the teacher as a provenance label. If `--hub` is non-HF but `--teacher` is not a repo id (e.g. the default `local-judge`), Soup prints a loud yellow warning rather than silently dropping the flag.
+
 
 ## Data Quality Scorecard
 

{soup_cli-0.71.4 → soup_cli-0.71.6}/docs/evaluation.md

@@ -82,6 +82,8 @@ soup advise compare
 4. Task is `factual_lookup` with high output variance → **RAG**.
 5. Otherwise → **SFT**.
 
+**Cross-project confidence bias (v0.71.5).** When `~/.soup/advise_history.jsonl` holds ≥3 prior verdicts for the *same choice* in the *same project*, `soup advise` nudges its confidence (not its decision) toward what worked before: a net-positive precedent record (you accepted it AND its recorded outcome was good) bumps confidence up by a small constant; a net-negative one bumps it down. The rubric verdict itself never changes — only how sure Soup is. Verdicts must be `--record`ed for the bias to kick in.
+
 **Why this command exists.** "Choose fine-tuning vs RAG vs prompt-engineering" is the most-mis-made decision in the space. Reddit, HN, IBM, and Google Cloud all converge on the same advice (start with prompts, escalate to RAG, fine-tune as last resort) and almost everyone ignores it because nobody has the data to prove their case is the exception. Soup `autopilot` picks hyperparameters AFTER you've decided to train; `soup advise` owns the layer above. No trainer library has an incentive to tell users *not to train* — Unsloth's funnel, Axolotl's hosted business, LLaMA-Factory's Alibaba alignment all monetise the training event.
 
 
@@ -213,6 +215,8 @@ soup ab --input ab.jsonl --metric judge_score --alpha 0.01 --beta 0.10 --effect-
 
 Input rows look like `{"arm": "control", "latency": 1.23}` or `{"arm": "treatment", "judge_score": 0.91}`. Decision is one of `continue` (keep collecting samples), `reject_h0` (real difference detected), `accept_h0` (no significant difference). Composes with `soup loop canary` (v0.58) — promote or roll back as soon as the LLR clears a decision boundary.
 
+`soup ab` accepts `--slack-url` / `--discord-url` (v0.71.5) and pings the webhook **only when the test actually decides** (`reject_h0` / `accept_h0`) — a still-running `continue` stays quiet so you're not paged on every peek. Same SSRF-hardened validator as `soup drift-alarm`.
+
 
 ## Drift Alarm (`soup drift-alarm`)
 
@@ -499,8 +503,11 @@ soup eval checklist tests.yaml --evidence responses.json
 ```bash
 # Pick top-info 30% of items (5-10x eval-bill cut without losing power)
 soup eval irt-subset per_item_correctness.jsonl --size small --output plan.json
+
+# Richer item models: 2PL learns per-item discrimination, 3PL adds a guessing floor
+soup eval irt-subset per_item_correctness.jsonl --size small --model 2pl
 ```
 
-Closed-form 1PL Rasch fit (`β̂_i = -log(p̂_i / (1 - p̂_i))`); ranks by `p̂ · (1-p̂)` info (maximised at 50/50 items, since extremes carry no new ranking information). `full` keeps 100%, `small` keeps 30%, `tiny` keeps 10%.
+`--model` picks the item-response model (default `1pl`). `1pl` is the closed-form Rasch fit (`β̂_i = -log(p̂_i / (1 - p̂_i))`); `2pl` adds a per-item discrimination parameter and `3pl` a guessing floor, both via joint coordinate-ascent MLE (v0.71.6). Items rank by Fisher information at θ=0 (`p̂ · (1-p̂)` in 1PL — maximised at 50/50 items, since extremes carry no new ranking information). `full` keeps 100%, `small` keeps 30%, `tiny` keeps 10%.
 
 

{soup_cli-0.71.4 → soup_cli-0.71.6}/docs/training.md

@@ -907,12 +907,15 @@ Layer dynamic re-weighting on top of the static `curriculum` bucketer. Every N s
 training:
   curriculum: true                          # static bucketer (v0.23.0)
   curriculum_buckets: 4
+  curriculum_metric: perplexity             # length (default) | loss | perplexity
   curriculum_dynamic: true                  # NEW — dynamic re-weighting
   curriculum_dynamic_recompute_steps: 50    # refresh every 50 global steps
   curriculum_dynamic_floor: 0.05            # min weight per bucket
   curriculum_dynamic_temperature: 1.0       # softmax temp on uncertainty
 ```
 
+**Bucketing by difficulty percentile (v0.71.5).** When `curriculum_metric` is `loss` or `perplexity`, the dynamic callback assigns each step's sample to a bucket by its *rank* within a rolling 512-step window of the difficulty signal (perplexity = `exp(min(loss, 50))`), instead of the round-robin fallback used for `length`. This keeps the buckets calibrated to the live loss distribution rather than a static length sort. `length` (the default) keeps the round-robin assignment.
+
 Visualise the recorded bucket-weight evolution with `soup runs curriculum-curve <run_id>`.
 
 DDP / grad-accum safety: multi-rank launches must wire an `all_reduce` hook on per-bucket stats (a cross-validator rejects un-coordinated multi-rank runs upfront). Multi-trainer expansion beyond `sft` / `pretrain` is tracked for v0.48.1.

{soup_cli-0.71.4 → soup_cli-0.71.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "soup-cli"
-version = "0.71.4"
+version = "0.71.6"
 description = "Fine-tune LLMs in one command. No SSH, no config hell."
 readme = "README.md"
 license = "Apache-2.0"

{soup_cli-0.71.4 → soup_cli-0.71.6}/src/soup_cli/__init__.py

@@ -1,3 +1,3 @@
 """Soup CLI — Fine-tune LLMs in one command."""
 
-__version__ = "0.71.4"
+__version__ = "0.71.6"

{soup_cli-0.71.4 → soup_cli-0.71.6}/src/soup_cli/commands/_eval_v0650.py

@@ -327,12 +327,20 @@ def register(app: typer.Typer, console: Console) -> None:
     @app.command(name="irt-subset")
     def irt_subset_cmd(
         responses_path: str = typer.Argument(
-            ..., help="Path to JSONL with rows {item_id, correct(bool), score?}.",
+            ...,
+            help=(
+                "Path to JSONL with rows {item_id, correct(bool), "
+                "respondent_id? (required for 2pl/3pl)}."
+            ),
         ),
         size: str = typer.Option(
             "small", "--size", "-z",
             help="Subset profile: full / small / tiny.",
         ),
+        model: str = typer.Option(
+            "1pl", "--model", "-m",
+            help="IRT model: 1pl (Rasch) / 2pl (discrimination) / 3pl (+ guessing).",
+        ),
         output: Optional[str] = typer.Option(
             None, "--output", "-o",
             help="Where to write the rendered IrtSubsetPlan JSON.",
@@ -341,7 +349,9 @@ def register(app: typer.Typer, console: Console) -> None:
         """Pick a minimum-cost eval subset that preserves ranking power."""
         from soup_cli.utils.irt import (
             IRT_PROFILES,
+            SUPPORTED_IRT_MODELS,
             fit_difficulty,
+            fit_irt,
             load_response_rows,
             pick_irt_subset,
         )
@@ -352,6 +362,12 @@ def register(app: typer.Typer, console: Console) -> None:
                 f"(valid: {', '.join(sorted(IRT_PROFILES))})[/]"
             )
             raise typer.Exit(2)
+        if model not in SUPPORTED_IRT_MODELS:
+            console.print(
+                f"[red]Invalid --model: {escape(model)} "
+                f"(valid: {', '.join(SUPPORTED_IRT_MODELS)})[/]"
+            )
+            raise typer.Exit(2)
 
         try:
             rows = load_response_rows(responses_path)
@@ -360,13 +376,20 @@ def register(app: typer.Typer, console: Console) -> None:
             raise typer.Exit(2) from exc
 
         try:
-            difficulty = fit_difficulty(rows)
-            plan = pick_irt_subset(difficulty, size=size)
+            if model == "1pl":
+                # 1PL keeps the single-respondent correct-rate fit (back-compat;
+                # rows need only {item_id, correct}).
+                items = fit_difficulty(rows)
+            else:
+                # 2PL/3PL need a response matrix ({respondent_id, item_id, correct}).
+                items = fit_irt(rows, model=model)
+            plan = pick_irt_subset(items, size=size)
         except (TypeError, ValueError) as exc:
             console.print(f"[red]IRT fit failed:[/] {escape(str(exc))}")
             raise typer.Exit(2) from exc
 
         console.print(Panel(
+            f"Model: [bold]{escape(model)}[/]\n"
             f"Profile: [bold]{escape(plan.size)}[/]\n"
             f"Selected: {len(plan.item_ids)} / {plan.total_items}\n"
             f"Approx cost cut: {plan.cost_ratio:.1%}",