codeccat-train 0.1.0__tar.gz

codeccat_train-0.1.0/.gitignore

@@ -0,0 +1,24 @@
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.venv/
+.alpha-env/
+dist/
+build/
+*.egg-info/
+research-results/
+.env
+.env.*
+# Generated model weights must never be committed as source files
+src/codeccat/models/
+
+# Native build/test products (generated only in GitHub Actions)
+native-prototype/python/codeccat/models/
+native-prototype/model/
+native-prototype/dist/
+native-prototype/target/
+ci-fixtures/
+.root-clean/
+.native-clean/
+.clean/
+.legacy/

codeccat_train-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Johnny Kao
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

codeccat_train-0.1.0/PKG-INFO

@@ -0,0 +1,128 @@
+Metadata-Version: 2.5
+Name: codeccat-train
+Version: 0.1.0
+Summary: Offline trainer and evaluator for CodecCat native encoding models
+Project-URL: Repository, https://github.com/Johnny-Kao/CodecCat
+Project-URL: Issues, https://github.com/Johnny-Kao/CodecCat/issues
+Project-URL: Documentation, https://github.com/Johnny-Kao/CodecCat/blob/main/native-prototype/training/docs/GETTING_STARTED.md
+Author: Johnny Kao
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Text Processing :: General
+Requires-Python: >=3.10
+Requires-Dist: numpy>=2.0
+Requires-Dist: scikit-learn<2,>=1.4
+Description-Content-Type: text/markdown
+
+# codeccat-train — independent offline training toolkit
+
+Independent Python distribution for training and exporting CodecCat native
+`.ccm` models. **Not a runtime dependency** of `codeccat`. The train
+environment requires NumPy and scikit-learn; production inference does not.
+
+**Release candidate: `codeccat-train 0.1.0` (independent package).** The core offline Trainer and end-to-end published-Runtime model-loading workflow are implemented and verified. This repository branch contains a stable release candidate; **PyPI availability is not implied until a separate verified publishing run succeeds**. Historical official L2/L3 reproduction establishes *reproducibility*, not a better model or permission to publish third-party training texts.
+
+The same Trainer is used for both workflows: private community-supplied datasets
+remain the user's responsibility; the project-maintained dataset is governed by
+a separate rights and default-model acceptance decision. No user dataset or
+generated model is automatically submitted to the CodecCat project.
+
+From a checkout (works before and after the first PyPI publication):
+
+```bash
+python -m pip install ./native-prototype/training
+python -m codeccat_train fit --manifest dataset/manifest.jsonl --output custom.ccm --report training-report.json
+```
+
+Dataset manifest format (one JSON object per line, paths relative to the
+manifest file):
+
+```json
+{"path":"samples/a.bin","encoding":"utf-8","split":"train","group":"document-a"}
+{"path":"samples/b.bin","encoding":"shift_jis","split":"validation","group":"document-b"}
+```
+
+The trainer checks strict decodability, rejects path traversal, conflicting
+duplicate contents and train/validation source-group overlap. Supply varied,
+lawfully obtained text samples with independently known encoding labels.
+The training report stores both `manifest_sha256` (the manifest file bytes) and
+`dataset_content_fingerprint_sha256` (a deterministic digest of accepted
+input-file hashes, codec labels, group IDs and split assignments). Unlike a
+manifest hash alone, the second value changes when an input file's content
+changes at the same path. This does **not** prove dataset licensing or
+independent holdout status, and does not expose raw training text.
+
+The generated report describes model provenance and validation; it is **not**
+a guarantee of accuracy on arbitrary real-world text.
+
+See the [CodecCat repository](https://github.com/Johnny-Kao/CodecCat)
+for the native runtime and [rights and provenance policy](https://github.com/Johnny-Kao/CodecCat/blob/main/RIGHTS_AND_PROVENANCE.md).
+
+## Verification and release boundary
+
+- **Implemented and CI-verified:** grouped source-manifest ingestion and isolation,
+  deterministic same-environment CCM1/CCM2 training, native Runtime loading,
+  source-content fingerprints, candidate-vs-Stable Unicode comparison,
+  optional L3 training, exact pinned official historical L2/L3 replay.
+- **Platform evidence:** training and Runtime interoperability checked on Linux
+  x64/ARM64 and macOS ARM64; custom-model Runtime portability checked on
+  macOS Intel and Windows x64.
+- **Stable-candidate release boundary:** the independent `0.1.0` source/Wheel and release CI must pass before tagging. It is not available via PyPI until its own Trusted Publisher is configured and a separate upload succeeds. Historical training provenance is documented with residual risk, not universally cleared.
+- **Not an automatic Runtime update:** installing or training with this tool
+  does not alter `codeccat` or the bundled Stable model. A candidate default
+  requires a separately approved evaluation and release.
+
+CI evidence: [Trainer E2E](https://github.com/Johnny-Kao/CodecCat/actions/runs/42240463787),
+[macOS ARM64 native trainer](https://github.com/Johnny-Kao/CodecCat/actions/runs/42237800042),
+and [model submission standards](../../docs/MODEL_SUBMISSIONS.md).
+
+## Evaluate a trained model
+
+Install the public `codeccat==0.1.0` Runtime only in the evaluation environment, then compare against the official bundled model:
+
+```bash
+python -m codeccat_train evaluate --model custom.ccm \
+  --manifest dataset/test-manifest.jsonl --split test --report comparison.json
+```
+
+This compares decoded Unicode text (primary) and exact codec labels (secondary), with per-codec results and paired improves/regressions. `--split validation` can be used for development, but **do not call it an independent holdout**. The command does not replace or update the official model.
+
+## Optional L3 and historical reproducibility
+
+For the optional L3 Unicode-trigram artifact, the **same offline Trainer**
+supports a train-only manifest. Unlike the L2 `--residual-focus` replay,
+L3 uses every eligible unfiltered input record:
+
+```bash
+codeccat-train fit-l3 --manifest dataset/train-only.jsonl \
+  --output language-l3.json --report l3-report.json
+```
+
+L3 stays disabled by default. The pinned historical L2 CCM2 and L3 JSON
+have both been reproduced byte-for-byte in GitHub Actions; see
+[official source and reproducibility status](../../datasets/official/README.md).
+The manual strict replay gate fails when required hashes, source
+fingerprints or numeric identity evidence differ. This is **not** rights
+clearance or authorization to replace a production model.
+
+See [quick start](docs/GETTING_STARTED.md). The legacy `python -m codeccat_train --manifest ...` training command remains supported.
+
+## Compact CCM2 export
+
+The same trainer can export `CCM2` float32 weights instead of the default `CCM1` float64 weights:
+
+```bash
+codeccat-train fit --manifest dataset/manifest.jsonl --output custom-f32.ccm \
+  --report train-f32.json --precision f32
+```
+
+`--precision f64` remains the default for compatibility. CCM2 stores IDF and bias as float64 while compressing the weight matrix to float32. A smaller model does not imply better accuracy; compare both models on independently held-out bytes before choosing one. This is a format option for custom models, **not** reproduction of the published official CCM2 model.

codeccat_train-0.1.0/README.md

@@ -0,0 +1,103 @@
+# codeccat-train — independent offline training toolkit
+
+Independent Python distribution for training and exporting CodecCat native
+`.ccm` models. **Not a runtime dependency** of `codeccat`. The train
+environment requires NumPy and scikit-learn; production inference does not.
+
+**Release candidate: `codeccat-train 0.1.0` (independent package).** The core offline Trainer and end-to-end published-Runtime model-loading workflow are implemented and verified. This repository branch contains a stable release candidate; **PyPI availability is not implied until a separate verified publishing run succeeds**. Historical official L2/L3 reproduction establishes *reproducibility*, not a better model or permission to publish third-party training texts.
+
+The same Trainer is used for both workflows: private community-supplied datasets
+remain the user's responsibility; the project-maintained dataset is governed by
+a separate rights and default-model acceptance decision. No user dataset or
+generated model is automatically submitted to the CodecCat project.
+
+From a checkout (works before and after the first PyPI publication):
+
+```bash
+python -m pip install ./native-prototype/training
+python -m codeccat_train fit --manifest dataset/manifest.jsonl --output custom.ccm --report training-report.json
+```
+
+Dataset manifest format (one JSON object per line, paths relative to the
+manifest file):
+
+```json
+{"path":"samples/a.bin","encoding":"utf-8","split":"train","group":"document-a"}
+{"path":"samples/b.bin","encoding":"shift_jis","split":"validation","group":"document-b"}
+```
+
+The trainer checks strict decodability, rejects path traversal, conflicting
+duplicate contents and train/validation source-group overlap. Supply varied,
+lawfully obtained text samples with independently known encoding labels.
+The training report stores both `manifest_sha256` (the manifest file bytes) and
+`dataset_content_fingerprint_sha256` (a deterministic digest of accepted
+input-file hashes, codec labels, group IDs and split assignments). Unlike a
+manifest hash alone, the second value changes when an input file's content
+changes at the same path. This does **not** prove dataset licensing or
+independent holdout status, and does not expose raw training text.
+
+The generated report describes model provenance and validation; it is **not**
+a guarantee of accuracy on arbitrary real-world text.
+
+See the [CodecCat repository](https://github.com/Johnny-Kao/CodecCat)
+for the native runtime and [rights and provenance policy](https://github.com/Johnny-Kao/CodecCat/blob/main/RIGHTS_AND_PROVENANCE.md).
+
+## Verification and release boundary
+
+- **Implemented and CI-verified:** grouped source-manifest ingestion and isolation,
+  deterministic same-environment CCM1/CCM2 training, native Runtime loading,
+  source-content fingerprints, candidate-vs-Stable Unicode comparison,
+  optional L3 training, exact pinned official historical L2/L3 replay.
+- **Platform evidence:** training and Runtime interoperability checked on Linux
+  x64/ARM64 and macOS ARM64; custom-model Runtime portability checked on
+  macOS Intel and Windows x64.
+- **Stable-candidate release boundary:** the independent `0.1.0` source/Wheel and release CI must pass before tagging. It is not available via PyPI until its own Trusted Publisher is configured and a separate upload succeeds. Historical training provenance is documented with residual risk, not universally cleared.
+- **Not an automatic Runtime update:** installing or training with this tool
+  does not alter `codeccat` or the bundled Stable model. A candidate default
+  requires a separately approved evaluation and release.
+
+CI evidence: [Trainer E2E](https://github.com/Johnny-Kao/CodecCat/actions/runs/42240463787),
+[macOS ARM64 native trainer](https://github.com/Johnny-Kao/CodecCat/actions/runs/42237800042),
+and [model submission standards](../../docs/MODEL_SUBMISSIONS.md).
+
+## Evaluate a trained model
+
+Install the public `codeccat==0.1.0` Runtime only in the evaluation environment, then compare against the official bundled model:
+
+```bash
+python -m codeccat_train evaluate --model custom.ccm \
+  --manifest dataset/test-manifest.jsonl --split test --report comparison.json
+```
+
+This compares decoded Unicode text (primary) and exact codec labels (secondary), with per-codec results and paired improves/regressions. `--split validation` can be used for development, but **do not call it an independent holdout**. The command does not replace or update the official model.
+
+## Optional L3 and historical reproducibility
+
+For the optional L3 Unicode-trigram artifact, the **same offline Trainer**
+supports a train-only manifest. Unlike the L2 `--residual-focus` replay,
+L3 uses every eligible unfiltered input record:
+
+```bash
+codeccat-train fit-l3 --manifest dataset/train-only.jsonl \
+  --output language-l3.json --report l3-report.json
+```
+
+L3 stays disabled by default. The pinned historical L2 CCM2 and L3 JSON
+have both been reproduced byte-for-byte in GitHub Actions; see
+[official source and reproducibility status](../../datasets/official/README.md).
+The manual strict replay gate fails when required hashes, source
+fingerprints or numeric identity evidence differ. This is **not** rights
+clearance or authorization to replace a production model.
+
+See [quick start](docs/GETTING_STARTED.md). The legacy `python -m codeccat_train --manifest ...` training command remains supported.
+
+## Compact CCM2 export
+
+The same trainer can export `CCM2` float32 weights instead of the default `CCM1` float64 weights:
+
+```bash
+codeccat-train fit --manifest dataset/manifest.jsonl --output custom-f32.ccm \
+  --report train-f32.json --precision f32
+```
+
+`--precision f64` remains the default for compatibility. CCM2 stores IDF and bias as float64 while compressing the weight matrix to float32. A smaller model does not imply better accuracy; compare both models on independently held-out bytes before choosing one. This is a format option for custom models, **not** reproduction of the published official CCM2 model.

codeccat_train-0.1.0/docs/GETTING_STARTED.md

@@ -0,0 +1,121 @@
+# Offline model training with codeccat-train
+
+CodecCat offers **one** offline training tool for anyone's dataset. The CLI and model format are shared; official and user-owned datasets differ in provenance and what is allowed to ship as an official release.
+
+**Release candidate (2026-11-07):** Offline training and candidate evaluation are E2E-verified against the published Runtime. The source tree is preparing independent `codeccat-train==0.1.0` Stable distribution, which must pass release CI and PyPI Trusted Publishing before it becomes publicly installable from PyPI. Exact historical official L2 CCM2 and optional L3 replay have passed under pinned sources and dependencies; this is not data-rights clearance, stronger real-world accuracy, or authorization to promote a new default model. The separately published Runtime remains `codeccat==0.1.0`.
+
+## Quick local example
+
+Use Python 3.12 and keep the trainer out of production environments:
+
+```bash
+python -m venv .trainer-env
+.trainer-env/bin/python -m pip install ./native-prototype/training
+.trainer-env/bin/python native-prototype/training/make_smoke_data.py
+.trainer-env/bin/python -m codeccat_train \
+  --manifest native-prototype/training/smoke-data/manifest.jsonl \
+  --output custom.ccm \
+  --report training-report.json
+```
+
+The synthetic smoke dataset consists of UTF-8 and Windows-1252 bytes. Its purpose is checking the **pipeline**, not measuring real-world encoding accuracy. Do not use its reported validation score to claim production quality.
+
+A separate environment can install and use the released Runtime:
+
+```bash
+python -m venv .runtime-env
+.runtime-env/bin/python -m pip install --only-binary=:all: codeccat==0.1.0
+.runtime-env/bin/python - <<'PY'
+from codeccat import CodecCat
+model = CodecCat.from_files("custom.ccm")
+result = model.detect("日本語".encode("utf-8"))
+print(result.encoding, result.text, result.ambiguous)
+PY
+```
+
+The default custom-model output is `CCM1`; use `--precision f32` for official-compatible `CCM2`. The pinned historical recipe has matched the official CCM2 bytes, but arbitrary custom datasets or versions do not guarantee that result.
+
+## Manifest format
+
+Each line of `manifest.jsonl` is a JSON object:
+
+```jsonl
+{"path":"samples/document-001.bin","encoding":"utf-8","split":"train","group":"document-001"}
+{"path":"samples/document-002.bin","encoding":"cp1252","split":"validation","group":"document-002"}
+```
+
+Paths are relative to the manifest directory. `group` identifies a document/source that must not cross the training/validation boundary. The loader rejects escaping paths, duplicate bytes with conflicting labels/splits, unknown codecs and non-decodable labeled bytes. **This is only a first barrier:** near-duplicates, source overlap and corpus licenses remain the user's responsibility.
+
+Use bytes with independently known encodings and valid redistribution/training rights. Don't upload raw confidential files or training data to GitHub Actions.
+
+## Acceptance and next steps
+
+- **P0:** [Trainer E2E CI](../../../.github/workflows/trainer-e2e.yml) installs Trainer separately, trains twice and checks same-env reproducibility, then loads the resulting model with an independent **PyPI `codeccat==0.1.0`** Runtime environment; it also tests some unsafe manifest cases. No model is promoted by this check.
+- **P1:** Pinned historical input manifest, exact L2 residual training, and optional unfiltered L3 replay are implemented and have matched frozen reference artifacts. Third-party corpus rights review and independent source-overlap assurance are not complete.
+- **P2:** Candidate-vs-Stable comparison and model submission checks exist; exact historical reproduction is guarded separately from any future model-quality or release approval. No independent candidate release is implied.
+
+See [RIGHTS_AND_PROVENANCE.md](../../../RIGHTS_AND_PROVENANCE.md), [ROADMAP.md](../../../ROADMAP.md), and [CONTRIBUTING.md](../../../CONTRIBUTING.md).
+
+## Compare your model against official Stable
+
+Install the public Runtime in the **evaluation environment** (not required for the training-only environment), then run:
+
+```bash
+codeccat-train evaluate \
+  --model custom.ccm \
+  --manifest my-data/test-manifest.jsonl \
+  --split test \
+  --report comparison.json
+```
+
+The `evaluation` command compares candidate and bundled Stable on identical bytes, reporting **correct decoded Unicode text**, correct codec labels, per-codec accuracy, and how many examples improve or regress. No sample text or raw bytes appear in the output. Optionally use `--baseline-model /path/to/another.ccm`.
+
+The evaluator scans all manifest splits, rejecting identical bytes and source groups reused across train, validation and test. This prevents exact within-manifest leakage, but it cannot identify near-duplicates or overlap with a separate training manifest. See the [official dataset provenance status](../../../datasets/official/README.md) before claiming official-model reproducibility.
+
+The evaluation manifest uses the same JSONL keys with `"split":"test"` (or explicitly `--split validation` for development tests). Never present validation or previously used training sources as blind independent test data. Results are descriptive, not an automatic official-model upgrade.
+
+## Compact numeric model output
+
+By default, `codeccat-train fit` writes `CCM1` (float64 weights). Supply `--precision f32` to write a `CCM2` model with float32 weights, compatible with the published Runtime:
+
+```bash
+codeccat-train fit --manifest my-data/manifest.jsonl --output custom-f32.ccm \
+  --report training-f32.json --precision f32
+```
+
+This changes only stored weight precision and may introduce minor numerical differences. Reproducing the official model additionally requires the pinned dataset manifest, `--train-only`, `--residual-focus` and verified environment, as shown by the historical research workflow.
+
+## Replaying historical residual-focused training (research only)
+
+For investigations that need to match the old Stable23 data-selection
+method, `--residual-focus` applies the historical sample filter before the
+**same** L2 fitting core. `--train-only` accepts a manifest with only
+`"split":"train"` records, which matches the old release-building choice
+to fit on every eligible example:
+
+```bash
+codeccat-train fit --manifest local/training.jsonl --train-only \
+  --residual-focus --precision f32 --output candidate.ccm \
+  --report candidate-report.json
+```
+
+In train-only mode, `validation_codec_label_accuracy` is `null` and no
+independent accuracy claim is made. The report records original training
+size, selected training size, removed counts per codec and dataset
+fingerprints before and after filtering. The default community path still
+**requires** validation records and does not filter samples.
+
+The full pinned-source replay has reproduced the existing L2 model exactly. The separate `fit-l3` command handles the optional, unfiltered L3 model and also matched the frozen reference. See [official reconstruction status](../../../datasets/official/README.md).
+
+
+## Training optional L3 (off by default)
+
+`fit-l3` uses the full, unfiltered training rows (not the residual L2 subset), preserving the historical Unicode-trigram statistics:
+
+```bash
+codeccat-train fit-l3 --manifest /path/to/train-only.jsonl \
+  --output /path/to/language-l3.json \
+  --report /path/to/l3-report.json
+```
+
+The [manual L2 + L3 reproduction gate](../../../.github/workflows/official-stable23-replay.yml) compares both generated artifacts against pinned released reference SHA256 values and fails if a required identity check differs. L3 stays disabled by default. Reproduction does **not** establish rights clearance or production accuracy improvements.

codeccat_train-0.1.0/make_smoke_data.py

@@ -0,0 +1,27 @@
+"""Generate grouped, manifest-driven training smoke fixtures, with no downloads.
+
+Training data here checks pipeline roundtrip only; NOT accuracy or generalization.
+"""
+from pathlib import Path
+import json
+
+def main():
+    root=Path("native-prototype/training/smoke-data")
+    root.mkdir(parents=True,exist_ok=True)
+    records=[]
+    for split,indices in (("train",range(24)),("validation",range(24,30))):
+        for i in indices:
+            for codec,text in (
+                ("utf-8",f"日本語の文字列とUTF8データ {i}. 日本語の文字列とUTF8データ。"),
+                ("cp1252",f"café crème façade déjà vu numéro {i}. café crème."),
+            ):
+                filename=f"{split}-{codec}-{i}.dat"
+                (root/filename).write_bytes(text.encode(codec))
+                records.append({"path":filename,"encoding":codec,
+                    "group":f"{split}-{codec}-{i}","split":split})
+    (root/"manifest.jsonl").write_text(
+        "".join(json.dumps(r,ensure_ascii=False)+"\n" for r in records),
+        encoding="utf-8")
+    print("TRAINER_SMOKE_FIXTURES_READY",len(records),flush=True)
+
+if __name__=="__main__":main()

codeccat_train-0.1.0/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[project]
+name = "codeccat-train"
+version = "0.1.0"
+description = "Offline trainer and evaluator for CodecCat native encoding models"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Johnny Kao" }]
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Programming Language :: Python :: 3",
+  "Operating System :: OS Independent",
+  "Intended Audience :: Developers",
+  "Topic :: Text Processing :: General",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+]
+dependencies = ["numpy>=2.0", "scikit-learn>=1.4,<2"]
+
+[project.urls]
+Repository = "https://github.com/Johnny-Kao/CodecCat"
+Issues = "https://github.com/Johnny-Kao/CodecCat/issues"
+Documentation = "https://github.com/Johnny-Kao/CodecCat/blob/main/native-prototype/training/docs/GETTING_STARTED.md"
+
+[project.scripts]
+codeccat-train = "codeccat_train.__main__:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/codeccat_train"]

codeccat_train-0.1.0/src/codeccat_train/__init__.py

@@ -0,0 +1 @@
+"""Offline training and native-model export; never imported by CodecCat inference."""