codeccat-train 0.1.0__py3-none-any.whl

codeccat_train/__init__.py

@@ -0,0 +1 @@
+"""Offline training and native-model export; never imported by CodecCat inference."""

codeccat_train/__main__.py

@@ -0,0 +1,242 @@
+"""Offline CodecCat trainer: safe dataset reading, group isolation, CCM1 export.
+
+No dependency on codeccat's public runtime or internal legacy R21 training.
+Users can install codeccat-train only in their training environment.
+"""
+from __future__ import annotations
+import argparse,codecs,hashlib,json,struct,sys
+from pathlib import Path
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.linear_model import SGDClassifier
+from sklearn.metrics import accuracy_score
+
+def read_manifest(manifest:Path,require_validation:bool=True):
+    root=manifest.resolve().parent
+    result=[];digests={};groups={}
+    for number,line in enumerate(manifest.read_text(encoding="utf-8").splitlines(),1):
+        if not line.strip():continue
+        try:record=json.loads(line)
+        except json.JSONDecodeError as ex:raise ValueError(f"line {number}: invalid JSON") from ex
+        for key in ("path","encoding","group","split"):
+            if not isinstance(record.get(key),str) or not record[key]:
+                raise ValueError(f"line {number}: missing {key}")
+        if record["split"] not in ("train","validation"):
+            raise ValueError(f"line {number}: split must be train or validation")
+        rel=Path(record["path"])
+        resolved=(root/rel).resolve()
+        if rel.is_absolute() or not resolved.is_relative_to(root):
+            raise ValueError(f"line {number}: path escapes manifest directory")
+        raw=resolved.read_bytes()
+        if not raw:raise ValueError(f"line {number}: empty input")
+        # Optional per-file attestations protect reproducible training against
+        # silent input changes after a manifest was generated.
+        for key,payload in (("sha256",raw),("prefix_sha256",raw[:4096])):
+            expected=record.get(key)
+            if expected is not None:
+                actual=hashlib.sha256(payload).hexdigest()
+                if not isinstance(expected,str) or expected!=actual:
+                    raise ValueError(f"line {number}: {key} mismatch")
+        try:codecs.lookup(record["encoding"])
+        except LookupError as ex:raise ValueError(f"line {number}: unknown codec") from ex
+        try:raw.decode(record["encoding"],"strict")
+        except UnicodeError as ex:raise ValueError(f"line {number}: label does not decode bytes") from ex
+        sha=hashlib.sha256(raw).hexdigest()
+        grp=record["group"];split=record["split"]
+        # Register and validate groups before deduplicating identical content.
+        # A duplicate can introduce a group that later appears in another split.
+        if grp in groups and groups[grp]!=split:
+            raise ValueError(f"line {number}: source-group leakage")
+        groups[grp]=split
+        if sha in digests:
+            old=digests[sha]
+            if old!=(record["encoding"],split):
+                raise ValueError(f"line {number}: duplicate bytes cross labels/splits")
+            continue
+        digests[sha]=(record["encoding"],split)
+        result.append((raw,record["encoding"],grp,split,sha))
+    train=[r for r in result if r[3]=="train"]
+    val=[r for r in result if r[3]=="validation"]
+    if not train or (require_validation and not val):
+        raise ValueError("both train and validation records required")
+    if len({r[1] for r in train})<2:raise ValueError("need >=2 training codec labels")
+    if any(r[1] not in {t[1] for t in train} for r in val):
+        raise ValueError("validation codec label absent from training")
+    return train,val
+
+def fingerprint_dataset(rows):
+    """Fingerprint accepted labeled bytes and assignments without storing texts."""
+    source_rows = [
+        {"sha256": row[4], "encoding": row[1], "group": row[2], "split": row[3]}
+        for row in rows
+    ]
+    return hashlib.sha256(json.dumps(
+        source_rows, sort_keys=True, separators=(",", ":"), ensure_ascii=False
+    ).encode("utf-8")).hexdigest()
+
+def historical_stable23_fingerprint(rows):
+    """Match the original TrainableDetector fingerprint on 4KB corpus prefixes.
+
+    This fingerprint is an audit value, not a trust or license certificate.
+    """
+    digest=hashlib.sha256()
+    for row in rows:
+        raw,encoding,group=row[:3]
+        prefix_hash=hashlib.sha256(raw[:4096]).hexdigest()
+        digest.update(prefix_hash.encode()+b"\0"+encoding.encode()+b"\0"+group.encode()+b"\0")
+    return digest.hexdigest()
+
+def write_string(parts,s):
+    payload=s.encode("utf-8")
+    if not 0<len(payload)<=256:raise ValueError("invalid model string length")
+    parts.extend((struct.pack("<H",len(payload)),payload))
+
+def export_ccm(path,vectorizer,classifier,precision="f64"):
+    if precision not in ("f64", "f32"):
+        raise ValueError("unsupported weight precision")
+    vocab=vectorizer.vocabulary_
+    width=len(vectorizer.idf_)
+    classes=[str(x) for x in classifier.classes_]
+    rows=1 if len(classes)==2 else len(classes)
+    if len(vocab)!=width or classifier.coef_.shape!=(rows,width):
+        raise ValueError("model tensor dimensions invalid")
+    if classifier.intercept_.shape!=(rows,):
+        raise ValueError("model bias dimensions invalid")
+    if width>200_000 or len(classes)>256:raise ValueError("model exceeds native limit")
+    low,high=vectorizer.ngram_range
+    flags=int(vectorizer.binary)|2*int(vectorizer.sublinear_tf)
+    schema=b"CCM2" if precision=="f32" else b"CCM1"
+    parts=[struct.pack("<4sIIBBBB",schema,width,len(classes),low,high,flags,0)]
+    for c in classes:write_string(parts,c)
+    for gram,index in sorted(vocab.items(),key=lambda item:item[1]):
+        parts.append(struct.pack("<I",index))
+        write_string(parts,gram)
+    for tensor in (vectorizer.idf_, classifier.intercept_):
+        parts.append(np.asarray(tensor,dtype="<f8").tobytes(order="C"))
+    weights_dtype="<f4" if precision=="f32" else "<f8"
+    parts.append(np.asarray(classifier.coef_,dtype=weights_dtype).tobytes(order="C"))
+    data=b"".join(parts)
+    data+=hashlib.sha256(data).digest()
+    path.parent.mkdir(parents=True,exist_ok=True)
+    path.write_bytes(data)
+    return hashlib.sha256(data).hexdigest(),len(data)
+
+def fit_l2(training_rows,seed:int):
+    """Stable23-equivalent L2 fitting on supplied rows; no official corpus implied."""
+    strings=[r[0][:4096].decode("latin1") for r in training_rows]
+    labels=[r[1] for r in training_rows]
+    vec=TfidfVectorizer(analyzer="char",ngram_range=(2,3),
+        min_df=2,max_features=45000,sublinear_tf=True)
+    x=vec.fit_transform(strings)
+    model=SGDClassifier(loss="modified_huber",alpha=.0001,
+        random_state=seed,max_iter=1500)
+    model.fit(x,labels)
+    return vec,model
+
+def fit(manifest:Path,output:Path,report:Path,seed:int,precision:str="f64",residual_focus:bool=False,train_only:bool=False):
+    train,validation=read_manifest(manifest,require_validation=not train_only)
+    if train_only and validation:
+        raise ValueError("--train-only requires a manifest containing train rows only")
+    original_train_n=len(train)
+    source_fingerprint=fingerprint_dataset(train + validation)
+    removed={}
+    if residual_focus:
+        from .residual import residual_subset
+        train,removed=residual_subset(train)
+    effective_fingerprint=fingerprint_dataset(train)
+    vec,model=fit_l2(train,seed)
+    accuracy=None
+    if validation:
+        xval=vec.transform([r[0][:4096].decode("latin1") for r in validation])
+        pred=model.predict(xval)
+        accuracy=float(accuracy_score([r[1] for r in validation],pred))
+    digest,size=export_ccm(output,vec,model,precision=precision)
+    result={
+        "schema":"codeccat-offline-train-v1",
+        "feature_profile":"stable23-equivalent",
+        "model_schema":"CCM2" if precision=="f32" else "CCM1",
+        "weight_precision":precision,
+        "seed":seed,
+        "training_n":len(train),"validation_n":len(validation),
+        "training_original_n":original_train_n,
+        "residual_focus_applied":residual_focus,
+        "training_removed_by_label":removed,
+        "training_groups":len({r[2] for r in train}),
+        "validation_groups":len({r[2] for r in validation}),
+        "classes":list(model.classes_),
+        "validation_codec_label_accuracy":accuracy,
+        "train_only":train_only,
+        "note":"Validation checks exact encoding LABEL, not full public Runtime text-level detection and not independent external generalization.",
+        "manifest_sha256":hashlib.sha256(manifest.read_bytes()).hexdigest(),
+        "dataset_content_fingerprint_sha256":source_fingerprint,
+        "effective_training_fingerprint_sha256":effective_fingerprint,
+        "historical_stable23_fingerprint":historical_stable23_fingerprint(train),
+        "dataset_fingerprint_scope":"SHA256 of ordered accepted bytes SHA256, encoding, group and split; no raw text",
+        "model_sha256":digest,
+        "model_bytes":size,
+        "training_tool_version":"0.1.0",
+        "python_version":sys.version.split()[0],
+        "numpy_version":np.__version__,
+        "sklearn_version":__import__("sklearn").__version__,
+        "training_config":{"min_df":2,"max_features":45000,"ngram_range":[2,3],
+                           "sublinear_tf":True,"loss":"modified_huber",
+                           "alpha":0.0001,"max_iter":1500,
+                           "byte_limit_per_document":4096}
+    }
+    report.parent.mkdir(parents=True,exist_ok=True)
+    report.write_text(json.dumps(result,indent=2,ensure_ascii=False)+"\n",encoding="utf-8")
+    return result
+
+def main():
+    import sys
+    args=sys.argv[1:]
+    if args and args[0]=="fit-l3":
+        from .l3 import save_l3
+        p=argparse.ArgumentParser(description="Fit optional Unicode trigram L3 from the unfiltered training corpus")
+        p.add_argument("--manifest",type=Path,required=True)
+        p.add_argument("--output",type=Path,required=True)
+        p.add_argument("--report",type=Path,required=True)
+        p.add_argument("--max-features",type=int,default=30000)
+        a=p.parse_args(args[1:])
+        train,validation=read_manifest(a.manifest,require_validation=False)
+        if validation:
+            raise ValueError("fit-l3 requires a train-only manifest; no validation rows may be used")
+        result=save_l3(train,a.output,max_features=a.max_features)
+        result["input_manifest_sha256"]=hashlib.sha256(a.manifest.read_bytes()).hexdigest()
+        result["input_content_fingerprint_sha256"]=fingerprint_dataset(train)
+        result["input_n"]=len(train)
+        a.report.parent.mkdir(parents=True,exist_ok=True)
+        a.report.write_text(json.dumps(result,sort_keys=True,indent=2)+"\n",encoding="utf-8")
+        print("CODECCAT_L3_TRAIN_RESULT",json.dumps(result,sort_keys=True),flush=True)
+        return
+    if args and args[0]=="evaluate":
+        from .evaluate import evaluate
+        p=argparse.ArgumentParser(description="Compare a custom model against the bundled Stable model on labeled bytes")
+        p.add_argument("--manifest",type=Path,required=True)
+        p.add_argument("--model",type=Path,required=True)
+        p.add_argument("--report",type=Path,required=True)
+        p.add_argument("--baseline-model",type=Path,default=None)
+        p.add_argument("--split",choices=("test","validation"),default="test")
+        a=p.parse_args(args[1:])
+        print("CODECCAT_MODEL_COMPARISON",
+              json.dumps(evaluate(a.manifest,a.model,a.report,a.split,a.baseline_model),sort_keys=True),
+              flush=True)
+        return
+    # Preserve the original CLI without subcommands for existing users.
+    if args and args[0]=="fit":
+        args=args[1:]
+    p=argparse.ArgumentParser(description="Offline-only CodecCat training; exports numeric CCM1/CCM2 for Native Runtime")
+    p.add_argument("--manifest",type=Path,required=True)
+    p.add_argument("--output",type=Path,required=True)
+    p.add_argument("--report",type=Path,required=True)
+    p.add_argument("--seed",type=int,default=20261015)
+    p.add_argument("--train-only",action="store_true",
+                   help="fit every labeled row; requires train-only manifest and reports no validation accuracy")
+    p.add_argument("--residual-focus",action="store_true",
+                   help="replay historical Stable23 residual sample filtering (not official release approval)")
+    p.add_argument("--precision",choices=("f64","f32"),default="f64",
+                   help="f64 exports CCM1 (default); f32 exports compact CCM2 weights")
+    a=p.parse_args(args)
+    print("CODECCAT_TRAIN_RESULT",json.dumps(fit(a.manifest,a.output,a.report,a.seed,precision=a.precision,residual_focus=a.residual_focus,train_only=a.train_only)),flush=True)
+
+if __name__=="__main__":main()

codeccat_train/evaluate.py

@@ -0,0 +1,123 @@
+"""Compare a user-trained numeric model with the published CodecCat default.
+
+Explicitly evaluates bytes using the public Runtime, never raw scikit-learn
+labels. No source bytes or decoded text are copied into the report.
+"""
+from __future__ import annotations
+import codecs
+import hashlib
+import json
+from collections import defaultdict
+from pathlib import Path
+
+
+def evaluate(manifest: Path, candidate: Path, report: Path,
+             split: str = "test", baseline: Path | None = None) -> dict:
+    # Optional dependency only for evaluation; offline fitting remains independent.
+    try:
+        from codeccat import CodecCat
+    except ImportError as ex:
+        raise RuntimeError(
+            "Evaluation needs the public Runtime: install 'codeccat' "
+            "in the evaluation environment"
+        ) from ex
+
+    if split not in ("test", "validation"):
+        raise ValueError("evaluation split must be 'test' or 'validation'")
+    base = manifest.resolve().parent
+    records = []
+    selected_hashes = set()
+    # Audit the full manifest, not just the requested evaluation split.
+    # Otherwise an example can occur in train and test while appearing unique
+    # in the filtered evaluation subset.
+    seen_content = {}
+    group_splits = {}
+    for index, line in enumerate(manifest.read_text(encoding="utf-8").splitlines(), 1):
+        if not line.strip():
+            continue
+        try:
+            r = json.loads(line)
+        except json.JSONDecodeError as ex:
+            raise ValueError(f"manifest line {index}: invalid JSON") from ex
+        for key in ("path", "encoding", "group", "split"):
+            if not isinstance(r.get(key), str) or not r[key]:
+                raise ValueError(f"line {index}: missing {key}")
+        record_split = r["split"]
+        if record_split not in ("train", "validation", "test"):
+            raise ValueError(f"line {index}: unsupported split")
+        rel = Path(r["path"])
+        path = (base / rel).resolve()
+        if rel.is_absolute() or not path.is_relative_to(base):
+            raise ValueError(f"line {index}: path escapes manifest directory")
+        raw = path.read_bytes()
+        if not raw:
+            raise ValueError(f"line {index}: empty input")
+        try:
+            canonical = codecs.lookup(r["encoding"]).name
+            truth = raw.decode(r["encoding"], "strict")
+        except (UnicodeError, LookupError) as ex:
+            raise ValueError(f"line {index}: invalid encoding or ground truth") from ex
+        group = r["group"]
+        previous_split = group_splits.setdefault(group, record_split)
+        if previous_split != record_split:
+            raise ValueError(f"line {index}: source-group leakage across splits")
+        digest = hashlib.sha256(raw).hexdigest()
+        previous = seen_content.setdefault(digest, (canonical, record_split))
+        if previous[1] != record_split:
+            raise ValueError(f"line {index}: duplicated bytes across splits")
+        if previous[0] != canonical:
+            raise ValueError(f"line {index}: conflicting labels for identical bytes")
+        if record_split == split:
+            if digest in selected_hashes:
+                raise ValueError(f"line {index}: duplicated evaluation content")
+            selected_hashes.add(digest)
+            records.append((raw, canonical, truth))
+    if not records:
+        raise ValueError(f"manifest has no '{split}' examples")
+
+    new=CodecCat.from_files(candidate)
+    old=CodecCat.from_files(baseline) if baseline else CodecCat()
+    both={"candidate":{"n":0,"accepted":0,"correct_text":0,"correct_label":0},
+          "baseline":{"n":0,"accepted":0,"correct_text":0,"correct_label":0}}
+    by_codec=defaultdict(lambda:{
+        "n":0,"candidate_correct_text":0,"baseline_correct_text":0,
+        "helps":0,"harms":0})
+    helps=harms=ties_correct=ties_wrong=0
+    for raw,canonical,truth in records:
+        a=new.detect(raw);b=old.detect(raw)
+        a_ok=a.text==truth;b_ok=b.text==truth
+        group=by_codec[canonical]
+        group["n"]+=1
+        group["candidate_correct_text"]+=int(a_ok)
+        group["baseline_correct_text"]+=int(b_ok)
+        group["helps"]+=int(a_ok and not b_ok)
+        group["harms"]+=int(b_ok and not a_ok)
+        for name,det,ok in (("candidate",a,a_ok),("baseline",b,b_ok)):
+            val=both[name]
+            val["n"]+=1
+            val["accepted"]+=int(det.accepted)
+            val["correct_text"]+=int(ok)
+            val["correct_label"]+=int(det.encoding is not None
+                                      and codecs.lookup(det.encoding).name==canonical)
+        helps+=int(a_ok and not b_ok)
+        harms+=int(b_ok and not a_ok)
+        ties_correct+=int(a_ok and b_ok)
+        ties_wrong+=int(not a_ok and not b_ok)
+
+    output={
+        "schema":"codeccat-model-comparison-v1",
+        "evaluation_split":split,
+        "warning":"Only a held-out source-disjoint test manifest is independent. Exact cross-split byte and group reuse is rejected; this tool cannot prove absence of near-duplicate or external training-source overlap.",
+        "manifest_sha256":hashlib.sha256(manifest.read_bytes()).hexdigest(),
+        "candidate_model_sha256":hashlib.sha256(candidate.read_bytes()).hexdigest(),
+        "candidate_fingerprint":new.model_fingerprint,
+        "baseline_model_sha256":hashlib.sha256(baseline.read_bytes()).hexdigest() if baseline else None,
+        "baseline_fingerprint":old.model_fingerprint,
+        "results":both,
+        "by_codec":dict(sorted(by_codec.items())),
+        "paired":{"helps":helps,"harms":harms,"both_correct":ties_correct,
+                  "both_wrong":ties_wrong},
+    }
+    report.parent.mkdir(parents=True,exist_ok=True)
+    report.write_text(json.dumps(output,indent=2,sort_keys=True)+"\n",encoding="utf-8")
+    return output

codeccat_train/l3.py

@@ -0,0 +1,80 @@
+"""Offline L3 unicode-trigram fitting for optional model research.
+
+Matches historical LanguageCoherence.fit behavior without importing CodecCat
+Runtime, legacy training code, or third-party corpus into this distribution.
+"""
+from __future__ import annotations
+
+import codecs
+import hashlib
+import json
+import re
+from collections import Counter
+from pathlib import Path
+
+SCHEMA = "vnext-l3-unicode-trigram-v1"
+_SPACES = re.compile(r"\s+")
+
+
+def _trigrams(text: str) -> Counter[str]:
+    text = _SPACES.sub(" ", text[:4096].casefold())
+    return Counter(text[i:i + 3] for i in range(max(0, len(text)-2)))
+
+
+def _decode(data: bytes, encoding: str) -> str | None:
+    try:
+        decoder = codecs.getincrementaldecoder(encoding)("strict")
+        return decoder.decode(data[:4096], final=len(data) < 4096)
+    except (UnicodeError, LookupError, ValueError):
+        return None
+
+
+def fit_l3(training_rows, max_features: int = 30000) -> dict:
+    if max_features < 100:
+        raise ValueError("max_features must be >= 100")
+    counts = Counter()
+    fingerprint = hashlib.sha256()
+    found = 0
+    for row in training_rows:
+        raw, codec, group = row[:3]
+        if not isinstance(raw, bytes) or not isinstance(codec, str) or not isinstance(group, str) or not codec or not group:
+            raise ValueError("expected (bytes, encoding, source_group)")
+        # Historical corpus() passed a truncated prefix to L3 training.
+        # Reproduce that stage when source files themselves exceed 4096 B.
+        raw = raw[:4096]
+        text = _decode(raw, codec)
+        if text is None:
+            continue
+        counts.update(_trigrams(text))
+        fingerprint.update(hashlib.sha256(raw).digest())
+        fingerprint.update(codec.encode() + b"\x00" + group.encode() + b"\x00")
+        found += 1
+    if not found or not counts:
+        raise ValueError("no decodable training text")
+    grams = dict(sorted(counts.most_common(max_features), key=lambda x: x[0]))
+    return {
+        "schema": SCHEMA,
+        "grams": grams,
+        "total": sum(grams.values()),
+        "fingerprint": fingerprint.hexdigest(),
+        "alpha": 0.5,
+    }
+
+
+def save_l3(training_rows, destination: Path, max_features: int = 30000) -> dict:
+    fitted = fit_l3(training_rows, max_features=max_features)
+    # Historical LanguageCoherence.save used json.dumps with these options
+    # and wrote no trailing newline.
+    payload = json.dumps(fitted, sort_keys=True, ensure_ascii=False)
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    destination.write_text(payload, encoding="utf-8")
+    return {
+        "schema": SCHEMA,
+        "model_sha256": hashlib.sha256(payload.encode("utf-8")).hexdigest(),
+        "model_bytes": len(payload.encode("utf-8")),
+        "training_fingerprint": fitted["fingerprint"],
+        "training_grams": len(fitted["grams"]),
+        "training_total_trigram_count": fitted["total"],
+        "max_features": max_features,
+        "optional_model_enabled_by_default": False,
+    }

codeccat_train/residual.py

@@ -0,0 +1,72 @@
+"""Stable23 residual-sample selection for offline replay.
+
+This is the *data-selection* stage used by the historical bundled-model
+recipe. It is separate from the fitting algorithm and does not download data.
+"""
+from __future__ import annotations
+from collections import Counter
+
+_BOMS = (
+    (b"\x00\x00\xfe\xff", "utf-32-be"),
+    (b"\xff\xfe\x00\x00", "utf-32-le"),
+    (b"\xef\xbb\xbf", "utf-8-sig"),
+    (b"\xfe\xff", "utf-16-be"),
+    (b"\xff\xfe", "utf-16-le"),
+)
+
+
+def stable_structural_guess(data: bytes, known: set[str]) -> str | None:
+    # Match the former Stable23 selection rule, not the full Runtime gate.
+    final = len(data) < 4096
+    if not final and data and any(
+        sig.startswith(data) and len(data) < len(sig) for sig, _ in _BOMS
+    ):
+        return None
+    for signature, encoding in _BOMS:
+        if data.startswith(signature):
+            family = "-".join(encoding.split("-")[:2])
+            preferred = (
+                family if family in ("utf-16", "utf-32") and family in known
+                else encoding
+            )
+            return preferred if preferred in known else None
+    if len(data) < 32:
+        return None
+    pairs = min(len(data), 4096) // 2
+    even = data[: pairs * 2 : 2].count(0)
+    odd = data[1 : pairs * 2 : 2].count(0)
+    if max(even, odd) >= max(12, int(pairs * 0.30)) and max(
+        even, odd
+    ) >= 4 * max(1, min(even, odd)):
+        predicted = "utf-16-be" if even > odd else "utf-16-le"
+        if predicted in known:
+            return predicted
+    return None
+
+
+def residual_subset(rows, min_anchors: int = 2, max_drop_fraction: float = 0.35):
+    """Return selected input rows and per-class drop counts, preserving order.
+
+    A row is any sequence starting with (bytes, encoding, group); remaining
+    metadata is preserved. Filtering applies to training samples only.
+    """
+    labels = {row[1] for row in rows}
+    per_label = Counter(row[1] for row in rows)
+    drop_limit = {
+        label: min(per_label[label] - min_anchors,
+                   int(per_label[label] * max_drop_fraction))
+        for label in per_label
+    }
+    covered = Counter()
+    dropped = Counter()
+    chosen = []
+    for row in rows:
+        data, label = row[:2]
+        if stable_structural_guess(data, labels) == label and covered[label] < drop_limit[label]:
+            covered[label] += 1
+            dropped[label] += 1
+        else:
+            chosen.append(row)
+    if {row[1] for row in chosen} != labels:
+        raise ValueError("residual selection removed a training codec class")
+    return chosen, dict(dropped)

codeccat_train-0.1.0.dist-info/METADATA

@@ -0,0 +1,128 @@
+Metadata-Version: 2.5
+Name: codeccat-train
+Version: 0.1.0
+Summary: Offline trainer and evaluator for CodecCat native encoding models
+Project-URL: Repository, https://github.com/Johnny-Kao/CodecCat
+Project-URL: Issues, https://github.com/Johnny-Kao/CodecCat/issues
+Project-URL: Documentation, https://github.com/Johnny-Kao/CodecCat/blob/main/native-prototype/training/docs/GETTING_STARTED.md
+Author: Johnny Kao
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Text Processing :: General
+Requires-Python: >=3.10
+Requires-Dist: numpy>=2.0
+Requires-Dist: scikit-learn<2,>=1.4
+Description-Content-Type: text/markdown
+
+# codeccat-train — independent offline training toolkit
+
+Independent Python distribution for training and exporting CodecCat native
+`.ccm` models. **Not a runtime dependency** of `codeccat`. The train
+environment requires NumPy and scikit-learn; production inference does not.
+
+**Release candidate: `codeccat-train 0.1.0` (independent package).** The core offline Trainer and end-to-end published-Runtime model-loading workflow are implemented and verified. This repository branch contains a stable release candidate; **PyPI availability is not implied until a separate verified publishing run succeeds**. Historical official L2/L3 reproduction establishes *reproducibility*, not a better model or permission to publish third-party training texts.
+
+The same Trainer is used for both workflows: private community-supplied datasets
+remain the user's responsibility; the project-maintained dataset is governed by
+a separate rights and default-model acceptance decision. No user dataset or
+generated model is automatically submitted to the CodecCat project.
+
+From a checkout (works before and after the first PyPI publication):
+
+```bash
+python -m pip install ./native-prototype/training
+python -m codeccat_train fit --manifest dataset/manifest.jsonl --output custom.ccm --report training-report.json
+```
+
+Dataset manifest format (one JSON object per line, paths relative to the
+manifest file):
+
+```json
+{"path":"samples/a.bin","encoding":"utf-8","split":"train","group":"document-a"}
+{"path":"samples/b.bin","encoding":"shift_jis","split":"validation","group":"document-b"}
+```
+
+The trainer checks strict decodability, rejects path traversal, conflicting
+duplicate contents and train/validation source-group overlap. Supply varied,
+lawfully obtained text samples with independently known encoding labels.
+The training report stores both `manifest_sha256` (the manifest file bytes) and
+`dataset_content_fingerprint_sha256` (a deterministic digest of accepted
+input-file hashes, codec labels, group IDs and split assignments). Unlike a
+manifest hash alone, the second value changes when an input file's content
+changes at the same path. This does **not** prove dataset licensing or
+independent holdout status, and does not expose raw training text.
+
+The generated report describes model provenance and validation; it is **not**
+a guarantee of accuracy on arbitrary real-world text.
+
+See the [CodecCat repository](https://github.com/Johnny-Kao/CodecCat)
+for the native runtime and [rights and provenance policy](https://github.com/Johnny-Kao/CodecCat/blob/main/RIGHTS_AND_PROVENANCE.md).
+
+## Verification and release boundary
+
+- **Implemented and CI-verified:** grouped source-manifest ingestion and isolation,
+  deterministic same-environment CCM1/CCM2 training, native Runtime loading,
+  source-content fingerprints, candidate-vs-Stable Unicode comparison,
+  optional L3 training, exact pinned official historical L2/L3 replay.
+- **Platform evidence:** training and Runtime interoperability checked on Linux
+  x64/ARM64 and macOS ARM64; custom-model Runtime portability checked on
+  macOS Intel and Windows x64.
+- **Stable-candidate release boundary:** the independent `0.1.0` source/Wheel and release CI must pass before tagging. It is not available via PyPI until its own Trusted Publisher is configured and a separate upload succeeds. Historical training provenance is documented with residual risk, not universally cleared.
+- **Not an automatic Runtime update:** installing or training with this tool
+  does not alter `codeccat` or the bundled Stable model. A candidate default
+  requires a separately approved evaluation and release.
+
+CI evidence: [Trainer E2E](https://github.com/Johnny-Kao/CodecCat/actions/runs/42240463787),
+[macOS ARM64 native trainer](https://github.com/Johnny-Kao/CodecCat/actions/runs/42237800042),
+and [model submission standards](../../docs/MODEL_SUBMISSIONS.md).
+
+## Evaluate a trained model
+
+Install the public `codeccat==0.1.0` Runtime only in the evaluation environment, then compare against the official bundled model:
+
+```bash
+python -m codeccat_train evaluate --model custom.ccm \
+  --manifest dataset/test-manifest.jsonl --split test --report comparison.json
+```
+
+This compares decoded Unicode text (primary) and exact codec labels (secondary), with per-codec results and paired improves/regressions. `--split validation` can be used for development, but **do not call it an independent holdout**. The command does not replace or update the official model.
+
+## Optional L3 and historical reproducibility
+
+For the optional L3 Unicode-trigram artifact, the **same offline Trainer**
+supports a train-only manifest. Unlike the L2 `--residual-focus` replay,
+L3 uses every eligible unfiltered input record:
+
+```bash
+codeccat-train fit-l3 --manifest dataset/train-only.jsonl \
+  --output language-l3.json --report l3-report.json
+```
+
+L3 stays disabled by default. The pinned historical L2 CCM2 and L3 JSON
+have both been reproduced byte-for-byte in GitHub Actions; see
+[official source and reproducibility status](../../datasets/official/README.md).
+The manual strict replay gate fails when required hashes, source
+fingerprints or numeric identity evidence differ. This is **not** rights
+clearance or authorization to replace a production model.
+
+See [quick start](docs/GETTING_STARTED.md). The legacy `python -m codeccat_train --manifest ...` training command remains supported.
+
+## Compact CCM2 export
+
+The same trainer can export `CCM2` float32 weights instead of the default `CCM1` float64 weights:
+
+```bash
+codeccat-train fit --manifest dataset/manifest.jsonl --output custom-f32.ccm \
+  --report train-f32.json --precision f32
+```
+
+`--precision f64` remains the default for compatibility. CCM2 stores IDF and bias as float64 while compressing the weight matrix to float32. A smaller model does not imply better accuracy; compare both models on independently held-out bytes before choosing one. This is a format option for custom models, **not** reproduction of the published official CCM2 model.

codeccat_train-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+codeccat_train/__init__.py,sha256=cC_9nJupVDVttZPpBDhWAp2K8OlqlpQPDfyY1dqgRn8,86
+codeccat_train/__main__.py,sha256=s6Bg2_6XFO6n6jW2Nw1vM9m6UuDaB6ZUyONZWu_bh2g,12676
+codeccat_train/evaluate.py,sha256=AhBUF7reBWKSGsMRtiBw8pWIZGXmgCmo2aB0kpZXzK8,5814
+codeccat_train/l3.py,sha256=46BXqOvcF6rBsCu7YMxhnbXWWTB4_QtyrK_6g4EY_Sk,3015
+codeccat_train/residual.py,sha256=vaShT0o4rhIvhHO_t9CF5oDIbsR90sPW6pQWSA_61ns,2601
+codeccat_train-0.1.0.dist-info/METADATA,sha256=SAqpYmdiU-a2JjVyPuIFTbfSthxb6id8QyU-MEMUFLk,7112
+codeccat_train-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+codeccat_train-0.1.0.dist-info/entry_points.txt,sha256=REQ1SgJWObuvHb_uvBN6xoB0UI639_W2xb_-zxHSdY4,64
+codeccat_train-0.1.0.dist-info/licenses/LICENSE,sha256=pFFx-WIl6v-GV7UQSH09uaQMNzjX4gPmjOGhtYi64ks,1067
+codeccat_train-0.1.0.dist-info/RECORD,,

codeccat_train-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

codeccat_train-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+codeccat-train = codeccat_train.__main__:main

codeccat_train-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Johnny Kao
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.