codeccat 0.1.0__cp310-abi3-win_amd64.whl

codeccat/__init__.py

@@ -0,0 +1,45 @@
+"""CodecCat public API with lazily loaded opt-in legacy R21 compatibility.
+
+Default import/detect/decode never imports NumPy. Legacy symbols preserve the
+previous API when the user explicitly installs codeccat[legacy].
+"""
+from __future__ import annotations
+
+from .usable import CodecCat, detect, decode
+from .vnext_decoding import Detection
+
+__all__ = [
+    "CodecCat", "detect", "decode", "Detection",
+    "DetectionResult", "Detector", "LinearModel", "RuntimeBundle",
+    "RuntimeStack", "load_bundle", "save_bundle",
+]
+
+_LEGACY = {
+    "DetectionResult": "api",
+    "Detector": "api",
+    "LinearModel": "model",
+    "RuntimeBundle": "model",
+    "RuntimeStack": "model",
+    "load_bundle": "model_io",
+    "save_bundle": "model_io",
+}
+
+
+def __getattr__(name: str):
+    mod = _LEGACY.get(name)
+    if mod is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    try:
+        from importlib import import_module
+        module = import_module(f".{mod}", __name__)
+    except ModuleNotFoundError as exc:
+        if exc.name == "numpy":
+            raise ImportError(
+                f"{name} is a legacy CodecCat API; "
+                "install optional compatibility support with "
+                "'pip install codeccat[legacy]'"
+            ) from exc
+        raise
+    result = getattr(module, name)
+    globals()[name] = result
+    return result

codeccat/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

codeccat/api.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .model import RuntimeBundle
+from .runtime import Runtime
+
+
+@dataclass(frozen=True)
+class DetectionResult:
+    encoding: str | None
+    alternatives: tuple[str, ...]
+
+
+class Detector:
+    """CodecCat runtime bound to one immutable fitted model bundle."""
+
+    def __init__(self, bundle: RuntimeBundle):
+        self._runtime = Runtime(bundle)
+
+    def rank(self, data: bytes) -> tuple[str, ...]:
+        if not isinstance(data, (bytes, bytearray, memoryview)):
+            raise TypeError("CodecCat expects a bytes-like input")
+        return self._runtime.rank(bytes(data))
+
+    def detect(self, data: bytes) -> DetectionResult:
+        ranked = self.rank(data)
+        return DetectionResult(
+            encoding=ranked[0] if ranked else None,
+            alternatives=ranked,
+        )

codeccat/cli.py

@@ -0,0 +1,59 @@
+"""CodecCat CLI: detect or strictly decode a local byte stream."""
+from __future__ import annotations
+import argparse
+import json
+from pathlib import Path
+import sys
+
+MAX_BYTES = 16 * 1024 * 1024
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(description="Detect text encoding with CodecCat's bundled native model")
+    p.add_argument("file", nargs="?", default="-", help="input file or - for stdin")
+    p.add_argument("--decode", action="store_true", help="write strictly decoded text to stdout")
+    p.add_argument("--json", action="store_true", help="write structured detection metadata")
+    p.add_argument("--bom-policy", choices=("strip", "preserve"), default="strip")
+    args = p.parse_args(argv)
+    if args.json and args.decode:
+        p.error("--json and --decode are mutually exclusive")
+    try:
+        if args.file == "-":
+            raw = sys.stdin.buffer.read(MAX_BYTES + 1)
+        else:
+            with Path(args.file).open("rb") as stream:
+                raw = stream.read(MAX_BYTES + 1)
+        if len(raw) > MAX_BYTES:
+            p.error("input exceeds 16 MiB limit; never silently truncate a stream")
+        from .usable import detect
+        result = detect(raw, bom_policy=args.bom_policy)
+    except (OSError, RuntimeError, ValueError) as exc:
+        p.exit(2, f"codeccat: {exc}\n")
+    if args.decode:
+        if result.text is None:
+            print("codeccat: unable to strictly decode input", file=sys.stderr)
+            return 2
+        # Decoded data is emitted as UTF-8 bytes on all platforms.
+        # Windows terminal defaults (e.g., cp1252) must not corrupt text.
+        sys.stdout.buffer.write(result.text.encode("utf-8", "strict"))
+    elif args.json:
+        print(json.dumps({
+            "encoding": result.encoding,
+            "accepted": result.accepted,
+            "ambiguous": result.ambiguous,
+            "candidates": result.candidates,
+            "bom_present": result.bom_present,
+            "reason": result.reason,
+        }, ensure_ascii=False))
+    else:
+        if not result.accepted:
+            print("unknown")
+            return 2
+        print(result.encoding)
+        if result.ambiguous:
+            print("codeccat: encoding label is ambiguous", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

codeccat/features.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+import math
+
+import numpy as np
+
+S3_THRESHOLD = 0.02
+BASE_WIDTH = 518
+EXTRA_WIDTH = 368
+FULL_WIDTH = BASE_WIDTH + EXTRA_WIDTH
+
+_REDUCE_IDX = np.asarray([32, 127, 128], dtype=np.intp)
+
+
+def hmt768(data: bytes) -> bytes:
+    if len(data) <= 768:
+        return data
+    middle = len(data) // 2
+    return data[:256] + data[middle - 128 : middle + 128] + data[-256:]
+
+
+def hmt768_array(data: bytes) -> np.ndarray:
+    return np.frombuffer(hmt768(data), dtype=np.uint8)
+
+
+def strict_utf8(data: bytes) -> bool:
+    try:
+        data[:4096].decode("utf-8", "strict")
+        return True
+    except UnicodeDecodeError:
+        return False
+
+
+def has_utf8_bom(data: bytes) -> bool:
+    return data.startswith(b"\xef\xbb\xbf")
+
+
+def route(data: bytes) -> str:
+    sample = data[:4096]
+    if strict_utf8(sample):
+        return "U"
+    if b"\x00" in sample:
+        return "N"
+    array = np.frombuffer(sample, dtype=np.uint8)
+    high_ratio = float(np.mean(array >= 128)) if len(array) else 0.0
+    return "RL" if high_ratio <= S3_THRESHOLD else "RH"
+
+
+def replacement_rate(data: bytes) -> float:
+    if not data:
+        return 0.0
+    text = data.decode("utf-8", errors="replace")
+    return text.count("\ufffd") / max(1, len(text))
+
+
+def byte_signals(data: bytes) -> tuple[bool, bool, bool, float, float, int]:
+    sample = data[:4096]
+    array = np.frombuffer(sample, dtype=np.uint8)
+    strict = strict_utf8(data)
+    bom = has_utf8_bom(data)
+    if len(array):
+        high_count = int(np.count_nonzero(array >= 128))
+        nul_count = int(np.count_nonzero(array == 0))
+        high = high_count / len(array)
+        nul = nul_count / len(array)
+        ascii_only = high_count == 0
+    else:
+        high = 0.0
+        nul = 0.0
+        ascii_only = True
+    return bom, strict, ascii_only, high, nul, len(array)
+
+
+def baseline_from_array(array: np.ndarray) -> np.ndarray:
+    n = max(1, len(array))
+    counts = np.bincount(array, minlength=256)
+    out = np.empty(BASE_WIDTH, dtype=np.float32)
+    out[:256] = counts
+    out[:256] *= 1.0 / n
+
+    if len(array) >= 2:
+        bins = array[:-1] + array[1:]
+        bigram_counts = np.bincount(bins, minlength=256)
+        out[256:512] = bigram_counts
+        out[256:512] *= 1.0 / (len(array) - 1)
+    else:
+        out[256:512] = 0.0
+
+    if len(array):
+        inv = 1.0 / len(array)
+        aggregate = np.add.reduceat(counts, _REDUCE_IDX)
+        printable = float(aggregate[0] * inv)
+        high = float(aggregate[2] * inv)
+        nul = float(counts[0] * inv)
+        lf = float(counts[10] * inv)
+        cr = float(counts[13] * inv)
+    else:
+        high = nul = printable = lf = cr = 0.0
+
+    out[512:] = (
+        math.log2(n + 1) / 16.0,
+        high,
+        nul,
+        printable,
+        lf,
+        cr,
+    )
+    return out
+
+
+def full_b_extra_from_array(array: np.ndarray, route_name: str) -> np.ndarray:
+    out = np.zeros(EXTRA_WIDTH, dtype=np.float32)
+    if route_name not in ("U", "RH"):
+        return out
+
+    if len(array) >= 2:
+        left = array[:-1]
+        right = array[1:]
+        x = left.astype(np.int16, copy=False)
+        y = right.astype(np.int16, copy=False)
+        diff = ((y - x) & 255).astype(np.intp, copy=False)
+        out[:256] = np.bincount(diff, minlength=256).astype(np.float32) / (len(array) - 1)
+
+        xb = (np.bitwise_xor(left, right) >> 2).astype(np.intp, copy=False)
+        out[256:320] = np.bincount(xb, minlength=64).astype(np.float32) / (len(array) - 1)
+
+    p = 320
+    for chunk in (array[:256], array[256:512], array[512:768]):
+        high = chunk[chunk >= 128]
+        if len(high):
+            idx = ((high.astype(np.uint16) - 128) >> 3).astype(np.intp, copy=False)
+            out[p : p + 16] = np.bincount(idx, minlength=16).astype(np.float32) / len(high)
+        p += 16
+    return out
+
+
+def feature_vector(data: bytes) -> np.ndarray:
+    return baseline_from_array(hmt768_array(data))
+
+
+def full_b_vector(data: bytes, route_name: str | None = None) -> np.ndarray:
+    route_name = route_name or route(data)
+    array = hmt768_array(data)
+    baseline = baseline_from_array(array)
+    extra = full_b_extra_from_array(array, route_name)
+    return np.concatenate([baseline, extra])

codeccat/legacy_portable.py

@@ -0,0 +1,78 @@
+"""Optional legacy stable23/ordered13 numeric ZIP model adapter.
+
+The original format stores JSON metadata plus compressed NumPy arrays, never
+pickle. Loading it requires only the opt-in NumPy dependency. Inference still
+uses the Rust-native L2 scorer; sklearn is not imported.
+"""
+from __future__ import annotations
+
+import io
+import json
+import zipfile
+from pathlib import Path
+
+SCHEMA_23 = "vnext-l2-tfidf-ordered23-sgd-huber-v1"
+SCHEMA_13 = "vnext-l2-tfidf-ordered13-sgd-huber-v2"
+
+
+def load_legacy_zip(path: str | Path):
+    try:
+        import numpy as np
+    except ImportError as exc:
+        raise ImportError(
+            "Legacy CodecCat .zip models require optional NumPy support; "
+            "install 'codeccat[legacy]' or export your model to .ccm."
+        ) from exc
+    from ._codeccat_native import NativeL2
+    from .vnext_strong_l2 import TrainableDetector
+
+    with zipfile.ZipFile(path, "r") as archive:
+        names = archive.namelist()
+        if sorted(names) != ["manifest.json", "weights.npz"]:
+            raise ValueError("unexpected or duplicated model archive entries")
+        manifest_info = archive.getinfo("manifest.json")
+        tensor_info = archive.getinfo("weights.npz")
+        if (manifest_info.file_size > 8_000_000 or
+                tensor_info.file_size > 200_000_000):
+            raise ValueError("legacy archive sections exceed maximum size")
+        manifest = json.loads(archive.read("manifest.json"))
+        schema = manifest.get("schema")
+        expected_range = {
+            SCHEMA_23: [2, 3],
+            SCHEMA_13: [1, 3],
+        }.get(schema)
+        if expected_range is None:
+            raise ValueError("unsupported legacy model schema")
+        if manifest.get("params") != {
+            "ngram_range": expected_range,
+            "sublinear_tf": True,
+            "lowercase": True,
+            "norm": "l2",
+        }:
+            raise ValueError("unsupported legacy feature transform")
+
+        with np.load(io.BytesIO(archive.read("weights.npz")),
+                     allow_pickle=False) as tensors:
+            if set(tensors.files) != {"idf", "coef", "intercept"}:
+                raise ValueError("unexpected numeric tensor keys")
+            idf = tensors["idf"].copy()
+            coef = tensors["coef"].copy()
+            intercept = tensors["intercept"].copy()
+
+    vocab = manifest.get("vocabulary")
+    classes = manifest.get("classes")
+    fingerprint = manifest.get("fingerprint")
+    if (not isinstance(vocab, dict) or not vocab or
+            not isinstance(classes, list) or len(classes) < 2 or
+            not all(isinstance(c, str) for c in classes) or
+            not isinstance(fingerprint, str) or len(fingerprint) != 64 or
+            sorted(vocab.values()) != list(range(len(vocab))) or
+            idf.shape != (len(vocab),) or
+            coef.shape != (1 if len(classes) == 2 else len(classes), len(vocab)) or
+            intercept.shape != (coef.shape[0],) or
+            not np.isfinite(idf).all() or not np.isfinite(coef).all() or
+            not np.isfinite(intercept).all()):
+        raise ValueError("invalid legacy model metadata or weights")
+    native = NativeL2(vocab, idf.tolist(), coef.tolist(), intercept.tolist(),
+                      expected_range[0], expected_range[1], False, True)
+    return TrainableDetector(native, fingerprint, classes=classes)

codeccat/model.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Mapping
+
+import numpy as np
+
+
+@dataclass(frozen=True)
+class LinearModel:
+    classes: tuple[str, ...]
+    weight: np.ndarray
+    bias: np.ndarray
+
+
+@dataclass(frozen=True)
+class RuntimeStack:
+    route_models: Mapping[str, LinearModel]
+    calibrator: LinearModel | None
+    triad: LinearModel | None
+    utf8_sig: LinearModel | None
+    utf8_gb: LinearModel | None
+
+    def __post_init__(self) -> None:
+        missing = {"U", "N", "RL", "RH"} - set(self.route_models)
+        if missing:
+            raise ValueError(f"missing route models: {sorted(missing)}")
+
+
+@dataclass(frozen=True)
+class RuntimeBundle:
+    baseline: RuntimeStack
+    full_b: RuntimeStack | None
+    rl_specialist: LinearModel | None
+    gate_thresholds: Mapping[str, float]
+    classes: tuple[str, ...]
+    families: tuple[str, ...]
+    r12_threshold: float = 0.65
+
+    def __post_init__(self) -> None:
+        if self.full_b is not None:
+            for route in ("U", "RH"):
+                if route not in self.full_b.route_models:
+                    raise ValueError(f"full-B stack missing route: {route}")
+        for route in self.gate_thresholds:
+            if route not in ("U", "RH"):
+                raise ValueError(f"unsupported gate route: {route}")
+
+    @property
+    def route_models(self) -> Mapping[str, LinearModel]:
+        """Compatibility alias for the baseline route models."""
+        return self.baseline.route_models
+
+    @property
+    def calibrator(self) -> LinearModel | None:
+        return self.baseline.calibrator
+
+    @property
+    def triad(self) -> LinearModel | None:
+        return self.baseline.triad
+
+    @property
+    def utf8_sig(self) -> LinearModel | None:
+        return self.baseline.utf8_sig
+
+    @property
+    def utf8_gb(self) -> LinearModel | None:
+        return self.baseline.utf8_gb

codeccat/model_io.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import numpy as np
+
+from .model import LinearModel, RuntimeBundle, RuntimeStack
+
+
+def _put_model(arrays: dict[str, np.ndarray], meta: dict, prefix: str, model: LinearModel | None) -> None:
+    if model is None:
+        meta[prefix] = None
+        return
+    meta[prefix] = {"classes": list(model.classes)}
+    arrays[f"{prefix}.weight"] = np.asarray(model.weight, dtype=np.float64)
+    arrays[f"{prefix}.bias"] = np.asarray(model.bias, dtype=np.float64)
+
+
+def _get_model(data, meta: dict, prefix: str) -> LinearModel | None:
+    info = meta[prefix]
+    if info is None:
+        return None
+    return LinearModel(
+        classes=tuple(info["classes"]),
+        weight=np.asarray(data[f"{prefix}.weight"], dtype=np.float64),
+        bias=np.asarray(data[f"{prefix}.bias"], dtype=np.float64),
+    )
+
+
+def _put_stack(arrays: dict[str, np.ndarray], meta: dict, prefix: str, stack: RuntimeStack | None) -> None:
+    if stack is None:
+        meta[prefix] = None
+        return
+    meta[prefix] = {"routes": list(stack.route_models)}
+    for route, model in stack.route_models.items():
+        _put_model(arrays, meta, f"{prefix}.route.{route}", model)
+    _put_model(arrays, meta, f"{prefix}.calibrator", stack.calibrator)
+    _put_model(arrays, meta, f"{prefix}.triad", stack.triad)
+    _put_model(arrays, meta, f"{prefix}.utf8_sig", stack.utf8_sig)
+    _put_model(arrays, meta, f"{prefix}.utf8_gb", stack.utf8_gb)
+
+
+def _get_stack(data, meta: dict, prefix: str) -> RuntimeStack | None:
+    info = meta[prefix]
+    if info is None:
+        return None
+    route_models = {
+        route: _get_model(data, meta, f"{prefix}.route.{route}")
+        for route in info["routes"]
+    }
+    if any(model is None for model in route_models.values()):
+        raise ValueError(f"{prefix} route model cannot be null")
+    return RuntimeStack(
+        route_models=route_models,  # type: ignore[arg-type]
+        calibrator=_get_model(data, meta, f"{prefix}.calibrator"),
+        triad=_get_model(data, meta, f"{prefix}.triad"),
+        utf8_sig=_get_model(data, meta, f"{prefix}.utf8_sig"),
+        utf8_gb=_get_model(data, meta, f"{prefix}.utf8_gb"),
+    )
+
+
+def save_bundle(path: str | Path, bundle: RuntimeBundle) -> None:
+    arrays: dict[str, np.ndarray] = {}
+    meta = {
+        "schema": 2,
+        "classes": list(bundle.classes),
+        "families": list(bundle.families),
+        "gate_thresholds": {k: float(v) for k, v in bundle.gate_thresholds.items()},
+        "r12_threshold": float(bundle.r12_threshold),
+    }
+    _put_stack(arrays, meta, "baseline", bundle.baseline)
+    _put_stack(arrays, meta, "full_b", bundle.full_b)
+    _put_model(arrays, meta, "rl_specialist", bundle.rl_specialist)
+    arrays["__manifest__"] = np.asarray(json.dumps(meta, sort_keys=True))
+    np.savez_compressed(Path(path), **arrays)
+
+
+def _load_schema1(data, meta: dict) -> RuntimeBundle:
+    route_models = {
+        route: _get_model(data, meta, f"route.{route}")
+        for route in meta["routes"]
+    }
+    if any(model is None for model in route_models.values()):
+        raise ValueError("route model cannot be null")
+    baseline = RuntimeStack(
+        route_models=route_models,  # type: ignore[arg-type]
+        calibrator=_get_model(data, meta, "calibrator"),
+        triad=_get_model(data, meta, "triad"),
+        utf8_sig=_get_model(data, meta, "utf8_sig"),
+        utf8_gb=_get_model(data, meta, "utf8_gb"),
+    )
+    return RuntimeBundle(
+        baseline=baseline,
+        full_b=None,
+        rl_specialist=None,
+        gate_thresholds={},
+        classes=tuple(meta["classes"]),
+        families=tuple(meta["families"]),
+    )
+
+
+def load_bundle(path: str | Path) -> RuntimeBundle:
+    with np.load(Path(path), allow_pickle=False) as data:
+        meta = json.loads(str(data["__manifest__"]))
+        schema = meta.get("schema")
+        if schema == 1:
+            return _load_schema1(data, meta)
+        if schema != 2:
+            raise ValueError(f"unsupported CodecCat model schema: {schema}")
+
+        baseline = _get_stack(data, meta, "baseline")
+        if baseline is None:
+            raise ValueError("baseline stack cannot be null")
+        return RuntimeBundle(
+            baseline=baseline,
+            full_b=_get_stack(data, meta, "full_b"),
+            rl_specialist=_get_model(data, meta, "rl_specialist"),
+            gate_thresholds={k: float(v) for k, v in meta.get("gate_thresholds", {}).items()},
+            classes=tuple(meta["classes"]),
+            families=tuple(meta["families"]),
+            r12_threshold=float(meta.get("r12_threshold", 0.65)),
+        )

codeccat/models/default.ccm.meta.json

@@ -0,0 +1 @@
+{"model_sha256": "16073306b5027bf93acc2883f57fbda0cb0cf2d77b9a7bb62cf1b07fcbb9748b", "schema": "codeccat-native-model-provenance-v1", "training_fingerprint": "936eb0331a0d0ac447b0f6c6135bec7cd397962544d053cda3220358dacba85c"}