cleanllm 0.4.0__py3-none-any.whl

cleanllm/__init__.py

@@ -0,0 +1,41 @@
+"""cleanllm: streaming JSONL cleaner for LLM fine-tuning datasets."""
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+from .scan import scan_jsonl
+from .fix import fix_jsonl, FixRules
+from .shard import shard_jsonl
+from .manifest import make_manifest
+from .validate import validate_jsonl
+from .dedup import dedup_jsonl
+from .sample import sample_jsonl
+from .audit import audit_bundle
+from .stats import stats_jsonl
+from .hf import download_from_hub, detect_hf_schema
+
+
+def _resolve_version() -> str:
+    try:
+        return version("cleanllm")
+    except PackageNotFoundError:
+        return "0.0.0"
+
+
+__version__ = _resolve_version()
+
+__all__ = [
+    "scan_jsonl",
+    "fix_jsonl",
+    "FixRules",
+    "shard_jsonl",
+    "make_manifest",
+    "validate_jsonl",
+    "dedup_jsonl",
+    "sample_jsonl",
+    "audit_bundle",
+    "stats_jsonl",
+    "download_from_hub",
+    "detect_hf_schema",
+    "__version__",
+]

cleanllm/__main__.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from .cli import app
+
+
+def main() -> None:
+    app()
+
+
+if __name__ == "__main__":
+    main()

cleanllm/audit.py

@@ -0,0 +1,287 @@
+from __future__ import annotations
+
+import csv
+import os
+from dataclasses import dataclass
+from typing import Any
+
+from .sample import sample_jsonl
+from .util import parse_json, sha256_file
+
+
+@dataclass(frozen=True)
+class AuditPaths:
+    outdir: str
+    sample_path: str
+    index_path: str
+    summary_path: str
+    readme_path: str
+    manifest_path: str
+
+
+def audit_bundle(
+    inp: str,
+    outdir: str,
+    num_rows: int,
+    seed: int | None = None,
+    stratify: list[str] | None = None,
+    schema: str | None = None,
+    overwrite: bool = False,
+    show_progress: bool = False,
+    started_at: str | None = None,
+    ended_at: str | None = None,
+    duration_seconds: float | None = None,
+    version: str | None = None,
+) -> dict[str, Any]:
+    _prepare_outdir(outdir, overwrite=overwrite)
+    paths = _bundle_paths(outdir)
+
+    sample_result = sample_jsonl(
+        inp,
+        paths.sample_path,
+        num_rows=num_rows,
+        seed=seed,
+        stratify=stratify,
+        schema=schema,
+        show_progress=show_progress,
+    )
+
+    sampled_line_numbers = sample_result.get("sampled_line_numbers", [])
+    index_rows = _write_index_csv(paths.sample_path, paths.index_path, sampled_line_numbers)
+    _write_readme(
+        paths.readme_path,
+        inp=inp,
+        requested_rows=num_rows,
+        sampled_rows=sample_result["sampled_rows"],
+        seed=seed,
+        stratify=stratify,
+        schema=schema,
+        paths=paths,
+    )
+    # Summary excludes summary+manifest to avoid hash loops.
+    summary_files = _bundle_files(paths, include_summary=False, include_manifest=False)
+    summary = _build_summary(
+        inp=inp,
+        outdir=outdir,
+        num_rows=num_rows,
+        seed=seed,
+        stratify=stratify,
+        schema=schema,
+        sample_result=sample_result,
+        index_rows=index_rows,
+        bundle_files=summary_files,
+        started_at=started_at,
+        ended_at=ended_at,
+        duration_seconds=duration_seconds,
+        version=version,
+    )
+    _write_summary(paths.summary_path, summary)
+
+    # Manifest includes summary but excludes itself to avoid self-referential hashing.
+    _write_manifest(paths.manifest_path, paths)
+
+    return summary
+
+
+def _prepare_outdir(outdir: str, overwrite: bool) -> None:
+    if os.path.exists(outdir):
+        if not overwrite and os.listdir(outdir):
+            raise ValueError(f"Outdir exists and is not empty: {outdir}")
+    os.makedirs(outdir, exist_ok=True)
+
+
+def _bundle_paths(outdir: str) -> AuditPaths:
+    return AuditPaths(
+        outdir=outdir,
+        sample_path=os.path.join(outdir, "audit_sample.jsonl"),
+        index_path=os.path.join(outdir, "audit_index.csv"),
+        summary_path=os.path.join(outdir, "audit_summary.json"),
+        readme_path=os.path.join(outdir, "AUDIT_README.md"),
+        manifest_path=os.path.join(outdir, "manifest.json"),
+    )
+
+
+def _write_index_csv(sample_path: str, index_path: str, sampled_line_numbers: list[int]) -> int:
+    columns = [
+        "sample_idx",
+        "input_line_number",
+        "id",
+        "source",
+        "problem_id",
+        "difficulty_bucket",
+        "has_tests",
+        "num_messages",
+        "num_tests",
+        "review_status",
+        "review_label",
+        "notes",
+    ]
+    count = 0
+    with open(sample_path, "r", encoding="utf-8", errors="replace") as f_in, open(
+        index_path, "w", encoding="utf-8", newline=""
+    ) as f_out:
+        writer = csv.DictWriter(f_out, fieldnames=columns)
+        writer.writeheader()
+        for line in f_in:
+            if not line.strip():
+                continue
+            line_number = sampled_line_numbers[count] if count < len(sampled_line_numbers) else ""
+            count += 1
+            try:
+                obj = parse_json(line)
+            except Exception:
+                continue
+
+            messages = obj.get("messages")
+            tests = obj.get("tests")
+            writer.writerow(
+                {
+                    "sample_idx": count,
+                    "input_line_number": line_number,
+                    "id": _as_str(obj.get("id")),
+                    "source": _as_str(obj.get("source")),
+                    "problem_id": _as_str(obj.get("problem_id")),
+                    "difficulty_bucket": _as_str(obj.get("difficulty_bucket")),
+                    "has_tests": _bool_str(isinstance(tests, list) and len(tests) > 0),
+                    "num_messages": str(len(messages)) if isinstance(messages, list) else "",
+                    "num_tests": str(len(tests)) if isinstance(tests, list) else "",
+                    "review_status": "",
+                    "review_label": "",
+                    "notes": "",
+                }
+            )
+    return count
+
+
+def _build_summary(
+    inp: str,
+    outdir: str,
+    num_rows: int,
+    seed: int | None,
+    stratify: list[str] | None,
+    schema: str | None,
+    sample_result: dict[str, Any],
+    index_rows: int,
+    bundle_files: list[dict[str, Any]],
+    started_at: str | None,
+    ended_at: str | None,
+    duration_seconds: float | None,
+    version: str | None,
+) -> dict[str, Any]:
+    return {
+        "command": "audit",
+        "input_path": inp,
+        "outdir": outdir,
+        "requested_rows": num_rows,
+        "sampled_rows": sample_result["sampled_rows"],
+        "seed": seed,
+        "stratify": stratify or None,
+        "schema": schema,
+        "valid_rows_seen": sample_result["valid_rows_seen"],
+        "invalid_json_rows": sample_result["invalid_json_rows"],
+        "invalid_schema_rows": sample_result["invalid_schema_rows"],
+        "missing_strata_field_rows": sample_result["missing_strata_field_rows"],
+        "error_counts_by_reason": sample_result["error_counts_by_reason"],
+        "issue_examples": [
+            {"line": issue.line, "reason": issue.reason} for issue in sample_result["issues"][:5]
+        ],
+        "strata_counts_seen": sample_result["strata_counts_seen"],
+        "strata_counts_sampled": sample_result["strata_counts_sampled"],
+        "bundle_files": bundle_files,
+        "index_rows": index_rows,
+        "started_at": started_at,
+        "ended_at": ended_at,
+        "duration_seconds": duration_seconds,
+        "version": version,
+    }
+
+
+def _bundle_files(
+    paths: AuditPaths,
+    include_summary: bool,
+    include_manifest: bool,
+) -> list[dict[str, Any]]:
+    files = []
+    candidates = [
+        paths.sample_path,
+        paths.index_path,
+        paths.readme_path,
+    ]
+    if include_summary:
+        candidates.append(paths.summary_path)
+    if include_manifest:
+        candidates.append(paths.manifest_path)
+    for path in candidates:
+        if not os.path.exists(path):
+            continue
+        files.append(
+            {
+                "path": os.path.basename(path),
+                "size_bytes": os.path.getsize(path),
+                "sha256": sha256_file(path),
+            }
+        )
+    return files
+
+
+def _write_summary(path: str, summary: dict[str, Any]) -> None:
+    import orjson
+
+    with open(path, "wb") as f:
+        f.write(orjson.dumps(summary, option=orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS))
+        f.write(b"\n")
+
+
+def _write_readme(
+    path: str,
+    inp: str,
+    requested_rows: int,
+    sampled_rows: int,
+    seed: int | None,
+    stratify: list[str] | None,
+    schema: str | None,
+    paths: AuditPaths,
+) -> None:
+    lines = [
+        "# Audit Bundle",
+        "",
+        "## Configuration",
+        f"- input: {inp}",
+        f"- requested_rows: {requested_rows}",
+        f"- sampled_rows: {sampled_rows}",
+        f"- seed: {seed}",
+        f"- stratify: {stratify}",
+        f"- schema: {schema}",
+        "",
+        "## Bundle Contents",
+        f"- {os.path.basename(paths.sample_path)}: sampled JSONL rows",
+        f"- {os.path.basename(paths.index_path)}: review index (edit review fields)",
+        f"- {os.path.basename(paths.summary_path)}: machine-readable summary",
+        f"- {os.path.basename(paths.manifest_path)}: file hashes",
+        "",
+        "## Review Workflow",
+        "1. Open audit_index.csv",
+        "2. Fill in review_status / review_label / notes",
+        "3. Use id to cross-reference audit_sample.jsonl",
+    ]
+    with open(path, "w", encoding="utf-8") as f:
+        f.write("\n".join(lines) + "\n")
+
+
+def _write_manifest(path: str, paths: AuditPaths) -> None:
+    import orjson
+
+    entries = _bundle_files(paths, include_summary=True, include_manifest=False)
+    with open(path, "wb") as f:
+        f.write(orjson.dumps({"files": entries}, option=orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS))
+        f.write(b"\n")
+
+
+def _as_str(value: Any) -> str:
+    if value is None:
+        return ""
+    return str(value)
+
+
+def _bool_str(value: bool) -> str:
+    return "true" if value else "false"