phys-ai-dataprofiler 0.3.1__py3-none-any.whl

phys_ai_dataprofiler/__init__.py

@@ -0,0 +1,13 @@
+"""phys_ai_dataprofiler: zero-copy profiling of physical-AI dataset files (MCAP, ROS bag, MATLAB, Parquet).
+
+Copyright (c) 2026 Anand Krishniyer. MIT License."""
+
+__version__ = "0.3.1"
+
+from phys_ai_dataprofiler.inspectors.mcap import McapFileInspector, McapInspectionResult  # noqa: F401
+from phys_ai_dataprofiler.inspectors.mat import MatFileInspector, MatInspectionResult  # noqa: F401
+from phys_ai_dataprofiler.inspectors.rich import RichInspector, RichProfile  # noqa: F401
+from phys_ai_dataprofiler.remote.range_file import HfRangeFile, LocalRangeFile, RangeFile  # noqa: F401
+from phys_ai_dataprofiler.remote.inspector import inspect_mcap_remote, inspect_mat_remote, sniff_format  # noqa: F401
+from phys_ai_dataprofiler.connectors.huggingface import HuggingFaceConnector  # noqa: F401
+from phys_ai_dataprofiler.remote.parquet import inspect_parquet, ParquetInspectionResult  # noqa: F401

phys_ai_dataprofiler/__main__.py

@@ -0,0 +1,5 @@
+"""`python -m phys_ai_dataprofiler` runs the Hugging Face connector CLI."""
+import sys
+from phys_ai_dataprofiler.connectors.huggingface import main
+
+sys.exit(main())

phys_ai_dataprofiler/cli.py

@@ -0,0 +1,214 @@
+"""
+Subcommands next to the default `phys-ai-dataprofiler org/dataset` profile:
+
+    phys-ai-dataprofiler check org/dataset -r req.yaml        profile + verdict; exit 0 pass, 1 fail, 4 unknown
+    phys-ai-dataprofiler check --report run.json -r req.yaml  verdict from an existing report, no network
+    phys-ai-dataprofiler compare a.json b.json -r req.yaml    rank saved reports against the same requirements
+    phys-ai-dataprofiler compare org/a org/b -r req.yaml      profile each repo, then rank
+    phys-ai-dataprofiler requirements list                    built-in presets
+    phys-ai-dataprofiler requirements show teleop-imitation   print a preset (start your own from it)
+    phys-ai-dataprofiler requirements validate req.yaml       check a file without profiling anything
+
+Exit codes: 0 pass (or success), 1 fail, 2 usage or requirements error,
+3 nothing to profile, 4 unknown (the profile could not settle every required check).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import datetime
+from pathlib import Path
+
+from phys_ai_dataprofiler.compare import compare_reports, compare_to_dict, print_compare, render_compare_markdown
+from phys_ai_dataprofiler.evaluate import evaluate, print_verdict, render_verdict_markdown, verdict_to_dict
+from phys_ai_dataprofiler.requirements import (
+    RequirementsError,
+    list_presets,
+    load_requirements,
+    preset_text,
+)
+
+PROG = "phys-ai-dataprofiler"
+
+
+def _log(quiet: bool):
+    return (lambda m: None) if quiet else (lambda m: print(m, file=sys.stderr))
+
+
+def _load_req_or_exit(spec: str):
+    try:
+        return load_requirements(spec)
+    except RequirementsError as exc:
+        print(str(exc), file=sys.stderr)
+        raise SystemExit(2)
+
+
+def _read_report(path: str) -> dict:
+    try:
+        rep = json.loads(Path(path).read_text())
+    except (OSError, json.JSONDecodeError) as exc:
+        print(f"Error: cannot read report {path}: {exc}", file=sys.stderr)
+        raise SystemExit(2)
+    if not isinstance(rep, dict) or "files" not in rep:
+        print(f"Error: {path} is not a profiler run report (no 'files')", file=sys.stderr)
+        raise SystemExit(2)
+    return rep
+
+
+# --------------------------------------------------------------------------- #
+# check
+# --------------------------------------------------------------------------- #
+
+def cmd_check(argv: list[str]) -> int:
+    from phys_ai_dataprofiler.connectors.huggingface import build_parser, profile_main
+
+    if "--report" in argv or any(a.startswith("--report=") for a in argv):
+        p = argparse.ArgumentParser(prog=f"{PROG} check",
+                                    description="Verdict for an existing run report (no network).")
+        p.add_argument("--report", required=True, help="JSON run report written by an earlier profile")
+        p.add_argument("--requirements", "-r", required=True, metavar="FILE|preset:NAME")
+        p.add_argument("--output", "-o", help="Write the verdict here (.md or .json; default: print only)")
+        p.add_argument("--quiet", action="store_true")
+        a = p.parse_args(argv)
+        req = _load_req_or_exit(a.requirements)
+        v = evaluate(_read_report(a.report), req)
+        print_verdict(v, _log(a.quiet))
+        if a.output:
+            out = Path(a.output)
+            out.write_text(json.dumps(verdict_to_dict(v), indent=2) if out.suffix == ".json"
+                           else render_verdict_markdown(v, level=1))
+            _log(a.quiet)(f"Verdict: {out}")
+        return v.exit_code
+
+    p = build_parser(prog=f"{PROG} check")
+    p.description = ("Profile a repo and check it against requirements. Exit code 0 pass, 1 fail, 4 unknown. "
+                     "With --report FILE, check an existing report instead (no network).")
+    args = p.parse_args(argv)
+    if not args.requirements:
+        p.error("check needs --requirements FILE or --requirements preset:NAME")
+    return profile_main(args)
+
+
+# --------------------------------------------------------------------------- #
+# compare
+# --------------------------------------------------------------------------- #
+
+def cmd_compare(argv: list[str]) -> int:
+    from phys_ai_dataprofiler.connectors.huggingface import build_parser, run, NothingToProfile, _to_jsonable
+
+    p = argparse.ArgumentParser(
+        prog=f"{PROG} compare",
+        description="Rank datasets against the same requirements. Each target is a saved JSON run report "
+                    "or a Hub repo id (profiled now with the default settings plus any options after --).",
+        epilog="Example: compare lerobot/pusht org/other -r preset:teleop-imitation -- --max-files 20",
+    )
+    p.add_argument("targets", nargs="+", help="report.json files and/or Hub repo ids (at least two)")
+    p.add_argument("--requirements", "-r", required=True, metavar="FILE|preset:NAME")
+    p.add_argument("--output", "-o", help="Write the comparison here (.md or .json; default: "
+                                          "reports/compare_<requirements>_<time>.md and .json)")
+    p.add_argument("--quiet", action="store_true")
+    if "--" in argv:
+        i = argv.index("--")
+        argv, profile_opts = argv[:i], argv[i + 1:]
+    else:
+        profile_opts = []
+    a = p.parse_args(argv)
+    if len(a.targets) < 2:
+        p.error("compare needs at least two targets")
+    log = _log(a.quiet)
+    req = _load_req_or_exit(a.requirements)
+
+    reports: list[tuple[str, dict]] = []
+    for t in a.targets:
+        if t.endswith(".json") or Path(t).is_file():
+            reports.append((Path(t).stem, _read_report(t)))
+            continue
+        log(f"== profiling {t}")
+        args = build_parser().parse_args([t, *profile_opts, *(["--quiet"] if a.quiet else [])])
+        if req.files_include and not args.include and not args.file:
+            args.include = list(req.files_include)
+        if req.files_exclude:
+            args.exclude = list(args.exclude or []) + list(req.files_exclude)
+        try:
+            rep = run(args)
+        except NothingToProfile as exc:
+            log(f"   {exc}")
+            rep = None
+        except RuntimeError as exc:
+            log(f"   Error: {exc}")
+            rep = None
+        if rep is None:
+            reports.append((t, {"repo_id": t, "files": [], "totals": {}, "files_skipped": []}))
+        else:
+            reports.append((t, _to_jsonable(rep)))
+
+    cands = compare_reports(reports, req)
+    print_compare(cands, log)
+    md = render_compare_markdown(cands, req)
+    data = compare_to_dict(cands, req)
+    if a.output:
+        out = Path(a.output)
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out.write_text(json.dumps(data, indent=2) if out.suffix == ".json" else md)
+        log(f"Comparison: {out}")
+    else:
+        d = Path("reports")
+        d.mkdir(exist_ok=True)
+        base = d / f"compare_{req.name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        base.with_suffix(".md").write_text(md)
+        base.with_suffix(".json").write_text(json.dumps(data, indent=2))
+        log(f"Comparison: {base}.md and .json")
+    best = cands[0].verdict.status
+    return {"pass": 0, "fail": 1, "unknown": 4}[best]
+
+
+# --------------------------------------------------------------------------- #
+# requirements
+# --------------------------------------------------------------------------- #
+
+def cmd_requirements(argv: list[str]) -> int:
+    p = argparse.ArgumentParser(prog=f"{PROG} requirements", description="Presets and requirements files.")
+    sp = p.add_subparsers(dest="action", required=True)
+    sp.add_parser("list", help="List built-in presets")
+    s = sp.add_parser("show", help="Print a preset's YAML (redirect to a file to start your own)")
+    s.add_argument("preset")
+    v = sp.add_parser("validate", help="Check a requirements file without profiling anything")
+    v.add_argument("spec", metavar="FILE|preset:NAME")
+    a = p.parse_args(argv)
+
+    if a.action == "list":
+        for name, desc in list_presets():
+            print(f"preset:{name}\n    {desc}")
+        return 0
+    if a.action == "show":
+        try:
+            print(preset_text(a.preset.removeprefix("preset:")), end="")
+        except RequirementsError as exc:
+            print(str(exc), file=sys.stderr)
+            return 2
+        return 0
+    try:
+        req = load_requirements(a.spec)
+    except RequirementsError as exc:
+        print(str(exc), file=sys.stderr)
+        return 2
+    print(f"OK: {req.name}: {len(req.streams)} stream requirement(s)"
+          f" ({sum(s.required for s in req.streams)} required), "
+          f"{len(req.dataset.active())} dataset requirement(s)")
+    for s in req.streams:
+        th = ", ".join(f"{k}={val:g}" for k, val in s.thresholds())
+        print(f"  stream '{s.name}'{'' if s.required else ' (optional)'}: [{s.match.describe()}]"
+              + (f" {th}" if th else ""))
+    for k in req.dataset.active():
+        print(f"  dataset {k} = {getattr(req.dataset, k)}")
+    return 0
+
+
+def main(argv: list[str]) -> int:
+    cmd, rest = argv[0], argv[1:]
+    try:
+        return {"check": cmd_check, "compare": cmd_compare, "requirements": cmd_requirements}[cmd](rest)
+    except SystemExit as exc:
+        return exc.code if isinstance(exc.code, int) else 2

phys_ai_dataprofiler/compare.py

@@ -0,0 +1,123 @@
+"""
+Put several datasets side by side against the same requirements.
+
+Each candidate is a run report (dict, as written by the CLI). The result has
+one row per check and one column per dataset, plus a ranking: fewer required
+failures first, then fewer unknowns, then more passes, then more hours.
+Ties keep the order the datasets were given in.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from phys_ai_dataprofiler.evaluate import BADGE, FAIL, PASS, UNKNOWN, Verdict, evaluate, verdict_to_dict
+from phys_ai_dataprofiler.requirements import Requirements
+
+
+@dataclass
+class Candidate:
+    label: str
+    report: dict
+    verdict: Verdict
+
+    @property
+    def hours(self) -> float:
+        return (self.report.get("totals") or {}).get("duration_s", 0) / 3600
+
+    def sort_key(self) -> tuple:
+        req = [c for c in self.verdict.checks if c.required]
+        return (sum(c.status == FAIL for c in req), sum(c.status == UNKNOWN for c in req),
+                -sum(c.status == PASS for c in self.verdict.checks), -self.hours)
+
+
+def compare_reports(reports: list[tuple[str, dict]], req: Requirements) -> list[Candidate]:
+    cands = [Candidate(label, rep, evaluate(rep, req)) for label, rep in reports]
+    order = sorted(range(len(cands)), key=lambda i: (cands[i].sort_key(), i))
+    return [cands[i] for i in order]
+
+
+def _size(n: int | None) -> str:
+    if not n:
+        return "0 B"
+    for unit in ("B", "KB", "MB", "GB", "TB"):
+        if abs(n) < 1024 or unit == "TB":
+            return f"{n:.1f} {unit}" if unit != "B" else f"{n} B"
+        n /= 1024
+    return f"{n:.1f} TB"
+
+
+def _cell(text: str) -> str:
+    return text.replace("|", "/").replace("\n", " ")
+
+
+def render_compare_markdown(cands: list[Candidate], req: Requirements) -> str:
+    L = [f"# Dataset comparison against `{req.name}`", ""]
+    if req.description:
+        L += [req.description, ""]
+    L.append(f"Requirements: `{req.source}`. Ranked best first: fewer required failures, then fewer unknowns, "
+             "then more passes, then more hours.")
+    L.append("")
+    head = "| | " + " | ".join(f"{i + 1}. `{_cell(c.label)}`" for i, c in enumerate(cands)) + " |"
+    L += [head, "|---|" + "---|" * len(cands)]
+    L.append("| **Verdict** | " + " | ".join(f"**{BADGE[c.verdict.status]}** "
+                                              f"({c.verdict.passed}/{c.verdict.failed}/{c.verdict.unknown})"
+                                              for c in cands) + " |")
+    totals = [c.report.get("totals") or {} for c in cands]
+    L.append("| Files profiled | " + " | ".join(f"{t.get('files', 0)} ({t.get('invalid_or_errored', 0)} errored)"
+                                                for t in totals) + " |")
+    L.append("| Hours | " + " | ".join(f"{c.hours:.2f}" for c in cands) + " |")
+    L.append("| Size / read | " + " | ".join(f"{_size(t.get('bytes'))} / {_size(t.get('bytes_fetched'))}"
+                                              for t in totals) + " |")
+    ids = list(dict.fromkeys(ch.id for c in cands for ch in c.verdict.checks))
+    for cid in ids:
+        label = next(ch.label for c in cands for ch in c.verdict.checks if ch.id == cid)
+        opt = next((not ch.required) for c in cands for ch in c.verdict.checks if ch.id == cid)
+        row = []
+        for c in cands:
+            ch = next((x for x in c.verdict.checks if x.id == cid), None)
+            row.append("-" if ch is None else _cell(f"{BADGE[ch.status]}: {ch.headline or ch.observed}"))
+        L.append(f"| {_cell(label)}{' (optional)' if opt else ''} | " + " | ".join(row) + " |")
+    L.append("")
+    L.append("Cells show status and what was found. (n/n/n) is passed/failed/unknown checks.")
+    from phys_ai_dataprofiler import __version__
+    L.append("")
+    L.append(f"Generated by [phys-ai-dataprofiler](https://github.com/akrishni/phys-ai-dataprofiler) {__version__}.")
+    L.append("")
+    for i, c in enumerate(cands):
+        bad = [ch for ch in c.verdict.checks if ch.status != PASS]
+        if not bad and not c.verdict.notes:
+            continue
+        L.append(f"## {i + 1}. `{c.label}`: why not pass")
+        L.append("")
+        for n in c.verdict.notes:
+            L.append(f"- Note: {n}")
+        for ch in bad:
+            L.append(f"- **{ch.label}** {BADGE[ch.status]}: {ch.observed}")
+            for e in ch.evidence[:3]:
+                L.append(f"  - {e}")
+            if ch.resolve:
+                L.append(f"  - To settle: {ch.resolve}")
+        L.append("")
+    return "\n".join(L)
+
+
+def print_compare(cands: list[Candidate], log) -> None:
+    log("")
+    log("COMPARISON (best first)")
+    for i, c in enumerate(cands, 1):
+        v = c.verdict
+        log(f"  {i}. {BADGE[v.status]:<8} {c.label}  ({v.passed} pass, {v.failed} fail, {v.unknown} unknown; "
+            f"{c.hours:.2f} h)")
+        for ch in v.checks:
+            if ch.status != PASS:
+                log(f"       {BADGE[ch.status]:<8} {ch.label}: {ch.observed}")
+
+
+def compare_to_dict(cands: list[Candidate], req: Requirements) -> dict:
+    return {
+        "requirements": req.name, "requirements_source": req.source,
+        "ranking": [{"rank": i + 1, "label": c.label, "repo_id": c.report.get("repo_id"),
+                     "status": c.verdict.status, "hours": round(c.hours, 4),
+                     "verdict": verdict_to_dict(c.verdict)} for i, c in enumerate(cands)],
+    }

phys_ai_dataprofiler/connectors/__init__.py

@@ -0,0 +1 @@
+"""Data-source connectors. Currently: Hugging Face Hub."""