bobframes 0.1.0__py3-none-any.whl

bobframes/tests/data/synthetic/_data/District 01/2026-05-27_r110565/_manifest.json

@@ -0,0 +1,51 @@
+{
+  "schema_version": 3,
+  "build_timestamp": "2026-01-01T00:00:00+00:00",
+  "area": "District 01",
+  "drop_date": "2026-05-27",
+  "drop_label": "r110565",
+  "captures": [
+    "1",
+    "2",
+    "3",
+    "4",
+    "5"
+  ],
+  "capture_status": {
+    "1": "ok",
+    "2": "ok",
+    "3": "ok",
+    "4": "ok",
+    "5": "ok"
+  },
+  "row_counts": {
+    "draws": 60,
+    "draw_bindings": 60,
+    "passes": 60,
+    "shaders": 60,
+    "textures": 60,
+    "render_targets": 45,
+    "rt_event_timeline": 60,
+    "buffers": 60,
+    "programs": 60,
+    "samplers": 55,
+    "fbos": 60,
+    "events": 60,
+    "clears": 40,
+    "dispatches": 45,
+    "state_change_events": 60,
+    "indirect_args": 0,
+    "vertex_inputs": 60,
+    "resource_creation": 60,
+    "counters_per_event": 60,
+    "descriptor_access": 60,
+    "program_transitions": 60,
+    "frame_totals": 5,
+    "pixel_history": 60,
+    "vbo_samples": 60,
+    "ibo_samples": 60,
+    "post_vs_samples": 60,
+    "texture_samples": 60
+  },
+  "rotated_from": null
+}

bobframes/tests/data/synthetic/_data/District 01/2026-05-28_r110600/_manifest.json

@@ -0,0 +1,51 @@
+{
+  "schema_version": 3,
+  "build_timestamp": "2026-01-01T00:00:00+00:00",
+  "area": "District 01",
+  "drop_date": "2026-05-28",
+  "drop_label": "r110600",
+  "captures": [
+    "1",
+    "2",
+    "3",
+    "4",
+    "5"
+  ],
+  "capture_status": {
+    "1": "ok",
+    "2": "ok",
+    "3": "ok",
+    "4": "ok",
+    "5": "ok"
+  },
+  "row_counts": {
+    "draws": 60,
+    "draw_bindings": 60,
+    "passes": 60,
+    "shaders": 60,
+    "textures": 60,
+    "render_targets": 45,
+    "rt_event_timeline": 60,
+    "buffers": 60,
+    "programs": 60,
+    "samplers": 55,
+    "fbos": 60,
+    "events": 60,
+    "clears": 40,
+    "dispatches": 45,
+    "state_change_events": 60,
+    "indirect_args": 0,
+    "vertex_inputs": 60,
+    "resource_creation": 60,
+    "counters_per_event": 60,
+    "descriptor_access": 60,
+    "program_transitions": 60,
+    "frame_totals": 5,
+    "pixel_history": 60,
+    "vbo_samples": 60,
+    "ibo_samples": 60,
+    "post_vs_samples": 60,
+    "texture_samples": 60
+  },
+  "rotated_from": null
+}

bobframes/tests/make_synthetic.py

@@ -0,0 +1,171 @@
+"""Generate the bundled synthetic `_data/` test fixture from a real ingest.
+
+Derives a tiny, content-scrubbed dataset from a real `_data/` tree so the golden-parity
+suite has something to render in CI without committing real captures (ADR-8). NOT run in
+CI — run by hand to (re)generate the fixture, then commit the result + refreshed golden.
+
+Scrubbing policy (ADR-8 + ADR-6):
+  * area name           -> generic ("District 01")
+  * marker/pass paths   -> KEYWORD-PRESERVING (keeps the draw_class keyword so the classifier
+                           still produces the same class distribution; strips game content)
+  * labels / names / file paths -> generic per-row tokens
+  * counter names, formats, enums, blend/depth numerics -> KEPT (not content; load-bearing,
+    e.g. 'GPU Duration' drives pass_gpu)
+  * row counts truncated to --rows per table
+
+Usage (from the venv):
+  python -m bobframes.tests.make_synthetic \
+      --src "c:/Users/vsiva/Downloads/RDC mainline r110565 25-05-2026/_data" \
+      --out bobframes/tests/data/synthetic --rows 60
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from .. import schemas
+
+SYNTH_AREA = "District 01"
+FIXED_TS = "2026-01-01T00:00:00+00:00"
+
+# Two synthetic drops (different build labels/dates) so trend/sparkline reports have >1 point.
+DROPS = [
+    {"date": "2026-05-27", "label": "r110565"},
+    {"date": "2026-05-28", "label": "r110600"},
+]
+
+# Derived tables are regenerated by render-only (derive/pass_class_breakdown/texture_usage); skip.
+SKIP_TABLES = {"pass_class_breakdown", "texture_usage"}
+
+# String columns that carry game content -> generic per-row token.
+GENERIC_SCRUB = {
+    "draw_name", "label", "src_file_path", "declared_label",
+    "attribute_name", "usage_name",
+}
+# Marker/pass-path columns -> keyword-preserving (keeps draw_class coverage).
+MARKER_SCRUB = {
+    "marker_path", "marker_path_norm", "parent_pass_path", "parent_pass_path_norm",
+    "parent_marker_path",
+}
+_STRING_TYPES = (pa.string(), pa.large_string())
+
+
+def _marker_token(mp: str) -> str:
+    """Return a class-preserving stand-in for a marker path (mirrors _classify_draw order)."""
+    s = (mp or "").lower()
+    if "shadow" in s:
+        return "shadow"
+    if "prepass" in s or "depthonly" in s:
+        return "prepass"
+    if "slate" in s or "/ui" in s or s.endswith("ui"):
+        return "slate"          # -> ui branch
+    if "postprocess" in s or "tonemap" in s or "bloom" in s or "eyeadapt" in s:
+        return "postprocess"
+    if "decal" in s:
+        return "decal"
+    if "translucen" in s:
+        return "translucent"
+    if "basepass" in s:
+        return "basepass"
+    return "scene"              # no keyword -> class falls to blend/depth (preserved)
+
+
+def _scrub_table(t: pa.Table, area: str, date: str, label: str) -> pa.Table:
+    n = t.num_rows
+    field_type = {f.name: f.type for f in t.schema}
+    out = {}
+    for name in t.column_names:
+        ft = field_type[name]
+        if name == "area":
+            out[name] = pa.array([area] * n, type=ft)
+        elif name == "drop_date":
+            out[name] = pa.array([date] * n, type=ft)
+        elif name == "drop_label":
+            out[name] = pa.array([label] * n, type=ft)
+        elif name in MARKER_SCRUB and ft in _STRING_TYPES:
+            orig = t.column(name).to_pylist()
+            out[name] = pa.array(
+                [f"Frame 1/{_marker_token(v)}/p{i}" if v is not None else None
+                 for i, v in enumerate(orig)],
+                type=ft,
+            )
+        elif name in GENERIC_SCRUB and ft in _STRING_TYPES:
+            orig = t.column(name).to_pylist()
+            out[name] = pa.array(
+                [f"{name}_{i}" if v is not None else None for i, v in enumerate(orig)],
+                type=ft,
+            )
+        else:
+            out[name] = t.column(name)
+    # Preserve column order exactly (schema regression checks names/order).
+    return pa.table({name: out[name] for name in t.column_names})
+
+
+def _build_drop(src_area_dir: str, src_drop: str, out_drop_dir: str,
+                date: str, label: str) -> dict:
+    os.makedirs(out_drop_dir, exist_ok=True)
+    src_drop_dir = os.path.join(src_area_dir, src_drop)
+    row_counts: dict[str, int] = {}
+    captures: set[str] = set()
+    for stem in schemas.TABLES:
+        if stem in SKIP_TABLES:
+            continue
+        src = os.path.join(src_drop_dir, f"{stem}.parquet")
+        if not os.path.exists(src):
+            continue
+        t = pq.read_table(src)
+        t = t.slice(0, ARGS.rows)
+        t = _scrub_table(t, SYNTH_AREA, date, label)
+        pq.write_table(t, os.path.join(out_drop_dir, f"{stem}.parquet"), compression="snappy")
+        row_counts[stem] = t.num_rows
+        if "capture" in t.column_names:
+            captures.update(str(c) for c in t.column("capture").to_pylist() if c is not None)
+    caps = sorted(captures, key=lambda x: (len(x), x))
+    manifest = {
+        "schema_version": schemas.SCHEMA_VERSION,
+        "build_timestamp": FIXED_TS,
+        "area": SYNTH_AREA,
+        "drop_date": date,
+        "drop_label": label,
+        "captures": caps,
+        "capture_status": {c: "ok" for c in caps},
+        "row_counts": row_counts,
+        "rotated_from": None,
+    }
+    with open(os.path.join(out_drop_dir, "_manifest.json"), "w", encoding="utf-8") as f:
+        json.dump(manifest, f, indent=2)
+        f.write("\n")
+    return {"label": label, "tables": len(row_counts), "captures": caps}
+
+
+def main() -> int:
+    src = ARGS.src
+    areas = sorted(d for d in os.listdir(src) if os.path.isdir(os.path.join(src, d)))
+    if len(areas) < len(DROPS):
+        raise SystemExit(f"need >= {len(DROPS)} source areas, found {len(areas)} in {src}")
+    out_area = os.path.join(ARGS.out, "_data", SYNTH_AREA)
+    for spec, src_area in zip(DROPS, areas):
+        src_area_dir = os.path.join(src, src_area)
+        src_drops = sorted(d for d in os.listdir(src_area_dir)
+                           if os.path.isdir(os.path.join(src_area_dir, d)))
+        if not src_drops:
+            raise SystemExit(f"no drops under {src_area_dir}")
+        out_drop = os.path.join(out_area, f"{spec['date']}_{spec['label']}")
+        info = _build_drop(src_area_dir, src_drops[0], out_drop, spec["date"], spec["label"])
+        print(f"  {SYNTH_AREA}/{spec['date']}_{spec['label']}  <- {src_area}/{src_drops[0]}  "
+              f"({info['tables']} tables, captures={info['captures']})")
+    print(f"synthetic _data written under {os.path.join(ARGS.out, '_data')}")
+    return 0
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser(prog="bobframes.tests.make_synthetic")
+    ap.add_argument("--src", required=True, help="real _data/ tree to derive from")
+    ap.add_argument("--out", default=os.path.join(os.path.dirname(__file__), "data", "synthetic"))
+    ap.add_argument("--rows", type=int, default=60, help="max rows per table")
+    ARGS = ap.parse_args()
+    raise SystemExit(main())

bobframes/tests/smoke.py

@@ -0,0 +1,199 @@
+"""End-to-end smoke test (c15 rewrite — G-12).
+
+Two modes, no hardcoded area/label/date (the old `Chor bazar` / `r110565` / `2026-05-27`
+constants + the `__file__`-walked project root are gone):
+
+  bobframes smoke                 render-only against the bundled synthetic `_data/` fixture
+                                  (no `.rdc`, no qrenderdoc/GPU). Runs everywhere, incl. CI.
+  bobframes smoke --data <DIR>    full ingest + render against a real capture root; auto-selects
+                                  area + latest drop via discovery.find_drops. Needs Windows +
+                                  RenderDoc; self-hosted / nightly only.
+
+Both modes assert: outputs exist, every Parquet matches schemas.expected_columns, entity tables
+carry a populated stable_key, catalog is rebuilt, and every emitted HTML is lint-clean.
+
+Run standalone:  python -m bobframes.tests.smoke [--data DIR]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import tempfile
+
+import pyarrow.parquet as papq
+
+from .. import discovery, lint, paths, schemas
+
+
+def _fail(msg: str) -> int:
+    print(f'FAIL: {msg}')
+    return 1
+
+
+def _lint_html(root: str) -> int:
+    """Lint every emitted HTML under `root` (skipping the parquet cache). Returns hit count."""
+    from . import _render_util as u
+    hits = 0
+    for rel in u.rendered_html_files(root):
+        for lineno, label, snip in lint.lint_file(os.path.join(root, rel)):
+            print(f'  LINT {rel}:{lineno}: [{label}] {snip}')
+            hits += 1
+    return hits
+
+
+def _assert_drop_parquet(out_dir: str, check_csv: bool) -> int:
+    """Schema match + stable_key population for one drop's `_data` dir. Returns error count.
+
+    `check_csv` asserts each Parquet has its `.csv` sidecar — true only for the full-ingest path;
+    the committed synthetic fixture is Parquet-only (ADR-8), so render-only skips it.
+    """
+    errors = 0
+    for stem in schemas.TABLES:
+        pq = os.path.join(out_dir, f'{stem}.parquet')
+        if not os.path.exists(pq):
+            continue
+        if check_csv and not os.path.exists(os.path.join(out_dir, f'{stem}.csv')):
+            print(f'  {stem}: .csv missing alongside .parquet')
+            errors += 1
+        cols = list(papq.read_schema(pq).names)
+        expected = list(schemas.expected_columns(stem))
+        if cols != expected:
+            print(f'  {stem}: cols={cols} != expected={expected}')
+            errors += 1
+        if schemas.is_entity_table(stem):
+            t = papq.read_table(pq, columns=['stable_key'])
+            if t.num_rows and not any(t.column('stable_key').to_pylist()):
+                print(f'  {stem}: stable_key all empty across {t.num_rows} rows')
+                errors += 1
+    return errors
+
+
+# --- render-only (default) ---------------------------------------------------
+
+def _render_only_smoke() -> int:
+    from . import _render_util as u
+
+    with tempfile.TemporaryDirectory(prefix='bobframes-smoke-') as td:
+        dest = os.path.join(td, 'root')
+        print(f'1. render-only against bundled synthetic -> {dest}')
+        try:
+            u.render_fresh(dest)
+        except RuntimeError as e:
+            return _fail(str(e))
+
+        print('2. html emitted')
+        html = u.rendered_html_files(dest)
+        if not os.path.exists(paths.root_index_html(dest)):
+            return _fail('root index.html missing')
+        drills = [r for r in html if r.endswith('/index.html') and 'drill' in r]
+        reports = [r for r in html if r.startswith('_reports/') and 'drill' not in r]
+        if not drills:
+            return _fail('no per-drop drill index.html emitted')
+        if not reports:
+            return _fail('no _reports/*.html emitted')
+
+        print('3. schema + stable_key (synthetic _data)')
+        errors = 0
+        for drop in discovery.find_drops(dest):
+            out_dir = paths.drop_data_dir(dest, drop.area, os.path.basename(drop.drop_dir))
+            errors += _assert_drop_parquet(out_dir, check_csv=False)
+        if errors:
+            return _fail(f'{errors} parquet schema/stable_key error(s)')
+
+        print('4. catalog')
+        if not os.path.exists(paths.catalog_parquet(dest)):
+            return _fail('catalog parquet missing')
+
+        print('5. lint')
+        if _lint_html(dest):
+            return _fail('lint hits in rendered HTML')
+
+        print(f'OK: render-only smoke - {len(drills)} drop(s), {len(html)} HTML pages, lint clean')
+        return 0
+
+
+# --- full ingest (--data) ----------------------------------------------------
+
+def _full_smoke(data_dir: str, pixel_grid: int) -> int:
+    data_dir = os.path.abspath(data_dir)
+    drops = discovery.find_drops(data_dir)
+    if not drops:
+        return _fail(f'no drops with .rdc found under {data_dir}')
+    print(f'1. discovered {len(drops)} drop(s): ' +
+          ', '.join(f'{d.area}/{os.path.basename(d.drop_dir)}' for d in drops))
+
+    print('2. ingest (full pipeline, --force)')
+    rc = subprocess.run(
+        [sys.executable, '-m', 'bobframes.run', '--root', data_dir, '--force',
+         '--pixel-grid', str(pixel_grid)],
+    ).returncode
+    if rc != 0:
+        return _fail(f'pipeline exited rc={rc}')
+
+    print('3. per-drop outputs')
+    errors = 0
+    for drop in drops:
+        dated = os.path.basename(drop.drop_dir)
+        out_dir = paths.drop_data_dir(data_dir, drop.area, dated)
+        tmp_dir = paths.drop_data_dir_tmp(data_dir, drop.area, dated)
+        if not os.path.isdir(out_dir):
+            errors += 1
+            print(f'  {out_dir} missing')
+            continue
+        if os.path.isdir(tmp_dir):
+            errors += 1
+            print(f'  {tmp_dir} should be gone after atomic commit')
+        errors += _assert_drop_parquet(out_dir, check_csv=True)
+
+        mf = os.path.join(out_dir, '_manifest.json')
+        with open(mf, encoding='utf-8') as f:
+            m = json.load(f)
+        if m.get('schema_version') != schemas.SCHEMA_VERSION:
+            errors += 1
+            print(f'  manifest schema_version={m.get("schema_version")} != {schemas.SCHEMA_VERSION}')
+        if not all(s == 'ok' for s in m.get('capture_status', {}).values()):
+            errors += 1
+            print(f'  capture_status not all ok: {m.get("capture_status")}')
+
+        drill = paths.drop_drill_dir(data_dir, drop.area, dated)
+        if not os.path.exists(os.path.join(drill, 'index.html')):
+            errors += 1
+            print(f'  drill index.html missing at {drill}')
+    if errors:
+        return _fail(f'{errors} ingest output error(s)')
+
+    print('4. catalog + root index')
+    if not os.path.exists(paths.catalog_parquet(data_dir)):
+        return _fail('catalog parquet missing')
+    if not os.path.exists(paths.root_index_html(data_dir)):
+        return _fail('root index.html missing')
+
+    print('5. lint')
+    if _lint_html(data_dir):
+        return _fail('lint hits in rendered HTML')
+
+    print(f'OK: full smoke - {len(drops)} drop(s) ingested + rendered, lint clean')
+    return 0
+
+
+def main(data: str | None = None, pixel_grid: int = 4) -> int:
+    if data:
+        return _full_smoke(data, pixel_grid)
+    return _render_only_smoke()
+
+
+def _parse(argv: list[str]) -> argparse.Namespace:
+    ap = argparse.ArgumentParser(prog='bobframes.tests.smoke',
+                                 description='render-only (default) or full ingest (--data) smoke')
+    ap.add_argument('--data', help='capture root for full ingest (default: bundled synthetic)')
+    ap.add_argument('--pixel-grid', type=int, default=4)
+    return ap.parse_args(argv)
+
+
+if __name__ == '__main__':
+    a = _parse(sys.argv[1:])
+    sys.exit(main(data=a.data, pixel_grid=a.pixel_grid))

bobframes/tests/test_determinism.py

@@ -0,0 +1,19 @@
+"""Determinism: rendering the same data twice produces byte-identical HTML (after masking the
+build timestamp). Catches dict-ordering, set-iteration, and other nondeterminism regressions."""
+from __future__ import annotations
+
+import os
+
+from . import _render_util as u
+
+
+def test_render_is_deterministic(tmp_path):
+    a = u.render_fresh(str(tmp_path / "a"))
+    b = u.render_fresh(str(tmp_path / "b"))
+
+    fa, fb = u.rendered_html_files(a), u.rendered_html_files(b)
+    assert fa == fb, f"page set differs between runs: {set(fa) ^ set(fb)}"
+    for rel in fa:
+        ca = u.normalize(open(os.path.join(a, rel), encoding="utf-8").read())
+        cb = u.normalize(open(os.path.join(b, rel), encoding="utf-8").read())
+        assert ca == cb, f"nondeterministic output: {rel}"