bobframes 0.1.0__py3-none-any.whl

bobframes/discovery.py

@@ -0,0 +1,132 @@
+"""Area + dated-drop walking.
+
+Layout:
+    <root>/
+      <Area>/
+        <YYYY-MM-DD>[_<label>]/
+          <capture>.rdc, .xml, .zip.xml, .zip
+
+Finds the newest dated drop per area; filters by --area / --label / --capture.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass
+
+DATED_RE = re.compile(r'^(\d{4}-\d{2}-\d{2})(?:_(.*))?$')
+
+
+@dataclass(frozen=True)
+class Drop:
+    area: str
+    drop_date: str          # 'YYYY-MM-DD'
+    drop_label: str         # text after the date, or '' if absent
+    drop_dir: str           # absolute path to the dated folder
+    captures: tuple[str, ...]  # rdc filenames without .rdc extension, sorted
+
+
+def _list_areas(root: str) -> list[tuple[str, str]]:
+    out = []
+    for entry in sorted(os.listdir(root)):
+        if entry.startswith('_') or entry.startswith('.'):
+            continue
+        full = os.path.join(root, entry)
+        if os.path.isdir(full):
+            out.append((entry, full))
+    return out
+
+
+def _latest_drop_dir(area_dir: str) -> tuple[str, str, str] | None:
+    """Return (drop_date, drop_label, drop_dir) for newest dated sub-dir, else None."""
+    dated = []
+    for entry in os.listdir(area_dir):
+        m = DATED_RE.match(entry)
+        if not m:
+            continue
+        full = os.path.join(area_dir, entry)
+        if not os.path.isdir(full):
+            continue
+        dated.append((m.group(1), m.group(2) or '', full))
+    if not dated:
+        return None
+    dated.sort(reverse=True)
+    return dated[0]
+
+
+def _captures(drop_dir: str) -> tuple[str, ...]:
+    names = []
+    for entry in sorted(os.listdir(drop_dir)):
+        if entry.endswith('.rdc') and os.path.isfile(os.path.join(drop_dir, entry)):
+            names.append(entry[:-4])
+    names.sort(key=lambda s: (len(s), s))
+    return tuple(names)
+
+
+def find_drops(
+    root: str,
+    area_filter: str | None = None,
+    label_filter: str | None = None,
+    capture_filter: str | None = None,
+) -> list[Drop]:
+    """Return drops to process. One per area (newest dated drop).
+
+    area_filter: exact area folder name to restrict to.
+    label_filter: only return drops whose drop_label matches.
+    capture_filter: restricts each drop's captures tuple to just this name.
+    """
+    root = os.path.abspath(root)
+    if not os.path.isdir(root):
+        raise FileNotFoundError(f'root not found: {root}')
+
+    drops: list[Drop] = []
+    for area, area_dir in _list_areas(root):
+        if area_filter and area != area_filter:
+            continue
+        latest = _latest_drop_dir(area_dir)
+        if latest is None:
+            continue
+        drop_date, drop_label, drop_dir = latest
+        if label_filter and drop_label != label_filter:
+            continue
+        captures = _captures(drop_dir)
+        if not captures:
+            continue
+        if capture_filter:
+            if capture_filter not in captures:
+                continue
+            captures = (capture_filter,)
+        drops.append(Drop(
+            area=area,
+            drop_date=drop_date,
+            drop_label=drop_label,
+            drop_dir=drop_dir,
+            captures=captures,
+        ))
+    return drops
+
+
+def parse_single_drop_arg(arg: str, root: str) -> Drop:
+    """Parse a positional argument like 'Chor bazar/2026-05-27_r110565/' into a Drop.
+
+    Used when the user passes a specific drop directory rather than --area.
+    """
+    arg = arg.rstrip('/\\')
+    parts = arg.replace('\\', '/').split('/')
+    if len(parts) < 2:
+        raise ValueError(f'expected <area>/<dated_drop>, got {arg!r}')
+    area, dated = parts[-2], parts[-1]
+    m = DATED_RE.match(dated)
+    if not m:
+        raise ValueError(f'not a dated drop folder: {dated!r}')
+    drop_dir = os.path.join(root, area, dated)
+    if not os.path.isdir(drop_dir):
+        raise FileNotFoundError(f'drop dir does not exist: {drop_dir}')
+    return Drop(
+        area=area,
+        drop_date=m.group(1),
+        drop_label=m.group(2) or '',
+        drop_dir=drop_dir,
+        captures=_captures(drop_dir),
+    )

bobframes/global_entities.py

@@ -0,0 +1,99 @@
+"""Emit <root>/_global_entities.parquet.
+
+One row per (stable_key, kind, area, drop_date, drop_label, capture, local_id).
+This is the canonical index for cross-drop entity joins. Layer 2 reports
+join into draws/passes/events tables using stable_key directly:
+
+    SELECT *
+    FROM read_parquet('**/draws.parquet') d
+    JOIN read_parquet('_global_entities.parquet') g
+      ON g.area = d.area
+     AND g.drop_date = d.drop_date
+     AND g.drop_label = d.drop_label
+     AND g.capture = d.capture
+     AND g.local_id = d.fs_shader_id
+     AND g.kind = 'shader'
+    WHERE g.stable_key = 'abc123...';
+
+The (kind, local_id) pair lets reports resolve any per-capture resource ID
+back to its drop-spanning identity.
+"""
+
+from __future__ import annotations
+
+import glob
+import os
+
+import pyarrow as pa
+import pyarrow.csv as pacsv
+import pyarrow.parquet as papq
+
+
+_ENTITY_TABLES = [
+    ('shaders', 'shader_id', 'shader'),
+    ('textures', 'tex_id', 'texture'),
+    ('render_targets', 'rt_id', 'texture'),  # RTs are textures
+    ('programs', 'program_id', 'program'),
+    ('samplers', 'sampler_id', 'sampler'),
+    ('fbos', 'fbo_id', 'fbo'),
+    ('buffers', 'buffer_id', 'buffer'),
+]
+
+
+_OUT_COLS = ('stable_key', 'kind', 'area', 'drop_date', 'drop_label',
+             'capture', 'local_id')
+
+
+def build_global_entities(root: str) -> int:
+    """Walk every entity parquet under <root>; emit _global_entities.parquet.
+
+    Returns the row count written.
+    """
+    cols: dict[str, list] = {c: [] for c in _OUT_COLS}
+
+    from . import paths as _paths
+    data_root = _paths.data_root(root)
+    for table, id_col, kind in _ENTITY_TABLES:
+        for path in sorted(glob.glob(os.path.join(data_root, '*', '*',
+                                                  f'{table}.parquet'))):
+            try:
+                t = papq.read_table(path, columns=['stable_key', 'area', 'drop_date',
+                                                   'drop_label', 'capture', id_col])
+            except Exception:
+                continue
+            sk = t.column('stable_key').to_pylist()
+            ar = t.column('area').to_pylist()
+            dd = t.column('drop_date').to_pylist()
+            dl = t.column('drop_label').to_pylist()
+            cp = t.column('capture').to_pylist()
+            lid = t.column(id_col).to_pylist()
+            for i in range(len(sk)):
+                if not sk[i]:
+                    continue  # skip entities without a stable_key (best-effort only)
+                cols['stable_key'].append(sk[i])
+                cols['kind'].append(kind)
+                cols['area'].append(ar[i])
+                cols['drop_date'].append(dd[i])
+                cols['drop_label'].append(dl[i])
+                cols['capture'].append(cp[i])
+                cols['local_id'].append(int(lid[i]) if lid[i] is not None else 0)
+
+    arrays: dict[str, pa.Array] = {}
+    for c in _OUT_COLS:
+        if c == 'local_id':
+            arrays[c] = pa.array(cols[c], type=pa.int64())
+        else:
+            arrays[c] = pa.array(cols[c], type=pa.string())
+    table = pa.table(arrays)
+
+    os.makedirs(_paths.data_root(root), exist_ok=True)
+    papq.write_table(table, _paths.global_entities_parquet(root), compression='snappy')
+    pacsv.write_csv(table, _paths.global_entities_csv(root))
+    return table.num_rows
+
+
+if __name__ == '__main__':
+    import sys
+    root = sys.argv[1] if len(sys.argv) > 1 else '.'
+    n = build_global_entities(root)
+    print(f'wrote _global_entities.parquet: {n} rows')