oopsie-data-tools 0.2.0__py3-none-any.whl

oopsie_data_tools/utils/conversion_utils.py

@@ -0,0 +1,220 @@
+"""Shared utilities for dataset-to-HDF5 conversion scripts.
+
+A converter writes ``oopsiedata_format_v1`` directly instead of going through
+:class:`~oopsie_data_tools.annotation_tool.episode_recorder.EpisodeRecorder`, so none of the
+recording-time checks run. These helpers exist so the parts that are easy to get silently
+wrong — the annotation layout and the image bounds — come from the same definitions the
+validator uses. See ``skill/reference/conversion.md``.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Sequence
+
+import cv2
+import h5py
+import numpy as np
+
+from oopsie_data_tools.annotation_tool.annotation_schema import (
+    OUTCOME_SUCCESS,
+    OUTCOMES,
+    SUCCESS_THRESHOLD,
+    success_to_outcome,
+    write_annotation_attrs,
+)
+from oopsie_data_tools.utils.h5 import decode_h5_scalar
+from oopsie_data_tools.utils.robot_profile.robot_profile import (
+    RobotProfile,
+    robot_profile_to_json,
+)
+from oopsie_data_tools.utils.validation.episode_loader import OOPSIE_DATA_SCHEMA_V1
+from oopsie_data_tools.utils.validation.episode_validator import (
+    MAX_IMAGE_SIZE,
+    MIN_IMAGE_SIZE,
+)
+
+#: Kept as an alias so existing converters keep working. Imported from the validator rather
+#: than restated: a local copy drifted to 1080 once already, which silently downscaled every
+#: video for a limit that was never 1080.
+SCHEMA_VERSION = OOPSIE_DATA_SCHEMA_V1
+MAX_DIM = MAX_IMAGE_SIZE
+
+
+def _resize_frames(frames: np.ndarray, max_dim: int = MAX_DIM) -> np.ndarray:
+    """Resize ``(T, H, W, 3)`` frames so that ``max(H, W) <= max_dim``.
+
+    Raises:
+        ValueError: If the result would fall below the validator's minimum side, which
+            would produce an episode rejected at upload rather than here.
+    """
+    h, w = frames.shape[1:3]
+    if w <= max_dim and h <= max_dim:
+        new_h, new_w = h, w
+    else:
+        scale = max_dim / max(w, h)
+        new_w, new_h = int(w * scale), int(h * scale)
+
+    if min(new_w, new_h) < MIN_IMAGE_SIZE:
+        raise ValueError(
+            f"Frames are {w}x{h}; at most {max_dim}px on the long side that becomes "
+            f"{new_w}x{new_h}, whose short side is under the {MIN_IMAGE_SIZE}px minimum "
+            "the validator enforces. Crop or re-render the source video instead."
+        )
+
+    if (new_w, new_h) == (w, h):
+        return frames
+    return np.stack(
+        [cv2.resize(f, (new_w, new_h), interpolation=cv2.INTER_AREA) for f in frames]
+    )
+
+
+def _parse_fps(control_freq: Any, default_fps: float = 15.0) -> float:
+    """Coerce a profile's ``control_freq`` to a usable FPS, falling back on nonsense."""
+    try:
+        parsed = float(control_freq)
+    except (TypeError, ValueError):
+        return default_fps
+    return parsed if parsed > 0 else default_fps
+
+
+def _decode_text(value: Any) -> str:
+    """Decode bytes, numpy scalars/arrays, or arbitrary values to a plain str."""
+    # Thin alias: decode_h5_scalar handles every case this did, plus None and str.
+    return decode_h5_scalar(value)
+
+
+def write_root_attrs(
+    file_handle: h5py.File,
+    *,
+    episode_id: str,
+    language_instruction: str,
+    lab_id: str,
+    operator_name: str,
+    robot_profile: RobotProfile,
+    timestamp: float | None = None,
+) -> None:
+    """Write the six required root attributes, plus the optional ``timestamp``.
+
+    All six are checked by the loader before anything else is read, and four of them are
+    additionally checked for emptiness. ``lab_id`` must be the real one from registration —
+    ``"your_lab_id"`` is rejected by name.
+    """
+    str_dtype = h5py.string_dtype(encoding="utf-8")
+    file_handle.attrs["schema"] = SCHEMA_VERSION
+    file_handle.attrs["episode_id"] = episode_id
+    file_handle.attrs["language_instruction"] = language_instruction
+    file_handle.attrs["lab_id"] = lab_id
+    file_handle.attrs["operator_name"] = operator_name
+    file_handle.attrs.create(
+        "robot_profile", robot_profile_to_json(robot_profile), dtype=str_dtype
+    )
+    if timestamp is not None:
+        file_handle.attrs["timestamp"] = float(timestamp)
+
+
+def write_episode_annotations(
+    file_handle: h5py.File,
+    *,
+    annotator_name: str,
+    success: float,
+    outcome: str | None = None,
+    episode_description: str = "",
+    side_effect_category: Sequence[str] | None = None,
+    severity: str = "",
+    additional_notes: str = "",
+    timestamp: str = "",
+) -> None:
+    """Write one annotator's annotation into ``episode_annotations/<annotator_name>/``.
+
+    The per-annotator subgroup is not optional: the loader iterates the subgroups of
+    ``episode_annotations`` and reads attributes off each one, so attributes written on the
+    parent group are invisible and the episode fails as unannotated. Identity fields
+    (``lab_id``, ``operator_name``) are root attributes — see :func:`write_root_attrs` —
+    not annotation fields.
+
+    ``success`` is stored as the exact float given, so qualified successes survive; it is
+    authoritative for magnitude, and ``outcome`` for which of the four branches this is.
+    Omit ``outcome`` and it is derived from the float, which can only ever yield the coarse
+    ``success``/``failure`` — pass it explicitly to record a qualified success.
+
+    Every taxonomy field is optional: a partial annotation is valid.
+    """
+    if not 0.0 <= float(success) <= 1.0:
+        raise ValueError(f"success must be in [0.0, 1.0], got {success!r}")
+
+    resolved_outcome = (outcome or "").strip().lower() or success_to_outcome(success)
+    if resolved_outcome not in OUTCOME_SUCCESS:
+        raise ValueError(f"outcome must be one of {OUTCOMES}, got {outcome!r}")
+    # A float and a slug that disagree would make every downstream reader pick a different
+    # answer depending on which one it happens to consult.
+    if (resolved_outcome == "failure") != (float(success) < SUCCESS_THRESHOLD):
+        raise ValueError(
+            f"outcome={resolved_outcome!r} disagrees with success={success!r} "
+            f"(threshold {SUCCESS_THRESHOLD})"
+        )
+
+    group = file_handle.require_group("episode_annotations").require_group(annotator_name)
+    write_annotation_attrs(
+        group,
+        {
+            "outcome": resolved_outcome,
+            "timestamp": timestamp,
+            "episode_description": episode_description,
+            "side_effect_category": list(side_effect_category or []),
+            "severity": severity,
+            "additional_notes": additional_notes,
+        },
+    )
+    # write_annotation_attrs derives a 1.0/0.0 success from the outcome; restore the exact
+    # value so a fractional success survives.
+    group.attrs["success"] = float(success)
+
+
+def write_actions(
+    file_handle: h5py.File,
+    actions: dict[str, np.ndarray],
+    action_space: Sequence[str],
+) -> None:
+    """Write ``/actions``: real arrays for ``action_space``, ``h5py.Empty`` for the rest.
+
+    Every canonical key must exist as a dataset. A key in ``action_space`` stored as Empty is
+    rejected, and a non-empty dataset the profile does not declare is rejected too, so the
+    split has to follow the profile exactly.
+    """
+    from oopsie_data_tools.annotation_tool.episode_recorder import VALID_ACTION_KEYS
+
+    declared = set(action_space)
+    unknown = declared - set(VALID_ACTION_KEYS)
+    if unknown:
+        raise ValueError(f"action_space contains unrecognized keys: {sorted(unknown)}")
+    missing = declared - set(actions)
+    if missing:
+        raise ValueError(f"No action array supplied for declared keys: {sorted(missing)}")
+
+    group = file_handle.create_group("actions")
+    for key in sorted(VALID_ACTION_KEYS):
+        if key in declared:
+            group.create_dataset(key, data=np.asarray(actions[key]), dtype=np.float64)
+        else:
+            group.create_dataset(key, data=h5py.Empty(dtype=np.float64))
+
+
+def write_video_paths(
+    file_handle: h5py.File,
+    video_paths: dict[str, str],
+    h5_path: Path | str,
+) -> None:
+    """Write ``/observations/video_paths`` as paths relative to the episode file."""
+    import os
+
+    str_dtype = h5py.string_dtype(encoding="utf-8")
+    episode_dir = Path(h5_path).resolve().parent
+    observations = file_handle.require_group("observations")
+    group = observations.require_group("video_paths")
+    for cam, raw in video_paths.items():
+        target = Path(raw).expanduser()
+        if not target.is_absolute():
+            target = episode_dir / target
+        rel = os.path.relpath(target.resolve(), start=episode_dir)
+        group.create_dataset(cam, data=rel.replace(os.sep, "/"), dtype=str_dtype)

oopsie_data_tools/utils/h5.py

@@ -0,0 +1,60 @@
+"""Small HDF5 helpers shared by the package and the scripts.
+
+Deliberately dependency-light — numpy only, no h5py or cv2 — so conversion and inspection
+scripts can use it without pulling in the validation stack.
+
+Both helpers replaced a family of near-identical local copies: four separate
+bytes-to-string decoders across the scripts, and three hand-rolled episode globs that
+disagreed about whether ``.hdf5`` counts.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Iterable
+
+import numpy as np
+
+#: Extensions an episode file may carry. ``validate`` accepts both.
+EPISODE_SUFFIXES = (".h5", ".hdf5")
+
+
+def decode_h5_scalar(value: Any) -> str:
+    """Decode an HDF5 attribute or dataset scalar to a plain ``str``.
+
+    Handles the shapes h5py hands back depending on how a value was written: raw ``bytes``,
+    ``str``, numpy scalars, and 0-d or single-element arrays. Anything else is stringified,
+    so this never raises.
+    """
+    if value is None:
+        return ""
+    if isinstance(value, bytes):
+        return value.decode("utf-8", errors="replace")
+    if isinstance(value, str):
+        return value
+    if isinstance(value, np.generic):
+        return decode_h5_scalar(value.item())
+    if isinstance(value, np.ndarray):
+        if value.shape == ():
+            return decode_h5_scalar(value.item())
+        if value.size == 0:
+            return ""
+        return decode_h5_scalar(value.reshape(-1)[0])
+    return str(value)
+
+
+def find_episode_files(
+    root: Path | str, suffixes: Iterable[str] = EPISODE_SUFFIXES
+) -> list[Path]:
+    """Every episode file under ``root``, recursively, in a stable order.
+
+    Args:
+        root: Directory to search.
+        suffixes: Extensions to accept. Pass a narrower set where downstream code only
+            handles one — the annotation server's path guard accepts ``.h5`` alone.
+    """
+    root = Path(root)
+    wanted = {s.lower() for s in suffixes}
+    return sorted(
+        p for p in root.rglob("*") if p.is_file() and p.suffix.lower() in wanted
+    )

oopsie_data_tools/utils/h5_inspect.py

@@ -0,0 +1,167 @@
+"""Human-readable dump of an HDF5 file: groups, datasets, shapes, dtypes, attributes.
+
+Backs ``oopsie-data inspect``. This is a debugging aid for looking at a recorded episode,
+not a validator — it never rejects anything and makes no assumptions about the schema, so
+it is equally useful on a file that fails ``oopsie-data validate``.
+
+Output goes to stdout via ``print`` rather than the logger: it is the command's result,
+not a progress report.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import math
+from pprint import pformat
+from typing import Any
+
+import h5py
+import numpy as np
+
+from oopsie_data_tools.utils.h5 import decode_h5_scalar
+
+_MAX_STR = 200  # truncation width for string scalars
+_MAX_ELEMS = 32  # array elements shown before eliding
+_MAX_LIST = 16  # list entries shown before eliding
+
+
+def _human_bytes(n: int | None) -> str:
+    if n is None:
+        return "?"
+    if n < 1024:
+        return f"{n} B"
+    units = ["KiB", "MiB", "GiB", "TiB", "PiB"]
+    f = float(n)
+    for u in units:
+        f /= 1024.0
+        if f < 1024.0:
+            return f"{f:.2f} {u}"
+    return f"{f:.2f} EiB"
+
+
+def _truncate(s: str) -> str:
+    return s[:_MAX_STR] + "…" if len(s) > _MAX_STR else s
+
+
+def _fmt_scalar(v: Any) -> str:
+    if isinstance(v, (np.generic,)):
+        try:
+            v = v.item()
+        except Exception:
+            pass
+    if isinstance(v, bytes):
+        return repr(_truncate(decode_h5_scalar(v)))
+    if isinstance(v, str):
+        return repr(_truncate(v))
+    if isinstance(v, (_dt.datetime, _dt.date)):
+        return v.isoformat()
+    return repr(v)
+
+
+def _fmt_array(a: np.ndarray, *, max_elems: int = _MAX_ELEMS) -> str:
+    # Keep output stable and short for big arrays.
+    if a.size == 0:
+        return f"array(shape={a.shape}, dtype={a.dtype}, empty)"
+
+    flat = a.ravel()
+    head = flat[: min(flat.size, max_elems)]
+    suffix = f", … (+{flat.size - max_elems} more)" if flat.size > max_elems else ""
+
+    try:
+        content = np.array2string(head, threshold=max_elems, edgeitems=math.inf)
+    except Exception:
+        content = repr(head)
+
+    return f"array(shape={a.shape}, dtype={a.dtype}, head={content}{suffix})"
+
+
+def _fmt_attr_value(v: Any) -> str:
+    # h5py may return scalars, bytes, numpy arrays, or lists.
+    if isinstance(v, np.ndarray):
+        return _fmt_array(v)
+    if isinstance(v, (list, tuple)):
+        if len(v) == 0:
+            return "[]"
+        if len(v) <= _MAX_LIST:
+            return pformat([_fmt_scalar(x) for x in v])
+        head = [_fmt_scalar(x) for x in v[:_MAX_LIST]]
+        return pformat(head)[:-1] + f", … (+{len(v) - _MAX_LIST} more)]"
+    return _fmt_scalar(v)
+
+
+def _print_attrs(obj: h5py.Group | h5py.Dataset, *, indent: int) -> None:
+    if len(obj.attrs) == 0:
+        return
+    print(" " * indent + "attrs:")
+    for k in sorted(obj.attrs.keys()):
+        try:
+            v = obj.attrs[k]
+        except Exception as e:
+            print(" " * (indent + 2) + f"- {k!r}: <error reading attr: {e}>")
+            continue
+        print(" " * (indent + 2) + f"- {k!r}: {_fmt_attr_value(v)}")
+
+
+def _describe_dataset(ds: h5py.Dataset) -> str:
+    parts = [f"shape={ds.shape}", f"dtype={ds.dtype}"]
+
+    try:
+        if ds.chunks is not None:
+            parts.append(f"chunks={ds.chunks}")
+    except Exception:
+        pass
+
+    try:
+        comp = ds.compression
+        if comp is not None:
+            parts.append(f"compression={comp!r}")
+    except Exception:
+        pass
+
+    try:
+        fill = ds.fillvalue
+        if fill is not None:
+            parts.append(f"fill={_fmt_scalar(fill)}")
+    except Exception:
+        pass
+
+    try:
+        nbytes = int(ds.size) * int(ds.dtype.itemsize)
+        parts.append(f"approx_nbytes={_human_bytes(nbytes)}")
+    except Exception:
+        pass
+
+    return ", ".join(parts)
+
+
+def _walk(name: str, obj: h5py.Group | h5py.Dataset, *, indent: int) -> None:
+    if isinstance(obj, h5py.Group):
+        title = "/" if name == "" else name
+        print(" " * indent + f"[group] {title}")
+        _print_attrs(obj, indent=indent + 2)
+
+        for k in sorted(obj.keys()):
+            # Show links explicitly rather than following them (common in some layouts).
+            try:
+                link = obj.get(k, getlink=True)
+                if isinstance(link, (h5py.SoftLink, h5py.ExternalLink)):
+                    print(" " * (indent + 2) + f"[link] {title.rstrip('/')}/{k} -> {link}")
+                    continue
+            except Exception:
+                pass
+
+            child_name = (title.rstrip("/") + "/" + k) if title != "/" else ("/" + k)
+            _walk(child_name, obj[k], indent=indent + 2)
+
+    elif isinstance(obj, h5py.Dataset):
+        print(" " * indent + f"[dataset] {name} ({_describe_dataset(obj)})")
+        _print_attrs(obj, indent=indent + 2)
+    else:
+        print(" " * indent + f"[unknown] {name}: {type(obj)}")
+
+
+def inspect_h5(path: str) -> None:
+    """Print the full structure of the HDF5 file at *path* to stdout."""
+    with h5py.File(path, "r") as f:
+        print(f"HDF5: {path}")
+        _walk("", f, indent=0)

oopsie_data_tools/utils/hf_limits.py

@@ -0,0 +1,22 @@
+"""HuggingFace Hub layout limits, and how we split a session to stay under them.
+
+These live apart from :mod:`oopsie_data_tools.utils.restructure` — their natural home —
+only so that ``oopsie-data --help`` can quote the numbers without importing h5py, which
+costs ~280 ms and is pure waste when all the user asked for was help text. Everything
+else imports them from here too, so there is still exactly one definition of each.
+
+Modules bind these names at import time, so a test that patches ``restructure.FILE_LIMIT``
+changes the limit only for that module. Patch each module you want affected.
+"""
+
+from __future__ import annotations
+
+#: Maximum files HuggingFace Hub accepts in a single directory. Exceeding it fails the
+#: upload at the Hub, so ``hf_upload.check_folder_size`` refuses before transferring
+#: anything.
+FILE_LIMIT = 10_000
+
+#: Episodes per numbered subfolder when splitting an oversized directory. A batch is this
+#: many HDF5 files plus their videos, so it stays under FILE_LIMIT as long as episodes have
+#: fewer than FILE_LIMIT / BATCH_SIZE - 1 cameras — roughly 19 at the values above.
+BATCH_SIZE = 500