datahive-tools 0.1.0__py3-none-any.whl

datahive/__init__.py

@@ -0,0 +1,31 @@
+"""datahive-tools: client package for collecting, validating, annotating and
+uploading HiveBoard manipulation episodes.
+
+Public library API (also used internally by the CLI and the local GUI, so
+all three surfaces stay in sync):
+
+    from datahive import EpisodeWriter, RobotProfile, TrialAnnotation
+"""
+
+from datahive.episode import EpisodeWriter, read_header, read_trajectory, resolve_episode
+from datahive.errors import DatahiveError
+from datahive.profile import RobotProfile, load_profile, write_profile_skeleton
+from datahive.schema import EpisodeHeader, FailureCause, Outcome, Strategy, TrialAnnotation
+
+__all__ = [
+    "EpisodeWriter",
+    "read_header",
+    "read_trajectory",
+    "resolve_episode",
+    "DatahiveError",
+    "RobotProfile",
+    "load_profile",
+    "write_profile_skeleton",
+    "EpisodeHeader",
+    "FailureCause",
+    "Outcome",
+    "Strategy",
+    "TrialAnnotation",
+]
+
+__version__ = "0.1.0"

datahive/annotate.py

@@ -0,0 +1,123 @@
+"""`datahive annotate`: fills outcome/failure_cause/etc. for an episode's
+trial, shared by the CLI's interactive prompts and the GUI's validation
+form (both end up calling `annotate_episode`)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from datahive.attachments import is_composed_assembly
+from datahive.episode import read_header
+from datahive.errors import AnnotationError
+from datahive.paths import resolve_episode_paths
+from datahive.schema import ANNOTATION_SCHEMA_CURRENT, TrialAnnotation
+from datahive.trials import upsert_row
+
+
+def annotate_episode(
+    samples_root: Path, episode_id: str, fields: dict[str, Any], *, validate_after: bool = True
+) -> TrialAnnotation:
+    """`fields` is a dict of TrialAnnotation column values (trial_id/lab_id/
+    platform_id/attachment_id/date are filled in automatically from the
+    episode header + config when omitted). Writes the row to trials.csv and,
+    by default, runs full validation afterward so the index status reflects
+    whether the episode is now `validated`."""
+    paths = resolve_episode_paths(samples_root, episode_id)
+    header = read_header(paths.h5)
+
+    data = dict(fields)
+    data.setdefault("trial_id", header.trial_id)
+    data.setdefault("lab_id", header.lab_id)
+    data.setdefault("platform_id", header.platform_id)
+    if "date" not in data or data["date"] is None:
+        from datetime import datetime, timezone
+
+        data["date"] = datetime.now(timezone.utc).date().isoformat()
+    for name_field in ("operator_name", "annotator_name", "failure_cause_detail"):
+        if data.get(name_field) is None:
+            data[name_field] = ""
+
+    if data.get("outcome") == "success" and data.get("completion_time_s") is None:
+        from datahive.episode import episode_stats
+
+        duration = episode_stats(paths.h5).get("duration_s")
+        if duration is not None:
+            data["completion_time_s"] = round(duration, 3)
+            data.setdefault("completion_source", "hdf5")
+
+    from datetime import datetime, timezone
+
+    data["annotated_at"] = datetime.now(timezone.utc).isoformat()
+    data["schema_version"] = ANNOTATION_SCHEMA_CURRENT
+
+    attachment_id = data.get("attachment_id")
+    composed = is_composed_assembly(attachment_id or "", samples_root)
+    context = {} if composed is None else {"composed_assembly": composed}
+
+    try:
+        annotation = TrialAnnotation.model_validate(data, context=context)
+    except Exception as e:
+        raise AnnotationError(f"Invalid annotation: {e}") from e
+
+    upsert_row(paths.trials_csv, annotation)
+    write_h5_annotation(paths.h5, annotation)
+
+    if validate_after:
+        from datahive.validate import validate_episode
+
+        try:
+            validate_episode(samples_root, episode_id)
+        except Exception:
+            pass  
+        
+    return annotation
+
+
+def _annotator_key(name: str) -> str:
+    return (name or "unknown").strip().replace("/", "_") or "unknown"
+
+
+def write_h5_annotation(h5_path: Path, annotation: TrialAnnotation) -> None:
+    """Stores the annotation inside the episode as
+    episode_annotations/<annotator>/ (attrs), stamped with schema_version, so
+    the file stays self-describing if trials.csv is lost or the episode moved."""
+    import json
+
+    import h5py
+
+    row = annotation.to_csv_row()
+    key = _annotator_key(annotation.annotator_name)
+    with h5py.File(h5_path, "r+") as f:
+        root = f.require_group("episode_annotations")
+        if key in root:
+            del root[key]
+        grp = root.create_group(key)
+        grp.attrs["schema_version"] = annotation.schema_version
+        grp.attrs["source"] = "human"
+        grp.attrs["timestamp"] = row["annotated_at"]
+        grp.attrs["annotation"] = json.dumps(row, sort_keys=True)
+
+
+def read_h5_annotations(h5_path: Path) -> dict[str, dict]:
+    """{annotator: row-dict} from the episode file. Groups without a
+    schema_version are upcast to the legacy version; nothing is rewritten."""
+    import json
+
+    import h5py
+
+    from datahive.schema import ANNOTATION_SCHEMA_LEGACY
+
+    out: dict[str, dict] = {}
+    with h5py.File(h5_path, "r") as f:
+        root = f.get("episode_annotations")
+        if root is None:
+            return out
+        for name, grp in root.items():
+            try:
+                row = json.loads(grp.attrs.get("annotation", "{}"))
+            except (TypeError, ValueError):
+                row = {}
+            row["schema_version"] = grp.attrs.get("schema_version") or row.get("schema_version") or ANNOTATION_SCHEMA_LEGACY
+            out[name] = row
+    return out

datahive/attachments.py

@@ -0,0 +1,71 @@
+"""Attachment registry: which attachment_ids are "composed-assembly"
+(multi-stage) attachments, for which stage_reached is meaningful.
+
+Bundled defaults live in datahive/attachments.yaml; a lab can add or
+override entries with samples/attachments.yaml.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+
+import yaml
+
+from datahive.paths import attachments_override_path
+
+_BUNDLED_PATH = Path(__file__).parent / "attachments.yaml"
+
+
+@dataclass
+class AttachmentInfo:
+    attachment_id: str
+    name: str
+    composed_assembly: bool
+    n_stages: int | None = None
+    family: str | None = None
+    timeout: int | None = None
+    success: str | None = None
+    reset: str | None = None
+    stages: list[str] | None = None
+    image: str | None = None
+
+
+@lru_cache(maxsize=1)
+def _bundled() -> dict:
+    return yaml.safe_load(_BUNDLED_PATH.read_text(encoding="utf-8")) or {}
+
+
+def load_registry(samples_root: Path | None = None) -> dict[str, AttachmentInfo]:
+    raw: dict = dict(_bundled())
+    if samples_root is not None:
+        override_path = attachments_override_path(samples_root)
+        if override_path.is_file():
+            override = yaml.safe_load(override_path.read_text(encoding="utf-8")) or {}
+            raw.update(override)
+    return {
+        aid: AttachmentInfo(
+            attachment_id=aid,
+            name=info.get("name", aid),
+            composed_assembly=bool(info.get("composed_assembly", False)),
+            n_stages=info.get("n_stages"),
+            family=info.get("family"),
+            timeout=info.get("timeout"),
+            success=info.get("success"),
+            reset=info.get("reset"),
+            stages=info.get("stages"),
+            image=info.get("image"),
+        )
+        for aid, info in raw.items()
+    }
+
+
+def is_composed_assembly(attachment_id: str, samples_root: Path | None = None) -> bool | None:
+    """Returns True/False if the attachment is known, or None if unknown
+    (caller should treat this as 'no constraint, but warn')."""
+    registry = load_registry(samples_root)
+    info = registry.get(attachment_id)
+    if info is None:
+        return None
+    return info.composed_assembly

datahive/attachments.yaml

@@ -0,0 +1,124 @@
+# Bundled HiveBoard attachment (task) registry.
+#
+# This mirrors the 13 official HiveBoard evaluation conditions from
+# HiveBoard's Evaluation Runner
+# (https://hiveboard-bench.github.io/hivedocs/benchmark/evaluation-runner),
+# so `attachment_id` here means the same thing as `attachment_id` in a
+# HiveBoard submission. `composed_assembly: true` marks attachments made
+# of multiple stages that must be completed in sequence (stage_reached is
+# meaningful for these and required in the trial annotation).
+#
+# Labs can override/extend this list by placing an attachments.yaml with
+# the same shape at the root of their samples/ directory.
+valve_ball:
+  name: Ball valve
+  family: Torque
+  timeout: 60
+  composed_assembly: false
+  success: Rotate the handle 90° from the closed state to the open state.
+  reset: Return the handle to the closed position and confirm that the attachment is fully seated.
+  image: torque_valve_noFriction_3d.png
+valve_ball_ring:
+  name: Ball valve with friction ring
+  family: Torque
+  timeout: 90
+  composed_assembly: false
+  success: Rotate the handle 90° from closed to open with the friction ring fitted.
+  reset: Fit the friction ring, return the handle to closed, and confirm that the attachment is seated.
+  image: torque_valve_noFriction_3d.png
+valve_gate_small:
+  name: Small gate valve
+  family: Torque
+  timeout: 90
+  composed_assembly: false
+  success: Complete one full turn of the valve stem.
+  reset: Return the stem to the marked initial orientation without changing the board position.
+  image: small_valve_3d.png
+valve_gate_large:
+  name: Large gate valve
+  family: Torque
+  timeout: 120
+  composed_assembly: false
+  success: Complete one full turn of the valve stem.
+  reset: Return the stem to the marked initial orientation without changing the board position.
+  image: big_valve_3d.png
+circuit_breaker:
+  name: Circuit breaker
+  family: Torque
+  timeout: 60
+  composed_assembly: false
+  success: Move the toggle to the opposite state and hold it there.
+  reset: Return the toggle to its initial state and confirm that it moves freely.
+  image: switch_3d.png
+light_bulb:
+  name: Light bulb and socket
+  family: Precision
+  timeout: 120
+  composed_assembly: false
+  success: Thread the bulb into the socket until it is seated.
+  reset: Remove the bulb, restore the documented starting pose, and inspect the thread.
+  image: lamp_3d.png
+thread_m8:
+  name: M8 threaded fastener
+  family: Precision
+  timeout: 120
+  composed_assembly: false
+  success: Thread the bolt along the available length.
+  reset: Return the bolt to the documented initial engagement and check that the thread is clear.
+  image: m8_3d.png
+thread_m30:
+  name: M30 threaded fastener
+  family: Precision
+  timeout: 120
+  composed_assembly: false
+  success: Thread the bolt along the available length.
+  reset: Return the bolt to the documented initial engagement and check that the thread is clear.
+  image: m30_3d.png
+peg_insertion:
+  name: Threaded peg insertion
+  family: Precision
+  timeout: 120
+  composed_assembly: false
+  success: Thread the free 8 mm peg into the empty socket until it is seated.
+  reset: Remove the peg and return it to the initial pose next to the empty socket.
+  image: peg_and_hole_3d.png
+button:
+  name: Covered button
+  family: Composed assembly
+  timeout: 60
+  composed_assembly: true
+  n_stages: 2
+  stages: [Open cover, Press button]
+  success: Open the cover and press the button.
+  reset: Close the cover and confirm that the button has returned.
+  image: button_3d.png
+lock:
+  name: Lock and key
+  family: Composed assembly
+  timeout: 180
+  composed_assembly: true
+  n_stages: 3
+  stages: [Grasp key, Insert key vertically, Rotate to unlock]
+  success: Grasp the key, insert it vertically, and rotate it to unlock.
+  reset: Remove the key, return the lock to its initial state, and restore the key pose.
+  image: key_3d.png
+drawer:
+  name: Sliding drawer
+  family: Composed assembly
+  timeout: 120
+  composed_assembly: true
+  n_stages: 3
+  stages: [Grasp handle, Pull open, Push closed]
+  success: Grasp the handle, pull the drawer open, and push it closed.
+  reset: Return the drawer to the fully closed initial position.
+  image: box_3d.png
+shock_absorber:
+  name: Shock absorber
+  family: Composed assembly
+  timeout: 180
+  composed_assembly: true
+  n_stages: 3
+  stages: [Grasp pin, Align with hole, Insert fully]
+  success: Grasp the pin, align it with the hole, and insert it fully.
+  reset: Remove the pin, restore its starting pose, and check both occupied board cells.
+  image: spring_3d.png

datahive/check.py

@@ -0,0 +1,192 @@
+"""Pre-annotation health check for collected episodes.
+
+Checks recording integrity (HDF5 structure, proprioception sample rate,
+monotonic timestamps, timestamp jitter/gaps, and camera video files)
+without requiring an annotation row in trials.csv.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import h5py
+import numpy as np
+
+from datahive.episode import episode_stats, read_header, sample_rate_hz
+from datahive.errors import DatahiveError, EpisodeNotFound
+from datahive.paths import resolve_episode_paths
+from datahive.profile import camera_consistency_problems, load_profile
+from datahive.validate import MIN_SAMPLE_RATE_HZ
+
+
+def check_episode(samples_root: Path, episode_id: str) -> dict[str, Any]:
+    """Runs a quick pre-annotation health check on a recorded episode.
+
+    Checks:
+      1. HDF5 exists and is a valid readable file.
+      2. Episode header attributes parse and contain required hardware metadata.
+      3. /proprioception/timestamp exists, has >= 2 steps, and is monotonically increasing.
+      4. Proprioception sample rate meets the HiveBoard 100 Hz minimum.
+      5. No severe timestamp dropouts/gaps.
+      6. All declared camera video files (.mp4) exist and are non-empty.
+      7. Consistency with robot_profile.yaml (if profile exists).
+
+    Returns a dict with:
+      ok: bool
+      episode_id: str
+      problems: list[str]  # hard failures preventing valid rollout
+      warnings: list[str]  # non-fatal recommendations / notices
+      stats: dict[str, Any]
+    """
+    problems: list[str] = []
+    warnings: list[str] = []
+    stats: dict[str, Any] = {
+        "n_steps": 0,
+        "duration_s": None,
+        "sample_rate_hz": None,
+        "cameras": [],
+        "timestamp_gaps": 0,
+    }
+
+    try:
+        paths = resolve_episode_paths(samples_root, episode_id)
+    except EpisodeNotFound:
+        return {
+            "ok": False,
+            "episode_id": episode_id,
+            "problems": [f"Episode '{episode_id}' not found under {samples_root}."],
+            "warnings": [],
+            "stats": stats,
+        }
+    except Exception as e:
+        return {
+            "ok": False,
+            "episode_id": episode_id,
+            "problems": [f"Could not resolve episode paths: {e}"],
+            "warnings": [],
+            "stats": stats,
+        }
+
+    if not paths.h5.is_file():
+        return {
+            "ok": False,
+            "episode_id": episode_id,
+            "problems": [f"HDF5 file does not exist: {paths.h5}"],
+            "warnings": [],
+            "stats": stats,
+        }
+
+    # 1. HDF5 readability & header parsing
+    try:
+        header = read_header(paths.h5)
+    except Exception as e:
+        return {
+            "ok": False,
+            "episode_id": episode_id,
+            "problems": [f"Could not read header from {paths.h5}: {e}"],
+            "warnings": [],
+            "stats": stats,
+        }
+
+    if not header.low_level.get("mode"):
+        problems.append("Header has no low_level.mode recorded.")
+    if not header.cameras:
+        problems.append("Header lists no cameras.")
+    problems.extend(camera_consistency_problems(header.cameras))
+
+    # 2. Proprioception datasets & timestamp sanity
+    try:
+        with h5py.File(paths.h5, "r") as f:
+            proprio = f.get("proprioception")
+            if proprio is None:
+                problems.append("Missing /proprioception group in HDF5 file.")
+            elif "timestamp" not in proprio:
+                problems.append("Missing /proprioception/timestamp dataset.")
+            else:
+                ts = np.asarray(proprio["timestamp"][:]).reshape(-1)
+                n_steps = int(len(ts))
+                stats["n_steps"] = n_steps
+                if n_steps < 2:
+                    problems.append(f"Episode has only {n_steps} step(s); minimum is 2.")
+                else:
+                    duration = float(ts[-1] - ts[0])
+                    stats["duration_s"] = duration
+                    if duration <= 0:
+                        problems.append(f"Invalid duration: {duration:.3f}s (timestamps not increasing).")
+                    
+                    diffs = np.diff(ts)
+                    negative_diffs = np.sum(diffs < 0)
+                    if negative_diffs > 0:
+                        problems.append(f"Timestamps are not monotonically increasing ({negative_diffs} non-positive step(s)).")
+
+                    median_dt = np.median(diffs)
+                    if median_dt > 0:
+                        rate = 1.0 / median_dt
+                        stats["sample_rate_hz"] = float(rate)
+                        if rate < MIN_SAMPLE_RATE_HZ * 0.99:
+                            problems.append(
+                                f"Proprioception sample rate is {rate:.1f} Hz, below the required {MIN_SAMPLE_RATE_HZ:.0f} Hz."
+                            )
+                        # Check for large gaps (> 3x median dt)
+                        large_gaps = int(np.sum(diffs > 3.0 * median_dt))
+                        stats["timestamp_gaps"] = large_gaps
+                        if large_gaps > 0:
+                            problems.append(f"Detected {large_gaps} severe timestamp gap(s) greater than 3x normal sampling interval.")
+    except Exception as e:
+        problems.append(f"Error inspecting datasets in {paths.h5}: {e}")
+
+    # 3. Camera video files
+    cam_names = []
+    for cam in header.cameras:
+        cname = cam.get("name") or "unnamed"
+        cam_names.append(cname)
+        fname = cam.get("file")
+        if not fname:
+            problems.append(f"Camera '{cname}' video file missing: no mp4 file recorded in header.")
+            continue
+        vpath = paths.h5.parent / fname
+        if not vpath.exists():
+            problems.append(f"Camera '{cname}' video file missing: {fname}")
+        elif vpath.stat().st_size == 0:
+            problems.append(f"Camera '{cname}' video file is empty (0 bytes): {fname}")
+    stats["cameras"] = cam_names
+
+    # 4. Consistency with robot_profile.yaml
+    try:
+        profile = load_profile(samples_root)
+        profile_cams = {c.name for c in profile.cameras}
+        header_cams = set(cam_names)
+        if profile_cams != header_cams:
+            warnings.append(
+                f"Camera names in header ({sorted(header_cams)}) differ from profile ({sorted(profile_cams)})."
+            )
+        profile_joints = set(profile.manipulator.joint_names or [])
+        header_joints = set(header.manipulator.get("joint_names") or [])
+        if profile_joints and header_joints and profile_joints != header_joints:
+            warnings.append(
+                f"Joint names in header ({sorted(header_joints)}) differ from profile ({sorted(profile_joints)})."
+            )
+    except Exception:
+        # Profile might not exist yet if just testing raw episodes; that's fine as a warning
+        pass
+
+    return {
+        "ok": len(problems) == 0,
+        "episode_id": episode_id,
+        "problems": problems,
+        "warnings": warnings,
+        "stats": stats,
+    }
+
+
+def check_all_episodes(samples_root: Path) -> list[dict[str, Any]]:
+    """Runs health check on all episodes found under samples_root."""
+    from datahive.index import Index
+
+    with Index(samples_root) as idx:
+        idx.scan()
+        records = idx.all()
+        episode_ids = [r.episode_id for r in records]
+
+    return [check_episode(samples_root, eid) for eid in episode_ids]