oopsie-data-tools 0.2.1__tar.gz → 0.3__tar.gz

{oopsie_data_tools-0.2.1 → oopsie_data_tools-0.3}/.gitignore

@@ -220,7 +220,8 @@ trajectory_*/
 
 # Claude Code memory file
 CLAUDE.md
-.claude
+.claude/*
+.claude/skills/*
 
 # Jekyll build output
 docs/_site/

{oopsie_data_tools-0.2.1 → oopsie_data_tools-0.3}/PKG-INFO

@@ -1,9 +1,10 @@
 Metadata-Version: 2.4
 Name: oopsie-data-tools
-Version: 0.2.1
+Version: 0.3
 Summary: Collect, annotate, validate and upload robotic manipulation rollout data.
 License-File: LICENSE
 Requires-Python: >=3.8
+Requires-Dist: click>=8.1.3
 Requires-Dist: flask>=3.0.0
 Requires-Dist: h5py>=3.7.0
 Requires-Dist: huggingface-hub>=0.36.2
@@ -24,18 +25,21 @@ Description-Content-Type: text/markdown
 
 Tools for collecting, annotating, inspecting, and converting robotic manipulation rollout data.
 
-This repository currently provides around:
+This repository currently provides tools for:
 
 - HDF5 episode recording (`EpisodeRecorder`)
 - Web annotation workflows
 - In-the-loop annotation during policy rollout
 
+as well as all the necessary utilities to validate, inspect, and upload Oopsie-Data to the official repositories.
+
+[Sign up today](https://forms.gle/9arwZHAvRjvbozoT7) and start contributing!
+
 ---
 
 For detailed explanations on how to use our tooling and contribute to the project, please visit [our website](https://oopsie-data.com/).
 
 For an overview of the steps necessary to integrate the tooling into your workflow and to contribute data to the official Oopsie Data repositories, check out [our quickstart guide](https://oopsie-data.com/quickstart).
-You can also use the information in AI_CONTEXT.md to guide a coding agent through the setup.
 
 ## Repository structure
 

{oopsie_data_tools-0.2.1 → oopsie_data_tools-0.3}/README.md

@@ -2,18 +2,21 @@
 
 Tools for collecting, annotating, inspecting, and converting robotic manipulation rollout data.
 
-This repository currently provides around:
+This repository currently provides tools for:
 
 - HDF5 episode recording (`EpisodeRecorder`)
 - Web annotation workflows
 - In-the-loop annotation during policy rollout
 
+as well as all the necessary utilities to validate, inspect, and upload Oopsie-Data to the official repositories.
+
+[Sign up today](https://forms.gle/9arwZHAvRjvbozoT7) and start contributing!
+
 ---
 
 For detailed explanations on how to use our tooling and contribute to the project, please visit [our website](https://oopsie-data.com/).
 
 For an overview of the steps necessary to integrate the tooling into your workflow and to contribute data to the official Oopsie Data repositories, check out [our quickstart guide](https://oopsie-data.com/quickstart).
-You can also use the information in AI_CONTEXT.md to guide a coding agent through the setup.
 
 ## Repository structure
 

{oopsie_data_tools-0.2.1 → oopsie_data_tools-0.3}/oopsie_data_tools/annotation_tool/annotator_server.py

@@ -14,17 +14,16 @@ Tool can also be launched as a standalone module to browse and annotate existing
 
 from __future__ import annotations
 
-import argparse
 import json
 import logging
-import os
+import socket
 import threading
 import webbrowser
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
-from urllib.parse import quote, unquote
+from urllib.parse import quote
 
 import h5py
 from flask import Flask, abort, jsonify, request, send_file
@@ -257,8 +256,6 @@ def api_get_annotation(sample_id: str):
 def api_save_annotation_json() -> Any:
     """Save annotation with ``sample_id`` in the JSON body (avoids slashes in URL paths)."""
     payload = _json_payload()
-    if not isinstance(payload, dict):
-        return jsonify({"error": "invalid JSON body"}), 400
     sample_id = str(payload.get("sample_id", "")).strip()
     if not sample_id:
         return jsonify({"error": "sample_id is required"}), 400
@@ -348,15 +345,19 @@ class H5PathError(ValueError):
 
     def __init__(self, message: str, status: int = 400) -> None:
         super().__init__(message)
-        self.message = message
         self.status = status
 
 
+@app.errorhandler(H5PathError)
+def _h5_path_error(e: H5PathError):
+    return jsonify({"error": str(e)}), e.status
+
+
 def _safe_h5_path_from_query(samples_root: Path) -> Path:
     raw = str(request.args.get("path", "")).strip()
     if not raw:
         raise H5PathError("path query parameter is required", 400)
-    rel = Path(unquote(raw))
+    rel = Path(raw)
     target = (samples_root.resolve() / rel).resolve()
     try:
         target.relative_to(samples_root.resolve())
@@ -443,21 +444,9 @@ def _h5_annotation_tick_level(h5f: h5py.File, annotator_name: str) -> int:
     level = _annotation_tick_level(ann)
     if level > 0:
         return level
-    raw = _read_h5_attr(ea, "failure_annotation", "")
-    raw_s = str(raw or "").strip()
-    if raw_s:
-        try:
-            parsed = json.loads(raw_s)
-            if isinstance(parsed, dict):
-                if len(parsed) > 0:
-                    return 1
-            elif bool(parsed):
-                return 1
-        except Exception:
-            return 1
-    if annotator_name in ea.keys() and isinstance(ea[annotator_name], h5py.Group):
-        if _annotator_subgroup_looks_annotated(ea[annotator_name]):
-            return 1
+    sub = ea.get(annotator_name)
+    if isinstance(sub, h5py.Group) and _annotator_subgroup_looks_annotated(sub):
+        return 1
     return 0
 
 
@@ -539,6 +528,49 @@ def _has_other_human_annotation(h5f: h5py.File, annotator_name: str) -> bool:
     return False
 
 
+_H5_META_CACHE: dict[tuple[str, str], tuple[tuple[float, int], dict[str, Any]]] = {}
+_H5_META_LOCK = threading.Lock()
+_H5_META_CACHE_MAX = 5000
+
+
+def _h5_meta(path: Path, annotator_name: str) -> dict[str, Any]:
+    """Annotation summary of one episode, cached on (mtime, size).
+
+    The browser polls ``/api/h5/list`` every 1.5 s per open tab, so without this every
+    poll would reopen every episode in the tree.
+    """
+    try:
+        st = path.stat()
+    except OSError:
+        return {"tick_level": 0, "annotated_by_others": False, "annotation": None}
+    key = (str(path), annotator_name)
+    stamp = (st.st_mtime, st.st_size)
+    with _H5_META_LOCK:
+        hit = _H5_META_CACHE.get(key)
+        if hit is not None and hit[0] == stamp:
+            return hit[1]
+
+    meta: dict[str, Any] = {"tick_level": 0, "annotated_by_others": False, "annotation": None}
+    try:
+        with h5py.File(path, "r") as h5f:
+            meta["tick_level"] = _h5_annotation_tick_level(h5f, annotator_name)
+            meta["annotated_by_others"] = _has_other_human_annotation(h5f, annotator_name)
+            ea = h5f.get("episode_annotations")
+            # Only the current annotator's OWN subgroup — never the legacy
+            # episode-level failure_annotation fallback, which isn't annotator-scoped
+            # and could surface someone else's labels (#27).
+            if isinstance(ea, h5py.Group) and isinstance(ea.get(annotator_name), h5py.Group):
+                meta["annotation"] = _read_existing_annotation_dict(ea, annotator_name)
+    except Exception:
+        meta = {"tick_level": 0, "annotated_by_others": False, "annotation": None}
+
+    with _H5_META_LOCK:
+        if len(_H5_META_CACHE) >= _H5_META_CACHE_MAX:
+            _H5_META_CACHE.clear()
+        _H5_META_CACHE[key] = (stamp, meta)
+    return meta
+
+
 @app.get("/api/h5/list")
 def api_h5_list():
     rt = _get_runtime()
@@ -548,15 +580,9 @@ def api_h5_list():
         if not p.is_file():
             continue
         rel = p.resolve().relative_to(root).as_posix()
-        tick_level = 0
-        others = False
-        try:
-            with h5py.File(p, "r") as h5f:
-                tick_level = _h5_annotation_tick_level(h5f, rt.cfg.annotator_name)
-                others = _has_other_human_annotation(h5f, rt.cfg.annotator_name)
-        except Exception:
-            tick_level = 0
-            others = False
+        meta = _h5_meta(p, rt.cfg.annotator_name)
+        tick_level = meta["tick_level"]
+        others = meta["annotated_by_others"]
         annotated = tick_level >= 1
         entries.append(
             {
@@ -593,18 +619,8 @@ def api_recent_annotations():
     for p in files:
         if len(recent) >= limit:
             break
-        try:
-            with h5py.File(p, "r") as h5f:
-                ea = h5f.get("episode_annotations")
-                if not isinstance(ea, h5py.Group):
-                    continue
-                # Only the current annotator's OWN subgroup — never the legacy
-                # episode-level failure_annotation fallback, which isn't
-                # annotator-scoped and could surface someone else's labels (#27).
-                if ann_name not in ea.keys() or not isinstance(ea[ann_name], h5py.Group):
-                    continue
-                ann = _read_existing_annotation_dict(ea, ann_name)
-        except Exception:
+        ann = _h5_meta(p, ann_name)["annotation"]
+        if ann is None:
             continue
         # Anything but a clean success has details worth copying — a suboptimal-execution
         # description is as reusable as a failure's.
@@ -632,10 +648,7 @@ def api_recent_annotations():
 def api_h5_sample():
     rt = _get_runtime()
     samples_root = rt.cfg.samples_dir.resolve()
-    try:
-        h5_path = _safe_h5_path_from_query(samples_root)
-    except H5PathError as e:
-        return jsonify({"error": e.message}), e.status
+    h5_path = _safe_h5_path_from_query(samples_root)
 
     video_urls: dict[str, str] = {}
     existing_annotation: dict[str, Any] = {}
@@ -705,14 +718,9 @@ def api_h5_sample():
 def api_h5_save_annotation():
     rt = _get_runtime()
     samples_root = rt.cfg.samples_dir.resolve()
-    try:
-        h5_path = _safe_h5_path_from_query(samples_root)
-    except H5PathError as e:
-        return jsonify({"error": e.message}), e.status
+    h5_path = _safe_h5_path_from_query(samples_root)
 
     payload = _json_payload()
-    if not isinstance(payload, dict):
-        return jsonify({"error": "invalid JSON body"}), 400
     error = validate_annotation_payload(payload)
     if error:
         return jsonify({"error": error}), 400
@@ -727,10 +735,8 @@ def api_h5_save_annotation():
             ea = f.require_group("episode_annotations")
             ag = ea.require_group(ann["annotator"])
             write_annotation_attrs(ag, ann)
-    except OSError as e:
-        return jsonify({"error": f"could not write HDF5: {e}"}), 500
     except Exception as e:
-        return jsonify({"error": f"could not update annotation: {e}"}), 500
+        return jsonify({"error": f"could not write annotation: {e}"}), 500
 
     return jsonify({"status": "saved", "annotation": ann})
 
@@ -740,10 +746,7 @@ def api_h5_set_instruction():
     """Overwrite an episode's ``language_instruction`` root attr (issue #31)."""
     rt = _get_runtime()
     samples_root = rt.cfg.samples_dir.resolve()
-    try:
-        h5_path = _safe_h5_path_from_query(samples_root)
-    except H5PathError as e:
-        return jsonify({"error": e.message}), e.status
+    h5_path = _safe_h5_path_from_query(samples_root)
 
     instruction = str(_json_payload().get("instruction", "")).strip()
     if not instruction:
@@ -752,10 +755,8 @@ def api_h5_set_instruction():
     try:
         with h5py.File(h5_path, "r+") as f:
             f.attrs["language_instruction"] = instruction
-    except OSError as e:
-        return jsonify({"error": f"could not write HDF5: {e}"}), 500
     except Exception as e:
-        return jsonify({"error": f"could not update instruction: {e}"}), 500
+        return jsonify({"error": f"could not write instruction: {e}"}), 500
 
     return jsonify({"status": "saved", "language_instruction": instruction})
 
@@ -770,21 +771,29 @@ def _load_template(filename: str) -> str:
     return path.read_text(encoding="utf-8")
 
 
+def _port_in_use(port: int, host: str = "localhost") -> bool:
+    """Whether something is already listening on ``host:port``.
+
+    Connecting rather than test-binding: a bind probe also trips on sockets in TIME_WAIT,
+    which werkzeug's SO_REUSEADDR would have accepted anyway.
+    """
+    try:
+        with socket.create_connection((host, port), timeout=0.5):
+            return True
+    except OSError:
+        return False
+
+
 def run_server(
     samples_dir: Path,
     annotator_name: str,
     port: int = 5001,
     open_browser: bool = True,
     with_rollouts: bool = False,
-    debug: bool = False,
 ) -> int:
     """Configure the runtime and serve the annotation UI (blocks until interrupted).
 
-    Shared by this module's ``main()`` and the ``oopsie-data annotate`` CLI command.
-
-    ``debug`` is off by default and should stay off outside development: it enables the
-    Werkzeug debugger, whose interactive console executes arbitrary Python on request, and
-    the reloader, which re-executes this process and so loses any state answered at startup.
+    Backs the ``oopsie-data annotate`` CLI command.
     """
     samples_dir = Path(samples_dir).resolve()
     samples_dir.mkdir(parents=True, exist_ok=True)
@@ -794,57 +803,21 @@ def run_server(
         browse_only=bool(not with_rollouts),
     )
 
-    # WERKZEUG_RUN_MAIN marks the reloader's child process; only the parent should open a
-    # browser. Only reachable with debug=True, but harmless and correct either way.
-    if open_browser and os.environ.get("WERKZEUG_RUN_MAIN") != "true":
+    # Checked before the browser opens: werkzeug would otherwise fail its own bind a
+    # moment later, leaving a tab pointed at whatever already owns the port.
+    if _port_in_use(port):
+        logging.getLogger(__name__).error(
+            "Port %d is already in use — another annotator is probably still running "
+            "(open http://localhost:%d/ to check). Stop it, or pass --port with a free port.",
+            port,
+            port,
+        )
+        return 1
+
+    if open_browser:
         webbrowser.open(f"http://localhost:{port}/")
 
     logging.getLogger("werkzeug").setLevel(logging.WARNING)
 
-    app.run(host="localhost", port=port, debug=debug)
+    app.run(host="localhost", port=port)
     return 0
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description="Annotator server")
-    parser.add_argument(
-        "--samples-dir",
-        type=Path,
-        default=Path("samples"),
-        help="Directory containing saved MP4s (and HDF5 episodes) for this session",
-    )
-    parser.add_argument("--port", type=int, default=5001)
-    parser.add_argument(
-        "--annotator-name",
-        type=str,
-        required=True,
-        help="Annotator name to stamp into saved annotations",
-    )
-    parser.add_argument("--no-browser", action="store_true")
-    parser.add_argument(
-        "--with-rollouts",
-        action="store_true",
-        help="HDF5 browser + annotation form + rollouts",
-    )
-    parser.add_argument(
-        "--debug",
-        action="store_true",
-        help=(
-            "Development only: enable the Werkzeug reloader and debugger. The debugger's "
-            "console executes arbitrary Python on request — never use it on a shared machine."
-        ),
-    )
-    args = parser.parse_args()
-
-    return run_server(
-        samples_dir=args.samples_dir,
-        annotator_name=args.annotator_name,
-        port=args.port,
-        open_browser=not args.no_browser,
-        with_rollouts=args.with_rollouts,
-        debug=args.debug,
-    )
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

{oopsie_data_tools-0.2.1 → oopsie_data_tools-0.3}/oopsie_data_tools/annotation_tool/episode_recorder.py

@@ -17,7 +17,7 @@ from oopsie_data_tools.annotation_tool.annotation_schema import (
     success_to_outcome,
     write_annotation_attrs,
 )
-from oopsie_data_tools.utils.contributor_config import read_contributor_config
+from oopsie_data_tools.utils import contributor_config
 from oopsie_data_tools.utils.robot_profile.robot_profile import RobotProfile, robot_profile_to_json
 from oopsie_data_tools.utils.robot_profile.rotation_utils import ActionQuatConversion
 from oopsie_data_tools.utils.validation.episode_data import EpisodeData, VideoInfo
@@ -88,7 +88,7 @@ class EpisodeRecorder:
             robot_profile (RobotProfile): Robot profile.
             data_root_dir (str): Base output directory for saved artifacts.
             operator_name (str): Name of the operator recording the episode.
-            session_name (str | None): Optional unique session name
+            resume_session_name (str | None): Optional unique session name
 
         Raises:
             ValueError: If ``data_root_dir`` is not a valid directory.
@@ -134,7 +134,7 @@ class EpisodeRecorder:
         # AttributeError on save_fname — an undocumented ordering requirement.
         self.reset_episode_recorder()
 
-        self.lab_id, _ = read_contributor_config()
+        self.lab_id, _ = contributor_config.read_contributor_config()
 
     def reset_episode_recorder(self) -> None:
         """Reset the buffers and start a new episode."""
@@ -180,24 +180,13 @@ class EpisodeRecorder:
         # Buffer frames for each configured camera (if available)
         for cam in self.camera_names:
             frame = self._get_camera_frame(observation["image_observation"], cam)
-            if frame is not None:
-                self.frames[cam].append(np.asarray(frame, dtype=np.uint8))
+            self.frames[cam].append(np.asarray(frame, dtype=np.uint8))
 
         # Buffer timestep data
         step_data = {"robot_state": {}, "action_dict": {}}
         for key in self.robot_profile.robot_state_keys:
             step_data["robot_state"][key] = np.asarray(robot_state[key], dtype=np.float32)
-        step_data["action_dict"] = {
-            "cartesian_position": action.get("cartesian_position", None),
-            "cartesian_velocity": action.get("cartesian_velocity", None),
-            "joint_position": action.get("joint_position", None),
-            "joint_velocity": action.get("joint_velocity", None),
-            "base_position": action.get("base_position", None),
-            "base_velocity": action.get("base_velocity", None),
-            "gripper_velocity": action.get("gripper_velocity", None),
-            "gripper_position": action.get("gripper_position", None),
-            "gripper_binary": action.get("gripper_binary", None),
-        }
+        step_data["action_dict"] = {k: action.get(k) for k in sorted(VALID_ACTION_KEYS)}
         self.timesteps.append(step_data)
 
     def _save_videos(self) -> dict[str, str]:
@@ -252,7 +241,7 @@ class EpisodeRecorder:
         """Persist the currently buffered episode to disk.
 
         Args:
-            metadata (dict[str, Any]): Save metadata containing language and
+            data (dict[str, Any]): Save metadata containing language and
                 annotation fields.
 
         Returns:
@@ -349,14 +338,6 @@ class EpisodeRecorder:
                 f"action must not be empty. Valid keys: {VALID_ACTION_KEYS}. Please pass it in your record_step() call. Double check that the passed keys match the robot profile you initialized the recorder with."
             )
 
-        # Make sure the action keys are valid
-        invalid_action = set(action.keys()) - set(VALID_ACTION_KEYS)
-        if invalid_action:
-            raise ValueError(
-                f"action contains unrecognized keys: {sorted(invalid_action)}. "
-                f"Valid keys: {VALID_ACTION_KEYS}"
-            )
-
         # Make sure the action keys agree between robot_profile and the action dict
         profile_action_keys = set(self.robot_profile.action_space)
         action_keys = set(action.keys())
@@ -429,7 +410,7 @@ class EpisodeRecorder:
 
         Args:
             path (Path): Target HDF5 file path.
-            metadata (dict[str, Any]): Normalized metadata payload.
+            data (dict[str, Any]): Normalized metadata payload.
 
         Returns:
             None: This method only performs file I/O side effects.
@@ -562,22 +543,8 @@ class EpisodeRecorder:
     def _get_camera_frame(
         self, observation: dict[str, Any], cam_name: str
     ) -> np.ndarray | None:
-        """Extract a camera frame from supported observation key patterns.
-
-        Args:
-            observation (dict[str, Any]): Observation dictionary for one
-                timestep.
-            cam_name (str): Canonical camera name to resolve.
-
-        Returns:
-            np.ndarray | None: Camera frame array when available, otherwise
-                ``None``.
-        """
-        candidates = (cam_name, f"image_{cam_name}", f"{cam_name}_image")
-        for key in candidates:
-            if key in observation:
-                return np.asarray(observation[key])
-        return None
+        """The frame for *cam_name*; the key is guaranteed present by the step check."""
+        return np.asarray(observation[cam_name])
 
     def _validate_pre_save(self, data: dict[str, Any]) -> None:
         """Perform final validation checks before saving the episode.