ls-algorithm-plugin-sdk 0.3.4__tar.gz → 0.3.6__tar.gz

{ls_algorithm_plugin_sdk-0.3.4 → ls_algorithm_plugin_sdk-0.3.6}/PKG-INFO

@@ -1,11 +1,12 @@
 Metadata-Version: 2.4
 Name: ls-algorithm-plugin-sdk
-Version: 0.3.4
+Version: 0.3.6
 Summary: Protocol-independent runtime SDK for dataset algorithms
 Author: Ling Robotics
 License: Proprietary
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+Requires-Dist: pyarrow>=14
 Provides-Extra: service
 Requires-Dist: fastapi<1,>=0.110; extra == "service"
 Requires-Dist: pydantic<3,>=2.0; extra == "service"
@@ -40,6 +41,16 @@ algorithm-plugin run camera-space-mano \
 
 `--gpu-ids` 和 `--parameters` 可省略，也可用 `--request-file` 读取完整请求 JSON。
 
+## 数据集裁剪
+
+将每个 episode 裁剪为最多指定分钟数，并同步更新 Parquet、视频、传感器 CSV 和元数据：
+
+```bash
+algorithm-plugin cut /data/input_dataset /data/output_dataset --minutes 1.5
+```
+
+支持 LeRobot v2/v3 的 `episode_*`、`file-*` 和多数据源目录布局。视频裁剪需要系统已安装 `ffmpeg` 与 `ffprobe`。
+
 ## 配置现有仓库
 
 目标算法仓库必须满足：

{ls_algorithm_plugin_sdk-0.3.4 → ls_algorithm_plugin_sdk-0.3.6}/README.md

@@ -25,6 +25,16 @@ algorithm-plugin run camera-space-mano \
 
 `--gpu-ids` 和 `--parameters` 可省略，也可用 `--request-file` 读取完整请求 JSON。
 
+## 数据集裁剪
+
+将每个 episode 裁剪为最多指定分钟数，并同步更新 Parquet、视频、传感器 CSV 和元数据：
+
+```bash
+algorithm-plugin cut /data/input_dataset /data/output_dataset --minutes 1.5
+```
+
+支持 LeRobot v2/v3 的 `episode_*`、`file-*` 和多数据源目录布局。视频裁剪需要系统已安装 `ffmpeg` 与 `ffprobe`。
+
 ## 配置现有仓库
 
 目标算法仓库必须满足：

{ls_algorithm_plugin_sdk-0.3.4 → ls_algorithm_plugin_sdk-0.3.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ls-algorithm-plugin-sdk"
-version = "0.3.4"
+version = "0.3.6"
 description = "Protocol-independent runtime SDK for dataset algorithms"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -12,7 +12,9 @@ license = {text = "Proprietary"}
 authors = [
   {name = "Ling Robotics"},
 ]
-dependencies = []
+dependencies = [
+  "pyarrow>=14",
+]
 
 [project.optional-dependencies]
 service = [

{ls_algorithm_plugin_sdk-0.3.4 → ls_algorithm_plugin_sdk-0.3.6}/src/algorithm_plugin_sdk/cli.py

@@ -5,6 +5,7 @@ import logging
 import sys
 
 from .cli_impl import configure as configure_command
+from .cli_impl import cut as cut_command
 from .cli_impl import run as run_command
 from .cli_impl import serve as serve_command
 
@@ -22,6 +23,9 @@ def build_parser() -> argparse.ArgumentParser:
     configure_command.configure_parser(
         subs.add_parser("configure", help="configure an existing Algorithm repository for systemd")
     )
+    cut_command.configure_parser(
+        subs.add_parser("cut", help="cut each dataset episode to a maximum duration")
+    )
     return parser
 
 

ls_algorithm_plugin_sdk-0.3.6/src/algorithm_plugin_sdk/cli_impl/cut.py

@@ -0,0 +1,551 @@
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import re
+import shutil
+import subprocess
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+
+EPISODE_FILE_PATTERN = re.compile(r"episode[-_](\d+)")
+FILE_PATTERN = re.compile(r"file-(\d+)")
+VIDEO_SUFFIXES = {".avi", ".mkv", ".mov", ".mp4", ".webm"}
+
+
+def positive_minutes(value: str) -> float:
+    try:
+        minutes = float(value)
+    except ValueError as exc:
+        raise argparse.ArgumentTypeError("minutes must be a number") from exc
+    if minutes <= 0:
+        raise argparse.ArgumentTypeError("minutes must be greater than zero")
+    return minutes
+
+
+def configure_parser(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("input_dataset", type=Path)
+    parser.add_argument("output_path", type=Path)
+    parser.add_argument(
+        "--minutes",
+        type=positive_minutes,
+        required=True,
+        help="maximum duration retained from the start of each episode",
+    )
+    parser.set_defaults(handler=execute)
+
+
+def _source_name(path: Path, data_root: Path) -> str:
+    relative = path.relative_to(data_root)
+    parts = []
+    for part in relative.parts[:-1]:
+        if part.startswith("chunk-"):
+            break
+        parts.append(part)
+    return "/".join(parts)
+
+
+def _replace_column(table: pa.Table, name: str, values: list[int]) -> pa.Table:
+    index = table.column_names.index(name)
+    return table.set_column(
+        index,
+        name,
+        pa.array(values, type=table.schema.field(name).type),
+    )
+
+
+def _read_json_lines(path: Path) -> list[dict[str, Any]]:
+    if not path.is_file():
+        return []
+    return [
+        json.loads(line)
+        for line in path.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+
+def _write_json_lines(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.write_text(
+        "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows),
+        encoding="utf-8",
+    )
+
+
+def _episode_tables(root: Path) -> list[tuple[Path, pa.Table]]:
+    result = []
+    for path in sorted((root / "meta" / "episodes").glob("**/*.parquet")):
+        result.append((path, pq.read_table(path)))
+    return result
+
+
+def _episode_rows(tables: list[tuple[Path, pa.Table]]) -> list[dict[str, Any]]:
+    return [row for _, table in tables for row in table.to_pylist()]
+
+
+def _table_time_values(table: pa.Table, fps: float) -> list[float]:
+    if "timestamp" in table.column_names:
+        return [float(value) for value in table["timestamp"].to_pylist()]
+    if "frame_index" in table.column_names:
+        return [float(value) / fps for value in table["frame_index"].to_pylist()]
+    return [index / fps for index in range(table.num_rows)]
+
+
+def _candidate_lengths(
+    paths: list[Path], data_root: Path, seconds: float, fps: float
+) -> dict[int, int]:
+    counts: dict[str, dict[int, int]] = defaultdict(lambda: defaultdict(int))
+    first_timestamps: dict[tuple[str, int], float] = {}
+    for path in paths:
+        schema = pq.read_schema(path)
+        if "episode_index" not in schema.names:
+            raise ValueError(f"data parquet lacks episode_index: {path}")
+        columns = ["episode_index"]
+        for candidate in ("timestamp", "frame_index"):
+            if candidate in schema.names:
+                columns.append(candidate)
+                break
+        table = pq.read_table(path, columns=columns)
+        source = _source_name(path, data_root)
+        episodes = [int(value) for value in table["episode_index"].to_pylist()]
+        timestamps = _table_time_values(table, fps)
+        for episode, timestamp in zip(episodes, timestamps):
+            key = (source, episode)
+            first = first_timestamps.setdefault(key, timestamp)
+            if timestamp <= first + seconds + 1e-9:
+                counts[source][episode] += 1
+
+    episodes = {episode for values in counts.values() for episode in values}
+    if not episodes:
+        raise ValueError("no episode rows found in data parquet files")
+    lengths = {
+        episode: min(
+            values[episode] for values in counts.values() if episode in values
+        )
+        for episode in episodes
+    }
+    return lengths
+
+
+def _probe_frames(path: Path) -> int:
+    command = [
+        "ffprobe",
+        "-v",
+        "error",
+        "-select_streams",
+        "v:0",
+        "-show_entries",
+        "stream=nb_frames",
+        "-of",
+        "json",
+        str(path),
+    ]
+    try:
+        result = subprocess.run(
+            command, check=True, capture_output=True, text=True
+        )
+        stream = json.loads(result.stdout)["streams"][0]
+        value = stream.get("nb_frames")
+        if value not in (None, "N/A"):
+            return int(value)
+
+        command.insert(command.index("-show_entries"), "-count_frames")
+        command[command.index("stream=nb_frames")] = "stream=nb_read_frames"
+        result = subprocess.run(
+            command, check=True, capture_output=True, text=True
+        )
+        value = json.loads(result.stdout)["streams"][0].get("nb_read_frames")
+        if value not in (None, "N/A"):
+            return int(value)
+    except (subprocess.CalledProcessError, KeyError, ValueError, json.JSONDecodeError) as exc:
+        raise RuntimeError(f"cannot inspect video frames: {path}") from exc
+    raise RuntimeError(f"video does not report a frame count: {path}")
+
+
+def _chunk_index(path: Path) -> int | None:
+    for part in path.parts:
+        if part.startswith("chunk-"):
+            try:
+                return int(part.removeprefix("chunk-"))
+            except ValueError:
+                return None
+    return None
+
+
+def _media_episodes(path: Path, root: Path, rows: list[dict[str, Any]]) -> list[int]:
+    direct = EPISODE_FILE_PATTERN.search(path.stem)
+    if direct:
+        return [int(direct.group(1))]
+
+    file_match = FILE_PATTERN.search(path.stem)
+    if not file_match:
+        raise ValueError(f"cannot determine episode for media file: {path}")
+    file_index = int(file_match.group(1))
+    chunk_index = _chunk_index(path)
+    relative = path.relative_to(root).as_posix()
+    kind = relative.split("/", 1)[0]
+    matches = []
+    for row in rows:
+        episode = int(row.get("episode_index", -1))
+        for key, value in row.items():
+            if not key.startswith(f"{kind}/") or not key.endswith("/file_index"):
+                continue
+            feature = key[len(kind) + 1 : -len("/file_index")]
+            chunk_key = key[: -len("file_index")] + "chunk_index"
+            if (
+                int(value) == file_index
+                and (chunk_index is None or int(row.get(chunk_key, -1)) == chunk_index)
+                and feature in relative
+            ):
+                matches.append(episode)
+                break
+    return sorted(set(matches)) or [file_index]
+
+
+def _copy_static_tree(source: Path, output: Path) -> None:
+    skipped = {"data", "videos", "depths", "sensors"}
+
+    def ignore(directory: str, names: list[str]) -> set[str]:
+        return skipped.intersection(names) if Path(directory) == source else set()
+
+    shutil.copytree(source, output, ignore=ignore)
+
+
+def _write_data(
+    paths: list[Path],
+    source_root: Path,
+    output_root: Path,
+    limits: dict[int, int],
+    fps: float,
+) -> tuple[dict[str, dict[int, dict[str, float | int]]], dict[str, int]]:
+    seen: dict[str, dict[int, int]] = defaultdict(lambda: defaultdict(int))
+    global_index: dict[str, int] = defaultdict(int)
+    metrics: dict[str, dict[int, dict[str, float | int]]] = defaultdict(dict)
+
+    for path in paths:
+        table = pq.read_table(path)
+        source = _source_name(path, source_root)
+        episodes = [int(value) for value in table["episode_index"].to_pylist()]
+        timestamps = _table_time_values(table, fps)
+        keep = []
+        frame_indexes = []
+        indexes = []
+        for row_index, (episode, timestamp) in enumerate(zip(episodes, timestamps)):
+            ordinal = seen[source][episode]
+            seen[source][episode] += 1
+            if ordinal >= limits[episode]:
+                continue
+            keep.append(row_index)
+            frame_indexes.append(ordinal)
+            indexes.append(global_index[source])
+            global_index[source] += 1
+            episode_metrics = metrics[source].setdefault(
+                episode,
+                {
+                    "from_index": indexes[-1],
+                    "to_index": indexes[-1] + 1,
+                    "from_timestamp": timestamp,
+                    "to_timestamp": timestamp,
+                    "length": 0,
+                },
+            )
+            episode_metrics["to_index"] = indexes[-1] + 1
+            episode_metrics["to_timestamp"] = timestamp
+            episode_metrics["length"] = int(episode_metrics["length"]) + 1
+
+        trimmed = table.take(pa.array(keep, type=pa.int64()))
+        if "frame_index" in trimmed.column_names:
+            trimmed = _replace_column(trimmed, "frame_index", frame_indexes)
+        if "index" in trimmed.column_names:
+            trimmed = _replace_column(trimmed, "index", indexes)
+        destination = output_root / path.relative_to(source_root.parent)
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        pq.write_table(trimmed, destination)
+    return metrics, dict(global_index)
+
+
+def _copy_or_cut_video(source: Path, destination: Path, frames: int) -> None:
+    source_frames = _probe_frames(source)
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    if source_frames <= frames:
+        shutil.copy2(source, destination)
+        return
+    depth_video = "depth" in source.as_posix().lower()
+    codec = ["-c:v", "libx264rgb", "-crf", "0"] if depth_video else [
+        "-c:v",
+        "libx264",
+        "-crf",
+        "20",
+    ]
+    subprocess.run(
+        [
+            "ffmpeg",
+            "-y",
+            "-loglevel",
+            "error",
+            "-i",
+            str(source),
+            "-map",
+            "0:v:0",
+            "-frames:v",
+            str(frames),
+            *codec,
+            "-preset",
+            "veryfast",
+            "-an",
+            str(destination),
+        ],
+        check=True,
+    )
+    actual = _probe_frames(destination)
+    if actual != frames:
+        raise RuntimeError(
+            f"video frame count mismatch for {destination}: expected {frames}, got {actual}"
+        )
+
+
+def _sensor_episode(path: Path, rows: list[dict[str, Any]], root: Path) -> int:
+    match = EPISODE_FILE_PATTERN.search(path.stem)
+    if match:
+        return int(match.group(1))
+    relative = path.relative_to(root).as_posix()
+    matches = {
+        int(row["episode_index"])
+        for row in rows
+        for key, value in row.items()
+        if key.startswith("sensors/") and key.endswith("/path") and value == relative
+    }
+    if len(matches) != 1:
+        raise ValueError(f"cannot determine episode for sensor file: {path}")
+    return matches.pop()
+
+
+def _copy_sensors(
+    source: Path,
+    output: Path,
+    episode_rows: list[dict[str, Any]],
+    end_times: dict[int, float],
+) -> dict[str, dict[str, float | int]]:
+    metrics = {}
+    sensor_root = source / "sensors"
+    if not sensor_root.exists():
+        return metrics
+    for path in sorted(sensor_root.glob("**/*")):
+        if not path.is_file():
+            continue
+        destination = output / path.relative_to(source)
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        if path.suffix.lower() != ".csv":
+            shutil.copy2(path, destination)
+            continue
+        episode = _sensor_episode(path, episode_rows, source)
+        with path.open(newline="", encoding="utf-8") as stream:
+            reader = csv.DictReader(stream)
+            rows = list(reader)
+            fieldnames = reader.fieldnames or []
+        if "timestamp" in fieldnames and rows:
+            first_timestamp = float(rows[0]["timestamp"])
+            maximum = first_timestamp + end_times[episode] + 1e-9
+            rows = [row for row in rows if float(row["timestamp"]) <= maximum]
+        if "imu_index" in fieldnames:
+            for index, row in enumerate(rows):
+                row["imu_index"] = str(index)
+        with destination.open("w", newline="", encoding="utf-8") as stream:
+            writer = csv.DictWriter(stream, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(rows)
+        relative = path.relative_to(source).as_posix()
+        metrics[relative] = {
+            "from_index": 0,
+            "to_index": len(rows),
+            "from_timestamp": float(rows[0]["timestamp"]) if rows and "timestamp" in fieldnames else 0.0,
+            "to_timestamp": float(rows[-1]["timestamp"]) if rows and "timestamp" in fieldnames else 0.0,
+        }
+    return metrics
+
+
+def _update_episode_row(
+    row: dict[str, Any],
+    limits: dict[int, int],
+    offsets: dict[int, int],
+    source_metrics: dict[str, dict[int, dict[str, float | int]]],
+    sensor_metrics: dict[str, dict[str, float | int]],
+    end_times: dict[int, float],
+) -> None:
+    episode = int(row["episode_index"])
+    length = limits[episode]
+    for key in list(row):
+        if key == "length" or key.endswith("_length"):
+            row[key] = length
+    for key in ("duration", "duration_s"):
+        if key in row:
+            row[key] = min(float(row[key]), end_times[episode])
+    if "dataset_from_index" in row:
+        row["dataset_from_index"] = offsets[episode]
+    if "dataset_to_index" in row:
+        row["dataset_to_index"] = offsets[episode] + length
+
+    for source, episodes in source_metrics.items():
+        values = episodes.get(episode)
+        if values is None:
+            continue
+        prefix = f"data/{source}/" if source else "data/"
+        for field in ("from_index", "to_index", "from_timestamp", "to_timestamp"):
+            key = prefix + field
+            if key in row:
+                row[key] = values[field]
+
+    for key, value in list(row.items()):
+        if key.startswith(("videos/", "depths/")) and key.endswith("/to_timestamp"):
+            prefix = key[: -len("to_timestamp")]
+            row[key] = float(row.get(prefix + "from_timestamp", 0.0)) + end_times[episode]
+        if key.startswith("sensors/") and key.endswith("/path") and value in sensor_metrics:
+            prefix = key[: -len("path")]
+            for field, metric_value in sensor_metrics[value].items():
+                metric_key = prefix + field
+                if metric_key in row:
+                    row[metric_key] = metric_value
+
+
+def cut_dataset(input_dataset: Path, output_path: Path, minutes: float) -> None:
+    source = input_dataset.expanduser().resolve()
+    output = output_path.expanduser().resolve()
+    if not source.is_dir():
+        raise NotADirectoryError(f"input dataset does not exist: {source}")
+    if output.exists():
+        raise FileExistsError(f"output path already exists: {output}")
+    if output.is_relative_to(source):
+        raise ValueError("output path must be outside the input dataset")
+    data_root = source / "data"
+    data_paths = sorted(data_root.glob("**/*.parquet"))
+    if not data_paths:
+        raise ValueError(f"no data parquet files found under {data_root}")
+
+    info_path = source / "meta" / "info.json"
+    info = json.loads(info_path.read_text(encoding="utf-8"))
+    fps = float(info.get("fps", 30.0))
+    limits = _candidate_lengths(data_paths, data_root, minutes * 60, fps)
+    episode_tables = _episode_tables(source)
+    episode_rows = _episode_rows(episode_tables)
+
+    media_paths = sorted(
+        path
+        for directory in (source / "videos", source / "depths")
+        if directory.exists()
+        for path in directory.glob("**/*")
+        if path.is_file() and path.suffix.lower() in VIDEO_SUFFIXES
+    )
+    media_episodes = {}
+    for path in media_paths:
+        episodes = _media_episodes(path, source, episode_rows)
+        if len(episodes) != 1:
+            raise ValueError(
+                f"media file contains multiple episodes and cannot be cut safely: {path}"
+            )
+        episode = episodes[0]
+        if episode not in limits:
+            raise ValueError(f"media references unknown episode {episode}: {path}")
+        media_episodes[path] = episode
+        limits[episode] = min(limits[episode], _probe_frames(path))
+
+    if any(length <= 0 for length in limits.values()):
+        raise ValueError("the requested cut produced an empty episode")
+
+    try:
+        _copy_static_tree(source, output)
+        source_metrics, source_totals = _write_data(
+            data_paths, data_root, output, limits, fps
+        )
+        end_times = {
+            episode: min(
+                float(episodes[episode]["to_timestamp"])
+                - float(episodes[episode]["from_timestamp"])
+                for episodes in source_metrics.values()
+                if episode in episodes
+            )
+            for episode in limits
+        }
+
+        for path, episode in media_episodes.items():
+            _copy_or_cut_video(
+                path, output / path.relative_to(source), limits[episode]
+            )
+        for directory in (source / "videos", source / "depths"):
+            if not directory.exists():
+                continue
+            for path in directory.glob("**/*"):
+                if path.is_file() and path not in media_episodes:
+                    destination = output / path.relative_to(source)
+                    destination.parent.mkdir(parents=True, exist_ok=True)
+                    shutil.copy2(path, destination)
+        sensor_metrics = _copy_sensors(
+            source, output, episode_rows, end_times
+        )
+
+        offsets = {}
+        offset = 0
+        for episode in sorted(limits):
+            offsets[episode] = offset
+            offset += limits[episode]
+
+        for original_path, table in episode_tables:
+            rows = table.to_pylist()
+            for row in rows:
+                _update_episode_row(
+                    row,
+                    limits,
+                    offsets,
+                    source_metrics,
+                    sensor_metrics,
+                    end_times,
+                )
+            destination = output / original_path.relative_to(source)
+            pq.write_table(pa.Table.from_pylist(rows, schema=table.schema), destination)
+
+        jsonl_path = output / "meta" / "episodes.jsonl"
+        jsonl_rows = _read_json_lines(jsonl_path)
+        for row in jsonl_rows:
+            _update_episode_row(
+                row,
+                limits,
+                offsets,
+                source_metrics,
+                sensor_metrics,
+                end_times,
+            )
+        if jsonl_path.exists():
+            _write_json_lines(jsonl_path, jsonl_rows)
+
+        info["total_frames"] = sum(limits.values())
+        if isinstance(info.get("source_frame_counts"), dict):
+            info["source_frame_counts"] = {
+                source_name: source_totals.get(source_name, 0)
+                for source_name in info["source_frame_counts"]
+            }
+        (output / "meta" / "info.json").write_text(
+            json.dumps(info, ensure_ascii=False, indent=2) + "\n",
+            encoding="utf-8",
+        )
+    except Exception:
+        shutil.rmtree(output, ignore_errors=True)
+        raise
+
+
+def execute(args: argparse.Namespace) -> int:
+    cut_dataset(args.input_dataset, args.output_path, args.minutes)
+    print(
+        json.dumps(
+            {
+                "input": str(args.input_dataset),
+                "output": str(args.output_path),
+                "minutes": args.minutes,
+            },
+            ensure_ascii=False,
+        )
+    )
+    return 0

{ls_algorithm_plugin_sdk-0.3.4 → ls_algorithm_plugin_sdk-0.3.6}/src/algorithm_plugin_sdk/cli_impl/run.py

@@ -34,7 +34,7 @@ def configure_parser(parser: argparse.ArgumentParser) -> None:
     parser.add_argument("--parameters", type=json_object)
     parser.add_argument("--request-file", type=Path)
     parser.add_argument("--scratch-dir")
-    parser.add_argument("--algorithm-log", type=Path)
+    parser.add_argument("--log", type=Path)
     parser.set_defaults(handler=execute)
 
 
@@ -139,10 +139,10 @@ def execute(args: argparse.Namespace) -> int:
     )
     try:
         with ExitStack() as stack:
-            if args.algorithm_log is not None:
-                args.algorithm_log.parent.mkdir(parents=True, exist_ok=True)
+            if args.log is not None:
+                args.log.parent.mkdir(parents=True, exist_ok=True)
                 stream = stack.enter_context(
-                    args.algorithm_log.open("w", encoding="utf-8", buffering=1)
+                    args.log.open("w", encoding="utf-8", buffering=1)
                 )
                 stack.enter_context(redirect_stdout(stream))
                 stack.enter_context(redirect_stderr(stream))

{ls_algorithm_plugin_sdk-0.3.4 → ls_algorithm_plugin_sdk-0.3.6}/src/ls_algorithm_plugin_sdk.egg-info/PKG-INFO

@@ -1,11 +1,12 @@
 Metadata-Version: 2.4
 Name: ls-algorithm-plugin-sdk
-Version: 0.3.4
+Version: 0.3.6
 Summary: Protocol-independent runtime SDK for dataset algorithms
 Author: Ling Robotics
 License: Proprietary
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+Requires-Dist: pyarrow>=14
 Provides-Extra: service
 Requires-Dist: fastapi<1,>=0.110; extra == "service"
 Requires-Dist: pydantic<3,>=2.0; extra == "service"
@@ -40,6 +41,16 @@ algorithm-plugin run camera-space-mano \
 
 `--gpu-ids` 和 `--parameters` 可省略，也可用 `--request-file` 读取完整请求 JSON。
 
+## 数据集裁剪
+
+将每个 episode 裁剪为最多指定分钟数，并同步更新 Parquet、视频、传感器 CSV 和元数据：
+
+```bash
+algorithm-plugin cut /data/input_dataset /data/output_dataset --minutes 1.5
+```
+
+支持 LeRobot v2/v3 的 `episode_*`、`file-*` 和多数据源目录布局。视频裁剪需要系统已安装 `ffmpeg` 与 `ffprobe`。
+
 ## 配置现有仓库
 
 目标算法仓库必须满足：

{ls_algorithm_plugin_sdk-0.3.4 → ls_algorithm_plugin_sdk-0.3.6}/src/ls_algorithm_plugin_sdk.egg-info/SOURCES.txt

@@ -17,6 +17,7 @@ src/algorithm_plugin_sdk/service.py
 src/algorithm_plugin_sdk/webui_app.py
 src/algorithm_plugin_sdk/cli_impl/__init__.py
 src/algorithm_plugin_sdk/cli_impl/configure.py
+src/algorithm_plugin_sdk/cli_impl/cut.py
 src/algorithm_plugin_sdk/cli_impl/parsing.py
 src/algorithm_plugin_sdk/cli_impl/run.py
 src/algorithm_plugin_sdk/cli_impl/serve.py
@@ -35,6 +36,7 @@ src/ls_algorithm_plugin_sdk.egg-info/requires.txt
 src/ls_algorithm_plugin_sdk.egg-info/top_level.txt
 tests/test_cli.py
 tests/test_configure.py
+tests/test_cut.py
 tests/test_deployment.py
 tests/test_gpu_isolation.py
 tests/test_models.py

{ls_algorithm_plugin_sdk-0.3.4 → ls_algorithm_plugin_sdk-0.3.6}/src/ls_algorithm_plugin_sdk.egg-info/requires.txt

@@ -1,3 +1,4 @@
+pyarrow>=14
 
 [dev]
 build>=1.2

ls_algorithm_plugin_sdk-0.3.6/tests/test_cut.py

@@ -0,0 +1,104 @@
+import csv
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from algorithm_plugin_sdk.cli import build_parser
+from algorithm_plugin_sdk.cli_impl.cut import cut_dataset, positive_minutes
+
+
+class CutTests(unittest.TestCase):
+    def test_cut_parser_binds_handler(self) -> None:
+        args = build_parser().parse_args(
+            ["cut", "input", "output", "--minutes", "1.5"]
+        )
+
+        self.assertEqual(args.minutes, 1.5)
+        self.assertEqual(
+            args.handler.__module__, "algorithm_plugin_sdk.cli_impl.cut"
+        )
+
+    def test_minutes_must_be_positive(self) -> None:
+        with self.assertRaises(Exception):
+            positive_minutes("0")
+
+    def test_rejects_output_inside_input_dataset(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary:
+            source = Path(temporary)
+            with self.assertRaisesRegex(ValueError, "outside"):
+                cut_dataset(source, source / "output", 1)
+
+    def test_cuts_v2_parquet_sensor_and_metadata(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            source = root / "input"
+            output = root / "output"
+            data_path = source / "data/chunk-000/episode_000000.parquet"
+            data_path.parent.mkdir(parents=True)
+            table = pa.table(
+                {
+                    "episode_index": pa.array([0] * 61, type=pa.int64()),
+                    "frame_index": pa.array(range(61), type=pa.int64()),
+                    "index": pa.array(range(61), type=pa.int64()),
+                    "timestamp": pa.array(
+                        [index / 30 for index in range(61)], type=pa.float64()
+                    ),
+                }
+            )
+            pq.write_table(table, data_path)
+
+            meta = source / "meta"
+            meta.mkdir()
+            (meta / "info.json").write_text(
+                json.dumps({"fps": 30, "total_frames": 61}),
+                encoding="utf-8",
+            )
+            (meta / "episodes.jsonl").write_text(
+                json.dumps(
+                    {
+                        "episode_index": 0,
+                        "length": 61,
+                        "sensors/imu/path": "sensors/imu/episode-000000.csv",
+                        "sensors/imu/from_index": 0,
+                        "sensors/imu/to_index": 201,
+                        "sensors/imu/from_timestamp": 0.0,
+                        "sensors/imu/to_timestamp": 2.0,
+                    }
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            sensor = source / "sensors/imu/episode-000000.csv"
+            sensor.parent.mkdir(parents=True)
+            with sensor.open("w", newline="", encoding="utf-8") as stream:
+                writer = csv.DictWriter(
+                    stream, fieldnames=["imu_index", "timestamp"]
+                )
+                writer.writeheader()
+                writer.writerows(
+                    {"imu_index": index, "timestamp": index / 100}
+                    for index in range(201)
+                )
+
+            cut_dataset(source, output, 1 / 60)
+
+            trimmed = pq.read_table(
+                output / "data/chunk-000/episode_000000.parquet"
+            )
+            self.assertEqual(trimmed.num_rows, 31)
+            self.assertEqual(trimmed["frame_index"].to_pylist(), list(range(31)))
+            info = json.loads((output / "meta/info.json").read_text())
+            self.assertEqual(info["total_frames"], 31)
+            episode = json.loads(
+                (output / "meta/episodes.jsonl").read_text().strip()
+            )
+            self.assertEqual(episode["length"], 31)
+            self.assertEqual(episode["sensors/imu/to_index"], 101)
+
+
+if __name__ == "__main__":
+    unittest.main()

{ls_algorithm_plugin_sdk-0.3.4 → ls_algorithm_plugin_sdk-0.3.6}/tests/test_runner.py

@@ -318,7 +318,7 @@ class AlgorithmRunnerTests(unittest.TestCase):
                 "gpu_ids": None,
                 "parameters": None,
                 "scratch_dir": None,
-                "algorithm_log": None,
+                "log": None,
                 "algorithm": "test:CancelledAlgorithm",
             },
         )()