pymcap-cli 0.6.0__tar.gz → 0.8.0__tar.gz

{pymcap_cli-0.6.0 → pymcap_cli-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pymcap-cli
-Version: 0.6.0
+Version: 0.8.0
 Summary: High-performance Python CLI for MCAP file processing with advanced recovery, filtering, and optimization capabilities
 Keywords: mcap,cli,robotics,ros,ros2,recovery,filtering,compression
 Author: Marko Bausch
@@ -22,22 +22,30 @@ Classifier: Topic :: Utilities
 Classifier: Typing :: Typed
 Requires-Dist: rich>=14.1.0
 Requires-Dist: small-mcap[compression]
+Requires-Dist: mcap-codec-support[ros2]
 Requires-Dist: mcap-ros2-support-fast
 Requires-Dist: cyclopts>=4
 Requires-Dist: ros-parser
 Requires-Dist: platformdirs>=4.0.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: typing-extensions>=4.15.0
-Requires-Dist: pymcap-cli[video,pointcloud,plot] ; extra == 'all'
+Requires-Dist: pymcap-cli[video,pointcloud,plot,parquet,image,draco] ; extra == 'all'
+Requires-Dist: mcap-codec-support[draco] ; extra == 'draco'
+Requires-Dist: mcap-codec-support[image] ; extra == 'image'
+Requires-Dist: pyarrow>=15.0.0 ; extra == 'parquet'
+Requires-Dist: numpy>=1.24.0 ; extra == 'parquet'
+Requires-Dist: mcap-codec-support[pointcloud] ; extra == 'parquet'
 Requires-Dist: plotly>=6.0.0 ; extra == 'plot'
-Requires-Dist: pureini ; extra == 'pointcloud'
-Requires-Dist: av>=12.0.0 ; extra == 'video'
-Requires-Dist: numpy>=1.24.0 ; extra == 'video'
+Requires-Dist: mcap-codec-support[pointcloud] ; extra == 'pointcloud'
+Requires-Dist: mcap-codec-support[video] ; extra == 'video'
 Requires-Python: >=3.10
 Project-URL: Homepage, https://github.com/mrkbac/robotic-tools
 Project-URL: Issues, https://github.com/mrkbac/robotic-tools/issues
 Project-URL: Repository, https://github.com/mrkbac/robotic-tools
 Provides-Extra: all
+Provides-Extra: draco
+Provides-Extra: image
+Provides-Extra: parquet
 Provides-Extra: plot
 Provides-Extra: pointcloud
 Provides-Extra: video
@@ -352,7 +360,7 @@ pymcap-cli video data.mcap --topic /lidar/image --output lidar.mp4 --codec h265
 
 ### `roscompress` — ROS Image Compression
 
-Compress ROS MCAP files by converting CompressedImage/Image topics to CompressedVideo format. Requires the `video` extra.
+Compress ROS MCAP files by converting CompressedImage/Image topics to CompressedVideo format and PointCloud2 topics to Cloudini or Draco compressed point clouds.
 
 ```bash
 # Basic compression
@@ -360,11 +368,14 @@ pymcap-cli roscompress data.mcap -o compressed.mcap
 
 # Specify quality and codec
 pymcap-cli roscompress data.mcap -o compressed.mcap --quality 28 --codec h265
+
+# Draco point cloud compression using the Foxglove compressed point cloud schema
+pymcap-cli roscompress data.mcap -o compressed.mcap --pc-format draco --pc-schema foxglove
 ```
 
 ### `rosdecompress` — ROS Decompression
 
-Decompress CompressedVideo and CompressedPointCloud2 topics back to standard ROS formats. Requires the `video` extra.
+Decompress CompressedVideo, CompressedPointCloud2, and Foxglove CompressedPointCloud topics back to standard ROS formats.
 
 ```bash
 # Decompress to CompressedImage (JPEG)

{pymcap_cli-0.6.0 → pymcap_cli-0.8.0}/README.md

@@ -307,7 +307,7 @@ pymcap-cli video data.mcap --topic /lidar/image --output lidar.mp4 --codec h265
 
 ### `roscompress` — ROS Image Compression
 
-Compress ROS MCAP files by converting CompressedImage/Image topics to CompressedVideo format. Requires the `video` extra.
+Compress ROS MCAP files by converting CompressedImage/Image topics to CompressedVideo format and PointCloud2 topics to Cloudini or Draco compressed point clouds.
 
 ```bash
 # Basic compression
@@ -315,11 +315,14 @@ pymcap-cli roscompress data.mcap -o compressed.mcap
 
 # Specify quality and codec
 pymcap-cli roscompress data.mcap -o compressed.mcap --quality 28 --codec h265
+
+# Draco point cloud compression using the Foxglove compressed point cloud schema
+pymcap-cli roscompress data.mcap -o compressed.mcap --pc-format draco --pc-schema foxglove
 ```
 
 ### `rosdecompress` — ROS Decompression
 
-Decompress CompressedVideo and CompressedPointCloud2 topics back to standard ROS formats. Requires the `video` extra.
+Decompress CompressedVideo, CompressedPointCloud2, and Foxglove CompressedPointCloud topics back to standard ROS formats.
 
 ```bash
 # Decompress to CompressedImage (JPEG)

{pymcap_cli-0.6.0 → pymcap_cli-0.8.0}/pyproject.toml

@@ -1,14 +1,21 @@
 [project]
 name = "pymcap-cli"
-version = "0.6.0"
+version = "0.8.0"
 description = "High-performance Python CLI for MCAP file processing with advanced recovery, filtering, and optimization capabilities"
 readme = "README.md"
 requires-python = ">=3.10"
-license = {text = "GPL-3.0"}
-authors = [
-    {name = "Marko Bausch"}
+license = { text = "GPL-3.0" }
+authors = [{ name = "Marko Bausch" }]
+keywords = [
+    "mcap",
+    "cli",
+    "robotics",
+    "ros",
+    "ros2",
+    "recovery",
+    "filtering",
+    "compression",
 ]
-keywords = ["mcap", "cli", "robotics", "ros", "ros2", "recovery", "filtering", "compression"]
 classifiers = [
     "Development Status :: 4 - Beta",
     "Environment :: Console",
@@ -29,6 +36,7 @@ classifiers = [
 dependencies = [
     "rich>=14.1.0",
     "small-mcap[compression]",
+    "mcap-codec-support[ros2]",
     "mcap-ros2-support-fast",
     "cyclopts>=4",
     "ros-parser",
@@ -43,13 +51,14 @@ Repository = "https://github.com/mrkbac/robotic-tools"
 Issues = "https://github.com/mrkbac/robotic-tools/issues"
 
 [project.optional-dependencies]
-video = ["av>=12.0.0", "numpy>=1.24.0"]
-pointcloud = ["pureini"]
-plot = [
-    "plotly>=6.0.0",
-]
-all = [
-    "pymcap-cli[video,pointcloud,plot]",
+video = ["mcap-codec-support[video]"]
+pointcloud = ["mcap-codec-support[pointcloud]"]
+plot = ["plotly>=6.0.0"]
+parquet = ["pyarrow>=15.0.0", "numpy>=1.24.0", "mcap-codec-support[pointcloud]"]
+image = ["mcap-codec-support[image]"]
+all = ["pymcap-cli[video,pointcloud,plot,parquet,image,draco]"]
+draco = [
+    "mcap-codec-support[draco]",
 ]
 
 [build-system]
@@ -65,3 +74,5 @@ small-mcap = { workspace = true }
 mcap-ros2-support-fast = { workspace = true }
 ros-parser = { workspace = true }
 pureini = { workspace = true }
+pointcloud2 = { workspace = true }
+mcap-codec-support = { workspace = true }

{pymcap_cli-0.6.0 → pymcap_cli-0.8.0}/src/pymcap_cli/cli.py

@@ -10,7 +10,11 @@ from pymcap_cli.cmd import (
     compress_cmd,
     convert_cmd,
     diag_cmd,
+    diff_cmd,
     du_cmd,
+    export_csv_cmd,
+    export_geo_cmd,
+    export_json_cmd,
     filter_cmd,
     info_cmd,
     info_json_cmd,
@@ -21,6 +25,7 @@ from pymcap_cli.cmd import (
     records_cmd,
     recover_cmd,
     recover_inplace_cmd,
+    split_cmd,
     tftree_cmd,
     topic_chunks_cmd,
 )
@@ -88,6 +93,65 @@ except ImportError:
         return 1
 
 
+try:
+    from pymcap_cli.cmd.export_parquet_cmd import export_parquet
+except ImportError:
+
+    def export_parquet() -> int:
+        """Export command is unavailable because 'pyarrow' is not installed.
+
+        To enable Parquet export, install pymcap-cli with the 'parquet' extra:
+
+            uv add 'pymcap-cli[parquet]'
+        """
+        print(  # noqa: T201
+            "Error:\n"
+            "Export to Parquet is unavailable because 'pyarrow' is not installed.\n"
+            "Install with:\n\n"
+            "    uv add 'pymcap-cli[parquet]'\n",
+            file=sys.stderr,
+        )
+        return 1
+
+
+try:
+    from pymcap_cli.cmd.export_pcd_cmd import export_pcd
+except ImportError:
+
+    def export_pcd() -> int:
+        """PCD export is unavailable because numpy / pointcloud2 are not installed.
+
+        Install with:
+
+            uv add 'pymcap-cli[pointcloud]'
+        """
+        print(  # noqa: T201
+            "Error:\nPCD export requires numpy + pointcloud2.\n"
+            "Install with:\n\n    uv add 'pymcap-cli[pointcloud]'\n",
+            file=sys.stderr,
+        )
+        return 1
+
+
+try:
+    from pymcap_cli.cmd.export_images_cmd import export_images
+except ImportError:
+
+    def export_images() -> int:
+        """Image export is unavailable because required image deps are missing.
+
+        Install with:
+
+            uv add 'pymcap-cli[image]'
+        """
+        print(  # noqa: T201
+            "Error:\nImage export requires the 'image' extra (imagecodecs).\n"
+            "Install with:\n\n    uv add 'pymcap-cli[image]'\n",
+            file=sys.stderr,
+        )
+        return 1
+
+
 try:
     from pymcap_cli.cmd.rosdecompress_cmd import rosdecompress
 except ImportError:
@@ -122,6 +186,7 @@ transform_group = Group("Transform", sort_key=1)
 app.command(name="cat", group=inspect_group)(cat_cmd.cat)
 app.command(name="diag", group=inspect_group)(diag_cmd.diag)
 app.command(name="du", group=inspect_group)(du_cmd.du)
+app.command(name="diff", group=inspect_group)(diff_cmd.diff_cmd)
 app.command(name="info", group=inspect_group)(info_cmd.info)
 app.command(name="info-json", group=inspect_group)(info_json_cmd.info_json)
 list_cmd.list_app.group = (inspect_group,)
@@ -138,9 +203,16 @@ app.command(name="filter", group=transform_group)(filter_cmd.filter_cmd)
 app.command(name="merge", group=transform_group)(merge_cmd.merge)
 app.command(name="process", group=transform_group)(process_cmd.process)
 app.command(name="rechunk", group=transform_group)(rechunk_cmd.rechunk)
+app.command(name="split", group=transform_group)(split_cmd.split)
 app.command(name="recover", group=transform_group)(recover_cmd.recover)
 app.command(name="recover-inplace", group=transform_group)(recover_inplace_cmd.recover_inplace)
 app.command(name="plot", group=inspect_group)(plot)
+app.command(name="export-csv", group=transform_group)(export_csv_cmd.export_csv)
+app.command(name="export-geo", group=transform_group)(export_geo_cmd.export_geo)
+app.command(name="export-json", group=transform_group)(export_json_cmd.export_json)
+app.command(name="export-pcd", group=transform_group)(export_pcd)
+app.command(name="export-images", group=transform_group)(export_images)
+app.command(name="export-parquet", group=transform_group)(export_parquet)
 app.command(name="roscompress", group=transform_group)(roscompress)
 app.command(name="rosdecompress", group=transform_group)(rosdecompress)
 app.command(name="video", group=transform_group)(video)

pymcap_cli-0.8.0/src/pymcap_cli/cmd/_run_processor.py

@@ -0,0 +1,81 @@
+"""Shared processor pipeline for transform commands."""
+
+import contextlib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import BinaryIO
+
+from pymcap_cli.core.input_handler import open_input
+from pymcap_cli.core.mcap_processor import (
+    InputFile,
+    InputOptions,
+    McapProcessor,
+    OutputOptions,
+    OverwriteCollisionPolicy,
+    ProcessingOptions,
+    ProcessingStats,
+)
+from pymcap_cli.utils import confirm_output_overwrite
+
+
+@dataclass(slots=True)
+class ProcessorResult:
+    """Result of a processor run."""
+
+    stats: ProcessingStats
+    processor: McapProcessor
+
+
+def resolve_overwrite_policy(*, force: bool, no_clobber: bool) -> OverwriteCollisionPolicy | None:
+    """Map CLI overwrite flags to the processor overwrite policy."""
+    if force and no_clobber:
+        return None
+    if force:
+        return OverwriteCollisionPolicy.OVERWRITE
+    if no_clobber:
+        return OverwriteCollisionPolicy.ERROR
+    return OverwriteCollisionPolicy.ASK
+
+
+def _open_output_stream(output: Path, overwrite_policy: OverwriteCollisionPolicy) -> BinaryIO:
+    """Open a single-output destination with the configured overwrite policy."""
+    if overwrite_policy == OverwriteCollisionPolicy.ASK:
+        confirm_output_overwrite(output, force=False)
+    elif overwrite_policy == OverwriteCollisionPolicy.ERROR and output.exists():
+        raise FileExistsError(f"Output file '{output}' already exists.")
+
+    return output.open("wb")
+
+
+def run_processor(
+    *,
+    files: list[str],
+    output: Path,
+    input_options: InputOptions,
+    output_options: OutputOptions,
+) -> ProcessorResult:
+    """Open files, build ProcessingOptions, run McapProcessor, return results.
+
+    Raises any exception from McapProcessor.process() to the caller.
+    """
+    with contextlib.ExitStack() as stack:
+        input_files: list[InputFile] = []
+
+        for f in files:
+            stream, size = stack.enter_context(open_input(f))
+            input_files.append(InputFile(stream=stream, size=size, options=input_options))
+
+        output_stream = stack.enter_context(
+            _open_output_stream(output, output_options.overwrite_policy)
+        )
+
+        processing_options = ProcessingOptions(
+            inputs=input_files,
+            input_options=InputOptions.from_args(),
+            output_options=output_options,
+        )
+
+        processor = McapProcessor(processing_options)
+        stats = processor.process(output_stream)
+
+    return ProcessorResult(stats=stats, processor=processor)

pymcap_cli-0.6.0/src/pymcap_cli/cmd/_run_processor.py → pymcap_cli-0.8.0/src/pymcap_cli/cmd/_run_processor_multi.py

@@ -1,8 +1,7 @@
-"""Shared processor pipeline for transform commands."""
+"""Shared processor pipeline for multi-output transform commands (splitting)."""
 
 import contextlib
 from dataclasses import dataclass
-from pathlib import Path
 
 from pymcap_cli.core.input_handler import open_input
 from pymcap_cli.core.mcap_processor import (
@@ -23,14 +22,15 @@ class ProcessorResult:
     processor: McapProcessor
 
 
-def run_processor(
+def run_processor_multi(
     *,
     files: list[str],
-    output: Path,
-    input_options: InputOptions,
     output_options: OutputOptions,
 ) -> ProcessorResult:
-    """Open files, build ProcessingOptions, run McapProcessor, return results.
+    """Open input files, build ProcessingOptions, run McapProcessor in multi-output mode.
+
+    Unlike run_processor(), this does not open an output stream. The OutputManager
+    creates and manages output files based on split routing.
 
     Raises any exception from McapProcessor.process() to the caller.
     """
@@ -39,9 +39,9 @@ def run_processor(
 
         for f in files:
             stream, size = stack.enter_context(open_input(f))
-            input_files.append(InputFile(stream=stream, size=size, options=input_options))
-
-        output_stream = stack.enter_context(output.open("wb"))
+            input_files.append(
+                InputFile(stream=stream, size=size, options=InputOptions.from_args())
+            )
 
         processing_options = ProcessingOptions(
             inputs=input_files,
@@ -50,6 +50,7 @@ def run_processor(
         )
 
         processor = McapProcessor(processing_options)
-        stats = processor.process(output_stream)
+        # Multi-output mode: pass None, OutputManager creates files
+        stats = processor.process(output_stream=None)
 
     return ProcessorResult(stats=stats, processor=processor)

{pymcap_cli-0.6.0 → pymcap_cli-0.8.0}/src/pymcap_cli/cmd/bag2mcap_cmd.py

@@ -17,8 +17,8 @@ from rich.progress import (
     TextColumn,
     TimeRemainingColumn,
 )
-from small_mcap.writer import CompressionType, McapWriter
-from small_mcap.writer import CompressionType as WriterCompressionType
+from small_mcap import CompressionType, McapWriter
+from small_mcap import CompressionType as WriterCompressionType
 
 from pymcap_cli.display.osc_utils import OSCProgressColumn
 from pymcap_cli.types.types_manual import (

{pymcap_cli-0.6.0 → pymcap_cli-0.8.0}/src/pymcap_cli/cmd/cat_cmd.py

@@ -10,7 +10,7 @@ from pathlib import Path
 from typing import IO, TYPE_CHECKING, Annotated, Any
 
 if TYPE_CHECKING:
-    from small_mcap.reader import DecodedMessage
+    from small_mcap import DecodedMessage
 
 from cyclopts import Group, Parameter
 from mcap_ros2_support_fast.decoder import DecoderFactory
@@ -21,9 +21,7 @@ from rich.panel import Panel
 from rich.text import Text
 from ros_parser import parse_schema_to_definitions
 from ros_parser.message_path import MessagePathError, ValidationError, parse_message_path
-from small_mcap import JSONDecoderFactory
-from small_mcap.reader import read_message_decoded
-from small_mcap.records import Channel
+from small_mcap import Channel, JSONDecoderFactory, read_message_decoded
 
 from pymcap_cli.core.input_handler import open_input
 from pymcap_cli.utils import MAX_INT64, ProgressTrackingIO, file_progress, parse_timestamp_args
@@ -35,11 +33,16 @@ FILTERING_GROUP = Group("Filtering")
 OUTPUT_GROUP = Group("Output")
 
 _TTY_BYTES_TRUNCATE = 32
+# Threshold (bytes) below which `smart` mode inlines the full payload as an int
+# list. Anything larger becomes a `<N bytes>` placeholder so large binary
+# payloads (Image, PointCloud2) don't drown out the rest of the message.
+_SMART_BYTES_INLINE_LIMIT = 64
 
 
 class BytesMode(str, Enum):
     """How to serialize bytes fields in JSON output."""
 
+    SMART = "smart"
     INTS = "ints"
     BASE64 = "base64"
     SKIP = "skip"
@@ -48,7 +51,7 @@ class BytesMode(str, Enum):
 def message_to_dict(
     obj: Any,
     *,
-    bytes_mode: BytesMode = BytesMode.INTS,
+    bytes_mode: BytesMode = BytesMode.SMART,
     truncate_bytes: int = 0,
 ) -> Any:
     """Recursively convert a message object to a JSON-serializable dict.
@@ -74,6 +77,12 @@ def message_to_dict(
             return f"<{total} bytes>"
         if bytes_mode == BytesMode.BASE64:
             return base64.b64encode(bytes(obj)).decode("ascii")
+        if bytes_mode == BytesMode.SMART:
+            # Small payloads inline as ints for easy inspection; large ones
+            # collapse to a placeholder so output stays grep/jq-friendly.
+            if total <= _SMART_BYTES_INLINE_LIMIT:
+                return list(obj)
+            return f"<{total} bytes>"
         if truncate_bytes and total > truncate_bytes:
             return [*list(obj[:truncate_bytes]), f"... ({total} bytes total)"]
         return list(obj)
@@ -152,8 +161,16 @@ def cat(
         Parameter(
             name=["--bytes"],
             group=OUTPUT_GROUP,
+            help=(
+                "How to render `bytes` fields in JSON output. `smart` (default) "
+                f"inlines payloads ≤{_SMART_BYTES_INLINE_LIMIT} bytes as int lists "
+                "and collapses larger ones to `<N bytes>` so `cat` stays readable "
+                "on messages with Image/PointCloud2 payloads. Use `ints` for the "
+                "full int list, `base64` for a compact serialisable string, or "
+                "`skip` to always drop the payload."
+            ),
         ),
-    ] = BytesMode.INTS,
+    ] = BytesMode.SMART,
 ) -> int:
     """Stream MCAP messages to stdout.
 

{pymcap_cli-0.6.0 → pymcap_cli-0.8.0}/src/pymcap_cli/cmd/compress_cmd.py

@@ -2,17 +2,20 @@
 
 from rich.console import Console
 
-from pymcap_cli.cmd._run_processor import run_processor
-from pymcap_cli.core.mcap_processor import InputOptions, OutputOptions
+from pymcap_cli.cmd._run_processor import resolve_overwrite_policy, run_processor
+from pymcap_cli.core.mcap_processor import (
+    InputOptions,
+    OutputOptions,
+)
 from pymcap_cli.types.types_manual import (
     DEFAULT_CHUNK_SIZE,
     DEFAULT_COMPRESSION,
     ChunkSizeOption,
     CompressionOption,
     ForceOverwriteOption,
+    NoClobberOption,
     OutputPathOption,
 )
-from pymcap_cli.utils import confirm_output_overwrite
 
 console = Console()
 
@@ -24,6 +27,7 @@ def compress(
     chunk_size: ChunkSizeOption = DEFAULT_CHUNK_SIZE,
     compression: CompressionOption = DEFAULT_COMPRESSION,
     force: ForceOverwriteOption = False,
+    no_clobber: NoClobberOption = False,
 ) -> int:
     """Create a compressed copy of an MCAP file.
 
@@ -41,6 +45,8 @@ def compress(
         Compression algorithm for output file.
     force
         Force overwrite of output file without confirmation.
+    no_clobber
+        Fail instead of prompting if the output file already exists.
 
     Examples
     --------
@@ -48,17 +54,24 @@ def compress(
     pymcap-cli compress in.mcap -o out.mcap
     ```
     """
-    confirm_output_overwrite(output, force)
+    overwrite_policy = resolve_overwrite_policy(force=force, no_clobber=no_clobber)
+    if overwrite_policy is None:
+        console.print("[red]Error: --force and --no-clobber cannot be used together.[/red]")
+        return 1
+
     console.print(f"[blue]Compressing '{file}' to '{output}'[/blue]")
 
     try:
         result = run_processor(
             files=[file],
             output=output,
-            input_options=InputOptions.from_args(always_decode_chunk=True),
+            # Do not force always_decode_chunk — the processor now has a
+            # chunk-level RECOMPRESS path that avoids per-message parsing.
+            input_options=InputOptions.from_args(),
             output_options=OutputOptions(
                 compression=compression.value,
                 chunk_size=chunk_size,
+                overwrite_policy=overwrite_policy,
             ),
         )
         console.print("[green]✓ Compression completed successfully![/green]")

{pymcap_cli-0.6.0 → pymcap_cli-0.8.0}/src/pymcap_cli/cmd/convert_cmd.py

@@ -18,8 +18,8 @@ from rich.progress import (
     TextColumn,
     TimeRemainingColumn,
 )
-from small_mcap.writer import CompressionType, McapWriter
-from small_mcap.writer import CompressionType as WriterCompressionType
+from small_mcap import CompressionType, McapWriter
+from small_mcap import CompressionType as WriterCompressionType
 
 from pymcap_cli.core.msg_resolver import ROS2Distro, get_message_definition
 from pymcap_cli.display.osc_utils import OSCProgressColumn

{pymcap_cli-0.6.0 → pymcap_cli-0.8.0}/src/pymcap_cli/cmd/diag_cmd.py

@@ -14,7 +14,7 @@ from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeEl
 from rich.table import Table
 from rich.text import Text
 from rich.tree import Tree
-from small_mcap.reader import include_topics, read_message_decoded
+from small_mcap import include_topics, read_message_decoded
 
 from pymcap_cli.core.input_handler import open_input
 from pymcap_cli.display.sparkline import sparkline