pymcap-cli 0.7.0__tar.gz → 0.9.0__tar.gz

{pymcap_cli-0.7.0 → pymcap_cli-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pymcap-cli
-Version: 0.7.0
+Version: 0.9.0
 Summary: High-performance Python CLI for MCAP file processing with advanced recovery, filtering, and optimization capabilities
 Keywords: mcap,cli,robotics,ros,ros2,recovery,filtering,compression
 Author: Marko Bausch
@@ -22,22 +22,30 @@ Classifier: Topic :: Utilities
 Classifier: Typing :: Typed
 Requires-Dist: rich>=14.1.0
 Requires-Dist: small-mcap[compression]
+Requires-Dist: mcap-codec-support[ros2]
 Requires-Dist: mcap-ros2-support-fast
 Requires-Dist: cyclopts>=4
 Requires-Dist: ros-parser
 Requires-Dist: platformdirs>=4.0.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: typing-extensions>=4.15.0
-Requires-Dist: pymcap-cli[video,pointcloud,plot] ; extra == 'all'
+Requires-Dist: pymcap-cli[video,pointcloud,plot,parquet,image,draco] ; extra == 'all'
+Requires-Dist: mcap-codec-support[draco] ; extra == 'draco'
+Requires-Dist: mcap-codec-support[image] ; extra == 'image'
+Requires-Dist: pyarrow>=15.0.0 ; extra == 'parquet'
+Requires-Dist: numpy>=1.24.0 ; extra == 'parquet'
+Requires-Dist: mcap-codec-support[pointcloud] ; extra == 'parquet'
 Requires-Dist: plotly>=6.0.0 ; extra == 'plot'
-Requires-Dist: pureini ; extra == 'pointcloud'
-Requires-Dist: av>=12.0.0 ; extra == 'video'
-Requires-Dist: numpy>=1.24.0 ; extra == 'video'
+Requires-Dist: mcap-codec-support[pointcloud] ; extra == 'pointcloud'
+Requires-Dist: mcap-codec-support[video] ; extra == 'video'
 Requires-Python: >=3.10
 Project-URL: Homepage, https://github.com/mrkbac/robotic-tools
 Project-URL: Issues, https://github.com/mrkbac/robotic-tools/issues
 Project-URL: Repository, https://github.com/mrkbac/robotic-tools
 Provides-Extra: all
+Provides-Extra: draco
+Provides-Extra: image
+Provides-Extra: parquet
 Provides-Extra: plot
 Provides-Extra: pointcloud
 Provides-Extra: video
@@ -352,7 +360,7 @@ pymcap-cli video data.mcap --topic /lidar/image --output lidar.mp4 --codec h265
 
 ### `roscompress` — ROS Image Compression
 
-Compress ROS MCAP files by converting CompressedImage/Image topics to CompressedVideo format. Requires the `video` extra.
+Compress ROS MCAP files by converting CompressedImage/Image topics to CompressedVideo format and PointCloud2 topics to Cloudini or Draco compressed point clouds.
 
 ```bash
 # Basic compression
@@ -360,11 +368,14 @@ pymcap-cli roscompress data.mcap -o compressed.mcap
 
 # Specify quality and codec
 pymcap-cli roscompress data.mcap -o compressed.mcap --quality 28 --codec h265
+
+# Draco point cloud compression using the Foxglove compressed point cloud schema
+pymcap-cli roscompress data.mcap -o compressed.mcap --pc-format draco --pc-schema foxglove
 ```
 
 ### `rosdecompress` — ROS Decompression
 
-Decompress CompressedVideo and CompressedPointCloud2 topics back to standard ROS formats. Requires the `video` extra.
+Decompress CompressedVideo, CompressedPointCloud2, and Foxglove CompressedPointCloud topics back to standard ROS formats.
 
 ```bash
 # Decompress to CompressedImage (JPEG)

{pymcap_cli-0.7.0 → pymcap_cli-0.9.0}/README.md

@@ -307,7 +307,7 @@ pymcap-cli video data.mcap --topic /lidar/image --output lidar.mp4 --codec h265
 
 ### `roscompress` — ROS Image Compression
 
-Compress ROS MCAP files by converting CompressedImage/Image topics to CompressedVideo format. Requires the `video` extra.
+Compress ROS MCAP files by converting CompressedImage/Image topics to CompressedVideo format and PointCloud2 topics to Cloudini or Draco compressed point clouds.
 
 ```bash
 # Basic compression
@@ -315,11 +315,14 @@ pymcap-cli roscompress data.mcap -o compressed.mcap
 
 # Specify quality and codec
 pymcap-cli roscompress data.mcap -o compressed.mcap --quality 28 --codec h265
+
+# Draco point cloud compression using the Foxglove compressed point cloud schema
+pymcap-cli roscompress data.mcap -o compressed.mcap --pc-format draco --pc-schema foxglove
 ```
 
 ### `rosdecompress` — ROS Decompression
 
-Decompress CompressedVideo and CompressedPointCloud2 topics back to standard ROS formats. Requires the `video` extra.
+Decompress CompressedVideo, CompressedPointCloud2, and Foxglove CompressedPointCloud topics back to standard ROS formats.
 
 ```bash
 # Decompress to CompressedImage (JPEG)

{pymcap_cli-0.7.0 → pymcap_cli-0.9.0}/pyproject.toml

@@ -1,14 +1,21 @@
 [project]
 name = "pymcap-cli"
-version = "0.7.0"
+version = "0.9.0"
 description = "High-performance Python CLI for MCAP file processing with advanced recovery, filtering, and optimization capabilities"
 readme = "README.md"
 requires-python = ">=3.10"
-license = {text = "GPL-3.0"}
-authors = [
-    {name = "Marko Bausch"}
+license = { text = "GPL-3.0" }
+authors = [{ name = "Marko Bausch" }]
+keywords = [
+    "mcap",
+    "cli",
+    "robotics",
+    "ros",
+    "ros2",
+    "recovery",
+    "filtering",
+    "compression",
 ]
-keywords = ["mcap", "cli", "robotics", "ros", "ros2", "recovery", "filtering", "compression"]
 classifiers = [
     "Development Status :: 4 - Beta",
     "Environment :: Console",
@@ -29,6 +36,7 @@ classifiers = [
 dependencies = [
     "rich>=14.1.0",
     "small-mcap[compression]",
+    "mcap-codec-support[ros2]",
     "mcap-ros2-support-fast",
     "cyclopts>=4",
     "ros-parser",
@@ -43,13 +51,14 @@ Repository = "https://github.com/mrkbac/robotic-tools"
 Issues = "https://github.com/mrkbac/robotic-tools/issues"
 
 [project.optional-dependencies]
-video = ["av>=12.0.0", "numpy>=1.24.0"]
-pointcloud = ["pureini"]
-plot = [
-    "plotly>=6.0.0",
-]
-all = [
-    "pymcap-cli[video,pointcloud,plot]",
+video = ["mcap-codec-support[video]"]
+pointcloud = ["mcap-codec-support[pointcloud]"]
+plot = ["plotly>=6.0.0"]
+parquet = ["pyarrow>=15.0.0", "numpy>=1.24.0", "mcap-codec-support[pointcloud]"]
+image = ["mcap-codec-support[image]"]
+all = ["pymcap-cli[video,pointcloud,plot,parquet,image,draco]"]
+draco = [
+    "mcap-codec-support[draco]",
 ]
 
 [build-system]
@@ -65,3 +74,5 @@ small-mcap = { workspace = true }
 mcap-ros2-support-fast = { workspace = true }
 ros-parser = { workspace = true }
 pureini = { workspace = true }
+pointcloud2 = { workspace = true }
+mcap-codec-support = { workspace = true }

{pymcap_cli-0.7.0 → pymcap_cli-0.9.0}/src/pymcap_cli/cli.py

@@ -1,8 +1,8 @@
 """Main CLI entry point for pymcap-cli using Cyclopts."""
 
-import sys
+from typing import Annotated
 
-from cyclopts import App, Group
+from cyclopts import App, Group, Parameter
 
 from pymcap_cli.cmd import (
     bag2mcap_cmd,
@@ -12,6 +12,10 @@ from pymcap_cli.cmd import (
     diag_cmd,
     diff_cmd,
     du_cmd,
+    duplicates_cmd,
+    export_csv_cmd,
+    export_geo_cmd,
+    export_json_cmd,
     filter_cmd,
     info_cmd,
     info_json_cmd,
@@ -22,9 +26,11 @@ from pymcap_cli.cmd import (
     records_cmd,
     recover_cmd,
     recover_inplace_cmd,
+    split_cmd,
     tftree_cmd,
     topic_chunks_cmd,
 )
+from pymcap_cli.log_setup import ERR, setup_logging
 
 try:
     from pymcap_cli.cmd.video_cmd import video
@@ -37,12 +43,11 @@ except ImportError:
 
             uv add --group video pymcap-cli
         """
-        print(  # noqa: T201
-            "Error:\n"
+        ERR.print(
+            "[red]Error:[/red]\n"
             "Video command is unavailable because the 'av' and/or 'numpy' are not installed.\n"
             "To enable video functionality, please install pymcap-cli with the 'video' extra:\n\n"
-            "    uv add --group video pymcap-cli\n",
-            file=sys.stderr,
+            "    uv add --group video pymcap-cli\n"
         )
         return 1
 
@@ -58,12 +63,11 @@ except ImportError:
 
             uv add --group plot pymcap-cli
         """
-        print(  # noqa: T201
-            "Error:\n"
+        ERR.print(
+            "[red]Error:[/red]\n"
             "Plot command is unavailable because 'plotly' is not installed.\n"
             "To enable plot functionality, please install pymcap-cli with the 'plot' extra:\n\n"
-            "    uv add --group plot pymcap-cli\n",
-            file=sys.stderr,
+            "    uv add --group plot pymcap-cli\n"
         )
         return 1
 
@@ -79,12 +83,67 @@ except ImportError:
 
             uv add --group video pymcap-cli
         """
-        print(  # noqa: T201
-            "Error:\n"
+        ERR.print(
+            "[red]Error:[/red]\n"
             "ROS compress command is unavailable because the 'av' package is not installed.\n"
             "To enable this functionality, please install pymcap-cli with the 'video' extra:\n\n"
-            "    uv add --group video pymcap-cli\n",
-            file=sys.stderr,
+            "    uv add --group video pymcap-cli\n"
+        )
+        return 1
+
+
+try:
+    from pymcap_cli.cmd.export_parquet_cmd import export_parquet
+except ImportError:
+
+    def export_parquet() -> int:
+        """Export command is unavailable because 'pyarrow' is not installed.
+
+        To enable Parquet export, install pymcap-cli with the 'parquet' extra:
+
+            uv add 'pymcap-cli[parquet]'
+        """
+        ERR.print(
+            "[red]Error:[/red]\n"
+            "Export to Parquet is unavailable because 'pyarrow' is not installed.\n"
+            "Install with:\n\n"
+            "    uv add 'pymcap-cli[parquet]'\n"
+        )
+        return 1
+
+
+try:
+    from pymcap_cli.cmd.export_pcd_cmd import export_pcd
+except ImportError:
+
+    def export_pcd() -> int:
+        """PCD export is unavailable because numpy / pointcloud2 are not installed.
+
+        Install with:
+
+            uv add 'pymcap-cli[pointcloud]'
+        """
+        ERR.print(
+            "[red]Error:[/red]\nPCD export requires numpy + pointcloud2.\n"
+            "Install with:\n\n    uv add 'pymcap-cli[pointcloud]'\n"
+        )
+        return 1
+
+
+try:
+    from pymcap_cli.cmd.export_images_cmd import export_images
+except ImportError:
+
+    def export_images() -> int:
+        """Image export is unavailable because required image deps are missing.
+
+        Install with:
+
+            uv add 'pymcap-cli[image]'
+        """
+        ERR.print(
+            "[red]Error:[/red]\nImage export requires the 'image' extra (imagecodecs).\n"
+            "Install with:\n\n    uv add 'pymcap-cli[image]'\n"
         )
         return 1
 
@@ -100,12 +159,11 @@ except ImportError:
 
             uv add --group video pymcap-cli
         """
-        print(  # noqa: T201
-            "Error:\n"
+        ERR.print(
+            "[red]Error:[/red]\n"
             "ROS decompress command is unavailable because the 'av' package is not installed.\n"
             "To enable this functionality, please install pymcap-cli with the 'video' extra:\n\n"
-            "    uv add --group video pymcap-cli\n",
-            file=sys.stderr,
+            "    uv add --group video pymcap-cli\n"
         )
         return 1
 
@@ -114,6 +172,7 @@ app = App(
     name="pymcap-cli",
     help="CLI tool for slicing and dicing MCAP files.",
     help_format="rich",
+    default_parameter=Parameter(negative_iterable=""),
 )
 
 inspect_group = Group("Inspect", sort_key=0)
@@ -124,6 +183,7 @@ app.command(name="cat", group=inspect_group)(cat_cmd.cat)
 app.command(name="diag", group=inspect_group)(diag_cmd.diag)
 app.command(name="du", group=inspect_group)(du_cmd.du)
 app.command(name="diff", group=inspect_group)(diff_cmd.diff_cmd)
+app.command(name="duplicates", group=inspect_group)(duplicates_cmd.duplicates)
 app.command(name="info", group=inspect_group)(info_cmd.info)
 app.command(name="info-json", group=inspect_group)(info_json_cmd.info_json)
 list_cmd.list_app.group = (inspect_group,)
@@ -140,16 +200,47 @@ app.command(name="filter", group=transform_group)(filter_cmd.filter_cmd)
 app.command(name="merge", group=transform_group)(merge_cmd.merge)
 app.command(name="process", group=transform_group)(process_cmd.process)
 app.command(name="rechunk", group=transform_group)(rechunk_cmd.rechunk)
+app.command(name="split", group=transform_group)(split_cmd.split)
 app.command(name="recover", group=transform_group)(recover_cmd.recover)
 app.command(name="recover-inplace", group=transform_group)(recover_inplace_cmd.recover_inplace)
 app.command(name="plot", group=inspect_group)(plot)
+app.command(name="export-csv", group=transform_group)(export_csv_cmd.export_csv)
+app.command(name="export-geo", group=transform_group)(export_geo_cmd.export_geo)
+app.command(name="export-json", group=transform_group)(export_json_cmd.export_json)
+app.command(name="export-pcd", group=transform_group)(export_pcd)
+app.command(name="export-images", group=transform_group)(export_images)
+app.command(name="export-parquet", group=transform_group)(export_parquet)
 app.command(name="roscompress", group=transform_group)(roscompress)
 app.command(name="rosdecompress", group=transform_group)(rosdecompress)
 app.command(name="video", group=transform_group)(video)
 
 
+@app.meta.default
+def launcher(
+    *tokens: Annotated[str, Parameter(allow_leading_hyphen=True)],
+    verbose: Annotated[
+        int,
+        Parameter(
+            name=["--verbose", "-v"],
+            count=True,
+            help="Increase log verbosity. -v: DEBUG.",
+        ),
+    ] = 0,
+    quiet: Annotated[
+        int,
+        Parameter(
+            name=["--quiet"],
+            count=True,
+            help="Decrease log verbosity. Once: WARNING; twice: ERROR.",
+        ),
+    ] = 0,
+) -> int | None:
+    setup_logging(verbose=verbose, quiet=quiet)
+    return app(tokens)
+
+
 def main() -> None:
-    app()
+    app.meta()
 
 
 if __name__ == "__main__":

pymcap_cli-0.9.0/src/pymcap_cli/cmd/_run_processor.py

@@ -0,0 +1,146 @@
+"""Shared processor pipeline for transform commands."""
+
+import contextlib
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import BinaryIO
+from urllib.parse import urlparse
+
+from small_mcap import InvalidMagicError, McapError
+
+from pymcap_cli.core.input_handler import open_input
+from pymcap_cli.core.mcap_processor import (
+    InputFile,
+    InputOptions,
+    McapProcessor,
+    OutputOptions,
+    OverwriteCollisionPolicy,
+    ProcessingOptions,
+    ProcessingStats,
+)
+from pymcap_cli.utils import confirm_output_overwrite, read_info
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(slots=True)
+class ProcessorResult:
+    """Result of a processor run."""
+
+    stats: ProcessingStats
+    processor: McapProcessor
+
+
+def resolve_overwrite_policy(*, force: bool, no_clobber: bool) -> OverwriteCollisionPolicy | None:
+    """Map CLI overwrite flags to the processor overwrite policy."""
+    if force and no_clobber:
+        return None
+    if force:
+        return OverwriteCollisionPolicy.OVERWRITE
+    if no_clobber:
+        return OverwriteCollisionPolicy.ERROR
+    return OverwriteCollisionPolicy.ASK
+
+
+def _open_output_stream(output: Path, overwrite_policy: OverwriteCollisionPolicy) -> BinaryIO:
+    """Open a single-output destination with the configured overwrite policy."""
+    if overwrite_policy == OverwriteCollisionPolicy.ASK:
+        confirm_output_overwrite(output, force=False)
+    elif overwrite_policy == OverwriteCollisionPolicy.ERROR and output.exists():
+        raise FileExistsError(f"Output file '{output}' already exists.")
+
+    return output.open("wb")
+
+
+def run_processor(
+    *,
+    files: list[str],
+    output: Path,
+    input_options: InputOptions,
+    output_options: OutputOptions,
+) -> ProcessorResult:
+    """Open files, build ProcessingOptions, run McapProcessor, return results.
+
+    Raises any exception from McapProcessor.process() to the caller.
+    """
+    with contextlib.ExitStack() as stack:
+        input_files: list[InputFile] = []
+
+        for f in files:
+            stream, size = stack.enter_context(open_input(f))
+            input_files.append(InputFile(stream=stream, size=size, options=input_options))
+
+        output_stream = stack.enter_context(
+            _open_output_stream(output, output_options.overwrite_policy)
+        )
+
+        processing_options = ProcessingOptions(
+            inputs=input_files,
+            input_options=InputOptions.from_args(),
+            output_options=output_options,
+        )
+
+        processor = McapProcessor(processing_options)
+        stats = processor.process(output_stream)
+
+    return ProcessorResult(stats=stats, processor=processor)
+
+
+def validate_mcap_output(path: Path) -> bool:
+    """Return True iff the MCAP at ``path`` has a readable header and summary."""
+    try:
+        with path.open("rb") as f:
+            read_info(f)
+    except (McapError, InvalidMagicError, OSError, AssertionError) as e:
+        logger.debug(f"Output validation failed for {path}: {e}")
+        return False
+    return True
+
+
+def delete_source_files(sources: list[str], outputs: list[Path]) -> None:
+    """Delete each local source file. Skip URLs and any source path that
+    resolves to one of ``outputs`` (with a warning).
+    """
+    output_resolved = {p.resolve() for p in outputs}
+    for src in sources:
+        scheme = urlparse(src).scheme
+        if scheme in ("http", "https"):
+            logger.warning(f"Skipping delete: '{src}' is a remote URL")
+            continue
+        path = Path(src)
+        try:
+            resolved = path.resolve()
+        except OSError as e:
+            logger.warning(f"Skipping delete '{src}': {e}")
+            continue
+        if resolved in output_resolved:
+            logger.warning(f"Skipping delete: source '{src}' is also an output")
+            continue
+        try:
+            path.unlink()
+            logger.info(f"Deleted source: {src}")
+        except FileNotFoundError:
+            logger.debug(f"Source already gone: {src}")
+        except OSError:
+            logger.exception(f"Failed to delete '{src}'")
+
+
+def finalize_delete_source(
+    *,
+    sources: list[str],
+    outputs: list[Path],
+) -> int:
+    """Validate every output and, if all valid, delete the eligible sources.
+
+    Returns 0 on success (sources deleted or skipped with warning) and 1 if
+    any output failed validation (no sources are deleted in that case).
+    """
+    invalid = [p for p in outputs if not validate_mcap_output(p)]
+    if invalid:
+        for p in invalid:
+            logger.error(f"[red]Output failed validation: {p}[/red]")
+        logger.error("Source file(s) preserved — output not safe to replace source.")
+        return 1
+    delete_source_files(sources, outputs)
+    return 0

pymcap_cli-0.7.0/src/pymcap_cli/cmd/_run_processor.py → pymcap_cli-0.9.0/src/pymcap_cli/cmd/_run_processor_multi.py

@@ -1,8 +1,7 @@
-"""Shared processor pipeline for transform commands."""
+"""Shared processor pipeline for multi-output transform commands (splitting)."""
 
 import contextlib
 from dataclasses import dataclass
-from pathlib import Path
 
 from pymcap_cli.core.input_handler import open_input
 from pymcap_cli.core.mcap_processor import (
@@ -23,14 +22,15 @@ class ProcessorResult:
     processor: McapProcessor
 
 
-def run_processor(
+def run_processor_multi(
     *,
     files: list[str],
-    output: Path,
-    input_options: InputOptions,
     output_options: OutputOptions,
 ) -> ProcessorResult:
-    """Open files, build ProcessingOptions, run McapProcessor, return results.
+    """Open input files, build ProcessingOptions, run McapProcessor in multi-output mode.
+
+    Unlike run_processor(), this does not open an output stream. The OutputManager
+    creates and manages output files based on split routing.
 
     Raises any exception from McapProcessor.process() to the caller.
     """
@@ -39,9 +39,9 @@ def run_processor(
 
         for f in files:
             stream, size = stack.enter_context(open_input(f))
-            input_files.append(InputFile(stream=stream, size=size, options=input_options))
-
-        output_stream = stack.enter_context(output.open("wb"))
+            input_files.append(
+                InputFile(stream=stream, size=size, options=InputOptions.from_args())
+            )
 
         processing_options = ProcessingOptions(
             inputs=input_files,
@@ -50,6 +50,7 @@ def run_processor(
         )
 
         processor = McapProcessor(processing_options)
-        stats = processor.process(output_stream)
+        # Multi-output mode: pass None, OutputManager creates files
+        stats = processor.process(output_stream=None)
 
     return ProcessorResult(stats=stats, processor=processor)

{pymcap_cli-0.7.0 → pymcap_cli-0.9.0}/src/pymcap_cli/cmd/bag2mcap_cmd.py

@@ -17,8 +17,8 @@ from rich.progress import (
     TextColumn,
     TimeRemainingColumn,
 )
-from small_mcap.writer import CompressionType, McapWriter
-from small_mcap.writer import CompressionType as WriterCompressionType
+from small_mcap import CompressionType, McapWriter
+from small_mcap import CompressionType as WriterCompressionType
 
 from pymcap_cli.display.osc_utils import OSCProgressColumn
 from pymcap_cli.types.types_manual import (
@@ -249,7 +249,7 @@ def bag2mcap(
     """
     input_path = Path(file)
     if not input_path.exists():
-        console.print(f"[red]Error: Input file '{file}' does not exist[/red]")
+        logger.error(f"Input file '{file}' does not exist")
         return 1
 
     confirm_output_overwrite(output, force)
@@ -266,20 +266,20 @@ def bag2mcap(
         compression=writer_compression,
     )
 
-    console.print(f"[blue]Converting '{file}' to '{output}'[/blue]")
+    logger.info(f"Converting '{file}' to '{output}'")
 
     with output.open("wb") as output_stream:
         try:
             stats = convert_bag_to_mcap(input_path, output_stream, options)
 
-            console.print("[green]Conversion completed successfully[/green]")
+            logger.info("[green]Conversion completed successfully[/green]")
             console.print(
                 f"Converted {stats.topic_count} topics, "
                 f"{stats.message_count:,} messages, "
                 f"{stats.schema_count} schemas"
             )
-        except Exception as e:  # noqa: BLE001
-            console.print(f"[red]Error during conversion: {e}[/red]")
+        except Exception:
+            logger.exception("Error during conversion")
             return 1
 
     return 0