pymcap-cli 0.10.0__tar.gz → 0.12.0__tar.gz

{pymcap_cli-0.10.0 → pymcap_cli-0.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pymcap-cli
-Version: 0.10.0
+Version: 0.12.0
 Summary: High-performance Python CLI for MCAP file processing with advanced recovery, filtering, and optimization capabilities
 Keywords: mcap,cli,robotics,ros,ros2,recovery,filtering,compression
 Author: Marko Bausch

{pymcap_cli-0.10.0 → pymcap_cli-0.12.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pymcap-cli"
-version = "0.10.0"
+version = "0.12.0"
 description = "High-performance Python CLI for MCAP file processing with advanced recovery, filtering, and optimization capabilities"
 readme = "README.md"
 requires-python = ">=3.10"

{pymcap_cli-0.10.0 → pymcap_cli-0.12.0}/src/pymcap_cli/cli.py

@@ -33,6 +33,7 @@ from pymcap_cli.cmd import (
     recover_inplace_cmd,
     split_cmd,
     tf_export_cmd,
+    tf_get_cmd,
     tftree_cmd,
     topic_chunks_cmd,
 )
@@ -120,6 +121,22 @@ def _load_optional_app(
     return cast("App", getattr(module, app_name))
 
 
+# Modules that go missing together when a given extra is absent. Used by the
+# optional-command loader to suppress ImportError for the *expected* shape of
+# "extra not installed."
+_VIDEO_MODULES: tuple[str, ...] = ("av", "mcap_codec_support", "numpy")
+_POINTCLOUD_MODULES: tuple[str, ...] = (
+    "DracoPy",
+    "mcap_codec_support",
+    "numpy",
+    "pointcloud2",
+    "pureini",
+)
+_VIDEO_AND_POINTCLOUD_MODULES: tuple[str, ...] = tuple(
+    sorted({*_VIDEO_MODULES, *_POINTCLOUD_MODULES})
+)
+
+
 bridge_app = _load_optional_app(
     "pymcap_cli.cmd.bridge",
     "bridge_app",
@@ -130,7 +147,7 @@ bridge_app = _load_optional_app(
 video = _load_optional_command(
     "pymcap_cli.cmd.video_cmd",
     "video",
-    expected_missing_modules=("av", "mcap_codec_support", "numpy"),
+    expected_missing_modules=_VIDEO_MODULES,
     message="Video command requires the 'video' extra.",
     install_command="uv add 'pymcap-cli[video]'",
 )
@@ -144,35 +161,21 @@ plot = _load_optional_command(
 roscompress = _load_optional_command(
     "pymcap_cli.cmd.roscompress_cmd",
     "roscompress",
-    expected_missing_modules=(
-        "av",
-        "DracoPy",
-        "mcap_codec_support",
-        "numpy",
-        "pointcloud2",
-        "pureini",
-    ),
+    expected_missing_modules=_VIDEO_AND_POINTCLOUD_MODULES,
     message="ROS compression requires the 'video' and 'pointcloud' extras.",
     install_command="uv add 'pymcap-cli[video,pointcloud]'",
 )
 export_parquet = _load_optional_command(
     "pymcap_cli.cmd.export_parquet_cmd",
     "export_parquet",
-    expected_missing_modules=(
-        "DracoPy",
-        "mcap_codec_support",
-        "numpy",
-        "pointcloud2",
-        "pureini",
-        "pyarrow",
-    ),
+    expected_missing_modules=(*_POINTCLOUD_MODULES, "pyarrow"),
     message="Parquet export requires the 'parquet' extra.",
     install_command="uv add 'pymcap-cli[parquet]'",
 )
 export_pcd = _load_optional_command(
     "pymcap_cli.cmd.export_pcd_cmd",
     "export_pcd",
-    expected_missing_modules=("DracoPy", "mcap_codec_support", "numpy", "pointcloud2", "pureini"),
+    expected_missing_modules=_POINTCLOUD_MODULES,
     message="PCD export requires the 'pointcloud' extra.",
     install_command="uv add 'pymcap-cli[pointcloud]'",
 )
@@ -186,23 +189,25 @@ export_images = _load_optional_command(
 rosdecompress = _load_optional_command(
     "pymcap_cli.cmd.rosdecompress_cmd",
     "rosdecompress",
-    expected_missing_modules=(
-        "av",
-        "DracoPy",
-        "mcap_codec_support",
-        "numpy",
-        "pointcloud2",
-        "pureini",
-    ),
+    expected_missing_modules=_VIDEO_AND_POINTCLOUD_MODULES,
     message="ROS decompression requires the 'video' and 'pointcloud' extras.",
     install_command="uv add 'pymcap-cli[video,pointcloud]'",
 )
+index_app = _load_optional_app(
+    "pymcap_cli.cmd.index_cmd",
+    "index_app",
+    expected_missing_modules=("xxhash",),
+    message="Index command requires the 'xxhash' extra "
+    "(fingerprint stability requires a single committed hash function).",
+    install_command="uv add 'pymcap-cli[xxhash]'",
+)
 
 
 app = App(
     name="pymcap-cli",
     help="CLI tool for slicing and dicing MCAP files.",
     help_format="rich",
+    help_on_error=True,
     default_parameter=Parameter(negative_iterable=""),
 )
 
@@ -220,11 +225,14 @@ app.command(name="diff", group=inspect_group)(diff_cmd.diff_cmd)
 app.command(name="duplicates", group=inspect_group)(duplicates_cmd.duplicates)
 app.command(name="info", group=inspect_group)(info_cmd.info)
 app.command(name="info-json", group=inspect_group)(info_json_cmd.info_json)
+index_app.group = (inspect_group,)
+app.command(index_app, name="index")
 get_cmd.get_app.group = (inspect_group,)
 app.command(get_cmd.get_app, name="get")
 list_cmd.list_app.group = (inspect_group,)
 app.command(list_cmd.list_app, name="list")
 app.command(name="records", group=inspect_group)(records_cmd.records)
+app.command(name="tf-get", group=inspect_group)(tf_get_cmd.tf_get)
 app.command(name="tftree", group=inspect_group)(tftree_cmd.tftree)
 app.command(name="topic-chunks", group=inspect_group)(topic_chunks_cmd.topic_chunks)
 

pymcap_cli-0.12.0/src/pymcap_cli/cmd/_rechunk_strategy.py

@@ -0,0 +1,42 @@
+"""CLI-level rechunking strategy enum.
+
+Lives in the cmd layer because it is purely a user-facing selector that the
+``rechunk`` and ``process`` commands translate into concrete
+``OutputProcessor`` instances. The core processor itself is strategy-agnostic.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import TYPE_CHECKING
+
+from pymcap_cli.core.processors.chunk_groupers import (
+    PatternGrouper,
+    PerChannelGrouper,
+)
+
+if TYPE_CHECKING:
+    from re import Pattern
+
+    from pymcap_cli.core.processors.base import OutputProcessor
+
+
+class RechunkStrategy(str, Enum):
+    """User-facing rechunking strategy."""
+
+    NONE = "none"  # No rechunking — fast-copy optimization when possible
+    PATTERN = "pattern"  # Group by topic / schema regex
+    ALL = "all"  # Each channel in its own chunk group
+
+
+def build_output_processors(
+    strategy: RechunkStrategy,
+    topic_patterns: list[Pattern[str]] | None = None,
+    schema_patterns: list[Pattern[str]] | None = None,
+) -> list[OutputProcessor]:
+    """Translate a CLI rechunking strategy into concrete OutputProcessors."""
+    if strategy == RechunkStrategy.NONE:
+        return []
+    if strategy == RechunkStrategy.ALL:
+        return [PerChannelGrouper()]
+    return [PatternGrouper(topic_patterns or [], schema_patterns or [])]

{pymcap_cli-0.10.0 → pymcap_cli-0.12.0}/src/pymcap_cli/cmd/diff_cmd.py

@@ -20,11 +20,14 @@ from pymcap_cli.core.mcap_compare import (
     IndexedCompareKind,
     IndexedComparison,
     MessageIndexIdentityReadResult,
+    PayloadLookup,
     collect_message_timestamps,
     compare_indexed_identities,
     message_index_identity_from_info,
+    payload_lookup_from_info,
     read_compare_file,
     split_timestamps_into_segments,
+    verify_comparison_payloads,
 )
 from pymcap_cli.display.time_ranges import (
     format_count,
@@ -236,8 +239,13 @@ def _extract_summary(path: str, label: str, info: RebuildInfo, file_size: int) -
 
 
 def _process_file(
-    path: str, label: str
-) -> tuple[FileSummary, dict[int, set[int]], MessageIndexIdentityReadResult]:
+    path: str, label: str, *, compare_payloads: bool
+) -> tuple[
+    FileSummary,
+    dict[int, set[int]],
+    MessageIndexIdentityReadResult,
+    PayloadLookup | None,
+]:
     result = read_compare_file(path, rebuild_missing=True)
     indexed = MessageIndexIdentityReadResult(
         path=path,
@@ -245,10 +253,16 @@ def _process_file(
         identity=message_index_identity_from_info(result.info),
         read_mode=result.read_mode,
     )
+    payload_lookup = (
+        payload_lookup_from_info(path, result.size_bytes, result.info, result.read_mode)
+        if compare_payloads
+        else None
+    )
     return (
         _extract_summary(path, label, result.info, result.size_bytes),
         collect_message_timestamps(result.info),
         indexed,
+        payload_lookup,
     )
 
 
@@ -568,6 +582,8 @@ def _verdict_text(
         return "[yellow]edge overlap[/yellow]"
     if comparison.kind is IndexedCompareKind.PARTIAL_OVERLAP:
         return "[yellow]partial overlap[/yellow]"
+    if comparison.kind is IndexedCompareKind.PAYLOAD_MISMATCH:
+        return "[red]payload mismatch[/red] [dim](timestamps/indexes match)[/dim]"
     return "[red]no indexed message overlap[/red]"
 
 
@@ -593,6 +609,13 @@ def _build_smart_diff_tree(
             f"left-only {format_count(comparison.left_extra_messages, 'msg')}, "
             f"right-only {format_count(comparison.right_extra_messages, 'msg')})[/dim]"
         )
+        if comparison.payload_mismatch is not None:
+            mismatch = comparison.payload_mismatch
+            node.add(
+                f"[red]first mismatch[/red] [cyan]{escape(mismatch.topic)}[/cyan] "
+                f"[dim]@ {format_ts_short(mismatch.log_time)}: "
+                f"{escape(mismatch.reason)}[/dim]"
+            )
 
         for topic in comparison.topics[:max_topic_rows]:
             overlap_window = format_optional_time_window(
@@ -623,21 +646,34 @@ def _compare_to_first(
     labels: list[str],
     *,
     max_range_preview: int,
+    payload_lookups: list[PayloadLookup | None] | None = None,
 ) -> list[LabeledIndexedComparison]:
     first = indexed_results[0]
     first_label = labels[0]
-    return [
-        LabeledIndexedComparison(
-            left_label=first_label,
-            right_label=labels[index],
-            comparison=compare_indexed_identities(
-                first,
-                indexed_result,
-                max_range_preview=max_range_preview,
-            ),
+    comparisons: list[LabeledIndexedComparison] = []
+    for index, indexed_result in enumerate(indexed_results[1:], start=1):
+        comparison = compare_indexed_identities(
+            first,
+            indexed_result,
+            max_range_preview=max_range_preview,
+        )
+        if payload_lookups is not None:
+            left_lookup = payload_lookups[0]
+            right_lookup = payload_lookups[index]
+            if left_lookup is not None and right_lookup is not None:
+                comparison = verify_comparison_payloads(
+                    comparison,
+                    left_lookup,
+                    right_lookup,
+                )
+        comparisons.append(
+            LabeledIndexedComparison(
+                left_label=first_label,
+                right_label=labels[index],
+                comparison=comparison,
+            )
         )
-        for index, indexed_result in enumerate(indexed_results[1:], start=1)
-    ]
+    return comparisons
 
 
 def diff_cmd(
@@ -663,6 +699,16 @@ def diff_cmd(
             help="Maximum number of timestamp ranges to show per channel",
         ),
     ] = 3,
+    compare_payloads: Annotated[
+        bool,
+        Parameter(
+            name=["--compare-payloads"],
+            help=(
+                "After indexes match, compare raw message payload bytes and fail "
+                "fast on the first payload/header mismatch."
+            ),
+        ),
+    ] = False,
 ) -> int:
     """Compare MCAP files using summary and message index timestamps.
 
@@ -678,6 +724,8 @@ def diff_cmd(
         Hide channels where all message timestamps match exactly
     max_ranges
         Maximum timestamp ranges to display per channel (default: 3)
+    compare_payloads
+        Verify raw message payloads after the fast index comparison passes.
 
     Examples
     --------
@@ -700,12 +748,17 @@ def diff_cmd(
     all_timestamps: dict[str, dict[int, set[int]]] = {}
     all_summaries: dict[str, FileSummary] = {}
     indexed_results: list[MessageIndexIdentityReadResult] = []
+    payload_lookups: list[PayloadLookup | None] = []
     seen_labels: dict[str, int] = {}
 
     for path in files:
         label = _unique_label(path, seen_labels)
         try:
-            fs, ts, indexed = _process_file(path, label)
+            fs, ts, indexed, payload_lookup = _process_file(
+                path,
+                label,
+                compare_payloads=compare_payloads,
+            )
         except Exception:
             logger.exception(f"Error reading {path}")
             return 1
@@ -713,14 +766,20 @@ def diff_cmd(
         all_timestamps[label] = ts
         all_summaries[label] = fs
         indexed_results.append(indexed)
+        payload_lookups.append(payload_lookup)
 
     labels = [fs.label for fs in summaries]
     first_label = labels[0]
-    smart_comparisons = _compare_to_first(
-        indexed_results,
-        labels,
-        max_range_preview=max_ranges,
-    )
+    try:
+        smart_comparisons = _compare_to_first(
+            indexed_results,
+            labels,
+            max_range_preview=max_ranges,
+            payload_lookups=payload_lookups if compare_payloads else None,
+        )
+    except Exception:
+        logger.exception("Error comparing message payloads")
+        return 1
 
     channel_diffs = _compare_channels(all_timestamps, all_summaries)
     schema_diffs = _compare_schemas(all_summaries)
@@ -732,6 +791,9 @@ def diff_cmd(
     has_diffs = total_added > 0 or total_removed > 0
     has_schema_diffs = any(not d.is_identical for d in schema_diffs.values())
     has_mismatches = bool(channel_schema_mismatches)
+    has_payload_mismatches = any(
+        item.comparison.kind is IndexedCompareKind.PAYLOAD_MISMATCH for item in smart_comparisons
+    )
 
     console.print()
     console.print(_build_smart_diff_tree(smart_comparisons))
@@ -768,11 +830,19 @@ def diff_cmd(
             console.print(mismatch_table)
 
     console.print()
-    all_good = not has_diffs and not has_schema_diffs and not has_mismatches
+    all_good = (
+        not has_diffs and not has_schema_diffs and not has_mismatches and not has_payload_mismatches
+    )
     if all_good:
-        console.print(
-            f"[green]✓ All {total_common:,} messages have identical timestamps and schemas[/]"
-        )
+        if compare_payloads:
+            console.print(
+                f"[green]✓ All {total_common:,} messages have identical timestamps, "
+                "schemas and payloads[/]"
+            )
+        else:
+            console.print(
+                f"[green]✓ All {total_common:,} messages have identical timestamps and schemas[/]"
+            )
     else:
         if not has_diffs:
             console.print(f"[green]✓ All {total_common:,} messages have identical timestamps[/]")
@@ -790,5 +860,7 @@ def diff_cmd(
                 f"[red]⚠ {len(channel_schema_mismatches)} "
                 f"channel(s) use different schemas across files[/]"
             )
+        if has_payload_mismatches:
+            console.print("[red]⚠ message payloads differ across files[/]")
 
     return 0