mcap-bag-parser 0.1.0__tar.gz

mcap_bag_parser-0.1.0/.gitattributes

@@ -0,0 +1 @@
+*.mcap filter=lfs diff=lfs merge=lfs -text

mcap_bag_parser-0.1.0/.gitignore

@@ -0,0 +1,3 @@
+.idea
+*.pickle
+__pycache__

mcap_bag_parser-0.1.0/.gitlab-ci.yml

@@ -0,0 +1,22 @@
+stages:
+    - linting
+    - testing
+
+flake8:
+    stage: linting
+    image: python:3.10
+
+    script:
+        - pip install flake8 pep8-naming
+        - flake8 . --count --max-complexity=20 --max-line-length=127 --statistics
+    only:
+        - merge_requests
+
+pytest:
+    stage: testing
+    image: python:3.10
+    script:
+        - pip install -r requirements.txt
+        - python -m pytest
+    only:
+        - merge_requests

mcap_bag_parser-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Neal Tanner
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

mcap_bag_parser-0.1.0/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.3
+Name: mcap_bag_parser
+Version: 0.1.0
+Summary: Parse mcap bagfiles into pandas dataframes
+Project-URL: Homepage, https://gitlab.com/nealtanner/mcap-bag-parser
+Project-URL: Issues, https://gitlab.com/nealtanner/mcap-bag-parser/-/issues
+Author: Neal Tanner, Teresa Gadda
+License-File: LICENSE
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+# MCAP Bag Parser
+
+Parse MCAP rosbags into pandas dataframes

mcap_bag_parser-0.1.0/README.md

@@ -0,0 +1,3 @@
+# MCAP Bag Parser
+
+Parse MCAP rosbags into pandas dataframes

mcap_bag_parser-0.1.0/pyproject.toml

@@ -0,0 +1,26 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "mcap_bag_parser"
+version = "0.1.0"
+authors = [
+  { name="Neal Tanner"},
+  { name="Teresa Gadda"},
+]
+description = "Parse mcap bagfiles into pandas dataframes"
+readme = "README.md"
+requires-python = ">=3.8"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+
+[project.urls]
+Homepage = "https://gitlab.com/nealtanner/mcap-bag-parser"
+Issues = "https://gitlab.com/nealtanner/mcap-bag-parser/-/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/mcap_bag_parser"]

mcap_bag_parser-0.1.0/requirements.txt

@@ -0,0 +1,5 @@
+mcap
+mcap-ros2-support
+pandas
+numpy
+pytest

mcap_bag_parser-0.1.0/src/mcap_bag_parser/mcap_bag_parser.py

@@ -0,0 +1,325 @@
+import functools
+import pathlib
+from collections import defaultdict
+import pickle
+from typing import Iterable, Optional
+
+import pandas as pd
+from mcap.reader import make_reader
+from mcap_ros2.decoder import DecoderFactory
+
+MAX_ARRAY_SIZE_TO_SPLIT = 10
+
+
+def read_messages(bag_file: str,
+                  topics: Optional[Iterable[str]] = None,
+                  start_time: Optional[int] = None,
+                  end_time: Optional[int] = None,
+                  log_time_order: bool = True,
+                  reverse: bool = False):
+    """ iterates through the messages in an MCAP.
+
+    Parameters:
+        bag_file: path/name of bag file
+        topics: if not None, only messages from these topics will be returned.
+        start_time: an integer nanosecond timestamp. messages logged before this timestamp are not included.
+        end_time: an integer nanosecond timestamp. messages logged after this timestamp are not included.
+        log_time_order: if True, messages will be yielded in ascending log time order.
+                        if False, messages will be yielded in the order they appear in the MCAP file.
+        reverse: if both `log_time_order` and `reverse` are True, messages will be
+                 yielded in descending log time order.
+    """
+    with open(bag_file, "rb") as fobj:
+        reader = make_reader(fobj, decoder_factories=[DecoderFactory()])
+        for schema, channel, message, ros_msg in reader.iter_decoded_messages(topics=topics,
+                                                                              start_time=start_time,
+                                                                              end_time=end_time,
+                                                                              log_time_order=log_time_order,
+                                                                              reverse=reverse):
+            yield channel.topic, ros_msg, message.log_time
+
+
+def members(msg):
+    """ Return the public members of a message """
+    return [attr for attr in dir(msg) if
+            not callable(getattr(msg, attr)) and
+            not attr.startswith("_") and
+            attr not in ['real', 'imag', 'numerator', 'denominator']]
+
+
+# https://stackoverflow.com/a/31174427
+def rgetattr(obj, attr, *args):
+    def _getattr(obj, attr):
+        return getattr(obj, attr, *args)
+    return functools.reduce(_getattr, [obj] + attr.split('.'))
+
+
+# https://docs.python.org/3/library/operator.html#operator.attrgetter
+def resolve_attr(obj, attr):
+    for name in attr.split("."):
+        if name in [f'{idx}' for idx in range(100)]:
+            # interpret field names of simple numbers as array indices
+            try:
+                idx = int(name)
+                obj = obj[idx]
+            except IndexError:
+                obj = None
+        else:
+            obj = getattr(obj, name)
+    return obj
+
+
+class MessageSchema:
+    """ Describes how we decompose a message into a DataFrame entry """
+
+    def __init__(self, topic, msg, prefix_with_topic=False):
+        """ Generate the schema from an example message """
+        self._topic = topic
+        self._name_indexed_fields = []
+        self._names = None
+        self._field_map = {}  # alternative approach
+
+        if 'name' in members(msg) and isinstance(getattr(msg, 'name'), Iterable):
+            # Lots of ROS messages use parallel arrays that are indexed by a `name` array
+            # We want to unwrap these for the user
+            # TODO: figure out how to handle multiple messages on same topic with different name lists
+            # TODO: Push parsing of name-indexed arrays down into recursive approach below so that
+            #       it works at more than just the top level
+            self._names = getattr(msg, 'name')
+            for member_name in members(msg):
+                member_var = getattr(msg, member_name)
+                if member_name == 'name':
+                    pass
+                elif isinstance(member_var, Iterable) \
+                        and len(member_var) == len(self._names) \
+                        and not isinstance(member_var, str):
+                    # This field is the same length as the name field, so we will assume they are paired
+                    self._name_indexed_fields.append(member_name)
+                    for idx, name in enumerate(self._names):
+                        self._parse_field(key_name=f'{member_name}.{name}',
+                                          data_name=f'{member_name}.{idx}',
+                                          var=member_var[idx])
+                else:
+                    # not name-index, so we'll deal with it separately
+                    pass
+
+        for member_name in members(msg):
+            member_var = getattr(msg, member_name)
+            if member_name in self._name_indexed_fields or member_name == 'name':
+                # already dealt with these
+                pass
+            else:
+                self._parse_field(key_name=member_name, data_name=member_name, var=member_var)
+
+        if prefix_with_topic:
+            prefixed_field_map = {}
+            for column, member_name in self._field_map.items():
+                if column in ['msg_timestamp', 'header', 'stamp']:
+                    # there are a few special fields that we do NOT want to be unique between topics
+                    prefixed_field_map[column] = member_name
+                else:
+                    prefixed_field_map[f'{self._topic}.{column}'] = member_name
+            self._field_map = prefixed_field_map
+
+        # print(f'Header for {self._topic} is {self.header}')
+
+    def _parse_field(self, key_name, data_name, var):
+        """ Parse an individual field in the data structure, likely to recurse! """
+        # print(f'parse_field({key_name}, {data_name}, {var}')
+        # print(f'members = {members(var)}')
+        # If this looks like a struct, recurse down into each named member
+        if len(members(var)):
+            for member in members(var):
+                member_var = getattr(var, member)
+                self._parse_field(key_name=f'{key_name}.{member}',
+                                  data_name=f'{data_name}.{member}',
+                                  var=member_var)
+
+        # If this looks like an array, recurse down into each numbered element
+        elif isinstance(var, Iterable) \
+                and len(var) <= MAX_ARRAY_SIZE_TO_SPLIT \
+                and not isinstance(var, str) \
+                and key_name not in ['arg', 'changed_parameters', 'deleted_parameters']:
+            # TODO: figure out how to handle arrays (such as `arg` field) with variable lengths message to message
+            for idx in range(len(var)):
+                # recurse down into each sub_member
+                self._parse_field(key_name=f'{key_name}.{idx}',
+                                  data_name=f'{data_name}.{idx}',
+                                  var=var[idx])
+            else:  # The thing in the array looks like a scalar, so go ahead and add it to the field map
+                self._field_map.update(
+                    {f'{key_name}.{idx}': f'{data_name}.{idx}' for idx in range(len(var))})
+
+        # Otherwise, presume that we have reached the end of the tree and add this field to the map
+        elif key_name != 'name':
+            # No special situations found, so just use each member as its own column
+            self._field_map[f'{key_name}'] = f'{data_name}'
+
+    @property
+    def header(self):
+        return tuple(['msg_timestamp'] + list(self._field_map.keys()))
+
+    def generate_row(self, timestamp, msg):
+        """ Use the pre-determined schema to generate one row in the dataframe """
+        row_data = [timestamp]
+
+        for header, member_name in self._field_map.items():
+            row_data.append(resolve_attr(msg, member_name))
+
+        return row_data
+
+
+class BagFileParser:
+    """ Class-based parser to mimic existing sql-based parser interface """
+
+    def __init__(self, bag_file, use_pickle_if_exists=False):
+        self._bag_file: pathlib.Path = bag_file
+        self._pickle_file = pathlib.Path(str(self._bag_file).replace('.mcap', '.pickle'))
+
+        # Write to/read from a locally stored dataframe for faster post-processing
+        self._use_pickle_if_exists: bool = use_pickle_if_exists
+
+    def _get_summary(self):
+        with open(self._bag_file, "rb") as fobj:
+            reader = make_reader(fobj, decoder_factories=[DecoderFactory()])
+            summary = reader.get_summary()
+        return summary
+
+    def set_pickle_path(self, path):
+        self._pickle_file = path
+
+    @property
+    def message_start_time(self):
+        return self._get_summary().statistics.message_start_time
+
+    @property
+    def message_end_time(self):
+        return self._get_summary().statistics.message_end_time
+
+    @property
+    def topics(self):
+        summary = self._get_summary()
+        topics = [channel.topic for idx, channel in summary.channels.items()]
+        return tuple(topics)
+
+    @property
+    def message_counts(self):
+        summary = self._get_summary()
+        message_counts = {}
+        for idx, count in summary.statistics.channel_message_counts.items():
+            message_counts[summary.channels[idx].topic] = count
+        return message_counts
+
+    def read_messages(self, *args, **kwargs):
+        return read_messages(self._bag_file, *args, **kwargs)
+
+    def topic_to_dataframe(self, topic: str,
+                           start_time: Optional[int] = None,  # [ns]
+                           end_time: Optional[int] = None,  # [ns]
+                           rel_start_time: Optional[float] = None,  # [s]
+                           rel_end_time: Optional[float] = None,):  # [s]
+        """ Generate a pandas.DataFrame from a single topic  """
+        if rel_start_time is not None:
+            start_time = self.message_start_time + int(1E9*rel_start_time)
+        if rel_end_time is not None:
+            end_time = self.message_start_time + int(1E9*rel_end_time)
+
+        # TODO: fix message counts for loading a partial bagfile
+        print(f'Expect {self.message_counts[topic]} messages for {topic}')
+        schema = None  # have to wait until we get the first message
+        data = []
+        msg_count = 0
+        for topic, msg, time in self.read_messages(topics=[topic], start_time=start_time, end_time=end_time):
+            if schema is None:
+                # figure out how we are going to represent this type of message in the dataframe
+                schema = MessageSchema(topic=topic, msg=msg)
+            data.append(schema.generate_row(timestamp=time, msg=msg))
+            msg_count += 1
+            if msg_count % 10000 == 0:
+                print(f'Processed {msg_count} messages')
+        print('Generating data frame')
+        df = pd.DataFrame(data, columns=schema.header)
+        df.set_index('msg_timestamp', inplace=True)
+        return df
+
+    def has_messages_for_topic(self, topic: str) -> bool:
+        return topic in self.message_counts
+
+    def check_for_existing_data(self) -> bool:
+        if pathlib.Path.exists(self._pickle_file):
+            return True
+        return False
+
+    def to_dataframe(self, topics: Optional[Iterable[str]] = None,
+                     start_time: Optional[int] = None,  # [ns]
+                     end_time: Optional[int] = None,  # [ns]
+                     rel_start_time: Optional[float] = None,  # [s]
+                     rel_end_time: Optional[float] = None,  # [s]
+                     fillna=False,
+                     generate_relative_timestamps=True):
+        """ Generate a pandas.DataFrame from a list of topics
+
+        DataFrame field names will be prepended with the topic name
+        """
+        if rel_start_time is not None:
+            start_time = self.message_start_time + int(1E9*rel_start_time)
+        if rel_end_time is not None:
+            end_time = self.message_start_time + int(1E9*rel_end_time)
+
+        if self._use_pickle_if_exists:
+            # Check if we already have the dataframe downloaded. If so, just use that, instead of re-parsing the MCAP.
+            if self.check_for_existing_data():
+                with open(self._pickle_file, 'rb') as f:
+                    # The protocol version used is detected automatically, so we do not
+                    # have to specify it.
+                    mega_data_frame = pickle.load(f)
+                    return mega_data_frame
+            else:
+                print(f'Data not found at {self._pickle_file}. Proceeding to MCAP parsing...')
+
+        if topics is None:
+            print(f'Expected message counts are {self.message_counts}')
+        else:
+            for topic in topics:
+                print(f'Expect {self.message_counts[topic]} messages for {topic}')
+
+        schemas = {}
+        data_tables = defaultdict(list)
+        msg_count = 0
+        for topic, msg, time in self.read_messages(topics=topics, start_time=start_time, end_time=end_time):
+            if topic not in schemas:
+                print(f'Trying to infer schema for {topic}')
+                # figure out how we are going to represent this type of message in the dataframe
+                schemas[topic] = MessageSchema(topic=topic, msg=msg, prefix_with_topic=True)
+            data_tables[topic].append(schemas[topic].generate_row(timestamp=time, msg=msg))
+            msg_count += 1
+            if msg_count % 10000 == 0:
+                print(f'Processed {msg_count} messages')
+
+        # create a dataframe for each topic, and then concatenate
+        print('Generating data frames')
+        dataframes = []
+        for topic in schemas:
+            dataframes.append(pd.DataFrame(data_tables[topic], columns=schemas[topic].header))
+            dataframes[-1].set_index('msg_timestamp', drop=False, inplace=True)
+        print('Concatenating data frames')
+        mega_data_frame = pd.concat(dataframes)
+        print('Sorting resultant data frame')
+        mega_data_frame.sort_index(inplace=True)
+
+        if generate_relative_timestamps:
+            print('Creating relative timestamps')
+            mega_data_frame['rel_timestamp'] = (mega_data_frame.index - self.message_start_time) / 1E9
+            mega_data_frame.set_index('rel_timestamp', drop=False, inplace=True)
+        if fillna:
+            print('Filling NaNs')
+            mega_data_frame.fillna(method='ffill', inplace=True)
+
+        if self._use_pickle_if_exists:
+            # If we've gotten here, it's because we didn't find an existing pickle, so let's create one for this data
+            with open(self._pickle_file, 'wb') as f:
+                pickle.dump(mega_data_frame, f, pickle.HIGHEST_PROTOCOL)
+                print(f'Stored data frame at: {self._pickle_file}')
+
+        print('Done!')
+        return mega_data_frame

mcap_bag_parser-0.1.0/src/mcap_bag_parser/test/bagfile/metadata.yaml

@@ -0,0 +1,272 @@
+rosbag2_bagfile_information:
+  version: 5
+  storage_identifier: mcap
+  duration:
+    nanoseconds: 42730368608
+  starting_time:
+    nanoseconds_since_epoch: 1694016529830946819
+  message_count: 25387
+  topics_with_message_count:
+    - topic_metadata:
+        name: /visualizer/secondary_camera_pose
+        type: geometry_msgs/msg/Pose
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 13
+    - topic_metadata:
+        name: /visualizer/primary_camera_pose
+        type: geometry_msgs/msg/Pose
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 13
+    - topic_metadata:
+        name: /robot/sensor_status
+        type: capstan/msg/RobotSensorStatus
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 76
+    - topic_metadata:
+        name: /device/state
+        type: capstan/msg/DeviceState
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 3999
+    - topic_metadata:
+        name: /operator_interface/input_event
+        type: capstan/msg/StateMachineEvent
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /reference_hardstops/_action/status
+        type: action_msgs/msg/GoalStatusArray
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 1\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /robot/simulate_device_installed
+        type: std_msgs/msg/Bool
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /system/state
+        type: std_msgs/msg/String
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 189
+    - topic_metadata:
+        name: /fault/action
+        type: capstan/msg/FaultAction
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /robot/event
+        type: std_msgs/msg/String
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /reference_hardstops/_action/feedback
+        type: capstan/action/ReferenceHardstops_FeedbackMessage
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /device/commanded_pose
+        type: capstan/msg/DevicePose
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 3998
+    - topic_metadata:
+        name: /device/command
+        type: capstan/msg/StateMachineEvent
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 6
+    - topic_metadata:
+        name: /device/status
+        type: capstan/msg/DeviceStatus
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 3999
+    - topic_metadata:
+        name: /robot/joint_command
+        type: capstan/msg/RobotJointCommand
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 3998
+    - topic_metadata:
+        name: /user_input/implant
+        type: capstan/msg/Implant
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /presence/state
+        type: std_msgs/msg/String
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 176
+    - topic_metadata:
+        name: /device/event
+        type: capstan/msg/StateMachineEvent
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 2
+    - topic_metadata:
+        name: /user_input/implant_twist
+        type: geometry_msgs/msg/Twist
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /rosout
+        type: rcl_interfaces/msg/Log
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 1\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 10\n    nsec: 0\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false\n- history: 3\n  depth: 0\n  reliability: 1\n  durability: 1\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 10\n    nsec: 0\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false\n- history: 3\n  depth: 0\n  reliability: 1\n  durability: 1\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 10\n    nsec: 0\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false\n- history: 3\n  depth: 0\n  reliability: 1\n  durability: 1\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 10\n    nsec: 0\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false\n- history: 3\n  depth: 0\n  reliability: 1\n  durability: 1\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 10\n    nsec: 0\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 166
+    - topic_metadata:
+        name: /user_input/pose_absolute
+        type: capstan/msg/DevicePose
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /robot/joint_override
+        type: capstan/msg/RobotJointState
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /user_input/lever
+        type: capstan/msg/Lever
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /robot/joint_state
+        type: capstan/msg/RobotJointState
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 4217
+    - topic_metadata:
+        name: /device/estimated_pose
+        type: capstan/msg/DevicePose
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 3998
+    - topic_metadata:
+        name: /fault/detail
+        type: capstan/msg/FaultDetail
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 85
+    - topic_metadata:
+        name: /joy
+        type: sensor_msgs/msg/Joy
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /carriage_clutch/state
+        type: std_msgs/msg/String
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 171
+    - topic_metadata:
+        name: /fault/config
+        type: capstan/msg/FaultConfig
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /imu/camera_orientation
+        type: capstan/msg/CameraOrientation
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /fault/status
+        type: capstan/msg/FaultStatus
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 85
+    - topic_metadata:
+        name: /motionless_self_test/_action/feedback
+        type: capstan/action/MotionlessSelfTest_FeedbackMessage
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /user/event
+        type: capstan/msg/StateMachineEvent
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 1
+    - topic_metadata:
+        name: /joy/set_feebback
+        type: sensor_msgs/msg/JoyFeedback
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /events/write_split
+        type: rosbag2_interfaces/msg/WriteSplitEvent
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /user_input/angle_plane
+        type: capstan/msg/AnglePlane
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /motionless_self_test/_action/status
+        type: action_msgs/msg/GoalStatusArray
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 1\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 2
+    - topic_metadata:
+        name: /visualizer/device_pose
+        type: geometry_msgs/msg/Pose
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 12
+    - topic_metadata:
+        name: /user_input/auto_relax
+        type: capstan/msg/AutoRelax
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 0
+    - topic_metadata:
+        name: /operator_interface/state
+        type: std_msgs/msg/String
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 171
+    - topic_metadata:
+        name: /parameter_events
+        type: rcl_interfaces/msg/ParameterEvent
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false\n- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false\n- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false\n- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 9
+    - topic_metadata:
+        name: /operator_interface/event
+        type: capstan/msg/StateMachineEvent
+        serialization_format: cdr
+        offered_qos_profiles: "- history: 3\n  depth: 0\n  reliability: 1\n  durability: 2\n  deadline:\n    sec: 9223372036\n    nsec: 854775807\n  lifespan:\n    sec: 9223372036\n    nsec: 854775807\n  liveliness: 1\n  liveliness_lease_duration:\n    sec: 9223372036\n    nsec: 854775807\n  avoid_ros_namespace_conventions: false"
+      message_count: 1
+  compression_format: ""
+  compression_mode: ""
+  relative_file_paths:
+    - bagfile_0.mcap
+  files:
+    - path: bagfile_0.mcap
+      starting_time:
+        nanoseconds_since_epoch: 1694016529830946819
+      duration:
+        nanoseconds: 42730368608
+      message_count: 25387

mcap_bag_parser-0.1.0/src/mcap_bag_parser/test/test_mcap_bag_parser.py

@@ -0,0 +1,113 @@
+import pytest
+import pandas as pd
+from collections import defaultdict
+import sys
+import os
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import mcap_bag_parser  # noqa: E402
+
+
+SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__))
+
+
+def test_always_passes():
+    assert True
+
+
+@pytest.fixture
+def parser():
+    return mcap_bag_parser.BagFileParser(os.path.join(SCRIPT_DIR, 'bagfile', 'bagfile_0.mcap'))
+
+
+def test_read_messages():
+    num_msgs = defaultdict(lambda: 0)
+    for topic, msg, timestamp in mcap_bag_parser.read_messages(
+            os.path.join(SCRIPT_DIR, 'bagfile', 'bagfile_0.mcap')):
+        num_msgs[topic] = num_msgs[topic] + 1
+        # if topic in '/device/status':
+        #     print(f"{topic} [{timestamp}]: '{msg}'")
+
+    print(f'Found {num_msgs}')
+    assert num_msgs['/rosout'] == 166
+    assert num_msgs['/parameter_events'] == 9
+    assert num_msgs['/robot/joint_command'] == 3998
+    assert num_msgs['/device/estimated_pose'] == 3998
+    assert num_msgs['/device/status'] == 3999
+    assert num_msgs['/device/state'] == 3999
+    assert num_msgs['/device/command'] == 6
+
+
+def test_read_messages_through_class(parser):
+    num_msgs = defaultdict(lambda: 0)
+    for topic, msg, timestamp in parser.read_messages():
+        num_msgs[topic] = num_msgs[topic] + 1
+        # if topic in '/device/status':
+        #     print(f"{topic} [{timestamp}]: '{msg}'")
+
+    print(f'Found {num_msgs}')
+    assert num_msgs['/rosout'] == 166
+    assert num_msgs['/parameter_events'] == 9
+    assert num_msgs['/robot/joint_command'] == 3998
+    assert num_msgs['/device/estimated_pose'] == 3998
+    assert num_msgs['/device/status'] == 3999
+    assert num_msgs['/device/state'] == 3999
+    assert num_msgs['/device/command'] == 6
+
+
+def test_topic_to_dataframe(parser):
+    df = parser.topic_to_dataframe(topic='/device/command')
+    print(f'\n/device/command = \n{df}')
+    assert isinstance(df, pd.DataFrame)
+    assert len(df) == 6
+    assert list(df.columns) == ['arg', 'event']
+
+    df = parser.topic_to_dataframe(topic='/device/status')
+    print(f'\n/device/status = \n{df}')
+    assert isinstance(df, pd.DataFrame)
+    assert len(df) == 3999
+    assert 'active_driving_view' in df.columns
+    assert 'state' in df.columns
+
+    df = parser.topic_to_dataframe(topic='/robot/joint_command')
+    print(f'\n/robot/joint_command = \n{df}')
+    assert isinstance(df, pd.DataFrame)
+    assert len(df) == 3998
+    print(df.columns)
+    assert 'enable.C1_E' in df.columns
+    assert 'position.C3_ROLL' in df.columns
+
+    df = parser.topic_to_dataframe(topic='/device/estimated_pose')
+    print(f'\n/device/estimated_pose = \n{df}')
+    assert isinstance(df, pd.DataFrame)
+    assert len(df) == 3998
+    assert 'section.DISTAL.insertion' in df.columns
+    assert 'section.PROXIMAL.roll' in df.columns
+
+
+def test_to_dataframe(parser):
+    df = parser.to_dataframe(topics=['/device/command', '/device/status', '/robot/joint_command'])
+    print(f'\ncombined dataframe = \n{df}')
+    assert '/device/command.event' in df.columns
+    assert '/device/status.state' in df.columns
+    assert '/robot/joint_command.enable.C1_E' in df.columns
+
+
+def test_topics(parser):
+    topics = parser.topics
+    print(topics)
+    assert '/rosout' in topics
+    assert '/parameter_events' in topics
+    assert '/robot/joint_command' in topics
+    assert '/device/estimated_pose' in topics
+    assert '/device/status' in topics
+    assert '/device/state' in topics
+    assert '/device/estimated_pose' in topics
+    assert '/device/command' in topics
+
+
+def test_message_counts(parser):
+    message_counts = parser.message_counts
+    print(message_counts)
+    assert message_counts['/device/command'] == 6
+    assert message_counts['/device/estimated_pose'] == 3998
+    assert message_counts['/device/status'] == 3999

mcap_bag_parser-0.1.0/src/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+qt_api=pyqt5