fmu-manipulation-toolbox 1.9.2b2__tar.gz → 1.9.2b4__tar.gz

{fmu_manipulation_toolbox-1.9.2b2 → fmu_manipulation_toolbox-1.9.2b4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmu_manipulation_toolbox
-Version: 1.9.2b2
+Version: 1.9.2b4
 Summary: FMU Manipulation Toolbox is a python package which helps to analyze, modify or combine Functional Mock-up Units (FMUs) without recompilation.
 Home-page: https://github.com/grouperenault/fmu_manipulation_toolbox/
 Author: Nicolas.LAURENT@Renault.com
@@ -23,7 +23,7 @@ Dynamic: requires-python
 Dynamic: summary
 
 FMU Manipulation Toolbox is a python package which helps to analyze, modify or combine
-[Functional Mock-up Units (FMUs)](http://fmi-standard.org/) without recompilation. It is highly customizable and comes with
+[Functional Mock-up Units (FMUs)](https://fmi-standard.org/) without recompilation. It is highly customizable and comes with
 a Python API.
 
 FMU Manipulation Toolbox can be used in different ways:

fmu_manipulation_toolbox-1.9.2b4/fmu_manipulation_toolbox/__version__.py

@@ -0,0 +1 @@
+'V1.9.2b4'

fmu_manipulation_toolbox-1.9.2b4/fmu_manipulation_toolbox/cli/datalog2pcap.py

@@ -0,0 +1,143 @@
+import argparse
+import csv
+import logging
+import sys
+
+from pathlib import Path
+from typing import *
+
+from .utils import setup_logger, close_logger, make_wide
+from ..version import __version__ as version
+
+logger = setup_logger()
+
+class DatalogConverter:
+    def __init__(self, cvs_filename: Union[Path, str]):
+        self.csv_filename = Path(cvs_filename)
+        self.pcap_filename = self.csv_filename.with_suffix(".pcap")
+
+    def open_pcap(self):
+        logger.info(f"Creating PCAP file '{self.pcap_filename}'...")
+        file = open(self.pcap_filename, "wb")
+        file.write(int(0xA1B2C3D4).to_bytes(4, byteorder="big"))  # Magic number
+                                                                  # meaning the timestamp are in min seconds and microseconds
+
+        file.write(int(2).to_bytes(2, byteorder="big"))           # Major Version of fileformat
+        file.write(int(4).to_bytes(2, byteorder="big"))           # Minor Version
+        file.write(int(0).to_bytes(4, byteorder="big"))           # Reserved. SHOULD BE 0.
+        file.write(int(0).to_bytes(4, byteorder="big"))           # Reserved. SHOULD BE 0.
+
+        file.write(int(0xFFFF).to_bytes(4, byteorder="big"))      # snaplen indicating the maximum number of octets
+                                                                  # captured from each packet.
+
+        file.write(int(227).to_bytes(4, byteorder="big"))         # link type. his field is defined in the Section
+                                                                  # # 8.1 IANA registry.
+        return file
+
+    def open_csv(self):
+        logger.debug(f"Loading '{self.csv_filename}'")
+        file = open(self.csv_filename, "rt")
+        return file
+
+    def decode_hexstring(self, hex_string: bytes, time_s, time_us):
+        opcode = int.from_bytes(hex_string[0:4], byteorder="little")
+        length = int.from_bytes(hex_string[4:8], byteorder="little")
+        can_id = int.from_bytes(hex_string[8:12], byteorder="little")
+        if opcode == 0x10:  # TRANSMIT
+            rtr = int.from_bytes(hex_string[13:14], byteorder="little")
+            ide = int.from_bytes(hex_string[12:13], byteorder="little")
+            data_length = int.from_bytes(hex_string[14:16], byteorder="little")
+            raw_data = hex_string[16:]
+
+            logger.debug(f"time={time_s}.{time_us:06d} OP=0x{opcode:04X} len={length} {data_length} id={can_id}"
+                         f" ide={ide} rtr={rtr} len={data_length} {raw_data}")
+
+            # TimeStamp
+            self.pcapfile.write(time_s.to_bytes(4, byteorder="big"))
+            self.pcapfile.write(time_us.to_bytes(4, byteorder="big"))
+
+            # Packet length
+            packet_length = data_length + 8
+            self.pcapfile.write(packet_length.to_bytes(4, byteorder="big"))
+            self.pcapfile.write(packet_length.to_bytes(4, byteorder="big"))
+
+            # Control and flags
+            control = (can_id & 0x1FFFFFFF) | ((rtr & 0b1) << 30) | ((ide & 0b1) << 31)
+            self.pcapfile.write(control.to_bytes(4, byteorder="big"))
+
+            # Frame Length
+            self.pcapfile.write(data_length.to_bytes(1, byteorder="big"))
+
+            # Reserved
+            self.pcapfile.write(int(0).to_bytes(2, byteorder="big"))
+
+            # DLC
+            dlc = int(data_length)  # Classic CAN
+            self.pcapfile.write(dlc.to_bytes(1, byteorder="big"))
+
+            # PAYLOAD
+            self.pcapfile.write(raw_data)
+
+    def convert(self):
+        with self.open_csv() as self.csvfile, self.open_pcap() as self.pcapfile:
+            csv_reader = csv.DictReader(self.csvfile)
+
+            data_column_names = [ name for name in csv_reader.fieldnames if "_Data" in name ]
+            clock_column_names = {}
+
+            for column_name in data_column_names:
+                splitted_name = column_name.split(".")
+                splitted_name[-1] = splitted_name[-1].replace("_Data", "_Clock")
+                clock_column_name = ".".join(splitted_name)
+
+                if clock_column_name in csv_reader.fieldnames:
+                    logger.debug(f"{column_name} is clocked by {clock_column_name}")
+                    clock_column_names[column_name] = clock_column_name
+
+            for row in csv_reader:
+                time_s, time_us = divmod(float(row["time"]), 1)
+                time_s = int(time_s)
+                time_us = int(time_us * 1000000)
+
+                for names in data_column_names:
+                    hex_data = row[names]
+                    if hex_data:
+                        try:
+                            clock = row[clock_column_names[names]] == "1"
+                        except KeyError:
+                            clock = True
+                        if clock:
+                            self.decode_hexstring(bytes.fromhex(hex_data), time_s, time_us)
+
+
+def datalog2pcap():
+    logger.info(f"FMUContainer version {version}")
+    logger.warning(f"Datalog2PCAP is still experimental.")
+
+    parser = argparse.ArgumentParser(prog="datalog2pcap", description="Convert datalog from container to PCAP file.",
+                                     formatter_class=make_wide(argparse.ArgumentDefaultsHelpFormatter),
+                                     add_help=False,
+                                     epilog="see: https://github.com/grouperenault/fmu_manipulation_toolbox/blob/main/"
+                                            "doc/datalog.md")
+
+    parser.add_argument('-h', '-help', action="help")
+
+    parser.add_argument("-can", action="store", dest="can_filename", default=None,
+                        metavar="can-datalog.csv", required=True,
+                        help="Datalog with CAN data and clocks.")
+
+    parser.add_argument("-debug", action="store_true", dest="debug",
+                        help="Add lot of useful log during the process.")
+
+    config = parser.parse_args(sys.argv[1:])
+
+    if config.debug:
+        logger.setLevel(logging.DEBUG)
+
+    DatalogConverter(config.can_filename).convert()
+
+    close_logger(logger)
+
+
+if __name__ == "__main__":
+    datalog2pcap()

{fmu_manipulation_toolbox-1.9.2b2 → fmu_manipulation_toolbox-1.9.2b4}/fmu_manipulation_toolbox/container.py

@@ -95,6 +95,7 @@ class EmbeddedFMUPort:
         self.start_value = attrs.get("start", None)
         self.initial = attrs.get("initial", None)
         self.clock = attrs.get("clocks", None)
+        self.interval_variability = attrs.get("intervalVariability", None)
 
 
     def xml(self, vr: int, name=None, causality=None, start=None, fmi_version=2) -> str:
@@ -1106,8 +1107,8 @@ class FMUContainer:
             else:
                 local_per_type["clock"].append(link.vr)
                 for cport_to in link.cport_to_list:
-                    if cport_to.fmu.ls.is_bus:
-                        logger.info(f"LS-BUS: importer scheduling for '{cport_to.fmu.name}' '{cport_to.port.name}' (clock={cport_to.port.vr}, {link.vr})")
+                    if cport_to.port.interval_variability == "countdown":
+                        logger.info(f"LS-BUS: importer scheduling for '{cport_to.fmu.name}' '{cport_to.port.name}' (clock={cport_to.port.vr}, vr={link.vr})")
                         clock_list.append(cport_to, link.vr)
                         break
 
@@ -1179,7 +1180,7 @@ class FMUContainer:
         clock_list.write_txt(txt_file)
 
     def make_datalog(self, datalog_file):
-        print(f"# Datalog filename")
+        print(f"# Datalog filename", file=datalog_file)
         print(f"{self.identifier}-datalog.csv", file=datalog_file)
 
         ports = defaultdict(list)
@@ -1188,7 +1189,11 @@ class FMUContainer:
         for output_port_name, output_port in self.outputs.items():
             ports[output_port.port.type_name].append((output_port.vr, output_port_name))
         for link in self.links.values():
-            ports[link.cport_from.port.type_name].append((link.vr, link.name))
+            if link.cport_from is None:
+                # LS-BUS allows to connected to input clocks.
+                ports[link.cport_to_list[0].port.type_name].append((link.vr, link.name))
+            else:
+                ports[link.cport_from.port.type_name].append((link.vr, link.name))
 
         for type_name in EmbeddedFMUPort.ALL_TYPES:
             print(f"# {type_name}: <VR> <NAME>" , file=datalog_file)

{fmu_manipulation_toolbox-1.9.2b2 → fmu_manipulation_toolbox-1.9.2b4}/fmu_manipulation_toolbox/operations.py

@@ -61,7 +61,7 @@ class FMUPort:
         self.dimension = None
 
     def dict_level(self, nb):
-        return " ".join([f'{key}="{value}"' for key, value in self.attrs_list[nb].items()])
+        return " ".join([f'{key}="{Manipulation.escape(value)}"' for key, value in self.attrs_list[nb].items()])
 
     def write_xml(self, fmi_version: int, file):
         if fmi_version == 2:
@@ -177,7 +177,7 @@ class Manipulation:
                 self.current_port = FMUPort()
                 self.current_port.push_attrs(attrs)
             elif self.fmu.fmi_version == 2 and name in self.fmu.FMI2_TYPES:
-                if self.current_port: # <Enumeration> can be found before port defition. Ignored.
+                if self.current_port: # <Enumeration> can be found before port definition. Ignored.
                     self.current_port.fmi_type = name
                     self.current_port.push_attrs(attrs)
             elif self.fmu.fmi_version == 3 and name in self.fmu.FMI3_TYPES:

{fmu_manipulation_toolbox-1.9.2b2 → fmu_manipulation_toolbox-1.9.2b4}/fmu_manipulation_toolbox.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmu_manipulation_toolbox
-Version: 1.9.2b2
+Version: 1.9.2b4
 Summary: FMU Manipulation Toolbox is a python package which helps to analyze, modify or combine Functional Mock-up Units (FMUs) without recompilation.
 Home-page: https://github.com/grouperenault/fmu_manipulation_toolbox/
 Author: Nicolas.LAURENT@Renault.com
@@ -23,7 +23,7 @@ Dynamic: requires-python
 Dynamic: summary
 
 FMU Manipulation Toolbox is a python package which helps to analyze, modify or combine
-[Functional Mock-up Units (FMUs)](http://fmi-standard.org/) without recompilation. It is highly customizable and comes with
+[Functional Mock-up Units (FMUs)](https://fmi-standard.org/) without recompilation. It is highly customizable and comes with
 a Python API.
 
 FMU Manipulation Toolbox can be used in different ways:

{fmu_manipulation_toolbox-1.9.2b2 → fmu_manipulation_toolbox-1.9.2b4}/fmu_manipulation_toolbox.egg-info/SOURCES.txt

@@ -22,6 +22,7 @@ fmu_manipulation_toolbox.egg-info/entry_points.txt
 fmu_manipulation_toolbox.egg-info/requires.txt
 fmu_manipulation_toolbox.egg-info/top_level.txt
 fmu_manipulation_toolbox/cli/__init__.py
+fmu_manipulation_toolbox/cli/datalog2pcap.py
 fmu_manipulation_toolbox/cli/fmucontainer.py
 fmu_manipulation_toolbox/cli/fmusplit.py
 fmu_manipulation_toolbox/cli/fmutool.py

{fmu_manipulation_toolbox-1.9.2b2 → fmu_manipulation_toolbox-1.9.2b4}/fmu_manipulation_toolbox.egg-info/entry_points.txt

@@ -1,4 +1,5 @@
 [console_scripts]
+datalog2pcap = fmu_manipulation_toolbox.cli.datalog2pcap:datalog2pcap
 fmucontainer = fmu_manipulation_toolbox.cli.fmucontainer:fmucontainer
 fmusplit = fmu_manipulation_toolbox.cli.fmusplit:fmusplit
 fmutool = fmu_manipulation_toolbox.cli.fmutool:fmutool

{fmu_manipulation_toolbox-1.9.2b2 → fmu_manipulation_toolbox-1.9.2b4}/setup.py

@@ -44,7 +44,8 @@ setup(
     ]},
     entry_points={"console_scripts": ["fmutool = fmu_manipulation_toolbox.cli.fmutool:fmutool",
                                       "fmucontainer = fmu_manipulation_toolbox.cli.fmucontainer:fmucontainer",
-                                      "fmusplit = fmu_manipulation_toolbox.cli.fmusplit:fmusplit"
+                                      "fmusplit = fmu_manipulation_toolbox.cli.fmusplit:fmusplit",
+                                      "datalog2pcap = fmu_manipulation_toolbox.cli.datalog2pcap:datalog2pcap",
                                       ],
                   "gui_scripts": ["fmutool-gui = fmu_manipulation_toolbox.gui.gui:main"]
                   },
@@ -53,7 +54,7 @@ setup(
     description="FMU Manipulation Toolbox is a python package which helps to analyze, modify or combine "
                 "Functional Mock-up Units (FMUs) without recompilation.",
     long_description="""FMU Manipulation Toolbox is a python package which helps to analyze, modify or combine
-[Functional Mock-up Units (FMUs)](http://fmi-standard.org/) without recompilation. It is highly customizable and comes with
+[Functional Mock-up Units (FMUs)](https://fmi-standard.org/) without recompilation. It is highly customizable and comes with
 a Python API.
 
 FMU Manipulation Toolbox can be used in different ways:

{fmu_manipulation_toolbox-1.9.2b2 → fmu_manipulation_toolbox-1.9.2b4}/tests/test_suite.py

@@ -1,3 +1,4 @@
+import hashlib
 import numpy as np
 import pytest
 import sys
@@ -14,6 +15,7 @@ from fmu_manipulation_toolbox.assembly import *
 from fmu_manipulation_toolbox.cli.fmusplit import fmusplit
 from fmu_manipulation_toolbox.cli.fmucontainer import fmucontainer
 from fmu_manipulation_toolbox.cli.fmutool import fmutool
+from fmu_manipulation_toolbox.cli.datalog2pcap import datalog2pcap
 
 
 class TestSuite:
@@ -49,17 +51,23 @@ class TestSuite:
         with open(log_filename, mode="rt", newline=None) as a, open(ref_filename, mode="rt", newline=None) as b:
             for i, (lineA, lineB) in enumerate(zip(a, b)):
                 if i > 10:
-                    assert(lineA == lineB)
+                    assert lineA == lineB, \
+                       f"files {log_filename} and {ref_filename} missmatch (excl. GUID):\n" \
+                       f"{lineA}\n" \
+                       f"vs.\n\n" \
+                       f"{lineB}"
 
     @staticmethod
     def assert_identical_files(filename1, filename2):
+        assert Path(filename1).exists(), f"{filename1} does not exist"
+        assert Path(filename2).exists(), f"{filename2} does not exist"
         with open(filename1, mode="rt", newline=None) as a, open(filename2, mode="rt", newline=None) as b:
             for lineA, lineB in zip(a, b):
-                assert(lineA == lineB,
-                       f"file {filename1} and {filename2} missmatch (excl. GUID):\n"
-                       f"{lineA}\n"
-                       f"vs.\n\n"
-                       f"{lineB}")
+                assert lineA == lineB, \
+                       f"file {filename1} and {filename2} missmatch (excl. GUID):\n" \
+                       f"{lineA}\n" \
+                       f"vs.\n\n" \
+                       f"{lineB}"
 
     @staticmethod
     def assert_identical_files_but_guid(filename1, filename2):
@@ -71,11 +79,11 @@ class TestSuite:
                     if keyword in lineA:
                         skip = True
                         break
-                assert(skip or lineA == lineB,
-                       f"file {filename1} and {filename2} missmatch:\n"
-                       f"{lineA}\n"
-                       f"vs.\n\n"
-                       f"{lineB}")
+                assert skip or lineA == lineB, \
+                       f"file {filename1} and {filename2} missmatch:\n" \
+                       f"{lineA}\n" \
+                       f"vs.\n\n" \
+                       f"{lineB}"
 
     def assert_names_match_ref(self, fmu_filename):
         fmu = FMU(fmu_filename)
@@ -91,9 +99,19 @@ class TestSuite:
         fmu.repack(fmu_filename)
         self.assert_names_match_ref(fmu_filename)
 
+    @staticmethod
+    def assert_md5(filename, expected_md5):
+        hash_md5 = hashlib.md5()
+        with open(filename, "rb") as f:
+            for chunk in iter(lambda: f.read(4096), b""):
+                hash_md5.update(chunk)
+        print(f"{filename}: {expected_md5} {hash_md5.hexdigest()}")
+        assert hash_md5.hexdigest() == expected_md5 ,\
+            f"Wrong md5 hash for {filename}. Expected {expected_md5} but got {hash_md5.hexdigest()}"
+
     @staticmethod
     def assert_file_exist(path):
-        assert(Path(path).exists())
+        assert Path(path).exists()
 
     def test_strip_top_level(self):
         self.assert_operation_match_ref("operations/bouncing_ball-no-tl.fmu", OperationStripTopLevel())
@@ -295,16 +313,21 @@ class TestSuite:
         sys.argv = ['fmusplit',
                     "-fmu", "containers/ssp/bouncing.fmu"]
         fmusplit()
-        assert(Path("containers/ssp/bouncing.dir/bb_position.fmu").exists())
-        assert(Path("containers/ssp/bouncing.dir/bb_velocity.fmu").exists())
+        assert Path("containers/ssp/bouncing.dir/bb_position.fmu").exists()
+        assert Path("containers/ssp/bouncing.dir/bb_velocity.fmu").exists()
         self.assert_identical_files("containers/ssp/REF-split-bouncing.json",
                                     "containers/ssp/bouncing.dir/bouncing.json")
 
-    def test_ls_bus(self):
+    def test_ls_bus_nodes_and_bus(self):
         assembly = Assembly("bus+nodes.json", fmu_directory=Path("ls-bus"))
         assembly.make_fmu(fmi_version=3)
         self.assert_simulation_log("ls-bus/bus+nodes.fmu", 0.1)
 
+    def test_ls_bus_nodes_only(self):
+        assembly = Assembly("nodes-only.json", fmu_directory=Path("ls-bus"))
+        assembly.make_fmu(fmi_version=3, datalog=True)
+        self.assert_simulation_log("ls-bus/nodes-only.fmu", 0.1)
+
     def test_datalog(self):
         assembly = Assembly("bouncing.csv", fmu_directory=Path("containers/bouncing_ball"), mt=True, debug=True)
         assembly.make_fmu(filename="bouncing-datalog.fmu", datalog=True)
@@ -322,3 +345,9 @@ class TestSuite:
         self.assert_simulation("containers/VanDerPol/VanDerPol-datalog.fmu", 0.1)
         self.assert_file_exist("VanDerPol-Container-datalog.csv")
 
+    def test_datalog_pcap(self):
+        sys.argv = ['datalog2pcap',
+                    '-can', 'ls-bus/REF-nodes-only-datalog.csv']
+        datalog2pcap()
+
+        self.assert_md5("ls-bus/REF-nodes-only-datalog.pcap", "ceab6b0161dbc93458bd47c057e80375")

fmu_manipulation_toolbox-1.9.2b2/fmu_manipulation_toolbox/__version__.py

@@ -1 +0,0 @@
-'V1.9.2b2'