fmu-manipulation-toolbox 1.9rc7__tar.gz → 1.9rc9__tar.gz

{fmu_manipulation_toolbox-1.9rc7 → fmu_manipulation_toolbox-1.9rc9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmu_manipulation_toolbox
-Version: 1.9rc7
+Version: 1.9rc9
 Summary: FMU Manipulation Toolbox is a python application for modifying Functional Mock-up Units (FMUs) without recompilation or bundling them into FMU Containers
 Home-page: https://github.com/grouperenault/fmu_manipulation_toolbox/
 Author: Nicolas.LAURENT@Renault.com

fmu_manipulation_toolbox-1.9rc9/fmu_manipulation_toolbox/__version__.py

@@ -0,0 +1 @@
+'V1.9-rc9'

{fmu_manipulation_toolbox-1.9rc7 → fmu_manipulation_toolbox-1.9rc9}/fmu_manipulation_toolbox/cli/fmucontainer.py

@@ -1,6 +1,8 @@
 import argparse
 import logging
+import sys
 
+from typing import *
 from pathlib import Path
 
 from .utils import setup_logger, make_wide
@@ -9,7 +11,7 @@ from ..container import FMUContainerError
 from ..version import __version__ as version
 
 
-def fmucontainer():
+def fmucontainer(command_options: Sequence[str]):
     logger = setup_logger()
 
     logger.info(f"FMUContainer version {version}")
@@ -63,7 +65,7 @@ def fmucontainer():
     parser.add_argument("-dump-json",  action="store_true", dest="dump", default=False,
                         help="Dump a JSON file for each container.")
 
-    config = parser.parse_args()
+    config = parser.parse_args(command_options)
 
     if config.debug:
         logger.setLevel(logging.DEBUG)
@@ -88,16 +90,16 @@ def fmucontainer():
                                 auto_parameter=config.auto_parameter)
         except FileNotFoundError as e:
             logger.fatal(f"Cannot read file: {e}")
-            continue
+            sys.exit(-1)
         except (FMUContainerError, AssemblyError) as e:
             logger.fatal(f"{filename}: {e}")
-            continue
+            sys.exit(-2)
 
         try:
             assembly.make_fmu(dump_json=config.dump, fmi_version=int(config.fmi_version))
         except FMUContainerError as e:
             logger.fatal(f"{filename}: {e}")
-            continue
+            sys.exit(-3)
 
 if __name__ == "__main__":
-    fmucontainer()
+    fmucontainer(sys.argv[1:])

{fmu_manipulation_toolbox-1.9rc7 → fmu_manipulation_toolbox-1.9rc9}/fmu_manipulation_toolbox/cli/fmusplit.py

@@ -1,12 +1,15 @@
 import argparse
 import logging
+import sys
+
+from typing import *
 
 from .utils import setup_logger, make_wide
 from ..split import FMUSplitter, FMUSplitterError
 from ..version import __version__ as version
 
 
-def fmusplit():
+def fmusplit(command_options: Sequence[str]):
     logger = setup_logger()
 
     logger.info(f"FMUSplit version {version}")
@@ -24,7 +27,7 @@ def fmusplit():
                         metavar="filename.fmu", required=True,
                         help="Description of the FMU container to split.")
 
-    config = parser.parse_args()
+    config = parser.parse_args(command_options)
 
     if config.debug:
         logger.setLevel(logging.DEBUG)
@@ -35,11 +38,11 @@ def fmusplit():
             splitter.split_fmu()
         except FMUSplitterError as e:
             logger.fatal(f"{fmu_filename}: {e}")
-            continue
+            sys.exit(-1)
         except FileNotFoundError as e:
             logger.fatal(f"Cannot read file: {e}")
-            continue
+            sys.exit(-2)
 
 
 if __name__ == "__main__":
-    fmusplit()
+    fmusplit(sys.argv[1:])

{fmu_manipulation_toolbox-1.9rc7 → fmu_manipulation_toolbox-1.9rc9}/fmu_manipulation_toolbox/cli/fmutool.py

@@ -1,6 +1,8 @@
 import argparse
 import sys
 
+from typing import *
+
 from .utils import setup_logger, make_wide
 from ..operations import (OperationSummary, OperationError, OperationRemoveRegexp,
                           OperationRemoveSources, OperationTrimUntil, OperationKeepOnlyRegexp, OperationMergeTopLevel,
@@ -12,7 +14,7 @@ from ..version import __version__ as version
 from ..help import Help
 
 
-def fmutool():
+def fmutool(command_options: Sequence[str]):
     logger = setup_logger()
 
     logger.info(f"FMU Manipulation Toolbox version {version}")
@@ -64,11 +66,12 @@ def fmutool():
     add_option('-only-parameters', action='append_const', dest='apply_on', const='parameter')
     add_option('-only-inputs', action='append_const', dest='apply_on', const='input')
     add_option('-only-outputs', action='append_const', dest='apply_on', const='output')
+    add_option('-only-locals', action='append_const', dest='apply_on', const='local')
     # Checker
     add_option('-summary', action='append_const', dest='operations_list', const=OperationSummary())
     add_option('-check', action='append_const', dest='operations_list', const=[checker() for checker in checker_list])
 
-    cli_options = parser.parse_args()
+    cli_options = parser.parse_args(command_options)
     # handle the "no operation" use case
     if not cli_options.operations_list:
         cli_options.operations_list = []
@@ -121,4 +124,4 @@ def fmutool():
         logger.info(f"INFO    Modified FMU is not saved. If necessary use '-output' option.")
 
 if __name__ == "__main__":
-    fmutool()
+    fmutool(sys.argv[1:])

{fmu_manipulation_toolbox-1.9rc7 → fmu_manipulation_toolbox-1.9rc9}/fmu_manipulation_toolbox/help.py

@@ -73,6 +73,9 @@ class Help:
         '-only-outputs': "apply operation only on ports with  causality = 'output'. This "
                          "option is available from version 1.3.",
 
+        '-only-locals': "apply operation only on ports with  causality = 'local'. This "
+                         "option is available from version 1.9.",
+
         '-summary': "display useful information regarding the FMU.",
 
         '-check': "performs some check of FMU and display Errors or Warnings. This is useful to avoid later "

{fmu_manipulation_toolbox-1.9rc7 → fmu_manipulation_toolbox-1.9rc9}/fmu_manipulation_toolbox/operations.py

@@ -112,6 +112,8 @@ class FMUError(Exception):
 
 class Manipulation:
     """Parse modelDescription.xml file and create a modified version"""
+    TAGS_MODEL_STRUCTURE = ("InitialUnknowns", "Derivatives", "Outputs")
+
     def __init__(self, operation, fmu):
         self.output_filename = tempfile.mktemp()
         self.out = None
@@ -120,7 +122,14 @@ class Manipulation:
         self.parser.StartElementHandler = self.start_element
         self.parser.EndElementHandler = self.end_element
         self.parser.CharacterDataHandler = self.char_data
+
+        # used for filter
         self.skip_until: Optional[str] = None
+
+        # used to remove empty sections
+        self.delayed_tag = None
+        self.delayed_tag_open = False
+
         self.operation.set_fmu(fmu)
         self.fmu = fmu
 
@@ -155,6 +164,7 @@ class Manipulation:
     def start_element(self, name, attrs):
         if self.skip_until:
             return
+
         try:
             if name == 'ScalarVariable': # FMI 2.0 only
                 self.current_port = FMUPort()
@@ -176,7 +186,7 @@ class Manipulation:
                 self.operation.fmi_attrs(attrs)
             elif name == 'Unknown': # FMI-2.0 only
                 self.unknown_attrs(attrs)
-            elif name == 'Output' or name == "ContinuousStateDerivative" or "InitialUnknown":
+            elif name == 'Output' or name == "ContinuousStateDerivative" or "InitialUnknown": #  FMI-3.0 only
                 self.handle_structure(attrs)
 
         except ManipulationSkipTag:
@@ -184,11 +194,19 @@ class Manipulation:
             return
 
         if self.current_port is None:
+            if self.delayed_tag and not self.delayed_tag_open:
+                print(f"<{self.delayed_tag}>", end='', file=self.out)
+                self.delayed_tag_open = True
+
             if attrs:
                 attrs_list = [f'{key}="{self.escape(value)}"' for (key, value) in attrs.items()]
                 print(f"<{name}", " ".join(attrs_list), ">", end='', file=self.out)
             else:
-                print(f"<{name}>", end='', file=self.out)
+                if name in self.TAGS_MODEL_STRUCTURE:
+                    self.delayed_tag = name
+                    self.delayed_tag_open = False
+                else:
+                    print(f"<{name}>", end='', file=self.out)
 
     def end_element(self, name):
         if self.skip_until:
@@ -205,7 +223,14 @@ class Manipulation:
                 self.current_port = None
 
             elif self.current_port is None:
-                print(f"</{name}>", end='', file=self.out)
+                if self.delayed_tag and name == self.delayed_tag:
+                    if self.delayed_tag_open:
+                        print(f"</{self.delayed_tag}>", end='', file=self.out)
+                    else:
+                        logger.debug(f"Remove tag <{self.delayed_tag}> from modelDescription.xml")
+                    self.delayed_tag = None
+                else:
+                    print(f"</{name}>", end='', file=self.out)
 
     def char_data(self, data):
         if not self.skip_until:
@@ -223,33 +248,71 @@ class Manipulation:
         self.port_translation.append(self.current_port_number)
 
     def unknown_attrs(self, attrs):
-        index = int(attrs['index']) - 1
-        new_index = self.port_translation[index]
-        if new_index:
-            attrs['index'] = self.port_translation[int(attrs['index']) - 1]
+        index = int(attrs['index'])
+        new_index = self.port_translation[index-1]
+        if new_index is not None:
+            attrs['index'] = str(new_index)
+            if attrs.get('dependencies', ""):
+                if 'dependenciesKind' in attrs:
+                    new_dependencies = []
+                    new_kinds = []
+                    for dependency, kind in zip(attrs['dependencies'].split(' '), attrs['dependenciesKind'].split(' ')):
+                        new_dependency = self.port_translation[int(dependency)-1]
+                        if new_dependency is not None:
+                            new_dependencies.append(str(new_dependency))
+                            new_kinds.append(kind)
+                    if new_dependencies:
+                        attrs['dependencies'] = " ".join(new_dependencies)
+                        attrs['dependenciesKind'] = " ".join(new_kinds)
+                    else:
+                        attrs.pop('dependencies')
+                        attrs.pop('dependenciesKind')
+                else:
+                    new_dependencies = []
+                    for dependency in attrs['dependencies'].split(' '):
+                        new_dependency = self.port_translation[int(dependency)-1]
+                        if new_dependency is not None:
+                            new_dependencies.append(str(new_dependency))
+                    if new_dependencies:
+                        attrs['dependencies'] = " ".join(new_dependencies)
+                    else:
+                        attrs.pop('dependencies')
         else:
-            logger.warning(f"Removed port '{self.port_names_list[index]}' is involved in dependencies tree.")
+            logger.warning(f"Removed port '{self.port_names_list[index-1]}' is involved in dependencies tree.")
             raise ManipulationSkipTag
 
     def handle_structure(self, attrs):
         try:
             vr = attrs['valueReference']
             if vr in self.port_removed_vr:
+                logger.warning(f"Removed port vr={vr} is involved in dependencies tree.")
                 raise ManipulationSkipTag
         except KeyError:
             return
 
-        try:
-            dependencies = attrs['dependencies']
-            new_dependencies = []
-            for dependency in dependencies.split(" "):
-                if dependency in self.port_removed_vr:
-                    logger.warning(f"Removed port 'vr={dependency}' was involved in dependencies tree.")
+        if attrs.get('dependencies', ""):
+            if 'dependenciesKind' in attrs:
+                new_dependencies = []
+                new_kinds = []
+                for dependency, kind in zip(attrs['dependencies'].split(' '), attrs['dependenciesKind'].split(' ')):
+                    if dependency not in self.port_removed_vr:
+                        new_dependencies.append(dependency)
+                        new_kinds.append(kind)
+                if new_dependencies:
+                    attrs['dependencies'] = " ".join(new_dependencies)
+                    attrs['dependenciesKind'] = " ".join(new_kinds)
                 else:
-                    new_dependencies.append(dependency)
-            attrs['dependencies'] = " ".join(new_dependencies)
-        except KeyError:
-            return
+                    attrs.pop('dependencies')
+                    attrs.pop('dependenciesKind')
+            else:
+                new_dependencies = []
+                for dependency in attrs['dependencies'].split(' '):
+                    if dependency not in self.port_removed_vr:
+                        new_dependencies.append(dependency)
+                if new_dependencies:
+                    attrs['dependencies'] = " ".join(new_dependencies)
+                else:
+                    attrs.pop('dependencies')
 
     def manipulate(self, descriptor_filename, apply_on=None):
         self.apply_on = apply_on
@@ -416,8 +479,8 @@ class OperationSummary(OperationAbstract):
                 hash_md5.update(chunk)
         digest = hash_md5.hexdigest()
         logger.info(f"| MD5Sum = {digest}")
-
-        logger.info(f"|\n| FMI properties: ")
+        logger.info(f"|")
+        logger.info(f"| FMI properties: ")
         for (k, v) in attrs.items():
             logger.info(f"|  - {k} = {v}")
         logger.info(f"|")
@@ -457,21 +520,25 @@ class OperationSummary(OperationAbstract):
 
         resource_dir = os.path.join(self.fmu.tmp_directory, "resources")
         if os.path.isdir(resource_dir):
-            logger.info("|\n| Embedded resources:")
+            logger.info("|")
+            logger.info("| Embedded resources:")
             for resource in os.listdir(resource_dir):
                 logger.info(f"|  - {resource}")
 
         extra_dir = os.path.join(self.fmu.tmp_directory, "extra")
         if os.path.isdir(extra_dir):
-            logger.info("|\n| Additional (meta-)data:")
+            logger.info("|")
+            logger.info("| Additional (meta-)data:")
             for extra in os.listdir(extra_dir):
                 logger.info(f"|  - {extra}")
 
-        logger.info("|\n| Number of ports")
+        logger.info("|")
+        logger.info("| Number of ports")
         for causality, nb_ports in self.nb_port_per_causality.items():
             logger.info(f"|  {causality} : {nb_ports}")
 
-        logger.info("|\n| [End of report]")
+        logger.info("|")
+        logger.info("| [End of report]")
 
 
 class OperationRemoveSources(OperationAbstract):

{fmu_manipulation_toolbox-1.9rc7 → fmu_manipulation_toolbox-1.9rc9}/fmu_manipulation_toolbox/split.py

@@ -6,6 +6,8 @@ import xml.parsers.expat
 from typing import *
 from pathlib import Path
 
+from .container import EmbeddedFMUPort
+
 logger = logging.getLogger("fmu_manipulation_toolbox")
 
 
@@ -66,7 +68,7 @@ class FMUSplitter:
     def split_fmu(self):
         logger.info(f"Splitting...")
         config = self._split_fmu(fmu_filename=str(self.fmu_filename), relative_path="")
-        config_filename = self.directory / self.fmu_filename.with_suffix(".json")
+        config_filename = self.directory / self.fmu_filename.with_suffix(".json").name
         with open(config_filename, "w") as file:
             json.dump(config, file, indent=2)
         logger.info(f"Container definition saved to '{config_filename}'")
@@ -77,7 +79,7 @@ class FMUSplitter:
         if txt_filename in self.filenames_list:
             description = FMUSplitterDescription(self.zip)
             config = description.parse_txt_file(txt_filename)
-            config["name"] = fmu_filename
+            config["name"] = Path(fmu_filename).name
             for i, fmu_filename in enumerate(config["candidate_fmu"]):
                 directory = f"{relative_path}resources/{i:02x}/"
                 if directory not in self.dir_set:
@@ -134,6 +136,7 @@ class FMUSplitterDescription:
         self.current_vr = None
         self.current_name = None
         self.current_causality = None
+        self.supported_fmi_types: Tuple[str] = ()
 
     @staticmethod
     def get_line(file):
@@ -182,10 +185,12 @@ class FMUSplitterDescription:
     def parse_txt_file_header(self, file, txt_filename):
         flags = self.get_line(file).split(" ")
         if len(flags) == 1:
+            self.supported_fmi_types = ("Real", "Integer", "Boolean", "String")
             self.config["mt"] = flags[0] == "1"
             self.config["profiling"] = self.get_line(file) == "1"
             self.config["sequential"] = False
         elif len(flags) == 3:
+            self.supported_fmi_types = EmbeddedFMUPort.ALL_TYPES
             self.config["mt"] = flags[0] == "1"
             self.config["profiling"] = flags[1] == "1"
             self.config["sequential"] = flags[2] == "1"
@@ -242,7 +247,7 @@ class FMUSplitterDescription:
             logger.debug(f"Adding container port {causality}: {definition}")
 
     def parse_txt_file_ports(self, file):
-        for fmi_type in ("Real", "Integer", "Boolean", "String"):
+        for fmi_type in self.supported_fmi_types:
             nb_port_variables = self.get_line(file).split(" ")[0]
             for i in range(int(nb_port_variables)):
                 tokens = self.get_line(file).split(" ")

{fmu_manipulation_toolbox-1.9rc7 → fmu_manipulation_toolbox-1.9rc9}/fmu_manipulation_toolbox.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmu_manipulation_toolbox
-Version: 1.9rc7
+Version: 1.9rc9
 Summary: FMU Manipulation Toolbox is a python application for modifying Functional Mock-up Units (FMUs) without recompilation or bundling them into FMU Containers
 Home-page: https://github.com/grouperenault/fmu_manipulation_toolbox/
 Author: Nicolas.LAURENT@Renault.com

{fmu_manipulation_toolbox-1.9rc7 → fmu_manipulation_toolbox-1.9rc9}/tests/test_suite.py

@@ -9,7 +9,9 @@ from fmu_manipulation_toolbox.operations import *
 from fmu_manipulation_toolbox.remoting import *
 from fmu_manipulation_toolbox.container import *
 from fmu_manipulation_toolbox.assembly import *
-
+from fmu_manipulation_toolbox.cli.fmutool import fmutool
+from fmu_manipulation_toolbox.cli.fmusplit import fmusplit
+from fmu_manipulation_toolbox.cli.fmucontainer import fmucontainer
 
 class FMUManipulationToolboxTestSuite(unittest.TestCase):
     def __init__(self, *args, **kwargs):
@@ -214,6 +216,28 @@ class FMUManipulationToolboxTestSuite(unittest.TestCase):
         for link in links_fmu0a:
             print(f"{link}")
 
+    def test_fmutool(self):
+        fmutool(['-input', 'operations/bouncing_ball.fmu', '-summary', '-check', '-dump-csv',
+                 'operations/cli-bouncing_ball.csv'])
+        self.assert_identical_files("operations/cli-bouncing_ball.csv", "operations/REF-bouncing_ball.csv")
+
+    def test_fmucontainer_csv(self):
+        fmucontainer(['-container', 'cli-bouncing.csv', '-fmu-directory', 'containers/bouncing_ball', '-mt', '-debug'])
+        self.assert_identical_files("containers/bouncing_ball/REF-container.txt",
+                                    "containers/bouncing_ball/cli-bouncing/resources/container.txt")
+
+    def test_fmucontainer_json(self):
+        fmucontainer(['-fmu-directory', 'containers/arch', '-container', 'cli-flat.json', '-dump'])
+        self.assert_identical_files("containers/arch/REF-cli-flat-dump.json",
+                                    "containers/arch/cli-flat-dump.json")
+
+    def test_fmusplit(self):
+        fmusplit(["-fmu", "containers/ssp/bouncing.fmu"])
+        self.assertTrue(Path("containers/ssp/bouncing.dir/bb_position.fmu").exists())
+        self.assertTrue(Path("containers/ssp/bouncing.dir/bb_velocity.fmu").exists())
+        self.assert_identical_files("containers/ssp/REF-split-bouncing.json",
+                                    "containers/ssp/bouncing.dir/bouncing.json")
+
 
 if __name__ == '__main__':
     unittest.main()

fmu_manipulation_toolbox-1.9rc7/fmu_manipulation_toolbox/__version__.py

@@ -1 +0,0 @@
-'V1.9-rc7'