fmu-manipulation-toolbox 1.8.4.3b0__py3-none-any.whl → 1.9__py3-none-any.whl

fmu_manipulation_toolbox/__init__.py

@@ -1 +0,0 @@
-

fmu_manipulation_toolbox/__main__.py

@@ -10,7 +10,7 @@ def gui():
 
 
 def cli():
-    from .cli import fmutool
+    from .cli.fmutool import fmutool
     fmutool()
 
 

fmu_manipulation_toolbox/__version__.py

@@ -1 +1 @@
-'V1.8.4.3-beta'
+'V1.9'

fmu_manipulation_toolbox/assembly.py

@@ -37,7 +37,7 @@ class Connection:
 
 
 class AssemblyNode:
-    def __init__(self, name: Optional[str], step_size: float = None, mt=False, profiling=False,
+    def __init__(self, name: Optional[str], step_size: float = None, mt=False, profiling=False, sequential=False,
                  auto_link=True, auto_input=True, auto_output=True, auto_parameter=False, auto_local=False):
         self.name = name
         if step_size:
@@ -50,6 +50,7 @@ class AssemblyNode:
             self.step_size = None
         self.mt = mt
         self.profiling = profiling
+        self.sequential = sequential
         self.auto_link = auto_link
         self.auto_input = auto_input
         self.auto_output = auto_output
@@ -100,12 +101,13 @@ class AssemblyNode:
     def add_start_value(self, fmu_filename: str, port_name: str, value: str):
         self.start_values[Port(fmu_filename, port_name)] = value
 
-    def make_fmu(self, fmu_directory: Path, debug=False, description_pathname=None):
+    def make_fmu(self, fmu_directory: Path, debug=False, description_pathname=None, fmi_version=2):
         for node in self.children.values():
-            node.make_fmu(fmu_directory, debug=debug)
+            node.make_fmu(fmu_directory, debug=debug, fmi_version=fmi_version)
 
         identifier = str(Path(self.name).stem)
-        container = FMUContainer(identifier, fmu_directory, description_pathname=description_pathname)
+        container = FMUContainer(identifier, fmu_directory, description_pathname=description_pathname,
+                                 fmi_version=fmi_version)
 
         for fmu_name in self.fmu_names_list:
             container.get_fmu(fmu_name)
@@ -138,7 +140,8 @@ class AssemblyNode:
         for link_rule in wired.rule_link:
             self.add_link(link_rule[0], link_rule[1], link_rule[2], link_rule[3])
 
-        container.make_fmu(self.name, self.step_size, mt=self.mt, profiling=self.profiling, debug=debug)
+        container.make_fmu(self.name, self.step_size, mt=self.mt, profiling=self.profiling, sequential=self.sequential,
+                           debug=debug)
 
         for node in self.children.values():
             logger.info(f"Deleting transient FMU Container '{node.name}'")
@@ -216,7 +219,7 @@ class AssemblyError(Exception):
 
 
 class Assembly:
-    def __init__(self, filename: str, step_size=None, auto_link=True,  auto_input=True, debug=False,
+    def __init__(self, filename: str, step_size=None, auto_link=True,  auto_input=True, debug=False, sequential=False,
                  auto_output=True, mt=False, profiling=False, fmu_directory: Path = Path("."), auto_parameter=False,
                  auto_local=False):
         self.filename = Path(filename)
@@ -228,6 +231,7 @@ class Assembly:
         self.default_auto_parameter = auto_parameter
         self.default_auto_local = auto_local
         self.default_mt = mt
+        self.default_sequential = sequential
         self.default_profiling = profiling
         self.fmu_directory = fmu_directory
         self.transient_filenames: List[Path] = []
@@ -238,7 +242,7 @@ class Assembly:
 
         self.input_pathname = fmu_directory / self.filename
         self.description_pathname = self.input_pathname   # For inclusion in FMU
-        self.root = None
+        self.root: Optional[AssemblyNode] = None
         self.read()
 
     def add_transient_file(self, filename: str):
@@ -283,8 +287,9 @@ class Assembly:
         name = str(self.filename.with_suffix(".fmu"))
         self.root = AssemblyNode(name, step_size=self.default_step_size, auto_link=self.default_auto_link,
                                  mt=self.default_mt, profiling=self.default_profiling,
-                                 auto_input=self.default_auto_input, auto_output=self.default_auto_output,
-                                 auto_parameter=self.default_auto_parameter, auto_local=self.default_auto_local)
+                                 sequential=self.default_sequential, auto_input=self.default_auto_input,
+                                 auto_output=self.default_auto_output, auto_parameter=self.default_auto_parameter,
+                                 auto_local=self.default_auto_local)
 
         with open(self.input_pathname) as file:
             reader = csv.reader(file, delimiter=';')
@@ -385,6 +390,7 @@ class Assembly:
         name = data.get("name", None)                                                       # 1
         mt = data.get("mt", self.default_mt)                                                # 2
         profiling = data.get("profiling", self.default_profiling)                           # 3
+        sequential = data.get("sequential", self.default_sequential)                        # 3b
         auto_link = data.get("auto_link", self.default_auto_link)                           # 4
         auto_input = data.get("auto_input", self.default_auto_input)                        # 5
         auto_output = data.get("auto_output", self.default_auto_output)                     # 6
@@ -393,11 +399,12 @@ class Assembly:
         step_size = data.get("step_size", self.default_step_size)                           # 7
 
         node = AssemblyNode(name, step_size=step_size, auto_link=auto_link, mt=mt, profiling=profiling,
+                            sequential=sequential,
                             auto_input=auto_input, auto_output=auto_output, auto_parameter=auto_parameter,
                             auto_local=auto_local)
 
         for key, value in data.items():
-            if key in ('name', 'step_size', 'auto_link', 'auto_input', 'auto_output', 'mt', 'profiling',
+            if key in ('name', 'step_size', 'auto_link', 'auto_input', 'auto_output', 'mt', 'profiling', 'sequential',
                        'auto_parameter', 'auto_local'):
                 continue  # Already read
 
@@ -455,6 +462,7 @@ class Assembly:
         json_node["name"] = node.name                      # 1
         json_node["mt"] = node.mt                          # 2
         json_node["profiling"] = node.profiling            # 3
+        json_node["sequential"] = node.sequential          # 3b
         json_node["auto_link"] = node.auto_link            # 4
         json_node["auto_input"] = node.auto_input          # 5
         json_node["auto_output"] = node.auto_output        # 6
@@ -510,8 +518,9 @@ class Assembly:
             self.root = sdd.parse(self.description_pathname)
             self.root.name = str(self.filename.with_suffix(".fmu"))
 
-    def make_fmu(self, dump_json=False):
-        self.root.make_fmu(self.fmu_directory, debug=self.debug, description_pathname=self.description_pathname)
+    def make_fmu(self, dump_json=False, fmi_version=2):
+        self.root.make_fmu(self.fmu_directory, debug=self.debug, description_pathname=self.description_pathname,
+                           fmi_version=fmi_version)
         if dump_json:
             dump_file = Path(self.input_pathname.stem + "-dump").with_suffix(".json")
             logger.info(f"Dump Json '{dump_file}'")

fmu_manipulation_toolbox/checker.py

@@ -1,10 +1,13 @@
 import importlib.util
 import inspect
+import logging
 import os
 import xmlschema
 from xmlschema.validators.exceptions import XMLSchemaValidationError
 from .operations import OperationAbstract
 
+logger = logging.getLogger("fmu_manipulation_toolbox")
+
 
 class OperationGenericCheck(OperationAbstract):
     SUPPORTED_FMI_VERSIONS = ('2.0', '3.0')
@@ -17,7 +20,7 @@ class OperationGenericCheck(OperationAbstract):
 
     def fmi_attrs(self, attrs):
         if attrs['fmiVersion'] not in self.SUPPORTED_FMI_VERSIONS:
-            print(f"ERROR: Expected FMI {','.join(self.SUPPORTED_FMI_VERSIONS)} versions.")
+            logger.error(f"Expected FMI {','.join(self.SUPPORTED_FMI_VERSIONS)} versions.")
             return
 
         fmi_name = f"fmi{attrs['fmiVersion'][0]}"
@@ -27,15 +30,15 @@ class OperationGenericCheck(OperationAbstract):
         try:
             xmlschema.validate(self.fmu.descriptor_filename, schema=xsd_filename)
         except XMLSchemaValidationError as error:
-            print(error.reason, error.msg)
+            logger.error(error.reason, error.msg)
         else:
             self.compliant_with_version = attrs['fmiVersion']
 
     def closure(self):
         if self.compliant_with_version:
-            print(f"INFO: This FMU seems to be compliant with FMI-{self.compliant_with_version}.")
+            logger.info(f"This FMU seems to be compliant with FMI-{self.compliant_with_version}.")
         else:
-            print(f"ERROR: This FMU does not validate with FMI standard.")
+            logger.error(f"This FMU does not validate with FMI standard.")
 
 
 checker_list = [OperationGenericCheck]
@@ -44,20 +47,20 @@ checker_list = [OperationGenericCheck]
 def add_from_file(checker_filename: str):
     spec = importlib.util.spec_from_file_location(checker_filename, checker_filename)
     if not spec:
-        print(f"ERROR: Cannot load {checker_filename}. Is this a python file?")
+        logger.error(f"Cannot load '{checker_filename}'. Is this a python file?")
         return
     try:
         checker_module = importlib.util.module_from_spec(spec)
         try:
             spec.loader.exec_module(checker_module)
         except (ModuleNotFoundError, SyntaxError) as error:
-            print(f"ERROR: Cannot load {checker_filename}: {error})")
+            logger.error(f"Cannot load '{checker_filename}': {error})")
             return
 
         for checker_name, checker_class in inspect.getmembers(checker_module, inspect.isclass):
             if OperationAbstract in checker_class.__bases__:
                 checker_list.append(checker_class)
-                print(f"Adding checker: {checker_filename}|{checker_name}")
+                logger.info(f"Adding checker: {checker_filename}|{checker_name}")
 
     except AttributeError:
-        print(f"ERROR: {checker_filename} should implement class 'OperationCheck'")
+        logger.error(f"'{checker_filename}' should implement class 'OperationCheck'")

fmu_manipulation_toolbox/cli/fmucontainer.py

@@ -0,0 +1,105 @@
+import argparse
+import logging
+import sys
+
+from typing import *
+from pathlib import Path
+
+from .utils import setup_logger, make_wide
+from ..assembly import Assembly, AssemblyError
+from ..container import FMUContainerError
+from ..version import __version__ as version
+
+
+def fmucontainer(command_options: Sequence[str]):
+    logger = setup_logger()
+
+    logger.info(f"FMUContainer version {version}")
+    parser = argparse.ArgumentParser(prog="fmucontainer", description="Generate FMU from FMU's",
+                                     formatter_class=make_wide(argparse.ArgumentDefaultsHelpFormatter),
+                                     add_help=False,
+                                     epilog="see: https://github.com/grouperenault/fmu_manipulation_toolbox/blob/main/"
+                                            "container/README.md")
+
+    parser.add_argument('-h', '-help', action="help")
+
+    parser.add_argument("-fmu-directory", action="store", dest="fmu_directory", required=False, default=".",
+                        help="Directory containing initial FMU’s and used to generate containers. "
+                             "If not defined, current directory is used.")
+
+    parser.add_argument("-fmi", action="store", dest="fmi_version", required=False, default="2",
+                        help="Define version of FMI to be used for container interface."
+                             "Only '2' or '3' is supported.")
+
+    parser.add_argument("-container", action="append", dest="container_descriptions_list", default=[],
+                        metavar="filename.{csv|json|ssp},[:step_size]", required=True,
+                        help="Description of the container to create.")
+
+    parser.add_argument("-debug", action="store_true", dest="debug",
+                        help="Add lot of useful log during the process.")
+
+    parser.add_argument("-no-auto-input", action="store_false", dest="auto_input", default=True,
+                        help="Create ONLY explicit input.")
+
+    parser.add_argument("-no-auto-output", action="store_false", dest="auto_output", default=True,
+                        help="Create ONLY explicit output.")
+
+    parser.add_argument("-auto-parameter", action="store_true", dest="auto_parameter", default=False,
+                        help="Expose parameters of the embedded fmu's.")
+
+    parser.add_argument("-auto-local", action="store_true", dest="auto_local", default=False,
+                        help="Expose local variables of the embedded fmu's.")
+
+    parser.add_argument("-no-auto-link", action="store_false", dest="auto_link", default=True,
+                        help="Create ONLY explicit links.")
+
+    parser.add_argument("-mt", action="store_true", dest="mt", default=False,
+                        help="Enable Multi-Threaded mode for the generated container.")
+
+    parser.add_argument("-profile", action="store_true", dest="profiling", default=False,
+                        help="Enable Profiling mode for the generated container.")
+
+    parser.add_argument("-sequential", action="store_true", dest="sequential", default=False,
+                        help="Use sequential mode to schedule embedded fmu's.")
+
+    parser.add_argument("-dump-json",  action="store_true", dest="dump", default=False,
+                        help="Dump a JSON file for each container.")
+
+    config = parser.parse_args(command_options)
+
+    if config.debug:
+        logger.setLevel(logging.DEBUG)
+
+    fmu_directory = Path(config.fmu_directory)
+    logger.info(f"FMU directory: '{fmu_directory}'")
+
+    for description in config.container_descriptions_list:
+        try:
+            tokens = description.split(":")
+            filename = ":".join(tokens[:-1])
+            step_size = float(tokens[-1])
+        except ValueError:
+            step_size = None
+            filename = description
+
+        try:
+            assembly = Assembly(filename, step_size=step_size, auto_link=config.auto_link,
+                                auto_input=config.auto_input, auto_output=config.auto_output,
+                                auto_local=config.auto_local, mt=config.mt, sequential=config.sequential,
+                                profiling=config.profiling, fmu_directory=fmu_directory, debug=config.debug,
+                                auto_parameter=config.auto_parameter)
+        except FileNotFoundError as e:
+            logger.fatal(f"Cannot read file: {e}")
+            sys.exit(-1)
+        except (FMUContainerError, AssemblyError) as e:
+            logger.fatal(f"{filename}: {e}")
+            sys.exit(-2)
+
+        try:
+            assembly.make_fmu(dump_json=config.dump, fmi_version=int(config.fmi_version))
+        except FMUContainerError as e:
+            logger.fatal(f"{filename}: {e}")
+            sys.exit(-3)
+
+if __name__ == "__main__":
+    fmucontainer(sys.argv[1:])

fmu_manipulation_toolbox/cli/fmusplit.py

@@ -0,0 +1,48 @@
+import argparse
+import logging
+import sys
+
+from typing import *
+
+from .utils import setup_logger, make_wide
+from ..split import FMUSplitter, FMUSplitterError
+from ..version import __version__ as version
+
+
+def fmusplit(command_options: Sequence[str]):
+    logger = setup_logger()
+
+    logger.info(f"FMUSplit version {version}")
+    parser = argparse.ArgumentParser(prog="fmusplit", description="Split FMU Container into FMU's",
+                                     formatter_class=make_wide(argparse.ArgumentDefaultsHelpFormatter),
+                                     add_help=False,
+                                     epilog="see: https://github.com/grouperenault/fmu_manipulation_toolbox/blob/main/"
+                                            "container/README.md")
+    parser.add_argument('-h', '-help', action="help")
+
+    parser.add_argument("-debug", action="store_true", dest="debug",
+                        help="Add lot of useful log during the process.")
+
+    parser.add_argument("-fmu", action="append", dest="fmu_filename_list", default=[],
+                        metavar="filename.fmu", required=True,
+                        help="Description of the FMU container to split.")
+
+    config = parser.parse_args(command_options)
+
+    if config.debug:
+        logger.setLevel(logging.DEBUG)
+
+    for fmu_filename in config.fmu_filename_list:
+        try:
+            splitter = FMUSplitter(fmu_filename)
+            splitter.split_fmu()
+        except FMUSplitterError as e:
+            logger.fatal(f"{fmu_filename}: {e}")
+            sys.exit(-1)
+        except FileNotFoundError as e:
+            logger.fatal(f"Cannot read file: {e}")
+            sys.exit(-2)
+
+
+if __name__ == "__main__":
+    fmusplit(sys.argv[1:])

fmu_manipulation_toolbox/cli/fmutool.py

@@ -0,0 +1,127 @@
+import argparse
+import sys
+
+from typing import *
+
+from .utils import setup_logger, make_wide
+from ..operations import (OperationSummary, OperationError, OperationRemoveRegexp,
+                          OperationRemoveSources, OperationTrimUntil, OperationKeepOnlyRegexp, OperationMergeTopLevel,
+                          OperationStripTopLevel, OperationRenameFromCSV, OperationSaveNamesToCSV, FMU, FMUError)
+from ..remoting import  (OperationAddFrontendWin32, OperationAddFrontendWin64, OperationAddRemotingWin32,
+                         OperationAddRemotingWin64)
+from ..checker import checker_list
+from ..version import __version__ as version
+from ..help import Help
+
+
+def fmutool(command_options: Sequence[str]):
+    logger = setup_logger()
+
+    logger.info(f"FMU Manipulation Toolbox version {version}")
+    help_message = Help()
+
+    parser = argparse.ArgumentParser(prog='fmutool',
+                                     description="Analyse and Manipulate a FMU by modifying its 'modelDescription.xml'",
+                                     formatter_class=make_wide(argparse.HelpFormatter),
+                                     add_help=False,
+                                     epilog="see: https://github.com/grouperenault/fmu_manipulation_toolbox/blob/main/README.md")
+
+    def add_option(option, *args, **kwargs):
+        parser.add_argument(option, *args, help=help_message.usage(option), **kwargs)
+
+    add_option('-h', '-help', action="help")
+
+    # I/O
+    add_option('-input', action='store', dest='fmu_input', default=None, required=True, metavar='path/to/module.fmu')
+    add_option('-output', action='store', dest='fmu_output', default=None, metavar='path/to/module-modified.fmu')
+
+    # Port name manipulation
+    add_option('-remove-toplevel', action='append_const', dest='operations_list', const=OperationStripTopLevel())
+    add_option('-merge-toplevel', action='append_const', dest='operations_list', const=OperationMergeTopLevel())
+    add_option('-trim-until', action='append', dest='operations_list', type=OperationTrimUntil, metavar='prefix')
+    add_option('-remove-regexp', action='append', dest='operations_list', type=OperationRemoveRegexp,
+               metavar='regular-expression')
+    add_option('-keep-only-regexp', action='append', dest='operations_list', type=OperationKeepOnlyRegexp,
+               metavar='regular-expression')
+    add_option('-remove-all', action='append_const', dest='operations_list', const=OperationRemoveRegexp('.*'))
+
+    # Batch Rename
+    add_option('-dump-csv', action='append', dest='operations_list', type=OperationSaveNamesToCSV,
+               metavar='path/to/list.csv')
+    add_option('-rename-from-csv', action='append', dest='operations_list', type=OperationRenameFromCSV,
+               metavar='path/to/translation.csv')
+
+    # Remoting
+    add_option('-add-remoting-win32', action='append_const', dest='operations_list', const=OperationAddRemotingWin32())
+    add_option('-add-remoting-win64', action='append_const', dest='operations_list', const=OperationAddRemotingWin64())
+    add_option('-add-frontend-win32', action='append_const', dest='operations_list', const=OperationAddFrontendWin32())
+    add_option('-add-frontend-win64', action='append_const', dest='operations_list', const=OperationAddFrontendWin64())
+
+    # Extraction / Removal
+    add_option('-extract-descriptor', action='store', dest='extract_description',
+               metavar='path/to/saved-modelDescriptor.xml')
+    add_option('-remove-sources', action='append_const', dest='operations_list',
+               const=OperationRemoveSources())
+    # Filter
+    add_option('-only-parameters', action='append_const', dest='apply_on', const='parameter')
+    add_option('-only-inputs', action='append_const', dest='apply_on', const='input')
+    add_option('-only-outputs', action='append_const', dest='apply_on', const='output')
+    add_option('-only-locals', action='append_const', dest='apply_on', const='local')
+    # Checker
+    add_option('-summary', action='append_const', dest='operations_list', const=OperationSummary())
+    add_option('-check', action='append_const', dest='operations_list', const=[checker() for checker in checker_list])
+
+    cli_options = parser.parse_args(command_options)
+    # handle the "no operation" use case
+    if not cli_options.operations_list:
+        cli_options.operations_list = []
+
+    if cli_options.fmu_input == cli_options.fmu_output:
+        logger.fatal(f"'-input' and '-output' should point to different files.")
+        sys.exit(-3)
+
+    logger.info(f"READING Input='{cli_options.fmu_input}'")
+    try:
+        fmu = FMU(cli_options.fmu_input)
+    except FMUError as reason:
+        logger.fatal(f"{reason}")
+        sys.exit(-4)
+
+    if cli_options.apply_on:
+        logger.info("Applying operation for :")
+        for causality in cli_options.apply_on:
+            logger.info(f"     - causality = {causality}")
+
+    # Checker operations are added as a list into operations_list
+    def operation_iterator():
+        for op in cli_options.operations_list:
+            if isinstance(op, list):
+                for sub_op in op:
+                    yield sub_op
+            else:
+                yield op
+
+    for operation in operation_iterator():
+        logger.info(f"     => {operation}")
+        try:
+            fmu.apply_operation(operation, cli_options.apply_on)
+        except OperationError as reason:
+            logger.fatal(f"{reason}")
+            sys.exit(-6)
+
+    if cli_options.extract_description:
+        logger.info(f"WRITING ModelDescriptor='{cli_options.extract_description}'")
+        fmu.save_descriptor(cli_options.extract_description)
+
+    if cli_options.fmu_output:
+        logger.info(f"WRITING Output='{cli_options.fmu_output}'")
+        try:
+            fmu.repack(cli_options.fmu_output)
+        except FMUError as reason:
+            logger.fatal(f"FATAL ERROR: {reason}")
+            sys.exit(-5)
+    else:
+        logger.info(f"INFO    Modified FMU is not saved. If necessary use '-output' option.")
+
+if __name__ == "__main__":
+    fmutool(sys.argv[1:])

fmu_manipulation_toolbox/cli/utils.py

@@ -0,0 +1,36 @@
+import logging
+import sys
+from colorama import Fore, Style, init
+
+def setup_logger():
+    class CustomFormatter(logging.Formatter):
+        def format(self, record):
+            log_format = "%(levelname)-8s | %(message)s"
+            format_per_level = {
+                logging.DEBUG: str(Fore.BLUE) + log_format,
+                logging.INFO: str(Fore.CYAN) + log_format,
+                logging.WARNING: str(Fore.YELLOW) + log_format,
+                logging.ERROR: str(Fore.RED) + log_format,
+                logging.CRITICAL: str(Fore.RED + Style.BRIGHT) + log_format,
+            }
+            formatter = logging.Formatter(format_per_level[record.levelno])
+            return formatter.format(record)
+    init()
+    logger = logging.getLogger("fmu_manipulation_toolbox")
+    handler = logging.StreamHandler(stream=sys.stdout)
+    handler.setFormatter(CustomFormatter())
+    logger.addHandler(handler)
+    logger.setLevel(logging.INFO)
+
+    return logger
+
+def make_wide(formatter, w=120, h=36):
+    """Return a wider HelpFormatter, if possible."""
+    try:
+        # https://stackoverflow.com/a/5464440
+        # beware: "Only the name of this class is considered a public API."
+        kwargs = {'width': w, 'max_help_position': h}
+        formatter(None, **kwargs)
+        return lambda prog: formatter(prog, **kwargs)
+    except TypeError:
+        return formatter