vss-tools 0.5.0.dev0__py3-none-any.whl

vspec/utils/idgen_utils.py

@@ -0,0 +1,97 @@
+# Copyright (c) 2023 Contributors to COVESA
+#
+# This program and the accompanying materials are made available under the
+# terms of the Mozilla Public License 2.0 which is available at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+# SPDX-License-Identifier: MPL-2.0
+
+
+def get_node_identifier_bytes(
+    qualified_name: str,
+    data_type: str,
+    node_type: str,
+    unit: str,
+    allowed: str,
+    minimum: str,
+    maximum: str,
+    strict_mode: bool,
+) -> bytes:
+    """Get a node identifier as bytes. Used as an input for hashing
+
+    @param qualified_name: full path to the node
+    @param data_type: its datatype
+    @param node_type: its node type
+    @param unit: the unit if it uses one
+    @param allowed: the enum for allowed values
+    @param minimum: min value for the data if exists
+    @param maximum: max value for the data if exists
+    @param strict_mode: strict mode means case sensitivity of node qualified names
+    @return: a bytes representation of the node
+    """
+
+    node_identifier: bytes = (
+        f"{qualified_name}: "
+        f"unit: {unit}, "
+        f"datatype: {data_type}, "
+        f"type: {node_type}"
+        f"allowed: {allowed}"
+        f"min: {minimum}"
+        f"max: {maximum}"
+    ).encode()
+
+    if strict_mode:
+        return node_identifier
+    else:
+        return node_identifier.lower()
+
+
+def fnv1_32_hash(identifier: bytes) -> int:
+    """32-bit hash of a node according to Fowler–Noll–Vo
+
+    @param identifier: a bytes representation of a node
+    @return: hashed value for the node as int
+    """
+    id_hash = 2166136261
+    for byte in identifier:
+        id_hash = (id_hash * 16777619) & 0xFFFFFFFF
+        id_hash ^= byte
+
+    return id_hash
+
+
+def fnv1_32_wrapper(name: str, source: dict, strict_mode: bool):
+    """A wrapper for the 32-bit hashing function if the input node
+     is represented as dict instead of VSSNode
+
+    @param name: full name aka qualified name
+    @param source:
+    @param strict_mode: strict mode means case sensitivity of node qualified names
+    @return:
+    """
+    # Verify and assign values from source dictionary using source.get
+    allowed: str = source.get("allowed", "")
+    minimum: str = source.get("min", "")
+    maximum: str = source.get("max", "")
+    datatype: str = source.get("datatype", "")
+    vsstype: str = source.get("type", "")
+    unit: str = source.get("unit", "")
+
+    identifier = get_node_identifier_bytes(
+        name,
+        datatype,
+        vsstype,
+        unit,
+        allowed,
+        minimum,
+        maximum,
+        strict_mode,
+    )
+    return format(fnv1_32_hash(identifier), "08X")
+
+
+def get_all_keys_values(d: dict):
+    for key, value in d.items():
+        yield key, value
+        if isinstance(value, dict):
+            yield from get_all_keys_values(value)

vspec/utils/stringstyle.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2023 Contributors to COVESA
+#
+# This program and the accompanying materials are made available under the
+# terms of the Mozilla Public License 2.0 which is available at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+# SPDX-License-Identifier: MPL-2.0
+
+import re
+
+
+def camel_case(st):
+    """Camel case string conversion"""
+    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', st)
+    s2 = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
+    return re.sub(r'(?:^|_)([a-z])', lambda x: x.group(1).upper(), s2)
+
+
+def camel_back(st):
+    """Camel back string conversion"""
+    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', st)
+    s2 = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
+    return re.sub(r'_([a-z])', lambda x: x.group(1).upper(), s2)

vspec/utils/vss2id_val.py

@@ -0,0 +1,165 @@
+# Copyright (c) 2023 Contributors to COVESA
+#
+# This program and the accompanying materials are made available under the
+# terms of the Mozilla Public License 2.0 which is available at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+# SPDX-License-Identifier: MPL-2.0
+
+import argparse
+import logging
+import sys
+from typing import Optional
+
+from anytree import PreOrderIter  # type: ignore
+
+from vspec.model.vsstree import VSSNode
+from vspec.utils.idgen_utils import fnv1_32_wrapper
+
+
+def validate_static_uids(
+    signals_dict: dict, validation_tree: VSSNode, config: argparse.Namespace
+):
+    """Check if static UIDs have changed or if new ones need to be added
+
+    @param signals_dict: to be exported dict of all signals containing static UID
+    @param validation_tree: tree loaded from validation file
+    @param config: the command line arguments the script was run with
+    @return: None
+    """
+
+    def check_description(k: str, v: dict, match_tuple: tuple):
+        validation_node: VSSNode = validation_tree_nodes[match_tuple[1]]
+
+        try:
+            assert v["description"] == validation_node.description
+
+        except AssertionError:
+            logging.warning(
+                "[Validation] "
+                f"DESCRIPTION MISMATCH: The description of {k} has changed from "
+                f"\n\t   Validation: '{validation_node.description}' to \n\t   Current "
+                f"vspec: '{v['description']}'"
+            )
+
+    def check_semantics(k: str, v: dict, strict_mode: bool) -> Optional[int]:
+        """Checks if the change was a semantic or path change. This can be triggered by
+        manually adding a fka (formerly known as) attribute to the vspec. The result
+        is that the old hash can be matched such that a node keeps the same UID.
+
+        @param k: the current key
+        @param v: the current value (dict)
+        @param strict_mode: strict mode means case sensitivity for static UID generation
+        @return: boolean if it was a semantic or path change
+        """
+        if "fka" in v.keys():
+            semantic_match: Optional[int] = None
+            for fka_val in v["fka"]:
+                old_static_uid = "0x" + fnv1_32_wrapper(fka_val, v, strict_mode)
+                for i, validation_node in enumerate(validation_tree_nodes):
+                    if (
+                        old_static_uid
+                        == validation_node.extended_attributes["staticUID"]
+                    ):
+                        logging.warning(
+                            f"[Validation] SEMANTIC NAME CHANGE or PATH CHANGE for '{k}', "
+                            f"it used to be '{validation_node.qualified_name()}'."
+                        )
+                        semantic_match = i
+            return semantic_match
+        else:
+            return None
+
+    def check_deprecation(k: str, v: dict, match_tuple: tuple):
+        if (
+            "deprecation" in v.keys()
+            and validation_tree_nodes[match_tuple[1]].deprecation
+        ):
+            if v["deprecation"] != validation_tree_nodes[match_tuple[1]].deprecation:
+                logging.warning(
+                    f"[Validation] DEPRECATION MSG CHANGE: Deprecation message "
+                    f"for '{k}' was "
+                    f"'{validation_tree_nodes[match_tuple[1]].deprecation}' "
+                    f"in validation but now is '{v['deprecation']}'."
+                )
+
+    def hashed_pipeline():
+        """This pipeline uses FNV-1 hash for static UIDs.
+
+        If no UID was matched we check if the user has written an old path or old name,
+        so we can tell if it was a semantic or path change. If semantic or path change
+        we trigger a corresponding warning.
+        If not semantic or path change we try to match the key:
+        If key was matched we know that it was a unit, datatype, type, enum, min/max change
+        which triggers `BREAKING CHANGE` warning. If key cannot be matched the node must have
+        been added since the last validation, so it triggers an `ADDED ATTRIBUTE` warning.
+        If there was a UID match we will continue to check for non-breaking changes and throw
+        corresponding logs.
+        In the end the remaining nodes correspond to deleted nodes, so we throw a
+        `DELETED ATTRIBUTE` warning.
+        """
+        nonlocal validation_tree_nodes
+
+        for key, value in signals_dict.items():
+            matched_uids = []
+            for id_validation_tree, other_node in enumerate(validation_tree_nodes):
+                if value["staticUID"] == other_node.extended_attributes["staticUID"]:
+                    if key != other_node.qualified_name():
+                        _ = check_semantics(key, value, config.strict_mode)
+                    matched_uids.append((key, id_validation_tree))
+            # if not matched via UID check semantics or path change
+            if len(matched_uids) == 0:
+                semantic_match = check_semantics(key, value, config.strict_mode)
+                if semantic_match is None:
+                    key_found: bool = False
+                    for i, node in enumerate(validation_tree_nodes):
+                        if key == node.qualified_name():
+                            key_found = True
+                            validation_tree_nodes.pop(i)
+                            break
+
+                    if key_found:
+                        logging.warning(
+                            f"[Validation] BREAKING CHANGE: "
+                            f"There was a breaking change for '{key}' which "
+                            f"means its name, unit, datatype, type, enum or "
+                            f"min/max has changed."
+                        )
+                    else:
+                        logging.warning(
+                            f"[Validation] ADDED ATTRIBUTE: "
+                            f"The node '{key}' was added since the last validation."
+                        )
+                else:
+                    validation_tree_nodes.pop(semantic_match)
+
+            elif len(matched_uids) == 1:
+                check_deprecation(key, value, matched_uids[0])
+                check_description(key, value, matched_uids[0])
+
+                validation_tree_nodes.pop(matched_uids[0][1])
+
+            else:
+                logging.error(
+                    "[Validation] Multiple matches do not make sense "
+                    "for the way we load the data. Please check your "
+                    "input vspec!"
+                )
+                sys.exit(-1)
+
+        for node in validation_tree_nodes:
+            logging.warning(
+                "[Validation] DELETED ATTRIBUTE: "
+                f"'{node.qualified_name()}' was not matched so it must have "
+                f"been deleted."
+            )
+
+    if validation_tree.parent:
+        while validation_tree.parent:
+            validation_tree = validation_tree.parent
+
+    validation_tree_nodes: list = []
+    for val_node in PreOrderIter(validation_tree):
+        validation_tree_nodes.append(val_node)
+
+    hashed_pipeline()

vspec/vspec2vss_config.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2023 Contributors to COVESA
+#
+# This program and the accompanying materials are made available under the
+# terms of the Mozilla Public License 2.0 which is available at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+# SPDX-License-Identifier: MPL-2.0
+
+class Vspec2VssConfig:
+    """
+    This class is intended to be a container for settings related to the conversion
+    from *.vspec files to the internal VSS model as well as for some generic vspec2x features.
+    A user may change settings before vspec2x.main is called.
+    Then vspec2x will use those settings to control which command line arguments
+    that will be accepted or considered, and based on those some of the settings in this
+    file may be overridden.
+    The config instance is later passed to the vss2x instance, so that it can consider
+    the settings as well.
+    """
+
+    def __init__(self):
+        """
+        Setting default/recommended features for vss-tools.
+        """
+
+        # Input to model parsing
+
+        # Is it possible to request that uuid shall be generated
+        self.uuid_supported = True
+        # Shall it be possible to specify which extended attributes to consider?
+        self.extended_attributes_supported = True
+        # Shall it be possible to give a type tree
+        self.type_tree_supported = True
+        # Is it supported to get type data generated to a separate file
+        # Only relevant if self.type_tree_supported is True
+        self.separate_output_type_file_supported = True
+
+        # shall vspec2vss expand the model (by default)
+        self.expand_model = True
+        # if so shall there anyway be an option to NOT expand
+        self.no_expand_option_supported = True
+
+        # Is an output file required as part of command line arguments
+        self.output_file_required = True
+
+        # As of now we have only one type of nodes in the type tree
+        # and that is structs, so if we support type tree we assume structs are supported
+
+        # Default values for features
+        # These are typically updated by vspec2x when reading command line arguments
+        self.generate_uuid = False

vspec/vspec2x.py

@@ -0,0 +1,191 @@
+
+
+# Copyright (c) 2016 Contributors to COVESA
+#
+# This program and the accompanying materials are made available under the
+# terms of the Mozilla Public License 2.0 which is available at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+# SPDX-License-Identifier: MPL-2.0
+
+#
+# Convert vspec files to various other formats
+#
+
+from vspec.model.vsstree import VSSNode
+from vspec.model.constants import VSSTreeType
+from vspec.loggingconfig import initLogging
+from vspec.vss2x import Vss2X
+from vspec.vspec2vss_config import Vspec2VssConfig
+import argparse
+import logging
+import sys
+import vspec
+
+import pkg_resources  # part of setuptools
+VERSION = pkg_resources.require("vss-tools")[0].version
+
+
+class Vspec2X():
+    """
+    Framework for translating from *.vspec files to first an internal VSS model,
+    and then to something else called X.
+    Users must provide a Vss2X generator that can generate X if they get a VSS model as input.
+    """
+    def __init__(self, generator: Vss2X, vspec2vss_config: Vspec2VssConfig):
+
+        self.generator = generator
+        self.vspec2vss_config = vspec2vss_config
+
+    def main(self, arguments):
+        initLogging()
+        parser = argparse.ArgumentParser(description="Convert vspec to other formats.")
+
+        parser.add_argument('--version', action='version', version=VERSION)
+        parser.add_argument('-I', '--include-dir', action='append', metavar='dir', type=str, default=[],
+                            help='Add include directory to search for included vspec files.')
+        if self.vspec2vss_config.extended_attributes_supported:
+            parser.add_argument('-e', '--extended-attributes', type=str, default="",
+                                help='Whitelisted extended attributes as comma separated list.')
+        parser.add_argument('-s', '--strict', action='store_true',
+                            help='Use strict checking: Terminate when anything not covered or not recommended '
+                                 'by VSS language or extensions is found.')
+        parser.add_argument('--abort-on-unknown-attribute', action='store_true',
+                            help="Terminate when an unknown attribute is found.")
+        parser.add_argument('--abort-on-name-style', action='store_true',
+                            help="Terminate if name style does not follow VSS standard catalog naming convention.")
+        if self.vspec2vss_config.uuid_supported:
+            parser.add_argument('--uuid', action='store_true',
+                                help='Include uuid in generated files. (Deprecated, will be removed in VSS-tools 6.0)')
+        if self.vspec2vss_config.expand_model and self.vspec2vss_config.no_expand_option_supported:
+            parser.add_argument('--no-expand', action='store_true',
+                                help='Do not expand tree.')
+        parser.add_argument('-o', '--overlays', action='append', metavar='overlays', type=str, default=[],
+                            help='Add overlay that will be layered on top of the VSS file in the order they appear.')
+        parser.add_argument('-q', '--quantity-file', action='append', metavar='quantity_file', type=str, default=[],
+                            help='Quantity file to be used for generation. Argument -q may be used multiple times.')
+        parser.add_argument('-u', '--unit-file', action='append', metavar='unit_file', type=str, default=[],
+                            help='Unit file to be used for generation. Argument -u may be used multiple times.')
+        parser.add_argument('vspec_file', metavar='<vspec_file>',
+                            help='The vehicle specification file to convert.')
+        if self.vspec2vss_config.output_file_required:
+            parser.add_argument('output_file', metavar='<output_file>',
+                                help='The file to write output to.')
+
+        if self.vspec2vss_config.type_tree_supported:
+            type_group = parser.add_argument_group('VSS Data Type Tree arguments',
+                                                   'Arguments related to struct/type support')
+
+            type_group.add_argument('-vt', '--vspec-types-file', action='append', metavar='vspec_types_file', type=str,
+                                    default=[],
+                                    help='Data types file in vspec format.')
+            if self.vspec2vss_config.separate_output_type_file_supported:
+                # Note we might get some odd errors if using -ot in a tool not supporting it due to conflict with -o
+                type_group.add_argument('-ot', '--types-output-file', metavar='<types_output_file>',
+                                        help='Output file for writing data types from vspec file. ' +
+                                        'If not specified, a single file is used where applicable. ' +
+                                        'In case of JSON and YAML, the data is exported under a ' +
+                                        'special key - "ComplexDataTypes"')
+
+        self.generator.add_arguments(parser.add_argument_group(
+            "Exporter specific arguments", ""))
+
+        args = parser.parse_args(arguments)
+
+        logging.info("VSS-tools version %s", VERSION)
+
+        include_dirs = ["."]
+        include_dirs.extend(args.include_dir)
+
+        abort_on_unknown_attribute = False
+        abort_on_namestyle = False
+
+        if args.abort_on_unknown_attribute or args.strict:
+            abort_on_unknown_attribute = True
+        if args.abort_on_name_style or args.strict:
+            abort_on_namestyle = True
+
+        if self.vspec2vss_config.extended_attributes_supported and (len(args.extended_attributes) > 0):
+            known_extended_attributes_list = args.extended_attributes.split(",")
+            vspec.model.vsstree.VSSNode.whitelisted_extended_attributes = known_extended_attributes_list
+            logging.info(f"Known extended attributes: {', '.join(known_extended_attributes_list)}")
+        else:
+            known_extended_attributes_list = list()
+
+        self.vspec2vss_config.generate_uuid = self.vspec2vss_config.uuid_supported and args.uuid
+
+        # Follow up to https://github.com/COVESA/vehicle_signal_specification/pull/721
+        # Deprecate --uuid
+        if self.vspec2vss_config.generate_uuid:
+            logging.warning("The argument --uuid is deprecated and the uuid feature is planned "
+                            "to be removed in VSS-tools 6.0")
+            logging.info("If you need static identifiers consider using the vspec2id tool")
+
+        self.vspec2vss_config.expand_model = (self.vspec2vss_config.expand_model and not
+                                              (self.vspec2vss_config.no_expand_option_supported and args.no_expand))
+
+        vspec.load_quantities(args.vspec_file, args.quantity_file)
+        vspec.load_units(args.vspec_file, args.unit_file)
+
+        # process data type tree
+        data_type_tree = None
+        if self.vspec2vss_config.type_tree_supported:
+            if self.vspec2vss_config.separate_output_type_file_supported:
+                if args.types_output_file is not None and not args.vspec_types_file:
+                    parser.error("An output file for data types was provided. Please also provide "
+                                 "the input vspec file for data types")
+            if args.vspec_types_file:
+                data_type_tree = self.processDataTypeTree(
+                    parser, args, include_dirs, abort_on_namestyle)
+                vspec.verify_mandatory_attributes(data_type_tree, abort_on_unknown_attribute)
+
+        try:
+            logging.info(f"Loading vspec from {args.vspec_file}...")
+            tree = vspec.load_tree(
+                args.vspec_file, include_dirs, VSSTreeType.SIGNAL_TREE,
+                break_on_name_style_violation=abort_on_namestyle,
+                expand_inst=False, data_type_tree=data_type_tree)
+
+            for overlay in args.overlays:
+                logging.info(f"Applying VSS overlay from {overlay}...")
+                othertree = vspec.load_tree(overlay, include_dirs, VSSTreeType.SIGNAL_TREE,
+                                            break_on_name_style_violation=abort_on_namestyle, expand_inst=False,
+                                            data_type_tree=data_type_tree)
+                vspec.merge_tree(tree, othertree)
+
+            vspec.check_type_usage(tree, VSSTreeType.SIGNAL_TREE, data_type_tree)
+            if self.vspec2vss_config.expand_model:
+                vspec.expand_tree_instances(tree)
+
+            vspec.clean_metadata(tree)
+            vspec.verify_mandatory_attributes(tree, abort_on_unknown_attribute)
+            logging.info("Calling exporter...")
+
+            self.generator.generate(args, tree, self.vspec2vss_config, data_type_tree)
+            logging.info("All done.")
+        except vspec.VSpecError as e:
+            logging.error(f"Error: {e}")
+            sys.exit(255)
+
+    def processDataTypeTree(self, parser: argparse.ArgumentParser, args, include_dirs,
+                            abort_on_namestyle: bool) -> VSSNode:
+        """
+        Helper function to process command line arguments and invoke logic for processing data
+        type information provided in vspec format
+        """
+        if self.vspec2vss_config.separate_output_type_file_supported and \
+           args.types_output_file is None:
+            logging.info("Sensors and custom data types will be consolidated into one file.")
+
+        first_tree = True
+        for type_file in args.vspec_types_file:
+            logging.info(f"Loading and processing struct/data type tree from {type_file}")
+            new_tree = vspec.load_tree(type_file, include_dirs, VSSTreeType.DATA_TYPE_TREE,
+                                       break_on_name_style_violation=abort_on_namestyle, expand_inst=False)
+            if first_tree:
+                tree = new_tree
+                first_tree = False
+            else:
+                vspec.merge_tree(tree, new_tree)
+        vspec.check_type_usage(tree, VSSTreeType.DATA_TYPE_TREE)
+        return tree

vspec/vss2x.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2023 Contributors to COVESA
+#
+# This program and the accompanying materials are made available under the
+# terms of the Mozilla Public License 2.0 which is available at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+# SPDX-License-Identifier: MPL-2.0
+
+from abc import ABC, abstractmethod
+import argparse
+from typing import Optional
+from vspec.model.vsstree import VSSNode
+from vspec.vspec2vss_config import Vspec2VssConfig
+
+
+class Vss2X(ABC):
+    """
+    Abstract class for something that takes a VSS model as input
+    (signal tree plus optionally a type tree)
+    and does "something". For now it is assumed that output is either files
+    or text output.
+    The generator may use both command line config as well as specific configs used for
+    creating the VSS model to control the generation.
+    """
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.description = "This generator does not support any additional arguments."
+
+    @abstractmethod
+    def generate(self, config: argparse.Namespace, signal_root: VSSNode, vspec2vss_config: Vspec2VssConfig,
+                 data_type_root: Optional[VSSNode] = None) -> None:
+        """
+        Previously called export, but changing to a more generic name.
+        Must be defined by the tool.
+        """
+        pass

vspec/vssexporters/__init__.py

@@ -0,0 +1,7 @@
+# Copyright (c) 2022 Contributors to COVESA
+#
+# This program and the accompanying materials are made available under the
+# terms of the Mozilla Public License 2.0 which is available at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+# SPDX-License-Identifier: MPL-2.0