fprime-gds 3.6.1__py3-none-any.whl → 4.0.0a1__py3-none-any.whl

fprime_gds/common/loaders/json_loader.py

@@ -71,6 +71,18 @@ class JsonLoader(dict_loader.DictLoader):
             self.json_dict["metadata"].get("projectVersion", "unknown"),
         )
 
+    def get_metadata(self):
+        """Get the metadata field of the JSON dictionary
+
+        Raises:
+            GdsDictionaryParsingException: if the dictionary has no metadata field
+        """
+        if "metadata" not in self.json_dict:
+            raise GdsDictionaryParsingException(
+                f"Dictionary has no metadata field: {self.json_file}"
+            )
+        return self.json_dict["metadata"]
+
     def parse_type(self, type_dict: dict) -> BaseType:
         type_name: str = type_dict.get("name", None)
 
@@ -102,6 +114,9 @@ class JsonLoader(dict_loader.DictLoader):
                 f"Dictionary type name has no corresponding type definition: {type_name}"
             )
 
+        if qualified_type.get("kind") == "alias":
+            return self.parse_type(qualified_type.get("underlyingType"))
+        
         if qualified_type.get("kind") == "array":
             return self.construct_array_type(type_name, qualified_type)
 

fprime_gds/common/loaders/pkt_json_loader.py

@@ -0,0 +1,121 @@
+"""
+pkt_json_loader.py:
+
+Loads flight dictionary (JSON) and returns Python dictionaries of telemetry packets
+
+@author jawest
+"""
+
+from fprime_gds.common.templates.pkt_template import PktTemplate
+from fprime_gds.common.loaders.json_loader import JsonLoader
+from fprime_gds.common.data_types.exceptions import GdsDictionaryParsingException
+
+
+class PktJsonLoader(JsonLoader):
+    """Class to load python based telemetry packet dictionaries"""
+
+    PACKETS_FIELD = "telemetryPacketSets"
+
+    SET_NAME = "name"
+    MEMBERS = "members"
+
+    def get_id_dict(self, path, packet_set_name: str, ch_name_dict: dict):
+        if path in self.saved_dicts and packet_set_name in self.saved_dicts[path]:
+            (id_dict, name_dict) = self.saved_dicts[path][packet_set_name]
+        else:
+            (id_dict, name_dict, self.versions) = self.construct_dicts(packet_set_name, ch_name_dict)
+            if path not in self.saved_dicts:
+                self.saved_dicts[path] = dict()
+            self.saved_dicts[path].update({packet_set_name: (id_dict, name_dict)})
+
+        return id_dict
+    
+    def get_name_dict(self, path, packet_set_name: str, ch_name_dict: dict):
+        if path in self.saved_dicts and packet_set_name in self.saved_dicts[path]:
+            (id_dict, name_dict) = self.saved_dicts[path][packet_set_name]
+        else:
+            (id_dict, name_dict, self.versions) = self.construct_dicts(packet_set_name, ch_name_dict)
+            if path not in self.saved_dicts:
+                self.saved_dicts[path] = dict()
+            self.saved_dicts[path].update({packet_set_name: (id_dict, name_dict)})
+
+        return name_dict
+
+
+    def construct_dicts(self, packet_set_name: str, ch_name_dict: dict):
+        """
+        Constructs and returns python dictionaries keyed on id and name
+
+        This function should not be called directly, instead, use
+        get_id_dict(path) and get_name_dict(path)
+
+        Args:
+            ch_name_dict (dict()): Channel dictionary with names as keys and
+                                   ChTemplate objects as values.
+
+        Returns:
+            A tuple with two packet dictionaries (type==dict()):
+            (id_dict, name_dict) and the dictionary version. The keys of the packet dictionaries should 
+            be the packets' id and name fields respectively and the values should be PktTemplate objects.
+        """
+        id_dict = {}
+        name_dict = {}
+
+        if self.PACKETS_FIELD not in self.json_dict:
+            raise GdsDictionaryParsingException(
+                f"Ground Dictionary missing '{self.PACKETS_FIELD}' field: {str(self.json_file)}"
+            )
+
+        for packet_dict in self.json_dict[self.PACKETS_FIELD]:
+            try:
+                if packet_set_name == packet_dict[self.SET_NAME]:
+                    for packet_group_dict in packet_dict.get(self.MEMBERS, []):
+                        packet_temp = self.construct_template_from_dict(packet_group_dict, ch_name_dict)
+                        id_dict[packet_temp.get_id()] = packet_temp
+                        name_dict[packet_temp.get_name()] = packet_temp
+
+                    return (
+                        dict(sorted(id_dict.items())),
+                        dict(sorted(name_dict.items())),
+                        self.get_versions(),
+                    )
+                
+            except KeyError as e:
+                raise GdsDictionaryParsingException(
+                    f"{str(e)} key missing from telemetry packet dictionary entry: {str(packet_dict)}"
+                )
+            
+        raise GdsDictionaryParsingException(
+            f"Ground Dictionary does not contain packet set '{packet_set_name}'"
+        )
+
+    def construct_template_from_dict(self, packet_group_dict: dict, ch_name_dict: dict):
+        """        
+        Args:
+            packet_group_dict (dict()): Packet group dictionary with group id, name, and members
+            ch_name_dict (dict()): Channel dictionary with names as keys and ChTemplate objects as values.
+        Returns:
+            A a PktTemplate object containing the packet group id, group name, and list of ChTemplate 
+            objects that represent each member in the packet.
+        """
+        try:
+            ch_list = []
+            group_name = packet_group_dict["name"]
+            group_id = packet_group_dict["id"]
+            group_members = packet_group_dict["members"]
+
+            for ch_name in group_members:
+                ch_template = ch_name_dict[ch_name]
+                ch_list.append(ch_template)
+            
+        except KeyError as e:
+            raise GdsDictionaryParsingException(
+                f"{str(e)} key missing from telemetry packet member or member is not a channel in the dictionary: {str(group_name)}"
+            )
+        
+        return PktTemplate(
+            group_id,
+            group_name,
+            ch_list
+        )
+

fprime_gds/common/loaders/prm_json_loader.py

@@ -0,0 +1,85 @@
+"""
+prm_json_loader.py:
+
+Loads flight dictionary (JSON) and returns id and mnemonic based Python dictionaries of params
+
+@author zimri.leisher
+"""
+
+from fprime_gds.common.templates.prm_template import PrmTemplate
+from fprime_gds.common.loaders.json_loader import JsonLoader
+from fprime_gds.common.data_types.exceptions import GdsDictionaryParsingException
+
+
+class PrmJsonLoader(JsonLoader):
+    """Class to load parameters from json dictionaries"""
+
+    PARAMS_FIELD = "parameters"
+
+    ID = "id"
+    NAME = "name"
+    TYPE = "type"
+    DESC = "annotation"
+    DEFAULT = "default"
+    
+
+    def construct_dicts(self, _):
+        """
+        Constructs and returns python dictionaries keyed on id and name
+
+        Args:
+            _: Unused argument (inherited)
+        Returns:
+            A tuple with two channel dictionaries (python type dict):
+            (id_dict, fqn_name_dict). The keys should be the channels' id and
+            fully qualified name fields respectively and the values should be PrmTemplate
+            objects.
+        """
+        id_dict = {}
+        fqn_name_dict = {}
+
+        if self.PARAMS_FIELD not in self.json_dict:
+            raise GdsDictionaryParsingException(
+                f"Ground Dictionary missing '{self.PARAMS_FIELD}' field: {str(self.json_file)}"
+            )
+
+        for prm_dict in self.json_dict[self.PARAMS_FIELD]:
+            # Create a channel template object
+            prm_temp = self.construct_template_from_dict(prm_dict)
+
+            id_dict[prm_temp.get_id()] = prm_temp
+            fqn_name_dict[prm_temp.get_full_name()] = prm_temp
+
+        return (
+            dict(sorted(id_dict.items())),
+            dict(sorted(fqn_name_dict.items())),
+            self.get_versions(),
+        )
+
+    def construct_template_from_dict(self, prm_dict: dict) -> PrmTemplate:
+        try:
+            prm_id = prm_dict[self.ID]
+            # The below assignment also raises a ValueError if the name does not contain a '.'
+            qualified_component_name, prm_name = prm_dict[self.NAME].rsplit('.', 1)
+            if not qualified_component_name or not prm_name:
+                raise ValueError()
+
+            type_obj = self.parse_type(prm_dict[self.TYPE])
+        except ValueError as e:
+            raise GdsDictionaryParsingException(
+                f"Parameter dictionary entry malformed, expected name of the form '<QUAL_COMP_NAME>.<PRM_NAME>' in : {str(prm_dict)}"
+            )
+        except KeyError as e:
+            raise GdsDictionaryParsingException(
+                f"{str(e)} key missing from parameter dictionary entry or its associated type in the dictionary: {str(prm_dict)}"
+            )
+
+        prm_default_val = prm_dict.get(self.DEFAULT, None)
+
+        return PrmTemplate(
+            prm_id,
+            prm_name,
+            qualified_component_name,
+            type_obj,
+            prm_default_val
+        )

fprime_gds/common/pipeline/dictionaries.py

@@ -7,25 +7,22 @@ class called "Dictionaries".
 @author mstarch
 """
 
-import os
 from pathlib import Path
 
-import fprime_gds.common.loaders.ch_py_loader
-import fprime_gds.common.loaders.ch_xml_loader
-
-# Py Loaders
-import fprime_gds.common.loaders.cmd_py_loader
 
 # XML Loaders
+import fprime_gds.common.loaders.ch_xml_loader
 import fprime_gds.common.loaders.cmd_xml_loader
-import fprime_gds.common.loaders.event_py_loader
 import fprime_gds.common.loaders.event_xml_loader
+import fprime_gds.common.loaders.fw_type_json_loader
+import fprime_gds.common.loaders.pkt_json_loader
 import fprime_gds.common.loaders.pkt_xml_loader
+
+# JSON Loaders
 import fprime_gds.common.loaders.ch_json_loader
 import fprime_gds.common.loaders.cmd_json_loader
 import fprime_gds.common.loaders.event_json_loader
 
-
 class Dictionaries:
     """
     Dictionaries class to encapsulate the many different dictionaries used in the system. This includes the following
@@ -49,9 +46,11 @@ class Dictionaries:
         self._event_name_dict = None
         self._channel_name_dict = None
         self._packet_dict = None
+        self._fw_type_name_dict = None
         self._versions = None
+        self._metadata = None
 
-    def load_dictionaries(self, dictionary, packet_spec):
+    def load_dictionaries(self, dictionary, packet_spec, packet_set_name):
         """
         Loads the dictionaries based on the dictionary path supplied. Optional packet_spec is allowed to specify the
         definitions of packets.
@@ -59,57 +58,44 @@ class Dictionaries:
         :param dictionary: dictionary path used for loading dictionaries
         :param packet_spec: specification for packets, or None, for packetized telemetry
         """
-        # Loading the dictionaries from a directory. A directory indicates heritage python dicts.
-        if os.path.isdir(dictionary):
-            # Events
-            event_loader = fprime_gds.common.loaders.event_py_loader.EventPyLoader()
-            self._event_id_dict = event_loader.get_id_dict(
-                os.path.join(dictionary, "events")
-            )
-            self._event_name_dict = event_loader.get_name_dict(
-                os.path.join(dictionary, "events")
-            )
-            # Commands
-            command_loader = fprime_gds.common.loaders.cmd_py_loader.CmdPyLoader()
-            self._command_id_dict = command_loader.get_id_dict(
-                os.path.join(dictionary, "commands")
-            )
-            self._command_name_dict = command_loader.get_name_dict(
-                os.path.join(dictionary, "commands")
-            )
-            # Channels
-            channel_loader = fprime_gds.common.loaders.ch_py_loader.ChPyLoader()
-            self._channel_id_dict = channel_loader.get_id_dict(
-                os.path.join(dictionary, "channels")
-            )
-            self._channel_name_dict = channel_loader.get_name_dict(
-                os.path.join(dictionary, "channels")
-            )
-        elif Path(dictionary).is_file() and ".json" in Path(dictionary).suffixes:
+        if Path(dictionary).is_file() and ".json" in Path(dictionary).suffixes:
             # Events
             json_event_loader = (
                 fprime_gds.common.loaders.event_json_loader.EventJsonLoader(dictionary)
             )
             self._event_name_dict = json_event_loader.get_name_dict(None)
             self._event_id_dict = json_event_loader.get_id_dict(None)
-            self._versions = json_event_loader.get_versions()
             # Commands
             json_command_loader = (
                 fprime_gds.common.loaders.cmd_json_loader.CmdJsonLoader(dictionary)
             )
             self._command_name_dict = json_command_loader.get_name_dict(None)
             self._command_id_dict = json_command_loader.get_id_dict(None)
-            assert (
-                self._versions == json_command_loader.get_versions()
-            ), "Version mismatch while loading"
             # Channels
             json_channel_loader = fprime_gds.common.loaders.ch_json_loader.ChJsonLoader(
                 dictionary
             )
             self._channel_name_dict = json_channel_loader.get_name_dict(None)
             self._channel_id_dict = json_channel_loader.get_id_dict(None)
+            # Fw Types
+            fw_types_loader = fprime_gds.common.loaders.fw_type_json_loader.FwTypeJsonLoader(
+                dictionary
+            )
+            self._fw_type_name_dict = fw_types_loader.get_name_dict(None)
+            # Metadata
+            self._versions = json_event_loader.get_versions()
+            self._metadata = json_event_loader.get_metadata().copy()
+            self._metadata["dictionary_type"] = "json"
+            # Each loaders should agree on metadata and versions
+            assert (
+                json_command_loader.get_metadata()
+                == json_channel_loader.get_metadata()
+                == json_event_loader.get_metadata()
+            ), "Metadata mismatch while loading"
             assert (
-                self._versions == json_channel_loader.get_versions()
+                json_command_loader.get_versions()
+                == json_channel_loader.get_versions()
+                == json_event_loader.get_versions()
             ), "Version mismatch while loading"
         # XML dictionaries
         elif Path(dictionary).is_file():
@@ -132,12 +118,23 @@ class Dictionaries:
             assert (
                 self._versions == channel_loader.get_versions()
             ), "Version mismatch while loading"
+            # versions are camelCase to match the metadata field of the JSON dictionaries
+            self._metadata = {
+                "frameworkVersion": self._versions[0],
+                "projectVersion": self._versions[1],
+                "dictionary_type": "xml",
+            }
         else:
             msg = f"[ERROR] Dictionary '{dictionary}' does not exist."
             raise Exception(msg)
         # Check for packet specification
-        if packet_spec is not None:
-            packet_loader = fprime_gds.common.loaders.pkt_xml_loader.PktXmlLoader()
+        if self._metadata["dictionary_type"] == "json" and packet_set_name is not None:
+            packet_loader = fprime_gds.common.loaders.pkt_json_loader.PktJsonLoader(dictionary)
+            self._packet_dict = packet_loader.get_id_dict(
+                None, packet_set_name, self._channel_name_dict
+            )
+        elif packet_spec is not None:
+            packet_loader = fprime_gds.common.loaders.pkt_xml_loader.PktXmlLoader(dictionary)
             self._packet_dict = packet_loader.get_id_dict(
                 packet_spec, self._channel_name_dict
             )
@@ -173,6 +170,11 @@ class Dictionaries:
     def channel_name(self):
         """Channel dictionary by name"""
         return self._channel_name_dict
+    
+    @property
+    def fw_type_name(self):
+        """Fw type name dictionary by name"""
+        return self._fw_type_name_dict
 
     @property
     def project_version(self):
@@ -184,6 +186,14 @@ class Dictionaries:
         """Framework version in dictionary"""
         return self._versions[0]
 
+    @property
+    def metadata(self):
+        """Dictionary metadata.
+
+        Note: framework_version and project_version are also available as separate properties
+        for legacy reasons. New code should use the metadata property."""
+        return self._metadata
+
     @property
     def packet(self):
         """Packet dictionary"""

fprime_gds/common/pipeline/encoding.py

@@ -6,6 +6,7 @@ and decoding into a single component that the be composed into the standard pipe
 
 @mstarch
 """
+
 import fprime_gds.common.decoders.ch_decoder
 import fprime_gds.common.decoders.event_decoder
 import fprime_gds.common.decoders.file_decoder
@@ -130,6 +131,24 @@ class EncodingDecoding:
         """
         return self.channel_decoder.deregister(consumer)
 
+    def register_file_consumer(self, consumer):
+        """
+        Registers a consumer with the file decoder.
+
+        :param consumer: consumer of file packets
+        """
+        self.file_decoder.register(consumer)
+
+    def remove_file_consumer(self, consumer):
+        """
+        Removes a consumer from the file decoder. Will raise an error if the history was not
+        previously registered.
+
+        :param consumer: consumer of channels
+        :return: a boolean indicating if the consumer was removed.
+        """
+        return self.file_decoder.deregister(consumer)
+
     def register_command_consumer(self, consumer):
         """
         Registers a history with the standard pipeline.

fprime_gds/common/pipeline/histories.py

@@ -6,6 +6,7 @@ to compose in this code.
 
 @author mstarch
 """
+
 from typing import Type
 
 from fprime_gds.common.history.history import History
@@ -37,6 +38,9 @@ class Histories:
         :param coders: coders object to register histories with
         """
         self.coders = coders
+        # Allow implementation type to disable histories
+        if self._implementation_type is None:
+            return
         # Create histories, RAM histories for now
         self.commands = self._implementation_type()
         self.events = self._implementation_type()

fprime_gds/common/pipeline/standard.py

@@ -8,6 +8,7 @@ below.
 
 :author: lestarch
 """
+
 import datetime
 import os.path
 from pathlib import Path
@@ -19,6 +20,7 @@ import fprime_gds.common.data_types.cmd_data
 import fprime_gds.common.distributor.distributor
 import fprime_gds.common.logger.data_logger
 from fprime_gds.common.transport import RoutingTag, ThreadedTCPSocketClient
+from fprime_gds.common.utils.config_manager import ConfigManager
 
 # Local imports for the sake of composition
 from . import dictionaries, encoding, files, histories
@@ -54,7 +56,13 @@ class StandardPipeline:
         self.__transport_type = ThreadedTCPSocketClient
 
     def setup(
-        self, config, dictionary, file_store, logging_prefix=None, packet_spec=None
+        self,
+        config: ConfigManager,
+        dictionary,
+        file_store,
+        logging_prefix=None,
+        packet_spec=None,
+        packet_set_name=None,
     ):
         """
         Setup the standard pipeline for moving data from the middleware layer through the GDS layers using the standard
@@ -84,7 +92,13 @@ class StandardPipeline:
         self.distributor = fprime_gds.common.distributor.distributor.Distributor(config)
         self.client_socket = self.__transport_type()
         # Setup dictionaries encoders and decoders
-        self.dictionaries.load_dictionaries(self.dictionary_path, packet_spec)
+        self.dictionaries.load_dictionaries(
+            self.dictionary_path, packet_spec, packet_set_name
+        )        
+        # Update config to use Fw types defined in the JSON dictionary
+        if self.dictionaries.fw_type_name:
+            for fw_type_name, fw_type in self.dictionaries.fw_type_name.items():
+                config.set("types", fw_type_name, fw_type)
         self.coders.setup_coders(
             self.dictionaries, self.distributor, self.client_socket, config
         )

fprime_gds/common/templates/prm_template.py

@@ -0,0 +1,81 @@
+"""
+@brief Params Template class
+
+Instances of this class describe a parameter of a component instance (not 
+including a specific value)
+
+@date Created January 27, 2025
+@author Zimri Leisher
+
+@bug Hopefully none
+"""
+
+from fprime.common.models.serialize.type_base import BaseType
+from fprime.common.models.serialize.type_exceptions import TypeMismatchException
+
+from . import data_template
+
+
+class PrmTemplate(data_template.DataTemplate):
+    """Class for param templates that describe parameters of component instances"""
+
+    def __init__(
+        self,
+        prm_id: int,
+        prm_name: str,
+        comp_name: str,
+        prm_type_obj: BaseType,
+        prm_default_val,
+    ):
+        """
+        Constructor
+
+        Args:
+            prm_id: the id of the parameter
+            prm_name: the name of the parameter
+            comp_name: the name of the component instance owning this parameter
+            prm_type_obj: the instance of BaseType corresponding to the type of this parameter
+            prm_default_val: the default value of this parameter, in raw JSON form
+        """
+        super().__init__()
+        # Make sure correct types are passed
+        if not isinstance(prm_id, int):
+            raise TypeMismatchException(int, type(prm_id))
+
+        if not isinstance(prm_name, str):
+            raise TypeMismatchException(str, type(prm_name))
+
+        if not isinstance(comp_name, str):
+            raise TypeMismatchException(str, type(comp_name))
+
+        if not issubclass(prm_type_obj, BaseType):
+            raise TypeMismatchException(BaseType, prm_type_obj)
+
+        # prm_default_val is an arbitrary type, likely a primitive or dict
+
+        self.prm_id = prm_id
+        self.prm_name = prm_name
+        self.comp_name = comp_name
+        self.prm_type_obj = prm_type_obj
+        self.prm_default_val = prm_default_val
+
+    def get_full_name(self):
+        """
+        Get the full name of this param
+
+        Returns:
+            The full name (component.param) for this param
+        """
+        return f"{self.comp_name}.{self.prm_name}"
+
+    def get_id(self):
+        return self.prm_id
+
+    def get_name(self):
+        return self.prm_name
+
+    def get_comp_name(self):
+        return self.comp_name
+
+    def get_type_obj(self):
+        return self.prm_type_obj

fprime_gds/common/testing_fw/api.py

@@ -9,6 +9,8 @@ telemetry and dictionaries.
 """
 import signal
 import time
+from pathlib import Path
+import shutil
 
 from fprime.common.models.serialize.time_type import TimeType
 
@@ -63,6 +65,15 @@ class IntegrationTestAPI(DataHandler):
         # Initialize the logger
         self.logger = TestLogger(logpath) if logpath is not None else None
 
+        # Copy dictionaries and binary file to output directory
+        if logpath is not None:
+            base_dir = Path(self.pipeline.dictionary_path).parents[1]
+            for subdir in ['bin', 'dict']:
+                dir_path = base_dir / subdir
+                if dir_path.is_dir():
+                    shutil.copytree(dir_path, Path(logpath) / subdir,
+                                    dirs_exist_ok=True)
+
         # A predicate used as a filter to choose which events to log automatically
         self.event_log_filter = self.get_event_pred()
 
@@ -215,6 +226,37 @@ class IntegrationTestAPI(DataHandler):
         """
         self.event_log_filter = self.get_event_pred(event, args, severity, time_pred)
 
+    def get_deployment(self):
+        """
+        Get the deployment of the target using the loaded FSW dictionary path
+        Returns:
+            The name of the deployment (str)
+        """
+        return Path(self.pipeline.dictionary_path).parent.parent.name
+
+    def wait_for_dataflow(self, count=1, channels=None, start=None, timeout=120):
+        """
+        Wait for data flow by checking for any telemetry updates within a specified timeout.
+
+        Args:
+            count: either an exact amount (int) or a predicate to specify how many objects to find
+            channels: a channel specifier or list of channel specifiers (mnemonic, ID, or predicate). All will count if None
+            start: an optional index or predicate to specify the earliest item to search
+            timeout: the number of seconds to wait before terminating the search (int)
+        """
+        if start is None:
+            start = self.get_latest_time()
+
+        history = self.get_telemetry_subhistory()
+        result = self.await_telemetry_count(
+            count, channels=channels, history=history, start=start, timeout=timeout
+        )
+        if not result:
+            msg = f'Failed to detect any data flow for {timeout} s.'
+            self.__log(msg, TestLogger.RED)
+            assert False, msg
+        self.remove_telemetry_subhistory(history)
+
     ######################################################################################
     #   History Functions
     ######################################################################################