fprime-gds 3.6.2a1__py3-none-any.whl → 4.0.0a2__py3-none-any.whl

fprime_gds/common/handlers.py

@@ -6,7 +6,10 @@ defines the "DataHandler" base class for handling data.
 
 @author mstarch
 """
+
 import abc
+from typing import List, Type
+from fprime_gds.plugin.definitions import gds_plugin_specification
 
 
 class DataHandler(abc.ABC):
@@ -27,6 +30,42 @@ class DataHandler(abc.ABC):
         """
 
 
+class DataHandlerPlugin(DataHandler, abc.ABC):
+    """PLugin class allowing for custom data handlers
+
+    This class acts as a DataHandler class with the addition that it can be used as a plugin and thus self reports the
+    data types it handles (whereas DataHandler leaves that up to the registration call). Users shall concretely subclass
+    this class with their own data handling functionality.
+    """
+
+    @abc.abstractmethod
+    def get_handled_descriptors() -> List[str]:
+        """Return a list of data descriptor names this plugin handles"""
+        raise NotImplementedError()
+
+    @classmethod
+    @gds_plugin_specification
+    def register_data_handler_plugin(cls) -> Type["DataHandlerPlugin"]:
+        """Register a plugin to provide post-decoding data handling capabilities
+
+        Plugin hook for registering a plugin that supplies a DataHandler implementation. Implementors of this hook must
+        return a non-abstract subclass of DataHandlerPlugin. This class will be provided as a data handling
+        that is automatically enabled. Users may disable this via the command line. This data handler will be supplied
+        all data types returned by the `get_data_types()` method.
+
+        This DataHandler will run within the standard GDS (UI) process. Users wanting a separate process shall use a
+        GdsApp plugin instead.
+
+        Note: users should return the class, not an instance of the class. Needed arguments for instantiation are
+        determined from class methods, solicited via the command line, and provided at construction time to the chosen
+        instantiation.
+
+        Returns:
+            DataHandlerPlugin subclass (not instance)
+        """
+        raise NotImplementedError()
+
+
 class HandlerRegistrar(abc.ABC):
     """
     Defines a class that will take in registrants and remember them for calling back later. These objects should be of

fprime_gds/common/loaders/fw_type_json_loader.py

@@ -0,0 +1,54 @@
+"""
+fw_type_json_loader.py:
+
+Loads flight dictionary (JSON) and returns name based Python dictionaries of Fw types
+
+@author jawest
+"""
+
+from fprime_gds.common.loaders.json_loader import JsonLoader
+from fprime_gds.common.data_types.exceptions import GdsDictionaryParsingException
+
+class FwTypeJsonLoader(JsonLoader):
+    """Class to load python based Fw type dictionaries"""
+
+    TYPE_DEFINITIONS_FIELD = "typeDefinitions"
+
+    def construct_dicts(self, _):
+        """
+        Constructs and returns python dictionaries keyed on id and name
+
+        Args:
+            _: Unused argument (inherited)
+        Returns:
+            A tuple with two Fw type dictionaries (python type dict):
+            (id_dict, name_dict). The keys should be the type id and
+            name fields respectively and the values should be type name
+            strings. Note: An empty id dictionary is returned since there 
+            are no id fields in the Fw type alias JSON dictionary entries.
+        """
+        id_dict = {}
+        name_dict = {}
+
+        if self.TYPE_DEFINITIONS_FIELD not in self.json_dict:
+            raise GdsDictionaryParsingException(
+                f"Ground Dictionary missing '{self.TYPE_DEFINITIONS_FIELD}' field: {str(self.json_file)}"
+            )
+
+        for type_def in self.json_dict[self.TYPE_DEFINITIONS_FIELD]:
+            try:
+                if type_def["kind"] == "alias":
+                    name = str(type_def["qualifiedName"])
+                    # Only consider names with the pattern Fw*Type
+                    if name.startswith("Fw") and name.endswith("Type"):
+                        name_dict[type_def["qualifiedName"]] = type_def["underlyingType"]["name"]
+            except KeyError as e:
+                raise GdsDictionaryParsingException(
+                    f"{str(e)} key missing from Type Definition dictionary entry: {str(type_def)}"
+                )
+
+        return (
+            dict(sorted(id_dict.items())),
+            dict(sorted(name_dict.items())),
+            self.get_versions(),
+        )

fprime_gds/common/loaders/pkt_json_loader.py

@@ -0,0 +1,121 @@
+"""
+pkt_json_loader.py:
+
+Loads flight dictionary (JSON) and returns Python dictionaries of telemetry packets
+
+@author jawest
+"""
+
+from fprime_gds.common.templates.pkt_template import PktTemplate
+from fprime_gds.common.loaders.json_loader import JsonLoader
+from fprime_gds.common.data_types.exceptions import GdsDictionaryParsingException
+
+
+class PktJsonLoader(JsonLoader):
+    """Class to load python based telemetry packet dictionaries"""
+
+    PACKETS_FIELD = "telemetryPacketSets"
+
+    SET_NAME = "name"
+    MEMBERS = "members"
+
+    def get_id_dict(self, path, packet_set_name: str, ch_name_dict: dict):
+        if path in self.saved_dicts and packet_set_name in self.saved_dicts[path]:
+            (id_dict, name_dict) = self.saved_dicts[path][packet_set_name]
+        else:
+            (id_dict, name_dict, self.versions) = self.construct_dicts(packet_set_name, ch_name_dict)
+            if path not in self.saved_dicts:
+                self.saved_dicts[path] = dict()
+            self.saved_dicts[path].update({packet_set_name: (id_dict, name_dict)})
+
+        return id_dict
+    
+    def get_name_dict(self, path, packet_set_name: str, ch_name_dict: dict):
+        if path in self.saved_dicts and packet_set_name in self.saved_dicts[path]:
+            (id_dict, name_dict) = self.saved_dicts[path][packet_set_name]
+        else:
+            (id_dict, name_dict, self.versions) = self.construct_dicts(packet_set_name, ch_name_dict)
+            if path not in self.saved_dicts:
+                self.saved_dicts[path] = dict()
+            self.saved_dicts[path].update({packet_set_name: (id_dict, name_dict)})
+
+        return name_dict
+
+
+    def construct_dicts(self, packet_set_name: str, ch_name_dict: dict):
+        """
+        Constructs and returns python dictionaries keyed on id and name
+
+        This function should not be called directly, instead, use
+        get_id_dict(path) and get_name_dict(path)
+
+        Args:
+            ch_name_dict (dict()): Channel dictionary with names as keys and
+                                   ChTemplate objects as values.
+
+        Returns:
+            A tuple with two packet dictionaries (type==dict()):
+            (id_dict, name_dict) and the dictionary version. The keys of the packet dictionaries should 
+            be the packets' id and name fields respectively and the values should be PktTemplate objects.
+        """
+        id_dict = {}
+        name_dict = {}
+
+        if self.PACKETS_FIELD not in self.json_dict:
+            raise GdsDictionaryParsingException(
+                f"Ground Dictionary missing '{self.PACKETS_FIELD}' field: {str(self.json_file)}"
+            )
+
+        for packet_dict in self.json_dict[self.PACKETS_FIELD]:
+            try:
+                if packet_set_name == packet_dict[self.SET_NAME]:
+                    for packet_group_dict in packet_dict.get(self.MEMBERS, []):
+                        packet_temp = self.construct_template_from_dict(packet_group_dict, ch_name_dict)
+                        id_dict[packet_temp.get_id()] = packet_temp
+                        name_dict[packet_temp.get_name()] = packet_temp
+
+                    return (
+                        dict(sorted(id_dict.items())),
+                        dict(sorted(name_dict.items())),
+                        self.get_versions(),
+                    )
+                
+            except KeyError as e:
+                raise GdsDictionaryParsingException(
+                    f"{str(e)} key missing from telemetry packet dictionary entry: {str(packet_dict)}"
+                )
+            
+        raise GdsDictionaryParsingException(
+            f"Ground Dictionary does not contain packet set '{packet_set_name}'"
+        )
+
+    def construct_template_from_dict(self, packet_group_dict: dict, ch_name_dict: dict):
+        """        
+        Args:
+            packet_group_dict (dict()): Packet group dictionary with group id, name, and members
+            ch_name_dict (dict()): Channel dictionary with names as keys and ChTemplate objects as values.
+        Returns:
+            A a PktTemplate object containing the packet group id, group name, and list of ChTemplate 
+            objects that represent each member in the packet.
+        """
+        try:
+            ch_list = []
+            group_name = packet_group_dict["name"]
+            group_id = packet_group_dict["id"]
+            group_members = packet_group_dict["members"]
+
+            for ch_name in group_members:
+                ch_template = ch_name_dict[ch_name]
+                ch_list.append(ch_template)
+            
+        except KeyError as e:
+            raise GdsDictionaryParsingException(
+                f"{str(e)} key missing from telemetry packet member or member is not a channel in the dictionary: {str(group_name)}"
+            )
+        
+        return PktTemplate(
+            group_id,
+            group_name,
+            ch_list
+        )
+

fprime_gds/common/loaders/prm_json_loader.py

@@ -0,0 +1,85 @@
+"""
+prm_json_loader.py:
+
+Loads flight dictionary (JSON) and returns id and mnemonic based Python dictionaries of params
+
+@author zimri.leisher
+"""
+
+from fprime_gds.common.templates.prm_template import PrmTemplate
+from fprime_gds.common.loaders.json_loader import JsonLoader
+from fprime_gds.common.data_types.exceptions import GdsDictionaryParsingException
+
+
+class PrmJsonLoader(JsonLoader):
+    """Class to load parameters from json dictionaries"""
+
+    PARAMS_FIELD = "parameters"
+
+    ID = "id"
+    NAME = "name"
+    TYPE = "type"
+    DESC = "annotation"
+    DEFAULT = "default"
+    
+
+    def construct_dicts(self, _):
+        """
+        Constructs and returns python dictionaries keyed on id and name
+
+        Args:
+            _: Unused argument (inherited)
+        Returns:
+            A tuple with two channel dictionaries (python type dict):
+            (id_dict, fqn_name_dict). The keys should be the channels' id and
+            fully qualified name fields respectively and the values should be PrmTemplate
+            objects.
+        """
+        id_dict = {}
+        fqn_name_dict = {}
+
+        if self.PARAMS_FIELD not in self.json_dict:
+            raise GdsDictionaryParsingException(
+                f"Ground Dictionary missing '{self.PARAMS_FIELD}' field: {str(self.json_file)}"
+            )
+
+        for prm_dict in self.json_dict[self.PARAMS_FIELD]:
+            # Create a channel template object
+            prm_temp = self.construct_template_from_dict(prm_dict)
+
+            id_dict[prm_temp.get_id()] = prm_temp
+            fqn_name_dict[prm_temp.get_full_name()] = prm_temp
+
+        return (
+            dict(sorted(id_dict.items())),
+            dict(sorted(fqn_name_dict.items())),
+            self.get_versions(),
+        )
+
+    def construct_template_from_dict(self, prm_dict: dict) -> PrmTemplate:
+        try:
+            prm_id = prm_dict[self.ID]
+            # The below assignment also raises a ValueError if the name does not contain a '.'
+            qualified_component_name, prm_name = prm_dict[self.NAME].rsplit('.', 1)
+            if not qualified_component_name or not prm_name:
+                raise ValueError()
+
+            type_obj = self.parse_type(prm_dict[self.TYPE])
+        except ValueError as e:
+            raise GdsDictionaryParsingException(
+                f"Parameter dictionary entry malformed, expected name of the form '<QUAL_COMP_NAME>.<PRM_NAME>' in : {str(prm_dict)}"
+            )
+        except KeyError as e:
+            raise GdsDictionaryParsingException(
+                f"{str(e)} key missing from parameter dictionary entry or its associated type in the dictionary: {str(prm_dict)}"
+            )
+
+        prm_default_val = prm_dict.get(self.DEFAULT, None)
+
+        return PrmTemplate(
+            prm_id,
+            prm_name,
+            qualified_component_name,
+            type_obj,
+            prm_default_val
+        )

fprime_gds/common/pipeline/dictionaries.py

@@ -14,6 +14,8 @@ from pathlib import Path
 import fprime_gds.common.loaders.ch_xml_loader
 import fprime_gds.common.loaders.cmd_xml_loader
 import fprime_gds.common.loaders.event_xml_loader
+import fprime_gds.common.loaders.fw_type_json_loader
+import fprime_gds.common.loaders.pkt_json_loader
 import fprime_gds.common.loaders.pkt_xml_loader
 
 # JSON Loaders
@@ -21,7 +23,6 @@ import fprime_gds.common.loaders.ch_json_loader
 import fprime_gds.common.loaders.cmd_json_loader
 import fprime_gds.common.loaders.event_json_loader
 
-
 class Dictionaries:
     """
     Dictionaries class to encapsulate the many different dictionaries used in the system. This includes the following
@@ -45,10 +46,11 @@ class Dictionaries:
         self._event_name_dict = None
         self._channel_name_dict = None
         self._packet_dict = None
+        self._fw_type_name_dict = None
         self._versions = None
         self._metadata = None
 
-    def load_dictionaries(self, dictionary, packet_spec):
+    def load_dictionaries(self, dictionary, packet_spec, packet_set_name):
         """
         Loads the dictionaries based on the dictionary path supplied. Optional packet_spec is allowed to specify the
         definitions of packets.
@@ -75,6 +77,11 @@ class Dictionaries:
             )
             self._channel_name_dict = json_channel_loader.get_name_dict(None)
             self._channel_id_dict = json_channel_loader.get_id_dict(None)
+            # Fw Types
+            fw_types_loader = fprime_gds.common.loaders.fw_type_json_loader.FwTypeJsonLoader(
+                dictionary
+            )
+            self._fw_type_name_dict = fw_types_loader.get_name_dict(None)
             # Metadata
             self._versions = json_event_loader.get_versions()
             self._metadata = json_event_loader.get_metadata().copy()
@@ -121,8 +128,13 @@ class Dictionaries:
             msg = f"[ERROR] Dictionary '{dictionary}' does not exist."
             raise Exception(msg)
         # Check for packet specification
-        if packet_spec is not None:
-            packet_loader = fprime_gds.common.loaders.pkt_xml_loader.PktXmlLoader()
+        if self._metadata["dictionary_type"] == "json" and packet_set_name is not None:
+            packet_loader = fprime_gds.common.loaders.pkt_json_loader.PktJsonLoader(dictionary)
+            self._packet_dict = packet_loader.get_id_dict(
+                None, packet_set_name, self._channel_name_dict
+            )
+        elif packet_spec is not None:
+            packet_loader = fprime_gds.common.loaders.pkt_xml_loader.PktXmlLoader(dictionary)
             self._packet_dict = packet_loader.get_id_dict(
                 packet_spec, self._channel_name_dict
             )
@@ -158,6 +170,11 @@ class Dictionaries:
     def channel_name(self):
         """Channel dictionary by name"""
         return self._channel_name_dict
+    
+    @property
+    def fw_type_name(self):
+        """Fw type name dictionary by name"""
+        return self._fw_type_name_dict
 
     @property
     def project_version(self):

fprime_gds/common/pipeline/encoding.py

@@ -6,6 +6,7 @@ and decoding into a single component that the be composed into the standard pipe
 
 @mstarch
 """
+
 import fprime_gds.common.decoders.ch_decoder
 import fprime_gds.common.decoders.event_decoder
 import fprime_gds.common.decoders.file_decoder
@@ -130,6 +131,24 @@ class EncodingDecoding:
         """
         return self.channel_decoder.deregister(consumer)
 
+    def register_file_consumer(self, consumer):
+        """
+        Registers a consumer with the file decoder.
+
+        :param consumer: consumer of file packets
+        """
+        self.file_decoder.register(consumer)
+
+    def remove_file_consumer(self, consumer):
+        """
+        Removes a consumer from the file decoder. Will raise an error if the history was not
+        previously registered.
+
+        :param consumer: consumer of channels
+        :return: a boolean indicating if the consumer was removed.
+        """
+        return self.file_decoder.deregister(consumer)
+
     def register_command_consumer(self, consumer):
         """
         Registers a history with the standard pipeline.

fprime_gds/common/pipeline/histories.py

@@ -6,6 +6,7 @@ to compose in this code.
 
 @author mstarch
 """
+
 from typing import Type
 
 from fprime_gds.common.history.history import History
@@ -37,6 +38,9 @@ class Histories:
         :param coders: coders object to register histories with
         """
         self.coders = coders
+        # Allow implementation type to disable histories
+        if self._implementation_type is None:
+            return
         # Create histories, RAM histories for now
         self.commands = self._implementation_type()
         self.events = self._implementation_type()

fprime_gds/common/pipeline/standard.py

@@ -8,6 +8,7 @@ below.
 
 :author: lestarch
 """
+
 import datetime
 import os.path
 from pathlib import Path
@@ -19,6 +20,7 @@ import fprime_gds.common.data_types.cmd_data
 import fprime_gds.common.distributor.distributor
 import fprime_gds.common.logger.data_logger
 from fprime_gds.common.transport import RoutingTag, ThreadedTCPSocketClient
+from fprime_gds.common.utils.config_manager import ConfigManager
 
 # Local imports for the sake of composition
 from . import dictionaries, encoding, files, histories
@@ -54,7 +56,13 @@ class StandardPipeline:
         self.__transport_type = ThreadedTCPSocketClient
 
     def setup(
-        self, config, dictionary, file_store, logging_prefix=None, packet_spec=None
+        self,
+        config: ConfigManager,
+        dictionary,
+        file_store,
+        logging_prefix=None,
+        packet_spec=None,
+        packet_set_name=None,
     ):
         """
         Setup the standard pipeline for moving data from the middleware layer through the GDS layers using the standard
@@ -84,7 +92,13 @@ class StandardPipeline:
         self.distributor = fprime_gds.common.distributor.distributor.Distributor(config)
         self.client_socket = self.__transport_type()
         # Setup dictionaries encoders and decoders
-        self.dictionaries.load_dictionaries(self.dictionary_path, packet_spec)
+        self.dictionaries.load_dictionaries(
+            self.dictionary_path, packet_spec, packet_set_name
+        )        
+        # Update config to use Fw types defined in the JSON dictionary
+        if self.dictionaries.fw_type_name:
+            for fw_type_name, fw_type in self.dictionaries.fw_type_name.items():
+                config.set("types", fw_type_name, fw_type)
         self.coders.setup_coders(
             self.dictionaries, self.distributor, self.client_socket, config
         )

fprime_gds/common/templates/prm_template.py

@@ -0,0 +1,81 @@
+"""
+@brief Params Template class
+
+Instances of this class describe a parameter of a component instance (not 
+including a specific value)
+
+@date Created January 27, 2025
+@author Zimri Leisher
+
+@bug Hopefully none
+"""
+
+from fprime.common.models.serialize.type_base import BaseType
+from fprime.common.models.serialize.type_exceptions import TypeMismatchException
+
+from . import data_template
+
+
+class PrmTemplate(data_template.DataTemplate):
+    """Class for param templates that describe parameters of component instances"""
+
+    def __init__(
+        self,
+        prm_id: int,
+        prm_name: str,
+        comp_name: str,
+        prm_type_obj: BaseType,
+        prm_default_val,
+    ):
+        """
+        Constructor
+
+        Args:
+            prm_id: the id of the parameter
+            prm_name: the name of the parameter
+            comp_name: the name of the component instance owning this parameter
+            prm_type_obj: the instance of BaseType corresponding to the type of this parameter
+            prm_default_val: the default value of this parameter, in raw JSON form
+        """
+        super().__init__()
+        # Make sure correct types are passed
+        if not isinstance(prm_id, int):
+            raise TypeMismatchException(int, type(prm_id))
+
+        if not isinstance(prm_name, str):
+            raise TypeMismatchException(str, type(prm_name))
+
+        if not isinstance(comp_name, str):
+            raise TypeMismatchException(str, type(comp_name))
+
+        if not issubclass(prm_type_obj, BaseType):
+            raise TypeMismatchException(BaseType, prm_type_obj)
+
+        # prm_default_val is an arbitrary type, likely a primitive or dict
+
+        self.prm_id = prm_id
+        self.prm_name = prm_name
+        self.comp_name = comp_name
+        self.prm_type_obj = prm_type_obj
+        self.prm_default_val = prm_default_val
+
+    def get_full_name(self):
+        """
+        Get the full name of this param
+
+        Returns:
+            The full name (component.param) for this param
+        """
+        return f"{self.comp_name}.{self.prm_name}"
+
+    def get_id(self):
+        return self.prm_id
+
+    def get_name(self):
+        return self.prm_name
+
+    def get_comp_name(self):
+        return self.comp_name
+
+    def get_type_obj(self):
+        return self.prm_type_obj

fprime_gds/common/testing_fw/api.py

@@ -9,6 +9,8 @@ telemetry and dictionaries.
 """
 import signal
 import time
+from pathlib import Path
+import shutil
 
 from fprime.common.models.serialize.time_type import TimeType
 
@@ -63,6 +65,15 @@ class IntegrationTestAPI(DataHandler):
         # Initialize the logger
         self.logger = TestLogger(logpath) if logpath is not None else None
 
+        # Copy dictionaries and binary file to output directory
+        if logpath is not None:
+            base_dir = Path(self.pipeline.dictionary_path).parents[1]
+            for subdir in ['bin', 'dict']:
+                dir_path = base_dir / subdir
+                if dir_path.is_dir():
+                    shutil.copytree(dir_path, Path(logpath) / subdir,
+                                    dirs_exist_ok=True)
+
         # A predicate used as a filter to choose which events to log automatically
         self.event_log_filter = self.get_event_pred()
 
@@ -215,6 +226,37 @@ class IntegrationTestAPI(DataHandler):
         """
         self.event_log_filter = self.get_event_pred(event, args, severity, time_pred)
 
+    def get_deployment(self):
+        """
+        Get the deployment of the target using the loaded FSW dictionary path
+        Returns:
+            The name of the deployment (str)
+        """
+        return Path(self.pipeline.dictionary_path).parent.parent.name
+
+    def wait_for_dataflow(self, count=1, channels=None, start=None, timeout=120):
+        """
+        Wait for data flow by checking for any telemetry updates within a specified timeout.
+
+        Args:
+            count: either an exact amount (int) or a predicate to specify how many objects to find
+            channels: a channel specifier or list of channel specifiers (mnemonic, ID, or predicate). All will count if None
+            start: an optional index or predicate to specify the earliest item to search
+            timeout: the number of seconds to wait before terminating the search (int)
+        """
+        if start is None:
+            start = self.get_latest_time()
+
+        history = self.get_telemetry_subhistory()
+        result = self.await_telemetry_count(
+            count, channels=channels, history=history, start=start, timeout=timeout
+        )
+        if not result:
+            msg = f'Failed to detect any data flow for {timeout} s.'
+            self.__log(msg, TestLogger.RED)
+            assert False, msg
+        self.remove_telemetry_subhistory(history)
+
     ######################################################################################
     #   History Functions
     ######################################################################################

fprime_gds/common/testing_fw/pytest_integration.py

@@ -15,7 +15,7 @@ Here a test (defined by starting the name with test_) uses the fprime_test_api f
 @author lestarch
 """
 import sys
-
+from pathlib import Path
 import pytest
 
 from fprime_gds.common.testing_fw.api import IntegrationTestAPI
@@ -38,7 +38,30 @@ def pytest_addoption(parser):
         # Reduce flags to only the long option (i.e. --something) form
         flags = [flag for flag in flags if flag.startswith("--")]
         parser.addoption(*flags, **specifiers)
-
+        
+    # Add an option to specify JUnit XML report file
+    parser.addoption(
+        "--junit-xml-file",
+        action="store",
+        default="report.xml",
+        help="File to store JUnit XML report. [default: %(default)s]",
+    )
+    # Add an option to enable JUnit XML report generation to a specified location
+    parser.addoption(
+        "--gen-junitxml",
+        action="store_true",
+        help="Enable JUnitXML report generation to a specified location"
+    )
+
+def pytest_configure(config):
+    """ This is a hook to allow plugins and conftest files to perform initial configuration
+    
+    This hook is called for every initial conftest file after command line options have been parsed. After that, the 
+    hook is called for other conftest files as they are registered.
+    """
+    # Create a JUnit XML report file to capture the test result in a specified location
+    if config.getoption("--gen-junitxml"):
+        config.option.xmlpath = Path(config.getoption("--logs")) / config.getoption("--junit-xml-file")
 
 @pytest.fixture(scope='session')
 def fprime_test_api_session(request):