fprime-gds 4.0.1__py3-none-any.whl → 4.0.2a2__py3-none-any.whl

fprime_gds/executables/apps.py

@@ -14,8 +14,8 @@ command line that will be spun into its own process.
 import subprocess
 import sys
 from abc import ABC, abstractmethod
-from argparse import Namespace
-from typing import final, List, Dict, Tuple, Type
+import argparse
+from typing import final, List, Dict, Tuple, Type, Optional
 
 from fprime_gds.plugin.definitions import gds_plugin_specification, gds_plugin
 from fprime_gds.plugin.system import Plugins
@@ -23,10 +23,13 @@ from fprime_gds.executables.cli import (
     CompositeParser,
     ParserBase,
     BareArgumentParser,
+    MiddleWareParser,
+    DictionaryParser,
     StandardPipelineParser,
     PluginArgumentParser,
 )
 from fprime_gds.common.pipeline.standard import StandardPipeline
+from fprime_gds.common.pipeline.publishing import PublishingPipeline
 
 
 class GdsBaseFunction(ABC):
@@ -40,7 +43,7 @@ class GdsBaseFunction(ABC):
     """
 
     @abstractmethod
-    def run(self):
+    def run(self, parsed_args):
         """Run the start-up function
 
         Run the start-up function unconstrained by the limitations of running in a dedicated subprocess.
@@ -110,13 +113,13 @@ class GdsApp(GdsBaseFunction):
         self.process = None
         self.arguments = arguments
 
-    def run(self):
+    def run(self, parsed_args):
         """Run the application as an isolated process
 
         GdsFunction objects require an implementation of the `run` command. This implementation will take the arguments
         provided from `get_process_invocation` function and supplies them as an invocation of the isolated subprocess.
         """
-        invocation_arguments = self.get_process_invocation()
+        invocation_arguments = self.get_process_invocation(parsed_args)
         self.process = subprocess.Popen(invocation_arguments)
 
     def wait(self, timeout=None):
@@ -137,7 +140,9 @@ class GdsApp(GdsBaseFunction):
         return self.process.returncode
 
     @abstractmethod
-    def get_process_invocation(self) -> List[str]:
+    def get_process_invocation(
+        self, namespace: Optional[argparse.Namespace] = None
+    ) -> List[str]:
         """Run the start-up function
 
         Run the start-up function unconstrained by the limitations of running in a dedicated subprocess.
@@ -199,6 +204,18 @@ class GdsStandardApp(GdsApp):
             dictionary of flag tuple to argparse kwargs
         """
         return {}
+    
+    @classmethod
+    def get_additional_cli_parsers(cls) -> List[ParserBase]:
+        """ Supply a list of CLI parser objects
+        
+        Supply a list of CLI parser objects to the CLI system. This allows use of full ParserBase objects instead of
+        the more restrictive dictionary approach seen in get_additional_arguments.
+
+        Returns:
+            list of parser objects as passed to ParserBase
+        """
+        return []
 
     @classmethod
     def init(cls):
@@ -230,7 +247,7 @@ class GdsStandardApp(GdsApp):
         """Start function to contain behavior based in standard pipeline"""
         raise NotImplementedError()
 
-    def get_process_invocation(self):
+    def get_process_invocation(self, namespace=None):
         """Return the process invocation for this class' main
 
         The process invocation of this application is to run cls.main and supply it a reproduced version of the
@@ -247,7 +264,8 @@ class GdsStandardApp(GdsApp):
         composite_parser = CompositeParser(
             [self.get_cli_parser(), StandardPipelineParser]
         )
-        namespace, _, _ = ParserBase.parse_known_args([composite_parser])
+        if namespace is None:
+            namespace, _, _ = ParserBase.parse_known_args([composite_parser], client=True)
         args = composite_parser.reproduce_cli_args(namespace)
         return [sys.executable, "-c", f"import {module}\n{module}.{cls}.main()"] + args
 
@@ -261,32 +279,39 @@ class GdsStandardApp(GdsApp):
                     []
                 )  # Disable plugin system unless specified through init
             # In the case where `init` sets up the plugin system, we want to pass the assertion
-            # triggered by the code above that turns it off in the not-setup case. 
+            # triggered by the code above that turns it off in the not-setup case.
             except AssertionError:
                 pass
             plugin_name = getattr(cls, "get_name", lambda: cls.__name__)()
+            plugin_composite = CompositeParser([cls.get_cli_parser()] + cls.get_additional_cli_parsers())
+
             parsed_arguments, _ = ParserBase.parse_args(
-                [cls.get_cli_parser(), StandardPipelineParser, PluginArgumentParser],
+                [ StandardPipelineParser, PluginArgumentParser, plugin_composite],
                 f"{plugin_name}: a standard app plugin",
+                client=True,
             )
             pipeline = StandardPipeline()
-            # Turn off history and filing
+            # Turn off history, file handling, and logging
             pipeline.histories.implementation = None
             pipeline.filing = None
+            parsed_arguments.disable_data_logging = True 
             pipeline = StandardPipelineParser.pipeline_factory(
                 parsed_arguments, pipeline
             )
             application = cls(
-                **cls.get_cli_parser().extract_arguments(parsed_arguments)
+                **cls.get_cli_parser().extract_arguments(parsed_arguments),
+                namespace=parsed_arguments, 
+
             )
             application.start(pipeline)
             sys.exit(0)
         except Exception as e:
             print(f"[ERROR] Error launching {cls.__name__}: {e}", file=sys.stderr)
-            raise
             sys.exit(148)
 
 
+
+
 @gds_plugin(GdsApp)
 class CustomDataHandlers(GdsStandardApp):
     """Run an app that registers all custom data handlers
@@ -294,10 +319,25 @@ class CustomDataHandlers(GdsStandardApp):
     A GdsApp plugin, built using the GdsStandardApp helper, that uses the provided standard pipeline to register each
     custom DataHandler plugin as a consumer of the appropriate type.
     """
+    PLUGIN_PARSER = CompositeParser([MiddleWareParser, DictionaryParser])
 
-    def __init__(self, **kwargs):
+    def __init__(self, namespace, **kwargs):
         """Required __init__ implementation"""
         super().__init__(**kwargs)
+        self.connection_transport = namespace.connection_transport
+        self.connection_uri = namespace.connection_uri
+        self.dictionaries  = namespace.dictionaries
+
+    @classmethod
+    def get_additional_arguments(cls):
+        """ Supplies additional arguments needed """
+        return {}
+
+    @classmethod
+    def get_additional_cli_parsers(cls):
+        """ Requires MiddleWareParser and Dictionary Parser"""
+        return [cls.PLUGIN_PARSER]
+
 
     @classmethod
     def init(cls):
@@ -312,10 +352,15 @@ class CustomDataHandlers(GdsStandardApp):
             "FW_PACKET_FILE": pipeline.coders.register_file_consumer,
             "FW_PACKET_PACKETIZED_TLM": pipeline.coders.register_packet_consumer,
         }
+        self.publisher = PublishingPipeline()
+        self.publisher.transport_implementation = self.connection_transport
+        self.publisher.setup(self.dictionaries)
+        self.publisher.connect(self.connection_uri)
 
         data_handlers = Plugins.system().get_feature_classes("data_handler")
         for data_handler_class in data_handlers:
             data_handler = data_handler_class()
+            data_handler.set_publisher(self.publisher)
             descriptors = data_handler.get_handled_descriptors()
             for descriptor in descriptors:
                 DESCRIPTOR_TO_FUNCTION.get(descriptor, lambda discard: discard)(

fprime_gds/executables/cli.py

@@ -27,9 +27,9 @@ from pathlib import Path
 from typing import Any, Dict, List, Tuple
 
 # Required to set the checksum as a module variable
-import fprime_gds.common.communication.checksum
 import fprime_gds.common.logger
 from fprime_gds.common.communication.adapters.ip import check_port
+from fprime_gds.common.models.dictionaries import Dictionaries
 from fprime_gds.common.pipeline.standard import StandardPipeline
 from fprime_gds.common.transport import ThreadedTCPSocketClient
 from fprime_gds.common.utils.config_manager import ConfigManager
@@ -176,6 +176,18 @@ class ParserBase(ABC):
         ]
         return list(itertools.chain.from_iterable(cli_pairs))
 
+    def handle_values(self, values: Dict[str, Any]):
+        """Post-process the parser's arguments in dictionary form
+
+        Handle arguments from the given parser in dictionary form. This will convert to/from the namespace and then
+        delegate to handle_arguments.
+
+        Args:
+            args: arguments namespace of processed arguments
+        Returns: dictionary with processed results of arguments.
+        """
+        return vars(self.handle_arguments(args=argparse.Namespace(**values), kwargs={}))
+
     @abstractmethod
     def handle_arguments(self, args, **kwargs):
         """Post-process the parser's arguments
@@ -851,6 +863,11 @@ class LogDeployParser(ParserBase):
                 "default": "INFO",
                 "help": "Set the logging level of GDS processes [default: %(default)s]",
             },
+            ("--disable-data-logging",): {
+                "action": "store_true",
+                "default": False,
+                "help": "Disable logging of each data item",
+            },
         }
 
     def handle_arguments(self, args, **kwargs):
@@ -1002,6 +1019,19 @@ class DictionaryParser(DetectionParser):
         elif args.dictionary is None:
             args = super().handle_arguments(args, **kwargs)
             args.dictionary = find_dict(args.deployment)
+        
+        # Setup dictionaries encoders and decoders
+        dictionaries = Dictionaries()
+
+        dictionaries.load_dictionaries(
+            args.dictionary, args.packet_spec, args.packet_set_name
+        )
+        config = ConfigManager.get_instance()
+        # Update config to use Fw types defined in the JSON dictionary
+        if dictionaries.fw_type_name:
+            for fw_type_name, fw_type in dictionaries.fw_type_name.items():
+                config.set("types", fw_type_name, fw_type)
+        args.dictionaries = dictionaries
         return args
 
 
@@ -1065,12 +1095,11 @@ class StandardPipelineParser(CompositeParser):
     def pipeline_factory(args_ns, pipeline=None) -> StandardPipeline:
         """A factory of the standard pipeline given the handled arguments"""
         pipeline_arguments = {
-            "config": ConfigManager(),
-            "dictionary": args_ns.dictionary,
+            "config": ConfigManager.get_instance(),
+            "dictionaries": args_ns.dictionaries,
             "file_store": args_ns.files_storage_directory,
-            "packet_spec": args_ns.packet_spec,
-            "packet_set_name": args_ns.packet_set_name,
             "logging_prefix": args_ns.logs,
+            "data_logging_enabled": not args_ns.disable_data_logging
         }
         pipeline = pipeline if pipeline else StandardPipeline()
         pipeline.transport_implementation = args_ns.connection_transport

fprime_gds/executables/dictionary_merge.py

@@ -0,0 +1,206 @@
+""" fprime_gds.executables.dictionary_merge: script to merge two F Prime dictionaries """
+
+import argparse
+import functools
+import json
+import re
+import sys
+from pathlib import Path
+
+
+def validate_metadata(metadata1, metadata2):
+    """ Check consistency between metadata blocks
+    
+    The JSON dictionary has multiple fields in the metadata block. This function will check that there is consistency
+    between these two blocks.
+
+    Args:
+        metadata1: metadata from the first dictionary
+        metadata2: metadata from the second dictionary
+    """
+    for field in ["projectVersion", "frameworkVersion", "dictionarySpecVersion"]:
+        value1 = metadata1[field]
+        value2 = metadata2[field]
+        if value1 != value2:
+            raise ValueError(f"Inconsistent metadata values for field '{field}'. ({value1} vs {value2})")
+
+def validate_non_unique(non_unique1, non_unique2):
+    """ Validate non-unique definitions are consistent between dictionaries """
+    indexed_non_unique1 = {value.get("qualifiedName"): value for value in non_unique1}
+
+    for value2 in non_unique2:
+        value1 = indexed_non_unique1.get(value2["qualifiedName"], None)
+        if value1 is not None and value1 != value2:
+            raise ValueError(f"'{value2['qualifiedName']}' has inconsistent definitions")
+
+def validate_unique(unique1, unique2):
+    """ Validate unique definitions have no duplication """
+    ids = {item.get("id", item.get("opcode", "")) for item in unique1}
+    names = {item.get("name") for item in unique1}
+
+
+    for value2 in unique2:
+        name = value2['name']
+        id = value2.get("id", value2.get("opcode", ""))
+        if name in names:
+            raise ValueError(f"'{name}' appears in both dictionaries")
+        if id and id in ids:
+            raise ValueError(f"ID/Opcode {id} used in both dictionaries")
+
+
+def merge_metadata(meta1, meta2, name=None, permissive=False):
+    """ Merge JSON dictionary metadata blocks
+    
+    The JSON dictionary starts with a metadata block. This function will merge the two metadata blocks preferring the
+    first when there is a discrepancy. 'name' will be supplied as the new name defaulting to "name1_name2_merged" when
+    not supplied. If 'permissive' is true, version discrepancies will be ignored otherwise this will throw a ValueError
+    if the versions do not match.
+
+    Args:
+        meta1: first metadata block
+        meta2: second metadata block
+        name: (optional) name for the new dictionary (Default: meta.name_meta2.name_merged)
+        permissive: (optional) True to allow version miss-matching. (Default: False)
+    Return:
+        merged metadata block
+    Throws:
+        ValueError on version miss-match without the permissive flag
+    """
+    if not permissive:
+        validate_metadata(meta1, meta2)
+    if name is None:
+        name = f"{meta1.get('deploymentName', 'unknown')}_{meta2.get('deploymentName', 'unknown')}_merged"
+    return {
+        **meta1,
+        **{
+            "deploymentName": name
+          }
+    }
+
+def merge_lists(list1, list2, validator):
+    """ Merge list-like entities
+    
+    This will merge two list-like entities using the supplied validator.
+
+    Args:
+        list1: first list-like
+        list2: second list-like
+        validator: validate the lists are consistent or non-colliding
+
+    """
+    validator(list1, list2)
+    singular = {item.get("qualifiedName", item.get("name", "")): item for item in list1 + list2}
+    return list(singular.values())
+
+def merge_non_unique(non_unique1, non_unique2):
+    """ Merge the non-unique blocks in JSON dictionaries
+
+    JSON dictionaries have some non-unique definitions (e.g. "typeDefinitions") that must be merged ensuring
+    consistency but ignoring duplication. This function will create a superset of the two blocks. Inconsistent
+    definitions will result in a ValueError.
+
+    Args:
+        non_unique1: first non unique block
+        non_unique2: second non unique block
+    """
+    return merge_lists(non_unique1, non_unique2, validate_non_unique)
+
+
+def merge_unique(unique1, unique2):
+    """ Merge the unique blocks in JSON dictionaries
+
+    JSON dictionaries have some unique definitions (e.g. "eventDefinitions") that must be merged ensuring that entries
+    are not duplicated between the sets. This function will create a superset of the two blocks. Duplicated definitions
+    will result in a ValueError.
+
+    Args:
+        unique1: first unique block
+        unique2: second unique block
+    """
+    return merge_lists(unique1, unique2, validate_unique)
+
+
+def merge_dictionaries(dictionary1, dictionary2, name=None, permissive=False):
+    """ Merge two dictionaries
+
+    This will merge two JSON dictionaries' major top-level sections. Unknown fields will be preserved preferring
+    dictionary1's content for unknown fields.
+
+    Args:
+        dictionary1: dictionary 1's content
+        dictionary2: dictionary 2's content
+        name: new 'deploymentName' field
+        permissive: allow miss-matched dictionary versions
+
+    Return: merged dictionaries
+        
+    """
+    merge_metadata_fn = functools.partial(merge_metadata, name=name, permissive=permissive)
+
+    stages = [
+        ("metadata", merge_metadata_fn),
+        ("typeDefinitions", merge_non_unique),
+        ("constants", merge_non_unique),
+        ("commands", merge_unique),
+        ("parameters", merge_unique),
+        ("events", merge_unique),
+        ("telemetryChannels", merge_unique),
+        ("records", merge_unique),
+        ("containers", merge_unique),
+        ("telemetryPacketSets", merge_unique),
+
+    ]
+
+    merged = {**dictionary2, **dictionary1}
+    for field, merger in stages:
+        object1 = dictionary1[field]
+        object2 = dictionary2[field]
+        try:
+            merged[field] = merger(object1, object2)
+        except ValueError as value_error:
+            raise ValueError(f"Merging '{field}' failed. {value_error}")
+        except KeyError as key_error:
+            raise ValueError(f"Malformed dictionary section '{field}'. Missing key: {key_error}")
+    return merged
+
+def parse_arguments():
+    """ Parse arguments for this script """
+    parser = argparse.ArgumentParser(description="Merge two dictionaries")
+    parser.add_argument("--name", type=str, default=None, help="Name to use as the new 'deploymentName' field")
+    parser.add_argument("--output", type=Path, default=Path("MergedAppDictionary.json"),
+                        help="Output dictionary path. Default: MergedAppDictionary.json")
+    parser.add_argument("--permissive", action="store_true", default=False,
+                        help="Ignore discrepancies between dictionaries")
+    parser.add_argument("dictionary1", type=Path, help="Primary dictionary to merge")
+    parser.add_argument("dictionary2", type=Path, help="Secondary dictionary to merge")
+
+    args = parser.parse_args()
+
+    # Validate arguments
+    if args.name is not None and not re.match("[a-zA-Z_][a-zA-Z_0-9]*"):
+        raise ValueError(f"--name '{args.name}' is an invalid identifier")
+    if not args.dictionary1.exists():
+        raise ValueError(f"'{args.dictionary1}' does not exist")
+    if not args.dictionary2.exists():
+        raise ValueError(f"'{args.dictionary2}' does not exist")
+    return args
+
+def main():
+    """ Main entry point """
+    try:
+        args = parse_arguments()
+        # Open dictionaries
+        with open(args.dictionary1, "r") as dictionary1_fh:
+            dictionary1 = json.load(dictionary1_fh)
+        with open(args.dictionary2, "r") as dictionary2_fh:
+            dictionary2 = json.load(dictionary2_fh)
+        output = merge_dictionaries(dictionary1, dictionary2, args.name, args.permissive)
+        with open(args.output, "w") as output_fh:
+            json.dump(output, output_fh, indent=2)
+    except Exception as exception:
+        print(f"[ERROR] {exception}", file=sys.stderr)
+        sys.exit(1)
+    sys.exit(0)
+
+if __name__ == "__main__":
+    main()

fprime_gds/executables/fprime_cli.py

@@ -227,7 +227,7 @@ class CommandSubparserInjector(CliSubparserInjectorBase):
             argcomplete.warn("No dictionary found to get command names from")
             return []
 
-        from fprime_gds.common.pipeline.dictionaries import Dictionaries
+        from fprime_gds.common.models.dictionaries import Dictionaries
 
         dictionary = Dictionaries()
         dictionary.load_dictionaries(dict_path, None, None)

fprime_gds/executables/run_deployment.py

@@ -5,6 +5,7 @@
 ####
 import os
 import sys
+import copy
 import webbrowser
 
 from fprime_gds.executables.cli import (
@@ -38,7 +39,9 @@ def parse_args():
         PluginArgumentParser,
     ]
     # Parse the arguments, and refine through all handlers
-    args, parser = ConfigDrivenParser.parse_args(arg_handlers, "Run F prime deployment and GDS")
+    args, parser = ConfigDrivenParser.parse_args(
+        arg_handlers, "Run F prime deployment and GDS"
+    )
     return args
 
 
@@ -175,11 +178,21 @@ def launch_comm(parsed_args):
     )
 
 
-def launch_plugin(plugin_class_instance):
+def launch_plugin(parsed_args, plugin_class_instance):
     """Launch a plugin instance"""
-    plugin_name = getattr(plugin_class_instance, "get_name", lambda: plugin_class_instance.__class__.__name__)()
+    plugin_name = getattr(
+        plugin_class_instance,
+        "get_name",
+        lambda: plugin_class_instance.__class__.__name__,
+    )()
+    plugin_args = copy.deepcopy(parsed_args)
+    # Set logging to use a subdirectory within the root logs directory
+    plugin_logs = os.path.join(plugin_args.logs, plugin_name)
+    os.mkdir(plugin_logs)
+    plugin_args.logs = plugin_logs
+    plugin_args.log_directly = True
     return launch_process(
-        plugin_class_instance.get_process_invocation(),
+        plugin_class_instance.get_process_invocation(plugin_args),
         name=f"{ plugin_name } Plugin App",
         launch_time=1,
     )
@@ -218,11 +231,11 @@ def main():
     try:
         procs = [launcher(parsed_args) for launcher in launchers]
         _ = [
-            launch_plugin(cls())
+            launch_plugin(parsed_args, cls(namespace=parsed_args))
             for cls in Plugins.system().get_feature_classes("gds_app")
         ]
         _ = [
-            instance().run()
+            instance().run(parsed_args)
             for instance in Plugins.system().get_feature_classes("gds_function")
         ]
 

{fprime_gds-4.0.1.dist-info → fprime_gds-4.0.2a2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fprime-gds
-Version: 4.0.1
+Version: 4.0.2a2
 Summary: F Prime Flight Software Ground Data System layer
 Author-email: Michael Starch <Michael.D.Starch@jpl.nasa.gov>, Thomas Boyer-Chammard <Thomas.Boyer.Chammard@jpl.nasa.gov>
 License: