fprime-gds 4.0.0a10__tar.gz → 4.0.2a1__tar.gz

{fprime_gds-4.0.0a10 → fprime_gds-4.0.2a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fprime-gds
-Version: 4.0.0a10
+Version: 4.0.2a1
 Summary: F Prime Flight Software Ground Data System layer
 Author-email: Michael Starch <Michael.D.Starch@jpl.nasa.gov>, Thomas Boyer-Chammard <Thomas.Boyer.Chammard@jpl.nasa.gov>
 License: 

{fprime_gds-4.0.0a10 → fprime_gds-4.0.2a1}/pyproject.toml

@@ -67,6 +67,7 @@ fprime-dp-write = "fprime_gds.executables.data_product_writer:main"
 fprime-gds = "fprime_gds.executables.run_deployment:main"
 fprime-prm-write = "fprime_gds.common.tools.params:main"
 fprime-fpyc = "fprime_gds.common.fpy.main:main"
+fprime-merge-dictionary = "fprime_gds.executables.dictionary_merge:main"
 
 # For Pytest fixtures
 [project.entry-points."pytest11"]

{fprime_gds-4.0.0a10 → fprime_gds-4.0.2a1}/src/fprime_gds/common/distributor/distributor.py

@@ -13,6 +13,7 @@ descriptor header will be passed on to the registered objects.
 
 @bug No known bugs
 """
+
 import logging
 
 from fprime_gds.common.decoders.decoder import DecodingException
@@ -193,18 +194,23 @@ class Distributor(DataHandler):
         self.__buf.extend(data)
 
         (leftover_data, raw_msgs) = self.parse_into_raw_msgs_api(self.__buf)
-        assert leftover_data == self.__buf, "Leftover data is not equivalent to the remaining data in buffer"
+        assert (
+            leftover_data == self.__buf
+        ), "Leftover data is not equivalent to the remaining data in buffer"
 
         for raw_msg in raw_msgs:
             (length, data_desc, msg) = self.parse_raw_msg_api(raw_msg)
 
             data_desc_key = data_desc_type.DataDescType(data_desc).name
-            for d in self.__decoders[data_desc_key]:
+            decoders = self.__decoders[data_desc_key]
+
+            if not decoders:
+                LOGGER.warning(f"No decoder registered for: {data_desc_key}")
+
+            for d in decoders:
                 try:
                     d.data_callback(msg)
                 except DecodingException as dexc:
-                    LOGGER.warning("Decoding error: %s", dexc)
+                    LOGGER.warning(f"Decoding error: {dexc}")
                 except Exception as exc:
-                    LOGGER.warning("Parsing error: %s", exc)
-            else:
-                LOGGER.warning("No decoder registered for: %s", data_desc_type.DataDescType(data_desc).name)
+                    LOGGER.warning(f"Parsing error: {exc}")

{fprime_gds-4.0.0a10 → fprime_gds-4.0.2a1}/src/fprime_gds/common/pipeline/standard.py

@@ -63,6 +63,7 @@ class StandardPipeline:
         logging_prefix=None,
         packet_spec=None,
         packet_set_name=None,
+        data_logging_enabled=True
     ):
         """
         Setup the standard pipeline for moving data from the middleware layer through the GDS layers using the standard
@@ -113,7 +114,7 @@ class StandardPipeline:
         # Register distributor to client socket
         self.client_socket.register(self.distributor)
         # Final setup step is to make a logging directory, and register in the logger
-        if logging_prefix:
+        if logging_prefix and data_logging_enabled:
             self.setup_logging(logging_prefix)
 
     @property

{fprime_gds-4.0.0a10 → fprime_gds-4.0.2a1}/src/fprime_gds/common/testing_fw/api.py

@@ -7,6 +7,7 @@ telemetry and dictionaries.
 
 :author: koran
 """
+
 import signal
 import time
 from pathlib import Path
@@ -71,11 +72,12 @@ class IntegrationTestAPI(DataHandler):
         # Copy dictionaries and binary file to output directory
         if logpath is not None:
             base_dir = Path(self.pipeline.dictionary_path).parents[1]
-            for subdir in ['bin', 'dict']:
+            for subdir in ["bin", "dict"]:
                 dir_path = base_dir / subdir
                 if dir_path.is_dir():
-                    shutil.copytree(dir_path, Path(logpath) / subdir,
-                                    dirs_exist_ok=True)
+                    shutil.copytree(
+                        dir_path, Path(logpath) / subdir, dirs_exist_ok=True
+                    )
 
         # A predicate used as a filter to choose which events to log automatically
         self.event_log_filter = self.get_event_pred()
@@ -84,7 +86,7 @@ class IntegrationTestAPI(DataHandler):
         self.last_evr = None
 
     def setup(self):
-        """ Set up the API, assumes pipeline is now setup """
+        """Set up the API, assumes pipeline is now setup"""
         self.pipeline.coders.register_event_consumer(self)
 
     def teardown(self):
@@ -162,10 +164,10 @@ class IntegrationTestAPI(DataHandler):
             fail_color = TestLogger.ORANGE
 
         if value:
-            ast_msg = f'{ast_msg} succeeded: {msg}'
+            ast_msg = f"{ast_msg} succeeded: {msg}"
             self.__log(ast_msg, TestLogger.GREEN)
         else:
-            ast_msg = f'{ast_msg} failed: {msg}'
+            ast_msg = f"{ast_msg} failed: {msg}"
             self.__log(ast_msg, fail_color)
 
         if not expect:
@@ -209,7 +211,7 @@ class IntegrationTestAPI(DataHandler):
             self.event_history.clear()
             self.telemetry_history.clear()
             msg = "Clearing Test Histories"
-            
+
         self.__log(msg, TestLogger.WHITE)
         self.command_history.clear()
 
@@ -239,9 +241,9 @@ class IntegrationTestAPI(DataHandler):
         dictionary = str(self.pipeline.dictionary_path)
 
         try:
-            with open(dictionary, 'r') as file:
+            with open(dictionary, "r") as file:
                 data = json.load(file)
-            return data['metadata'].get("deploymentName")
+            return data["metadata"].get("deploymentName")
         except FileNotFoundError:
             msg = f"Error: File not found at path: {dictionary}"
             self.__log(msg, TestLogger.YELLOW)
@@ -273,7 +275,7 @@ class IntegrationTestAPI(DataHandler):
             count, channels=channels, history=history, start=start, timeout=timeout
         )
         if not result:
-            msg = f'Failed to detect any data flow for {timeout} s.'
+            msg = f"Failed to detect any data flow for {timeout} s."
             self.__log(msg, TestLogger.RED)
             assert False, msg
         self.remove_telemetry_subhistory(history)
@@ -300,7 +302,7 @@ class IntegrationTestAPI(DataHandler):
         config_file = self.get_config_file_path()
 
         try:
-            with open(config_file, 'r') as file:
+            with open(config_file, "r") as file:
                 result = json.load(file)
             return result
         except FileNotFoundError:
@@ -351,7 +353,7 @@ class IntegrationTestAPI(DataHandler):
         if data:
             try:
                 filepath = data["Svc.PrmDb.filename"]
-                if filepath.startswith('/'):
+                if filepath.startswith("/"):
                     return filepath
                 else:
                     msg = f"Error: {filepath} did not start with a forward slash"
@@ -549,7 +551,15 @@ class IntegrationTestAPI(DataHandler):
             return self.await_event_sequence(events, start=start, timeout=timeout)
         return self.await_event(events, start=start, timeout=timeout)
 
-    def send_and_assert_command(self, command, args=[], max_delay=None, timeout=5, events=None, commander="cmdDisp"):
+    def send_and_assert_command(
+        self,
+        command,
+        args=[],
+        max_delay=None,
+        timeout=5,
+        events=None,
+        commander="cmdDisp",
+    ):
         """
         This helper will send a command and verify that the command was dispatched and completed
         within the F' deployment. This helper can retroactively check that the delay between
@@ -566,7 +576,9 @@ class IntegrationTestAPI(DataHandler):
             returns a list of the EventData objects found by the search
         """
         cmd_id = self.translate_command_name(command)
-        dispatch = [self.get_event_pred(f"{commander}.OpCodeDispatched", [cmd_id, None])]
+        dispatch = [
+            self.get_event_pred(f"{commander}.OpCodeDispatched", [cmd_id, None])
+        ]
         complete = [self.get_event_pred(f"{commander}.OpCodeCompleted", [cmd_id])]
         events = dispatch + (events if events else []) + complete
         results = self.send_and_assert_event(command, args, events, timeout=timeout)
@@ -576,7 +588,6 @@ class IntegrationTestAPI(DataHandler):
             assert delay < max_delay, msg
         return results
 
-
     ######################################################################################
     #   Command Asserts
     ######################################################################################
@@ -652,14 +663,18 @@ class IntegrationTestAPI(DataHandler):
         """
         if isinstance(channel, str):
             ch_dict = self.pipeline.dictionaries.channel_name
-            matching = [ch_dict[name].get_id() for name in ch_dict.keys() if name.endswith(f".{channel}")] 
+            matching = [
+                ch_dict[name].get_id()
+                for name in ch_dict.keys()
+                if name.endswith(f".{channel}")
+            ]
             if channel in ch_dict:
                 return ch_dict[channel].get_id()
             if force_component or not matching:
                 msg = f"The telemetry mnemonic, {channel}, wasn't in the dictionary"
                 raise KeyError(msg)
             return matching
-        
+
         ch_dict = self.pipeline.dictionaries.channel_id
         if channel in ch_dict:
             return channel
@@ -689,7 +704,11 @@ class IntegrationTestAPI(DataHandler):
 
         if not predicates.is_predicate(channel) and channel is not None:
             channel = self.translate_telemetry_name(channel, force_component=False)
-            channel = predicates.is_a_member_of(channel) if isinstance(channel, list) else predicates.equal_to(channel)
+            channel = (
+                predicates.is_a_member_of(channel)
+                if isinstance(channel, list)
+                else predicates.equal_to(channel)
+            )
 
         if not predicates.is_predicate(value) and value is not None:
             value = predicates.equal_to(value)
@@ -873,7 +892,11 @@ class IntegrationTestAPI(DataHandler):
         """
         if isinstance(event, str):
             event_dict = self.pipeline.dictionaries.event_name
-            matching = [event_dict[name].get_id() for name in event_dict.keys() if name.endswith(f".{event}")] 
+            matching = [
+                event_dict[name].get_id()
+                for name in event_dict.keys()
+                if name.endswith(f".{event}")
+            ]
             if event in event_dict:
                 return event_dict[event].get_id()
             if force_component or not matching:
@@ -910,7 +933,11 @@ class IntegrationTestAPI(DataHandler):
 
         if not predicates.is_predicate(event) and event is not None:
             event = self.translate_event_name(event, force_component=False)
-            event = predicates.is_a_member_of(event) if isinstance(event, list) else predicates.equal_to(event)
+            event = (
+                predicates.is_a_member_of(event)
+                if isinstance(event, list)
+                else predicates.equal_to(event)
+            )
 
         if not predicates.is_predicate(args) and args is not None:
             args = predicates.args_predicate(args)
@@ -1097,6 +1124,38 @@ class IntegrationTestAPI(DataHandler):
         self.__assert_pred("Event count", count_pred, len(results), msg)
         return results
 
+    ######################################################################################
+    #   File Uplink functions
+    ######################################################################################
+
+    def uplink_file_and_await_completion(self, file_path, destination=None, timeout=10):
+        """
+        This function will upload a file and wait for its completion, awaiting for the
+        FileReceived event.
+
+        Args:
+            file_path: the path to the file to upload
+            destination: the destination path for the uploaded file
+            timeout: the maximum time to wait for the event
+        """
+        self.uplink_file(file_path, destination)
+        self.await_event("FileReceived", timeout=timeout)
+
+    def uplink_file(self, file_path, destination=None):
+        """
+        This function will upload a file to the specified location.
+
+        Note: this will simply put the file on the outgoing queue. No guarantee
+        is made on when the file will be delivered. To wait for the completion of
+        the file uplink, use uplink_file_and_await_completion()
+
+        Args:
+            file_path: the path to the file to upload
+        """
+        uplink_file = Path(self.pipeline.up_store) / Path(file_path).name
+        shutil.copy2(file_path, uplink_file)
+        self.pipeline.files.uplinker.enqueue(str(uplink_file), destination)
+
     ######################################################################################
     #   History Searches
     ######################################################################################
@@ -1207,11 +1266,13 @@ class IntegrationTestAPI(DataHandler):
                             return searcher.get_return_value()
                     time.sleep(0.1)
             except self.TimeoutException:
-                self.__log(f'{name} timed out and ended unsuccessfully.', TestLogger.YELLOW)
+                self.__log(
+                    f"{name} timed out and ended unsuccessfully.", TestLogger.YELLOW
+                )
             finally:
                 signal.alarm(0)
         else:
-            self.__log(f'{name} ended unsuccessfully.', TestLogger.YELLOW)
+            self.__log(f"{name} ended unsuccessfully.", TestLogger.YELLOW)
         return searcher.get_return_value()
 
     def find_history_item(self, search_pred, history, start=None, timeout=0):
@@ -1365,7 +1426,9 @@ class IntegrationTestAPI(DataHandler):
                     self.log(f"Count search counted another item: {item}")
                     self.ret_val.append(item)
                     if self.count_pred(len(self.ret_val)):
-                        msg = f"Count search found a correct amount: {len(self.ret_val)}"
+                        msg = (
+                            f"Count search found a correct amount: {len(self.ret_val)}"
+                        )
                         self.log(msg, TestLogger.YELLOW)
                         return True
                 return False

{fprime_gds-4.0.0a10 → fprime_gds-4.0.2a1}/src/fprime_gds/executables/cli.py

@@ -851,6 +851,11 @@ class LogDeployParser(ParserBase):
                 "default": "INFO",
                 "help": "Set the logging level of GDS processes [default: %(default)s]",
             },
+            ("--disable-data-logging",): {
+                "action": "store_true",
+                "default": False,
+                "help": "Disable logging of each data item",
+            },
         }
 
     def handle_arguments(self, args, **kwargs):
@@ -1071,6 +1076,7 @@ class StandardPipelineParser(CompositeParser):
             "packet_spec": args_ns.packet_spec,
             "packet_set_name": args_ns.packet_set_name,
             "logging_prefix": args_ns.logs,
+            "data_logging_enabled": not args_ns.disable_data_logging
         }
         pipeline = pipeline if pipeline else StandardPipeline()
         pipeline.transport_implementation = args_ns.connection_transport

fprime_gds-4.0.2a1/src/fprime_gds/executables/dictionary_merge.py

@@ -0,0 +1,206 @@
+""" fprime_gds.executables.dictionary_merge: script to merge two F Prime dictionaries """
+
+import argparse
+import functools
+import json
+import re
+import sys
+from pathlib import Path
+
+
+def validate_metadata(metadata1, metadata2):
+    """ Check consistency between metadata blocks
+    
+    The JSON dictionary has multiple fields in the metadata block. This function will check that there is consistency
+    between these two blocks.
+
+    Args:
+        metadata1: metadata from the first dictionary
+        metadata2: metadata from the second dictionary
+    """
+    for field in ["projectVersion", "frameworkVersion", "dictionarySpecVersion"]:
+        value1 = metadata1[field]
+        value2 = metadata2[field]
+        if value1 != value2:
+            raise ValueError(f"Inconsistent metadata values for field '{field}'. ({value1} vs {value2})")
+
+def validate_non_unique(non_unique1, non_unique2):
+    """ Validate non-unique definitions are consistent between dictionaries """
+    indexed_non_unique1 = {value.get("qualifiedName"): value for value in non_unique1}
+
+    for value2 in non_unique2:
+        value1 = indexed_non_unique1.get(value2["qualifiedName"], None)
+        if value1 is not None and value1 != value2:
+            raise ValueError(f"'{value2['qualifiedName']}' has inconsistent definitions")
+
+def validate_unique(unique1, unique2):
+    """ Validate unique definitions have no duplication """
+    ids = {item.get("id", item.get("opcode", "")) for item in unique1}
+    names = {item.get("name") for item in unique1}
+
+
+    for value2 in unique2:
+        name = value2['name']
+        id = value2.get("id", value2.get("opcode", ""))
+        if name in names:
+            raise ValueError(f"'{name}' appears in both dictionaries")
+        if id and id in ids:
+            raise ValueError(f"ID/Opcode {id} used in both dictionaries")
+
+
+def merge_metadata(meta1, meta2, name=None, permissive=False):
+    """ Merge JSON dictionary metadata blocks
+    
+    The JSON dictionary starts with a metadata block. This function will merge the two metadata blocks preferring the
+    first when there is a discrepancy. 'name' will be supplied as the new name defaulting to "name1_name2_merged" when
+    not supplied. If 'permissive' is true, version discrepancies will be ignored otherwise this will throw a ValueError
+    if the versions do not match.
+
+    Args:
+        meta1: first metadata block
+        meta2: second metadata block
+        name: (optional) name for the new dictionary (Default: meta.name_meta2.name_merged)
+        permissive: (optional) True to allow version miss-matching. (Default: False)
+    Return:
+        merged metadata block
+    Throws:
+        ValueError on version miss-match without the permissive flag
+    """
+    if not permissive:
+        validate_metadata(meta1, meta2)
+    if name is None:
+        name = f"{meta1.get('deploymentName', 'unknown')}_{meta2.get('deploymentName', 'unknown')}_merged"
+    return {
+        **meta1,
+        **{
+            "deploymentName": name
+          }
+    }
+
+def merge_lists(list1, list2, validator):
+    """ Merge list-like entities
+    
+    This will merge two list-like entities using the supplied validator.
+
+    Args:
+        list1: first list-like
+        list2: second list-like
+        validator: validate the lists are consistent or non-colliding
+
+    """
+    validator(list1, list2)
+    singular = {item.get("qualifiedName", item.get("name", "")): item for item in list1 + list2}
+    return list(singular.values())
+
+def merge_non_unique(non_unique1, non_unique2):
+    """ Merge the non-unique blocks in JSON dictionaries
+
+    JSON dictionaries have some non-unique definitions (e.g. "typeDefinitions") that must be merged ensuring
+    consistency but ignoring duplication. This function will create a superset of the two blocks. Inconsistent
+    definitions will result in a ValueError.
+
+    Args:
+        non_unique1: first non unique block
+        non_unique2: second non unique block
+    """
+    return merge_lists(non_unique1, non_unique2, validate_non_unique)
+
+
+def merge_unique(unique1, unique2):
+    """ Merge the unique blocks in JSON dictionaries
+
+    JSON dictionaries have some unique definitions (e.g. "eventDefinitions") that must be merged ensuring that entries
+    are not duplicated between the sets. This function will create a superset of the two blocks. Duplicated definitions
+    will result in a ValueError.
+
+    Args:
+        unique1: first unique block
+        unique2: second unique block
+    """
+    return merge_lists(unique1, unique2, validate_unique)
+
+
+def merge_dictionaries(dictionary1, dictionary2, name=None, permissive=False):
+    """ Merge two dictionaries
+
+    This will merge two JSON dictionaries' major top-level sections. Unknown fields will be preserved preferring
+    dictionary1's content for unknown fields.
+
+    Args:
+        dictionary1: dictionary 1's content
+        dictionary2: dictionary 2's content
+        name: new 'deploymentName' field
+        permissive: allow miss-matched dictionary versions
+
+    Return: merged dictionaries
+        
+    """
+    merge_metadata_fn = functools.partial(merge_metadata, name=name, permissive=permissive)
+
+    stages = [
+        ("metadata", merge_metadata_fn),
+        ("typeDefinitions", merge_non_unique),
+        ("constants", merge_non_unique),
+        ("commands", merge_unique),
+        ("parameters", merge_unique),
+        ("events", merge_unique),
+        ("telemetryChannels", merge_unique),
+        ("records", merge_unique),
+        ("containers", merge_unique),
+        ("telemetryPacketSets", merge_unique),
+
+    ]
+
+    merged = {**dictionary2, **dictionary1}
+    for field, merger in stages:
+        object1 = dictionary1[field]
+        object2 = dictionary2[field]
+        try:
+            merged[field] = merger(object1, object2)
+        except ValueError as value_error:
+            raise ValueError(f"Merging '{field}' failed. {value_error}")
+        except KeyError as key_error:
+            raise ValueError(f"Malformed dictionary section '{field}'. Missing key: {key_error}")
+    return merged
+
+def parse_arguments():
+    """ Parse arguments for this script """
+    parser = argparse.ArgumentParser(description="Merge two dictionaries")
+    parser.add_argument("--name", type=str, default=None, help="Name to use as the new 'deploymentName' field")
+    parser.add_argument("--output", type=Path, default=Path("MergedAppDictionary.json"),
+                        help="Output dictionary path. Default: MergedAppDictionary.json")
+    parser.add_argument("--permissive", action="store_true", default=False,
+                        help="Ignore discrepancies between dictionaries")
+    parser.add_argument("dictionary1", type=Path, help="Primary dictionary to merge")
+    parser.add_argument("dictionary2", type=Path, help="Secondary dictionary to merge")
+
+    args = parser.parse_args()
+
+    # Validate arguments
+    if args.name is not None and not re.match("[a-zA-Z_][a-zA-Z_0-9]*"):
+        raise ValueError(f"--name '{args.name}' is an invalid identifier")
+    if not args.dictionary1.exists():
+        raise ValueError(f"'{args.dictionary1}' does not exist")
+    if not args.dictionary2.exists():
+        raise ValueError(f"'{args.dictionary2}' does not exist")
+    return args
+
+def main():
+    """ Main entry point """
+    try:
+        args = parse_arguments()
+        # Open dictionaries
+        with open(args.dictionary1, "r") as dictionary1_fh:
+            dictionary1 = json.load(dictionary1_fh)
+        with open(args.dictionary2, "r") as dictionary2_fh:
+            dictionary2 = json.load(dictionary2_fh)
+        output = merge_dictionaries(dictionary1, dictionary2, args.name, args.permissive)
+        with open(args.output, "w") as output_fh:
+            json.dump(output, output_fh, indent=2)
+    except Exception as exception:
+        print(f"[ERROR] {exception}", file=sys.stderr)
+        sys.exit(1)
+    sys.exit(0)
+
+if __name__ == "__main__":
+    main()

{fprime_gds-4.0.0a10 → fprime_gds-4.0.2a1}/src/fprime_gds.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fprime-gds
-Version: 4.0.0a10
+Version: 4.0.2a1
 Summary: F Prime Flight Software Ground Data System layer
 Author-email: Michael Starch <Michael.D.Starch@jpl.nasa.gov>, Thomas Boyer-Chammard <Thomas.Boyer.Chammard@jpl.nasa.gov>
 License: 

{fprime_gds-4.0.0a10 → fprime_gds-4.0.2a1}/src/fprime_gds.egg-info/SOURCES.txt

@@ -159,6 +159,7 @@ src/fprime_gds/executables/apps.py
 src/fprime_gds/executables/cli.py
 src/fprime_gds/executables/comm.py
 src/fprime_gds/executables/data_product_writer.py
+src/fprime_gds/executables/dictionary_merge.py
 src/fprime_gds/executables/fprime_cli.py
 src/fprime_gds/executables/run_deployment.py
 src/fprime_gds/executables/tcpserver.py

{fprime_gds-4.0.0a10 → fprime_gds-4.0.2a1}/src/fprime_gds.egg-info/entry_points.txt

@@ -3,6 +3,7 @@ fprime-cli = fprime_gds.executables.fprime_cli:main
 fprime-dp-write = fprime_gds.executables.data_product_writer:main
 fprime-fpyc = fprime_gds.common.fpy.main:main
 fprime-gds = fprime_gds.executables.run_deployment:main
+fprime-merge-dictionary = fprime_gds.executables.dictionary_merge:main
 fprime-prm-write = fprime_gds.common.tools.params:main
 fprime-seqgen = fprime_gds.common.tools.seqgen:main