omnata-plugin-devkit 0.13.0a158__tar.gz → 0.13.1__tar.gz

{omnata_plugin_devkit-0.13.0a158 → omnata_plugin_devkit-0.13.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omnata-plugin-devkit
-Version: 0.13.0a158
+Version: 0.13.1
 Summary: 
 License-File: LICENSE
 Author: James Weakley

{omnata_plugin_devkit-0.13.0a158 → omnata_plugin_devkit-0.13.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "omnata-plugin-devkit"
-version = "0.13.0a158"
+version = "0.13.1"
 description = ""
 authors = ["James Weakley <james.weakley@omnata.com>"]
 readme = "README.md"

{omnata_plugin_devkit-0.13.0a158 → omnata_plugin_devkit-0.13.1}/src/omnata_plugin_devkit/development_session.py

@@ -28,6 +28,7 @@ from omnata_plugin_runtime.omnata_plugin import (
     InboundSyncRequest,
     OmnataPlugin,
     OutboundSyncRequest,
+    RecordsToUploadResult,
     SyncRequest,
 )
 from omnata_plugin_runtime.logging import OmnataPluginLogHandler
@@ -103,7 +104,7 @@ class SyncScenario:
         )
         os.makedirs(os.path.dirname(self.inbound_results_file), exist_ok=True)
         self.scenario_records = scenario_records
-        self.results:Optional[pandas.DataFrame] = None
+        self.results: Optional[pandas.DataFrame] = None
         self.initial_stream_state = initial_stream_state
 
 
@@ -136,9 +137,9 @@ class DevelopmentSession:
         self.vcr = None
         self.recorded_http = recorded_http
         self.inject_snowpark_session = inject_snowpark_session
-        self.stream_index_fields:Dict[str,str] = {}
-        self.stream_excluded_fields:Dict[str,List[str]] = {}
-        self.exclude_records_by_field_value:Dict[str,str] = {}
+        self.stream_index_fields: Dict[str, str] = {}
+        self.stream_excluded_fields: Dict[str, List[str]] = {}
+        self.exclude_records_by_field_value: Dict[str, str] = {}
         self.plugin_id = plugin_instance.get_manifest().plugin_id
         self.feature_filename = os.path.join("features", f"{self.plugin_id}.feature")
         self.current_scenario = None
@@ -153,8 +154,8 @@ class DevelopmentSession:
         Collects the full set of records which were previously staged
         """
         return self.current_scenario.scenario_records
-    
-    def set_inbound_stream_index_field(self,stream_name:str,index_field:str):
+
+    def set_inbound_stream_index_field(self, stream_name: str, index_field: str):
         """
         Sets the field to be used as the index field for a given stream.
         By default, it is set to the primary key field of the stream, but this may
@@ -163,18 +164,22 @@ class DevelopmentSession:
         """
         self.stream_index_fields[stream_name] = index_field
 
-    def exclude_inbound_result_fields(self, stream_name: str, field_names: Union[str,List[str]]):
+    def exclude_inbound_result_fields(
+        self, stream_name: str, field_names: Union[str, List[str]]
+    ):
         """
         Excludes a field from the comparison of inbound results.
         """
         if stream_name not in self.stream_excluded_fields:
             self.stream_excluded_fields[stream_name] = []
-        if isinstance(field_names,str):
+        if isinstance(field_names, str):
             self.stream_excluded_fields[stream_name].append(field_names)
         else:
             self.stream_excluded_fields[stream_name].extend(field_names)
-    
-    def exclude_inbound_result_records_by_field_value(self, field_name: str, field_value: str):
+
+    def exclude_inbound_result_records_by_field_value(
+        self, field_name: str, field_value: str
+    ):
         """
         Excludes records from the comparison of inbound results where the field matches the value.
         """
@@ -235,13 +240,13 @@ class DevelopmentSession:
         if not click.globals.get_current_context(silent=True):
             # set an empty one so snowcli's app config doesn't error
             click.globals.push_context(click.core.Context(click.core.Group()))
-        self.session: Session = get_snowpark_session(connection_name=snowcli_environment)
+        self.session: Session = get_snowpark_session(
+            connection_name=snowcli_environment
+        )
         self.sync_slug = sync_slug
-        session_run_response = self.session.sql(
-            f"""
+        session_run_response = self.session.sql(f"""
             call {self.omnata_app_name}.API.CREATE_DEVELOPMENT_SESSION_RUN('{sync_slug}')
-            """
-        ).collect()
+            """).collect()
         response_parsed = json.loads(session_run_response[0][0])
         if response_parsed["success"] is False:
             raise ValueError(f"Failed: {response_parsed['error']}")
@@ -276,16 +281,15 @@ class DevelopmentSession:
             else "null"
         )
 
-        secrets_fetch_response = self.session.sql(
-            f"""
+        secrets_fetch_response = self.session.sql(f"""
             call {plugin_app_name}.PLUGIN.RETRIEVE_SECRETS({oauth_secret_param},{other_secrets_param})
-            """
-        ).collect()
+            """).collect()
         secrets_response_parsed = json.loads(secrets_fetch_response[0][0])
         if secrets_response_parsed["success"] is False:
             raise ValueError(f"Failed: {secrets_response_parsed['error']}")
-        secrets_data = TypeAdapter(
-            Dict[str, StoredConfigurationValue]).validate_python(secrets_response_parsed["data"])
+        secrets_data = TypeAdapter(Dict[str, StoredConfigurationValue]).validate_python(
+            secrets_response_parsed["data"]
+        )
 
         source_database = self.omnata_app_name
         source_schema = outbound_payload.records_schema_name
@@ -298,17 +302,20 @@ class DevelopmentSession:
         configuration_parameters = OutboundSyncConfigurationParameters(
             connection_method=outbound_payload.connection_method,
             connection_parameters=TypeAdapter(
-                Dict[str, StoredConfigurationValue]).validate_python(
+                Dict[str, StoredConfigurationValue]
+            ).validate_python(
                 outbound_payload.connection_parameters,
             ),
             connection_secrets=TypeAdapter(
-                Dict[str, StoredConfigurationValue]).validate_python(secrets_data
-            ),
+                Dict[str, StoredConfigurationValue]
+            ).validate_python(secrets_data),
             sync_parameters=TypeAdapter(
-                Dict[str, StoredConfigurationValue]).validate_python(outbound_payload.sync_parameters
-            ),
+                Dict[str, StoredConfigurationValue]
+            ).validate_python(outbound_payload.sync_parameters),
             sync_strategy=outbound_payload.sync_strategy,
-            field_mappings=TypeAdapter(StoredMappingValue).validate_python(outbound_payload.field_mappings),
+            field_mappings=TypeAdapter(StoredMappingValue).validate_python(
+                outbound_payload.field_mappings
+            ),
             current_form_parameters={},
         )
         outbound_sync_request = OutboundSyncRequest(
@@ -323,7 +330,8 @@ class DevelopmentSession:
             api_limits=self.api_limits,
             rate_limit_state_all={},
             rate_limit_state_this_sync_and_branch={},
-            run_deadline=datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=4),
+            run_deadline=datetime.datetime.now(datetime.timezone.utc)
+            + datetime.timedelta(hours=4),
             development_mode=True,
         )
 
@@ -359,20 +367,20 @@ class DevelopmentSession:
         if not click.globals.get_current_context(silent=True):
             # set an empty one so snowcli's app config doesn't error
             click.globals.push_context(click.core.Context(click.core.Group()))
-        
-        self.session: Session = get_snowpark_session(connection_name=snowcli_environment)
+
+        self.session: Session = get_snowpark_session(
+            connection_name=snowcli_environment
+        )
         # we set the current database to the plugin app name, if it's set in the environment
         # this means any plugins which call their own UDFs will work
-        if 'PLUGIN_APP_NAME' in os.environ:
-            plugin_app_name = os.environ['PLUGIN_APP_NAME']
+        if "PLUGIN_APP_NAME" in os.environ:
+            plugin_app_name = os.environ["PLUGIN_APP_NAME"]
             self.session.sql(f"USE DATABASE {plugin_app_name}").collect()
 
         self.sync_slug = sync_slug
-        session_run_response = self.session.sql(
-            f"""
+        session_run_response = self.session.sql(f"""
             call {self.omnata_app_name}.API.CREATE_DEVELOPMENT_SESSION_RUN('{sync_slug}')
-            """
-        ).collect()
+            """).collect()
         response_parsed = json.loads(session_run_response[0][0])
         if response_parsed["success"] is False:
             raise ValueError(f"Failed: {response_parsed['error']}")
@@ -408,31 +416,31 @@ class DevelopmentSession:
             else "null"
         )
 
-        secrets_fetch_response = self.session.sql(
-            f"""
+        secrets_fetch_response = self.session.sql(f"""
             call {plugin_app_name}.PLUGIN.RETRIEVE_SECRETS({oauth_secret_param},{other_secrets_param})
-            """
-        ).collect()
+            """).collect()
         secrets_response_parsed = json.loads(secrets_fetch_response[0][0])
         if secrets_response_parsed["success"] is False:
             raise ValueError(f"Failed: {secrets_response_parsed['error']}")
-        secrets_data = TypeAdapter(
-            Dict[str, StoredConfigurationValue]).validate_python(secrets_response_parsed["data"])
+        secrets_data = TypeAdapter(Dict[str, StoredConfigurationValue]).validate_python(
+            secrets_response_parsed["data"]
+        )
 
         # full_source_table_name = f"{source_database}.{source_schema}.{source_table}"
         # self.api_limits.concurrency = 1 # we have to set this because pyvcr isn't thread safe
         configuration_parameters = InboundSyncConfigurationParameters(
             connection_method=inbound_payload.connection_method,
             connection_parameters=TypeAdapter(
-                Dict[str, StoredConfigurationValue]).validate_python(
+                Dict[str, StoredConfigurationValue]
+            ).validate_python(
                 inbound_payload.connection_parameters,
             ),
             connection_secrets=TypeAdapter(
-                Dict[str, StoredConfigurationValue]).validate_python(secrets_data
-            ),
+                Dict[str, StoredConfigurationValue]
+            ).validate_python(secrets_data),
             sync_parameters=TypeAdapter(
-                Dict[str, StoredConfigurationValue]).validate_python(inbound_payload.sync_parameters
-            ),
+                Dict[str, StoredConfigurationValue]
+            ).validate_python(inbound_payload.sync_parameters),
             current_form_parameters={},
         )
 
@@ -446,7 +454,9 @@ class DevelopmentSession:
         )
         for stream in streams_list:
             if stream.stream.source_defined_primary_key is not None:
-                self.stream_index_fields[stream.stream_name] = stream.stream.source_defined_primary_key
+                self.stream_index_fields[stream.stream_name] = (
+                    stream.stream.source_defined_primary_key
+                )
         omnata_log_handler = OmnataPluginLogHandler(
             session=self.session,
             sync_id=None,
@@ -464,12 +474,13 @@ class DevelopmentSession:
             api_limits=self.api_limits,
             rate_limit_state_all={},
             rate_limit_state_this_sync_and_branch={},
-            run_deadline=datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=4),
+            run_deadline=datetime.datetime.now(datetime.timezone.utc)
+            + datetime.timedelta(hours=4),
             development_mode=True,
             # this skips over the part where we check for new streams etc,
             # so development sessions will only get requests for explicitly configured streams
             streams=streams_list,
-            omnata_log_handler=omnata_log_handler
+            omnata_log_handler=omnata_log_handler,
         )
 
         self.current_scenario = SyncScenario(
@@ -479,14 +490,14 @@ class DevelopmentSession:
             configuration_parameters=configuration_parameters,
             sync_request=inbound_sync_request,
             scenario_records=None,
-            initial_stream_state=initial_stream_state
+            initial_stream_state=initial_stream_state,
         )
         self.run_id = inbound_payload.run_id
         self.plugin_instance._sync_request = inbound_sync_request
         if self.recorded_http:
             self._start_recording()
         return configuration_parameters, inbound_sync_request
-    
+
     def cancel_scenario(self):
         """
         Cancels the current scenario, and stops recording traffic.
@@ -498,7 +509,7 @@ class DevelopmentSession:
             self._stop_recording()
         print("Success: cancelled scenario")
 
-    def complete_scenario(self,apply_results:bool=True):
+    def complete_scenario(self, apply_results: bool = True):
         """
         Marks the scenario as complete, and stops recording traffic.
         """
@@ -511,14 +522,18 @@ class DevelopmentSession:
             self._stop_recording()
         if apply_results:
             self.current_scenario.sync_request.apply_results_queue()
-            if self.current_scenario.sync_request.__class__.__name__ == "InboundSyncRequest":
-                print(f'States at completion: {self.current_scenario.sync_request._latest_states}')
-                self.current_scenario.sync_request.apply_progress_updates(ignore_errors=False)
-        session_run_response = self.session.sql(
-            f"""
+            if (
+                self.current_scenario.sync_request.__class__.__name__
+                == "InboundSyncRequest"
+            ):
+                # Checkpointing have removed this field
+                # print(f"States at completion: {self.current_scenario.sync_request._latest_states}")
+                self.current_scenario.sync_request.apply_progress_updates(
+                    ignore_errors=False
+                )
+        session_run_response = self.session.sql(f"""
             call {self.omnata_app_name}.API.COMPLETE_DEVELOPMENT_SESSION_RUN({self.run_id},{apply_results})
-            """
-        ).collect()
+            """).collect()
         response_parsed = json.loads(session_run_response[0][0])
         if response_parsed["success"] is False:
             raise ValueError(f"Failed: {response_parsed['error']}")
@@ -576,7 +591,9 @@ class DevelopmentSession:
             if self.current_scenario is None:
                 print("Failed: no active scenario")
                 return
-            results_dataframe = pandas.concat(outbound_sync_request._apply_results).copy()
+            results_dataframe = pandas.concat(
+                outbound_sync_request._apply_results
+            ).copy()
 
             originally_provided_records = self.current_scenario.scenario_records
 
@@ -625,7 +642,10 @@ class DevelopmentSession:
             print("Success: results are valid")
         elif sync_request.__class__.__name__ == "InboundSyncRequest":
             inbound_sync_request: InboundSyncRequest = sync_request
-            if inbound_sync_request._apply_results is None or len(inbound_sync_request._apply_results) == 0:
+            if (
+                inbound_sync_request._apply_results is None
+                or len(inbound_sync_request._apply_results) == 0
+            ):
                 # if there are no results, we check the real results table
                 full_table_name = inbound_sync_request._full_results_table_name
                 # usually if results are provided in a table, RECORD_OBJECT (object) is used instead of RECORD_DATA (varchar)
@@ -637,27 +657,37 @@ class DevelopmentSession:
                         COALESCE(PARSE_JSON(RECORD_DATA), RECORD_OBJECT) as RECORD_OBJECT,
                         IS_DELETED
                     from {full_table_name}""").collect()
-                
+
                 if len(results_from_table) == 0:
-                    print(f"Failed: No results were enqueued to the sync request, and no results found in {full_table_name}")
+                    print(
+                        f"Failed: No results were enqueued to the sync request, and no results found in {full_table_name}"
+                    )
                 else:
                     results_dataframe = pandas.DataFrame(results_from_table)
-                    results_dataframe["RECORD_OBJECT"] = results_dataframe["RECORD_OBJECT"].apply(
-                        lambda x: json.loads(x) if x is not None else None
-                    )
+                    results_dataframe["RECORD_OBJECT"] = results_dataframe[
+                        "RECORD_OBJECT"
+                    ].apply(lambda x: json.loads(x) if x is not None else None)
                     self.current_scenario.results = results_dataframe
                     print("Success: results are valid (retrieved from table)")
             else:
                 # inbound_sync_request._apply_results is a dictionary of stream_name -> list of dataframes
                 # we want to flatten this into a single dataframe
+
+                # In Checkpointing, we need to loop each dataframe again as each dataframe holds a RecordsToUploadResult object
                 results_dataframe = pandas.concat(
-                    [df for df_list in inbound_sync_request._apply_results.values() for df in df_list]
+                    [
+                        item.records
+                        for df_list in inbound_sync_request._apply_results.values()
+                        for item in df_list
+                        if isinstance(item, RecordsToUploadResult)
+                    ]
                 ).copy()
+
                 # typically, the results table will have a RECORD_DATA column, which is a varchar
                 # we need to parse this as JSON into an object column named RECORD_OBJECT, allowing for nulls
-                results_dataframe["RECORD_OBJECT"] = results_dataframe["RECORD_DATA"].apply(
-                    lambda x: json.loads(x) if x is not None else None
-                )
+                results_dataframe["RECORD_OBJECT"] = results_dataframe[
+                    "RECORD_DATA"
+                ].apply(lambda x: json.loads(x) if x is not None else None)
                 # we can drop the RECORD_DATA column now
                 results_dataframe = results_dataframe.drop(columns=["RECORD_DATA"])
                 self.current_scenario.results = results_dataframe
@@ -766,20 +796,18 @@ class DevelopmentSession:
                         {
                             "Property": "connection_parameters",
                             "Value": json.dumps(
-                                to_jsonable_python(outbound_params.connection_parameters)
+                                to_jsonable_python(
+                                    outbound_params.connection_parameters
+                                )
                             ),
                         },
                         {
                             "Property": "connection_secrets",
-                            "Value": json.dumps(
-                                to_jsonable_python(secrets_redacted)
-                            ),
+                            "Value": json.dumps(to_jsonable_python(secrets_redacted)),
                         },
                         {
                             "Property": "api_limits",
-                            "Value": json.dumps(
-                                to_jsonable_python(self.api_limits)
-                            ),
+                            "Value": json.dumps(to_jsonable_python(self.api_limits)),
                         },
                         {
                             "Property": "sync_parameters",
@@ -816,15 +844,11 @@ class DevelopmentSession:
                         },
                         {
                             "Property": "connection_secrets",
-                            "Value": json.dumps(
-                                to_jsonable_python(secrets_redacted)
-                            ),
+                            "Value": json.dumps(to_jsonable_python(secrets_redacted)),
                         },
                         {
                             "Property": "api_limits",
-                            "Value": json.dumps(
-                                to_jsonable_python(self.api_limits)
-                            ),
+                            "Value": json.dumps(to_jsonable_python(self.api_limits)),
                         },
                         {
                             "Property": "sync_parameters",
@@ -846,7 +870,11 @@ class DevelopmentSession:
             f.write(f"\tScenario: {recorded_scenario.scenario_name}\n")
             if recorded_scenario.sync_direction == SyncDirection.OUTBOUND:
                 f.write("\t\tGiven the following records:\n")
-                for col in ["RESULT","TRANSFORMED_RECORD", "TRANSFORMED_RECORD_PREVIOUS"]:
+                for col in [
+                    "RESULT",
+                    "TRANSFORMED_RECORD",
+                    "TRANSFORMED_RECORD_PREVIOUS",
+                ]:
                     recorded_scenario.scenario_records[col] = (
                         recorded_scenario.scenario_records[col]
                         .apply(lambda x: json.loads(x) if x is not None else None)
@@ -859,7 +887,14 @@ class DevelopmentSession:
                     )
                 table_indented = self.dataframe_to_behave_table(
                     recorded_scenario.scenario_records[
-                        ["IDENTIFIER", "APP_IDENTIFIER", "RESULT", "SYNC_ACTION", "TRANSFORMED_RECORD","TRANSFORMED_RECORD_PREVIOUS"]
+                        [
+                            "IDENTIFIER",
+                            "APP_IDENTIFIER",
+                            "RESULT",
+                            "SYNC_ACTION",
+                            "TRANSFORMED_RECORD",
+                            "TRANSFORMED_RECORD_PREVIOUS",
+                        ]
                     ],
                     3,
                 )
@@ -871,7 +906,10 @@ class DevelopmentSession:
                     recorded_scenario.sync_request
                 )
                 for stream in inbound_sync_request.streams:
-                    if recorded_scenario.initial_stream_state is not None and stream.stream_name in recorded_scenario.initial_stream_state:
+                    if (
+                        recorded_scenario.initial_stream_state is not None
+                        and stream.stream_name in recorded_scenario.initial_stream_state
+                    ):
                         f.write(
                             f"\t\t\t| {stream.stream_name} | {json.dumps(recorded_scenario.initial_stream_state[stream.stream_name],separators=(',', ':'),default=str)} | \n"
                         )
@@ -892,9 +930,7 @@ class DevelopmentSession:
                 f"\t\tAnd we use the {self.plugin_instance.__class__.__name__} class from the {self.plugin_module} module\n"
             )
             if self.inject_snowpark_session:
-                f.write(
-                    f"\t\tAnd we attach a Snowpark session to the sync request\n"
-                )
+                f.write(f"\t\tAnd we attach a Snowpark session to the sync request\n")
                 f.write(
                     f"\t\tAnd the plugin returns inbound sync results directly to a real table\n"
                 )
@@ -907,7 +943,9 @@ class DevelopmentSession:
                     "\t\tWhen we perform an inbound sync with configuration parameters:\n"
                 )
             f.write(f"{table_indented}\n")
-            f.write(f"\t\tAnd we use the recorded results from {recorded_scenario.inbound_results_file_name}\n")
+            f.write(
+                f"\t\tAnd we use the recorded results from {recorded_scenario.inbound_results_file_name}\n"
+            )
             if recorded_scenario.sync_direction == SyncDirection.OUTBOUND:
                 f.write("\t\tAnd the response will be:\n")
                 # self.create_results['TRANSFORMED_RECORD']=self.create_results['TRANSFORMED_RECORD'].apply(json.dumps)
@@ -942,9 +980,7 @@ class DevelopmentSession:
                             "RECORD_OBJECT",
                         ].apply(
                             lambda x: {
-                                k: v
-                                for k, v in x.items()
-                                if k not in excluded_fields
+                                k: v for k, v in x.items() if k not in excluded_fields
                             }
                         )
                         f.write(
@@ -952,56 +988,88 @@ class DevelopmentSession:
                         )
                 # any records excluded by field value are removed both in the recording and the test
                 if len(self.exclude_records_by_field_value) > 0:
-                    for field_name, field_value in self.exclude_records_by_field_value.items():
-                        if field_value == 'true':
+                    for (
+                        field_name,
+                        field_value,
+                    ) in self.exclude_records_by_field_value.items():
+                        if field_value == "true":
                             field_value = True
-                        elif field_value == 'false':
+                        elif field_value == "false":
                             field_value = False
-                        recorded_scenario.results["value_test"] = recorded_scenario.results["RECORD_OBJECT"].apply(
-                            lambda x: x[field_name] if x is not None and field_name in x else None
+                        recorded_scenario.results[
+                            "value_test"
+                        ] = recorded_scenario.results["RECORD_OBJECT"].apply(
+                            lambda x: (
+                                x[field_name]
+                                if x is not None and field_name in x
+                                else None
+                            )
                         )
-                        matching_records = recorded_scenario.results[recorded_scenario.results["value_test"] == field_value]
-                        print(f"Excluded {len(matching_records)} records where {field_name} is {field_value}")
-                        recorded_scenario.results = recorded_scenario.results[recorded_scenario.results["value_test"] != field_value]
+                        matching_records = recorded_scenario.results[
+                            recorded_scenario.results["value_test"] == field_value
+                        ]
+                        print(
+                            f"Excluded {len(matching_records)} records where {field_name} is {field_value}"
+                        )
+                        recorded_scenario.results = recorded_scenario.results[
+                            recorded_scenario.results["value_test"] != field_value
+                        ]
                         # remove the temporary column
-                        recorded_scenario.results.drop(columns=["value_test"],inplace=True)
-
+                        recorded_scenario.results.drop(
+                            columns=["value_test"], inplace=True
+                        )
 
                 # convert the RECORD_OBJECT to a json string without newlines
                 recorded_scenario.results["RECORD_OBJECT"] = recorded_scenario.results[
                     "RECORD_OBJECT"
                 ].apply(json.dumps)
                 # also, drop the RETRIEVE_DATE column
-                recorded_scenario.results.drop(columns=["RETRIEVE_DATE"],inplace=True)
+                recorded_scenario.results.drop(columns=["RETRIEVE_DATE"], inplace=True)
                 # create the results file if it doesn't exist
-                os.makedirs(os.path.dirname(recorded_scenario.inbound_results_file), exist_ok=True)
-                recorded_scenario.results.to_csv(recorded_scenario.inbound_results_file, sep="|", index=False)
+                os.makedirs(
+                    os.path.dirname(recorded_scenario.inbound_results_file),
+                    exist_ok=True,
+                )
+                recorded_scenario.results.to_csv(
+                    recorded_scenario.inbound_results_file, sep="|", index=False
+                )
 
                 if len(self.exclude_records_by_field_value) > 0:
-                    for field_name, field_value in self.exclude_records_by_field_value.items():
-                        f.write(f"\t\tAnd we exclude all inbound results where the field {field_name} has the value {field_value}\n")
+                    for (
+                        field_name,
+                        field_value,
+                    ) in self.exclude_records_by_field_value.items():
+                        f.write(
+                            f"\t\tAnd we exclude all inbound results where the field {field_name} has the value {field_value}\n"
+                        )
                 # group the results by stream name
                 stream_names = recorded_scenario.results["STREAM_NAME"].unique()
 
                 for stream_name in stream_names:
                     if stream_name not in self.stream_index_fields:
-                        print(f"Failed: no index field set for stream {stream_name}. Use the set_inbound_stream_index_field function to set one")
+                        print(
+                            f"Failed: no index field set for stream {stream_name}. Use the set_inbound_stream_index_field function to set one"
+                        )
                         return
                     f.write(
                         f"\t\tAnd when indexed on the {self.stream_index_fields[stream_name]} RECORD_OBJECT field, we assert that the recorded results for the {stream_name} stream are correct\n"
                     )
                     # count the number of records in the results that have a STREAM_NAME of stream_name
-                    stream_results = recorded_scenario.results[recorded_scenario.results["STREAM_NAME"] == stream_name]
+                    stream_results = recorded_scenario.results[
+                        recorded_scenario.results["STREAM_NAME"] == stream_name
+                    ]
                     if len(stream_results) == 0:
-                        raise ValueError(f"Recorded results for scenario {recorded_scenario.scenario_name} did not contain any results for stream {stream_name}")
-                    #results = pandas.concat(recorded_scenario.results[stream_name])
+                        raise ValueError(
+                            f"Recorded results for scenario {recorded_scenario.scenario_name} did not contain any results for stream {stream_name}"
+                        )
+                    # results = pandas.concat(recorded_scenario.results[stream_name])
                     ## no need to do this anymore, as the results are already json
                     ##results["RECORD_DATA"] = results["RECORD_DATA"].apply(json.dumps)
-                    #results["RECORD_DATA"] = results["RECORD_DATA"].replace("|", "\\|")
-                    #results.loc[
+                    # results["RECORD_DATA"] = results["RECORD_DATA"].replace("|", "\\|")
+                    # results.loc[
                     #    results["RECORD_DATA"] == '"null"', "RECORD_DATA"
-                    #] = "null"
-                    #results = results[
+                    # ] = "null"
+                    # results = results[
                     #    results.columns.intersection(
                     #        [
                     #            "APP_IDENTIFIER",
@@ -1010,9 +1078,9 @@ class DevelopmentSession:
                     #            "STREAM_NAME",
                     #        ]
                     #    )
-                    #]
-                    #table_indented = self.dataframe_to_behave_table(results, 3)
-                    #f.write(f"{table_indented}\n")
+                    # ]
+                    # table_indented = self.dataframe_to_behave_table(results, 3)
+                    # f.write(f"{table_indented}\n")
 
         f.close()
         print("Finished writing feature file")