omnata-plugin-runtime 0.10.33a297__tar.gz → 0.11.0__tar.gz

{omnata_plugin_runtime-0.10.33a297 → omnata_plugin_runtime-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: omnata-plugin-runtime
-Version: 0.10.33a297
+Version: 0.11.0
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.10.33a297 → omnata_plugin_runtime-0.11.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "omnata-plugin-runtime"
-version = "0.10.33-a297"
+version = "0.11.0"
 description = "Classes and common runtime components for building and running Omnata Plugins"
 authors = ["James Weakley <james.weakley@omnata.com>"]
 readme = "README.md"

{omnata_plugin_runtime-0.10.33a297 → omnata_plugin_runtime-0.11.0}/src/omnata_plugin_runtime/json_schema.py

@@ -405,6 +405,20 @@ class FullyQualifiedTable(BaseModel):
         return self.get_fully_qualified_name(
             table_override=f"{self.table_name}_CRITERIA_DELETES"
         )
+    
+    def get_fully_qualified_state_register_table_name(self) -> str:
+        """
+        Returns the fully qualified name of the state register table.
+        This is used to store state values for syncs, paired with query IDs to use with time travel.
+        """
+        return self.get_fully_qualified_name(table_override=f"{self.table_name}_STATE_REGISTER")
+
+    def get_fully_qualified_state_register_table_sequence_name(self) -> str:
+        """
+        Returns the fully qualified name of the state register table.
+        This is used to store state values for syncs, paired with query IDs to use with time travel.
+        """
+        return self.get_fully_qualified_name(table_override=f"{self.table_name}_STATE_REGISTER_SEQ")
 
 class SnowflakeViewPart(BaseModel):
     """
@@ -456,10 +470,20 @@ class SnowflakeViewPart(BaseModel):
             c.name_with_comment(binding_list) for c in self.columns
         ]
     
-    def cte_text(self,original_name: bool = False, include_only_columns:Optional[List[str]] = None) -> str:
+    def cte_text(self,original_name: bool = False,
+            include_only_columns:Optional[List[str]] = None,
+            include_extra_columns:Optional[List[str]] = None
+            ) -> str:
         """
         Returns the CTE text for this view part.
         """
+        if include_extra_columns is not None:
+            # includes direct columns plus any extra specified
+            return f""" "{self.stream_name}" as (
+    select {', '.join([c.definition(original_name=original_name,remove_stream_prefix=self.stream_name) for c in self.columns
+                       if c.original_name in include_extra_columns or not c.is_join_column])} 
+    from {self.raw_table_location.get_fully_qualified_name()}
+) """
         if include_only_columns is None:
             return f""" "{self.stream_name}" as (
     select {', '.join([c.definition(original_name=original_name,remove_stream_prefix=self.stream_name) for c in self.direct_columns()])} 
@@ -490,6 +514,29 @@ class SnowflakeViewParts(BaseModel):
         ..., description="The other streams that are joined to the main stream"
     )
 
+    def column_indirectly_references_other_streams(
+        self,
+        all_view_parts:List[SnowflakeViewPart],
+        stream_name:str,column_name:str) -> bool:
+
+        for part in all_view_parts:
+            if part.stream_name == stream_name:
+                for col in part.columns:
+                    if col.original_name == column_name:
+                        if col.referenced_columns:
+                            for ref_stream, ref_cols in col.referenced_columns.items():
+                                if ref_stream != stream_name:
+                                    return True
+                                else:
+                                    # we have to call this recursively in case the referenced column also references other streams
+                                    result = any(
+                                        self.column_indirectly_references_other_streams(
+                                            all_view_parts, ref_stream, ref_col
+                                        ) for ref_col in ref_cols
+                                    )
+                                    return result
+        return False
+
     def view_body(self):
         """
         Creates a view definition from the parts.
@@ -505,31 +552,40 @@ class SnowflakeViewParts(BaseModel):
 
         # first, we need to collapse all referenced columns into a single map
         all_referenced_columns:Dict[str,List[str]] = {}
+        
+        # if a column references other columns, but there are no dependencies outside of its own stream, we can include those columns in the initial CTE for that stream
+        # because they can be calculated directly without needing joins
+        columns_only_referencing_own_stream:Dict[str,List[str]] = {}
+ 
+        
         for part in [self.main_part] + self.joined_parts:
-            # if the main part references any columns in this part in its joins, we need to include those columns
+            # if the main part references any columns in this part in its joins, we need to include those columns because they are used in the join condition
             aliases_for_stream = [j.join_stream_alias for j in self.main_part.joins 
                 if j.join_stream_name == part.stream_name]
             columns_used_in_joins = [
                 j.left_column for j in self.main_part.joins if j.left_alias in aliases_for_stream
             ]
-            if part.stream_name not in all_referenced_columns:
-                all_referenced_columns[part.stream_name] = []
-            all_referenced_columns[part.stream_name] += columns_used_in_joins
+            all_referenced_columns.setdefault(part.stream_name, []).extend(columns_used_in_joins)
+            # now, for each column in the part, if it references columns in other streams, we need to include those columns
             for column in part.columns:
                 if column.referenced_columns:
                     for stream_name, referenced_columns in column.referenced_columns.items():
-                        if stream_name not in all_referenced_columns:
-                            all_referenced_columns[stream_name] = []
-                        all_referenced_columns[stream_name] += referenced_columns
+                        all_referenced_columns.setdefault(stream_name, []).extend(referenced_columns)
+                        # populate columns_only_referencing_own_stream by following the chain of references until we reach a column that references another stream or has no references
+                        if self.column_indirectly_references_other_streams(
+                            [self.main_part] + self.joined_parts, part.stream_name, column.original_name
+                        ) == False:
+                            columns_only_referencing_own_stream.setdefault(part.stream_name, []).append(column.original_name)
+                else:
+                    # if the column has no references, it can be included in the initial CTE for its own stream
+                    columns_only_referencing_own_stream.setdefault(part.stream_name, []).append(column.original_name)
+            # if this part has joins to other streams, we need to include the join columns
             for join in part.joins:
-                if join.join_stream_name not in all_referenced_columns:
-                    all_referenced_columns[join.join_stream_name] = []
-                all_referenced_columns[join.join_stream_name].append(join.join_stream_column)
-                all_referenced_columns[part.stream_name].append(join.left_column)
-
+                all_referenced_columns.setdefault(join.join_stream_name, []).append(join.join_stream_column)
+                all_referenced_columns.setdefault(part.stream_name, []).append(join.left_column)
 
         ctes = [
-                self.main_part.cte_text(original_name=True)
+                self.main_part.cte_text(original_name=True,include_extra_columns=columns_only_referencing_own_stream.get(self.main_part.stream_name))
             ] + [
                 part.cte_text(original_name=True,include_only_columns=all_referenced_columns.get(part.stream_name)) 
             for part in joined_parts_deduped
@@ -539,9 +595,9 @@ class SnowflakeViewParts(BaseModel):
         final_cte = f""" OMNATA_FINAL_CTE as (
             select {', '.join(
             [
-                f'"{self.main_part.stream_name}"."{c.original_name}"' for c in self.main_part.direct_columns()
+                f'"{self.main_part.stream_name}"."{c.original_name}"' for c in self.main_part.columns if not c.is_join_column or c.original_name in columns_only_referencing_own_stream.get(self.main_part.stream_name,[])
             ]+[
-                c.definition(original_name=True) for c in self.main_part.join_columns()
+                c.definition(original_name=True) for c in self.main_part.columns if c.is_join_column and c.original_name not in columns_only_referencing_own_stream.get(self.main_part.stream_name,[])
             ])}
             from "{self.main_part.stream_name}" """
         if len(self.main_part.joins) > 0:
@@ -747,6 +803,19 @@ class SnowflakeViewParts(BaseModel):
 
         return cls(main_part=main_stream_view_part, joined_parts=joined_parts)
 
+
+# Helper function to find a view part by stream name
+def find_part(view_part: SnowflakeViewPart, joined_parts: List[SnowflakeViewPart], stream_name: str) -> Optional[SnowflakeViewPart]:
+    if stream_name == view_part.stream_name:
+        return view_part
+    for part in joined_parts:
+        if part.stream_name == stream_name:
+            return part
+    for join in view_part.joins:
+        if join.join_stream_alias == stream_name:
+            return view_part
+    return None
+
 def prune(view_part: SnowflakeViewPart, joined_parts: List[SnowflakeViewPart]) -> bool:
     """
     Prunes columns from view parts that reference fields that don't exist in the referenced streams.
@@ -759,12 +828,6 @@ def prune(view_part: SnowflakeViewPart, joined_parts: List[SnowflakeViewPart]) -
     Raises ValueError if a cyclic dependency is detected.
     """
     columns_removed = False
-    
-    # Helper function to find a view part by stream name
-    def find_part(stream_name: str) -> Optional[SnowflakeViewPart]:
-        if stream_name == view_part.stream_name:
-            return view_part
-        return next((p for p in joined_parts if p.stream_name == stream_name), None)
 
     # Helper function to check if a column should be kept or removed
     def should_keep_column(column: SnowflakeViewColumn, part: SnowflakeViewPart) -> bool:
@@ -779,7 +842,7 @@ def prune(view_part: SnowflakeViewPart, joined_parts: List[SnowflakeViewPart]) -
         # Check each referenced stream and its fields
         for ref_stream_name, ref_fields in column.referenced_columns.items():
             # Find the referenced part
-            ref_part = find_part(ref_stream_name)
+            ref_part = find_part(view_part, joined_parts,ref_stream_name)
             
             # If referenced stream doesn't exist, remove the column
             if ref_part is None:

{omnata_plugin_runtime-0.10.33a297 → omnata_plugin_runtime-0.11.0}/src/omnata_plugin_runtime/omnata_plugin.py

@@ -93,6 +93,9 @@ from .rate_limiting import (
     RateLimitState,
     RateLimitedSession
 )
+from .json_schema import (
+    FullyQualifiedTable
+)
 
 SortDirectionType = Literal["asc", "desc"]
 
@@ -1055,7 +1058,6 @@ class InboundSyncRequest(SyncRequest):
         
         # These are similar to the results, but represent requests to delete records by some criteria
         self._apply_results_criteria_deletes: Dict[str, List[pandas.DataFrame]] = {}
-        self._latest_states: Dict[str, Any] = {}
         self._temp_tables = {}
         self._temp_table_lock = threading.Lock()
         self._results_exist: Dict[
@@ -1085,24 +1087,47 @@ class InboundSyncRequest(SyncRequest):
             sync_id=sync_id,
             branch_name=branch_name
         )
-        # named by convention, see SyncRunProcessor.enqueue
-        self._criteria_deletes_table_name = (
-            f"{self._source_app_name}.{self._results_schema_name}.{self._results_table_name}_CRITERIA_DELETES"
+        # The results table name is also used to derive several other table/stage names
+        results_table = FullyQualifiedTable(
+            database_name= self._source_app_name,
+            schema_name= self._results_schema_name,
+            table_name= self._results_table_name
         )
-        self._apply_results: Dict[str, List[pandas.DataFrame]] = {}
+        self._criteria_deletes_table_name = results_table.get_fully_qualified_criteria_deletes_table_name()
+        self.state_register_table_name = results_table.get_fully_qualified_state_register_table_name()
+        # this is keyed on stream name, each containing a list of dataframes and state updates mixed
+        self._apply_results: Dict[str, List[pandas.DataFrame | Dict]] = {}
 
     def apply_results_queue(self):
         """
-        Merges all of the queued results and applies them
+        Merges all of the queued results and applies them, including state updates.
         """
-        logger.debug("InboundSyncRequest apply_results_queue ")
+        logger.debug("InboundSyncRequest apply_results_queue")
         if self._apply_results is not None:
             with self._apply_results_lock:
                 results:List[pandas.DataFrame] = []
-                stream_names:List[str] = []
+                stream_states_for_upload:Dict[str, Dict[str, Any]] = {}
                 for stream_name, stream_results in self._apply_results.items():
+                    # the stream results contains an ordered sequence of dataframes and state updates (append only)
+                    # we only want to apply the dataframes up until the most recent state update
+                    # so first, we iterate backwards to find the last state update
+                    last_state_index = -1
+                    for i in range(len(stream_results) - 1, -1, -1):
+                        if isinstance(stream_results[i], dict):
+                            last_state_index = i
+                            stream_states_for_upload[stream_name] = stream_results[i]
+                            break
+                    # if there are no state updates, we can't do anything with this stream
+                    if last_state_index == -1:
+                        logger.debug(
+                            f"No state updates for stream {stream_name}, skipping"
+                        )
+                        continue
+                    assert isinstance(stream_states_for_upload[stream_name], dict), "Latest state must be a dictionary"
+                    # now we can take the dataframes up to the last state update
+                    dfs = stream_results[:last_state_index]
                     non_empty_dfs = [
-                        x for x in stream_results if x is not None and len(x) > 0
+                        x for x in dfs if x is not None and isinstance(x, pandas.DataFrame) and len(x) > 0
                     ]
                     # get the total length of all the dataframes
                     total_length = sum([len(x) for x in non_empty_dfs])
@@ -1110,22 +1135,28 @@ class InboundSyncRequest(SyncRequest):
                     self._stream_record_counts[
                         stream_name
                     ] = self._stream_record_counts[stream_name] + total_length
-                    results.extend(non_empty_dfs)  # remove any None/empty dataframes
-                    stream_names.append(stream_name)
+                    results.extend(non_empty_dfs)
+                    # now remove everything up to the last state update
+                    # we do this so that we don't apply the same state update multiple times
+                    self._apply_results[stream_name] = stream_results[
+                        last_state_index + 1 :
+                    ]  # keep everything after the last state update
                 if len(results) > 0:
                     logger.debug(
                         f"Applying {len(results)} batches of queued results"
                     )
                     # upload all cached apply results
                     all_dfs = pandas.concat(results)
-                    self._apply_results_dataframe(stream_names, all_dfs)
-                # update the stream state object too
-                self._apply_latest_states()
-                for stream_name in stream_names:
-                    self._apply_results[stream_name] = None
-                self._apply_results = {}
-                
+                    query_id = self._apply_results_dataframe(list(stream_states_for_upload.keys()), all_dfs)
+                    # now that the results have been updated, we need to insert records into the state register table
+                    # we do this by inserting the latest state for each stream
+                    self._directly_insert_to_state_register(
+                        stream_states_for_upload, query_id=query_id
+                    )
+        
         # also take care of uploading delete requests
+        # technically these should be managed along with the state, however there aren't any scenarios where checkpointing is done
+        # and deletes have an impact. This is because we only checkpoint in scenarios where the target table is empty first
         if hasattr(self,'_apply_results_criteria_deletes') and self._apply_results_criteria_deletes is not None:
             with self._apply_results_lock:
                 results:List[pandas.DataFrame] = []
@@ -1149,27 +1180,57 @@ class InboundSyncRequest(SyncRequest):
         # so we need to make sure all the results are applied first
         self.apply_progress_updates()
     
+    def _directly_insert_to_state_register(
+            self, stream_states_for_upload: Dict[str, Dict[str, Any]],
+            query_id: Optional[str] = None
+    ) -> str:
+        binding_values = []
+        select_clauses = []
+        
+        with self._snowflake_query_lock:
+            if query_id is None:
+                query_id = self._get_query_id_for_now()
+            for stream_name, latest_state in stream_states_for_upload.items():
+                binding_values.extend([stream_name, query_id, json.dumps(latest_state)])
+                select_clauses.append(
+                    f"select ?, ?, PARSE_JSON(?)"
+                )
+            final_query = f"""INSERT INTO {self.state_register_table_name} (STREAM_NAME, QUERY_ID, STATE_VALUE)
+                {' union all '.join(select_clauses)}"""
+            self._session.sql(final_query, binding_values).collect()
+            streams_included = list(stream_states_for_upload.keys())
+            logger.debug(f"Inserted state for streams: {streams_included} with query ID {query_id}")
+    
     def apply_progress_updates(self, ignore_errors:bool = True):
         """
         Sends a message to the plugin with the current progress of the sync run, if it has changed since last time.
         """
-        if self._apply_results is not None:
-            with self._apply_results_lock:
-                new_progress_update = PluginMessageStreamProgressUpdate(
-                        stream_total_counts=self._stream_record_counts,
-                        # records could have been marked as completed, but still have results to apply
-                        completed_streams=[s for s in self._completed_streams if s not in self._apply_results or self._apply_results[s] is None],
-                        stream_errors=self._omnata_log_handler.stream_global_errors,
-                        total_records_estimate=self._total_records_estimate
-                    )
-            if self._last_stream_progress_update is None or new_progress_update != self._last_stream_progress_update:
-                result = self._plugin_message(
-                    message=new_progress_update,
-                    ignore_errors=ignore_errors
+        with self._apply_results_lock:
+            new_progress_update = PluginMessageStreamProgressUpdate(
+                    stream_total_counts=self._stream_record_counts,
+                    # records could have been marked as completed, but still have results to apply
+                    completed_streams=[s for s in self._completed_streams 
+                        if s not in self._apply_results 
+                            or self._apply_results[s] is None
+                            or len(self._apply_results[s]) == 0],
+                    stream_errors=self._omnata_log_handler.stream_global_errors,
+                    total_records_estimate=self._total_records_estimate
                 )
-                if result is None:
-                    return False
-                self._last_stream_progress_update = new_progress_update
+        if self._last_stream_progress_update is None or new_progress_update != self._last_stream_progress_update:
+            result = self._plugin_message(
+                message=new_progress_update,
+                ignore_errors=ignore_errors
+            )
+            if result is None:
+                return False
+            self._last_stream_progress_update = new_progress_update
+        completed_streams_awaiting_results_upload = [
+            s for s in self._completed_streams if s in self._apply_results and self._apply_results[s] is not None
+        ]
+        if len(completed_streams_awaiting_results_upload) > 0:
+            logger.debug(
+                f"Streams marked as completed but awaiting upload: {', '.join(completed_streams_awaiting_results_upload)}"
+            )
         return True
 
     def apply_cancellation(self):
@@ -1224,9 +1285,9 @@ class InboundSyncRequest(SyncRequest):
             if stream_name in self._apply_results:
                 existing_results = self._apply_results[stream_name]
             existing_results.append(self._preprocess_results_list(stream_name, results, is_delete))
+            if new_state is not None:
+                existing_results.append(new_state)  # append the new state at the end
             self._apply_results[stream_name] = existing_results
-            current_latest = self._latest_states or {}
-            self._latest_states = {**current_latest, **{stream_name: new_state}}
         # if the total size of all the dataframes exceeds 200MB, apply the results immediately
         # we'll use df.memory_usage(index=True) for this
         if self.development_mode is False:
@@ -1236,7 +1297,7 @@ class InboundSyncRequest(SyncRequest):
             # if the total exceeds 200MB, we apply the results immediately
             all_df_lists:List[List[pandas.DataFrame]] = list(self._apply_results.values())
             # flatten
-            all_dfs:List[pandas.DataFrame] = [x for sublist in all_df_lists for x in sublist]
+            all_dfs:List[pandas.DataFrame] = [x for sublist in all_df_lists for x in sublist if isinstance(x, pandas.DataFrame)]
             combined_length = sum([len(x) for x in all_dfs])
             # first, don't bother if the count is less than 10000, since it's unlikely to be even close
             if combined_length > 10000:
@@ -1286,7 +1347,7 @@ class InboundSyncRequest(SyncRequest):
             combined_length = sum([len(x) for x in all_dfs])
             # first, don't both if the count is less than 10000, since it's unlikely to be even close
             if combined_length > 10000:
-                if sum([x.memory_usage(index=True).sum() for x in all_dfs]) > 200000000:
+                if sum([x.memory_usage(index=True).sum() for x in all_dfs if isinstance(x, pandas.DataFrame)]) > 200000000:
                     logger.debug(f"Applying criteria deletes queue immediately due to combined dataframe size")
                     self.apply_results_queue() 
 
@@ -1295,9 +1356,11 @@ class InboundSyncRequest(SyncRequest):
         Marks a stream as completed, this is called automatically per stream when using @managed_inbound_processing.
         If @managed_inbound_processing is not used, call this whenever a stream has finished recieving records.
         """
-        self._completed_streams.append(stream_name)
-        # dedup just in case it's called twice
-        self._completed_streams = list(set(self._completed_streams))
+        logger.debug(f"Marking stream {stream_name} as completed locally")
+        with self._apply_results_lock:
+            self._completed_streams.append(stream_name)
+            # dedup just in case it's called twice
+            self._completed_streams = list(set(self._completed_streams))
     
     def set_stream_record_count(self, stream_name: str, count: int):
         """
@@ -1321,9 +1384,42 @@ class InboundSyncRequest(SyncRequest):
         instead you should store state using the new_state parameter in the enqueue_results
         method to ensure it's applied along with the associated new records.
         """
+        self.enqueue_state(
+            stream_name=stream_name,
+            new_state=new_state,
+            query_id=None  # query_id will be generated automatically if not provided
+        )
+    
+    def enqueue_state(self, stream_name: str, new_state: Any, query_id: Optional[str] = None):
+        """
+        Enqueues some new stream state to be stored. This method should be called whenever the state of a stream changes.
+        
+        If there have been records enqueued here for this stream, it is assumed that the state is related to those records.
+        In this case, the state will be applied after the records are applied.
+        If there are no records enqueued for this stream, the state will be applied immediately as it is assumed that the results
+        were directly inserted, and therefore we need to capture the current query ID before more results are inserted.
+        """
         with self._apply_results_lock:
-            current_latest = self._latest_states or {}
-            self._latest_states = {**current_latest, **{stream_name: new_state}}
+            if stream_name in self._apply_results:
+                if len(self._apply_results[stream_name]) > 0:
+                    self._apply_results[stream_name].append(new_state)
+                    return
+        
+        self._directly_insert_to_state_register(
+            {
+                stream_name: new_state
+            }, query_id=query_id
+        )
+        
+
+    def _get_query_id_for_now(self):
+        """
+        Gets a Snowflake query ID right now. Note that this does not require a Snowflake lock, the caller 
+        should ensure that this is called in a thread-safe manner.
+        """
+        job=self._session.sql("select 1").collect_nowait()
+        job.result()
+        return job.query_id
 
     def get_queued_results(self, stream_name: str):
         """
@@ -1337,7 +1433,7 @@ class InboundSyncRequest(SyncRequest):
                 "get_queued_results was called, but no results have been queued"
             )
         concat_results = pandas.concat(self._apply_results[stream_name])
-        return concat_results
+        return [c for c in concat_results if c is not None and isinstance(c, pandas.DataFrame) and len(c) > 0]
 
     def _convert_by_json_schema(
         self, stream_name: str, data: Dict, json_schema: Dict
@@ -1512,10 +1608,11 @@ class InboundSyncRequest(SyncRequest):
         hash_object = hashlib.sha256(key_string.encode())
         return hash_object.hexdigest()
 
-    def _apply_results_dataframe(self, stream_names: List[str], results_df: pandas.DataFrame):
+    def _apply_results_dataframe(self, stream_names: List[str], results_df: pandas.DataFrame) -> Optional[str]:
         """
         Applies results for an inbound sync. The results are staged into a temporary
         table in Snowflake, so that we can make an atomic commit at the end.
+        Returns a query ID that can be used for checkpointing after the copy into command has run.
         """
         if len(results_df) > 0:
             with self._snowflake_query_lock:
@@ -1538,6 +1635,7 @@ class InboundSyncRequest(SyncRequest):
                                 raise ValueError(
                                     f"Failed to write results to table {self._full_results_table_name}"
                                 )
+                            query_id = self._get_query_id_for_now()
                             logger.debug(
                                 f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
                             )
@@ -1550,19 +1648,10 @@ class InboundSyncRequest(SyncRequest):
                             #                                )
                             for stream_name in stream_names:
                                 self._results_exist[stream_name] = True
+                            return query_id
         else:
             logger.debug("Results dataframe is empty, not applying")
 
-    def _apply_latest_states(self):
-        """
-        Updates the SYNC table to have the latest stream states.
-        TODO: This should be done in concert with the results, revisit
-        """
-        if self._last_states_update is None or json.dumps(self._latest_states) != json.dumps(self._last_states_update):
-            self._last_states_update = json.loads(json.dumps(self._latest_states))
-            self._plugin_message(PluginMessageStreamState(stream_state=self._latest_states))
-        
-
     def _apply_criteria_deletes_dataframe(self, results_df: pandas.DataFrame):
         """
         Applies results for an inbound sync. The results are staged into a temporary

{omnata_plugin_runtime-0.10.33a297 → omnata_plugin_runtime-0.11.0}/src/omnata_plugin_runtime/plugin_entrypoints.py

@@ -250,7 +250,6 @@ class PluginEntrypoint:
                 self._plugin_instance._configuration_parameters = parameters
 
                 inbound_sync_request.update_activity("Invoking plugin")
-                logger.info(f"inbound sync request: {inbound_sync_request}")
                 # plugin_instance._inbound_sync_request = outbound_sync_request
                 with tracer.start_as_current_span("invoke_plugin"):
                     with HttpRateLimiting(inbound_sync_request, parameters):
@@ -283,6 +282,11 @@ class PluginEntrypoint:
                     # token is set. We throw it here as an error since that's currently how it flows back to the engine with a DELAYED state
                     raise DeadlineReachedException()
             finally:
+                # try to upload any remaining results
+                try:
+                    inbound_sync_request.apply_results_queue()
+                except Exception as e:
+                    logger.warning(f"Error uploading remaining results: {str(e)}", exc_info=True)
                 # cancel the thread so we don't leave anything hanging around and cop a nasty error
                 try:
                     inbound_sync_request._thread_cancellation_token.set()  # pylint: disable=protected-access