omnata-plugin-runtime 0.11.0a310__tar.gz → 0.11.0a312__tar.gz

{omnata_plugin_runtime-0.11.0a310 → omnata_plugin_runtime-0.11.0a312}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: omnata-plugin-runtime
-Version: 0.11.0a310
+Version: 0.11.0a312
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.11.0a310 → omnata_plugin_runtime-0.11.0a312}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "omnata-plugin-runtime"
-version = "0.11.0-a310"
+version = "0.11.0-a312"
 description = "Classes and common runtime components for building and running Omnata Plugins"
 authors = ["James Weakley <james.weakley@omnata.com>"]
 readme = "README.md"

{omnata_plugin_runtime-0.11.0a310 → omnata_plugin_runtime-0.11.0a312}/src/omnata_plugin_runtime/json_schema.py

@@ -470,10 +470,20 @@ class SnowflakeViewPart(BaseModel):
             c.name_with_comment(binding_list) for c in self.columns
         ]
     
-    def cte_text(self,original_name: bool = False, include_only_columns:Optional[List[str]] = None) -> str:
+    def cte_text(self,original_name: bool = False,
+            include_only_columns:Optional[List[str]] = None,
+            include_extra_columns:Optional[List[str]] = None
+            ) -> str:
         """
         Returns the CTE text for this view part.
         """
+        if include_extra_columns is not None:
+            # includes direct columns plus any extra specified
+            return f""" "{self.stream_name}" as (
+    select {', '.join([c.definition(original_name=original_name,remove_stream_prefix=self.stream_name) for c in self.columns
+                       if c.original_name in include_extra_columns or not c.is_join_column])} 
+    from {self.raw_table_location.get_fully_qualified_name()}
+) """
         if include_only_columns is None:
             return f""" "{self.stream_name}" as (
     select {', '.join([c.definition(original_name=original_name,remove_stream_prefix=self.stream_name) for c in self.direct_columns()])} 
@@ -543,7 +553,7 @@ class SnowflakeViewParts(BaseModel):
 
 
         ctes = [
-                self.main_part.cte_text(original_name=True)
+                self.main_part.cte_text(original_name=True,include_extra_columns=all_referenced_columns.get(self.main_part.stream_name))
             ] + [
                 part.cte_text(original_name=True,include_only_columns=all_referenced_columns.get(part.stream_name)) 
             for part in joined_parts_deduped

{omnata_plugin_runtime-0.11.0a310 → omnata_plugin_runtime-0.11.0a312}/src/omnata_plugin_runtime/omnata_plugin.py

@@ -1297,7 +1297,7 @@ class InboundSyncRequest(SyncRequest):
             # if the total exceeds 200MB, we apply the results immediately
             all_df_lists:List[List[pandas.DataFrame]] = list(self._apply_results.values())
             # flatten
-            all_dfs:List[pandas.DataFrame] = [x for sublist in all_df_lists for x in sublist]
+            all_dfs:List[pandas.DataFrame] = [x for sublist in all_df_lists for x in sublist if isinstance(x, pandas.DataFrame)]
             combined_length = sum([len(x) for x in all_dfs])
             # first, don't bother if the count is less than 10000, since it's unlikely to be even close
             if combined_length > 10000:
@@ -1347,7 +1347,7 @@ class InboundSyncRequest(SyncRequest):
             combined_length = sum([len(x) for x in all_dfs])
             # first, don't both if the count is less than 10000, since it's unlikely to be even close
             if combined_length > 10000:
-                if sum([x.memory_usage(index=True).sum() for x in all_dfs]) > 200000000:
+                if sum([x.memory_usage(index=True).sum() for x in all_dfs if isinstance(x, pandas.DataFrame)]) > 200000000:
                     logger.debug(f"Applying criteria deletes queue immediately due to combined dataframe size")
                     self.apply_results_queue()