omnata-plugin-runtime 0.11.0a313__py3-none-any.whl → 0.11.1__py3-none-any.whl

omnata_plugin_runtime/json_schema.py

@@ -470,10 +470,20 @@ class SnowflakeViewPart(BaseModel):
             c.name_with_comment(binding_list) for c in self.columns
         ]
     
-    def cte_text(self,original_name: bool = False, include_only_columns:Optional[List[str]] = None) -> str:
+    def cte_text(self,original_name: bool = False,
+            include_only_columns:Optional[List[str]] = None,
+            include_extra_columns:Optional[List[str]] = None
+            ) -> str:
         """
         Returns the CTE text for this view part.
         """
+        if include_extra_columns is not None:
+            # includes direct columns plus any extra specified
+            return f""" "{self.stream_name}" as (
+    select {', '.join([c.definition(original_name=original_name,remove_stream_prefix=self.stream_name) for c in self.columns
+                       if c.original_name in include_extra_columns or not c.is_join_column])} 
+    from {self.raw_table_location.get_fully_qualified_name()}
+) """
         if include_only_columns is None:
             return f""" "{self.stream_name}" as (
     select {', '.join([c.definition(original_name=original_name,remove_stream_prefix=self.stream_name) for c in self.direct_columns()])} 
@@ -504,6 +514,29 @@ class SnowflakeViewParts(BaseModel):
         ..., description="The other streams that are joined to the main stream"
     )
 
+    def column_indirectly_references_other_streams(
+        self,
+        all_view_parts:List[SnowflakeViewPart],
+        stream_name:str,column_name:str) -> bool:
+
+        for part in all_view_parts:
+            if part.stream_name == stream_name:
+                for col in part.columns:
+                    if col.original_name == column_name:
+                        if col.referenced_columns:
+                            for ref_stream, ref_cols in col.referenced_columns.items():
+                                if ref_stream != stream_name:
+                                    return True
+                                else:
+                                    # we have to call this recursively in case the referenced column also references other streams
+                                    result = any(
+                                        self.column_indirectly_references_other_streams(
+                                            all_view_parts, ref_stream, ref_col
+                                        ) for ref_col in ref_cols
+                                    )
+                                    return result
+        return False
+
     def view_body(self):
         """
         Creates a view definition from the parts.
@@ -519,31 +552,44 @@ class SnowflakeViewParts(BaseModel):
 
         # first, we need to collapse all referenced columns into a single map
         all_referenced_columns:Dict[str,List[str]] = {}
+        
+        # if a column references other columns, but there are no dependencies outside of its own stream, we can include those columns in the initial CTE for that stream
+        # because they can be calculated directly without needing joins
+        columns_only_referencing_own_stream:Dict[str,List[str]] = {}
+ 
         for part in [self.main_part] + self.joined_parts:
-            # if the main part references any columns in this part in its joins, we need to include those columns
+            # if the main part references any columns in this part in its joins, we need to include those columns because they are used in the join condition
             aliases_for_stream = [j.join_stream_alias for j in self.main_part.joins 
                 if j.join_stream_name == part.stream_name]
             columns_used_in_joins = [
                 j.left_column for j in self.main_part.joins if j.left_alias in aliases_for_stream
             ]
-            if part.stream_name not in all_referenced_columns:
-                all_referenced_columns[part.stream_name] = []
-            all_referenced_columns[part.stream_name] += columns_used_in_joins
+            all_referenced_columns.setdefault(part.stream_name, []).extend(columns_used_in_joins)
+            # now, for each column in the part, if it references columns in other streams, we need to include those columns
             for column in part.columns:
                 if column.referenced_columns:
                     for stream_name, referenced_columns in column.referenced_columns.items():
-                        if stream_name not in all_referenced_columns:
-                            all_referenced_columns[stream_name] = []
-                        all_referenced_columns[stream_name] += referenced_columns
+                        aliases_for_referenced_stream = [j.join_stream_name for j in self.main_part.joins 
+                            if j.join_stream_alias == stream_name]
+                        all_referenced_columns.setdefault(stream_name, []).extend(referenced_columns)
+                        # the stream name could be an alias, so we need to check if it's one of the aliases for this part
+                        for stream_name_for_alias in aliases_for_referenced_stream:
+                            all_referenced_columns.setdefault(stream_name_for_alias, []).extend(referenced_columns)
+                        # populate columns_only_referencing_own_stream by following the chain of references until we reach a column that references another stream or has no references
+                        if self.column_indirectly_references_other_streams(
+                            [self.main_part] + self.joined_parts, part.stream_name, column.original_name
+                        ) == False:
+                            columns_only_referencing_own_stream.setdefault(part.stream_name, []).append(column.original_name)
+                else:
+                    # if the column has no references, it can be included in the initial CTE for its own stream
+                    columns_only_referencing_own_stream.setdefault(part.stream_name, []).append(column.original_name)
+            # if this part has joins to other streams, we need to include the join columns
             for join in part.joins:
-                if join.join_stream_name not in all_referenced_columns:
-                    all_referenced_columns[join.join_stream_name] = []
-                all_referenced_columns[join.join_stream_name].append(join.join_stream_column)
-                all_referenced_columns[part.stream_name].append(join.left_column)
-
-
+                all_referenced_columns.setdefault(join.join_stream_name, []).append(join.join_stream_column)
+                all_referenced_columns.setdefault(join.join_stream_alias, []).append(join.join_stream_column)
+                all_referenced_columns.setdefault(part.stream_name, []).append(join.left_column)
         ctes = [
-                self.main_part.cte_text(original_name=True)
+                self.main_part.cte_text(original_name=True,include_extra_columns=columns_only_referencing_own_stream.get(self.main_part.stream_name))
             ] + [
                 part.cte_text(original_name=True,include_only_columns=all_referenced_columns.get(part.stream_name)) 
             for part in joined_parts_deduped
@@ -553,9 +599,9 @@ class SnowflakeViewParts(BaseModel):
         final_cte = f""" OMNATA_FINAL_CTE as (
             select {', '.join(
             [
-                f'"{self.main_part.stream_name}"."{c.original_name}"' for c in self.main_part.direct_columns()
+                f'"{self.main_part.stream_name}"."{c.original_name}"' for c in self.main_part.columns if not c.is_join_column or c.original_name in columns_only_referencing_own_stream.get(self.main_part.stream_name,[])
             ]+[
-                c.definition(original_name=True) for c in self.main_part.join_columns()
+                c.definition(original_name=True) for c in self.main_part.columns if c.is_join_column and c.original_name not in columns_only_referencing_own_stream.get(self.main_part.stream_name,[])
             ])}
             from "{self.main_part.stream_name}" """
         if len(self.main_part.joins) > 0:
@@ -761,6 +807,25 @@ class SnowflakeViewParts(BaseModel):
 
         return cls(main_part=main_stream_view_part, joined_parts=joined_parts)
 
+
+# Helper function to find a view part by stream name
+def find_part(view_part: SnowflakeViewPart, joined_parts: List[SnowflakeViewPart], stream_name: str) -> Optional[SnowflakeViewPart]:
+    if stream_name == view_part.stream_name:
+        return view_part
+    for part in joined_parts:
+        if part.stream_name == stream_name:
+            return part
+    for join in view_part.joins:
+        if join.join_stream_alias == stream_name:
+            # this is the join, we need to find the actual stream
+            for part in joined_parts:
+                if part.stream_name == join.join_stream_name:
+                    return part
+            logger.warning(
+                f"Join alias {stream_name} maps to stream {join.join_stream_name}, but that stream is not in the joined parts"
+            )
+    return None
+
 def prune(view_part: SnowflakeViewPart, joined_parts: List[SnowflakeViewPart]) -> bool:
     """
     Prunes columns from view parts that reference fields that don't exist in the referenced streams.
@@ -773,12 +838,6 @@ def prune(view_part: SnowflakeViewPart, joined_parts: List[SnowflakeViewPart]) -
     Raises ValueError if a cyclic dependency is detected.
     """
     columns_removed = False
-    
-    # Helper function to find a view part by stream name
-    def find_part(stream_name: str) -> Optional[SnowflakeViewPart]:
-        if stream_name == view_part.stream_name:
-            return view_part
-        return next((p for p in joined_parts if p.stream_name == stream_name), None)
 
     # Helper function to check if a column should be kept or removed
     def should_keep_column(column: SnowflakeViewColumn, part: SnowflakeViewPart) -> bool:
@@ -793,7 +852,7 @@ def prune(view_part: SnowflakeViewPart, joined_parts: List[SnowflakeViewPart]) -
         # Check each referenced stream and its fields
         for ref_stream_name, ref_fields in column.referenced_columns.items():
             # Find the referenced part
-            ref_part = find_part(ref_stream_name)
+            ref_part = find_part(view_part, joined_parts,ref_stream_name)
             
             # If referenced stream doesn't exist, remove the column
             if ref_part is None:

{omnata_plugin_runtime-0.11.0a313.dist-info → omnata_plugin_runtime-0.11.1.dist-info}/METADATA

@@ -1,7 +1,8 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: omnata-plugin-runtime
-Version: 0.11.0a313
+Version: 0.11.1
 Summary: Classes and common runtime components for building and running Omnata Plugins
+License-File: LICENSE
 Author: James Weakley
 Author-email: james.weakley@omnata.com
 Requires-Python: >=3.8,<=3.11

{omnata_plugin_runtime-0.11.0a313.dist-info → omnata_plugin_runtime-0.11.1.dist-info}/RECORD

@@ -2,12 +2,12 @@ omnata_plugin_runtime/__init__.py,sha256=MS9d1whnfT_B3-ThqZ7l63QeC_8OEKTuaYV5wTw
 omnata_plugin_runtime/api.py,sha256=5gbjbnFy72Xjf0E3kbG23G0V2J3CorvD5kpBn_BkdlI,8084
 omnata_plugin_runtime/configuration.py,sha256=SffokJfgvy6V3kUsoEjXcK3GdNgHo6U3mgBEs0qBv4I,46972
 omnata_plugin_runtime/forms.py,sha256=Lrbr3otsFDrvHWJw7v-slsW4PvEHJ6BG1Yl8oaJfiDo,20529
-omnata_plugin_runtime/json_schema.py,sha256=xI5YeoxdV1e4fYLUYB-1ZGXZrLAIrSnKj91189wf8cQ,49943
+omnata_plugin_runtime/json_schema.py,sha256=_3oLravvrbIM8dy-bOZSON22Di1FvNnDAsiehwfoH-E,54125
 omnata_plugin_runtime/logging.py,sha256=WBuZt8lF9E5oFWM4KYQbE8dDJ_HctJ1pN3BHwU6rcd0,4461
 omnata_plugin_runtime/omnata_plugin.py,sha256=PV1BzsqI2yi3pd6DBSRc1s1QJ_My-b6pewrB7ad4sms,140256
 omnata_plugin_runtime/plugin_entrypoints.py,sha256=_1pDLov3iQorGmfcae8Sw2bVjxw1vYeowBaKKNzRclQ,32629
 omnata_plugin_runtime/rate_limiting.py,sha256=qpr5esU4Ks8hMzuMpSR3gLFdor2ZUXYWCjmsQH_K6lQ,25882
-omnata_plugin_runtime-0.11.0a313.dist-info/LICENSE,sha256=rGaMQG3R3F5-JGDp_-rlMKpDIkg5n0SI4kctTk8eZSI,56
-omnata_plugin_runtime-0.11.0a313.dist-info/METADATA,sha256=cKUI7DscZ7fka1sfOEUJ0T9p3usHMohmiuBc9TPVM-E,2211
-omnata_plugin_runtime-0.11.0a313.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-omnata_plugin_runtime-0.11.0a313.dist-info/RECORD,,
+omnata_plugin_runtime-0.11.1.dist-info/METADATA,sha256=d040ESGm3ElaDUf4pdLsIsXE21J5h2xuwDHwbpnbrNg,2229
+omnata_plugin_runtime-0.11.1.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+omnata_plugin_runtime-0.11.1.dist-info/licenses/LICENSE,sha256=rGaMQG3R3F5-JGDp_-rlMKpDIkg5n0SI4kctTk8eZSI,56
+omnata_plugin_runtime-0.11.1.dist-info/RECORD,,

{omnata_plugin_runtime-0.11.0a313.dist-info → omnata_plugin_runtime-0.11.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 2.2.1
 Root-Is-Purelib: true
 Tag: py3-none-any