omnata-plugin-runtime 0.10.12a263__tar.gz → 0.10.13a266__tar.gz

{omnata_plugin_runtime-0.10.12a263 → omnata_plugin_runtime-0.10.13a266}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: omnata-plugin-runtime
-Version: 0.10.12a263
+Version: 0.10.13a266
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.10.12a263 → omnata_plugin_runtime-0.10.13a266}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "omnata-plugin-runtime"
-version = "0.10.12-a263"
+version = "0.10.13-a266"
 description = "Classes and common runtime components for building and running Omnata Plugins"
 authors = ["James Weakley <james.weakley@omnata.com>"]
 readme = "README.md"

{omnata_plugin_runtime-0.10.12a263 → omnata_plugin_runtime-0.10.13a266}/src/omnata_plugin_runtime/json_schema.py

@@ -270,7 +270,9 @@ class SnowflakeViewColumn(BaseModel):
         This is allowed in Snowflake, as long as the aliased column is defined before it's used in a later column
         So we need to sort the columns so that if the name of the column appears (in quotes) in the expression of another column, it is ordered first
         """
-        
+        logger.debug(
+            f"Ordering columns by reference for stream: {current_stream_name} ({len(columns)} columns)"
+        )
         # Collect columns to be moved
         columns_to_move:List[Self] = []
         # Collect Omnata System columns and keep them at the front
@@ -562,10 +564,32 @@ class SnowflakeViewParts(BaseModel):
         # Until this generate function is called with the raw stream names, we don't know which streams the user has actually selected, nor which
         # fields are actually available (some may be dropped due to something like an unsupported formula).
         # So now there's a pruning process where we remove columns that reference fields that are not available.
-        # We'll start by doing a first pass and removing unavailable columns from other streams
+        # First, explicitly check for circular references between tables, erroring if they are found.
+        circular_refs = {}
+        for part in [main_stream_view_part] + joined_parts:
+            for column in part.columns:
+                if column.referenced_columns:
+                    for ref_stream_name, ref_fields in column.referenced_columns.items():
+                        # Record this reference
+                        if (part.stream_name, ref_stream_name) not in circular_refs:
+                            circular_refs[(part.stream_name, ref_stream_name)] = []
+                        circular_refs[(part.stream_name, ref_stream_name)].append((column.original_name, ref_fields))
+        
+        # Check for circular references
+        for (stream1, stream2), refs1 in circular_refs.items():
+            if (stream2, stream1) in circular_refs:
+                # Found a potential circular reference between stream1 and stream2
+                refs2 = circular_refs[(stream2, stream1)]
+                raise ValueError(f"""Cyclic dependency detected: Circular reference between {stream1} and {stream2}.
+{stream1} -> {stream2}: {refs1}
+{stream2} -> {stream1}: {refs2}""")
+        
+        # Now proceed with the actual pruning process
+        # First, removing unavailable columns from other streams
         # then, we can do a final pass and remove columns that reference fields that are not available in the current stream
+
         prune_count = 0
-        while prune(main_stream_view_part,joined_parts):
+        while prune(main_stream_view_part, joined_parts):
             prune_count += 1
             if prune_count > 10000:
                 raise ValueError("Pruning of columns from the view has entered an infinite loop")
@@ -574,37 +598,75 @@ class SnowflakeViewParts(BaseModel):
 
 def prune(view_part: SnowflakeViewPart, joined_parts: List[SnowflakeViewPart]) -> bool:
     """
-    Iteratively prunes columns from the main view part that reference fields
-    that are not available in the joined parts.
+    Prunes columns from view parts that reference fields that don't exist in the referenced streams.
+    
+    This function handles:
+    1. Direct dependencies - removing columns that directly reference non-existent columns
+    2. Transitive dependencies - removing columns that depend on columns that were removed
+    
+    Returns True if any columns were removed, False otherwise.
+    Raises ValueError if a cyclic dependency is detected.
     """
-    stack = [(view_part, joined_parts)]
     columns_removed = False
-
-    while stack:
-        current_part, current_joined_parts = stack.pop()
-        for column in current_part.columns[:]:  # Iterate over a copy to allow safe removal
-            if column.referenced_columns:
-                for referenced_stream_name, referenced_fields in column.referenced_columns.items():
-                    if referenced_stream_name == current_part.stream_name:
-                        part = current_part
-                    else:
-                        part = next((p for p in current_joined_parts if p.stream_name == referenced_stream_name), None)
-                        if part is None:
-                            logger.warning(f"Column {column.name} in stream {current_part.stream_name} references stream {referenced_stream_name}, but it was not provided")
-                            current_part.columns.remove(column)
-                            columns_removed = True
-                            break
-
-                    columns_missing_from_join = part.columns_missing(referenced_fields)
-                    if columns_missing_from_join:
-                        logger.warning(f"Column {column.name} in stream {current_part.stream_name} references fields {columns_missing_from_join} in stream {referenced_stream_name}, but they were not provided")
-                        current_part.columns.remove(column)
-                        columns_removed = True
-                        break
-                    else:
-                        if part != current_part:
-                            stack.append((part, current_joined_parts))
-
+    
+    # Helper function to find a view part by stream name
+    def find_part(stream_name: str) -> Optional[SnowflakeViewPart]:
+        if stream_name == view_part.stream_name:
+            return view_part
+        return next((p for p in joined_parts if p.stream_name == stream_name), None)
+
+    # Helper function to check if a column should be kept or removed
+    def should_keep_column(column: SnowflakeViewColumn, part: SnowflakeViewPart) -> bool:
+        """
+        Checks if a column should be kept based on its dependencies.
+        Returns True if the column should be kept, False if it should be removed.
+        """
+        # If no references, keep the column
+        if not column.referenced_columns:
+            return True
+            
+        # Check each referenced stream and its fields
+        for ref_stream_name, ref_fields in column.referenced_columns.items():
+            # Find the referenced part
+            ref_part = find_part(ref_stream_name)
+            
+            # If referenced stream doesn't exist, remove the column
+            if ref_part is None:
+                logger.warning(
+                    f"Column {column.name} in stream {part.stream_name} references stream "
+                    f"{ref_stream_name}, but it was not provided"
+                )
+                return False
+                
+            # Check each referenced field
+            for ref_field in ref_fields:
+                # Find the referenced column
+                ref_column = next((c for c in ref_part.columns if c.original_name == ref_field), None)
+                
+                # If referenced column doesn't exist, remove the column
+                if ref_column is None:
+                    logger.warning(
+                        f"Column {column.name} in stream {part.stream_name} references field "
+                        f"{ref_field} in stream {ref_stream_name}, but it was not provided"
+                    )
+                    return False
+                    
+        # All dependencies are satisfied
+        return True
+    
+    # Process columns for removal
+    for column in view_part.columns[:]:  # Use a copy to allow safe removal
+        if not should_keep_column(column, view_part):
+            view_part.columns.remove(column)
+            columns_removed = True
+    
+    # Process joined parts
+    for joined_part in joined_parts:
+        for column in joined_part.columns[:]:  # Use a copy to allow safe removal
+            if not should_keep_column(column, joined_part):
+                joined_part.columns.remove(column)
+                columns_removed = True
+    
     return columns_removed
 
 class JsonSchemaTopLevel(BaseModel):
@@ -711,6 +773,9 @@ def normalized_view_part(
     - A list of SnowflakeViewColumn objects, representing the columns to create in the view
     - A list of SnowflakeViewJoin objects, representing the joins to create in the view
     """
+    logger.debug(
+        f"Building normalized view part for stream: {stream_name}"
+    )
     snowflake_columns: List[SnowflakeViewColumn] = []
     if include_default_columns:
         snowflake_columns.append(
@@ -757,6 +822,9 @@ def normalized_view_part(
     joins = []
     comment = None
     if stream_schema is not None:
+        logger.debug(
+            f"Building view columns for stream: {stream_name}"
+        )
         json_schema = JsonSchemaTopLevel.model_validate(stream_schema)
         view_columns += json_schema.build_view_columns(
             column_name_environment=column_name_environment,