omnata-plugin-runtime 0.10.8a255__py3-none-any.whl → 0.10.8a257__py3-none-any.whl

omnata_plugin_runtime/json_schema.py

@@ -423,6 +423,12 @@ class SnowflakeViewPart(BaseModel):
     select {', '.join([c.definition(original_name=original_name) for c in self.direct_columns()])} 
     from {self.raw_table_location.get_fully_qualified_name()}
 ) """
+    
+    def columns_missing(self,columns_to_check:List[str]) -> List[str]:
+        """
+        Returns a list of columns that are missing from the view part.
+        """
+        return [c for c in columns_to_check if c not in [c.original_name for c in self.columns]]
 
 class SnowflakeViewParts(BaseModel):
     """
@@ -494,7 +500,7 @@ class SnowflakeViewParts(BaseModel):
             column_name_environment=column_name_environment,
             column_name_expression=column_name_expression
         )
-        joined_parts = []
+        joined_parts:List[SnowflakeViewPart] = []
         # remove the joins from the main part if they are not in the raw stream locations
         main_stream_view_part.joins = [join for join in main_stream_view_part.joins 
                                        if join.join_stream_name in raw_stream_locations
@@ -509,21 +515,50 @@ class SnowflakeViewParts(BaseModel):
                 column_name_environment=column_name_environment,
                 column_name_expression=column_name_expression
             ))
-        # For each column, the plugin can advise which streams are required for the join, which comes through as required_stream_names
+        # For each column, the plugin can advise which fields (of the same stream or joined) are required for the join, which comes through as referenced_columns
         # on the SnowflakeViewColumn object.
-        # Until this generate function is called with the raw stream names, we don't know which streams the user has actually selected.
-        # So now there's a pruning process where we remove columns from the main view part that depend on streams that are not selected
-        for column in main_stream_view_part.columns:
-            if column.required_stream_names:
-                for required_stream_name in column.required_stream_names:
-                    if required_stream_name not in raw_stream_locations:
-                        logger.warning(f"Column {column.name} in stream {stream_name} requires stream {required_stream_name} to be selected, but it was not provided")
-                        main_stream_view_part.columns.remove(column)
-                        break
+        # Until this generate function is called with the raw stream names, we don't know which streams the user has actually selected, nor which
+        # fields are actually available (some may be dropped due to something like an unsupported formula).
+        # So now there's a pruning process where we remove columns that reference fields that are not available.
+        # We'll start by doing a first pass and removing unavailable columns from other streams
+        # then, we can do a final pass and remove columns that reference fields that are not available in the current stream
+        prune_count = 0
+        while prune(main_stream_view_part,joined_parts):
+            prune_count += 1
+            if prune_count > 10000:
+                raise ValueError("Pruning of columns from the view has entered an infinite loop")
 
         return cls(main_part=main_stream_view_part, joined_parts=joined_parts)
 
-    
+def prune(view_part:SnowflakeViewPart,joined_parts:List[SnowflakeViewPart]) -> bool:
+    """
+    Prunes columns from the main view part that reference fields that are not available in the joined parts.
+    Returns True if columns were removed, False otherwise.
+    """
+    for column in view_part.columns:
+        if column.referenced_columns:
+            for referenced_stream_name, referenced_fields in column.referenced_columns.items():
+                
+                if referenced_stream_name == view_part.stream_name:
+                    part = view_part
+                else:
+                    part = next((part for part in joined_parts if part.stream_name==referenced_stream_name),None)
+                    if part is None:
+                        logger.warning(f"Column {column.name} in stream {view_part.stream_name} references stream {referenced_stream_name}, but it was not provided")
+                        view_part.columns.remove(column)
+                        return True
+                
+                columns_missing_from_join = part.columns_missing(referenced_fields)
+                if len(columns_missing_from_join) > 0:
+                    logger.warning(f"Column {column.name} in stream {view_part.stream_name} references fields {columns_missing_from_join} in stream {referenced_stream_name}, but they were not provided")
+                    view_part.columns.remove(column)
+                    return True
+                else:
+                    # no columns were removed, but we need to check if the columns that are referenced are not themselves referencing other missing columns
+                    if part != view_part:
+                        return prune(part,joined_parts)
+
+    return False
 
 class JsonSchemaTopLevel(BaseModel):
     """

{omnata_plugin_runtime-0.10.8a255.dist-info → omnata_plugin_runtime-0.10.8a257.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: omnata-plugin-runtime
-Version: 0.10.8a255
+Version: 0.10.8a257
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.10.8a255.dist-info → omnata_plugin_runtime-0.10.8a257.dist-info}/RECORD

@@ -2,12 +2,12 @@ omnata_plugin_runtime/__init__.py,sha256=MS9d1whnfT_B3-ThqZ7l63QeC_8OEKTuaYV5wTw
 omnata_plugin_runtime/api.py,sha256=baGraSMiD4Yvi3ZWrEv_TKh8Ktd1U8riBdOpe9j0Puw,8202
 omnata_plugin_runtime/configuration.py,sha256=0rfIGv8rCu8OwL7m-VOXIBjd05iyaBWRdt2h9o6scwo,46754
 omnata_plugin_runtime/forms.py,sha256=9YHJ_T17lT-rwyDaUg_0yj_YMPda4DRCw_wrvf8hE0E,19964
-omnata_plugin_runtime/json_schema.py,sha256=EBTIVFXFXsrV9D1M6Dhp_IJ8tO9CtUeu8swWoX6xgjg,32063
+omnata_plugin_runtime/json_schema.py,sha256=6dLkOC_aeD0iwi1neCnjcwHlqj-_IfcTlVDUr-1den0,34158
 omnata_plugin_runtime/logging.py,sha256=WBuZt8lF9E5oFWM4KYQbE8dDJ_HctJ1pN3BHwU6rcd0,4461
 omnata_plugin_runtime/omnata_plugin.py,sha256=M0b6f9lKKEoEI0zf-ZwZcIPKPQTmHTIMhvcrBc94Mhg,133278
 omnata_plugin_runtime/plugin_entrypoints.py,sha256=iqGl8_nEEnPGKg3Aem4YLSQ6d5xS3ju5gq8MJbx6sCA,31968
 omnata_plugin_runtime/rate_limiting.py,sha256=qpr5esU4Ks8hMzuMpSR3gLFdor2ZUXYWCjmsQH_K6lQ,25882
-omnata_plugin_runtime-0.10.8a255.dist-info/LICENSE,sha256=rGaMQG3R3F5-JGDp_-rlMKpDIkg5n0SI4kctTk8eZSI,56
-omnata_plugin_runtime-0.10.8a255.dist-info/METADATA,sha256=GHsZU1A2Ad6KCV3l_YOzMtRjG_ywoEM1A8SE3SAWnHQ,2211
-omnata_plugin_runtime-0.10.8a255.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-omnata_plugin_runtime-0.10.8a255.dist-info/RECORD,,
+omnata_plugin_runtime-0.10.8a257.dist-info/LICENSE,sha256=rGaMQG3R3F5-JGDp_-rlMKpDIkg5n0SI4kctTk8eZSI,56
+omnata_plugin_runtime-0.10.8a257.dist-info/METADATA,sha256=kfPT1sT9Jhby1FyZuD6bgVchFaJjc-ayKB7TrCtH_Uc,2211
+omnata_plugin_runtime-0.10.8a257.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+omnata_plugin_runtime-0.10.8a257.dist-info/RECORD,,