omnata-plugin-runtime 0.10.3__py3-none-any.whl → 0.10.4__py3-none-any.whl

omnata_plugin_runtime/json_schema.py

@@ -7,6 +7,7 @@ from typing import Any, Dict, Optional, Literal, List, Union
 from typing_extensions import Self
 from pydantic import BaseModel, Field, model_validator, computed_field
 from jinja2 import Environment
+from .logging import logger
 
 class JsonSchemaProperty(BaseModel):
     """
@@ -48,6 +49,9 @@ class JsonSchemaProperty(BaseModel):
     isJoinColumn: Optional[bool] = Field(
         False, description="Whether this column is sourced from a joined stream"
     )
+    requiredStreamNames: Optional[List[str]] = Field(
+        None, description="The names of the streams that are depended upon by this column, via joins. If these streams are not selected, the column will be omitted."
+    )
 
     @model_validator(mode='after')
     def validate(self) -> Self:
@@ -155,6 +159,9 @@ class SnowflakeViewColumn(BaseModel):
     is_join_column: Optional[bool] = Field(
         default=False, description="Whether this column is sourced from a joined stream"
     )
+    required_stream_names: Optional[List[str]] = Field(
+        default=None, description="The names of the streams that are depended upon by this column, via joins. If these streams are not selected, the column will be omitted"
+    )
 
     def __repr__(self) -> str:
         return "SnowflakeViewColumn(name=%r, definition=%r, comment=%r)" % (
@@ -210,11 +217,15 @@ class SnowflakeViewColumn(BaseModel):
         else:
             if not json_schema_property.snowflakeColumnExpression:
                 expression=f"""{expression}::{json_schema_property.snowflake_data_type}"""
+        required_stream_names = None
+        if json_schema_property.requiredStreamNames:
+            required_stream_names = json_schema_property.requiredStreamNames
         return cls(
             name=final_column_name,
             expression=expression,
             comment=comment,
             is_join_column=json_schema_property.isJoinColumn,
+            required_stream_names=required_stream_names
         )
     
     @classmethod
@@ -435,11 +446,12 @@ class SnowflakeViewParts(BaseModel):
             column_name_expression=column_name_expression
         )
         joined_parts = []
+        # remove the joins from the main part if they are not in the raw stream locations
+        main_stream_view_part.joins = [join for join in main_stream_view_part.joins 
+                                       if join.join_stream_name in raw_stream_locations
+                                       and join.join_stream_name in stream_schemas]
+
         for join in main_stream_view_part.joins:
-            if join.join_stream_name not in raw_stream_locations:
-                raise ValueError(f"Stream {join.join_stream_name} is required as a join for stream {stream_name}, but its location was not provided")
-            if join.join_stream_name not in stream_schemas:
-                raise ValueError(f"Stream {join.join_stream_name} is required as a join for stream {stream_name}, but its schema was not provided")
             joined_parts.append(normalized_view_part(
                 stream_name=join.join_stream_name,
                 raw_table_location=raw_stream_locations[join.join_stream_name],
@@ -448,6 +460,18 @@ class SnowflakeViewParts(BaseModel):
                 column_name_environment=column_name_environment,
                 column_name_expression=column_name_expression
             ))
+        # For each column, the plugin can advise which streams are required for the join, which comes through as required_stream_names
+        # on the SnowflakeViewColumn object.
+        # Until this generate function is called with the raw stream names, we don't know which streams the user has actually selected.
+        # So now there's a pruning process where we remove columns from the main view part that depend on streams that are not selected
+        for column in main_stream_view_part.columns:
+            if column.required_stream_names:
+                for required_stream_name in column.required_stream_names:
+                    if required_stream_name not in raw_stream_locations:
+                        logger.warning(f"Column {column.name} in stream {stream_name} requires stream {required_stream_name} to be selected, but it was not provided")
+                        main_stream_view_part.columns.remove(column)
+                        break
+
         return cls(main_part=main_stream_view_part, joined_parts=joined_parts)
 
     

{omnata_plugin_runtime-0.10.3.dist-info → omnata_plugin_runtime-0.10.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: omnata-plugin-runtime
-Version: 0.10.3
+Version: 0.10.4
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.10.3.dist-info → omnata_plugin_runtime-0.10.4.dist-info}/RECORD

@@ -2,12 +2,12 @@ omnata_plugin_runtime/__init__.py,sha256=MS9d1whnfT_B3-ThqZ7l63QeC_8OEKTuaYV5wTw
 omnata_plugin_runtime/api.py,sha256=baGraSMiD4Yvi3ZWrEv_TKh8Ktd1U8riBdOpe9j0Puw,8202
 omnata_plugin_runtime/configuration.py,sha256=f4ssL8ckqlG6vrUZEaX5SvirS4Tdrstl66ZZxVzBado,42457
 omnata_plugin_runtime/forms.py,sha256=9YHJ_T17lT-rwyDaUg_0yj_YMPda4DRCw_wrvf8hE0E,19964
-omnata_plugin_runtime/json_schema.py,sha256=7nsluzRHn5gmlV4V2G1ostOBstUuIbacBbQr3WsqGcU,26795
+omnata_plugin_runtime/json_schema.py,sha256=NujSQBtyfnvSyw13sSFiSKmZeXkSk5cqCDVyuNwOACg,28395
 omnata_plugin_runtime/logging.py,sha256=WBuZt8lF9E5oFWM4KYQbE8dDJ_HctJ1pN3BHwU6rcd0,4461
 omnata_plugin_runtime/omnata_plugin.py,sha256=M0b6f9lKKEoEI0zf-ZwZcIPKPQTmHTIMhvcrBc94Mhg,133278
 omnata_plugin_runtime/plugin_entrypoints.py,sha256=iqGl8_nEEnPGKg3Aem4YLSQ6d5xS3ju5gq8MJbx6sCA,31968
 omnata_plugin_runtime/rate_limiting.py,sha256=qpr5esU4Ks8hMzuMpSR3gLFdor2ZUXYWCjmsQH_K6lQ,25882
-omnata_plugin_runtime-0.10.3.dist-info/LICENSE,sha256=rGaMQG3R3F5-JGDp_-rlMKpDIkg5n0SI4kctTk8eZSI,56
-omnata_plugin_runtime-0.10.3.dist-info/METADATA,sha256=ugqO2OoACEKX2BUe-WuZ5qePhtV5W-OB6hdyar15ZVU,2207
-omnata_plugin_runtime-0.10.3.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-omnata_plugin_runtime-0.10.3.dist-info/RECORD,,
+omnata_plugin_runtime-0.10.4.dist-info/LICENSE,sha256=rGaMQG3R3F5-JGDp_-rlMKpDIkg5n0SI4kctTk8eZSI,56
+omnata_plugin_runtime-0.10.4.dist-info/METADATA,sha256=qnnOuM4eS9eMC7eXXOaYJ5hM_iVVZk1L4iMlDJsTTi4,2207
+omnata_plugin_runtime-0.10.4.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+omnata_plugin_runtime-0.10.4.dist-info/RECORD,,