omnata-plugin-runtime 0.10.4a230__py3-none-any.whl → 0.10.5a233__py3-none-any.whl

omnata_plugin_runtime/configuration.py

@@ -50,6 +50,20 @@ class InboundSyncStrategy(str, Enum):
     INCREMENTAL = "Incremental"
     AUTO = "Auto" # not valid as an individual stream choice, only when referring to a forward preference (new_stream_sync_strategy)
 
+class InboundSyncStrategyBulkConfiguration(str, Enum):
+    """
+    Supercedes InboundSyncBulkConfiguration, which was a combination of sync strategy and storage behaviour.
+    """
+    AUTO = "auto"
+    MANUAL = "manual"
+
+class InboundStorageBehaviourBulkConfiguration(str, Enum):
+    """
+    Supercedes InboundSyncBulkConfiguration, which was a combination of sync strategy and storage behaviour.
+    """
+    MERGE = "merge"
+    APPEND = "append"
+    MANUAL = "manual"
 
 class InboundSyncBulkConfiguration(str, Enum):
     """
@@ -479,7 +493,52 @@ class InboundSyncStreamsConfiguration(SubscriptableBaseModel):
     new_stream_storage_behaviour: Optional[InboundStorageBehaviour] = Field(default=None)
     included_streams: Dict[str, StoredStreamConfiguration]
     excluded_streams: List[str]
-    bulk_configuration: InboundSyncBulkConfiguration = InboundSyncBulkConfiguration.CUSTOMIZE
+    bulk_configuration: Optional[InboundSyncBulkConfiguration] = None
+    sync_strategy_bulk_configuration: InboundSyncStrategyBulkConfiguration
+    storage_behaviour_bulk_configuration: InboundStorageBehaviourBulkConfiguration
+
+    """
+    We use a model_validator to manage the migration from bulk_configuration to the new
+    sync_strategy_bulk_configuration and storage_behaviour_bulk_configuration objects.
+    This means that if bulk_configuration is set, it will be used to populate the new objects.
+    """
+    @model_validator(mode='before')
+    @classmethod
+    def migrate_bulk_configuration(cls, data: Any) -> Any:
+        if 'bulk_configuration' in data:
+            # Recently deprecated values
+            # AUTO_MERGE = "Auto / Merge changes" # Merge
+            # AUTO_APPEND = "Auto / Keep history"  # Append
+            # CUSTOMIZE = "Customize - choose sync behaviours for each object"
+            # Deprecated longer ago
+            # ALL_FULL_REPLACE = "All Objects - Full refresh / Replace"
+            # ALL_FULL_APPEND = "All Objects - Full refresh / Append"
+            # ALL_INCR_MERGE = "All Objects - Incremental / Merge"
+            # ALL_INCR_APPEND = "All Objects - Incremental / Append"
+            if data['bulk_configuration']== InboundSyncBulkConfiguration.AUTO_MERGE.value:
+                data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.AUTO.value
+                data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.MERGE.value
+            elif data['bulk_configuration']== InboundSyncBulkConfiguration.AUTO_APPEND.value:
+                data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.AUTO.value
+                data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.APPEND.value
+            elif data['bulk_configuration']== InboundSyncBulkConfiguration.CUSTOMIZE.value:
+                data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.MANUAL.value
+                data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.MANUAL.value
+            elif data['bulk_configuration']== InboundSyncBulkConfiguration.ALL_FULL_REPLACE.value:
+                data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.MANUAL.value
+                data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.MERGE.value
+            elif data['bulk_configuration']== InboundSyncBulkConfiguration.ALL_FULL_APPEND.value:
+                data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.MANUAL.value
+                data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.APPEND.value
+            elif data['bulk_configuration']== InboundSyncBulkConfiguration.ALL_INCR_MERGE.value:
+                data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.AUTO.value
+                data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.MERGE.value
+            elif data['bulk_configuration']== InboundSyncBulkConfiguration.ALL_INCR_APPEND.value:
+                data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.AUTO.value
+                data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.APPEND.value
+            # remove this once the old UI is deprecated
+            #del data['bulk_configuration']
+        return data
 
 
 class StoredStreamConfiguration(SubscriptableBaseModel):

omnata_plugin_runtime/json_schema.py

@@ -446,11 +446,12 @@ class SnowflakeViewParts(BaseModel):
             column_name_expression=column_name_expression
         )
         joined_parts = []
+        # remove the joins from the main part if they are not in the raw stream locations
+        main_stream_view_part.joins = [join for join in main_stream_view_part.joins 
+                                       if join.join_stream_name in raw_stream_locations
+                                       and join.join_stream_name in stream_schemas]
+
         for join in main_stream_view_part.joins:
-            if join.join_stream_name not in raw_stream_locations:
-                continue
-            if join.join_stream_name not in stream_schemas:
-                continue
             joined_parts.append(normalized_view_part(
                 stream_name=join.join_stream_name,
                 raw_table_location=raw_stream_locations[join.join_stream_name],

{omnata_plugin_runtime-0.10.4a230.dist-info → omnata_plugin_runtime-0.10.5a233.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: omnata-plugin-runtime
-Version: 0.10.4a230
+Version: 0.10.5a233
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.10.4a230.dist-info → omnata_plugin_runtime-0.10.5a233.dist-info}/RECORD

@@ -1,13 +1,13 @@
 omnata_plugin_runtime/__init__.py,sha256=MS9d1whnfT_B3-ThqZ7l63QeC_8OEKTuaYV5wTwRpBA,1576
 omnata_plugin_runtime/api.py,sha256=baGraSMiD4Yvi3ZWrEv_TKh8Ktd1U8riBdOpe9j0Puw,8202
-omnata_plugin_runtime/configuration.py,sha256=f4ssL8ckqlG6vrUZEaX5SvirS4Tdrstl66ZZxVzBado,42457
+omnata_plugin_runtime/configuration.py,sha256=8rt_NW0wlJDNbZGkYjz-vKKP3JzIP6HFkgeuG_fRf_Y,46398
 omnata_plugin_runtime/forms.py,sha256=9YHJ_T17lT-rwyDaUg_0yj_YMPda4DRCw_wrvf8hE0E,19964
-omnata_plugin_runtime/json_schema.py,sha256=iBeOo-D7bFQQ6aIFOtkNYYx16GEFozr9-iHxjAJ16FY,28223
+omnata_plugin_runtime/json_schema.py,sha256=NujSQBtyfnvSyw13sSFiSKmZeXkSk5cqCDVyuNwOACg,28395
 omnata_plugin_runtime/logging.py,sha256=WBuZt8lF9E5oFWM4KYQbE8dDJ_HctJ1pN3BHwU6rcd0,4461
 omnata_plugin_runtime/omnata_plugin.py,sha256=M0b6f9lKKEoEI0zf-ZwZcIPKPQTmHTIMhvcrBc94Mhg,133278
 omnata_plugin_runtime/plugin_entrypoints.py,sha256=iqGl8_nEEnPGKg3Aem4YLSQ6d5xS3ju5gq8MJbx6sCA,31968
 omnata_plugin_runtime/rate_limiting.py,sha256=qpr5esU4Ks8hMzuMpSR3gLFdor2ZUXYWCjmsQH_K6lQ,25882
-omnata_plugin_runtime-0.10.4a230.dist-info/LICENSE,sha256=rGaMQG3R3F5-JGDp_-rlMKpDIkg5n0SI4kctTk8eZSI,56
-omnata_plugin_runtime-0.10.4a230.dist-info/METADATA,sha256=o59eFbaituLMTRsAiEnxjx5erzQbydFFwi9Fb7wMB7k,2211
-omnata_plugin_runtime-0.10.4a230.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-omnata_plugin_runtime-0.10.4a230.dist-info/RECORD,,
+omnata_plugin_runtime-0.10.5a233.dist-info/LICENSE,sha256=rGaMQG3R3F5-JGDp_-rlMKpDIkg5n0SI4kctTk8eZSI,56
+omnata_plugin_runtime-0.10.5a233.dist-info/METADATA,sha256=7g-buTqGgenl_DcE7T9d4Es8juGY79-XlI5Ixmhhip4,2211
+omnata_plugin_runtime-0.10.5a233.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+omnata_plugin_runtime-0.10.5a233.dist-info/RECORD,,