omnata-plugin-runtime 0.10.6a246__tar.gz → 0.10.7__tar.gz

{omnata_plugin_runtime-0.10.6a246 → omnata_plugin_runtime-0.10.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: omnata-plugin-runtime
-Version: 0.10.6a246
+Version: 0.10.7
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.10.6a246 → omnata_plugin_runtime-0.10.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "omnata-plugin-runtime"
-version = "0.10.6-a246"
+version = "0.10.7"
 description = "Classes and common runtime components for building and running Omnata Plugins"
 authors = ["James Weakley <james.weakley@omnata.com>"]
 readme = "README.md"

{omnata_plugin_runtime-0.10.6a246 → omnata_plugin_runtime-0.10.7}/src/omnata_plugin_runtime/configuration.py

@@ -494,8 +494,8 @@ class InboundSyncStreamsConfiguration(SubscriptableBaseModel):
     included_streams: Dict[str, StoredStreamConfiguration]
     excluded_streams: List[str]
     bulk_configuration: Optional[InboundSyncBulkConfiguration] = None
-    sync_strategy_bulk_configuration: InboundSyncStrategyBulkConfiguration
-    storage_behaviour_bulk_configuration: InboundStorageBehaviourBulkConfiguration
+    sync_strategy_bulk_configuration: Optional[InboundSyncStrategyBulkConfiguration] = None
+    storage_behaviour_bulk_configuration: Optional[InboundStorageBehaviourBulkConfiguration] = None
 
     """
     We use a model_validator to manage the migration from bulk_configuration to the new
@@ -505,39 +505,44 @@ class InboundSyncStreamsConfiguration(SubscriptableBaseModel):
     @model_validator(mode='before')
     @classmethod
     def migrate_bulk_configuration(cls, data: Any) -> Any:
-        if 'sync_strategy_bulk_configuration' in data or 'storage_behaviour_bulk_configuration' in data:
+        if 'sync_strategy_bulk_configuration' in data \
+                and data['sync_strategy_bulk_configuration'] is not None \
+                and 'storage_behaviour_bulk_configuration' in data \
+                and data['storage_behaviour_bulk_configuration'] is not None:
             return data
-        if 'bulk_configuration' in data:
-            # Recently deprecated values
-            # AUTO_MERGE = "Auto / Merge changes" # Merge
-            # AUTO_APPEND = "Auto / Keep history"  # Append
-            # CUSTOMIZE = "Customize - choose sync behaviours for each object"
-            # Deprecated longer ago
-            # ALL_FULL_REPLACE = "All Objects - Full refresh / Replace"
-            # ALL_FULL_APPEND = "All Objects - Full refresh / Append"
-            # ALL_INCR_MERGE = "All Objects - Incremental / Merge"
-            # ALL_INCR_APPEND = "All Objects - Incremental / Append"
-            if data['bulk_configuration']== InboundSyncBulkConfiguration.AUTO_MERGE.value:
-                data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.AUTO.value
-                data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.MERGE.value
-            elif data['bulk_configuration']== InboundSyncBulkConfiguration.AUTO_APPEND.value:
-                data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.AUTO.value
-                data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.APPEND.value
-            elif data['bulk_configuration']== InboundSyncBulkConfiguration.CUSTOMIZE.value:
-                data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.MANUAL.value
-                data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.MANUAL.value
-            elif data['bulk_configuration']== InboundSyncBulkConfiguration.ALL_FULL_REPLACE.value:
-                data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.MANUAL.value
-                data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.MERGE.value
-            elif data['bulk_configuration']== InboundSyncBulkConfiguration.ALL_FULL_APPEND.value:
-                data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.MANUAL.value
-                data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.APPEND.value
-            elif data['bulk_configuration']== InboundSyncBulkConfiguration.ALL_INCR_MERGE.value:
-                data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.AUTO.value
-                data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.MERGE.value
-            elif data['bulk_configuration']== InboundSyncBulkConfiguration.ALL_INCR_APPEND.value:
-                data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.AUTO.value
-                data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.APPEND.value
+        if 'bulk_configuration' not in data:
+            # This was previously a default value
+            data['bulk_configuration'] = InboundSyncBulkConfiguration.CUSTOMIZE.value
+        # Recently deprecated values
+        # AUTO_MERGE = "Auto / Merge changes" # Merge
+        # AUTO_APPEND = "Auto / Keep history"  # Append
+        # CUSTOMIZE = "Customize - choose sync behaviours for each object"
+        # Deprecated longer ago
+        # ALL_FULL_REPLACE = "All Objects - Full refresh / Replace"
+        # ALL_FULL_APPEND = "All Objects - Full refresh / Append"
+        # ALL_INCR_MERGE = "All Objects - Incremental / Merge"
+        # ALL_INCR_APPEND = "All Objects - Incremental / Append"
+        if data['bulk_configuration']== InboundSyncBulkConfiguration.AUTO_MERGE.value:
+            data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.AUTO.value
+            data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.MERGE.value
+        elif data['bulk_configuration']== InboundSyncBulkConfiguration.AUTO_APPEND.value:
+            data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.AUTO.value
+            data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.APPEND.value
+        elif data['bulk_configuration']== InboundSyncBulkConfiguration.CUSTOMIZE.value:
+            data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.MANUAL.value
+            data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.MANUAL.value
+        elif data['bulk_configuration']== InboundSyncBulkConfiguration.ALL_FULL_REPLACE.value:
+            data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.MANUAL.value
+            data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.MERGE.value
+        elif data['bulk_configuration']== InboundSyncBulkConfiguration.ALL_FULL_APPEND.value:
+            data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.MANUAL.value
+            data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.APPEND.value
+        elif data['bulk_configuration']== InboundSyncBulkConfiguration.ALL_INCR_MERGE.value:
+            data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.AUTO.value
+            data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.MERGE.value
+        elif data['bulk_configuration']== InboundSyncBulkConfiguration.ALL_INCR_APPEND.value:
+            data['sync_strategy_bulk_configuration']= InboundSyncStrategyBulkConfiguration.AUTO.value
+            data['storage_behaviour_bulk_configuration']= InboundStorageBehaviourBulkConfiguration.APPEND.value
             # remove this once the old UI is deprecated
             #del data['bulk_configuration']
         return data

{omnata_plugin_runtime-0.10.6a246 → omnata_plugin_runtime-0.10.7}/src/omnata_plugin_runtime/json_schema.py

@@ -242,7 +242,7 @@ class SnowflakeViewColumn(BaseModel):
         )
     
     @classmethod
-    def order_by_reference(cls,join_columns:List[Self]) -> List[Self]:
+    def order_by_reference(cls,columns:List[Self]) -> List[Self]:
         """
         In some situations, column expressions may reference the alias of another column
         This is allowed in Snowflake, as long as the aliased column is defined before it's used in a later column
@@ -251,17 +251,26 @@ class SnowflakeViewColumn(BaseModel):
         
         # Collect columns to be moved
         columns_to_move:List[Self] = []
-        for column in join_columns:
-            for other_column in join_columns:
+        # Collect Omnata System columns and keep them at the front
+        omnata_system_columns_start = []
+        for column in columns[:]:
+            if column.original_name.startswith("OMNATA_"):
+                columns.remove(column)
+                omnata_system_columns_start.append(column)
+
+        for column in columns:
+            for other_column in columns:
+                if column==other_column:
+                    continue
                 if f'"{column.original_name}"' in other_column.expression:
                     if column not in columns_to_move:
                         columns_to_move.append(column)
         
         # Move collected columns to the front
         for column in columns_to_move:
-            join_columns.remove(column)
-            join_columns.insert(0, column)
-        return join_columns
+            columns.remove(column)
+            columns.insert(0, column)
+        return omnata_system_columns_start + columns
 
 
 class SnowflakeViewJoin(BaseModel):
@@ -369,13 +378,13 @@ class SnowflakeViewPart(BaseModel):
         """
         Returns the columns that are not sourced from joins.
         """
-        return SnowflakeViewColumn.order_by_reference([c for c in self.columns if not c.is_join_column])
+        return [c for c in self.columns if not c.is_join_column]
 
     def join_columns(self) -> List[SnowflakeViewColumn]:
         """
         Returns the columns that are sourced from joins.
         """
-        return SnowflakeViewColumn.order_by_reference([c for c in self.columns if c.is_join_column])
+        return [c for c in self.columns if c.is_join_column]
 
     def comment_clause(self) -> str:
         """
@@ -393,7 +402,7 @@ class SnowflakeViewPart(BaseModel):
         # the outer view definition has all of the column names and comments, but with the direct columns
         # first and the join columns last, same as they are ordered in the inner query
         return [
-            c.name_with_comment(binding_list) for c in (self.direct_columns() + self.join_columns())
+            c.name_with_comment(binding_list) for c in self.columns
         ]
     
     def cte_text(self,original_name: bool = False) -> str:
@@ -431,25 +440,26 @@ class SnowflakeViewParts(BaseModel):
         final_cte = f""" OMNATA_FINAL_CTE as (
             select {', '.join(
             [
-                f'"{c.original_name}"' for c in self.main_part.direct_columns()
+                f'"{self.main_part.stream_name}"."{c.original_name}"' for c in self.main_part.direct_columns()
             ]+[
                 c.definition(original_name=True) for c in self.main_part.join_columns()
             ])}
-            from "{self.main_part.stream_name}"
-        ) """
+            from "{self.main_part.stream_name}" """
+        if len(self.main_part.joins) > 0:
+            join_clauses = [join.definition() for join in self.main_part.joins]
+            final_cte += "\n" + ("\n".join(join_clauses))
+        final_cte += ")"
+
         ctes.append(final_cte)
         all_ctes = "\n,".join(ctes)
-        main_columns:List[SnowflakeViewColumn] = SnowflakeViewColumn.order_by_reference(self.main_part.direct_columns() +
-                                                              self.main_part.join_columns())
+        main_columns:List[SnowflakeViewColumn] = self.main_part.columns
         column_clauses = [f"\"OMNATA_FINAL_CTE\"."+c.original_to_transformed()
                           for c in main_columns]
         
         view_body = f"""with {all_ctes}
     select {', '.join(column_clauses)}
     from OMNATA_FINAL_CTE """
-        if len(self.main_part.joins) > 0:
-            join_clauses = [join.definition() for join in self.main_part.joins]
-            view_body += "\n" + ("\n".join(join_clauses))
+        
         return view_body
 
     @classmethod
@@ -662,7 +672,15 @@ def normalized_view_part(
         if json_schema.joins:
             joins = json_schema.joins
         comment = json_schema.description
-
+    
+    direct_view_columns = [c for c in view_columns if c.is_join_column]
+    join_view_columns = [c for c in view_columns if not c.is_join_column]
+    # The final order of view columns is:
+    #- APP_IDENTIFIER
+    #- Direct and joined columns, ordered so that columns that reference other columns are defined after the columns they reference
+    #- OMNATA_RETRIEVE_DATE, OMNATA_RAW_RECORD, OMNATA_IS_DELETED, OMNATA_RUN_ID
+    view_columns = SnowflakeViewColumn.order_by_reference(direct_view_columns +
+                                                              join_view_columns)
     return SnowflakeViewPart(
         stream_name=stream_name,
         raw_table_location=raw_table_location,