dasl-client 1.0.22__py3-none-any.whl → 1.0.24__py3-none-any.whl

test/test_marshaling.py

@@ -1,910 +0,0 @@
-from dasl_client import *
-
-from .constants import *
-
-
-def test_workspace_config_marshal_unmarshal():
-    workspace_config = WorkspaceConfig(
-        metadata=Metadata(
-            name="config",
-            workspace=workspace,
-            comment="a random comment",
-            annotations={"key1": "value1", "key2": "value2"},
-            created_timestamp=datetime(2023, 1, 1, 12, 30),
-            created_by="creator_user",
-            modified_timestamp=datetime(2023, 1, 2, 13, 45),
-            last_successful_run_timestamp=datetime(2023, 1, 3, 14, 0),
-            modified_by="modifier_user",
-            version=1,
-            deleted=False,
-            resource_status="none",
-            ui_status="ok",
-            client_of_origin=get_client_identifier(),
-        ),
-        system_tables_config=SystemTablesConfig(
-            catalog_name="catalog_random", var_schema="schema_random"
-        ),
-        default_sql_warehouse="warehouse_random",
-        detection_rule_metadata=DetectionRuleMetadata(
-            detection_categories=["category1", "category2"]
-        ),
-        notable_export=ExportConfig(
-            destination="webhook",
-            export_open_only=True,
-            webhook_config=ExportConfig.WebhookConfig(
-                destination=ExportConfig.WebhookDestination(
-                    value="wh_value1", scope="wh_scope1", key="wh_key1"
-                )
-            ),
-            slack_config=ExportConfig.SlackConfig(
-                token=ExportConfig.WebhookDestination(
-                    value="slack_token1", scope="slack_scope1", key="slack_key1"
-                ),
-                channel="#channel1",
-                message="slack message 1",
-            ),
-        ),
-        operational_alert_export=ExportConfig(
-            destination="slack",
-            export_open_only=False,
-            webhook_config=ExportConfig.WebhookConfig(
-                destination=ExportConfig.WebhookDestination(
-                    value="wh_value2", scope="wh_scope2", key="wh_key2"
-                )
-            ),
-            slack_config=ExportConfig.SlackConfig(
-                token=ExportConfig.WebhookDestination(
-                    value="slack_token2", scope="slack_scope2", key="slack_key2"
-                ),
-                channel="#channel2",
-                message="slack message 2",
-            ),
-        ),
-        observables=WorkspaceConfigObservables(
-            kinds=[
-                WorkspaceConfigObservables.ObservablesKinds(
-                    name="kind1", sql_type="type1"
-                ),
-                WorkspaceConfigObservables.ObservablesKinds(
-                    name="kind2", sql_type="type2"
-                ),
-            ],
-            relationships=["rel1", "rel2"],
-        ),
-        dasl_storage_path="/random/storage/path",
-        default_config=DefaultConfig(
-            datasources=DefaultConfig.Config(
-                notebook_location="notebook_ds",
-                bronze_schema="bronze_ds",
-                silver_schema="silver_ds",
-                gold_schema="gold_ds",
-                catalog_name="catalog_ds",
-                default_max_resources_per_job=5,
-                checkpoint_location="checkpoint_ds",
-                compute_group_overrides={
-                    "override1": DefaultConfig.Config.ComputeGroupOverrides(
-                        max_resources_per_job=2
-                    ),
-                    "override2": DefaultConfig.Config.ComputeGroupOverrides(
-                        max_resources_per_job=3
-                    ),
-                },
-            ),
-            transforms=DefaultConfig.Config(
-                notebook_location="notebook_trans",
-                bronze_schema="bronze_trans",
-                silver_schema="silver_trans",
-                gold_schema="gold_trans",
-                catalog_name="catalog_trans",
-                default_max_resources_per_job=6,
-                checkpoint_location="checkpoint_trans",
-                compute_group_overrides={
-                    "override3": DefaultConfig.Config.ComputeGroupOverrides(
-                        max_resources_per_job=4
-                    ),
-                    "override4": DefaultConfig.Config.ComputeGroupOverrides(
-                        max_resources_per_job=5
-                    ),
-                },
-            ),
-            rules=DefaultConfig.Config(
-                notebook_location="notebook_rules",
-                bronze_schema="bronze_rules",
-                silver_schema="silver_rules",
-                gold_schema="gold_rules",
-                catalog_name="catalog_rules",
-                default_max_resources_per_job=7,
-                checkpoint_location="checkpoint_rules",
-                compute_group_overrides={
-                    "override5": DefaultConfig.Config.ComputeGroupOverrides(
-                        max_resources_per_job=6
-                    ),
-                    "override6": DefaultConfig.Config.ComputeGroupOverrides(
-                        max_resources_per_job=7
-                    ),
-                },
-            ),
-            var_global=DefaultConfig.Config(
-                notebook_location="notebook_var",
-                bronze_schema="bronze_var",
-                silver_schema="silver_var",
-                gold_schema="gold_var",
-                catalog_name="catalog_var",
-                default_max_resources_per_job=8,
-                checkpoint_location="checkpoint_var",
-                compute_group_overrides={
-                    "override7": DefaultConfig.Config.ComputeGroupOverrides(
-                        max_resources_per_job=8
-                    ),
-                    "override8": DefaultConfig.Config.ComputeGroupOverrides(
-                        max_resources_per_job=9
-                    ),
-                },
-            ),
-        ),
-        managed_retention=[
-            ManagedRetention(
-                catalog="catalog_ret1",
-                var_schema="schema_ret1",
-                column="col_ret1",
-                duration="1d",
-                overrides=[
-                    ManagedRetention.Overrides(
-                        table="table1", column="colA", duration="1h"
-                    ),
-                    ManagedRetention.Overrides(
-                        table="table2", column="colB", duration="2h"
-                    ),
-                ],
-            ),
-            ManagedRetention(
-                catalog="catalog_ret2",
-                var_schema="schema_ret2",
-                column="col_ret2",
-                duration="2d",
-                overrides=[
-                    ManagedRetention.Overrides(
-                        table="table3", column="colC", duration="3h"
-                    ),
-                    ManagedRetention.Overrides(
-                        table="table4", column="colD", duration="4h"
-                    ),
-                ],
-            ),
-        ],
-        status=ResourceStatus(
-            job_id=123,
-            job_name="job_random",
-            enabled=True,
-            notebook_path="/path/to/notebook",
-            created_at=datetime(2023, 1, 4, 15, 0),
-            job_status="scheduled",
-            events=[
-                ResourceStatus.StatusEvent(
-                    action="create",
-                    message="job started",
-                    recorded_at=datetime(2023, 1, 4, 15, 0),
-                ),
-                ResourceStatus.StatusEvent(
-                    action="update",
-                    message="job finished",
-                    recorded_at=datetime(2023, 1, 4, 16, 0),
-                ),
-            ],
-        ),
-    )
-
-    assert workspace_config == WorkspaceConfig.from_api_obj(
-        workspace_config.to_api_obj()
-    )
-
-
-def test_data_source_marshal_unmarshal():
-    data_source = DataSource(
-        metadata=Metadata(
-            name="data_source_name",
-            workspace="workspace1",
-            comment="A sample data source",
-            annotations={"env": "prod", "version": "v1"},
-            created_timestamp=datetime(2023, 4, 1, 10, 0),
-            created_by="creator_ds",
-            modified_timestamp=datetime(2023, 4, 2, 11, 30),
-            last_successful_run_timestamp=datetime(2023, 4, 3, 12, 45),
-            modified_by="modifier_ds",
-            version=2,
-            deleted=False,
-            resource_status="deletionPending",
-            ui_status="ok",
-            client_of_origin=get_client_identifier(),
-        ),
-        source="s3://data-bucket/source",
-        source_type="custom",
-        schedule=Schedule(
-            at_least_every="1h",
-            exactly="1h",
-            continuous=True,
-            compute_group="compute_group1",
-            enabled=True,
-        ),
-        custom=DataSource.CustomNotebook(
-            notebook="print('Hello from custom notebook')"
-        ),
-        use_preset="preset_alpha",
-        autoloader=DataSource.Autoloader(
-            format="json",
-            location="s3://data-bucket/autoloader",
-            schema_file="schema_autoloader.json",
-            cloud_files=DataSource.Autoloader.CloudFiles(
-                schema_hints_file="hints.txt", schema_hints="hint1, hint2"
-            ),
-        ),
-        bronze=BronzeSpec(
-            clustering=BronzeSpec.Clustering(
-                column_names=["cluster_col1", "cluster_col2"], time_column="timestamp"
-            ),
-            bronze_table="bronze_table_1",
-            skip_bronze_loading=False,
-        ),
-        silver=SilverSpec(
-            bronze_tables=[
-                SilverSpec.BronzeTable(
-                    name="silver_bronze_table_1",
-                    streaming=True,
-                    watermark=SilverSpec.BronzeTable.Watermark(
-                        event_time_column="event_time",
-                        delay_threshold="10m",
-                        drop_duplicates=["dup1", "dup2"],
-                    ),
-                    alias="silver_alias",
-                    join_type="inner",
-                    join_expr="a.id = b.id",
-                ),
-            ],
-            pre_transform=SilverSpec.PreTransform(
-                use_preset="silver_pre",
-                skip_pre_transform=False,
-                custom=SilverSpec.PreTransform.Custom(
-                    function="pre_transform_func",
-                    options={"option1": "value1", "option2": "value2"},
-                ),
-                filter="col > 0",
-                post_filter="col < 100",
-                preset_overrides=SilverSpec.PreTransform.PresetOverrides(
-                    omit_fields=["omit_field1", "omit_field2"]
-                ),
-                add_fields=[
-                    FieldSpec(
-                        name="pre_field1",
-                        comment="Pre transform field 1",
-                        var_assert=[
-                            FieldSpec.Assert(expr="x > 0", message="x must be positive")
-                        ],
-                        var_from="source_x",
-                        alias="alias_x",
-                        expr="x + 1",
-                        literal="10",
-                        join=FieldSpec.Join(
-                            with_table="join_table_1",
-                            with_csv=FieldSpec.Join.WithCSV(path="/path/to/join1.csv"),
-                            lhs="x",
-                            rhs="y",
-                            select="x, y",
-                        ),
-                    ),
-                    FieldSpec(
-                        name="pre_field2",
-                        comment="Pre transform field 2",
-                        var_assert=[
-                            FieldSpec.Assert(expr="y != 0", message="y must be nonzero")
-                        ],
-                        var_from="source_y",
-                        alias="alias_y",
-                        expr="y * 2",
-                        literal="20",
-                        join=FieldSpec.Join(
-                            with_table="join_table_2",
-                            with_csv=FieldSpec.Join.WithCSV(path="/path/to/join2.csv"),
-                            lhs="y",
-                            rhs="z",
-                            select="y, z",
-                        ),
-                    ),
-                ],
-            ),
-            transform=SilverSpec.Transform(
-                skip_silver_transform=False,
-                preset_overrides=SilverSpec.Transform.PresetOverrides(
-                    modify_tables=[
-                        SilverSpec.Transform.PresetOverrides.ModifyTables(
-                            name="modify_table1",
-                            custom=SilverSpec.Transform.PresetOverrides.Custom(
-                                function="modify_func1",
-                                options={"mod_opt1": "val1", "mod_opt2": "val2"},
-                            ),
-                            omit_fields=["mod_omit1", "mod_omit2"],
-                            override_liquid_columns=["liq1", "liq2"],
-                            add_fields=[
-                                FieldSpec(
-                                    name="mod_field1",
-                                    comment="Modify field 1",
-                                    var_assert=[
-                                        FieldSpec.Assert(
-                                            expr="a < b",
-                                            message="a should be less than b",
-                                        )
-                                    ],
-                                    var_from="mod_source1",
-                                    alias="mod_alias1",
-                                    expr="a - b",
-                                    literal="5",
-                                    join=FieldSpec.Join(
-                                        with_table="mod_join_table",
-                                        with_csv=FieldSpec.Join.WithCSV(
-                                            path="/path/to/mod_join.csv"
-                                        ),
-                                        lhs="a",
-                                        rhs="b",
-                                        select="a, b",
-                                    ),
-                                )
-                            ],
-                            filter="mod_filter > 0",
-                            post_filter="mod_post_filter < 100",
-                            utils=FieldUtils(
-                                unreferenced_columns=FieldUtils.UnreferencedColumns(
-                                    preserve=False,
-                                    embed_column="mod_embed",
-                                    omit_columns=["mod_omit_col1", "mod_omit_col2"],
-                                    duplicate_prefix="mod_dup_",
-                                ),
-                                json_extract=[
-                                    FieldUtils.JsonExtract(
-                                        source="mod_json_source",
-                                        omit_fields=[
-                                            "mod_json_omit1",
-                                            "mod_json_omit2",
-                                        ],
-                                        duplicate_prefix="mod_json_dup_",
-                                        embed_column="mod_json_embed",
-                                    )
-                                ],
-                            ),
-                        )
-                    ],
-                    omit_tables=["omit_table1", "omit_table2"],
-                    add_tables=[
-                        SilverSpec.Transform.PresetOverrides.AddTables(
-                            custom=SilverSpec.Transform.PresetOverrides.Custom(
-                                function="add_func1",
-                                options={"add_opt1": "val1", "add_opt2": "val2"},
-                            ),
-                            name="add_table1",
-                            filter="add_filter_condition",
-                            post_filter="add_post_filter_condition",
-                            override_liquid_columns=["add_liq1", "add_liq2"],
-                            fields=[
-                                FieldSpec(
-                                    name="add_field1",
-                                    comment="Add table field 1",
-                                    var_assert=[
-                                        FieldSpec.Assert(
-                                            expr="c == 1", message="c must equal 1"
-                                        )
-                                    ],
-                                    var_from="add_source1",
-                                    alias="add_alias1",
-                                    expr="c + 10",
-                                    literal="15",
-                                    join=FieldSpec.Join(
-                                        with_table="add_join_table",
-                                        with_csv=FieldSpec.Join.WithCSV(
-                                            path="/path/to/add_join.csv"
-                                        ),
-                                        lhs="c",
-                                        rhs="d",
-                                        select="c, d",
-                                    ),
-                                )
-                            ],
-                            utils=FieldUtils(
-                                unreferenced_columns=FieldUtils.UnreferencedColumns(
-                                    preserve=True,
-                                    embed_column="add_embed",
-                                    omit_columns=["add_omit1", "add_omit2"],
-                                    duplicate_prefix="add_dup_",
-                                ),
-                                json_extract=[
-                                    FieldUtils.JsonExtract(
-                                        source="add_json_source",
-                                        omit_fields=[
-                                            "add_json_omit1",
-                                            "add_json_omit2",
-                                        ],
-                                        duplicate_prefix="add_json_dup_",
-                                        embed_column="add_json_embed",
-                                    )
-                                ],
-                            ),
-                        )
-                    ],
-                ),
-            ),
-        ),
-        gold=GoldSpec(
-            omit_tables=["gold_omit1", "gold_omit2"],
-            modify_tables=[
-                GoldSpec.ModifyTables(
-                    name="gold_modify_table1",
-                    source_table="gold_source_table1",
-                    custom=GoldSpec.ModifyTables.Custom(
-                        function="gold_modify_func",
-                        options={"gold_opt1": "val1", "gold_opt2": "val2"},
-                    ),
-                    omit_fields=["gold_field_omit1", "gold_field_omit2"],
-                    add_fields=[
-                        FieldSpec(
-                            name="gold_field1",
-                            comment="Gold modify field 1",
-                            var_assert=[
-                                FieldSpec.Assert(
-                                    expr="z != 0", message="z must not be zero"
-                                )
-                            ],
-                            var_from="gold_source",
-                            alias="gold_alias",
-                            expr="z / 2",
-                            literal="3.14",
-                            join=FieldSpec.Join(
-                                with_table="gold_join_table",
-                                with_csv=FieldSpec.Join.WithCSV(
-                                    path="/path/to/gold_join.csv"
-                                ),
-                                lhs="z",
-                                rhs="w",
-                                select="z, w",
-                            ),
-                        )
-                    ],
-                    filter="gold_filter_condition",
-                    post_filter="gold_post_filter_condition",
-                )
-            ],
-            add_tables=[
-                GoldSpec.AddTables(
-                    name="gold_add_table1",
-                    source_table="gold_add_source_table1",
-                    custom=GoldSpec.AddTables.Custom(
-                        function="gold_add_func",
-                        options={"gold_add_opt1": "val1", "gold_add_opt2": "val2"},
-                    ),
-                    filter="gold_add_filter_condition",
-                    post_filter="gold_add_post_filter_condition",
-                    fields=[
-                        FieldSpec(
-                            name="gold_add_field1",
-                            comment="Gold add field 1",
-                            var_assert=[
-                                FieldSpec.Assert(
-                                    expr="a > 0", message="a must be positive"
-                                )
-                            ],
-                            var_from="gold_add_source",
-                            alias="gold_add_alias",
-                            expr="a + 10",
-                            literal="20",
-                            join=FieldSpec.Join(
-                                with_table="gold_add_join_table",
-                                with_csv=FieldSpec.Join.WithCSV(
-                                    path="/path/to/gold_add_join.csv"
-                                ),
-                                lhs="a",
-                                rhs="b",
-                                select="a, b",
-                            ),
-                        )
-                    ],
-                ),
-            ],
-        ),
-        status=ResourceStatus(
-            job_id=789,
-            job_name="data_source_job",
-            enabled=True,
-            notebook_path="/path/to/datasource/notebook",
-            created_at=datetime(2023, 5, 1, 8, 0),
-            job_status="unscheduled",
-            events=[
-                ResourceStatus.StatusEvent(
-                    action="create",
-                    message="Data source job started",
-                    recorded_at=datetime(2023, 5, 1, 8, 0),
-                ),
-                ResourceStatus.StatusEvent(
-                    action="update",
-                    message="Data source job finished",
-                    recorded_at=datetime(2023, 5, 1, 8, 30),
-                ),
-            ],
-        ),
-    )
-
-    assert data_source == DataSource.from_api_obj(data_source.to_api_obj())
-
-
-def test_rule_marshal_unmarshal():
-    rule = Rule(
-        metadata=Metadata(
-            name="rule_meta_name",
-            workspace="example_workspace",
-            comment="This is a sample rule metadata comment.",
-            annotations={"env": "prod", "source": "system"},
-            created_timestamp=datetime(2023, 1, 1, 9, 0),
-            created_by="rule_creator",
-            modified_timestamp=datetime(2023, 1, 2, 10, 0),
-            last_successful_run_timestamp=datetime(2023, 1, 3, 11, 0),
-            modified_by="rule_modifier",
-            version=1,
-            deleted=False,
-            resource_status="none",
-            ui_status="ok",
-            client_of_origin=get_client_identifier(),
-        ),
-        rule_metadata=Rule.RuleMetadata(
-            version=1.0,
-            category="Security",
-            severity="High",
-            fidelity="Investigative",
-            mitre=[
-                Rule.RuleMetadata.Mitre(
-                    taxonomy="MITRE ATT&CK",
-                    tactic="Initial Access",
-                    technique_id="T1190",
-                    technique="Exploit Public-Facing Application",
-                    sub_technique_id="T1190.001",
-                    sub_technique="Example Sub-technique",
-                )
-            ],
-            objective="Detect unauthorized access attempts",
-            response=Rule.RuleMetadata.Response(
-                guidelines="Follow the incident response plan immediately.",
-                playbooks=[
-                    Rule.RuleMetadata.Response.Playbook(
-                        notebook="incident_response.ipynb",
-                        options={"notify": "email", "severity": "high"},
-                    )
-                ],
-            ),
-        ),
-        schedule=Schedule(
-            at_least_every="15m",
-            exactly="15m",
-            continuous=True,
-            compute_group="rule_compute_group",
-            enabled=True,
-        ),
-        input=Rule.Input(
-            stream=Rule.Input.Stream(
-                tables=[
-                    Rule.Input.Stream.Table(
-                        name="access_logs",
-                        watermark=Rule.Input.Stream.Table.Watermark(
-                            event_time_column="timestamp",
-                            delay_threshold="5m",
-                            drop_duplicates=["ip", "user_id"],
-                        ),
-                        alias="logs",
-                        join_type="inner",
-                        join_expr="access_logs.user_id = user_info.id",
-                    ),
-                    Rule.Input.Stream.Table(
-                        name="user_info",
-                        watermark=Rule.Input.Stream.Table.Watermark(
-                            event_time_column="event_time",
-                            delay_threshold="10m",
-                            drop_duplicates=["id"],
-                        ),
-                        alias="users",
-                        join_type="left",
-                        join_expr="user_info.id = access_logs.user_id",
-                    ),
-                ],
-                filter="status = 'active'",
-                sql="SELECT * FROM streaming_source",
-                custom=Rule.Input.CustomStream(
-                    notebook="stream_custom.ipynb",
-                    options={"filter": "recent", "limit": "1000"},
-                ),
-            ),
-            batch=Rule.Input.Batch(
-                sql="SELECT * FROM historical_source",
-                custom=Rule.Input.CustomBatch(
-                    notebook="batch_custom.ipynb",
-                    options={"start_date": "2022-01-01", "end_date": "2022-12-31"},
-                ),
-            ),
-        ),
-        observables=[
-            Rule.Observable(
-                kind="ip",
-                value="192.168.0.1",
-                relationship="suspicious",
-                risk=Rule.Observable.Risk(
-                    impact="High",
-                    confidence="Medium",
-                ),
-            ),
-            Rule.Observable(
-                kind="domain",
-                value="malicious.com",
-                relationship="malicious",
-                risk=Rule.Observable.Risk(
-                    impact="Critical",
-                    confidence="High",
-                ),
-            ),
-        ],
-        output=Rule.Output(
-            summary="Unauthorized access detected from multiple sources.",
-            context={"alert": "Multiple failed logins", "severity": "high"},
-        ),
-        collate=Rule.Collate(
-            collate_on=["ip", "user_id"],
-            within="1h",
-            action="append",
-        ),
-        status=ResourceStatus(
-            job_id=101,
-            job_name="rule_evaluation_job",
-            enabled=True,
-            notebook_path="/rules/evaluate_rule.ipynb",
-            created_at=datetime(2023, 1, 5, 12, 0),
-            job_status="scheduled",
-            events=[
-                ResourceStatus.StatusEvent(
-                    action="create",
-                    message="Rule evaluation started",
-                    recorded_at=datetime(2023, 1, 5, 12, 0),
-                ),
-                ResourceStatus.StatusEvent(
-                    action="update",
-                    message="Rule evaluation finished",
-                    recorded_at=datetime(2023, 1, 5, 12, 15),
-                ),
-            ],
-        ),
-    )
-
-    assert rule == Rule.from_api_obj(rule.to_api_obj())
-
-
-def test_transform_request_marshal_unmarshal():
-    request = TransformRequest(
-        input=TransformRequest.Input(
-            columns=[
-                Dbui.TableColumnDetails(
-                    name="col1",
-                    type_name="int",
-                    type_detail="integer",
-                    position=1,
-                    nullable=False,
-                ),
-                Dbui.TableColumnDetails(
-                    name="col2",
-                    type_name="varchar",
-                    type_detail="string",
-                    position=2,
-                    nullable=True,
-                ),
-            ],
-            data=[{"col1": "1", "col2": "a"}, {"col1": "2", "col2": "b"}],
-        ),
-        autoloader_input=TransformRequest.Autoloader(
-            format="csv",
-            location="s3://bucket/data",
-            schema_file="schema.json",
-            cloud_files=TransformRequest.Autoloader.CloudFiles(
-                schema_hints_file="hints_file.csv", schema_hints="hint1, hint2"
-            ),
-            row_count=1,
-            row_offset=5,
-        ),
-        use_preset="preset_value",
-        transforms=[
-            TransformRequest.Transform(
-                transform_type="Gold",
-                use_preset_table="table_name",
-                filter="col > 0",
-                post_filter="col < 100",
-                preset_overrides=TransformRequest.Transform.PresetOverrides(
-                    omit_fields=["field1", "field2"]
-                ),
-                add_fields=[
-                    FieldSpec(
-                        name="field1",
-                        comment="comment1",
-                        var_assert=[
-                            FieldSpec.Assert(expr="1=1", message="assertion passed"),
-                            FieldSpec.Assert(expr="2=2", message="assertion passed 2"),
-                        ],
-                        var_from="source_field1",
-                        alias="alias1",
-                        expr="expr1",
-                        literal="literal1",
-                        join=FieldSpec.Join(
-                            with_table="table_join1",
-                            with_csv=FieldSpec.Join.WithCSV(path="csv_path1"),
-                            lhs="left1",
-                            rhs="right1",
-                            select="select_expr1",
-                        ),
-                    ),
-                    FieldSpec(
-                        name="field2",
-                        comment="comment2",
-                        var_assert=[
-                            FieldSpec.Assert(expr="a=b", message="assertion ok"),
-                            FieldSpec.Assert(expr="c=d", message="assertion ok 2"),
-                        ],
-                        var_from="source_field2",
-                        alias="alias2",
-                        expr="expr2",
-                        literal="literal2",
-                        join=FieldSpec.Join(
-                            with_table="table_join2",
-                            with_csv=FieldSpec.Join.WithCSV(path="csv_path2"),
-                            lhs="left2",
-                            rhs="right2",
-                            select="select_expr2",
-                        ),
-                    ),
-                ],
-                utils=FieldUtils(
-                    unreferenced_columns=FieldUtils.UnreferencedColumns(
-                        preserve=True,
-                        embed_column="all_data",
-                        omit_columns=["omit1", "omit2"],
-                        duplicate_prefix="dup_",
-                    ),
-                    json_extract=[
-                        FieldUtils.JsonExtract(
-                            source="json_column1",
-                            omit_fields=["omitA", "omitB"],
-                            duplicate_prefix="dup1",
-                            embed_column="embed1",
-                        ),
-                        FieldUtils.JsonExtract(
-                            source="json_column2",
-                            omit_fields=["omitC", "omitD"],
-                            duplicate_prefix="dup2",
-                            embed_column="embed2",
-                        ),
-                    ],
-                ),
-            ),
-            TransformRequest.Transform(
-                transform_type="SilverTransform",
-                use_preset_table="table_b",
-                filter="col >= 10",
-                post_filter="col <= 50",
-                preset_overrides=TransformRequest.Transform.PresetOverrides(
-                    omit_fields=["fieldX", "fieldY"]
-                ),
-                add_fields=[
-                    FieldSpec(
-                        name="field3",
-                        comment="comment3",
-                        var_assert=[
-                            FieldSpec.Assert(expr="assert_expr3a", message="message3a"),
-                            FieldSpec.Assert(expr="assert_expr3b", message="message3b"),
-                        ],
-                        var_from="source_field3",
-                        alias="alias3",
-                        expr="expr3",
-                        literal="literal3",
-                        join=FieldSpec.Join(
-                            with_table="table_join3",
-                            with_csv=FieldSpec.Join.WithCSV(path="csv_path3"),
-                            lhs="left3",
-                            rhs="right3",
-                            select="select_expr3",
-                        ),
-                    ),
-                    FieldSpec(
-                        name="field4",
-                        comment="comment4",
-                        var_assert=[
-                            FieldSpec.Assert(expr="assert_expr4a", message="message4a"),
-                            FieldSpec.Assert(expr="assert_expr4b", message="message4b"),
-                        ],
-                        var_from="source_field4",
-                        alias="alias4",
-                        expr="expr4",
-                        literal="literal4",
-                        join=FieldSpec.Join(
-                            with_table="table_join4",
-                            with_csv=FieldSpec.Join.WithCSV(path="csv_path4"),
-                            lhs="left4",
-                            rhs="right4",
-                            select="select_expr4",
-                        ),
-                    ),
-                ],
-                utils=FieldUtils(
-                    unreferenced_columns=FieldUtils.UnreferencedColumns(
-                        preserve=False,
-                        embed_column="extra_data",
-                        omit_columns=["omitX", "omitY", "omitZ"],
-                        duplicate_prefix="dupB",
-                    ),
-                    json_extract=[
-                        FieldUtils.JsonExtract(
-                            source="json_column3",
-                            omit_fields=["omitE", "omitF"],
-                            duplicate_prefix="dup3",
-                            embed_column="embed3",
-                        ),
-                        FieldUtils.JsonExtract(
-                            source="json_column4",
-                            omit_fields=["omitG", "omitH"],
-                            duplicate_prefix="dup4",
-                            embed_column="embed4",
-                        ),
-                    ],
-                ),
-            ),
-        ],
-    )
-
-    assert request == TransformRequest.from_api_obj(request.to_api_obj())
-
-
-def test_transform_response_marshal_unmarshal():
-    response = TransformResponse(
-        stages=[
-            TransformResponse.Stages(
-                transform_type="Gold",
-                columns=[
-                    Dbui.TableColumnDetails(
-                        name="id",
-                        type_name="int",
-                        type_detail="integer",
-                        position=1,
-                        nullable=False,
-                    ),
-                    Dbui.TableColumnDetails(
-                        name="name",
-                        type_name="varchar",
-                        type_detail="text",
-                        position=2,
-                        nullable=True,
-                    ),
-                ],
-                data=[{"id": "1", "name": "Alice"}, {"id": "2", "name": "Bob"}],
-            ),
-            TransformResponse.Stages(
-                transform_type="SilverPreTransform",
-                columns=[
-                    Dbui.TableColumnDetails(
-                        name="price",
-                        type_name="float",
-                        type_detail="decimal",
-                        position=3,
-                        nullable=False,
-                    ),
-                    Dbui.TableColumnDetails(
-                        name="quantity",
-                        type_name="int",
-                        type_detail="integer",
-                        position=4,
-                        nullable=True,
-                    ),
-                ],
-                data=[
-                    {"price": "9.99", "quantity": "5"},
-                    {"price": "19.99", "quantity": "10"},
-                ],
-            ),
-        ]
-    )
-
-    assert response == TransformResponse.from_api_obj(response.to_api_obj())