sqlmesh 0.213.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl

sqlmesh/migrations/v0042_trim_indirect_versions.py

@@ -1,66 +0,0 @@
-"""Trim irrelevant attributes from indirect versions."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-from sqlmesh.utils.migration import blob_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-
-    for name, identifier, version, snapshot, kind_name, expiration_ts in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name", "expiration_ts").from_(
-            snapshots_table
-        ),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        for indirect_versions in parsed_snapshot["indirect_versions"].values():
-            for indirect_version in indirect_versions:
-                # Only keep version and change_category.
-                version = indirect_version.get("version")
-                change_category = indirect_version.get("change_category")
-                indirect_version.clear()
-                indirect_version["version"] = version
-                indirect_version["change_category"] = change_category
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-                "expiration_ts": expiration_ts,
-            }
-        )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-        blob_type = blob_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build(blob_type),
-                "kind_name": exp.DataType.build(index_type),
-                "expiration_ts": exp.DataType.build("bigint"),
-            },
-        )

sqlmesh/migrations/v0043_fix_remove_obsolete_attributes_in_plan_dags.py

@@ -1,61 +0,0 @@
-"""Trim irrelevant attributes from the plan DAGs state."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-from sqlmesh.utils.migration import blob_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    plan_dags_table = "_plan_dags"
-    if schema:
-        plan_dags_table = f"{schema}.{plan_dags_table}"
-
-    new_dag_specs = []
-
-    for request_id, dag_id, dag_spec in engine_adapter.fetchall(
-        exp.select("request_id", "dag_id", "dag_spec").from_(plan_dags_table),
-        quote_identifiers=True,
-    ):
-        parsed_dag_spec = json.loads(dag_spec)
-        for snapshot in parsed_dag_spec.get("new_snapshots", []):
-            snapshot["node"].pop("hash_raw_query", None)
-
-            for indirect_versions in snapshot.get("indirect_versions", {}).values():
-                for indirect_version in indirect_versions:
-                    # Only keep version and change_category.
-                    version = indirect_version.get("version")
-                    change_category = indirect_version.get("change_category")
-                    indirect_version.clear()
-                    indirect_version["version"] = version
-                    indirect_version["change_category"] = change_category
-
-        new_dag_specs.append(
-            {
-                "request_id": request_id,
-                "dag_id": dag_id,
-                "dag_spec": json.dumps(parsed_dag_spec),
-            }
-        )
-
-    if new_dag_specs:
-        engine_adapter.delete_from(plan_dags_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-        blob_type = blob_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            plan_dags_table,
-            pd.DataFrame(new_dag_specs),
-            target_columns_to_types={
-                "request_id": exp.DataType.build(index_type),
-                "dag_id": exp.DataType.build(index_type),
-                "dag_spec": exp.DataType.build(blob_type),
-            },
-        )

sqlmesh/migrations/v0044_quote_identifiers_in_model_attributes.py

@@ -1,5 +0,0 @@
-"""Quoted identifiers in model SQL attributes."""
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    pass

sqlmesh/migrations/v0045_move_gateway_variable.py

@@ -1,70 +0,0 @@
-"""Move the gateway variable."""
-
-import ast
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-from sqlmesh.utils.migration import blob_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    migration_needed = False
-    new_snapshots = []
-
-    for name, identifier, version, snapshot, kind_name, expiration_ts in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name", "expiration_ts").from_(
-            snapshots_table
-        ),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        python_env = parsed_snapshot["node"].get("python_env")
-        if python_env:
-            gateway = python_env.pop("gateway", None)
-            if gateway is not None:
-                migration_needed = True
-                sqlmesh_vars = {"gateway": ast.literal_eval(gateway["payload"])}
-                python_env["__sqlmesh__vars__"] = {
-                    "payload": repr(sqlmesh_vars),
-                    "kind": "value",
-                }
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-                "expiration_ts": expiration_ts,
-            }
-        )
-
-    if migration_needed and new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-        blob_type = blob_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build(blob_type),
-                "kind_name": exp.DataType.build(index_type),
-                "expiration_ts": exp.DataType.build("bigint"),
-            },
-        )

sqlmesh/migrations/v0046_add_batch_concurrency.py

@@ -1,8 +0,0 @@
-"""Add the batch_concurrency attribute to the incremental model kinds.
-
-This results in a change to the metadata hash.
-"""
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    pass

sqlmesh/migrations/v0047_change_scd_string_to_column.py

@@ -1,5 +0,0 @@
-"""Changes the SCD Type 2 columns from strings to columns."""
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    pass

sqlmesh/migrations/v0048_drop_indirect_versions.py

@@ -1,59 +0,0 @@
-"""Drop the indirect_versions attribute in snapshots."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-from sqlmesh.utils.migration import blob_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-
-    for name, identifier, version, snapshot, kind_name, expiration_ts in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name", "expiration_ts").from_(
-            snapshots_table
-        ),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        parsed_snapshot.pop("indirect_versions", None)
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-                "expiration_ts": expiration_ts,
-            }
-        )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-        blob_type = blob_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build(blob_type),
-                "kind_name": exp.DataType.build(index_type),
-                "expiration_ts": exp.DataType.build("bigint"),
-            },
-        )

sqlmesh/migrations/v0049_replace_identifier_with_version_in_seeds_table.py

@@ -1,57 +0,0 @@
-"""Use version instead of identifier in the seeds table."""
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    engine_adapter = state_sync.engine_adapter
-
-    snapshots_table = "_snapshots"
-    seeds_table = "_seeds"
-    new_seeds_table = f"{seeds_table}_v49"
-
-    if state_sync.schema:
-        snapshots_table = f"{state_sync.schema}.{snapshots_table}"
-        seeds_table = f"{state_sync.schema}.{seeds_table}"
-        new_seeds_table = f"{state_sync.schema}.{new_seeds_table}"
-
-    index_type = index_text_type(engine_adapter.dialect)
-
-    engine_adapter.drop_table(new_seeds_table)
-    engine_adapter.create_state_table(
-        new_seeds_table,
-        {
-            "name": exp.DataType.build(index_type),
-            "version": exp.DataType.build(index_type),
-            "content": exp.DataType.build("text"),
-        },
-        primary_key=("name", "version"),
-    )
-
-    name_col = exp.column("name", table="seeds")
-    version_col = exp.column("version", table="snapshots")
-    query = (
-        exp.select(
-            name_col,
-            version_col,
-            exp.func("MAX", exp.column("content", table="seeds")).as_("content"),
-        )
-        .from_(exp.to_table(seeds_table).as_("seeds"))
-        .join(
-            exp.to_table(snapshots_table).as_("snapshots"),
-            on=exp.and_(
-                exp.column("name", table="seeds").eq(exp.column("name", table="snapshots")),
-                exp.column("identifier", table="seeds").eq(
-                    exp.column("identifier", table="snapshots")
-                ),
-            ),
-        )
-        .where(exp.column("version", table="snapshots").is_(exp.null()).not_())
-        .group_by(name_col, version_col)
-    )
-
-    engine_adapter.insert_append(new_seeds_table, query)
-    engine_adapter.drop_table(seeds_table)
-    engine_adapter.rename_table(new_seeds_table, seeds_table)

sqlmesh/migrations/v0050_drop_seeds_table.py

@@ -1,11 +0,0 @@
-"""Drop the seeds table."""
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    engine_adapter = state_sync.engine_adapter
-
-    seeds_table = "_seeds"
-    if state_sync.schema:
-        seeds_table = f"{state_sync.schema}.{seeds_table}"
-
-    engine_adapter.drop_table(seeds_table)

sqlmesh/migrations/v0051_rename_column_descriptions.py

@@ -1,65 +0,0 @@
-"""Rename the node attribute `column_descriptions_` to `column_descriptions` in snapshots."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-from sqlmesh.utils.migration import blob_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-    found_col_descriptions = False
-
-    for name, identifier, version, snapshot, kind_name, expiration_ts in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name", "expiration_ts").from_(
-            snapshots_table
-        ),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-
-        if "column_descriptions_" in parsed_snapshot["node"]:
-            found_col_descriptions = True
-            parsed_snapshot["node"]["column_descriptions"] = parsed_snapshot["node"].pop(
-                "column_descriptions_"
-            )
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-                "expiration_ts": expiration_ts,
-            }
-        )
-
-    if found_col_descriptions:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-        blob_type = blob_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build(blob_type),
-                "kind_name": exp.DataType.build(index_type),
-                "expiration_ts": exp.DataType.build("bigint"),
-            },
-        )

sqlmesh/migrations/v0052_add_normalize_name_in_environment_naming_info.py

@@ -1,28 +0,0 @@
-"""Add flag that controls whether environment names will be normalized."""
-
-from sqlglot import exp
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    engine_adapter = state_sync.engine_adapter
-    environments_table = "_environments"
-    if state_sync.schema:
-        environments_table = f"{state_sync.schema}.{environments_table}"
-
-    alter_table_exp = exp.Alter(
-        this=exp.to_table(environments_table),
-        kind="TABLE",
-        actions=[
-            exp.ColumnDef(
-                this=exp.to_column("normalize_name"),
-                kind=exp.DataType.build("boolean"),
-            )
-        ],
-    )
-    engine_adapter.execute(alter_table_exp)
-
-    state_sync.engine_adapter.update_table(
-        environments_table,
-        {"normalize_name": False},
-        where=exp.true(),
-    )

sqlmesh/migrations/v0053_custom_model_kind_extra_attributes.py

@@ -1,5 +0,0 @@
-"""Add batch_size, batch_concurrency, and batch_interval to the CUSTOM model kind."""
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    pass

sqlmesh/migrations/v0054_fix_trailing_comments.py

@@ -1,5 +0,0 @@
-"""Fix support for trailing comments in SQL model definitions."""
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    pass

sqlmesh/migrations/v0055_add_updated_ts_unpaused_ts_ttl_ms_unrestorable_to_snapshot.py

@@ -1,132 +0,0 @@
-"""Add updated_ts, unpaused_ts, ttl_ms, and unrestorable columns to the snapshots table."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.date import to_datetime, to_timestamp
-from sqlmesh.utils.migration import index_text_type
-from sqlmesh.utils.migration import blob_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    index_type = index_text_type(engine_adapter.dialect)
-    blob_type = blob_text_type(engine_adapter.dialect)
-
-    add_column_exps = [
-        exp.Alter(
-            this=exp.to_table(snapshots_table),
-            kind="TABLE",
-            actions=[
-                exp.ColumnDef(
-                    this=exp.to_column(column_name),
-                    kind=exp.DataType.build("bigint"),
-                )
-            ],
-        )
-        for column_name in ["updated_ts", "unpaused_ts", "ttl_ms"]
-    ] + [
-        exp.Alter(
-            this=exp.to_table(snapshots_table),
-            kind="TABLE",
-            actions=[
-                exp.ColumnDef(
-                    this=exp.to_column("unrestorable"),
-                    kind=exp.DataType.build("boolean"),
-                )
-            ],
-        )
-    ]
-    engine_adapter.execute(add_column_exps)
-
-    if engine_adapter.dialect == "databricks":
-        # Databricks will throw an error like:
-        # > databricks.sql.exc.ServerOperationError: [DELTA_UNSUPPORTED_DROP_COLUMN] DROP COLUMN is not supported for your Delta table.
-        # when we try to drop `expiration_ts` below unless we set delta.columnMapping.mode to 'name'
-        alter_table_exp = exp.Alter(
-            this=exp.to_table(snapshots_table),
-            kind="TABLE",
-            actions=[
-                exp.AlterSet(
-                    expressions=[
-                        exp.Properties(
-                            expressions=[
-                                exp.Property(
-                                    this=exp.Literal.string("delta.columnMapping.mode"),
-                                    value=exp.Literal.string("name"),
-                                )
-                            ]
-                        )
-                    ]
-                )
-            ],
-        )
-        engine_adapter.execute(alter_table_exp)
-
-    drop_column_exp = exp.Alter(
-        this=exp.to_table(snapshots_table),
-        kind="TABLE",
-        actions=[exp.Drop(this=exp.to_column("expiration_ts"), kind="COLUMN")],
-    )
-    engine_adapter.execute(drop_column_exp)
-
-    new_snapshots = []
-
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        updated_ts = parsed_snapshot.pop("updated_ts")
-        unpaused_ts = parsed_snapshot.pop("unpaused_ts", None)
-        ttl_ms = max(
-            to_timestamp(
-                parsed_snapshot["ttl"],
-                relative_base=to_datetime(updated_ts),
-                check_categorical_relative_expression=False,
-            )
-            - updated_ts,
-            0,
-        )
-        unrestorable = parsed_snapshot.pop("unrestorable", False)
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-                "updated_ts": updated_ts,
-                "unpaused_ts": unpaused_ts,
-                "ttl_ms": ttl_ms,
-                "unrestorable": unrestorable,
-            }
-        )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build(blob_type),
-                "kind_name": exp.DataType.build(index_type),
-                "updated_ts": exp.DataType.build("bigint"),
-                "unpaused_ts": exp.DataType.build("bigint"),
-                "ttl_ms": exp.DataType.build("bigint"),
-                "unrestorable": exp.DataType.build("boolean"),
-            },
-        )