sqlmesh 0.213.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl

sqlmesh/migrations/v0023_fix_added_models_with_forward_only_parents.py

@@ -1,65 +0,0 @@
-"""Fix snapshots of added models with forward only parents."""
-
-import json
-import typing as t
-
-from sqlglot import exp
-
-from sqlmesh.utils.dag import DAG
-
-
-def migrate(state_sync: t.Any, **kwargs) -> None:  # type: ignore
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    environments_table = "_environments"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-        environments_table = f"{schema}.{environments_table}"
-
-    dag: DAG[t.Tuple[str, str]] = DAG()
-    snapshot_mapping: t.Dict[t.Tuple[str, str], t.Dict[str, t.Any]] = {}
-
-    for identifier, snapshot in engine_adapter.fetchall(
-        exp.select("identifier", "snapshot").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-
-        snapshot_id = (parsed_snapshot["name"], identifier)
-        snapshot_mapping[snapshot_id] = parsed_snapshot
-
-        parent_ids = [
-            (parent["name"], parent["identifier"]) for parent in parsed_snapshot["parents"]
-        ]
-        dag.add(snapshot_id, parent_ids)
-
-    snapshots_to_delete = set()
-
-    for snapshot_id in dag:
-        if snapshot_id not in snapshot_mapping:
-            continue
-        parsed_snapshot = snapshot_mapping[snapshot_id]
-        is_breaking = parsed_snapshot.get("change_category") == 1
-        has_previous_versions = bool(parsed_snapshot.get("previous_versions", []))
-
-        has_paused_forward_only_parent = False
-        if is_breaking and not has_previous_versions:
-            for upstream_id in dag.upstream(snapshot_id):
-                if upstream_id not in snapshot_mapping:
-                    continue
-                upstream_snapshot = snapshot_mapping[upstream_id]
-                upstream_change_category = upstream_snapshot.get("change_category")
-                is_forward_only_upstream = upstream_change_category == 3
-                if is_forward_only_upstream and not upstream_snapshot.get("unpaused_ts"):
-                    has_paused_forward_only_parent = True
-                    break
-
-        if has_paused_forward_only_parent:
-            snapshots_to_delete.add(snapshot_id)
-
-    if snapshots_to_delete:
-        where = t.cast(exp.Tuple, exp.convert((exp.column("name"), exp.column("identifier")))).isin(
-            *snapshots_to_delete
-        )
-        engine_adapter.delete_from(snapshots_table, where)

sqlmesh/migrations/v0024_replace_model_kind_name_enum_with_value.py

@@ -1,55 +0,0 @@
-"""Replace snapshot model_kind_name enum with value."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        corrected_kind_name = None
-        parsed_snapshot = json.loads(snapshot)
-        if "kind" in parsed_snapshot["node"]:
-            corrected_kind_name = parsed_snapshot["node"]["kind"].get("name")
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": snapshot,
-                "kind_name": corrected_kind_name,
-            }
-        )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )

sqlmesh/migrations/v0025_fix_intervals_and_missing_change_category.py

@@ -1,117 +0,0 @@
-"""Normalize intervals and fix missing change category."""
-
-import json
-import zlib
-
-from sqlglot import exp
-
-from sqlmesh.utils import random_id
-from sqlmesh.utils.date import now_timestamp
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    intervals_table = "_intervals"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-        intervals_table = f"{schema}.{intervals_table}"
-
-    migration_required = False
-    new_snapshots = []
-    new_intervals = []
-
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-
-        if not parsed_snapshot.get("change_category"):
-            fingerprint = parsed_snapshot.get("fingerprint")
-            version = _hash(
-                [
-                    fingerprint["data_hash"],
-                    fingerprint["parent_data_hash"],
-                ]
-            )
-            parsed_snapshot["change_category"] = (
-                4 if version == parsed_snapshot.get("version") else 5
-            )
-            migration_required = True
-
-        def _add_interval(start_ts: int, end_ts: int, is_dev: bool) -> None:
-            new_intervals.append(
-                {
-                    "id": random_id(),
-                    "created_ts": now_timestamp(),
-                    "name": name,
-                    "identifier": identifier,
-                    "version": version,
-                    "start_ts": start_ts,
-                    "end_ts": end_ts,
-                    "is_dev": is_dev,
-                    "is_removed": False,
-                    "is_compacted": True,
-                }
-            )
-
-        for interval in parsed_snapshot.pop("intervals", []):
-            _add_interval(interval[0], interval[1], False)
-            migration_required = True
-
-        for interval in parsed_snapshot.pop("dev_intervals", []):
-            _add_interval(interval[0], interval[1], True)
-            migration_required = True
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-            }
-        )
-
-    if migration_required:
-        index_type = index_text_type(engine_adapter.dialect)
-
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )
-
-        if new_intervals:
-            engine_adapter.insert_append(
-                intervals_table,
-                pd.DataFrame(new_intervals),
-                target_columns_to_types={
-                    "id": exp.DataType.build(index_type),
-                    "created_ts": exp.DataType.build("bigint"),
-                    "name": exp.DataType.build(index_type),
-                    "identifier": exp.DataType.build(index_type),
-                    "version": exp.DataType.build(index_type),
-                    "start_ts": exp.DataType.build("bigint"),
-                    "end_ts": exp.DataType.build("bigint"),
-                    "is_dev": exp.DataType.build("boolean"),
-                    "is_removed": exp.DataType.build("boolean"),
-                    "is_compacted": exp.DataType.build("boolean"),
-                },
-            )
-
-
-def _hash(data):  # type: ignore
-    return str(zlib.crc32(";".join("" if d is None else d for d in data).encode("utf-8")))

sqlmesh/migrations/v0026_remove_dialect_from_seed.py

@@ -1,55 +0,0 @@
-"""Remove dialect from seeds."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        node = parsed_snapshot["node"]
-        if "seed" in node:
-            node["seed"].pop("dialect", None)
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-            }
-        )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )

sqlmesh/migrations/v0027_minute_interval_to_five.py

@@ -1,57 +0,0 @@
-"""Change any interval unit of minute to five_minute."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-
-        node = parsed_snapshot["node"]
-
-        if node.get("interval_unit") == "minute":
-            node["interval_unit"] = "five_minute"
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-            }
-        )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )

sqlmesh/migrations/v0028_add_plan_dags_table.py

@@ -1,29 +0,0 @@
-"""Creates the '_plan_dags' table if Airflow is used."""
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    plan_dags_table = "_plan_dags"
-
-    if schema:
-        engine_adapter.create_schema(schema)
-        plan_dags_table = f"{schema}.{plan_dags_table}"
-
-    index_type = index_text_type(engine_adapter.dialect)
-
-    engine_adapter.create_state_table(
-        plan_dags_table,
-        {
-            "request_id": exp.DataType.build(index_type),
-            "dag_id": exp.DataType.build(index_type),
-            "dag_spec": exp.DataType.build("text"),
-        },
-        primary_key=("request_id",),
-    )
-
-    engine_adapter.create_index(plan_dags_table, "dag_id_idx", ("dag_id",))

sqlmesh/migrations/v0029_generate_schema_types_using_dialect.py

@@ -1,69 +0,0 @@
-"""Generate mapping schema data types using the corresponding model's dialect."""
-
-import json
-
-from sqlglot import exp, parse_one
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        node = parsed_snapshot["node"]
-
-        mapping_schema = node.get("mapping_schema")
-        if mapping_schema:
-            node["mapping_schema"] = _convert_schema_types(mapping_schema, node["dialect"])
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-            }
-        )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )
-
-
-def _convert_schema_types(schema, dialect):  # type: ignore
-    if not schema:
-        return schema
-
-    for k, v in schema.items():
-        if isinstance(v, dict):
-            _convert_schema_types(v, dialect)
-        else:
-            schema[k] = parse_one(v).sql(dialect=dialect)
-
-    return schema

sqlmesh/migrations/v0030_update_unrestorable_snapshots.py

@@ -1,65 +0,0 @@
-"""Update unrestorable snapshots."""
-
-import json
-import typing as t
-from collections import defaultdict
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync: t.Any, **kwargs: t.Any) -> None:  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-    snapshots_by_version = defaultdict(list)
-
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        snapshots_by_version[(name, version)].append((identifier, kind_name, parsed_snapshot))
-
-    for (name, version), snapshots in snapshots_by_version.items():
-        has_forward_only = any(s["change_category"] == 3 for _, _, s in snapshots)
-        for identifier, kind_name, snapshot in snapshots:
-            if (
-                has_forward_only
-                and snapshot["change_category"] != 3
-                and not snapshot.get("unpaused_ts")
-            ):
-                snapshot["unrestorable"] = True
-            new_snapshots.append(
-                {
-                    "name": name,
-                    "identifier": identifier,
-                    "version": version,
-                    "snapshot": json.dumps(snapshot),
-                    "kind_name": kind_name,
-                }
-            )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )

sqlmesh/migrations/v0031_remove_dbt_target_fields.py

@@ -1,65 +0,0 @@
-"""Remove dbt target fields from snapshots outside of limited list of approved fields"""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-    found_dbt_target = False
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        node = parsed_snapshot["node"]
-        dbt_target = node.get("jinja_macros", {}).get("global_objs", {}).get("target", {})
-        # Double check that `target_name` exists as a field since we know that all dbt targets have `target_name`
-        # We do this in case someone has a target macro defined that is not related to dbt
-        if dbt_target and dbt_target.get("target_name"):
-            found_dbt_target = True
-            node["jinja_macros"]["global_objs"]["target"] = {
-                "type": dbt_target.get("type", "None"),
-                "name": dbt_target.get("name", "None"),
-                "schema": dbt_target.get("schema", "None"),
-                "database": dbt_target.get("database", "None"),
-                "target_name": dbt_target["target_name"],
-            }
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-            }
-        )
-
-    if found_dbt_target:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )

sqlmesh/migrations/v0032_add_sqlmesh_version.py

@@ -1,25 +0,0 @@
-"""Add new 'sqlmesh_version' column to the version state table."""
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    engine_adapter = state_sync.engine_adapter
-    versions_table = "_versions"
-    if state_sync.schema:
-        versions_table = f"{state_sync.schema}.{versions_table}"
-    index_type = index_text_type(engine_adapter.dialect)
-    alter_table_exp = exp.Alter(
-        this=exp.to_table(versions_table),
-        kind="TABLE",
-        actions=[
-            exp.ColumnDef(
-                this=exp.to_column("sqlmesh_version"),
-                kind=exp.DataType.build(index_type),
-            )
-        ],
-    )
-
-    engine_adapter.execute(alter_table_exp)

sqlmesh/migrations/v0033_mysql_fix_blob_text_type.py

@@ -1,45 +0,0 @@
-"""Use LONGTEXT type for blob fields in MySQL."""
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import blob_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    engine_adapter = state_sync.engine_adapter
-    if engine_adapter.dialect != "mysql":
-        return
-
-    schema = state_sync.schema
-    environments_table = "_environments"
-    snapshots_table = "_snapshots"
-    seeds_table = "_seeds"
-    plan_dags_table = "_plan_dags"
-
-    if schema:
-        environments_table = f"{schema}.{environments_table}"
-        snapshots_table = f"{schema}.{snapshots_table}"
-        seeds_table = f"{state_sync.schema}.{seeds_table}"
-        plan_dags_table = f"{schema}.{plan_dags_table}"
-
-    targets = [
-        (environments_table, "snapshots"),
-        (snapshots_table, "snapshot"),
-        (seeds_table, "content"),
-        (plan_dags_table, "dag_spec"),
-    ]
-
-    for table_name, column_name in targets:
-        blob_type = blob_text_type(engine_adapter.dialect)
-        alter_table_exp = exp.Alter(
-            this=exp.to_table(table_name),
-            kind="TABLE",
-            actions=[
-                exp.AlterColumn(
-                    this=exp.to_column(column_name),
-                    dtype=exp.DataType.build(blob_type),
-                )
-            ],
-        )
-
-        engine_adapter.execute(alter_table_exp)