sqlmesh 0.213.1.dev1__py3-none-any.whl → 0.227.2.dev4__py3-none-any.whl

sqlmesh/migrations/v0011_add_model_kind_name.py

@@ -1,63 +0,0 @@
-"""Add the kind_name column to the snapshots table."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    index_type = index_text_type(engine_adapter.dialect)
-
-    alter_table_exp = exp.Alter(
-        this=exp.to_table(snapshots_table),
-        kind="TABLE",
-        actions=[
-            exp.ColumnDef(
-                this=exp.to_column("kind_name"),
-                kind=exp.DataType.build(index_type),
-            )
-        ],
-    )
-    engine_adapter.execute(alter_table_exp)
-
-    new_snapshots = []
-
-    for name, identifier, version, snapshot in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": snapshot,
-                "kind_name": parsed_snapshot["model"]["kind"]["name"],
-            }
-        )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )

sqlmesh/migrations/v0012_update_jinja_expressions.py

@@ -1,86 +0,0 @@
-"""Fix expressions that contain jinja."""
-
-import json
-import typing as t
-
-from sqlglot import exp
-
-from sqlmesh.utils.jinja import has_jinja
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        audits = parsed_snapshot.get("audits", [])
-        model = parsed_snapshot["model"]
-
-        if "query" in model and has_jinja(model["query"]):
-            model["query"] = _wrap_query(model["query"])
-
-        _wrap_statements(model, "pre_statements")
-        _wrap_statements(model, "post_statements")
-
-        for audit in audits:
-            if has_jinja(audit["query"]):
-                audit["query"] = _wrap_query(audit["query"])
-            _wrap_statements(audit, "expressions")
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-            }
-        )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )
-
-
-def _wrap_statements(obj: t.Dict, key: str) -> None:
-    updated_statements = []
-    for statement in obj.get(key, []):
-        if has_jinja(statement):
-            statement = _wrap_statement(statement)
-        updated_statements.append(statement)
-
-    if updated_statements:
-        obj[key] = updated_statements
-
-
-def _wrap_query(sql: str) -> str:
-    return f"JINJA_QUERY_BEGIN;\n{sql}\nJINJA_END;"
-
-
-def _wrap_statement(sql: str) -> str:
-    return f"JINJA_STATEMENT_BEGIN;\n{sql}\nJINJA_END;"

sqlmesh/migrations/v0013_serde_using_model_dialects.py

@@ -1,87 +0,0 @@
-"""Serialize SQL using the dialect of each model."""
-
-import json
-import typing as t
-
-from sqlglot import exp, parse_one
-
-from sqlmesh.utils.jinja import has_jinja
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        model = parsed_snapshot["model"]
-        dialect = model["dialect"]
-
-        _update_expression(model, "query", dialect)
-        _update_expression_list(model, "pre_statements", dialect)
-        _update_expression_list(model, "post_statements", dialect)
-
-        for audit in parsed_snapshot.get("audits", []):
-            dialect = audit["dialect"]
-            _update_expression(audit, "query", dialect)
-            _update_expression_list(audit, "expressions", dialect)
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-            }
-        )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )
-
-
-# Note: previously we used to do serde using the SQLGlot dialect, so we need to parse the
-# stored queries using that dialect and then write them back using the correct dialect.
-
-
-def _update_expression(obj: t.Dict, key: str, dialect: str) -> None:
-    if key in obj and not has_jinja(obj[key]):
-        obj[key] = parse_one(obj[key]).sql(dialect=dialect)
-
-
-def _update_expression_list(obj: t.Dict, key: str, dialect: str) -> None:
-    if key in obj:
-        obj[key] = [
-            (
-                parse_one(expression).sql(dialect=dialect)
-                if not has_jinja(expression)
-                else expression
-            )
-            for expression in obj[key]
-            if expression
-        ]

sqlmesh/migrations/v0014_fix_dev_intervals.py

@@ -1,14 +0,0 @@
-"""Fix snapshot intervals that have been erroneously marked as dev."""
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    schema = state_sync.schema
-    intervals_table = "_intervals"
-    if schema:
-        intervals_table = f"{schema}.{intervals_table}"
-
-    state_sync.engine_adapter.update_table(
-        intervals_table,
-        {"is_dev": False},
-        where="1=1",
-    )

sqlmesh/migrations/v0015_environment_add_promoted_snapshot_ids.py

@@ -1,26 +0,0 @@
-"""Include a set of snapshot IDs filtered for promotion."""
-
-from sqlglot import exp
-from sqlmesh.utils.migration import blob_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    engine_adapter = state_sync.engine_adapter
-    environments_table = "_environments"
-    if state_sync.schema:
-        environments_table = f"{state_sync.schema}.{environments_table}"
-
-    blob_type = blob_text_type(engine_adapter.dialect)
-
-    alter_table_exp = exp.Alter(
-        this=exp.to_table(environments_table),
-        kind="TABLE",
-        actions=[
-            exp.ColumnDef(
-                this=exp.to_column("promoted_snapshot_ids"),
-                kind=exp.DataType.build(blob_type),
-            )
-        ],
-    )
-
-    engine_adapter.execute(alter_table_exp)

sqlmesh/migrations/v0016_fix_windows_path.py

@@ -1,59 +0,0 @@
-"""Fix paths that have a Windows forward slash in them."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        model = parsed_snapshot["model"]
-        python_env = model.get("python_env")
-        if python_env:
-            for py_definition in python_env.values():
-                path = py_definition.get("path")
-                if path:
-                    py_definition["path"] = path.replace("\\", "/")
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-            }
-        )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )

sqlmesh/migrations/v0017_fix_windows_seed_path.py

@@ -1,55 +0,0 @@
-"""Fix seed paths that have a Windows forward slash in them."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        model_kind = parsed_snapshot["model"]["kind"]
-        if "path" in model_kind:
-            model_kind["path"] = model_kind["path"].replace("\\", "/")
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-            }
-        )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )

sqlmesh/migrations/v0018_rename_snapshot_model_to_node.py

@@ -1,53 +0,0 @@
-"""Replace snapshot model field with node."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        parsed_snapshot["node"] = parsed_snapshot.pop("model")
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-            }
-        )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )

sqlmesh/migrations/v0019_add_env_suffix_target.py

@@ -1,28 +0,0 @@
-"""Add support for environment suffix target."""
-
-from sqlglot import exp
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    engine_adapter = state_sync.engine_adapter
-    environments_table = "_environments"
-    if state_sync.schema:
-        environments_table = f"{state_sync.schema}.{environments_table}"
-
-    alter_table_exp = exp.Alter(
-        this=exp.to_table(environments_table),
-        kind="TABLE",
-        actions=[
-            exp.ColumnDef(
-                this=exp.to_column("suffix_target"),
-                kind=exp.DataType.build("text"),
-            )
-        ],
-    )
-    engine_adapter.execute(alter_table_exp)
-
-    state_sync.engine_adapter.update_table(
-        environments_table,
-        {"suffix_target": "schema"},
-        where="1=1",
-    )

sqlmesh/migrations/v0020_remove_redundant_attributes_from_dbt_models.py

@@ -1,80 +0,0 @@
-"""Remove redundant attributes from dbt models."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        jinja_macros_global_objs = parsed_snapshot["node"]["jinja_macros"]["global_objs"]
-        if "config" in jinja_macros_global_objs and isinstance(
-            jinja_macros_global_objs["config"], dict
-        ):
-            for key in CONFIG_ATTRIBUTE_KEYS_TO_REMOVE:
-                jinja_macros_global_objs["config"].pop(key, None)
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-            }
-        )
-
-    if new_snapshots:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )
-
-
-CONFIG_ATTRIBUTE_KEYS_TO_REMOVE = [
-    "config",
-    "config_call_dict",
-    "depends_on",
-    "dependencies",
-    "metrics",
-    "original_file_path",
-    "packages",
-    "patch_path",
-    "path",
-    "post-hook",
-    "pre-hook",
-    "raw_code",
-    "refs",
-    "resource_type",
-    "sources",
-    "sql",
-    "tests",
-    "unrendered_config",
-]

sqlmesh/migrations/v0021_fix_table_properties.py

@@ -1,62 +0,0 @@
-"""Fix table properties that have extra quoting due to a bug."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.core import dialect as d
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-    found_table_properties = False
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-        table_properties = parsed_snapshot["node"].get("table_properties")
-        if table_properties:
-            found_table_properties = True
-            dialect = parsed_snapshot["node"].get("dialect")
-            parsed_snapshot["node"]["table_properties"] = exp.Tuple(
-                expressions=[
-                    exp.Literal.string(k).eq(d.parse_one(v)) for k, v in table_properties.items()
-                ]
-            ).sql(dialect=dialect)
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-            }
-        )
-
-    if found_table_properties:
-        engine_adapter.delete_from(snapshots_table, "TRUE")
-
-        index_type = index_text_type(engine_adapter.dialect)
-
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )

sqlmesh/migrations/v0022_move_project_to_model.py

@@ -1,54 +0,0 @@
-"""Move project attr from snapshot to model."""
-
-import json
-
-from sqlglot import exp
-
-from sqlmesh.utils.migration import index_text_type
-
-
-def migrate(state_sync, **kwargs):  # type: ignore
-    import pandas as pd
-
-    engine_adapter = state_sync.engine_adapter
-    schema = state_sync.schema
-    snapshots_table = "_snapshots"
-    if schema:
-        snapshots_table = f"{schema}.{snapshots_table}"
-
-    new_snapshots = []
-
-    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
-        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
-        quote_identifiers=True,
-    ):
-        parsed_snapshot = json.loads(snapshot)
-
-        parsed_snapshot["node"]["project"] = parsed_snapshot.pop("project", "")
-
-        new_snapshots.append(
-            {
-                "name": name,
-                "identifier": identifier,
-                "version": version,
-                "snapshot": json.dumps(parsed_snapshot),
-                "kind_name": kind_name,
-            }
-        )
-
-    engine_adapter.delete_from(snapshots_table, "TRUE")
-
-    index_type = index_text_type(engine_adapter.dialect)
-
-    if new_snapshots:
-        engine_adapter.insert_append(
-            snapshots_table,
-            pd.DataFrame(new_snapshots),
-            target_columns_to_types={
-                "name": exp.DataType.build(index_type),
-                "identifier": exp.DataType.build(index_type),
-                "version": exp.DataType.build(index_type),
-                "snapshot": exp.DataType.build("text"),
-                "kind_name": exp.DataType.build(index_type),
-            },
-        )