sqlmesh 0.199.1.dev34__py3-none-any.whl → 0.199.1.dev35__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.199.1.dev34'
-__version_tuple__ = version_tuple = (0, 199, 1, 'dev34')
+__version__ = version = '0.199.1.dev35'
+__version_tuple__ = version_tuple = (0, 199, 1, 'dev35')

{sqlmesh-0.199.1.dev34.dist-info → sqlmesh-0.199.1.dev35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.199.1.dev34
+Version: 0.199.1.dev35
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.199.1.dev34.dist-info → sqlmesh-0.199.1.dev35.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=3sW3IsgIqRv-yyS7RFt31EBonk0WMykh--mr6eY-8SE,530
+sqlmesh/_version.py,sha256=S9lEVv3QVWQs4d4mxXKYzMCuqOotm2m6JhCf7IGStXs,530
 sqlmesh/magics.py,sha256=tdkXrxkZug6LZ5F8TH4lx_XNV3T8bHGYTJUlGJfPQ4A,41777
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -150,9 +150,7 @@ sqlmesh/dbt/converter/jinja.py,sha256=LwtMRWcVYv16cVilOekBjoFfnD-OBfaP90JpYy34ji
 sqlmesh/dbt/converter/jinja_builtins.py,sha256=EVWyDNn0wnSrn9T5TkmKdBBVdw0TzROlfwnSVgnMehM,4589
 sqlmesh/dbt/converter/jinja_transforms.py,sha256=MkgL3Wky2mHzLaEIqkCwiYQk99jL19Wt2pOoXhsqtqQ,17597
 sqlmesh/engines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sqlmesh/engines/commands.py,sha256=Fjy_seH1JL2EZxFNJBr0-eIaOn5PNY-JEb0nBbcr6Go,6094
 sqlmesh/engines/spark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sqlmesh/engines/spark/app.py,sha256=du4Hh-3A7yHak54_CjL0W87X_Kl3btzHpUy9koIQXf4,3748
 sqlmesh/engines/spark/db_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/engines/spark/db_api/errors.py,sha256=LSXuHHhkzHvKlcdwmhCM9qm51KyKoP8gQH-p_W3Oow4,148
 sqlmesh/engines/spark/db_api/spark_session.py,sha256=uQP0VFFYtp3N9mK7zKw6SoiGVhJDDYPylqx46yDwGEc,4667
@@ -284,7 +282,7 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
 sqlmesh/utils/rich.py,sha256=BpB47G0-JcU41LkGkeGKroexj4NOV5fox4-GOIXPgIc,3545
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.199.1.dev34.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.199.1.dev35.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=DAv6ABGogw4xRLb0T_92WP_S7XWyqVhwBgGKWtuQwU4,5472
 web/client/dist/assets/Audits-BuHAmFO8.js,sha256=zbYUF6WdUGMbnMm01GMNPb7xAJwj-o5wMdL56vPo2qE,1041
@@ -402,8 +400,8 @@ web/server/api/endpoints/models.py,sha256=kwj0s7uve3iZSMfmjkoPVMFMeY1sD0peTeyrWf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.199.1.dev34.dist-info/METADATA,sha256=q_vQrhtmnSFSZk0NLlSXgPVsNk_om42U68wH4ejEOkk,26659
-sqlmesh-0.199.1.dev34.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.199.1.dev34.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.199.1.dev34.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.199.1.dev34.dist-info/RECORD,,
+sqlmesh-0.199.1.dev35.dist-info/METADATA,sha256=2O2l5ufMQveHcekXHm4RlUfVObh5Pq_sSG_gm5BLqWE,26659
+sqlmesh-0.199.1.dev35.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.199.1.dev35.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.199.1.dev35.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.199.1.dev35.dist-info/RECORD,,

sqlmesh/engines/commands.py

@@ -1,184 +0,0 @@
-import typing as t
-from enum import Enum
-
-from sqlglot import exp
-from sqlmesh.core.environment import Environment, EnvironmentNamingInfo
-from sqlmesh.core.snapshot import (
-    DeployabilityIndex,
-    Snapshot,
-    SnapshotEvaluator,
-    SnapshotId,
-    SnapshotTableCleanupTask,
-    SnapshotTableInfo,
-)
-from sqlmesh.core.state_sync import cleanup_expired_views
-from sqlmesh.utils.date import TimeLike
-from sqlmesh.utils.errors import AuditError
-from sqlmesh.utils.pydantic import PydanticModel
-
-COMMAND_PAYLOAD_FILE_NAME = "payload.json"
-
-
-class CommandType(str, Enum):
-    EVALUATE = "evaluate"
-    PROMOTE = "promote"
-    DEMOTE = "demote"
-    CLEANUP = "cleanup"
-    CREATE_TABLES = "create_tables"
-    MIGRATE_TABLES = "migrate_tables"
-
-    # This makes it easy to integrate with argparse
-    def __str__(self) -> str:
-        return self.value
-
-
-class EvaluateCommandPayload(PydanticModel):
-    snapshot: Snapshot
-    parent_snapshots: t.Dict[str, Snapshot]
-    start: TimeLike
-    end: TimeLike
-    execution_time: TimeLike
-    deployability_index: DeployabilityIndex
-    batch_index: int
-
-
-class PromoteCommandPayload(PydanticModel):
-    snapshots: t.List[Snapshot]
-    environment_naming_info: EnvironmentNamingInfo
-    deployability_index: DeployabilityIndex
-
-
-class DemoteCommandPayload(PydanticModel):
-    snapshots: t.List[SnapshotTableInfo]
-    environment_naming_info: EnvironmentNamingInfo
-
-
-class CleanupCommandPayload(PydanticModel):
-    environments: t.List[Environment]
-    tasks: t.List[SnapshotTableCleanupTask]
-
-
-class CreateTablesCommandPayload(PydanticModel):
-    target_snapshot_ids: t.List[SnapshotId]
-    snapshots: t.List[Snapshot]
-    deployability_index: DeployabilityIndex
-    allow_destructive_snapshots: t.Set[str]
-
-
-class MigrateTablesCommandPayload(PydanticModel):
-    target_snapshot_ids: t.List[SnapshotId]
-    snapshots: t.List[Snapshot]
-    allow_destructive_snapshots: t.Set[str]
-
-
-def evaluate(
-    evaluator: SnapshotEvaluator, command_payload: t.Union[str, EvaluateCommandPayload]
-) -> None:
-    if isinstance(command_payload, str):
-        command_payload = EvaluateCommandPayload.parse_raw(command_payload)
-
-    parent_snapshots = command_payload.parent_snapshots
-    parent_snapshots[command_payload.snapshot.name] = command_payload.snapshot
-
-    wap_id = evaluator.evaluate(
-        command_payload.snapshot,
-        start=command_payload.start,
-        end=command_payload.end,
-        execution_time=command_payload.execution_time,
-        snapshots=parent_snapshots,
-        deployability_index=command_payload.deployability_index,
-        batch_index=command_payload.batch_index,
-    )
-    audit_results = evaluator.audit(
-        snapshot=command_payload.snapshot,
-        start=command_payload.start,
-        end=command_payload.end,
-        execution_time=command_payload.execution_time,
-        snapshots=parent_snapshots,
-        deployability_index=command_payload.deployability_index,
-        wap_id=wap_id,
-    )
-
-    failed_audit_result = next((r for r in audit_results if r.count and r.blocking), None)
-    if failed_audit_result:
-        raise AuditError(
-            audit_name=failed_audit_result.audit.name,
-            audit_args=failed_audit_result.audit_args,
-            model=command_payload.snapshot.model_or_none,
-            count=t.cast(int, failed_audit_result.count),
-            query=t.cast(exp.Query, failed_audit_result.query),
-            adapter_dialect=evaluator.adapter.dialect,
-        )
-
-
-def promote(
-    evaluator: SnapshotEvaluator, command_payload: t.Union[str, PromoteCommandPayload]
-) -> None:
-    if isinstance(command_payload, str):
-        command_payload = PromoteCommandPayload.parse_raw(command_payload)
-    evaluator.promote(
-        command_payload.snapshots,
-        command_payload.environment_naming_info,
-        deployability_index=command_payload.deployability_index,
-    )
-
-
-def demote(
-    evaluator: SnapshotEvaluator, command_payload: t.Union[str, DemoteCommandPayload]
-) -> None:
-    if isinstance(command_payload, str):
-        command_payload = DemoteCommandPayload.parse_raw(command_payload)
-    evaluator.demote(
-        command_payload.snapshots,
-        command_payload.environment_naming_info,
-    )
-
-
-def cleanup(
-    evaluator: SnapshotEvaluator, command_payload: t.Union[str, CleanupCommandPayload]
-) -> None:
-    if isinstance(command_payload, str):
-        command_payload = CleanupCommandPayload.parse_raw(command_payload)
-
-    cleanup_expired_views(evaluator.adapter, evaluator.adapters, command_payload.environments)
-    evaluator.cleanup(command_payload.tasks)
-
-
-def create_tables(
-    evaluator: SnapshotEvaluator,
-    command_payload: t.Union[str, CreateTablesCommandPayload],
-) -> None:
-    if isinstance(command_payload, str):
-        command_payload = CreateTablesCommandPayload.parse_raw(command_payload)
-
-    snapshots_by_id = {s.snapshot_id: s for s in command_payload.snapshots}
-    target_snapshots = [snapshots_by_id[sid] for sid in command_payload.target_snapshot_ids]
-    evaluator.create(
-        target_snapshots,
-        snapshots_by_id,
-        deployability_index=command_payload.deployability_index,
-        allow_destructive_snapshots=command_payload.allow_destructive_snapshots,
-    )
-
-
-def migrate_tables(
-    evaluator: SnapshotEvaluator,
-    command_payload: t.Union[str, MigrateTablesCommandPayload],
-) -> None:
-    if isinstance(command_payload, str):
-        command_payload = MigrateTablesCommandPayload.parse_raw(command_payload)
-    snapshots_by_id = {s.snapshot_id: s for s in command_payload.snapshots}
-    target_snapshots = [snapshots_by_id[sid] for sid in command_payload.target_snapshot_ids]
-    evaluator.migrate(
-        target_snapshots, snapshots_by_id, command_payload.allow_destructive_snapshots
-    )
-
-
-COMMAND_HANDLERS: t.Dict[CommandType, t.Callable[[SnapshotEvaluator, str], None]] = {
-    CommandType.EVALUATE: evaluate,
-    CommandType.PROMOTE: promote,
-    CommandType.DEMOTE: demote,
-    CommandType.CLEANUP: cleanup,
-    CommandType.CREATE_TABLES: create_tables,
-    CommandType.MIGRATE_TABLES: migrate_tables,
-}

sqlmesh/engines/spark/app.py

@@ -1,114 +0,0 @@
-import argparse
-import logging
-import os
-import tempfile
-
-from pyspark import SparkFiles
-from pyspark.sql import SparkSession
-
-from sqlmesh.core.engine_adapter import create_engine_adapter
-from sqlmesh.core.snapshot import SnapshotEvaluator
-from sqlmesh.engines import commands
-from sqlmesh.engines.spark.db_api import spark_session as spark_session_db
-from sqlmesh.engines.spark.db_api.errors import NotSupportedError
-from sqlmesh.utils.errors import SQLMeshError
-
-logger = logging.getLogger(__name__)
-
-
-def get_or_create_spark_session(dialect: str) -> SparkSession:
-    if dialect == "databricks":
-        spark = SparkSession.getActiveSession()
-        if not spark:
-            raise SQLMeshError("Could not find an active SparkSession.")
-        return spark
-    return (
-        SparkSession.builder.config("spark.scheduler.mode", "FAIR")
-        .enableHiveSupport()
-        .getOrCreate()
-    )
-
-
-def main(
-    dialect: str,
-    default_catalog: str,
-    command_type: commands.CommandType,
-    ddl_concurrent_tasks: int,
-    payload_path: str,
-) -> None:
-    if dialect not in ("databricks", "spark"):
-        raise NotSupportedError(
-            f"Dialect '{dialect}' not supported. Must be either 'databricks' or 'spark'"
-        )
-    logging.basicConfig(
-        format="%(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)",
-        level=logging.INFO,
-    )
-    command_handler = commands.COMMAND_HANDLERS.get(command_type)
-    if not command_handler:
-        raise NotSupportedError(f"Command '{command_type.value}' not supported")
-
-    spark = get_or_create_spark_session(dialect)
-
-    evaluator = SnapshotEvaluator(
-        create_engine_adapter(
-            lambda: spark_session_db.connection(spark),
-            dialect,
-            default_catalog=default_catalog,
-            multithreaded=ddl_concurrent_tasks > 1,
-            execute_log_level=logging.INFO,
-        ),
-        ddl_concurrent_tasks=ddl_concurrent_tasks,
-    )
-    if dialect == "spark":
-        with open(SparkFiles.get(payload_path), "r", encoding="utf-8") as payload_fd:
-            command_payload = payload_fd.read()
-    else:
-        from pyspark.dbutils import DBUtils  # type: ignore
-
-        dbutils = DBUtils(spark)
-        with tempfile.TemporaryDirectory() as tmp:
-            local_payload_path = os.path.join(tmp, commands.COMMAND_PAYLOAD_FILE_NAME)
-            dbutils.fs.cp(payload_path, f"file://{local_payload_path}")
-            with open(local_payload_path, "r", encoding="utf-8") as payload_fd:
-                command_payload = payload_fd.read()
-    logger.info("Command payload:\n %s", command_payload)
-    command_handler(evaluator, command_payload)
-
-    evaluator.close()
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="SQLMesh Spark Submit App")
-    parser.add_argument(
-        "--dialect",
-        help="The dialect to use when creating the engine adapter.",
-    )
-    parser.add_argument(
-        "--default_catalog",
-        help="The default catalog to use when creating the engine adapter.",
-    )
-    parser.add_argument(
-        "--command_type",
-        type=commands.CommandType,
-        choices=list(commands.CommandType),
-        help="The type of command that is being run",
-    )
-    parser.add_argument(
-        "--ddl_concurrent_tasks",
-        type=int,
-        default=1,
-        help="The number of ddl concurrent tasks to use. Default to 1.",
-    )
-    parser.add_argument(
-        "--payload_path",
-        help="Path to the payload object. Can be a local or remote path.",
-    )
-    args = parser.parse_args()
-    main(
-        args.dialect,
-        args.default_catalog,
-        args.command_type,
-        args.ddl_concurrent_tasks,
-        args.payload_path,
-    )