sqlmesh 0.181.1.dev5__py3-none-any.whl → 0.181.2.dev1__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.181.1.dev5'
-__version_tuple__ = version_tuple = (0, 181, 1, 'dev5')
+__version__ = version = '0.181.2.dev1'
+__version_tuple__ = version_tuple = (0, 181, 2, 'dev1')

sqlmesh/cli/main.py

@@ -546,6 +546,19 @@ def janitor(ctx: click.Context, ignore_ttl: bool, **kwargs: t.Any) -> None:
     ctx.obj.run_janitor(ignore_ttl, **kwargs)
 
 
+@cli.command("destroy")
+@click.pass_context
+@error_handler
+@cli_analytics
+def destroy(ctx: click.Context, **kwargs: t.Any) -> None:
+    """
+    The destroy command removes all project resources.
+
+    This includes engine-managed objects, state tables, the SQLMesh cache and any build artifacts.
+    """
+    ctx.obj.destroy(**kwargs)
+
+
 @cli.command("dag")
 @click.argument("file", required=True)
 @click.option(

sqlmesh/core/console.py

@@ -181,6 +181,26 @@ class JanitorConsole(abc.ABC):
         """
 
 
+class DestroyConsole(abc.ABC):
+    """Console for describing a destroy operation"""
+
+    @abc.abstractmethod
+    def start_destroy(self) -> bool:
+        """Start a destroy operation.
+
+        Returns:
+            Whether or not the destroy operation should proceed
+        """
+
+    @abc.abstractmethod
+    def stop_destroy(self, success: bool = True) -> None:
+        """Indicates the destroy operation has ended
+
+        Args:
+            success: Whether or not the cleanup completed successfully
+        """
+
+
 class EnvironmentsConsole(abc.ABC):
     """Console for displaying environments"""
 
@@ -304,6 +324,7 @@ class Console(
     StateExporterConsole,
     StateImporterConsole,
     JanitorConsole,
+    DestroyConsole,
     EnvironmentsConsole,
     DifferenceConsole,
     TableDiffConsole,
@@ -744,6 +765,12 @@ class NoopConsole(Console):
     ) -> None:
         pass
 
+    def start_destroy(self) -> bool:
+        return True
+
+    def stop_destroy(self, success: bool = True) -> None:
+        pass
+
 
 def make_progress_bar(
     message: str,
@@ -1092,6 +1119,25 @@ class TerminalConsole(Console):
         else:
             self.log_error("Cleanup failed!")
 
+    def start_destroy(self) -> bool:
+        self.log_warning(
+            (
+                "This will permanently delete all engine-managed objects, state tables and SQLMesh cache.\n"
+                "The operation is irreversible and may disrupt any currently running or scheduled plans.\n"
+                "Use this command only when you intend to fully reset the project."
+            )
+        )
+        if not self._confirm("Proceed?"):
+            self.log_error("Destroy aborted!")
+            return False
+        return True
+
+    def stop_destroy(self, success: bool = False) -> None:
+        if success:
+            self.log_success("Destroy completed successfully.")
+        else:
+            self.log_error("Destroy failed!")
+
     def start_promotion_progress(
         self,
         snapshots: t.List[SnapshotTableInfo],
@@ -3472,6 +3518,53 @@ class DebuggerTerminalConsole(TerminalConsole):
     ) -> None:
         self._write(row_diff)
 
+    def show_table_diff(
+        self,
+        table_diffs: t.List[TableDiff],
+        show_sample: bool = True,
+        skip_grain_check: bool = False,
+        temp_schema: t.Optional[str] = None,
+    ) -> None:
+        for table_diff in table_diffs:
+            self.show_table_diff_summary(table_diff)
+            self.show_schema_diff(table_diff.schema_diff())
+            self.show_row_diff(
+                table_diff.row_diff(temp_schema=temp_schema, skip_grain_check=skip_grain_check),
+                show_sample=show_sample,
+                skip_grain_check=skip_grain_check,
+            )
+
+    def update_table_diff_progress(self, model: str) -> None:
+        self._write(f"Finished table diff for: {model}")
+
+    def start_table_diff_progress(self, models_to_diff: int) -> None:
+        self._write("Table diff started")
+
+    def start_table_diff_model_progress(self, model: str) -> None:
+        self._write(f"Calculating differences for: {model}")
+
+    def stop_table_diff_progress(self, success: bool) -> None:
+        self._write(f"Table diff finished with success={success}")
+
+    def show_table_diff_details(
+        self,
+        models_to_diff: t.List[str],
+    ) -> None:
+        if models_to_diff:
+            models = "\n".join(models_to_diff)
+            self._write(f"Models to compare: {models}")
+
+    def show_table_diff_summary(self, table_diff: TableDiff) -> None:
+        if table_diff.model_name:
+            self._write(f"Model: {table_diff.model_name}")
+            self._write(f"Source env: {table_diff.source_alias}")
+            self._write(f"Target env: {table_diff.target_alias}")
+        self._write(f"Source table: {table_diff.source}")
+        self._write(f"Target table: {table_diff.target}")
+        _, _, key_column_names = table_diff.key_columns
+        keys = ", ".join(key_column_names)
+        self._write(f"Join On: {keys}")
+
 
 _CONSOLE: Console = NoopConsole()
 

sqlmesh/core/context.py

@@ -832,6 +832,19 @@ class GenericContext(BaseContext, t.Generic[C]):
 
         return success
 
+    @python_api_analytics
+    def destroy(self) -> bool:
+        success = False
+
+        if self.console.start_destroy():
+            try:
+                self._destroy()
+                success = True
+            finally:
+                self.console.stop_destroy(success=success)
+
+        return success
+
     @t.overload
     def get_model(
         self, model_or_snapshot: ModelOrSnapshot, raise_if_missing: Literal[True] = True
@@ -2537,6 +2550,22 @@ class GenericContext(BaseContext, t.Generic[C]):
             gateway_managed_virtual_layer=self.gateway_managed_virtual_layer,
         )
 
+    def _destroy(self) -> None:
+        # Invalidate all environments, including prod
+        for environment in self.state_reader.get_environments():
+            self.state_sync.invalidate_environment(name=environment.name, protect_prod=False)
+            self.console.log_success(f"Environment '{environment.name}' invalidated.")
+
+        # Run janitor to clean up all objects
+        self._run_janitor(ignore_ttl=True)
+
+        # Remove state tables, including backup tables
+        self.state_sync.remove_state(including_backup=True)
+        self.console.log_status_update("State tables removed.")
+
+        # Finally clear caches
+        self.clear_caches()
+
     def _run_janitor(self, ignore_ttl: bool = False) -> None:
         current_ts = now_timestamp()
 

sqlmesh/core/state_sync/base.py

@@ -347,13 +347,18 @@ class StateSync(StateReader, abc.ABC):
         """
 
     @abc.abstractmethod
-    def invalidate_environment(self, name: str) -> None:
+    def invalidate_environment(self, name: str, protect_prod: bool = True) -> None:
         """Invalidates the target environment by setting its expiration timestamp to now.
 
         Args:
             name: The name of the environment to invalidate.
+            protect_prod: If True, prevents invalidation of the production environment.
         """
 
+    @abc.abstractmethod
+    def remove_state(self, including_backup: bool = False) -> None:
+        """Removes the state store objects."""
+
     @abc.abstractmethod
     def remove_intervals(
         self,

sqlmesh/core/state_sync/db/environment.py

@@ -108,14 +108,15 @@ class EnvironmentState:
                 columns_to_types=self._environment_statements_columns_to_types,
             )
 
-    def invalidate_environment(self, name: str) -> None:
+    def invalidate_environment(self, name: str, protect_prod: bool = True) -> None:
         """Invalidates the environment.
 
         Args:
             name: The name of the environment
+            protect_prod: If True, prevents invalidation of the production environment.
         """
         name = name.lower()
-        if name == c.PROD:
+        if protect_prod and name == c.PROD:
             raise SQLMeshError("Cannot invalidate the production environment.")
 
         filter_expr = exp.column("name").eq(name)

sqlmesh/core/state_sync/db/facade.py

@@ -59,7 +59,7 @@ from sqlmesh.core.state_sync.db.interval import IntervalState
 from sqlmesh.core.state_sync.db.environment import EnvironmentState
 from sqlmesh.core.state_sync.db.snapshot import SnapshotState
 from sqlmesh.core.state_sync.db.version import VersionState
-from sqlmesh.core.state_sync.db.migrator import StateMigrator
+from sqlmesh.core.state_sync.db.migrator import StateMigrator, _backup_table_name
 from sqlmesh.utils.date import TimeLike, to_timestamp, time_like_to_str, now_timestamp
 from sqlmesh.utils.errors import ConflictingPlanError, SQLMeshError
 
@@ -270,8 +270,8 @@ class EngineAdapterStateSync(StateSync):
     ) -> None:
         self.snapshot_state.unpause_snapshots(snapshots, unpaused_dt, self.interval_state)
 
-    def invalidate_environment(self, name: str) -> None:
-        self.environment_state.invalidate_environment(name)
+    def invalidate_environment(self, name: str, protect_prod: bool = True) -> None:
+        self.environment_state.invalidate_environment(name, protect_prod)
 
     def get_expired_snapshots(
         self, current_ts: int, ignore_ttl: bool = False
@@ -313,18 +313,26 @@ class EngineAdapterStateSync(StateSync):
     def nodes_exist(self, names: t.Iterable[str], exclude_external: bool = False) -> t.Set[str]:
         return self.snapshot_state.nodes_exist(names, exclude_external)
 
-    def reset(self, default_catalog: t.Optional[str]) -> None:
-        """Resets the state store to the state when it was first initialized."""
+    def remove_state(self, including_backup: bool = False) -> None:
+        """Removes the state store objects."""
         for table in (
             self.snapshot_state.snapshots_table,
             self.snapshot_state.auto_restatements_table,
             self.environment_state.environments_table,
+            self.environment_state.environment_statements_table,
             self.interval_state.intervals_table,
             self.plan_dags_table,
             self.version_state.versions_table,
         ):
             self.engine_adapter.drop_table(table)
+            if including_backup:
+                self.engine_adapter.drop_table(_backup_table_name(table))
+
         self.snapshot_state.clear_cache()
+
+    def reset(self, default_catalog: t.Optional[str]) -> None:
+        """Resets the state store to the state when it was first initialized."""
+        self.remove_state()
         self.migrate(default_catalog)
 
     @transactional()

sqlmesh/magics.py

@@ -1111,6 +1111,13 @@ class SQLMeshMagics(Magics):
         args = parse_argstring(self.lint, line)
         context.lint_models(args.models)
 
+    @magic_arguments()
+    @line_magic
+    @pass_sqlmesh_context
+    def destroy(self, context: Context, line: str) -> None:
+        """Removes all project resources, engine-managed objects, state tables and clears the SQLMesh cache."""
+        context.destroy()
+
 
 def register_magics() -> None:
     try:

{sqlmesh-0.181.1.dev5.dist-info → sqlmesh-0.181.2.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.181.1.dev5
+Version: 0.181.2.dev1
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.181.1.dev5.dist-info → sqlmesh-0.181.2.dev1.dist-info}/RECORD

@@ -1,19 +1,19 @@
 sqlmesh/__init__.py,sha256=uPXld_xTZ2NTruSfVJ_vR_w-uAxR3T-gMXAb-EphTe8,7167
-sqlmesh/_version.py,sha256=V8bpUP0oeVbvVW84PK7nOlJ-2zYX3cciqF_2AM_17dM,528
-sqlmesh/magics.py,sha256=7BU9ec9zE4tjvZ3hdEqwFX2VVV-CI0h06TGZo_Ssi8M,39070
+sqlmesh/_version.py,sha256=m2QOcgt0SG7_0zqrgNZNVrrFlbqjzxcFKbczXRa2uTQ,528
+sqlmesh/magics.py,sha256=P5JY9OEpIwKPZa61Nje9f2HQjsIeyVAtPxM85RGMo7s,39334
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/bot.py,sha256=2zlbn-DXkqQzr3lA0__IGU4XaIfXBXBKLWXNI2DRJX8,759
 sqlmesh/cicd/config.py,sha256=mpgkisSBBovlNrNZir2unT8Wes6b38RW2UsU_o4e8vY,109
 sqlmesh/cli/__init__.py,sha256=MAXAyO7isbyM-PaSdFjkJNcjn32WjOl0XzxT5gvp_vo,1690
 sqlmesh/cli/example_project.py,sha256=QAFQbw_kzqn30GLTthqsuWXbFDBpX8W6RsCgQ-jx65M,9308
-sqlmesh/cli/main.py,sha256=foSIAx3hyhCWLIG7XQuBE-zH7lNqSvazkgpydtJados,31564
+sqlmesh/cli/main.py,sha256=Fd9nFPMJShtzLH1HXFVO1tPUATts1sziSJGYREWxmSE,31899
 sqlmesh/cli/options.py,sha256=aC_L5WnMScKzMU55BD1Y-8R-QzqCkPSYrlCI-payHRA,1333
 sqlmesh/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/core/_typing.py,sha256=e1f7xO2aZCy38M_3VoDy-XI3shyd99-IFctgIFSgXWM,497
-sqlmesh/core/console.py,sha256=ERAYluCseNnRxWXpgtNE0MYut5CCWGoyHdPKMnYv0Fc,146135
+sqlmesh/core/console.py,sha256=qPMcYwr4ktv8BJe7-2vYI3N5534mdviihK4p9YipuMQ,149415
 sqlmesh/core/constants.py,sha256=oqt6zjepUhQoWlgapKn_HpG7Pu9mrFwepdX5caNfJUI,2499
-sqlmesh/core/context.py,sha256=CPrmMHceymyDjjgKfkszwWxAkjTUqO90TbGfozkZJWw,113720
+sqlmesh/core/context.py,sha256=kLKXwOLc_5Orv-SxvI0zbpL9ZJ0XkB9HZgd9R6uh-9Y,114691
 sqlmesh/core/context_diff.py,sha256=s9toP17d4tJfOpFH91LcjGK2CM64PVuB0MPkeFgF4hQ,20415
 sqlmesh/core/dialect.py,sha256=fPTV517IOS0nWMxYznNNQnXkQkFFrPOSKe8ghsQ0o8A,49759
 sqlmesh/core/environment.py,sha256=0TTQf4C4LXwf37fm-dfYv1qyG7vtZfQI5rizgm5g0j0,11442
@@ -102,13 +102,13 @@ sqlmesh/core/snapshot/categorizer.py,sha256=qGG5870mu1jR2whKisSBH_GFbsyh96RnzJDg
 sqlmesh/core/snapshot/definition.py,sha256=PKzBLiQ1yIycGbmslEoQyJEwtfnJHsQuY_sl1xkBcvk,84081
 sqlmesh/core/snapshot/evaluator.py,sha256=NZVgQoaSH-D7Q7PAHTRzQ5O2pdra3MUToLi9p8qAyVg,89265
 sqlmesh/core/state_sync/__init__.py,sha256=B60xj-_Xk-4jas3UphLvwkuf04G-9AyGckcblXE0aRg,869
-sqlmesh/core/state_sync/base.py,sha256=1gQzl8GCf0yYVfAtGYnNF86lU6rW1MAXB6Lcctd2mBs,18186
+sqlmesh/core/state_sync/base.py,sha256=M3EsZ_HyV_6aCKucwdmZPWV9QFNvHm5mCCgSaQ-mE7c,18441
 sqlmesh/core/state_sync/cache.py,sha256=xKLPOrPiQeRDPMkUtSlZaMMVlUnnK2ytxGE2rilHRBY,5446
 sqlmesh/core/state_sync/common.py,sha256=THK-6AGHFBAuLXydD8QhzmzCdyl0BMrz2rB2xpmC99Y,6532
 sqlmesh/core/state_sync/export_import.py,sha256=WJ7sFX2b_RPAFE4yspJRi_4JQnaO23xpJgRxwXD0agA,8709
 sqlmesh/core/state_sync/db/__init__.py,sha256=jGCqKYB7npq4eh6H2LyfBAJWdOP-IFu0DC4CMiQgYvw,107
-sqlmesh/core/state_sync/db/environment.py,sha256=7HVRkRugHsGgyHZuSMil6NyTtx7B02QQN-AE18GSJWM,13382
-sqlmesh/core/state_sync/db/facade.py,sha256=2HcuQt8PrHXHbv3VqvzHhXHDW6_aRwTfY8oX_ntGsAs,25045
+sqlmesh/core/state_sync/db/environment.py,sha256=pwc2eRxZnIiTM8ZTHSfiOslO4FqFz9ch91J3VK6U1b8,13514
+sqlmesh/core/state_sync/db/facade.py,sha256=F3B6M1x3rM9XNq6L4UvpXKOf-uZUGgRPlJdIp_-mrWo,25423
 sqlmesh/core/state_sync/db/interval.py,sha256=QgdqoVyLYw15SnX8O6icHUF3ZHnI6a9rX0505-FgFNg,17964
 sqlmesh/core/state_sync/db/migrator.py,sha256=2LNUwOBNFkA1ntp-Z93S_OWxFQhOBj3ZD_ZqPCyPEAE,18139
 sqlmesh/core/state_sync/db/snapshot.py,sha256=04b2lan3KamXuX3bdnhOiaoQpXgwRFMjW08tgWIL0Qk,29596
@@ -265,7 +265,7 @@ sqlmesh/utils/pandas.py,sha256=j_tYhuPcWpzuPceZZ6IhdTVbmxYyGJW7l4FHr4bQqjM,1824
 sqlmesh/utils/pydantic.py,sha256=ioaAzYF42PutEJRyAS11-AHguiCG3yWI6VRcTVa2G-g,10470
 sqlmesh/utils/rich.py,sha256=xNZyPCqEc4TAlJlsEthg-5eYUWpaizz0QjBzO7a1Yv4,1509
 sqlmesh/utils/yaml.py,sha256=np-WVcwmwsKghCgAj1OBBMsK1SXSRn4wR-H37cGiFr8,2282
-sqlmesh-0.181.1.dev5.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.181.2.dev1.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=cQRIlDgkl6OgeWuHI9hM8bJhKUE9-Ji2NpQa2hlgNM8,5472
 web/client/dist/assets/Audits-4U9LnWyN.js,sha256=reXu-sKJ76kiaAZIGkCPvgO68Km9tj1PN_9mx6X6WAY,997
@@ -382,8 +382,8 @@ web/server/api/endpoints/models.py,sha256=SzS5bshzdXyM6hb948sZ_3k-xhpGCHkxkb7pHz
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=OYuQ_4dgbhG9ibTcX1a8mvYgH3ANxw9Fhbb4wg6QWqc,9222
 web/server/api/endpoints/table_diff.py,sha256=Wd5YxeKbZB7AkrDvqFgvV9ZQlnKYOF1n5W_w2EHE_Zw,2085
-sqlmesh-0.181.1.dev5.dist-info/METADATA,sha256=4zzJW4BjZ10b_SHIa919RjR9Auj8Wfi-V2P402pQ-0c,25307
-sqlmesh-0.181.1.dev5.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-sqlmesh-0.181.1.dev5.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.181.1.dev5.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.181.1.dev5.dist-info/RECORD,,
+sqlmesh-0.181.2.dev1.dist-info/METADATA,sha256=Qre3CG503vJupcINvEMW5Wx5UsIBtEyX451cTlnIL4E,25307
+sqlmesh-0.181.2.dev1.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+sqlmesh-0.181.2.dev1.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.181.2.dev1.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.181.2.dev1.dist-info/RECORD,,