sqlbuild 0.70.0__tar.gz → 0.71.0__tar.gz

{sqlbuild-0.70.0 → sqlbuild-0.71.0}/Cargo.lock

@@ -572,7 +572,7 @@ checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
 
 [[package]]
 name = "sqlbuild-kata-native"
-version = "0.70.0"
+version = "0.71.0"
 dependencies = [
  "fensu-policy",
  "globset",

{sqlbuild-0.70.0 → sqlbuild-0.71.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlbuild
-Version: 0.70.0
+Version: 0.71.0
 Requires-Dist: duckdb>=1.3.2
 Requires-Dist: filelock>=3.20.3
 Requires-Dist: polyglot-sql>=0.9.2

{sqlbuild-0.70.0 → sqlbuild-0.71.0}/crates/sqlbuild-kata-native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sqlbuild-kata-native"
-version = "0.70.0"
+version = "0.71.0"
 edition.workspace = true
 license.workspace = true
 publish.workspace = true

{sqlbuild-0.70.0 → sqlbuild-0.71.0}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "maturin"
 
 [project]
 name = "sqlbuild"
-version = "0.70.0"
+version = "0.71.0"
 description = "Typed, test-first SQL pipelines with local E2E testing"
 readme = "README.md"
 requires-python = ">=3.12"

{sqlbuild-0.70.0 → sqlbuild-0.71.0}/src/sqlbuild/executor/run/_helpers/execution/final_audits.py

@@ -66,11 +66,15 @@ def run_final_model_audits(
     return FinalAuditRun(results=tuple(results), has_error=has_error)
 
 
-def run_final_scope_audits(*, context: ModelMaterializationContext) -> FinalAuditRun:
+def run_final_scope_audits(
+    *, context: ModelMaterializationContext, relation_override: str | None = None
+) -> FinalAuditRun:
     """Run all model audits at FINAL scope against promoted relations."""
 
     return run_final_model_audits(
-        relation_overrides=None,
+        relation_overrides=(
+            None if relation_override is None else {context.entry.name: relation_override}
+        ),
         model_audits=context.model_audits,
         reuse_origin_fingerprint=None,
         adapter=context.adapter,

sqlbuild-0.71.0/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py

@@ -0,0 +1,135 @@
+"""Shared build-aside full-refresh relation lifecycle."""
+
+from __future__ import annotations
+
+import hashlib
+from typing import Any
+
+from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
+from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder
+from sqlbuild.adapter.contract.types import BuiltinAdapter
+from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import (
+    resolve_qualified_name_parts,
+)
+from sqlbuild.executor.run.models import FullRefreshRelations
+
+_PREVIOUS_PREFIX: str = "__sqb_prev__"
+_REBUILD_PREFIX: str = "__sqb_rebuild__"
+
+
+def resolve_full_refresh_relations(
+    *,
+    adapter: BaseAdapter,
+    database: str | None,
+    schema: str | None,
+    target_name: str,
+) -> FullRefreshRelations:
+    """Resolve identifier-fitted build-aside relation names."""
+
+    identifier_limit: int = adapter.maximum_identifier_length()
+    rebuild_name: str = _artifact_name(
+        logical_name=target_name,
+        fixed_prefix=_REBUILD_PREFIX,
+        identifier_limit=identifier_limit,
+    )
+    previous_name: str = _artifact_name(
+        logical_name=target_name,
+        fixed_prefix=_PREVIOUS_PREFIX,
+        identifier_limit=identifier_limit,
+    )
+    return FullRefreshRelations(
+        target_name=target_name,
+        target_qualified=_qualified(
+            adapter=adapter, database=database, schema=schema, name=target_name
+        ),
+        rebuild_name=rebuild_name,
+        rebuild_qualified=_qualified(
+            adapter=adapter, database=database, schema=schema, name=rebuild_name
+        ),
+        previous_name=previous_name,
+        previous_qualified=_qualified(
+            adapter=adapter, database=database, schema=schema, name=previous_name
+        ),
+    )
+
+
+def relation_exists(
+    *,
+    adapter: BaseAdapter,
+    connection: Any,
+    database: str | None,
+    schema: str | None,
+    name: str,
+) -> bool:
+    """Inspect whether one full-refresh relation exists."""
+
+    return adapter.relation_exists(
+        connection=connection, database=database, schema=schema, name=name
+    )
+
+
+def promote_full_refresh_rebuild(
+    *,
+    adapter: BaseAdapter,
+    connection: Any,
+    relations: FullRefreshRelations,
+    target_exists: bool,
+    statement_recorder: StatementRecorder,
+) -> None:
+    """Promote a rebuild; dropping prev at the next swap is retention pruning only."""
+
+    if not target_exists:
+        adapter.rename(
+            connection=connection,
+            origin=relations.rebuild_qualified,
+            destination=relations.target_qualified,
+            statement_recorder=statement_recorder,
+        )
+        return
+    adapter.drop(
+        connection=connection,
+        destination=relations.previous_qualified,
+        if_exists=True,
+        statement_recorder=statement_recorder,
+    )
+    if adapter.adapter_name == BuiltinAdapter.SNOWFLAKE:
+        adapter.swap(
+            connection=connection,
+            left=relations.target_qualified,
+            right=relations.rebuild_qualified,
+            statement_recorder=statement_recorder,
+        )
+        adapter.rename(
+            connection=connection,
+            origin=relations.rebuild_qualified,
+            destination=relations.previous_qualified,
+            statement_recorder=statement_recorder,
+        )
+        return
+    adapter.rename(
+        connection=connection,
+        origin=relations.target_qualified,
+        destination=relations.previous_qualified,
+        statement_recorder=statement_recorder,
+    )
+    adapter.rename(
+        connection=connection,
+        origin=relations.rebuild_qualified,
+        destination=relations.target_qualified,
+        statement_recorder=statement_recorder,
+    )
+
+
+def _artifact_name(*, logical_name: str, fixed_prefix: str, identifier_limit: int) -> str:
+    available: int = identifier_limit - len(fixed_prefix)
+    if len(logical_name) <= available:
+        return fixed_prefix + logical_name
+    digest: str = hashlib.sha256(logical_name.encode()).hexdigest()[:12]
+    fitted: str = logical_name[: max(1, available - len(digest) - 1)]
+    return f"{fixed_prefix}{fitted}_{digest}"
+
+
+def _qualified(*, adapter: BaseAdapter, database: str | None, schema: str | None, name: str) -> str:
+    return resolve_qualified_name_parts(
+        adapter=adapter, database=database, schema=schema, name=name
+    )

{sqlbuild-0.70.0 → sqlbuild-0.71.0}/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py

@@ -39,6 +39,7 @@ from sqlbuild.compiler.planner.types import (
     CursorType,
     IncrementalStrategy,
     OnSchemaChange,
+    PlanReason,
 )
 from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context
 from sqlbuild.diagnostics.main.log_debug_event import log_debug_event
@@ -56,6 +57,11 @@ from sqlbuild.executor.run._helpers.execution.results import (
     build_failed_result,
     build_skipped_result,
 )
+from sqlbuild.executor.run._helpers.materializations.full_refresh import (
+    promote_full_refresh_rebuild,
+    relation_exists,
+    resolve_full_refresh_relations,
+)
 from sqlbuild.executor.run._helpers.materializations.incremental import (
     _apply_schema_change,
     _execute_dml,
@@ -72,6 +78,7 @@ from sqlbuild.executor.run._helpers.validation.type_enforcement import enforce_t
 from sqlbuild.executor.run.models import (
     BatchWindow,
     FinalAuditRun,
+    FullRefreshRelations,
     MicrobatchAccountingInterval,
     MicrobatchLifecycleState,
     MicrobatchPhaseOutcome,
@@ -184,6 +191,19 @@ def execute_microbatch_entry(
     """Execute one microbatch incremental model through batched delta/DML."""
 
     targets: MicrobatchTargets = _resolve_microbatch_targets(context=context)
+    full_refresh_relations: FullRefreshRelations | None = None
+    if is_full_refresh:
+        full_refresh_relations = resolve_full_refresh_relations(
+            adapter=context.adapter,
+            database=context.entry.destination.database,
+            schema=context.entry.destination.schema,
+            target_name=context.entry.destination.name,
+        )
+        targets = replace(
+            targets,
+            target_table=full_refresh_relations.rebuild_name,
+            target_qualified=full_refresh_relations.rebuild_qualified,
+        )
     state: MicrobatchLifecycleState = MicrobatchLifecycleState(
         warnings=[],
         audit_results=[],
@@ -232,6 +252,14 @@ def execute_microbatch_entry(
     )
     if isinstance(history_context, ModelExecutionResult):
         return history_context
+    if full_refresh_relations is not None:
+        preparation_failure: ModelExecutionResult | None = _prepare_full_refresh_rebuild(
+            context=context,
+            state=state,
+            relations=full_refresh_relations,
+        )
+        if preparation_failure is not None:
+            return preparation_failure
     if history_context.run_type == MicrobatchRunType.REPLAY_ON_CHANGE:
         replay_batches: tuple[BatchWindow, ...] = compute_batch_windows(
             start=history_context.run_start,
@@ -303,16 +331,6 @@ def execute_microbatch_entry(
                 cursor_grain=context.entry.cursor_grain,
             )
         )
-    full_refresh_exit: ModelExecutionResult | None = _drop_target_for_full_refresh(
-        context=context,
-        is_full_refresh=is_full_refresh,
-        target_qualified=targets.target_qualified,
-        warnings=state.warnings,
-        audit_results=state.audit_results,
-        statement_recorder=state.statement_recorder,
-    )
-    if full_refresh_exit is not None:
-        return full_refresh_exit
     batch_outcome: MicrobatchPhaseOutcome = _execute_microbatch_batches(
         context=context,
         declared_columns=declared_columns,
@@ -328,6 +346,7 @@ def execute_microbatch_entry(
         context=context,
         history=history_context,
         batches=batch_plan.batches,
+        physical_target_name=targets.target_table,
     )
     if batch_outcome.failure is not None:
         return replace(
@@ -337,7 +356,12 @@ def execute_microbatch_entry(
             rows_affected=batch_outcome.rows_affected,
             **_microbatch_result_fields(history=history_context, succeeded=False),
         )
-    final_audit_run: FinalAuditRun = run_final_scope_audits(context=context)
+    final_audit_run: FinalAuditRun = run_final_scope_audits(
+        context=context,
+        relation_override=(
+            None if full_refresh_relations is None else full_refresh_relations.rebuild_qualified
+        ),
+    )
     state.audit_results.extend(final_audit_run.results)
     if final_audit_run.has_error:
         return replace(
@@ -345,9 +369,11 @@ def execute_microbatch_entry(
                 entry=context.entry,
                 phase=ExecutionPhase.AUDIT,
                 error=(
-                    f"final audit for '{context.entry.name}' failed after target update "
-                    "with severity level: error"
-                ),
+                    f"final audit for '{context.entry.name}' failed before full-refresh promotion "
+                    if full_refresh_relations is not None
+                    else f"final audit for '{context.entry.name}' failed after target update "
+                )
+                + ("with severity level: error"),
                 promoted_relation=targets.target_qualified,
                 warnings=state.warnings,
                 audit_results=state.audit_results,
@@ -355,6 +381,14 @@ def execute_microbatch_entry(
             ),
             **_microbatch_result_fields(history=history_context, succeeded=True),
         )
+    if full_refresh_relations is not None:
+        promotion_failure: ModelExecutionResult | None = _promote_microbatch_full_refresh(
+            context=context,
+            state=state,
+            relations=full_refresh_relations,
+        )
+        if promotion_failure is not None:
+            return promotion_failure
     post_hook_outcome: PostHookPhaseOutcome = run_post_hook_phase(
         context=context,
         warnings=state.warnings,
@@ -577,6 +611,7 @@ def _execute_microbatch_batches(
         completion_failure: ModelExecutionResult | None = _record_microbatch_completion(
             context=context,
             batch=batch,
+            is_full_refresh=is_full_refresh,
             rows_affected=dml_result if isinstance(dml_result, int) else None,
             history=history_context,
             state=state,
@@ -1056,6 +1091,7 @@ def _record_microbatch_completion(
     *,
     context: ModelMaterializationContext,
     batch: BatchWindow,
+    is_full_refresh: bool,
     rows_affected: int | None,
     history: _MicrobatchHistoryContext,
     state: MicrobatchLifecycleState,
@@ -1064,7 +1100,11 @@ def _record_microbatch_completion(
     now: datetime = datetime.now(tz=UTC)
     recovery_origin: _RecoveryOrigin | None = history.recovery_origins.get((batch.start, batch.end))
     completion_scope: MicrobatchScope = _current_completion_scope(
-        context=context, scope=history.scope
+        context=context,
+        scope=history.scope,
+        physical_target_name=(
+            targets.target_table if is_full_refresh else context.entry.destination.name
+        ),
     )
     event: MicrobatchEvent = MicrobatchEvent(
         event_id=uuid4().hex,
@@ -1145,15 +1185,21 @@ def _expected_model_version_hash(*, context: ModelMaterializationContext) -> str
 
 
 def _current_completion_scope(
-    *, context: ModelMaterializationContext, scope: MicrobatchScope
+    *,
+    context: ModelMaterializationContext,
+    scope: MicrobatchScope,
+    physical_target_name: str | None = None,
 ) -> MicrobatchScope:
-    if scope.physical_generation_id.startswith(MICROBATCH_REPLAY_GENERATION_PREFIX):
+    if (
+        scope.physical_generation_id.startswith(MICROBATCH_REPLAY_GENERATION_PREFIX)
+        or context.entry.reason == PlanReason.FULL_REFRESH
+    ):
         return scope
     generation: str | None = context.adapter.physical_relation_generation(
         connection=context.connection,
         database=context.entry.destination.database,
         schema=context.entry.destination.schema,
-        name=context.entry.destination.name,
+        name=physical_target_name or context.entry.destination.name,
     )
     if generation is None:
         return scope
@@ -1775,13 +1821,16 @@ def _refresh_microbatch_result_history(
     context: ModelMaterializationContext,
     history: _MicrobatchHistoryContext,
     batches: tuple[BatchWindow, ...],
+    physical_target_name: str,
 ) -> _MicrobatchHistoryContext:
     if not batches:
         return history
     expected: tuple[MicrobatchInterval, ...] = tuple(
         MicrobatchInterval(start=batch.start, end=batch.end) for batch in batches
     )
-    current_scope: MicrobatchScope = _current_completion_scope(context=context, scope=history.scope)
+    current_scope: MicrobatchScope = _current_completion_scope(
+        context=context, scope=history.scope, physical_target_name=physical_target_name
+    )
     try:
         events: tuple[MicrobatchEvent, ...] = history.store.read_scope_history(current_scope)
     except Exception:
@@ -2164,35 +2213,93 @@ def _complete_microbatch_batch(
     )
 
 
-def _drop_target_for_full_refresh(
+def _prepare_full_refresh_rebuild(
     *,
     context: ModelMaterializationContext,
-    is_full_refresh: bool,
-    target_qualified: str,
-    warnings: list[str],
-    audit_results: list[AuditExecutionResult],
-    statement_recorder: StatementRecorder,
+    state: MicrobatchLifecycleState,
+    relations: FullRefreshRelations,
 ) -> ModelExecutionResult | None:
-    """Drop the target before a full-refresh run; return a failure result on error."""
+    """Inspect or initialize the deterministic full-refresh rebuild relation."""
 
-    if not is_full_refresh:
-        return None
     try:
-        with diagnostics_context(sqlbuild_phase="cleanup", sqlbuild_action_name="drop_target"):
+        live_exists: bool = relation_exists(
+            adapter=context.adapter,
+            connection=context.connection,
+            database=context.entry.destination.database,
+            schema=context.entry.destination.schema,
+            name=relations.target_name,
+        )
+        rebuild_exists: bool = relation_exists(
+            adapter=context.adapter,
+            connection=context.connection,
+            database=context.entry.destination.database,
+            schema=context.entry.destination.schema,
+            name=relations.rebuild_name,
+        )
+        if not live_exists and rebuild_exists:
+            context.adapter.rename(
+                connection=context.connection,
+                origin=relations.rebuild_qualified,
+                destination=relations.target_qualified,
+                statement_recorder=state.statement_recorder,
+            )
+        elif not live_exists and context.entry.reason == PlanReason.FULL_REFRESH:
+            raise ExecutorInputError(
+                "full-refresh reconciliation found neither the live target nor its rebuild "
+                f"relation for '{context.entry.name}'"
+            )
+        elif rebuild_exists:
             context.adapter.drop(
                 connection=context.connection,
-                destination=target_qualified,
+                destination=relations.rebuild_qualified,
                 if_exists=True,
-                statement_recorder=statement_recorder,
+                statement_recorder=state.statement_recorder,
             )
     except Exception as exc:
         return build_failed_result(
             entry=context.entry,
             phase=ExecutionPhase.STAGING,
-            error=f"failed to drop target for full-refresh microbatch: {exc}",
-            warnings=warnings,
-            audit_results=audit_results,
-            statement_recorder=statement_recorder,
+            error=str(exc),
+            warnings=state.warnings,
+            audit_results=state.audit_results,
+            statement_recorder=state.statement_recorder,
+        )
+    return None
+
+
+def _promote_microbatch_full_refresh(
+    *,
+    context: ModelMaterializationContext,
+    state: MicrobatchLifecycleState,
+    relations: FullRefreshRelations,
+) -> ModelExecutionResult | None:
+    """Promote a complete rebuild while retaining the outgoing generation."""
+
+    try:
+        live_exists: bool = relation_exists(
+            adapter=context.adapter,
+            connection=context.connection,
+            database=context.entry.destination.database,
+            schema=context.entry.destination.schema,
+            name=relations.target_name,
+        )
+        promote_full_refresh_rebuild(
+            adapter=context.adapter,
+            connection=context.connection,
+            relations=relations,
+            target_exists=live_exists,
+            statement_recorder=state.statement_recorder,
+        )
+    except Exception as exc:
+        return build_failed_result(
+            entry=context.entry,
+            phase=ExecutionPhase.DML,
+            error=f"failed to promote full-refresh rebuild: {exc}",
+            staging_relation=relations.rebuild_qualified,
+            promoted_relation=relations.target_qualified,
+            warnings=state.warnings,
+            audit_results=state.audit_results,
+            statement_recorder=state.statement_recorder,
         )
     return None