sqlbuild 0.112.2__tar.gz → 0.113.0__tar.gz

{sqlbuild-0.112.2 → sqlbuild-0.113.0}/Cargo.lock

@@ -592,7 +592,7 @@ checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
 
 [[package]]
 name = "sqlbuild-rules-native"
-version = "0.112.2"
+version = "0.113.0"
 dependencies = [
  "fensu-policy",
  "globset",

{sqlbuild-0.112.2 → sqlbuild-0.113.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlbuild
-Version: 0.112.2
+Version: 0.113.0
 Requires-Dist: duckdb>=1.3.2
 Requires-Dist: filelock>=3.20.3
 Requires-Dist: orjson>=3.10.0

{sqlbuild-0.112.2 → sqlbuild-0.113.0}/crates/sqlbuild-rules-native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sqlbuild-rules-native"
-version = "0.112.2"
+version = "0.113.0"
 edition.workspace = true
 license.workspace = true
 publish.workspace = true

{sqlbuild-0.112.2 → sqlbuild-0.113.0}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "maturin"
 
 [project]
 name = "sqlbuild"
-version = "0.112.2"
+version = "0.113.0"
 description = "Typed, test-first SQL pipelines with local E2E testing"
 readme = "README.md"
 requires-python = ">=3.12"

{sqlbuild-0.112.2 → sqlbuild-0.113.0}/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py

@@ -147,6 +147,9 @@ from sqlbuild.microbatches.types import (
     ReplayRequirementState,
     UnaccountedPartitionPolicy,
 )
+from sqlbuild.runtime.observability.classes.microbatch_lifecycle import (
+    MicrobatchLifecycle,
+)
 from sqlbuild.runtime.observability.classes.operation_lifecycle import (
     OperationAttributes,
     OperationLifecycle,
@@ -154,6 +157,7 @@ from sqlbuild.runtime.observability.classes.operation_lifecycle import (
 from sqlbuild.runtime.observability.main.canonicalize_operation_adapter import (
     canonicalize_operation_adapter,
 )
+from sqlbuild.runtime.observability.models import MicrobatchLifecycleContext
 from sqlbuild.spec.contracts.types import MicrobatchLimitAction, TableType
 
 _DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS
@@ -225,6 +229,24 @@ class _PreparedMicrobatchExecution:
     history: _MicrobatchHistoryContext
 
 
+@dataclass(frozen=True)
+class _MicrobatchLifecyclePlan:
+    """Stable planned context repeated on each active microbatch fact."""
+
+    batch_count: int
+    cursor_start: str
+    cursor_end_exclusive: str
+
+
+@dataclass(frozen=True)
+class _MicrobatchBatchPreparation:
+    """Prepared batch state or the failure that prevents target DML."""
+
+    state: MicrobatchLifecycleState
+    schema_checked: bool
+    failure: ModelExecutionResult | None = None
+
+
 class _SerialMicrobatchEventStore:
     """Reject accidental durable-state use from the serial execution path."""
 
@@ -494,6 +516,10 @@ def execute_microbatch_entry(
         state=state,
         history_context=history_context,
         on_progress=on_progress,
+        lifecycle_plan=_microbatch_lifecycle_plan(
+            batch_plan=batch_plan,
+            cursor_type=context.entry.cursor_type or "",
+        ),
     )
     state = batch_outcome.state
     if context.microbatch_event_store is not None:
@@ -994,7 +1020,13 @@ def _execute_microbatch_batches(
     state: MicrobatchLifecycleState,
     history_context: _MicrobatchHistoryContext,
     on_progress: Callable[[str], None] | None = None,
+    lifecycle_plan: _MicrobatchLifecyclePlan | None = None,
 ) -> MicrobatchPhaseOutcome:
+    plan: _MicrobatchLifecyclePlan = lifecycle_plan or _MicrobatchLifecyclePlan(
+        batch_count=len(batches),
+        cursor_start=render(value=batches[0].start if batches else history_context.run_start),
+        cursor_end_exclusive=render(value=batches[-1].end if batches else history_context.run_end),
+    )
     if context.microbatch_batch_runner is not None and not is_full_refresh and len(batches) > 1:
         return _execute_microbatch_batches_concurrently(
             context=context,
@@ -1004,15 +1036,22 @@ def _execute_microbatch_batches(
             state=state,
             history_context=history_context,
             on_progress=on_progress,
+            lifecycle_plan=plan,
         )
     schema_checked: bool = False
     completed_batches: int = 0
     applied_intervals: list[tuple[str, str]] = []
     total_rows: int = 0
     row_count_known: bool = False
-    total_batches: int = len(batches)
     batch: BatchWindow
     for batch in batches:
+        lifecycle: MicrobatchLifecycle = _microbatch_lifecycle(
+            context=context,
+            batch=batch,
+            history=history_context,
+            plan=plan,
+        )
+        lifecycle.start()
         lease_failure: ModelExecutionResult | None = _microbatch_lease_failure(
             context=context,
             state=state,
@@ -1020,6 +1059,7 @@ def _execute_microbatch_batches(
             boundary="batch staging",
         )
         if lease_failure is not None:
+            lifecycle.failed(error_code=lease_failure.error_code)
             return MicrobatchPhaseOutcome(
                 state=state,
                 failure=lease_failure,
@@ -1032,53 +1072,26 @@ def _execute_microbatch_batches(
         window_text: str = f"{render(value=batch.start)}..{render(value=batch.end)}"
         display_window: str = _format_batch_window_for_display(batch=batch, entry=context.entry)
         if on_progress is not None:
-            on_progress(f"batch {completed_batches + 1}/{total_batches} {display_window}")
-        stage_failure: ModelExecutionResult | None = _stage_microbatch_delta(
-            context=context,
-            batch=batch,
-            window_text=window_text,
-            targets=batch_targets,
-            state=state,
-        )
-        if stage_failure is not None:
-            return MicrobatchPhaseOutcome(
-                state=state, failure=stage_failure, completed_batches=completed_batches
-            )
-        schema_outcome: MicrobatchSchemaPhaseOutcome = _apply_microbatch_schema_change(
-            context=context,
-            is_full_refresh=is_full_refresh,
-            schema_checked=schema_checked,
-            window_text=window_text,
-            targets=batch_targets,
-            state=state,
-        )
-        state = schema_outcome.state
-        if schema_outcome.failure is not None:
-            return MicrobatchPhaseOutcome(
-                state=state, failure=schema_outcome.failure, completed_batches=completed_batches
-            )
-        schema_checked = schema_outcome.schema_checked
-        type_failure: ModelExecutionResult | None = _enforce_microbatch_types(
+            on_progress(f"batch {batch.index + 1}/{plan.batch_count} {display_window}")
+        preparation: _MicrobatchBatchPreparation = _prepare_microbatch_batch(
             context=context,
             declared_columns=declared_columns,
+            is_full_refresh=is_full_refresh,
             batch=batch,
+            schema_checked=schema_checked,
             window_text=window_text,
             targets=batch_targets,
             state=state,
         )
-        if type_failure is not None:
+        state = preparation.state
+        if preparation.failure is not None:
+            lifecycle.failed(error_code=preparation.failure.error_code)
             return MicrobatchPhaseOutcome(
-                state=state, failure=type_failure, completed_batches=completed_batches
+                state=state,
+                failure=preparation.failure,
+                completed_batches=completed_batches,
             )
-        audit_outcome: MicrobatchPhaseOutcome = _run_microbatch_delta_audits(
-            context=context,
-            batch=batch,
-            targets=batch_targets,
-            state=state,
-        )
-        state = audit_outcome.state
-        if audit_outcome.failure is not None:
-            return replace(audit_outcome, completed_batches=completed_batches)
+        schema_checked = preparation.schema_checked
         lease_failure = _microbatch_lease_failure(
             context=context,
             state=state,
@@ -1086,6 +1099,7 @@ def _execute_microbatch_batches(
             boundary="target DML",
         )
         if lease_failure is not None:
+            lifecycle.failed(error_code=lease_failure.error_code)
             return MicrobatchPhaseOutcome(
                 state=state,
                 failure=lease_failure,
@@ -1101,6 +1115,7 @@ def _execute_microbatch_batches(
             state=state,
         )
         if isinstance(dml_result, ModelExecutionResult):
+            lifecycle.failed(error_code=dml_result.error_code)
             return MicrobatchPhaseOutcome(
                 state=state,
                 failure=dml_result,
@@ -1121,6 +1136,7 @@ def _execute_microbatch_batches(
             boundary="completion publication",
         )
         if lease_failure is not None:
+            lifecycle.failed(error_code=lease_failure.error_code)
             return MicrobatchPhaseOutcome(
                 state=state,
                 failure=lease_failure,
@@ -1145,6 +1161,7 @@ def _execute_microbatch_batches(
             )
         )
         if completion_failure is not None:
+            lifecycle.failed(error_code=completion_failure.error_code)
             return MicrobatchPhaseOutcome(
                 state=state,
                 failure=completion_failure,
@@ -1153,18 +1170,23 @@ def _execute_microbatch_batches(
                     total_rows=total_rows, row_count_known=row_count_known
                 ),
             )
-        _complete_microbatch_batch(
-            context=context,
-            window_text=window_text,
-            targets=batch_targets,
-            state=state,
-        )
+        try:
+            _complete_microbatch_batch(
+                context=context,
+                window_text=window_text,
+                targets=batch_targets,
+                state=state,
+            )
+        except BaseException as error:
+            lifecycle.failed(error=error)
+            raise
+        lifecycle.completed(affected_rows=dml_result if isinstance(dml_result, int) else None)
         completed_batches += 1
         applied_intervals.append((render(value=batch.start), render(value=batch.end)))
         if on_progress is not None:
             batch_elapsed: float = time.monotonic() - batch_start_time
             on_progress(
-                f"batch {completed_batches}/{total_batches} {display_window} {batch_elapsed:.1f}s"
+                f"batch {batch.index + 1}/{plan.batch_count} {display_window} {batch_elapsed:.1f}s"
             )
     return MicrobatchPhaseOutcome(
         state=state,
@@ -1177,6 +1199,119 @@ def _execute_microbatch_batches(
     )
 
 
+def _microbatch_lifecycle(
+    *,
+    context: ModelMaterializationContext,
+    batch: BatchWindow,
+    history: _MicrobatchHistoryContext,
+    plan: _MicrobatchLifecyclePlan,
+) -> MicrobatchLifecycle:
+    start: str = render(value=batch.start)
+    end: str = render(value=batch.end)
+    return MicrobatchLifecycle(
+        context=MicrobatchLifecycleContext(
+            cursor_start=start,
+            cursor_end_exclusive=end,
+            planned_cursor_start=plan.cursor_start,
+            planned_cursor_end_exclusive=plan.cursor_end_exclusive,
+            batch_index=batch.index + 1,
+            batch_count=plan.batch_count,
+            configured_batch_size=context.entry.batch_size or history.batch_size,
+            effective_batch_size=history.batch_size,
+            microbatch_strategy=context.entry.microbatch_strategy or "watermark",
+            microbatch_run_type=history.run_type.value,
+            batch_kind=("recovery" if (start, end) in history.recovery_intervals else "ordinary"),
+        )
+    )
+
+
+def _microbatch_lifecycle_plan(
+    *, batch_plan: _MicrobatchPlan, cursor_type: str
+) -> _MicrobatchLifecyclePlan:
+    selected_intervals: tuple[MicrobatchInterval, ...] = merge_intervals(
+        intervals=tuple(
+            MicrobatchInterval(start=render(value=batch.start), end=render(value=batch.end))
+            for batch in batch_plan.batches
+        ),
+        cursor_type=cursor_type,
+    )
+    if selected_intervals:
+        return _MicrobatchLifecyclePlan(
+            batch_count=len(batch_plan.batches),
+            cursor_start=selected_intervals[0].start,
+            cursor_end_exclusive=selected_intervals[-1].end,
+        )
+    resolved_range: CursorBounds | None = batch_plan.resolved_range
+    if resolved_range is None:
+        raise ExecutorInputError("microbatch lifecycle requires resolved cursor bounds")
+    return _MicrobatchLifecyclePlan(
+        batch_count=len(batch_plan.batches),
+        cursor_start=render(value=_concrete_bound(bound=resolved_range.start)),
+        cursor_end_exclusive=render(value=_concrete_bound(bound=resolved_range.end)),
+    )
+
+
+def _prepare_microbatch_batch(
+    *,
+    context: ModelMaterializationContext,
+    declared_columns: tuple[ColumnInfo, ...],
+    is_full_refresh: bool,
+    batch: BatchWindow,
+    schema_checked: bool,
+    window_text: str,
+    targets: MicrobatchTargets,
+    state: MicrobatchLifecycleState,
+) -> _MicrobatchBatchPreparation:
+    stage_failure: ModelExecutionResult | None = _stage_microbatch_delta(
+        context=context,
+        batch=batch,
+        window_text=window_text,
+        targets=targets,
+        state=state,
+    )
+    if stage_failure is not None:
+        return _MicrobatchBatchPreparation(state, schema_checked, stage_failure)
+    schema_outcome: MicrobatchSchemaPhaseOutcome = _apply_microbatch_schema_change(
+        context=context,
+        is_full_refresh=is_full_refresh,
+        schema_checked=schema_checked,
+        window_text=window_text,
+        targets=targets,
+        state=state,
+    )
+    if schema_outcome.failure is not None:
+        return _MicrobatchBatchPreparation(
+            schema_outcome.state,
+            schema_outcome.schema_checked,
+            schema_outcome.failure,
+        )
+    type_failure: ModelExecutionResult | None = _enforce_microbatch_types(
+        context=context,
+        declared_columns=declared_columns,
+        batch=batch,
+        window_text=window_text,
+        targets=targets,
+        state=schema_outcome.state,
+    )
+    if type_failure is not None:
+        return _MicrobatchBatchPreparation(
+            schema_outcome.state,
+            schema_outcome.schema_checked,
+            type_failure,
+        )
+    audit_outcome: MicrobatchPhaseOutcome = _run_microbatch_delta_audits(
+        context=context,
+        batch=batch,
+        targets=targets,
+        state=schema_outcome.state,
+    )
+    return _MicrobatchBatchPreparation(
+        audit_outcome.state,
+        schema_outcome.schema_checked,
+        audit_outcome.failure,
+    )
+
+
 def _microbatch_lease_failure(
     *,
     context: ModelMaterializationContext,
@@ -1211,6 +1346,7 @@ def _execute_microbatch_batches_concurrently(
     state: MicrobatchLifecycleState,
     history_context: _MicrobatchHistoryContext,
     on_progress: Callable[[str], None] | None,
+    lifecycle_plan: _MicrobatchLifecyclePlan,
 ) -> MicrobatchPhaseOutcome:
     recovery_batches: tuple[BatchWindow, ...] = tuple(
         batch
@@ -1239,6 +1375,7 @@ def _execute_microbatch_batches_concurrently(
             aggregate_state=state,
             history_context=history_context,
             on_progress=on_progress,
+            lifecycle_plan=lifecycle_plan,
         )
         state = phase_outcome.state
         completed_batches += phase_outcome.completed_batches
@@ -1275,6 +1412,7 @@ def _execute_concurrent_microbatch_phase(
     aggregate_state: MicrobatchLifecycleState,
     history_context: _MicrobatchHistoryContext,
     on_progress: Callable[[str], None] | None,
+    lifecycle_plan: _MicrobatchLifecyclePlan,
 ) -> MicrobatchPhaseOutcome:
     first: BatchWindow = batches[0]
     serial_context: ModelMaterializationContext = replace(context, microbatch_batch_runner=None)
@@ -1287,6 +1425,7 @@ def _execute_concurrent_microbatch_phase(
         state=aggregate_state,
         history_context=history_context,
         on_progress=on_progress,
+        lifecycle_plan=lifecycle_plan,
     )
     if first_outcome.failure is not None or len(batches) == 1:
         return first_outcome
@@ -1317,6 +1456,7 @@ def _execute_concurrent_microbatch_phase(
             state=worker_state,
             history_context=replace(history_context, store=worker_store),
             on_progress=on_progress,
+            lifecycle_plan=lifecycle_plan,
         )
 
     runner: MicrobatchBatchRunner | None = context.microbatch_batch_runner

{sqlbuild-0.112.2 → sqlbuild-0.113.0}/src/sqlbuild/runtime/event_exporting/types.py

@@ -9,6 +9,7 @@ class LifecycleEventKind(StrEnum):
     INVOCATION = "invocation"
     RUN = "run"
     RESOURCE = "resource"
+    MICROBATCH = "microbatch"
     OPERATION = "operation"
     STATEMENT = "statement"
     RETRY = "retry"

{sqlbuild-0.112.2 → sqlbuild-0.113.0}/src/sqlbuild/runtime/observability/_helpers/validation.py

@@ -23,6 +23,10 @@ from sqlbuild.runtime.observability.constants import (
     LIFECYCLE_EVENT_CATALOGS,
     MAX_METADATA_BYTES,
     METADATA_FIELD,
+    MICROBATCH_EVENT_PREFIX,
+    MICROBATCH_KINDS,
+    MICROBATCH_RUN_TYPES,
+    MICROBATCH_STRATEGIES,
     NONNEGATIVE_INTEGER_PAYLOAD_FIELDS,
     OPERATION_ADAPTERS,
     OPERATION_EVENT_PREFIX,
@@ -295,6 +299,8 @@ def validate_known_lifecycle_event(*, event: LifecycleEvent) -> None:
             )
     if event.event_type.startswith(OPERATION_EVENT_PREFIX):
         _validate_operation_payload(payload=event.payload)
+    if event.event_type.startswith(MICROBATCH_EVENT_PREFIX):
+        _validate_microbatch_payload(payload=event.payload)
     if event.event_type == RESOURCE_ATTEMPT_SKIPPED_EVENT:
         if event.payload.get("skip_code") not in RESOURCE_SKIP_CODES:
             raise ObservabilityValidationError("skip_code must be a catalogued value")
@@ -323,6 +329,25 @@ def validate_known_lifecycle_event(*, event: LifecycleEvent) -> None:
     _validate_run_payload(event=event)
 
 
+def _validate_microbatch_payload(*, payload: Mapping[str, JSONValue]) -> None:
+    batch_index: JSONValue | None = payload.get("batch_index")
+    batch_count: JSONValue | None = payload.get("batch_count")
+    if isinstance(batch_index, bool) or not isinstance(batch_index, int) or batch_index < 1:
+        raise ObservabilityValidationError("batch_index must be a positive integer")
+    if isinstance(batch_count, bool) or not isinstance(batch_count, int) or batch_count < 1:
+        raise ObservabilityValidationError("batch_count must be a positive integer")
+    if batch_index > batch_count:
+        raise ObservabilityValidationError("batch_index must not exceed batch_count")
+    for field_name, allowed in (
+        ("microbatch_strategy", MICROBATCH_STRATEGIES),
+        ("microbatch_run_type", MICROBATCH_RUN_TYPES),
+        ("batch_kind", MICROBATCH_KINDS),
+    ):
+        value: JSONValue | None = payload.get(field_name)
+        if not isinstance(value, str) or value not in allowed:
+            raise ObservabilityValidationError(f"{field_name} must be a catalogued value")
+
+
 def _validate_run_payload(*, event: LifecycleEvent) -> None:
     payload: Mapping[str, JSONValue] = event.payload
     if event.event_type == RUN_STARTED_EVENT and CONFIGURED_CONCURRENCY_FIELD in payload:

sqlbuild-0.113.0/src/sqlbuild/runtime/observability/classes/microbatch_lifecycle.py

@@ -0,0 +1,125 @@
+"""Lifecycle facts for one active microbatch interval."""
+
+from __future__ import annotations
+
+import re
+import time
+from collections.abc import Mapping
+from types import TracebackType
+
+from sqlbuild.runtime.observability._helpers.dispatcher import current_event_dispatcher
+from sqlbuild.runtime.observability._helpers.factory import create_lifecycle_event
+from sqlbuild.runtime.observability._helpers.identity import current_execution_identity
+from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher
+from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError
+from sqlbuild.runtime.observability.models import ExecutionIdentity, MicrobatchLifecycleContext
+from sqlbuild.runtime.observability.types import JSONValue
+
+_ERROR_TOKEN_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
+
+
+class MicrobatchLifecycle:
+    """Publish the active bounds and terminal outcome for one microbatch."""
+
+    def __init__(
+        self,
+        *,
+        context: MicrobatchLifecycleContext,
+    ) -> None:
+        self._payload: dict[str, JSONValue] = context.payload()
+        self._dispatcher: EventDispatcher | None = None
+        self._started_monotonic: float | None = None
+        self._terminal: bool = False
+        self._entered: bool = False
+
+    def __enter__(self) -> MicrobatchLifecycle:
+        if self._entered:
+            raise ObservabilityValidationError(
+                "microbatch lifecycle cannot be entered more than once"
+            )
+        self._entered = True
+        identity: ExecutionIdentity | None = current_execution_identity()
+        dispatcher: EventDispatcher | None = current_event_dispatcher()
+        if (
+            identity is None
+            or identity.run_id is None
+            or identity.resource_id is None
+            or identity.resource_attempt_id is None
+            or dispatcher is None
+        ):
+            return self
+        self._dispatcher = dispatcher
+        self._started_monotonic = time.monotonic()
+        self._publish(event_type="microbatch_started", payload=self._payload)
+        return self
+
+    def start(self) -> None:
+        """Publish the start fact without requiring context-manager ownership."""
+
+        self.__enter__()
+
+    def completed(self, *, affected_rows: int | None = None) -> None:
+        payload: dict[str, JSONValue] = self._terminal_payload()
+        if affected_rows is not None:
+            payload["affected_rows"] = affected_rows
+        self._publish_terminal(event_type="microbatch_completed", payload=payload)
+
+    def failed(
+        self,
+        *,
+        error: BaseException | None = None,
+        error_code: str | None = None,
+    ) -> None:
+        payload: dict[str, JSONValue] = self._terminal_payload()
+        payload["error_type"] = _safe_error_token(
+            value="ExecutionFailed" if error is None else type(error).__name__,
+            fallback="ExecutionFailed",
+        )
+        safe_error_code: str | None = _safe_optional_error_code(error_code)
+        if safe_error_code is not None:
+            payload["error_code"] = safe_error_code
+        self._publish_terminal(event_type="microbatch_failed", payload=payload)
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        del exc_type, traceback
+        if not self._terminal:
+            if exc_value is None:
+                self.completed()
+            else:
+                self.failed(error=exc_value)
+
+    def _terminal_payload(self) -> dict[str, JSONValue]:
+        payload: dict[str, JSONValue] = dict(self._payload)
+        started: float = self._started_monotonic if self._started_monotonic is not None else 0.0
+        payload["duration_ms"] = max(0.0, (time.monotonic() - started) * 1000.0)
+        return payload
+
+    def _publish(self, *, event_type: str, payload: Mapping[str, JSONValue]) -> None:
+        if self._dispatcher is None:
+            return
+        self._dispatcher.publish_lifecycle(
+            create_lifecycle_event(event_type=event_type, payload=payload)
+        )
+
+    def _publish_terminal(self, *, event_type: str, payload: Mapping[str, JSONValue]) -> None:
+        if self._terminal:
+            return
+        self._terminal = True
+        self._publish(event_type=event_type, payload=payload)
+
+
+def _safe_error_token(*, value: object, fallback: str) -> str:
+    if not isinstance(value, str) or _ERROR_TOKEN_PATTERN.fullmatch(value) is None:
+        return fallback
+    return value
+
+
+def _safe_optional_error_code(value: str | None) -> str | None:
+    if value is None or _ERROR_TOKEN_PATTERN.fullmatch(value) is None:
+        return None
+    return value

{sqlbuild-0.112.2 → sqlbuild-0.113.0}/src/sqlbuild/runtime/observability/constants.py

@@ -31,6 +31,25 @@ RESOURCE_ATTEMPT_SKIPPED_EVENT: str = "resource_attempt_skipped"
 RESOURCE_TERMINALS: frozenset[str] = frozenset(
     {"resource_attempt_completed", "resource_attempt_failed", RESOURCE_ATTEMPT_SKIPPED_EVENT}
 )
+MICROBATCH_EVENT_PREFIX: str = "microbatch_"
+MICROBATCH_FIELDS: frozenset[str] = frozenset(
+    {
+        "cursor_start",
+        "cursor_end_exclusive",
+        "planned_cursor_start",
+        "planned_cursor_end_exclusive",
+        "batch_index",
+        "batch_count",
+        "configured_batch_size",
+        "effective_batch_size",
+        "microbatch_strategy",
+        "microbatch_run_type",
+        "batch_kind",
+    }
+)
+MICROBATCH_STRATEGIES: frozenset[str] = frozenset({"rolling_window", "watermark"})
+MICROBATCH_RUN_TYPES: frozenset[str] = frozenset({"normal", "backfill", "replay_on_change"})
+MICROBATCH_KINDS: frozenset[str] = frozenset({"ordinary", "recovery"})
 HOOK_PHASES: frozenset[str] = frozenset({"post_hooks", "pre_hooks"})
 HOOK_TYPES: frozenset[str] = frozenset({"python", "sql"})
 OPERATION_FIELDS: frozenset[str] = frozenset(
@@ -253,6 +272,15 @@ STRING_PAYLOAD_FIELDS: frozenset[str] = frozenset(
         "executed_sql",
         "execution_error",
         "result_id",
+        "cursor_start",
+        "cursor_end_exclusive",
+        "planned_cursor_start",
+        "planned_cursor_end_exclusive",
+        "configured_batch_size",
+        "effective_batch_size",
+        "microbatch_strategy",
+        "microbatch_run_type",
+        "batch_kind",
     }
 )
 NONNEGATIVE_INTEGER_PAYLOAD_FIELDS: frozenset[str] = frozenset(
@@ -280,6 +308,8 @@ NONNEGATIVE_INTEGER_PAYLOAD_FIELDS: frozenset[str] = frozenset(
         "sample_count",
         "minimum_samples",
         "evidence_count",
+        "batch_index",
+        "batch_count",
     }
 )
 FINITE_NUMBER_PAYLOAD_FIELDS: frozenset[str] = frozenset({"measured_value"})
@@ -523,7 +553,29 @@ LIFECYCLE_EVENT_CATALOG_V1: Mapping[str, LifecycleEventDefinition] = MappingProx
         ),
     }
 )
+LIFECYCLE_EVENT_CATALOG_V2: Mapping[str, LifecycleEventDefinition] = MappingProxyType(
+    {
+        **LIFECYCLE_EVENT_CATALOG_V1,
+        "microbatch_started": LifecycleEventDefinition.create(
+            required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}),
+            required_payload=MICROBATCH_FIELDS,
+            allowed=MICROBATCH_FIELDS,
+        ),
+        "microbatch_completed": LifecycleEventDefinition.create(
+            required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}),
+            required_payload=MICROBATCH_FIELDS,
+            allowed=MICROBATCH_FIELDS | DURATION_FIELDS | frozenset({"affected_rows"}),
+            terminal=True,
+        ),
+        "microbatch_failed": LifecycleEventDefinition.create(
+            required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}),
+            required_payload=MICROBATCH_FIELDS | frozenset({"error_type"}),
+            allowed=MICROBATCH_FIELDS | DURATION_FIELDS | ERROR_FIELDS,
+            terminal=True,
+        ),
+    }
+)
 LIFECYCLE_EVENT_CATALOGS: Mapping[int, Mapping[str, LifecycleEventDefinition]] = MappingProxyType(
-    {1: LIFECYCLE_EVENT_CATALOG_V1, 2: LIFECYCLE_EVENT_CATALOG_V1}
+    {1: LIFECYCLE_EVENT_CATALOG_V1, 2: LIFECYCLE_EVENT_CATALOG_V2}
 )
-LIFECYCLE_EVENT_CATALOG: Mapping[str, LifecycleEventDefinition] = LIFECYCLE_EVENT_CATALOG_V1
+LIFECYCLE_EVENT_CATALOG: Mapping[str, LifecycleEventDefinition] = LIFECYCLE_EVENT_CATALOG_V2