sqlmesh-dag-generator 0.9.5__tar.gz → 0.9.7__tar.gz

{sqlmesh_dag_generator-0.9.5/sqlmesh_dag_generator.egg-info → sqlmesh_dag_generator-0.9.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh-dag-generator
-Version: 0.9.5
+Version: 0.9.7
 Summary: Open-source Airflow DAG generator for SQLMesh projects
 Home-page: https://github.com/kubolko/sqlmesh-dag-generator
 Author: Jakub Sumionka
@@ -128,9 +128,9 @@ If you run sub-hourly incremental models with `catchup=False`, outages can leave
 
 The package now supports an explicit recovery policy:
 
-- `recovery_mode="disabled"` (default): no runtime recovery tasks are added.
+- `recovery_mode="disabled"`: no runtime recovery tasks are added.
 - `recovery_mode="warn"`: add an integrity guard task that detects missing intervals and logs them.
-- `recovery_mode="bounded_auto"`: add the same guard task plus a bounded recovery task that replays missing intervals when the gap is within `recovery_max_intervals`.
+- `recovery_mode="bounded_auto"` (default): add the same guard task plus a bounded recovery task that replays missing intervals when the gap is within `recovery_max_intervals`.
 
 Example:
 
@@ -150,6 +150,41 @@ When `recovery_mode` is enabled, the package adds stable helper tasks to the DAG
 
 This keeps recovery explicit and observable in Airflow while preserving the default "no surprise backfills" behavior.
 
+### Manual Backfill Task
+
+For larger historical gaps, the package also exposes a first-class manual backfill task helper.
+Use it in a separate unscheduled DAG and trigger it only when you need to replay a broader window:
+
+```python
+from datetime import datetime, timedelta
+from airflow import DAG
+from sqlmesh_dag_generator import SQLMeshDAGGenerator
+
+generator = SQLMeshDAGGenerator(
+  sqlmesh_project_path="/path/to/project",
+  dag_id="my_pipeline",
+  gateway="prod",
+)
+
+with DAG(
+  dag_id="my_pipeline_manual_backfill",
+  schedule=None,
+  start_date=datetime(2024, 1, 1),
+  catchup=False,
+) as dag:
+  generator.create_manual_backfill_task(
+    dag,
+    default_start="2024-01-01T00:00:00",
+    execution_timeout=timedelta(hours=12),
+  )
+```
+
+The task reads optional `dag_run.conf` overrides:
+
+- `start`: ISO-8601 start boundary
+- `end`: ISO-8601 end boundary. If omitted, the task backfills up to the current UTC time.
+- `models`: one model name or a list of model names. If omitted, the task backfills all models.
+
 ## 💡 What You Get
 
 ### Your SQLMesh Project:

{sqlmesh_dag_generator-0.9.5 → sqlmesh_dag_generator-0.9.7}/README.md

@@ -81,9 +81,9 @@ If you run sub-hourly incremental models with `catchup=False`, outages can leave
 
 The package now supports an explicit recovery policy:
 
-- `recovery_mode="disabled"` (default): no runtime recovery tasks are added.
+- `recovery_mode="disabled"`: no runtime recovery tasks are added.
 - `recovery_mode="warn"`: add an integrity guard task that detects missing intervals and logs them.
-- `recovery_mode="bounded_auto"`: add the same guard task plus a bounded recovery task that replays missing intervals when the gap is within `recovery_max_intervals`.
+- `recovery_mode="bounded_auto"` (default): add the same guard task plus a bounded recovery task that replays missing intervals when the gap is within `recovery_max_intervals`.
 
 Example:
 
@@ -103,6 +103,41 @@ When `recovery_mode` is enabled, the package adds stable helper tasks to the DAG
 
 This keeps recovery explicit and observable in Airflow while preserving the default "no surprise backfills" behavior.
 
+### Manual Backfill Task
+
+For larger historical gaps, the package also exposes a first-class manual backfill task helper.
+Use it in a separate unscheduled DAG and trigger it only when you need to replay a broader window:
+
+```python
+from datetime import datetime, timedelta
+from airflow import DAG
+from sqlmesh_dag_generator import SQLMeshDAGGenerator
+
+generator = SQLMeshDAGGenerator(
+  sqlmesh_project_path="/path/to/project",
+  dag_id="my_pipeline",
+  gateway="prod",
+)
+
+with DAG(
+  dag_id="my_pipeline_manual_backfill",
+  schedule=None,
+  start_date=datetime(2024, 1, 1),
+  catchup=False,
+) as dag:
+  generator.create_manual_backfill_task(
+    dag,
+    default_start="2024-01-01T00:00:00",
+    execution_timeout=timedelta(hours=12),
+  )
+```
+
+The task reads optional `dag_run.conf` overrides:
+
+- `start`: ISO-8601 start boundary
+- `end`: ISO-8601 end boundary. If omitted, the task backfills up to the current UTC time.
+- `models`: one model name or a list of model names. If omitted, the task backfills all models.
+
 ## 💡 What You Get
 
 ### Your SQLMesh Project:

{sqlmesh_dag_generator-0.9.5 → sqlmesh_dag_generator-0.9.7}/setup.py

@@ -20,7 +20,7 @@ if requirements_file.exists():
 
 setup(
     name="sqlmesh-dag-generator",
-    version="0.9.5",
+    version="0.9.7",
     description="Open-source Airflow DAG generator for SQLMesh projects",
     long_description=long_description,
     long_description_content_type="text/markdown",

{sqlmesh_dag_generator-0.9.5 → sqlmesh_dag_generator-0.9.7}/sqlmesh_dag_generator/__init__.py

@@ -2,7 +2,7 @@
 SQLMesh DAG Generator - Open Source Airflow Integration for SQLMesh
 """
 
-__version__ = "0.9.5"
+__version__ = "0.9.7"
 
 
 from sqlmesh_dag_generator.generator import SQLMeshDAGGenerator

{sqlmesh_dag_generator-0.9.5 → sqlmesh_dag_generator-0.9.7}/sqlmesh_dag_generator/config.py

@@ -84,7 +84,7 @@ class SQLMeshConfig:
 @dataclass
 class RecoveryConfig:
     """Optional runtime recovery behavior for missed Airflow intervals."""
-    mode: str = "disabled"  # disabled, warn, bounded_auto
+    mode: str = "bounded_auto"  # disabled, warn, bounded_auto
     max_intervals: int = 6
     fail_on_excess_gap: bool = False
 
@@ -137,7 +137,7 @@ class GenerationConfig:
     dry_run: bool = False
     include_source_tables: bool = True  # Include upstream source tables as dummy tasks
     return_value: bool = True  # Whether to return execution result (XCom)
-    auto_replan_on_change: bool = False  # Automatically replan if models change
+    auto_replan_on_change: bool = True  # Automatically plan+apply before running models
     replan_timeout_hours: int = 6  # Timeout for replan task (initial backfills can be long!)
     skip_audits: bool = False  # Skip audit checks during execution
     enable_health_check: bool = False  # Add a pre-flight health check task

{sqlmesh_dag_generator-0.9.5 → sqlmesh_dag_generator-0.9.7}/sqlmesh_dag_generator/generator.py

@@ -1,7 +1,7 @@
 """
 Core DAG generator module
 """
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 import logging
 from pathlib import Path
 from typing import Dict, Optional, Union, Any, List
@@ -182,7 +182,7 @@ class SQLMeshDAGGenerator:
                 model_pattern=kwargs.get("model_pattern"),
                 include_source_tables=kwargs.get("include_source_tables", True),  # Default: enabled
                 return_value=kwargs.get("return_value", True),
-                auto_replan_on_change=kwargs.get("auto_replan_on_change", False),
+                auto_replan_on_change=kwargs.get("auto_replan_on_change", True),
                 replan_timeout_hours=kwargs.get("replan_timeout_hours", 6),  # Default: 6 hours for backfills
                 skip_audits=kwargs.get("skip_audits", False),
                 enable_health_check=kwargs.get("enable_health_check", False),
@@ -1337,6 +1337,162 @@ class SQLMeshDAGGenerator:
         all_tasks = {**tasks, **source_table_tasks}
         return all_tasks
 
+    def _build_runtime_context_kwargs(self) -> Dict[str, Any]:
+        """Build SQLMesh Context kwargs using merged runtime connection config when available."""
+        context_kwargs = {
+            "paths": self.config.sqlmesh.project_path,
+        }
+
+        if self.merged_config is not None:
+            context_kwargs["config"] = self.merged_config
+            if self.runtime_gateway is not None:
+                context_kwargs["gateway"] = self.runtime_gateway
+        else:
+            context_kwargs["gateway"] = self.config.sqlmesh.gateway
+
+        return context_kwargs
+
+    @staticmethod
+    def _parse_manual_backfill_datetime(value: Union[str, datetime], field_name: str) -> datetime:
+        """Parse a manual backfill boundary into a timezone-aware UTC datetime."""
+        if isinstance(value, datetime):
+            parsed = value
+        else:
+            raw_value = str(value).strip()
+            if not raw_value:
+                raise ValueError(f"Manual backfill requires a non-empty '{field_name}' value.")
+            parsed = datetime.fromisoformat(raw_value.replace("Z", "+00:00"))
+
+        if parsed.tzinfo is None:
+            return parsed.replace(tzinfo=timezone.utc)
+        return parsed.astimezone(timezone.utc)
+
+    @staticmethod
+    def _normalize_manual_backfill_models(
+        raw_models: Optional[Union[str, List[str]]]
+    ) -> Optional[List[str]]:
+        """Normalize manual backfill model selection into a list of model names."""
+        if raw_models is None:
+            return None
+        if isinstance(raw_models, str):
+            return [raw_models]
+        if not isinstance(raw_models, list) or not all(isinstance(model, str) for model in raw_models):
+            raise ValueError("Manual backfill 'models' must be a string or a list of model names.")
+        if not raw_models:
+            raise ValueError("Manual backfill 'models' cannot be an empty list.")
+        return raw_models
+
+    def create_manual_backfill_task(
+        self,
+        dag,
+        task_id: str = "sqlmesh_manual_backfill",
+        default_start: Optional[Union[str, datetime]] = None,
+        default_end: Optional[Union[str, datetime]] = None,
+        default_models: Optional[Union[str, List[str]]] = None,
+        execution_timeout: Optional[timedelta] = None,
+    ):
+        """
+        Create a manual SQLMesh backfill task for ad hoc historical replay.
+
+        The returned task reads optional overrides from ``dag_run.conf``:
+
+        - ``start``: ISO-8601 start boundary
+        - ``end``: ISO-8601 end boundary (defaults to current UTC time when omitted)
+        - ``models``: single model name or list of model names to replay
+
+        Args:
+            dag: Airflow DAG object.
+            task_id: Task ID for the manual backfill operator.
+            default_start: Default start boundary when ``dag_run.conf.start`` is omitted.
+            default_end: Default end boundary when ``dag_run.conf.end`` is omitted.
+            default_models: Optional default model selection.
+            execution_timeout: Optional Airflow execution timeout for the task.
+
+        Returns:
+            PythonOperator configured for manual SQLMesh backfill.
+        """
+        from airflow.exceptions import AirflowException
+        from airflow.operators.python import PythonOperator
+
+        normalized_default_models = self._normalize_manual_backfill_models(default_models)
+
+        def run_manual_backfill(**context):
+            dag_run = context.get("dag_run")
+            conf = dag_run.conf if dag_run and dag_run.conf else {}
+
+            requested_start = conf.get("start", default_start)
+            requested_end = conf.get("end", default_end)
+            requested_models = conf.get("models", normalized_default_models)
+
+            if requested_start is None:
+                raise AirflowException(
+                    "Manual backfill requires a start boundary. Provide default_start or dag_run.conf['start']."
+                )
+
+            try:
+                backfill_start = self._parse_manual_backfill_datetime(requested_start, "start")
+                if requested_end is None:
+                    backfill_end = datetime.now(timezone.utc).replace(microsecond=0)
+                else:
+                    backfill_end = self._parse_manual_backfill_datetime(requested_end, "end")
+                selected_models = self._normalize_manual_backfill_models(requested_models)
+            except ValueError as exc:
+                raise AirflowException(str(exc)) from exc
+
+            if backfill_start >= backfill_end:
+                raise AirflowException(
+                    f"Manual backfill start must be earlier than end. Got {backfill_start} >= {backfill_end}."
+                )
+
+            logger.info(
+                "Starting manual SQLMesh backfill for %s -> %s (models=%s)",
+                backfill_start,
+                backfill_end,
+                selected_models or "ALL",
+            )
+
+            self.load_sqlmesh_context()
+            if selected_models is not None:
+                if not self.models:
+                    self.extract_models()
+                unknown_models = sorted(set(selected_models) - set(self.models))
+                if unknown_models:
+                    raise AirflowException(
+                        f"Unknown SQLMesh models requested for manual backfill: {unknown_models}"
+                    )
+
+            run_ctx = Context(**self._build_runtime_context_kwargs())
+
+            run_kwargs = {
+                "environment": self.config.sqlmesh.environment,
+                "start": backfill_start,
+                "end": backfill_end,
+            }
+            if selected_models is not None:
+                run_kwargs["select_models"] = selected_models
+
+            run_sig = inspect.signature(run_ctx.run)
+            if "skip_audits" in run_sig.parameters and self.config.generation.skip_audits:
+                run_kwargs["skip_audits"] = True
+
+            result = run_ctx.run(**run_kwargs)
+            status = result.name if hasattr(result, "name") else str(result)
+            logger.info("Manual SQLMesh backfill finished with status: %s", status)
+
+            return {
+                "status": status,
+                "start": backfill_start.isoformat(),
+                "end": backfill_end.isoformat(),
+                "models": selected_models or "ALL",
+            }
+
+        return PythonOperator(
+            task_id=task_id,
+            python_callable=run_manual_backfill,
+            execution_timeout=execution_timeout,
+            dag=dag,
+        )
+
     def _get_output_path(self) -> Path:
         """Get the output file path for the generated DAG"""
         output_dir = Path(self.config.generation.output_dir)

{sqlmesh_dag_generator-0.9.5 → sqlmesh_dag_generator-0.9.7/sqlmesh_dag_generator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh-dag-generator
-Version: 0.9.5
+Version: 0.9.7
 Summary: Open-source Airflow DAG generator for SQLMesh projects
 Home-page: https://github.com/kubolko/sqlmesh-dag-generator
 Author: Jakub Sumionka
@@ -128,9 +128,9 @@ If you run sub-hourly incremental models with `catchup=False`, outages can leave
 
 The package now supports an explicit recovery policy:
 
-- `recovery_mode="disabled"` (default): no runtime recovery tasks are added.
+- `recovery_mode="disabled"`: no runtime recovery tasks are added.
 - `recovery_mode="warn"`: add an integrity guard task that detects missing intervals and logs them.
-- `recovery_mode="bounded_auto"`: add the same guard task plus a bounded recovery task that replays missing intervals when the gap is within `recovery_max_intervals`.
+- `recovery_mode="bounded_auto"` (default): add the same guard task plus a bounded recovery task that replays missing intervals when the gap is within `recovery_max_intervals`.
 
 Example:
 
@@ -150,6 +150,41 @@ When `recovery_mode` is enabled, the package adds stable helper tasks to the DAG
 
 This keeps recovery explicit and observable in Airflow while preserving the default "no surprise backfills" behavior.
 
+### Manual Backfill Task
+
+For larger historical gaps, the package also exposes a first-class manual backfill task helper.
+Use it in a separate unscheduled DAG and trigger it only when you need to replay a broader window:
+
+```python
+from datetime import datetime, timedelta
+from airflow import DAG
+from sqlmesh_dag_generator import SQLMeshDAGGenerator
+
+generator = SQLMeshDAGGenerator(
+  sqlmesh_project_path="/path/to/project",
+  dag_id="my_pipeline",
+  gateway="prod",
+)
+
+with DAG(
+  dag_id="my_pipeline_manual_backfill",
+  schedule=None,
+  start_date=datetime(2024, 1, 1),
+  catchup=False,
+) as dag:
+  generator.create_manual_backfill_task(
+    dag,
+    default_start="2024-01-01T00:00:00",
+    execution_timeout=timedelta(hours=12),
+  )
+```
+
+The task reads optional `dag_run.conf` overrides:
+
+- `start`: ISO-8601 start boundary
+- `end`: ISO-8601 end boundary. If omitted, the task backfills up to the current UTC time.
+- `models`: one model name or a list of model names. If omitted, the task backfills all models.
+
 ## 💡 What You Get
 
 ### Your SQLMesh Project:

{sqlmesh_dag_generator-0.9.5 → sqlmesh_dag_generator-0.9.7}/tests/test_config.py

@@ -10,6 +10,7 @@ from sqlmesh_dag_generator.config import (
     AirflowConfig,
     GenerationConfig,
     DAGGeneratorConfig,
+    RecoveryConfig,
 )
 
 
@@ -39,6 +40,15 @@ def test_airflow_config_creation():
     assert "test" in config.tags
 
 
+def test_recovery_config_defaults_to_bounded_auto():
+    """Test default recovery mode enables bounded auto replay."""
+    config = RecoveryConfig()
+
+    assert config.mode == "bounded_auto"
+    assert config.max_intervals == 6
+    assert config.fail_on_excess_gap is False
+
+
 def test_generation_config_defaults():
     """Test GenerationConfig default values"""
     config = GenerationConfig()

{sqlmesh_dag_generator-0.9.5 → sqlmesh_dag_generator-0.9.7}/tests/test_generator.py

@@ -383,6 +383,92 @@ class TestSQLMeshDAGGenerator:
         assert run_kwargs["select_models"] == ["dwh.raw_5m"]
         assert recovery_result["gap_intervals"] == 3
 
+    @patch("sqlmesh_dag_generator.generator.Context")
+    def test_create_manual_backfill_task_uses_defaults_and_conf_overrides(self, mock_context):
+        """Test manual backfill task supports default windows and dag_run.conf overrides."""
+        from airflow import DAG
+
+        run_ctx = MagicMock()
+        run_result = MagicMock()
+        run_result.name = "SUCCESS"
+        run_ctx.run.return_value = run_result
+        mock_context.return_value = run_ctx
+
+        generator = SQLMeshDAGGenerator(
+            sqlmesh_project_path="/tmp/project",
+            dag_id="test",
+        )
+        generator.load_sqlmesh_context = MagicMock(return_value=run_ctx)
+        generator.models = {
+            "dwh.raw_5m": SQLMeshModelInfo(
+                name="dwh.raw_5m",
+                dependencies=set(),
+                interval_unit="FIVE_MINUTE",
+                kind="INCREMENTAL_BY_TIME_RANGE",
+            ),
+            "dwh.report": SQLMeshModelInfo(
+                name="dwh.report",
+                dependencies={"dwh.raw_5m"},
+                interval_unit="HOUR",
+                kind="FULL",
+            ),
+        }
+
+        with DAG("test", start_date=datetime(2024, 1, 1)) as dag:
+            task = generator.create_manual_backfill_task(
+                dag,
+                default_start="2024-01-01T00:00:00",
+            )
+
+        dag_run = MagicMock()
+        dag_run.conf = {
+            "end": "2024-01-01T02:00:00",
+            "models": ["dwh.raw_5m"],
+        }
+        result = task.python_callable(dag_run=dag_run)
+
+        run_ctx.run.assert_called_once()
+        _, run_kwargs = run_ctx.run.call_args
+        assert run_kwargs["start"].isoformat() == "2024-01-01T00:00:00+00:00"
+        assert run_kwargs["end"].isoformat() == "2024-01-01T02:00:00+00:00"
+        assert run_kwargs["select_models"] == ["dwh.raw_5m"]
+        assert result["status"] == "SUCCESS"
+        assert result["models"] == ["dwh.raw_5m"]
+
+    def test_create_manual_backfill_task_rejects_unknown_models(self):
+        """Test manual backfill task validates requested model names."""
+        from airflow import DAG
+        from airflow.exceptions import AirflowException
+
+        generator = SQLMeshDAGGenerator(
+            sqlmesh_project_path="/tmp/project",
+            dag_id="test",
+        )
+        generator.load_sqlmesh_context = MagicMock()
+        generator.models = {
+            "dwh.raw_5m": SQLMeshModelInfo(
+                name="dwh.raw_5m",
+                dependencies=set(),
+                interval_unit="FIVE_MINUTE",
+                kind="INCREMENTAL_BY_TIME_RANGE",
+            ),
+        }
+
+        with DAG("test", start_date=datetime(2024, 1, 1)) as dag:
+            task = generator.create_manual_backfill_task(
+                dag,
+                default_start="2024-01-01T00:00:00",
+            )
+
+        dag_run = MagicMock()
+        dag_run.conf = {
+            "end": "2024-01-01T02:00:00",
+            "models": ["dwh.missing"],
+        }
+
+        with pytest.raises(AirflowException, match="Unknown SQLMesh models"):
+            task.python_callable(dag_run=dag_run)
+
 
 class TestDynamicFeatures:
     """Test dynamic DAG specific features"""