sqlmesh-dag-generator 0.9.7__tar.gz → 0.9.9__tar.gz

{sqlmesh_dag_generator-0.9.7/sqlmesh_dag_generator.egg-info → sqlmesh_dag_generator-0.9.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh-dag-generator
-Version: 0.9.7
+Version: 0.9.9
 Summary: Open-source Airflow DAG generator for SQLMesh projects
 Home-page: https://github.com/kubolko/sqlmesh-dag-generator
 Author: Jakub Sumionka

{sqlmesh_dag_generator-0.9.7 → sqlmesh_dag_generator-0.9.9}/setup.py

@@ -20,7 +20,7 @@ if requirements_file.exists():
 
 setup(
     name="sqlmesh-dag-generator",
-    version="0.9.7",
+    version="0.9.9",
     description="Open-source Airflow DAG generator for SQLMesh projects",
     long_description=long_description,
     long_description_content_type="text/markdown",

{sqlmesh_dag_generator-0.9.7 → sqlmesh_dag_generator-0.9.9}/sqlmesh_dag_generator/__init__.py

@@ -2,7 +2,7 @@
 SQLMesh DAG Generator - Open Source Airflow Integration for SQLMesh
 """
 
-__version__ = "0.9.7"
+__version__ = "0.9.9"
 
 
 from sqlmesh_dag_generator.generator import SQLMeshDAGGenerator

{sqlmesh_dag_generator-0.9.7 → sqlmesh_dag_generator-0.9.9}/sqlmesh_dag_generator/config.py

@@ -138,7 +138,7 @@ class GenerationConfig:
     include_source_tables: bool = True  # Include upstream source tables as dummy tasks
     return_value: bool = True  # Whether to return execution result (XCom)
     auto_replan_on_change: bool = True  # Automatically plan+apply before running models
-    replan_timeout_hours: int = 6  # Timeout for replan task (initial backfills can be long!)
+    replan_timeout_hours: Optional[int] = 6  # None disables the replan task timeout.
     skip_audits: bool = False  # Skip audit checks during execution
     enable_health_check: bool = False  # Add a pre-flight health check task
     # Tag-based filtering - only include models with any of these tags

{sqlmesh_dag_generator-0.9.7 → sqlmesh_dag_generator-0.9.9}/sqlmesh_dag_generator/generator.py

@@ -2,6 +2,7 @@
 Core DAG generator module
 """
 from datetime import datetime, timedelta, timezone
+import inspect
 import logging
 from pathlib import Path
 from typing import Dict, Optional, Union, Any, List
@@ -183,7 +184,7 @@ class SQLMeshDAGGenerator:
                 include_source_tables=kwargs.get("include_source_tables", True),  # Default: enabled
                 return_value=kwargs.get("return_value", True),
                 auto_replan_on_change=kwargs.get("auto_replan_on_change", True),
-                replan_timeout_hours=kwargs.get("replan_timeout_hours", 6),  # Default: 6 hours for backfills
+                replan_timeout_hours=kwargs.get("replan_timeout_hours", 6),  # None disables the replan timeout.
                 skip_audits=kwargs.get("skip_audits", False),
                 enable_health_check=kwargs.get("enable_health_check", False),
                 include_tags=kwargs.get("include_tags"),
@@ -991,14 +992,25 @@ class SQLMeshDAGGenerator:
                 }
 
             from datetime import timedelta as td
+
+            replan_timeout_hours = self.config.generation.replan_timeout_hours
             replan_task = PythonOperator(
                 task_id="sqlmesh_plan_apply",
                 python_callable=run_replan,
                 dag=dag,
-                # Use custom timeout for replan task - initial backfills can take hours!
-                execution_timeout=td(hours=self.config.generation.replan_timeout_hours),
+                execution_timeout=(
+                    td(hours=replan_timeout_hours)
+                    if replan_timeout_hours is not None
+                    else None
+                ),
             )
-            logger.info(f"Created auto-replan task: sqlmesh_plan_apply (timeout: {self.config.generation.replan_timeout_hours}h)")
+            if replan_timeout_hours is None:
+                logger.info("Created auto-replan task: sqlmesh_plan_apply (timeout: disabled)")
+            else:
+                logger.info(
+                    "Created auto-replan task: sqlmesh_plan_apply (timeout: %sh)",
+                    replan_timeout_hours,
+                )
 
             # Link health check to replan if both exist
             if health_check_task:
@@ -1184,7 +1196,7 @@ class SQLMeshDAGGenerator:
             task_id = model_info.get_task_id()
 
             # Create the execution function
-            def make_callable(m_name, m_fqn):
+            def make_callable(m_name, m_fqn, m_interval_minutes=None):
                 def execute_model(**context):
                     from sqlmesh import Context
 
@@ -1212,16 +1224,42 @@ class SQLMeshDAGGenerator:
                     start = context.get('data_interval_start') or context.get('execution_date')
                     end = context.get('data_interval_end') or context.get('execution_date')
 
+                    # When this model's interval is COARSER than the DAG tick,
+                    # the tick's data_interval window (e.g. 5 minutes on a mixed
+                    # sub-hourly + hourly project) spans no full model interval.
+                    # SQLMesh then returns NOTHING_TO_DO and the model's state
+                    # silently freezes while the task still reports success.
+                    # In that case omit start/end so SQLMesh selects the correct
+                    # interval(s) from the model's own cron instead.
+                    if (
+                        m_interval_minutes
+                        and expected_interval_minutes
+                        and m_interval_minutes > expected_interval_minutes
+                    ):
+                        logger.info(
+                            "Model %s interval (%s min) is coarser than the DAG tick "
+                            "(%s min); running without an explicit window so SQLMesh "
+                            "picks the due interval(s) from the model cron.",
+                            m_fqn, m_interval_minutes, expected_interval_minutes,
+                        )
+                        start = None
+                        end = None
+
                     # Run the model with proper time range
                     try:
                         # Build run kwargs - only include skip_audits if enabled
                         # (some SQLMesh versions may not support this parameter)
                         run_kwargs = {
                             "environment": self.config.sqlmesh.environment,
-                            "start": start,
-                            "end": end,
                             "select_models": [m_fqn],
                         }
+                        # Only pass an explicit window when we have one. For
+                        # coarser-than-tick models start/end were cleared above so
+                        # SQLMesh selects the due interval(s) from the model cron.
+                        if start is not None:
+                            run_kwargs["start"] = start
+                        if end is not None:
+                            run_kwargs["end"] = end
 
                         # Check if skip_audits is supported by inspecting the method signature
                         import inspect
@@ -1299,7 +1337,13 @@ class SQLMeshDAGGenerator:
             # Create PythonOperator
             task = PythonOperator(
                 task_id=task_id,
-                python_callable=make_callable(model_name, model_info.name),
+                python_callable=make_callable(
+                    model_name,
+                    model_info.name,
+                    get_interval_frequency_minutes(model_info.interval_unit)
+                    if model_info.interval_unit is not None
+                    else None,
+                ),
                 dag=dag,
             )
 

{sqlmesh_dag_generator-0.9.7 → sqlmesh_dag_generator-0.9.9/sqlmesh_dag_generator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh-dag-generator
-Version: 0.9.7
+Version: 0.9.9
 Summary: Open-source Airflow DAG generator for SQLMesh projects
 Home-page: https://github.com/kubolko/sqlmesh-dag-generator
 Author: Jakub Sumionka

{sqlmesh_dag_generator-0.9.7 → sqlmesh_dag_generator-0.9.9}/tests/test_generator.py

@@ -204,6 +204,82 @@ class TestSQLMeshDAGGenerator:
                 assert hasattr(task, 'task_id')
                 assert hasattr(task, 'python_callable')
 
+    @patch("sqlmesh.Context")
+    @patch("sqlmesh_dag_generator.generator.Context")
+    def test_execute_model_omits_window_for_coarser_than_tick_models(self, mock_mod_context, mock_sqlmesh_context):
+        """A model whose interval is coarser than the DAG tick must not be run
+        with the tick's narrow window.
+
+        Root cause regression: when a project mixes sub-hourly and hourly models,
+        the DAG schedule is the global minimum (5 min). execute_model passed the
+        tick's data_interval (e.g. a 5-minute window) as start/end to EVERY model,
+        including hourly ones. SQLMesh returns NOTHING_TO_DO for an hourly model
+        given a sub-hour window that spans no full hourly boundary, so the hourly
+        model's state froze while its task still reported success. The fix: for a
+        model whose interval is coarser than the scheduler tick, omit start/end so
+        SQLMesh selects the correct interval(s) from the model cron.
+        """
+        from airflow import DAG
+
+        run_ctx = MagicMock()
+        run_result = MagicMock()
+        run_result.name = "SUCCESS"
+        run_ctx.run.return_value = run_result
+        # execute_model does `from sqlmesh import Context` locally, so the
+        # sqlmesh.Context patch is the one that matters here.
+        mock_sqlmesh_context.return_value = run_ctx
+        mock_mod_context.return_value = run_ctx
+
+        generator = SQLMeshDAGGenerator(
+            sqlmesh_project_path="/tmp/project",
+            dag_id="test",
+        )
+        # Mixed project: 5-min model sets the DAG tick to 5 minutes; the hourly
+        # model is coarser than the tick.
+        generator.models = {
+            "dwh.raw_5m": SQLMeshModelInfo(
+                name="dwh.raw_5m",
+                dependencies=set(),
+                interval_unit="FIVE_MINUTE",
+                kind="INCREMENTAL_BY_TIME_RANGE",
+            ),
+            "dwh.f_hourly": SQLMeshModelInfo(
+                name="dwh.f_hourly",
+                dependencies=set(),
+                interval_unit="HOUR",
+                kind="INCREMENTAL_BY_TIME_RANGE",
+            ),
+        }
+
+        with DAG("test", start_date=datetime(2024, 1, 1)) as dag:
+            tasks = generator.create_tasks_in_dag(dag)
+
+        # 5-minute tick window (what the scheduler hands every task).
+        tick_start = datetime(2024, 1, 6, 12, 25)
+        tick_end = datetime(2024, 1, 6, 12, 30)
+
+        # Hourly model (coarser than tick): must be run WITHOUT start/end.
+        run_ctx.run.reset_mock()
+        tasks["dwh.f_hourly"].python_callable(
+            data_interval_start=tick_start, data_interval_end=tick_end,
+        )
+        run_ctx.run.assert_called_once()
+        _, hourly_kwargs = run_ctx.run.call_args
+        assert hourly_kwargs.get("start") is None
+        assert hourly_kwargs.get("end") is None
+        assert hourly_kwargs["select_models"] == ["dwh.f_hourly"]
+
+        # 5-min model (matches tick): keeps the explicit window (unchanged).
+        run_ctx.run.reset_mock()
+        tasks["dwh.raw_5m"].python_callable(
+            data_interval_start=tick_start, data_interval_end=tick_end,
+        )
+        run_ctx.run.assert_called_once()
+        _, raw_kwargs = run_ctx.run.call_args
+        assert raw_kwargs["start"] == tick_start
+        assert raw_kwargs["end"] == tick_end
+        assert raw_kwargs["select_models"] == ["dwh.raw_5m"]
+
     def test_static_dag_generation(self, demo_sqlmesh_project):
         """Test static DAG generation (alternative mode)"""
         config = DAGGeneratorConfig.from_dict({