dbt-cube-sync 0.1.0a10__py3-none-any.whl → 0.1.0a11__py3-none-any.whl

dbt_cube_sync/cli.py

@@ -357,21 +357,22 @@ def sync_all(
         modified_models = set()
         removed_models = set()
 
-        # ============================================================
-        # STEP 1: Incremental dbt → Cube.js sync
-        # ============================================================
-        click.echo("\n[1/3] dbt → Cube.js schemas")
-        click.echo("-" * 40)
-
-        # Initialize state manager
+        # Initialize state manager and load previous state
         state_manager = StateManager(state_path)
         previous_state = None
+        current_state = None
 
         if not force_full_sync:
             previous_state = state_manager.load_state()
             if previous_state:
                 click.echo(f"  Loaded state from {state_path}")
 
+        # ============================================================
+        # STEP 1: Incremental dbt → Cube.js sync
+        # ============================================================
+        click.echo("\n[1/3] dbt → Cube.js schemas")
+        click.echo("-" * 40)
+
         # Parse manifest
         parser = DbtParser(
             manifest_path=manifest,
@@ -414,65 +415,112 @@ def sync_all(
 
         # Generate Cube.js files for changed models
         generated_files = {}
-        if node_ids_to_process:
-            parsed_models = parser.parse_models(node_ids_filter=node_ids_to_process)
-
-            if parsed_models:
-                generator = CubeGenerator('./cube/templates', output)
-                generated_files = generator.generate_cube_files(parsed_models)
-                click.echo(f"  Generated {len(generated_files)} Cube.js files")
-
-        # Save state
+        cube_sync_error = None
+        try:
+            if node_ids_to_process:
+                parsed_models = parser.parse_models(node_ids_filter=node_ids_to_process)
+
+                if parsed_models:
+                    generator = CubeGenerator('./cube/templates', output)
+                    generated_files = generator.generate_cube_files(parsed_models)
+                    click.echo(f"  Generated {len(generated_files)} Cube.js files")
+        except Exception as e:
+            cube_sync_error = str(e)
+            click.echo(f"  Error: {cube_sync_error}", err=True)
+
+        # Build/update state
         if changes_detected or force_full_sync:
             if previous_state and not force_full_sync:
-                new_state = state_manager.merge_state(
+                current_state = state_manager.merge_state(
                     previous_state, manifest, manifest_nodes, generated_files, removed_models
                 )
             else:
-                new_state = state_manager.create_state_from_results(
+                current_state = state_manager.create_state_from_results(
                     manifest, manifest_nodes, generated_files
                 )
-            state_manager.save_state(new_state)
-            click.echo(f"  State saved to {state_path}")
+        else:
+            # No changes - use previous state or create empty one
+            current_state = previous_state or state_manager.create_state_from_results(
+                manifest, manifest_nodes, {}
+            )
+
+        # Update cube_sync step state
+        current_state = state_manager.update_step_state(
+            current_state,
+            'cube_sync',
+            'failed' if cube_sync_error else 'success',
+            cube_sync_error
+        )
+        state_manager.save_state(current_state)
+        click.echo(f"  State saved to {state_path}")
 
         # ============================================================
         # STEP 2: Sync to Superset (if configured)
         # ============================================================
-        if superset_url and superset_username and superset_password:
-            click.echo("\n[2/3] Cube.js → Superset")
-            click.echo("-" * 40)
+        click.echo("\n[2/3] Cube.js → Superset")
+        click.echo("-" * 40)
 
-            if not changes_detected and not force_full_sync:
-                click.echo("  Skipped - no changes detected")
-            else:
-                connector_config = {
-                    'url': superset_url,
-                    'username': superset_username,
-                    'password': superset_password,
-                    'database_name': cube_connection_name
-                }
-
-                connector = ConnectorRegistry.get_connector('superset', **connector_config)
-                results = connector.sync_cube_schemas(output)
-
-                successful = sum(1 for r in results if r.status == 'success')
-                failed = sum(1 for r in results if r.status == 'failed')
-                click.echo(f"  Synced: {successful} successful, {failed} failed")
-        else:
-            click.echo("\n[2/3] Cube.js → Superset")
-            click.echo("-" * 40)
+        if not superset_url or not superset_username or not superset_password:
             click.echo("  Skipped - no Superset credentials provided")
+            current_state = state_manager.update_step_state(current_state, 'superset_sync', 'skipped')
+            state_manager.save_state(current_state)
+        else:
+            should_run_superset = state_manager.should_run_step(
+                'superset_sync', previous_state, changes_detected
+            ) or force_full_sync
+
+            if not should_run_superset:
+                click.echo("  Skipped - no changes and previous sync succeeded")
+            else:
+                superset_error = None
+                try:
+                    connector_config = {
+                        'url': superset_url,
+                        'username': superset_username,
+                        'password': superset_password,
+                        'database_name': cube_connection_name
+                    }
+
+                    connector = ConnectorRegistry.get_connector('superset', **connector_config)
+                    results = connector.sync_cube_schemas(output)
+
+                    successful = sum(1 for r in results if r.status == 'success')
+                    failed = sum(1 for r in results if r.status == 'failed')
+                    click.echo(f"  Synced: {successful} successful, {failed} failed")
+
+                    if failed > 0:
+                        superset_error = f"{failed} datasets failed to sync"
+                except Exception as e:
+                    superset_error = str(e)
+                    click.echo(f"  Error: {superset_error}", err=True)
+
+                current_state = state_manager.update_step_state(
+                    current_state,
+                    'superset_sync',
+                    'failed' if superset_error else 'success',
+                    superset_error
+                )
+                state_manager.save_state(current_state)
 
         # ============================================================
         # STEP 3: Update RAG embeddings (if configured)
         # ============================================================
-        if rag_api_url:
-            click.echo("\n[3/3] Update RAG embeddings")
-            click.echo("-" * 40)
+        click.echo("\n[3/3] Update RAG embeddings")
+        click.echo("-" * 40)
 
-            if not changes_detected and not force_full_sync:
-                click.echo("  Skipped - no changes detected")
+        if not rag_api_url:
+            click.echo("  Skipped - no RAG API URL provided")
+            current_state = state_manager.update_step_state(current_state, 'rag_sync', 'skipped')
+            state_manager.save_state(current_state)
+        else:
+            should_run_rag = state_manager.should_run_step(
+                'rag_sync', previous_state, changes_detected
+            ) or force_full_sync
+
+            if not should_run_rag:
+                click.echo("  Skipped - no changes and previous sync succeeded")
             else:
+                rag_error = None
                 try:
                     # Call the RAG API to re-ingest embeddings
                     response = requests.post(
@@ -485,13 +533,19 @@ def sync_all(
                         result = response.json()
                         click.echo(f"  Ingested {result.get('schemas_ingested', 0)} schema documents")
                     else:
-                        click.echo(f"  Warning: RAG API returned {response.status_code}", err=True)
+                        rag_error = f"RAG API returned {response.status_code}"
+                        click.echo(f"  Error: {rag_error}", err=True)
                 except requests.RequestException as e:
-                    click.echo(f"  Warning: Could not reach RAG API: {e}", err=True)
-        else:
-            click.echo("\n[3/3] Update RAG embeddings")
-            click.echo("-" * 40)
-            click.echo("  Skipped - no RAG API URL provided")
+                    rag_error = str(e)
+                    click.echo(f"  Error: Could not reach RAG API: {rag_error}", err=True)
+
+                current_state = state_manager.update_step_state(
+                    current_state,
+                    'rag_sync',
+                    'failed' if rag_error else 'success',
+                    rag_error
+                )
+                state_manager.save_state(current_state)
 
         # ============================================================
         # Summary
@@ -500,12 +554,27 @@ def sync_all(
         click.echo("SYNC COMPLETE")
         click.echo("=" * 60)
 
+        # Show step statuses
+        click.echo(f"  Cube sync:     {current_state.cube_sync.status if current_state.cube_sync else 'unknown'}")
+        click.echo(f"  Superset sync: {current_state.superset_sync.status if current_state.superset_sync else 'unknown'}")
+        click.echo(f"  RAG sync:      {current_state.rag_sync.status if current_state.rag_sync else 'unknown'}")
+
         if changes_detected or force_full_sync:
             click.echo(f"  Models processed: {len(added_models) + len(modified_models)}")
             click.echo(f"  Models removed: {len(removed_models)}")
             click.echo(f"  Cube.js files generated: {len(generated_files)}")
         else:
-            click.echo("  No changes - everything is up to date")
+            click.echo("  No model changes detected")
+
+        # Exit with error if any step failed
+        any_failed = (
+            (current_state.cube_sync and current_state.cube_sync.status == 'failed') or
+            (current_state.superset_sync and current_state.superset_sync.status == 'failed') or
+            (current_state.rag_sync and current_state.rag_sync.status == 'failed')
+        )
+        if any_failed:
+            click.echo("\n  ⚠️  Some steps failed - they will be retried on next run")
+            sys.exit(1)
 
     except Exception as e:
         click.echo(f"Error: {str(e)}", err=True)

dbt_cube_sync/core/models.py

@@ -114,9 +114,20 @@ class ModelState(BaseModel):
     output_file: str
 
 
+class StepState(BaseModel):
+    """Represents the state of a pipeline step"""
+    status: str  # 'success', 'failed', 'skipped'
+    last_run: Optional[str] = None
+    error: Optional[str] = None
+
+
 class SyncState(BaseModel):
     """Represents the overall state for incremental sync"""
-    version: str = "1.0"
+    version: str = "1.1"
     last_sync_timestamp: str
     manifest_path: str
-    models: Dict[str, ModelState] = {}
+    models: Dict[str, ModelState] = {}
+    # Step states for tracking pipeline progress
+    cube_sync: Optional[StepState] = None
+    superset_sync: Optional[StepState] = None
+    rag_sync: Optional[StepState] = None

dbt_cube_sync/core/state_manager.py

@@ -10,7 +10,7 @@ from datetime import datetime
 from pathlib import Path
 from typing import Dict, List, Optional, Set, Tuple
 
-from .models import ModelState, SyncState
+from .models import ModelState, StepState, SyncState
 
 
 class StateManager:
@@ -219,3 +219,65 @@ class StateManager:
                     files_to_delete.append(output_file)
 
         return files_to_delete
+
+    def should_run_step(
+        self,
+        step_name: str,
+        previous_state: Optional[SyncState],
+        changes_detected: bool,
+    ) -> bool:
+        """
+        Determine if a pipeline step should run.
+
+        A step should run if:
+        - There are changes detected, OR
+        - The previous run of this step failed
+
+        Args:
+            step_name: Name of the step ('cube_sync', 'superset_sync', 'rag_sync')
+            previous_state: Previous sync state
+            changes_detected: Whether model changes were detected
+
+        Returns:
+            True if the step should run
+        """
+        if changes_detected:
+            return True
+
+        if previous_state is None:
+            return True
+
+        step_state = getattr(previous_state, step_name, None)
+        if step_state is None:
+            return True
+
+        # Re-run if previous attempt failed
+        return step_state.status == 'failed'
+
+    def update_step_state(
+        self,
+        state: SyncState,
+        step_name: str,
+        status: str,
+        error: Optional[str] = None,
+    ) -> SyncState:
+        """
+        Update the state of a pipeline step.
+
+        Args:
+            state: Current sync state
+            step_name: Name of the step ('cube_sync', 'superset_sync', 'rag_sync')
+            status: Step status ('success', 'failed', 'skipped')
+            error: Error message if failed
+
+        Returns:
+            Updated SyncState
+        """
+        timestamp = datetime.utcnow().isoformat() + "Z"
+        step_state = StepState(
+            status=status,
+            last_run=timestamp,
+            error=error,
+        )
+        setattr(state, step_name, step_state)
+        return state

{dbt_cube_sync-0.1.0a10.dist-info → dbt_cube_sync-0.1.0a11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dbt-cube-sync
-Version: 0.1.0a10
+Version: 0.1.0a11
 Summary: Synchronization tool for dbt models to Cube.js schemas and BI tools
 Author: Ponder
 Requires-Python: >=3.9,<4.0

{dbt_cube_sync-0.1.0a10.dist-info → dbt_cube_sync-0.1.0a11.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dbt_cube_sync/__init__.py,sha256=aifkfgUDRPL5v0LZzceH2LXu66YDkJjdpvKwXsdikbI,113
-dbt_cube_sync/cli.py,sha256=AxSVF3hJJqovk51mjA8Nyyte5NkfukSF3sAjk_VYJ6Y,20992
+dbt_cube_sync/cli.py,sha256=bD7rOPstM9qESCTaovEbfdl80wz5yhk8GDnHnb-hBd0,24117
 dbt_cube_sync/config.py,sha256=qhGE7CxTmh0RhPizgd3x3Yj-3L2LoC00UQIDT0q9FlQ,3858
 dbt_cube_sync/connectors/__init__.py,sha256=NG6tYZ3CYD5bG_MfNLZrUM8YoBEKArG8-AOmJ8pwvQI,52
 dbt_cube_sync/connectors/base.py,sha256=JLzerxJdt34z0kWuyieL6UQhf5_dUYPGmwkiRWBuSPY,2802
@@ -10,9 +10,9 @@ dbt_cube_sync/core/__init__.py,sha256=kgsawtU5dqEvnHz6dU8qwJbH3rtIV7QlK2MhtYVDCa
 dbt_cube_sync/core/cube_generator.py,sha256=DtmaA_dtWmBVJnSWHVoQi-3KEsRc0axHZpCUEcKeYAk,11061
 dbt_cube_sync/core/db_inspector.py,sha256=V_cd12FBXj-1gB2JZeYmkQluUO-UYufy_tvfYoJXCGI,5073
 dbt_cube_sync/core/dbt_parser.py,sha256=KbhDoB0ULP6JDUPZPDVbm9yCtRKrW17ptGoJvVLtueY,12763
-dbt_cube_sync/core/models.py,sha256=2s5iZ9MEBGfSzkB4HJB5vG0mZqNXNJSfAD3Byw1IVe4,3203
-dbt_cube_sync/core/state_manager.py,sha256=7uXJtlZBIWj6s6XgAhNlP6UHdfhH0y461iyQlfidqGI,7233
-dbt_cube_sync-0.1.0a10.dist-info/METADATA,sha256=Foy8KI7-ILdZwTvejfjukArtnGNJJg85vwIlQoMS31w,10681
-dbt_cube_sync-0.1.0a10.dist-info/WHEEL,sha256=3ny-bZhpXrU6vSQ1UPG34FoxZBp3lVcvK0LkgUz6VLk,88
-dbt_cube_sync-0.1.0a10.dist-info/entry_points.txt,sha256=iEAB_nZ1AoSeFwSHPY2tr02xmTHLVFKp5CJeFh0AfCw,56
-dbt_cube_sync-0.1.0a10.dist-info/RECORD,,
+dbt_cube_sync/core/models.py,sha256=KoYrIRjnUPwDigrJmvldv0kb0V6rY141XavjRKfgB3A,3578
+dbt_cube_sync/core/state_manager.py,sha256=MoEzAkyVPvSEiiiPaBz_V8X5hqZqzZ4t1w-Sjf_3oUA,8963
+dbt_cube_sync-0.1.0a11.dist-info/METADATA,sha256=Ayr_7pO1GjQpZ1btSdmsMsm8n4ueiviwuY5ot7roh2Q,10681
+dbt_cube_sync-0.1.0a11.dist-info/WHEEL,sha256=3ny-bZhpXrU6vSQ1UPG34FoxZBp3lVcvK0LkgUz6VLk,88
+dbt_cube_sync-0.1.0a11.dist-info/entry_points.txt,sha256=iEAB_nZ1AoSeFwSHPY2tr02xmTHLVFKp5CJeFh0AfCw,56
+dbt_cube_sync-0.1.0a11.dist-info/RECORD,,