dbt-cube-sync 0.1.0a3__tar.gz → 0.1.0a5__tar.gz

{dbt_cube_sync-0.1.0a3 → dbt_cube_sync-0.1.0a5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dbt-cube-sync
-Version: 0.1.0a3
+Version: 0.1.0a5
 Summary: Synchronization tool for dbt models to Cube.js schemas and BI tools
 Author: Ponder
 Requires-Python: >=3.9,<4.0

{dbt_cube_sync-0.1.0a3 → dbt_cube_sync-0.1.0a5}/dbt_cube_sync/connectors/superset.py

@@ -337,12 +337,26 @@ class SupersetConnector(BaseConnector):
             'count_distinct': 'COUNT(DISTINCT'
         }
         
+        # Remove Cube.js ${} syntax and convert to plain SQL column references
+        cleaned_expression = self._clean_cube_expression(sql_expression)
+        
         agg_func = agg_mapping.get(agg_type, 'SUM')
         
         if agg_type == 'count_distinct':
-            return f"{agg_func} {sql_expression})"
+            return f"{agg_func} {cleaned_expression})"
         else:
-            return f"{agg_func}({sql_expression})"
+            return f"{agg_func}({cleaned_expression})"
+    
+    def _clean_cube_expression(self, expression: str) -> str:
+        """Convert Cube.js expressions to SQL column references for Superset"""
+        import re
+        
+        # Remove ${} syntax - convert ${column_name} to column_name
+        cleaned = re.sub(r'\$\{([^}]+)\}', r'\1', expression)
+        
+        # Handle more complex expressions like arithmetic
+        # Keep parentheses and operators but clean column references
+        return cleaned
     
     def _create_or_update_dataset(self, schema_info: Dict[str, Any]) -> int:
         """Create a new dataset or update existing one"""
@@ -506,37 +520,41 @@ class SupersetConnector(BaseConnector):
     
     def _update_metrics(self, existing_metrics: List[dict], measures: List[dict]) -> List[dict]:
         """Update metrics with new measures"""
-        # Clean existing metrics
+        # Clean existing metrics and create a lookup by name
         updated_metrics = []
+        existing_metric_names = {}
+        
         for metric in existing_metrics:
             clean_metric = {k: v for k, v in metric.items() 
                           if k not in ['created_on', 'changed_on', 'uuid']}
+            existing_metric_names[metric.get('metric_name')] = len(updated_metrics)
             updated_metrics.append(clean_metric)
         
-        # Add new metrics
-        existing_metric_names = {m.get('metric_name') for m in existing_metrics}
-        added_count = 0
-        
+        # Add or update metrics
         for measure in measures:
             metric_name = measure['metric_name']
             
-            if metric_name not in existing_metric_names:
-                new_metric = {
-                    'metric_name': metric_name,
-                    'verbose_name': measure['verbose_name'],
-                    'expression': measure['expression'],
-                    'description': measure['description'],
-                    'metric_type': 'simple',
-                    'currency': None,
-                    'd3format': None,
-                    'extra': None,
-                    'warning_text': None
-                }
-                updated_metrics.append(new_metric)
-                print(f"  ✓ Prepared '{metric_name}': {measure['expression']}")
-                added_count += 1
+            new_metric = {
+                'metric_name': metric_name,
+                'verbose_name': measure['verbose_name'],
+                'expression': measure['expression'],
+                'description': measure['description'],
+                'metric_type': 'simple',
+                'currency': None,
+                'd3format': None,
+                'extra': None,
+                'warning_text': None
+            }
+            
+            if metric_name in existing_metric_names:
+                # Update existing metric
+                index = existing_metric_names[metric_name]
+                updated_metrics[index].update(new_metric)
+                print(f"  ✓ Updated '{metric_name}': {measure['expression']}")
             else:
-                print(f"  ⊘ Skipping '{metric_name}' (already exists)")
+                # Add new metric
+                updated_metrics.append(new_metric)
+                print(f"  ✓ Added '{metric_name}': {measure['expression']}")
         
         return updated_metrics
 

{dbt_cube_sync-0.1.0a3 → dbt_cube_sync-0.1.0a5}/dbt_cube_sync/core/cube_generator.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from typing import List, Dict, Any
 from jinja2 import Environment, FileSystemLoader, Template
 
-from .models import DbtModel, CubeSchema, CubeDimension, CubeMeasure
+from .models import DbtModel, CubeSchema, CubeDimension, CubeMeasure, CubePreAggregation, CubeRefreshKey
 from .dbt_parser import DbtParser
 
 
@@ -98,11 +98,36 @@ class CubeGenerator:
             )
             measures.append(measure)
         
+        # Convert pre-aggregations
+        pre_aggregations = []
+        for pre_agg_name, pre_agg_data in model.pre_aggregations.items():
+            # Convert refresh_key if present
+            refresh_key = None
+            if pre_agg_data.refresh_key:
+                refresh_key = CubeRefreshKey(
+                    every=pre_agg_data.refresh_key.every,
+                    sql=pre_agg_data.refresh_key.sql,
+                    incremental=pre_agg_data.refresh_key.incremental,
+                    update_window=pre_agg_data.refresh_key.update_window
+                )
+            
+            pre_aggregation = CubePreAggregation(
+                name=pre_agg_name,
+                type=pre_agg_data.type,
+                measures=pre_agg_data.measures,
+                dimensions=pre_agg_data.dimensions,
+                time_dimension=pre_agg_data.time_dimension,
+                granularity=pre_agg_data.granularity,
+                refresh_key=refresh_key
+            )
+            pre_aggregations.append(pre_aggregation)
+        
         return CubeSchema(
             cube_name=cube_name,
             sql=sql,
             dimensions=dimensions,
-            measures=measures
+            measures=measures,
+            pre_aggregations=pre_aggregations
         )
     
     def _write_cube_file(self, cube_schema: CubeSchema) -> Path:
@@ -116,7 +141,8 @@ class CubeGenerator:
                 cube_name=cube_schema.cube_name,
                 sql=cube_schema.sql,
                 dimensions=cube_schema.dimensions,
-                measures=cube_schema.measures
+                measures=cube_schema.measures,
+                pre_aggregations=cube_schema.pre_aggregations
             )
         else:
             # Fallback to hardcoded template
@@ -131,7 +157,12 @@ class CubeGenerator:
     
     def _generate_cube_content(self, cube_schema: CubeSchema) -> str:
         """Generate Cube.js content using hardcoded template"""
-        
+
+        # Extract table name from SQL for refresh_key replacement
+        import re
+        table_name_match = re.search(r'FROM\s+([^\s,;]+)', cube_schema.sql, re.IGNORECASE)
+        table_name = table_name_match.group(1) if table_name_match else None
+
         # Generate dimensions
         dimensions_content = []
         for dim in cube_schema.dimensions:
@@ -152,11 +183,79 @@ class CubeGenerator:
     }}"""
             measures_content.append(measure_content)
         
+        # Generate pre-aggregations
+        pre_aggregations_content = []
+        for pre_agg in cube_schema.pre_aggregations:
+            pre_agg_parts = [f"      type: `{pre_agg.type}`"]
+            
+            if pre_agg.measures:
+                measures_list = ', '.join([f'CUBE.{measure}' for measure in pre_agg.measures])
+                pre_agg_parts.append(f"      measures: [{measures_list}]")
+
+            if pre_agg.dimensions:
+                dims_list = ', '.join([f'CUBE.{dim}' for dim in pre_agg.dimensions])
+                pre_agg_parts.append(f"      dimensions: [{dims_list}]")
+
+            if pre_agg.time_dimension:
+                pre_agg_parts.append(f"      time_dimension: CUBE.{pre_agg.time_dimension}")
+                
+            if pre_agg.granularity:
+                pre_agg_parts.append(f"      granularity: `{pre_agg.granularity}`")
+                
+            if pre_agg.refresh_key:
+                refresh_key_parts = []
+                if pre_agg.refresh_key.every:
+                    refresh_key_parts.append(f"        every: `{pre_agg.refresh_key.every}`")
+                if pre_agg.refresh_key.sql:
+                    # Replace ${CUBE} and ${this} with actual table name
+                    refresh_sql = pre_agg.refresh_key.sql
+                    if table_name:
+                        refresh_sql = refresh_sql.replace('${CUBE}', table_name)
+                        refresh_sql = refresh_sql.replace('${this}', table_name)
+                    refresh_key_parts.append(f"        sql: `{refresh_sql}`")
+                if pre_agg.refresh_key.incremental is not None:
+                    refresh_key_parts.append(f"        incremental: {str(pre_agg.refresh_key.incremental).lower()}")
+                if pre_agg.refresh_key.update_window:
+                    refresh_key_parts.append(f"        update_window: `{pre_agg.refresh_key.update_window}`")
+                
+                if refresh_key_parts:
+                    refresh_key_content = ',\n'.join(refresh_key_parts)
+                    pre_agg_parts.append(f"      refresh_key: {{\n{refresh_key_content}\n      }}")
+            
+            parts_joined = ',\n'.join(pre_agg_parts)
+            pre_agg_content = f"""    {pre_agg.name}: {{
+{parts_joined}
+    }}"""
+            pre_aggregations_content.append(pre_agg_content)
+        
         # Combine into full cube definition
         dimensions_joined = ',\n\n'.join(dimensions_content)
         measures_joined = ',\n\n'.join(measures_content)
         
-        content = f"""cube(`{cube_schema.cube_name}`, {{
+        # Ensure we have measures (required for a useful Cube.js schema)
+        if not measures_content:
+            raise ValueError(f"Cube {cube_schema.cube_name} has no measures defined. Measures are required for Cube.js schemas.")
+        
+        if pre_aggregations_content:
+            pre_aggregations_joined = ',\n\n'.join(pre_aggregations_content)
+            content = f"""cube(`{cube_schema.cube_name}`, {{
+  sql: `{cube_schema.sql}`,
+  
+  dimensions: {{
+{dimensions_joined}
+  }},
+  
+  measures: {{
+{measures_joined}
+  }},
+  
+  pre_aggregations: {{
+{pre_aggregations_joined}
+  }}
+}});
+"""
+        else:
+            content = f"""cube(`{cube_schema.cube_name}`, {{
   sql: `{cube_schema.sql}`,
   
   dimensions: {{

{dbt_cube_sync-0.1.0a3 → dbt_cube_sync-0.1.0a5}/dbt_cube_sync/core/dbt_parser.py

@@ -6,7 +6,7 @@ import os
 from typing import Dict, List
 from pathlib import Path
 
-from .models import DbtModel, DbtColumn, DbtMetric
+from .models import DbtModel, DbtColumn, DbtMetric, DbtPreAggregation, DbtRefreshKey
 
 
 class DbtParser:
@@ -61,7 +61,8 @@ class DbtParser:
                 continue
             
             model = self._parse_model(node_id, node_data)
-            if model and model.columns and model.metrics:  # Only include models with BOTH columns AND metrics
+            # Include models that have columns AND metrics (measures are required for useful Cube.js schemas)
+            if model and model.columns and model.metrics:
                 models.append(model)
         
         return models
@@ -78,13 +79,17 @@ class DbtParser:
         # Parse metrics from config.meta.metrics
         metrics = self._parse_metrics(node_data)
         
+        # Parse pre-aggregations from config.meta.pre_aggregations
+        pre_aggregations = self._parse_pre_aggregations(node_data)
+        
         return DbtModel(
             name=model_name,
             database=model_database,
             schema_name=model_schema,
             node_id=node_id,
             columns=columns,
-            metrics=metrics
+            metrics=metrics,
+            pre_aggregations=pre_aggregations
         )
     
     def _parse_columns(self, node_id: str, node_data: dict) -> Dict[str, DbtColumn]:
@@ -145,6 +150,40 @@ class DbtParser:
         
         return metrics
     
+    def _parse_pre_aggregations(self, node_data: dict) -> Dict[str, DbtPreAggregation]:
+        """Parse pre-aggregations from model configuration"""
+        pre_aggregations = {}
+        
+        # Look for pre-aggregations in config.meta.pre_aggregations
+        config = node_data.get('config', {})
+        meta = config.get('meta', {})
+        pre_aggs_data = meta.get('pre_aggregations', {})
+        
+        for pre_agg_name, pre_agg_config in pre_aggs_data.items():
+            if isinstance(pre_agg_config, dict):
+                # Parse refresh_key if present
+                refresh_key = None
+                refresh_key_config = pre_agg_config.get('refresh_key')
+                if refresh_key_config and isinstance(refresh_key_config, dict):
+                    refresh_key = DbtRefreshKey(
+                        every=refresh_key_config.get('every'),
+                        sql=refresh_key_config.get('sql'),
+                        incremental=refresh_key_config.get('incremental'),
+                        update_window=refresh_key_config.get('update_window')
+                    )
+                
+                pre_aggregations[pre_agg_name] = DbtPreAggregation(
+                    name=pre_agg_name,
+                    type=pre_agg_config.get('type', 'rollup'),
+                    measures=pre_agg_config.get('measures', []),
+                    dimensions=pre_agg_config.get('dimensions', []),
+                    time_dimension=pre_agg_config.get('time_dimension'),
+                    granularity=pre_agg_config.get('granularity'),
+                    refresh_key=refresh_key
+                )
+        
+        return pre_aggregations
+    
     @staticmethod
     def map_dbt_type_to_cube_type(dbt_type: str) -> str:
         """Map dbt metric types to Cube.js measure types"""

{dbt_cube_sync-0.1.0a3 → dbt_cube_sync-0.1.0a5}/dbt_cube_sync/core/models.py

@@ -22,6 +22,25 @@ class DbtMetric(BaseModel):
     description: Optional[str] = None
 
 
+class DbtRefreshKey(BaseModel):
+    """Represents a refresh_key configuration for pre-aggregations"""
+    every: Optional[str] = None
+    sql: Optional[str] = None
+    incremental: Optional[bool] = None
+    update_window: Optional[str] = None
+
+
+class DbtPreAggregation(BaseModel):
+    """Represents a dbt pre-aggregation configuration"""
+    name: str
+    type: str = "rollup"
+    measures: Optional[List[str]] = None
+    dimensions: Optional[List[str]] = None
+    time_dimension: Optional[str] = None
+    granularity: Optional[str] = None
+    refresh_key: Optional[DbtRefreshKey] = None
+
+
 class DbtModel(BaseModel):
     """Represents a parsed dbt model"""
     name: str
@@ -30,6 +49,7 @@ class DbtModel(BaseModel):
     node_id: str
     columns: Dict[str, DbtColumn]
     metrics: Dict[str, DbtMetric]
+    pre_aggregations: Dict[str, DbtPreAggregation] = {}
 
 
 class CubeDimension(BaseModel):
@@ -50,12 +70,32 @@ class CubeMeasure(BaseModel):
     description: Optional[str] = None
 
 
+class CubeRefreshKey(BaseModel):
+    """Represents a Cube.js refresh_key configuration"""
+    every: Optional[str] = None
+    sql: Optional[str] = None
+    incremental: Optional[bool] = None
+    update_window: Optional[str] = None
+
+
+class CubePreAggregation(BaseModel):
+    """Represents a Cube.js pre-aggregation"""
+    name: str
+    type: str = "rollup"
+    measures: Optional[List[str]] = None
+    dimensions: Optional[List[str]] = None
+    time_dimension: Optional[str] = None
+    granularity: Optional[str] = None
+    refresh_key: Optional[CubeRefreshKey] = None
+
+
 class CubeSchema(BaseModel):
     """Represents a complete Cube.js schema"""
     cube_name: str
     sql: str
     dimensions: List[CubeDimension]
     measures: List[CubeMeasure]
+    pre_aggregations: List[CubePreAggregation] = []
 
 
 class SyncResult(BaseModel):

{dbt_cube_sync-0.1.0a3 → dbt_cube_sync-0.1.0a5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dbt-cube-sync"
-version = "0.1.0a3"
+version = "0.1.0a5"
 description = "Synchronization tool for dbt models to Cube.js schemas and BI tools"
 authors = ["Ponder"]
 readme = "README.md"