dbt-cube-sync 0.1.0a4__tar.gz → 0.1.0a5__tar.gz

{dbt_cube_sync-0.1.0a4 → dbt_cube_sync-0.1.0a5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dbt-cube-sync
-Version: 0.1.0a4
+Version: 0.1.0a5
 Summary: Synchronization tool for dbt models to Cube.js schemas and BI tools
 Author: Ponder
 Requires-Python: >=3.9,<4.0

{dbt_cube_sync-0.1.0a4 → dbt_cube_sync-0.1.0a5}/dbt_cube_sync/core/cube_generator.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from typing import List, Dict, Any
 from jinja2 import Environment, FileSystemLoader, Template
 
-from .models import DbtModel, CubeSchema, CubeDimension, CubeMeasure
+from .models import DbtModel, CubeSchema, CubeDimension, CubeMeasure, CubePreAggregation, CubeRefreshKey
 from .dbt_parser import DbtParser
 
 
@@ -98,11 +98,36 @@ class CubeGenerator:
             )
             measures.append(measure)
         
+        # Convert pre-aggregations
+        pre_aggregations = []
+        for pre_agg_name, pre_agg_data in model.pre_aggregations.items():
+            # Convert refresh_key if present
+            refresh_key = None
+            if pre_agg_data.refresh_key:
+                refresh_key = CubeRefreshKey(
+                    every=pre_agg_data.refresh_key.every,
+                    sql=pre_agg_data.refresh_key.sql,
+                    incremental=pre_agg_data.refresh_key.incremental,
+                    update_window=pre_agg_data.refresh_key.update_window
+                )
+            
+            pre_aggregation = CubePreAggregation(
+                name=pre_agg_name,
+                type=pre_agg_data.type,
+                measures=pre_agg_data.measures,
+                dimensions=pre_agg_data.dimensions,
+                time_dimension=pre_agg_data.time_dimension,
+                granularity=pre_agg_data.granularity,
+                refresh_key=refresh_key
+            )
+            pre_aggregations.append(pre_aggregation)
+        
         return CubeSchema(
             cube_name=cube_name,
             sql=sql,
             dimensions=dimensions,
-            measures=measures
+            measures=measures,
+            pre_aggregations=pre_aggregations
         )
     
     def _write_cube_file(self, cube_schema: CubeSchema) -> Path:
@@ -116,7 +141,8 @@ class CubeGenerator:
                 cube_name=cube_schema.cube_name,
                 sql=cube_schema.sql,
                 dimensions=cube_schema.dimensions,
-                measures=cube_schema.measures
+                measures=cube_schema.measures,
+                pre_aggregations=cube_schema.pre_aggregations
             )
         else:
             # Fallback to hardcoded template
@@ -131,7 +157,12 @@ class CubeGenerator:
     
     def _generate_cube_content(self, cube_schema: CubeSchema) -> str:
         """Generate Cube.js content using hardcoded template"""
-        
+
+        # Extract table name from SQL for refresh_key replacement
+        import re
+        table_name_match = re.search(r'FROM\s+([^\s,;]+)', cube_schema.sql, re.IGNORECASE)
+        table_name = table_name_match.group(1) if table_name_match else None
+
         # Generate dimensions
         dimensions_content = []
         for dim in cube_schema.dimensions:
@@ -152,11 +183,79 @@ class CubeGenerator:
     }}"""
             measures_content.append(measure_content)
         
+        # Generate pre-aggregations
+        pre_aggregations_content = []
+        for pre_agg in cube_schema.pre_aggregations:
+            pre_agg_parts = [f"      type: `{pre_agg.type}`"]
+            
+            if pre_agg.measures:
+                measures_list = ', '.join([f'CUBE.{measure}' for measure in pre_agg.measures])
+                pre_agg_parts.append(f"      measures: [{measures_list}]")
+
+            if pre_agg.dimensions:
+                dims_list = ', '.join([f'CUBE.{dim}' for dim in pre_agg.dimensions])
+                pre_agg_parts.append(f"      dimensions: [{dims_list}]")
+
+            if pre_agg.time_dimension:
+                pre_agg_parts.append(f"      time_dimension: CUBE.{pre_agg.time_dimension}")
+                
+            if pre_agg.granularity:
+                pre_agg_parts.append(f"      granularity: `{pre_agg.granularity}`")
+                
+            if pre_agg.refresh_key:
+                refresh_key_parts = []
+                if pre_agg.refresh_key.every:
+                    refresh_key_parts.append(f"        every: `{pre_agg.refresh_key.every}`")
+                if pre_agg.refresh_key.sql:
+                    # Replace ${CUBE} and ${this} with actual table name
+                    refresh_sql = pre_agg.refresh_key.sql
+                    if table_name:
+                        refresh_sql = refresh_sql.replace('${CUBE}', table_name)
+                        refresh_sql = refresh_sql.replace('${this}', table_name)
+                    refresh_key_parts.append(f"        sql: `{refresh_sql}`")
+                if pre_agg.refresh_key.incremental is not None:
+                    refresh_key_parts.append(f"        incremental: {str(pre_agg.refresh_key.incremental).lower()}")
+                if pre_agg.refresh_key.update_window:
+                    refresh_key_parts.append(f"        update_window: `{pre_agg.refresh_key.update_window}`")
+                
+                if refresh_key_parts:
+                    refresh_key_content = ',\n'.join(refresh_key_parts)
+                    pre_agg_parts.append(f"      refresh_key: {{\n{refresh_key_content}\n      }}")
+            
+            parts_joined = ',\n'.join(pre_agg_parts)
+            pre_agg_content = f"""    {pre_agg.name}: {{
+{parts_joined}
+    }}"""
+            pre_aggregations_content.append(pre_agg_content)
+        
         # Combine into full cube definition
         dimensions_joined = ',\n\n'.join(dimensions_content)
         measures_joined = ',\n\n'.join(measures_content)
         
-        content = f"""cube(`{cube_schema.cube_name}`, {{
+        # Ensure we have measures (required for a useful Cube.js schema)
+        if not measures_content:
+            raise ValueError(f"Cube {cube_schema.cube_name} has no measures defined. Measures are required for Cube.js schemas.")
+        
+        if pre_aggregations_content:
+            pre_aggregations_joined = ',\n\n'.join(pre_aggregations_content)
+            content = f"""cube(`{cube_schema.cube_name}`, {{
+  sql: `{cube_schema.sql}`,
+  
+  dimensions: {{
+{dimensions_joined}
+  }},
+  
+  measures: {{
+{measures_joined}
+  }},
+  
+  pre_aggregations: {{
+{pre_aggregations_joined}
+  }}
+}});
+"""
+        else:
+            content = f"""cube(`{cube_schema.cube_name}`, {{
   sql: `{cube_schema.sql}`,
   
   dimensions: {{

{dbt_cube_sync-0.1.0a4 → dbt_cube_sync-0.1.0a5}/dbt_cube_sync/core/dbt_parser.py

@@ -6,7 +6,7 @@ import os
 from typing import Dict, List
 from pathlib import Path
 
-from .models import DbtModel, DbtColumn, DbtMetric
+from .models import DbtModel, DbtColumn, DbtMetric, DbtPreAggregation, DbtRefreshKey
 
 
 class DbtParser:
@@ -61,7 +61,8 @@ class DbtParser:
                 continue
             
             model = self._parse_model(node_id, node_data)
-            if model and model.columns and model.metrics:  # Only include models with BOTH columns AND metrics
+            # Include models that have columns AND metrics (measures are required for useful Cube.js schemas)
+            if model and model.columns and model.metrics:
                 models.append(model)
         
         return models
@@ -78,13 +79,17 @@ class DbtParser:
         # Parse metrics from config.meta.metrics
         metrics = self._parse_metrics(node_data)
         
+        # Parse pre-aggregations from config.meta.pre_aggregations
+        pre_aggregations = self._parse_pre_aggregations(node_data)
+        
         return DbtModel(
             name=model_name,
             database=model_database,
             schema_name=model_schema,
             node_id=node_id,
             columns=columns,
-            metrics=metrics
+            metrics=metrics,
+            pre_aggregations=pre_aggregations
         )
     
     def _parse_columns(self, node_id: str, node_data: dict) -> Dict[str, DbtColumn]:
@@ -145,6 +150,40 @@ class DbtParser:
         
         return metrics
     
+    def _parse_pre_aggregations(self, node_data: dict) -> Dict[str, DbtPreAggregation]:
+        """Parse pre-aggregations from model configuration"""
+        pre_aggregations = {}
+        
+        # Look for pre-aggregations in config.meta.pre_aggregations
+        config = node_data.get('config', {})
+        meta = config.get('meta', {})
+        pre_aggs_data = meta.get('pre_aggregations', {})
+        
+        for pre_agg_name, pre_agg_config in pre_aggs_data.items():
+            if isinstance(pre_agg_config, dict):
+                # Parse refresh_key if present
+                refresh_key = None
+                refresh_key_config = pre_agg_config.get('refresh_key')
+                if refresh_key_config and isinstance(refresh_key_config, dict):
+                    refresh_key = DbtRefreshKey(
+                        every=refresh_key_config.get('every'),
+                        sql=refresh_key_config.get('sql'),
+                        incremental=refresh_key_config.get('incremental'),
+                        update_window=refresh_key_config.get('update_window')
+                    )
+                
+                pre_aggregations[pre_agg_name] = DbtPreAggregation(
+                    name=pre_agg_name,
+                    type=pre_agg_config.get('type', 'rollup'),
+                    measures=pre_agg_config.get('measures', []),
+                    dimensions=pre_agg_config.get('dimensions', []),
+                    time_dimension=pre_agg_config.get('time_dimension'),
+                    granularity=pre_agg_config.get('granularity'),
+                    refresh_key=refresh_key
+                )
+        
+        return pre_aggregations
+    
     @staticmethod
     def map_dbt_type_to_cube_type(dbt_type: str) -> str:
         """Map dbt metric types to Cube.js measure types"""

{dbt_cube_sync-0.1.0a4 → dbt_cube_sync-0.1.0a5}/dbt_cube_sync/core/models.py

@@ -22,6 +22,25 @@ class DbtMetric(BaseModel):
     description: Optional[str] = None
 
 
+class DbtRefreshKey(BaseModel):
+    """Represents a refresh_key configuration for pre-aggregations"""
+    every: Optional[str] = None
+    sql: Optional[str] = None
+    incremental: Optional[bool] = None
+    update_window: Optional[str] = None
+
+
+class DbtPreAggregation(BaseModel):
+    """Represents a dbt pre-aggregation configuration"""
+    name: str
+    type: str = "rollup"
+    measures: Optional[List[str]] = None
+    dimensions: Optional[List[str]] = None
+    time_dimension: Optional[str] = None
+    granularity: Optional[str] = None
+    refresh_key: Optional[DbtRefreshKey] = None
+
+
 class DbtModel(BaseModel):
     """Represents a parsed dbt model"""
     name: str
@@ -30,6 +49,7 @@ class DbtModel(BaseModel):
     node_id: str
     columns: Dict[str, DbtColumn]
     metrics: Dict[str, DbtMetric]
+    pre_aggregations: Dict[str, DbtPreAggregation] = {}
 
 
 class CubeDimension(BaseModel):
@@ -50,12 +70,32 @@ class CubeMeasure(BaseModel):
     description: Optional[str] = None
 
 
+class CubeRefreshKey(BaseModel):
+    """Represents a Cube.js refresh_key configuration"""
+    every: Optional[str] = None
+    sql: Optional[str] = None
+    incremental: Optional[bool] = None
+    update_window: Optional[str] = None
+
+
+class CubePreAggregation(BaseModel):
+    """Represents a Cube.js pre-aggregation"""
+    name: str
+    type: str = "rollup"
+    measures: Optional[List[str]] = None
+    dimensions: Optional[List[str]] = None
+    time_dimension: Optional[str] = None
+    granularity: Optional[str] = None
+    refresh_key: Optional[CubeRefreshKey] = None
+
+
 class CubeSchema(BaseModel):
     """Represents a complete Cube.js schema"""
     cube_name: str
     sql: str
     dimensions: List[CubeDimension]
     measures: List[CubeMeasure]
+    pre_aggregations: List[CubePreAggregation] = []
 
 
 class SyncResult(BaseModel):

{dbt_cube_sync-0.1.0a4 → dbt_cube_sync-0.1.0a5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dbt-cube-sync"
-version = "0.1.0a4"
+version = "0.1.0a5"
 description = "Synchronization tool for dbt models to Cube.js schemas and BI tools"
 authors = ["Ponder"]
 readme = "README.md"