dbt-cube-sync 0.1.0a4__tar.gz → 0.1.0a6__tar.gz

{dbt_cube_sync-0.1.0a4 → dbt_cube_sync-0.1.0a6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dbt-cube-sync
-Version: 0.1.0a4
+Version: 0.1.0a6
 Summary: Synchronization tool for dbt models to Cube.js schemas and BI tools
 Author: Ponder
 Requires-Python: >=3.9,<4.0
@@ -16,6 +16,7 @@ Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
 Requires-Dist: pydantic (>=2.5.0,<3.0.0)
 Requires-Dist: pyyaml (>=6.0,<7.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
+Requires-Dist: sqlalchemy (>=2.0.0,<3.0.0)
 Description-Content-Type: text/markdown
 
 # dbt-cube-sync
@@ -25,6 +26,8 @@ A powerful synchronization tool that creates a seamless pipeline from dbt models
 ## Features
 
 - 🔄 **dbt → Cube.js**: Auto-generate Cube.js schemas from dbt models with metrics
+- 🗃️ **Flexible Data Type Source**: Get column types from catalog OR directly from database via SQLAlchemy
+- 🎯 **Model Filtering**: Process specific models instead of all models
 - 📊 **Cube.js → BI Tools**: Sync schemas to multiple BI platforms
 - 🏗️ **Extensible Architecture**: Plugin-based connector system for easy BI tool integration
 - 🐳 **Docker Support**: Containerized execution with orchestration support
@@ -46,6 +49,27 @@ poetry install
 poetry run dbt-cube-sync --help
 ```
 
+### Database Drivers (for SQLAlchemy URI feature)
+
+If you want to use the `--sqlalchemy-uri` option to fetch column types directly from your database, you'll need to install the appropriate database driver:
+
+```bash
+# PostgreSQL
+poetry add psycopg2-binary
+
+# MySQL
+poetry add pymysql
+
+# Snowflake
+poetry add snowflake-sqlalchemy
+
+# BigQuery
+poetry add sqlalchemy-bigquery
+
+# Redshift
+poetry add sqlalchemy-redshift
+```
+
 ### Using Docker
 
 ```bash
@@ -55,42 +79,43 @@ docker run --rm dbt-cube-sync --help
 
 ## Quick Start
 
-### 1. Create Configuration File
+### 1. Generate Cube.js Schemas from dbt
 
+**Option A: Using catalog file (traditional method)**
 ```bash
-# Create sample config
-dbt-cube-sync create-config sync-config.yaml
-
-# Edit the config file with your BI tool credentials
+dbt-cube-sync dbt-to-cube \
+  --manifest ./target/manifest.json \
+  --catalog ./target/catalog.json \
+  --output ./cube_output
 ```
 
-### 2. Generate Cube.js Schemas
-
+**Option B: Using database connection (no catalog needed)**
 ```bash
-# Generate from dbt manifest
-dbt-cube-sync generate-cubes \\
-  --dbt-manifest ./DbtEducationalDataProject/target/manifest.json \\
-  --output-dir ./cube/conf/cube_output
+dbt-cube-sync dbt-to-cube \
+  --manifest ./target/manifest.json \
+  --sqlalchemy-uri postgresql://user:password@localhost:5432/mydb \
+  --output ./cube_output
 ```
 
-### 3. Sync to BI Tool
-
+**Option C: Filter specific models**
 ```bash
-# Sync to Superset
-dbt-cube-sync sync-bi superset \\
-  --cube-dir ./cube/conf/cube_output \\
-  --config-file ./sync-config.yaml
+dbt-cube-sync dbt-to-cube \
+  --manifest ./target/manifest.json \
+  --sqlalchemy-uri postgresql://user:password@localhost:5432/mydb \
+  --models orders,customers,products \
+  --output ./cube_output
 ```
 
-### 4. Full Pipeline
+### 2. Sync to BI Tool (Optional)
 
 ```bash
-# Complete dbt → Cube.js → Superset pipeline
-dbt-cube-sync full-sync \\
-  --dbt-manifest ./DbtEducationalDataProject/target/manifest.json \\
-  --cube-dir ./cube/conf/cube_output \\
-  --bi-connector superset \\
-  --config-file ./sync-config.yaml
+# Sync to Superset
+dbt-cube-sync cube-to-bi superset \
+  --cube-files ./cube_output \
+  --url http://localhost:8088 \
+  --username admin \
+  --password admin \
+  --cube-connection-name Cube
 ```
 
 ## Configuration
@@ -119,23 +144,50 @@ connectors:
 
 ## CLI Commands
 
-### `generate-cubes`
+### `dbt-to-cube`
 Generate Cube.js schema files from dbt models.
 
 **Options:**
-- `--dbt-manifest` / `-m`: Path to dbt manifest.json file
-- `--output-dir` / `-o`: Output directory for Cube.js files
-- `--template-dir` / `-t`: Directory containing Cube.js templates
+- `--manifest` / `-m`: Path to dbt manifest.json file (required)
+- `--catalog` / `-c`: Path to dbt catalog.json file (optional if --sqlalchemy-uri is provided)
+- `--sqlalchemy-uri` / `-s`: SQLAlchemy database URI for fetching column types (optional if --catalog is provided)
+  - Example: `postgresql://user:password@localhost:5432/database`
+  - Example: `mysql://user:password@localhost:3306/database`
+  - Example: `snowflake://user:password@account/database/schema`
+- `--models`: Comma-separated list of model names to process (optional, processes all if not specified)
+  - Example: `--models model1,model2,model3`
+- `--output` / `-o`: Output directory for Cube.js files (required)
+- `--template-dir` / `-t`: Directory containing Cube.js templates (default: ./cube/templates)
+
+**Examples:**
+```bash
+# Using catalog file
+dbt-cube-sync dbt-to-cube -m manifest.json -c catalog.json -o output/
+
+# Using database connection (no catalog needed)
+dbt-cube-sync dbt-to-cube -m manifest.json -s postgresql://user:pass@localhost/db -o output/
 
-### `sync-bi`
+# Filter specific models
+dbt-cube-sync dbt-to-cube -m manifest.json -s postgresql://user:pass@localhost/db --models users,orders -o output/
+```
+
+### `cube-to-bi`
 Sync Cube.js schemas to BI tool datasets.
 
 **Arguments:**
-- `connector`: BI tool type (`superset`, `tableau`, `powerbi`)
+- `bi_tool`: BI tool type (`superset`, `tableau`, `powerbi`)
 
 **Options:**
-- `--cube-dir` / `-c`: Directory containing Cube.js files
-- `--config-file` / `-f`: Configuration file for BI tool connection
+- `--cube-files` / `-c`: Directory containing Cube.js files (required)
+- `--url` / `-u`: BI tool URL (required)
+- `--username` / `-n`: BI tool username (required)
+- `--password` / `-p`: BI tool password (required)
+- `--cube-connection-name` / `-d`: Name of Cube database connection in BI tool (default: Cube)
+
+**Example:**
+```bash
+dbt-cube-sync cube-to-bi superset -c cube_output/ -u http://localhost:8088 -n admin -p admin -d Cube
+```
 
 ### `full-sync`
 Complete pipeline: dbt models → Cube.js schemas → BI tool datasets.
@@ -170,6 +222,7 @@ dbt-cube-sync/
 │   ├── config.py             # Configuration management
 │   ├── core/
 │   │   ├── dbt_parser.py     # dbt manifest parser
+│   │   ├── db_inspector.py   # Database column type inspector (SQLAlchemy)
 │   │   ├── cube_generator.py # Cube.js generator
 │   │   └── models.py         # Pydantic data models
 │   └── connectors/

{dbt_cube_sync-0.1.0a4 → dbt_cube_sync-0.1.0a6}/README.md

@@ -5,6 +5,8 @@ A powerful synchronization tool that creates a seamless pipeline from dbt models
 ## Features
 
 - 🔄 **dbt → Cube.js**: Auto-generate Cube.js schemas from dbt models with metrics
+- 🗃️ **Flexible Data Type Source**: Get column types from catalog OR directly from database via SQLAlchemy
+- 🎯 **Model Filtering**: Process specific models instead of all models
 - 📊 **Cube.js → BI Tools**: Sync schemas to multiple BI platforms
 - 🏗️ **Extensible Architecture**: Plugin-based connector system for easy BI tool integration
 - 🐳 **Docker Support**: Containerized execution with orchestration support
@@ -26,6 +28,27 @@ poetry install
 poetry run dbt-cube-sync --help
 ```
 
+### Database Drivers (for SQLAlchemy URI feature)
+
+If you want to use the `--sqlalchemy-uri` option to fetch column types directly from your database, you'll need to install the appropriate database driver:
+
+```bash
+# PostgreSQL
+poetry add psycopg2-binary
+
+# MySQL
+poetry add pymysql
+
+# Snowflake
+poetry add snowflake-sqlalchemy
+
+# BigQuery
+poetry add sqlalchemy-bigquery
+
+# Redshift
+poetry add sqlalchemy-redshift
+```
+
 ### Using Docker
 
 ```bash
@@ -35,42 +58,43 @@ docker run --rm dbt-cube-sync --help
 
 ## Quick Start
 
-### 1. Create Configuration File
+### 1. Generate Cube.js Schemas from dbt
 
+**Option A: Using catalog file (traditional method)**
 ```bash
-# Create sample config
-dbt-cube-sync create-config sync-config.yaml
-
-# Edit the config file with your BI tool credentials
+dbt-cube-sync dbt-to-cube \
+  --manifest ./target/manifest.json \
+  --catalog ./target/catalog.json \
+  --output ./cube_output
 ```
 
-### 2. Generate Cube.js Schemas
-
+**Option B: Using database connection (no catalog needed)**
 ```bash
-# Generate from dbt manifest
-dbt-cube-sync generate-cubes \\
-  --dbt-manifest ./DbtEducationalDataProject/target/manifest.json \\
-  --output-dir ./cube/conf/cube_output
+dbt-cube-sync dbt-to-cube \
+  --manifest ./target/manifest.json \
+  --sqlalchemy-uri postgresql://user:password@localhost:5432/mydb \
+  --output ./cube_output
 ```
 
-### 3. Sync to BI Tool
-
+**Option C: Filter specific models**
 ```bash
-# Sync to Superset
-dbt-cube-sync sync-bi superset \\
-  --cube-dir ./cube/conf/cube_output \\
-  --config-file ./sync-config.yaml
+dbt-cube-sync dbt-to-cube \
+  --manifest ./target/manifest.json \
+  --sqlalchemy-uri postgresql://user:password@localhost:5432/mydb \
+  --models orders,customers,products \
+  --output ./cube_output
 ```
 
-### 4. Full Pipeline
+### 2. Sync to BI Tool (Optional)
 
 ```bash
-# Complete dbt → Cube.js → Superset pipeline
-dbt-cube-sync full-sync \\
-  --dbt-manifest ./DbtEducationalDataProject/target/manifest.json \\
-  --cube-dir ./cube/conf/cube_output \\
-  --bi-connector superset \\
-  --config-file ./sync-config.yaml
+# Sync to Superset
+dbt-cube-sync cube-to-bi superset \
+  --cube-files ./cube_output \
+  --url http://localhost:8088 \
+  --username admin \
+  --password admin \
+  --cube-connection-name Cube
 ```
 
 ## Configuration
@@ -99,23 +123,50 @@ connectors:
 
 ## CLI Commands
 
-### `generate-cubes`
+### `dbt-to-cube`
 Generate Cube.js schema files from dbt models.
 
 **Options:**
-- `--dbt-manifest` / `-m`: Path to dbt manifest.json file
-- `--output-dir` / `-o`: Output directory for Cube.js files
-- `--template-dir` / `-t`: Directory containing Cube.js templates
+- `--manifest` / `-m`: Path to dbt manifest.json file (required)
+- `--catalog` / `-c`: Path to dbt catalog.json file (optional if --sqlalchemy-uri is provided)
+- `--sqlalchemy-uri` / `-s`: SQLAlchemy database URI for fetching column types (optional if --catalog is provided)
+  - Example: `postgresql://user:password@localhost:5432/database`
+  - Example: `mysql://user:password@localhost:3306/database`
+  - Example: `snowflake://user:password@account/database/schema`
+- `--models`: Comma-separated list of model names to process (optional, processes all if not specified)
+  - Example: `--models model1,model2,model3`
+- `--output` / `-o`: Output directory for Cube.js files (required)
+- `--template-dir` / `-t`: Directory containing Cube.js templates (default: ./cube/templates)
+
+**Examples:**
+```bash
+# Using catalog file
+dbt-cube-sync dbt-to-cube -m manifest.json -c catalog.json -o output/
+
+# Using database connection (no catalog needed)
+dbt-cube-sync dbt-to-cube -m manifest.json -s postgresql://user:pass@localhost/db -o output/
 
-### `sync-bi`
+# Filter specific models
+dbt-cube-sync dbt-to-cube -m manifest.json -s postgresql://user:pass@localhost/db --models users,orders -o output/
+```
+
+### `cube-to-bi`
 Sync Cube.js schemas to BI tool datasets.
 
 **Arguments:**
-- `connector`: BI tool type (`superset`, `tableau`, `powerbi`)
+- `bi_tool`: BI tool type (`superset`, `tableau`, `powerbi`)
 
 **Options:**
-- `--cube-dir` / `-c`: Directory containing Cube.js files
-- `--config-file` / `-f`: Configuration file for BI tool connection
+- `--cube-files` / `-c`: Directory containing Cube.js files (required)
+- `--url` / `-u`: BI tool URL (required)
+- `--username` / `-n`: BI tool username (required)
+- `--password` / `-p`: BI tool password (required)
+- `--cube-connection-name` / `-d`: Name of Cube database connection in BI tool (default: Cube)
+
+**Example:**
+```bash
+dbt-cube-sync cube-to-bi superset -c cube_output/ -u http://localhost:8088 -n admin -p admin -d Cube
+```
 
 ### `full-sync`
 Complete pipeline: dbt models → Cube.js schemas → BI tool datasets.
@@ -150,6 +201,7 @@ dbt-cube-sync/
 │   ├── config.py             # Configuration management
 │   ├── core/
 │   │   ├── dbt_parser.py     # dbt manifest parser
+│   │   ├── db_inspector.py   # Database column type inspector (SQLAlchemy)
 │   │   ├── cube_generator.py # Cube.js generator
 │   │   └── models.py         # Pydantic data models
 │   └── connectors/

{dbt_cube_sync-0.1.0a4 → dbt_cube_sync-0.1.0a6}/dbt_cube_sync/cli.py

@@ -24,10 +24,12 @@ class CustomGroup(click.Group):
             click.echo("\nAvailable commands:")
             click.echo("  dbt-cube-sync --help                                    # Show help")
             click.echo("  dbt-cube-sync --version                                 # Show version")
-            click.echo("  dbt-cube-sync dbt-to-cube -m manifest -c catalog -o output # Generate Cube.js schemas")
+            click.echo("  dbt-cube-sync dbt-to-cube -m manifest -c catalog -o output # Generate with catalog")
+            click.echo("  dbt-cube-sync dbt-to-cube -m manifest -s postgresql://user:pass@host/db -o output # Generate with database")
+            click.echo("  dbt-cube-sync dbt-to-cube -m manifest -s <uri> --models model1,model2 -o output # Filter specific models")
             click.echo("  dbt-cube-sync cube-to-bi superset -c cubes -u url -n user -p pass -d Cube # Sync to BI tool")
             ctx.exit(1)
-        
+
         return super().get_command(ctx, cmd_name)
 
 
@@ -39,35 +41,66 @@ def main():
 
 
 @main.command()
-@click.option('--manifest', '-m', 
+@click.option('--manifest', '-m',
               required=True,
               help='Path to dbt manifest.json file')
 @click.option('--catalog', '-c',
-              required=True,
-              help='Path to dbt catalog.json file')
+              required=False,
+              default=None,
+              help='Path to dbt catalog.json file (optional if --sqlalchemy-uri is provided)')
+@click.option('--sqlalchemy-uri', '-s',
+              required=False,
+              default=None,
+              help='SQLAlchemy database URI for fetching column types (e.g., postgresql://user:pass@host:port/db)')
+@click.option('--models',
+              required=False,
+              default=None,
+              help='Comma-separated list of model names to process (e.g., model1,model2). If not specified, processes all models')
 @click.option('--output', '-o',
               required=True,
               help='Output directory for Cube.js files')
 @click.option('--template-dir', '-t',
               default='./cube/templates',
               help='Directory containing Cube.js templates')
-def dbt_to_cube(manifest: str, catalog: str, output: str, template_dir: str):
+def dbt_to_cube(manifest: str, catalog: Optional[str], sqlalchemy_uri: Optional[str], models: Optional[str], output: str, template_dir: str):
     """Generate Cube.js schemas from dbt models"""
     try:
+        # Validate that at least one source of column types is provided
+        if not catalog and not sqlalchemy_uri:
+            click.echo("❌ Error: You must provide either --catalog or --sqlalchemy-uri to get column data types", err=True)
+            click.echo("💡 Example with catalog: dbt-cube-sync dbt-to-cube -m manifest.json -c catalog.json -o output/", err=True)
+            click.echo("💡 Example with database: dbt-cube-sync dbt-to-cube -m manifest.json -s postgresql://user:pass@host:port/db -o output/", err=True)
+            sys.exit(1)
+
+        # Parse model filter if provided
+        model_filter = None
+        if models:
+            model_filter = [m.strip() for m in models.split(',')]
+            click.echo(f"🎯 Filtering models: {', '.join(model_filter)}")
+
         click.echo("🔄 Parsing dbt manifest...")
-        parser = DbtParser(manifest, catalog)
-        models = parser.parse_models()
-        
-        click.echo(f"📊 Found {len(models)} dbt models")
-        
+        parser = DbtParser(
+            manifest_path=manifest,
+            catalog_path=catalog,
+            sqlalchemy_uri=sqlalchemy_uri,
+            model_filter=model_filter
+        )
+        parsed_models = parser.parse_models()
+
+        click.echo(f"📊 Found {len(parsed_models)} dbt models")
+
+        if len(parsed_models) == 0:
+            click.echo("⚠️  No models found. Make sure your models have both columns and metrics defined.")
+            sys.exit(0)
+
         click.echo("🏗️  Generating Cube.js schemas...")
         generator = CubeGenerator(template_dir, output)
-        generated_files = generator.generate_cube_files(models)
-        
+        generated_files = generator.generate_cube_files(parsed_models)
+
         click.echo(f"✅ Generated {len(generated_files)} Cube.js files:")
         for file_path in generated_files:
             click.echo(f"   • {file_path}")
-            
+
     except Exception as e:
         click.echo(f"❌ Error: {str(e)}", err=True)
         sys.exit(1)

{dbt_cube_sync-0.1.0a4 → dbt_cube_sync-0.1.0a6}/dbt_cube_sync/core/cube_generator.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from typing import List, Dict, Any
 from jinja2 import Environment, FileSystemLoader, Template
 
-from .models import DbtModel, CubeSchema, CubeDimension, CubeMeasure
+from .models import DbtModel, CubeSchema, CubeDimension, CubeMeasure, CubePreAggregation, CubeRefreshKey
 from .dbt_parser import DbtParser
 
 
@@ -98,11 +98,36 @@ class CubeGenerator:
             )
             measures.append(measure)
         
+        # Convert pre-aggregations
+        pre_aggregations = []
+        for pre_agg_name, pre_agg_data in model.pre_aggregations.items():
+            # Convert refresh_key if present
+            refresh_key = None
+            if pre_agg_data.refresh_key:
+                refresh_key = CubeRefreshKey(
+                    every=pre_agg_data.refresh_key.every,
+                    sql=pre_agg_data.refresh_key.sql,
+                    incremental=pre_agg_data.refresh_key.incremental,
+                    update_window=pre_agg_data.refresh_key.update_window
+                )
+            
+            pre_aggregation = CubePreAggregation(
+                name=pre_agg_name,
+                type=pre_agg_data.type,
+                measures=pre_agg_data.measures,
+                dimensions=pre_agg_data.dimensions,
+                time_dimension=pre_agg_data.time_dimension,
+                granularity=pre_agg_data.granularity,
+                refresh_key=refresh_key
+            )
+            pre_aggregations.append(pre_aggregation)
+        
         return CubeSchema(
             cube_name=cube_name,
             sql=sql,
             dimensions=dimensions,
-            measures=measures
+            measures=measures,
+            pre_aggregations=pre_aggregations
         )
     
     def _write_cube_file(self, cube_schema: CubeSchema) -> Path:
@@ -116,7 +141,8 @@ class CubeGenerator:
                 cube_name=cube_schema.cube_name,
                 sql=cube_schema.sql,
                 dimensions=cube_schema.dimensions,
-                measures=cube_schema.measures
+                measures=cube_schema.measures,
+                pre_aggregations=cube_schema.pre_aggregations
             )
         else:
             # Fallback to hardcoded template
@@ -131,7 +157,12 @@ class CubeGenerator:
     
     def _generate_cube_content(self, cube_schema: CubeSchema) -> str:
         """Generate Cube.js content using hardcoded template"""
-        
+
+        # Extract table name from SQL for refresh_key replacement
+        import re
+        table_name_match = re.search(r'FROM\s+([^\s,;]+)', cube_schema.sql, re.IGNORECASE)
+        table_name = table_name_match.group(1) if table_name_match else None
+
         # Generate dimensions
         dimensions_content = []
         for dim in cube_schema.dimensions:
@@ -152,11 +183,79 @@ class CubeGenerator:
     }}"""
             measures_content.append(measure_content)
         
+        # Generate pre-aggregations
+        pre_aggregations_content = []
+        for pre_agg in cube_schema.pre_aggregations:
+            pre_agg_parts = [f"      type: `{pre_agg.type}`"]
+            
+            if pre_agg.measures:
+                measures_list = ', '.join([f'CUBE.{measure}' for measure in pre_agg.measures])
+                pre_agg_parts.append(f"      measures: [{measures_list}]")
+
+            if pre_agg.dimensions:
+                dims_list = ', '.join([f'CUBE.{dim}' for dim in pre_agg.dimensions])
+                pre_agg_parts.append(f"      dimensions: [{dims_list}]")
+
+            if pre_agg.time_dimension:
+                pre_agg_parts.append(f"      time_dimension: CUBE.{pre_agg.time_dimension}")
+                
+            if pre_agg.granularity:
+                pre_agg_parts.append(f"      granularity: `{pre_agg.granularity}`")
+                
+            if pre_agg.refresh_key:
+                refresh_key_parts = []
+                if pre_agg.refresh_key.every:
+                    refresh_key_parts.append(f"        every: `{pre_agg.refresh_key.every}`")
+                if pre_agg.refresh_key.sql:
+                    # Replace ${CUBE} and ${this} with actual table name
+                    refresh_sql = pre_agg.refresh_key.sql
+                    if table_name:
+                        refresh_sql = refresh_sql.replace('${CUBE}', table_name)
+                        refresh_sql = refresh_sql.replace('${this}', table_name)
+                    refresh_key_parts.append(f"        sql: `{refresh_sql}`")
+                if pre_agg.refresh_key.incremental is not None:
+                    refresh_key_parts.append(f"        incremental: {str(pre_agg.refresh_key.incremental).lower()}")
+                if pre_agg.refresh_key.update_window:
+                    refresh_key_parts.append(f"        update_window: `{pre_agg.refresh_key.update_window}`")
+                
+                if refresh_key_parts:
+                    refresh_key_content = ',\n'.join(refresh_key_parts)
+                    pre_agg_parts.append(f"      refresh_key: {{\n{refresh_key_content}\n      }}")
+            
+            parts_joined = ',\n'.join(pre_agg_parts)
+            pre_agg_content = f"""    {pre_agg.name}: {{
+{parts_joined}
+    }}"""
+            pre_aggregations_content.append(pre_agg_content)
+        
         # Combine into full cube definition
         dimensions_joined = ',\n\n'.join(dimensions_content)
         measures_joined = ',\n\n'.join(measures_content)
         
-        content = f"""cube(`{cube_schema.cube_name}`, {{
+        # Ensure we have measures (required for a useful Cube.js schema)
+        if not measures_content:
+            raise ValueError(f"Cube {cube_schema.cube_name} has no measures defined. Measures are required for Cube.js schemas.")
+        
+        if pre_aggregations_content:
+            pre_aggregations_joined = ',\n\n'.join(pre_aggregations_content)
+            content = f"""cube(`{cube_schema.cube_name}`, {{
+  sql: `{cube_schema.sql}`,
+  
+  dimensions: {{
+{dimensions_joined}
+  }},
+  
+  measures: {{
+{measures_joined}
+  }},
+  
+  pre_aggregations: {{
+{pre_aggregations_joined}
+  }}
+}});
+"""
+        else:
+            content = f"""cube(`{cube_schema.cube_name}`, {{
   sql: `{cube_schema.sql}`,
   
   dimensions: {{

dbt_cube_sync-0.1.0a6/dbt_cube_sync/core/db_inspector.py

@@ -0,0 +1,51 @@
+"""
+Database inspector - fetches column types using SQLAlchemy
+"""
+from typing import Dict, Optional
+from sqlalchemy import create_engine, inspect, MetaData, Table
+from sqlalchemy.engine import Engine
+
+
+class DatabaseInspector:
+    """Inspects database schema to extract column type information"""
+
+    def __init__(self, sqlalchemy_uri: str):
+        """
+        Initialize the database inspector
+
+        Args:
+            sqlalchemy_uri: SQLAlchemy connection URI (e.g., postgresql://user:pass@host:port/db)
+        """
+        self.engine: Engine = create_engine(sqlalchemy_uri)
+        self.inspector = inspect(self.engine)
+
+    def get_table_columns(self, schema: str, table_name: str) -> Dict[str, str]:
+        """
+        Get column names and their data types for a specific table
+
+        Args:
+            schema: Database schema name
+            table_name: Table name
+
+        Returns:
+            Dictionary mapping column names to data types
+        """
+        columns = {}
+
+        try:
+            # Get columns from the database
+            table_columns = self.inspector.get_columns(table_name, schema=schema)
+
+            for column in table_columns:
+                col_name = column['name']
+                col_type = str(column['type'])
+                columns[col_name] = col_type
+
+        except Exception as e:
+            print(f"Warning: Could not inspect table {schema}.{table_name}: {e}")
+
+        return columns
+
+    def close(self):
+        """Close the database connection"""
+        self.engine.dispose()

{dbt_cube_sync-0.1.0a4 → dbt_cube_sync-0.1.0a6}/dbt_cube_sync/core/dbt_parser.py

@@ -3,27 +3,39 @@ dbt manifest parser - extracts models, metrics, and column information
 """
 import json
 import os
-from typing import Dict, List
+from typing import Dict, List, Optional
 from pathlib import Path
 
-from .models import DbtModel, DbtColumn, DbtMetric
+from .models import DbtModel, DbtColumn, DbtMetric, DbtPreAggregation, DbtRefreshKey
+from .db_inspector import DatabaseInspector
 
 
 class DbtParser:
     """Parses dbt manifest.json to extract model and metric information"""
-    
-    def __init__(self, manifest_path: str, catalog_path: str = None):
+
+    def __init__(
+        self,
+        manifest_path: str,
+        catalog_path: Optional[str] = None,
+        sqlalchemy_uri: Optional[str] = None,
+        model_filter: Optional[List[str]] = None
+    ):
         """
         Initialize the parser
-        
+
         Args:
             manifest_path: Path to dbt manifest.json file
             catalog_path: Optional path to dbt catalog.json for column types
+            sqlalchemy_uri: Optional SQLAlchemy URI to connect to database for column types
+            model_filter: Optional list of model names to process (if None, processes all models)
         """
         self.manifest_path = manifest_path
         self.catalog_path = catalog_path
+        self.sqlalchemy_uri = sqlalchemy_uri
+        self.model_filter = model_filter
         self.manifest = self._load_manifest()
         self.catalog = self._load_catalog() if catalog_path else None
+        self.db_inspector = DatabaseInspector(sqlalchemy_uri) if sqlalchemy_uri else None
     
     def _load_manifest(self) -> dict:
         """Load the dbt manifest.json file"""
@@ -48,22 +60,32 @@ class DbtParser:
     def parse_models(self) -> List[DbtModel]:
         """
         Extract models with metrics and columns from manifest
-        
+
         Returns:
             List of DbtModel instances
         """
         models = []
         nodes = self.manifest.get('nodes', {})
-        
+
         for node_id, node_data in nodes.items():
             # Only process models
             if node_data.get('resource_type') != 'model':
                 continue
-            
+
+            # Apply model filter if specified
+            model_name = node_data.get('name', '')
+            if self.model_filter and model_name not in self.model_filter:
+                continue
+
             model = self._parse_model(node_id, node_data)
-            if model and model.columns and model.metrics:  # Only include models with BOTH columns AND metrics
+            # Include models that have columns AND metrics (measures are required for useful Cube.js schemas)
+            if model and model.columns and model.metrics:
                 models.append(model)
-        
+
+        # Close database inspector if it was used
+        if self.db_inspector:
+            self.db_inspector.close()
+
         return models
     
     def _parse_model(self, node_id: str, node_data: dict) -> DbtModel:
@@ -78,34 +100,49 @@ class DbtParser:
         # Parse metrics from config.meta.metrics
         metrics = self._parse_metrics(node_data)
         
+        # Parse pre-aggregations from config.meta.pre_aggregations
+        pre_aggregations = self._parse_pre_aggregations(node_data)
+        
         return DbtModel(
             name=model_name,
             database=model_database,
             schema_name=model_schema,
             node_id=node_id,
             columns=columns,
-            metrics=metrics
+            metrics=metrics,
+            pre_aggregations=pre_aggregations
         )
     
     def _parse_columns(self, node_id: str, node_data: dict) -> Dict[str, DbtColumn]:
-        """Parse columns for a model, enhanced with catalog data if available"""
+        """Parse columns for a model, enhanced with catalog or database data if available"""
         columns = {}
         manifest_columns = node_data.get('columns', {})
-        
-        # Get catalog columns for type information
+
+        # Get catalog columns for type information (if catalog is available)
         catalog_columns = {}
         if self.catalog and node_id in self.catalog.get('nodes', {}):
             catalog_columns = self.catalog['nodes'][node_id].get('columns', {})
-        
-        # If manifest has columns, use them with catalog type info
+
+        # Get database columns for type information (if db_inspector is available)
+        db_columns = {}
+        if self.db_inspector and not self.catalog:
+            schema = node_data.get('schema', '')
+            table_name = node_data.get('name', '')
+            if schema and table_name:
+                db_columns = self.db_inspector.get_table_columns(schema, table_name)
+
+        # If manifest has columns, use them with catalog or database type info
         if manifest_columns:
             for col_name, col_data in manifest_columns.items():
                 data_type = None
-                
-                # Try to get data type from catalog
+
+                # Try to get data type from catalog first
                 if col_name in catalog_columns:
                     data_type = catalog_columns[col_name].get('type', '')
-                
+                # Otherwise try database
+                elif col_name in db_columns:
+                    data_type = db_columns[col_name]
+
                 columns[col_name] = DbtColumn(
                     name=col_name,
                     data_type=data_type,
@@ -113,15 +150,24 @@ class DbtParser:
                     meta=col_data.get('meta', {})
                 )
         else:
-            # If no manifest columns, use all catalog columns
-            for col_name, col_data in catalog_columns.items():
+            # If no manifest columns, use catalog or database columns
+            source_columns = catalog_columns or db_columns
+            for col_name in source_columns:
+                if catalog_columns:
+                    col_data = catalog_columns[col_name]
+                    data_type = col_data.get('type', '')
+                    description = f"Column from catalog: {col_name}"
+                else:
+                    data_type = db_columns[col_name]
+                    description = f"Column from database: {col_name}"
+
                 columns[col_name] = DbtColumn(
                     name=col_name,
-                    data_type=col_data.get('type', ''),
-                    description=f"Column from catalog: {col_name}",
+                    data_type=data_type,
+                    description=description,
                     meta={}
                 )
-        
+
         return columns
     
     def _parse_metrics(self, node_data: dict) -> Dict[str, DbtMetric]:
@@ -145,6 +191,40 @@ class DbtParser:
         
         return metrics
     
+    def _parse_pre_aggregations(self, node_data: dict) -> Dict[str, DbtPreAggregation]:
+        """Parse pre-aggregations from model configuration"""
+        pre_aggregations = {}
+        
+        # Look for pre-aggregations in config.meta.pre_aggregations
+        config = node_data.get('config', {})
+        meta = config.get('meta', {})
+        pre_aggs_data = meta.get('pre_aggregations', {})
+        
+        for pre_agg_name, pre_agg_config in pre_aggs_data.items():
+            if isinstance(pre_agg_config, dict):
+                # Parse refresh_key if present
+                refresh_key = None
+                refresh_key_config = pre_agg_config.get('refresh_key')
+                if refresh_key_config and isinstance(refresh_key_config, dict):
+                    refresh_key = DbtRefreshKey(
+                        every=refresh_key_config.get('every'),
+                        sql=refresh_key_config.get('sql'),
+                        incremental=refresh_key_config.get('incremental'),
+                        update_window=refresh_key_config.get('update_window')
+                    )
+                
+                pre_aggregations[pre_agg_name] = DbtPreAggregation(
+                    name=pre_agg_name,
+                    type=pre_agg_config.get('type', 'rollup'),
+                    measures=pre_agg_config.get('measures', []),
+                    dimensions=pre_agg_config.get('dimensions', []),
+                    time_dimension=pre_agg_config.get('time_dimension'),
+                    granularity=pre_agg_config.get('granularity'),
+                    refresh_key=refresh_key
+                )
+        
+        return pre_aggregations
+    
     @staticmethod
     def map_dbt_type_to_cube_type(dbt_type: str) -> str:
         """Map dbt metric types to Cube.js measure types"""

{dbt_cube_sync-0.1.0a4 → dbt_cube_sync-0.1.0a6}/dbt_cube_sync/core/models.py

@@ -22,6 +22,25 @@ class DbtMetric(BaseModel):
     description: Optional[str] = None
 
 
+class DbtRefreshKey(BaseModel):
+    """Represents a refresh_key configuration for pre-aggregations"""
+    every: Optional[str] = None
+    sql: Optional[str] = None
+    incremental: Optional[bool] = None
+    update_window: Optional[str] = None
+
+
+class DbtPreAggregation(BaseModel):
+    """Represents a dbt pre-aggregation configuration"""
+    name: str
+    type: str = "rollup"
+    measures: Optional[List[str]] = None
+    dimensions: Optional[List[str]] = None
+    time_dimension: Optional[str] = None
+    granularity: Optional[str] = None
+    refresh_key: Optional[DbtRefreshKey] = None
+
+
 class DbtModel(BaseModel):
     """Represents a parsed dbt model"""
     name: str
@@ -30,6 +49,7 @@ class DbtModel(BaseModel):
     node_id: str
     columns: Dict[str, DbtColumn]
     metrics: Dict[str, DbtMetric]
+    pre_aggregations: Dict[str, DbtPreAggregation] = {}
 
 
 class CubeDimension(BaseModel):
@@ -50,12 +70,32 @@ class CubeMeasure(BaseModel):
     description: Optional[str] = None
 
 
+class CubeRefreshKey(BaseModel):
+    """Represents a Cube.js refresh_key configuration"""
+    every: Optional[str] = None
+    sql: Optional[str] = None
+    incremental: Optional[bool] = None
+    update_window: Optional[str] = None
+
+
+class CubePreAggregation(BaseModel):
+    """Represents a Cube.js pre-aggregation"""
+    name: str
+    type: str = "rollup"
+    measures: Optional[List[str]] = None
+    dimensions: Optional[List[str]] = None
+    time_dimension: Optional[str] = None
+    granularity: Optional[str] = None
+    refresh_key: Optional[CubeRefreshKey] = None
+
+
 class CubeSchema(BaseModel):
     """Represents a complete Cube.js schema"""
     cube_name: str
     sql: str
     dimensions: List[CubeDimension]
     measures: List[CubeMeasure]
+    pre_aggregations: List[CubePreAggregation] = []
 
 
 class SyncResult(BaseModel):

{dbt_cube_sync-0.1.0a4 → dbt_cube_sync-0.1.0a6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dbt-cube-sync"
-version = "0.1.0a4"
+version = "0.1.0a6"
 description = "Synchronization tool for dbt models to Cube.js schemas and BI tools"
 authors = ["Ponder"]
 readme = "README.md"
@@ -13,6 +13,7 @@ pyyaml = "^6.0"
 click = "^8.1.7"
 pydantic = "^2.5.0"
 jinja2 = "^3.1.2"
+sqlalchemy = "^2.0.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.4.0"