dbt-cube-sync 0.1.0a1__py3-none-any.whl

dbt_cube_sync/core/cube_generator.py

@@ -0,0 +1,188 @@
+"""
+Cube.js schema generator - creates Cube.js files from dbt models
+"""
+import os
+import re
+from pathlib import Path
+from typing import List, Dict, Any
+from jinja2 import Environment, FileSystemLoader, Template
+
+from .models import DbtModel, CubeSchema, CubeDimension, CubeMeasure
+from .dbt_parser import DbtParser
+
+
+class CubeGenerator:
+    """Generates Cube.js schema files from dbt models"""
+    
+    def __init__(self, template_dir: str, output_dir: str):
+        """
+        Initialize the generator
+        
+        Args:
+            template_dir: Directory containing Jinja2 templates
+            output_dir: Directory to write generated Cube.js files
+        """
+        self.template_dir = Path(template_dir)
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Initialize Jinja2 environment
+        self.env = Environment(loader=FileSystemLoader(str(self.template_dir)))
+    
+    def generate_cube_files(self, models: List[DbtModel]) -> List[str]:
+        """
+        Generate Cube.js files for all models
+        
+        Args:
+            models: List of DbtModel instances
+            
+        Returns:
+            List of generated file paths
+        """
+        generated_files = []
+        
+        for model in models:
+            try:
+                cube_schema = self._convert_model_to_cube(model)
+                file_path = self._write_cube_file(cube_schema)
+                generated_files.append(str(file_path))
+                print(f"✓ Generated: {file_path.name}")
+            except Exception as e:
+                print(f"✗ Error generating cube for {model.name}: {str(e)}")
+        
+        return generated_files
+    
+    def _convert_model_to_cube(self, model: DbtModel) -> CubeSchema:
+        """Convert a dbt model to a Cube.js schema"""
+        
+        # Generate cube name (PascalCase)
+        cube_name = self._to_pascal_case(model.name)
+        
+        # Generate SQL reference
+        sql = f"SELECT * FROM {model.schema_name}.{model.name}"
+        
+        # Convert columns to dimensions
+        dimensions = []
+        for col_name, col_data in model.columns.items():
+            cube_type = DbtParser.map_data_type_to_cube_type(col_data.data_type or '')
+            
+            dimension = CubeDimension(
+                name=col_name,
+                sql=col_name,
+                type=cube_type,
+                title=col_data.description or col_name.replace('_', ' ').title(),
+                description=col_data.description or col_name.replace('_', ' ').title()
+            )
+            dimensions.append(dimension)
+        
+        # Convert explicitly defined metrics to measures
+        measures = []
+        for metric_name, metric_data in model.metrics.items():
+            cube_type = DbtParser.map_dbt_type_to_cube_type(metric_data.type)
+            
+            # Generate SQL expression
+            if metric_data.sql:
+                sql_expr = metric_data.sql
+            elif metric_data.type == 'count':
+                sql_expr = "*"
+            else:
+                # Default to the metric name if no SQL provided
+                sql_expr = metric_name
+            
+            measure = CubeMeasure(
+                name=metric_name,
+                type=cube_type,
+                sql=sql_expr,
+                title=metric_data.title or metric_name.replace('_', ' ').title(),
+                description=metric_data.description or metric_name.replace('_', ' ').title()
+            )
+            measures.append(measure)
+        
+        return CubeSchema(
+            cube_name=cube_name,
+            sql=sql,
+            dimensions=dimensions,
+            measures=measures
+        )
+    
+    def _write_cube_file(self, cube_schema: CubeSchema) -> Path:
+        """Write a Cube.js schema to file"""
+        
+        # Try to use template if available
+        template_path = self.template_dir / 'cube_template.js'
+        if template_path.exists():
+            template = self.env.get_template('cube_template.js')
+            content = template.render(
+                cube_name=cube_schema.cube_name,
+                sql=cube_schema.sql,
+                dimensions=cube_schema.dimensions,
+                measures=cube_schema.measures
+            )
+        else:
+            # Fallback to hardcoded template
+            content = self._generate_cube_content(cube_schema)
+        
+        # Write to file
+        file_path = self.output_dir / f"{cube_schema.cube_name}.js"
+        with open(file_path, 'w') as f:
+            f.write(content)
+        
+        return file_path
+    
+    def _generate_cube_content(self, cube_schema: CubeSchema) -> str:
+        """Generate Cube.js content using hardcoded template"""
+        
+        # Generate dimensions
+        dimensions_content = []
+        for dim in cube_schema.dimensions:
+            dim_content = f"""    {dim.name}: {{
+      sql: `{dim.sql}`,
+      type: `{dim.type}`,
+      title: '{dim.title}'
+    }}"""
+            dimensions_content.append(dim_content)
+        
+        # Generate measures  
+        measures_content = []
+        for measure in cube_schema.measures:
+            measure_content = f"""    {measure.name}: {{
+      type: `{measure.type}`,
+      sql: `{measure.sql}`,
+      title: '{measure.title}'
+    }}"""
+            measures_content.append(measure_content)
+        
+        # Combine into full cube definition
+        dimensions_joined = ',\n\n'.join(dimensions_content)
+        measures_joined = ',\n\n'.join(measures_content)
+        
+        content = f"""cube(`{cube_schema.cube_name}`, {{
+  sql: `{cube_schema.sql}`,
+  
+  dimensions: {{
+{dimensions_joined}
+  }},
+  
+  measures: {{
+{measures_joined}
+  }}
+}});
+"""
+        
+        return content
+    
+    @staticmethod
+    def _to_pascal_case(text: str) -> str:
+        """Convert text to PascalCase"""
+        # Remove non-alphanumeric characters and split
+        words = re.sub(r'[^a-zA-Z0-9]', ' ', text).split()
+        # Capitalize first letter of each word and join
+        return ''.join(word.capitalize() for word in words if word)
+    
+    @staticmethod
+    def _to_snake_case(text: str) -> str:
+        """Convert text to snake_case"""
+        # Replace non-alphanumeric with underscores and lowercase
+        result = re.sub(r'[^a-zA-Z0-9]', '_', text).lower()
+        # Remove multiple underscores
+        return re.sub(r'_+', '_', result).strip('_')

dbt_cube_sync/core/dbt_parser.py

@@ -0,0 +1,178 @@
+"""
+dbt manifest parser - extracts models, metrics, and column information
+"""
+import json
+import os
+from typing import Dict, List
+from pathlib import Path
+
+from .models import DbtModel, DbtColumn, DbtMetric
+
+
+class DbtParser:
+    """Parses dbt manifest.json to extract model and metric information"""
+    
+    def __init__(self, manifest_path: str, catalog_path: str = None):
+        """
+        Initialize the parser
+        
+        Args:
+            manifest_path: Path to dbt manifest.json file
+            catalog_path: Optional path to dbt catalog.json for column types
+        """
+        self.manifest_path = manifest_path
+        self.catalog_path = catalog_path
+        self.manifest = self._load_manifest()
+        self.catalog = self._load_catalog() if catalog_path else None
+    
+    def _load_manifest(self) -> dict:
+        """Load the dbt manifest.json file"""
+        if not os.path.exists(self.manifest_path):
+            raise FileNotFoundError(f"Manifest file not found: {self.manifest_path}")
+        
+        with open(self.manifest_path, 'r') as f:
+            return json.load(f)
+    
+    def _load_catalog(self) -> dict:
+        """Load the dbt catalog.json file if available"""
+        if not self.catalog_path or not os.path.exists(self.catalog_path):
+            return None
+        
+        try:
+            with open(self.catalog_path, 'r') as f:
+                return json.load(f)
+        except Exception as e:
+            print(f"Warning: Could not load catalog file {self.catalog_path}: {e}")
+            return None
+    
+    def parse_models(self) -> List[DbtModel]:
+        """
+        Extract models with metrics and columns from manifest
+        
+        Returns:
+            List of DbtModel instances
+        """
+        models = []
+        nodes = self.manifest.get('nodes', {})
+        
+        for node_id, node_data in nodes.items():
+            # Only process models
+            if node_data.get('resource_type') != 'model':
+                continue
+            
+            model = self._parse_model(node_id, node_data)
+            if model and model.columns and model.metrics:  # Only include models with BOTH columns AND metrics
+                models.append(model)
+        
+        return models
+    
+    def _parse_model(self, node_id: str, node_data: dict) -> DbtModel:
+        """Parse a single model from the manifest"""
+        model_name = node_data.get('name', '')
+        model_schema = node_data.get('schema', '')
+        model_database = node_data.get('database', '')
+        
+        # Parse columns
+        columns = self._parse_columns(node_id, node_data)
+        
+        # Parse metrics from config.meta.metrics
+        metrics = self._parse_metrics(node_data)
+        
+        return DbtModel(
+            name=model_name,
+            database=model_database,
+            schema_name=model_schema,
+            node_id=node_id,
+            columns=columns,
+            metrics=metrics
+        )
+    
+    def _parse_columns(self, node_id: str, node_data: dict) -> Dict[str, DbtColumn]:
+        """Parse columns for a model, enhanced with catalog data if available"""
+        columns = {}
+        manifest_columns = node_data.get('columns', {})
+        
+        # Get catalog columns for type information
+        catalog_columns = {}
+        if self.catalog and node_id in self.catalog.get('nodes', {}):
+            catalog_columns = self.catalog['nodes'][node_id].get('columns', {})
+        
+        # If manifest has columns, use them with catalog type info
+        if manifest_columns:
+            for col_name, col_data in manifest_columns.items():
+                data_type = None
+                
+                # Try to get data type from catalog
+                if col_name in catalog_columns:
+                    data_type = catalog_columns[col_name].get('type', '')
+                
+                columns[col_name] = DbtColumn(
+                    name=col_name,
+                    data_type=data_type,
+                    description=col_data.get('description'),
+                    meta=col_data.get('meta', {})
+                )
+        else:
+            # If no manifest columns, use all catalog columns
+            for col_name, col_data in catalog_columns.items():
+                columns[col_name] = DbtColumn(
+                    name=col_name,
+                    data_type=col_data.get('type', ''),
+                    description=f"Column from catalog: {col_name}",
+                    meta={}
+                )
+        
+        return columns
+    
+    def _parse_metrics(self, node_data: dict) -> Dict[str, DbtMetric]:
+        """Parse metrics from model configuration"""
+        metrics = {}
+        
+        # Look for metrics in config.meta.metrics
+        config = node_data.get('config', {})
+        meta = config.get('meta', {})
+        metrics_data = meta.get('metrics', {})
+        
+        for metric_name, metric_config in metrics_data.items():
+            if isinstance(metric_config, dict):
+                metrics[metric_name] = DbtMetric(
+                    name=metric_name,
+                    type=metric_config.get('type', 'sum'),
+                    sql=metric_config.get('sql'),
+                    title=metric_config.get('title', metric_name.replace('_', ' ').title()),
+                    description=metric_config.get('description', metric_name.replace('_', ' ').title())
+                )
+        
+        return metrics
+    
+    @staticmethod
+    def map_dbt_type_to_cube_type(dbt_type: str) -> str:
+        """Map dbt metric types to Cube.js measure types"""
+        type_mapping = {
+            'sum': 'sum',
+            'average': 'avg',
+            'avg': 'avg',
+            'count': 'count',
+            'count_distinct': 'countDistinct',
+            'min': 'min',
+            'max': 'max',
+            'number': 'number',
+        }
+        return type_mapping.get(dbt_type.lower(), 'sum')
+    
+    @staticmethod
+    def map_data_type_to_cube_type(data_type: str) -> str:
+        """Map SQL/dbt data types to Cube.js dimension types"""
+        if not data_type:
+            return 'string'
+        
+        data_type = data_type.lower()
+        
+        if any(t in data_type for t in ['int', 'bigint', 'decimal', 'numeric', 'float', 'double']):
+            return 'number'
+        elif any(t in data_type for t in ['timestamp', 'datetime', 'date']):
+            return 'time'
+        elif any(t in data_type for t in ['bool']):
+            return 'boolean'
+        else:
+            return 'string'

dbt_cube_sync/core/models.py

@@ -0,0 +1,66 @@
+"""
+Pydantic models for data structures
+"""
+from typing import Dict, List, Optional, Any
+from pydantic import BaseModel
+
+
+class DbtColumn(BaseModel):
+    """Represents a dbt model column"""
+    name: str
+    data_type: Optional[str] = None
+    description: Optional[str] = None
+    meta: Optional[Dict[str, Any]] = None
+
+
+class DbtMetric(BaseModel):
+    """Represents a dbt metric"""
+    name: str
+    type: str
+    sql: Optional[str] = None
+    title: Optional[str] = None
+    description: Optional[str] = None
+
+
+class DbtModel(BaseModel):
+    """Represents a parsed dbt model"""
+    name: str
+    database: str
+    schema_name: str  # Renamed to avoid shadowing BaseModel.schema
+    node_id: str
+    columns: Dict[str, DbtColumn]
+    metrics: Dict[str, DbtMetric]
+
+
+class CubeDimension(BaseModel):
+    """Represents a Cube.js dimension"""
+    name: str
+    sql: str
+    type: str
+    title: Optional[str] = None
+    description: Optional[str] = None
+
+
+class CubeMeasure(BaseModel):
+    """Represents a Cube.js measure"""
+    name: str
+    type: str
+    sql: Optional[str] = None
+    title: Optional[str] = None
+    description: Optional[str] = None
+
+
+class CubeSchema(BaseModel):
+    """Represents a complete Cube.js schema"""
+    cube_name: str
+    sql: str
+    dimensions: List[CubeDimension]
+    measures: List[CubeMeasure]
+
+
+class SyncResult(BaseModel):
+    """Represents the result of a sync operation"""
+    file_or_dataset: str
+    status: str  # 'success' or 'failed'
+    message: Optional[str] = None
+    error: Optional[str] = None

dbt_cube_sync-0.1.0a1.dist-info/METADATA

@@ -0,0 +1,230 @@
+Metadata-Version: 2.4
+Name: dbt-cube-sync
+Version: 0.1.0a1
+Summary: Synchronization tool for dbt models to Cube.js schemas and BI tools
+Author: Ponder
+Requires-Python: >=3.9,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
+Requires-Dist: pydantic (>=2.5.0,<3.0.0)
+Requires-Dist: pyyaml (>=6.0,<7.0)
+Requires-Dist: requests (>=2.31.0,<3.0.0)
+Description-Content-Type: text/markdown
+
+# dbt-cube-sync
+
+A powerful synchronization tool that creates a seamless pipeline from dbt models to Cube.js schemas and BI tools (Superset, Tableau, PowerBI).
+
+## Features
+
+- 🔄 **dbt → Cube.js**: Auto-generate Cube.js schemas from dbt models with metrics
+- 📊 **Cube.js → BI Tools**: Sync schemas to multiple BI platforms
+- 🏗️ **Extensible Architecture**: Plugin-based connector system for easy BI tool integration
+- 🐳 **Docker Support**: Containerized execution with orchestration support
+- 🎯 **CLI Interface**: Simple command-line tools for automation
+
+## Supported BI Tools
+
+- ✅ **Apache Superset** - Full implementation
+- 🚧 **Tableau** - Placeholder (coming soon)
+- 🚧 **PowerBI** - Placeholder (coming soon)
+
+## Installation
+
+### Using Poetry (Development)
+
+```bash
+cd dbt-cube-sync
+poetry install
+poetry run dbt-cube-sync --help
+```
+
+### Using Docker
+
+```bash
+docker build -t dbt-cube-sync .
+docker run --rm dbt-cube-sync --help
+```
+
+## Quick Start
+
+### 1. Create Configuration File
+
+```bash
+# Create sample config
+dbt-cube-sync create-config sync-config.yaml
+
+# Edit the config file with your BI tool credentials
+```
+
+### 2. Generate Cube.js Schemas
+
+```bash
+# Generate from dbt manifest
+dbt-cube-sync generate-cubes \\
+  --dbt-manifest ./DbtEducationalDataProject/target/manifest.json \\
+  --output-dir ./cube/conf/cube_output
+```
+
+### 3. Sync to BI Tool
+
+```bash
+# Sync to Superset
+dbt-cube-sync sync-bi superset \\
+  --cube-dir ./cube/conf/cube_output \\
+  --config-file ./sync-config.yaml
+```
+
+### 4. Full Pipeline
+
+```bash
+# Complete dbt → Cube.js → Superset pipeline
+dbt-cube-sync full-sync \\
+  --dbt-manifest ./DbtEducationalDataProject/target/manifest.json \\
+  --cube-dir ./cube/conf/cube_output \\
+  --bi-connector superset \\
+  --config-file ./sync-config.yaml
+```
+
+## Configuration
+
+### Sample Configuration (`sync-config.yaml`)
+
+```yaml
+connectors:
+  superset:
+    type: superset
+    url: http://localhost:8088
+    username: admin
+    password: admin
+    database_name: Cube
+    
+  tableau:
+    type: tableau
+    url: https://your-tableau-server.com
+    username: your-username
+    password: your-password
+    
+  powerbi:
+    type: powerbi
+    # PowerBI specific configuration
+```
+
+## CLI Commands
+
+### `generate-cubes`
+Generate Cube.js schema files from dbt models.
+
+**Options:**
+- `--dbt-manifest` / `-m`: Path to dbt manifest.json file
+- `--output-dir` / `-o`: Output directory for Cube.js files
+- `--template-dir` / `-t`: Directory containing Cube.js templates
+
+### `sync-bi`
+Sync Cube.js schemas to BI tool datasets.
+
+**Arguments:**
+- `connector`: BI tool type (`superset`, `tableau`, `powerbi`)
+
+**Options:**
+- `--cube-dir` / `-c`: Directory containing Cube.js files
+- `--config-file` / `-f`: Configuration file for BI tool connection
+
+### `full-sync`
+Complete pipeline: dbt models → Cube.js schemas → BI tool datasets.
+
+**Options:**
+- `--dbt-manifest` / `-m`: Path to dbt manifest.json file
+- `--cube-dir` / `-c`: Directory for Cube.js files
+- `--template-dir` / `-t`: Directory containing Cube.js templates
+- `--bi-connector` / `-b`: BI tool to sync to
+- `--config-file` / `-f`: Configuration file for BI tool connection
+
+## Architecture
+
+```
+dbt models (with metrics) 
+    ↓
+dbt-cube-sync generate-cubes
+    ↓
+Cube.js schemas
+    ↓
+dbt-cube-sync sync-bi [connector]
+    ↓
+BI Tool Datasets (Superset/Tableau/PowerBI)
+```
+
+### Project Structure
+
+```
+dbt-cube-sync/
+├── dbt_cube_sync/
+│   ├── cli.py                 # CLI interface
+│   ├── config.py             # Configuration management
+│   ├── core/
+│   │   ├── dbt_parser.py     # dbt manifest parser
+│   │   ├── cube_generator.py # Cube.js generator
+│   │   └── models.py         # Pydantic data models
+│   └── connectors/
+│       ├── base.py           # Abstract base connector
+│       ├── superset.py       # Superset implementation
+│       ├── tableau.py        # Tableau placeholder
+│       └── powerbi.py        # PowerBI placeholder
+├── Dockerfile                # Container definition
+├── pyproject.toml            # Poetry configuration
+└── README.md
+```
+
+## Adding New BI Connectors
+
+1. Create a new connector class inheriting from `BaseConnector`
+2. Implement the required abstract methods
+3. Register the connector using `ConnectorRegistry.register()`
+
+Example:
+```python
+from .base import BaseConnector, ConnectorRegistry
+
+class MyBIConnector(BaseConnector):
+    def _validate_config(self):
+        # Validation logic
+        pass
+    
+    def connect(self):
+        # Connection logic
+        pass
+    
+    def sync_cube_schemas(self, cube_dir):
+        # Sync implementation
+        pass
+
+# Register the connector
+ConnectorRegistry.register('mybi', MyBIConnector)
+```
+
+## Docker Integration
+
+The tool is designed to work in containerized environments with proper dependency orchestration:
+
+1. **dbt docs**: Runs `dbt build` then serves documentation
+2. **dbt-cube-sync**: Runs sync pipeline after dbt and Cube.js are ready  
+3. **BI Tools**: Receive synced datasets after sync completes
+
+## Contributing
+
+1. Fork the repository
+2. Create a feature branch
+3. Implement your changes
+4. Add tests if applicable
+5. Submit a pull request
+
+## License
+
+MIT License - see LICENSE file for details.

dbt_cube_sync-0.1.0a1.dist-info/RECORD

@@ -0,0 +1,16 @@
+dbt_cube_sync/__init__.py,sha256=aifkfgUDRPL5v0LZzceH2LXu66YDkJjdpvKwXsdikbI,113
+dbt_cube_sync/cli.py,sha256=hcZzKqmSFEvEXoDWbbluna5mqdWVC6Cej6imWob0ml4,4983
+dbt_cube_sync/config.py,sha256=qhGE7CxTmh0RhPizgd3x3Yj-3L2LoC00UQIDT0q9FlQ,3858
+dbt_cube_sync/connectors/__init__.py,sha256=NG6tYZ3CYD5bG_MfNLZrUM8YoBEKArG8-AOmJ8pwvQI,52
+dbt_cube_sync/connectors/base.py,sha256=JLzerxJdt34z0kWuyieL6UQhf5_dUYPGmwkiRWBuSPY,2802
+dbt_cube_sync/connectors/powerbi.py,sha256=2Y8fTfh_6Q_Myma1ymipPh1U3HsfQKcktVequXXnIXI,1275
+dbt_cube_sync/connectors/superset.py,sha256=7-tWo5yBK-kF9mRH9jLJXJtfjVkM44LMKn7qreWy9GM,20846
+dbt_cube_sync/connectors/tableau.py,sha256=jKve1zErzTbgPOtmPB92ZwZl4I6uEySedM51JiwlGrE,1261
+dbt_cube_sync/core/__init__.py,sha256=kgsawtU5dqEvnHz6dU8qwJbH3rtIV7QlK2MhtYVDCaY,46
+dbt_cube_sync/core/cube_generator.py,sha256=JtFEqlr9_ShS0e9LxjE0I1HlU13x7MX5sQ3O1G4r9ns,6444
+dbt_cube_sync/core/dbt_parser.py,sha256=Woed3RCuluVFaRZgjG1m0m-v2c4_Zb721lQq5hLKXqM,6521
+dbt_cube_sync/core/models.py,sha256=PvfE4A_1K_fSutO_KdzseBuRp-ZQtyKCW3Qonzu0NuU,1543
+dbt_cube_sync-0.1.0a1.dist-info/METADATA,sha256=Aou7v6ErpetsOzTHcRK_4R5yFs0lX-Jc260_z3WfJ7A,6116
+dbt_cube_sync-0.1.0a1.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+dbt_cube_sync-0.1.0a1.dist-info/entry_points.txt,sha256=iEAB_nZ1AoSeFwSHPY2tr02xmTHLVFKp5CJeFh0AfCw,56
+dbt_cube_sync-0.1.0a1.dist-info/RECORD,,

dbt_cube_sync-0.1.0a1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.2.1
+Root-Is-Purelib: true
+Tag: py3-none-any

dbt_cube_sync-0.1.0a1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+dbt-cube-sync=dbt_cube_sync.cli:main
+