duckrun 0.2.9.dev0__tar.gz → 0.2.9.dev1__tar.gz

{duckrun-0.2.9.dev0 → duckrun-0.2.9.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.9.dev0
+Version: 0.2.9.dev1
 Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
 Author: mim
 License: MIT

{duckrun-0.2.9.dev0 → duckrun-0.2.9.dev1}/duckrun/core.py

@@ -696,6 +696,45 @@ class Duckrun:
             print(f"❌ Error creating lakehouse '{lakehouse_name}': {e}")
             return False
 
+    def deploy(self, bim_url: str, dataset_name: Optional[str] = None, 
+               wait_seconds: int = 5) -> int:
+        """
+        Deploy a semantic model from a BIM file using DirectLake mode.
+        
+        Args:
+            bim_url: URL to the BIM file (e.g., GitHub raw URL)
+            dataset_name: Name for the semantic model (default: lakehouse_schema)
+            wait_seconds: Seconds to wait for permission propagation (default: 5)
+        
+        Returns:
+            1 for success, 0 for failure
+        
+        Examples:
+            dr = Duckrun.connect("My Workspace/My Lakehouse.lakehouse/dbo")
+            
+            # Deploy with auto-generated name
+            dr.deploy("https://raw.githubusercontent.com/.../model.bim")
+            
+            # Deploy with custom name
+            dr.deploy("https://raw.githubusercontent.com/.../model.bim", 
+                     dataset_name="Sales Model")
+        """
+        from .semantic_model import deploy_semantic_model
+        
+        # Auto-generate dataset name if not provided
+        if dataset_name is None:
+            dataset_name = f"{self.lakehouse_name}_{self.schema}"
+        
+        # Call the deployment function (DirectLake only)
+        return deploy_semantic_model(
+            workspace_name=self.workspace,
+            lakehouse_name=self.lakehouse_name,
+            schema_name=self.schema,
+            dataset_name=dataset_name,
+            bim_url=bim_url,
+            wait_seconds=wait_seconds
+        )
+
     def _get_workspace_id_by_name(self, token: str, workspace_name: str) -> Optional[str]:
         """Helper method to get workspace ID from name"""
         try:

duckrun-0.2.9.dev1/duckrun/semantic_model.py

@@ -0,0 +1,367 @@
+"""
+Semantic Model Deployer - DirectLake mode for Fabric Lakehouses
+Uses duckrun's authentication. Works anywhere duckrun works.
+"""
+
+import requests
+import json
+import time
+import base64
+
+
+class FabricRestClient:
+    """Fabric REST API client using duckrun's authentication."""
+    
+    def __init__(self):
+        self.base_url = "https://api.fabric.microsoft.com"
+        self.token = None
+        self._get_token()
+    
+    def _get_token(self):
+        """Get Fabric API token using duckrun's auth module"""
+        from duckrun.auth import get_fabric_api_token
+        self.token = get_fabric_api_token()
+        if not self.token:
+            raise Exception("Failed to get Fabric API token")
+    
+    def _get_headers(self):
+        return {
+            "Authorization": f"Bearer {self.token}",
+            "Content-Type": "application/json"
+        }
+    
+    def get(self, endpoint: str):
+        url = f"{self.base_url}{endpoint}"
+        response = requests.get(url, headers=self._get_headers())
+        response.raise_for_status()
+        return response
+    
+    def post(self, endpoint: str, json: dict = None):
+        url = f"{self.base_url}{endpoint}"
+        response = requests.post(url, headers=self._get_headers(), json=json)
+        response.raise_for_status()
+        return response
+
+
+def get_workspace_id(workspace_name, client):
+    """Get workspace ID by name"""
+    response = client.get("/v1/workspaces")
+    workspaces = response.json().get('value', [])
+    
+    workspace_match = next((ws for ws in workspaces if ws.get('displayName') == workspace_name), None)
+    if not workspace_match:
+        raise ValueError(f"Workspace '{workspace_name}' not found")
+    
+    workspace_id = workspace_match['id']
+    print(f"✓ Found workspace: {workspace_name}")
+    return workspace_id
+
+
+def get_lakehouse_id(lakehouse_name, workspace_id, client):
+    """Get lakehouse ID by name"""
+    response = client.get(f"/v1/workspaces/{workspace_id}/lakehouses")
+    items = response.json().get('value', [])
+    
+    lakehouse_match = next((item for item in items if item.get('displayName') == lakehouse_name), None)
+    if not lakehouse_match:
+        raise ValueError(f"Lakehouse '{lakehouse_name}' not found")
+    
+    lakehouse_id = lakehouse_match['id']
+    print(f"✓ Found lakehouse: {lakehouse_name}")
+    return lakehouse_id
+
+
+def get_dataset_id(dataset_name, workspace_id, client):
+    """Get dataset ID by name"""
+    response = client.get(f"/v1/workspaces/{workspace_id}/semanticModels")
+    items = response.json().get('value', [])
+    
+    dataset_match = next((item for item in items if item.get('displayName') == dataset_name), None)
+    if not dataset_match:
+        raise ValueError(f"Dataset '{dataset_name}' not found")
+    
+    return dataset_match['id']
+
+
+def check_dataset_exists(dataset_name, workspace_id, client):
+    """Check if dataset already exists"""
+    try:
+        get_dataset_id(dataset_name, workspace_id, client)
+        print(f"⚠️  Dataset '{dataset_name}' already exists")
+        return True
+    except:
+        print(f"✓ Dataset name '{dataset_name}' is available")
+        return False
+
+
+def refresh_dataset(dataset_name, workspace_id, client):
+    """Refresh a dataset and monitor progress"""
+    dataset_id = get_dataset_id(dataset_name, workspace_id, client)
+    
+    payload = {
+        "type": "full",
+        "commitMode": "transactional",
+        "maxParallelism": 10,
+        "retryCount": 2,
+        "objects": []
+    }
+    
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/semanticModels/{dataset_id}/refreshes",
+        json=payload
+    )
+    
+    if response.status_code in [200, 202]:
+        print(f"✓ Refresh initiated")
+        
+        refresh_id = response.json().get('id')
+        if refresh_id:
+            print("   Monitoring refresh progress...")
+            max_attempts = 60
+            for attempt in range(max_attempts):
+                time.sleep(5)
+                
+                status_response = client.get(
+                    f"/v1/workspaces/{workspace_id}/semanticModels/{dataset_id}/refreshes/{refresh_id}"
+                )
+                status = status_response.json().get('status')
+                
+                if status == 'Completed':
+                    print(f"✓ Refresh completed successfully")
+                    return
+                elif status == 'Failed':
+                    error = status_response.json().get('error', {})
+                    raise Exception(f"Refresh failed: {error.get('message', 'Unknown error')}")
+                elif status == 'Cancelled':
+                    raise Exception("Refresh was cancelled")
+                
+                if attempt % 6 == 0:
+                    print(f"   Status: {status}...")
+            
+            raise Exception(f"Refresh timed out")
+
+
+def download_bim_from_github(url):
+    """Download BIM file from URL"""
+    print(f"Downloading BIM file...")
+    response = requests.get(url)
+    response.raise_for_status()
+    bim_content = response.json()
+    print(f"✓ BIM file downloaded")
+    print(f"  - Tables: {len(bim_content.get('model', {}).get('tables', []))}")
+    print(f"  - Relationships: {len(bim_content.get('model', {}).get('relationships', []))}")
+    return bim_content
+
+
+def update_bim_for_directlake(bim_content, workspace_id, lakehouse_id, schema_name):
+    """Update BIM file for DirectLake mode"""
+    
+    new_url = f"https://onelake.dfs.fabric.microsoft.com/{workspace_id}/{lakehouse_id}"
+    expression_name = None
+    
+    # Update or create DirectLake expression
+    if 'model' in bim_content and 'expressions' in bim_content['model']:
+        for expr in bim_content['model']['expressions']:
+            if 'DirectLake' in expr['name'] or expr.get('kind') == 'm':
+                expression_name = expr['name']
+                expr['expression'] = [
+                    "let",
+                    f"    Source = AzureStorage.DataLake(\"{new_url}\", [HierarchicalNavigation=true])",
+                    "in",
+                    "    Source"
+                ]
+                break
+    
+    if not expression_name:
+        expression_name = f"DirectLake - {schema_name}"
+        if 'expressions' not in bim_content['model']:
+            bim_content['model']['expressions'] = []
+        
+        bim_content['model']['expressions'].append({
+            "name": expression_name,
+            "kind": "m",
+            "expression": [
+                "let",
+                f"    Source = AzureStorage.DataLake(\"{new_url}\", [HierarchicalNavigation=true])",
+                "in",
+                "    Source"
+            ],
+            "lineageTag": f"directlake-{schema_name}-source"
+        })
+    
+    # Update table partitions for DirectLake
+    if 'tables' in bim_content['model']:
+        for table in bim_content['model']['tables']:
+            if 'partitions' in table:
+                for partition in table['partitions']:
+                    if 'source' in partition:
+                        partition['mode'] = 'directLake'
+                        partition['source'] = {
+                            "type": "entity",
+                            "entityName": partition['source'].get('entityName', table['name']),
+                            "expressionSource": expression_name,
+                            "schemaName": schema_name
+                        }
+    
+    print(f"✓ Updated BIM for DirectLake")
+    print(f"  - OneLake URL: {new_url}")
+    print(f"  - Schema: {schema_name}")
+    
+    return bim_content
+
+
+def create_dataset_from_bim(dataset_name, bim_content, workspace_id, client):
+    """Create semantic model from BIM using Fabric REST API"""
+    # Convert to base64
+    bim_json = json.dumps(bim_content, indent=2)
+    bim_base64 = base64.b64encode(bim_json.encode('utf-8')).decode('utf-8')
+    
+    pbism_content = {"version": "1.0"}
+    pbism_json = json.dumps(pbism_content)
+    pbism_base64 = base64.b64encode(pbism_json.encode('utf-8')).decode('utf-8')
+    
+    payload = {
+        "displayName": dataset_name,
+        "definition": {
+            "parts": [
+                {
+                    "path": "model.bim",
+                    "payload": bim_base64,
+                    "payloadType": "InlineBase64"
+                },
+                {
+                    "path": "definition.pbism",
+                    "payload": pbism_base64,
+                    "payloadType": "InlineBase64"
+                }
+            ]
+        }
+    }
+    
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/semanticModels",
+        json=payload
+    )
+    
+    print(f"✓ Semantic model created")
+    
+    # Handle long-running operation
+    if response.status_code == 202:
+        operation_id = response.headers.get('x-ms-operation-id')
+        print(f"   Waiting for operation to complete...")
+        
+        max_attempts = 30
+        for attempt in range(max_attempts):
+            time.sleep(2)
+            status_response = client.get(f"/v1/operations/{operation_id}")
+            status = status_response.json().get('status')
+            
+            if status == 'Succeeded':
+                print(f"✓ Operation completed")
+                break
+            elif status == 'Failed':
+                error = status_response.json().get('error', {})
+                raise Exception(f"Operation failed: {error.get('message')}")
+            elif attempt == max_attempts - 1:
+                raise Exception(f"Operation timed out")
+
+
+def deploy_semantic_model(workspace_name, lakehouse_name, schema_name, dataset_name, 
+                         bim_url, wait_seconds=5):
+    """
+    Deploy a semantic model using DirectLake mode.
+    
+    Args:
+        workspace_name: Name of the target workspace
+        lakehouse_name: Name of the lakehouse
+        schema_name: Schema name (e.g., 'dbo', 'staging')
+        dataset_name: Name for the semantic model
+        bim_url: URL to the BIM file
+        wait_seconds: Seconds to wait before refresh (default: 5)
+    
+    Returns:
+        1 for success, 0 for failure
+    
+    Examples:
+        dr = Duckrun.connect("My Workspace/My Lakehouse.lakehouse/dbo")
+        dr.deploy("https://raw.githubusercontent.com/.../model.bim")
+    """
+    print("=" * 70)
+    print("Semantic Model Deployment (DirectLake)")
+    print("=" * 70)
+    
+    client = FabricRestClient()
+    
+    try:
+        # Step 1: Get workspace ID
+        print("\n[Step 1/6] Getting workspace information...")
+        workspace_id = get_workspace_id(workspace_name, client)
+        
+        # Step 2: Check if dataset exists
+        print(f"\n[Step 2/6] Checking if dataset '{dataset_name}' exists...")
+        dataset_exists = check_dataset_exists(dataset_name, workspace_id, client)
+        
+        if dataset_exists:
+            print(f"\n✓ Dataset exists - refreshing...")
+            
+            if wait_seconds > 0:
+                print(f"   Waiting {wait_seconds} seconds...")
+                time.sleep(wait_seconds)
+            
+            print("\n[Step 6/6] Refreshing semantic model...")
+            refresh_dataset(dataset_name, workspace_id, client)
+            
+            print("\n" + "=" * 70)
+            print("🎉 Refresh Completed!")
+            print("=" * 70)
+            print(f"Dataset: {dataset_name}")
+            print("=" * 70)
+            return 1
+        
+        # Step 3: Get lakehouse ID
+        print(f"\n[Step 3/6] Finding lakehouse...")
+        lakehouse_id = get_lakehouse_id(lakehouse_name, workspace_id, client)
+        
+        # Step 4: Download and update BIM
+        print("\n[Step 4/6] Downloading and configuring BIM file...")
+        bim_content = download_bim_from_github(bim_url)
+        
+        modified_bim = update_bim_for_directlake(bim_content, workspace_id, lakehouse_id, schema_name)
+        modified_bim['name'] = dataset_name
+        modified_bim['id'] = dataset_name
+        
+        # Step 5: Deploy
+        print("\n[Step 5/6] Deploying semantic model...")
+        create_dataset_from_bim(dataset_name, modified_bim, workspace_id, client)
+        
+        if wait_seconds > 0:
+            print(f"   Waiting {wait_seconds} seconds for permissions...")
+            time.sleep(wait_seconds)
+        
+        # Step 6: Refresh
+        print("\n[Step 6/6] Refreshing semantic model...")
+        refresh_dataset(dataset_name, workspace_id, client)
+        
+        print("\n" + "=" * 70)
+        print("🎉 Deployment Completed!")
+        print("=" * 70)
+        print(f"Dataset: {dataset_name}")
+        print(f"Workspace: {workspace_name}")
+        print(f"Lakehouse: {lakehouse_name}")
+        print(f"Schema: {schema_name}")
+        print("=" * 70)
+        
+        return 1
+        
+    except Exception as e:
+        print("\n" + "=" * 70)
+        print("❌ Deployment Failed")
+        print("=" * 70)
+        print(f"Error: {str(e)}")
+        print("\n💡 Troubleshooting:")
+        print(f"  - Verify workspace '{workspace_name}' exists")
+        print(f"  - Verify lakehouse '{lakehouse_name}' exists")
+        print(f"  - Ensure tables exist in '{schema_name}' schema")
+        print(f"  - Check tables are in Delta format")
+        print("=" * 70)
+        return 0

{duckrun-0.2.9.dev0 → duckrun-0.2.9.dev1}/duckrun/stats.py

@@ -147,7 +147,20 @@ def get_stats(duckrun_instance, source: str):
         
         try:
             dt = DeltaTable(table_path)
-            xx = dt.get_add_actions(flatten=True).to_pydict()
+            add_actions = dt.get_add_actions(flatten=True)
+            
+            # Convert to dict - compatible with both old and new deltalake versions
+            # Try to_pydict() first (old versions), fall back to to_pylist() (new versions)
+            try:
+                xx = add_actions.to_pydict()
+            except AttributeError:
+                # New version with arro3: use to_pylist() and convert to dict of lists
+                records = add_actions.to_pylist()
+                if records:
+                    # Convert list of dicts to dict of lists
+                    xx = {key: [record[key] for record in records] for key in records[0].keys()}
+                else:
+                    xx = {}
             
             # Check if VORDER exists
             vorder = 'tags.VORDER' in xx.keys()

{duckrun-0.2.9.dev0 → duckrun-0.2.9.dev1}/duckrun/writer.py

@@ -1,18 +1,36 @@
 """
 Delta Lake writer functionality for duckrun - Spark-style write API
 """
-from deltalake import DeltaTable, write_deltalake
+from deltalake import DeltaTable, write_deltalake, __version__ as deltalake_version
 
 
 # Row Group configuration for optimal Delta Lake performance
 RG = 8_000_000
 
+# Check deltalake version once at module load
+# Version 0.18.x and 0.19.x support engine parameter and row group optimization
+# Version 0.20+ removed these features (rust only, no row groups)
+_DELTALAKE_VERSION = tuple(map(int, deltalake_version.split('.')[:2]))
+_IS_OLD_DELTALAKE = _DELTALAKE_VERSION < (0, 20)
+
 
 def _build_write_deltalake_args(path, df, mode, schema_mode=None, partition_by=None):
     """
-    Build arguments for write_deltalake based on requirements:
-    - If schema_mode='merge': use rust engine (no row group params)
-    - Otherwise: use pyarrow engine with row group optimization (if supported)
+    Build arguments for write_deltalake based on requirements and version:
+    
+    deltalake 0.18.2 - 0.19.x:
+    - Has 'engine' parameter (defaults to 'pyarrow')
+    - Has max_rows_per_file/max_rows_per_group/min_rows_per_group for optimization
+    - When mergeSchema=True: must set schema_mode='merge' + engine='rust', NO row group params
+    - When mergeSchema=False: use row group params, DON'T set engine (pyarrow is default)
+    
+    deltalake 0.20+:
+    - Does NOT have 'engine' parameter (everything is rust, pyarrow deprecated)
+    - Does NOT have max_rows_per_file (row group optimization removed)
+    - When mergeSchema=True: must set schema_mode='merge'
+    - When mergeSchema=False: just write normally (no special params)
+    
+    Uses version detection for simpler logic.
     """
     args = {
         'table_or_uri': path,
@@ -24,23 +42,24 @@ def _build_write_deltalake_args(path, df, mode, schema_mode=None, partition_by=N
     if partition_by:
         args['partition_by'] = partition_by
     
-    # Engine selection based on schema_mode
     if schema_mode == 'merge':
-        # Use rust engine for schema merging (no row group params supported)
+        # Schema merging mode - must explicitly set schema_mode='merge'
         args['schema_mode'] = 'merge'
-        args['engine'] = 'rust'
-    else:
-        # Try to use pyarrow engine with row group optimization
-        # Check if row group parameters are supported by inspecting function signature
-        import inspect
-        sig = inspect.signature(write_deltalake)
         
-        if 'max_rows_per_file' in sig.parameters:
-            # Older deltalake version - use row group optimization
+        if _IS_OLD_DELTALAKE:
+            # deltalake 0.18.2-0.19.x: must also set engine='rust' for schema merging
+            # Do NOT use row group params (they conflict with rust engine)
+            args['engine'] = 'rust'
+        # For version 0.20+: just schema_mode='merge' is enough, rust is default
+    else:
+        # Normal write mode (no schema merging)
+        if _IS_OLD_DELTALAKE:
+            # deltalake 0.18.2-0.19.x: use row group optimization
+            # DON'T set engine parameter - pyarrow is the default and works with row groups
             args['max_rows_per_file'] = RG
             args['max_rows_per_group'] = RG
             args['min_rows_per_group'] = RG
-        # For newer versions, just use default parameters
+        # For version 0.20+: no optimization available (rust by default, no row group params supported)
     
     return args
 
@@ -113,7 +132,18 @@ class DeltaWriter:
             partition_by=self._partition_by
         )
         
-        engine_info = f" (engine=rust, schema_mode=merge)" if self._schema_mode == 'merge' else " (engine=pyarrow)"
+        # Prepare info message based on version and settings
+        if self._schema_mode == 'merge':
+            if _IS_OLD_DELTALAKE:
+                engine_info = " (engine=rust, schema_mode=merge)"
+            else:
+                engine_info = " (schema_mode=merge, rust by default)"
+        else:
+            if _IS_OLD_DELTALAKE:
+                engine_info = " (engine=pyarrow, optimized row groups)"
+            else:
+                engine_info = " (engine=rust by default)"
+        
         partition_info = f" partitioned by {self._partition_by}" if self._partition_by else ""
         print(f"Writing to Delta table: {schema}.{table} (mode={self._mode}){engine_info}{partition_info}")
         

{duckrun-0.2.9.dev0 → duckrun-0.2.9.dev1}/duckrun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.9.dev0
+Version: 0.2.9.dev1
 Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
 Author: mim
 License: MIT

{duckrun-0.2.9.dev0 → duckrun-0.2.9.dev1}/duckrun.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ duckrun/core.py
 duckrun/files.py
 duckrun/lakehouse.py
 duckrun/runner.py
+duckrun/semantic_model.py
 duckrun/stats.py
 duckrun/writer.py
 duckrun.egg-info/PKG-INFO

{duckrun-0.2.9.dev0 → duckrun-0.2.9.dev1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "duckrun"
-version = "0.2.9.dev0"
+version = "0.2.9.dev1"
 description = "Lakehouse task runner powered by DuckDB for Microsoft Fabric"
 readme = "README.md"
 license = {text = "MIT"}