duckrun 0.2.4__py3-none-any.whl → 0.2.5.dev1__py3-none-any.whl

duckrun/__init__.py

@@ -4,7 +4,7 @@ from duckrun.core import Duckrun
 
 __version__ = "0.1.0"
 
-# Expose connect at module level for: import duckrun as dr
+# Expose unified connect method at module level
 connect = Duckrun.connect
 
 __all__ = ["Duckrun", "connect"]

duckrun/core.py

@@ -49,17 +49,27 @@ class Duckrun:
         ]
     """
 
-    def __init__(self, workspace: str, lakehouse_name: str, schema: str = "dbo", 
+    def __init__(self, workspace_id: str, lakehouse_id: str, schema: str = "dbo", 
                  sql_folder: Optional[str] = None, compaction_threshold: int = 10,
                  scan_all_schemas: bool = False, storage_account: str = "onelake"):
-        self.workspace = workspace
-        self.lakehouse_name = lakehouse_name
+        # Store GUIDs for internal use
+        self.workspace_id = workspace_id
+        self.lakehouse_id = lakehouse_id
         self.schema = schema
         self.sql_folder = sql_folder.strip() if sql_folder else None
         self.compaction_threshold = compaction_threshold
         self.scan_all_schemas = scan_all_schemas
         self.storage_account = storage_account
-        self.table_base_url = f'abfss://{workspace}@{storage_account}.dfs.fabric.microsoft.com/{lakehouse_name}.Lakehouse/Tables/'
+        
+        # Construct proper ABFSS URLs using GUIDs
+        # Both Tables and Files use lakehouse GUID directly (no .Lakehouse suffix)
+        self.table_base_url = f'abfss://{workspace_id}@{storage_account}.dfs.fabric.microsoft.com/{lakehouse_id}/Tables/'
+        self.files_base_url = f'abfss://{workspace_id}@{storage_account}.dfs.fabric.microsoft.com/{lakehouse_id}/Files/'
+        
+        # Keep legacy properties for backward compatibility
+        self.workspace = workspace_id  
+        self.lakehouse_name = lakehouse_id
+        
         self.con = duckdb.connect()
         self.con.sql("SET preserve_insertion_order = false")
         self._attach_lakehouse()
@@ -68,57 +78,218 @@ class Duckrun:
     def connect(cls, connection_string: str, sql_folder: Optional[str] = None,
                 compaction_threshold: int = 100, storage_account: str = "onelake"):
         """
-        Create and connect to lakehouse.
+        Create and connect to lakehouse or workspace.
         
-        Uses compact format: connect("ws/lh.lakehouse/schema") or connect("ws/lh.lakehouse")
+        Smart detection based on connection string format:
+        - "workspace" → workspace management only
+        - "ws/lh.lakehouse/schema" → full lakehouse connection  
+        - "ws/lh.lakehouse" → lakehouse connection (defaults to dbo schema)
         
         Args:
-            connection_string: OneLake path "ws/lh.lakehouse/schema" or "ws/lh.lakehouse"
-            sql_folder: Optional path or URL to SQL files folder
+            connection_string: OneLake path or workspace name
+            sql_folder: Optional path or URL to SQL files folder  
             compaction_threshold: File count threshold for compaction
             storage_account: Storage account name (default: "onelake")
         
         Examples:
-            dr = Duckrun.connect("ws/lh.lakehouse/schema", sql_folder="./sql")
-            dr = Duckrun.connect("ws/lh.lakehouse/schema")  # no SQL folder
-            dr = Duckrun.connect("ws/lh.lakehouse")  # defaults to dbo schema
-            dr = Duckrun.connect("ws/lh.lakehouse", storage_account="xxx-onelake")  # custom storage
+            # Workspace management only (supports spaces in names)
+            ws = Duckrun.connect("My Workspace Name")
+            ws.list_lakehouses()
+            ws.create_lakehouse_if_not_exists("New Lakehouse")
+            
+            # Full lakehouse connections (supports spaces in names)
+            dr = Duckrun.connect("My Workspace/My Lakehouse.lakehouse/schema", sql_folder="./sql")
+            dr = Duckrun.connect("Data Workspace/Sales Data.lakehouse/analytics")  # spaces supported
+            dr = Duckrun.connect("My Workspace/My Lakehouse.lakehouse")  # defaults to dbo schema
+            dr = Duckrun.connect("workspace/lakehouse.lakehouse", storage_account="xxx-onelake")  # custom storage
+            
+        Note:
+            Internally resolves friendly names (with spaces) to GUIDs and constructs proper ABFSS URLs:
+            "My Workspace/My Lakehouse.lakehouse/schema" becomes
+            "abfss://workspace_guid@onelake.dfs.fabric.microsoft.com/lakehouse_guid/Tables/schema"
         """
+        
+        # Check if it's a workspace-only connection (no "/" means workspace name only)
+        if "/" not in connection_string:
+            print(f"Connecting to workspace '{connection_string}' for management operations...")
+            return WorkspaceConnection(connection_string)
+        
         print("Connecting to Lakehouse...")
         
         scan_all_schemas = False
         
-        # Only support compact format: "ws/lh.lakehouse/schema" or "ws/lh.lakehouse"
-        if not connection_string or "/" not in connection_string:
-            raise ValueError(
-                "Invalid connection string format. "
-                "Expected format: 'workspace/lakehouse.lakehouse/schema' or 'workspace/lakehouse.lakehouse'"
-            )
-        
+        # Parse lakehouse connection string: "ws/lh.lakehouse/schema" or "ws/lh.lakehouse"  
+        # Support workspace and lakehouse names with spaces
         parts = connection_string.split("/")
         if len(parts) == 2:
-            workspace, lakehouse_name = parts
+            workspace_name, lakehouse_name = parts
             scan_all_schemas = True
             schema = "dbo"
         elif len(parts) == 3:
-            workspace, lakehouse_name, schema = parts
+            workspace_name, lakehouse_name, schema = parts
         else:
             raise ValueError(
                 f"Invalid connection string format: '{connection_string}'. "
-                "Expected format: 'workspace/lakehouse.lakehouse' or 'workspace/lakehouse.lakehouse/schema'"
+                "Expected formats:\n"
+                "  'workspace name' (workspace management only)\n"
+                "  'workspace name/lakehouse name.lakehouse' (lakehouse with dbo schema)\n"
+                "  'workspace name/lakehouse name.lakehouse/schema' (lakehouse with specific schema)"
             )
         
         if lakehouse_name.endswith(".lakehouse"):
             lakehouse_name = lakehouse_name[:-10]
         
-        if not workspace or not lakehouse_name:
+        if not workspace_name or not lakehouse_name:
             raise ValueError(
-                "Missing required parameters. Use compact format:\n"
-                "  connect('workspace/lakehouse.lakehouse/schema', 'sql_folder')\n"
-                "  connect('workspace/lakehouse.lakehouse')  # defaults to dbo"
+                "Missing required parameters. Use one of these formats:\n"
+                "  connect('workspace name')  # workspace management\n"
+                "  connect('workspace name/lakehouse name.lakehouse/schema')  # full lakehouse\n"
+                "  connect('workspace name/lakehouse name.lakehouse')  # defaults to dbo"
             )
         
-        return cls(workspace, lakehouse_name, schema, sql_folder, compaction_threshold, scan_all_schemas, storage_account)
+        # Resolve friendly names to GUIDs and construct proper ABFSS path
+        workspace_id, lakehouse_id = cls._resolve_names_to_guids(workspace_name, lakehouse_name)
+        
+        return cls(workspace_id, lakehouse_id, schema, sql_folder, compaction_threshold, scan_all_schemas, storage_account)
+
+    @classmethod
+    def _resolve_names_to_guids(cls, workspace_name: str, lakehouse_name: str) -> tuple[str, str]:
+        """
+        Resolve friendly workspace and lakehouse names to their GUIDs.
+        
+        Optimization: If names don't contain spaces, use them directly (no API calls needed).
+        Only resolve to GUIDs when names contain spaces or are already GUIDs.
+        
+        Args:
+            workspace_name: Display name of the workspace (can contain spaces)
+            lakehouse_name: Display name of the lakehouse (can contain spaces)
+            
+        Returns:
+            Tuple of (workspace_id, lakehouse_id) - either resolved GUIDs or original names
+        """
+        
+        # Check if names are already GUIDs first
+        import re
+        guid_pattern = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', re.IGNORECASE)
+        
+        if guid_pattern.match(workspace_name) and guid_pattern.match(lakehouse_name):
+            print(f"✅ Names are already GUIDs: workspace={workspace_name}, lakehouse={lakehouse_name}")
+            return workspace_name, lakehouse_name
+        
+        # Optimization: If workspace name has no spaces, use both names directly (old behavior)
+        # Note: Lakehouse names cannot contain spaces in Microsoft Fabric, only workspace names can
+        if " " not in workspace_name:
+            print(f"✅ Using names directly (workspace has no spaces): workspace={workspace_name}, lakehouse={lakehouse_name}")
+            return workspace_name, lakehouse_name
+        
+        # Workspace name contains spaces - need to resolve both to GUIDs for proper ABFSS URLs
+        print(f"🔍 Resolving '{workspace_name}' workspace and '{lakehouse_name}' lakehouse to GUIDs (workspace has spaces)...")
+        
+        try:
+            # Get authentication token (try notebook environment first, then azure-identity)
+            try:
+                import notebookutils  # type: ignore
+                token = notebookutils.credentials.getToken("pbi")
+                current_workspace_id = notebookutils.runtime.context.get("workspaceId")
+            except ImportError:
+                current_workspace_id = None
+                # Fallback to azure-identity for external environments
+                from azure.identity import AzureCliCredential, InteractiveBrowserCredential, ChainedTokenCredential
+                credential = ChainedTokenCredential(AzureCliCredential(), InteractiveBrowserCredential())
+                token_obj = credential.get_token("https://api.fabric.microsoft.com/.default")
+                token = token_obj.token
+            
+            # Resolve workspace name to ID 
+            if current_workspace_id:
+                # In notebook environment, we could use current workspace ID
+                # but we should validate it matches the requested workspace name
+                workspace_id = cls._resolve_workspace_id_by_name(token, workspace_name)
+                if not workspace_id:
+                    # Fallback to current workspace if name resolution fails
+                    print(f"⚠️ Could not validate workspace name '{workspace_name}', using current workspace")
+                    workspace_id = current_workspace_id
+            else:
+                # External environment - must resolve by name
+                workspace_id = cls._resolve_workspace_id_by_name(token, workspace_name)
+                if not workspace_id:
+                    raise ValueError(f"Workspace '{workspace_name}' not found")
+            
+            # Resolve lakehouse name to ID (required for ABFSS URLs with spaces)
+            lakehouse_id = cls._resolve_lakehouse_id_by_name(token, workspace_id, lakehouse_name)
+            if not lakehouse_id:
+                raise ValueError(f"Lakehouse '{lakehouse_name}' not found in workspace '{workspace_name}'")
+            
+            print(f"✅ Resolved: {workspace_name} → {workspace_id}, {lakehouse_name} → {lakehouse_id}")
+            return workspace_id, lakehouse_id
+            
+        except Exception as e:
+            print(f"❌ Failed to resolve names to GUIDs: {e}")
+            print(f"❌ Cannot use friendly names with spaces '{workspace_name}'/'{lakehouse_name}' in ABFSS URLs without GUID resolution")
+            print("❌ Microsoft Fabric requires actual workspace and lakehouse GUIDs for ABFSS access when names contain spaces")
+            raise ValueError(
+                f"Unable to resolve workspace '{workspace_name}' and lakehouse '{lakehouse_name}' to GUIDs. "
+                f"ABFSS URLs require actual GUIDs when names contain spaces. "
+                f"Please ensure you have proper authentication and the workspace/lakehouse names are correct."
+            )
+
+    @classmethod  
+    def _resolve_workspace_id_by_name(cls, token: str, workspace_name: str) -> Optional[str]:
+        """Get workspace ID from display name"""
+        try:
+            import requests
+            url = "https://api.fabric.microsoft.com/v1/workspaces"
+            headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+            
+            response = requests.get(url, headers=headers)
+            response.raise_for_status()
+            
+            workspaces = response.json().get("value", [])
+            for workspace in workspaces:
+                if workspace.get("displayName") == workspace_name:
+                    return workspace.get("id")
+            
+            return None
+        except Exception:
+            return None
+    
+    @classmethod
+    def _resolve_lakehouse_id_by_name(cls, token: str, workspace_id: str, lakehouse_name: str) -> Optional[str]:
+        """Get lakehouse ID from display name within a workspace"""
+        try:
+            import requests
+            url = f"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/lakehouses"
+            headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+            
+            response = requests.get(url, headers=headers)
+            response.raise_for_status()
+            
+            lakehouses = response.json().get("value", [])
+            for lakehouse in lakehouses:
+                if lakehouse.get("displayName") == lakehouse_name:
+                    return lakehouse.get("id")
+            
+            return None
+        except Exception:
+            return None
+
+    @classmethod
+    def connect_workspace(cls, workspace_name: str):
+        """
+        Connect to a workspace without a specific lakehouse.
+        Used for lakehouse management operations.
+        
+        Args:
+            workspace_name: Name of the workspace
+            
+        Returns:
+            WorkspaceConnection object with lakehouse management methods
+            
+        Example:
+            con = duckrun.connect_workspace("MyWorkspace")
+            con.list_lakehouses()
+            con.create_lakehouse_if_not_exists("newlakehouse")
+        """
+        return WorkspaceConnection(workspace_name)
 
     def _get_storage_token(self):
         return os.environ.get("AZURE_STORAGE_TOKEN", "PLACEHOLDER_TOKEN_TOKEN_NOT_AVAILABLE")
@@ -155,7 +326,7 @@ class Duckrun:
         url = f"abfss://{self.workspace}@{self.storage_account}.dfs.fabric.microsoft.com/"
         store = AzureStore.from_url(url, bearer_token=token)
         
-        base_path = f"{self.lakehouse_name}.Lakehouse/Tables/"
+        base_path = f"{self.lakehouse_name}/Tables/"
         tables_found = []
         
         if self.scan_all_schemas:
@@ -198,9 +369,9 @@ class Duckrun:
             
             if not tables:
                 if self.scan_all_schemas:
-                    print(f"No Delta tables found in {self.lakehouse_name}.Lakehouse/Tables/")
+                    print(f"No Delta tables found in {self.lakehouse_name}/Tables/")
                 else:
-                    print(f"No Delta tables found in {self.lakehouse_name}.Lakehouse/Tables/{self.schema}/")
+                    print(f"No Delta tables found in {self.lakehouse_name}/Tables/{self.schema}/")
                 return
             
             # Group tables by schema for display
@@ -358,8 +529,268 @@ class Duckrun:
         """
         return _get_stats(self, source)
 
+    def list_lakehouses(self) -> List[str]:
+        """
+        List all lakehouses in the current workspace.
+        
+        Returns:
+            List of lakehouse names
+        """
+        try:
+            # Try to get token from notebook environment first
+            try:
+                import notebookutils  # type: ignore
+                token = notebookutils.credentials.getToken("pbi")
+                workspace_id = notebookutils.runtime.context.get("workspaceId")
+            except ImportError:
+                # Fallback to azure-identity
+                print("Getting authentication token...")
+                from azure.identity import AzureCliCredential, InteractiveBrowserCredential, ChainedTokenCredential
+                credential = ChainedTokenCredential(AzureCliCredential(), InteractiveBrowserCredential())
+                token_obj = credential.get_token("https://api.fabric.microsoft.com/.default")
+                token = token_obj.token
+                
+                # Get workspace ID by name
+                workspace_id = self._get_workspace_id_by_name(token, self.workspace)
+                if not workspace_id:
+                    print(f"Workspace '{self.workspace}' not found")
+                    return []
+            
+            # List lakehouses
+            url = f"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/lakehouses"
+            headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+            
+            response = requests.get(url, headers=headers)
+            response.raise_for_status()
+            
+            lakehouses = response.json().get("value", [])
+            lakehouse_names = [lh.get("displayName", "") for lh in lakehouses]
+            
+            print(f"Found {len(lakehouse_names)} lakehouses: {lakehouse_names}")
+            return lakehouse_names
+            
+        except Exception as e:
+            print(f"Error listing lakehouses: {e}")
+            return []
+
+    def create_lakehouse_if_not_exists(self, lakehouse_name: str) -> bool:
+        """
+        Create a lakehouse if it doesn't already exist.
+        
+        Args:
+            lakehouse_name: Name of the lakehouse to create
+            
+        Returns:
+            True if lakehouse exists or was created successfully, False otherwise
+        """
+        try:
+            # Try to get token from notebook environment first
+            try:
+                import notebookutils  # type: ignore
+                token = notebookutils.credentials.getToken("pbi")
+                workspace_id = notebookutils.runtime.context.get("workspaceId")
+            except ImportError:
+                # Fallback to azure-identity
+                print("Getting authentication token...")
+                from azure.identity import AzureCliCredential, InteractiveBrowserCredential, ChainedTokenCredential
+                credential = ChainedTokenCredential(AzureCliCredential(), InteractiveBrowserCredential())
+                token_obj = credential.get_token("https://api.fabric.microsoft.com/.default")
+                token = token_obj.token
+                
+                # Get workspace ID by name
+                workspace_id = self._get_workspace_id_by_name(token, self.workspace)
+                if not workspace_id:
+                    print(f"Workspace '{self.workspace}' not found")
+                    return False
+            
+            # Check if lakehouse already exists
+            url = f"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/lakehouses"
+            headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+            
+            response = requests.get(url, headers=headers)
+            response.raise_for_status()
+            
+            lakehouses = response.json().get("value", [])
+            existing_names = [lh.get("displayName", "") for lh in lakehouses]
+            
+            if lakehouse_name in existing_names:
+                print(f"Lakehouse '{lakehouse_name}' already exists")
+                return True
+            
+            # Create lakehouse
+            print(f"Creating lakehouse '{lakehouse_name}'...")
+            payload = {
+                "displayName": lakehouse_name,
+                "description": f"Lakehouse {lakehouse_name} created via duckrun"
+            }
+            
+            response = requests.post(url, headers=headers, json=payload)
+            response.raise_for_status()
+            
+            print(f"✅ Lakehouse '{lakehouse_name}' created successfully")
+            return True
+            
+        except Exception as e:
+            print(f"❌ Error creating lakehouse '{lakehouse_name}': {e}")
+            return False
+
+    def _get_workspace_id_by_name(self, token: str, workspace_name: str) -> Optional[str]:
+        """Helper method to get workspace ID from name"""
+        try:
+            url = "https://api.fabric.microsoft.com/v1/workspaces"
+            headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+            
+            response = requests.get(url, headers=headers)
+            response.raise_for_status()
+            
+            workspaces = response.json().get("value", [])
+            for workspace in workspaces:
+                if workspace.get("displayName") == workspace_name:
+                    return workspace.get("id")
+            
+            return None
+            
+        except Exception:
+            return None
+
     def close(self):
         """Close DuckDB connection"""
         if self.con:
             self.con.close()
-            print("Connection closed")
+            print("Connection closed")
+
+
+class WorkspaceConnection:
+    """
+    Simple workspace connection for lakehouse management operations.
+    """
+    
+    def __init__(self, workspace_name: str):
+        self.workspace_name = workspace_name
+    
+    def list_lakehouses(self) -> List[str]:
+        """
+        List all lakehouses in the workspace.
+        
+        Returns:
+            List of lakehouse names
+        """
+        try:
+            # Try to get token from notebook environment first
+            try:
+                import notebookutils  # type: ignore
+                token = notebookutils.credentials.getToken("pbi")
+                workspace_id = notebookutils.runtime.context.get("workspaceId")
+            except ImportError:
+                # Fallback to azure-identity
+                print("Getting authentication token...")
+                from azure.identity import AzureCliCredential, InteractiveBrowserCredential, ChainedTokenCredential
+                credential = ChainedTokenCredential(AzureCliCredential(), InteractiveBrowserCredential())
+                token_obj = credential.get_token("https://api.fabric.microsoft.com/.default")
+                token = token_obj.token
+                
+                # Get workspace ID by name
+                workspace_id = self._get_workspace_id_by_name(token, self.workspace_name)
+                if not workspace_id:
+                    print(f"Workspace '{self.workspace_name}' not found")
+                    return []
+            
+            # List lakehouses
+            url = f"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/lakehouses"
+            headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+            
+            response = requests.get(url, headers=headers)
+            response.raise_for_status()
+            
+            lakehouses = response.json().get("value", [])
+            lakehouse_names = [lh.get("displayName", "") for lh in lakehouses]
+            
+            print(f"Found {len(lakehouse_names)} lakehouses: {lakehouse_names}")
+            return lakehouse_names
+            
+        except Exception as e:
+            print(f"Error listing lakehouses: {e}")
+            return []
+
+    def create_lakehouse_if_not_exists(self, lakehouse_name: str) -> bool:
+        """
+        Create a lakehouse if it doesn't already exist.
+        
+        Args:
+            lakehouse_name: Name of the lakehouse to create
+            
+        Returns:
+            True if lakehouse exists or was created successfully, False otherwise
+        """
+        try:
+            # Try to get token from notebook environment first
+            try:
+                import notebookutils  # type: ignore
+                token = notebookutils.credentials.getToken("pbi")
+                workspace_id = notebookutils.runtime.context.get("workspaceId")
+            except ImportError:
+                # Fallback to azure-identity
+                print("Getting authentication token...")
+                from azure.identity import AzureCliCredential, InteractiveBrowserCredential, ChainedTokenCredential
+                credential = ChainedTokenCredential(AzureCliCredential(), InteractiveBrowserCredential())
+                token_obj = credential.get_token("https://api.fabric.microsoft.com/.default")
+                token = token_obj.token
+                
+                # Get workspace ID by name
+                workspace_id = self._get_workspace_id_by_name(token, self.workspace_name)
+                if not workspace_id:
+                    print(f"Workspace '{self.workspace_name}' not found")
+                    return False
+            
+            # Check if lakehouse already exists
+            url = f"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/lakehouses"
+            headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+            
+            response = requests.get(url, headers=headers)
+            response.raise_for_status()
+            
+            lakehouses = response.json().get("value", [])
+            existing_names = [lh.get("displayName", "") for lh in lakehouses]
+            
+            if lakehouse_name in existing_names:
+                print(f"Lakehouse '{lakehouse_name}' already exists")
+                return True
+            
+            # Create lakehouse
+            print(f"Creating lakehouse '{lakehouse_name}'...")
+            payload = {
+                "displayName": lakehouse_name,
+                "description": f"Lakehouse {lakehouse_name} created via duckrun",
+                "creationPayload": {
+                    "enableSchemas": True
+                }
+            }
+            
+            response = requests.post(url, headers=headers, json=payload)
+            response.raise_for_status()
+            
+            print(f"✅ Lakehouse '{lakehouse_name}' created successfully")
+            return True
+            
+        except Exception as e:
+            print(f"❌ Error creating lakehouse '{lakehouse_name}': {e}")
+            return False
+    
+    def _get_workspace_id_by_name(self, token: str, workspace_name: str) -> Optional[str]:
+        """Helper method to get workspace ID from name"""
+        try:
+            url = "https://api.fabric.microsoft.com/v1/workspaces"
+            headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+            
+            response = requests.get(url, headers=headers)
+            response.raise_for_status()
+            
+            workspaces = response.json().get("value", [])
+            for workspace in workspaces:
+                if workspace.get("displayName") == workspace_name:
+                    return workspace.get("id")
+            
+            return None
+            
+        except Exception:
+            return None

duckrun/files.py

@@ -51,8 +51,8 @@ def copy(duckrun_instance, local_folder: str, remote_folder: str,
         token = token_obj.token
         os.environ["AZURE_STORAGE_TOKEN"] = token
     
-    # Setup OneLake Files URL (not Tables)
-    files_base_url = f'abfss://{duckrun_instance.workspace}@{duckrun_instance.storage_account}.dfs.fabric.microsoft.com/{duckrun_instance.lakehouse_name}.Lakehouse/Files/'
+    # Setup OneLake Files URL (use correct format without .Lakehouse suffix)
+    files_base_url = duckrun_instance.files_base_url
     store = AzureStore.from_url(files_base_url, bearer_token=token)
     
     # Collect files to upload
@@ -160,8 +160,8 @@ def download(duckrun_instance, remote_folder: str = "", local_folder: str = "./d
         token = token_obj.token
         os.environ["AZURE_STORAGE_TOKEN"] = token
     
-    # Setup OneLake Files URL (not Tables)
-    files_base_url = f'abfss://{duckrun_instance.workspace}@{duckrun_instance.storage_account}.dfs.fabric.microsoft.com/{duckrun_instance.lakehouse_name}.Lakehouse/Files/'
+    # Setup OneLake Files URL (use correct format without .Lakehouse suffix)
+    files_base_url = duckrun_instance.files_base_url
     store = AzureStore.from_url(files_base_url, bearer_token=token)
     
     # Create local directory

duckrun/lakehouse.py

@@ -0,0 +1,402 @@
+import requests
+import time
+from typing import Optional
+
+class FabricLakehouseManager:
+    """
+    Manage Microsoft Fabric Lakehouses using REST API only.
+    Works on any machine with Python and internet access.
+    """
+    
+    def __init__(self, access_token: str):
+        """
+        Initialize with Azure AD access token.
+        
+        Args:
+            access_token: Bearer token for Fabric API authentication
+        """
+        self.base_url = "https://api.fabric.microsoft.com/v1"
+        self.headers = {
+            "Authorization": f"Bearer {access_token}",
+            "Content-Type": "application/json"
+        }
+    
+    def get_workspace_id(self, workspace_name: str) -> Optional[str]:
+        """
+        Get workspace ID from workspace name.
+        
+        Args:
+            workspace_name: Name of the workspace
+            
+        Returns:
+            Workspace ID if found, None otherwise
+        """
+        if not workspace_name:
+            return None
+        
+        try:
+            url = f"{self.base_url}/workspaces"
+            response = requests.get(url, headers=self.headers)
+            response.raise_for_status()
+            
+            workspaces = response.json().get("value", [])
+            for workspace in workspaces:
+                if workspace.get("displayName") == workspace_name:
+                    return workspace.get("id")
+            
+            print(f"Workspace '{workspace_name}' not found")
+            return None
+            
+        except Exception as e:
+            print(f"Error getting workspace ID: {e}")
+            return None
+    
+    def get_lakehouse(self, lakehouse_name: str, workspace_id: str) -> Optional[dict]:
+        """
+        Get lakehouse details if it exists.
+        
+        Args:
+            lakehouse_name: Name of the lakehouse
+            workspace_id: ID of the workspace
+            
+        Returns:
+            Lakehouse details if found, None otherwise
+        """
+        try:
+            url = f"{self.base_url}/workspaces/{workspace_id}/lakehouses"
+            response = requests.get(url, headers=self.headers)
+            response.raise_for_status()
+            
+            lakehouses = response.json().get("value", [])
+            for lakehouse in lakehouses:
+                if lakehouse.get("displayName") == lakehouse_name:
+                    return lakehouse
+            
+            return None
+            
+        except Exception as e:
+            print(f"Error getting lakehouse: {e}")
+            return None
+    
+    def create_lakehouse(self, lakehouse_name: str, workspace_id: str, 
+                        enable_schemas: bool = True) -> Optional[dict]:
+        """
+        Create a new lakehouse.
+        
+        Args:
+            lakehouse_name: Name of the lakehouse
+            workspace_id: ID of the workspace
+            enable_schemas: Whether to enable schemas
+            
+        Returns:
+            Created lakehouse details if successful, None otherwise
+        """
+        try:
+            url = f"{self.base_url}/workspaces/{workspace_id}/lakehouses"
+            payload = {
+                "displayName": lakehouse_name,
+                "description": f"Lakehouse {lakehouse_name}"
+            }
+            
+            if enable_schemas:
+                payload["creationPayload"] = {
+                    "enableSchemas": True
+                }
+            
+            response = requests.post(url, headers=self.headers, json=payload)
+            response.raise_for_status()
+            
+            # Wait a bit for the lakehouse to be fully provisioned
+            time.sleep(2)
+            
+            return response.json()
+            
+        except Exception as e:
+            print(f"Error creating lakehouse: {e}")
+            if hasattr(e, 'response') and e.response is not None:
+                print(f"Response: {e.response.text}")
+            return None
+    
+    def create_lakehouse_if_not_exists(self, lakehouse_name: str, 
+                                       workspace_name: Optional[str] = None,
+                                       workspace_id: Optional[str] = None) -> int:
+        """
+        Create a lakehouse if it doesn't exist.
+        
+        Args:
+            lakehouse_name: Name of the lakehouse
+            workspace_name: Optional workspace name
+            workspace_id: Optional workspace ID (takes precedence over workspace_name)
+            
+        Returns:
+            1 if successful (lakehouse exists or was created)
+            0 if failed
+        """
+        # Resolve workspace ID
+        if workspace_id is None and workspace_name:
+            workspace_id = self.get_workspace_id(workspace_name)
+            if workspace_id is None:
+                print(f"Workspace '{workspace_name}' not found - returning 0")
+                return 0
+        elif workspace_id is None:
+            print("No workspace specified - returning 0")
+            return 0
+        
+        print(f"Attempting to get lakehouse '{lakehouse_name}' in workspace '{workspace_id}'")
+        
+        # Check if lakehouse exists
+        lakehouse = self.get_lakehouse(lakehouse_name, workspace_id)
+        
+        if lakehouse:
+            print(f"Lakehouse '{lakehouse_name}' found - returning 1")
+            return 1
+        
+        # Create lakehouse if it doesn't exist
+        print(f"Lakehouse not found, attempting to create...")
+        created = self.create_lakehouse(lakehouse_name, workspace_id)
+        
+        if created:
+            # Verify creation
+            lakehouse = self.get_lakehouse(lakehouse_name, workspace_id)
+            if lakehouse:
+                print(f"Lakehouse '{lakehouse_name}' created successfully - returning 1")
+                return 1
+        
+        print(f"Failed to create lakehouse '{lakehouse_name}' - returning 0")
+        return 0
+
+
+# Example usage with Azure Identity:
+def main():
+    """
+    Example of how to use the FabricLakehouseManager with azure-identity.
+    """
+    from azure.identity import AzureCliCredential, InteractiveBrowserCredential, ChainedTokenCredential
+    
+    print("Authenticating with Azure (trying CLI, will fallback to browser if needed)...")
+    
+    # Create credential chain (CLI first, then interactive browser)
+    credential = ChainedTokenCredential(
+        AzureCliCredential(),
+        InteractiveBrowserCredential()
+    )
+    
+    # Get token for Fabric API (not storage!)
+    # Note: Use Fabric API scope, not storage scope
+    token = credential.get_token("https://api.fabric.microsoft.com/.default")
+    
+    print("✓ Authentication successful!")
+    
+    # Initialize manager with Fabric token
+    manager = FabricLakehouseManager(token.token)
+    
+    # Create lakehouse if not exists
+    result = manager.create_lakehouse_if_not_exists(
+        lakehouse_name="MyLakehouse",
+        workspace_name="MyWorkspace"
+    )
+    
+    if result == 1:
+        print("✓ Lakehouse operation successful!")
+    else:
+        print("✗ Lakehouse operation failed!")
+    
+    return result
+
+
+def get_fabric_token():
+    """
+    Helper function to get Fabric API token.
+    Returns the token string.
+    """
+    from azure.identity import AzureCliCredential, InteractiveBrowserCredential, ChainedTokenCredential
+    
+    credential = ChainedTokenCredential(
+        AzureCliCredential(),
+        InteractiveBrowserCredential()
+    )
+    
+    # Get token for Fabric API
+    token = credential.get_token("https://api.fabric.microsoft.com/.default")
+    return token.token
+
+
+def create_lakehouse_in_notebook(lakehouse_name: str, workspace_name: Optional[str] = None) -> int:
+    """
+    Create a lakehouse in a Fabric notebook environment.
+    This function uses the notebook's built-in authentication.
+    
+    Args:
+        lakehouse_name: Name of the lakehouse to create
+        workspace_name: Optional workspace name (uses current workspace if None)
+        
+    Returns:
+        1 if successful (lakehouse exists or was created)
+        0 if failed
+    """
+    try:
+        # Try to import fabric notebook utilities (only available in Fabric notebooks)
+        import notebookutils  # type: ignore
+        
+        # Get authentication token from notebook environment
+        token = notebookutils.credentials.getToken("https://api.fabric.microsoft.com/.default")
+        
+        # Initialize manager with notebook token
+        manager = FabricLakehouseManager(token)
+        
+        # Get current workspace ID if no workspace specified
+        workspace_id = None
+        if workspace_name:
+            workspace_id = manager.get_workspace_id(workspace_name)
+        else:
+            # In Fabric notebooks, we can get the current workspace from context
+            try:
+                workspace_id = notebookutils.runtime.context.get("workspaceId")
+            except:
+                print("Could not get current workspace ID from notebook context")
+                return 0
+        
+        if not workspace_id:
+            print(f"Could not resolve workspace ID")
+            return 0
+        
+        # Create lakehouse if not exists
+        return manager.create_lakehouse_if_not_exists(
+            lakehouse_name=lakehouse_name,
+            workspace_id=workspace_id
+        )
+        
+    except ImportError:
+        print("notebookutils not available - not running in Fabric notebook environment")
+        print("Use FabricLakehouseManager class directly with proper authentication")
+        return 0
+    except Exception as e:
+        print(f"Error creating lakehouse in notebook: {e}")
+        return 0
+
+
+def create_lakehouse_simple(lakehouse_name: str, access_token: str, workspace_id: str) -> dict:
+    """
+    Simple function to create a lakehouse with minimal dependencies.
+    Perfect for Fabric notebook environments.
+    
+    Args:
+        lakehouse_name: Name of the lakehouse to create
+        access_token: Bearer token for authentication
+        workspace_id: ID of the target workspace
+        
+    Returns:
+        Dictionary with creation result
+    """
+    import requests
+    import time
+    
+    base_url = "https://api.fabric.microsoft.com/v1"
+    headers = {
+        "Authorization": f"Bearer {access_token}",
+        "Content-Type": "application/json"
+    }
+    
+    try:
+        # First check if lakehouse already exists
+        list_url = f"{base_url}/workspaces/{workspace_id}/lakehouses"
+        response = requests.get(list_url, headers=headers)
+        response.raise_for_status()
+        
+        lakehouses = response.json().get("value", [])
+        for lakehouse in lakehouses:
+            if lakehouse.get("displayName") == lakehouse_name:
+                return {
+                    "success": True,
+                    "message": f"Lakehouse '{lakehouse_name}' already exists",
+                    "lakehouse": lakehouse,
+                    "created": False
+                }
+        
+        # Create new lakehouse
+        create_url = f"{base_url}/workspaces/{workspace_id}/lakehouses"
+        payload = {
+            "displayName": lakehouse_name,
+            "description": f"Lakehouse {lakehouse_name} created via API"
+        }
+        
+        response = requests.post(create_url, headers=headers, json=payload)
+        response.raise_for_status()
+        
+        # Wait for provisioning
+        time.sleep(3)
+        
+        created_lakehouse = response.json()
+        return {
+            "success": True,
+            "message": f"Lakehouse '{lakehouse_name}' created successfully",
+            "lakehouse": created_lakehouse,
+            "created": True
+        }
+        
+    except requests.exceptions.RequestException as e:
+        error_msg = f"HTTP error creating lakehouse: {e}"
+        if hasattr(e, 'response') and e.response is not None:
+            error_msg += f" Response: {e.response.text}"
+        
+        return {
+            "success": False,
+            "message": error_msg,
+            "lakehouse": None,
+            "created": False
+        }
+    except Exception as e:
+        return {
+            "success": False,
+            "message": f"Unexpected error: {e}",
+            "lakehouse": None,
+            "created": False
+        }
+
+
+if __name__ == "__main__":
+    # Uncomment to run the example
+    # main()
+    pass
+
+
+# Usage Examples:
+"""
+# Example 1: In a Fabric Notebook (simplest approach)
+from duckrun.lakehouse import create_lakehouse_in_notebook
+
+result = create_lakehouse_in_notebook("MyNewLakehouse")
+if result == 1:
+    print("Lakehouse created or already exists!")
+
+# Example 2: In a Fabric Notebook with explicit token
+import notebookutils
+from duckrun.lakehouse import create_lakehouse_simple
+
+token = notebookutils.credentials.getToken("https://api.fabric.microsoft.com/.default")
+workspace_id = notebookutils.runtime.context.get("workspaceId")
+
+result = create_lakehouse_simple("MyLakehouse", token, workspace_id)
+print(f"Result: {result['message']}")
+
+# Example 3: Outside Fabric (requires azure-identity package)
+from duckrun.lakehouse import FabricLakehouseManager, get_fabric_token
+
+token = get_fabric_token()
+manager = FabricLakehouseManager(token)
+result = manager.create_lakehouse_if_not_exists("MyLakehouse", workspace_name="MyWorkspace")
+
+# Example 4: With explicit workspace and lakehouse details
+from duckrun.lakehouse import FabricLakehouseManager
+
+# Get your token however you prefer
+token = "your_bearer_token_here"
+manager = FabricLakehouseManager(token)
+
+# Create lakehouse in specific workspace
+workspace_id = manager.get_workspace_id("Production Workspace")
+lakehouse = manager.create_lakehouse("DataLake2024", workspace_id, enable_schemas=True)
+
+if lakehouse:
+    print(f"Created lakehouse with ID: {lakehouse['id']}")
+"""

duckrun/runner.py

@@ -235,11 +235,28 @@ def _read_sql_file(duckrun_instance, table_name: str, params: Optional[Dict] = N
         print(f"SQL file is empty: {table_name}.sql")
         return None
 
+    import re
+    # Determine if lakehouse_name is a GUID
+    guid_pattern = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', re.IGNORECASE)
+    lakehouse_is_guid = bool(guid_pattern.match(duckrun_instance.lakehouse_name))
+
+    # Smart substitution for ${lh}.Lakehouse
+    # If template contains ${lh}.Lakehouse, replace with correct value
+    if '${lh}.Lakehouse' in content:
+        if lakehouse_is_guid:
+            # If GUID, use just the GUID
+            content = content.replace('${lh}.Lakehouse', duckrun_instance.lakehouse_name)
+        else:
+            # If not GUID, use legacy format
+            content = content.replace('${lh}.Lakehouse', f'{duckrun_instance.lakehouse_name}.Lakehouse')
+
     full_params = {
         'ws': duckrun_instance.workspace,
         'lh': duckrun_instance.lakehouse_name,
         'schema': duckrun_instance.schema,
-        'storage_account': duckrun_instance.storage_account
+        'storage_account': duckrun_instance.storage_account,
+        'tables_url': duckrun_instance.table_base_url,
+        'files_url': duckrun_instance.files_base_url
     }
     if params:
         full_params.update(params)
@@ -247,6 +264,10 @@ def _read_sql_file(duckrun_instance, table_name: str, params: Optional[Dict] = N
     try:
         template = Template(content)
         content = template.substitute(full_params)
+        # After substitution, remove .Lakehouse if it follows a GUID in any ABFSS URL
+        import re
+        # Pattern: GUID.Lakehouse or GUID.lakehouse (in URLs)
+        content = re.sub(r'([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.(Lakehouse|lakehouse)', r'\1', content)
     except KeyError as e:
         print(f"Missing parameter in SQL file: ${e}")
         return None

duckrun/stats.py

@@ -142,8 +142,8 @@ def get_stats(duckrun_instance, source: str):
     print(f"Processing {len(list_tables)} tables: {list_tables}")
     
     for idx, tbl in enumerate(list_tables):
-        # Construct lakehouse path using ABFSS URL
-        table_path = f"abfss://{duckrun_instance.workspace}@{duckrun_instance.storage_account}.dfs.fabric.microsoft.com/{duckrun_instance.lakehouse_name}.Lakehouse/Tables/{schema_name}/{tbl}"
+        # Construct lakehouse path using correct ABFSS URL format (no .Lakehouse suffix)
+        table_path = f"{duckrun_instance.table_base_url}{schema_name}/{tbl}"
         
         try:
             dt = DeltaTable(table_path)

{duckrun-0.2.4.dist-info → duckrun-0.2.5.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.4
+Version: 0.2.5.dev1
 Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
 Author: mim
 License: MIT

duckrun-0.2.5.dev1.dist-info/RECORD

@@ -0,0 +1,12 @@
+duckrun/__init__.py,sha256=XA85pL2vK1AkmBic8e7WxeqNvcd6SjFX4zsQpImDO6E,230
+duckrun/core.py,sha256=-UAsAOlOmVkVuQCkbCWTH7aiFSHP0OenAAWSl0i0inY,37107
+duckrun/files.py,sha256=piWRU5w9jHrW-wuV4Gf-SKY_jhFv9eflxgWO8AZCQTI,10495
+duckrun/lakehouse.py,sha256=j--Z3zo8AOWt1GF9VzRosmmTAy6ey2D0LVubti58twU,14109
+duckrun/runner.py,sha256=lfwNoU1CZXh6bPTHvGWVaUWjzG5crvT7Pzq4onMEVjw,12576
+duckrun/stats.py,sha256=2FTqoQNVjD84-H1HjStHxZkOpAGKXS79M55B00pOlok,9804
+duckrun/writer.py,sha256=eWrGtDQTbXi8H3sSt2WucYTdEQUjK97KmQxzCbqAuMs,6221
+duckrun-0.2.5.dev1.dist-info/licenses/LICENSE,sha256=-DeQQwdbCbkB4507ZF3QbocysB-EIjDtaLexvqRkGZc,1083
+duckrun-0.2.5.dev1.dist-info/METADATA,sha256=ZQfd7I-J08MC5ytjKt789PqHdroHI0ld36aMNp-57yE,18344
+duckrun-0.2.5.dev1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+duckrun-0.2.5.dev1.dist-info/top_level.txt,sha256=BknMEwebbUHrVAp3SC92ps8MPhK7XSYsaogTvi_DmEU,8
+duckrun-0.2.5.dev1.dist-info/RECORD,,

duckrun-0.2.4.dist-info/RECORD

@@ -1,11 +0,0 @@
-duckrun/__init__.py,sha256=L0jRtD9Ld8Ti4e6GRvPDdHvkQCFAPHM43GSP7ARh6EM,241
-duckrun/core.py,sha256=m_9DuSZNZ5DOETnkjNGn8HJBYheCgs_7NewcbM9VECI,16500
-duckrun/files.py,sha256=xba0juMEQPgaznDudmXcwaGH0wv-6aCoHmV_cNF6Y7I,10665
-duckrun/runner.py,sha256=X5g-57OCHQZ7USKpcBbhYGUcZwLQny2x147DLKrV32c,11417
-duckrun/stats.py,sha256=B9UfGOndRNfcB2AhOVjuSqgfmF2x-uRmdmBn3usx_jQ,9881
-duckrun/writer.py,sha256=eWrGtDQTbXi8H3sSt2WucYTdEQUjK97KmQxzCbqAuMs,6221
-duckrun-0.2.4.dist-info/licenses/LICENSE,sha256=-DeQQwdbCbkB4507ZF3QbocysB-EIjDtaLexvqRkGZc,1083
-duckrun-0.2.4.dist-info/METADATA,sha256=2t7-pNzcPCeseXTjp6Bc18_V41MpjDarG0z-2IzY-Lk,18339
-duckrun-0.2.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-duckrun-0.2.4.dist-info/top_level.txt,sha256=BknMEwebbUHrVAp3SC92ps8MPhK7XSYsaogTvi_DmEU,8
-duckrun-0.2.4.dist-info/RECORD,,