duckrun 0.1.5__py3-none-any.whl → 0.1.5.2__py3-none-any.whl

duckrun/core.py

@@ -13,7 +13,7 @@ class DeltaWriter:
     def __init__(self, relation, duckrun_instance):
         self.relation = relation
         self.duckrun = duckrun_instance
-        self._format = None
+        self._format = "delta"  # Default to delta format
         self._mode = "overwrite"
     
     def format(self, format_type: str):
@@ -32,8 +32,9 @@ class DeltaWriter:
     
     def saveAsTable(self, table_name: str):
         """Save query result as Delta table"""
+        # Format defaults to "delta", so no need to check
         if self._format != "delta":
-            raise RuntimeError("Must call .format('delta') before saveAsTable()")
+            raise RuntimeError(f"Only 'delta' format is supported, got '{self._format}'")
         
         # Parse schema.table or use default schema
         if "." in table_name:
@@ -116,7 +117,7 @@ class Duckrun:
         # For data exploration with Spark-style API:
         dr = Duckrun.connect(workspace, lakehouse, schema)
         dr.sql("SELECT * FROM table").show()
-        dr.sql("SELECT 43").write.format("delta").mode("append").saveAsTable("aemo.test")
+        dr.sql("SELECT 43").write.mode("append").saveAsTable("test")
     """
 
     def __init__(self, workspace: str, lakehouse_name: str, schema: str, 
@@ -413,9 +414,9 @@ class Duckrun:
             dr.sql("SELECT * FROM table").show()
             df = dr.sql("SELECT * FROM table").df()
             
-            # New Spark-style write API
-            dr.sql("SELECT 43 as value").write.format("delta").mode("append").saveAsTable("aemo.test")
-            dr.sql("SELECT * FROM source").write.format("delta").mode("overwrite").saveAsTable("target")
+            # New Spark-style write API (format is optional, defaults to delta)
+            dr.sql("SELECT 43 as value").write.mode("append").saveAsTable("test")
+            dr.sql("SELECT * FROM source").write.mode("overwrite").saveAsTable("target")
         """
         relation = self.con.sql(query)
         return QueryResult(relation, self)

duckrun-0.1.5.2.dist-info/METADATA

@@ -0,0 +1,303 @@
+Metadata-Version: 2.4
+Name: duckrun
+Version: 0.1.5.2
+Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
+Author: mim
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/djouallah/duckrun
+Project-URL: Repository, https://github.com/djouallah/duckrun
+Project-URL: Issues, https://github.com/djouallah/duckrun/issues
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: duckdb>=1.2.0
+Requires-Dist: deltalake>=0.18.2
+Requires-Dist: requests>=2.28.0
+Dynamic: license-file
+
+<img src="duckrun.png" width="400" alt="Duckrun">
+
+Simple task runner for Microsoft Fabric Python notebooks, powered by DuckDB and Delta Lake.
+
+## Important Notes
+
+**Requirements:**
+- Lakehouse must have a schema (e.g., `dbo`, `sales`, `analytics`)
+- Workspace and lakehouse names cannot contain spaces
+
+**Why no spaces?** Duckrun uses simple name-based paths instead of GUIDs. This keeps the code clean and readable, which is perfect for data engineering workspaces where naming conventions are already well-established. Just use underscores or hyphens instead: `my_workspace` or `my-lakehouse`.
+
+## Installation
+
+```bash
+pip install duckrun
+```
+
+## Quick Start
+
+```python
+import duckrun
+
+# Connect to your Fabric lakehouse
+con = duckrun.connect(
+    workspace="my_workspace",
+    lakehouse_name="my_lakehouse", 
+    schema="dbo"
+)
+
+# Explore data
+con.sql("SELECT * FROM my_table LIMIT 10").show()
+
+# Write to Delta tables (Spark-style API)
+con.sql("SELECT * FROM source").write.mode("overwrite").saveAsTable("target")
+```
+
+That's it! No `sql_folder` needed for data exploration.
+
+## Two Ways to Use Duckrun
+
+### 1. Data Exploration (Spark-Style API)
+
+Perfect for ad-hoc analysis and interactive notebooks:
+
+```python
+con = duckrun.connect("workspace", "lakehouse", "dbo")
+
+# Query existing tables
+con.sql("SELECT * FROM sales WHERE year = 2024").show()
+
+# Get DataFrame
+df = con.sql("SELECT COUNT(*) FROM orders").df()
+
+# Write results to Delta tables
+con.sql("""
+    SELECT 
+        customer_id,
+        SUM(amount) as total
+    FROM orders
+    GROUP BY customer_id
+""").write.mode("overwrite").saveAsTable("customer_totals")
+
+# Append mode
+con.sql("SELECT * FROM new_orders").write.mode("append").saveAsTable("orders")
+```
+
+**Note:** `.format("delta")` is optional - Delta is the default format!
+
+### 2. Pipeline Orchestration
+
+For production workflows with reusable SQL and Python tasks:
+
+```python
+con = duckrun.connect(
+    workspace="my_workspace",
+    lakehouse_name="my_lakehouse", 
+    schema="dbo",
+    sql_folder="./sql"  # folder with .sql and .py files
+)
+
+# Define pipeline
+pipeline = [
+    ('download_data', (url, path)),    # Python task
+    ('clean_data', 'overwrite'),       # SQL task  
+    ('aggregate', 'append')            # SQL task
+]
+
+# Run it
+con.run(pipeline)
+```
+
+## Pipeline Tasks
+
+### Python Tasks
+
+**Format:** `('function_name', (arg1, arg2, ...))`
+
+Create `sql_folder/function_name.py`:
+
+```python
+# sql_folder/download_data.py
+def download_data(url, path):
+    # your code here
+    return 1  # 1 = success, 0 = failure
+```
+
+### SQL Tasks
+
+**Format:** `('table_name', 'mode')` or `('table_name', 'mode', {params})`
+
+Create `sql_folder/table_name.sql`:
+
+```sql
+-- sql_folder/clean_data.sql
+SELECT 
+    id,
+    TRIM(name) as name,
+    date
+FROM raw_data
+WHERE date >= '2024-01-01'
+```
+
+**Write Modes:**
+- `overwrite` - Replace table completely
+- `append` - Add to existing table  
+- `ignore` - Create only if doesn't exist
+
+### Parameterized SQL
+
+Built-in parameters (always available):
+- `$ws` - workspace name
+- `$lh` - lakehouse name
+- `$schema` - schema name
+
+Custom parameters:
+
+```python
+pipeline = [
+    ('sales', 'append', {'start_date': '2024-01-01', 'end_date': '2024-12-31'})
+]
+```
+
+```sql
+-- sql_folder/sales.sql
+SELECT * FROM transactions
+WHERE date BETWEEN '$start_date' AND '$end_date'
+```
+
+## Advanced Features
+
+### Table Name Variants
+
+Use `__` to create multiple versions of the same table:
+
+```python
+pipeline = [
+    ('sales__initial', 'overwrite'),     # writes to 'sales'
+    ('sales__incremental', 'append'),    # appends to 'sales'
+]
+```
+
+Both tasks write to the `sales` table but use different SQL files (`sales__initial.sql` and `sales__incremental.sql`).
+
+### Remote SQL Files
+
+Load tasks from GitHub or any URL:
+
+```python
+con = duckrun.connect(
+    workspace="Analytics",
+    lakehouse_name="Sales", 
+    schema="dbo",
+    sql_folder="https://raw.githubusercontent.com/user/repo/main/sql"
+)
+```
+
+### Early Exit on Failure
+
+**Pipelines automatically stop when any task fails** - subsequent tasks won't run.
+
+For **SQL tasks**, failure is automatic:
+- If the query has a syntax error or runtime error, the task fails
+- The pipeline stops immediately
+
+For **Python tasks**, you control success/failure by returning:
+- `1` = Success → pipeline continues to next task
+- `0` = Failure → pipeline stops, remaining tasks are skipped
+
+Example:
+
+```python
+# sql_folder/download_data.py
+def download_data(url, path):
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        # save data...
+        return 1  # Success - pipeline continues
+    except Exception as e:
+        print(f"Download failed: {e}")
+        return 0  # Failure - pipeline stops here
+```
+
+```python
+pipeline = [
+    ('download_data', (url, path)),     # If returns 0, stops here
+    ('clean_data', 'overwrite'),        # Won't run if download failed
+    ('aggregate', 'append')             # Won't run if download failed
+]
+
+success = con.run(pipeline)  # Returns True only if ALL tasks succeed
+```
+
+This prevents downstream tasks from processing incomplete or corrupted data.
+
+### Delta Lake Optimization
+
+Duckrun automatically:
+- Compacts small files when file count exceeds threshold (default: 100)
+- Vacuums old versions on overwrite
+- Cleans up metadata
+
+Customize compaction threshold:
+
+```python
+con = duckrun.connect(
+    workspace="workspace",
+    lakehouse_name="lakehouse",
+    schema="dbo",
+    compaction_threshold=50  # compact after 50 files
+)
+```
+
+## Complete Example
+
+```python
+import duckrun
+
+# Connect
+con = duckrun.connect("Analytics", "Sales", "dbo", "./sql")
+
+# Pipeline with mixed tasks
+pipeline = [
+    # Download raw data (Python)
+    ('fetch_api_data', ('https://api.example.com/sales', 'raw')),
+    
+    # Clean and transform (SQL)
+    ('clean_sales', 'overwrite'),
+    
+    # Aggregate by region (SQL with params)
+    ('regional_summary', 'overwrite', {'min_amount': 1000}),
+    
+    # Append to history (SQL)
+    ('sales_history', 'append')
+]
+
+# Run
+success = con.run(pipeline)
+
+# Explore results
+con.sql("SELECT * FROM regional_summary").show()
+
+# Export to new table
+con.sql("""
+    SELECT region, SUM(total) as grand_total
+    FROM regional_summary
+    GROUP BY region
+""").write.mode("overwrite").saveAsTable("region_totals")
+```
+
+## How It Works
+
+1. **Connection**: Duckrun connects to your Fabric lakehouse using OneLake and Azure authentication
+2. **Table Discovery**: Automatically scans for Delta tables in your schema and creates DuckDB views
+3. **Query Execution**: Run SQL queries directly against Delta tables using DuckDB's speed
+4. **Write Operations**: Results are written back as Delta tables with automatic optimization
+5. **Pipelines**: Orchestrate complex workflows with reusable SQL and Python tasks
+
+## Real-World Example
+
+For a complete production example, see [fabric_demo](https://github.com/djouallah/fabric_demo).
+
+## License
+
+MIT

duckrun-0.1.5.2.dist-info/RECORD

@@ -0,0 +1,7 @@
+duckrun/__init__.py,sha256=L0jRtD9Ld8Ti4e6GRvPDdHvkQCFAPHM43GSP7ARh6EM,241
+duckrun/core.py,sha256=EoXlQsx7i3BS2a26zB90n4xDBy_WQu1sNicPNYU3DgY,18110
+duckrun-0.1.5.2.dist-info/licenses/LICENSE,sha256=-DeQQwdbCbkB4507ZF3QbocysB-EIjDtaLexvqRkGZc,1083
+duckrun-0.1.5.2.dist-info/METADATA,sha256=UWtYkoj5E5CsqHKUiJOOVJTQywdTsyzoJpnbiAd28cI,7792
+duckrun-0.1.5.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+duckrun-0.1.5.2.dist-info/top_level.txt,sha256=BknMEwebbUHrVAp3SC92ps8MPhK7XSYsaogTvi_DmEU,8
+duckrun-0.1.5.2.dist-info/RECORD,,

duckrun-0.1.5.dist-info/METADATA

@@ -1,172 +0,0 @@
-Metadata-Version: 2.4
-Name: duckrun
-Version: 0.1.5
-Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
-License-Expression: MIT
-Project-URL: Homepage, https://github.com/djouallah/duckrun
-Project-URL: Repository, https://github.com/djouallah/duckrun
-Project-URL: Issues, https://github.com/djouallah/duckrun/issues
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: duckdb>=1.2.0
-Requires-Dist: deltalake>=0.18.2
-Requires-Dist: requests>=2.28.0
-Dynamic: license-file
-
-
-<img src="duckrun.png" width="400" alt="Duckrun">
-
-Simple  task runner for Microsoft Fabric Python notebook, powered by DuckDB and Delta_rs.
-
-
-## Known Limitation
-
-Support only Lakehouse with schema, Workspace and lakehouse names should not contains space
-
-## Installation
-
-```bash
-pip install duckrun
-```
-
-
-
-## Quick Start
-
-```python
-import duckrun
-
-# Connect to your Fabric lakehouse (using `con` pattern)
-con = duckrun.connect(
-    workspace="my_workspace",
-    lakehouse_name="my_lakehouse", 
-    schema="dbo",
-    sql_folder="./sql"  # optional: folder containing your .sql and .py files (only needed for pipeline tasks)
-)
-
-# Define your pipeline
-pipeline = [
-    ('load_data', (url, path)),           # Python task
-    ('clean_data', 'overwrite'),          # SQL task  
-    ('aggregate', 'append')               # SQL task
-]
-
-# Run it
-con.run(pipeline)
-```
-
-Note: the `sql/` folder is optional — if all you want to do is explore data with SQL (for example by calling `con.sql(...)`), you don't need to provide a `sql_folder`.
-
-## Early Exit
-
-In a pipeline run, if a task fails, the pipeline will stop without running the subsequent tasks.
-
-## How It Works
-
-Duckrun runs two types of tasks:
-
-### 1. Python Tasks
-Format: `('function_name', (arg1, arg2, ...))`
-
-Create a file `sql_folder/function_name.py` with a function matching the name:
-
-```python
-# sql_folder/load_data.py
-def load_data(url, path):
-    # your code here
-    # IMPORTANT: Must return 1 for success, 0 for failure
-    return 1
-```
-
-### 2. SQL Tasks  
-Format: `('table_name', 'mode')` or `('table_name', 'mode', {params})`
-
-Create a file `sql_folder/table_name.sql`:
-
-```sql
--- sql_folder/clean_data.sql
-SELECT 
-    id,
-    TRIM(name) as name,
-    date
-FROM raw_data
-WHERE date >= '2024-01-01'
-```
-
-**Modes:**
-- `overwrite` - Replace table completely
-- `append` - Add to existing table
-- `ignore` - Create only if doesn't exist
-
-## Task Files
-
-The `sql_folder` can contain a mixture of both `.sql` and `.py` files. This allows you to combine SQL transformations and Python logic in your pipelines.
-
-### SQL Files
-Your SQL files automatically have access to:
-- `$ws` - workspace name
-- `$lh` - lakehouse name
-- `$schema` - schema name
-
-Pass custom parameters:
-
-```python
-pipeline = [
-    ('sales', 'append', {'start_date': '2024-01-01', 'end_date': '2024-12-31'})
-]
-```
-
-```sql
--- sql_folder/sales.sql
-SELECT * FROM transactions
-WHERE date BETWEEN '$start_date' AND '$end_date'
-```
-
-## Table Name Convention
-
-Use `__` to create variants of the same table:
-
-```python
-pipeline = [
-    ('sales__initial', 'overwrite'),    # writes to 'sales' table
-    ('sales__incremental', 'append'),   # appends to 'sales' table
-]
-```
-
-Both write to the same `sales` table, but use different SQL files.
-
-## Query Data
-
-```python
-# Run queries
-con.sql("SELECT * FROM my_table LIMIT 10").show()
-
-# Get as DataFrame
-df = con.sql("SELECT COUNT(*) FROM sales").df()
-```
-
-Explanation: DuckDB is connected to the lakehouse through `con`, so it is aware of the tables in that lakehouse (including tables created by your pipelines). That means you can query those tables directly with `con.sql(...)` just like any other DuckDB query. If you don't provide a `sql_folder`, you can still use `con.sql(...)` to explore existing tables.
-
-
-
-## Remote SQL Files
-
-You can load SQL/Python files from a URL:
-
-```python
-con = duckrun.connect(
-    workspace="Analytics",
-    lakehouse_name="Sales", 
-    schema="dbo",
-    sql_folder="https://raw.githubusercontent.com/user/repo/main/sql"
-)
-```
-
-## Real-Life Usage
-
-For a complete, production-style example, see [fabric_demo](https://github.com/djouallah/fabric_demo).
-
-## License
-
-MIT

duckrun-0.1.5.dist-info/RECORD

@@ -1,7 +0,0 @@
-duckrun/__init__.py,sha256=L0jRtD9Ld8Ti4e6GRvPDdHvkQCFAPHM43GSP7ARh6EM,241
-duckrun/core.py,sha256=6kjFoxjucRgb3qV9R6ZDy1oHcLE2JXPH5E0WCDKgpZM,18031
-duckrun-0.1.5.dist-info/licenses/LICENSE,sha256=-DeQQwdbCbkB4507ZF3QbocysB-EIjDtaLexvqRkGZc,1083
-duckrun-0.1.5.dist-info/METADATA,sha256=U2NQWJKm02rG4YvgAkDCJ9434OC_sJE-cuSUHu3eSvM,4377
-duckrun-0.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-duckrun-0.1.5.dist-info/top_level.txt,sha256=BknMEwebbUHrVAp3SC92ps8MPhK7XSYsaogTvi_DmEU,8
-duckrun-0.1.5.dist-info/RECORD,,