duckrun 0.1.0__tar.gz → 0.1.2__tar.gz

duckrun-0.1.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Mimoune
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

duckrun-0.1.2/PKG-INFO

@@ -0,0 +1,168 @@
+Metadata-Version: 2.4
+Name: duckrun
+Version: 0.1.2
+Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/djouallah/duckrun
+Project-URL: Repository, https://github.com/djouallah/duckrun
+Project-URL: Issues, https://github.com/djouallah/duckrun/issues
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: duckdb>=1.2.0
+Requires-Dist: deltalake>=0.18.2
+Requires-Dist: requests>=2.28.0
+Dynamic: license-file
+
+
+<img src="duckrun.png" width="400" alt="Duckrun">
+
+Simple  task runner for Microsoft Fabric Python notebook, powered by DuckDB and Delta_rs.
+
+
+## Known Limitation
+
+Support only Lakehouse with schema, Workspace and lakehouse names should not contains space
+
+## Installation
+
+```bash
+pip install duckrun
+```
+
+
+
+## Quick Start
+
+```python
+import duckrun as dr
+
+# Connect to your Fabric lakehouse
+lakehouse = dr.connect(
+    workspace="my_workspace",
+    lakehouse_name="my_lakehouse", 
+    schema="dbo",
+    sql_folder="./sql"  # folder containing your .sql and .py files
+)
+
+# Define your pipeline
+pipeline = [
+    ('load_data', (url, path)),           # Python task
+    ('clean_data', 'overwrite'),          # SQL task  
+    ('aggregate', 'append')               # SQL task
+]
+
+# Run it
+lakehouse.run(pipeline)
+```
+
+## Early Exit
+
+In a pipeline run, if a task fails, the pipeline will stop without running the subsequent tasks.
+
+## How It Works
+
+Duckrun runs two types of tasks:
+
+### 1. Python Tasks
+Format: `('function_name', (arg1, arg2, ...))`
+
+Create a file `sql_folder/function_name.py` with a function matching the name:
+
+```python
+# sql_folder/load_data.py
+def load_data(url, path):
+    # your code here
+    # IMPORTANT: Must return 1 for success, 0 for failure
+    return 1
+```
+
+### 2. SQL Tasks  
+Format: `('table_name', 'mode')` or `('table_name', 'mode', {params})`
+
+Create a file `sql_folder/table_name.sql`:
+
+```sql
+-- sql_folder/clean_data.sql
+SELECT 
+    id,
+    TRIM(name) as name,
+    date
+FROM raw_data
+WHERE date >= '2024-01-01'
+```
+
+**Modes:**
+- `overwrite` - Replace table completely
+- `append` - Add to existing table
+- `ignore` - Create only if doesn't exist
+
+## Task Files
+
+The `sql_folder` can contain a mixture of both `.sql` and `.py` files. This allows you to combine SQL transformations and Python logic in your pipelines.
+
+### SQL Files
+Your SQL files automatically have access to:
+- `$ws` - workspace name
+- `$lh` - lakehouse name
+- `$schema` - schema name
+
+Pass custom parameters:
+
+```python
+pipeline = [
+    ('sales', 'append', {'start_date': '2024-01-01', 'end_date': '2024-12-31'})
+]
+```
+
+```sql
+-- sql_folder/sales.sql
+SELECT * FROM transactions
+WHERE date BETWEEN '$start_date' AND '$end_date'
+```
+
+## Table Name Convention
+
+Use `__` to create variants of the same table:
+
+```python
+pipeline = [
+    ('sales__initial', 'overwrite'),    # writes to 'sales' table
+    ('sales__incremental', 'append'),   # appends to 'sales' table
+]
+```
+
+Both write to the same `sales` table, but use different SQL files.
+
+## Query Data
+
+```python
+# Run queries
+lakehouse.sql("SELECT * FROM my_table LIMIT 10").show()
+
+# Get as DataFrame
+df = lakehouse.sql("SELECT COUNT(*) FROM sales").df()
+```
+
+
+
+## Remote SQL Files
+
+You can load SQL/Python files from a URL:
+
+```python
+lakehouse = dr.connect(
+    workspace="Analytics",
+    lakehouse_name="Sales", 
+    schema="dbo",
+    sql_folder="https://raw.githubusercontent.com/user/repo/main/sql"
+)
+```
+
+## Real-Life Usage
+
+For a complete, production-style example, see [fabric_demo](https://github.com/djouallah/fabric_demo).
+
+## License
+
+MIT

{duckrun-0.1.0 → duckrun-0.1.2}/README.md

@@ -1,6 +1,12 @@
-# 🦆 Duckrun
 
-Simple lakehouse task runner for Microsoft Fabric, powered by DuckDB.
+<img src="duckrun.png" width="400" alt="Duckrun">
+
+Simple  task runner for Microsoft Fabric Python notebook, powered by DuckDB and Delta_rs.
+
+
+## Known Limitation
+
+Support only Lakehouse with schema, Workspace and lakehouse names should not contains space
 
 ## Installation
 
@@ -8,10 +14,7 @@ Simple lakehouse task runner for Microsoft Fabric, powered by DuckDB.
 pip install duckrun
 ```
 
-For local development (enables Azure CLI authentication):
-```bash
-pip install duckrun[local]
-```
+
 
 ## Quick Start
 
@@ -37,6 +40,10 @@ pipeline = [
 lakehouse.run(pipeline)
 ```
 
+## Early Exit
+
+In a pipeline run, if a task fails, the pipeline will stop without running the subsequent tasks.
+
 ## How It Works
 
 Duckrun runs two types of tasks:
@@ -50,7 +57,8 @@ Create a file `sql_folder/function_name.py` with a function matching the name:
 # sql_folder/load_data.py
 def load_data(url, path):
     # your code here
-    return result
+    # IMPORTANT: Must return 1 for success, 0 for failure
+    return 1
 ```
 
 ### 2. SQL Tasks  
@@ -73,8 +81,11 @@ WHERE date >= '2024-01-01'
 - `append` - Add to existing table
 - `ignore` - Create only if doesn't exist
 
-## SQL Parameters
+## Task Files
+
+The `sql_folder` can contain a mixture of both `.sql` and `.py` files. This allows you to combine SQL transformations and Python logic in your pipelines.
 
+### SQL Files
 Your SQL files automatically have access to:
 - `$ws` - workspace name
 - `$lh` - lakehouse name
@@ -100,8 +111,8 @@ Use `__` to create variants of the same table:
 
 ```python
 pipeline = [
-    ('sales__initial', 'overwrite', {}),    # writes to 'sales' table
-    ('sales__incremental', 'append', {}),   # appends to 'sales' table
+    ('sales__initial', 'overwrite'),    # writes to 'sales' table
+    ('sales__incremental', 'append'),   # appends to 'sales' table
 ]
 ```
 
@@ -117,32 +128,7 @@ lakehouse.sql("SELECT * FROM my_table LIMIT 10").show()
 df = lakehouse.sql("SELECT COUNT(*) FROM sales").df()
 ```
 
-## Real-World Example
 
-```python
-import duckrun as dr
-
-lakehouse = dr.connect(
-    workspace="Analytics",
-    lakehouse_name="Sales", 
-    schema="dbo",
-    sql_folder="./etl"
-)
-
-# Daily pipeline
-daily = [
-    ('download_files', (api_url, local_path)),
-    ('staging_orders', 'overwrite', {'run_date': '2024-06-01'}),
-    ('staging_customers', 'overwrite', {'run_date': '2024-06-01'}),
-    ('fact_sales', 'append'),
-    ('dim_customer', 'overwrite')
-]
-
-lakehouse.run(daily)
-
-# Check results
-lakehouse.sql("SELECT COUNT(*) FROM fact_sales").show()
-```
 
 ## Remote SQL Files
 
@@ -157,6 +143,10 @@ lakehouse = dr.connect(
 )
 ```
 
+## Real-Life Usage
+
+For a complete, production-style example, see [fabric_demo](https://github.com/djouallah/fabric_demo).
+
 ## License
 
 MIT

{duckrun-0.1.0 → duckrun-0.1.2}/duckrun/core.py

@@ -64,9 +64,14 @@ class Duckrun:
     def _attach_lakehouse(self):
         self._create_onelake_secret()
         try:
+            # Exclude Iceberg metadata folders when scanning for Delta tables
             list_tables_query = f"""
                 SELECT DISTINCT(split_part(file, '_delta_log', 1)) as tables
                 FROM glob ("abfss://{self.workspace}@onelake.dfs.fabric.microsoft.com/{self.lakehouse_name}.Lakehouse/Tables/*/*/_delta_log/*.json")
+                WHERE file NOT LIKE '%/metadata/%'
+                  AND file NOT LIKE '%/iceberg/%'
+                  AND split_part(file, '_delta_log', 1) NOT LIKE '%/metadata'
+                  AND split_part(file, '_delta_log', 1) NOT LIKE '%/iceberg'
             """
             list_tables_df = self.con.sql(list_tables_query).df()
             list_tables = list_tables_df['tables'].tolist() if not list_tables_df.empty else []
@@ -82,18 +87,27 @@ class Duckrun:
                 if len(parts) >= 2:
                     potential_schema = parts[-2]
                     table = parts[-1]
+                    
+                    # Skip Iceberg-related folders
+                    if table in ('metadata', 'iceberg') or potential_schema in ('metadata', 'iceberg'):
+                        continue
+                        
                     if potential_schema == self.schema:
                         try:
                             self.con.sql(f"""
                                 CREATE OR REPLACE VIEW {table}
                                 AS SELECT * FROM delta_scan('{self.table_base_url}{self.schema}/{table}');
                             """)
+                            print(f"  ✓ Attached: {table}")
                         except Exception as e:
-                            print(f"Error creating view for table {table}: {e}")
+                            print(f"  ⚠ Skipped {table}: {str(e)[:100]}")
+                            continue
+            
             print("\nAttached tables (views) in DuckDB:")
             self.con.sql("SELECT name FROM (SHOW ALL TABLES) WHERE database='memory'").show()
         except Exception as e:
             print(f"Error attaching lakehouse: {e}")
+            print("Continuing without pre-attached tables.")
 
     def _normalize_table_name(self, name: str) -> str:
         """Extract base table name before first '__'"""

duckrun-0.1.2/duckrun.egg-info/PKG-INFO

@@ -0,0 +1,168 @@
+Metadata-Version: 2.4
+Name: duckrun
+Version: 0.1.2
+Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/djouallah/duckrun
+Project-URL: Repository, https://github.com/djouallah/duckrun
+Project-URL: Issues, https://github.com/djouallah/duckrun/issues
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: duckdb>=1.2.0
+Requires-Dist: deltalake>=0.18.2
+Requires-Dist: requests>=2.28.0
+Dynamic: license-file
+
+
+<img src="duckrun.png" width="400" alt="Duckrun">
+
+Simple  task runner for Microsoft Fabric Python notebook, powered by DuckDB and Delta_rs.
+
+
+## Known Limitation
+
+Support only Lakehouse with schema, Workspace and lakehouse names should not contains space
+
+## Installation
+
+```bash
+pip install duckrun
+```
+
+
+
+## Quick Start
+
+```python
+import duckrun as dr
+
+# Connect to your Fabric lakehouse
+lakehouse = dr.connect(
+    workspace="my_workspace",
+    lakehouse_name="my_lakehouse", 
+    schema="dbo",
+    sql_folder="./sql"  # folder containing your .sql and .py files
+)
+
+# Define your pipeline
+pipeline = [
+    ('load_data', (url, path)),           # Python task
+    ('clean_data', 'overwrite'),          # SQL task  
+    ('aggregate', 'append')               # SQL task
+]
+
+# Run it
+lakehouse.run(pipeline)
+```
+
+## Early Exit
+
+In a pipeline run, if a task fails, the pipeline will stop without running the subsequent tasks.
+
+## How It Works
+
+Duckrun runs two types of tasks:
+
+### 1. Python Tasks
+Format: `('function_name', (arg1, arg2, ...))`
+
+Create a file `sql_folder/function_name.py` with a function matching the name:
+
+```python
+# sql_folder/load_data.py
+def load_data(url, path):
+    # your code here
+    # IMPORTANT: Must return 1 for success, 0 for failure
+    return 1
+```
+
+### 2. SQL Tasks  
+Format: `('table_name', 'mode')` or `('table_name', 'mode', {params})`
+
+Create a file `sql_folder/table_name.sql`:
+
+```sql
+-- sql_folder/clean_data.sql
+SELECT 
+    id,
+    TRIM(name) as name,
+    date
+FROM raw_data
+WHERE date >= '2024-01-01'
+```
+
+**Modes:**
+- `overwrite` - Replace table completely
+- `append` - Add to existing table
+- `ignore` - Create only if doesn't exist
+
+## Task Files
+
+The `sql_folder` can contain a mixture of both `.sql` and `.py` files. This allows you to combine SQL transformations and Python logic in your pipelines.
+
+### SQL Files
+Your SQL files automatically have access to:
+- `$ws` - workspace name
+- `$lh` - lakehouse name
+- `$schema` - schema name
+
+Pass custom parameters:
+
+```python
+pipeline = [
+    ('sales', 'append', {'start_date': '2024-01-01', 'end_date': '2024-12-31'})
+]
+```
+
+```sql
+-- sql_folder/sales.sql
+SELECT * FROM transactions
+WHERE date BETWEEN '$start_date' AND '$end_date'
+```
+
+## Table Name Convention
+
+Use `__` to create variants of the same table:
+
+```python
+pipeline = [
+    ('sales__initial', 'overwrite'),    # writes to 'sales' table
+    ('sales__incremental', 'append'),   # appends to 'sales' table
+]
+```
+
+Both write to the same `sales` table, but use different SQL files.
+
+## Query Data
+
+```python
+# Run queries
+lakehouse.sql("SELECT * FROM my_table LIMIT 10").show()
+
+# Get as DataFrame
+df = lakehouse.sql("SELECT COUNT(*) FROM sales").df()
+```
+
+
+
+## Remote SQL Files
+
+You can load SQL/Python files from a URL:
+
+```python
+lakehouse = dr.connect(
+    workspace="Analytics",
+    lakehouse_name="Sales", 
+    schema="dbo",
+    sql_folder="https://raw.githubusercontent.com/user/repo/main/sql"
+)
+```
+
+## Real-Life Usage
+
+For a complete, production-style example, see [fabric_demo](https://github.com/djouallah/fabric_demo).
+
+## License
+
+MIT

{duckrun-0.1.0 → duckrun-0.1.2}/duckrun.egg-info/SOURCES.txt

@@ -6,4 +6,5 @@ duckrun/core.py
 duckrun.egg-info/PKG-INFO
 duckrun.egg-info/SOURCES.txt
 duckrun.egg-info/dependency_links.txt
+duckrun.egg-info/requires.txt
 duckrun.egg-info/top_level.txt

duckrun-0.1.2/duckrun.egg-info/requires.txt

@@ -0,0 +1,3 @@
+duckdb>=1.2.0
+deltalake>=0.18.2
+requests>=2.28.0

{duckrun-0.1.0 → duckrun-0.1.2}/pyproject.toml

@@ -5,10 +5,16 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "duckrun"
-version = "0.1.0"
+version = "0.1.2"
 description = "Lakehouse task runner powered by DuckDB for Microsoft Fabric"
+readme = "README.md"
 license = "MIT"
 requires-python = ">=3.9"
+dependencies = [
+    "duckdb>=1.2.0",
+    "deltalake>=0.18.2",
+    "requests>=2.28.0"
+]
 
 [project.urls]
 Homepage = "https://github.com/djouallah/duckrun"

duckrun-0.1.0/LICENSE

@@ -1 +0,0 @@
-### **5. `LICENSE`**

duckrun-0.1.0/PKG-INFO

@@ -1,11 +0,0 @@
-Metadata-Version: 2.4
-Name: duckrun
-Version: 0.1.0
-Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
-License-Expression: MIT
-Project-URL: Homepage, https://github.com/djouallah/duckrun
-Project-URL: Repository, https://github.com/djouallah/duckrun
-Project-URL: Issues, https://github.com/djouallah/duckrun/issues
-Requires-Python: >=3.9
-License-File: LICENSE
-Dynamic: license-file

duckrun-0.1.0/duckrun.egg-info/PKG-INFO

@@ -1,11 +0,0 @@
-Metadata-Version: 2.4
-Name: duckrun
-Version: 0.1.0
-Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
-License-Expression: MIT
-Project-URL: Homepage, https://github.com/djouallah/duckrun
-Project-URL: Repository, https://github.com/djouallah/duckrun
-Project-URL: Issues, https://github.com/djouallah/duckrun/issues
-Requires-Python: >=3.9
-License-File: LICENSE
-Dynamic: license-file