duckrun 0.1.7__tar.gz → 0.1.9__tar.gz

{duckrun-0.1.7 → duckrun-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.1.7
+Version: 0.1.9
 Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
 Author: mim
 License: MIT
@@ -11,7 +11,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: duckdb>=1.2.0
-Requires-Dist: deltalake>=0.18.2
+Requires-Dist: deltalake<=0.18.2
 Requires-Dist: requests>=2.28.0
 Requires-Dist: obstore>=0.2.0
 Provides-Extra: local

{duckrun-0.1.7 → duckrun-0.1.9}/duckrun/core.py

@@ -8,6 +8,9 @@ from string import Template
 import obstore as obs
 from obstore.store import AzureStore
 
+# Row Group configuration for optimal Delta Lake performance
+RG = 8_000_000
+
 
 class DeltaWriter:
     """Spark-style write API for Delta Lake"""
@@ -48,7 +51,7 @@ class DeltaWriter:
         df = self.relation.record_batch()
         
         print(f"Writing to Delta table: {schema}.{table} (mode={self._mode})")
-        write_deltalake(path, df, mode=self._mode)
+        write_deltalake(path, df, mode=self._mode, max_rows_per_file=RG, max_rows_per_group=RG, min_rows_per_group=RG)
         
         self.duckrun.con.sql(f"DROP VIEW IF EXISTS {table}")
         self.duckrun.con.sql(f"""
@@ -406,7 +409,7 @@ class Duckrun:
         if mode == 'overwrite':
             self.con.sql(f"DROP VIEW IF EXISTS {normalized_table}")
             df = self.con.sql(sql).record_batch()
-            write_deltalake(path, df, mode='overwrite')
+            write_deltalake(path, df, mode='overwrite', max_rows_per_file=RG, max_rows_per_group=RG, min_rows_per_group=RG)
             self.con.sql(f"CREATE OR REPLACE VIEW {normalized_table} AS SELECT * FROM delta_scan('{path}')")
             dt = DeltaTable(path)
             dt.vacuum(retention_hours=0, dry_run=False, enforce_retention_duration=False)
@@ -414,7 +417,7 @@ class Duckrun:
 
         elif mode == 'append':
             df = self.con.sql(sql).record_batch()
-            write_deltalake(path, df, mode='append')
+            write_deltalake(path, df, mode='append', max_rows_per_file=RG, max_rows_per_group=RG, min_rows_per_group=RG)
             self.con.sql(f"CREATE OR REPLACE VIEW {normalized_table} AS SELECT * FROM delta_scan('{path}')")
             dt = DeltaTable(path)
             if len(dt.file_uris()) > self.compaction_threshold:
@@ -431,7 +434,7 @@ class Duckrun:
                 print(f"Table {normalized_table} doesn't exist. Creating...")
                 self.con.sql(f"DROP VIEW IF EXISTS {normalized_table}")
                 df = self.con.sql(sql).record_batch()
-                write_deltalake(path, df, mode='overwrite')
+                write_deltalake(path, df, mode='overwrite', max_rows_per_file=RG, max_rows_per_group=RG, min_rows_per_group=RG)
                 self.con.sql(f"CREATE OR REPLACE VIEW {normalized_table} AS SELECT * FROM delta_scan('{path}')")
                 dt = DeltaTable(path)
                 dt.vacuum(dry_run=False)
@@ -450,6 +453,7 @@ class Duckrun:
         
         Returns:
             True if all tasks succeeded
+            False if any task failed (exception) or Python task returned 0 (early exit)
         """
         if self.sql_folder is None:
             raise RuntimeError("sql_folder is not configured. Cannot run pipelines.")
@@ -460,22 +464,33 @@ class Duckrun:
             print('='*60)
             
             try:
+                result = None
+                
                 if len(task) == 2:
                     name, second = task
                     if isinstance(second, str) and second in {'overwrite', 'append', 'ignore'}:
-                        self._run_sql(name, second, {})
+                        result = self._run_sql(name, second, {})
                     else:
                         args = second if isinstance(second, (tuple, list)) else (second,)
-                        self._run_python(name, tuple(args))
+                        result = self._run_python(name, tuple(args))
                     
                 elif len(task) == 3:
                     table, mode, params = task
                     if not isinstance(params, dict):
                         raise ValueError(f"Expected dict for params, got {type(params)}")
-                    self._run_sql(table, mode, params)
+                    result = self._run_sql(table, mode, params)
                     
                 else:
                     raise ValueError(f"Invalid task format: {task}")
+                
+                # Check if Python task returned 0 (early exit condition)
+                # Only check for Python tasks as SQL tasks return table names (strings) and only stop on exceptions
+                if (len(task) == 2 and 
+                    not isinstance(task[1], str) and 
+                    result == 0):
+                    print(f"\n⏹️  Python task {i} returned 0 - stopping pipeline execution")
+                    print(f"   Remaining tasks ({len(pipeline) - i}) will not be executed")
+                    return False
                     
             except Exception as e:
                 print(f"\n❌ Task {i} failed: {e}")

{duckrun-0.1.7 → duckrun-0.1.9}/duckrun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.1.7
+Version: 0.1.9
 Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
 Author: mim
 License: MIT
@@ -11,7 +11,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: duckdb>=1.2.0
-Requires-Dist: deltalake>=0.18.2
+Requires-Dist: deltalake<=0.18.2
 Requires-Dist: requests>=2.28.0
 Requires-Dist: obstore>=0.2.0
 Provides-Extra: local

{duckrun-0.1.7 → duckrun-0.1.9}/duckrun.egg-info/requires.txt

@@ -1,5 +1,5 @@
 duckdb>=1.2.0
-deltalake>=0.18.2
+deltalake<=0.18.2
 requests>=2.28.0
 obstore>=0.2.0
 

{duckrun-0.1.7 → duckrun-0.1.9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "duckrun"
-version = "0.1.7"
+version = "0.1.9"
 description = "Lakehouse task runner powered by DuckDB for Microsoft Fabric"
 readme = "README.md"
 license = {text = "MIT"}
@@ -14,7 +14,7 @@ authors = [
 requires-python = ">=3.9"
 dependencies = [
     "duckdb>=1.2.0",
-    "deltalake>=0.18.2",
+    "deltalake<=0.18.2",
     "requests>=2.28.0",
     "obstore>=0.2.0"
 ]