duckrun 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

duckrun/core.py

@@ -8,6 +8,9 @@ from string import Template
 import obstore as obs
 from obstore.store import AzureStore
 
+# Row Group configuration for optimal Delta Lake performance
+RG = 8_000_000
+
 
 class DeltaWriter:
     """Spark-style write API for Delta Lake"""
@@ -48,7 +51,7 @@ class DeltaWriter:
         df = self.relation.record_batch()
         
         print(f"Writing to Delta table: {schema}.{table} (mode={self._mode})")
-        write_deltalake(path, df, mode=self._mode)
+        write_deltalake(path, df, mode=self._mode, max_rows_per_file=RG, max_rows_per_group=RG, min_rows_per_group=RG)
         
         self.duckrun.con.sql(f"DROP VIEW IF EXISTS {table}")
         self.duckrun.con.sql(f"""
@@ -406,7 +409,7 @@ class Duckrun:
         if mode == 'overwrite':
             self.con.sql(f"DROP VIEW IF EXISTS {normalized_table}")
             df = self.con.sql(sql).record_batch()
-            write_deltalake(path, df, mode='overwrite')
+            write_deltalake(path, df, mode='overwrite', max_rows_per_file=RG, max_rows_per_group=RG, min_rows_per_group=RG)
             self.con.sql(f"CREATE OR REPLACE VIEW {normalized_table} AS SELECT * FROM delta_scan('{path}')")
             dt = DeltaTable(path)
             dt.vacuum(retention_hours=0, dry_run=False, enforce_retention_duration=False)
@@ -414,7 +417,7 @@ class Duckrun:
 
         elif mode == 'append':
             df = self.con.sql(sql).record_batch()
-            write_deltalake(path, df, mode='append')
+            write_deltalake(path, df, mode='append', max_rows_per_file=RG, max_rows_per_group=RG, min_rows_per_group=RG)
             self.con.sql(f"CREATE OR REPLACE VIEW {normalized_table} AS SELECT * FROM delta_scan('{path}')")
             dt = DeltaTable(path)
             if len(dt.file_uris()) > self.compaction_threshold:
@@ -431,7 +434,7 @@ class Duckrun:
                 print(f"Table {normalized_table} doesn't exist. Creating...")
                 self.con.sql(f"DROP VIEW IF EXISTS {normalized_table}")
                 df = self.con.sql(sql).record_batch()
-                write_deltalake(path, df, mode='overwrite')
+                write_deltalake(path, df, mode='overwrite', max_rows_per_file=RG, max_rows_per_group=RG, min_rows_per_group=RG)
                 self.con.sql(f"CREATE OR REPLACE VIEW {normalized_table} AS SELECT * FROM delta_scan('{path}')")
                 dt = DeltaTable(path)
                 dt.vacuum(dry_run=False)
@@ -450,6 +453,7 @@ class Duckrun:
         
         Returns:
             True if all tasks succeeded
+            False if any task failed (exception) or Python task returned 0 (early exit)
         """
         if self.sql_folder is None:
             raise RuntimeError("sql_folder is not configured. Cannot run pipelines.")
@@ -460,22 +464,33 @@ class Duckrun:
             print('='*60)
             
             try:
+                result = None
+                
                 if len(task) == 2:
                     name, second = task
                     if isinstance(second, str) and second in {'overwrite', 'append', 'ignore'}:
-                        self._run_sql(name, second, {})
+                        result = self._run_sql(name, second, {})
                     else:
                         args = second if isinstance(second, (tuple, list)) else (second,)
-                        self._run_python(name, tuple(args))
+                        result = self._run_python(name, tuple(args))
                     
                 elif len(task) == 3:
                     table, mode, params = task
                     if not isinstance(params, dict):
                         raise ValueError(f"Expected dict for params, got {type(params)}")
-                    self._run_sql(table, mode, params)
+                    result = self._run_sql(table, mode, params)
                     
                 else:
                     raise ValueError(f"Invalid task format: {task}")
+                
+                # Check if Python task returned 0 (early exit condition)
+                # Only check for Python tasks as SQL tasks return table names (strings) and only stop on exceptions
+                if (len(task) == 2 and 
+                    not isinstance(task[1], str) and 
+                    result == 0):
+                    print(f"\n⏹️  Python task {i} returned 0 - stopping pipeline execution")
+                    print(f"   Remaining tasks ({len(pipeline) - i}) will not be executed")
+                    return False
                     
             except Exception as e:
                 print(f"\n❌ Task {i} failed: {e}")

{duckrun-0.1.7.dist-info → duckrun-0.1.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.1.7
+Version: 0.1.9
 Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
 Author: mim
 License: MIT
@@ -11,7 +11,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: duckdb>=1.2.0
-Requires-Dist: deltalake>=0.18.2
+Requires-Dist: deltalake<=0.18.2
 Requires-Dist: requests>=2.28.0
 Requires-Dist: obstore>=0.2.0
 Provides-Extra: local

duckrun-0.1.9.dist-info/RECORD

@@ -0,0 +1,7 @@
+duckrun/__init__.py,sha256=L0jRtD9Ld8Ti4e6GRvPDdHvkQCFAPHM43GSP7ARh6EM,241
+duckrun/core.py,sha256=0ShFCe9tPmrd2fsH_AHCwzSQ03gXbaWWwmgNTyi5pYo,34064
+duckrun-0.1.9.dist-info/licenses/LICENSE,sha256=-DeQQwdbCbkB4507ZF3QbocysB-EIjDtaLexvqRkGZc,1083
+duckrun-0.1.9.dist-info/METADATA,sha256=vno1WFSAviBvJfBc09zPmELxYD6LTmpIWj52sqUYRaE,13847
+duckrun-0.1.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+duckrun-0.1.9.dist-info/top_level.txt,sha256=BknMEwebbUHrVAp3SC92ps8MPhK7XSYsaogTvi_DmEU,8
+duckrun-0.1.9.dist-info/RECORD,,

duckrun-0.1.7.dist-info/RECORD

@@ -1,7 +0,0 @@
-duckrun/__init__.py,sha256=L0jRtD9Ld8Ti4e6GRvPDdHvkQCFAPHM43GSP7ARh6EM,241
-duckrun/core.py,sha256=PzeY1WJVhAGTOuN5Yf86oNhKpK_zw6GYdylZ_BdSJfg,32982
-duckrun-0.1.7.dist-info/licenses/LICENSE,sha256=-DeQQwdbCbkB4507ZF3QbocysB-EIjDtaLexvqRkGZc,1083
-duckrun-0.1.7.dist-info/METADATA,sha256=BIsqAq6Z1JwSv7RwJ6wthzTC7xKSDeigZfVom5RJH0s,13847
-duckrun-0.1.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-duckrun-0.1.7.dist-info/top_level.txt,sha256=BknMEwebbUHrVAp3SC92ps8MPhK7XSYsaogTvi_DmEU,8
-duckrun-0.1.7.dist-info/RECORD,,