duckrun 0.1.7__tar.gz → 0.1.8__tar.gz

{duckrun-0.1.7 → duckrun-0.1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.1.7
+Version: 0.1.8
 Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
 Author: mim
 License: MIT
@@ -11,7 +11,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: duckdb>=1.2.0
-Requires-Dist: deltalake>=0.18.2
+Requires-Dist: deltalake<=0.18.2
 Requires-Dist: requests>=2.28.0
 Requires-Dist: obstore>=0.2.0
 Provides-Extra: local

{duckrun-0.1.7 → duckrun-0.1.8}/duckrun/core.py

@@ -8,6 +8,9 @@ from string import Template
 import obstore as obs
 from obstore.store import AzureStore
 
+# Row Group configuration for optimal Delta Lake performance
+RG = 8_000_000
+
 
 class DeltaWriter:
     """Spark-style write API for Delta Lake"""
@@ -48,7 +51,7 @@ class DeltaWriter:
         df = self.relation.record_batch()
         
         print(f"Writing to Delta table: {schema}.{table} (mode={self._mode})")
-        write_deltalake(path, df, mode=self._mode)
+        write_deltalake(path, df, mode=self._mode, max_rows_per_file=RG, max_rows_per_group=RG, min_rows_per_group=RG)
         
         self.duckrun.con.sql(f"DROP VIEW IF EXISTS {table}")
         self.duckrun.con.sql(f"""
@@ -406,7 +409,7 @@ class Duckrun:
         if mode == 'overwrite':
             self.con.sql(f"DROP VIEW IF EXISTS {normalized_table}")
             df = self.con.sql(sql).record_batch()
-            write_deltalake(path, df, mode='overwrite')
+            write_deltalake(path, df, mode='overwrite', max_rows_per_file=RG, max_rows_per_group=RG, min_rows_per_group=RG)
             self.con.sql(f"CREATE OR REPLACE VIEW {normalized_table} AS SELECT * FROM delta_scan('{path}')")
             dt = DeltaTable(path)
             dt.vacuum(retention_hours=0, dry_run=False, enforce_retention_duration=False)
@@ -414,7 +417,7 @@ class Duckrun:
 
         elif mode == 'append':
             df = self.con.sql(sql).record_batch()
-            write_deltalake(path, df, mode='append')
+            write_deltalake(path, df, mode='append', max_rows_per_file=RG, max_rows_per_group=RG, min_rows_per_group=RG)
             self.con.sql(f"CREATE OR REPLACE VIEW {normalized_table} AS SELECT * FROM delta_scan('{path}')")
             dt = DeltaTable(path)
             if len(dt.file_uris()) > self.compaction_threshold:
@@ -431,7 +434,7 @@ class Duckrun:
                 print(f"Table {normalized_table} doesn't exist. Creating...")
                 self.con.sql(f"DROP VIEW IF EXISTS {normalized_table}")
                 df = self.con.sql(sql).record_batch()
-                write_deltalake(path, df, mode='overwrite')
+                write_deltalake(path, df, mode='overwrite', max_rows_per_file=RG, max_rows_per_group=RG, min_rows_per_group=RG)
                 self.con.sql(f"CREATE OR REPLACE VIEW {normalized_table} AS SELECT * FROM delta_scan('{path}')")
                 dt = DeltaTable(path)
                 dt.vacuum(dry_run=False)

{duckrun-0.1.7 → duckrun-0.1.8}/duckrun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.1.7
+Version: 0.1.8
 Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
 Author: mim
 License: MIT
@@ -11,7 +11,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: duckdb>=1.2.0
-Requires-Dist: deltalake>=0.18.2
+Requires-Dist: deltalake<=0.18.2
 Requires-Dist: requests>=2.28.0
 Requires-Dist: obstore>=0.2.0
 Provides-Extra: local

{duckrun-0.1.7 → duckrun-0.1.8}/duckrun.egg-info/requires.txt

@@ -1,5 +1,5 @@
 duckdb>=1.2.0
-deltalake>=0.18.2
+deltalake<=0.18.2
 requests>=2.28.0
 obstore>=0.2.0
 

{duckrun-0.1.7 → duckrun-0.1.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "duckrun"
-version = "0.1.7"
+version = "0.1.8"
 description = "Lakehouse task runner powered by DuckDB for Microsoft Fabric"
 readme = "README.md"
 license = {text = "MIT"}
@@ -14,7 +14,7 @@ authors = [
 requires-python = ">=3.9"
 dependencies = [
     "duckdb>=1.2.0",
-    "deltalake>=0.18.2",
+    "deltalake<=0.18.2",
     "requests>=2.28.0",
     "obstore>=0.2.0"
 ]