sqlframe 3.1.0__tar.gz → 3.2.0__tar.gz

{sqlframe-3.1.0 → sqlframe-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.1.0
+Version: 3.2.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.1.0 → sqlframe-3.2.0}/docs/configuration.md

@@ -71,7 +71,7 @@ spark = SparkSession.builder.getOrCreate()
 # "spark" is a SQLFrame DuckDBSession and will run directly on DuckDB with input dialect set to DuckDB
 ```
 
-#### Deactivating
+### Deactivating
 
 If you want to deactivate SQLFrame and revert back to PySpark, you can use the `deactivate` function.
 

{sqlframe-3.1.0 → sqlframe-3.2.0}/docs/standalone.md

@@ -84,7 +84,7 @@ from sqlframe.standalone import StandaloneSession
 session = StandaloneSession()
 
 session.catalog.add_table("test_table", column_mapping={"a": "int", "b": "int", "c": "int"})
-df = session.read.table("my_table")
+df = session.read.table("test_table")
 df.sql()
 ```
 

{sqlframe-3.1.0 → sqlframe-3.2.0}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.8",
     install_requires=[
         "prettytable<3.11.1",
-        "sqlglot>=24.0.0,<25.18",
+        "sqlglot>=24.0.0,<25.19",
         "typing_extensions>=4.8,<5",
     ],
     extras_require={
@@ -32,7 +32,7 @@ setup(
             "duckdb>=0.9,<1.1",
             "findspark>=2,<3",
             "mypy>=1.10.0,<1.12",
-            "openai>=1.30,<1.43",
+            "openai>=1.30,<1.44",
             "pandas>=2,<3",
             "pandas-stubs>=2,<3",
             "psycopg>=3.1,<4",
@@ -58,7 +58,7 @@ setup(
             "pandas>=2,<3",
         ],
         "openai": [
-            "openai>=1.30,<1.43",
+            "openai>=1.30,<1.44",
         ],
         "pandas": [
             "pandas>=2,<3",

{sqlframe-3.1.0 → sqlframe-3.2.0}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.1.0'
-__version_tuple__ = version_tuple = (3, 1, 0)
+__version__ = version = '3.2.0'
+__version_tuple__ = version_tuple = (3, 2, 0)

{sqlframe-3.1.0 → sqlframe-3.2.0}/sqlframe/base/group.py

@@ -68,7 +68,7 @@ class _BaseGroupedData(t.Generic[DF]):
                 )
                 group_by_cols.extend(grouping_set)  # type: ignore
             group_by_cols = list(dict.fromkeys(group_by_cols))
-            group_by = exp.Group(grouping_sets=all_grouping_sets)
+            group_by = exp.Group(grouping_sets=[exp.GroupingSets(expressions=all_grouping_sets)])
             expression.set("group", group_by)
         for col in cols:
             # Spark supports having an empty grouping_id which means all of the columns but other dialects

{sqlframe-3.1.0 → sqlframe-3.2.0}/sqlframe/duckdb/readwriter.py

@@ -85,7 +85,9 @@ class DuckDBDataFrameReader(_BaseDataFrameReader["DuckDBSession", "DuckDBDataFra
                 options["columns"] = "{" + duckdb_columns + "}"
         else:
             select_columns = [exp.Star()]
-        if format:
+        if format == "delta":
+            from_clause = f"delta_scan('{path}')"
+        elif format:
             paths = ",".join([f"'{path}'" for path in ensure_list(path)])
             from_clause = f"read_{format}([{paths}], {to_csv(options)})"
         else:

{sqlframe-3.1.0 → sqlframe-3.2.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.1.0
+Version: 3.2.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.1.0 → sqlframe-3.2.0}/sqlframe.egg-info/SOURCES.txt

@@ -171,6 +171,20 @@ tests/fixtures/employee.csv
 tests/fixtures/employee.json
 tests/fixtures/employee.parquet
 tests/fixtures/employee_extra_line.csv
+tests/fixtures/employee_delta/.part-00000-e5965c7b-e58f-4d3c-ad56-002876814e3a-c000.snappy.parquet.crc
+tests/fixtures/employee_delta/.part-00002-3fed7f18-370f-4b16-b232-504d6194eb52-c000.snappy.parquet.crc
+tests/fixtures/employee_delta/.part-00004-143c5da1-d5ab-4706-8e84-0d2a324c6894-c000.snappy.parquet.crc
+tests/fixtures/employee_delta/.part-00006-64f07e25-c30e-4075-acc6-b3c69c4ce80b-c000.snappy.parquet.crc
+tests/fixtures/employee_delta/.part-00008-89ccad8d-df73-4ad5-8850-82ef3884db60-c000.snappy.parquet.crc
+tests/fixtures/employee_delta/.part-00010-812b3382-8c7f-4c4e-9bcd-09ce8664f6e0-c000.snappy.parquet.crc
+tests/fixtures/employee_delta/part-00000-e5965c7b-e58f-4d3c-ad56-002876814e3a-c000.snappy.parquet
+tests/fixtures/employee_delta/part-00002-3fed7f18-370f-4b16-b232-504d6194eb52-c000.snappy.parquet
+tests/fixtures/employee_delta/part-00004-143c5da1-d5ab-4706-8e84-0d2a324c6894-c000.snappy.parquet
+tests/fixtures/employee_delta/part-00006-64f07e25-c30e-4075-acc6-b3c69c4ce80b-c000.snappy.parquet
+tests/fixtures/employee_delta/part-00008-89ccad8d-df73-4ad5-8850-82ef3884db60-c000.snappy.parquet
+tests/fixtures/employee_delta/part-00010-812b3382-8c7f-4c4e-9bcd-09ce8664f6e0-c000.snappy.parquet
+tests/fixtures/employee_delta/_delta_log/.00000000000000000000.json.crc
+tests/fixtures/employee_delta/_delta_log/00000000000000000000.json
 tests/integration/__init__.py
 tests/integration/fixtures.py
 tests/integration/test_int_dataframe.py

{sqlframe-3.1.0 → sqlframe-3.2.0}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<3.11.1
-sqlglot<25.18,>=24.0.0
+sqlglot<25.19,>=24.0.0
 typing_extensions<5,>=4.8
 
 [bigquery]
@@ -10,7 +10,7 @@ google-cloud-bigquery[pandas]<4,>=3
 duckdb<1.1,>=0.9
 findspark<3,>=2
 mypy<1.12,>=1.10.0
-openai<1.43,>=1.30
+openai<1.44,>=1.30
 pandas-stubs<3,>=2
 pandas<3,>=2
 psycopg<4,>=3.1
@@ -40,7 +40,7 @@ duckdb<1.1,>=0.9
 pandas<3,>=2
 
 [openai]
-openai<1.43,>=1.30
+openai<1.44,>=1.30
 
 [pandas]
 pandas<3,>=2

sqlframe-3.2.0/tests/fixtures/employee_delta/_delta_log/00000000000000000000.json

@@ -0,0 +1,8 @@
+{"commitInfo":{"timestamp":1725137989496,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"6","numOutputRows":"5","numOutputBytes":"7800"},"engineInfo":"Apache-Spark/3.5.2 Delta-Lake/3.2.0","txnId":"17633aa0-b065-49d6-9893-022103f7364e"}}
+{"metaData":{"id":"73709a6c-bb75-4ac7-a78e-635400886d74","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"employee_id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"fname\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"lname\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"store_id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1725137987547}}
+{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
+{"add":{"path":"part-00002-3fed7f18-370f-4b16-b232-504d6194eb52-c000.snappy.parquet","partitionValues":{},"size":1437,"modificationTime":1725137989427,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"employee_id\":1,\"fname\":\"Jack\",\"lname\":\"Shephard\",\"age\":37,\"store_id\":1},\"maxValues\":{\"employee_id\":1,\"fname\":\"Jack\",\"lname\":\"Shephard\",\"age\":37,\"store_id\":1},\"nullCount\":{\"employee_id\":0,\"fname\":0,\"lname\":0,\"age\":0,\"store_id\":0}}"}}
+{"add":{"path":"part-00004-143c5da1-d5ab-4706-8e84-0d2a324c6894-c000.snappy.parquet","partitionValues":{},"size":1417,"modificationTime":1725137989428,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"employee_id\":2,\"fname\":\"John\",\"lname\":\"Locke\",\"age\":65,\"store_id\":1},\"maxValues\":{\"employee_id\":2,\"fname\":\"John\",\"lname\":\"Locke\",\"age\":65,\"store_id\":1},\"nullCount\":{\"employee_id\":0,\"fname\":0,\"lname\":0,\"age\":0,\"store_id\":0}}"}}
+{"add":{"path":"part-00006-64f07e25-c30e-4075-acc6-b3c69c4ce80b-c000.snappy.parquet","partitionValues":{},"size":1424,"modificationTime":1725137989428,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"employee_id\":3,\"fname\":\"Kate\",\"lname\":\"Austen\",\"age\":37,\"store_id\":2},\"maxValues\":{\"employee_id\":3,\"fname\":\"Kate\",\"lname\":\"Austen\",\"age\":37,\"store_id\":2},\"nullCount\":{\"employee_id\":0,\"fname\":0,\"lname\":0,\"age\":0,\"store_id\":0}}"}}
+{"add":{"path":"part-00008-89ccad8d-df73-4ad5-8850-82ef3884db60-c000.snappy.parquet","partitionValues":{},"size":1459,"modificationTime":1725137989427,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"employee_id\":4,\"fname\":\"Claire\",\"lname\":\"Littleton\",\"age\":27,\"store_id\":2},\"maxValues\":{\"employee_id\":4,\"fname\":\"Claire\",\"lname\":\"Littleton\",\"age\":27,\"store_id\":2},\"nullCount\":{\"employee_id\":0,\"fname\":0,\"lname\":0,\"age\":0,\"store_id\":0}}"}}
+{"add":{"path":"part-00010-812b3382-8c7f-4c4e-9bcd-09ce8664f6e0-c000.snappy.parquet","partitionValues":{},"size":1417,"modificationTime":1725137989428,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"employee_id\":5,\"fname\":\"Hugo\",\"lname\":\"Reyes\",\"age\":29,\"store_id\":100},\"maxValues\":{\"employee_id\":5,\"fname\":\"Hugo\",\"lname\":\"Reyes\",\"age\":29,\"store_id\":100},\"nullCount\":{\"employee_id\":0,\"fname\":0,\"lname\":0,\"age\":0,\"store_id\":0}}"}}

{sqlframe-3.1.0 → sqlframe-3.2.0}/tests/integration/engines/duck/test_duckdb_reader.py

@@ -98,3 +98,19 @@ def test_employee_extra_line_csv_multiple(duckdb_session: DuckDBSession):
         ),
         Row(**{"employee_id": 5, "fname": "Hugo", "lname": "Reyes", "age": 29, "store_id": 100}),
     ]
+
+
+def test_employee_delta(duckdb_session: DuckDBSession):
+    df = duckdb_session.read.load(
+        "tests/fixtures/employee_delta",
+        format="delta",
+    )
+    assert df.collect() == [
+        Row(**{"employee_id": 1, "fname": "Jack", "lname": "Shephard", "age": 37, "store_id": 1}),
+        Row(**{"employee_id": 2, "fname": "John", "lname": "Locke", "age": 65, "store_id": 1}),
+        Row(**{"employee_id": 3, "fname": "Kate", "lname": "Austen", "age": 37, "store_id": 2}),
+        Row(
+            **{"employee_id": 4, "fname": "Claire", "lname": "Littleton", "age": 27, "store_id": 2}
+        ),
+        Row(**{"employee_id": 5, "fname": "Hugo", "lname": "Reyes", "age": 29, "store_id": 100}),
+    ]