sqlframe 3.40.1__tar.gz → 3.41.0__tar.gz

{sqlframe-3.40.1 → sqlframe-3.41.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.40.1
+Version: 3.41.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.40.1 → sqlframe-3.41.0}/docs/duckdb.md

@@ -435,6 +435,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [rank](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.rank.html)
 * [regexp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp.html)
 * [regexp_extract](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_extract.html)
+* [regexp_extract_all](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_extract_all.html)
 * [regexp_like](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_like.html)
 * [regexp_replace](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_replace.html)
 * [repeat](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.repeat.html)

{sqlframe-3.40.1 → sqlframe-3.41.0}/docs/snowflake.md

@@ -472,6 +472,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [rand](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.rand.html)
 * [rank](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.rank.html)
 * [regexp_extract](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_extract.html)
+* [regexp_extract_all](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_extract_all.html)
 * [regexp_replace](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.regexp_replace.html)
 * [repeat](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.repeat.html)
 * [replace](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.replace.html)

{sqlframe-3.40.1 → sqlframe-3.41.0}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<27.13",
+        "sqlglot>=24.0.0,<27.15",
         "typing_extensions",
         "more-itertools",
     ],
@@ -32,7 +32,7 @@ setup(
         "dev": [
             "duckdb>=1.2,<1.4",
             "findspark>=2,<3",
-            "mypy>=1.10.0,<1.18",
+            "mypy>=1.10.0,<1.19",
             "openai>=1.30,<2",
             "pandas>=2,<3",
             "pandas-stubs>=2,<3",
@@ -45,7 +45,7 @@ setup(
             "pytest-rerunfailures",
             "pytest-xdist>=3.6,<3.9",
             "pre-commit>=3.7,<5",
-            "ruff>=0.4.4,<0.13",
+            "ruff>=0.4.4,<0.14",
             "types-psycopg2>=2.9,<3",
         ],
         "docs": [

{sqlframe-3.40.1 → sqlframe-3.41.0}/sqlframe/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.40.1'
-__version_tuple__ = version_tuple = (3, 40, 1)
+__version__ = version = '3.41.0'
+__version_tuple__ = version_tuple = (3, 41, 0)
 
-__commit_id__ = commit_id = 'g032251aae'
+__commit_id__ = commit_id = 'g961d3fda2'

{sqlframe-3.40.1 → sqlframe-3.41.0}/sqlframe/base/functions.py

@@ -2133,6 +2133,11 @@ def bit_xor(col: ColumnOrName) -> Column:
 
 @meta(unsupported_engines=["postgres", "snowflake"])
 def bit_count(col: ColumnOrName) -> Column:
+    session = _get_session()
+
+    if session._is_duckdb:
+        return Column.invoke_anonymous_function(col, "BIT_COUNT")
+
     return Column.invoke_expression_over_column(col, expression.BitwiseCountAgg)
 
 
@@ -5216,7 +5221,7 @@ def regexp_count(str: ColumnOrName, regexp: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(str, "regexp_count", regexp)
 
 
-@meta(unsupported_engines="*")
+@meta(unsupported_engines=["bigquery", "postgres"])
 def regexp_extract_all(
     str: ColumnOrName, regexp: ColumnOrName, idx: t.Optional[t.Union[int, Column]] = None
 ) -> Column:
@@ -5251,6 +5256,9 @@ def regexp_extract_all(
     >>> df.select(regexp_extract_all('str', col("regexp")).alias('d')).collect()
     [Row(d=['100', '300'])]
     """
+    if idx is None:
+        idx = 1
+
     return Column.invoke_expression_over_column(
         str, expression.RegexpExtractAll, expression=regexp, group=idx
     )

{sqlframe-3.40.1 → sqlframe-3.41.0}/sqlframe/base/readerwriter.py

@@ -61,14 +61,17 @@ class _BaseDataFrameReader(t.Generic[SESSION, DF, TABLE]):
     def _to_casted_columns(self, column_mapping: t.Dict) -> t.List[Column]:
         from sqlframe.base.column import Column
 
-        return [
-            Column(
-                exp.cast(
-                    exp.to_column(k), to=exp.DataType.build(v, dialect=self.session.input_dialect)
-                ).as_(k)
+        results = []
+        for k, v in column_mapping.items():
+            column = exp.to_column(k, dialect=self.session.input_dialect)
+            results.append(
+                Column(
+                    exp.cast(
+                        column, to=exp.DataType.build(v, dialect=self.session.input_dialect)
+                    ).as_(column.this)
+                )
             )
-            for k, v in column_mapping.items()
-        ]
+        return results
 
     def format(self, source: str) -> "Self":
         """Specifies the input data source format.

{sqlframe-3.40.1 → sqlframe-3.41.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.40.1
+Version: 3.41.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.40.1 → sqlframe-3.41.0}/sqlframe.egg-info/SOURCES.txt

@@ -193,6 +193,7 @@ tests/fixtures/employee.json
 tests/fixtures/employee.parquet
 tests/fixtures/employee_extra_line.csv
 tests/fixtures/issue_219.csv
+tests/fixtures/netflix_daily_top_10.parquet
 tests/fixtures/employee_delta/.part-00000-e5965c7b-e58f-4d3c-ad56-002876814e3a-c000.snappy.parquet.crc
 tests/fixtures/employee_delta/.part-00002-3fed7f18-370f-4b16-b232-504d6194eb52-c000.snappy.parquet.crc
 tests/fixtures/employee_delta/.part-00004-143c5da1-d5ab-4706-8e84-0d2a324c6894-c000.snappy.parquet.crc

{sqlframe-3.40.1 → sqlframe-3.41.0}/sqlframe.egg-info/requires.txt

@@ -1,6 +1,6 @@
 more-itertools
 prettytable<4
-sqlglot<27.13,>=24.0.0
+sqlglot<27.15,>=24.0.0
 typing_extensions
 
 [bigquery]
@@ -13,7 +13,7 @@ databricks-sql-connector[pyarrow]<5,>=3.6
 [dev]
 duckdb<1.4,>=1.2
 findspark<3,>=2
-mypy<1.18,>=1.10.0
+mypy<1.19,>=1.10.0
 openai<2,>=1.30
 pandas-stubs<3,>=2
 pandas<3,>=2
@@ -26,7 +26,7 @@ pytest-postgresql<8,>=6
 pytest-rerunfailures
 pytest-xdist<3.9,>=3.6
 pytest<8.5,>=8.2.0
-ruff<0.13,>=0.4.4
+ruff<0.14,>=0.4.4
 types-psycopg2<3,>=2.9
 
 [docs]

{sqlframe-3.40.1 → sqlframe-3.41.0}/tests/integration/engines/duck/test_duckdb_reader.py

@@ -145,3 +145,21 @@ def test_read_parquet_optimize(duckdb_session: DuckDBSession):
     )  # Contains a `person_id` and a `person_source_value` column
     df.createOrReplaceTempView("employee")
     duckdb_session.table("employee").sql(optimize=True)  # type: ignore
+
+
+# https://github.com/eakmanrq/sqlframe/issues/499
+def test_quoted_columns_parquet(duckdb_session: DuckDBSession):
+    df = duckdb_session.read.parquet("tests/fixtures/netflix_daily_top_10.parquet")
+    df = (
+        df.filter(F.col("Type") == "TV Show")
+        .groupBy("Title")
+        .agg(F.max(F.col("Days In Top 10")).alias("max_days_in_top_10"))
+        .orderBy(F.desc("max_days_in_top_10"))
+        .limit(4)
+    )
+    assert df.collect() == [
+        Row(**{"Title": "Cocomelon", "max_days_in_top_10": 428}),
+        Row(**{"Title": "Ozark", "max_days_in_top_10": 89}),
+        Row(**{"Title": "Cobra Kai", "max_days_in_top_10": 81}),
+        Row(**{"Title": "Manifest", "max_days_in_top_10": 80}),
+    ]

{sqlframe-3.40.1 → sqlframe-3.41.0}/tests/unit/standalone/test_dataframe.py

@@ -178,7 +178,7 @@ def test_lineage(standalone_employee: StandaloneDataFrame):
 def test_unquoted_identifiers(standalone_employee: StandaloneDataFrame):
     assert (
         standalone_employee.sql(dialect="snowflake", pretty=False, quote_identifiers=False)
-        == """SELECT A1.EMPLOYEE_ID AS "employee_id", CAST(A1.FNAME AS TEXT) AS "fname", CAST(A1.LNAME AS TEXT) AS "lname", A1.AGE AS "age", A1.STORE_ID AS "store_id" FROM (VALUES (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100)) AS A1(EMPLOYEE_ID, FNAME, LNAME, AGE, STORE_ID)"""
+        == """SELECT A1.EMPLOYEE_ID AS "employee_id", CAST(A1.FNAME AS VARCHAR) AS "fname", CAST(A1.LNAME AS VARCHAR) AS "lname", A1.AGE AS "age", A1.STORE_ID AS "store_id" FROM (VALUES (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100)) AS A1(EMPLOYEE_ID, FNAME, LNAME, AGE, STORE_ID)"""
     )
 
 

{sqlframe-3.40.1 → sqlframe-3.41.0}/tests/unit/standalone/test_functions.py

@@ -19,6 +19,7 @@ def test_invoke_anonymous(name, func):
     # ltrim/rtrim - don't seem to convert correctly on some engines
     # unix_micros - it is actually supported just an engine specific override uses an anonymous function
     # format_string - seemed like a complex match that had overrides so not worth it
+    # bit_count - requires an override for DuckDB
     ignore_funcs = {
         "array_size",
         "exists",
@@ -30,6 +31,7 @@ def test_invoke_anonymous(name, func):
         "current_schema",
         "unix_micros",
         "format_string",
+        "bit_count",
     }
     if "invoke_anonymous_function" in inspect.getsource(func) and name not in ignore_funcs:
         func = parse_one(f"{name}()", read="spark", error_level=ErrorLevel.IGNORE)