sqlframe 3.13.1__tar.gz → 3.13.3__tar.gz

{sqlframe-3.13.1 → sqlframe-3.13.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.13.1
+Version: 3.13.3
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.13.1 → sqlframe-3.13.3}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<26.1",
+        "sqlglot>=24.0.0,<26.3",
         "typing_extensions",
     ],
     extras_require={
@@ -36,7 +36,7 @@ setup(
             "pandas>=2,<3",
             "pandas-stubs>=2,<3",
             "psycopg>=3.1,<4",
-            "pyarrow>=10,<19",
+            "pyarrow>=10,<20",
             "pyspark>=2,<3.6",
             "pytest>=8.2.0,<8.4",
             "pytest-forked",

{sqlframe-3.13.1 → sqlframe-3.13.3}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.13.1'
-__version_tuple__ = version_tuple = (3, 13, 1)
+__version__ = version = '3.13.3'
+__version_tuple__ = version_tuple = (3, 13, 3)

{sqlframe-3.13.1 → sqlframe-3.13.3}/sqlframe/base/dataframe.py

@@ -202,6 +202,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
     _na: t.Type[NA]
     _stat: t.Type[STAT]
     _group_data: t.Type[GROUP_DATA]
+    _EXPLAIN_PREFIX = "EXPLAIN"
 
     def __init__(
         self,
@@ -1144,6 +1145,18 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         final_df = filtered_df.select(*all_columns)
         return final_df
 
+    def _get_explain_plan_rows(self) -> t.List[Row]:
+        sql_queries = self.sql(
+            pretty=False, optimize=False, as_list=True, dialect=self.session.execution_dialect
+        )
+        if len(sql_queries) > 1:
+            raise ValueError("Cannot explain a DataFrame with multiple queries")
+        sql_query = " ".join([self._EXPLAIN_PREFIX, sql_queries[0]])
+        results = self.session._collect(sql_query)
+        if len(results) != 1:
+            raise ValueError("Got more than one result from explain query")
+        return results
+
     def explain(
         self, extended: t.Optional[t.Union[bool, str]] = None, mode: t.Optional[str] = None
     ) -> None:
@@ -1212,11 +1225,8 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         ...Statistics...
         ...
         """
-        sql_queries = self.sql(pretty=False, optimize=False, as_list=True)
-        if len(sql_queries) > 1:
-            raise ValueError("Cannot explain a DataFrame with multiple queries")
-        sql_query = "EXPLAIN " + sql_queries[0]
-        self.session._execute(sql_query)
+        results = self._get_explain_plan_rows()
+        print(results[0][0])
 
     @operation(Operation.FROM)
     def fillna(

{sqlframe-3.13.1 → sqlframe-3.13.3}/sqlframe/base/session.py

@@ -548,7 +548,7 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, CONN, UDF_REGISTRATION
         normalize_identifiers(expression, dialect=dialect)
         if quote_identifiers:
             quote_identifiers_func(expression, dialect=dialect)
-        return optimize(expression, dialect=dialect, schema=self.catalog._schema)
+        return optimize(expression, dialect=dialect, schema=self.catalog._schema, infer_schema=True)
 
     def _execute(self, sql: str) -> None:
         self._cur.execute(sql)

{sqlframe-3.13.1 → sqlframe-3.13.3}/sqlframe/bigquery/dataframe.py

@@ -72,3 +72,8 @@ class BigQueryDataFrame(
         sql = self.session._to_sql(self.expression)
         query_job = self.session._client.query(sql, job_config=job_config)
         return [field_to_column(field) for field in query_job.schema]
+
+    def explain(
+        self, extended: t.Optional[t.Union[bool, str]] = None, mode: t.Optional[str] = None
+    ) -> None:
+        raise NotImplementedError("BigQuery does not support EXPLAIN")

{sqlframe-3.13.1 → sqlframe-3.13.3}/sqlframe/duckdb/dataframe.py

@@ -46,6 +46,12 @@ class DuckDBDataFrame(
     _stat = DuckDBDataFrameStatFunctions
     _group_data = DuckDBGroupedData
 
+    def explain(
+        self, extended: t.Optional[t.Union[bool, str]] = None, mode: t.Optional[str] = None
+    ) -> None:
+        results = self._get_explain_plan_rows()
+        print(results[0][1])
+
     @t.overload
     def toArrow(self) -> ArrowTable: ...
 

{sqlframe-3.13.1 → sqlframe-3.13.3}/sqlframe/snowflake/dataframe.py

@@ -43,6 +43,7 @@ class SnowflakeDataFrame(
     _na = SnowflakeDataFrameNaFunctions
     _stat = SnowflakeDataFrameStatFunctions
     _group_data = SnowflakeGroupedData
+    _EXPLAIN_PREFIX = "EXPLAIN USING TEXT"
 
     @property
     def _typed_columns(self) -> t.List[CatalogColumn]:

{sqlframe-3.13.1 → sqlframe-3.13.3}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.13.1
+Version: 3.13.3
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.13.1 → sqlframe-3.13.3}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<4
-sqlglot<26.1,>=24.0.0
+sqlglot<26.3,>=24.0.0
 typing_extensions
 
 [bigquery]
@@ -18,7 +18,7 @@ pandas-stubs<3,>=2
 pandas<3,>=2
 pre-commit<5,>=3.7
 psycopg<4,>=3.1
-pyarrow<19,>=10
+pyarrow<20,>=10
 pyspark<3.6,>=2
 pytest-forked
 pytest-postgresql<7,>=6

{sqlframe-3.13.1 → sqlframe-3.13.3}/tests/integration/engines/bigquery/test_bigquery_dataframe.py

@@ -157,3 +157,8 @@ def test_schema_nested(bigquery_datatypes: BigQueryDataFrame):
     assert struct_fields[8].dataType == types.TimestampType()
     assert struct_fields[9].name == "boolean_col"
     assert struct_fields[9].dataType == types.BooleanType()
+
+
+def test_explain(bigquery_employee: BigQueryDataFrame):
+    with pytest.raises(NotImplementedError):
+        bigquery_employee.explain()

{sqlframe-3.13.1 → sqlframe-3.13.3}/tests/integration/engines/databricks/test_databricks_dataframe.py

@@ -167,3 +167,11 @@ def test_schema_nested(databricks_datatypes: DatabricksDataFrame):
     assert struct_fields[9].dataType == types.TimestampType()
     assert struct_fields[10].name == "boolean_col"
     assert struct_fields[10].dataType == types.BooleanType()
+
+
+def test_explain(databricks_employee: DatabricksDataFrame, capsys):
+    databricks_employee.explain()
+    output = capsys.readouterr().out.strip()
+    assert "== Physical Plan ==" in output
+    assert "LocalTableScan" in output
+    assert "== Photon Explanation ==" in output

{sqlframe-3.13.1 → sqlframe-3.13.3}/tests/integration/engines/duck/test_duckdb_dataframe.py

@@ -252,3 +252,17 @@ def test_to_arrow_batch(duckdb_employee: DuckDBDataFrame):
     assert fifth_batch.column(4).to_pylist() == [100]
     with pytest.raises(StopIteration):
         record_batch_reader.read_next_batch()
+
+
+def test_explain(duckdb_employee: DuckDBDataFrame, capsys):
+    duckdb_employee.explain()
+    assert (
+        capsys.readouterr().out.strip()
+        == """
+┌───────────────────────────┐
+│      COLUMN_DATA_SCAN     │
+│    ────────────────────   │
+│          ~5 Rows          │
+└───────────────────────────┘
+""".strip()
+    )

{sqlframe-3.13.1 → sqlframe-3.13.3}/tests/integration/engines/duck/test_duckdb_reader.py

@@ -127,3 +127,12 @@ def test_issue_219(duckdb_session: DuckDBSession):
     lightbulbs = duckdb_session.read.csv("tests/fixtures/issue_219.csv")
     min_prices = lightbulbs.groupBy("kind", "make").agg(F.min("price"))
     lightbulbs.join(min_prices, "make").show()
+
+
+# https://github.com/eakmanrq/sqlframe/issues/242
+def test_read_parquet_optimize(duckdb_session: DuckDBSession):
+    df = duckdb_session.read.parquet(
+        "tests/fixtures/employee.parquet"
+    )  # Contains a `person_id` and a `person_source_value` column
+    df.createOrReplaceTempView("employee")
+    duckdb_session.table("employee").sql(optimize=True)  # type: ignore

{sqlframe-3.13.1 → sqlframe-3.13.3}/tests/integration/engines/postgres/test_postgres_dataframe.py

@@ -120,3 +120,11 @@ def test_schema_nested(postgres_datatypes: PostgresDataFrame):
     assert struct_fields[6].dataType == types.TimestampType()
     assert struct_fields[7].name == "boolean_col"
     assert struct_fields[7].dataType == types.BooleanType()
+
+
+def test_explain(postgres_employee: PostgresDataFrame, capsys):
+    postgres_employee.explain()
+    assert (
+        capsys.readouterr().out.strip()
+        == """Values Scan on "*VALUES*"  (cost=0.00..0.06 rows=5 width=76)""".strip()
+    )

{sqlframe-3.13.1 → sqlframe-3.13.3}/tests/integration/engines/snowflake/test_snowflake_dataframe.py

@@ -156,3 +156,19 @@ def test_schema_nested(snowflake_datatypes: SnowflakeDataFrame):
     assert struct_fields[9].dataType == types.TimestampType()
     assert struct_fields[10].name == "boolean_col"
     assert struct_fields[10].dataType == types.BooleanType()
+
+
+def test_explain(snowflake_employee: SnowflakeDataFrame, capsys):
+    snowflake_employee.explain()
+    assert (
+        capsys.readouterr().out.strip()
+        == """
+GlobalStats:
+    partitionsTotal=0
+    partitionsAssigned=0
+    bytesAssigned=0
+Operations:
+1:0     ->Result  A1.EMPLOYEE_ID, A1.FNAME, A1.LNAME, A1.AGE, A1.STORE_ID  
+1:1          ->ValuesClause  (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100)
+""".strip()
+    )

{sqlframe-3.13.1 → sqlframe-3.13.3}/tests/integration/engines/spark/test_spark_dataframe.py

@@ -163,3 +163,10 @@ def test_schema_nested(spark_datatypes: SparkDataFrame):
     assert struct_fields[9].dataType == types.TimestampType()
     assert struct_fields[10].name == "boolean_col"
     assert struct_fields[10].dataType == types.BooleanType()
+
+
+def test_explain(spark_employee: SparkDataFrame, capsys):
+    spark_employee.explain()
+    output = capsys.readouterr().out.strip()
+    assert "== Physical Plan ==" in output
+    assert "LocalTableScan" in output

{sqlframe-3.13.1 → sqlframe-3.13.3}/tests/unit/standalone/test_session_case_sensitivity.py

@@ -25,7 +25,7 @@ tests = [
         '"Test"',
         {"name": "VARCHAR"},
         "name",
-        '''SELECT "Test"."NAME" AS "NAME" FROM "Test" AS "Test"''',
+        '''SELECT "TEST"."NAME" AS "NAME" FROM "Test" AS "TEST"''',
     ),
     (
         "Column has CS while table does not",
@@ -41,7 +41,7 @@ tests = [
         '"Test"',
         {'"Name"': "VARCHAR"},
         '"Name"',
-        '''SELECT "Test"."Name" AS "Name" FROM "Test" AS "Test"''',
+        '''SELECT "TEST"."Name" AS "Name" FROM "Test" AS "TEST"''',
     ),
     (
         "Lowercase CS table and column",
@@ -49,7 +49,7 @@ tests = [
         '"test"',
         {'"name"': "VARCHAR"},
         '"name"',
-        '''SELECT "test"."name" AS "name" FROM "test" AS "test"''',
+        '''SELECT "TEST"."name" AS "name" FROM "test" AS "TEST"''',
     ),
     (
         "CS table and column and query table but no CS in query column",