sqlframe 3.13.0__tar.gz → 3.13.2__tar.gz

{sqlframe-3.13.0 → sqlframe-3.13.2}/PKG-INFO

@@ -1,12 +1,11 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.13.0
+Version: 3.13.2
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman
 Author-email: eakmanrq@gmail.com
 License: MIT
-Platform: UNKNOWN
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -192,5 +191,3 @@ LIMIT 5
 | 1975 |          868,985          |     10.92      |
 +------+---------------------------+----------------+
 ```
-
-

{sqlframe-3.13.0 → sqlframe-3.13.2}/docs/spark.md

@@ -8,13 +8,7 @@ pip install "sqlframe[spark]"
 
 ## Enabling SQLFrame
 
-SQLFrame can be used in two ways:
-
-* Directly importing the `sqlframe.spark` package 
-* Using the [activate](./configuration.md#activating-sqlframe) function to allow for continuing to use `pyspark.sql` but have it use SQLFrame behind the scenes.
-
-### Import
-
+SQLFrame can be enabled by directly importing the `sqlframe.spark` package.
 If converting a PySpark pipeline, all `pyspark.sql` should be replaced with `sqlframe.spark`.
 In addition, many classes will have a `Spark` prefix. 
 For example, `SparkDataFrame` instead of `DataFrame`.
@@ -30,58 +24,18 @@ from sqlframe.spark import functions as F
 from sqlframe.spark import SparkDataFrame
 ```
 
-### Activate
-
-If you would like to continue using `pyspark.sql` but have it use SQLFrame behind the scenes, you can use the [activate](./configuration.md#activating-sqlframe) function.
-
-```python
-from sqlframe import activate
-activate("spark")
-
-from pyspark.sql import SparkSession
-```
-
 ## Creating a Session
 
 SQLFrame's SparkSession is created the same way you would normally create a SparkSession.
 The configuration you apply to the builder will be applied the the SparkSession that SQLFrame will create.
 
-=== "Import"
-
-    ```python
-    from sqlframe.spark import SparkSession
-    
-    spark = SparkSession.builder.appName("MyApp").getOrCreate()
-    
-    # Now you can use SQLFrame
-    ```
-
-=== "Activate + Without Providing SparkSession"
-
-    ```python
-    from sqlframe import activate
-    activate("spark")
-    
-    from pyspark.sql import SparkSession
-    
-    spark = SparkSession.builder.appName("MyApp").getOrCreate()
-    
-    # Now you can use SQLFrame
-    ```
-
-=== "Activate + Providing SparkSession"
-
-    ```python
-    from pyspark.sql import SparkSession
-    from sqlframe import activate
-    activate("spark", conn=SparkSession.builder.appName("MyApp").getOrCreate())
-    
-    from pyspark.sql import SparkSession
-    
-    spark = SparkSession.getOrCreate()
-    
-    # Now you can use SQLFrame
-    ```
+```python
+from sqlframe.spark import SparkSession
+
+spark = SparkSession.builder.appName("MyApp").getOrCreate()
+
+# Now you can use SQLFrame
+```
 
 ## Example Usage
 

{sqlframe-3.13.0 → sqlframe-3.13.2}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<26.1",
+        "sqlglot>=24.0.0,<26.3",
         "typing_extensions",
     ],
     extras_require={
@@ -36,14 +36,14 @@ setup(
             "pandas>=2,<3",
             "pandas-stubs>=2,<3",
             "psycopg>=3.1,<4",
-            "pyarrow>=10,<19",
+            "pyarrow>=10,<20",
             "pyspark>=2,<3.6",
             "pytest>=8.2.0,<8.4",
             "pytest-forked",
             "pytest-postgresql>=6,<7",
             "pytest-xdist>=3.6,<3.7",
             "pre-commit>=3.7,<5",
-            "ruff>=0.4.4,<0.9",
+            "ruff>=0.4.4,<0.10",
             "types-psycopg2>=2.9,<3",
         ],
         "docs": [

{sqlframe-3.13.0 → sqlframe-3.13.2}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.13.0'
-__version_tuple__ = version_tuple = (3, 13, 0)
+__version__ = version = '3.13.2'
+__version_tuple__ = version_tuple = (3, 13, 2)

{sqlframe-3.13.0 → sqlframe-3.13.2}/sqlframe/base/dataframe.py

@@ -202,6 +202,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
     _na: t.Type[NA]
     _stat: t.Type[STAT]
     _group_data: t.Type[GROUP_DATA]
+    _EXPLAIN_PREFIX = "EXPLAIN"
 
     def __init__(
         self,
@@ -806,6 +807,8 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
             )
         else:
             col = self._ensure_and_normalize_col(column)
+        if isinstance(col.expression, exp.Alias):
+            col.expression = col.expression.this
         return self.copy(expression=self.expression.where(col.expression))
 
     filter = where
@@ -1018,9 +1021,9 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         elif not isinstance(ascending, list):
             ascending = [ascending] * len(columns)
         ascending = [bool(x) for i, x in enumerate(ascending)]
-        assert len(columns) == len(
-            ascending
-        ), "The length of items in ascending must equal the number of columns provided"
+        assert len(columns) == len(ascending), (
+            "The length of items in ascending must equal the number of columns provided"
+        )
         col_and_ascending = list(zip(columns, ascending))
         order_by_columns = [
             (
@@ -1142,6 +1145,18 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         final_df = filtered_df.select(*all_columns)
         return final_df
 
+    def _get_explain_plan_rows(self) -> t.List[Row]:
+        sql_queries = self.sql(
+            pretty=False, optimize=False, as_list=True, dialect=self.session.execution_dialect
+        )
+        if len(sql_queries) > 1:
+            raise ValueError("Cannot explain a DataFrame with multiple queries")
+        sql_query = " ".join([self._EXPLAIN_PREFIX, sql_queries[0]])
+        results = self.session._collect(sql_query)
+        if len(results) != 1:
+            raise ValueError("Got more than one result from explain query")
+        return results
+
     def explain(
         self, extended: t.Optional[t.Union[bool, str]] = None, mode: t.Optional[str] = None
     ) -> None:
@@ -1210,11 +1225,8 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         ...Statistics...
         ...
         """
-        sql_queries = self.sql(pretty=False, optimize=False, as_list=True)
-        if len(sql_queries) > 1:
-            raise ValueError("Cannot explain a DataFrame with multiple queries")
-        sql_query = "EXPLAIN " + sql_queries[0]
-        self.session._execute(sql_query)
+        results = self._get_explain_plan_rows()
+        print(results[0][0])
 
     @operation(Operation.FROM)
     def fillna(
@@ -1282,9 +1294,9 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
             new_values = list(to_replace.values())
         elif not old_values and isinstance(to_replace, list):
             assert isinstance(value, list), "value must be a list since the replacements are a list"
-            assert len(to_replace) == len(
-                value
-            ), "the replacements and values must be the same length"
+            assert len(to_replace) == len(value), (
+                "the replacements and values must be the same length"
+            )
             old_values = to_replace
             new_values = value
         else:

{sqlframe-3.13.0 → sqlframe-3.13.2}/sqlframe/bigquery/dataframe.py

@@ -72,3 +72,8 @@ class BigQueryDataFrame(
         sql = self.session._to_sql(self.expression)
         query_job = self.session._client.query(sql, job_config=job_config)
         return [field_to_column(field) for field in query_job.schema]
+
+    def explain(
+        self, extended: t.Optional[t.Union[bool, str]] = None, mode: t.Optional[str] = None
+    ) -> None:
+        raise NotImplementedError("BigQuery does not support EXPLAIN")

{sqlframe-3.13.0 → sqlframe-3.13.2}/sqlframe/duckdb/dataframe.py

@@ -46,6 +46,12 @@ class DuckDBDataFrame(
     _stat = DuckDBDataFrameStatFunctions
     _group_data = DuckDBGroupedData
 
+    def explain(
+        self, extended: t.Optional[t.Union[bool, str]] = None, mode: t.Optional[str] = None
+    ) -> None:
+        results = self._get_explain_plan_rows()
+        print(results[0][1])
+
     @t.overload
     def toArrow(self) -> ArrowTable: ...
 

{sqlframe-3.13.0 → sqlframe-3.13.2}/sqlframe/snowflake/dataframe.py

@@ -43,6 +43,7 @@ class SnowflakeDataFrame(
     _na = SnowflakeDataFrameNaFunctions
     _stat = SnowflakeDataFrameStatFunctions
     _group_data = SnowflakeGroupedData
+    _EXPLAIN_PREFIX = "EXPLAIN USING TEXT"
 
     @property
     def _typed_columns(self) -> t.List[CatalogColumn]:

{sqlframe-3.13.0 → sqlframe-3.13.2}/sqlframe.egg-info/PKG-INFO

@@ -1,12 +1,11 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.13.0
+Version: 3.13.2
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman
 Author-email: eakmanrq@gmail.com
 License: MIT
-Platform: UNKNOWN
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -192,5 +191,3 @@ LIMIT 5
 | 1975 |          868,985          |     10.92      |
 +------+---------------------------+----------------+
 ```
-
-

{sqlframe-3.13.0 → sqlframe-3.13.2}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<4
-sqlglot<26.1,>=24.0.0
+sqlglot<26.3,>=24.0.0
 typing_extensions
 
 [bigquery]
@@ -18,13 +18,13 @@ pandas-stubs<3,>=2
 pandas<3,>=2
 pre-commit<5,>=3.7
 psycopg<4,>=3.1
-pyarrow<19,>=10
+pyarrow<20,>=10
 pyspark<3.6,>=2
 pytest-forked
 pytest-postgresql<7,>=6
 pytest-xdist<3.7,>=3.6
 pytest<8.4,>=8.2.0
-ruff<0.9,>=0.4.4
+ruff<0.10,>=0.4.4
 types-psycopg2<3,>=2.9
 
 [docs]

{sqlframe-3.13.0 → sqlframe-3.13.2}/tests/integration/engines/bigquery/test_bigquery_dataframe.py

@@ -157,3 +157,8 @@ def test_schema_nested(bigquery_datatypes: BigQueryDataFrame):
     assert struct_fields[8].dataType == types.TimestampType()
     assert struct_fields[9].name == "boolean_col"
     assert struct_fields[9].dataType == types.BooleanType()
+
+
+def test_explain(bigquery_employee: BigQueryDataFrame):
+    with pytest.raises(NotImplementedError):
+        bigquery_employee.explain()

{sqlframe-3.13.0 → sqlframe-3.13.2}/tests/integration/engines/databricks/test_databricks_dataframe.py

@@ -167,3 +167,11 @@ def test_schema_nested(databricks_datatypes: DatabricksDataFrame):
     assert struct_fields[9].dataType == types.TimestampType()
     assert struct_fields[10].name == "boolean_col"
     assert struct_fields[10].dataType == types.BooleanType()
+
+
+def test_explain(databricks_employee: DatabricksDataFrame, capsys):
+    databricks_employee.explain()
+    output = capsys.readouterr().out.strip()
+    assert "== Physical Plan ==" in output
+    assert "LocalTableScan" in output
+    assert "== Photon Explanation ==" in output

{sqlframe-3.13.0 → sqlframe-3.13.2}/tests/integration/engines/duck/test_duckdb_dataframe.py

@@ -252,3 +252,17 @@ def test_to_arrow_batch(duckdb_employee: DuckDBDataFrame):
     assert fifth_batch.column(4).to_pylist() == [100]
     with pytest.raises(StopIteration):
         record_batch_reader.read_next_batch()
+
+
+def test_explain(duckdb_employee: DuckDBDataFrame, capsys):
+    duckdb_employee.explain()
+    assert (
+        capsys.readouterr().out.strip()
+        == """
+┌───────────────────────────┐
+│      COLUMN_DATA_SCAN     │
+│    ────────────────────   │
+│          ~5 Rows          │
+└───────────────────────────┘
+""".strip()
+    )

{sqlframe-3.13.0 → sqlframe-3.13.2}/tests/integration/engines/postgres/test_postgres_dataframe.py

@@ -120,3 +120,11 @@ def test_schema_nested(postgres_datatypes: PostgresDataFrame):
     assert struct_fields[6].dataType == types.TimestampType()
     assert struct_fields[7].name == "boolean_col"
     assert struct_fields[7].dataType == types.BooleanType()
+
+
+def test_explain(postgres_employee: PostgresDataFrame, capsys):
+    postgres_employee.explain()
+    assert (
+        capsys.readouterr().out.strip()
+        == """Values Scan on "*VALUES*"  (cost=0.00..0.06 rows=5 width=76)""".strip()
+    )

{sqlframe-3.13.0 → sqlframe-3.13.2}/tests/integration/engines/snowflake/test_snowflake_dataframe.py

@@ -156,3 +156,19 @@ def test_schema_nested(snowflake_datatypes: SnowflakeDataFrame):
     assert struct_fields[9].dataType == types.TimestampType()
     assert struct_fields[10].name == "boolean_col"
     assert struct_fields[10].dataType == types.BooleanType()
+
+
+def test_explain(snowflake_employee: SnowflakeDataFrame, capsys):
+    snowflake_employee.explain()
+    assert (
+        capsys.readouterr().out.strip()
+        == """
+GlobalStats:
+    partitionsTotal=0
+    partitionsAssigned=0
+    bytesAssigned=0
+Operations:
+1:0     ->Result  A1.EMPLOYEE_ID, A1.FNAME, A1.LNAME, A1.AGE, A1.STORE_ID  
+1:1          ->ValuesClause  (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100)
+""".strip()
+    )

{sqlframe-3.13.0 → sqlframe-3.13.2}/tests/integration/engines/spark/test_spark_dataframe.py

@@ -163,3 +163,10 @@ def test_schema_nested(spark_datatypes: SparkDataFrame):
     assert struct_fields[9].dataType == types.TimestampType()
     assert struct_fields[10].name == "boolean_col"
     assert struct_fields[10].dataType == types.BooleanType()
+
+
+def test_explain(spark_employee: SparkDataFrame, capsys):
+    spark_employee.explain()
+    output = capsys.readouterr().out.strip()
+    assert "== Physical Plan ==" in output
+    assert "LocalTableScan" in output

{sqlframe-3.13.0 → sqlframe-3.13.2}/tests/integration/test_int_dataframe.py

@@ -2278,3 +2278,17 @@ def test_self_join(
     )
 
     compare_frames(df_joined, dfs_joined, compare_schema=False)
+
+
+# https://github.com/eakmanrq/sqlframe/issues/232
+def test_filter_alias(
+    pyspark_employee: PySparkDataFrame,
+    get_df: t.Callable[[str], BaseDataFrame],
+    compare_frames: t.Callable,
+):
+    df_filtered = pyspark_employee.where((F.col("age") > 40).alias("age_gt_40"))
+
+    employee = get_df("employee")
+    dfs_filtered = employee.where((SF.col("age") > 40).alias("age_gt_40"))
+
+    compare_frames(df_filtered, dfs_filtered, compare_schema=False)

{sqlframe-3.13.0 → sqlframe-3.13.2}/tests/unit/standalone/test_session_case_sensitivity.py

@@ -25,7 +25,7 @@ tests = [
         '"Test"',
         {"name": "VARCHAR"},
         "name",
-        '''SELECT "Test"."NAME" AS "NAME" FROM "Test" AS "Test"''',
+        '''SELECT "TEST"."NAME" AS "NAME" FROM "Test" AS "TEST"''',
     ),
     (
         "Column has CS while table does not",
@@ -41,7 +41,7 @@ tests = [
         '"Test"',
         {'"Name"': "VARCHAR"},
         '"Name"',
-        '''SELECT "Test"."Name" AS "Name" FROM "Test" AS "Test"''',
+        '''SELECT "TEST"."Name" AS "Name" FROM "Test" AS "TEST"''',
     ),
     (
         "Lowercase CS table and column",
@@ -49,7 +49,7 @@ tests = [
         '"test"',
         {'"name"': "VARCHAR"},
         '"name"',
-        '''SELECT "test"."name" AS "name" FROM "test" AS "test"''',
+        '''SELECT "TEST"."name" AS "name" FROM "test" AS "TEST"''',
     ),
     (
         "CS table and column and query table but no CS in query column",