sqlframe 3.0.0__tar.gz → 3.1.0__tar.gz

{sqlframe-3.0.0 → sqlframe-3.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.0.0
+Version: 3.1.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.0.0 → sqlframe-3.1.0}/docs/configuration.md

@@ -71,6 +71,16 @@ spark = SparkSession.builder.getOrCreate()
 # "spark" is a SQLFrame DuckDBSession and will run directly on DuckDB with input dialect set to DuckDB
 ```
 
+#### Deactivating
+
+If you want to deactivate SQLFrame and revert back to PySpark, you can use the `deactivate` function.
+
+```python
+from sqlframe import deactivate
+
+deactivate()
+```
+
 ## Generated SQL
 
 ### Pretty

{sqlframe-3.0.0 → sqlframe-3.1.0}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.8",
     install_requires=[
         "prettytable<3.11.1",
-        "sqlglot>=24.0.0,<25.17",
+        "sqlglot>=24.0.0,<25.18",
         "typing_extensions>=4.8,<5",
     ],
     extras_require={

{sqlframe-3.0.0 → sqlframe-3.1.0}/sqlframe/__init__.py

@@ -81,3 +81,19 @@ def activate(
                 sys.modules[f"pyspark.sql.{file}"] = engine_file
                 resolved_files.add(engine_file)
             setattr(engine_file, name_without_prefix, obj)
+
+
+def deactivate() -> None:
+    pyspark_imports = [k for k in sys.modules if k.startswith("pyspark")]
+
+    for k, v in sys.modules.copy().items():
+        if k in pyspark_imports:
+            del sys.modules[k]
+    # Try importing the pyspark imports again and see if pyspark is installed and therefore available
+    # if not then nothing will change
+    for k in pyspark_imports:
+        try:
+            sys.modules[k] = importlib.import_module(k)
+        except ImportError:
+            pass
+    ACTIVATE_CONFIG.clear()

{sqlframe-3.0.0 → sqlframe-3.1.0}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.0.0'
-__version_tuple__ = version_tuple = (3, 0, 0)
+__version__ = version = '3.1.0'
+__version_tuple__ = version_tuple = (3, 1, 0)

{sqlframe-3.0.0 → sqlframe-3.1.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.0.0
+Version: 3.1.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.0.0 → sqlframe-3.1.0}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<3.11.1
-sqlglot<25.17,>=24.0.0
+sqlglot<25.18,>=24.0.0
 typing_extensions<5,>=4.8
 
 [bigquery]

{sqlframe-3.0.0 → sqlframe-3.1.0}/tests/unit/bigquery/test_activate.py

@@ -21,7 +21,7 @@ from sqlframe.bigquery import types as BigQueryTypes
 
 
 @pytest.mark.forked
-def test_activate_bigquery():
+def test_activate_bigquery(check_pyspark_imports):
     check_pyspark_imports(
         "bigquery",
         sqlf_session=BigQuerySession,
@@ -47,5 +47,6 @@ def test_activate_bigquery_default_dataset():
     activate("bigquery", config={"default_dataset": "sqlframe.sqlframe_test"})
     from pyspark.sql import SparkSession
 
+    assert SparkSession == BigQuerySession
     spark = SparkSession.builder.appName("test").getOrCreate()
     assert spark.default_dataset == "sqlframe.sqlframe_test"

{sqlframe-3.0.0 → sqlframe-3.1.0}/tests/unit/conftest.py

@@ -1,13 +1,11 @@
-import sys
-
 import findspark
 import pytest
 
-from sqlframe import activate
+from sqlframe import activate, deactivate
 
 
 @pytest.fixture()
-def check_pyspark_imports(temporary_sys_modules):
+def check_pyspark_imports():
     def _check_pyspark_imports(
         engine_name,
         sqlf_session,
@@ -28,7 +26,6 @@ def check_pyspark_imports(temporary_sys_modules):
     ):
         activate(engine=engine_name)
         findspark.init()
-        # TODO: functions, types, udf
         from pyspark.sql import (
             Catalog,
             Column,
@@ -121,4 +118,101 @@ def check_pyspark_imports(temporary_sys_modules):
         assert F == sqlf_functions
         assert types == sqlf_types
 
+        deactivate()
+        findspark.init()
+        from pyspark.sql import (
+            Catalog,
+            Column,
+            DataFrame,
+            DataFrameNaFunctions,
+            DataFrameReader,
+            DataFrameStatFunctions,
+            DataFrameWriter,
+            GroupedData,
+            Row,
+            SparkSession,
+            UDFRegistration,
+            Window,
+            WindowSpec,
+            types,
+        )
+        from pyspark.sql import functions as F
+
+        assert SparkSession != sqlf_session
+        assert Catalog != sqlf_catalog
+        assert Column != sqlf_column
+        assert GroupedData != sqlf_grouped_data
+        assert DataFrame != sqlf_dataframe
+        assert Window != sqlf_window
+        assert WindowSpec != sqlf_window_spec
+        assert F != sqlf_functions
+        assert types != sqlf_types
+        assert UDFRegistration != sqlf_udf_registration
+        assert DataFrameNaFunctions != sqlf_dataframe_na_functions
+        assert DataFrameStatFunctions != sqlf_dataframe_stat_functions
+        assert DataFrameReader != sqlf_dataframe_reader
+        assert DataFrameWriter != sqlf_dataframe_writer
+        assert Row != sqlf_row
+
+        assert F.JVMView is not None
+        assert SparkSession.readStream is not None
+
+        from pyspark.sql.session import SparkSession as SparkSession2
+
+        assert SparkSession2 != sqlf_session
+
+        from pyspark.sql.catalog import Catalog as Catalog2
+
+        assert Catalog2 != sqlf_catalog
+
+        from pyspark.sql.column import Column as Column2
+
+        assert Column2 != sqlf_column
+
+        from pyspark.sql.dataframe import DataFrame as DataFrame2
+        from pyspark.sql.dataframe import DataFrameNaFunctions as DataFrameNaFunctions2
+        from pyspark.sql.dataframe import (
+            DataFrameStatFunctions as DataFrameStatFunctions2,
+        )
+
+        assert DataFrame2 != sqlf_dataframe
+        assert DataFrameNaFunctions2 != sqlf_dataframe_na_functions
+        assert DataFrameStatFunctions2 != sqlf_dataframe_stat_functions
+
+        from pyspark.sql.group import GroupedData as GroupedData2
+
+        assert GroupedData2 != sqlf_grouped_data
+
+        from pyspark.sql.window import WindowSpec as WindowSpec2
+
+        assert WindowSpec2 != sqlf_window_spec
+
+        from pyspark.sql.readwriter import DataFrameReader as DataFrameReader2
+        from pyspark.sql.readwriter import DataFrameWriter as DataFrameWriter2
+
+        assert DataFrameReader2 != sqlf_dataframe_reader
+        assert DataFrameWriter2 != sqlf_dataframe_writer
+
+        from pyspark.sql.window import Window, WindowSpec
+
+        assert Window != sqlf_window
+        assert WindowSpec != sqlf_window_spec
+
+        from pyspark.sql import functions as F
+        from pyspark.sql import types
+
+        assert F != sqlf_functions
+        assert types != sqlf_types
+        assert types.Row != sqlf_row
+
+        from pyspark.sql import UDFRegistration
+
+        assert UDFRegistration != sqlf_udf_registration
+
+        import pyspark.sql.functions as F
+        import pyspark.sql.types as types
+
+        assert F != sqlf_functions
+        assert types != sqlf_types
+
     return _check_pyspark_imports

{sqlframe-3.0.0 → sqlframe-3.1.0}/tests/unit/duck/test_activate.py

@@ -20,7 +20,7 @@ from sqlframe.duckdb import types as DuckDBTypes
 
 
 @pytest.mark.forked
-def test_activate_duckdb():
+def test_activate_duckdb(check_pyspark_imports):
     check_pyspark_imports(
         "duckdb",
         sqlf_session=DuckDBSession,

{sqlframe-3.0.0 → sqlframe-3.1.0}/tests/unit/snowflake/test_activate.py

@@ -20,7 +20,7 @@ from sqlframe.snowflake import types as SnowflakeTypes
 
 
 @pytest.mark.forked
-def test_activate_snowflake():
+def test_activate_snowflake(check_pyspark_imports):
     check_pyspark_imports(
         "snowflake",
         sqlf_session=SnowflakeSession,

{sqlframe-3.0.0 → sqlframe-3.1.0}/tests/unit/standalone/test_activate.py

@@ -20,7 +20,7 @@ from sqlframe.standalone import types as StandaloneTypes
 
 
 @pytest.mark.forked
-def test_activate_standalone():
+def test_activate_standalone(check_pyspark_imports):
     check_pyspark_imports(
         "standalone",
         sqlf_session=StandaloneSession,

{sqlframe-3.0.0 → sqlframe-3.1.0}/tests/unit/standalone/test_functions.py

@@ -14,7 +14,8 @@ from sqlframe.standalone import functions as SF
 def test_invoke_anonymous(name, func):
     # array_size - converts to `size` but `array_size` and `size` behave differently
     # to_char - convert to a cast that ignores the format provided
-    ignore_funcs = {"array_size", "to_char"}
+    # ltrim/rtrim - don't seem to convert correctly on some engines
+    ignore_funcs = {"array_size", "to_char", "ltrim", "rtrim"}
     if "invoke_anonymous_function" in inspect.getsource(func) and name not in ignore_funcs:
         func = parse_one(f"{name}()", read="spark", error_level=ErrorLevel.IGNORE)
         assert isinstance(func, exp.Anonymous)

{sqlframe-3.0.0 → sqlframe-3.1.0}/tests/unit/test_activate.py

@@ -4,7 +4,7 @@ from unittest.mock import MagicMock
 import findspark
 import pytest
 
-from sqlframe import activate
+from sqlframe import activate, deactivate
 from sqlframe import testing as SQLFrameTesting
 
 
@@ -35,3 +35,11 @@ def test_activate_no_engine():
     from pyspark import context
 
     assert isinstance(context, MagicMock)
+
+    deactivate()
+
+    from pyspark import context
+
+    assert not isinstance(context, MagicMock)
+    assert context is not MagicMock
+    assert context is not None