sqlframe 3.8.2__tar.gz → 3.9.1__tar.gz

{sqlframe-3.8.2 → sqlframe-3.9.1}/Makefile

@@ -1,5 +1,5 @@
 install-dev:
-	pip install -e ".[bigquery,dev,docs,duckdb,pandas,postgres,redshift,snowflake,spark]"
+	pip install -e ".[bigquery,dev,docs,duckdb,pandas,postgres,redshift,snowflake,databricks,spark]"
 
 install-pre-commit:
 	pre-commit install
@@ -33,6 +33,7 @@ stubs:
 	stubgen sqlframe/duckdb/functions.py --output ./ --inspect-mode
 	stubgen sqlframe/postgres/functions.py --output ./ --inspect-mode
 	stubgen sqlframe/snowflake/functions.py --output ./ --inspect-mode
+	stubgen sqlframe/databricks/functions.py --output ./ --inspect-mode
 	stubgen sqlframe/spark/functions.py --output ./ --inspect-mode
 
 package:

{sqlframe-3.8.2 → sqlframe-3.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.8.2
+Version: 3.9.1
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 Provides-Extra: bigquery
+Provides-Extra: databricks
 Provides-Extra: dev
 Provides-Extra: docs
 Provides-Extra: duckdb
@@ -42,6 +43,11 @@ SQLFrame currently supports the following engines (many more in development):
 * [Snowflake](https://sqlframe.readthedocs.io/en/stable/snowflake)
 * [Spark](https://sqlframe.readthedocs.io/en/stable/spark)
 
+There are also two engines in development. These engines lack test coverage and robust documentation, but are available for early testing:
+
+* [Redshift](https://sqlframe.readthedocs.io/en/stable/redshift)
+* [Databricks](https://sqlframe.readthedocs.io/en/stable/databricks)
+
 SQLFrame also has a "Standalone" session that be used to generate SQL without any connection to a database engine.
 
 * [Standalone](https://sqlframe.readthedocs.io/en/stable/standalone)
@@ -66,6 +72,10 @@ pip install "sqlframe[postgres]"
 pip install "sqlframe[snowflake]"
 # Spark
 pip install "sqlframe[spark]"
+# Redshift (in development)
+pip install "sqlframe[redshift]"
+# Databricks (in development)
+pip install "sqlframe[databricks]"
 # Standalone
 pip install sqlframe
 ```

{sqlframe-3.8.2 → sqlframe-3.9.1}/README.md

@@ -12,6 +12,11 @@ SQLFrame currently supports the following engines (many more in development):
 * [Snowflake](https://sqlframe.readthedocs.io/en/stable/snowflake)
 * [Spark](https://sqlframe.readthedocs.io/en/stable/spark)
 
+There are also two engines in development. These engines lack test coverage and robust documentation, but are available for early testing:
+
+* [Redshift](https://sqlframe.readthedocs.io/en/stable/redshift)
+* [Databricks](https://sqlframe.readthedocs.io/en/stable/databricks)
+
 SQLFrame also has a "Standalone" session that be used to generate SQL without any connection to a database engine.
 
 * [Standalone](https://sqlframe.readthedocs.io/en/stable/standalone)
@@ -36,6 +41,10 @@ pip install "sqlframe[postgres]"
 pip install "sqlframe[snowflake]"
 # Spark
 pip install "sqlframe[spark]"
+# Redshift (in development)
+pip install "sqlframe[redshift]"
+# Databricks (in development)
+pip install "sqlframe[databricks]"
 # Standalone
 pip install sqlframe
 ```

sqlframe-3.9.1/docs/databricks.md

@@ -0,0 +1,157 @@
+from test import auth_type
+
+# Databricks (In Development)
+
+## Installation
+
+```bash
+pip install "sqlframe[databricks]"
+```
+
+## Enabling SQLFrame
+
+SQLFrame can be used in two ways:
+
+* Directly importing the `sqlframe.databricks` package 
+* Using the [activate](./configuration.md#activating-sqlframe) function to allow for continuing to use `pyspark.sql` but have it use SQLFrame behind the scenes.
+
+### Import
+
+If converting a PySpark pipeline, all `pyspark.sql` should be replaced with `sqlframe.databricks`.
+In addition, many classes will have a `Databricks` prefix. 
+For example, `DatabricksDataFrame` instead of `DataFrame`.
+
+
+```python
+# PySpark import
+# from pyspark.sql import SparkSession
+# from pyspark.sql import functions as F
+# from pyspark.sql.dataframe import DataFrame
+# SQLFrame import
+from sqlframe.databricks import DatabricksSession
+from sqlframe.databricks import functions as F
+from sqlframe.databricks import DatabricksDataFrame
+```
+
+### Activate
+
+If you would like to continue using `pyspark.sql` but have it use SQLFrame behind the scenes, you can use the [activate](./configuration.md#activating-sqlframe) function.
+
+```python
+import os
+
+from databricks.sql import connect
+from sqlframe import activate
+conn = connect(
+    server_hostname="dbc-xxxxxxxx-xxxx.cloud.databricks.com",
+    http_path="/sql/1.0/warehouses/xxxxxxxxxxxxxxxx",
+    access_token=os.environ["ACCESS_TOKEN"],  # Replace this with how you get your databricks access token
+    auth_type="access_token",
+    catalog="catalog",
+    schema="schema",
+)
+activate("databricks", conn=conn)
+
+from pyspark.sql import SparkSession
+```
+
+`SparkSession` will now be a SQLFrame `DatabricksSession` object and everything will be run on Databricks directly.
+
+See [activate configuration](./configuration.md#activating-sqlframe) for information on how to pass in a connection and config options.
+
+## Creating a Session
+
+SQLFrame uses [Databricks SQL Connector for Python](https://github.com/databricks/databricks-sql-python) to connect to Databricks. 
+A DatabricksSession, which implements the PySpark Session API, is created by passing in a `databricks.sql.client.Connection` object.
+
+=== "Import"
+
+    ```python
+    import os
+   
+    from databricks.sql import connect
+    from sqlframe.databricks import DatabricksSession
+    
+    conn = connect(
+        server_hostname="dbc-xxxxxxxx-xxxx.cloud.databricks.com",
+        http_path="/sql/1.0/warehouses/xxxxxxxxxxxxxxxx",
+        access_token=os.environ["ACCESS_TOKEN"],  # Replace this with how you get your databricks access token
+        auth_type="access_token",
+        catalog="catalog",
+        schema="schema",
+    )
+    session = DatabricksSession(conn=conn)
+    ```
+
+=== "Activate"
+
+    ```python
+    import os
+
+    from databricks.sql import connect
+    from sqlframe import activate
+
+    conn = connect(
+        server_hostname="dbc-xxxxxxxx-xxxx.cloud.databricks.com",
+        http_path="/sql/1.0/warehouses/xxxxxxxxxxxxxxxx",
+        access_token=os.environ["ACCESS_TOKEN"],  # Replace this with how you get your databricks access token
+        auth_type="access_token",
+        catalog="catalog",
+        schema="schema",
+    )
+    activate("databricks", conn=conn)
+
+    from pyspark.sql import SparkSession
+    session = SparkSession.builder.getOrCreate()
+    ```
+
+## Example Usage
+
+```python
+import os
+
+from databricks.sql import connect
+from sqlframe import activate
+
+conn = connect(
+    server_hostname="dbc-xxxxxxxx-xxxx.cloud.databricks.com",
+    http_path="/sql/1.0/warehouses/xxxxxxxxxxxxxxxx",
+    access_token=os.environ["ACCESS_TOKEN"],  # Replace this with how you get your databricks access token
+    auth_type="access_token",
+    catalog="catalog",
+    schema="schema",
+)
+activate("databricks", conn=conn)
+
+from pyspark.sql import SparkSession
+from pyspark.sql import functions as F
+
+session = SparkSession.builder.getOrCreate()
+table_path = "samples.nyctaxi.trips"
+# Get columns in the table
+print(session.catalog.listColumns(table_path))
+# Get the number of rides per hour
+(
+    session.table(table_path)
+    .where(F.col("tpep_pickup_datetime").between("2016-01-01", "2016-01-16"))
+    .withColumn("dropoff_hour", F.hour(F.col("tpep_dropoff_datetime")))
+    .groupBy("dropoff_hour").count()
+    .select(
+        F.format_string('%02d:00', F.col("dropoff_hour")).alias("dropoff Hour"),
+        F.col("count").alias("number of rides")
+    ).orderBy("dropoff Hour")
+    .limit(5)
+    .show()
+)
+"""
++----------------+-------------------+
+| `dropoff hour` | `number of rides` |
++----------------+-------------------+
+|     00:00      |        205        |
+|     01:00      |        159        |
+|     02:00      |        117        |
+|     03:00      |         88        |
+|     04:00      |         73        |
++----------------+-------------------+
+"""
+```

sqlframe-3.9.1/docs/redshift.md

@@ -0,0 +1,162 @@
+# Redshift (In Development)
+
+## Installation
+
+```bash
+pip install "sqlframe[redshift]"
+```
+
+## Enabling SQLFrame
+
+SQLFrame can be used in two ways:
+
+* Directly importing the `sqlframe.redshift` package 
+* Using the [activate](./configuration.md#activating-sqlframe) function to allow for continuing to use `pyspark.sql` but have it use SQLFrame behind the scenes.
+
+### Import
+
+If converting a PySpark pipeline, all `pyspark.sql` should be replaced with `sqlframe.redshift`.
+In addition, many classes will have a `Redshift` prefix. 
+For example, `RedshiftDataFrame` instead of `DataFrame`.
+
+
+```python
+# PySpark import
+# from pyspark.sql import SparkSession
+# from pyspark.sql import functions as F
+# from pyspark.sql.dataframe import DataFrame
+# SQLFrame import
+from sqlframe.redshift import RedshiftSession
+from sqlframe.redshift import functions as F
+from sqlframe.redshift import RedshiftDataFrame
+```
+
+### Activate
+
+If you would like to continue using `pyspark.sql` but have it use SQLFrame behind the scenes, you can use the [activate](./configuration.md#activating-sqlframe) function.
+
+```python
+import os
+
+from redshift_connector import connect
+from sqlframe import activate
+conn = connect(
+    user="user",
+    password=os.environ["PASSWORD"],  # Replace this with how you get your password
+    database="database",
+    host="xxxxx.xxxxxx.region.redshift-serverless.amazonaws.com",
+    port=5439,
+)
+activate("redshift", conn=conn)
+
+from pyspark.sql import SparkSession
+```
+
+`SparkSession` will now be a SQLFrame `RedshiftSession` object and everything will be run on Redshift directly.
+
+See [activate configuration](./configuration.md#activating-sqlframe) for information on how to pass in a connection and config options.
+
+## Creating a Session
+
+SQLFrame uses [Redshift DBAPI Python Connector](https://github.com/aws/amazon-redshift-python-driver) to connect to Redshift. 
+A RedshiftSession, which implements the PySpark Session API, is created by passing in a `redshift_connector.Connection` object.
+
+=== "Import"
+
+    ```python
+    import os
+   
+    from redshift_connector import connect
+    from sqlframe.redshift import RedshiftSession
+    
+    conn = connect(
+        user="user",
+        password=os.environ["PASSWORD"],  # Replace this with how you get your password
+        database="database",
+        host="xxxxx.xxxxxx.region.redshift-serverless.amazonaws.com",
+        port=5439,
+    )
+    session = RedshiftSession(conn=conn)
+    ```
+
+=== "Activate"
+
+    ```python
+    import os
+
+    from redshift_connector import connect
+    from sqlframe import activate
+
+    conn = connect(
+        user="user",
+        password=os.environ["PASSWORD"],  # Replace this with how you get your password
+        database="database",
+        host="xxxxx.xxxxxx.region.redshift-serverless.amazonaws.com",
+        port=5439,
+    )
+    activate("redshift", conn=conn)
+
+    from pyspark.sql import SparkSession
+    session = SparkSession.builder.getOrCreate()
+    ```
+
+## Example Usage
+
+```python
+import os
+
+from redshift_connector import connect
+from sqlframe import activate
+
+conn = connect(
+    user="user",
+    password=os.environ["PASSWORD"],  # Replace this with how you get your password
+    database="database",
+    host="xxxxx.xxxxxx.region.redshift-serverless.amazonaws.com",
+    port=5439,
+)
+activate("redshift", conn=conn)
+
+from pyspark.sql import SparkSession
+from pyspark.sql import functions as F
+
+session = SparkSession.builder.getOrCreate()
+table_path = '"catalog.db.table"'
+# Get columns in the table
+print(session.catalog.listColumns(table_path))
+# Get the top 5 years with the greatest year-over-year % change in new families with a single child
+(
+    session.table(table_path)
+    .where(F.col("ever_born") == 1)
+    .groupBy("year")
+    .agg(F.count("*").alias("num_single_child_families"))
+    .withColumn(
+      "last_year_num_single_child_families", 
+      F.lag(F.col("num_single_child_families"), 1).over(Window.orderBy("year"))
+    )
+    .withColumn(
+      "percent_change", 
+      (F.col("num_single_child_families") - F.col("last_year_num_single_child_families")) 
+      / F.col("last_year_num_single_child_families")
+    )
+    .orderBy(F.abs(F.col("percent_change")).desc())
+    .select(
+        F.col("year").alias("year"),
+        F.format_number("num_single_child_families", 0).alias("new families single child"),
+        F.format_number(F.col("percent_change") * 100, 2).alias("percent change"),
+    )
+    .limit(5)
+    .show()
+)
+"""
++------+---------------------------+----------------+
+| year | new families single child | percent change |
++------+---------------------------+----------------+
+| 1989 |         1,650,246         |     25.02      |
+| 1974 |          783,448          |     14.49      |
+| 1977 |         1,057,379         |     11.38      |
+| 1985 |         1,308,476         |     11.15      |
+| 1975 |          868,985          |     10.92      |
++------+---------------------------+----------------+
+"""
+```

{sqlframe-3.8.2 → sqlframe-3.9.1}/mkdocs.yml

@@ -9,6 +9,8 @@ nav:
   - "Spark": spark.md
   - "Standalone": standalone.md
   - "Configuration": configuration.md
+  - "Redshift (In-Development)": redshift.md
+  - "Databricks (In-Development)": databricks.md
 theme:
   name: material
   logo: images/SF.png

{sqlframe-3.8.2 → sqlframe-3.9.1}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.8",
     install_requires=[
         "prettytable<3.12.1",
-        "sqlglot>=24.0.0,<25.32",
+        "sqlglot>=24.0.0,<25.33",
         "typing_extensions>=4.8,<5",
     ],
     extras_require={
@@ -32,7 +32,7 @@ setup(
             "duckdb>=0.9,<1.2",
             "findspark>=2,<3",
             "mypy>=1.10.0,<1.14",
-            "openai>=1.30,<1.55",
+            "openai>=1.30,<1.56",
             "pandas>=2,<3",
             "pandas-stubs>=2,<3",
             "psycopg>=3.1,<4",
@@ -43,7 +43,7 @@ setup(
             "pytest-xdist>=3.6,<3.7",
             "pre-commit>=3.5;python_version=='3.8'",
             "pre-commit>=3.7,<4.1;python_version>='3.9'",
-            "ruff>=0.4.4,<0.8",
+            "ruff>=0.4.4,<0.9",
             "types-psycopg2>=2.9,<3",
         ],
         "docs": [
@@ -58,7 +58,7 @@ setup(
             "pandas>=2,<3",
         ],
         "openai": [
-            "openai>=1.30,<1.55",
+            "openai>=1.30,<1.56",
         ],
         "pandas": [
             "pandas>=2,<3",
@@ -75,6 +75,9 @@ setup(
         "spark": [
             "pyspark>=2,<3.6",
         ],
+        "databricks": [
+            "databricks-sql-connector>=3.6,<4",
+        ],
     },
     classifiers=[
         "Development Status :: 5 - Production/Stable",

{sqlframe-3.8.2 → sqlframe-3.9.1}/sqlframe/__init__.py

@@ -15,6 +15,7 @@ ENGINE_TO_PREFIX = {
     "redshift": "Redshift",
     "snowflake": "Snowflake",
     "spark": "Spark",
+    "databricks": "Databricks",
     "standalone": "Standalone",
 }
 

{sqlframe-3.8.2 → sqlframe-3.9.1}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.8.2'
-__version_tuple__ = version_tuple = (3, 8, 2)
+__version__ = version = '3.9.1'
+__version_tuple__ = version_tuple = (3, 9, 1)

{sqlframe-3.8.2 → sqlframe-3.9.1}/sqlframe/base/dataframe.py

@@ -629,10 +629,10 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                 # We will drop the "view" if it exists before running the cache table
                 output_expressions.append(exp.Drop(this=cache_table, exists=True, kind="VIEW"))
             elif expression_type == exp.Create:
-                expression = df.output_expression_container.copy()
+                expression = df.output_expression_container.copy()  # type: ignore
                 expression.set("expression", select_expression)
             elif expression_type == exp.Insert:
-                expression = df.output_expression_container.copy()
+                expression = df.output_expression_container.copy()  # type: ignore
                 select_without_ctes = select_expression.copy()
                 select_without_ctes.set("with", None)
                 expression.set("expression", select_without_ctes)
@@ -866,6 +866,7 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         if on is None:
             logger.warning("Got no value for on. This appears to change the join to a cross join.")
             how = "cross"
+
         other_df = other_df._convert_leaf_to_cte()
         join_expression = self._add_ctes_to_expression(self.expression, other_df.expression.ctes)
         # We will determine actual "join on" expression later so we don't provide it at first
@@ -875,6 +876,13 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         self_columns = self._get_outer_select_columns(join_expression)
         other_columns = self._get_outer_select_columns(other_df.expression)
         join_columns = self._ensure_and_normalize_cols(on)
+        # If the two dataframes being joined come from the same branch and reference the same table,
+        # then really one of the tables was supposed to reference the other dataframe so we update it to do that.
+        if self.branch_id == other_df.branch_id:
+            for col in join_columns:
+                for eq in col.expression.find_all(exp.EQ):
+                    if eq.this.table == eq.expression.table:
+                        eq.expression.set("table", exp.to_identifier(other_df.latest_cte_name))
         # Determines the join clause and select columns to be used passed on what type of columns were provided for
         # the join. The columns returned changes based on how the on expression is provided.
         if how != "cross":

sqlframe-3.9.1/sqlframe/databricks/__init__.py

@@ -0,0 +1,32 @@
+from sqlframe.databricks.catalog import DatabricksCatalog
+from sqlframe.databricks.column import Column
+from sqlframe.databricks.dataframe import (
+    DatabricksDataFrame,
+    DatabricksDataFrameNaFunctions,
+    DatabricksDataFrameStatFunctions,
+)
+from sqlframe.databricks.group import DatabricksGroupedData
+from sqlframe.databricks.readwriter import (
+    DatabricksDataFrameReader,
+    DatabricksDataFrameWriter,
+)
+from sqlframe.databricks.session import DatabricksSession
+from sqlframe.databricks.types import Row
+from sqlframe.databricks.udf import DatabricksUDFRegistration
+from sqlframe.databricks.window import Window, WindowSpec
+
+__all__ = [
+    "Column",
+    "Row",
+    "DatabricksCatalog",
+    "DatabricksDataFrame",
+    "DatabricksDataFrameNaFunctions",
+    "DatabricksGroupedData",
+    "DatabricksDataFrameReader",
+    "DatabricksDataFrameWriter",
+    "DatabricksSession",
+    "DatabricksDataFrameStatFunctions",
+    "DatabricksUDFRegistration",
+    "Window",
+    "WindowSpec",
+]