sqlframe 3.14.0__tar.gz → 3.14.2__tar.gz

{sqlframe-3.14.0 → sqlframe-3.14.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.14.0
+Version: 3.14.2
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.14.0 → sqlframe-3.14.2}/docs/bigquery.md

@@ -641,7 +641,7 @@ df_employee = session.createDataFrame(
 
 df_employee.write.mode("overwrite").saveAsTable("employee")
 
-table_employee = session.table("employee")  # This object is of Type DatabricksTable
+table_employee = session.table("employee")  # This object is of Type BigqueryTable
 ```
 
 #### Update Statement

{sqlframe-3.14.0 → sqlframe-3.14.2}/docs/duckdb.md

@@ -592,7 +592,7 @@ df_employee = session.createDataFrame(
 
 df_employee.write.mode("overwrite").saveAsTable("employee")
 
-table_employee = session.table("employee")  # This object is of Type DatabricksTable
+table_employee = session.table("employee")  # This object is of Type DuckDBTable
 ```
 
 #### Update Statement

{sqlframe-3.14.0 → sqlframe-3.14.2}/docs/postgres.md

@@ -587,7 +587,7 @@ df_employee = session.createDataFrame(
 
 df_employee.write.mode("overwrite").saveAsTable("employee")
 
-table_employee = session.table("employee")  # This object is of Type DatabricksTable
+table_employee = session.table("employee")  # This object is of Type PostgresTable
 ```
 
 #### Update Statement

{sqlframe-3.14.0 → sqlframe-3.14.2}/docs/redshift.md

@@ -196,7 +196,7 @@ df_employee = session.createDataFrame(
 
 df_employee.write.mode("overwrite").saveAsTable("employee")
 
-table_employee = session.table("employee")  # This object is of Type DatabricksTable
+table_employee = session.table("employee")  # This object is of Type RedshiftTable
 ```
 
 #### Update Statement

{sqlframe-3.14.0 → sqlframe-3.14.2}/docs/snowflake.md

@@ -642,7 +642,7 @@ df_employee = session.createDataFrame(
 
 df_employee.write.mode("overwrite").saveAsTable("employee")
 
-table_employee = session.table("employee")  # This object is of Type DatabricksTable
+table_employee = session.table("employee")  # This object is of Type SnowflakeTable
 ```
 
 #### Update Statement

{sqlframe-3.14.0 → sqlframe-3.14.2}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<26.3",
+        "sqlglot>=24.0.0,<26.4",
         "typing_extensions",
     ],
     extras_require={
@@ -70,13 +70,13 @@ setup(
             "redshift_connector>=2.1.1,<2.2.0",
         ],
         "snowflake": [
-            "snowflake-connector-python[secure-local-storage]>=3.10.0,<3.13",
+            "snowflake-connector-python[secure-local-storage]>=3.10.0,<3.14",
         ],
         "spark": [
             "pyspark>=2,<3.6",
         ],
         "databricks": [
-            "databricks-sql-connector>=3.6,<4",
+            "databricks-sql-connector>=3.6,<5",
         ],
     },
     classifiers=[

{sqlframe-3.14.0 → sqlframe-3.14.2}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.14.0'
-__version_tuple__ = version_tuple = (3, 14, 0)
+__version__ = version = '3.14.2'
+__version_tuple__ = version_tuple = (3, 14, 2)

{sqlframe-3.14.0 → sqlframe-3.14.2}/sqlframe/base/dataframe.py

@@ -9,6 +9,7 @@ import json
 import logging
 import sys
 import typing as t
+import uuid
 import zlib
 from copy import copy
 from dataclasses import dataclass
@@ -79,6 +80,23 @@ JOIN_HINTS = {
     "SHUFFLE_REPLICATE_NL",
 }
 
+JOIN_TYPE_MAPPING = {
+    "inner": "inner",
+    "cross": "cross",
+    "outer": "full_outer",
+    "full": "full_outer",
+    "fullouter": "full_outer",
+    "left": "left_outer",
+    "leftouter": "left_outer",
+    "right": "right_outer",
+    "rightouter": "right_outer",
+    "semi": "left_semi",
+    "leftsemi": "left_semi",
+    "left_semi": "left_semi",
+    "anti": "left_anti",
+    "leftanti": "left_anti",
+    "left_anti": "left_anti",
+}
 
 DF = t.TypeVar("DF", bound="BaseDataFrame")
 
@@ -474,22 +492,20 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         with_expression = expression.args.get("with")
         if with_expression:
             existing_ctes = with_expression.expressions
-            existing_cte_counts = {x.alias_or_name: 0 for x in existing_ctes}
+            existing_cte_names = {x.alias_or_name for x in existing_ctes}
             replaced_cte_names = {}  # type: ignore
             for cte in ctes:
                 if replaced_cte_names:
                     cte = cte.transform(replace_id_value, replaced_cte_names)  # type: ignore
-                if cte.alias_or_name in existing_cte_counts:
-                    existing_cte_counts[cte.alias_or_name] += 10
+                if cte.alias_or_name in existing_cte_names:
+                    random_filter = exp.Literal.string(uuid.uuid4().hex)
                     # Add unique where filter to ensure that the hash of the CTE is unique
                     cte.set(
                         "this",
                         cte.this.where(
                             exp.EQ(
-                                this=exp.Literal.number(existing_cte_counts[cte.alias_or_name]),
-                                expression=exp.Literal.number(
-                                    existing_cte_counts[cte.alias_or_name]
-                                ),
+                                this=random_filter,
+                                expression=random_filter,
                             )
                         ),
                     )
@@ -503,7 +519,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                             new_cte_alias, dialect=self.session.input_dialect, into=exp.TableAlias
                         ),
                     )
-                    existing_cte_counts[new_cte_alias] = 0
+                    existing_cte_names.add(new_cte_alias)
                 existing_ctes.append(cte)
         else:
             existing_ctes = ctes
@@ -944,16 +960,20 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
     ) -> Self:
         from sqlframe.base.functions import coalesce
 
-        if on is None:
+        if (on is None) and ("cross" not in how):
             logger.warning("Got no value for on. This appears to change the join to a cross join.")
             how = "cross"
+        if (on is not None) and ("cross" in how):
+            # Not a lot of doc, but Spark handles cross with predicate as an inner join
+            # https://learn.microsoft.com/en-us/dotnet/api/microsoft.spark.sql.dataframe.join
+            logger.warning("Got cross join with an 'on' value. This will result in an inner join.")
+            how = "inner"
 
         other_df = other_df._convert_leaf_to_cte()
         join_expression = self._add_ctes_to_expression(self.expression, other_df.expression.ctes)
         # We will determine actual "join on" expression later so we don't provide it at first
-        join_expression = join_expression.join(
-            join_expression.ctes[-1].alias, join_type=how.replace("_", " ")
-        )
+        join_type = JOIN_TYPE_MAPPING.get(how, how).replace("_", " ")
+        join_expression = join_expression.join(join_expression.ctes[-1].alias, join_type=join_type)
         self_columns = self._get_outer_select_columns(join_expression)
         other_columns = self._get_outer_select_columns(other_df.expression)
         join_columns = self._ensure_and_normalize_cols(on)
@@ -961,7 +981,12 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
 
         # Determines the join clause and select columns to be used passed on what type of columns were provided for
         # the join. The columns returned changes based on how the on expression is provided.
-        if how != "cross":
+        select_columns = (
+            self_columns
+            if join_type in ["left anti", "left semi"]
+            else self_columns + other_columns
+        )
+        if join_type != "cross":
             if isinstance(join_columns[0].expression, exp.Column):
                 """
                 Unique characteristics of join on column names only:
@@ -992,7 +1017,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                         if not isinstance(column.expression.this, exp.Star)
                         else column.sql()
                     )
-                    for column in self_columns + other_columns
+                    for column in select_columns
                 ]
                 select_column_names = [
                     column_name
@@ -1010,13 +1035,11 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                 * The left join dataframe columns go first and right come after. No sort preference is given to join columns
                 """
                 join_clause = self._normalize_join_clause(join_columns, join_expression)
-                select_column_names = [
-                    column.alias_or_name for column in self_columns + other_columns
-                ]
+                select_column_names = [column.alias_or_name for column in select_columns]
 
             # Update the on expression with the actual join clause to replace the dummy one from before
         else:
-            select_column_names = [column.alias_or_name for column in self_columns + other_columns]
+            select_column_names = [column.alias_or_name for column in select_columns]
             join_clause = None
         join_expression.args["joins"][-1].set("on", join_clause.expression if join_clause else None)
         new_df = self.copy(expression=join_expression)

{sqlframe-3.14.0 → sqlframe-3.14.2}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.14.0
+Version: 3.14.2
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.14.0 → sqlframe-3.14.2}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<4
-sqlglot<26.3,>=24.0.0
+sqlglot<26.4,>=24.0.0
 typing_extensions
 
 [bigquery]
@@ -7,7 +7,7 @@ google-cloud-bigquery-storage<3,>=2
 google-cloud-bigquery[pandas]<4,>=3
 
 [databricks]
-databricks-sql-connector<4,>=3.6
+databricks-sql-connector<5,>=3.6
 
 [dev]
 duckdb<1.2,>=0.9
@@ -51,7 +51,7 @@ psycopg2<3,>=2.8
 redshift_connector<2.2.0,>=2.1.1
 
 [snowflake]
-snowflake-connector-python[secure-local-storage]<3.13,>=3.10.0
+snowflake-connector-python[secure-local-storage]<3.14,>=3.10.0
 
 [spark]
 pyspark<3.6,>=2

{sqlframe-3.14.0 → sqlframe-3.14.2}/tests/integration/test_int_dataframe.py

@@ -435,6 +435,43 @@ def test_join_inner(
     compare_frames(df_joined, dfs_joined, sort=True)
 
 
+@pytest.mark.parametrize(
+    "how",
+    [
+        "inner",
+        "cross",
+        "outer",
+        "full",
+        "fullouter",
+        "full_outer",
+        "left",
+        "leftouter",
+        "left_outer",
+        "right",
+        "rightouter",
+        "right_outer",
+        "semi",
+        "leftsemi",
+        "left_semi",
+        "anti",
+        "leftanti",
+        "left_anti",
+    ],
+)
+def test_join_various_how(
+    pyspark_employee: PySparkDataFrame,
+    pyspark_store: PySparkDataFrame,
+    get_df: t.Callable[[str], BaseDataFrame],
+    compare_frames: t.Callable,
+    how: str,
+):
+    employee = get_df("employee")
+    store = get_df("store")
+    df_joined = pyspark_employee.join(pyspark_store, on=["store_id"], how=how)
+    dfs_joined = employee.join(store, on=["store_id"], how=how)
+    compare_frames(df_joined, dfs_joined, sort=True)
+
+
 def test_join_inner_no_select(
     pyspark_employee: PySparkDataFrame,
     pyspark_store: PySparkDataFrame,
@@ -2318,3 +2355,29 @@ def test_union_common_root(
     dfs_final = dfs_1.union(dfs_2)
 
     compare_frames(df_final, dfs_final, compare_schema=False)
+
+
+# https://github.com/eakmanrq/sqlframe/issues/253
+def test_union_common_root_again(
+    pyspark_employee: PySparkDataFrame,
+    get_df: t.Callable[[str], BaseDataFrame],
+    compare_frames: t.Callable,
+):
+    df_1 = pyspark_employee.filter(F.col("age") > 40)
+    df_2 = df_1.join(
+        pyspark_employee.select("employee_id").distinct(),
+        on="employee_id",
+        how="right",
+    )
+    df_final = df_1.union(df_2).union(pyspark_employee)
+
+    employee = get_df("employee")
+    dfs_1 = employee.filter(SF.col("age") > 40)
+    dfs_2 = dfs_1.join(
+        employee.select("employee_id").distinct(),
+        on="employee_id",
+        how="right",
+    )
+    dfs_final = dfs_1.union(dfs_2).union(employee)
+
+    compare_frames(df_final, dfs_final, compare_schema=False)