sqlframe 3.14.0__tar.gz → 3.14.1__tar.gz

{sqlframe-3.14.0 → sqlframe-3.14.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.14.0
+Version: 3.14.1
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.14.0 → sqlframe-3.14.1}/docs/bigquery.md

@@ -641,7 +641,7 @@ df_employee = session.createDataFrame(
 
 df_employee.write.mode("overwrite").saveAsTable("employee")
 
-table_employee = session.table("employee")  # This object is of Type DatabricksTable
+table_employee = session.table("employee")  # This object is of Type BigqueryTable
 ```
 
 #### Update Statement

{sqlframe-3.14.0 → sqlframe-3.14.1}/docs/duckdb.md

@@ -592,7 +592,7 @@ df_employee = session.createDataFrame(
 
 df_employee.write.mode("overwrite").saveAsTable("employee")
 
-table_employee = session.table("employee")  # This object is of Type DatabricksTable
+table_employee = session.table("employee")  # This object is of Type DuckDBTable
 ```
 
 #### Update Statement

{sqlframe-3.14.0 → sqlframe-3.14.1}/docs/postgres.md

@@ -587,7 +587,7 @@ df_employee = session.createDataFrame(
 
 df_employee.write.mode("overwrite").saveAsTable("employee")
 
-table_employee = session.table("employee")  # This object is of Type DatabricksTable
+table_employee = session.table("employee")  # This object is of Type PostgresTable
 ```
 
 #### Update Statement

{sqlframe-3.14.0 → sqlframe-3.14.1}/docs/redshift.md

@@ -196,7 +196,7 @@ df_employee = session.createDataFrame(
 
 df_employee.write.mode("overwrite").saveAsTable("employee")
 
-table_employee = session.table("employee")  # This object is of Type DatabricksTable
+table_employee = session.table("employee")  # This object is of Type RedshiftTable
 ```
 
 #### Update Statement

{sqlframe-3.14.0 → sqlframe-3.14.1}/docs/snowflake.md

@@ -642,7 +642,7 @@ df_employee = session.createDataFrame(
 
 df_employee.write.mode("overwrite").saveAsTable("employee")
 
-table_employee = session.table("employee")  # This object is of Type DatabricksTable
+table_employee = session.table("employee")  # This object is of Type SnowflakeTable
 ```
 
 #### Update Statement

{sqlframe-3.14.0 → sqlframe-3.14.1}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<26.3",
+        "sqlglot>=24.0.0,<26.4",
         "typing_extensions",
     ],
     extras_require={
@@ -70,13 +70,13 @@ setup(
             "redshift_connector>=2.1.1,<2.2.0",
         ],
         "snowflake": [
-            "snowflake-connector-python[secure-local-storage]>=3.10.0,<3.13",
+            "snowflake-connector-python[secure-local-storage]>=3.10.0,<3.14",
         ],
         "spark": [
             "pyspark>=2,<3.6",
         ],
         "databricks": [
-            "databricks-sql-connector>=3.6,<4",
+            "databricks-sql-connector>=3.6,<5",
         ],
     },
     classifiers=[

{sqlframe-3.14.0 → sqlframe-3.14.1}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.14.0'
-__version_tuple__ = version_tuple = (3, 14, 0)
+__version__ = version = '3.14.1'
+__version_tuple__ = version_tuple = (3, 14, 1)

{sqlframe-3.14.0 → sqlframe-3.14.1}/sqlframe/base/dataframe.py

@@ -79,6 +79,23 @@ JOIN_HINTS = {
     "SHUFFLE_REPLICATE_NL",
 }
 
+JOIN_TYPE_MAPPING = {
+    "inner": "inner",
+    "cross": "cross",
+    "outer": "full_outer",
+    "full": "full_outer",
+    "fullouter": "full_outer",
+    "left": "left_outer",
+    "leftouter": "left_outer",
+    "right": "right_outer",
+    "rightouter": "right_outer",
+    "semi": "left_semi",
+    "leftsemi": "left_semi",
+    "left_semi": "left_semi",
+    "anti": "left_anti",
+    "leftanti": "left_anti",
+    "left_anti": "left_anti",
+}
 
 DF = t.TypeVar("DF", bound="BaseDataFrame")
 
@@ -944,16 +961,20 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
     ) -> Self:
         from sqlframe.base.functions import coalesce
 
-        if on is None:
+        if (on is None) and ("cross" not in how):
             logger.warning("Got no value for on. This appears to change the join to a cross join.")
             how = "cross"
+        if (on is not None) and ("cross" in how):
+            # Not a lot of doc, but Spark handles cross with predicate as an inner join
+            # https://learn.microsoft.com/en-us/dotnet/api/microsoft.spark.sql.dataframe.join
+            logger.warning("Got cross join with an 'on' value. This will result in an inner join.")
+            how = "inner"
 
         other_df = other_df._convert_leaf_to_cte()
         join_expression = self._add_ctes_to_expression(self.expression, other_df.expression.ctes)
         # We will determine actual "join on" expression later so we don't provide it at first
-        join_expression = join_expression.join(
-            join_expression.ctes[-1].alias, join_type=how.replace("_", " ")
-        )
+        join_type = JOIN_TYPE_MAPPING.get(how, how).replace("_", " ")
+        join_expression = join_expression.join(join_expression.ctes[-1].alias, join_type=join_type)
         self_columns = self._get_outer_select_columns(join_expression)
         other_columns = self._get_outer_select_columns(other_df.expression)
         join_columns = self._ensure_and_normalize_cols(on)
@@ -961,7 +982,12 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
 
         # Determines the join clause and select columns to be used passed on what type of columns were provided for
         # the join. The columns returned changes based on how the on expression is provided.
-        if how != "cross":
+        select_columns = (
+            self_columns
+            if join_type in ["left anti", "left semi"]
+            else self_columns + other_columns
+        )
+        if join_type != "cross":
             if isinstance(join_columns[0].expression, exp.Column):
                 """
                 Unique characteristics of join on column names only:
@@ -992,7 +1018,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                         if not isinstance(column.expression.this, exp.Star)
                         else column.sql()
                     )
-                    for column in self_columns + other_columns
+                    for column in select_columns
                 ]
                 select_column_names = [
                     column_name
@@ -1010,13 +1036,11 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                 * The left join dataframe columns go first and right come after. No sort preference is given to join columns
                 """
                 join_clause = self._normalize_join_clause(join_columns, join_expression)
-                select_column_names = [
-                    column.alias_or_name for column in self_columns + other_columns
-                ]
+                select_column_names = [column.alias_or_name for column in select_columns]
 
             # Update the on expression with the actual join clause to replace the dummy one from before
         else:
-            select_column_names = [column.alias_or_name for column in self_columns + other_columns]
+            select_column_names = [column.alias_or_name for column in select_columns]
             join_clause = None
         join_expression.args["joins"][-1].set("on", join_clause.expression if join_clause else None)
         new_df = self.copy(expression=join_expression)

{sqlframe-3.14.0 → sqlframe-3.14.1}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.14.0
+Version: 3.14.1
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.14.0 → sqlframe-3.14.1}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<4
-sqlglot<26.3,>=24.0.0
+sqlglot<26.4,>=24.0.0
 typing_extensions
 
 [bigquery]
@@ -7,7 +7,7 @@ google-cloud-bigquery-storage<3,>=2
 google-cloud-bigquery[pandas]<4,>=3
 
 [databricks]
-databricks-sql-connector<4,>=3.6
+databricks-sql-connector<5,>=3.6
 
 [dev]
 duckdb<1.2,>=0.9
@@ -51,7 +51,7 @@ psycopg2<3,>=2.8
 redshift_connector<2.2.0,>=2.1.1
 
 [snowflake]
-snowflake-connector-python[secure-local-storage]<3.13,>=3.10.0
+snowflake-connector-python[secure-local-storage]<3.14,>=3.10.0
 
 [spark]
 pyspark<3.6,>=2

{sqlframe-3.14.0 → sqlframe-3.14.1}/tests/integration/test_int_dataframe.py

@@ -435,6 +435,43 @@ def test_join_inner(
     compare_frames(df_joined, dfs_joined, sort=True)
 
 
+@pytest.mark.parametrize(
+    "how",
+    [
+        "inner",
+        "cross",
+        "outer",
+        "full",
+        "fullouter",
+        "full_outer",
+        "left",
+        "leftouter",
+        "left_outer",
+        "right",
+        "rightouter",
+        "right_outer",
+        "semi",
+        "leftsemi",
+        "left_semi",
+        "anti",
+        "leftanti",
+        "left_anti",
+    ],
+)
+def test_join_various_how(
+    pyspark_employee: PySparkDataFrame,
+    pyspark_store: PySparkDataFrame,
+    get_df: t.Callable[[str], BaseDataFrame],
+    compare_frames: t.Callable,
+    how: str,
+):
+    employee = get_df("employee")
+    store = get_df("store")
+    df_joined = pyspark_employee.join(pyspark_store, on=["store_id"], how=how)
+    dfs_joined = employee.join(store, on=["store_id"], how=how)
+    compare_frames(df_joined, dfs_joined, sort=True)
+
+
 def test_join_inner_no_select(
     pyspark_employee: PySparkDataFrame,
     pyspark_store: PySparkDataFrame,