sqlframe 3.14.0__py3-none-any.whl → 3.14.2__py3-none-any.whl

sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.14.0'
-__version_tuple__ = version_tuple = (3, 14, 0)
+__version__ = version = '3.14.2'
+__version_tuple__ = version_tuple = (3, 14, 2)

sqlframe/base/dataframe.py

@@ -9,6 +9,7 @@ import json
 import logging
 import sys
 import typing as t
+import uuid
 import zlib
 from copy import copy
 from dataclasses import dataclass
@@ -79,6 +80,23 @@ JOIN_HINTS = {
     "SHUFFLE_REPLICATE_NL",
 }
 
+JOIN_TYPE_MAPPING = {
+    "inner": "inner",
+    "cross": "cross",
+    "outer": "full_outer",
+    "full": "full_outer",
+    "fullouter": "full_outer",
+    "left": "left_outer",
+    "leftouter": "left_outer",
+    "right": "right_outer",
+    "rightouter": "right_outer",
+    "semi": "left_semi",
+    "leftsemi": "left_semi",
+    "left_semi": "left_semi",
+    "anti": "left_anti",
+    "leftanti": "left_anti",
+    "left_anti": "left_anti",
+}
 
 DF = t.TypeVar("DF", bound="BaseDataFrame")
 
@@ -474,22 +492,20 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         with_expression = expression.args.get("with")
         if with_expression:
             existing_ctes = with_expression.expressions
-            existing_cte_counts = {x.alias_or_name: 0 for x in existing_ctes}
+            existing_cte_names = {x.alias_or_name for x in existing_ctes}
             replaced_cte_names = {}  # type: ignore
             for cte in ctes:
                 if replaced_cte_names:
                     cte = cte.transform(replace_id_value, replaced_cte_names)  # type: ignore
-                if cte.alias_or_name in existing_cte_counts:
-                    existing_cte_counts[cte.alias_or_name] += 10
+                if cte.alias_or_name in existing_cte_names:
+                    random_filter = exp.Literal.string(uuid.uuid4().hex)
                     # Add unique where filter to ensure that the hash of the CTE is unique
                     cte.set(
                         "this",
                         cte.this.where(
                             exp.EQ(
-                                this=exp.Literal.number(existing_cte_counts[cte.alias_or_name]),
-                                expression=exp.Literal.number(
-                                    existing_cte_counts[cte.alias_or_name]
-                                ),
+                                this=random_filter,
+                                expression=random_filter,
                             )
                         ),
                     )
@@ -503,7 +519,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                             new_cte_alias, dialect=self.session.input_dialect, into=exp.TableAlias
                         ),
                     )
-                    existing_cte_counts[new_cte_alias] = 0
+                    existing_cte_names.add(new_cte_alias)
                 existing_ctes.append(cte)
         else:
             existing_ctes = ctes
@@ -944,16 +960,20 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
     ) -> Self:
         from sqlframe.base.functions import coalesce
 
-        if on is None:
+        if (on is None) and ("cross" not in how):
             logger.warning("Got no value for on. This appears to change the join to a cross join.")
             how = "cross"
+        if (on is not None) and ("cross" in how):
+            # Not a lot of doc, but Spark handles cross with predicate as an inner join
+            # https://learn.microsoft.com/en-us/dotnet/api/microsoft.spark.sql.dataframe.join
+            logger.warning("Got cross join with an 'on' value. This will result in an inner join.")
+            how = "inner"
 
         other_df = other_df._convert_leaf_to_cte()
         join_expression = self._add_ctes_to_expression(self.expression, other_df.expression.ctes)
         # We will determine actual "join on" expression later so we don't provide it at first
-        join_expression = join_expression.join(
-            join_expression.ctes[-1].alias, join_type=how.replace("_", " ")
-        )
+        join_type = JOIN_TYPE_MAPPING.get(how, how).replace("_", " ")
+        join_expression = join_expression.join(join_expression.ctes[-1].alias, join_type=join_type)
         self_columns = self._get_outer_select_columns(join_expression)
         other_columns = self._get_outer_select_columns(other_df.expression)
         join_columns = self._ensure_and_normalize_cols(on)
@@ -961,7 +981,12 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
 
         # Determines the join clause and select columns to be used passed on what type of columns were provided for
         # the join. The columns returned changes based on how the on expression is provided.
-        if how != "cross":
+        select_columns = (
+            self_columns
+            if join_type in ["left anti", "left semi"]
+            else self_columns + other_columns
+        )
+        if join_type != "cross":
             if isinstance(join_columns[0].expression, exp.Column):
                 """
                 Unique characteristics of join on column names only:
@@ -992,7 +1017,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                         if not isinstance(column.expression.this, exp.Star)
                         else column.sql()
                     )
-                    for column in self_columns + other_columns
+                    for column in select_columns
                 ]
                 select_column_names = [
                     column_name
@@ -1010,13 +1035,11 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                 * The left join dataframe columns go first and right come after. No sort preference is given to join columns
                 """
                 join_clause = self._normalize_join_clause(join_columns, join_expression)
-                select_column_names = [
-                    column.alias_or_name for column in self_columns + other_columns
-                ]
+                select_column_names = [column.alias_or_name for column in select_columns]
 
             # Update the on expression with the actual join clause to replace the dummy one from before
         else:
-            select_column_names = [column.alias_or_name for column in self_columns + other_columns]
+            select_column_names = [column.alias_or_name for column in select_columns]
             join_clause = None
         join_expression.args["joins"][-1].set("on", join_clause.expression if join_clause else None)
         new_df = self.copy(expression=join_expression)

{sqlframe-3.14.0.dist-info → sqlframe-3.14.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.14.0
+Version: 3.14.2
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman
@@ -17,13 +17,13 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: prettytable <4
-Requires-Dist: sqlglot <26.3,>=24.0.0
+Requires-Dist: sqlglot <26.4,>=24.0.0
 Requires-Dist: typing-extensions
 Provides-Extra: bigquery
 Requires-Dist: google-cloud-bigquery-storage <3,>=2 ; extra == 'bigquery'
 Requires-Dist: google-cloud-bigquery[pandas] <4,>=3 ; extra == 'bigquery'
 Provides-Extra: databricks
-Requires-Dist: databricks-sql-connector <4,>=3.6 ; extra == 'databricks'
+Requires-Dist: databricks-sql-connector <5,>=3.6 ; extra == 'databricks'
 Provides-Extra: dev
 Requires-Dist: duckdb <1.2,>=0.9 ; extra == 'dev'
 Requires-Dist: findspark <3,>=2 ; extra == 'dev'
@@ -59,7 +59,7 @@ Requires-Dist: psycopg2 <3,>=2.8 ; extra == 'postgres'
 Provides-Extra: redshift
 Requires-Dist: redshift-connector <2.2.0,>=2.1.1 ; extra == 'redshift'
 Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python[secure-local-storage] <3.13,>=3.10.0 ; extra == 'snowflake'
+Requires-Dist: snowflake-connector-python[secure-local-storage] <3.14,>=3.10.0 ; extra == 'snowflake'
 Provides-Extra: spark
 Requires-Dist: pyspark <3.6,>=2 ; extra == 'spark'
 

{sqlframe-3.14.0.dist-info → sqlframe-3.14.2.dist-info}/RECORD

@@ -1,10 +1,10 @@
 sqlframe/__init__.py,sha256=wfqm98eLoLid9oV_FzzpG5loKC6LxOhj2lXpfN7SARo,3138
-sqlframe/_version.py,sha256=Ipjekae6alpGZC2b94mJAE2S2ZyJybTBe3oNCWsIFS4,413
+sqlframe/_version.py,sha256=jOmVUgfrjHuKbVMclbnyeOg5hq5CSFD43rU-r3QVgI8,413
 sqlframe/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlframe/base/_typing.py,sha256=b2clI5HI1zEZKB_3Msx3FeAJQyft44ubUifJwQRVXyQ,1298
 sqlframe/base/catalog.py,sha256=SzFQalTWdhWzxUY-4ut1f9TfOECp_JmJEgNPfrRKCe0,38457
 sqlframe/base/column.py,sha256=wRghgieYAA51aw4WuFQWOvl0TFOToZbBhBuIamEzxx4,18011
-sqlframe/base/dataframe.py,sha256=DuvAT_xBqhiOVZgyYCXL5J01ahHEPp_qvx_62uHqbu4,75768
+sqlframe/base/dataframe.py,sha256=E1zWlB_a2FNOxjTcQ68MtL_A4c8fnLiHY3MeZttK4Xk,76570
 sqlframe/base/decorators.py,sha256=P56cgs8DANxGRIwVs5uOMnDy-BlXZZYMbf4fdnkpWPI,1889
 sqlframe/base/exceptions.py,sha256=9Uwvqn2eAkDpqm4BrRgbL61qM-GMCbJEMAW8otxO46s,370
 sqlframe/base/function_alternatives.py,sha256=8kDCh1cOXtdCcBPYBQ8byXxRAZvphS9N8GDs4txBzGg,52544
@@ -129,8 +129,8 @@ sqlframe/standalone/udf.py,sha256=azmgtUjHNIPs0WMVNId05SHwiYn41MKVBhKXsQJ5dmY,27
 sqlframe/standalone/window.py,sha256=6GKPzuxeSapJakBaKBeT9VpED1ACdjggDv9JRILDyV0,35
 sqlframe/testing/__init__.py,sha256=VVCosQhitU74A3NnE52O4mNtGZONapuEXcc20QmSlnQ,132
 sqlframe/testing/utils.py,sha256=PFsGZpwNUE_4-g_f43_vstTqsK0AQ2lBneb5Eb6NkFo,13008
-sqlframe-3.14.0.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
-sqlframe-3.14.0.dist-info/METADATA,sha256=Gvp37AedPVOp_1Rh4qf5B4s8fkReVysUqCySQesSl6s,8970
-sqlframe-3.14.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-sqlframe-3.14.0.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
-sqlframe-3.14.0.dist-info/RECORD,,
+sqlframe-3.14.2.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
+sqlframe-3.14.2.dist-info/METADATA,sha256=jaarma0pQSOhwGo8XtkdteTdJadSB4CIiVrjLLQovu0,8970
+sqlframe-3.14.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+sqlframe-3.14.2.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
+sqlframe-3.14.2.dist-info/RECORD,,