sqlframe 1.6.0__tar.gz → 1.6.2__tar.gz

{sqlframe-1.6.0 → sqlframe-1.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 1.6.0
+Version: 1.6.2
 Summary: Taking the Spark out of PySpark by converting to SQL
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-1.6.0 → sqlframe-1.6.2}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.8",
     install_requires=[
         "prettytable<3.11.0",
-        "sqlglot>=24.0.0,<24.2",
+        "sqlglot>=24.0.0,<25.1",
     ],
     extras_require={
         "bigquery": [

{sqlframe-1.6.0 → sqlframe-1.6.2}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.6.0'
-__version_tuple__ = version_tuple = (1, 6, 0)
+__version__ = version = '1.6.2'
+__version_tuple__ = version_tuple = (1, 6, 2)

{sqlframe-1.6.0 → sqlframe-1.6.2}/sqlframe/base/column.py

@@ -247,6 +247,10 @@ class Column:
     def alias_or_name(self) -> str:
         return quote_preserving_alias_or_name(self.expression)  # type: ignore
 
+    @property
+    def column_alias_or_name(self) -> str:
+        return quote_preserving_alias_or_name(self.column_expression)  # type: ignore
+
     @classmethod
     def ensure_literal(cls, value) -> Column:
         from sqlframe.base.functions import lit

{sqlframe-1.6.0 → sqlframe-1.6.2}/sqlframe/base/dataframe.py

@@ -668,7 +668,7 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                         cte
                         for cte in self.expression.ctes
                         if cte.alias_or_name in cte_names_in_join
-                        and ambiguous_col.alias_or_name in cte.this.named_selects
+                        and ambiguous_col.column_alias_or_name in cte.this.named_selects
                     ]
                     # Check if there is a CTE with this column that we haven't used before. If so, use it. Otherwise,
                     # use the same CTE we used before
@@ -677,7 +677,9 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                         resolved_column_position[ambiguous_col] += 1
                     else:
                         cte = ctes_with_column[resolved_column_position[ambiguous_col]]
-                    ambiguous_col.expression.set("table", exp.to_identifier(cte.alias_or_name))
+                    ambiguous_col.column_expression.set(
+                        "table", exp.to_identifier(cte.alias_or_name)
+                    )
         # If an expression is `CAST(x AS DATETYPE)` then we want to alias so that `x` is the result column name
         columns = [
             col.alias(col.expression.alias_or_name)
@@ -774,6 +776,8 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         how: str = "inner",
         **kwargs,
     ) -> Self:
+        from sqlframe.base.functions import coalesce
+
         if on is None:
             logger.warning("Got no value for on. This appears change the join to a cross join.")
             how = "cross"
@@ -833,7 +837,15 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                         for left_column, right_column in join_column_pairs
                     ],
                 )
-                join_column_names = [left_col.alias_or_name for left_col, _ in join_column_pairs]
+                join_column_names = [
+                    coalesce(
+                        left_col.sql(dialect=self.session.input_dialect),
+                        right_col.sql(dialect=self.session.input_dialect),
+                    ).alias(left_col.alias_or_name)
+                    if how == "full"
+                    else left_col.alias_or_name
+                    for left_col, right_col in join_column_pairs
+                ]
                 # To match spark behavior only the join clause gets deduplicated and it gets put in the front of the column list
                 select_column_names = [
                     (
@@ -846,7 +858,10 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                 select_column_names = [
                     column_name
                     for column_name in select_column_names
-                    if column_name not in join_column_names
+                    if column_name
+                    not in [
+                        x.alias_or_name if not isinstance(x, str) else x for x in join_column_names
+                    ]
                 ]
                 select_column_names = join_column_names + select_column_names
             else:

{sqlframe-1.6.0 → sqlframe-1.6.2}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 1.6.0
+Version: 1.6.2
 Summary: Taking the Spark out of PySpark by converting to SQL
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-1.6.0 → sqlframe-1.6.2}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<3.11.0
-sqlglot<24.2,>=24.0.0
+sqlglot<25.1,>=24.0.0
 
 [bigquery]
 google-cloud-bigquery-storage<3,>=2

{sqlframe-1.6.0 → sqlframe-1.6.2}/tests/integration/test_int_dataframe.py

@@ -375,7 +375,7 @@ def test_join_inner(
         pyspark_employee["fname"],
         F.col("lname"),
         F.col("age"),
-        F.col("store_id"),
+        F.col("store_id").alias("renamed_store_id"),
         pyspark_store.store_name,
         pyspark_store["num_sales"],
     )
@@ -384,7 +384,7 @@ def test_join_inner(
         employee["fname"],
         SF.col("lname"),
         SF.col("age"),
-        SF.col("store_id"),
+        SF.col("store_id").alias("renamed_store_id"),
         store.store_name,
         store["num_sales"],
     )
@@ -2057,3 +2057,48 @@ def test_transform(
     df = pyspark_employee.transform(cast_all_to_int_pyspark).transform(sort_columns_asc)
     dfs = employee.transform(cast_all_to_int_sqlframe).transform(sort_columns_asc)
     compare_frames(df, dfs)
+
+
+# https://github.com/eakmanrq/sqlframe/issues/51
+def test_join_full_outer_no_match(
+    pyspark_employee: PySparkDataFrame,
+    get_df: t.Callable[[str], _BaseDataFrame],
+    compare_frames: t.Callable,
+):
+    spark = pyspark_employee._session
+    initial = spark.createDataFrame(
+        [
+            (1, "data"),
+            (2, "data"),
+        ],
+        ["id", "data"],
+    )
+    for_join = spark.createDataFrame(
+        [
+            (1, "other_data"),
+            (2, "other_data"),
+            (3, "other_data"),
+        ],
+        ["id", "other_data"],
+    )
+    df = initial.join(for_join, on="id", how="full")
+
+    session = get_df("employee").session
+    dfs_initial = session.createDataFrame(
+        [
+            (1, "data"),
+            (2, "data"),
+        ],
+        ["id", "data"],
+    )
+    dfs_for_join = session.createDataFrame(
+        [
+            (1, "other_data"),
+            (2, "other_data"),
+            (3, "other_data"),
+        ],
+        ["id", "other_data"],
+    )
+    dfs = dfs_initial.join(dfs_for_join, on="id", how="full")
+
+    compare_frames(df, dfs)