sqlframe 3.9.0__tar.gz → 3.9.2__tar.gz

{sqlframe-3.9.0 → sqlframe-3.9.2}/.github/workflows/publish.workflow.yaml

@@ -5,17 +5,20 @@ on:
       - 'v[0-9]+.[0-9]+.[0-9]+'
 permissions:
   contents: write
+  id-token: write
 jobs:
   deploy:
     runs-on: ubuntu-latest
-    env:
-      TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
-      TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+    environment:
+      name: publish
+      url: https://pypi.org/p/sqlframe
     steps:
     - name: Checkout
       uses: actions/checkout@v4
+    - name: Package
+      run: make package
     - name: Publish
-      run: make publish
+      uses: pypa/gh-action-pypi-publish@release/v1
     - name: Create release
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

{sqlframe-3.9.0 → sqlframe-3.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.9.0
+Version: 3.9.2
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.9.0 → sqlframe-3.9.2}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.9.0'
-__version_tuple__ = version_tuple = (3, 9, 0)
+__version__ = version = '3.9.2'
+__version_tuple__ = version_tuple = (3, 9, 2)

{sqlframe-3.9.0 → sqlframe-3.9.2}/sqlframe/base/dataframe.py

@@ -12,6 +12,7 @@ import typing as t
 import zlib
 from copy import copy
 from dataclasses import dataclass
+from uuid import uuid4
 
 import sqlglot
 from prettytable import PrettyTable
@@ -208,6 +209,8 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         expression: exp.Select,
         branch_id: t.Optional[str] = None,
         sequence_id: t.Optional[str] = None,
+        join_on_uuid: t.Optional[str] = None,
+        known_uuids: t.Optional[t.Set[str]] = None,
         last_op: Operation = Operation.INIT,
         pending_hints: t.Optional[t.List[exp.Expression]] = None,
         output_expression_container: t.Optional[OutputExpressionContainer] = None,
@@ -217,6 +220,9 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         self.expression: exp.Select = expression
         self.branch_id = branch_id or self.session._random_branch_id
         self.sequence_id = sequence_id or self.session._random_sequence_id
+        self.join_on_uuid = join_on_uuid or str(uuid4())
+        self.known_uuids = known_uuids or set()
+        self.known_uuids.add(self.join_on_uuid)
         self.last_op = last_op
         self.pending_hints = pending_hints or []
         self.output_expression_container = output_expression_container or exp.Select()
@@ -228,10 +234,12 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
     def __getitem__(self, column_name: str) -> Column:
         from sqlframe.base.util import get_func_from_session
 
-        col = get_func_from_session("col", self.session)
+        col_func = get_func_from_session("col", self.session)
 
         column_name = f"{self.branch_id}.{column_name}"
-        return col(column_name)
+        col = col_func(column_name)
+        col.expression.meta["join_on_uuid"] = self.join_on_uuid
+        return col
 
     def __copy__(self):
         return self.copy()
@@ -715,6 +723,7 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         return ";\n".join(results)
 
     def copy(self, **kwargs) -> Self:
+        kwargs["join_on_uuid"] = str(uuid4())
         return self.__class__(**object_to_dict(self, **kwargs))
 
     @operation(Operation.SELECT)
@@ -866,6 +875,7 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         if on is None:
             logger.warning("Got no value for on. This appears to change the join to a cross join.")
             how = "cross"
+
         other_df = other_df._convert_leaf_to_cte()
         join_expression = self._add_ctes_to_expression(self.expression, other_df.expression.ctes)
         # We will determine actual "join on" expression later so we don't provide it at first
@@ -875,6 +885,21 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         self_columns = self._get_outer_select_columns(join_expression)
         other_columns = self._get_outer_select_columns(other_df.expression)
         join_columns = self._ensure_and_normalize_cols(on)
+        # If the two dataframes being joined come from the same branch, we then check if they have any columns that
+        # were created using the "branch_id" (df["column_name"]). If so, we know that we need to differentiate
+        # the two columns since they would end up with the same table name. We do this by checking for the unique
+        # uuids in the other df and finding columns that have metadata on them that match the uuids. If so, we know
+        # it comes from the other df and we change the table name to the other df's table name.
+        # See `test_self_join` for an example of this.
+        if self.branch_id == other_df.branch_id:
+            other_df_unique_uuids = other_df.known_uuids - self.known_uuids
+            for col in join_columns:
+                for col_expr in col.expression.find_all(exp.Column):
+                    if (
+                        "join_on_uuid" in col_expr.meta
+                        and col_expr.meta["join_on_uuid"] in other_df_unique_uuids
+                    ):
+                        col_expr.set("table", exp.to_identifier(other_df.latest_cte_name))
         # Determines the join clause and select columns to be used passed on what type of columns were provided for
         # the join. The columns returned changes based on how the on expression is provided.
         if how != "cross":

{sqlframe-3.9.0 → sqlframe-3.9.2}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.9.0
+Version: 3.9.2
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.9.0 → sqlframe-3.9.2}/tests/integration/test_int_dataframe.py

@@ -2247,3 +2247,34 @@ def test_chaining_joins_with_selects(
     )
 
     compare_frames(df, dfs, compare_schema=False)
+
+
+# https://github.com/eakmanrq/sqlframe/issues/210
+# https://github.com/eakmanrq/sqlframe/issues/212
+def test_self_join(
+    pyspark_employee: PySparkDataFrame,
+    get_df: t.Callable[[str], _BaseDataFrame],
+    compare_frames: t.Callable,
+    is_spark: t.Callable,
+):
+    if is_spark():
+        pytest.skip(
+            "This test is not supported in Spark. This is related to how duplicate columns are handled in Spark"
+        )
+    df_filtered = pyspark_employee.where(F.col("age") > 40)
+    df_joined = pyspark_employee.join(
+        df_filtered,
+        pyspark_employee["employee_id"].eqNullSafe(df_filtered["employee_id"]),
+        how="inner",
+    )
+
+    employee = get_df("employee")
+
+    dfs_filtered = employee.alias("dfs_filtered").where(SF.col("age") > 40)
+    dfs_joined = employee.join(
+        dfs_filtered,
+        employee["employee_id"].eqNullSafe(dfs_filtered["employee_id"]),
+        how="inner",
+    )
+
+    compare_frames(df_joined, dfs_joined, compare_schema=False)