sqlframe 3.15.0__tar.gz → 3.16.0__tar.gz

{sqlframe-3.15.0 → sqlframe-3.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.15.0
+Version: 3.16.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.15.0 → sqlframe-3.16.0}/docs/bigquery.md

@@ -334,6 +334,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [bool_and](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bool_and.html)
 * [bool_or](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bool_or.html)
 * [bround](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bround.html)
+* [btrim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.btrim.html)
 * [call_function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.call_function.html)
 * [cbrt](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.cbrt.html)
 * [ceil](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.ceil.html)

{sqlframe-3.15.0 → sqlframe-3.16.0}/docs/duckdb.md

@@ -296,6 +296,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [bitwise_not](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bitwise_not.html)
 * [bool_and](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bool_and.html)
 * [bool_or](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bool_or.html)
+* [btrim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.btrim.html)
 * [call_function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.call_function.html)
 * [cbrt](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.cbrt.html)
 * [ceil](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.ceil.html)

{sqlframe-3.15.0 → sqlframe-3.16.0}/docs/postgres.md

@@ -306,6 +306,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [bitwise_not](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bitwise_not.html)
 * [bool_and](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bool_and.html)
 * [bool_or](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bool_or.html)
+* [btrim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.btrim.html)
 * [call_function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.call_function.html)
 * [cbrt](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.cbrt.html)
 * [ceil](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.ceil.html)

{sqlframe-3.15.0 → sqlframe-3.16.0}/docs/snowflake.md

@@ -332,6 +332,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [bool_or](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bool_or.html)
 * [bround](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.bround.html)
   * [Input must be a fixed-point nnumber](https://docs.snowflake.com/en/sql-reference/data-types-numeric.html#label-data-types-for-fixed-point-numbers)
+* [btrim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.btrim.html)
 * [call_function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.call_function.html)
 * [cbrt](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.cbrt.html)
 * [ceil](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.ceil.html)

{sqlframe-3.15.0 → sqlframe-3.16.0}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.15.0'
-__version_tuple__ = version_tuple = (3, 15, 0)
+__version__ = version = '3.16.0'
+__version_tuple__ = version_tuple = (3, 16, 0)

{sqlframe-3.15.0 → sqlframe-3.16.0}/sqlframe/base/dataframe.py

@@ -391,7 +391,9 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
 
         cols = self._ensure_list_of_columns(cols)
         normalize(self.session, expression or self.expression, cols)
-        return list(flatten([self._expand_star(col) for col in cols]))
+        cols = list(flatten([self._expand_star(col) for col in cols]))
+        self._resolve_ambiguous_columns(cols)
+        return cols
 
     def _ensure_and_normalize_col(self, col):
         from sqlframe.base.column import Column
@@ -399,6 +401,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
 
         col = Column.ensure_col(col)
         normalize(self.session, self.expression, col)
+        self._resolve_ambiguous_columns(col)
         return col
 
     def _convert_leaf_to_cte(
@@ -745,10 +748,55 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         kwargs["join_on_uuid"] = str(uuid4())
         return self.__class__(**object_to_dict(self, **kwargs))
 
+    def _resolve_ambiguous_columns(self, columns: t.Union[Column, t.List[Column]]) -> None:
+        if "joins" not in self.expression.args:
+            return
+
+        columns = ensure_list(columns)
+        ambiguous_cols: t.List[exp.Column] = list(
+            flatten(
+                [
+                    sub_col
+                    for col in columns
+                    for sub_col in col.expression.find_all(exp.Column)
+                    if not sub_col.table
+                ]
+            )
+        )
+        if ambiguous_cols:
+            join_table_identifiers = [
+                x.this for x in get_tables_from_expression_with_join(self.expression)
+            ]
+            cte_names_in_join = [x.this for x in join_table_identifiers]
+            # If we have columns that resolve to multiple CTE expressions then we want to use each CTE left-to-right
+            # (or right to left if a right join) and therefore we allow multiple columns with the same
+            # name in the result. This matches the behavior of Spark.
+            resolved_column_position: t.Dict[exp.Column, int] = {
+                col.copy(): -1 for col in ambiguous_cols
+            }
+            for ambiguous_col in ambiguous_cols:
+                ctes = (
+                    list(reversed(self.expression.ctes))
+                    if self.expression.args["joins"][0].args.get("side", "") == "right"
+                    else self.expression.ctes
+                )
+                ctes_with_column = [
+                    cte
+                    for cte in ctes
+                    if cte.alias_or_name in cte_names_in_join
+                    and ambiguous_col.alias_or_name in cte.this.named_selects
+                ]
+                # Check if there is a CTE with this column that we haven't used before. If so, use it. Otherwise,
+                # use the same CTE we used before
+                cte = seq_get(ctes_with_column, resolved_column_position[ambiguous_col] + 1)
+                if cte:
+                    resolved_column_position[ambiguous_col] += 1
+                else:
+                    cte = ctes_with_column[resolved_column_position[ambiguous_col]]
+                ambiguous_col.set("table", exp.to_identifier(cte.alias_or_name))
+
     @operation(Operation.SELECT)
     def select(self, *cols, **kwargs) -> Self:
-        from sqlframe.base.column import Column
-
         if not cols:
             return self
 
@@ -756,48 +804,6 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
             cols = cols[0]  # type: ignore
         columns = self._ensure_and_normalize_cols(cols)
         kwargs["append"] = kwargs.get("append", False)
-        if self.expression.args.get("joins"):
-            ambiguous_cols: t.List[exp.Column] = list(
-                flatten(
-                    [
-                        sub_col
-                        for col in columns
-                        for sub_col in col.expression.find_all(exp.Column)
-                        if not sub_col.table
-                    ]
-                )
-            )
-            if ambiguous_cols:
-                join_table_identifiers = [
-                    x.this for x in get_tables_from_expression_with_join(self.expression)
-                ]
-                cte_names_in_join = [x.this for x in join_table_identifiers]
-                # If we have columns that resolve to multiple CTE expressions then we want to use each CTE left-to-right
-                # (or right to left if a right join) and therefore we allow multiple columns with the same
-                # name in the result. This matches the behavior of Spark.
-                resolved_column_position: t.Dict[exp.Column, int] = {
-                    col.copy(): -1 for col in ambiguous_cols
-                }
-                for ambiguous_col in ambiguous_cols:
-                    ctes = (
-                        list(reversed(self.expression.ctes))
-                        if self.expression.args["joins"][0].args.get("side", "") == "right"
-                        else self.expression.ctes
-                    )
-                    ctes_with_column = [
-                        cte
-                        for cte in ctes
-                        if cte.alias_or_name in cte_names_in_join
-                        and ambiguous_col.alias_or_name in cte.this.named_selects
-                    ]
-                    # Check if there is a CTE with this column that we haven't used before. If so, use it. Otherwise,
-                    # use the same CTE we used before
-                    cte = seq_get(ctes_with_column, resolved_column_position[ambiguous_col] + 1)
-                    if cte:
-                        resolved_column_position[ambiguous_col] += 1
-                    else:
-                        cte = ctes_with_column[resolved_column_position[ambiguous_col]]
-                    ambiguous_col.set("table", exp.to_identifier(cte.alias_or_name))
         # If an expression is `CAST(x AS DATETYPE)` then we want to alias so that `x` is the result column name
         columns = [
             col.alias(col.expression.alias_or_name)

{sqlframe-3.15.0 → sqlframe-3.16.0}/sqlframe/base/decorators.py

@@ -43,7 +43,7 @@ def func_metadata(unsupported_engines: t.Optional[t.Union[str, t.List[str]]] = N
                         col_name = col_name.this
                 alias_name = f"{func.__name__}__{col_name or ''}__"
                 # BigQuery has restrictions on alias names so we constrain it to alphanumeric characters and underscores
-                return result.alias(re.sub("\W", "_", alias_name))  # type: ignore
+                return result.alias(re.sub(r"\W", "_", alias_name))  # type: ignore
             return result
 
         wrapper.unsupported_engines = (  # type: ignore

{sqlframe-3.15.0 → sqlframe-3.16.0}/sqlframe/base/functions.py

@@ -2851,12 +2851,14 @@ def bool_or(col: ColumnOrName) -> Column:
     return Column.invoke_expression_over_column(col, expression.LogicalOr)
 
 
-@meta(unsupported_engines="*")
+@meta()
 def btrim(str: ColumnOrName, trim: t.Optional[ColumnOrName] = None) -> Column:
     if trim is not None:
-        return Column.invoke_anonymous_function(str, "btrim", trim)
+        return Column.invoke_expression_over_column(
+            str, expression.Trim, expression=Column.ensure_col(trim).column_expression
+        )
     else:
-        return Column.invoke_anonymous_function(str, "btrim")
+        return Column.invoke_expression_over_column(str, expression.Trim)
 
 
 @meta(unsupported_engines="*")

{sqlframe-3.15.0 → sqlframe-3.16.0}/sqlframe/base/session.py

@@ -108,6 +108,11 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, TABLE, CONN, UDF_REGIS
         if not getattr(self, "schema", None) or schema:
             self._schema = schema
 
+    # https://github.com/eakmanrq/sqlframe/issues/262
+    @property
+    def execution_dialect_name(self) -> str:
+        return self.execution_dialect.__class__.__name__.lower()
+
     @property
     def read(self) -> READER:
         return self._reader(self)

{sqlframe-3.15.0 → sqlframe-3.16.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.15.0
+Version: 3.16.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.15.0 → sqlframe-3.16.0}/tests/integration/engines/bigquery/test_bigquery_session.py

@@ -18,3 +18,4 @@ def test_session_from_config():
     session = BigQuerySession.builder.config("default_dataset", "sqlframe.db1").getOrCreate()
     columns = session.catalog.get_columns("db1.test_table")
     assert columns == {"cola": exp.DataType.build("BIGINT"), "colb": exp.DataType.build("TEXT")}
+    assert session.execution_dialect_name == "bigquery"

{sqlframe-3.15.0 → sqlframe-3.16.0}/tests/integration/engines/databricks/test_databricks_session.py

@@ -45,3 +45,4 @@ def test_session_from_config(cleanup_connector: DatabricksConnection):
         "cola": exp.DataType.build("INT", dialect=session.output_dialect),
         "colb": exp.DataType.build("STRING", dialect=session.output_dialect),
     }
+    assert session.execution_dialect_name == "databricks"

{sqlframe-3.15.0 → sqlframe-3.16.0}/tests/integration/engines/duck/test_duckdb_session.py

@@ -11,3 +11,4 @@ def test_session_from_config():
     session = DuckDBSession.builder.config("sqlframe.conn", conn).getOrCreate()
     columns = session.catalog.get_columns("test_table")
     assert columns == {"cola": exp.DataType.build("INT"), "colb": exp.DataType.build("TEXT")}
+    assert session.execution_dialect_name == "duckdb"

{sqlframe-3.15.0 → sqlframe-3.16.0}/tests/integration/engines/postgres/test_postgres_session.py

@@ -17,3 +17,4 @@ def test_session_from_config(function_scoped_postgres):
         "cola": exp.DataType.build("INT", dialect=session.output_dialect),
         "colb": exp.DataType.build("STRING", dialect=session.output_dialect),
     }
+    assert session.execution_dialect_name == "postgres"

{sqlframe-3.15.0 → sqlframe-3.16.0}/tests/integration/engines/redshift/test_redshift_session.py

@@ -45,3 +45,4 @@ def test_session_from_config(cleanup_connector: RedshiftConnection):
         "cola": exp.DataType.build("INT", dialect=session.output_dialect),
         "colb": exp.DataType.build("STRING", dialect=session.output_dialect),
     }
+    assert session.execution_dialect_name == "redshift"

{sqlframe-3.15.0 → sqlframe-3.16.0}/tests/integration/engines/snowflake/test_snowflake_session.py

@@ -45,3 +45,4 @@ def test_session_from_config(cleanup_connector: SnowflakeConnection):
         "cola": exp.DataType.build("DECIMAL(38, 0)", dialect=session.output_dialect),
         "colb": exp.DataType.build("TEXT", dialect=session.output_dialect),
     }
+    assert session.execution_dialect_name == "snowflake"

{sqlframe-3.15.0 → sqlframe-3.16.0}/tests/integration/test_int_dataframe.py

@@ -2381,3 +2381,27 @@ def test_union_common_root_again(
     dfs_final = dfs_1.union(dfs_2).union(employee)
 
     compare_frames(df_final, dfs_final, compare_schema=False)
+
+
+# https://github.com/eakmanrq/sqlframe/issues/277
+def test_filtering_join_key(
+    pyspark_employee: PySparkDataFrame,
+    pyspark_store: PySparkDataFrame,
+    get_df: t.Callable[[str], BaseDataFrame],
+    compare_frames: t.Callable,
+):
+    df = pyspark_employee.join(
+        pyspark_store,
+        on="store_id",
+        how="inner",
+    ).filter(F.col("store_id") > 1)
+
+    employee = get_df("employee")
+    store = get_df("store")
+    dfs = employee.join(
+        store,
+        on="store_id",
+        how="inner",
+    ).filter(SF.col("store_id") > 1)
+
+    compare_frames(df, dfs, compare_schema=False, sort=True)

{sqlframe-3.15.0 → sqlframe-3.16.0}/tests/unit/standalone/test_functions.py

@@ -3228,10 +3228,10 @@ def test_bool_or(expression, expected):
 @pytest.mark.parametrize(
     "expression, expected",
     [
-        (SF.btrim("cola"), "BTRIM(cola)"),
-        (SF.btrim(SF.col("cola")), "BTRIM(cola)"),
-        (SF.btrim("cola", "chars"), "BTRIM(cola, chars)"),
-        (SF.btrim("cola", SF.lit("chars")), "BTRIM(cola, 'chars')"),
+        (SF.btrim("cola"), "TRIM(cola)"),
+        (SF.btrim(SF.col("cola")), "TRIM(cola)"),
+        (SF.btrim("cola", "chars"), "TRIM(chars FROM cola)"),
+        (SF.btrim("cola", SF.lit("chars")), "TRIM('chars' FROM cola)"),
     ],
 )
 def test_btrim(expression, expected):