sqlframe 3.39.3__tar.gz → 3.40.0__tar.gz

{sqlframe-3.39.3 → sqlframe-3.40.0}/.github/workflows/main.workflow.yaml

@@ -29,7 +29,7 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v5
     - name: Install Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

{sqlframe-3.39.3 → sqlframe-3.40.0}/Makefile

@@ -1,5 +1,11 @@
+ifdef UV
+    PIP := uv pip
+else
+    PIP := pip3
+endif
+
 install-dev:
-	pip install -e ".[bigquery,dev,docs,duckdb,pandas,postgres,redshift,snowflake,databricks,spark]"
+	$(PIP) install -e ".[bigquery,dev,docs,duckdb,pandas,postgres,redshift,snowflake,databricks,spark]"
 
 install-pre-commit:
 	pre-commit install
@@ -31,7 +37,7 @@ stubs:
 	stubgen sqlframe/spark/functions.py --output ./ --inspect-mode
 
 package:
-	pip3 install wheel && python3 setup.py sdist bdist_wheel
+	$(PIP) install wheel && python3 setup.py sdist bdist_wheel
 
 publish: package
-	pip3 install twine && python3 -m twine upload dist/*
+	$(PIP) install twine && python3 -m twine upload dist/*

{sqlframe-3.39.3 → sqlframe-3.40.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.39.3
+Version: 3.40.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.39.3 → sqlframe-3.40.0}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<27.9",
+        "sqlglot>=24.0.0,<27.13",
         "typing_extensions",
         "more-itertools",
     ],

{sqlframe-3.39.3 → sqlframe-3.40.0}/sqlframe/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.39.3'
-__version_tuple__ = version_tuple = (3, 39, 3)
+__version__ = version = '3.40.0'
+__version_tuple__ = version_tuple = (3, 40, 0)
 
-__commit_id__ = commit_id = 'g9d915cb1e'
+__commit_id__ = commit_id = 'g93abcd907'

{sqlframe-3.39.3 → sqlframe-3.40.0}/sqlframe/base/dataframe.py

@@ -16,7 +16,6 @@ from dataclasses import dataclass
 from uuid import uuid4
 
 import sqlglot
-from more_itertools import partition
 from prettytable import PrettyTable
 from sqlglot import Dialect, maybe_parse
 from sqlglot import expressions as exp
@@ -397,12 +396,21 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         return Column.ensure_cols(ensure_list(cols))  # type: ignore
 
     def _ensure_and_normalize_cols(
-        self, cols, expression: t.Optional[exp.Select] = None, skip_star_expansion: bool = False
+        self,
+        cols,
+        expression: t.Optional[exp.Select] = None,
+        skip_star_expansion: bool = False,
+        remove_identifier_if_possible: bool = True,
     ) -> t.List[Column]:
         from sqlframe.base.normalize import normalize
 
         cols = self._ensure_list_of_columns(cols)
-        normalize(self.session, expression or self.expression, cols)
+        normalize(
+            self.session,
+            expression or self.expression,
+            cols,
+            remove_identifier_if_possible=remove_identifier_if_possible,
+        )
         if not skip_star_expansion:
             cols = list(flatten([self._expand_star(col) for col in cols]))
         self._resolve_ambiguous_columns(cols)
@@ -542,23 +550,16 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         expression.set("with", exp.With(expressions=existing_ctes))
         return expression
 
-    @classmethod
-    def _get_outer_select_expressions(
-        cls, item: exp.Expression
-    ) -> t.List[t.Union[exp.Column, exp.Alias]]:
-        outer_select = item.find(exp.Select)
-        if outer_select:
-            return outer_select.expressions
-        return []
-
     @classmethod
     def _get_outer_select_columns(cls, item: exp.Expression) -> t.List[Column]:
         from sqlframe.base.session import _BaseSession
 
         col = get_func_from_session("col", _BaseSession())
 
-        outer_expressions = cls._get_outer_select_expressions(item)
-        return [col(quote_preserving_alias_or_name(x)) for x in outer_expressions]
+        outer_select = item.find(exp.Select)
+        if outer_select:
+            return [col(quote_preserving_alias_or_name(x)) for x in outer_select.expressions]
+        return []
 
     def _create_hash_from_expression(self, expression: exp.Expression) -> str:
         from sqlframe.base.session import _BaseSession
@@ -1025,9 +1026,17 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         return join_column_pairs, join_clause
 
     def _normalize_join_clause(
-        self, join_columns: t.List[Column], join_expression: t.Optional[exp.Select]
+        self,
+        join_columns: t.List[Column],
+        join_expression: t.Optional[exp.Select],
+        *,
+        remove_identifier_if_possible: bool = True,
     ) -> Column:
-        join_columns = self._ensure_and_normalize_cols(join_columns, join_expression)
+        join_columns = self._ensure_and_normalize_cols(
+            join_columns,
+            join_expression,
+            remove_identifier_if_possible=remove_identifier_if_possible,
+        )
         if len(join_columns) > 1:
             join_columns = [functools.reduce(lambda x, y: x & y, join_columns)]
         join_clause = join_columns[0]
@@ -1512,23 +1521,20 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         """
         return func(self, *args, **kwargs)  # type: ignore
 
-    @operation(Operation.SELECT_CONSTRAINED)
+    @operation(Operation.SELECT)
     def withColumn(self, colName: str, col: Column) -> Self:
         return self.withColumns.__wrapped__(self, {colName: col})  # type: ignore
 
-    @operation(Operation.SELECT_CONSTRAINED)
+    @operation(Operation.SELECT)
     def withColumnRenamed(self, existing: str, new: str) -> Self:
-        col_func = get_func_from_session("col", self.session)
         expression = self.expression.copy()
         existing = self.session._normalize_string(existing)
-        outer_expressions = self._get_outer_select_expressions(expression)
+        columns = self._get_outer_select_columns(expression)
         results = []
         found_match = False
-        for expr in outer_expressions:
-            column = col_func(expr.copy())
-            if existing == quote_preserving_alias_or_name(expr):
-                if isinstance(column.expression, exp.Alias):
-                    column.expression.set("alias", exp.to_identifier(new))
+        for column in columns:
+            if column.alias_or_name == existing:
+                column = column.alias(new)
                 self._update_display_name_mapping([column], [new])
                 found_match = True
             results.append(column)
@@ -1536,7 +1542,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
             raise ValueError("Tried to rename a column that doesn't exist")
         return self.select.__wrapped__(self, *results, skip_update_display_name_mapping=True)  # type: ignore
 
-    @operation(Operation.SELECT_CONSTRAINED)
+    @operation(Operation.SELECT)
     def withColumnsRenamed(self, colsMap: t.Dict[str, str]) -> Self:
         """
         Returns a new :class:`DataFrame` by renaming multiple columns. If a non-existing column is
@@ -1582,7 +1588,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
 
         return self.select.__wrapped__(self, *results, skip_update_display_name_mapping=True)  # type: ignore
 
-    @operation(Operation.SELECT_CONSTRAINED)
+    @operation(Operation.SELECT)
     def withColumns(self, *colsMap: t.Dict[str, Column]) -> Self:
         """
         Returns a new :class:`DataFrame` by adding multiple columns or replacing the
@@ -1620,14 +1626,13 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         """
         if len(colsMap) != 1:
             raise ValueError("Only a single map is supported")
-        col_func = get_func_from_session("col")
         col_map = {
             self._ensure_and_normalize_col(k): (self._ensure_and_normalize_col(v), k)
             for k, v in colsMap[0].items()
         }
-        existing_expr = self._get_outer_select_expressions(self.expression)
-        existing_col_names = [x.alias_or_name for x in existing_expr]
-        select_columns = [col_func(x) for x in existing_expr]
+        existing_cols = self._get_outer_select_columns(self.expression)
+        existing_col_names = [x.alias_or_name for x in existing_cols]
+        select_columns = existing_cols
         for col, (col_value, display_name) in col_map.items():
             column_name = col.alias_or_name
             existing_col_index = (
@@ -1644,7 +1649,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         )
         return self.select.__wrapped__(self, *select_columns, skip_update_display_name_mapping=True)  # type: ignore
 
-    @operation(Operation.SELECT_CONSTRAINED)
+    @operation(Operation.SELECT)
     def drop(self, *cols: t.Union[str, Column]) -> Self:
         # Separate string column names from Column objects for different handling
         column_objs, column_names = partition_to(lambda x: isinstance(x, str), cols, list, set)

{sqlframe-3.39.3 → sqlframe-3.40.0}/sqlframe/base/functions.py

@@ -37,9 +37,7 @@ def _get_session() -> _BaseSession:
 
 @meta()
 def col(column_name: t.Union[ColumnOrName, t.Any]) -> Column:
-    from sqlframe.base.session import _BaseSession
-
-    dialect = _BaseSession().input_dialect
+    dialect = _get_session().input_dialect
     if isinstance(column_name, str):
         col_expression = expression.to_column(column_name, dialect=dialect).transform(
             dialect.normalize_identifier
@@ -192,27 +190,27 @@ sum_distinct = sumDistinct
 
 @meta()
 def acos(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "ACOS")
+    return Column.invoke_expression_over_column(col, expression.Acos)
 
 
 @meta(unsupported_engines="duckdb")
 def acosh(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "ACOSH")
+    return Column.invoke_expression_over_column(col, expression.Acosh)
 
 
 @meta()
 def asin(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "ASIN")
+    return Column.invoke_expression_over_column(col, expression.Asin)
 
 
 @meta(unsupported_engines="duckdb")
 def asinh(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "ASINH")
+    return Column.invoke_expression_over_column(col, expression.Asinh)
 
 
 @meta()
 def atan(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "ATAN")
+    return Column.invoke_expression_over_column(col, expression.Atan)
 
 
 @meta()
@@ -220,12 +218,12 @@ def atan2(col1: t.Union[ColumnOrName, float], col2: t.Union[ColumnOrName, float]
     col1_value = lit(col1) if isinstance(col1, (int, float)) else col1
     col2_value = lit(col2) if isinstance(col2, (int, float)) else col2
 
-    return Column.invoke_anonymous_function(col1_value, "ATAN2", col2_value)
+    return Column.invoke_expression_over_column(col1_value, expression.Atan2, expression=col2_value)
 
 
 @meta(unsupported_engines="duckdb")
 def atanh(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "ATANH")
+    return Column.invoke_expression_over_column(col, expression.Atanh)
 
 
 @meta()
@@ -253,12 +251,12 @@ def cosh(col: ColumnOrName) -> Column:
 
 @meta()
 def cot(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "COT")
+    return Column.invoke_expression_over_column(col, expression.Cot)
 
 
 @meta(unsupported_engines=["duckdb", "postgres", "snowflake"])
 def csc(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "CSC")
+    return Column.invoke_expression_over_column(col, expression.Csc)
 
 
 @meta()
@@ -364,7 +362,7 @@ def rint(col: ColumnOrName) -> Column:
 
 @meta(unsupported_engines=["duckdb", "postgres", "snowflake"])
 def sec(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "SEC")
+    return Column.invoke_expression_over_column(col, expression.Sec)
 
 
 @meta()
@@ -374,12 +372,12 @@ def signum(col: ColumnOrName) -> Column:
 
 @meta()
 def sin(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "SIN")
+    return Column.invoke_expression_over_column(col, expression.Sin)
 
 
 @meta(unsupported_engines="duckdb")
 def sinh(col: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(col, "SINH")
+    return Column.invoke_expression_over_column(col, expression.Sinh)
 
 
 @meta()
@@ -662,9 +660,7 @@ def grouping_id(*cols: ColumnOrName) -> Column:
 
 @meta()
 def input_file_name() -> Column:
-    from sqlframe.base.session import _BaseSession
-
-    return Column(expression.Literal.string(_BaseSession()._last_loaded_file or ""))
+    return Column(expression.Literal.string(_get_session()._last_loaded_file or ""))
 
 
 @meta()
@@ -944,7 +940,7 @@ def nth_value(
 
 @meta()
 def ntile(n: int) -> Column:
-    return Column.invoke_anonymous_function(None, "NTILE", lit(n))
+    return Column.invoke_expression_over_column(lit(n), expression.Ntile)
 
 
 @meta()
@@ -959,12 +955,10 @@ def current_timestamp() -> Column:
 
 @meta()
 def date_format(col: ColumnOrName, format: str) -> Column:
-    from sqlframe.base.session import _BaseSession
-
     return Column.invoke_expression_over_column(
         Column(expression.TimeStrToTime(this=Column.ensure_col(col).column_expression)),
         expression.TimeToStr,
-        format=_BaseSession().format_time(format),
+        format=_get_session().format_time(format),
     )
 
 
@@ -2832,7 +2826,7 @@ def make_interval(
 
 @meta(unsupported_engines="*")
 def try_add(left: ColumnOrName, right: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(left, "TRY_ADD", right)
+    return Column.invoke_expression_over_column(left, expression.SafeAdd, expression=right)
 
 
 @meta(unsupported_engines="*")
@@ -2849,12 +2843,12 @@ def try_divide(left: ColumnOrName, right: ColumnOrName) -> Column:
 
 @meta(unsupported_engines="*")
 def try_multiply(left: ColumnOrName, right: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(left, "TRY_MULTIPLY", right)
+    return Column.invoke_expression_over_column(left, expression.SafeMultiply, expression=right)
 
 
 @meta(unsupported_engines="*")
 def try_subtract(left: ColumnOrName, right: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(left, "TRY_SUBTRACT", right)
+    return Column.invoke_expression_over_column(left, expression.SafeSubtract, expression=right)
 
 
 @meta(unsupported_engines="*")
@@ -3378,10 +3372,9 @@ def get(col: ColumnOrName, index: t.Union[ColumnOrName, int]) -> Column:
 def get_active_spark_context() -> SparkContext:
     """Raise RuntimeError if SparkContext is not initialized,
     otherwise, returns the active SparkContext."""
-    from sqlframe.base.session import _BaseSession
     from sqlframe.spark.session import SparkSession
 
-    session: _BaseSession = _BaseSession()
+    session = _get_session()
     if not isinstance(session, SparkSession):
         raise RuntimeError("This function is only available in SparkSession.")
     return session.spark_session.sparkContext
@@ -5263,7 +5256,7 @@ def regexp_extract_all(
     )
 
 
-@meta(unsupported_engines="*")
+@meta(unsupported_engines=["duckdb", "bigquery", "postgres", "snowflake"])
 def regexp_instr(
     str: ColumnOrName, regexp: ColumnOrName, idx: t.Optional[t.Union[int, Column]] = None
 ) -> Column:
@@ -5298,11 +5291,9 @@ def regexp_instr(
     >>> df.select(regexp_instr('str', col("regexp")).alias('d')).collect()
     [Row(d=1)]
     """
-    if idx is None:
-        return Column.invoke_anonymous_function(str, "regexp_instr", regexp)
-    else:
-        idx = lit(idx) if isinstance(idx, int) else idx
-        return Column.invoke_anonymous_function(str, "regexp_instr", regexp, idx)
+    return Column.invoke_expression_over_column(
+        str, expression.RegexpInstr, expression=regexp, group=idx
+    )
 
 
 @meta(unsupported_engines="snowflake")
@@ -6344,7 +6335,7 @@ def to_unix_timestamp(
     session = _get_session()
 
     if session._is_duckdb:
-        format = format or _BaseSession().default_time_format
+        format = format or session.default_time_format
         timestamp = Column.ensure_col(timestamp).cast("string")
 
     if format is not None:

{sqlframe-3.39.3 → sqlframe-3.40.0}/sqlframe/base/mixins/table_mixins.py

@@ -275,7 +275,9 @@ class MergeSupportMixin(_BaseTable, t.Generic[DF]):
         join_expression = self._add_ctes_to_expression(
             self.expression, other_df.expression.copy().ctes
         )
-        condition = self._ensure_and_normalize_cols(condition, self.expression)
+        condition = self._ensure_and_normalize_cols(
+            condition, self.expression, remove_identifier_if_possible=False
+        )
         self._handle_self_join(other_df, condition)
 
         if isinstance(condition[0].expression, exp.Column) and not clause:
@@ -291,7 +293,9 @@ class MergeSupportMixin(_BaseTable, t.Generic[DF]):
                 condition, join_expression, other_df, table_names
             )
         else:
-            join_clause = self._normalize_join_clause(condition, join_expression)
+            join_clause = self._normalize_join_clause(
+                condition, join_expression, remove_identifier_if_possible=False
+            )
         return join_clause
 
     def _ensure_and_normalize_assignments(