sqlframe 1.11.0__tar.gz → 1.13.0__tar.gz

{sqlframe-1.11.0 → sqlframe-1.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 1.11.0
+Version: 1.13.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-1.11.0 → sqlframe-1.13.0}/docs/bigquery.md

@@ -307,7 +307,6 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [date_diff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_diff.html)
 * [datediff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.datediff.html)
 * [date_format](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_format.html)
-    * [The format string should be in BigQuery syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements)
 * [date_sub](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_sub.html)
 * [date_trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_trunc.html)
 * [dayofmonth](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.dayofmonth.html)
@@ -442,9 +441,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [toDegrees](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toDegrees.html)
 * [toRadians](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toRadians.html)
 * [to_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_date.html)
-    * [The format string should be in BigQuery syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements)
 * [to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html)
-    * [The format string should be in BigQuery syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)
@@ -454,7 +451,6 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [unbase64](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unbase64.html)
 * [unhex](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unhex.html)
 * [unix_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unix_timestamp.html)
-    * [The format string should be in BigQuery syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements)
 * [upper](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.upper.html)
 * [var_pop](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.var_pop.html)
 * [var_samp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.var_samp.html)

{sqlframe-1.11.0 → sqlframe-1.13.0}/docs/duckdb.md

@@ -276,8 +276,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [dateadd](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.dateadd.html)
 * [date_diff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_diff.html)
 * [datediff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.datediff.html)
-* [date_format](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_format.html)                     
-    * [The format string should be in DuckDB syntax](https://duckdb.org/docs/sql/functions/dateformat.html#format-specifiers)
+* [date_format](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_format.html)
 * [date_sub](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_sub.html)
 * [date_trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_trunc.html)
 * [dayofmonth](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.dayofmonth.html)
@@ -405,11 +404,8 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [toDegrees](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toDegrees.html)
 * [toRadians](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toRadians.html)
 * [to_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_date.html)
-    * [The format string should be in DuckDB syntax](https://duckdb.org/docs/sql/functions/dateformat.html#format-specifiers)
 * [to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html)
-    * [The format string should be in DuckDB syntax](https://duckdb.org/docs/sql/functions/dateformat.html#format-specifiers)
 * [to_unix_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_unix_timestamp.html)
-    * [The format string should be in DuckDB syntax](https://duckdb.org/docs/sql/functions/dateformat.html#format-specifiers
     * The values must match the format string (null will not be returned if they do not)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
@@ -420,7 +416,6 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [unbase64](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unbase64.html)
 * [unhex](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unhex.html)
 * [unix_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unix_timestamp.html)
-    * [The format string should be in DuckDB syntax](https://duckdb.org/docs/sql/functions/dateformat.html#format-specifiers)
 * [upper](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.upper.html)
 * [var_pop](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.var_pop.html)
 * [var_samp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.var_samp.html)

{sqlframe-1.11.0 → sqlframe-1.13.0}/docs/postgres.md

@@ -284,7 +284,6 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [date_diff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_diff.html)
 * [datediff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.datediff.html)
 * [date_format](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_format.html)
-    * [The format string should be in Postgres syntax](https://www.postgresql.org/docs/current/functions-formatting.html#FUNCTIONS-FORMATTING-DATETIME-TABLE)
 * [date_sub](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_sub.html)
 * [date_trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_trunc.html)
     * Rounded whole number is returned
@@ -397,10 +396,8 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [toDegrees](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toDegrees.html)
 * [toRadians](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toRadians.html)
 * [to_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_date.html)
-    * [The format string should be in Postgres syntax](https://www.postgresql.org/docs/current/functions-formatting.html#FUNCTIONS-FORMATTING-DATETIME-TABLE)
 * [to_number](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_number.html)
 * [to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html)
-    * [The format string should be in Postgres syntax](https://www.postgresql.org/docs/current/functions-formatting.html#FUNCTIONS-FORMATTING-DATETIME-TABLE)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)

{sqlframe-1.11.0 → sqlframe-1.13.0}/docs/snowflake.md

@@ -307,7 +307,6 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [date_diff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_diff.html)
 * [datediff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.datediff.html)
 * [date_format](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_format.html)
-  * [The format string should be in Snowflake syntax](https://docs.snowflake.com/en/sql-reference/functions-conversion#label-date-time-format-conversion)
 * [date_sub](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_sub.html)
 * [date_trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_trunc.html)
 * [dayofmonth](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.dayofmonth.html)
@@ -440,10 +439,8 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [toDegrees](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toDegrees.html)
 * [toRadians](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toRadians.html)
 * [to_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_date.html)
-  * [The format string should be in Snowflake syntax](https://docs.snowflake.com/en/sql-reference/functions-conversion#label-date-time-format-conversion)
 * [to_number](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_number.html)
 * [to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html)
-  * [The format string should be in Snowflake syntax](https://docs.snowflake.com/en/sql-reference/functions-conversion#label-date-time-format-conversion)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)
@@ -451,7 +448,6 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [unbase64](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unbase64.html)
 * [unhex](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unhex.html)
 * [unix_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unix_timestamp.html)
-  * [The format string should be in Snowflake syntax](https://docs.snowflake.com/en/sql-reference/functions-conversion#label-date-time-format-conversion)
 * [upper](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.upper.html)
 * [var_pop](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.var_pop.html)
 * [var_samp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.var_samp.html)

{sqlframe-1.11.0 → sqlframe-1.13.0}/setup.py

@@ -42,7 +42,7 @@ setup(
             "pytest-xdist>=3.6,<3.7",
             "pre-commit>=3.5;python_version=='3.8'",
             "pre-commit>=3.7,<3.8;python_version>='3.9'",
-            "ruff>=0.4.4,<0.5",
+            "ruff>=0.4.4,<0.6",
             "types-psycopg2>=2.9,<3",
         ],
         "docs": [

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.11.0'
-__version_tuple__ = version_tuple = (1, 11, 0)
+__version__ = version = '1.13.0'
+__version_tuple__ = version_tuple = (1, 13, 0)

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/base/column.py

@@ -407,3 +407,44 @@ class Column:
         window_expression = window.expression.copy()
         window_expression.set("this", self.column_expression)
         return Column(window_expression)
+
+    def getItem(self, key: t.Any) -> Column:
+        """
+        An expression that gets an item at position ``ordinal`` out of a list,
+        or gets an item by key out of a dict.
+
+        .. versionadded:: 1.3.0
+
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
+
+        Parameters
+        ----------
+        key
+            a literal value, or a :class:`Column` expression.
+            The result will only be true at a location if the item matches in the column.
+
+             .. deprecated:: 3.0.0
+                 :class:`Column` as a parameter is deprecated.
+
+        Returns
+        -------
+        :class:`Column`
+            Column representing the item(s) got at position out of a list or by key out of a dict.
+
+        Examples
+        --------
+        >>> df = spark.createDataFrame([([1, 2], {"key": "value"})], ["l", "d"])
+        >>> df.select(df.l.getItem(0), df.d.getItem("key")).show()
+        +----+------+
+        |l[0]|d[key]|
+        +----+------+
+        |   1| value|
+        +----+------+
+        """
+        element_at = get_func_from_session("element_at")
+        lit = get_func_from_session("lit")
+        key = lit(key) if not isinstance(key, Column) else key
+        if isinstance(key.expression, exp.Literal) and key.expression.is_number:
+            key = key + lit(1)
+        return element_at(self, key)

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/base/dataframe.py

@@ -361,7 +361,7 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
 
         cols = self._ensure_list_of_columns(cols)
         normalize(self.session, expression or self.expression, cols)
-        return cols
+        return list(flatten([self._expand_star(col) for col in cols]))
 
     def _ensure_and_normalize_col(self, col):
         from sqlframe.base.column import Column
@@ -514,6 +514,27 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         select_expressions.append(expression_select_pair)  # type: ignore
         return select_expressions
 
+    def _expand_star(self, col: Column) -> t.List[Column]:
+        from sqlframe.base.column import Column
+
+        if isinstance(col.column_expression, exp.Star):
+            return self._get_outer_select_columns(self.expression)
+        elif (
+            isinstance(col.column_expression, exp.Column)
+            and isinstance(col.column_expression.this, exp.Star)
+            and col.column_expression.args.get("table")
+        ):
+            for cte in self.expression.ctes:
+                if cte.alias_or_name == col.column_expression.args["table"].this:
+                    return [
+                        Column.ensure_col(exp.column(x.column_alias_or_name, cte.alias_or_name))
+                        for x in self._get_outer_select_columns(cte)
+                    ]
+            raise ValueError(
+                f"Could not find table to expand star: {col.column_expression.args['table']}"
+            )
+        return [col]
+
     @t.overload
     def sql(
         self,
@@ -1555,7 +1576,7 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
             result = self.session._fetch_rows(sql)
         table = PrettyTable()
         if row := seq_get(result, 0):
-            table.field_names = list(row.asDict().keys())
+            table.field_names = row._unique_field_names
             for row in result:
                 table.add_row(list(row))
         print(table)

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/base/function_alternatives.py

@@ -6,11 +6,16 @@ import re
 import typing as t
 
 from sqlglot import exp as expression
+from sqlglot.dialects.dialect import build_formatted_time
 from sqlglot.helper import ensure_list
 from sqlglot.helper import flatten as _flatten
 
 from sqlframe.base.column import Column
-from sqlframe.base.util import get_func_from_session
+from sqlframe.base.util import (
+    format_time_from_spark,
+    get_func_from_session,
+    spark_default_time_format,
+)
 
 if t.TYPE_CHECKING:
     from sqlframe.base._typing import ColumnOrLiteral, ColumnOrName
@@ -715,14 +720,10 @@ def months_between_cast_as_date_cast_roundoff(
 
 
 def from_unixtime_from_timestamp(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
-    from sqlframe.base.session import _BaseSession
-
-    session: _BaseSession = _BaseSession()
     lit = get_func_from_session("lit")
     col_func = get_func_from_session("col")
 
-    if format is None:
-        format = session.DEFAULT_TIME_FORMAT
+    format = lit(format or spark_default_time_format())
     return Column.invoke_expression_over_column(
         Column(
             expression.Anonymous(
@@ -731,7 +732,7 @@ def from_unixtime_from_timestamp(col: ColumnOrName, format: t.Optional[str] = No
             )
         ),
         expression.TimeToStr,
-        format=lit(format),
+        format=format_time_from_spark(format),  # type: ignore
     )
 
 
@@ -1135,13 +1136,11 @@ def array_intersect_using_intersection(col1: ColumnOrName, col2: ColumnOrName) -
 def element_at_using_brackets(col: ColumnOrName, value: ColumnOrLiteral) -> Column:
     col_func = get_func_from_session("col")
     lit = get_func_from_session("lit")
-    #  SQLGlot will auto add 1 to whatever we pass in for the brackets even though the value is already 1 based.
-    if not isinstance(value, int):
-        raise ValueError("This dialect requires the value must be an integer")
-    value_lit = lit(value - 1)
-    return Column(
-        expression.Bracket(this=col_func(col).expression, expressions=[value_lit.expression])
-    )
+    # SQLGlot will auto add 1 to whatever we pass in for the brackets even though the value is already 1 based.
+    value = value if isinstance(value, Column) else lit(value)
+    if [x for x in value.expression.find_all(expression.Literal) if x.is_number]:
+        value = value - lit(1)
+    return Column(expression.Bracket(this=col_func(col).expression, expressions=[value.expression]))  # type: ignore
 
 
 def array_remove_using_filter(col: ColumnOrName, value: ColumnOrLiteral) -> Column:
@@ -1513,10 +1512,10 @@ def to_unix_timestamp_include_default_format(
     format: t.Optional[ColumnOrName] = None,
 ) -> Column:
     from sqlframe.base.functions import to_unix_timestamp
-
-    lit = get_func_from_session("lit")
+    from sqlframe.base.session import _BaseSession
 
     if not format:
-        format = lit("%Y-%m-%d %H:%M:%S")
-
+        format = _BaseSession().output_dialect.TIME_FORMAT
+    else:
+        format = format_time_from_spark(format)
     return to_unix_timestamp(timestamp, format)

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/base/functions.py

@@ -6,12 +6,14 @@ import decimal
 import logging
 import typing as t
 
+from sqlglot import Dialect
 from sqlglot import exp as expression
 from sqlglot.helper import ensure_list
 from sqlglot.helper import flatten as _flatten
 
 from sqlframe.base.column import Column
 from sqlframe.base.decorators import func_metadata as meta
+from sqlframe.base.util import format_time_from_spark, spark_default_time_format
 
 if t.TYPE_CHECKING:
     from pyspark.sql.session import SparkContext
@@ -695,7 +697,7 @@ def date_format(col: ColumnOrName, format: str) -> Column:
     return Column.invoke_expression_over_column(
         Column(expression.TimeStrToTime(this=Column.ensure_col(col).expression)),
         expression.TimeToStr,
-        format=lit(format),
+        format=format_time_from_spark(format),
     )
 
 
@@ -875,17 +877,21 @@ def months_between(
 
 @meta()
 def to_date(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
+    format = lit(format or spark_default_time_format())
     if format is not None:
         return Column.invoke_expression_over_column(
-            col, expression.TsOrDsToDate, format=lit(format)
+            col, expression.TsOrDsToDate, format=format_time_from_spark(format)
         )
     return Column.invoke_expression_over_column(col, expression.TsOrDsToDate)
 
 
 @meta()
 def to_timestamp(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
+    format = lit(format or spark_default_time_format())
     if format is not None:
-        return Column.invoke_expression_over_column(col, expression.StrToTime, format=lit(format))
+        return Column.invoke_expression_over_column(
+            col, expression.StrToTime, format=format_time_from_spark(format)
+        )
 
     return Column.ensure_col(col).cast("timestamp")
 
@@ -916,23 +922,23 @@ def last_day(col: ColumnOrName) -> Column:
 
 @meta()
 def from_unixtime(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
-    from sqlframe.base.session import _BaseSession
-
-    if format is None:
-        format = _BaseSession().DEFAULT_TIME_FORMAT
-    return Column.invoke_expression_over_column(col, expression.UnixToStr, format=lit(format))
+    format = lit(format or spark_default_time_format())
+    return Column.invoke_expression_over_column(
+        col,
+        expression.UnixToStr,
+        format=format_time_from_spark(format),  # type: ignore
+    )
 
 
 @meta()
 def unix_timestamp(
     timestamp: t.Optional[ColumnOrName] = None, format: t.Optional[str] = None
 ) -> Column:
-    from sqlframe.base.session import _BaseSession
-
-    if format is None:
-        format = _BaseSession().DEFAULT_TIME_FORMAT
+    format = lit(format or spark_default_time_format())
     return Column.invoke_expression_over_column(
-        timestamp, expression.StrToUnix, format=lit(format)
+        timestamp,
+        expression.StrToUnix,
+        format=format_time_from_spark(format),  # type: ignore
     ).cast("bigint")
 
 
@@ -1923,7 +1929,9 @@ def call_function(funcName: str, *cols: ColumnOrName) -> Column:
     cols = ensure_list(cols)  # type: ignore
     if len(cols) > 1:
         return Column.invoke_anonymous_function(cols[0], funcName, *cols[1:])
-    return Column.invoke_anonymous_function(cols[0], funcName)
+    elif len(cols) == 1:
+        return Column.invoke_anonymous_function(cols[0], funcName)
+    return Column.invoke_anonymous_function(None, funcName)
 
 
 # @meta(unsupported_engines="*")
@@ -2028,7 +2036,7 @@ def character_length(str: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(str, "character_length")
 
 
-@meta(unsupported_engines="*")
+@meta()
 def contains(left: ColumnOrName, right: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(left, "contains", right)
 
@@ -5104,8 +5112,11 @@ def to_unix_timestamp(
     [Row(r=None)]
     >>> spark.conf.unset("spark.sql.session.timeZone")
     """
+    format = lit(spark_default_time_format()) if format is None else format
     if format is not None:
-        return Column.invoke_expression_over_column(timestamp, expression.StrToUnix, format=format)
+        return Column.invoke_expression_over_column(
+            timestamp, expression.StrToUnix, format=format_time_from_spark(format)
+        )
     else:
         return Column.invoke_expression_over_column(timestamp, expression.StrToUnix)
 
@@ -5322,7 +5333,7 @@ def ucase(str: ColumnOrName) -> Column:
     return Column.invoke_expression_over_column(str, expression.Upper)
 
 
-@meta()
+@meta(unsupported_engines=["bigquery", "snowflake"])
 def unix_date(col: ColumnOrName) -> Column:
     """Returns the number of days since 1970-01-01.
 

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/base/session.py

@@ -72,7 +72,6 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, CONN]):
     _df: t.Type[DF]
 
     SANITIZE_COLUMN_NAMES = False
-    DEFAULT_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss"
 
     def __init__(
         self,
@@ -114,6 +113,10 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, CONN]):
     def _cur(self) -> DBAPICursorWithPandas:
         return self._conn.cursor()
 
+    @property
+    def default_time_format(self) -> str:
+        return self.output_dialect.TIME_FORMAT.strip("'")
+
     def _sanitize_column_name(self, name: str) -> str:
         if self.SANITIZE_COLUMN_NAMES:
             return name.replace("(", "_").replace(")", "_")

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/base/types.py

@@ -416,3 +416,13 @@ class Row(tuple):
             )
         else:
             return "<Row(%s)>" % ", ".join(repr(field) for field in self)
+
+    # SQLFrame Specific
+    @property
+    def _unique_field_names(self) -> t.List[str]:
+        fields = []
+        for i, field in enumerate(self.__fields__):
+            if field in fields:
+                field = field + "_" + str(i)
+            fields.append(field)
+        return fields

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/base/util.py

@@ -13,7 +13,12 @@ if t.TYPE_CHECKING:
     from pyspark.sql.dataframe import SparkSession as PySparkSession
 
     from sqlframe.base import types
-    from sqlframe.base._typing import OptionalPrimitiveType, SchemaInput
+    from sqlframe.base._typing import (
+        ColumnOrLiteral,
+        OptionalPrimitiveType,
+        SchemaInput,
+    )
+    from sqlframe.base.column import Column
     from sqlframe.base.session import _BaseSession
     from sqlframe.base.types import StructType
 
@@ -342,3 +347,21 @@ def sqlglot_to_spark(sqlglot_dtype: exp.DataType) -> types.DataType:
             ]
         )
     raise NotImplementedError(f"Unsupported data type: {sqlglot_dtype}")
+
+
+def format_time_from_spark(value: ColumnOrLiteral) -> Column:
+    from sqlframe.base.column import Column
+    from sqlframe.base.session import _BaseSession
+
+    lit = get_func_from_session("lit")
+    value = lit(value) if not isinstance(value, Column) else value
+    formatted_time = Dialect["spark"].format_time(value.expression)
+    return Column(
+        _BaseSession()
+        .output_dialect.generator()
+        .format_time(exp.StrToTime(this=exp.Null(), format=formatted_time))
+    )
+
+
+def spark_default_time_format() -> str:
+    return Dialect["spark"].TIME_FORMAT.strip("'")

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/bigquery/functions.py

@@ -7,7 +7,11 @@ import typing as t
 from sqlglot import exp as sqlglot_expression
 
 import sqlframe.base.functions
-from sqlframe.base.util import get_func_from_session
+from sqlframe.base.util import (
+    format_time_from_spark,
+    get_func_from_session,
+    spark_default_time_format,
+)
 from sqlframe.bigquery.column import Column
 
 if t.TYPE_CHECKING:
@@ -52,6 +56,7 @@ from sqlframe.base.function_alternatives import (  # noqa
     make_date_from_date_func as make_date,
     to_date_from_timestamp as to_date,
     last_day_with_cast as last_day,
+    sha1_force_sha1_and_to_hex as sha,
     sha1_force_sha1_and_to_hex as sha1,
     hash_from_farm_fingerprint as hash,
     base64_from_blob as base64,
@@ -147,23 +152,15 @@ def from_unixtime(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
 
     session: _BaseSession = _BaseSession()
     lit = get_func_from_session("lit")
-    to_timestamp = get_func_from_session("to_timestamp")
 
     expressions = [Column.ensure_col(col).expression]
-    if format is not None:
-        expressions.append(lit(format).expression)
     return Column(
         sqlglot_expression.Anonymous(
             this="FORMAT_TIMESTAMP",
             expressions=[
-                lit(session.DEFAULT_TIME_FORMAT).expression,
-                to_timestamp(
-                    Column(
-                        sqlglot_expression.Anonymous(
-                            this="TIMESTAMP_SECONDS", expressions=expressions
-                        )
-                    ),
-                    format,
+                lit(session.default_time_format).expression,
+                Column(
+                    sqlglot_expression.Anonymous(this="TIMESTAMP_SECONDS", expressions=expressions)
                 ).expression,
             ],
         )
@@ -173,12 +170,9 @@ def from_unixtime(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
 def unix_timestamp(
     timestamp: t.Optional[ColumnOrName] = None, format: t.Optional[str] = None
 ) -> Column:
-    from sqlframe.base.session import _BaseSession
-
     lit = get_func_from_session("lit")
 
-    if format is None:
-        format = _BaseSession().DEFAULT_TIME_FORMAT
+    format = lit(format or spark_default_time_format())
     return Column(
         sqlglot_expression.Anonymous(
             this="UNIX_SECONDS",
@@ -186,7 +180,7 @@ def unix_timestamp(
                 sqlglot_expression.Anonymous(
                     this="PARSE_TIMESTAMP",
                     expressions=[
-                        lit(format).expression,
+                        format_time_from_spark(format).expression,
                         Column.ensure_col(timestamp).expression,
                         lit("UTC").expression,
                     ],

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/bigquery/functions.pyi

@@ -267,7 +267,6 @@ from sqlframe.base.functions import trunc as trunc
 from sqlframe.base.functions import ucase as ucase
 from sqlframe.base.functions import unbase64 as unbase64
 from sqlframe.base.functions import unhex as unhex
-from sqlframe.base.functions import unix_date as unix_date
 from sqlframe.base.functions import upper as upper
 from sqlframe.base.functions import user as user
 from sqlframe.base.functions import var_pop as var_pop

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/bigquery/session.py

@@ -32,7 +32,6 @@ class BigQuerySession(
     _writer = BigQueryDataFrameWriter
     _df = BigQueryDataFrame
 
-    DEFAULT_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
     QUALIFY_INFO_SCHEMA_WITH_DATABASE = True
     SANITIZE_COLUMN_NAMES = True
 

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/duckdb/functions.py

@@ -46,5 +46,4 @@ from sqlframe.base.function_alternatives import (  # noqa
     array_max_from_sort as array_max,
     sequence_from_generate_series as sequence,
     try_element_at_zero_based as try_element_at,
-    to_unix_timestamp_include_default_format as to_unix_timestamp,
 )

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/duckdb/session.py

@@ -33,8 +33,6 @@ class DuckDBSession(
     _writer = DuckDBDataFrameWriter
     _df = DuckDBDataFrame
 
-    DEFAULT_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
-
     def __init__(self, conn: t.Optional[DuckDBPyConnection] = None, *args, **kwargs):
         import duckdb
         from duckdb.typing import VARCHAR

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/postgres/session.py

@@ -34,8 +34,6 @@ class PostgresSession(
     _writer = PostgresDataFrameWriter
     _df = PostgresDataFrame
 
-    DEFAULT_TIME_FORMAT = "yyyy-MM-dd HH:MI:SS"
-
     def __init__(self, conn: t.Optional[psycopg2_connection] = None):
         if not hasattr(self, "_conn"):
             super().__init__(conn)

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/snowflake/functions.pyi

@@ -207,7 +207,6 @@ from sqlframe.base.functions import (
     trim as trim,
     trunc as trunc,
     ucase as ucase,
-    unix_date as unix_date,
     upper as upper,
     user as user,
     var_pop as var_pop,

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe/snowflake/session.py

@@ -57,8 +57,6 @@ class SnowflakeSession(
     _writer = SnowflakeDataFrameWriter
     _df = SnowflakeDataFrame
 
-    DEFAULT_TIME_FORMAT = "YYYY-MM-DD HH:MI:SS"
-
     def __init__(self, conn: t.Optional[SnowflakeConnection] = None):
         import snowflake
 

{sqlframe-1.11.0 → sqlframe-1.13.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 1.11.0
+Version: 1.13.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman