sqlframe 1.12.0__tar.gz → 1.14.0__tar.gz

{sqlframe-1.12.0 → sqlframe-1.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 1.12.0
+Version: 1.14.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-1.12.0 → sqlframe-1.14.0}/docs/bigquery.md

@@ -307,7 +307,6 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [date_diff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_diff.html)
 * [datediff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.datediff.html)
 * [date_format](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_format.html)
-    * [The format string should be in BigQuery syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements)
 * [date_sub](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_sub.html)
 * [date_trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_trunc.html)
 * [dayofmonth](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.dayofmonth.html)
@@ -442,19 +441,17 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [toDegrees](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toDegrees.html)
 * [toRadians](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toRadians.html)
 * [to_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_date.html)
-    * [The format string should be in BigQuery syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements)
 * [to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html)
-    * [The format string should be in BigQuery syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)
     * Shorthand expressions not supported. Ex: Use `month` instead of `mon`
+* [try_to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.try_to_timestamp.html)
 * [typeof](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.typeof.html)
 * [ucase](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.ucase.html)
 * [unbase64](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unbase64.html)
 * [unhex](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unhex.html)
 * [unix_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unix_timestamp.html)
-    * [The format string should be in BigQuery syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements)
 * [upper](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.upper.html)
 * [var_pop](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.var_pop.html)
 * [var_samp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.var_samp.html)

{sqlframe-1.12.0 → sqlframe-1.14.0}/docs/duckdb.md

@@ -276,10 +276,10 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [dateadd](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.dateadd.html)
 * [date_diff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_diff.html)
 * [datediff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.datediff.html)
-* [date_format](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_format.html)                     
-    * [The format string should be in DuckDB syntax](https://duckdb.org/docs/sql/functions/dateformat.html#format-specifiers)
+* [date_format](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_format.html)
 * [date_sub](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_sub.html)
 * [date_trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_trunc.html)
+* [day](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.day.html)
 * [dayofmonth](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.dayofmonth.html)
 * [dayofweek](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.dayofweek.html)
 * [dayofyear](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.dayofyear.html)
@@ -405,22 +405,19 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [toDegrees](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toDegrees.html)
 * [toRadians](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toRadians.html)
 * [to_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_date.html)
-    * [The format string should be in DuckDB syntax](https://duckdb.org/docs/sql/functions/dateformat.html#format-specifiers)
 * [to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html)
-    * [The format string should be in DuckDB syntax](https://duckdb.org/docs/sql/functions/dateformat.html#format-specifiers)
 * [to_unix_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_unix_timestamp.html)
-    * [The format string should be in DuckDB syntax](https://duckdb.org/docs/sql/functions/dateformat.html#format-specifiers
     * The values must match the format string (null will not be returned if they do not)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)
 * [try_element_at](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.try_element_at.html)
+* [try_to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.try_to_timestamp.html)
 * [typeof](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.typeof.html)
 * [ucase](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.ucase.html)
 * [unbase64](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unbase64.html)
 * [unhex](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unhex.html)
 * [unix_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unix_timestamp.html)
-    * [The format string should be in DuckDB syntax](https://duckdb.org/docs/sql/functions/dateformat.html#format-specifiers)
 * [upper](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.upper.html)
 * [var_pop](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.var_pop.html)
 * [var_samp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.var_samp.html)

{sqlframe-1.12.0 → sqlframe-1.14.0}/docs/postgres.md

@@ -284,7 +284,6 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [date_diff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_diff.html)
 * [datediff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.datediff.html)
 * [date_format](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_format.html)
-    * [The format string should be in Postgres syntax](https://www.postgresql.org/docs/current/functions-formatting.html#FUNCTIONS-FORMATTING-DATETIME-TABLE)
 * [date_sub](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_sub.html)
 * [date_trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_trunc.html)
     * Rounded whole number is returned
@@ -397,15 +396,15 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [toDegrees](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toDegrees.html)
 * [toRadians](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toRadians.html)
 * [to_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_date.html)
-    * [The format string should be in Postgres syntax](https://www.postgresql.org/docs/current/functions-formatting.html#FUNCTIONS-FORMATTING-DATETIME-TABLE)
 * [to_number](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_number.html)
 * [to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html)
-    * [The format string should be in Postgres syntax](https://www.postgresql.org/docs/current/functions-formatting.html#FUNCTIONS-FORMATTING-DATETIME-TABLE)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)
 * [try_element_at](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.try_element_at.html)
   * Negative index returns null and cannot lookup elements in maps
+* [try_to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.try_to_timestamp.html)
+* [typeof](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.typeof.html)
 * [ucase](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.ucase.html)
 * [unbase64](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unbase64.html)
 * [unix_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unix_timestamp.html)

{sqlframe-1.12.0 → sqlframe-1.14.0}/docs/snowflake.md

@@ -307,7 +307,6 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [date_diff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_diff.html)
 * [datediff](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.datediff.html)
 * [date_format](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_format.html)
-  * [The format string should be in Snowflake syntax](https://docs.snowflake.com/en/sql-reference/functions-conversion#label-date-time-format-conversion)
 * [date_sub](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_sub.html)
 * [date_trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.date_trunc.html)
 * [dayofmonth](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.dayofmonth.html)
@@ -440,18 +439,17 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [toDegrees](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toDegrees.html)
 * [toRadians](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.toRadians.html)
 * [to_date](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_date.html)
-  * [The format string should be in Snowflake syntax](https://docs.snowflake.com/en/sql-reference/functions-conversion#label-date-time-format-conversion)
 * [to_number](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_number.html)
 * [to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html)
-  * [The format string should be in Snowflake syntax](https://docs.snowflake.com/en/sql-reference/functions-conversion#label-date-time-format-conversion)
 * [translate](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.translate.html)
 * [trim](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trim.html)
 * [trunc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.trunc.html)
+* [try_to_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.try_to_timestamp.html)
+* [typeof](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.typeof.html)
 * [ucase](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.ucase.html)
 * [unbase64](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unbase64.html)
 * [unhex](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unhex.html)
 * [unix_timestamp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.unix_timestamp.html)
-  * [The format string should be in Snowflake syntax](https://docs.snowflake.com/en/sql-reference/functions-conversion#label-date-time-format-conversion)
 * [upper](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.upper.html)
 * [var_pop](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.var_pop.html)
 * [var_samp](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.var_samp.html)

{sqlframe-1.12.0 → sqlframe-1.14.0}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.8",
     install_requires=[
         "prettytable<3.11.0",
-        "sqlglot>=24.0.0,<25.4",
+        "sqlglot>=24.0.0,<25.5",
         "typing_extensions>=4.8,<5",
     ],
     extras_require={
@@ -42,7 +42,7 @@ setup(
             "pytest-xdist>=3.6,<3.7",
             "pre-commit>=3.5;python_version=='3.8'",
             "pre-commit>=3.7,<3.8;python_version>='3.9'",
-            "ruff>=0.4.4,<0.5",
+            "ruff>=0.4.4,<0.6",
             "types-psycopg2>=2.9,<3",
         ],
         "docs": [

{sqlframe-1.12.0 → sqlframe-1.14.0}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.12.0'
-__version_tuple__ = version_tuple = (1, 12, 0)
+__version__ = version = '1.14.0'
+__version_tuple__ = version_tuple = (1, 14, 0)

{sqlframe-1.12.0 → sqlframe-1.14.0}/sqlframe/base/function_alternatives.py

@@ -6,11 +6,16 @@ import re
 import typing as t
 
 from sqlglot import exp as expression
+from sqlglot.dialects.dialect import build_formatted_time
 from sqlglot.helper import ensure_list
 from sqlglot.helper import flatten as _flatten
 
 from sqlframe.base.column import Column
-from sqlframe.base.util import get_func_from_session
+from sqlframe.base.util import (
+    format_time_from_spark,
+    get_func_from_session,
+    spark_default_time_format,
+)
 
 if t.TYPE_CHECKING:
     from sqlframe.base._typing import ColumnOrLiteral, ColumnOrName
@@ -556,6 +561,14 @@ def to_date_from_timestamp(col: ColumnOrName, format: t.Optional[str] = None) ->
     return to_date(to_timestamp(col, format))
 
 
+def to_date_time_format(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
+    from sqlframe.base.functions import to_date
+
+    lit = get_func_from_session("lit")
+    format = lit(format or spark_default_time_format())
+    return to_date(col, format=format)
+
+
 def last_day_with_cast(col: ColumnOrName) -> Column:
     from sqlframe.base.functions import last_day
 
@@ -715,14 +728,10 @@ def months_between_cast_as_date_cast_roundoff(
 
 
 def from_unixtime_from_timestamp(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
-    from sqlframe.base.session import _BaseSession
-
-    session: _BaseSession = _BaseSession()
     lit = get_func_from_session("lit")
     col_func = get_func_from_session("col")
 
-    if format is None:
-        format = session.DEFAULT_TIME_FORMAT
+    format = lit(format or spark_default_time_format())
     return Column.invoke_expression_over_column(
         Column(
             expression.Anonymous(
@@ -731,7 +740,7 @@ def from_unixtime_from_timestamp(col: ColumnOrName, format: t.Optional[str] = No
             )
         ),
         expression.TimeToStr,
-        format=lit(format),
+        format=format_time_from_spark(format),  # type: ignore
     )
 
 
@@ -1511,10 +1520,106 @@ def to_unix_timestamp_include_default_format(
     format: t.Optional[ColumnOrName] = None,
 ) -> Column:
     from sqlframe.base.functions import to_unix_timestamp
+    from sqlframe.base.session import _BaseSession
+
+    if not format:
+        format = _BaseSession().output_dialect.TIME_FORMAT
+    else:
+        format = format_time_from_spark(format)
+    return to_unix_timestamp(timestamp, format)
+
+
+def array_append_list_append(col: ColumnOrName, value: ColumnOrLiteral) -> Column:
+    lit = get_func_from_session("lit")
+    value = value if isinstance(value, Column) else lit(value)
+    return Column.invoke_anonymous_function(col, "LIST_APPEND", value)
 
+
+def array_append_using_array_cat(col: ColumnOrName, value: ColumnOrLiteral) -> Column:
     lit = get_func_from_session("lit")
+    array = get_func_from_session("array")
+    value = value if isinstance(value, Column) else lit(value)
+    return Column.invoke_anonymous_function(col, "ARRAY_CONCAT", array(value))
 
-    if not format:
-        format = lit("%Y-%m-%d %H:%M:%S")
 
-    return to_unix_timestamp(timestamp, format)
+def day_with_try_to_timestamp(col: ColumnOrName) -> Column:
+    from sqlframe.base.functions import day
+
+    try_to_timestamp = get_func_from_session("try_to_timestamp")
+    to_date = get_func_from_session("to_date")
+    when = get_func_from_session("when")
+    _is_string = get_func_from_session("_is_string")
+    coalesce = get_func_from_session("coalesce")
+    return day(
+        when(
+            _is_string(col),
+            coalesce(try_to_timestamp(col), to_date(col)),
+        ).otherwise(col)
+    )
+
+
+def try_to_timestamp_strptime(col: ColumnOrName, format: t.Optional[ColumnOrName] = None) -> Column:
+    lit = get_func_from_session("lit")
+
+    format = lit(format or spark_default_time_format())
+    return Column.invoke_anonymous_function(col, "TRY_STRPTIME", format_time_from_spark(format))  # type: ignore
+
+
+def try_to_timestamp_safe(col: ColumnOrName, format: t.Optional[ColumnOrName] = None) -> Column:
+    lit = get_func_from_session("lit")
+
+    format = lit(format or spark_default_time_format())
+    return Column.invoke_anonymous_function(
+        format_time_from_spark(format),  # type: ignore
+        "SAFE.PARSE_TIMESTAMP",
+        col,  # type: ignore
+    )
+
+
+def try_to_timestamp_pgtemp(col: ColumnOrName, format: t.Optional[ColumnOrName] = None) -> Column:
+    lit = get_func_from_session("lit")
+
+    format = lit(format or spark_default_time_format())
+    return Column.invoke_anonymous_function(
+        col,
+        "pg_temp.TRY_TO_TIMESTAMP",
+        format_time_from_spark(format),  # type: ignore
+    )
+
+
+def typeof_pg_typeof(col: ColumnOrName) -> Column:
+    return Column.invoke_anonymous_function(col, "pg_typeof").cast("regtype").cast("text")
+
+
+def typeof_from_variant(col: ColumnOrName) -> Column:
+    col = Column.invoke_anonymous_function(col, "TO_VARIANT")
+    return Column.invoke_anonymous_function(col, "TYPEOF")
+
+
+def _is_string_using_typeof_varchar(col: ColumnOrName) -> Column:
+    typeof = get_func_from_session("typeof")
+    lit = get_func_from_session("lit")
+    return lit(typeof(col) == lit("VARCHAR"))
+
+
+def _is_string_using_typeof_char_varying(col: ColumnOrName) -> Column:
+    typeof = get_func_from_session("typeof")
+    lit = get_func_from_session("lit")
+    return lit(
+        (typeof(col) == lit("text"))
+        | (typeof(col) == lit("character varying"))
+        | (typeof(col) == lit("unknown"))
+        | (typeof(col) == lit("text"))
+    )
+
+
+def _is_string_using_typeof_string(col: ColumnOrName) -> Column:
+    typeof = get_func_from_session("typeof")
+    lit = get_func_from_session("lit")
+    return lit(typeof(col) == lit("STRING"))
+
+
+def _is_string_using_typeof_string_lcase(col: ColumnOrName) -> Column:
+    typeof = get_func_from_session("typeof")
+    lit = get_func_from_session("lit")
+    return lit(typeof(col) == lit("string"))

{sqlframe-1.12.0 → sqlframe-1.14.0}/sqlframe/base/functions.py

@@ -6,12 +6,19 @@ import decimal
 import logging
 import typing as t
 
+from sqlglot import Dialect
 from sqlglot import exp as expression
 from sqlglot.helper import ensure_list
 from sqlglot.helper import flatten as _flatten
 
 from sqlframe.base.column import Column
 from sqlframe.base.decorators import func_metadata as meta
+from sqlframe.base.util import (
+    format_time_from_spark,
+    get_func_from_session,
+    spark_default_date_format,
+    spark_default_time_format,
+)
 
 if t.TYPE_CHECKING:
     from pyspark.sql.session import SparkContext
@@ -695,7 +702,7 @@ def date_format(col: ColumnOrName, format: str) -> Column:
     return Column.invoke_expression_over_column(
         Column(expression.TimeStrToTime(this=Column.ensure_col(col).expression)),
         expression.TimeToStr,
-        format=lit(format),
+        format=format_time_from_spark(format),
     )
 
 
@@ -875,17 +882,21 @@ def months_between(
 
 @meta()
 def to_date(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
+    format = lit(format or spark_default_date_format())
     if format is not None:
         return Column.invoke_expression_over_column(
-            col, expression.TsOrDsToDate, format=lit(format)
+            col, expression.TsOrDsToDate, format=format_time_from_spark(format)
         )
     return Column.invoke_expression_over_column(col, expression.TsOrDsToDate)
 
 
 @meta()
 def to_timestamp(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
+    format = lit(format or spark_default_time_format())
     if format is not None:
-        return Column.invoke_expression_over_column(col, expression.StrToTime, format=lit(format))
+        return Column.invoke_expression_over_column(
+            col, expression.StrToTime, format=format_time_from_spark(format)
+        )
 
     return Column.ensure_col(col).cast("timestamp")
 
@@ -916,23 +927,23 @@ def last_day(col: ColumnOrName) -> Column:
 
 @meta()
 def from_unixtime(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
-    from sqlframe.base.session import _BaseSession
-
-    if format is None:
-        format = _BaseSession().DEFAULT_TIME_FORMAT
-    return Column.invoke_expression_over_column(col, expression.UnixToStr, format=lit(format))
+    format = lit(format or spark_default_time_format())
+    return Column.invoke_expression_over_column(
+        col,
+        expression.UnixToStr,
+        format=format_time_from_spark(format),  # type: ignore
+    )
 
 
 @meta()
 def unix_timestamp(
     timestamp: t.Optional[ColumnOrName] = None, format: t.Optional[str] = None
 ) -> Column:
-    from sqlframe.base.session import _BaseSession
-
-    if format is None:
-        format = _BaseSession().DEFAULT_TIME_FORMAT
+    format = lit(format or spark_default_time_format())
     return Column.invoke_expression_over_column(
-        timestamp, expression.StrToUnix, format=lit(format)
+        timestamp,
+        expression.StrToUnix,
+        format=format_time_from_spark(format),  # type: ignore
     ).cast("bigint")
 
 
@@ -1289,7 +1300,7 @@ def array_agg(col: ColumnOrName) -> Column:
     return Column.invoke_expression_over_column(col, expression.ArrayAgg)
 
 
-@meta(unsupported_engines="*")
+@meta()
 def array_append(col: ColumnOrName, value: ColumnOrLiteral) -> Column:
     value = value if isinstance(value, Column) else lit(value)
     return Column.invoke_anonymous_function(col, "ARRAY_APPEND", value)
@@ -1737,7 +1748,7 @@ def map_zip_with(
     return Column.invoke_anonymous_function(col1, "MAP_ZIP_WITH", col2, Column(f_expression))
 
 
-@meta(unsupported_engines=["postgres", "snowflake"])
+@meta()
 def typeof(col: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(col, "TYPEOF")
 
@@ -2156,7 +2167,7 @@ def datepart(field: ColumnOrName, source: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(field, "datepart", source)
 
 
-@meta(unsupported_engines="*")
+@meta(unsupported_engines=["bigquery", "postgres", "snowflake"])
 def day(col: ColumnOrName) -> Column:
     return Column.invoke_expression_over_column(col, expression.Day)
 
@@ -5106,8 +5117,11 @@ def to_unix_timestamp(
     [Row(r=None)]
     >>> spark.conf.unset("spark.sql.session.timeZone")
     """
+    format = lit(spark_default_time_format()) if format is None else format
     if format is not None:
-        return Column.invoke_expression_over_column(timestamp, expression.StrToUnix, format=format)
+        return Column.invoke_expression_over_column(
+            timestamp, expression.StrToUnix, format=format_time_from_spark(format)
+        )
     else:
         return Column.invoke_expression_over_column(timestamp, expression.StrToUnix)
 
@@ -5268,7 +5282,7 @@ def try_element_at(col: ColumnOrName, extraction: ColumnOrName) -> Column:
     )
 
 
-@meta(unsupported_engines="*")
+@meta()
 def try_to_timestamp(col: ColumnOrName, format: t.Optional[ColumnOrName] = None) -> Column:
     """
     Parses the `col` with the `format` to a timestamp. The function always
@@ -5293,10 +5307,8 @@ def try_to_timestamp(col: ColumnOrName, format: t.Optional[ColumnOrName] = None)
     >>> df.select(try_to_timestamp(df.t, lit('yyyy-MM-dd HH:mm:ss')).alias('dt')).collect()
     [Row(dt=datetime.datetime(1997, 2, 28, 10, 30))]
     """
-    if format is not None:
-        return Column.invoke_anonymous_function(col, "try_to_timestamp", format)
-    else:
-        return Column.invoke_anonymous_function(col, "try_to_timestamp")
+    format = lit(format or spark_default_time_format())
+    return Column.invoke_anonymous_function(col, "try_to_timestamp", format_time_from_spark(format))  # type: ignore
 
 
 @meta()
@@ -5324,7 +5336,7 @@ def ucase(str: ColumnOrName) -> Column:
     return Column.invoke_expression_over_column(str, expression.Upper)
 
 
-@meta()
+@meta(unsupported_engines=["bigquery", "snowflake"])
 def unix_date(col: ColumnOrName) -> Column:
     """Returns the number of days since 1970-01-01.
 
@@ -5788,6 +5800,20 @@ def years(col: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(col, "years")
 
 
+# SQLFrame specific
+@meta()
+def _is_string(col: ColumnOrName) -> Column:
+    col = Column.invoke_anonymous_function(col, "TO_VARIANT")
+    return Column.invoke_anonymous_function(col, "IS_VARCHAR")
+
+
+@meta()
+def _is_date(col: ColumnOrName) -> Column:
+    typeof = get_func_from_session("typeof")
+    upper = get_func_from_session("upper")
+    return lit(upper(typeof(col)) == lit("DATE"))
+
+
 @meta()
 def _lambda_quoted(value: str) -> t.Optional[bool]:
     return False if value == "_" else None

{sqlframe-1.12.0 → sqlframe-1.14.0}/sqlframe/base/session.py

@@ -72,7 +72,6 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, CONN]):
     _df: t.Type[DF]
 
     SANITIZE_COLUMN_NAMES = False
-    DEFAULT_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss"
 
     def __init__(
         self,
@@ -114,6 +113,10 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, CONN]):
     def _cur(self) -> DBAPICursorWithPandas:
         return self._conn.cursor()
 
+    @property
+    def default_time_format(self) -> str:
+        return self.output_dialect.TIME_FORMAT.strip("'")
+
     def _sanitize_column_name(self, name: str) -> str:
         if self.SANITIZE_COLUMN_NAMES:
             return name.replace("(", "_").replace(")", "_")

{sqlframe-1.12.0 → sqlframe-1.14.0}/sqlframe/base/util.py

@@ -13,7 +13,12 @@ if t.TYPE_CHECKING:
     from pyspark.sql.dataframe import SparkSession as PySparkSession
 
     from sqlframe.base import types
-    from sqlframe.base._typing import OptionalPrimitiveType, SchemaInput
+    from sqlframe.base._typing import (
+        ColumnOrLiteral,
+        OptionalPrimitiveType,
+        SchemaInput,
+    )
+    from sqlframe.base.column import Column
     from sqlframe.base.session import _BaseSession
     from sqlframe.base.types import StructType
 
@@ -342,3 +347,25 @@ def sqlglot_to_spark(sqlglot_dtype: exp.DataType) -> types.DataType:
             ]
         )
     raise NotImplementedError(f"Unsupported data type: {sqlglot_dtype}")
+
+
+def format_time_from_spark(value: ColumnOrLiteral) -> Column:
+    from sqlframe.base.column import Column
+    from sqlframe.base.session import _BaseSession
+
+    lit = get_func_from_session("lit")
+    value = lit(value) if not isinstance(value, Column) else value
+    formatted_time = Dialect["spark"].format_time(value.expression)
+    return Column(
+        _BaseSession()
+        .output_dialect.generator()
+        .format_time(exp.StrToTime(this=exp.Null(), format=formatted_time))
+    )
+
+
+def spark_default_time_format() -> str:
+    return Dialect["spark"].TIME_FORMAT.strip("'")
+
+
+def spark_default_date_format() -> str:
+    return Dialect["spark"].DATE_FORMAT.strip("'")

{sqlframe-1.12.0 → sqlframe-1.14.0}/sqlframe/bigquery/functions.py

@@ -7,7 +7,11 @@ import typing as t
 from sqlglot import exp as sqlglot_expression
 
 import sqlframe.base.functions
-from sqlframe.base.util import get_func_from_session
+from sqlframe.base.util import (
+    format_time_from_spark,
+    get_func_from_session,
+    spark_default_time_format,
+)
 from sqlframe.bigquery.column import Column
 
 if t.TYPE_CHECKING:
@@ -68,6 +72,9 @@ from sqlframe.base.function_alternatives import (  # noqa
     array_union_using_array_concat as array_union,
     sequence_from_generate_array as sequence,
     position_as_strpos as position,
+    try_to_timestamp_safe as try_to_timestamp,
+    _is_string_using_typeof_string as _is_string,
+    array_append_using_array_cat as array_append,
 )
 
 
@@ -148,23 +155,15 @@ def from_unixtime(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
 
     session: _BaseSession = _BaseSession()
     lit = get_func_from_session("lit")
-    to_timestamp = get_func_from_session("to_timestamp")
 
     expressions = [Column.ensure_col(col).expression]
-    if format is not None:
-        expressions.append(lit(format).expression)
     return Column(
         sqlglot_expression.Anonymous(
             this="FORMAT_TIMESTAMP",
             expressions=[
-                lit(session.DEFAULT_TIME_FORMAT).expression,
-                to_timestamp(
-                    Column(
-                        sqlglot_expression.Anonymous(
-                            this="TIMESTAMP_SECONDS", expressions=expressions
-                        )
-                    ),
-                    format,
+                lit(session.default_time_format).expression,
+                Column(
+                    sqlglot_expression.Anonymous(this="TIMESTAMP_SECONDS", expressions=expressions)
                 ).expression,
             ],
         )
@@ -174,12 +173,9 @@ def from_unixtime(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
 def unix_timestamp(
     timestamp: t.Optional[ColumnOrName] = None, format: t.Optional[str] = None
 ) -> Column:
-    from sqlframe.base.session import _BaseSession
-
     lit = get_func_from_session("lit")
 
-    if format is None:
-        format = _BaseSession().DEFAULT_TIME_FORMAT
+    format = lit(format or spark_default_time_format())
     return Column(
         sqlglot_expression.Anonymous(
             this="UNIX_SECONDS",
@@ -187,7 +183,7 @@ def unix_timestamp(
                 sqlglot_expression.Anonymous(
                     this="PARSE_TIMESTAMP",
                     expressions=[
-                        lit(format).expression,
+                        format_time_from_spark(format).expression,
                         Column.ensure_col(timestamp).expression,
                         lit("UTC").expression,
                     ],