sqlframe 3.32.1__tar.gz → 3.33.0__tar.gz

{sqlframe-3.32.1 → sqlframe-3.33.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.32.1
+Version: 3.33.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.32.1 → sqlframe-3.33.0}/docs/bigquery.md

@@ -219,6 +219,8 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [asc_nulls_last](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.asc_nulls_last.html)
 * [between](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.between.html)
 * [cast](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.cast.html)
+* [contains](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.contains.html)
+  * The argument to `contains` must be a string
 * [desc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.desc.html)
 * [desc_nulls_first](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.desc_nulls_first.html)
 * [desc_nulls_last](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.desc_nulls_last.html)
@@ -351,6 +353,8 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [concat](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.concat.html)
     * Only works on strings (does not work on arrays)
 * [concat_ws](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.concat_ws.html)
+* [contains](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.contains.html)
+  * The argument to `contains` must be a string
 * [corr](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.corr.html)
 * [cos](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.cos.html)
 * [cosh](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.cosh.html)

{sqlframe-3.32.1 → sqlframe-3.33.0}/docs/duckdb.md

@@ -183,6 +183,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [asc_nulls_last](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.asc_nulls_last.html)
 * [between](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.between.html)
 * [cast](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.cast.html)
+* [contains](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.contains.html)
 * [desc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.desc.html)
 * [desc_nulls_first](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.desc_nulls_first.html)
 * [desc_nulls_last](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.desc_nulls_last.html)

{sqlframe-3.32.1 → sqlframe-3.33.0}/docs/snowflake.md

@@ -215,6 +215,7 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [asc_nulls_last](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.asc_nulls_last.html)
 * [between](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.between.html)
 * [cast](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.cast.html)
+* [contains](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.contains.html)
 * [desc](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.desc.html)
 * [desc_nulls_first](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.desc_nulls_first.html)
 * [desc_nulls_last](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.Column.desc_nulls_last.html)

{sqlframe-3.32.1 → sqlframe-3.33.0}/sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.32.1'
-__version_tuple__ = version_tuple = (3, 32, 1)
+__version__ = version = '3.33.0'
+__version_tuple__ = version_tuple = (3, 33, 0)

{sqlframe-3.32.1 → sqlframe-3.33.0}/sqlframe/base/column.py

@@ -517,3 +517,7 @@ class Column:
         +---+
         """
         return self.getItem(name)
+
+    def contains(self, value: t.Union[str, Column]) -> Column:
+        value = self._lit(value) if not isinstance(value, Column) else value
+        return self.invoke_expression_over_column(self, exp.Contains, expression=value.expression)

{sqlframe-3.32.1 → sqlframe-3.33.0}/sqlframe/base/function_alternatives.py

@@ -78,17 +78,6 @@ def to_timestamp_tz(col: ColumnOrName, format: t.Optional[str] = None) -> Column
     return Column.ensure_col(col).cast("timestamptz", dialect="duckdb")
 
 
-def to_timestamp_just_timestamp(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
-    from sqlframe.base.session import _BaseSession
-
-    if format is not None:
-        return Column.invoke_expression_over_column(
-            col, expression.StrToTime, format=_BaseSession().format_time(format)
-        )
-
-    return Column.ensure_col(col).cast("datetime", dialect="bigquery")
-
-
 def bitwise_not_from_bitnot(col: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(col, "BITNOT")
 

{sqlframe-3.32.1 → sqlframe-3.33.0}/sqlframe/base/functions.py

@@ -1356,7 +1356,6 @@ def to_date(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
 @meta()
 def to_timestamp(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
     from sqlframe.base.function_alternatives import (
-        to_timestamp_just_timestamp,
         to_timestamp_tz,
         to_timestamp_with_time_zone,
     )
@@ -1366,9 +1365,6 @@ def to_timestamp(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
     if session._is_duckdb:
         return to_timestamp_tz(col, format)
 
-    if session._is_bigquery:
-        return to_timestamp_just_timestamp(col, format)
-
     if session._is_postgres:
         return to_timestamp_with_time_zone(col, format)
 
@@ -3068,7 +3064,7 @@ def character_length(str: ColumnOrName) -> Column:
     return Column.invoke_expression_over_column(str, expression.Length)
 
 
-@meta(unsupported_engines=["bigquery", "postgres"])
+@meta(unsupported_engines=["postgres"])
 def contains(left: ColumnOrName, right: ColumnOrName) -> Column:
     return Column.invoke_expression_over_column(
         left, expression.Contains, expression=Column.ensure_col(right).column_expression
@@ -6594,27 +6590,16 @@ def unix_micros(col: ColumnOrName) -> Column:
     """
     from sqlframe.base.function_alternatives import unix_micros_multiply_epoch
 
+    to_timestamp = get_func_from_session("to_timestamp")
+
     if _get_session()._is_duckdb:
         return Column.invoke_anonymous_function(col, "epoch_us")
 
-    if _get_session()._is_bigquery:
-        return Column(
-            expression.Anonymous(
-                this="UNIX_MICROS",
-                expressions=[
-                    expression.Anonymous(
-                        this="TIMESTAMP",
-                        expressions=[
-                            Column.ensure_col(col).column_expression,
-                        ],
-                    )
-                ],
-            )
-        )
-
     if _get_session()._is_postgres or _get_session()._is_snowflake:
         return unix_micros_multiply_epoch(col)
 
+    col = to_timestamp(col)
+
     return Column.invoke_anonymous_function(col, "unix_micros")
 
 
@@ -6666,22 +6651,6 @@ def unix_seconds(col: ColumnOrName) -> Column:
     if _get_session()._is_postgres:
         return unix_seconds_extract_epoch(col)
 
-    if _get_session()._is_bigquery:
-        return Column(
-            expression.Anonymous(
-                this="UNIX_SECONDS",
-                expressions=[
-                    expression.Anonymous(
-                        this="TIMESTAMP",
-                        expressions=[
-                            Column.ensure_col(col).column_expression,
-                            expression.Literal.string("UTC"),
-                        ],
-                    )
-                ],
-            )
-        )
-
     return Column.invoke_expression_over_column(col, expression.UnixSeconds)
 
 

{sqlframe-3.32.1 → sqlframe-3.33.0}/sqlframe/base/session.py

@@ -304,7 +304,10 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, TABLE, CONN, UDF_REGIS
             elif isinstance(value, float):
                 return "double"
             elif isinstance(value, datetime.datetime):
-                return "timestamp"
+                if value.tzinfo:
+                    # Spark defaults `timestamp` to be a timestamp with timezone
+                    return "timestamp"
+                return "timestampntz"
             elif isinstance(value, datetime.date):
                 return "date"
             elif isinstance(value, str):

{sqlframe-3.32.1 → sqlframe-3.33.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.32.1
+Version: 3.33.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.32.1 → sqlframe-3.33.0}/tests/integration/engines/duck/test_duckdb_dataframe.py

@@ -35,7 +35,7 @@ def duckdb_datatypes(duckdb_session: DuckDBSession) -> DuckDBDataFrame:
             "array<bigint>_col",
             "struct<a:bigint>_col",
             "date_col",
-            "timestamp_col",
+            "timestampntz_col",
             "timestamptz_col",
             "boolean_col",
         ],
@@ -79,7 +79,7 @@ root
  |-- struct<a:bigint>_col: struct<a: bigint> (nullable = true)
  |    |-- a: bigint (nullable = true)
  |-- date_col: date (nullable = true)
- |-- timestamp_col: timestamp (nullable = true)
+ |-- timestampntz_col: timestamp_ntz (nullable = true)
  |-- timestamptz_col: timestamp (nullable = true)
  |-- boolean_col: boolean (nullable = true)""".strip()
     )
@@ -157,8 +157,8 @@ def test_schema_nested(duckdb_datatypes: DuckDBDataFrame):
     )
     assert struct_fields[7].name == "date_col"
     assert struct_fields[7].dataType == types.DateType()
-    assert struct_fields[8].name == "timestamp_col"
-    assert struct_fields[8].dataType == types.TimestampType()
+    assert struct_fields[8].name == "timestampntz_col"
+    assert struct_fields[8].dataType == types.TimestampNTZType()
     assert struct_fields[9].name == "timestamptz_col"
     assert struct_fields[9].dataType == types.TimestampType()
     assert struct_fields[10].name == "boolean_col"

{sqlframe-3.32.1 → sqlframe-3.33.0}/tests/integration/engines/spark/test_spark_dataframe.py

@@ -35,7 +35,7 @@ def spark_datatypes(spark_session: SparkSession) -> SparkDataFrame:
             "array<bigint>_col",
             "struct<a:bigint>_col",
             "date_col",
-            "timestamp_col",
+            "timestampntz_col",
             "timestamptz_col",
             "boolean_col",
         ],
@@ -79,7 +79,7 @@ root
  |-- struct<a:bigint>_col: struct<a: bigint> (nullable = false)
  |    |-- a: bigint (nullable = true)
  |-- date_col: date (nullable = true)
- |-- timestamp_col: timestamp (nullable = true)
+ |-- timestampntz_col: timestamp_ntz (nullable = true)
  |-- timestamptz_col: timestamp (nullable = true)
  |-- boolean_col: boolean (nullable = false)""".strip()
     )
@@ -157,8 +157,8 @@ def test_schema_nested(spark_datatypes: SparkDataFrame):
     )
     assert struct_fields[7].name == "date_col"
     assert struct_fields[7].dataType == types.DateType()
-    assert struct_fields[8].name == "timestamp_col"
-    assert struct_fields[8].dataType == types.TimestampType()
+    assert struct_fields[8].name == "timestampntz_col"
+    assert struct_fields[8].dataType == types.TimestampNTZType()
     assert struct_fields[9].name == "timestamptz_col"
     assert struct_fields[9].dataType == types.TimestampType()
     assert struct_fields[10].name == "boolean_col"

{sqlframe-3.32.1 → sqlframe-3.33.0}/tests/integration/engines/test_engine_column.py

@@ -68,3 +68,23 @@ def test_column_get_field_struct(get_session: t.Callable[[], _BaseSession]):
     result3 = df.select(df.r.a.alias("a_dot")).collect()
     assert result3[0][0] == 1
     assert result3[1][0] == 2
+
+
+def test_contains(get_session: t.Callable[[], _BaseSession], get_func):
+    session = get_session()
+    if session._is_postgres:
+        pytest.skip("Postgres does not support the contains function")
+    lit = get_func("lit", session)
+    df = session.createDataFrame([Row(a="foo"), Row(a="bar")])
+    df_foo = df.select(df.a.contains("foo")).collect()
+    assert df_foo[0][0] is True
+    assert df_foo[1][0] is False
+    df_foo_lit = df.select(df.a.contains(lit("foo"))).collect()
+    assert df_foo_lit[0][0] is True
+    assert df_foo_lit[1][0] is False
+    df_bar = df.select(df.a.contains("bar")).collect()
+    assert df_bar[0][0] is False
+    assert df_bar[1][0] is True
+    df_bar_lit = df.select(df.a.contains(lit("bar"))).collect()
+    assert df_bar_lit[0][0] is False
+    assert df_bar_lit[1][0] is True

{sqlframe-3.32.1 → sqlframe-3.33.0}/tests/integration/engines/test_int_functions.py

@@ -172,7 +172,7 @@ def test_col(get_session_and_func, input, output):
         ([1, 2, 3], "array<bigint>"),
         (Row(a=1), "struct<a:bigint>"),
         (datetime.date(2022, 1, 1), "date"),
-        (datetime.datetime(2022, 1, 1, 0, 0, 0), "timestamptz"),
+        (datetime.datetime(2022, 1, 1, 0, 0, 0), "timestampntz"),
         (datetime.datetime(2022, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc), "timestamptz"),
         (True, "boolean"),
         (bytes("test", "utf-8"), "binary"),
@@ -188,9 +188,15 @@ def test_typeof(get_session_and_func, get_types, arg, expected):
         if isinstance(session, PySparkSession)
         else dialect_to_string(session.execution_dialect)
     )
-    if isinstance(session, (SparkSession, PySparkSession, DatabricksSession)):
+    if isinstance(session, (SparkSession, PySparkSession, DatabricksSession, BigQuerySession)):
         if expected == "timestamptz":
             expected = "timestamp"
+    if isinstance(session, PostgresSession):
+        if expected == "timestampntz":
+            expected = "timestamp"
+    if isinstance(session, BigQuerySession):
+        if expected == "timestampntz":
+            expected = "datetime"
     if isinstance(session, DuckDBSession):
         if expected == "binary":
             pytest.skip("DuckDB doesn't support binary")
@@ -218,6 +224,10 @@ def test_typeof(get_session_and_func, get_types, arg, expected):
             expected = "object"
         elif expected.startswith("array"):
             pytest.skip("Snowflake doesn't handle arrays properly in values clause")
+    # https://github.com/eakmanrq/sqlframe/issues/383#issuecomment-2870750972
+    if isinstance(session, PySparkSession):
+        if expected == "timestampntz":
+            expected = "timestamp"
     result = df.select(typeof("col").alias("test")).first()[0]
     assert exp.DataType.build(result, dialect=dialect) == exp.DataType.build(
         expected, dialect=dialect
@@ -3505,9 +3515,11 @@ def test_character_length(get_session_and_func, get_func):
 
 def test_contains(get_session_and_func, get_func):
     session, contains = get_session_and_func("contains")
-    to_binary = get_func("to_binary", session)
+    if isinstance(session, BigQuerySession):
+        pytest.skip("BigQuery just supports constaints for the contains function")
     df = session.createDataFrame([("Spark SQL", "Spark")], ["a", "b"])
     assert df.select(contains(df.a, df.b).alias("r")).collect() == [Row(r=True)]
+    to_binary = get_func("to_binary", session)
     df = session.createDataFrame(
         [
             (

{sqlframe-3.32.1 → sqlframe-3.33.0}/tests/unit/standalone/test_column.py

@@ -228,3 +228,7 @@ def test_over():
 
 def test_get_item():
     assert F.col("cola").getItem(1).sql() == "ELEMENT_AT(cola, (1 + 1)) AS element_at__cola__"
+
+
+def test_contains():
+    assert F.col("cola").contains("foo").sql() == "CONTAINS(cola, 'foo')"

{sqlframe-3.32.1 → sqlframe-3.33.0}/tests/unit/standalone/test_functions.py

@@ -4701,8 +4701,8 @@ def test_unix_date(expression, expected):
 @pytest.mark.parametrize(
     "expression, expected",
     [
-        (SF.unix_micros("cola"), "UNIX_MICROS(cola)"),
-        (SF.unix_micros(SF.col("cola")), "UNIX_MICROS(cola)"),
+        (SF.unix_micros("cola"), "UNIX_MICROS(CAST(cola AS TIMESTAMP_LTZ))"),
+        (SF.unix_micros(SF.col("cola")), "UNIX_MICROS(CAST(cola AS TIMESTAMP_LTZ))"),
     ],
 )
 def test_unix_micros(expression, expected):