sqlframe 3.32.0__py3-none-any.whl → 3.33.0__py3-none-any.whl

sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.32.0'
-__version_tuple__ = version_tuple = (3, 32, 0)
+__version__ = version = '3.33.0'
+__version_tuple__ = version_tuple = (3, 33, 0)

sqlframe/base/catalog.py

@@ -6,9 +6,16 @@ import typing as t
 from collections import defaultdict
 
 from sqlglot import MappingSchema, exp
+from sqlglot.helper import seq_get
 
+from sqlframe.base import types
 from sqlframe.base.exceptions import TableSchemaError
-from sqlframe.base.util import ensure_column_mapping, normalize_string, to_schema
+from sqlframe.base.util import (
+    ensure_column_mapping,
+    normalize_string,
+    spark_to_sqlglot,
+    to_schema,
+)
 
 if t.TYPE_CHECKING:
     from sqlglot.schema import ColumnMapping
@@ -99,6 +106,10 @@ class _BaseCatalog(t.Generic[SESSION, DF, TABLE]):
                     "This session does not have access to a catalog that can lookup column information. See docs for explicitly defining columns or using a session that can automatically determine this."
                 )
         column_mapping = ensure_column_mapping(column_mapping)  # type: ignore
+        if isinstance(column_mapping, dict) and isinstance(
+            seq_get(list(column_mapping.values()), 0), types.DataType
+        ):
+            column_mapping = {k: spark_to_sqlglot(v) for k, v in column_mapping.items()}
         for column_name in column_mapping:
             column = exp.to_column(column_name, dialect=self.session.input_dialect)
             if column.this.quoted:

sqlframe/base/column.py

@@ -517,3 +517,7 @@ class Column:
         +---+
         """
         return self.getItem(name)
+
+    def contains(self, value: t.Union[str, Column]) -> Column:
+        value = self._lit(value) if not isinstance(value, Column) else value
+        return self.invoke_expression_over_column(self, exp.Contains, expression=value.expression)

sqlframe/base/dataframe.py

@@ -260,10 +260,6 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
     def __copy__(self):
         return self.copy()
 
-    def __repr__(self) -> str:
-        fields = [f"{field.name}: {field.dataType}" for field in self.schema]
-        return "DataFrame[" + ", ".join(fields) + "]"
-
     def _display_(self) -> str:
         return self.__repr__()
 

sqlframe/base/function_alternatives.py

@@ -78,17 +78,6 @@ def to_timestamp_tz(col: ColumnOrName, format: t.Optional[str] = None) -> Column
     return Column.ensure_col(col).cast("timestamptz", dialect="duckdb")
 
 
-def to_timestamp_just_timestamp(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
-    from sqlframe.base.session import _BaseSession
-
-    if format is not None:
-        return Column.invoke_expression_over_column(
-            col, expression.StrToTime, format=_BaseSession().format_time(format)
-        )
-
-    return Column.ensure_col(col).cast("datetime", dialect="bigquery")
-
-
 def bitwise_not_from_bitnot(col: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(col, "BITNOT")
 

sqlframe/base/functions.py

@@ -1356,7 +1356,6 @@ def to_date(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
 @meta()
 def to_timestamp(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
     from sqlframe.base.function_alternatives import (
-        to_timestamp_just_timestamp,
         to_timestamp_tz,
         to_timestamp_with_time_zone,
     )
@@ -1366,9 +1365,6 @@ def to_timestamp(col: ColumnOrName, format: t.Optional[str] = None) -> Column:
     if session._is_duckdb:
         return to_timestamp_tz(col, format)
 
-    if session._is_bigquery:
-        return to_timestamp_just_timestamp(col, format)
-
     if session._is_postgres:
         return to_timestamp_with_time_zone(col, format)
 
@@ -3068,7 +3064,7 @@ def character_length(str: ColumnOrName) -> Column:
     return Column.invoke_expression_over_column(str, expression.Length)
 
 
-@meta(unsupported_engines=["bigquery", "postgres"])
+@meta(unsupported_engines=["postgres"])
 def contains(left: ColumnOrName, right: ColumnOrName) -> Column:
     return Column.invoke_expression_over_column(
         left, expression.Contains, expression=Column.ensure_col(right).column_expression
@@ -6594,27 +6590,16 @@ def unix_micros(col: ColumnOrName) -> Column:
     """
     from sqlframe.base.function_alternatives import unix_micros_multiply_epoch
 
+    to_timestamp = get_func_from_session("to_timestamp")
+
     if _get_session()._is_duckdb:
         return Column.invoke_anonymous_function(col, "epoch_us")
 
-    if _get_session()._is_bigquery:
-        return Column(
-            expression.Anonymous(
-                this="UNIX_MICROS",
-                expressions=[
-                    expression.Anonymous(
-                        this="TIMESTAMP",
-                        expressions=[
-                            Column.ensure_col(col).column_expression,
-                        ],
-                    )
-                ],
-            )
-        )
-
     if _get_session()._is_postgres or _get_session()._is_snowflake:
         return unix_micros_multiply_epoch(col)
 
+    col = to_timestamp(col)
+
     return Column.invoke_anonymous_function(col, "unix_micros")
 
 
@@ -6666,22 +6651,6 @@ def unix_seconds(col: ColumnOrName) -> Column:
     if _get_session()._is_postgres:
         return unix_seconds_extract_epoch(col)
 
-    if _get_session()._is_bigquery:
-        return Column(
-            expression.Anonymous(
-                this="UNIX_SECONDS",
-                expressions=[
-                    expression.Anonymous(
-                        this="TIMESTAMP",
-                        expressions=[
-                            Column.ensure_col(col).column_expression,
-                            expression.Literal.string("UTC"),
-                        ],
-                    )
-                ],
-            )
-        )
-
     return Column.invoke_expression_over_column(col, expression.UnixSeconds)
 
 

sqlframe/base/session.py

@@ -304,7 +304,10 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, TABLE, CONN, UDF_REGIS
             elif isinstance(value, float):
                 return "double"
             elif isinstance(value, datetime.datetime):
-                return "timestamp"
+                if value.tzinfo:
+                    # Spark defaults `timestamp` to be a timestamp with timezone
+                    return "timestamp"
+                return "timestampntz"
             elif isinstance(value, datetime.date):
                 return "date"
             elif isinstance(value, str):

sqlframe/base/util.py

@@ -347,6 +347,93 @@ def sqlglot_to_spark(sqlglot_dtype: exp.DataType) -> types.DataType:
     raise NotImplementedError(f"Unsupported data type: {sqlglot_dtype}")
 
 
+def spark_to_sqlglot(spark_dtype: types.DataType) -> exp.DataType:
+    """
+    Convert a Spark data type to a SQLGlot data type.
+
+    This function is the opposite of sqlglot_to_spark.
+
+    Args:
+        spark_dtype: A Spark data type
+
+    Returns:
+        The equivalent SQLGlot data type
+    """
+    from sqlframe.base import types
+
+    # Handle primitive types
+    if isinstance(spark_dtype, types.StringType):
+        return exp.DataType(this=exp.DataType.Type.TEXT)
+    elif isinstance(spark_dtype, types.VarcharType):
+        return exp.DataType(
+            this=exp.DataType.Type.VARCHAR,
+            expressions=[exp.DataTypeParam(this=exp.Literal.number(spark_dtype.length))],
+        )
+    elif isinstance(spark_dtype, types.CharType):
+        return exp.DataType(
+            this=exp.DataType.Type.CHAR,
+            expressions=[exp.DataTypeParam(this=exp.Literal.number(spark_dtype.length))],
+        )
+    elif isinstance(spark_dtype, types.BinaryType):
+        return exp.DataType(this=exp.DataType.Type.BINARY)
+    elif isinstance(spark_dtype, types.BooleanType):
+        return exp.DataType(this=exp.DataType.Type.BOOLEAN)
+    elif isinstance(spark_dtype, types.IntegerType):
+        return exp.DataType(this=exp.DataType.Type.INT)
+    elif isinstance(spark_dtype, types.LongType):
+        return exp.DataType(this=exp.DataType.Type.BIGINT)
+    elif isinstance(spark_dtype, types.ShortType):
+        return exp.DataType(this=exp.DataType.Type.SMALLINT)
+    elif isinstance(spark_dtype, types.ByteType):
+        return exp.DataType(this=exp.DataType.Type.TINYINT)
+    elif isinstance(spark_dtype, types.FloatType):
+        return exp.DataType(this=exp.DataType.Type.FLOAT)
+    elif isinstance(spark_dtype, types.DoubleType):
+        return exp.DataType(this=exp.DataType.Type.DOUBLE)
+    elif isinstance(spark_dtype, types.DecimalType):
+        if spark_dtype.precision is not None and spark_dtype.scale is not None:
+            return exp.DataType(
+                this=exp.DataType.Type.DECIMAL,
+                expressions=[
+                    exp.DataTypeParam(this=exp.Literal.number(spark_dtype.precision)),
+                    exp.DataTypeParam(this=exp.Literal.number(spark_dtype.scale)),
+                ],
+            )
+        return exp.DataType(this=exp.DataType.Type.DECIMAL)
+    elif isinstance(spark_dtype, types.TimestampType):
+        return exp.DataType(this=exp.DataType.Type.TIMESTAMP)
+    elif isinstance(spark_dtype, types.TimestampNTZType):
+        return exp.DataType(this=exp.DataType.Type.TIMESTAMPNTZ)
+    elif isinstance(spark_dtype, types.DateType):
+        return exp.DataType(this=exp.DataType.Type.DATE)
+
+    # Handle complex types
+    elif isinstance(spark_dtype, types.ArrayType):
+        return exp.DataType(
+            this=exp.DataType.Type.ARRAY, expressions=[spark_to_sqlglot(spark_dtype.elementType)]
+        )
+    elif isinstance(spark_dtype, types.MapType):
+        return exp.DataType(
+            this=exp.DataType.Type.MAP,
+            expressions=[
+                spark_to_sqlglot(spark_dtype.keyType),
+                spark_to_sqlglot(spark_dtype.valueType),
+            ],
+        )
+    elif isinstance(spark_dtype, types.StructType):
+        return exp.DataType(
+            this=exp.DataType.Type.STRUCT,
+            expressions=[
+                exp.ColumnDef(
+                    this=exp.to_identifier(field.name), kind=spark_to_sqlglot(field.dataType)
+                )
+                for field in spark_dtype
+            ],
+        )
+
+    raise NotImplementedError(f"Unsupported data type: {spark_dtype}")
+
+
 def normalize_string(
     value: t.Union[str, exp.Expression],
     from_dialect: DialectType = None,

{sqlframe-3.32.0.dist-info → sqlframe-3.33.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.32.0
+Version: 3.33.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.32.0.dist-info → sqlframe-3.33.0.dist-info}/RECORD

@@ -1,25 +1,25 @@
 sqlframe/__init__.py,sha256=SB80yLTITBXHI2GCDS6n6bN5ObHqgPjfpRPAUwxaots,3403
-sqlframe/_version.py,sha256=EatXYAvXc8eLZh8r-obXLCaLyBPqfgPtx9AXEI2rZ_E,513
+sqlframe/_version.py,sha256=ov9Fneoqycyrmto-1SaECibEXFbQJlZrt2fFCQsBGtY,513
 sqlframe/py.typed,sha256=Nqnn8clbgv-5l0PgxcTOldg8mkMKrFn4TvPL-rYUUGg,1
 sqlframe/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlframe/base/_typing.py,sha256=b2clI5HI1zEZKB_3Msx3FeAJQyft44ubUifJwQRVXyQ,1298
-sqlframe/base/catalog.py,sha256=ZuU_qmt4yjSoTYgecSGnOhitOdh3rJbGCUjnUBp5mlc,38564
-sqlframe/base/column.py,sha256=sp3fJstA49FslE2CcgvVFHyi7Jxsxk8qHTd-Z0cAEWc,19932
-sqlframe/base/dataframe.py,sha256=V_qRAPsdUji12PwWg7Ce8Cln9fsBoVY-3s4J4KwbINs,84538
+sqlframe/base/catalog.py,sha256=-YulM2BMK8MoWbXi05AsJIPxd4AuiZDBCZuk4HoeMlE,38900
+sqlframe/base/column.py,sha256=zDG9YT-5M7H8tDISOkJ6uMBU7Q3enTmc6d7rzZ08q40,20172
+sqlframe/base/dataframe.py,sha256=6L8xTdwwQCkUzpJ6K3QlCcz5zqk2QQmGzteI-1EJ23A,84374
 sqlframe/base/decorators.py,sha256=IhE5xNQDkwJHacCvulq5WpUKyKmXm7dL2A3o5WuKGP4,2131
 sqlframe/base/exceptions.py,sha256=9Uwvqn2eAkDpqm4BrRgbL61qM-GMCbJEMAW8otxO46s,370
-sqlframe/base/function_alternatives.py,sha256=Bs1bwl25fN3Yy9rb4GnUWBGunQ1C_yelkb2yV9DSZIY,53918
-sqlframe/base/functions.py,sha256=i93fc9t7HooXMo8p35VLHd3FeYazVZztVIWqGBmsMYA,227188
+sqlframe/base/function_alternatives.py,sha256=dEymHSOQgUzhoYtfY5acC9AxpMoGoHXX7v6yTadKzn8,53527
+sqlframe/base/functions.py,sha256=jNuCezcQl3j7hj1JsukaZLIvJqDtgQetiHnnTK5LU5w,226189
 sqlframe/base/group.py,sha256=OY4w1WRsCqLgW-Pi7DjF63zbbxSLISCF3qjAbzI2CQ4,4283
 sqlframe/base/normalize.py,sha256=nXAJ5CwxVf4DV0GsH-q1w0p8gmjSMlv96k_ez1eVul8,3880
 sqlframe/base/operations.py,sha256=g-YNcbvNKTOBbYm23GKfB3fmydlR7ZZDAuZUtXIHtzw,4438
 sqlframe/base/readerwriter.py,sha256=Nb2VJ_HBmLQp5mK8JhnFooZh2ydAaboCAFVPb-4MNX4,31241
-sqlframe/base/session.py,sha256=tSNlIlo7XeZUQFsZ3wnffqFLdxynY9i7oz60GUF-V14,27104
+sqlframe/base/session.py,sha256=djXPmuW0cIQYuoE7hegfyvZuKC2D3ABZCjvw-fa1C24,27260
 sqlframe/base/table.py,sha256=rCeh1W5SWbtEVfkLAUiexzrZwNgmZeptLEmLcM1ABkE,6961
 sqlframe/base/transforms.py,sha256=y0j3SGDz3XCmNGrvassk1S-owllUWfkHyMgZlY6SFO4,467
 sqlframe/base/types.py,sha256=iBNk9bpFtb2NBIogYS8i7OlQZMRvpR6XxqzBebsjQDU,12280
 sqlframe/base/udf.py,sha256=O6hMhBUy9NVv-mhJRtfFhXTIa_-Z8Y_FkmmuOHu0l90,1117
-sqlframe/base/util.py,sha256=P8NcogrbNGAS69uWfsBx-50c-4QgVQxUqZC2IrWX_Ts,15522
+sqlframe/base/util.py,sha256=gv_kRc3LxCuQy3t4dHFldV7elB8RU5PMqIN5-xSkWSo,19107
 sqlframe/base/window.py,sha256=7NaKDTlhun-95LEghukBCjFBwq0RHrPaajWQNCsLxok,4818
 sqlframe/base/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlframe/base/mixins/catalog_mixins.py,sha256=9fZGWToz9xMJSzUl1vsVtj6TH3TysP3fBCKJLnGUQzE,23353
@@ -130,8 +130,8 @@ sqlframe/standalone/udf.py,sha256=azmgtUjHNIPs0WMVNId05SHwiYn41MKVBhKXsQJ5dmY,27
 sqlframe/standalone/window.py,sha256=6GKPzuxeSapJakBaKBeT9VpED1ACdjggDv9JRILDyV0,35
 sqlframe/testing/__init__.py,sha256=VVCosQhitU74A3NnE52O4mNtGZONapuEXcc20QmSlnQ,132
 sqlframe/testing/utils.py,sha256=PFsGZpwNUE_4-g_f43_vstTqsK0AQ2lBneb5Eb6NkFo,13008
-sqlframe-3.32.0.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
-sqlframe-3.32.0.dist-info/METADATA,sha256=JXzQRL-VpYCTvuSF7QmUH0spMzGT2B5CoCIDDUNwq40,8987
-sqlframe-3.32.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-sqlframe-3.32.0.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
-sqlframe-3.32.0.dist-info/RECORD,,
+sqlframe-3.33.0.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
+sqlframe-3.33.0.dist-info/METADATA,sha256=I0vB586w0J4WQoBE4hd1YDnh2fger8Ra7zEXdcbe9AY,8987
+sqlframe-3.33.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+sqlframe-3.33.0.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
+sqlframe-3.33.0.dist-info/RECORD,,