sqlframe 3.31.4__py3-none-any.whl → 3.32.1__py3-none-any.whl

sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.31.4'
-__version_tuple__ = version_tuple = (3, 31, 4)
+__version__ = version = '3.32.1'
+__version_tuple__ = version_tuple = (3, 32, 1)

sqlframe/base/catalog.py

@@ -6,9 +6,16 @@ import typing as t
 from collections import defaultdict
 
 from sqlglot import MappingSchema, exp
+from sqlglot.helper import seq_get
 
+from sqlframe.base import types
 from sqlframe.base.exceptions import TableSchemaError
-from sqlframe.base.util import ensure_column_mapping, normalize_string, to_schema
+from sqlframe.base.util import (
+    ensure_column_mapping,
+    normalize_string,
+    spark_to_sqlglot,
+    to_schema,
+)
 
 if t.TYPE_CHECKING:
     from sqlglot.schema import ColumnMapping
@@ -99,6 +106,10 @@ class _BaseCatalog(t.Generic[SESSION, DF, TABLE]):
                     "This session does not have access to a catalog that can lookup column information. See docs for explicitly defining columns or using a session that can automatically determine this."
                 )
         column_mapping = ensure_column_mapping(column_mapping)  # type: ignore
+        if isinstance(column_mapping, dict) and isinstance(
+            seq_get(list(column_mapping.values()), 0), types.DataType
+        ):
+            column_mapping = {k: spark_to_sqlglot(v) for k, v in column_mapping.items()}
         for column_name in column_mapping:
             column = exp.to_column(column_name, dialect=self.session.input_dialect)
             if column.this.quoted:

sqlframe/base/dataframe.py

@@ -260,6 +260,9 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
     def __copy__(self):
         return self.copy()
 
+    def _display_(self) -> str:
+        return self.__repr__()
+
     @property
     def _typed_columns(self) -> t.List[CatalogColumn]:
         raise NotImplementedError

sqlframe/base/functions.py

@@ -494,21 +494,21 @@ def skewness(col: ColumnOrName) -> Column:
         func_name = "SKEW"
 
     if session._is_duckdb or session._is_snowflake:
+        col = Column.ensure_col(col)
         when_func = get_func_from_session("when")
         count_func = get_func_from_session("count")
-        count_star = count_func("*")
+        count_col = count_func(col)
         lit_func = get_func_from_session("lit")
         sqrt_func = get_func_from_session("sqrt")
-        col = Column.ensure_col(col)
         full_calc = (
             Column.invoke_anonymous_function(col, func_name)
-            * (count_star - lit_func(2))
-            / (sqrt_func(count_star * (count_star - lit_func(1))))
+            * (count_col - lit_func(2))
+            / (sqrt_func(count_col * (count_col - lit_func(1))))
         )
         return (
-            when_func(count_star == lit_func(0), lit_func(None))
-            .when(count_star == lit_func(1), lit_func(float("nan")))
-            .when(count_star == lit_func(2), lit_func(None))
+            when_func(count_col == lit_func(0), lit_func(None))
+            .when(count_col == lit_func(1), lit_func(None))
+            .when(count_col == lit_func(2), lit_func(0.0))
             .otherwise(full_calc)
         )
 

sqlframe/base/util.py

@@ -347,6 +347,93 @@ def sqlglot_to_spark(sqlglot_dtype: exp.DataType) -> types.DataType:
     raise NotImplementedError(f"Unsupported data type: {sqlglot_dtype}")
 
 
+def spark_to_sqlglot(spark_dtype: types.DataType) -> exp.DataType:
+    """
+    Convert a Spark data type to a SQLGlot data type.
+
+    This function is the opposite of sqlglot_to_spark.
+
+    Args:
+        spark_dtype: A Spark data type
+
+    Returns:
+        The equivalent SQLGlot data type
+    """
+    from sqlframe.base import types
+
+    # Handle primitive types
+    if isinstance(spark_dtype, types.StringType):
+        return exp.DataType(this=exp.DataType.Type.TEXT)
+    elif isinstance(spark_dtype, types.VarcharType):
+        return exp.DataType(
+            this=exp.DataType.Type.VARCHAR,
+            expressions=[exp.DataTypeParam(this=exp.Literal.number(spark_dtype.length))],
+        )
+    elif isinstance(spark_dtype, types.CharType):
+        return exp.DataType(
+            this=exp.DataType.Type.CHAR,
+            expressions=[exp.DataTypeParam(this=exp.Literal.number(spark_dtype.length))],
+        )
+    elif isinstance(spark_dtype, types.BinaryType):
+        return exp.DataType(this=exp.DataType.Type.BINARY)
+    elif isinstance(spark_dtype, types.BooleanType):
+        return exp.DataType(this=exp.DataType.Type.BOOLEAN)
+    elif isinstance(spark_dtype, types.IntegerType):
+        return exp.DataType(this=exp.DataType.Type.INT)
+    elif isinstance(spark_dtype, types.LongType):
+        return exp.DataType(this=exp.DataType.Type.BIGINT)
+    elif isinstance(spark_dtype, types.ShortType):
+        return exp.DataType(this=exp.DataType.Type.SMALLINT)
+    elif isinstance(spark_dtype, types.ByteType):
+        return exp.DataType(this=exp.DataType.Type.TINYINT)
+    elif isinstance(spark_dtype, types.FloatType):
+        return exp.DataType(this=exp.DataType.Type.FLOAT)
+    elif isinstance(spark_dtype, types.DoubleType):
+        return exp.DataType(this=exp.DataType.Type.DOUBLE)
+    elif isinstance(spark_dtype, types.DecimalType):
+        if spark_dtype.precision is not None and spark_dtype.scale is not None:
+            return exp.DataType(
+                this=exp.DataType.Type.DECIMAL,
+                expressions=[
+                    exp.DataTypeParam(this=exp.Literal.number(spark_dtype.precision)),
+                    exp.DataTypeParam(this=exp.Literal.number(spark_dtype.scale)),
+                ],
+            )
+        return exp.DataType(this=exp.DataType.Type.DECIMAL)
+    elif isinstance(spark_dtype, types.TimestampType):
+        return exp.DataType(this=exp.DataType.Type.TIMESTAMP)
+    elif isinstance(spark_dtype, types.TimestampNTZType):
+        return exp.DataType(this=exp.DataType.Type.TIMESTAMPNTZ)
+    elif isinstance(spark_dtype, types.DateType):
+        return exp.DataType(this=exp.DataType.Type.DATE)
+
+    # Handle complex types
+    elif isinstance(spark_dtype, types.ArrayType):
+        return exp.DataType(
+            this=exp.DataType.Type.ARRAY, expressions=[spark_to_sqlglot(spark_dtype.elementType)]
+        )
+    elif isinstance(spark_dtype, types.MapType):
+        return exp.DataType(
+            this=exp.DataType.Type.MAP,
+            expressions=[
+                spark_to_sqlglot(spark_dtype.keyType),
+                spark_to_sqlglot(spark_dtype.valueType),
+            ],
+        )
+    elif isinstance(spark_dtype, types.StructType):
+        return exp.DataType(
+            this=exp.DataType.Type.STRUCT,
+            expressions=[
+                exp.ColumnDef(
+                    this=exp.to_identifier(field.name), kind=spark_to_sqlglot(field.dataType)
+                )
+                for field in spark_dtype
+            ],
+        )
+
+    raise NotImplementedError(f"Unsupported data type: {spark_dtype}")
+
+
 def normalize_string(
     value: t.Union[str, exp.Expression],
     from_dialect: DialectType = None,

{sqlframe-3.31.4.dist-info → sqlframe-3.32.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.31.4
+Version: 3.32.1
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.31.4.dist-info → sqlframe-3.32.1.dist-info}/RECORD

@@ -1,15 +1,15 @@
 sqlframe/__init__.py,sha256=SB80yLTITBXHI2GCDS6n6bN5ObHqgPjfpRPAUwxaots,3403
-sqlframe/_version.py,sha256=ChXgGAAbqvs9VsUGEq1WDjWaF9LZTEF8w01X7cgcZUs,513
+sqlframe/_version.py,sha256=Pcai4MtywlDCWdUgy7BkNFFkadRh0SVOsjFMYSHbJLU,513
 sqlframe/py.typed,sha256=Nqnn8clbgv-5l0PgxcTOldg8mkMKrFn4TvPL-rYUUGg,1
 sqlframe/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlframe/base/_typing.py,sha256=b2clI5HI1zEZKB_3Msx3FeAJQyft44ubUifJwQRVXyQ,1298
-sqlframe/base/catalog.py,sha256=ZuU_qmt4yjSoTYgecSGnOhitOdh3rJbGCUjnUBp5mlc,38564
+sqlframe/base/catalog.py,sha256=-YulM2BMK8MoWbXi05AsJIPxd4AuiZDBCZuk4HoeMlE,38900
 sqlframe/base/column.py,sha256=sp3fJstA49FslE2CcgvVFHyi7Jxsxk8qHTd-Z0cAEWc,19932
-sqlframe/base/dataframe.py,sha256=D2N2Kvh_tiF60fYODUikq0xRCJYY4WB2aHbEcq5NIUo,84310
+sqlframe/base/dataframe.py,sha256=6L8xTdwwQCkUzpJ6K3QlCcz5zqk2QQmGzteI-1EJ23A,84374
 sqlframe/base/decorators.py,sha256=IhE5xNQDkwJHacCvulq5WpUKyKmXm7dL2A3o5WuKGP4,2131
 sqlframe/base/exceptions.py,sha256=9Uwvqn2eAkDpqm4BrRgbL61qM-GMCbJEMAW8otxO46s,370
 sqlframe/base/function_alternatives.py,sha256=Bs1bwl25fN3Yy9rb4GnUWBGunQ1C_yelkb2yV9DSZIY,53918
-sqlframe/base/functions.py,sha256=sEBPrJ8f86fjFEXohnMq7JK7Qh2-51OUIq-KFnjNCIg,227204
+sqlframe/base/functions.py,sha256=i93fc9t7HooXMo8p35VLHd3FeYazVZztVIWqGBmsMYA,227188
 sqlframe/base/group.py,sha256=OY4w1WRsCqLgW-Pi7DjF63zbbxSLISCF3qjAbzI2CQ4,4283
 sqlframe/base/normalize.py,sha256=nXAJ5CwxVf4DV0GsH-q1w0p8gmjSMlv96k_ez1eVul8,3880
 sqlframe/base/operations.py,sha256=g-YNcbvNKTOBbYm23GKfB3fmydlR7ZZDAuZUtXIHtzw,4438
@@ -19,7 +19,7 @@ sqlframe/base/table.py,sha256=rCeh1W5SWbtEVfkLAUiexzrZwNgmZeptLEmLcM1ABkE,6961
 sqlframe/base/transforms.py,sha256=y0j3SGDz3XCmNGrvassk1S-owllUWfkHyMgZlY6SFO4,467
 sqlframe/base/types.py,sha256=iBNk9bpFtb2NBIogYS8i7OlQZMRvpR6XxqzBebsjQDU,12280
 sqlframe/base/udf.py,sha256=O6hMhBUy9NVv-mhJRtfFhXTIa_-Z8Y_FkmmuOHu0l90,1117
-sqlframe/base/util.py,sha256=P8NcogrbNGAS69uWfsBx-50c-4QgVQxUqZC2IrWX_Ts,15522
+sqlframe/base/util.py,sha256=gv_kRc3LxCuQy3t4dHFldV7elB8RU5PMqIN5-xSkWSo,19107
 sqlframe/base/window.py,sha256=7NaKDTlhun-95LEghukBCjFBwq0RHrPaajWQNCsLxok,4818
 sqlframe/base/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlframe/base/mixins/catalog_mixins.py,sha256=9fZGWToz9xMJSzUl1vsVtj6TH3TysP3fBCKJLnGUQzE,23353
@@ -130,8 +130,8 @@ sqlframe/standalone/udf.py,sha256=azmgtUjHNIPs0WMVNId05SHwiYn41MKVBhKXsQJ5dmY,27
 sqlframe/standalone/window.py,sha256=6GKPzuxeSapJakBaKBeT9VpED1ACdjggDv9JRILDyV0,35
 sqlframe/testing/__init__.py,sha256=VVCosQhitU74A3NnE52O4mNtGZONapuEXcc20QmSlnQ,132
 sqlframe/testing/utils.py,sha256=PFsGZpwNUE_4-g_f43_vstTqsK0AQ2lBneb5Eb6NkFo,13008
-sqlframe-3.31.4.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
-sqlframe-3.31.4.dist-info/METADATA,sha256=XPgc6hiVWeBojzJ_UD6VAHhxIQ3aFTt2U2m-vYSIvao,8987
-sqlframe-3.31.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-sqlframe-3.31.4.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
-sqlframe-3.31.4.dist-info/RECORD,,
+sqlframe-3.32.1.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
+sqlframe-3.32.1.dist-info/METADATA,sha256=fRuHZ0SdYY3p5ns78Az8S5qfa6irmVZe2S14ixRYaBE,8987
+sqlframe-3.32.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+sqlframe-3.32.1.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
+sqlframe-3.32.1.dist-info/RECORD,,