sqlframe 3.6.0__py3-none-any.whl → 3.8.0__py3-none-any.whl

sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.6.0'
-__version_tuple__ = version_tuple = (3, 6, 0)
+__version__ = version = '3.8.0'
+__version_tuple__ = version_tuple = (3, 8, 0)

sqlframe/base/function_alternatives.py

@@ -1593,7 +1593,7 @@ def try_to_timestamp_pgtemp(col: ColumnOrName, format: t.Optional[ColumnOrName]
 def typeof_pg_typeof(col: ColumnOrName) -> Column:
     return (
         Column.invoke_anonymous_function(col, "pg_typeof")
-        .cast(expression.DataType.build("regtype", dialect="postgres"))
+        .cast(expression.DataType(this=expression.DataType.Type.USERDEFINED, kind="regtype"))
         .cast("text")
     )
 

sqlframe/base/functions.py

@@ -2069,9 +2069,11 @@ def character_length(str: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(str, "character_length")
 
 
-@meta()
+@meta(unsupported_engines=["bigquery", "postgres"])
 def contains(left: ColumnOrName, right: ColumnOrName) -> Column:
-    return Column.invoke_anonymous_function(left, "contains", right)
+    return Column.invoke_expression_over_column(
+        left, expression.Contains, expression=Column.ensure_col(right).expression
+    )
 
 
 @meta(unsupported_engines=["bigquery", "postgres"])
@@ -3484,7 +3486,7 @@ def mask(
     )
 
 
-@meta(unsupported_engines="*")
+@meta(unsupported_engines=["bigquery"])
 def median(col: ColumnOrName) -> Column:
     """
     Returns the median of the values in a group.
@@ -3520,7 +3522,7 @@ def median(col: ColumnOrName) -> Column:
     |dotNET|         10000.0|
     +------+----------------+
     """
-    return Column.invoke_anonymous_function(col, "median")
+    return Column.invoke_expression_over_column(col, expression.Median)
 
 
 @meta(unsupported_engines="*")
@@ -4106,11 +4108,9 @@ def regexp_extract_all(
     >>> df.select(regexp_extract_all('str', col("regexp")).alias('d')).collect()
     [Row(d=['100', '300'])]
     """
-    if idx is None:
-        return Column.invoke_anonymous_function(str, "regexp_extract_all", regexp)
-    else:
-        idx = lit(idx) if isinstance(idx, int) else idx
-        return Column.invoke_anonymous_function(str, "regexp_extract_all", regexp, idx)
+    return Column.invoke_expression_over_column(
+        str, expression.RegexpExtractAll, expression=regexp, group=idx
+    )
 
 
 @meta(unsupported_engines="*")
@@ -5426,7 +5426,7 @@ def unix_millis(col: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(col, "unix_millis")
 
 
-@meta(unsupported_engines="*")
+@meta(unsupported_engines=["bigquery", "duckdb", "postgres"])
 def unix_seconds(col: ColumnOrName) -> Column:
     """Returns the number of seconds since 1970-01-01 00:00:00 UTC.
     Truncates higher levels of precision.
@@ -5441,7 +5441,7 @@ def unix_seconds(col: ColumnOrName) -> Column:
     [Row(n=1437584400)]
     >>> spark.conf.unset("spark.sql.session.timeZone")
     """
-    return Column.invoke_anonymous_function(col, "unix_seconds")
+    return Column.invoke_expression_over_column(col, expression.UnixSeconds)
 
 
 @meta(unsupported_engines="*")

sqlframe/base/mixins/readwriter_mixins.py

@@ -75,7 +75,7 @@ class PandasLoaderMixin(_BaseDataFrameReader, t.Generic[SESSION, DF]):
 
         assert path is not None, "path is required"
         assert isinstance(path, str), "path must be a string"
-        format = format or _infer_format(path)
+        format = format or self.state_format_to_read or _infer_format(path)
         kwargs = {k: v for k, v in options.items() if v is not None}
         if format == "json":
             df = pd.read_json(path, lines=True, **kwargs)  # type: ignore

sqlframe/base/readerwriter.py

@@ -36,6 +36,7 @@ logger = logging.getLogger(__name__)
 class _BaseDataFrameReader(t.Generic[SESSION, DF]):
     def __init__(self, spark: SESSION):
         self._session = spark
+        self.state_format_to_read: t.Optional[str] = None
 
     @property
     def session(self) -> SESSION:
@@ -67,6 +68,44 @@ class _BaseDataFrameReader(t.Generic[SESSION, DF]):
             for k, v in column_mapping.items()
         ]
 
+    def format(self, source: str) -> "Self":
+        """Specifies the input data source format.
+
+        .. versionadded:: 1.4.0
+
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
+
+        Parameters
+        ----------
+        source : str
+            string, name of the data source, e.g. 'json', 'parquet'.
+
+        Examples
+        --------
+        >>> spark.read.format('json')
+        <...readwriter.DataFrameReader object ...>
+
+        Write a DataFrame into a JSON file and read it back.
+
+        >>> import tempfile
+        >>> with tempfile.TemporaryDirectory() as d:
+        ...     # Write a DataFrame into a JSON file
+        ...     spark.createDataFrame(
+        ...         [{"age": 100, "name": "Hyukjin Kwon"}]
+        ...     ).write.mode("overwrite").format("json").save(d)
+        ...
+        ...     # Read the JSON file as a DataFrame.
+        ...     spark.read.format('json').load(d).show()
+        +---+------------+
+        |age|        name|
+        +---+------------+
+        |100|Hyukjin Kwon|
+        +---+------------+
+        """
+        self.state_format_to_read = source
+        return self
+
     def load(
         self,
         path: t.Optional[PathOrPaths] = None,

sqlframe/duckdb/readwriter.py

@@ -72,6 +72,7 @@ class DuckDBDataFrameReader(_BaseDataFrameReader["DuckDBSession", "DuckDBDataFra
         |100|NULL|
         +---+----+
         """
+        format = format or self.state_format_to_read
         if schema:
             column_mapping = ensure_column_mapping(schema)
             select_column_mapping = column_mapping.copy()

{sqlframe-3.6.0.dist-info → sqlframe-3.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.6.0
+Version: 3.8.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman
@@ -17,8 +17,8 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: prettytable (<3.11.1)
-Requires-Dist: sqlglot (<25.29,>=24.0.0)
+Requires-Dist: prettytable (<3.12.1)
+Requires-Dist: sqlglot (<25.32,>=24.0.0)
 Requires-Dist: typing-extensions (<5,>=4.8)
 Provides-Extra: bigquery
 Requires-Dist: google-cloud-bigquery-storage (<3,>=2) ; extra == 'bigquery'
@@ -27,7 +27,7 @@ Provides-Extra: dev
 Requires-Dist: duckdb (<1.2,>=0.9) ; extra == 'dev'
 Requires-Dist: findspark (<3,>=2) ; extra == 'dev'
 Requires-Dist: mypy (<1.14,>=1.10.0) ; extra == 'dev'
-Requires-Dist: openai (<1.53,>=1.30) ; extra == 'dev'
+Requires-Dist: openai (<1.55,>=1.30) ; extra == 'dev'
 Requires-Dist: pandas-stubs (<3,>=2) ; extra == 'dev'
 Requires-Dist: pandas (<3,>=2) ; extra == 'dev'
 Requires-Dist: psycopg (<4,>=3.1) ; extra == 'dev'
@@ -50,7 +50,7 @@ Provides-Extra: duckdb
 Requires-Dist: duckdb (<1.2,>=0.9) ; extra == 'duckdb'
 Requires-Dist: pandas (<3,>=2) ; extra == 'duckdb'
 Provides-Extra: openai
-Requires-Dist: openai (<1.53,>=1.30) ; extra == 'openai'
+Requires-Dist: openai (<1.55,>=1.30) ; extra == 'openai'
 Provides-Extra: pandas
 Requires-Dist: pandas (<3,>=2) ; extra == 'pandas'
 Provides-Extra: postgres

{sqlframe-3.6.0.dist-info → sqlframe-3.8.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlframe/__init__.py,sha256=E3qCJ4PSEgKz6Hg3ves6LWt3JrQOV8c9HVLSIUOzKNc,3106
-sqlframe/_version.py,sha256=Z7ZPT2d9N4AepQhV8KR5f2xj-SC1GJPAoD3gGnvo1ZM,411
+sqlframe/_version.py,sha256=hvmaOw9yeb_zs_ePB0RUeYnhDHdIDSo_FxV0X8T7LEk,411
 sqlframe/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlframe/base/_typing.py,sha256=b2clI5HI1zEZKB_3Msx3FeAJQyft44ubUifJwQRVXyQ,1298
 sqlframe/base/catalog.py,sha256=SzFQalTWdhWzxUY-4ut1f9TfOECp_JmJEgNPfrRKCe0,38457
@@ -7,12 +7,12 @@ sqlframe/base/column.py,sha256=06fhVZ2nCn2QLxnfjdK-oYKeTFJC_smgSxu7u2UYlVg,17878
 sqlframe/base/dataframe.py,sha256=coeUwntwYbT1g6YKVwk3ZfWMfJqAzd1ECYabBSsNsV0,72892
 sqlframe/base/decorators.py,sha256=Jy4bf8MhZ-AJ6CWTj59bBJRqamtLbPC0USUMFrY6g0w,449
 sqlframe/base/exceptions.py,sha256=9Uwvqn2eAkDpqm4BrRgbL61qM-GMCbJEMAW8otxO46s,370
-sqlframe/base/function_alternatives.py,sha256=Cqt-OjyWewCXZJ8PV-2vc8yrDG2ennLPm6yWaxWVXRc,50746
-sqlframe/base/functions.py,sha256=JDVULt3WI1cwWYboU8ybenIXZPdKSNtnzg7A4xC1Lao,189978
+sqlframe/base/function_alternatives.py,sha256=jofb2-nweefqcjUsd4xVqfRmJSZ-T_0Iq5roW2pL0OA,50768
+sqlframe/base/functions.py,sha256=0pfNPCgFs7qKopkM9Q-E_BxCGJHUqlk3YqIzKg4jdQ4,190024
 sqlframe/base/group.py,sha256=fsyG5990_Pd7gFPjTFrH9IEoAquL_wEkVpIlBAIkZJU,4091
 sqlframe/base/normalize.py,sha256=nXAJ5CwxVf4DV0GsH-q1w0p8gmjSMlv96k_ez1eVul8,3880
 sqlframe/base/operations.py,sha256=-AhNuEzcV7ZExoP1oY3blaKip-joQyJeQVvfBTs_2g4,3456
-sqlframe/base/readerwriter.py,sha256=FkImnUR_qNASmXktk0JDsFeOzfh799hd09vI2uznNH8,25350
+sqlframe/base/readerwriter.py,sha256=9cgOZuB3phZbQufY98JRDBVWRww3hsULc6Or1HK2Onk,26554
 sqlframe/base/session.py,sha256=6vQ2bJa357J1bycx1vMxkIUdcA2YbMIYq956bpytztI,25256
 sqlframe/base/transforms.py,sha256=y0j3SGDz3XCmNGrvassk1S-owllUWfkHyMgZlY6SFO4,467
 sqlframe/base/types.py,sha256=iBNk9bpFtb2NBIogYS8i7OlQZMRvpR6XxqzBebsjQDU,12280
@@ -22,7 +22,7 @@ sqlframe/base/window.py,sha256=8hOv-ignPPIsZA9FzvYzcLE9J_glalVaYjIAUdRUX3o,4943
 sqlframe/base/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlframe/base/mixins/catalog_mixins.py,sha256=9tn0mK8oPoqIIjNItystD5tdBMdK9YpkxTG7G9KQl8k,18619
 sqlframe/base/mixins/dataframe_mixins.py,sha256=3MMQu2hdHG_qtDQ6jDHo0Iy5KtLj4lHePfovCVxTqbo,1411
-sqlframe/base/mixins/readwriter_mixins.py,sha256=QnxGVL8ftZfYlBNG0Bl24N_bnA2YioSxUsTSgKIbuvQ,4723
+sqlframe/base/mixins/readwriter_mixins.py,sha256=ap8j_g7PoUGHaHKCPMnRPbXofOsUhUzlaF7Loxy2m-I,4752
 sqlframe/bigquery/__init__.py,sha256=kbaomhYAANPdxeDQhajv8IHfMg_ENKivtYK-rPwaV08,939
 sqlframe/bigquery/catalog.py,sha256=8d36IzT5GPWd1FdxJ9vEljOdbIDepHnFOBjwP0bX6FE,11625
 sqlframe/bigquery/column.py,sha256=E1tUa62Y5HajkhgFuebU9zohrGyieudcHzTT8gfalio,40
@@ -42,7 +42,7 @@ sqlframe/duckdb/dataframe.py,sha256=HZg_uMAz4RsubZJT4-MslUQS_0-InF0_P5Yq5HyJ3wE,
 sqlframe/duckdb/functions.py,sha256=Ee8o6YFtRdEiq0jNLXxgu5lcbc7Tsg0-lK6oRyxdcjo,1920
 sqlframe/duckdb/functions.pyi,sha256=bWfQl7Cm1eecI39LJAyyRcC4z7epDJ-h9JOozPsEc34,5879
 sqlframe/duckdb/group.py,sha256=IkhbW42Ng1U5YT3FkIdiB4zBqRkW4QyTb-1detY1e_4,383
-sqlframe/duckdb/readwriter.py,sha256=iY0Wsms35gymilETOhAQGQCHfH8LCj9_uY8aCdF77ds,4806
+sqlframe/duckdb/readwriter.py,sha256=6nGnz2SE-tpuPDQXPI23SQQXRre_raUG5cou3s3NrpA,4859
 sqlframe/duckdb/session.py,sha256=b5IrKbTkYUVNQGSG2EJPNV9MTdJw4onN-9aMrskjxck,2721
 sqlframe/duckdb/types.py,sha256=KwNyuXIo-2xVVd4bZED3YrQOobKCtemlxGrJL7DrTC8,34
 sqlframe/duckdb/udf.py,sha256=Du9LnOtT1lJvB90D4HSR2tB7MXy179jZngDR-EjVjQk,656
@@ -107,8 +107,8 @@ sqlframe/standalone/udf.py,sha256=azmgtUjHNIPs0WMVNId05SHwiYn41MKVBhKXsQJ5dmY,27
 sqlframe/standalone/window.py,sha256=6GKPzuxeSapJakBaKBeT9VpED1ACdjggDv9JRILDyV0,35
 sqlframe/testing/__init__.py,sha256=VVCosQhitU74A3NnE52O4mNtGZONapuEXcc20QmSlnQ,132
 sqlframe/testing/utils.py,sha256=9DDYVuocO7tygee3RaajuJNZ24sJwf_LY556kKg7kTw,13011
-sqlframe-3.6.0.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
-sqlframe-3.6.0.dist-info/METADATA,sha256=Vax4ozsDNRqn-MBnTi1jW4WVkvZPDKsf_XGePysEw98,8639
-sqlframe-3.6.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-sqlframe-3.6.0.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
-sqlframe-3.6.0.dist-info/RECORD,,
+sqlframe-3.8.0.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
+sqlframe-3.8.0.dist-info/METADATA,sha256=SYeasQhk4sDbmww5wM0FUhqX63gTihxdZjM8t3qyi3Q,8639
+sqlframe-3.8.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+sqlframe-3.8.0.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
+sqlframe-3.8.0.dist-info/RECORD,,