sqlframe 3.26.0__py3-none-any.whl → 3.27.0__py3-none-any.whl

sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.26.0'
-__version_tuple__ = version_tuple = (3, 26, 0)
+__version__ = version = '3.27.0'
+__version_tuple__ = version_tuple = (3, 27, 0)

sqlframe/base/functions.py

@@ -4944,7 +4944,7 @@ def printf(format: ColumnOrName, *cols: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(format, "printf", *cols)
 
 
-@meta(unsupported_engines=["*", "spark", "databricks"])
+@meta(unsupported_engines=["bigquery", "postgres", "redshift", "snowflake", "spark", "databricks"])
 def product(col: ColumnOrName) -> Column:
     """
     Aggregate function: returns the product of the values in a group.

sqlframe/base/readerwriter.py

@@ -38,6 +38,7 @@ class _BaseDataFrameReader(t.Generic[SESSION, DF, TABLE]):
     def __init__(self, spark: SESSION):
         self._session = spark
         self.state_format_to_read: t.Optional[str] = None
+        self.state_options: t.Dict[str, OptionalPrimitiveType] = {}
 
     @property
     def session(self) -> SESSION:
@@ -107,6 +108,88 @@ class _BaseDataFrameReader(t.Generic[SESSION, DF, TABLE]):
         self.state_format_to_read = source
         return self
 
+    def options(self, **options: OptionalPrimitiveType) -> "Self":
+        """Adds input options for the underlying data source.
+
+        .. versionadded:: 1.4.0
+
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
+
+        Parameters
+        ----------
+        **options : dict
+            The dictionary of string keys and primitive-type values.
+
+        Examples
+        --------
+        >>> spark.read.options(inferSchema=True, header=True)
+        <...readwriter.DataFrameReader object ...>
+
+        Specify the option 'nullValue' and 'header' with reading a CSV file.
+
+        >>> import tempfile
+        >>> with tempfile.TemporaryDirectory() as d:
+        ...     # Write a DataFrame into a CSV file with a header.
+        ...     df = spark.createDataFrame([{"age": 100, "name": "Hyukjin Kwon"}])
+        ...     df.write.option("header", True).mode("overwrite").format("csv").save(d)
+        ...
+        ...     # Read the CSV file as a DataFrame with 'nullValue' option set to 'Hyukjin Kwon',
+        ...     # and 'header' option set to `True`.
+        ...     spark.read.options(
+        ...         nullValue="Hyukjin Kwon",
+        ...         header=True
+        ...     ).format('csv').load(d).show()
+        +---+----+
+        |age|name|
+        +---+----+
+        |100|NULL|
+        +---+----+
+        """
+
+        self.state_options = {**self.state_options, **options}
+        return self
+
+    def option(self, key: str, value: OptionalPrimitiveType) -> "Self":
+        """Adds an input option for the underlying data source.
+
+        .. versionadded:: 1.4.0
+
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
+
+        Parameters
+        ----------
+        key : str
+            The key of the option.
+        value :
+            The value of the option.
+
+        Examples
+        --------
+        >>> spark.read.option("inferSchema", True)
+        <...readwriter.DataFrameReader object ...>
+
+        Specify the option 'nullValue' and 'header' with reading a CSV file.
+
+        >>> import tempfile
+        >>> with tempfile.TemporaryDirectory() as d:
+        ...     # Write a DataFrame into a CSV file with a header.
+        ...     df = spark.createDataFrame([{"age": 100, "name": "Hyukjin Kwon"}])
+        ...     df.write.option("header", True).mode("overwrite").format("csv").save(d)
+        ...
+        ...     # Read the CSV file as a DataFrame with 'nullValue' option set to 'Hyukjin Kwon',
+        ...     # and 'header' option set to `True`.
+        ...     spark.read.option("nullValue", "Hyukjin Kwon").option("header", True).format('csv').load(d).show()
+        +---+----+
+        |age|name|
+        +---+----+
+        |100|NULL|
+        +---+----+
+        """
+        self.state_options[key] = value
+        return self
+
     def load(
         self,
         path: t.Optional[PathOrPaths] = None,
@@ -220,7 +303,9 @@ class _BaseDataFrameReader(t.Generic[SESSION, DF, TABLE]):
             modifiedAfter=modifiedAfter,
             allowNonNumericNumbers=allowNonNumericNumbers,
         )
-        return self.load(path=path, format="json", schema=schema, **options)
+        # Apply previously set options, with method-specific options taking precedence
+        all_options = {**self.state_options, **{k: v for k, v in options.items() if v is not None}}
+        return self.load(path=path, format="json", schema=schema, **all_options)
 
     def parquet(self, *paths: str, **options: OptionalPrimitiveType) -> DF:
         """
@@ -263,7 +348,8 @@ class _BaseDataFrameReader(t.Generic[SESSION, DF, TABLE]):
         |100|Hyukjin Kwon|
         +---+------------+
         """
-        dfs = [self.load(path=path, format="parquet", **options) for path in paths]  # type: ignore
+        all_options = {**self.state_options, **{k: v for k, v in options.items() if v is not None}}
+        dfs = [self.load(path=path, format="parquet", **all_options) for path in paths]  # type: ignore
         return reduce(lambda a, b: a.union(b), dfs)
 
     def csv(
@@ -384,7 +470,9 @@ class _BaseDataFrameReader(t.Generic[SESSION, DF, TABLE]):
             modifiedAfter=modifiedAfter,
             unescapedQuoteHandling=unescapedQuoteHandling,
         )
-        return self.load(path=path, format="csv", schema=schema, **options)
+        # Apply previously set options, with method-specific options taking precedence
+        all_options = {**self.state_options, **{k: v for k, v in options.items() if v is not None}}
+        return self.load(path=path, format="csv", schema=schema, **all_options)
 
 
 class _BaseDataFrameWriter(t.Generic[SESSION, DF]):

sqlframe/base/session.py

@@ -193,22 +193,21 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, TABLE, CONN, UDF_REGIS
     def getActiveSession(self) -> Self:
         return self
 
-    def range(self, *args):
-        start = 0
-        step = 1
-        numPartitions = None
-        if len(args) == 1:
-            end = args[0]
-        elif len(args) == 2:
-            start, end = args
-        elif len(args) == 3:
-            start, end, step = args
-        elif len(args) == 4:
-            start, end, step, numPartitions = args
-        else:
-            raise ValueError(
-                "range() takes 1 to 4 positional arguments but {} were given".format(len(args))
-            )
+    def range(
+        self,
+        start: int,
+        end: t.Optional[int] = None,
+        step: int = 1,
+        numPartitions: t.Optional[int] = None,
+    ):
+        # Ensure end is provided by either args or kwargs
+        if end is None:
+            if start:
+                end = start
+                start = 0
+            else:
+                raise ValueError("range() requires an 'end' value")
+
         if numPartitions is not None:
             logger.warning("numPartitions is not supported")
         return self.createDataFrame([[x] for x in range(start, end, step)], schema={"id": "long"})

sqlframe/databricks/readwriter.py

@@ -94,11 +94,15 @@ class DatabricksDataFrameReader(
         """
         assert path is not None, "path is required"
         assert isinstance(path, str), "path must be a string"
+
+        # Merge state_options with provided options, with provided options taking precedence
+        merged_options = {**self.state_options, **options}
+
         format = format or self.state_format_to_read or _infer_format(path)
         fs_prefix, filepath = split_filepath(path)
 
         if fs_prefix == "":
-            return super().load(path, format, schema, **options)
+            return super().load(path, format, schema, **merged_options)
 
         if schema:
             column_mapping = ensure_column_mapping(schema)
@@ -116,7 +120,7 @@ class DatabricksDataFrameReader(
             paths = ",".join([f"{path}" for path in ensure_list(path)])
 
             format_options: dict[str, OptionalPrimitiveType] = {
-                k: v for k, v in options.items() if v is not None
+                k: v for k, v in merged_options.items() if v is not None
             }
             format_options["format"] = format
             format_options["schemaEvolutionMode"] = "none"
@@ -137,7 +141,7 @@ class DatabricksDataFrameReader(
             qualify=False,
         )
         if select_columns == [exp.Star()] and df.schema:
-            return self.load(path=path, format=format, schema=df.schema, **options)
+            return self.load(path=path, format=format, schema=df.schema, **merged_options)
         self.session._last_loaded_file = path  # type: ignore
         return df
 

sqlframe/duckdb/functions.pyi

@@ -148,6 +148,7 @@ from sqlframe.base.functions import percentile_approx as percentile_approx
 from sqlframe.base.functions import position as position
 from sqlframe.base.functions import pow as pow
 from sqlframe.base.functions import power as power
+from sqlframe.base.functions import product as product
 from sqlframe.base.functions import quarter as quarter
 from sqlframe.base.functions import radians as radians
 from sqlframe.base.functions import rand as rand

sqlframe/duckdb/readwriter.py

@@ -75,31 +75,34 @@ class DuckDBDataFrameReader(
         |100|NULL|
         +---+----+
         """
+        # Merge state_options with provided options, with provided options taking precedence
+        merged_options = {**self.state_options, **options}
+
         format = format or self.state_format_to_read
         if schema:
             column_mapping = ensure_column_mapping(schema)
             select_column_mapping = column_mapping.copy()
-            if options.get("filename"):
+            if merged_options.get("filename"):
                 select_column_mapping["filename"] = "VARCHAR"
             select_columns = [x.expression for x in self._to_casted_columns(select_column_mapping)]
             if format == "csv":
                 duckdb_columns = ", ".join(
                     [f"'{column}': '{dtype}'" for column, dtype in column_mapping.items()]
                 )
-                options["columns"] = "{" + duckdb_columns + "}"
+                merged_options["columns"] = "{" + duckdb_columns + "}"
         else:
             select_columns = [exp.Star()]
         if format == "delta":
             from_clause = f"delta_scan('{path}')"
         elif format:
-            options.pop("inferSchema", None)
+            merged_options.pop("inferSchema", None)
             paths = ",".join([f"'{path}'" for path in ensure_list(path)])
-            from_clause = f"read_{format}([{paths}], {to_csv(options)})"
+            from_clause = f"read_{format}([{paths}], {to_csv(merged_options)})"
         else:
             from_clause = f"'{path}'"
         df = self.session.sql(exp.select(*select_columns).from_(from_clause), qualify=False)
         if select_columns == [exp.Star()]:
-            return self.load(path=path, format=format, schema=df.schema, **options)
+            return self.load(path=path, format=format, schema=df.schema, **merged_options)
         self.session._last_loaded_file = path  # type: ignore
         return df
 

sqlframe/spark/readwriter.py

@@ -78,6 +78,10 @@ class SparkDataFrameReader(
         """
         assert path is not None, "path is required"
         assert isinstance(path, str), "path must be a string"
+
+        # Merge state_options with provided options, with provided options taking precedence
+        merged_options = {**self.state_options, **options}
+
         format = format or self.state_format_to_read or _infer_format(path)
         if schema:
             column_mapping = ensure_column_mapping(schema)
@@ -93,11 +97,13 @@ class SparkDataFrameReader(
             from_clause = f"delta.`{path}`"
         elif format:
             paths = ",".join([f"{path}" for path in ensure_list(path)])
-            tmp_view_key = options.get("_tmp_view_key_", f"{generate_random_identifier()}_vw")
-            options["_tmp_view_key_"] = tmp_view_key
+            tmp_view_key = merged_options.get(
+                "_tmp_view_key_", f"{generate_random_identifier()}_vw"
+            )
+            merged_options["_tmp_view_key_"] = tmp_view_key
 
             format_options: dict[str, OptionalPrimitiveType] = {
-                k: v for k, v in options.items() if v is not None
+                k: v for k, v in merged_options.items() if v is not None
             }
             format_options.pop("_tmp_view_key_")
             format_options["path"] = paths
@@ -121,7 +127,7 @@ class SparkDataFrameReader(
             qualify=False,
         )
         if select_columns == [exp.Star()] and df.schema:
-            return self.load(path=path, format=format, schema=df.schema, **options)
+            return self.load(path=path, format=format, schema=df.schema, **merged_options)
         self.session._last_loaded_file = path  # type: ignore
         return df
 

{sqlframe-3.26.0.dist-info → sqlframe-3.27.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.26.0
+Version: 3.27.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.26.0.dist-info → sqlframe-3.27.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlframe/__init__.py,sha256=SB80yLTITBXHI2GCDS6n6bN5ObHqgPjfpRPAUwxaots,3403
-sqlframe/_version.py,sha256=O47f8PhfWsbiZ4mjHJkrL81FZEFxis7lNo0FJknkGus,513
+sqlframe/_version.py,sha256=Ax3LD7DzZA_hEyXtX5hoIc_sZjRSealrt3DnfvJEnEg,513
 sqlframe/py.typed,sha256=Nqnn8clbgv-5l0PgxcTOldg8mkMKrFn4TvPL-rYUUGg,1
 sqlframe/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlframe/base/_typing.py,sha256=b2clI5HI1zEZKB_3Msx3FeAJQyft44ubUifJwQRVXyQ,1298
@@ -9,12 +9,12 @@ sqlframe/base/dataframe.py,sha256=OgEUlDI5Y4rWrVngW5LttCUMC40WR-Pyr5af6aqbMNU,84
 sqlframe/base/decorators.py,sha256=ms-CvDOIW3T8IVB9VqDmLwAiaEsqXLYRXEqVQaxktiM,1890
 sqlframe/base/exceptions.py,sha256=9Uwvqn2eAkDpqm4BrRgbL61qM-GMCbJEMAW8otxO46s,370
 sqlframe/base/function_alternatives.py,sha256=Bs1bwl25fN3Yy9rb4GnUWBGunQ1C_yelkb2yV9DSZIY,53918
-sqlframe/base/functions.py,sha256=mSwASQe1WGyMzcwYudioJCR-bvOoqasuqNDFQ2GCC6g,224299
+sqlframe/base/functions.py,sha256=mazXkUuyvs_8YWD_ssesDXPO0TZ2sIjvUsaeUpIeNE0,224343
 sqlframe/base/group.py,sha256=4R9sOZm4ZRlTfShq2j3_HQOiL_Tj1bYkouenYsgnlII,4115
 sqlframe/base/normalize.py,sha256=nXAJ5CwxVf4DV0GsH-q1w0p8gmjSMlv96k_ez1eVul8,3880
 sqlframe/base/operations.py,sha256=g-YNcbvNKTOBbYm23GKfB3fmydlR7ZZDAuZUtXIHtzw,4438
-sqlframe/base/readerwriter.py,sha256=xjNRmHndfUAtqeVbNdcmTcjOc14LUpgxs23q7RUw1Ls,27893
-sqlframe/base/session.py,sha256=G5_bI_z1iJtAGm2SgEdjkKiyJmS0yOUopx9P5TEGdR4,27273
+sqlframe/base/readerwriter.py,sha256=Nb2VJ_HBmLQp5mK8JhnFooZh2ydAaboCAFVPb-4MNX4,31241
+sqlframe/base/session.py,sha256=vR0ESFGH6US0F-AxUw1TPDDpYbpCFGgOC049iKhq_yQ,27178
 sqlframe/base/table.py,sha256=rCeh1W5SWbtEVfkLAUiexzrZwNgmZeptLEmLcM1ABkE,6961
 sqlframe/base/transforms.py,sha256=y0j3SGDz3XCmNGrvassk1S-owllUWfkHyMgZlY6SFO4,467
 sqlframe/base/types.py,sha256=iBNk9bpFtb2NBIogYS8i7OlQZMRvpR6XxqzBebsjQDU,12280
@@ -46,7 +46,7 @@ sqlframe/databricks/dataframe.py,sha256=uBJ8GxjfiODto2PvhgmsslIwKArW7tcLBA70tvtu
 sqlframe/databricks/functions.py,sha256=La8rjAwO0hD4FBO0QxW5CtZtFAPvOrVc6lG4OtPGgbc,336
 sqlframe/databricks/functions.pyi,sha256=FzVBpzXCJzxIp73sIAo_R8Wx8uOJrix-W12HsgyeTcQ,23799
 sqlframe/databricks/group.py,sha256=dU3g0DVLRlfOSCamKchQFXRd1WTFbdxoXkpEX8tPD6Y,399
-sqlframe/databricks/readwriter.py,sha256=zLnRztTnS0onRFoC4ElSypxnVW1Q_vs75bd8FRB2GKg,14602
+sqlframe/databricks/readwriter.py,sha256=cuGRI1G627JEZgGNtirrT8LAwT6xQCdgkSAETmLKNXU,14777
 sqlframe/databricks/session.py,sha256=iw4uczkJHkpVO8vusEEmfCrhxHWyAHpCFmOZ-0qlkms,2343
 sqlframe/databricks/table.py,sha256=Q0Vnrl5aUqnqFTQpTwfWMRyQ9AQnagtpnSnXmP6IKRs,678
 sqlframe/databricks/types.py,sha256=KwNyuXIo-2xVVd4bZED3YrQOobKCtemlxGrJL7DrTC8,34
@@ -57,9 +57,9 @@ sqlframe/duckdb/catalog.py,sha256=89FCSJglMbOxonk3IXmlkMcdXCfMdePpGfqlbkkB_d0,51
 sqlframe/duckdb/column.py,sha256=E1tUa62Y5HajkhgFuebU9zohrGyieudcHzTT8gfalio,40
 sqlframe/duckdb/dataframe.py,sha256=Z8_K69UQGZVeBfVGXVwIJP8OMuIvNBB3DPKTP3Lfu4w,1908
 sqlframe/duckdb/functions.py,sha256=ix2efGGD4HLaY1rtCtEd3IrsicGEVGiBAeKOo5OD8rA,424
-sqlframe/duckdb/functions.pyi,sha256=dileIrfRspqWPBFUlXn3R8YLBQqhq0XnH6_6Rp8x1PA,12487
+sqlframe/duckdb/functions.pyi,sha256=hDjpT-tGDO8LyElcno5YYRUnJg1dXXbGcRjJ69Zqk_U,12542
 sqlframe/duckdb/group.py,sha256=IkhbW42Ng1U5YT3FkIdiB4zBqRkW4QyTb-1detY1e_4,383
-sqlframe/duckdb/readwriter.py,sha256=-_Ama7evadIa3PYvynKDK6RcTMTDBHpHJzfANTine7g,4983
+sqlframe/duckdb/readwriter.py,sha256=WEfUSKI68BFwAt4xwQX-GO8ZSGuUQYgYKkmWE55DmJo,5171
 sqlframe/duckdb/session.py,sha256=H1qjMYmhpwUHmf6jOPA6IhPIEIeX8rlvOl3MTIEijG0,2719
 sqlframe/duckdb/table.py,sha256=AmEKoH2TZo98loS5NbNaTuqv0eg76SY_OckVBMmQ6Co,410
 sqlframe/duckdb/types.py,sha256=KwNyuXIo-2xVVd4bZED3YrQOobKCtemlxGrJL7DrTC8,34
@@ -110,7 +110,7 @@ sqlframe/spark/dataframe.py,sha256=WyXHWsH8Ldu2cWTNmsLy5hEFrjJvQh_Aqv3JJcbDy6k,1
 sqlframe/spark/functions.py,sha256=MYCgHsjRQWylT-rezWRBuLV6BivcaVarbaQtP4T0toQ,331
 sqlframe/spark/functions.pyi,sha256=GyOdUzv2Z7Qt99JAKEPKgV2t2Rn274OuqwAfcoAXlN0,24259
 sqlframe/spark/group.py,sha256=MrvV_v-YkBc6T1zz882WrEqtWjlooWIyHBCmTQg3fCA,379
-sqlframe/spark/readwriter.py,sha256=YVGgkYpmuQj8wGHxJx6ivAAKSqyGes-0GhCezvd7kYU,6565
+sqlframe/spark/readwriter.py,sha256=VZHFWnhuFjlI-PMHclp1RmoIiISeT6IIp5O8lDn5TBY,6777
 sqlframe/spark/session.py,sha256=irlsTky06pKRKAyPLwVzUtLGe4O8mALSgxIqLvqJNF8,5675
 sqlframe/spark/table.py,sha256=puWV8h_CqA64zwpzq0ydY9LoygMAvprkODyxyzZeF9M,186
 sqlframe/spark/types.py,sha256=KwNyuXIo-2xVVd4bZED3YrQOobKCtemlxGrJL7DrTC8,34
@@ -130,8 +130,8 @@ sqlframe/standalone/udf.py,sha256=azmgtUjHNIPs0WMVNId05SHwiYn41MKVBhKXsQJ5dmY,27
 sqlframe/standalone/window.py,sha256=6GKPzuxeSapJakBaKBeT9VpED1ACdjggDv9JRILDyV0,35
 sqlframe/testing/__init__.py,sha256=VVCosQhitU74A3NnE52O4mNtGZONapuEXcc20QmSlnQ,132
 sqlframe/testing/utils.py,sha256=PFsGZpwNUE_4-g_f43_vstTqsK0AQ2lBneb5Eb6NkFo,13008
-sqlframe-3.26.0.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
-sqlframe-3.26.0.dist-info/METADATA,sha256=v5x-leHKJrTGK_cyWjXlJRBwLr0VrX1av511o4tKI-E,8971
-sqlframe-3.26.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-sqlframe-3.26.0.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
-sqlframe-3.26.0.dist-info/RECORD,,
+sqlframe-3.27.0.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
+sqlframe-3.27.0.dist-info/METADATA,sha256=OxTHyxBe656grcjYrGVqBA6VpyGAk36IvFaKnt1CIjk,8971
+sqlframe-3.27.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+sqlframe-3.27.0.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
+sqlframe-3.27.0.dist-info/RECORD,,