duckdb 1.4.1.dev113__cp311-cp311-macosx_10_9_universal2.whl → 1.5.0.dev37__cp311-cp311-macosx_10_9_universal2.whl

duckdb/bytes_io_wrapper.py

@@ -1,5 +1,5 @@
-from io import StringIO, TextIOBase  # noqa: D100
-from typing import Any, Union
+from io import StringIO, TextIOBase
+from typing import Union
 
 """
 BSD 3-Clause License
@@ -36,10 +36,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 """
 
 
-class BytesIOWrapper:  # noqa: D101
+class BytesIOWrapper:
     # Wrapper that wraps a StringIO buffer and reads bytes from it
     # Created for compat with pyarrow read_csv
-    def __init__(self, buffer: Union[StringIO, TextIOBase], encoding: str = "utf-8") -> None:  # noqa: D107
+    def __init__(self, buffer: Union[StringIO, TextIOBase], encoding: str = "utf-8") -> None:
         self.buffer = buffer
         self.encoding = encoding
         # Because a character can be represented by more than 1 byte,
@@ -48,10 +48,10 @@ class BytesIOWrapper:  # noqa: D101
         # overflow to the front of the bytestring the next time reading is performed
         self.overflow = b""
 
-    def __getattr__(self, attr: str) -> Any:  # noqa: D105, ANN401
+    def __getattr__(self, attr: str):
         return getattr(self.buffer, attr)
 
-    def read(self, n: Union[int, None] = -1) -> bytes:  # noqa: D102
+    def read(self, n: Union[int, None] = -1) -> bytes:
         assert self.buffer is not None
         bytestring = self.buffer.read(n).encode(self.encoding)
         # When n=-1/n greater than remaining bytes: Read entire file/rest of file
@@ -63,3 +63,4 @@ class BytesIOWrapper:  # noqa: D101
             to_return = combined_bytestring[:n]
             self.overflow = combined_bytestring[n:]
             return to_return
+

duckdb/experimental/__init__.py

@@ -1,3 +1,2 @@
-from . import spark  # noqa: D104
-
+from . import spark
 __all__ = spark.__all__

duckdb/experimental/spark/__init__.py

@@ -1,6 +1,7 @@
-from .conf import SparkConf  # noqa: D104
+from .sql import SparkSession, DataFrame
+from .conf import SparkConf
 from .context import SparkContext
+from ._globals import _NoValue
 from .exception import ContributionsAcceptedError
-from .sql import DataFrame, SparkSession
 
-__all__ = ["ContributionsAcceptedError", "DataFrame", "SparkConf", "SparkContext", "SparkSession"]
+__all__ = ["SparkSession", "DataFrame", "SparkConf", "SparkContext", "ContributionsAcceptedError"]

duckdb/experimental/spark/_globals.py

@@ -15,7 +15,8 @@
 # limitations under the License.
 #
 
-"""Module defining global singleton classes.
+"""
+Module defining global singleton classes.
 
 This module raises a RuntimeError if an attempt to reload it is made. In that
 way the identities of the classes defined here are fixed and will remain so
@@ -37,8 +38,7 @@ __ALL__ = ["_NoValue"]
 # Disallow reloading this module so as to preserve the identities of the
 # classes defined here.
 if "_is_loaded" in globals():
-    msg = "Reloading duckdb.experimental.spark._globals is not allowed"
-    raise RuntimeError(msg)
+    raise RuntimeError("Reloading duckdb.experimental.spark._globals is not allowed")
 _is_loaded = True
 
 
@@ -54,23 +54,23 @@ class _NoValueType:
 
     __instance = None
 
-    def __new__(cls) -> "_NoValueType":
+    def __new__(cls):
         # ensure that only one instance exists
         if not cls.__instance:
-            cls.__instance = super().__new__(cls)
+            cls.__instance = super(_NoValueType, cls).__new__(cls)
         return cls.__instance
 
     # Make the _NoValue instance falsey
-    def __nonzero__(self) -> bool:
+    def __nonzero__(self):
         return False
 
     __bool__ = __nonzero__
 
     # needed for python 2 to preserve identity through a pickle
-    def __reduce__(self) -> tuple[type, tuple]:
+    def __reduce__(self):
         return (self.__class__, ())
 
-    def __repr__(self) -> str:
+    def __repr__(self):
         return "<no value>"
 
 

duckdb/experimental/spark/_typing.py

@@ -16,11 +16,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from collections.abc import Iterable, Sized
-from typing import Callable, TypeVar, Union
+from typing import Callable, Iterable, Sized, TypeVar, Union
+from typing_extensions import Literal, Protocol
 
-from numpy import float32, float64, int32, int64, ndarray
-from typing_extensions import Literal, Protocol, Self
+from numpy import int32, int64, float32, float64, ndarray
 
 F = TypeVar("F", bound=Callable)
 T_co = TypeVar("T_co", covariant=True)
@@ -31,14 +30,17 @@ NonUDFType = Literal[0]
 
 
 class SupportsIAdd(Protocol):
-    def __iadd__(self, other: "SupportsIAdd") -> Self: ...
+    def __iadd__(self, other: "SupportsIAdd") -> "SupportsIAdd":
+        ...
 
 
 class SupportsOrdering(Protocol):
-    def __lt__(self, other: "SupportsOrdering") -> bool: ...
+    def __lt__(self, other: "SupportsOrdering") -> bool:
+        ...
 
 
-class SizedIterable(Protocol, Sized, Iterable[T_co]): ...
+class SizedIterable(Protocol, Sized, Iterable[T_co]):
+    ...
 
 
 S = TypeVar("S", bound=SupportsOrdering)

duckdb/experimental/spark/conf.py

@@ -1,45 +1,44 @@
-from typing import Optional  # noqa: D100
-
+from typing import Optional, List, Tuple
 from duckdb.experimental.spark.exception import ContributionsAcceptedError
 
 
-class SparkConf:  # noqa: D101
-    def __init__(self) -> None:  # noqa: D107
+class SparkConf:
+    def __init__(self):
         raise NotImplementedError
 
-    def contains(self, key: str) -> bool:  # noqa: D102
+    def contains(self, key: str) -> bool:
         raise ContributionsAcceptedError
 
-    def get(self, key: str, defaultValue: Optional[str] = None) -> Optional[str]:  # noqa: D102
+    def get(self, key: str, defaultValue: Optional[str] = None) -> Optional[str]:
         raise ContributionsAcceptedError
 
-    def getAll(self) -> list[tuple[str, str]]:  # noqa: D102
+    def getAll(self) -> List[Tuple[str, str]]:
         raise ContributionsAcceptedError
 
-    def set(self, key: str, value: str) -> "SparkConf":  # noqa: D102
+    def set(self, key: str, value: str) -> "SparkConf":
         raise ContributionsAcceptedError
 
-    def setAll(self, pairs: list[tuple[str, str]]) -> "SparkConf":  # noqa: D102
+    def setAll(self, pairs: List[Tuple[str, str]]) -> "SparkConf":
         raise ContributionsAcceptedError
 
-    def setAppName(self, value: str) -> "SparkConf":  # noqa: D102
+    def setAppName(self, value: str) -> "SparkConf":
         raise ContributionsAcceptedError
 
-    def setExecutorEnv(  # noqa: D102
-        self, key: Optional[str] = None, value: Optional[str] = None, pairs: Optional[list[tuple[str, str]]] = None
+    def setExecutorEnv(
+        self, key: Optional[str] = None, value: Optional[str] = None, pairs: Optional[List[Tuple[str, str]]] = None
     ) -> "SparkConf":
         raise ContributionsAcceptedError
 
-    def setIfMissing(self, key: str, value: str) -> "SparkConf":  # noqa: D102
+    def setIfMissing(self, key: str, value: str) -> "SparkConf":
         raise ContributionsAcceptedError
 
-    def setMaster(self, value: str) -> "SparkConf":  # noqa: D102
+    def setMaster(self, value: str) -> "SparkConf":
         raise ContributionsAcceptedError
 
-    def setSparkHome(self, value: str) -> "SparkConf":  # noqa: D102
+    def setSparkHome(self, value: str) -> "SparkConf":
         raise ContributionsAcceptedError
 
-    def toDebugString(self) -> str:  # noqa: D102
+    def toDebugString(self) -> str:
         raise ContributionsAcceptedError
 
 

duckdb/experimental/spark/context.py

@@ -1,42 +1,42 @@
-from typing import Optional  # noqa: D100
-
+from typing import Optional
 import duckdb
 from duckdb import DuckDBPyConnection
-from duckdb.experimental.spark.conf import SparkConf
+
 from duckdb.experimental.spark.exception import ContributionsAcceptedError
+from duckdb.experimental.spark.conf import SparkConf
 
 
-class SparkContext:  # noqa: D101
-    def __init__(self, master: str) -> None:  # noqa: D107
-        self._connection = duckdb.connect(":memory:")
+class SparkContext:
+    def __init__(self, master: str):
+        self._connection = duckdb.connect(':memory:')
         # This aligns the null ordering with Spark.
         self._connection.execute("set default_null_order='nulls_first_on_asc_last_on_desc'")
 
     @property
-    def connection(self) -> DuckDBPyConnection:  # noqa: D102
+    def connection(self) -> DuckDBPyConnection:
         return self._connection
 
-    def stop(self) -> None:  # noqa: D102
+    def stop(self) -> None:
         self._connection.close()
 
     @classmethod
-    def getOrCreate(cls, conf: Optional[SparkConf] = None) -> "SparkContext":  # noqa: D102
+    def getOrCreate(cls, conf: Optional[SparkConf] = None) -> "SparkContext":
         raise ContributionsAcceptedError
 
     @classmethod
-    def setSystemProperty(cls, key: str, value: str) -> None:  # noqa: D102
+    def setSystemProperty(cls, key: str, value: str) -> None:
         raise ContributionsAcceptedError
 
     @property
-    def applicationId(self) -> str:  # noqa: D102
+    def applicationId(self) -> str:
         raise ContributionsAcceptedError
 
     @property
-    def defaultMinPartitions(self) -> int:  # noqa: D102
+    def defaultMinPartitions(self) -> int:
         raise ContributionsAcceptedError
 
     @property
-    def defaultParallelism(self) -> int:  # noqa: D102
+    def defaultParallelism(self) -> int:
         raise ContributionsAcceptedError
 
     # @property
@@ -44,35 +44,33 @@ class SparkContext:  # noqa: D101
     # 	raise ContributionsAcceptedError
 
     @property
-    def startTime(self) -> str:  # noqa: D102
+    def startTime(self) -> str:
         raise ContributionsAcceptedError
 
     @property
-    def uiWebUrl(self) -> str:  # noqa: D102
+    def uiWebUrl(self) -> str:
         raise ContributionsAcceptedError
 
     @property
-    def version(self) -> str:  # noqa: D102
+    def version(self) -> str:
         raise ContributionsAcceptedError
 
-    def __repr__(self) -> str:  # noqa: D105
+    def __repr__(self) -> str:
         raise ContributionsAcceptedError
 
-    # def accumulator(self, value: ~T, accum_param: Optional[ForwardRef('AccumulatorParam[T]')] = None
-    #     ) -> 'Accumulator[T]':
+    # def accumulator(self, value: ~T, accum_param: Optional[ForwardRef('AccumulatorParam[T]')] = None) -> 'Accumulator[T]':
     # 	pass
 
-    def addArchive(self, path: str) -> None:  # noqa: D102
+    def addArchive(self, path: str) -> None:
         raise ContributionsAcceptedError
 
-    def addFile(self, path: str, recursive: bool = False) -> None:  # noqa: D102
+    def addFile(self, path: str, recursive: bool = False) -> None:
         raise ContributionsAcceptedError
 
-    def addPyFile(self, path: str) -> None:  # noqa: D102
+    def addPyFile(self, path: str) -> None:
         raise ContributionsAcceptedError
 
-    # def binaryFiles(self, path: str, minPartitions: Optional[int] = None
-    #     ) -> duckdb.experimental.spark.rdd.RDD[typing.Tuple[str, bytes]]:
+    # def binaryFiles(self, path: str, minPartitions: Optional[int] = None) -> duckdb.experimental.spark.rdd.RDD[typing.Tuple[str, bytes]]:
     # 	pass
 
     # def binaryRecords(self, path: str, recordLength: int) -> duckdb.experimental.spark.rdd.RDD[bytes]:
@@ -81,45 +79,37 @@ class SparkContext:  # noqa: D101
     # def broadcast(self, value: ~T) -> 'Broadcast[T]':
     # 	pass
 
-    def cancelAllJobs(self) -> None:  # noqa: D102
+    def cancelAllJobs(self) -> None:
         raise ContributionsAcceptedError
 
-    def cancelJobGroup(self, groupId: str) -> None:  # noqa: D102
+    def cancelJobGroup(self, groupId: str) -> None:
         raise ContributionsAcceptedError
 
-    def dump_profiles(self, path: str) -> None:  # noqa: D102
+    def dump_profiles(self, path: str) -> None:
         raise ContributionsAcceptedError
 
     # def emptyRDD(self) -> duckdb.experimental.spark.rdd.RDD[typing.Any]:
     # 	pass
 
-    def getCheckpointDir(self) -> Optional[str]:  # noqa: D102
+    def getCheckpointDir(self) -> Optional[str]:
         raise ContributionsAcceptedError
 
-    def getConf(self) -> SparkConf:  # noqa: D102
+    def getConf(self) -> SparkConf:
         raise ContributionsAcceptedError
 
-    def getLocalProperty(self, key: str) -> Optional[str]:  # noqa: D102
+    def getLocalProperty(self, key: str) -> Optional[str]:
         raise ContributionsAcceptedError
 
-    # def hadoopFile(self, path: str, inputFormatClass: str, keyClass: str, valueClass: str,
-    #     keyConverter: Optional[str] = None, valueConverter: Optional[str] = None,
-    #     conf: Optional[Dict[str, str]] = None, batchSize: int = 0) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
+    # def hadoopFile(self, path: str, inputFormatClass: str, keyClass: str, valueClass: str, keyConverter: Optional[str] = None, valueConverter: Optional[str] = None, conf: Optional[Dict[str, str]] = None, batchSize: int = 0) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
     # 	pass
 
-    # def hadoopRDD(self, inputFormatClass: str, keyClass: str, valueClass: str, keyConverter: Optional[str] = None,
-    #     valueConverter: Optional[str] = None, conf: Optional[Dict[str, str]] = None, batchSize: int = 0
-    #     ) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
+    # def hadoopRDD(self, inputFormatClass: str, keyClass: str, valueClass: str, keyConverter: Optional[str] = None, valueConverter: Optional[str] = None, conf: Optional[Dict[str, str]] = None, batchSize: int = 0) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
     # 	pass
 
-    # def newAPIHadoopFile(self, path: str, inputFormatClass: str, keyClass: str, valueClass: str,
-    #     keyConverter: Optional[str] = None, valueConverter: Optional[str] = None,
-    #     conf: Optional[Dict[str, str]] = None, batchSize: int = 0) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
+    # def newAPIHadoopFile(self, path: str, inputFormatClass: str, keyClass: str, valueClass: str, keyConverter: Optional[str] = None, valueConverter: Optional[str] = None, conf: Optional[Dict[str, str]] = None, batchSize: int = 0) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
     # 	pass
 
-    # def newAPIHadoopRDD(self, inputFormatClass: str, keyClass: str, valueClass: str,
-    #     keyConverter: Optional[str] = None, valueConverter: Optional[str] = None,
-    #     conf: Optional[Dict[str, str]] = None, batchSize: int = 0) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
+    # def newAPIHadoopRDD(self, inputFormatClass: str, keyClass: str, valueClass: str, keyConverter: Optional[str] = None, valueConverter: Optional[str] = None, conf: Optional[Dict[str, str]] = None, batchSize: int = 0) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
     # 	pass
 
     # def parallelize(self, c: Iterable[~T], numSlices: Optional[int] = None) -> pyspark.rdd.RDD[~T]:
@@ -128,52 +118,46 @@ class SparkContext:  # noqa: D101
     # def pickleFile(self, name: str, minPartitions: Optional[int] = None) -> pyspark.rdd.RDD[typing.Any]:
     # 	pass
 
-    # def range(self, start: int, end: Optional[int] = None, step: int = 1, numSlices: Optional[int] = None
-    #     ) -> pyspark.rdd.RDD[int]:
+    # def range(self, start: int, end: Optional[int] = None, step: int = 1, numSlices: Optional[int] = None) -> pyspark.rdd.RDD[int]:
     # 	pass
 
-    # def runJob(self, rdd: pyspark.rdd.RDD[~T], partitionFunc: Callable[[Iterable[~T]], Iterable[~U]],
-    #     partitions: Optional[Sequence[int]] = None, allowLocal: bool = False) -> List[~U]:
+    # def runJob(self, rdd: pyspark.rdd.RDD[~T], partitionFunc: Callable[[Iterable[~T]], Iterable[~U]], partitions: Optional[Sequence[int]] = None, allowLocal: bool = False) -> List[~U]:
     # 	pass
 
-    # def sequenceFile(self, path: str, keyClass: Optional[str] = None, valueClass: Optional[str] = None,
-    #     keyConverter: Optional[str] = None, valueConverter: Optional[str] = None, minSplits: Optional[int] = None,
-    #     batchSize: int = 0) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
+    # def sequenceFile(self, path: str, keyClass: Optional[str] = None, valueClass: Optional[str] = None, keyConverter: Optional[str] = None, valueConverter: Optional[str] = None, minSplits: Optional[int] = None, batchSize: int = 0) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
     # 	pass
 
-    def setCheckpointDir(self, dirName: str) -> None:  # noqa: D102
+    def setCheckpointDir(self, dirName: str) -> None:
         raise ContributionsAcceptedError
 
-    def setJobDescription(self, value: str) -> None:  # noqa: D102
+    def setJobDescription(self, value: str) -> None:
         raise ContributionsAcceptedError
 
-    def setJobGroup(self, groupId: str, description: str, interruptOnCancel: bool = False) -> None:  # noqa: D102
+    def setJobGroup(self, groupId: str, description: str, interruptOnCancel: bool = False) -> None:
         raise ContributionsAcceptedError
 
-    def setLocalProperty(self, key: str, value: str) -> None:  # noqa: D102
+    def setLocalProperty(self, key: str, value: str) -> None:
         raise ContributionsAcceptedError
 
-    def setLogLevel(self, logLevel: str) -> None:  # noqa: D102
+    def setLogLevel(self, logLevel: str) -> None:
         raise ContributionsAcceptedError
 
-    def show_profiles(self) -> None:  # noqa: D102
+    def show_profiles(self) -> None:
         raise ContributionsAcceptedError
 
-    def sparkUser(self) -> str:  # noqa: D102
+    def sparkUser(self) -> str:
         raise ContributionsAcceptedError
 
     # def statusTracker(self) -> duckdb.experimental.spark.status.StatusTracker:
     # 	raise ContributionsAcceptedError
 
-    # def textFile(self, name: str, minPartitions: Optional[int] = None, use_unicode: bool = True
-    #     ) -> pyspark.rdd.RDD[str]:
+    # def textFile(self, name: str, minPartitions: Optional[int] = None, use_unicode: bool = True) -> pyspark.rdd.RDD[str]:
     # 	pass
 
     # def union(self, rdds: List[pyspark.rdd.RDD[~T]]) -> pyspark.rdd.RDD[~T]:
     # 	pass
 
-    # def wholeTextFiles(self, path: str, minPartitions: Optional[int] = None, use_unicode: bool = True
-    #     ) -> pyspark.rdd.RDD[typing.Tuple[str, str]]:
+    # def wholeTextFiles(self, path: str, minPartitions: Optional[int] = None, use_unicode: bool = True) -> pyspark.rdd.RDD[typing.Tuple[str, str]]:
     # 	pass
 
 

duckdb/experimental/spark/errors/__init__.py

@@ -15,56 +15,58 @@
 # limitations under the License.
 #
 
-"""PySpark exceptions."""
-
-from .exceptions.base import (
+"""
+PySpark exceptions.
+"""
+from .exceptions.base import (  # noqa: F401
+    PySparkException,
     AnalysisException,
+    TempTableAlreadyExistsException,
+    ParseException,
+    IllegalArgumentException,
     ArithmeticException,
+    UnsupportedOperationException,
     ArrayIndexOutOfBoundsException,
     DateTimeException,
-    IllegalArgumentException,
     NumberFormatException,
-    ParseException,
-    PySparkAssertionError,
-    PySparkAttributeError,
-    PySparkException,
-    PySparkIndexError,
-    PySparkNotImplementedError,
-    PySparkRuntimeError,
-    PySparkTypeError,
-    PySparkValueError,
-    PythonException,
+    StreamingQueryException,
     QueryExecutionException,
+    PythonException,
+    UnknownException,
     SparkRuntimeException,
     SparkUpgradeException,
-    StreamingQueryException,
-    TempTableAlreadyExistsException,
-    UnknownException,
-    UnsupportedOperationException,
+    PySparkTypeError,
+    PySparkValueError,
+    PySparkIndexError,
+    PySparkAttributeError,
+    PySparkRuntimeError,
+    PySparkAssertionError,
+    PySparkNotImplementedError,
 )
 
+
 __all__ = [
+    "PySparkException",
     "AnalysisException",
+    "TempTableAlreadyExistsException",
+    "ParseException",
+    "IllegalArgumentException",
     "ArithmeticException",
+    "UnsupportedOperationException",
     "ArrayIndexOutOfBoundsException",
     "DateTimeException",
-    "IllegalArgumentException",
     "NumberFormatException",
-    "ParseException",
-    "PySparkAssertionError",
-    "PySparkAttributeError",
-    "PySparkException",
-    "PySparkIndexError",
-    "PySparkNotImplementedError",
-    "PySparkRuntimeError",
-    "PySparkTypeError",
-    "PySparkValueError",
-    "PythonException",
+    "StreamingQueryException",
     "QueryExecutionException",
+    "PythonException",
+    "UnknownException",
     "SparkRuntimeException",
     "SparkUpgradeException",
-    "StreamingQueryException",
-    "TempTableAlreadyExistsException",
-    "UnknownException",
-    "UnsupportedOperationException",
+    "PySparkTypeError",
+    "PySparkValueError",
+    "PySparkIndexError",
+    "PySparkAttributeError",
+    "PySparkRuntimeError",
+    "PySparkAssertionError",
+    "PySparkNotImplementedError",
 ]

duckdb/experimental/spark/errors/error_classes.py

@@ -1,4 +1,4 @@
-# ruff: noqa: D100, E501
+#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.

duckdb/experimental/spark/errors/exceptions/__init__.py

@@ -1,4 +1,4 @@
-#  # noqa: D104
+#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.