duckdb 1.5.0.dev86__cp314-cp314-macosx_10_15_universal2.whl

duckdb/experimental/spark/_typing.py

@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from collections.abc import Iterable, Sized
+from typing import Callable, TypeVar, Union
+
+from numpy import float32, float64, int32, int64, ndarray
+from typing_extensions import Literal, Protocol, Self
+
+F = TypeVar("F", bound=Callable)
+T_co = TypeVar("T_co", covariant=True)
+
+PrimitiveType = Union[bool, float, int, str]
+
+NonUDFType = Literal[0]
+
+
+class SupportsIAdd(Protocol):
+    def __iadd__(self, other: "SupportsIAdd") -> Self: ...
+
+
+class SupportsOrdering(Protocol):
+    def __lt__(self, other: "SupportsOrdering") -> bool: ...
+
+
+class SizedIterable(Protocol, Sized, Iterable[T_co]): ...
+
+
+S = TypeVar("S", bound=SupportsOrdering)
+
+NumberOrArray = TypeVar("NumberOrArray", float, int, complex, int32, int64, float32, float64, ndarray)

duckdb/experimental/spark/conf.py

@@ -0,0 +1,46 @@
+from typing import Optional  # noqa: D100
+
+from duckdb.experimental.spark.exception import ContributionsAcceptedError
+
+
+class SparkConf:  # noqa: D101
+    def __init__(self) -> None:  # noqa: D107
+        raise NotImplementedError
+
+    def contains(self, key: str) -> bool:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def get(self, key: str, defaultValue: Optional[str] = None) -> Optional[str]:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def getAll(self) -> list[tuple[str, str]]:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def set(self, key: str, value: str) -> "SparkConf":  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def setAll(self, pairs: list[tuple[str, str]]) -> "SparkConf":  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def setAppName(self, value: str) -> "SparkConf":  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def setExecutorEnv(  # noqa: D102
+        self, key: Optional[str] = None, value: Optional[str] = None, pairs: Optional[list[tuple[str, str]]] = None
+    ) -> "SparkConf":
+        raise ContributionsAcceptedError
+
+    def setIfMissing(self, key: str, value: str) -> "SparkConf":  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def setMaster(self, value: str) -> "SparkConf":  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def setSparkHome(self, value: str) -> "SparkConf":  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def toDebugString(self) -> str:  # noqa: D102
+        raise ContributionsAcceptedError
+
+
+__all__ = ["SparkConf"]

duckdb/experimental/spark/context.py

@@ -0,0 +1,180 @@
+from typing import Optional  # noqa: D100
+
+import duckdb
+from duckdb import DuckDBPyConnection
+from duckdb.experimental.spark.conf import SparkConf
+from duckdb.experimental.spark.exception import ContributionsAcceptedError
+
+
+class SparkContext:  # noqa: D101
+    def __init__(self, master: str) -> None:  # noqa: D107
+        self._connection = duckdb.connect(":memory:")
+        # This aligns the null ordering with Spark.
+        self._connection.execute("set default_null_order='nulls_first_on_asc_last_on_desc'")
+
+    @property
+    def connection(self) -> DuckDBPyConnection:  # noqa: D102
+        return self._connection
+
+    def stop(self) -> None:  # noqa: D102
+        self._connection.close()
+
+    @classmethod
+    def getOrCreate(cls, conf: Optional[SparkConf] = None) -> "SparkContext":  # noqa: D102
+        raise ContributionsAcceptedError
+
+    @classmethod
+    def setSystemProperty(cls, key: str, value: str) -> None:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    @property
+    def applicationId(self) -> str:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    @property
+    def defaultMinPartitions(self) -> int:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    @property
+    def defaultParallelism(self) -> int:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    # @property
+    # def resources(self) -> Dict[str, ResourceInformation]:
+    # 	raise ContributionsAcceptedError
+
+    @property
+    def startTime(self) -> str:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    @property
+    def uiWebUrl(self) -> str:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    @property
+    def version(self) -> str:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def __repr__(self) -> str:  # noqa: D105
+        raise ContributionsAcceptedError
+
+    # def accumulator(self, value: ~T, accum_param: Optional[ForwardRef('AccumulatorParam[T]')] = None
+    #     ) -> 'Accumulator[T]':
+    # 	pass
+
+    def addArchive(self, path: str) -> None:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def addFile(self, path: str, recursive: bool = False) -> None:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def addPyFile(self, path: str) -> None:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    # def binaryFiles(self, path: str, minPartitions: Optional[int] = None
+    #     ) -> duckdb.experimental.spark.rdd.RDD[typing.Tuple[str, bytes]]:
+    # 	pass
+
+    # def binaryRecords(self, path: str, recordLength: int) -> duckdb.experimental.spark.rdd.RDD[bytes]:
+    # 	pass
+
+    # def broadcast(self, value: ~T) -> 'Broadcast[T]':
+    # 	pass
+
+    def cancelAllJobs(self) -> None:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def cancelJobGroup(self, groupId: str) -> None:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def dump_profiles(self, path: str) -> None:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    # def emptyRDD(self) -> duckdb.experimental.spark.rdd.RDD[typing.Any]:
+    # 	pass
+
+    def getCheckpointDir(self) -> Optional[str]:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def getConf(self) -> SparkConf:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def getLocalProperty(self, key: str) -> Optional[str]:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    # def hadoopFile(self, path: str, inputFormatClass: str, keyClass: str, valueClass: str,
+    #     keyConverter: Optional[str] = None, valueConverter: Optional[str] = None,
+    #     conf: Optional[Dict[str, str]] = None, batchSize: int = 0) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
+    # 	pass
+
+    # def hadoopRDD(self, inputFormatClass: str, keyClass: str, valueClass: str, keyConverter: Optional[str] = None,
+    #     valueConverter: Optional[str] = None, conf: Optional[Dict[str, str]] = None, batchSize: int = 0
+    #     ) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
+    # 	pass
+
+    # def newAPIHadoopFile(self, path: str, inputFormatClass: str, keyClass: str, valueClass: str,
+    #     keyConverter: Optional[str] = None, valueConverter: Optional[str] = None,
+    #     conf: Optional[Dict[str, str]] = None, batchSize: int = 0) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
+    # 	pass
+
+    # def newAPIHadoopRDD(self, inputFormatClass: str, keyClass: str, valueClass: str,
+    #     keyConverter: Optional[str] = None, valueConverter: Optional[str] = None,
+    #     conf: Optional[Dict[str, str]] = None, batchSize: int = 0) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
+    # 	pass
+
+    # def parallelize(self, c: Iterable[~T], numSlices: Optional[int] = None) -> pyspark.rdd.RDD[~T]:
+    # 	pass
+
+    # def pickleFile(self, name: str, minPartitions: Optional[int] = None) -> pyspark.rdd.RDD[typing.Any]:
+    # 	pass
+
+    # def range(self, start: int, end: Optional[int] = None, step: int = 1, numSlices: Optional[int] = None
+    #     ) -> pyspark.rdd.RDD[int]:
+    # 	pass
+
+    # def runJob(self, rdd: pyspark.rdd.RDD[~T], partitionFunc: Callable[[Iterable[~T]], Iterable[~U]],
+    #     partitions: Optional[Sequence[int]] = None, allowLocal: bool = False) -> List[~U]:
+    # 	pass
+
+    # def sequenceFile(self, path: str, keyClass: Optional[str] = None, valueClass: Optional[str] = None,
+    #     keyConverter: Optional[str] = None, valueConverter: Optional[str] = None, minSplits: Optional[int] = None,
+    #     batchSize: int = 0) -> pyspark.rdd.RDD[typing.Tuple[~T, ~U]]:
+    # 	pass
+
+    def setCheckpointDir(self, dirName: str) -> None:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def setJobDescription(self, value: str) -> None:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def setJobGroup(self, groupId: str, description: str, interruptOnCancel: bool = False) -> None:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def setLocalProperty(self, key: str, value: str) -> None:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def setLogLevel(self, logLevel: str) -> None:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def show_profiles(self) -> None:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def sparkUser(self) -> str:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    # def statusTracker(self) -> duckdb.experimental.spark.status.StatusTracker:
+    # 	raise ContributionsAcceptedError
+
+    # def textFile(self, name: str, minPartitions: Optional[int] = None, use_unicode: bool = True
+    #     ) -> pyspark.rdd.RDD[str]:
+    # 	pass
+
+    # def union(self, rdds: List[pyspark.rdd.RDD[~T]]) -> pyspark.rdd.RDD[~T]:
+    # 	pass
+
+    # def wholeTextFiles(self, path: str, minPartitions: Optional[int] = None, use_unicode: bool = True
+    #     ) -> pyspark.rdd.RDD[typing.Tuple[str, str]]:
+    # 	pass
+
+
+__all__ = ["SparkContext"]

duckdb/experimental/spark/errors/__init__.py

@@ -0,0 +1,70 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""PySpark exceptions."""
+
+from .exceptions.base import (
+    AnalysisException,
+    ArithmeticException,
+    ArrayIndexOutOfBoundsException,
+    DateTimeException,
+    IllegalArgumentException,
+    NumberFormatException,
+    ParseException,
+    PySparkAssertionError,
+    PySparkAttributeError,
+    PySparkException,
+    PySparkIndexError,
+    PySparkNotImplementedError,
+    PySparkRuntimeError,
+    PySparkTypeError,
+    PySparkValueError,
+    PythonException,
+    QueryExecutionException,
+    SparkRuntimeException,
+    SparkUpgradeException,
+    StreamingQueryException,
+    TempTableAlreadyExistsException,
+    UnknownException,
+    UnsupportedOperationException,
+)
+
+__all__ = [
+    "AnalysisException",
+    "ArithmeticException",
+    "ArrayIndexOutOfBoundsException",
+    "DateTimeException",
+    "IllegalArgumentException",
+    "NumberFormatException",
+    "ParseException",
+    "PySparkAssertionError",
+    "PySparkAttributeError",
+    "PySparkException",
+    "PySparkIndexError",
+    "PySparkNotImplementedError",
+    "PySparkRuntimeError",
+    "PySparkTypeError",
+    "PySparkValueError",
+    "PythonException",
+    "QueryExecutionException",
+    "SparkRuntimeException",
+    "SparkUpgradeException",
+    "StreamingQueryException",
+    "TempTableAlreadyExistsException",
+    "UnknownException",
+    "UnsupportedOperationException",
+]