duckdb 0.8.2.dev3007__cp311-cp311-win_amd64.whl → 1.4.3.dev8__cp311-cp311-win_amd64.whl

pyduckdb/spark/sql/readwriter.py

@@ -1,180 +0,0 @@
-from typing import TYPE_CHECKING, Iterable, Union, List, Optional, cast
-from pyduckdb.spark.sql.types import StructType
-from ..exception import ContributionsAcceptedError
-
-PrimitiveType = Union[bool, float, int, str]
-OptionalPrimitiveType = Optional[PrimitiveType]
-
-if TYPE_CHECKING:
-    from pyduckdb.spark.sql.dataframe import DataFrame
-    from pyduckdb.spark.sql.session import SparkSession
-
-
-class DataFrameWriter:
-    def __init__(self, dataframe: "DataFrame"):
-        self.dataframe = dataframe
-
-    def saveAsTable(self, table_name: str) -> None:
-        relation = self.dataframe.relation
-        relation.create(table_name)
-
-
-class DataFrameReader:
-    def __init__(self, session: "SparkSession"):
-        self.session = session
-
-    def load(
-        self,
-        path: Optional[Union[str, List[str]]] = None,
-        format: Optional[str] = None,
-        schema: Optional[Union[StructType, str]] = None,
-        **options: OptionalPrimitiveType,
-    ) -> "DataFrame":
-        from pyduckdb.spark.sql.dataframe import DataFrame
-
-        if not isinstance(path, str):
-            raise ImportError
-        if options:
-            raise ContributionsAcceptedError
-
-        rel = None
-        if format:
-            format = format.lower()
-            if format == 'csv' or format == 'tsv':
-                rel = self.session.conn.read_csv(path)
-            elif format == 'json':
-                rel = self.session.conn.read_json(path)
-            elif format == 'parquet':
-                rel = self.session.conn.read_parquet(path)
-            else:
-                raise ContributionsAcceptedError
-        else:
-            rel = self.session.conn.sql(f'select * from {path}')
-        df = DataFrame(rel, self.session)
-        if schema:
-            if not isinstance(schema, StructType):
-                raise ContributionsAcceptedError
-            schema = cast(StructType, schema)
-            types, names = schema.extract_types_and_names()
-            df = df._cast_types(types)
-            df = df.toDF(names)
-        raise NotImplementedError
-
-    def csv(
-        self,
-        path: Union[str, List[str]],
-        schema: Optional[Union[StructType, str]] = None,
-        sep: Optional[str] = None,
-        encoding: Optional[str] = None,
-        quote: Optional[str] = None,
-        escape: Optional[str] = None,
-        comment: Optional[str] = None,
-        header: Optional[Union[bool, str]] = None,
-        inferSchema: Optional[Union[bool, str]] = None,
-        ignoreLeadingWhiteSpace: Optional[Union[bool, str]] = None,
-        ignoreTrailingWhiteSpace: Optional[Union[bool, str]] = None,
-        nullValue: Optional[str] = None,
-        nanValue: Optional[str] = None,
-        positiveInf: Optional[str] = None,
-        negativeInf: Optional[str] = None,
-        dateFormat: Optional[str] = None,
-        timestampFormat: Optional[str] = None,
-        maxColumns: Optional[Union[int, str]] = None,
-        maxCharsPerColumn: Optional[Union[int, str]] = None,
-        maxMalformedLogPerPartition: Optional[Union[int, str]] = None,
-        mode: Optional[str] = None,
-        columnNameOfCorruptRecord: Optional[str] = None,
-        multiLine: Optional[Union[bool, str]] = None,
-        charToEscapeQuoteEscaping: Optional[str] = None,
-        samplingRatio: Optional[Union[float, str]] = None,
-        enforceSchema: Optional[Union[bool, str]] = None,
-        emptyValue: Optional[str] = None,
-        locale: Optional[str] = None,
-        lineSep: Optional[str] = None,
-        pathGlobFilter: Optional[Union[bool, str]] = None,
-        recursiveFileLookup: Optional[Union[bool, str]] = None,
-        modifiedBefore: Optional[Union[bool, str]] = None,
-        modifiedAfter: Optional[Union[bool, str]] = None,
-        unescapedQuoteHandling: Optional[str] = None,
-    ) -> "DataFrame":
-        if not isinstance(path, str):
-            raise NotImplementedError
-        if schema and not isinstance(schema, StructType):
-            raise ContributionsAcceptedError
-        if comment:
-            raise ContributionsAcceptedError
-        if inferSchema:
-            raise ContributionsAcceptedError
-        if ignoreLeadingWhiteSpace:
-            raise ContributionsAcceptedError
-        if ignoreTrailingWhiteSpace:
-            raise ContributionsAcceptedError
-        if nanValue:
-            raise ConnectionAbortedError
-        if positiveInf:
-            raise ConnectionAbortedError
-        if negativeInf:
-            raise ConnectionAbortedError
-        if negativeInf:
-            raise ConnectionAbortedError
-        if maxColumns:
-            raise ContributionsAcceptedError
-        if maxCharsPerColumn:
-            raise ContributionsAcceptedError
-        if maxMalformedLogPerPartition:
-            raise ContributionsAcceptedError
-        if mode:
-            raise ContributionsAcceptedError
-        if columnNameOfCorruptRecord:
-            raise ContributionsAcceptedError
-        if multiLine:
-            raise ContributionsAcceptedError
-        if charToEscapeQuoteEscaping:
-            raise ContributionsAcceptedError
-        if samplingRatio:
-            raise ContributionsAcceptedError
-        if enforceSchema:
-            raise ContributionsAcceptedError
-        if emptyValue:
-            raise ContributionsAcceptedError
-        if locale:
-            raise ContributionsAcceptedError
-        if pathGlobFilter:
-            raise ContributionsAcceptedError
-        if recursiveFileLookup:
-            raise ContributionsAcceptedError
-        if modifiedBefore:
-            raise ContributionsAcceptedError
-        if modifiedAfter:
-            raise ContributionsAcceptedError
-        if unescapedQuoteHandling:
-            raise ContributionsAcceptedError
-        if lineSep:
-            # We have support for custom newline, just needs to be ported to 'read_csv'
-            raise NotImplementedError
-
-        dtype = None
-        names = None
-        if schema:
-            schema = cast(StructType, schema)
-            dtype, names = schema.extract_types_and_names()
-
-        rel = self.session.conn.read_csv(
-            path,
-            header=header if isinstance(header, bool) else header == "True",
-            sep=sep,
-            dtype=dtype,
-            na_values=nullValue,
-            quotechar=quote,
-            escapechar=escape,
-            encoding=encoding,
-            date_format=dateFormat,
-            timestamp_format=timestampFormat,
-        )
-        df = DataFrame(rel, self.session)
-        if names:
-            df = df.toDF(*names)
-        return df
-
-
-__all__ = ["DataFrameWriter", "DataFrameReader"]

pyduckdb/spark/sql/session.py

@@ -1,249 +0,0 @@
-from typing import Optional, List, Tuple, Any, Union, Iterable, TYPE_CHECKING
-import uuid
-
-if TYPE_CHECKING:
-    from pyduckdb.spark.sql.catalog import Catalog
-    from pandas.core.frame import DataFrame as PandasDataFrame
-
-from pyduckdb.spark.exception import ContributionsAcceptedError
-
-from pyduckdb.spark.sql.types import StructType, AtomicType, DataType
-from pyduckdb.spark.conf import SparkConf
-from pyduckdb.spark.sql.dataframe import DataFrame
-from pyduckdb.spark.sql.conf import RuntimeConfig
-from pyduckdb.spark.sql.readwriter import DataFrameReader
-from pyduckdb.spark.context import SparkContext
-from pyduckdb.spark.sql.udf import UDFRegistration
-from pyduckdb.spark.sql.streaming import DataStreamReader
-import duckdb
-
-# In spark:
-# SparkSession holds a SparkContext
-# SparkContext gets created from SparkConf
-# At this level the check is made to determine whether the instance already exists and just needs to be retrieved or it needs to be created
-
-# For us this is done inside of `duckdb.connect`, based on the passed in path + configuration
-# SparkContext can be compared to our Connection class, and SparkConf to our ClientContext class
-
-
-# data is a List of rows
-# every value in each row needs to be turned into a Value
-def _combine_data_and_schema(data: Iterable[Any], schema: StructType):
-    from pyduckdb import Value
-
-    new_data = []
-    for row in data:
-        new_row = [Value(x, dtype.duckdb_type) for x, dtype in zip(row, [y.dataType for y in schema])]
-        new_data.append(new_row)
-    return new_data
-
-
-class SparkSession:
-    def __init__(self, context: SparkContext):
-        self.conn = context.connection
-        self._context = context
-        self._conf = RuntimeConfig(self.conn)
-
-    def _create_dataframe(self, data: Union[Iterable[Any], "PandasDataFrame"]) -> DataFrame:
-        try:
-            import pandas
-
-            has_pandas = True
-        except:
-            has_pandas = False
-        if has_pandas and isinstance(data, pandas.DataFrame):
-            unique_name = f'pyspark_pandas_df_{uuid.uuid1()}'
-            self.conn.register(unique_name, data)
-            return DataFrame(self.conn.sql(f'select * from "{unique_name}"'), self)
-
-        def verify_tuple_integrity(tuples):
-            if len(tuples) <= 1:
-                return
-            assert all([len(x) == len(tuples[0]) for x in tuples[1:]])
-
-        if not isinstance(data, list):
-            data = list(data)
-        verify_tuple_integrity(data)
-
-        def construct_query(tuples) -> str:
-            def construct_values_list(row, start_param_idx):
-                parameter_count = len(row)
-                parameters = [f'${x+start_param_idx}' for x in range(parameter_count)]
-                parameters = '(' + ', '.join(parameters) + ')'
-                return parameters
-
-            row_size = len(tuples[0])
-            values_list = [construct_values_list(x, 1 + (i * row_size)) for i, x in enumerate(tuples)]
-            values_list = ', '.join(values_list)
-
-            query = f"""
-                select * from (values {values_list})
-            """
-            return query
-
-        query = construct_query(data)
-
-        def construct_parameters(tuples):
-            parameters = []
-            for row in tuples:
-                parameters.extend(list(row))
-            return parameters
-
-        parameters = construct_parameters(data)
-
-        rel = self.conn.sql(query, params=parameters)
-        return DataFrame(rel, self)
-
-    def createDataFrame(
-        self,
-        data: Union["PandasDataFrame", Iterable[Any]],
-        schema: Optional[Union[StructType, List[str]]] = None,
-        samplingRatio: Optional[float] = None,
-        verifySchema: bool = True,
-    ) -> DataFrame:
-        if samplingRatio:
-            raise NotImplementedError
-        if not verifySchema:
-            raise NotImplementedError
-        types = None
-        names = None
-        if schema:
-            if isinstance(schema, StructType):
-                types, names = schema.extract_types_and_names()
-            else:
-                names = schema
-
-        try:
-            import pandas
-
-            has_pandas = True
-        except:
-            has_pandas = False
-        # Falsey check on pandas dataframe is not defined, so first check if it's not a pandas dataframe
-        # Then check if 'data' is None or []
-        # Finally check if a schema was provided
-        is_empty = False
-        if (not has_pandas or (has_pandas and not isinstance(data, pandas.DataFrame))) and not data and names:
-            # Create NULLs for every type in our the dataframe
-            is_empty = True
-            data = [tuple(None for _ in names)]
-
-        if schema and isinstance(schema, StructType):
-            # Transform the data into Values to combine the data+schema
-            data = _combine_data_and_schema(data, schema)
-
-        df = self._create_dataframe(data)
-        if is_empty:
-            rel = df.relation
-            # Add impossible where clause
-            rel = rel.filter('1=0')
-            df = DataFrame(rel, self)
-
-        # Cast to types
-        if types:
-            df = df._cast_types(*types)
-        # Alias to names
-        if names:
-            df = df.toDF(*names)
-        return df
-
-    def newSession(self) -> "SparkSession":
-        return SparkSession(self._context)
-
-    def range(
-        self, start: int, end: Optional[int] = None, step: int = 1, numPartitions: Optional[int] = None
-    ) -> "DataFrame":
-        raise ContributionsAcceptedError
-
-    def sql(self, sqlQuery: str, **kwargs: Any) -> DataFrame:
-        if kwargs:
-            raise NotImplementedError
-        relation = self.conn.sql(sqlQuery)
-        return DataFrame(relation, self)
-
-    def stop(self) -> None:
-        self._context.stop()
-
-    def table(self, tableName: str) -> DataFrame:
-        relation = self.conn.table(tableName)
-        return DataFrame(relation, self)
-
-    def getActiveSession(self) -> "SparkSession":
-        return self
-
-    @property
-    def catalog(self) -> "Catalog":
-        if not hasattr(self, "_catalog"):
-            from pyduckdb.spark.sql.catalog import Catalog
-
-            self._catalog = Catalog(self)
-        return self._catalog
-
-    @property
-    def conf(self) -> RuntimeConfig:
-        return self._conf
-
-    @property
-    def read(self) -> DataFrameReader:
-        return DataFrameReader(self)
-
-    @property
-    def readStream(self) -> DataStreamReader:
-        return DataStreamReader(self)
-
-    @property
-    def sparkContext(self) -> SparkContext:
-        return self._context
-
-    @property
-    def streams(self) -> Any:
-        raise ContributionsAcceptedError
-
-    @property
-    def udf(self) -> UDFRegistration:
-        return UDFRegistration()
-
-    @property
-    def version(self) -> str:
-        return '1.0.0'
-
-    class Builder:
-        def __init__(self):
-            self.name = "builder"
-            self._master = ':memory:'
-            self._config = {}
-
-        def master(self, name: str) -> "SparkSession.Builder":
-            self._master = name
-            return self
-
-        def appName(self, name: str) -> "SparkSession.Builder":
-            # no-op
-            return self
-
-        def remote(self, url: str) -> "SparkSession.Builder":
-            # no-op
-            return self
-
-        def getOrCreate(self) -> "SparkSession":
-            # TODO: use the config to pass in methods to 'connect'
-            context = SparkContext(self._master)
-            return SparkSession(context)
-
-        def config(
-            self, key: Optional[str] = None, value: Optional[Any] = None, conf: Optional[SparkConf] = None
-        ) -> "SparkSession.Builder":
-            if conf:
-                raise NotImplementedError
-            if key and value:
-                self._config[key] = value
-            return self
-
-        def enableHiveSupport(self) -> "SparkSession.Builder":
-            # no-op
-            return self
-
-    builder = Builder()
-
-
-__all__ = ["SparkSession"]

pyduckdb/spark/sql/streaming.py

@@ -1,37 +0,0 @@
-from typing import TYPE_CHECKING, Optional, Union
-from pyduckdb.spark.sql.types import StructType
-
-if TYPE_CHECKING:
-    from pyduckdb.spark.sql.dataframe import DataFrame
-    from pyduckdb.spark.sql.session import SparkSession
-
-PrimitiveType = Union[bool, float, int, str]
-OptionalPrimitiveType = Optional[PrimitiveType]
-
-
-class DataStreamWriter:
-    def __init__(self, dataframe: "DataFrame"):
-        self.dataframe = dataframe
-
-    def toTable(self, table_name: str) -> None:
-        # Should we register the dataframe or create a table from the contents?
-        raise NotImplementedError
-
-
-class DataStreamReader:
-    def __init__(self, session: "SparkSession"):
-        self.session = session
-
-    def load(
-        self,
-        path: Optional[str] = None,
-        format: Optional[str] = None,
-        schema: Union[StructType, str, None] = None,
-        **options: OptionalPrimitiveType
-    ) -> "DataFrame":
-        from pyduckdb.spark.sql.dataframe import DataFrame
-
-        raise NotImplementedError
-
-
-__all__ = ["DataStreamReader", "DataStreamWriter"]

pyduckdb/spark/sql/type_utils.py

@@ -1,104 +0,0 @@
-import typing
-from duckdb.typing import DuckDBPyType
-from typing import List, Tuple, cast
-from .types import (
-    DataType,
-    StringType,
-    BinaryType,
-    BitstringType,
-    UUIDType,
-    BooleanType,
-    DateType,
-    TimestampType,
-    TimestampNTZType,
-    TimeType,
-    TimeNTZType,
-    TimestampNanosecondNTZType,
-    TimestampMilisecondNTZType,
-    TimestampSecondNTZType,
-    DecimalType,
-    DoubleType,
-    FloatType,
-    ByteType,
-    UnsignedByteType,
-    ShortType,
-    UnsignedShortType,
-    IntegerType,
-    UnsignedIntegerType,
-    LongType,
-    UnsignedLongType,
-    HugeIntegerType,
-    DayTimeIntervalType,
-    ArrayType,
-    MapType,
-    StructField,
-    StructType,
-)
-
-_sqltype_to_spark_class = {
-    'boolean': BooleanType,
-    'utinyint': UnsignedByteType,
-    'tinyint': ByteType,
-    'usmallint': UnsignedShortType,
-    'smallint': ShortType,
-    'uinteger': UnsignedIntegerType,
-    'integer': IntegerType,
-    'ubigint': UnsignedLongType,
-    'bigint': LongType,
-    'hugeint': HugeIntegerType,
-    'varchar': StringType,
-    'blob': BinaryType,
-    'bit': BitstringType,
-    'uuid': UUIDType,
-    'date': DateType,
-    'time': TimeNTZType,
-    'time with time zone': TimeType,
-    'timestamp': TimestampNTZType,
-    'timestamp with time zone': TimestampType,
-    'timestamp_ms': TimestampNanosecondNTZType,
-    'timestamp_ns': TimestampMilisecondNTZType,
-    'timestamp_s': TimestampSecondNTZType,
-    'interval': DayTimeIntervalType,
-    'list': ArrayType,
-    'struct': StructType,
-    'map': MapType,
-    # union
-    # enum
-    # null (???)
-    'float': FloatType,
-    'double': DoubleType,
-    'decimal': DecimalType,
-}
-
-
-def convert_nested_type(dtype: DuckDBPyType) -> DataType:
-    id = dtype.id
-    if id == 'list':
-        children = dtype.children
-        return ArrayType(convert_type(children[0][1]))
-    # TODO: add support for 'union'
-    if id == 'struct':
-        children: List[Tuple[str, DuckDBPyType]] = dtype.children
-        fields = [StructField(x[0], convert_type(x[1])) for x in children]
-        return StructType(fields)
-    if id == 'map':
-        return MapType(convert_type(dtype.key), convert_type(dtype.value))
-    raise NotImplementedError
-
-
-def convert_type(dtype: DuckDBPyType) -> DataType:
-    id = dtype.id
-    if id in ['list', 'struct', 'map']:
-        return convert_nested_type(dtype)
-    if id == 'decimal':
-        children: List[Tuple[str, DuckDBPyType]] = dtype.children
-        precision = cast(int, children[0][1])
-        scale = cast(int, children[1][1])
-        return DecimalType(precision, scale)
-    spark_type = _sqltype_to_spark_class[id]
-    return spark_type()
-
-
-def duckdb_to_spark_schema(names: List[str], types: List[DuckDBPyType]) -> StructType:
-    fields = [StructField(name, dtype) for name, dtype in zip(names, [convert_type(x) for x in types])]
-    return StructType(fields)

pyduckdb/spark/sql/udf.py

@@ -1,9 +0,0 @@
-# https://sparkbyexamples.com/pyspark/pyspark-udf-user-defined-function/
-
-
-class UDFRegistration:
-    def __init__(self):
-        raise NotImplementedError
-
-
-__all__ = ["UDFRegistration"]