fugue 0.9.7.dev1__tar.gz → 0.9.8__tar.gz

{fugue-0.9.7.dev1 → fugue-0.9.8}/PKG-INFO

@@ -1,35 +1,34 @@
 Metadata-Version: 2.4
 Name: fugue
-Version: 0.9.7.dev1
+Version: 0.9.8
 Summary: An abstraction layer for distributed computing
 Author-email: The Fugue Development Team <hello@fugue.ai>
 License: Apache-2.0
 Project-URL: Homepage, http://github.com/fugue-project/fugue
 Project-URL: Repository, http://github.com/fugue-project/fugue
-Keywords: distributed,spark,dask,ray,sql,dsl,domain specific language
+Keywords: distributed,spark,dask,ray,duckdb,sql,dsl,domain specific language
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: 3 :: Only
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: triad>=1.0.1
 Requires-Dist: adagio>=0.2.6
 Requires-Dist: pandas<3
 Provides-Extra: sql
-Requires-Dist: qpd>=0.4.4; extra == "sql"
-Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "sql"
+Requires-Dist: duckdb>=0.5.0; extra == "sql"
+Requires-Dist: fugue-sql-antlr>=0.2.4; extra == "sql"
 Requires-Dist: sqlglot<28; extra == "sql"
 Requires-Dist: jinja2; extra == "sql"
 Provides-Extra: cpp-sql-parser
-Requires-Dist: fugue-sql-antlr[cpp]>=0.2.0; extra == "cpp-sql-parser"
+Requires-Dist: fugue-sql-antlr[cpp]>=0.2.4; extra == "cpp-sql-parser"
 Provides-Extra: spark
 Requires-Dist: pyspark>=3.1.1; extra == "spark"
 Requires-Dist: zstandard>=0.25.0; extra == "spark"
@@ -43,8 +42,7 @@ Requires-Dist: duckdb>=0.5.0; extra == "ray"
 Requires-Dist: pyarrow>=7.0.0; extra == "ray"
 Requires-Dist: pandas; extra == "ray"
 Provides-Extra: duckdb
-Requires-Dist: qpd>=0.4.4; extra == "duckdb"
-Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "duckdb"
+Requires-Dist: fugue-sql-antlr>=0.2.4; extra == "duckdb"
 Requires-Dist: sqlglot<28; extra == "duckdb"
 Requires-Dist: jinja2; extra == "duckdb"
 Requires-Dist: duckdb>=0.5.0; extra == "duckdb"
@@ -52,8 +50,7 @@ Requires-Dist: numpy; extra == "duckdb"
 Provides-Extra: polars
 Requires-Dist: polars; extra == "polars"
 Provides-Extra: ibis
-Requires-Dist: qpd>=0.4.4; extra == "ibis"
-Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "ibis"
+Requires-Dist: fugue-sql-antlr>=0.2.4; extra == "ibis"
 Requires-Dist: sqlglot<28; extra == "ibis"
 Requires-Dist: jinja2; extra == "ibis"
 Requires-Dist: ibis-framework[pandas]; extra == "ibis"
@@ -62,8 +59,7 @@ Requires-Dist: notebook; extra == "notebook"
 Requires-Dist: jupyterlab; extra == "notebook"
 Requires-Dist: ipython>=7.10.0; extra == "notebook"
 Provides-Extra: all
-Requires-Dist: qpd>=0.4.4; extra == "all"
-Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "all"
+Requires-Dist: fugue-sql-antlr>=0.2.4; extra == "all"
 Requires-Dist: sqlglot<28; extra == "all"
 Requires-Dist: jinja2; extra == "all"
 Requires-Dist: pyspark>=3.1.1; extra == "all"
@@ -85,7 +81,7 @@ Dynamic: license-file
 [![PyPI version](https://badge.fury.io/py/fugue.svg)](https://pypi.python.org/pypi/fugue/)
 [![PyPI pyversions](https://img.shields.io/pypi/pyversions/fugue.svg)](https://pypi.python.org/pypi/fugue/)
 [![PyPI license](https://img.shields.io/pypi/l/fugue.svg)](https://pypi.python.org/pypi/fugue/)
-[![codecov](https://codecov.io/gh/fugue-project/fugue/branch/main/graph/badge.svg?token=ZO9YD5N3IA)](https://codecov.io/gh/fugue-project/fugue)
+[![codecov](https://codecov.io/gh/fugue-project/fugue/graph/badge.svg?token=ZO9YD5N3IA)](https://codecov.io/gh/fugue-project/fugue)
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/4fa5f2f53e6f48aaa1218a89f4808b91)](https://www.codacy.com/gh/fugue-project/fugue/dashboard?utm_source=github.com&utm_medium=referral&utm_content=fugue-project/fugue&utm_campaign=Badge_Grade)
 [![Downloads](https://static.pepy.tech/badge/fugue)](https://pepy.tech/project/fugue)
 

{fugue-0.9.7.dev1 → fugue-0.9.8}/README.md

@@ -3,7 +3,7 @@
 [![PyPI version](https://badge.fury.io/py/fugue.svg)](https://pypi.python.org/pypi/fugue/)
 [![PyPI pyversions](https://img.shields.io/pypi/pyversions/fugue.svg)](https://pypi.python.org/pypi/fugue/)
 [![PyPI license](https://img.shields.io/pypi/l/fugue.svg)](https://pypi.python.org/pypi/fugue/)
-[![codecov](https://codecov.io/gh/fugue-project/fugue/branch/main/graph/badge.svg?token=ZO9YD5N3IA)](https://codecov.io/gh/fugue-project/fugue)
+[![codecov](https://codecov.io/gh/fugue-project/fugue/graph/badge.svg?token=ZO9YD5N3IA)](https://codecov.io/gh/fugue-project/fugue)
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/4fa5f2f53e6f48aaa1218a89f4808b91)](https://www.codacy.com/gh/fugue-project/fugue/dashboard?utm_source=github.com&utm_medium=referral&utm_content=fugue-project/fugue&utm_campaign=Badge_Grade)
 [![Downloads](https://static.pepy.tech/badge/fugue)](https://pepy.tech/project/fugue)
 

{fugue-0.9.7.dev1 → fugue-0.9.8}/fugue/__init__.py

@@ -55,7 +55,7 @@ from fugue.execution.factory import (
 from fugue.execution.native_execution_engine import (
     NativeExecutionEngine,
     PandasMapEngine,
-    QPDPandasEngine,
+    DefaultSQLEngine,
 )
 from fugue.extensions.creator import Creator, creator, register_creator
 from fugue.extensions.outputter import Outputter, outputter, register_outputter

{fugue-0.9.7.dev1 → fugue-0.9.8}/fugue/_utils/misc.py

@@ -23,6 +23,4 @@ def import_or_throw(package_name: str, message: str) -> Any:
 
 
 def import_fsql_dependency(package_name: str) -> Any:
-    return import_or_throw(
-        package_name, "Please try to install the package by `pip install fugue[sql]`."
-    )
+    return import_or_throw(package_name, "Please pip install with extra `fugue[sql]`.")

{fugue-0.9.7.dev1 → fugue-0.9.8}/fugue/constants.py

@@ -37,7 +37,7 @@ _FUGUE_GLOBAL_CONF = ParamDict(
         FUGUE_CONF_WORKFLOW_CONCURRENCY: 1,
         FUGUE_CONF_WORKFLOW_AUTO_PERSIST: False,
         FUGUE_CONF_WORKFLOW_EXCEPTION_HIDE: "fugue.,six,adagio.,pandas,"
-        "fugue_dask.,dask.,fugue_spark.,pyspark.,antlr4,_qpd_antlr,qpd,triad,"
+        "fugue_dask.,dask.,fugue_spark.,pyspark.,antlr4,triad,"
         "fugue_notebook.,ipython.,jupyter.,ipykernel,_pytest,pytest,fugue_ibis.",
         FUGUE_CONF_WORKFLOW_EXCEPTION_INJECT: 3,
         FUGUE_CONF_WORKFLOW_EXCEPTION_OPTIMIZE: True,

{fugue-0.9.7.dev1 → fugue-0.9.8}/fugue/dev.py

@@ -32,7 +32,7 @@ from fugue.execution.factory import (
     register_execution_engine,
     register_sql_engine,
 )
-from fugue.execution.native_execution_engine import PandasMapEngine, QPDPandasEngine
+from fugue.execution.native_execution_engine import PandasMapEngine, DefaultSQLEngine
 from fugue.rpc import (
     EmptyRPCHandler,
     RPCClient,

{fugue-0.9.7.dev1 → fugue-0.9.8}/fugue/execution/__init__.py

@@ -10,4 +10,4 @@ from .factory import (
     register_execution_engine,
     register_sql_engine,
 )
-from .native_execution_engine import NativeExecutionEngine, QPDPandasEngine
+from .native_execution_engine import NativeExecutionEngine, DefaultSQLEngine

{fugue-0.9.7.dev1 → fugue-0.9.8}/fugue/execution/native_execution_engine.py

@@ -11,7 +11,6 @@ from triad.utils.io import makedirs
 from triad.utils.pandas_like import PandasUtils
 
 from fugue._utils.io import load_df, save_df
-from fugue._utils.misc import import_fsql_dependency
 from fugue.collections.partition import (
     PartitionCursor,
     PartitionSpec,
@@ -21,6 +20,7 @@ from fugue.collections.sql import StructuredRawSQL
 from fugue.constants import KEYWORD_PARALLELISM, KEYWORD_ROWCOUNT
 from fugue.dataframe import (
     AnyDataFrame,
+    ArrowDataFrame,
     DataFrame,
     DataFrames,
     LocalBoundedDataFrame,
@@ -39,31 +39,53 @@ from .execution_engine import (
 )
 
 
-class QPDPandasEngine(SQLEngine):
-    """QPD execution implementation.
+class DefaultSQLEngine(SQLEngine):
+    """DuckDB SQL backend implementation.
 
     :param execution_engine: the execution engine this sql engine will run on
     """
 
     @property
     def dialect(self) -> Optional[str]:
-        return "spark"
-
-    def to_df(self, df: AnyDataFrame, schema: Any = None) -> DataFrame:
-        return _to_native_execution_engine_df(df, schema)
-
-    @property
-    def is_distributed(self) -> bool:
-        return False
+        return "duckdb"
 
     def select(self, dfs: DataFrames, statement: StructuredRawSQL) -> DataFrame:
-        qpd_pandas = import_fsql_dependency("qpd_pandas")
+        import duckdb
+
+        from fugue_duckdb.dataframe import _duck_as_arrow
 
         _dfs, _sql = self.encode(dfs, statement)
-        _dd = {k: self.to_df(v).as_pandas() for k, v in _dfs.items()}  # type: ignore
 
-        df = qpd_pandas.run_sql_on_pandas(_sql, _dd, ignore_case=True)
-        return self.to_df(df)
+        with duckdb.connect() as conn:
+            for k, v in _dfs.items():
+                duckdb.from_arrow(v.as_arrow(), connection=conn).create_view(k)
+            return ArrowDataFrame(_duck_as_arrow(conn.sql(_sql)))
+
+    def table_exists(self, table: str) -> bool:  # pragma: no cover
+        raise NotImplementedError(
+            "table_exists can only be used with DuckExecutionEngine"
+        )
+
+    def save_table(
+        self,
+        df: DataFrame,
+        table: str,
+        mode: str = "overwrite",
+        partition_spec: Optional[PartitionSpec] = None,
+        **kwargs: Any,
+    ) -> None:  # pragma: no cover
+        raise NotImplementedError(
+            "save_table can only be used with DuckExecutionEngine"
+        )
+
+    def load_table(self, table: str, **kwargs: Any) -> DataFrame:  # pragma: no cover
+        raise NotImplementedError(
+            "load_table can only be used with DuckExecutionEngine"
+        )
+
+    @property
+    def is_distributed(self) -> bool:
+        return False
 
 
 class PandasMapEngine(MapEngine):
@@ -148,8 +170,9 @@ class PandasMapEngine(MapEngine):
                 output_df = PandasDataFrame(output_df.native, output_schema)
             assert_or_throw(
                 output_df.schema == output_schema,
-                lambda: f"map output {output_df.schema} "
-                f"mismatches given {output_schema}",
+                lambda: (
+                    f"map output {output_df.schema} mismatches given {output_schema}"
+                ),
             )
             return self.to_df(output_df)  # type: ignore
 
@@ -194,7 +217,7 @@ class NativeExecutionEngine(ExecutionEngine):
         return False
 
     def create_default_sql_engine(self) -> SQLEngine:
-        return QPDPandasEngine(self)
+        return DefaultSQLEngine(self)
 
     def create_default_map_engine(self) -> MapEngine:
         return PandasMapEngine(self)

{fugue-0.9.7.dev1 → fugue-0.9.8}/fugue/registry.py

@@ -1,7 +1,7 @@
 from fugue.execution.factory import register_execution_engine, register_sql_engine
 from fugue.execution.native_execution_engine import (
+    DefaultSQLEngine,
     NativeExecutionEngine,
-    QPDPandasEngine,
 )
 
 
@@ -25,8 +25,5 @@ def _register_engines() -> None:
         "pandas", lambda conf: NativeExecutionEngine(conf), on_dup="ignore"
     )
     register_sql_engine(
-        "qpdpandas", lambda engine: QPDPandasEngine(engine), on_dup="ignore"
-    )
-    register_sql_engine(
-        "qpd_pandas", lambda engine: QPDPandasEngine(engine), on_dup="ignore"
+        "default", lambda engine: DefaultSQLEngine(engine), on_dup="ignore"
     )

{fugue-0.9.7.dev1 → fugue-0.9.8}/fugue.egg-info/PKG-INFO

@@ -1,35 +1,34 @@
 Metadata-Version: 2.4
 Name: fugue
-Version: 0.9.7.dev1
+Version: 0.9.8
 Summary: An abstraction layer for distributed computing
 Author-email: The Fugue Development Team <hello@fugue.ai>
 License: Apache-2.0
 Project-URL: Homepage, http://github.com/fugue-project/fugue
 Project-URL: Repository, http://github.com/fugue-project/fugue
-Keywords: distributed,spark,dask,ray,sql,dsl,domain specific language
+Keywords: distributed,spark,dask,ray,duckdb,sql,dsl,domain specific language
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: 3 :: Only
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: triad>=1.0.1
 Requires-Dist: adagio>=0.2.6
 Requires-Dist: pandas<3
 Provides-Extra: sql
-Requires-Dist: qpd>=0.4.4; extra == "sql"
-Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "sql"
+Requires-Dist: duckdb>=0.5.0; extra == "sql"
+Requires-Dist: fugue-sql-antlr>=0.2.4; extra == "sql"
 Requires-Dist: sqlglot<28; extra == "sql"
 Requires-Dist: jinja2; extra == "sql"
 Provides-Extra: cpp-sql-parser
-Requires-Dist: fugue-sql-antlr[cpp]>=0.2.0; extra == "cpp-sql-parser"
+Requires-Dist: fugue-sql-antlr[cpp]>=0.2.4; extra == "cpp-sql-parser"
 Provides-Extra: spark
 Requires-Dist: pyspark>=3.1.1; extra == "spark"
 Requires-Dist: zstandard>=0.25.0; extra == "spark"
@@ -43,8 +42,7 @@ Requires-Dist: duckdb>=0.5.0; extra == "ray"
 Requires-Dist: pyarrow>=7.0.0; extra == "ray"
 Requires-Dist: pandas; extra == "ray"
 Provides-Extra: duckdb
-Requires-Dist: qpd>=0.4.4; extra == "duckdb"
-Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "duckdb"
+Requires-Dist: fugue-sql-antlr>=0.2.4; extra == "duckdb"
 Requires-Dist: sqlglot<28; extra == "duckdb"
 Requires-Dist: jinja2; extra == "duckdb"
 Requires-Dist: duckdb>=0.5.0; extra == "duckdb"
@@ -52,8 +50,7 @@ Requires-Dist: numpy; extra == "duckdb"
 Provides-Extra: polars
 Requires-Dist: polars; extra == "polars"
 Provides-Extra: ibis
-Requires-Dist: qpd>=0.4.4; extra == "ibis"
-Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "ibis"
+Requires-Dist: fugue-sql-antlr>=0.2.4; extra == "ibis"
 Requires-Dist: sqlglot<28; extra == "ibis"
 Requires-Dist: jinja2; extra == "ibis"
 Requires-Dist: ibis-framework[pandas]; extra == "ibis"
@@ -62,8 +59,7 @@ Requires-Dist: notebook; extra == "notebook"
 Requires-Dist: jupyterlab; extra == "notebook"
 Requires-Dist: ipython>=7.10.0; extra == "notebook"
 Provides-Extra: all
-Requires-Dist: qpd>=0.4.4; extra == "all"
-Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "all"
+Requires-Dist: fugue-sql-antlr>=0.2.4; extra == "all"
 Requires-Dist: sqlglot<28; extra == "all"
 Requires-Dist: jinja2; extra == "all"
 Requires-Dist: pyspark>=3.1.1; extra == "all"
@@ -85,7 +81,7 @@ Dynamic: license-file
 [![PyPI version](https://badge.fury.io/py/fugue.svg)](https://pypi.python.org/pypi/fugue/)
 [![PyPI pyversions](https://img.shields.io/pypi/pyversions/fugue.svg)](https://pypi.python.org/pypi/fugue/)
 [![PyPI license](https://img.shields.io/pypi/l/fugue.svg)](https://pypi.python.org/pypi/fugue/)
-[![codecov](https://codecov.io/gh/fugue-project/fugue/branch/main/graph/badge.svg?token=ZO9YD5N3IA)](https://codecov.io/gh/fugue-project/fugue)
+[![codecov](https://codecov.io/gh/fugue-project/fugue/graph/badge.svg?token=ZO9YD5N3IA)](https://codecov.io/gh/fugue-project/fugue)
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/4fa5f2f53e6f48aaa1218a89f4808b91)](https://www.codacy.com/gh/fugue-project/fugue/dashboard?utm_source=github.com&utm_medium=referral&utm_content=fugue-project/fugue&utm_campaign=Badge_Grade)
 [![Downloads](https://static.pepy.tech/badge/fugue)](https://pepy.tech/project/fugue)
 

{fugue-0.9.7.dev1 → fugue-0.9.8}/fugue.egg-info/requires.txt

@@ -3,8 +3,7 @@ adagio>=0.2.6
 pandas<3
 
 [all]
-qpd>=0.4.4
-fugue-sql-antlr>=0.2.0
+fugue-sql-antlr>=0.2.4
 sqlglot<28
 jinja2
 pyspark>=3.1.1
@@ -23,7 +22,7 @@ polars
 ray[data]>=2.30.0
 
 [cpp_sql_parser]
-fugue-sql-antlr[cpp]>=0.2.0
+fugue-sql-antlr[cpp]>=0.2.4
 
 [dask]
 dask[dataframe,distributed]>=2024.4.0
@@ -31,16 +30,14 @@ pyarrow>=7.0.0
 pandas>=2.0.2
 
 [duckdb]
-qpd>=0.4.4
-fugue-sql-antlr>=0.2.0
+fugue-sql-antlr>=0.2.4
 sqlglot<28
 jinja2
 duckdb>=0.5.0
 numpy
 
 [ibis]
-qpd>=0.4.4
-fugue-sql-antlr>=0.2.0
+fugue-sql-antlr>=0.2.4
 sqlglot<28
 jinja2
 ibis-framework[pandas]
@@ -66,7 +63,7 @@ pyspark>=3.1.1
 zstandard>=0.25.0
 
 [sql]
-qpd>=0.4.4
-fugue-sql-antlr>=0.2.0
+duckdb>=0.5.0
+fugue-sql-antlr>=0.2.4
 sqlglot<28
 jinja2

{fugue-0.9.7.dev1 → fugue-0.9.8}/fugue_ray/_utils/io.py

@@ -1,17 +1,17 @@
 import os
 import pathlib
-from typing import Any, Callable, Dict, Iterable, List, Optional, Union
+from collections.abc import Callable, Iterable
+from typing import Any
 
 import pyarrow as pa
 import ray.data as rd
 from packaging import version
 from pyarrow import csv as pacsv
 from pyarrow import json as pajson
-
 from triad.collections import Schema
 from triad.collections.dict import ParamDict
 from triad.utils.assertion import assert_or_throw
-from triad.utils.io import exists, makedirs, rm, isfile
+from triad.utils.io import exists, isfile, makedirs, rm
 
 from fugue import ExecutionEngine
 from fugue._utils.io import FileParser, save_df
@@ -25,9 +25,7 @@ try:
     from ray.data.datasource import FileExtensionFilter
 
     class _FileFiler(FileExtensionFilter):  # pragma: no cover
-        def __init__(
-            self, file_extensions: Union[str, List[str]], exclude: Iterable[str]
-        ):
+        def __init__(self, file_extensions: str | list[str], exclude: Iterable[str]):
             super().__init__(file_extensions, allow_if_no_extension=True)
             self._exclude = set(exclude)
 
@@ -36,23 +34,23 @@ try:
                 path
             ).name not in self._exclude and self._file_has_extension(path)
 
-        def __call__(self, paths: List[str]) -> List[str]:
+        def __call__(self, paths: list[str]) -> list[str]:
             return [path for path in paths if self._is_valid(path)]
 
 except ImportError:  # pragma: no cover
     pass  # ray >=2.10
 
 
-class RayIO(object):
+class RayIO:
     def __init__(self, engine: ExecutionEngine):
         self._engine = engine
         self._logger = engine.log
-        self._loads: Dict[str, Callable[..., DataFrame]] = {
+        self._loads: dict[str, Callable[..., DataFrame]] = {
             "csv": self._load_csv,
             "parquet": self._load_parquet,
             "json": self._load_json,
         }
-        self._saves: Dict[str, Callable[..., None]] = {
+        self._saves: dict[str, Callable[..., None]] = {
             "csv": self._save_csv,
             "parquet": self._save_parquet,
             "json": self._save_json,
@@ -60,8 +58,8 @@ class RayIO(object):
 
     def load_df(
         self,
-        uri: Union[str, List[str]],
-        format_hint: Optional[str] = None,
+        uri: str | list[str],
+        format_hint: str | None = None,
         columns: Any = None,
         **kwargs: Any,
     ) -> DataFrame:
@@ -81,8 +79,8 @@ class RayIO(object):
         self,
         df: RayDataFrame,
         uri: str,
-        format_hint: Optional[str] = None,
-        partition_spec: Optional[PartitionSpec] = None,
+        format_hint: str | None = None,
+        partition_spec: PartitionSpec | None = None,
         mode: str = "overwrite",
         force_single: bool = False,
         **kwargs: Any,
@@ -122,7 +120,7 @@ class RayIO(object):
         if "header" in kw:
             kw["include_header"] = kw.pop("header")
 
-        def _fn() -> Dict[str, Any]:  # pragma: no cover
+        def _fn() -> dict[str, Any]:  # pragma: no cover
             return dict(write_options=pacsv.WriteOptions(**kw))
 
         df.native.write_csv(
@@ -152,13 +150,25 @@ class RayIO(object):
         )
 
     def _load_parquet(
-        self, p: List[str], columns: Any = None, **kwargs: Any
+        self, p: list[str], columns: Any = None, **kwargs: Any
     ) -> DataFrame:
         # in 2.52.0 the default changes to ["parquet"]
         if "file_extensions" not in kwargs:
             kwargs = kwargs.copy()
             kwargs["file_extensions"] = None
-        sdf = rd.read_parquet(p, ray_remote_args=self._remote_args(), **kwargs)
+        # ray>=2.60 routes read_parquet to the V2 datasource by default, which
+        # (1) forces file_extensions=None to ["parquet"], so single files without
+        # extension can't be read, and (2) may raise ImportError from a threaded
+        # fsspec HTTPFileSystem import when listing multiple paths. Use V1.
+        ctx = rd.DataContext.get_current()
+        use_v2 = getattr(ctx, "use_datasource_v2", None)
+        if use_v2:  # pragma: no cover
+            ctx.use_datasource_v2 = False
+        try:
+            sdf = rd.read_parquet(p, ray_remote_args=self._remote_args(), **kwargs)
+        finally:
+            if use_v2:  # pragma: no cover
+                ctx.use_datasource_v2 = use_v2
         if columns is None:
             return RayDataFrame(sdf)
         if isinstance(columns, list):  # column names
@@ -166,20 +176,18 @@ class RayIO(object):
         schema = Schema(columns)
         return RayDataFrame(sdf)[schema.names].alter_columns(schema)
 
-    def _load_csv(  # noqa: C901
-        self, p: List[str], columns: Any = None, **kwargs: Any
-    ) -> DataFrame:
+    def _load_csv(self, p: list[str], columns: Any = None, **kwargs: Any) -> DataFrame:
         kw = ParamDict(kwargs)
         infer_schema = kw.get("infer_schema", False)
-        read_options: Dict[str, Any] = {"use_threads": False}
-        parse_options: Dict[str, Any] = {}
-        convert_options: Dict[str, Any] = {}
+        read_options: dict[str, Any] = {"use_threads": False}
+        parse_options: dict[str, Any] = {}
+        convert_options: dict[str, Any] = {}
         if infer_schema and columns is not None and not isinstance(columns, list):
             raise ValueError("can't set columns as a schema when infer schema is true")
 
         if RAY_VERSION >= version.parse("2.10"):
             if len(p) == 1 and isfile(p[0]):  # TODO: very hacky
-                params: Dict[str, Any] = {}
+                params: dict[str, Any] = {}
             else:
                 params = {"file_extensions": ["csv"]}
         else:  # pragma: no cover
@@ -230,14 +238,14 @@ class RayIO(object):
         else:  # pragma: no cover
             raise NotImplementedError(f"{header} is not supported")
 
-    def _load_json(self, p: List[str], columns: Any = None, **kwargs: Any) -> DataFrame:
-        read_options: Dict[str, Any] = {"use_threads": False}
-        parse_options: Dict[str, Any] = {}
+    def _load_json(self, p: list[str], columns: Any = None, **kwargs: Any) -> DataFrame:
+        read_options: dict[str, Any] = {"use_threads": False}
+        parse_options: dict[str, Any] = {}
 
         def _read_json() -> RayDataFrame:  # pragma: no cover
             if RAY_VERSION >= version.parse("2.10"):
                 if len(p) == 1 and isfile(p[0]):  # TODO: very hacky
-                    params: Dict[str, Any] = {"file_extensions": None}
+                    params: dict[str, Any] = {"file_extensions": None}
                 else:
                     params = {"file_extensions": ["json"]}
             elif RAY_VERSION >= version.parse("2.9"):  # pragma: no cover
@@ -272,5 +280,5 @@ class RayIO(object):
             schema = Schema(columns)
             return _read_json()[schema.names].alter_columns(schema)
 
-    def _remote_args(self) -> Dict[str, Any]:
+    def _remote_args(self) -> dict[str, Any]:
         return {"num_cpus": 1}

{fugue-0.9.7.dev1 → fugue-0.9.8}/fugue_spark/_utils/misc.py

@@ -1,25 +1,23 @@
 import pyspark.sql as ps
-import pyspark
 from typing import Any
 
 
-SPARK_VERSION = pyspark.__version__.split(".")
+try:
+    from pyspark.sql import is_remote as _is_remote
+except ImportError:  # pragma: no cover
+    _is_remote = None
 
 try:
-    if int(SPARK_VERSION[0]) >= 4:
-        from pyspark.sql import SparkSession as SparkConnectSession
-        from pyspark.sql import DataFrame as SparkConnectDataFrame
-    else:  # pragma: no cover
-        from pyspark.sql.connect.session import SparkSession as SparkConnectSession
-        from pyspark.sql.connect.dataframe import DataFrame as SparkConnectDataFrame
+    from pyspark.sql.connect.session import SparkSession as SparkConnectSession
+    from pyspark.sql.connect.dataframe import DataFrame as SparkConnectDataFrame
 except Exception:  # pragma: no cover
     SparkConnectSession = None
     SparkConnectDataFrame = None
 
 
 def is_spark_connect(session: Any) -> bool:  # pragma: no cover
-    if int(SPARK_VERSION[0]) >= 4:
-        return False
+    if _is_remote is not None:
+        return _is_remote()
     return SparkConnectSession is not None and isinstance(
         session, (SparkConnectSession, SparkConnectDataFrame)
     )

{fugue-0.9.7.dev1 → fugue-0.9.8}/fugue_spark/execution_engine.py

@@ -403,10 +403,17 @@ class SparkExecutionEngine(ExecutionEngine):
     def get_current_parallelism(self) -> int:
         spark = self.spark_session
         if self.is_spark_connect:  # pragma: no cover
-            num = spark.conf.get("spark.default.parallelism", "")
-            if num != "":
-                return int(num)
-            return int(spark.conf.get("spark.sql.shuffle.partitions", "200"))
+            for key in (
+                "spark.default.parallelism",
+                "spark.sql.shuffle.partitions",
+            ):
+                try:
+                    value = spark.conf.get(key, "")
+                except Exception:
+                    continue
+                if value != "":
+                    return int(value)
+            return 200
         e_cores = int(spark.conf.get("spark.executor.cores", "1"))
         tc = int(spark.conf.get("spark.task.cpus", "1"))
         sc = spark._jsc.sc()