fugue 0.9.7.dev2__tar.gz → 0.9.8__tar.gz

{fugue-0.9.7.dev2 → fugue-0.9.8}/PKG-INFO

@@ -1,23 +1,22 @@
 Metadata-Version: 2.4
 Name: fugue
-Version: 0.9.7.dev2
+Version: 0.9.8
 Summary: An abstraction layer for distributed computing
 Author-email: The Fugue Development Team <hello@fugue.ai>
 License: Apache-2.0
 Project-URL: Homepage, http://github.com/fugue-project/fugue
 Project-URL: Repository, http://github.com/fugue-project/fugue
-Keywords: distributed,spark,dask,ray,sql,dsl,domain specific language
+Keywords: distributed,spark,dask,ray,duckdb,sql,dsl,domain specific language
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: 3 :: Only
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: triad>=1.0.1
@@ -82,7 +81,7 @@ Dynamic: license-file
 [![PyPI version](https://badge.fury.io/py/fugue.svg)](https://pypi.python.org/pypi/fugue/)
 [![PyPI pyversions](https://img.shields.io/pypi/pyversions/fugue.svg)](https://pypi.python.org/pypi/fugue/)
 [![PyPI license](https://img.shields.io/pypi/l/fugue.svg)](https://pypi.python.org/pypi/fugue/)
-[![codecov](https://codecov.io/gh/fugue-project/fugue/branch/main/graph/badge.svg?token=ZO9YD5N3IA)](https://codecov.io/gh/fugue-project/fugue)
+[![codecov](https://codecov.io/gh/fugue-project/fugue/graph/badge.svg?token=ZO9YD5N3IA)](https://codecov.io/gh/fugue-project/fugue)
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/4fa5f2f53e6f48aaa1218a89f4808b91)](https://www.codacy.com/gh/fugue-project/fugue/dashboard?utm_source=github.com&utm_medium=referral&utm_content=fugue-project/fugue&utm_campaign=Badge_Grade)
 [![Downloads](https://static.pepy.tech/badge/fugue)](https://pepy.tech/project/fugue)
 

{fugue-0.9.7.dev2 → fugue-0.9.8}/README.md

@@ -3,7 +3,7 @@
 [![PyPI version](https://badge.fury.io/py/fugue.svg)](https://pypi.python.org/pypi/fugue/)
 [![PyPI pyversions](https://img.shields.io/pypi/pyversions/fugue.svg)](https://pypi.python.org/pypi/fugue/)
 [![PyPI license](https://img.shields.io/pypi/l/fugue.svg)](https://pypi.python.org/pypi/fugue/)
-[![codecov](https://codecov.io/gh/fugue-project/fugue/branch/main/graph/badge.svg?token=ZO9YD5N3IA)](https://codecov.io/gh/fugue-project/fugue)
+[![codecov](https://codecov.io/gh/fugue-project/fugue/graph/badge.svg?token=ZO9YD5N3IA)](https://codecov.io/gh/fugue-project/fugue)
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/4fa5f2f53e6f48aaa1218a89f4808b91)](https://www.codacy.com/gh/fugue-project/fugue/dashboard?utm_source=github.com&utm_medium=referral&utm_content=fugue-project/fugue&utm_campaign=Badge_Grade)
 [![Downloads](https://static.pepy.tech/badge/fugue)](https://pepy.tech/project/fugue)
 

{fugue-0.9.7.dev2 → fugue-0.9.8}/fugue.egg-info/PKG-INFO

@@ -1,23 +1,22 @@
 Metadata-Version: 2.4
 Name: fugue
-Version: 0.9.7.dev2
+Version: 0.9.8
 Summary: An abstraction layer for distributed computing
 Author-email: The Fugue Development Team <hello@fugue.ai>
 License: Apache-2.0
 Project-URL: Homepage, http://github.com/fugue-project/fugue
 Project-URL: Repository, http://github.com/fugue-project/fugue
-Keywords: distributed,spark,dask,ray,sql,dsl,domain specific language
+Keywords: distributed,spark,dask,ray,duckdb,sql,dsl,domain specific language
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: 3 :: Only
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: triad>=1.0.1
@@ -82,7 +81,7 @@ Dynamic: license-file
 [![PyPI version](https://badge.fury.io/py/fugue.svg)](https://pypi.python.org/pypi/fugue/)
 [![PyPI pyversions](https://img.shields.io/pypi/pyversions/fugue.svg)](https://pypi.python.org/pypi/fugue/)
 [![PyPI license](https://img.shields.io/pypi/l/fugue.svg)](https://pypi.python.org/pypi/fugue/)
-[![codecov](https://codecov.io/gh/fugue-project/fugue/branch/main/graph/badge.svg?token=ZO9YD5N3IA)](https://codecov.io/gh/fugue-project/fugue)
+[![codecov](https://codecov.io/gh/fugue-project/fugue/graph/badge.svg?token=ZO9YD5N3IA)](https://codecov.io/gh/fugue-project/fugue)
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/4fa5f2f53e6f48aaa1218a89f4808b91)](https://www.codacy.com/gh/fugue-project/fugue/dashboard?utm_source=github.com&utm_medium=referral&utm_content=fugue-project/fugue&utm_campaign=Badge_Grade)
 [![Downloads](https://static.pepy.tech/badge/fugue)](https://pepy.tech/project/fugue)
 

{fugue-0.9.7.dev2 → fugue-0.9.8}/fugue_ray/_utils/io.py

@@ -1,17 +1,17 @@
 import os
 import pathlib
-from typing import Any, Callable, Dict, Iterable, List, Optional, Union
+from collections.abc import Callable, Iterable
+from typing import Any
 
 import pyarrow as pa
 import ray.data as rd
 from packaging import version
 from pyarrow import csv as pacsv
 from pyarrow import json as pajson
-
 from triad.collections import Schema
 from triad.collections.dict import ParamDict
 from triad.utils.assertion import assert_or_throw
-from triad.utils.io import exists, makedirs, rm, isfile
+from triad.utils.io import exists, isfile, makedirs, rm
 
 from fugue import ExecutionEngine
 from fugue._utils.io import FileParser, save_df
@@ -25,9 +25,7 @@ try:
     from ray.data.datasource import FileExtensionFilter
 
     class _FileFiler(FileExtensionFilter):  # pragma: no cover
-        def __init__(
-            self, file_extensions: Union[str, List[str]], exclude: Iterable[str]
-        ):
+        def __init__(self, file_extensions: str | list[str], exclude: Iterable[str]):
             super().__init__(file_extensions, allow_if_no_extension=True)
             self._exclude = set(exclude)
 
@@ -36,23 +34,23 @@ try:
                 path
             ).name not in self._exclude and self._file_has_extension(path)
 
-        def __call__(self, paths: List[str]) -> List[str]:
+        def __call__(self, paths: list[str]) -> list[str]:
             return [path for path in paths if self._is_valid(path)]
 
 except ImportError:  # pragma: no cover
     pass  # ray >=2.10
 
 
-class RayIO(object):
+class RayIO:
     def __init__(self, engine: ExecutionEngine):
         self._engine = engine
         self._logger = engine.log
-        self._loads: Dict[str, Callable[..., DataFrame]] = {
+        self._loads: dict[str, Callable[..., DataFrame]] = {
             "csv": self._load_csv,
             "parquet": self._load_parquet,
             "json": self._load_json,
         }
-        self._saves: Dict[str, Callable[..., None]] = {
+        self._saves: dict[str, Callable[..., None]] = {
             "csv": self._save_csv,
             "parquet": self._save_parquet,
             "json": self._save_json,
@@ -60,8 +58,8 @@ class RayIO(object):
 
     def load_df(
         self,
-        uri: Union[str, List[str]],
-        format_hint: Optional[str] = None,
+        uri: str | list[str],
+        format_hint: str | None = None,
         columns: Any = None,
         **kwargs: Any,
     ) -> DataFrame:
@@ -81,8 +79,8 @@ class RayIO(object):
         self,
         df: RayDataFrame,
         uri: str,
-        format_hint: Optional[str] = None,
-        partition_spec: Optional[PartitionSpec] = None,
+        format_hint: str | None = None,
+        partition_spec: PartitionSpec | None = None,
         mode: str = "overwrite",
         force_single: bool = False,
         **kwargs: Any,
@@ -122,7 +120,7 @@ class RayIO(object):
         if "header" in kw:
             kw["include_header"] = kw.pop("header")
 
-        def _fn() -> Dict[str, Any]:  # pragma: no cover
+        def _fn() -> dict[str, Any]:  # pragma: no cover
             return dict(write_options=pacsv.WriteOptions(**kw))
 
         df.native.write_csv(
@@ -152,13 +150,25 @@ class RayIO(object):
         )
 
     def _load_parquet(
-        self, p: List[str], columns: Any = None, **kwargs: Any
+        self, p: list[str], columns: Any = None, **kwargs: Any
     ) -> DataFrame:
         # in 2.52.0 the default changes to ["parquet"]
         if "file_extensions" not in kwargs:
             kwargs = kwargs.copy()
             kwargs["file_extensions"] = None
-        sdf = rd.read_parquet(p, ray_remote_args=self._remote_args(), **kwargs)
+        # ray>=2.60 routes read_parquet to the V2 datasource by default, which
+        # (1) forces file_extensions=None to ["parquet"], so single files without
+        # extension can't be read, and (2) may raise ImportError from a threaded
+        # fsspec HTTPFileSystem import when listing multiple paths. Use V1.
+        ctx = rd.DataContext.get_current()
+        use_v2 = getattr(ctx, "use_datasource_v2", None)
+        if use_v2:  # pragma: no cover
+            ctx.use_datasource_v2 = False
+        try:
+            sdf = rd.read_parquet(p, ray_remote_args=self._remote_args(), **kwargs)
+        finally:
+            if use_v2:  # pragma: no cover
+                ctx.use_datasource_v2 = use_v2
         if columns is None:
             return RayDataFrame(sdf)
         if isinstance(columns, list):  # column names
@@ -166,20 +176,18 @@ class RayIO(object):
         schema = Schema(columns)
         return RayDataFrame(sdf)[schema.names].alter_columns(schema)
 
-    def _load_csv(  # noqa: C901
-        self, p: List[str], columns: Any = None, **kwargs: Any
-    ) -> DataFrame:
+    def _load_csv(self, p: list[str], columns: Any = None, **kwargs: Any) -> DataFrame:
         kw = ParamDict(kwargs)
         infer_schema = kw.get("infer_schema", False)
-        read_options: Dict[str, Any] = {"use_threads": False}
-        parse_options: Dict[str, Any] = {}
-        convert_options: Dict[str, Any] = {}
+        read_options: dict[str, Any] = {"use_threads": False}
+        parse_options: dict[str, Any] = {}
+        convert_options: dict[str, Any] = {}
         if infer_schema and columns is not None and not isinstance(columns, list):
             raise ValueError("can't set columns as a schema when infer schema is true")
 
         if RAY_VERSION >= version.parse("2.10"):
             if len(p) == 1 and isfile(p[0]):  # TODO: very hacky
-                params: Dict[str, Any] = {}
+                params: dict[str, Any] = {}
             else:
                 params = {"file_extensions": ["csv"]}
         else:  # pragma: no cover
@@ -230,14 +238,14 @@ class RayIO(object):
         else:  # pragma: no cover
             raise NotImplementedError(f"{header} is not supported")
 
-    def _load_json(self, p: List[str], columns: Any = None, **kwargs: Any) -> DataFrame:
-        read_options: Dict[str, Any] = {"use_threads": False}
-        parse_options: Dict[str, Any] = {}
+    def _load_json(self, p: list[str], columns: Any = None, **kwargs: Any) -> DataFrame:
+        read_options: dict[str, Any] = {"use_threads": False}
+        parse_options: dict[str, Any] = {}
 
         def _read_json() -> RayDataFrame:  # pragma: no cover
             if RAY_VERSION >= version.parse("2.10"):
                 if len(p) == 1 and isfile(p[0]):  # TODO: very hacky
-                    params: Dict[str, Any] = {"file_extensions": None}
+                    params: dict[str, Any] = {"file_extensions": None}
                 else:
                     params = {"file_extensions": ["json"]}
             elif RAY_VERSION >= version.parse("2.9"):  # pragma: no cover
@@ -272,5 +280,5 @@ class RayIO(object):
             schema = Schema(columns)
             return _read_json()[schema.names].alter_columns(schema)
 
-    def _remote_args(self) -> Dict[str, Any]:
+    def _remote_args(self) -> dict[str, Any]:
         return {"num_cpus": 1}

{fugue-0.9.7.dev2 → fugue-0.9.8}/fugue_spark/_utils/misc.py

@@ -1,25 +1,23 @@
 import pyspark.sql as ps
-import pyspark
 from typing import Any
 
 
-SPARK_VERSION = pyspark.__version__.split(".")
+try:
+    from pyspark.sql import is_remote as _is_remote
+except ImportError:  # pragma: no cover
+    _is_remote = None
 
 try:
-    if int(SPARK_VERSION[0]) >= 4:
-        from pyspark.sql import SparkSession as SparkConnectSession
-        from pyspark.sql import DataFrame as SparkConnectDataFrame
-    else:  # pragma: no cover
-        from pyspark.sql.connect.session import SparkSession as SparkConnectSession
-        from pyspark.sql.connect.dataframe import DataFrame as SparkConnectDataFrame
+    from pyspark.sql.connect.session import SparkSession as SparkConnectSession
+    from pyspark.sql.connect.dataframe import DataFrame as SparkConnectDataFrame
 except Exception:  # pragma: no cover
     SparkConnectSession = None
     SparkConnectDataFrame = None
 
 
 def is_spark_connect(session: Any) -> bool:  # pragma: no cover
-    if int(SPARK_VERSION[0]) >= 4:
-        return False
+    if _is_remote is not None:
+        return _is_remote()
     return SparkConnectSession is not None and isinstance(
         session, (SparkConnectSession, SparkConnectDataFrame)
     )

{fugue-0.9.7.dev2 → fugue-0.9.8}/fugue_spark/execution_engine.py

@@ -403,10 +403,17 @@ class SparkExecutionEngine(ExecutionEngine):
     def get_current_parallelism(self) -> int:
         spark = self.spark_session
         if self.is_spark_connect:  # pragma: no cover
-            num = spark.conf.get("spark.default.parallelism", "")
-            if num != "":
-                return int(num)
-            return int(spark.conf.get("spark.sql.shuffle.partitions", "200"))
+            for key in (
+                "spark.default.parallelism",
+                "spark.sql.shuffle.partitions",
+            ):
+                try:
+                    value = spark.conf.get(key, "")
+                except Exception:
+                    continue
+                if value != "":
+                    return int(value)
+            return 200
         e_cores = int(spark.conf.get("spark.executor.cores", "1"))
         tc = int(spark.conf.get("spark.task.cpus", "1"))
         sc = spark._jsc.sc()

{fugue-0.9.7.dev2 → fugue-0.9.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fugue"
-version = "0.9.7.dev2"
+version = "0.9.8"
 description = "An abstraction layer for distributed computing"
 readme = "README.md"
 license = { text = "Apache-2.0" }
@@ -14,6 +14,7 @@ keywords = [
     "spark",
     "dask",
     "ray",
+    "duckdb",
     "sql",
     "dsl",
     "domain specific language",
@@ -23,14 +24,13 @@ classifiers = [
     "Intended Audience :: Developers",
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
     "Programming Language :: Python :: 3.14",
     "Programming Language :: Python :: 3 :: Only",
 ]
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = ["triad>=1.0.1", "adagio>=0.2.6", "pandas<3"]
 
 [project.urls]
@@ -114,6 +114,7 @@ dev = [
     "s3fs",
     # Specific versions for compatibility
     "pyspark[connect]",
+    "pyarrow<23",  # pyarrow>=23 needs numpy>=2; pyspark 3.x pins numpy<2
     "duckdb-engine>=0.6.4",
     "sqlalchemy==2.0.10",
     "ray[data]>=2.30.0; python_version<'3.14'",
@@ -202,7 +203,7 @@ max-complexity = 10
 [tool.tox]
 legacy_tox_ini = """
 [tox]
-envlist = py39,py310,py311,py312
+envlist = py311,py312,py313,py314
 
 [testenv]
 setenv =