fugue 0.8.7.dev8__tar.gz → 0.9.0__tar.gz

{fugue-0.8.7.dev8 → fugue-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fugue
-Version: 0.8.7.dev8
+Version: 0.9.0
 Summary: An abstraction layer for distributed computation
 Home-page: http://github.com/fugue-project/fugue
 Author: The Fugue Development Team
@@ -292,6 +292,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown

{fugue-0.8.7.dev8 → fugue-0.9.0}/fugue/collections/sql.py

@@ -15,7 +15,7 @@ class TempTableName:
     """Generating a temporary, random and globaly unique table name"""
 
     def __init__(self):
-        self.key = "_" + str(uuid4())[:5]
+        self.key = "_" + str(uuid4())[:5].upper()
 
     def __repr__(self) -> str:
         return _TEMP_TABLE_EXPR_PREFIX + self.key + _TEMP_TABLE_EXPR_SUFFIX

{fugue-0.8.7.dev8 → fugue-0.9.0}/fugue/dataframe/utils.py

@@ -21,22 +21,6 @@ normalize_dataframe_column_names = normalize_column_names
 rename_dataframe_column_names = rename
 
 
-def _pa_type_eq(t1: pa.DataType, t2: pa.DataType) -> bool:
-    # should ignore the name difference of list
-    # e.g. list<item: string> == list<l: string>
-    if pa.types.is_list(t1) and pa.types.is_list(t2):  # pragma: no cover
-        return _pa_type_eq(t1.value_type, t2.value_type)
-    return t1 == t2
-
-
-def _schema_eq(s1: Schema, s2: Schema) -> bool:
-    if s1 == s2:
-        return True
-    return s1.names == s2.names and all(
-        _pa_type_eq(f1.type, f2.type) for f1, f2 in zip(s1.fields, s2.fields)
-    )
-
-
 def _df_eq(
     df: DataFrame,
     data: Any,
@@ -46,6 +30,7 @@ def _df_eq(
     check_schema: bool = True,
     check_content: bool = True,
     no_pandas: bool = False,
+    equal_type_groups: Optional[List[List[Any]]] = None,
     throw=False,
 ) -> bool:
     """Compare if two dataframes are equal. Is for internal, unit test
@@ -66,6 +51,7 @@ def _df_eq(
     :param no_pandas: if true, it will compare the string representations of the
       dataframes, otherwise, it will convert both to pandas dataframe to compare,
       defaults to False
+    :param equal_type_groups: the groups to treat as equal types, defaults to None.
     :param throw: if to throw error if not equal, defaults to False
     :return: if they equal
     """
@@ -78,8 +64,8 @@ def _df_eq(
         assert (
             df1.count() == df2.count()
         ), f"count mismatch {df1.count()}, {df2.count()}"
-        assert not check_schema or _schema_eq(
-            df.schema, df2.schema
+        assert not check_schema or df.schema.is_like(
+            df2.schema, equal_groups=equal_type_groups
         ), f"schema mismatch {df.schema.pa_schema}, {df2.schema.pa_schema}"
         if not check_content:
             return True

fugue-0.9.0/fugue/test/__init__.py

@@ -0,0 +1,11 @@
+# flake8: noqa
+from .pandas_tester import NativeTestBackend, PandasTestBackend
+from .plugins import (
+    FugueTestBackend,
+    FugueTestContext,
+    FugueTestSuite,
+    extract_conf,
+    fugue_test_backend,
+    fugue_test_suite,
+    with_backend,
+)

fugue-0.9.0/fugue/test/pandas_tester.py

@@ -0,0 +1,24 @@
+from contextlib import contextmanager
+from typing import Any, Dict, Iterator
+
+from .plugins import FugueTestBackend, fugue_test_backend
+
+
+@fugue_test_backend
+class PandasTestBackend(FugueTestBackend):
+    name = "pandas"
+
+    @classmethod
+    @contextmanager
+    def session_context(cls, session_conf: Dict[str, Any]) -> Iterator[Any]:
+        yield "pandas"  # pragma: no cover
+
+
+@fugue_test_backend
+class NativeTestBackend(FugueTestBackend):
+    name = "native"
+
+    @classmethod
+    @contextmanager
+    def session_context(cls, session_conf: Dict[str, Any]) -> Iterator[Any]:
+        yield "native"  # pragma: no cover

fugue-0.9.0/fugue/test/plugins.py

@@ -0,0 +1,393 @@
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Type
+from fugue.dataframe.utils import _df_eq
+from triad import assert_or_throw, run_once
+from triad.utils.entry_points import load_entry_point
+
+try:
+    import pytest
+
+    _HAS_PYTEST = True
+except ImportError:  # pragma: no cover
+    _HAS_PYTEST = False
+
+
+_FUGUE_TEST_BACKENDS: Dict[str, Type["FugueTestBackend"]] = {}
+_FUGUE_TEST_ALL_INI_CONF: Dict[str, Any] = {}
+_FUGUE_TEST_INI_FUGUE_CONF: Dict[str, Any] = {}
+
+
+def _set_global_conf(conf: Dict[str, Any]) -> None:
+    global _FUGUE_TEST_ALL_INI_CONF, _FUGUE_TEST_INI_FUGUE_CONF
+    _FUGUE_TEST_ALL_INI_CONF = conf
+    _FUGUE_TEST_INI_FUGUE_CONF = _extract_fugue_conf(conf)
+
+
+def _get_all_ini_conf() -> Dict[str, Any]:
+    return _FUGUE_TEST_ALL_INI_CONF
+
+
+@run_once
+def _load_all_backends() -> None:
+    from fugue.constants import FUGUE_ENTRYPOINT
+
+    load_entry_point(FUGUE_ENTRYPOINT)
+
+
+def with_backend(ctx: Any, *other: Any, skip_missing: bool = False) -> Any:
+    """The decorator to set the backend context for the test function
+
+    :param ctx: the object to specify the backend
+    :param other: more objects to specify the additional backends
+    :param skip_missing: skip if an object can't be converted to a backend,
+        defaults to False
+
+    .. admonition:: Examples
+
+        .. code-block:: python
+
+            import fugue.test as ft
+            import fugue.api as fa
+
+            @ft.with_backend("pandas", "spark")
+            def test_spark(path_fixture):
+                # test load and count under spark and pandas
+                df = fa.load(path_fixture)
+                assert fa.count(df) == 3
+
+    .. note::
+
+        ``ctx`` can be a string or a tuple of string and dict. If it contains a
+        dict, the dict will be merged with the default configuration for the backend.
+
+        the configuration of each backend is extracted from the pytest ini configuration
+        under ``fugue_test_conf``. The configuration must prefixed with the backend
+        name.
+
+        ``fugue_test_conf`` is defined at the parallel place where you define pytest
+        ``addopts``. For example, if you define them in ``setup.cfg``, then it
+        looks like:
+
+        .. code-block:: ini
+
+            [tool:pytest]
+            addopts =
+                -p pytest_cov
+                --cov=fugue
+            fugue_test_conf =
+                fugue.test.dummy=dummy
+                fugue.test:bool=true
+                # ray backend settings
+                ray.num_cpus:int=2
+                # dask backend settings
+                dask.processes:bool=true
+                dask.n_workers:int=3
+                dask.threads_per_worker:int=1
+    """
+    import pytest
+
+    _load_all_backends()
+
+    _ctx = _construct_parameterized_fixture([ctx] + list(other), skip_missing)
+    return lambda f: pytest.mark.parametrize("backend_context", _ctx, indirect=True)(
+        pytest.mark.usefixtures("backend_context")(f)
+    )
+
+
+def fugue_test_backend(cls: Type["FugueTestBackend"]) -> Type["FugueTestBackend"]:
+    """The decorator to register a Fugue test backend. Most Fugue users don't need
+    to use this decorator.
+
+    :param cls: the FugueTestBackend class
+
+    .. admonition:: Examples
+
+        .. code-block:: python
+
+            import fugue.test as ft
+
+            @ft.fugue_test_backend
+            class MyBackend(ft.FugueTestBackend):
+                name = "my_backend"  # the name of the backend
+
+                @classmethod
+                @contextmanager
+                def session_context(cls, session_conf: Dict[str, Any]) -> Iterator[Any]:
+                    # create a session object
+                    yield session
+
+    """
+    assert_or_throw(
+        issubclass(cls, FugueTestBackend),
+        ValueError(f"{cls} is not a FugueTestBackend"),
+    )
+    name = cls.name.strip().lower()
+    assert_or_throw(
+        name != "" and name != "fugue",
+        ValueError(f"Fugue test backend name cannot be empty or fugue: {cls}"),
+    )
+    assert_or_throw(
+        name not in _FUGUE_TEST_BACKENDS,
+        ValueError(f"Duplicate Fugue test backend name: {name}"),
+    )
+    _FUGUE_TEST_BACKENDS[name] = cls
+    return cls
+
+
+class FugueTestSuite:
+    """The base class for Fugue test suite. Most Fugue users don't need to use this
+    class or the decorator :func:`~.fugue_test_suite`. The dccorator and the class
+    are often used together.
+
+    .. admonition:: Examples
+
+        .. code-block:: python
+
+            import fugue.test as ft
+
+            @ft.fugue_test_suite("pandas")
+            class MyTest(ft.FugueTestSuite):
+                def test_pandas(self):
+                    # test pandas
+                    pass
+
+                def test_spark(self):
+                    # test spark
+                    pass
+    """
+
+    backend: Any
+    tmp_path: Path
+    equal_type_groups: Any = None
+
+    __test__ = False
+    _test_context: Any = None
+
+    if _HAS_PYTEST:
+
+        @pytest.fixture(autouse=True)
+        def init_builtin_per_func_context(self, tmp_path):
+            self.tmp_path = tmp_path
+
+    @property
+    def context(self) -> "FugueTestContext":
+        """The ``FugueTestContext`` object"""
+        return self._test_context
+
+    @property
+    def engine(self) -> Any:
+        """The engine object inside the ``FugueTestContext``"""
+        return self.context.engine
+
+    def get_equal_type_groups(self) -> Optional[List[List[Any]]]:
+        return None  # pragma: no cover
+
+    def df_eq(self, *args: Any, **kwargs: Any) -> bool:
+        """A wrapper of :func:`~fugue.dataframe.utils.df_eq`"""
+        if "equal_type_groups" not in kwargs:
+            kwargs["equal_type_groups"] = self.equal_type_groups
+        return _df_eq(*args, **kwargs)
+
+
+def fugue_test_suite(backend: Any, mark_test: Optional[bool] = None) -> Any:
+    def deco(cls: Type["FugueTestSuite"]) -> Type["FugueTestSuite"]:
+        import pytest
+
+        assert_or_throw(
+            issubclass(cls, FugueTestSuite),
+            ValueError(f"{cls} is not a FugueTestSuite"),
+        )
+        if mark_test is not None:
+            cls.__test__ = mark_test
+        c, extra_conf = _parse_backend(backend)
+        return pytest.mark.parametrize(
+            "backend_context", [pytest.param((c, extra_conf), id=c)], indirect=True
+        )(pytest.mark.usefixtures("_class_backend_context")(cls))
+
+    return deco
+
+
+@dataclass
+class FugueTestContext:
+    """The context object for Fugue test
+
+    :param engine: the Fugue ExecutionEngine object
+    :param session: the original session wrapped by the Fugue ExecutionEngine
+    :param name: the backend name
+    """
+
+    engine: Any
+    session: Any
+    name: str
+
+
+class FugueTestBackend:
+    """The base class for Fugue test backend. Most Fugue users don't need to use this
+    class or the decorator :func:`~.fugue_test_backend`. The dccorator and the class
+    are often used together.
+
+    .. admonition:: Examples
+
+        .. code-block:: python
+
+            import fugue.test as ft
+
+            @ft.fugue_test_backend
+            class MyBackend(ft.FugueTestBackend):
+                name = "my_backend"  # the name of the backend
+
+                @classmethod
+                @contextmanager
+                def session_context(cls, session_conf: Dict[str, Any]) -> Iterator[Any]:
+                    # create a session object
+                    yield session
+    """
+
+    name = ""
+    default_session_conf: Dict[str, Any] = {}
+    default_fugue_conf: Dict[str, Any] = {}
+    session_conf: Dict[str, Any] = {}
+    fugue_conf: Dict[str, Any] = {}
+
+    @classmethod
+    def transform_session_conf(cls, conf: Dict[str, Any]) -> Dict[str, Any]:
+        """Extract and transform the ``conf`` and keep the ones specific to
+        this backend. The default implementation will extract the configuration
+        prefixed with ``<backend name>.`` and remove the prefix.
+
+        :param conf: the raw configuration
+        :return: the transformed configuration
+        """
+        return extract_conf(conf, cls.name + ".", remove_prefix=True)
+
+    @classmethod
+    @contextmanager
+    def session_context(cls, session_conf: Dict[str, Any]) -> Iterator[Any]:
+        """Create the backend native session object
+
+        :param session_conf: the configuration for this backend
+        :yield: the session object
+        """
+        raise NotImplementedError  # pragma: no cover
+
+    @classmethod
+    @contextmanager
+    def generate_session_fixture(cls) -> Iterator[Any]:
+        """Generate the session fixture for this backend from the default backend
+        configuration and then the pytest ini configuration under ``fugue_test_conf``
+
+        :yield: the session object
+        """
+        session_conf = _merge_dicts(
+            cls.default_session_conf,
+            cls.transform_session_conf(_FUGUE_TEST_ALL_INI_CONF),
+        )
+        with cls.session_context(session_conf) as session:
+            yield session
+
+    @classmethod
+    @contextmanager
+    def generate_context_fixture(
+        cls, session: object, extra_fugue_conf: Dict[str, Any]
+    ) -> Iterator[FugueTestContext]:
+        """Generate the ``FugueTestContext`` fixture for this backend from the
+        default backend configuration and then the pytest ini configuration under
+        ``fugue_test_conf`` and then ``extra_fugue_conf``.
+
+        :param session: the session object
+        :param extra_fugue_conf: the extra fugue configuration
+
+        :yield: the ``FugueTestContext`` object
+        """
+        import fugue.api as fa
+
+        fugue_conf = _merge_dicts(
+            cls.default_fugue_conf,
+            _FUGUE_TEST_INI_FUGUE_CONF,
+            _extract_fugue_conf(extra_fugue_conf),
+        )
+        with fa.engine_context(session, fugue_conf) as engine:
+            yield FugueTestContext(engine=engine, session=session, name=cls.name)
+
+
+def extract_conf(
+    conf: Dict[str, Any], prefix: str, remove_prefix: bool
+) -> Dict[str, Any]:
+    """Extract the configuration prefixed with ``prefix`` and optionally remove the
+    prefix
+
+    :param conf: the raw configuration
+    :param prefix: the prefix
+    :param remove_prefix: whether to remove the prefix
+
+    :return: the extracted configuration
+    """
+    res: Dict[str, Any] = {}
+    for k, v in conf.items():
+        if k.startswith(prefix):
+            if remove_prefix:
+                k = k[len(prefix) :]
+            res[k] = v
+    return res
+
+
+@contextmanager
+def _make_backend_context(obj: Any, session: Any) -> Iterator[Any]:
+    _load_all_backends()
+    key, extra_conf = _parse_backend(obj)
+    assert_or_throw(
+        key in _FUGUE_TEST_BACKENDS,
+        lambda: ValueError(
+            f"Undefined Fugue test backend: {key}, "
+            f"available backends: {list(_FUGUE_TEST_BACKENDS.keys())}"
+        ),
+    )
+    with _FUGUE_TEST_BACKENDS[key].generate_context_fixture(session, extra_conf) as ctx:
+        yield ctx
+
+
+def _extract_fugue_conf(conf: Dict[str, Any]) -> Dict[str, Any]:
+    return extract_conf(conf, "fugue.", remove_prefix=False)
+
+
+def _construct_parameterized_fixture(ctx: List[Any], skip_missing: bool) -> List[Any]:
+    import pytest
+
+    _ctx: List[Tuple[str, Dict[str, Any]]] = []
+    for x in ctx:
+        c, extra_conf = _parse_backend(x)
+        if c not in _FUGUE_TEST_BACKENDS:
+            if not skip_missing:
+                raise ValueError(
+                    f"Undefined Fugue test backend: {c}, "
+                    f"available backends: {list(_FUGUE_TEST_BACKENDS.keys())}"
+                )
+            else:
+                _ctx.append(
+                    pytest.param(
+                        c,
+                        marks=pytest.mark.xfail(
+                            reason="Undefined Fugue test backend", run=False
+                        ),
+                        id=c,
+                    )
+                )
+        else:
+            _ctx.append(pytest.param((c, extra_conf), id=c))
+    return _ctx
+
+
+def _merge_dicts(*dicts: Dict[str, Any]) -> Dict[str, Any]:
+    res: Dict[str, Any] = {}
+    for d in dicts:
+        res.update(d)
+    return res
+
+
+def _parse_backend(ctx: Any) -> Tuple[str, Dict[str, Any]]:
+    if isinstance(ctx, str):
+        return ctx, {}
+    else:
+        return ctx[0], ctx[1]

{fugue-0.8.7.dev8 → fugue-0.9.0}/fugue.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fugue
-Version: 0.8.7.dev8
+Version: 0.9.0
 Summary: An abstraction layer for distributed computation
 Home-page: http://github.com/fugue-project/fugue
 Author: The Fugue Development Team
@@ -292,6 +292,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown

{fugue-0.8.7.dev8 → fugue-0.9.0}/fugue.egg-info/SOURCES.txt

@@ -81,6 +81,9 @@ fugue/sql/_utils.py
 fugue/sql/_visitors.py
 fugue/sql/api.py
 fugue/sql/workflow.py
+fugue/test/__init__.py
+fugue/test/pandas_tester.py
+fugue/test/plugins.py
 fugue/workflow/__init__.py
 fugue/workflow/_checkpoint.py
 fugue/workflow/_tasks.py
@@ -100,25 +103,21 @@ fugue_dask/_io.py
 fugue_dask/_utils.py
 fugue_dask/dataframe.py
 fugue_dask/execution_engine.py
-fugue_dask/ibis_engine.py
 fugue_dask/registry.py
+fugue_dask/tester.py
 fugue_duckdb/__init__.py
 fugue_duckdb/_io.py
 fugue_duckdb/_utils.py
 fugue_duckdb/dask.py
 fugue_duckdb/dataframe.py
 fugue_duckdb/execution_engine.py
-fugue_duckdb/ibis_engine.py
 fugue_duckdb/registry.py
+fugue_duckdb/tester.py
 fugue_ibis/__init__.py
 fugue_ibis/_compat.py
 fugue_ibis/_utils.py
 fugue_ibis/dataframe.py
 fugue_ibis/execution_engine.py
-fugue_ibis/extensions.py
-fugue_ibis/execution/__init__.py
-fugue_ibis/execution/ibis_engine.py
-fugue_ibis/execution/pandas_backend.py
 fugue_notebook/__init__.py
 fugue_notebook/env.py
 fugue_notebook/nbextension/README.md
@@ -134,6 +133,7 @@ fugue_ray/_constants.py
 fugue_ray/dataframe.py
 fugue_ray/execution_engine.py
 fugue_ray/registry.py
+fugue_ray/tester.py
 fugue_ray/_utils/__init__.py
 fugue_ray/_utils/cluster.py
 fugue_ray/_utils/dataframe.py
@@ -142,8 +142,8 @@ fugue_spark/__init__.py
 fugue_spark/_constants.py
 fugue_spark/dataframe.py
 fugue_spark/execution_engine.py
-fugue_spark/ibis_engine.py
 fugue_spark/registry.py
+fugue_spark/tester.py
 fugue_spark/_utils/__init__.py
 fugue_spark/_utils/convert.py
 fugue_spark/_utils/io.py
@@ -156,14 +156,5 @@ fugue_test/bag_suite.py
 fugue_test/builtin_suite.py
 fugue_test/dataframe_suite.py
 fugue_test/execution_suite.py
-fugue_test/ibis_suite.py
-fugue_test/plugins/__init__.py
-fugue_test/plugins/dask/__init__.py
-fugue_test/plugins/dask/fixtures.py
-fugue_test/plugins/duckdb/__init__.py
-fugue_test/plugins/duckdb/fixtures.py
-fugue_test/plugins/misc/__init__.py
-fugue_test/plugins/misc/fixtures.py
-fugue_test/plugins/ray/__init__.py
-fugue_test/plugins/ray/fixtures.py
+fugue_test/fixtures.py
 fugue_version/__init__.py

fugue-0.9.0/fugue.egg-info/entry_points.txt

@@ -0,0 +1,12 @@
+[fugue.plugins]
+dask = fugue_dask.registry [dask]
+duckdb = fugue_duckdb.registry [duckdb]
+ibis = fugue_ibis [ibis]
+polars = fugue_polars.registry [polars]
+ray = fugue_ray.registry [ray]
+spark = fugue_spark.registry [spark]
+
+[pytest11]
+fugue_test = fugue_test
+fugue_test_fixtures = fugue_test.fixtures
+

{fugue-0.8.7.dev8 → fugue-0.9.0}/fugue.egg-info/requires.txt

@@ -1,41 +1,50 @@
-triad==0.9.2
+triad>=0.9.6
 adagio>=0.2.4
-qpd>=0.4.4
-fugue-sql-antlr>=0.1.6
-sqlglot
-jinja2
 
 [all]
+qpd>=0.4.4
+fugue-sql-antlr>=0.2.0
 sqlglot
 jinja2
-fugue-sql-antlr[cpp]>=0.1.6
 pyspark>=3.1.1
 dask[dataframe,distributed]>=2023.5.0
 dask-sql
-ray[data]>=2.4.0
+ray[data]>=2.5.0
 notebook
 jupyterlab
 ipython>=7.10.0
 duckdb>=0.5.0
 pyarrow>=6.0.1
-pandas>=2.0.2
-ibis-framework<6,>=3.2.0
+pandas<2.2,>=2.0.2
+ibis-framework
 polars
 
 [cpp_sql_parser]
-fugue-sql-antlr[cpp]>=0.1.6
+fugue-sql-antlr[cpp]>=0.2.0
 
 [dask]
 dask[dataframe,distributed]>=2023.5.0
 pyarrow>=7.0.0
 pandas>=2.0.2
 
+[dask:python_version >= "3.11.9"]
+dask[dataframe,distributed]>=2024.4.0
+
 [duckdb]
+qpd>=0.4.4
+fugue-sql-antlr>=0.2.0
+sqlglot
+jinja2
 duckdb>=0.5.0
 numpy
 
 [ibis]
-ibis-framework<6,>=3.2.0
+qpd>=0.4.4
+fugue-sql-antlr>=0.2.0
+sqlglot
+jinja2
+ibis-framework
+pandas<2.2
 
 [notebook]
 notebook
@@ -46,15 +55,16 @@ ipython>=7.10.0
 polars
 
 [ray]
-ray[data]>=2.4.0
+ray[data]>=2.5.0
 duckdb>=0.5.0
-pyarrow>=6.0.1
+pyarrow>=7.0.0
+pandas<2.2
 
 [spark]
 pyspark>=3.1.1
 
 [sql]
 qpd>=0.4.4
-fugue-sql-antlr>=0.1.6
+fugue-sql-antlr>=0.2.0
 sqlglot
 jinja2