boti-dask 1.0.0__tar.gz

boti_dask-1.0.0/PKG-INFO

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: boti-dask
+Version: 1.0.0
+Summary: Dask runtime and resilience utilities for the Boti ecosystem
+Author-email: Luis Valverde <lvalverdeb@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/lvalverdeb/boti-dask
+Project-URL: Repository, https://github.com/lvalverdeb/boti-dask
+Project-URL: Documentation, https://github.com/lvalverdeb/boti-dask#readme
+Project-URL: Issues, https://github.com/lvalverdeb/boti-dask/issues
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: boti>=0.1.0
+Requires-Dist: dask[dataframe,distributed]>=2026.3.0
+Requires-Dist: pandas>=3.0.2
+Provides-Extra: extra
+Requires-Dist: numpy>=2.3.0; extra == "extra"
+Requires-Dist: polars>=1.29.0; extra == "extra"
+Requires-Dist: pyarrow>=23.0.1; extra == "extra"
+
+# boti-dask
+
+Dask runtime/session/resilience utilities for the Boti ecosystem.
+
+## API sections
+
+- Session: `docs/api/session.md`
+- Resilience: `docs/api/resilience.md`
+- Diagnostics: `docs/api/diagnostics.md`
+
+## Phase-1 scope
+
+This initial bootstrap provides:
+
+- `DaskSession` and `dask_session(...)` helpers
+- `DaskSessionSettings` and `dask_session_from_env_prefix(...)` helpers
+- shared-session lifecycle support
+- recommended Dask config profile helpers
+- resilient execution wrappers:
+  - `safe_compute`, `safe_persist`, `safe_wait`, `safe_head`, `safe_gather`
+  - async counterparts
+- diagnostics helpers:
+  - `inspect_graph`, `describe_frame`, `diagnostics_logger`
+- Dask emptiness and unique-value helpers:
+  - `dask_is_probably_empty`, `dask_is_empty`, `UniqueValuesExtractor`
+
+## Quick start
+
+```python
+from boti_dask import dask_session, inspect_graph, safe_compute
+import dask
+
+with dask_session(cluster_kwargs={"n_workers": 1, "threads_per_worker": 1, "processes": False, "dashboard_address": ":0"}) as client:
+    value = dask.delayed(lambda: 6 * 7)()
+    print(safe_compute(value, dask_client=client))
+    print(inspect_graph(value))
+```
+
+If `dashboard_address` is omitted for local managed sessions, `boti-dask` defaults it to `":0"`.
+
+Load session defaults from prefixed environment variables:
+
+```python
+from boti_dask import DaskSessionSettings, dask_session_from_env_prefix
+
+settings = DaskSessionSettings.from_env_prefix("BOTI_DASK_", env_file=".env")
+session = dask_session_from_env_prefix("BOTI_DASK_", env_file=".env", verify_connectivity=True)
+```
+
+## Distributed computing scenarios
+
+### Multi-worker cluster with grouped aggregation
+
+Launch a 4-worker cluster, distribute a DataFrame across 8 partitions, and
+run a grouped sum that executes in parallel across all workers:
+
+```python
+from boti_dask import dask_session, describe_client, safe_persist, safe_compute
+import dask.dataframe as dd, pandas as pd
+
+with dask_session(
+    cluster_kwargs={"n_workers": 4, "threads_per_worker": 2, "processes": False, "dashboard_address": ":0"},
+) as client:
+    print(describe_client(client))  # "workers": 4, "threads": 8
+
+    ddf = dd.from_pandas(pd.DataFrame({"v": range(10_000), "g": ["X","Y","Z"] * 3334}), npartitions=8)
+    persisted = safe_persist(ddf, dask_client=client)
+    grouped = safe_compute(persisted.groupby("g")["v"].sum(), dask_client=client)
+    print(grouped)  # {"X": 16665000, "Y": 16665000, "Z": 16665000}
+```
+
+### Shared session with nested contexts
+
+Multiple ``with`` blocks share the same Dask client via reference counting.
+The client stays alive until the last holder exits:
+
+```python
+from boti_dask import dask_session
+
+kwargs = {"n_workers": 2, "threads_per_worker": 1, "processes": False, "dashboard_address": ":0"}
+with dask_session(cluster_kwargs=kwargs, shared=True, shared_key="my-cluster") as outer:
+    with dask_session(shared=True, shared_key="my-cluster") as inner:
+        print(outer is inner)  # True
+    # outer still alive here
+```
+
+### Async distributed pipeline
+
+Persist, wait, head, compute, and gather — all through async-safe wrappers:
+
+```python
+import asyncio
+from boti_dask import dask_session, async_safe_persist, async_safe_head, async_safe_gather
+import dask.dataframe as dd, pandas as pd, dask
+
+async def pipeline():
+    ddf = dd.from_pandas(pd.DataFrame({"id": range(100)}), npartitions=4)
+    with dask_session(cluster_kwargs={"n_workers": 2, "processes": False, "dashboard_address": ":0"}) as client:
+        p = await async_safe_persist(ddf, dask_client=client)
+        preview = await async_safe_head(p, n=3, dask_client=client)
+        gathered = await async_safe_gather([dask.delayed(lambda: 42)()], dask_client=client)
+        return preview["id"].tolist(), gathered
+
+asyncio.run(pipeline())
+```
+
+### Orphaned graph detection
+
+When a Dask collection outlives the session that created it, operations
+fail with a clear error instead of a cryptic Dask traceback:
+
+```python
+with dask_session(cluster_kwargs={...}) as client:
+    persisted = safe_persist(ddf, dask_client=client)
+# session closed — persisted is now orphaned
+safe_head(persisted)  # RuntimeError: orphaned from its original client/worker state
+```
+
+Pass an external Dask `Client` to keep the client alive across session boundaries:
+
+```python
+from dask.distributed import LocalCluster, Client
+cluster = LocalCluster(...)
+external = Client(cluster)
+with dask_session(client=external) as session_client:
+    result = safe_compute(dask.delayed(lambda: 42)(), dask_client=session_client)
+# external.close()  # manual lifecycle
+```
+
+## Development
+
+```bash
+uv sync --dev
+uv run pytest -q
+```
+
+## Examples
+
+```bash
+uv run python examples/data_facade_dask_resilience.py
+uv run python examples/smoke_all_examples.py
+```
+
+See `examples/README.md` for details.

boti_dask-1.0.0/README.md

@@ -0,0 +1,144 @@
+# boti-dask
+
+Dask runtime/session/resilience utilities for the Boti ecosystem.
+
+## API sections
+
+- Session: `docs/api/session.md`
+- Resilience: `docs/api/resilience.md`
+- Diagnostics: `docs/api/diagnostics.md`
+
+## Phase-1 scope
+
+This initial bootstrap provides:
+
+- `DaskSession` and `dask_session(...)` helpers
+- `DaskSessionSettings` and `dask_session_from_env_prefix(...)` helpers
+- shared-session lifecycle support
+- recommended Dask config profile helpers
+- resilient execution wrappers:
+  - `safe_compute`, `safe_persist`, `safe_wait`, `safe_head`, `safe_gather`
+  - async counterparts
+- diagnostics helpers:
+  - `inspect_graph`, `describe_frame`, `diagnostics_logger`
+- Dask emptiness and unique-value helpers:
+  - `dask_is_probably_empty`, `dask_is_empty`, `UniqueValuesExtractor`
+
+## Quick start
+
+```python
+from boti_dask import dask_session, inspect_graph, safe_compute
+import dask
+
+with dask_session(cluster_kwargs={"n_workers": 1, "threads_per_worker": 1, "processes": False, "dashboard_address": ":0"}) as client:
+    value = dask.delayed(lambda: 6 * 7)()
+    print(safe_compute(value, dask_client=client))
+    print(inspect_graph(value))
+```
+
+If `dashboard_address` is omitted for local managed sessions, `boti-dask` defaults it to `":0"`.
+
+Load session defaults from prefixed environment variables:
+
+```python
+from boti_dask import DaskSessionSettings, dask_session_from_env_prefix
+
+settings = DaskSessionSettings.from_env_prefix("BOTI_DASK_", env_file=".env")
+session = dask_session_from_env_prefix("BOTI_DASK_", env_file=".env", verify_connectivity=True)
+```
+
+## Distributed computing scenarios
+
+### Multi-worker cluster with grouped aggregation
+
+Launch a 4-worker cluster, distribute a DataFrame across 8 partitions, and
+run a grouped sum that executes in parallel across all workers:
+
+```python
+from boti_dask import dask_session, describe_client, safe_persist, safe_compute
+import dask.dataframe as dd, pandas as pd
+
+with dask_session(
+    cluster_kwargs={"n_workers": 4, "threads_per_worker": 2, "processes": False, "dashboard_address": ":0"},
+) as client:
+    print(describe_client(client))  # "workers": 4, "threads": 8
+
+    ddf = dd.from_pandas(pd.DataFrame({"v": range(10_000), "g": ["X","Y","Z"] * 3334}), npartitions=8)
+    persisted = safe_persist(ddf, dask_client=client)
+    grouped = safe_compute(persisted.groupby("g")["v"].sum(), dask_client=client)
+    print(grouped)  # {"X": 16665000, "Y": 16665000, "Z": 16665000}
+```
+
+### Shared session with nested contexts
+
+Multiple ``with`` blocks share the same Dask client via reference counting.
+The client stays alive until the last holder exits:
+
+```python
+from boti_dask import dask_session
+
+kwargs = {"n_workers": 2, "threads_per_worker": 1, "processes": False, "dashboard_address": ":0"}
+with dask_session(cluster_kwargs=kwargs, shared=True, shared_key="my-cluster") as outer:
+    with dask_session(shared=True, shared_key="my-cluster") as inner:
+        print(outer is inner)  # True
+    # outer still alive here
+```
+
+### Async distributed pipeline
+
+Persist, wait, head, compute, and gather — all through async-safe wrappers:
+
+```python
+import asyncio
+from boti_dask import dask_session, async_safe_persist, async_safe_head, async_safe_gather
+import dask.dataframe as dd, pandas as pd, dask
+
+async def pipeline():
+    ddf = dd.from_pandas(pd.DataFrame({"id": range(100)}), npartitions=4)
+    with dask_session(cluster_kwargs={"n_workers": 2, "processes": False, "dashboard_address": ":0"}) as client:
+        p = await async_safe_persist(ddf, dask_client=client)
+        preview = await async_safe_head(p, n=3, dask_client=client)
+        gathered = await async_safe_gather([dask.delayed(lambda: 42)()], dask_client=client)
+        return preview["id"].tolist(), gathered
+
+asyncio.run(pipeline())
+```
+
+### Orphaned graph detection
+
+When a Dask collection outlives the session that created it, operations
+fail with a clear error instead of a cryptic Dask traceback:
+
+```python
+with dask_session(cluster_kwargs={...}) as client:
+    persisted = safe_persist(ddf, dask_client=client)
+# session closed — persisted is now orphaned
+safe_head(persisted)  # RuntimeError: orphaned from its original client/worker state
+```
+
+Pass an external Dask `Client` to keep the client alive across session boundaries:
+
+```python
+from dask.distributed import LocalCluster, Client
+cluster = LocalCluster(...)
+external = Client(cluster)
+with dask_session(client=external) as session_client:
+    result = safe_compute(dask.delayed(lambda: 42)(), dask_client=session_client)
+# external.close()  # manual lifecycle
+```
+
+## Development
+
+```bash
+uv sync --dev
+uv run pytest -q
+```
+
+## Examples
+
+```bash
+uv run python examples/data_facade_dask_resilience.py
+uv run python examples/smoke_all_examples.py
+```
+
+See `examples/README.md` for details.

boti_dask-1.0.0/pyproject.toml

@@ -0,0 +1,75 @@
+[project]
+name = "boti-dask"
+version = "1.0.0"
+description = "Dask runtime and resilience utilities for the Boti ecosystem"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [
+    {name = "Luis Valverde", email = "lvalverdeb@gmail.com"}
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3 :: Only",
+    "Operating System :: OS Independent",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "boti>=0.1.0",
+    "dask[dataframe,distributed]>=2026.3.0",
+    "pandas>=3.0.2",
+]
+
+[project.optional-dependencies]
+extra = [
+    "numpy>=2.3.0",
+    "polars>=1.29.0",
+    "pyarrow>=23.0.1",
+]
+
+[project.urls]
+Homepage = "https://github.com/lvalverdeb/boti-dask"
+Repository = "https://github.com/lvalverdeb/boti-dask"
+Documentation = "https://github.com/lvalverdeb/boti-dask#readme"
+Issues = "https://github.com/lvalverdeb/boti-dask/issues"
+
+[build-system]
+requires = ["setuptools>=80", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.3",
+    "pytest-asyncio>=1.3.0",
+    "ruff>=0.11.0",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "UP"]
+ignore = ["E501"]
+
+[tool.pytest.ini_options]
+addopts = "-q"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.package-data]
+boti_dask = ["py.typed"]
+
+[tool.setuptools.package-dir]
+"" = "src"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["boti_dask*"]

boti_dask-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

boti_dask-1.0.0/src/boti_dask/__init__.py

@@ -0,0 +1,62 @@
+"""Dask runtime and resilience utilities for the Boti ecosystem."""
+
+from boti_dask.diagnostics import (
+    UniqueValuesExtractor,
+    describe_frame,
+    diagnostics_logger,
+    inspect_graph,
+)
+from boti_dask.resilience import (
+    RECOVERABLE_DASK_ERRORS,
+    async_safe_compute,
+    async_safe_gather,
+    async_safe_head,
+    async_safe_persist,
+    async_safe_wait,
+    dask_is_empty,
+    dask_is_probably_empty,
+    safe_compute,
+    safe_gather,
+    safe_head,
+    safe_persist,
+    safe_wait,
+)
+from boti_dask.session import (
+    DaskSession,
+    DaskSessionSettings,
+    apply_recommended_dask_config,
+    current_client_summary,
+    dask_session,
+    dask_session_from_env_prefix,
+    describe_client,
+    recommended_dask_config,
+)
+
+__all__ = [
+    "DaskSession",
+    "DaskSessionSettings",
+    "RECOVERABLE_DASK_ERRORS",
+    "UniqueValuesExtractor",
+    "apply_recommended_dask_config",
+    "async_safe_compute",
+    "async_safe_gather",
+    "async_safe_head",
+    "async_safe_persist",
+    "async_safe_wait",
+    "current_client_summary",
+    "dask_is_empty",
+    "dask_is_probably_empty",
+    "dask_session",
+    "dask_session_from_env_prefix",
+    "describe_frame",
+    "describe_client",
+    "diagnostics_logger",
+    "inspect_graph",
+    "recommended_dask_config",
+    "safe_compute",
+    "safe_gather",
+    "safe_head",
+    "safe_persist",
+    "safe_wait",
+]
+

boti_dask-1.0.0/src/boti_dask/_internal.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from typing import Any
+
+import dask.dataframe as dd
+
+
+def _log(logger: Any | None, level: str, message: str) -> None:
+    if logger is None:
+        return
+    log_fn = getattr(logger, level, None)
+    if callable(log_fn):
+        log_fn(message)
+
+
+def _is_dask_dataframe_like(obj: Any) -> bool:
+    return isinstance(obj, (dd.DataFrame, dd.Series)) or hasattr(obj, "_meta")
+
+
+def _is_running_client(client: Any | None) -> bool:
+    if client is None:
+        return False
+    return getattr(client, "status", "running") in {"running", "started"}

boti_dask-1.0.0/src/boti_dask/diagnostics.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import dask.dataframe as dd
+import pandas as pd
+
+try:
+    import pyarrow as pa
+except ImportError:
+    pa = None
+
+try:
+    import polars as pl
+except ImportError:
+    pl = None
+
+from boti.core.logger import Logger
+
+from ._internal import _log
+
+
+def _has_dask_graph(obj: Any) -> bool:
+    return hasattr(obj, "__dask_graph__")
+
+
+def inspect_graph(obj: Any, *, logger: Any | None = None) -> dict[str, Any]:
+    """Return compact Dask graph metrics for diagnostics and dry-run usage."""
+    if not _has_dask_graph(obj):
+        metrics = {"type": type(obj).__name__, "is_dask": False}
+        _log(logger, "info", f"Dask graph inspection metrics={metrics}")
+        return metrics
+
+    graph = obj.__dask_graph__()
+    graph_layers: int | None = None
+    if hasattr(getattr(obj, "dask", None), "layers"):
+        graph_layers = len(obj.dask.layers)
+    elif hasattr(graph, "layers"):
+        graph_layers = len(graph.layers)
+    metrics = {
+        "type": type(obj).__name__,
+        "is_dask": True,
+        "task_count": len(graph) if hasattr(graph, "__len__") else None,
+        "npartitions": getattr(obj, "npartitions", None),
+        "graph_layers": graph_layers,
+    }
+    _log(logger, "info", f"Dask graph inspection metrics={metrics}")
+    return metrics
+
+
+def describe_frame(frame: Any) -> dict[str, Any]:
+    """Return compact frame metrics suitable for runtime diagnostics logs."""
+    if isinstance(frame, dd.DataFrame):
+        graph = frame.__dask_graph__()
+        return {
+            "engine": "dask",
+            "columns": len(frame.columns),
+            "npartitions": frame.npartitions,
+            "graph_tasks": len(graph) if hasattr(graph, "__len__") else None,
+            "graph_layers": len(frame.dask.layers) if hasattr(frame.dask, "layers") else None,
+            "known_divisions": frame.known_divisions,
+        }
+    if isinstance(frame, pd.DataFrame):
+        return {
+            "engine": "pandas",
+            "rows": len(frame.index),
+            "columns": len(frame.columns),
+        }
+    if pa is not None and isinstance(frame, pa.Table):
+        return {
+            "engine": "arrow",
+            "rows": frame.num_rows,
+            "columns": len(frame.column_names),
+        }
+    if pl is not None and isinstance(frame, pl.DataFrame):
+        return {
+            "engine": "polars",
+            "rows": frame.height,
+            "columns": frame.width,
+        }
+    return {"engine": type(frame).__name__}
+
+
+def diagnostics_logger(logger: Any | None, *, name: str) -> Any:
+    """Return provided logger or a :class:`boti.Logger` fallback scoped by *name*."""
+    if logger is not None:
+        return logger
+    return Logger.default_logger(logger_name=name)
+
+
+class UniqueValuesExtractor:
+    """Best-effort unique value extraction for Dask-backed columns."""
+
+    def __init__(self, *, dask_client: Any | None = None, logger: Any | None = None) -> None:
+        self.dask_client = dask_client
+        self.logger = logger
+
+    def _extract_one(self, frame: dd.DataFrame, column: str, limit: int) -> tuple[str, list[Any]]:
+        from .resilience import safe_compute
+
+        if column not in frame.columns:
+            _log(self.logger, "warning", f"Column '{column}' not found for unique extraction.")
+            return column, []
+
+        expression = frame[column].dropna().drop_duplicates()
+        if self.dask_client is not None:
+            values = safe_compute(expression, dask_client=self.dask_client, logger=self.logger)
+        else:
+            values = expression.compute(scheduler="threads")
+        as_series = pd.Series(values)
+        unique_values = as_series.dropna().unique().tolist()
+        if len(unique_values) > limit:
+            _log(
+                self.logger,
+                "warning",
+                f"Unique value extraction for column '{column}' truncated at {limit} items.",
+            )
+            unique_values = unique_values[:limit]
+        return column, unique_values
+
+    async def extract_unique_values(
+        self,
+        frame: dd.DataFrame,
+        *columns: str,
+        limit: int = 100_000,
+    ) -> dict[str, list[Any]]:
+        from .resilience import safe_persist
+
+        if self.dask_client is not None:
+            frame = safe_persist(frame, dask_client=self.dask_client, logger=self.logger)
+        else:
+            frame = frame.persist()
+        _log(self.logger, "info", f"Persisted {frame.npartitions}-partition frame for unique extraction.")
+        pairs = await asyncio.gather(
+            *(asyncio.to_thread(self._extract_one, frame, column, limit) for column in columns)
+        )
+        return dict(pairs)
+
+
+__all__ = [
+    "UniqueValuesExtractor",
+    "describe_frame",
+    "diagnostics_logger",
+    "inspect_graph",
+]