sclab 0.1.7__py3-none-any.whl

sclab/dataset/processor/step/__init__.py

@@ -0,0 +1,7 @@
+from ._basic_processor_step import BasicProcessorStep
+from ._processor_step_base import ProcessorStepBase
+
+__all__ = [
+    "BasicProcessorStep",
+    "ProcessorStepBase",
+]

sclab/dataset/processor/step/_basic_processor_step.py

@@ -0,0 +1,109 @@
+import traceback
+from typing import Any
+
+from IPython.display import display
+from ipywidgets.widgets import (
+    HTML,
+    Button,
+    Output,
+    VBox,
+)
+from ipywidgets.widgets.valuewidget import ValueWidget
+from ipywidgets.widgets.widget_description import DescriptionWidget
+
+from ....event import EventClient
+from .._processor import Processor
+
+
+class BasicProcessorStep(EventClient):
+    events: list[str] = None
+    parent: Processor
+    description: str
+    function_name: str
+    function: callable
+    fixed_params: dict[str, Any]
+    variable_controls: dict[str, DescriptionWidget | ValueWidget]
+    output: Output
+    run_button: Button
+    controls: VBox
+
+    def __init__(
+        self,
+        parent: Processor,
+        description: str,
+        function: callable,
+        fixed_params: dict[str, Any] = {},
+        variable_controls: dict[str, DescriptionWidget | ValueWidget] = {},
+        use_run_button: bool = True,
+    ):
+        self.parent = parent
+        self.description = description
+        self.function = function
+        self.function_name = function.__name__
+
+        self.events = [
+            f"step_{self.function_name}_started",
+            f"step_{self.function_name}_ended",
+        ]
+
+        self.fixed_params = fixed_params
+        self.variable_controls = variable_controls
+        self.output = Output()
+
+        controls = []
+        for control in self.variable_controls.values():
+            control.layout.width = "98%"
+            self.parent.all_controls_list.append(control)
+            controls.append(control)
+
+        self.use_run_button = use_run_button
+        if use_run_button:
+            self.run_button = Button(description="Run", button_style="primary")
+            self.run_button.on_click(self.callback)
+            controls.append(self.run_button)
+
+        controls.append(self.output)
+        self.controls = VBox(controls)
+        super().__init__(parent.broker)
+
+    @property
+    def variable_params(self):
+        return {key: control.value for key, control in self.variable_controls.items()}
+
+    def callback(self, _: Button | None = None):
+        self.run()
+
+    def run(self, **extra_params):
+        try:
+            # extra params will override fixed and variable params
+            params = {**self.fixed_params, **self.variable_params, **extra_params}
+
+            if self.use_run_button:
+                self.run_button.disabled = True
+                self.run_button.button_style = "warning"
+                self.run_button.description = "Running..."
+            self.broker.publish(f"step_{self.function_name}_started")
+
+            self.function(**params)
+            self.parent.append_to_step_history(self.description, params)
+
+            if self.use_run_button:
+                self.run_button.button_style = "success"
+
+            self.broker.publish(f"step_{self.function_name}_ended", status="success")
+
+        except Exception as e:
+            if self.use_run_button:
+                self.run_button.button_style = "danger"
+
+            info = dict(status="failed", error=e, traceback=traceback.format_exc())
+            self.broker.publish(f"step_{self.function_name}_ended", **info)
+            self.broker.exceptions_log.append(traceback.format_exc())
+
+            with self.broker.exceptions_output.output:
+                display(HTML(f"<pre>{traceback.format_exc()}</pre>"))
+
+        finally:
+            if self.use_run_button:
+                self.run_button.description = "Run"
+                self.run_button.disabled = False

sclab/dataset/processor/step/_processor_step_base.py

@@ -0,0 +1,120 @@
+import traceback
+from typing import Any
+
+from IPython.display import HTML, Markdown, display
+from ipywidgets import Button, Output, VBox
+from ipywidgets.widgets.valuewidget import ValueWidget
+from ipywidgets.widgets.widget_description import DescriptionWidget
+
+from ....event import EventClient
+from .._processor import Processor
+
+
+class ProcessorStepBase(EventClient):
+    events: list[str] = None
+    parent: Processor
+    name: str
+    description: str
+    fixed_params: dict[str, Any]
+    variable_controls: dict[str, DescriptionWidget | ValueWidget]
+    output: Output
+    run_button: Button
+    controls_list: list[DescriptionWidget | ValueWidget | Button]
+    controls: VBox
+
+    run_button_description = "Run"
+
+    def __init__(
+        self,
+        parent: Processor,
+        name: str,
+        description: str,
+        fixed_params: dict[str, Any],
+        variable_controls: dict[str, DescriptionWidget | ValueWidget],
+    ):
+        self.parent = parent
+        self.name = name
+        self.description = description
+        self.fixed_params = fixed_params
+        self.variable_controls = variable_controls
+
+        self.events = [
+            f"step_{self.name}_started",
+            f"step_{self.name}_ended",
+        ]
+
+        self.output = Output()
+        self.run_button = Button(
+            description=self.run_button_description, button_style="primary"
+        )
+        self.run_button.on_click(self.button_callback)
+
+        self.controls_list = [
+            *self.variable_controls.values(),
+            self.run_button,
+            self.output,
+        ]
+        self.make_controls()
+
+        super().__init__(parent.broker)
+
+    def make_controls(self):
+        for control in self.controls_list:
+            control.layout.width = "98%"
+            self.parent.all_controls_list.append(control)
+
+        self.controls = VBox(children=self.controls_list)
+
+    @property
+    def variable_params(self):
+        return {key: control.value for key, control in self.variable_controls.items()}
+
+    def button_callback(self, _: Button | None = None):
+        self.run()
+
+    def function(self, *pargs, **kwargs):
+        raise NotImplementedError
+
+    def run(self, **extra_params):
+        self.output.clear_output(wait=False)
+        try:
+            # extra params will override fixed and variable params
+            params = {**self.fixed_params, **self.variable_params, **extra_params}
+
+            self.run_button.disabled = True
+            self.run_button.button_style = "warning"
+            self.run_button.description = "..."
+
+            self.broker.publish(f"step_{self.name}_started")
+            self.function(**params)
+
+            self.parent.append_to_step_history(self.description, params)
+            info = dict(status="success")
+            self.run_button.button_style = "success"
+
+        except Exception as e:
+            self.run_button.button_style = "danger"
+
+            info = dict(status="failed", error=e, traceback=traceback.format_exc())
+            self.broker.exceptions_log.append(traceback.format_exc())
+            with self.broker.exceptions_output.output:
+                display(HTML(f"<pre>{traceback.format_exc()}</pre>"))
+            self.update_output(f"{type(e)}: {e}")
+
+        finally:
+            self.run_button.description = self.run_button_description
+            self.run_button.disabled = False
+            self.broker.publish(f"step_{self.name}_ended", **info)
+
+    def update_output(self, message: str | Any | None, clear: bool = True):
+        if clear:
+            self.output.clear_output(wait=True)
+
+        if isinstance(message, str):
+            message = Markdown(message)
+
+        elif message is None:
+            message = Markdown("")
+
+        with self.output:
+            display(message)

sclab/event/__init__.py

@@ -0,0 +1,7 @@
+from ._broker import EventBroker
+from ._client import EventClient
+
+__all__ = [
+    "EventBroker",
+    "EventClient",
+]

sclab/event/_broker.py

@@ -0,0 +1,201 @@
+import traceback
+from contextlib import contextmanager
+from uuid import uuid4
+
+from IPython.display import HTML, display
+from ipywidgets.widgets import Layout, Output, Tab
+
+from ._client import EventClient
+from ._utils import LogOutput
+
+
+class EventBroker:
+    """Simple event broker for publishing and subscribing to events.
+
+    Example:
+    ```
+    broker = EventBroker()
+
+    def callback(*args, **kwargs):
+        print(args, kwargs)
+
+    broker.subscribe("event", callback)
+    broker.publish("event", 1, 2, 3, a=4, b=5)
+    ```
+    """
+
+    clients: list[EventClient]
+    available_events: set[str]
+    preemptions: dict[str, list[str]]
+    _subscribers: dict[str, list[callable]]
+    _disabled_events: list[str]
+
+    def __init__(self):
+        self.clients = []
+        self.available_events = set()
+        self.preemptions = {}
+        self._subscribers = {}
+        self._disabled_events = []
+        self.depth = 0
+        self.std_output = Output(layout=Layout(width="auto", height="500px"))
+        self.execution_log = []
+        self.execution_output = LogOutput()
+        self.exceptions_log = []
+        self.exceptions_output = LogOutput()
+        self.id = uuid4()
+        self.event_log: Output = Output(
+            layout=Layout(height="200px"),
+            # style={"overflow-y": "scroll"},  TODO: it seems it the way to set this has changed
+        )
+        self.logs_tab = Tab(
+            [
+                self.std_output,
+                self.execution_output,
+                self.exceptions_output,
+            ],
+            titles=[
+                "Standard Output",
+                "Events",
+                "Exceptions",
+            ],
+        )
+
+    def register_client(self, client: EventClient):
+        if not isinstance(client, EventClient):
+            raise TypeError("client must be an instance of EventClient")
+
+        if client in self.clients:
+            return
+
+        self.clients.append(client)
+        self.available_events.update(client.events)
+        self.preemptions.update(client.preemptions)
+        client.broker = self
+
+    def unregister_client(self, client: EventClient):
+        # TODO: ensure full cleanup of subscriptions
+
+        if client in self.clients:
+            self.clients.remove(client)
+
+        self.available_events.difference_update(client.events)
+
+        self._disabled_events = [
+            e for e in self._disabled_events if e not in client.events
+        ]
+
+        for event in client.events:
+            self._subscribers.pop(event, None)
+            self.preemptions.pop(event, None)
+
+    def update_subscriptions(self):
+        for client in self.clients:
+            for event in self.available_events:
+                if callback := getattr(client, f"{event}_callback", None):
+                    self.subscribe(event, callback)
+                    client.subscriptions[event] = callback
+
+    def subscribe(self, event: str, callback: callable):
+        if event not in self._subscribers:
+            self._subscribers[event] = []
+
+        if callback not in self._subscribers[event]:
+            # Prevent duplicate subscriptions
+            self._subscribers[event].append(callback)
+
+    def unsubscribe(self, event: str, callback: callable):
+        if event in self._subscribers and callback in self._subscribers[event]:
+            self._subscribers[event].remove(callback)
+
+    def publish(self, event: str, *args, **kwargs):
+        if event not in self.available_events:
+            raise ValueError(f"Event '{event}' is not available.")
+
+        try:
+            self.depth += 1
+            tab = " " * self.depth * 4
+            txt_args = []
+            for arg in args:
+                if isinstance(arg, str | float | int | bool | tuple | Exception | None):
+                    txt_args.append(arg)
+                else:
+                    txt_args.append(type(arg))
+            txt_kwargs = {}
+            for k, v in kwargs.items():
+                if isinstance(v, str | float | int | bool | tuple | Exception | None):
+                    txt_kwargs[k] = v
+                else:
+                    txt_kwargs[k] = type(v)
+
+            if event in self._disabled_events:
+                msg = f"{tab}Disabled Event: {event}."
+                self.execution_log.append(msg)
+                with self.execution_output.output:
+                    display(HTML(f"<pre>{msg}</pre>"))
+                return
+
+            msg = f"{tab}{event}. Args: {tuple(txt_args)}. Kwargs: {txt_kwargs}"
+            self.execution_log.append(msg)
+            with self.execution_output.output:
+                display(HTML(f"<pre>{msg}</pre>"))
+
+            preempt = self.preemptions.get(event, [])
+            with self.disable(preempt):
+                if event in self._subscribers:
+                    for callback in self._subscribers[event]:
+                        if hasattr(callback, "__self__"):
+                            parent_class = callback.__self__.__class__.__name__
+                        else:
+                            parent_class = "<local context>"
+                        msg = f"{tab}{event} --> {parent_class}.{callback.__name__}()"
+                        self.execution_log.append(msg)
+                        with self.execution_output.output:
+                            display(HTML(f"<pre>{msg}</pre>"))
+
+                        callback(*args, **kwargs)
+        except Exception as e:
+            msg = f"{tab}    Exception: {e}"
+            self.execution_log.append(msg)
+            with self.execution_output.output:
+                display(HTML(f"<pre>{msg}</pre>"))
+
+            msg = f"{msg}{tab}    Exception: {e}"
+            msg += f"\n\n{traceback.format_exc()}"
+            self.exceptions_log.append(msg)
+
+            with self.exceptions_output.output:
+                display(HTML(f"<pre>{traceback.format_exc()}</pre>"))
+
+        finally:
+            self.depth -= 1
+
+    def disable_event(self, event: str):
+        self._disabled_events.append(event)
+
+    def enable_event(self, event: str):
+        if event in self._disabled_events:
+            self._disabled_events.remove(event)
+
+    @contextmanager
+    def disable(self, events: str | list[str]):
+        if isinstance(events, str):
+            events = [events]
+
+        try:
+            for event in events:
+                self.disable_event(event)
+            yield
+        finally:
+            for event in events:
+                self.enable_event(event)
+
+    @contextmanager
+    def delay(self, events: str | list[str]):
+        if isinstance(events, str):
+            events = [events]
+
+        # TODO: implement delay context manager
+        # the idea is to catch these events and execute them after the current event is done
+        # if the delayed event is duplicated, it should be executed only once
+        # we could generate a execution tree and execute the events in the correct order
+        pass

sclab/event/_client.py

@@ -0,0 +1,81 @@
+from uuid import uuid4
+
+from ipywidgets.widgets import Button, Output
+
+
+# forward declaration
+class EventBroker:
+    clients: list["EventClient"]
+    available_events: set[str]
+    preemptions: dict[str, list[str]]
+    _subscribers: dict[str, list[callable]]
+    _disabled_events: list[str]
+    std_output: Output
+
+    def register_client(self, client: "EventClient"):
+        ...
+
+    def unregister_client(self, client: "EventClient"):
+        ...
+
+    def update_subscriptions(self):
+        ...
+
+    def subscribe(self, event: str, callback: callable):
+        ...
+
+    def unsubscribe(self, event: str, callback: callable):
+        ...
+
+    def publish(self, event: str, *args, **kwargs):
+        ...
+
+    def disable_event(self, event: str):
+        ...
+
+    def enable_event(self, event: str):
+        ...
+
+
+class EventClient:
+    subscriptions: dict[str, callable]
+    preemptions: dict[str, list[str]] = {}  # override this in subclasses if needed
+    broker: "EventBroker"
+    events: list[str]
+
+    def __init__(self, broker: "EventBroker"):
+        self.uuid = uuid4().hex
+        self.subscriptions = {}
+        broker.register_client(self)
+        broker.update_subscriptions()
+
+    def button_click_event_publisher(self, control_prefix: str, control_name: str):
+        event_str = f"{control_prefix}_{control_name}_click"
+        assert (
+            event_str in self.events
+        ), f"Event {event_str} not in {self.__class__.__name__}.events list."
+
+        def publisher(_: Button):
+            self.broker.publish(event_str)
+
+        return publisher
+
+    def control_value_change_event_publisher(
+        self, control_prefix: str, control_name: str
+    ):
+        event_str = f"{control_prefix}_{control_name}_change"
+        assert (
+            event_str in self.events
+        ), f"Event {event_str} not in {self.__class__.__name__}.events list."
+
+        def publisher(event: dict):
+            if event["type"] != "change":
+                raise ValueError("Event type must be 'change'")
+            new_value = event["new"]
+            self.broker.publish(event_str, new_value=new_value)
+
+        return publisher
+
+    def __del__(self):
+        for event, callback in self.subscriptions.items():
+            self.broker.unsubscribe(event, callback)

sclab/event/_utils.py

@@ -0,0 +1,14 @@
+from ipywidgets.widgets import Button, Layout, Output, VBox
+
+
+class LogOutput(VBox):
+    def __init__(self):
+        self.clear_button = Button(description="Clear", button_style="warning")
+        self.output = Output(layout=Layout(width="auto", height="500px"))
+
+        super().__init__(children=[self.clear_button, self.output])
+
+        self.clear_button.on_click(self._clear)
+
+    def _clear(self, _):
+        self.output.clear_output()

sclab/examples/__init__.py

@@ -0,0 +1,5 @@
+from . import processor_steps
+
+__all__ = [
+    "processor_steps",
+]

sclab/examples/processor_steps/__init__.py

@@ -0,0 +1,15 @@
+from ._cluster import Cluster
+from ._neighbors import Neighbors
+from ._pca import PCA
+from ._preprocess import Preprocess
+from ._qc import QC
+from ._umap import UMAP
+
+__all__ = [
+    "QC",
+    "Preprocess",
+    "PCA",
+    "Neighbors",
+    "UMAP",
+    "Cluster",
+]

sclab/examples/processor_steps/_cluster.py

@@ -0,0 +1,37 @@
+from ipywidgets import FloatSlider
+
+from sclab.dataset.processor import Processor
+from sclab.dataset.processor.step import ProcessorStepBase
+
+
+class Cluster(ProcessorStepBase):
+    parent: Processor
+
+    def __init__(self, parent: Processor) -> None:
+        try:
+            import scanpy as sc  # noqa: F401
+        except ImportError:
+            raise ImportError("Please install scanpy: `pip install scanpy`")
+
+        variable_controls = dict(
+            resolution=FloatSlider(
+                value=1.0, min=0.1, max=10.0, step=0.1, description="Resolution"
+            )
+        )
+
+        super().__init__(
+            parent=parent,
+            name="cluster",
+            description="Cluster",
+            fixed_params={},
+            variable_controls=variable_controls,
+        )
+
+    def function(self, resolution: float = 1.0):
+        import scanpy as sc
+
+        dataset = self.parent.dataset
+        adata = self.parent.dataset.adata
+        sc.tl.leiden(adata, resolution=resolution)
+
+        self.broker.publish("dset_metadata_change", dataset.metadata, "leiden")

sclab/examples/processor_steps/_neighbors.py

@@ -0,0 +1,72 @@
+from ipywidgets import Dropdown, IntText
+
+from sclab.dataset.processor import Processor
+from sclab.dataset.processor.step import ProcessorStepBase
+
+
+class Neighbors(ProcessorStepBase):
+    parent: Processor
+
+    def __init__(self, parent: Processor) -> None:
+        try:
+            import scanpy as sc  # noqa: F401
+        except ImportError:
+            raise ImportError("Please install scanpy: `pip install scanpy`")
+
+        variable_controls = dict(
+            use_rep=Dropdown(
+                options=tuple(parent.dataset.adata.obsm.keys()),
+                value=None,
+                description="Use rep.",
+            ),
+            n_neighbors=IntText(value=20, description="N neighbors"),
+            n_dims=IntText(value=10, description="N Dims"),
+            metric=Dropdown(
+                options=["euclidean", "cosine"],
+                value="euclidean",
+                description="Metric",
+            ),
+            **parent.make_groupbybatch_checkbox(),
+        )
+
+        super().__init__(
+            parent=parent,
+            name="neighbors",
+            description="Neighbors",
+            fixed_params={},
+            variable_controls=variable_controls,
+        )
+
+    def function(
+        self,
+        n_neighbors: int = 20,
+        use_rep: str = "X_pca",
+        n_dims: int = 10,
+        metric: str = "euclidean",
+        group_by_batch: bool = False,
+    ):
+        import scanpy as sc
+
+        adata = self.parent.dataset.adata
+
+        if group_by_batch and self.parent.batch_key:
+            group_by = self.parent.batch_key
+            sc.external.pp.bbknn(
+                adata,
+                batch_key=group_by,
+                use_rep=use_rep,
+                n_pcs=n_dims,
+                use_annoy=False,
+                metric=metric,
+                pynndescent_n_neighbors=n_neighbors,
+            )
+        else:
+            sc.pp.neighbors(
+                adata,
+                n_neighbors=n_neighbors,
+                use_rep=use_rep,
+                n_pcs=n_dims,
+                metric=metric,
+            )
+
+        self.broker.publish("dset_anndata_neighbors_change")