decider 0.0.1__py3-none-any.whl

decider/__init__.py

@@ -0,0 +1,7 @@
+from .initialization import initialize_decider
+
+try:
+    from importlib.metadata import version, PackageNotFoundError
+    __version__ = version("decider")
+except PackageNotFoundError:
+    __version__ = "unknown"

decider/_ext.py

@@ -0,0 +1,106 @@
+import inspect
+import typing as t
+from abc import ABC
+from pydantic import create_model, RootModel, BaseModel, Field, model_validator
+from warnings import warn
+
+
+class TypeDiscriminatedBaseModule(BaseModel, ABC):
+    type: str 
+
+    _CLASS_TYPE_IDENTIFIER: t.ClassVar[str]
+
+    def __init_subclass__(cls, **kwargs):
+        """
+        We are basically using the below to ensure:
+        1. the class implements a type: Literal['value'] so we can use that as a discriminator for the union of all implementations of this class
+        2. We dont want there to be type: Literal['value'] = 'value' on the class because we making use of pydantic.model_dump(exclude_defaults=True) to exclude the type field when saving out modules, and if there is a default value then it will not be included in the dumdecider dict which breaks loading it back in.
+        3. We want to store what the value of Literal is so we can automatically initialise it when we construct the model Model() rather than needing Model(type='value') every time
+        """
+        super().__init_subclass__(**kwargs)
+
+        # Skip abstract classes. as this will be used as a base for multiple implementations.
+        if inspect.isabstract(cls):
+            return
+
+        # Ensure `type` declared
+        if "type" not in cls.__annotations__:
+            raise TypeError(f"{cls.__name__} must define a 'type' annotation")
+
+        annotation = cls.__annotations__["type"]
+
+        if t.get_origin(annotation) is not t.Literal:
+            raise TypeError(
+                f"{cls.__name__}.type must be typing.Literal[...]"
+            )
+
+        literal_values = t.get_args(annotation)
+
+        if len(literal_values) != 1:
+            raise TypeError(
+                f"{cls.__name__}.type must be a single-value Literal"
+            )
+
+        if "type" in cls.__dict__:
+            raise TypeError(
+                f"{cls.__name__}.type must not define a default value"
+            )
+
+        cls._CLASS_TYPE_IDENTIFIER = literal_values[0]
+
+    @model_validator(mode="before")
+    @classmethod
+    def auto_set_type(cls, values):
+        if isinstance(values, dict) and not inspect.isabstract(cls):
+            values.setdefault("type", cls._CLASS_TYPE_IDENTIFIER)
+        return values
+
+
+_TExtenableRootType = t.TypeVar("_TExtenableRootType")
+
+class TExtendableModel(RootModel[_TExtenableRootType], t.Generic[_TExtenableRootType]):
+    root: _TExtenableRootType
+
+
+def create_extendable_model(
+    base_class: t.Type, 
+    discriminator_field: str = "type", 
+    model_name: str = "ExtendableModel"
+) -> t.Tuple[t.Type[TExtendableModel], t.Callable[[t.Type], None]]:
+    """
+    Creates an extendable model pattern that allows external packages to register
+    new types without creating hard dependencies.
+    
+    Returns a tuple of (Model class, register function).
+    """
+    
+    ExtendableModel = create_model(
+        model_name,
+        __base__=RootModel,
+        root=("RootType", ...)
+    )
+    
+    _registered: t.Dict[str, t.Type] = {}
+
+    def _rebuild():
+        classes = list(_registered.values())
+        if not classes:
+            return
+        union = classes[0] if len(classes) == 1 else t.Union[tuple(classes)]
+        annotated = t.Annotated[union, Field(discriminator=discriminator_field)]
+        ExtendableModel.__annotations__["root"] = annotated
+        ExtendableModel.model_fields["root"].annotation = annotated
+        was_rebuilt = ExtendableModel.model_rebuild(
+            force=True,
+            _types_namespace={"RootType": annotated},
+        )
+        if was_rebuilt is not True:
+            warn(f"model_rebuild did not return True for {ExtendableModel.__name__}")
+
+    def register_provider(provider_class: t.Type):
+        assert issubclass(provider_class, base_class), f"Provider must be a subclass of {base_class.__name__}"
+        type_id = getattr(provider_class, "_CLASS_TYPE_IDENTIFIER", None) or provider_class.__name__
+        _registered[type_id] = provider_class
+        _rebuild()
+    
+    return ExtendableModel, register_provider

decider/cli/__init__.py

@@ -0,0 +1,15 @@
+import click
+
+from .template import template
+from .serve import serve
+from .visualise import visualise
+
+
+@click.group()
+def cli():
+    """Decider — build, serve and inspect decision pipelines."""
+
+
+cli.add_command(template)
+cli.add_command(serve)
+cli.add_command(visualise)

decider/cli/_graph.py

@@ -0,0 +1,310 @@
+"""
+Walk a BaseModule tree and produce graph structures for the visualiser.
+
+Two graph types:
+  build_graph(module)            — module-level structural graph (pipeline view)
+  build_expression_graph(module) — expression node DAG inside one ExpressionModule
+
+Both return ModuleGraph.  The module_ref on each GraphNode holds the live
+module object so the app can drill into it.
+"""
+
+import typing as t
+from dataclasses import dataclass, field
+
+
+@dataclass
+class GraphNode:
+    id: str
+    label: str
+    kind: str          # "expression" | "sequential" | "join" | "union" | "col" | "config" | "unknown"
+    type_id: str
+    parent: t.Optional[str] = None
+    fields: t.Dict[str, t.Any] = field(default_factory=dict)
+    module_ref: t.Any = None       # live BaseModule if drillable
+    drillable: bool = False
+
+
+@dataclass
+class GraphEdge:
+    source: str
+    target: str
+    label: str = ""
+
+
+@dataclass
+class ModuleGraph:
+    nodes: t.List[GraphNode] = field(default_factory=list)
+    edges: t.List[GraphEdge] = field(default_factory=list)
+
+    def to_graphviz(self) -> "graphviz.Digraph":
+        import graphviz
+        dot = graphviz.Digraph(graph_attr={"rankdir": "TB", "splines": "ortho"})
+
+        _KIND_COLOURS = {
+            "expression": "#4C9BE8",
+            "sequential": "#E8884C",
+            "join":       "#4CE8A0",
+            "union":      "#9B4CE8",
+            "col":        "#888888",
+            "config":     "#C8A850",
+            "unknown":    "#AAAAAA",
+        }
+
+        for n in self.nodes:
+            colour = _KIND_COLOURS.get(n.kind, "#AAAAAA")
+            tooltip = "\n".join(f"{k}: {v}" for k, v in n.fields.items()) or n.type_id
+            shape = "ellipse" if n.kind in ("col", "config") else "box"
+            border = "bold" if n.drillable else ""
+            dot.node(
+                n.id,
+                label=n.label,
+                shape=shape,
+                style=f"filled,rounded,{border}".strip(","),
+                fillcolor=colour,
+                fontcolor="white",
+                tooltip=tooltip,
+            )
+        for e in self.edges:
+            dot.edge(e.source, e.target, label=e.label)
+        return dot
+
+
+# ── helpers ───────────────────────────────────────────────────────────────────
+
+def _kind(module) -> str:
+    type_id = getattr(module, "type", "")
+    if type_id == "sequential":
+        return "sequential"
+    if type_id == "join":
+        return "join"
+    if type_id == "union":
+        return "union"
+    if hasattr(module, "expand_nodes"):
+        return "expression"
+    return "unknown"
+
+
+def _config_fields(module) -> t.Dict[str, t.Any]:
+    _SKIP = {"type", "name", "steps", "modules", "left", "right", "on", "how"}
+    try:
+        raw = module.model_dump(exclude_defaults=False)
+    except Exception:
+        return {}
+    return {k: v for k, v in raw.items() if k not in _SKIP and not k.startswith("_")}
+
+
+# ── module-level graph ────────────────────────────────────────────────────────
+
+def _walk(
+    module,
+    graph: ModuleGraph,
+    parent_id: t.Optional[str] = None,
+    counter: t.Optional[t.List[int]] = None,
+) -> str:
+    if counter is None:
+        counter = [0]
+
+    counter[0] += 1
+    node_id = f"node_{counter[0]}"
+    type_id = getattr(module, "type", type(module).__name__)
+    name = getattr(module, "name", type_id)
+    kind = _kind(module)
+    drillable = kind in ("expression", "sequential", "join", "union")
+
+    graph.nodes.append(GraphNode(
+        id=node_id,
+        label=name,
+        kind=kind,
+        type_id=type_id,
+        parent=parent_id,
+        fields=_config_fields(module),
+        module_ref=module,
+        drillable=drillable,
+    ))
+
+    if parent_id is not None:
+        graph.edges.append(GraphEdge(source=parent_id, target=node_id))
+
+    if kind == "sequential":
+        prev = node_id
+        for step in module.steps:
+            child_id = _walk(step, graph, parent_id=node_id, counter=counter)
+            if graph.edges and graph.edges[-1].source == node_id:
+                graph.edges[-1] = GraphEdge(source=prev, target=child_id, label="then")
+            prev = child_id
+
+    elif kind == "join":
+        for side, ref in (("left", module.left), ("right", module.right)):
+            if hasattr(ref, "type"):
+                child_id = _walk(ref, graph, parent_id=node_id, counter=counter)
+                if graph.edges:
+                    graph.edges[-1].label = side
+            else:
+                fid = f"frame_{ref}_{counter[0]}"
+                counter[0] += 1
+                graph.nodes.append(GraphNode(
+                    id=fid, label=f'"{ref}"', kind="col", type_id="frame", parent=node_id,
+                ))
+                graph.edges.append(GraphEdge(source=node_id, target=fid, label=side))
+
+    elif kind == "union":
+        for child_mod in module.modules:
+            _walk(child_mod, graph, parent_id=node_id, counter=counter)
+
+    return node_id
+
+
+def build_graph(module) -> ModuleGraph:
+    """Module-level structural graph for any BaseModule tree."""
+    g = ModuleGraph()
+    _walk(module, g)
+    return g
+
+
+# ── expression node DAG ───────────────────────────────────────────────────────
+
+def build_expression_graph(module) -> ModuleGraph:
+    """
+    Return a computation DAG for an ExpressionModule showing individual
+    function nodes, their column inputs, and config injections.
+    """
+    from decider.modules.expression import ExternalInputNode, StaticValueNode, Node as ExprNode
+
+    g = ModuleGraph()
+    nodes = module.expand_nodes()
+
+    # add a function node for every expression node
+    for name, expr_node in nodes.items():
+        g.nodes.append(GraphNode(
+            id=f"fn_{name}",
+            label=name,
+            kind="expression",
+            type_id="function",
+            drillable=False,
+        ))
+
+    # add edges: inputs → function nodes
+    for name, expr_node in nodes.items():
+        for param, ref in expr_node.input_map.items():
+            if isinstance(ref, ExprNode):
+                g.edges.append(GraphEdge(source=f"fn_{ref.name}", target=f"fn_{name}", label=param))
+            elif isinstance(ref, ExternalInputNode):
+                col_id = f"col_{ref.input_name}"
+                if not any(n.id == col_id for n in g.nodes):
+                    g.nodes.append(GraphNode(
+                        id=col_id,
+                        label=ref.input_name,
+                        kind="col",
+                        type_id="column",
+                        drillable=False,
+                    ))
+                g.edges.append(GraphEdge(source=col_id, target=f"fn_{name}", label=param))
+            elif isinstance(ref, StaticValueNode):
+                val = ref.value
+                cfg_id = f"cfg_{name}_{param}"
+                # show the config type name, not the full repr
+                cfg_label = type(val).__name__ if hasattr(val, "__class__") else str(val)
+                if not any(n.id == cfg_id for n in g.nodes):
+                    g.nodes.append(GraphNode(
+                        id=cfg_id,
+                        label=cfg_label,
+                        kind="config",
+                        type_id="config",
+                        drillable=False,
+                        fields=val.model_dump() if hasattr(val, "model_dump") else {},
+                    ))
+                g.edges.append(GraphEdge(source=cfg_id, target=f"fn_{name}", label=param))
+
+    return g
+
+
+# ── intermediate value extraction ─────────────────────────────────────────────
+
+def run_with_intermediates(
+    module,
+    inputs: t.Dict[str, "pl.DataFrame"],
+) -> t.List[t.Tuple[str, "pl.DataFrame"]]:
+    """
+    Execute module and return a list of (label, DataFrame) pairs, one per
+    logical step, in execution order.
+
+    - ExpressionModule  → one entry per compiled expression column, accumulated
+    - SequentialModule  → one entry per step
+    - Others            → single entry with final output
+    """
+    import polars as pl
+
+    kind = _kind(module)
+
+    if kind == "expression":
+        return _run_expression_intermediates(module, inputs)
+    elif kind == "sequential":
+        return _run_sequential_intermediates(module, inputs)
+    elif kind == "join":
+        return _run_join_intermediates(module, inputs)
+    else:
+        out = module(inputs)
+        if isinstance(out, pl.LazyFrame):
+            out = out.collect()
+        return [(getattr(module, "name", "output"), out)]
+
+
+def _run_expression_intermediates(
+    module,
+    inputs: t.Dict[str, "pl.DataFrame"],
+) -> t.List[t.Tuple[str, "pl.DataFrame"]]:
+    import polars as pl
+
+    module.compile_expressions()
+    ce = module._compiled_expressions
+    frame = inputs.get(ce.input_frame)
+    if frame is None:
+        frame = next(iter(inputs.values()))
+    if isinstance(frame, pl.DataFrame):
+        frame = frame.lazy()
+
+    results = []
+    accumulated = frame
+    for col_name, expr in ce.expressions.items():
+        accumulated = accumulated.with_columns(expr.alias(col_name))
+        snapshot = accumulated.collect()
+        results.append((col_name, snapshot))
+
+    return results
+
+
+def _run_sequential_intermediates(
+    module,
+    inputs: t.Dict[str, "pl.DataFrame"],
+) -> t.List[t.Tuple[str, "pl.DataFrame"]]:
+    import polars as pl
+
+    frames = {
+        k: v.lazy() if isinstance(v, pl.DataFrame) else v
+        for k, v in inputs.items()
+    }
+    current = frames.get("input") if "input" in frames else next(iter(frames.values()))
+
+    results = []
+    for step in module.steps:
+        frames["input"] = current
+        out = step(frames)
+        if isinstance(out, pl.LazyFrame):
+            out = out.collect()
+        current = out.lazy()
+        results.append((getattr(step, "name", step.type), out))
+
+    return results
+
+
+def _run_join_intermediates(
+    module,
+    inputs: t.Dict[str, "pl.DataFrame"],
+) -> t.List[t.Tuple[str, "pl.DataFrame"]]:
+    import polars as pl
+
+    out = module(inputs)
+    if isinstance(out, pl.LazyFrame):
+        out = out.collect()
+    return [(getattr(module, "name", "join"), out)]

decider/cli/_visualise_app.py

@@ -0,0 +1,236 @@
+"""
+Streamlit app — launched by `decider visualise`.
+
+Env vars:
+    DECIDER_VISUALISE_PROJECT_DIR
+    DECIDER_VISUALISE_EXT_DIR       (optional)
+    DECIDER_VISUALISE_CONFIG_DIR    (optional)
+    DECIDER_VISUALISE_ROOT_MODULE   (optional, default "main")
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+import polars as pl
+import streamlit as st
+
+# ── bootstrap ─────────────────────────────────────────────────────────────────
+
+_project_dir = Path(os.environ.get("DECIDER_VISUALISE_PROJECT_DIR", ".")).resolve()
+_repo_root = _project_dir.parent.parent
+if str(_repo_root) not in sys.path:
+    sys.path.insert(0, str(_repo_root))
+
+_ext_dir = os.environ.get("DECIDER_VISUALISE_EXT_DIR",
+                           str(_project_dir / "decider_extensions"))
+_configs_dir = os.environ.get("DECIDER_VISUALISE_CONFIG_DIR",
+                               str(_project_dir / "configs"))
+_root_module = os.environ.get("DECIDER_VISUALISE_ROOT_MODULE", "main")
+
+from decider.initialization import initialize_decider
+from decider.config.file import JsonFileConfigManager
+from decider.modules import GraphModule
+from decider.cli._graph import (
+    build_graph,
+    build_expression_graph,
+    run_with_intermediates,
+    _kind,
+)
+
+# ── page setup ────────────────────────────────────────────────────────────────
+
+st.set_page_config(
+    page_title="Decider Visualise",
+    layout="wide",
+    initial_sidebar_state="expanded",
+)
+
+# ── session state init ────────────────────────────────────────────────────────
+
+if "breadcrumb" not in st.session_state:
+    # Each entry: {"label": str, "module": BaseModule}
+    st.session_state.breadcrumb = []
+
+if "run_inputs" not in st.session_state:
+    st.session_state.run_inputs = None   # Dict[str, pl.DataFrame] when set
+
+# ── load root module (cached) ─────────────────────────────────────────────────
+
+@st.cache_resource
+def _load_root(root_key: str):
+    initialize_decider(extension_path=_ext_dir)
+    import asyncio
+    mgr = JsonFileConfigManager(basepath=_configs_dir)
+    versioned = asyncio.run(mgr.get_latest())
+    module = GraphModule.model_validate(versioned.config[root_key]).root
+    return module, versioned
+
+
+# ── sidebar ───────────────────────────────────────────────────────────────────
+
+with st.sidebar:
+    st.header("Project")
+    st.caption(str(_project_dir))
+
+    root_key = st.text_input("Root module key", value=_root_module)
+    if st.button("↺  Reload config"):
+        st.cache_resource.clear()
+        st.session_state.breadcrumb = []
+        st.session_state.run_inputs = None
+        st.rerun()
+
+try:
+    root_module, versioned = _load_root(root_key)
+except Exception as e:
+    st.error(f"Could not load module: {e}")
+    st.stop()
+
+with st.sidebar:
+    st.divider()
+    st.caption(f"version  {versioned.version}")
+    st.caption(f"type     {root_module.type}")
+
+    # ── input data entry ──────────────────────────────────────────────────────
+    st.divider()
+    st.subheader("Run data")
+    st.caption("Paste JSON (column-oriented) to push data through the pipeline.")
+
+    default_cols = root_module.get_input_frame_keys()
+    json_placeholder = json.dumps(
+        {k: ["value1", "value2"] for k in
+         (root_module._compute_input_frame_keys() if hasattr(root_module, '_compute_input_frame_keys') else ["input"])},
+        indent=2,
+    )
+    raw_json = st.text_area("Input JSON", value="", height=180,
+                             placeholder=json_placeholder)
+    if st.button("▶  Run"):
+        try:
+            parsed = json.loads(raw_json)
+            # support both {col: [...]} (single frame) and {"frame": {col: [...]}}
+            if parsed and isinstance(next(iter(parsed.values())), dict):
+                st.session_state.run_inputs = {
+                    k: pl.DataFrame(v) for k, v in parsed.items()
+                }
+            else:
+                st.session_state.run_inputs = {"input": pl.DataFrame(parsed)}
+        except Exception as e:
+            st.error(f"Invalid JSON: {e}")
+
+    if st.session_state.run_inputs is not None:
+        if st.button("✕  Clear run"):
+            st.session_state.run_inputs = None
+            st.rerun()
+
+
+# ── breadcrumb navigation ─────────────────────────────────────────────────────
+
+# current module is root unless the user has drilled in
+crumb_stack = st.session_state.breadcrumb
+current_module = crumb_stack[-1]["module"] if crumb_stack else root_module
+
+# render breadcrumb bar
+crumb_parts = [{"label": root_key, "module": root_module}] + crumb_stack
+cols = st.columns([1] * len(crumb_parts) + [8])
+for i, crumb in enumerate(crumb_parts):
+    with cols[i]:
+        is_last = i == len(crumb_parts) - 1
+        if is_last:
+            st.markdown(f"**{crumb['label']}**")
+        else:
+            if st.button(crumb["label"], key=f"crumb_{i}"):
+                st.session_state.breadcrumb = crumb_stack[: i]   # pop back to i
+                st.rerun()
+
+if crumb_stack:
+    st.caption(f"type: {current_module.type}  ·  name: {current_module.name}")
+
+st.divider()
+
+# ── main content: tabs ────────────────────────────────────────────────────────
+
+tab_graph, tab_run, tab_config = st.tabs(["Graph", "Run output", "Config"])
+
+# ── TAB: Graph ────────────────────────────────────────────────────────────────
+
+with tab_graph:
+    kind = _kind(current_module)
+
+    if kind == "expression":
+        # show the intra-module expression DAG
+        st.caption("Expression node DAG — functions, column inputs and config injections")
+        eg = build_expression_graph(current_module)
+        dot = eg.to_graphviz()
+        st.graphviz_chart(dot.source, use_container_width=True)
+
+        # node table
+        rows = [{"node": n.label, "kind": n.kind,
+                 **{f"cfg:{k}": v for k, v in n.fields.items()}}
+                for n in eg.nodes]
+        if rows:
+            st.dataframe(pl.DataFrame(rows, infer_schema_length=len(rows)),
+                         use_container_width=True)
+
+    else:
+        # show the module-level structural graph
+        g = build_graph(current_module)
+        col_g, col_d = st.columns([2, 1])
+
+        with col_g:
+            dot = g.to_graphviz()
+            st.graphviz_chart(dot.source, use_container_width=True)
+
+        with col_d:
+            st.subheader("Modules")
+            for n in g.nodes:
+                if not n.drillable:
+                    continue
+                c1, c2 = st.columns([4, 1])
+                with c1:
+                    tag = f"`{n.type_id}`"
+                    cfg = "  ·  " + "  ".join(f"{k}={v}" for k, v in n.fields.items()) if n.fields else ""
+                    st.markdown(f"**{n.label}** {tag}{cfg}")
+                with c2:
+                    if st.button("→", key=f"drill_{n.id}",
+                                 help=f"Drill into {n.label}"):
+                        st.session_state.breadcrumb = crumb_stack + [
+                            {"label": n.label, "module": n.module_ref}
+                        ]
+                        st.rerun()
+
+# ── TAB: Run output ───────────────────────────────────────────────────────────
+
+with tab_run:
+    if st.session_state.run_inputs is None:
+        st.info("Paste input data in the sidebar and click **▶ Run** to see intermediate outputs.")
+    else:
+        inputs = st.session_state.run_inputs
+
+        st.subheader("Input")
+        for frame_key, df in inputs.items():
+            st.caption(f"frame: `{frame_key}`")
+            st.dataframe(df, use_container_width=True)
+
+        st.subheader("Intermediates")
+        try:
+            intermediates = run_with_intermediates(current_module, inputs)
+        except Exception as e:
+            st.error(f"Execution error: {e}")
+            intermediates = []
+
+        for label, df in intermediates:
+            with st.expander(f"after  **{label}**", expanded=True):
+                # highlight newly-added columns vs the input
+                input_cols = set(next(iter(inputs.values())).columns)
+                new_cols = [c for c in df.columns if c not in input_cols]
+                st.caption(f"new columns: {', '.join(new_cols) if new_cols else '(none)'}")
+                st.dataframe(df, use_container_width=True)
+
+# ── TAB: Config ───────────────────────────────────────────────────────────────
+
+with tab_config:
+    try:
+        st.json(current_module.model_dump())
+    except Exception:
+        st.json(versioned.config)