vedana-backoffice 0.1.0__py3-none-any.whl

vedana_backoffice/Caddyfile

@@ -0,0 +1,17 @@
+:9000
+
+encode gzip
+
+@backend_routes path /_event/* /ping /_upload /_upload/*
+handle @backend_routes {
+	reverse_proxy localhost:8000 {
+		header_up Upgrade {http.request.header.upgrade}
+		header_up Connection {http.request.header.connection}
+	}
+}
+
+root * /srv
+route {
+	try_files {path} {path}/ /404.html
+	file_server
+}

vedana_backoffice/components/etl_graph.py

@@ -0,0 +1,132 @@
+import reflex as rx
+
+from vedana_backoffice.states.etl import EtlState
+
+
+def _node_card(node: dict) -> rx.Component:
+    is_table = node.get("node_type") == "table"  # step for transform, table for table
+    return rx.card(
+        rx.vstack(
+            rx.hstack(
+                rx.hstack(
+                    rx.cond(
+                        is_table,
+                        rx.box(),
+                        rx.badge(node.get("step_type", ""), color_scheme="indigo", variant="soft"),
+                    ),
+                    spacing="2",
+                ),
+                rx.text(node.get("name", ""), weight="medium"),
+                justify="between",
+                width="100%",
+            ),
+            rx.text(node.get("labels_str", ""), size="1", color="gray"),
+            rx.cond(
+                is_table,
+                rx.hstack(
+                    rx.tooltip(rx.text(node.get("last_run", "—"), size="1", color="gray"), content="last update time"),
+                    rx.hstack(
+                        rx.tooltip(
+                            rx.text(node.get("row_count", "—"), size="1", color="gray", weight="bold"),
+                            content="rows total",
+                        ),
+                        rx.tooltip(
+                            rx.text(node.get("last_add", "—"), size="1", color="green"), content="added during last run"
+                        ),
+                        rx.text("/", size="1", color="gray"),
+                        rx.tooltip(
+                            rx.text(node.get("last_upd", "—"), size="1", color="gray"),
+                            content="updated during last run",
+                        ),
+                        rx.text("/", size="1", color="gray"),
+                        rx.tooltip(
+                            rx.text(node.get("last_rm", "—"), size="1", color="red"), content="deleted during last run"
+                        ),
+                        spacing="1",
+                    ),
+                    width="100%",
+                    justify="between",
+                ),
+                # Step view: show last run time and rows processed
+                rx.cond(
+                    node.get("step_type") != "BatchGenerate",  # BatchGenerate has no meta table
+                    rx.hstack(
+                        rx.tooltip(
+                            rx.text(node.get("last_run", "—"), size="1", color="gray"),
+                            content="last run time (that produced changes)",
+                        ),
+                        rx.hstack(
+                            rx.tooltip(
+                                rx.text(node.get("rows_processed", 0), size="1", color="gray"),
+                                content="rows processed in last run",
+                            ),
+                            rx.text("/", size="1", color="gray"),
+                            rx.tooltip(
+                                rx.text(node.get("total_success", 0), size="1", color="gray", weight="bold"),
+                                content="rows processed total",
+                            ),
+                            rx.cond(
+                                node.get("has_total_failed", False),
+                                rx.tooltip(
+                                    rx.text(node.get("total_failed_str", ""), size="1", color="red"),
+                                    content="total failed rows (all time)",
+                                ),
+                                rx.box(),
+                            ),
+                            spacing="1",
+                        ),
+                        width="100%",
+                        justify="between",
+                    ),
+                    rx.box(),
+                ),
+            ),
+            spacing="2",
+            width="100%",
+        ),
+        padding="0.75em",
+        style={
+            "position": "absolute",
+            "left": node.get("left", "0px"),
+            "top": node.get("top", "0px"),
+            "width": node.get("width", "420px"),
+            "height": "auto",
+            "border": node.get("border_css", "1px solid #e5e7eb"),
+            "overflow": "visible",
+            "boxSizing": "border-box",
+        },
+        variant="surface",
+        on_click=rx.cond(
+            is_table,
+            # no direct return values here and that's ok, handled in state
+            EtlState.preview_table(table_name=node.get("name", "")),  # type: ignore
+            EtlState.toggle_node_selection(index=node.get("index_value")),  # type: ignore
+        ),
+    )
+
+
+def etl_graph() -> rx.Component:
+    svg = rx.box(
+        rx.html(EtlState.graph_svg),
+        style={
+            "position": "absolute",
+            "left": 0,
+            "top": 0,
+            "pointerEvents": "none",
+            "width": "100%",
+            "height": "100%",
+        },
+    )
+
+    nodes_layer = rx.box(
+        rx.foreach(EtlState.graph_nodes, _node_card),
+        style={
+            "position": "absolute",
+            "left": 0,
+            "top": 0,
+            "width": "100%",
+            "height": "100%",
+        },
+    )
+
+    return rx.box(svg, nodes_layer, style={"position": "relative", "width": "100%", "height": "100%"})

vedana_backoffice/components/ui_chat.py

@@ -0,0 +1,236 @@
+import reflex as rx
+
+
+def render_message_bubble(
+    msg: dict,
+    on_toggle_details,
+    extras: rx.Component | None = None,
+    corner_tags_component: rx.Component | None = None,
+) -> rx.Component:  # type: ignore[valid-type]
+    """Render a chat-style message bubble.
+
+    Expects msg dict with keys:
+      - content, is_assistant (bool-like), created_at_fmt or created_at
+      - has_tech, has_logs, show_details
+      - has_models, has_vts, has_cypher, models_str, vts_str, cypher_str (optional)
+      - logs_str (optional)
+    """
+
+    tech_block = rx.cond(
+        msg.get("has_tech"),
+        rx.card(
+            rx.vstack(
+                rx.cond(
+                    msg.get("has_models"),
+                    rx.vstack(
+                        rx.text("Models", weight="medium"),
+                        rx.code_block(
+                            msg.get("models_str", ""),
+                            font_size="11px",
+                            language="json",
+                            # (bug in reflex?) code_block does not pass some custom styling (wordBreak, whiteSpace)
+                            # https://github.com/reflex-dev/reflex/issues/6051
+                            code_tag_props={"style": {"whiteSpace": "pre-wrap"}},
+                            style={
+                                "display": "block",
+                                "maxWidth": "100%",
+                                "boxSizing": "border-box",
+                            },
+                        ),
+                        spacing="1",
+                        width="100%",
+                    ),
+                ),
+                rx.cond(
+                    msg.get("has_vts"),
+                    rx.vstack(
+                        rx.text("VTS Queries", weight="medium"),
+                        rx.code_block(
+                            msg.get("vts_str", ""),
+                            font_size="11px",
+                            language="python",
+                            code_tag_props={"style": {"whiteSpace": "pre-wrap"}},
+                            style={
+                                "display": "block",
+                                "maxWidth": "100%",
+                                "boxSizing": "border-box",
+                            },
+                        ),
+                        spacing="1",
+                        width="100%",
+                    ),
+                ),
+                rx.cond(
+                    msg.get("has_cypher"),
+                    rx.vstack(
+                        rx.text("Cypher Queries", weight="medium"),
+                        rx.code_block(
+                            msg.get("cypher_str", ""),
+                            font_size="11px",
+                            language="cypher",
+                            code_tag_props={"style": {"whiteSpace": "pre-wrap"}},
+                            style={
+                                "display": "block",
+                                "maxWidth": "100%",
+                                "boxSizing": "border-box",
+                            },
+                        ),
+                        spacing="1",
+                        width="100%",
+                    ),
+                ),
+                spacing="2",
+                width="100%",
+            ),
+            padding="0.75em",
+            variant="surface",
+        ),
+        rx.box(),
+    )
+
+    generic_details_block = rx.cond(
+        msg.get("generic_meta"),
+        rx.code_block(
+            msg.get("event_data_str", ""),
+            font_size="11px",
+            language="json",
+            code_tag_props={"style": {"whiteSpace": "pre-wrap"}},
+            style={
+                "display": "block",
+                "maxWidth": "100%",
+                "boxSizing": "border-box",
+            },
+        ),
+        rx.box(),
+    )
+
+    logs_block = rx.cond(
+        msg.get("has_logs"),
+        rx.card(
+            rx.vstack(
+                rx.text("Logs", weight="medium"),
+                rx.scroll_area(
+                    rx.code_block(
+                        msg.get("logs_str", ""),
+                        font_size="11px",
+                        language="log",
+                        wrap_long_lines=True,
+                        style={
+                            "display": "block",
+                            "maxWidth": "100%",
+                            "boxSizing": "border-box",
+                        },
+                        code_tag_props={"style": {"whiteSpace": "pre-wrap"}},  # styling workaround
+                    ),
+                    type="always",
+                    scrollbars="vertical",
+                    style={
+                        "maxHeight": "25vh",
+                        "width": "100%",
+                    },
+                ),
+                spacing="1",
+                width="100%",
+            ),
+            padding="0.75em",
+            width="100%",
+            variant="surface",
+        ),
+        rx.box(),
+    )
+
+    details_block = rx.vstack(
+        tech_block,
+        generic_details_block,
+        logs_block,
+        spacing="2",
+        width="100%",
+    )
+
+    # Tag badges for feedback
+    tags_box = corner_tags_component or rx.box()
+
+    header_left = rx.hstack(
+        # event type label at the very left
+        rx.cond(
+            msg.get("tag_label", "") != "",
+            rx.badge(msg.get("tag_label", ""), variant="soft", size="1", color_scheme="gray"),
+            rx.box(),
+        ),
+        rx.cond(
+            msg.get("has_tech") | msg.get("has_logs") | msg.get("generic_meta"),  # type: ignore[operator]
+            rx.button(
+                "Details",
+                variant="ghost",
+                color_scheme="gray",
+                size="1",
+                on_click=on_toggle_details,  # type: ignore[arg-type]
+            ),
+        ),
+        rx.text(msg.get("created_at", ""), size="1", color="gray"),
+        spacing="2",
+        align="center",
+    )
+
+    body = rx.vstack(
+        rx.hstack(header_left, tags_box, justify="between", width="100%", align="center"),  # header
+        rx.text(
+            msg.get("content", ""),
+            style={
+                "whiteSpace": "pre-wrap",
+                "wordBreak": "break-word",
+            },
+        ),
+        rx.cond(msg.get("show_details"), details_block),
+        rx.cond(extras is not None, extras or rx.box()),
+        spacing="2",
+        width="100%",
+        style={
+            "maxWidth": "100%",
+        },
+    )
+
+    assistant_bubble = rx.card(
+        body,
+        padding="0.75em",
+        style={
+            "maxWidth": "70%",  # 35 vw is 70% of 50% parent card width in vw terms
+            "backgroundColor": "#11182714",
+            "border": "1px solid #e5e7eb",
+            "borderRadius": "12px",
+            "wordBreak": "break-word",
+            "overflowX": "hidden",
+        },
+    )
+    user_bubble = rx.card(
+        body,
+        padding="0.75em",
+        style={
+            "maxWidth": "70%",
+            "backgroundColor": "#3b82f614",
+            "border": "1px solid #e5e7eb",
+            "borderRadius": "12px",
+            "wordBreak": "break-word",
+            "overflowX": "hidden",
+        },
+    )
+
+    return rx.cond(
+        msg.get("is_assistant"),
+        rx.hstack(
+            rx.avatar(fallback="A", size="2", radius="full"),
+            assistant_bubble,
+            spacing="2",
+            width="100%",
+            justify="start",
+            align="start",
+        ),
+        rx.hstack(
+            user_bubble,
+            rx.avatar(fallback="U", size="2", radius="full"),
+            spacing="2",
+            width="100%",
+            justify="end",
+            align="start",
+        ),
+    )

vedana_backoffice/graph/build.py

@@ -0,0 +1,169 @@
+from dataclasses import dataclass
+from typing import Any, Dict, Iterable, List, Set, Tuple
+
+
+@dataclass(frozen=True)
+class StepMeta:
+    index: int
+    name: str
+    step_type: str
+    inputs: List[str]
+    outputs: List[str]
+    labels: List[Tuple[str, str]]
+
+
+NodeDict = Dict[str, Any]
+EdgeDict = Dict[str, Any]
+
+
+@dataclass
+class CanonicalGraph:
+    steps: List[StepMeta]
+    # Indexes for fast joins
+    table_to_producers: Dict[str, Set[int]]
+    table_to_consumers: Dict[str, Set[int]]
+    step_name_by_index: Dict[int, str]
+    step_type_by_index: Dict[int, str]
+
+
+def build_canonical(steps_meta: Iterable[dict]) -> CanonicalGraph:
+    """Build a pipeline representation from step metadata.
+
+    - All steps with their metadata (name, type, inputs, outputs, labels)
+    - Indexed mappings for fast lookups (table -> producers/consumers)
+    - Pre-computed relationships for efficient edge derivation
+
+    Args:
+        steps_meta: Iterable of step metadata dictionaries from pipeline introspection
+
+    Returns:
+        CanonicalGraph: Normalized pipeline representation with indexed relationships
+    """
+    steps: List[StepMeta] = []
+    table_to_producers: Dict[str, Set[int]] = {}
+    table_to_consumers: Dict[str, Set[int]] = {}
+    step_name_by_index: Dict[int, str] = {}
+    step_type_by_index: Dict[int, str] = {}
+
+    for m in steps_meta:
+        try:
+            idx = int(m.get("index", -1))
+        except Exception:
+            idx = -1
+        if idx < 0:
+            continue
+        name = str(m.get("name", f"step_{idx}"))
+        step_type = str(m.get("step_type", ""))
+        inputs = [str(x) for x in (m.get("inputs") or [])]
+        outputs = [str(x) for x in (m.get("outputs") or [])]
+        labels_raw = m.get("labels") or []
+        labels = [(str(a), str(b)) for a, b in labels_raw]
+        sm = StepMeta(index=idx, name=name, step_type=step_type, inputs=inputs, outputs=outputs, labels=labels)
+        steps.append(sm)
+        step_name_by_index[idx] = name
+        step_type_by_index[idx] = step_type
+
+        for t in outputs:
+            table_to_producers.setdefault(t, set()).add(idx)
+        for t in inputs:
+            table_to_consumers.setdefault(t, set()).add(idx)
+
+    return CanonicalGraph(
+        steps=steps,
+        table_to_producers=table_to_producers,
+        table_to_consumers=table_to_consumers,
+        step_name_by_index=step_name_by_index,
+        step_type_by_index=step_type_by_index,
+    )
+
+
+def derive_step_edges(cg: CanonicalGraph) -> List[Tuple[int, int, List[str]]]:
+    """Edges between steps, labeled by shared tables.
+
+    Use table indexes to connect producers -> consumers.
+    """
+    edges_map: Dict[Tuple[int, int], Set[str]] = {}
+    for table, producers in cg.table_to_producers.items():
+        consumers = cg.table_to_consumers.get(table, set())
+        if not consumers:
+            continue
+        for s in producers:
+            for t in consumers:
+                if s == t:
+                    continue
+                edges_map.setdefault((s, t), set()).add(table)
+    edges: List[Tuple[int, int, List[str]]] = []
+    for (s, t), tables in edges_map.items():
+        edges.append((s, t, sorted(list(tables))))
+    return edges
+
+
+def derive_table_edges(cg: CanonicalGraph) -> List[Tuple[int, int, str]]:
+    """Edges between tables; labeled by step name.
+
+    For each step, connect each input table (or -1 for BatchGenerate's) to each output table.
+    """
+    table_names = sorted(list(set([t for s in cg.steps for t in s.inputs + s.outputs])))
+    name_to_id: Dict[str, int] = {name: i for i, name in enumerate(table_names)}
+
+    edges: List[Tuple[int, int, str]] = []
+    for sm in cg.steps:
+        in_ids = [name_to_id[t] for t in sm.inputs if t in name_to_id]
+        out_ids = [name_to_id[t] for t in sm.outputs if t in name_to_id]
+        if not in_ids:
+            for ot in out_ids:
+                edges.append((-1, ot, sm.name))
+        else:
+            for it in in_ids:
+                for ot in out_ids:
+                    edges.append((it, ot, sm.name))
+    return edges
+
+
+def refine_layer_orders(
+    layers_dict: dict[int, list[int]],
+    parent_map: dict[int, list[int]],
+    child_map: dict[int, list[int]],
+    max_layer_idx: int,
+    passes: int = 2,
+):
+    """Apply iterative barycentric ordering to reduce edge length."""
+
+    if not layers_dict:
+        return
+
+    order_map: dict[int, float] = {}
+
+    def _snapshot_layer(layer_idx: int) -> None:
+        nodes_in_layer = layers_dict.get(layer_idx, [])
+        for pos, node_id in enumerate(nodes_in_layer):
+            order_map[node_id] = float(pos)
+
+    def _barycenter(nodes: list[int]) -> float | None:
+        valid = [order_map[n] for n in nodes if n in order_map]
+        if not valid:
+            return None
+        return sum(valid) / len(valid)
+
+    def _sort_layer(layer_idx: int, neighbor_map: dict[int, list[int]]) -> None:
+        nodes_in_layer = layers_dict.get(layer_idx)
+        if not nodes_in_layer:
+            return
+
+        def _key(node_id: int) -> tuple[float, float]:
+            bc = _barycenter(neighbor_map.get(node_id, []))
+            fallback = order_map.get(node_id, 0.0)
+            return bc if bc is not None else fallback, fallback
+
+        nodes_in_layer.sort(key=_key)
+        for pos, node_id in enumerate(nodes_in_layer):
+            order_map[node_id] = float(pos)
+
+    for layer_idx in range(0, max_layer_idx + 1):
+        _snapshot_layer(layer_idx)
+
+    for _ in range(max(1, passes)):
+        for layer_idx in range(1, max_layer_idx + 1):
+            _sort_layer(layer_idx, parent_map)
+        for layer_idx in range(max_layer_idx - 1, -1, -1):
+            _sort_layer(layer_idx, child_map)