tkati-dashboard 0.4.0__py3-none-any.whl

tkati_dashboard/__init__.py

@@ -0,0 +1,9 @@
+from tkati_dashboard.app import create_app
+from tkati_dashboard.dataflow import Dataflow, DataflowValidationError, load_dataflow
+
+__all__ = [
+    "Dataflow",
+    "DataflowValidationError",
+    "create_app",
+    "load_dataflow",
+]

tkati_dashboard/__main__.py

@@ -0,0 +1,4 @@
+from .main import main
+
+if __name__ == "__main__":
+    main()

tkati_dashboard/_kafka_metadata.py

@@ -0,0 +1,31 @@
+"""Shared "resolve a live topic's metadata" helpers for snapshot.py, lag.py, and topic_stats.py."""
+
+from typing import Any
+
+from confluent_kafka import Consumer
+
+
+def resolve_topic_metadata(
+    consumer: Consumer, broker: str, topic: str, timeout_sec: float
+) -> Any:
+    """Return `topic`'s TopicMetadata (partitions, each with leader/replicas/isrs), or raise
+    RuntimeError with a message naming `broker`/`topic` if the broker is unreachable or the
+    topic doesn't exist.
+    """
+    try:
+        metadata = consumer.list_topics(topic, timeout=timeout_sec)
+    except Exception as e:
+        raise RuntimeError(f"Could not reach broker {broker!r}: {e}") from e
+
+    topic_metadata = metadata.topics.get(topic)
+    if topic_metadata is None or topic_metadata.error is not None:
+        raise RuntimeError(f"Topic {topic!r} not found on {broker!r}")
+
+    return topic_metadata
+
+
+def resolve_partitions(
+    consumer: Consumer, broker: str, topic: str, timeout_sec: float
+) -> list[int]:
+    """Return the partition ids of `topic`. See resolve_topic_metadata for error behavior."""
+    return list(resolve_topic_metadata(consumer, broker, topic, timeout_sec).partitions)

tkati_dashboard/app.py

@@ -0,0 +1,155 @@
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import FileResponse
+
+from tkati_dashboard import lag, snapshot, topic_stats
+from tkati_dashboard.dataflow import (
+    SOURCE_SINK_TYPES,
+    DataflowValidationError,
+    NodeDef,
+    load_dataflow,
+)
+from tkati_dashboard.flows import FlowConfigError
+
+STATIC_DIR = Path(__file__).parent / "static"
+
+
+def _list_flows(list_flows: Callable[[], dict[str, Path]]) -> dict[str, Path]:
+    try:
+        return list_flows()
+    except FlowConfigError as e:
+        raise HTTPException(status_code=500, detail=str(e)) from e
+
+
+def _resolve_flow_dir(list_flows: Callable[[], dict[str, Path]], flow_id: str) -> Path:
+    directory = _list_flows(list_flows).get(flow_id)
+    if directory is None:
+        raise HTTPException(status_code=404, detail=f"Unknown flow {flow_id!r}")
+    return directory
+
+
+def _graph_json(directory: Path) -> dict[str, Any]:
+    dataflow = load_dataflow(directory)
+
+    nodes = [
+        {
+            "id": node_id,
+            "label": node.name or node_id,
+            "type": node.type,
+            "group": "source-sink"
+            if node.type in SOURCE_SINK_TYPES
+            else "processing-node",
+            "schema": node.schema,
+            "connection": node.connection,
+            "config": node.config,
+        }
+        for node_id, node in dataflow.nodes.items()
+    ]
+    edges = [
+        {
+            "from": edge.from_,
+            "to": edge.to,
+            "kind": edge.kind,
+            "consumer": edge.consumer,
+        }
+        for edge in dataflow.edges
+    ]
+    return {"name": dataflow.name, "nodes": nodes, "edges": edges}
+
+
+def _get_node(directory: Path, node_id: str) -> NodeDef:
+    dataflow = load_dataflow(directory)
+    node = dataflow.nodes.get(node_id)
+    if node is None:
+        raise HTTPException(status_code=404, detail=f"Unknown node {node_id!r}")
+    return node
+
+
+def _require_kafka_connection(
+    node_id: str, node: NodeDef, feature: str
+) -> tuple[str, str]:
+    """Common gate for the two live-Kafka endpoints: node must be a kafka-topic with a
+    broker/topic to connect to. Returns (broker, topic) or raises HTTPException."""
+    if node.type != "kafka-topic":
+        raise HTTPException(
+            status_code=404,
+            detail=f"No {feature} available for node type {node.type!r}",
+        )
+    connection = node.connection or {}
+    broker, topic = connection.get("broker"), connection.get("topic")
+    if not broker or not topic:
+        raise HTTPException(
+            status_code=422,
+            detail=f"Node {node_id!r} is missing connection.broker/connection.topic",
+        )
+    return broker, topic
+
+
+def create_app(list_flows: Callable[[], dict[str, Path]]) -> FastAPI:
+    app = FastAPI(title="tkati-dashboard")
+
+    @app.get("/")
+    def index() -> FileResponse:
+        return FileResponse(STATIC_DIR / "index.html")
+
+    @app.get("/api/flows")
+    def flows() -> list[dict[str, str]]:
+        resolved = _list_flows(list_flows)
+        return [{"id": flow_id, "name": flow_id} for flow_id in sorted(resolved, key=str.lower)]
+
+    @app.get("/api/flows/{flow_id}/graph")
+    def graph(flow_id: str) -> dict[str, Any]:
+        directory = _resolve_flow_dir(list_flows, flow_id)
+        try:
+            return _graph_json(directory)
+        except DataflowValidationError as e:
+            raise HTTPException(status_code=422, detail=str(e)) from e
+
+    @app.get("/api/flows/{flow_id}/nodes/{node_id}/snapshot")
+    def node_snapshot(flow_id: str, node_id: str) -> dict[str, Any]:
+        directory = _resolve_flow_dir(list_flows, flow_id)
+        try:
+            node = _get_node(directory, node_id)
+        except DataflowValidationError as e:
+            raise HTTPException(status_code=422, detail=str(e)) from e
+        broker, topic = _require_kafka_connection(node_id, node, "live snapshot")
+
+        try:
+            events = snapshot.fetch_kafka_snapshot(broker, topic)
+        except snapshot.SnapshotError as e:
+            raise HTTPException(status_code=502, detail=str(e)) from e
+
+        return {"events": events}
+
+    @app.get("/api/flows/{flow_id}/nodes/{node_id}/consumer-lag")
+    def node_consumer_lag(flow_id: str, node_id: str, group_id: str) -> dict[str, Any]:
+        directory = _resolve_flow_dir(list_flows, flow_id)
+        try:
+            node = _get_node(directory, node_id)
+        except DataflowValidationError as e:
+            raise HTTPException(status_code=422, detail=str(e)) from e
+        broker, topic = _require_kafka_connection(node_id, node, "consumer lag")
+
+        try:
+            return lag.fetch_consumer_lag(broker, topic, group_id)
+        except lag.LagError as e:
+            raise HTTPException(status_code=502, detail=str(e)) from e
+
+    @app.get("/api/flows/{flow_id}/nodes/{node_id}/topic-stats")
+    def node_topic_stats(flow_id: str, node_id: str) -> dict[str, Any]:
+        directory = _resolve_flow_dir(list_flows, flow_id)
+        try:
+            node = _get_node(directory, node_id)
+        except DataflowValidationError as e:
+            raise HTTPException(status_code=422, detail=str(e)) from e
+        broker, topic = _require_kafka_connection(node_id, node, "topic stats")
+
+        try:
+            return topic_stats.fetch_topic_stats(broker, topic)
+        except topic_stats.TopicStatsError as e:
+            raise HTTPException(status_code=502, detail=str(e)) from e
+
+    return app

tkati_dashboard/dataflow.py

@@ -0,0 +1,168 @@
+"""Load and validate a serialized tkati dataflow directory.
+
+See docs/dataflow-serialization.md for the format this module implements: a directory of JSON or
+YAML fragments merged into one graph of nodes and edges. There is no manifest file — every
+`*.json`/`*.yaml`/`*.yml` file directly inside the directory is a fragment.
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field
+from tkati_core.type_mapping import TYPE_MAPPING
+
+# Node types that represent data at rest (as opposed to a processing step) and therefore require
+# a `schema`. Kept as a heuristic, not a closed registry: an unrecognized type is still accepted,
+# it just isn't schema-checked.
+SOURCE_SINK_TYPES = {"kafka-topic", "clickhouse-table"}
+
+FRAGMENT_GLOBS = ("*.json", "*.yaml", "*.yml")
+YAML_SUFFIXES = (".yaml", ".yml")
+
+
+class DataflowValidationError(ValueError):
+    """A serialized dataflow directory failed validation."""
+
+
+class NodeDef(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    type: str
+    name: str | None = None
+    schema: dict[str, str] | None = None
+    connection: dict[str, Any] | None = None
+    config: dict[str, Any] | None = None
+
+
+class EdgeDef(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    from_: str = Field(alias="from")
+    to: str
+    kind: str = "stream"
+    consumer: dict[str, Any] | None = None
+
+
+class Dataflow(BaseModel):
+    name: str
+    nodes: dict[str, NodeDef]
+    edges: list[EdgeDef]
+
+
+def find_fragment_paths(directory: Path) -> list[Path]:
+    """Return every `*.json`/`*.yaml`/`*.yml` file directly inside `directory`, sorted by name
+    (mixing extensions in one alphabetical list, so merge order is purely filename-driven)."""
+    return sorted(p for pattern in FRAGMENT_GLOBS for p in directory.glob(pattern))
+
+
+def _read_fragment(path: Path) -> dict[str, Any]:
+    try:
+        text = path.read_text()
+    except FileNotFoundError as e:
+        raise DataflowValidationError(f"Missing dataflow file: {path}") from e
+
+    if path.suffix in YAML_SUFFIXES:
+        try:
+            data = yaml.safe_load(text)
+        except yaml.YAMLError as e:
+            raise DataflowValidationError(f"Invalid YAML in {path}: {e}") from e
+    else:
+        try:
+            data = json.loads(text)
+        except json.JSONDecodeError as e:
+            raise DataflowValidationError(f"Invalid JSON in {path}: {e}") from e
+
+    if data is None:
+        return {}
+    if not isinstance(data, dict):
+        raise DataflowValidationError(
+            f"{path}: fragment must be a JSON/YAML object, got {type(data).__name__}"
+        )
+    return data
+
+
+def _validate_node(node_id: str, node: NodeDef) -> None:
+    # `schema` is optional even for source/sink nodes: a real-world fragment (e.g. one written
+    # by hand, or discovered from a live cluster without introspecting its columns) may not
+    # have one on hand. When it is present, its field types are still checked.
+    if node.schema is not None:
+        for field_name, field_type in node.schema.items():
+            if field_type not in TYPE_MAPPING:
+                raise DataflowValidationError(
+                    f"Node {node_id!r} field {field_name!r} has unknown schema type {field_type!r}"
+                )
+
+
+def _merge_node(
+    nodes: dict[str, NodeDef],
+    node_sources: dict[str, str],
+    fragment_name: str,
+    node_id: str,
+    raw_node: dict[str, Any],
+) -> None:
+    node = NodeDef.model_validate(raw_node)
+    if node_id in nodes:
+        if nodes[node_id] != node:
+            raise DataflowValidationError(
+                f"Node {node_id!r} is defined differently in "
+                f"{node_sources[node_id]!r} and {fragment_name!r}"
+            )
+        return
+    nodes[node_id] = node
+    node_sources[node_id] = fragment_name
+
+
+def load_dataflow(directory: Path) -> Dataflow:
+    """Read every JSON/YAML fragment directly inside `directory`, merge, and validate them.
+
+    There is no manifest: any `*.json`/`*.yaml`/`*.yml` file in the directory is a fragment
+    contributing to the graph. The dataflow's name is the directory's own name.
+    """
+    if not directory.is_dir():
+        raise DataflowValidationError(f"Not a directory: {directory}")
+
+    fragment_paths = find_fragment_paths(directory)
+    if not fragment_paths:
+        raise DataflowValidationError(
+            f"No dataflow fragments ({'/'.join(FRAGMENT_GLOBS)}) found in {directory}"
+        )
+
+    nodes: dict[str, NodeDef] = {}
+    node_sources: dict[
+        str, str
+    ] = {}  # node id -> fragment it was first seen in, for error messages
+    edges: list[EdgeDef] = []
+
+    for fragment_path in fragment_paths:
+        fragment = _read_fragment(fragment_path)
+
+        for node_id, raw_node in fragment.get("nodes", {}).items():
+            _merge_node(nodes, node_sources, fragment_path.name, node_id, raw_node)
+
+        # A fragment may also declare a single node via a top-level "node" object carrying
+        # its own "id", instead of keying it under "nodes" — e.g. one file per node.
+        if "node" in fragment:
+            raw_node = fragment["node"]
+            node_id = raw_node.get("id")
+            if not node_id:
+                raise DataflowValidationError(
+                    f"{fragment_path.name!r} has a 'node' object with no 'id'"
+                )
+            _merge_node(nodes, node_sources, fragment_path.name, node_id, raw_node)
+
+        for raw_edge in fragment.get("edges", []):
+            edges.append(EdgeDef.model_validate(raw_edge))
+
+    for node_id, node in nodes.items():
+        _validate_node(node_id, node)
+
+    for edge in edges:
+        for endpoint in (edge.from_, edge.to):
+            if endpoint not in nodes:
+                raise DataflowValidationError(
+                    f"Edge {edge.from_!r} -> {edge.to!r} references unknown node {endpoint!r}"
+                )
+
+    return Dataflow(name=directory.name, nodes=nodes, edges=edges)

tkati_dashboard/flows.py

@@ -0,0 +1,54 @@
+"""Resolve the set of dataflow directories a dashboard instance observes.
+
+A dashboard instance can watch more than one dataflow at once: some directories are named
+explicitly on the command line, others are discovered as immediate subdirectories of a
+`--flows-root`. Either way, a flow's id is simply its directory's own basename — the same
+identity `dataflow.load_dataflow` already gives a `Dataflow.name`.
+"""
+
+from collections.abc import Callable
+from pathlib import Path
+
+from tkati_dashboard.dataflow import find_fragment_paths
+
+
+class FlowConfigError(ValueError):
+    """The configured dataflow directories/roots don't resolve to an unambiguous flow set."""
+
+
+def discover_flows_root(root: Path) -> dict[str, Path]:
+    """Every immediate subdirectory of `root` that contains at least one dataflow fragment is a
+    flow, keyed by its own basename. A subdirectory with no fragments (e.g. a stray `README` or
+    `.git`) is silently skipped rather than treated as an error."""
+    return {
+        path.name: path
+        for path in sorted(root.iterdir())
+        if path.is_dir() and find_fragment_paths(path)
+    }
+
+
+def make_flow_lister(
+    dataflow_dirs: list[Path], flows_roots: list[Path]
+) -> Callable[[], dict[str, Path]]:
+    """Build a `list_flows()` callable returning the current flow id -> directory map.
+
+    Explicit `dataflow_dirs` are fixed once, at build time. Each `flows_roots` entry is
+    re-scanned on every call, so adding or removing a flow subdirectory there is picked up
+    without restarting the server — the same "always read fresh, never cached" philosophy
+    `load_dataflow` already applies to a single flow's fragments.
+    """
+    fixed = {directory.name: directory for directory in dataflow_dirs}
+
+    def list_flows() -> dict[str, Path]:
+        flows = dict(fixed)
+        for root in flows_roots:
+            for flow_id, path in discover_flows_root(root).items():
+                if flow_id in flows and flows[flow_id] != path:
+                    raise FlowConfigError(
+                        f"Flow id {flow_id!r} is ambiguous: {flows[flow_id]} and {path} both "
+                        "resolve to it — rename one of the directories"
+                    )
+                flows[flow_id] = path
+        return flows
+
+    return list_flows

tkati_dashboard/lag.py

@@ -0,0 +1,78 @@
+"""Live consumer-group lag lookup for a dataflow edge, for the dashboard's edge labels.
+
+Like tkati_dashboard.snapshot, this is a best-effort, on-demand look at a live broker — it never
+subscribes or polls as the consumer group, only reads its committed offsets via `.committed()`,
+so it can never join the group, trigger a rebalance, or otherwise disrupt a real pipeline.
+"""
+
+from typing import Any
+
+from confluent_kafka import Consumer, TopicPartition
+
+from tkati_dashboard._kafka_metadata import resolve_partitions
+
+DEFAULT_TIMEOUT_SEC = 5.0
+
+
+class LagError(RuntimeError):
+    """Consumer lag could not be computed."""
+
+
+def fetch_consumer_lag(
+    broker: str,
+    topic: str,
+    group_id: str,
+    timeout_sec: float = DEFAULT_TIMEOUT_SEC,
+) -> dict[str, Any]:
+    """Return per-partition and total lag of `group_id` on `topic`.
+
+    A partition `group_id` has never committed an offset on counts as fully behind (lag =
+    partition size), since that's the backlog the group would have to process from scratch.
+    """
+    consumer = Consumer(
+        {
+            "bootstrap.servers": broker,
+            "group.id": group_id,
+            "enable.auto.commit": False,
+        }
+    )
+    try:
+        try:
+            partition_ids = resolve_partitions(consumer, broker, topic, timeout_sec)
+        except RuntimeError as e:
+            raise LagError(str(e)) from e
+
+        if not partition_ids:
+            return {"total_lag": 0, "partitions": []}
+
+        try:
+            committed = consumer.committed(
+                [TopicPartition(topic, pid) for pid in partition_ids],
+                timeout=timeout_sec,
+            )
+        except Exception as e:
+            raise LagError(
+                f"Could not fetch committed offsets for group {group_id!r}: {e}"
+            ) from e
+
+        partitions = []
+        total_lag = 0
+        for tp in committed:
+            low, high = consumer.get_watermark_offsets(
+                TopicPartition(topic, tp.partition), timeout=timeout_sec, cached=False
+            )
+            has_committed = tp.offset is not None and tp.offset >= 0
+            current = tp.offset if has_committed else low
+            partition_lag = max(high - current, 0)
+            total_lag += partition_lag
+            partitions.append(
+                {
+                    "partition": tp.partition,
+                    "committed_offset": tp.offset if has_committed else None,
+                    "high_watermark": high,
+                    "lag": partition_lag,
+                }
+            )
+        return {"total_lag": total_lag, "partitions": partitions}
+    finally:
+        consumer.close()

tkati_dashboard/main.py

@@ -0,0 +1,76 @@
+import argparse
+from pathlib import Path
+
+import uvicorn
+
+from tkati_dashboard.app import create_app
+from tkati_dashboard.dataflow import FRAGMENT_GLOBS, find_fragment_paths
+from tkati_dashboard.flows import FlowConfigError, make_flow_lister
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="tkati-dashboard",
+        description="Serve a graph view of one or more serialized tkati dataflow directories",
+    )
+    parser.add_argument(
+        "dataflow_dir",
+        type=Path,
+        nargs="*",
+        default=[],
+        help=(
+            "Path to a dataflow directory (a directory of *.json/*.yaml/*.yml fragments). Pass "
+            "more than one to observe multiple flows from one dashboard; see --flows-root to "
+            "auto-discover a whole directory of them instead of naming each one"
+        ),
+    )
+    parser.add_argument(
+        "--flows-root",
+        type=Path,
+        action="append",
+        default=[],
+        metavar="DIR",
+        help=(
+            "Treat every immediate subdirectory of DIR that contains dataflow fragments as its "
+            "own flow (id = subdirectory name). Re-scanned on every request, so adding or "
+            "removing a flow directory there is picked up without restarting. Repeatable."
+        ),
+    )
+    parser.add_argument("--host", default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=8000)
+    args = parser.parse_args()
+
+    if not args.dataflow_dir and not args.flows_root:
+        parser.error("pass at least one dataflow_dir or --flows-root")
+
+    for directory in args.dataflow_dir:
+        if not directory.is_dir():
+            parser.error(f"{directory} is not a directory")
+        if not find_fragment_paths(directory):
+            parser.error(
+                f"{directory} contains no dataflow fragments ({'/'.join(FRAGMENT_GLOBS)})"
+            )
+
+    seen: dict[str, Path] = {}
+    for directory in args.dataflow_dir:
+        flow_id = directory.name
+        if flow_id in seen and seen[flow_id] != directory:
+            parser.error(
+                f"flow id {flow_id!r} is ambiguous: {seen[flow_id]} and {directory} both have "
+                "this directory name — rename one of them"
+            )
+        seen[flow_id] = directory
+
+    for root in args.flows_root:
+        if not root.is_dir():
+            parser.error(f"{root} is not a directory")
+
+    list_flows = make_flow_lister(args.dataflow_dir, args.flows_root)
+    try:
+        if not list_flows():
+            parser.error("no flows found: check the given dataflow_dir(s)/--flows-root")
+    except FlowConfigError as e:
+        parser.error(str(e))
+
+    app = create_app(list_flows)
+    uvicorn.run(app, host=args.host, port=args.port)