flextool 4.0.0__py3-none-any.whl

flextool/gui/network_graph.py

@@ -0,0 +1,215 @@
+"""NetworkGraphBuilder -- reads Spine DB entities and renders a network graph."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from matplotlib.figure import Figure
+
+try:
+    import networkx as nx
+except ImportError:
+    nx = None  # type: ignore
+
+try:
+    import spinedb_api as api
+except ImportError:
+    api = None  # type: ignore
+
+logger = logging.getLogger(__name__)
+
+
+def build_network_figure(db_url: str) -> Figure | None:
+    """Build a network graph *Figure* from the Spine database at *db_url*.
+
+    Returns ``None`` if no lat/lon data is available or if required
+    dependencies (``networkx``, ``spinedb_api``) are missing.
+    """
+    if nx is None:
+        logger.warning("networkx is not installed -- cannot render network graph")
+        return None
+    if api is None:
+        logger.warning("spinedb_api is not installed -- cannot render network graph")
+        return None
+
+    try:
+        db = api.DatabaseMapping(db_url)
+    except Exception:
+        logger.exception("Failed to open database: %s", db_url)
+        return None
+
+    try:
+        # 1. Get nodes with lat/lon
+        nodes: dict[str, dict[str, float | None]] = {}
+        for entity in db.find_entities(entity_class_name="node"):
+            name = entity["entity_byname"][0]
+            nodes[name] = {"lat": None, "lon": None}
+
+        for pv in db.find_parameter_values(entity_class_name="node"):
+            name = pv["entity_byname"][0]
+            param = pv["parameter_definition_name"]
+            value = api.from_database(pv["value"], pv["type"])
+            if param in ("latitude", "lat") and name in nodes:
+                nodes[name]["lat"] = float(value)
+            elif param in ("longitude", "lon") and name in nodes:
+                nodes[name]["lon"] = float(value)
+
+        # Check if we have coordinates -- abort if not
+        if not any(
+            n["lat"] is not None and n["lon"] is not None for n in nodes.values()
+        ):
+            return None
+
+        # Filter to nodes that have coordinates
+        positioned_nodes = {
+            k: v
+            for k, v in nodes.items()
+            if v["lat"] is not None and v["lon"] is not None
+        }
+
+        # 2. Get units
+        units = [
+            e["entity_byname"][0]
+            for e in db.find_entities(entity_class_name="unit")
+        ]
+
+        # 3. Get relationships
+        unit_inputs: dict[str, list[str]] = {}
+        for e in db.find_entities(entity_class_name="unit__inputNode"):
+            unit, node = e["entity_byname"]
+            unit_inputs.setdefault(unit, []).append(node)
+
+        unit_outputs: dict[str, list[str]] = {}
+        for e in db.find_entities(entity_class_name="unit__outputNode"):
+            unit, node = e["entity_byname"]
+            unit_outputs.setdefault(unit, []).append(node)
+
+        conn_endpoints: dict[str, tuple[str, str]] = {}
+        for e in db.find_entities(entity_class_name="connection__node__node"):
+            conn, n1, n2 = e["entity_byname"]
+            conn_endpoints[conn] = (n1, n2)
+
+        # 4. Build networkx graph and render
+        return _render_network(
+            positioned_nodes, units, unit_inputs, unit_outputs, conn_endpoints
+        )
+    except Exception:
+        logger.exception("Error building network graph from %s", db_url)
+        return None
+    finally:
+        db.close()
+
+
+def _render_network(
+    positioned_nodes: dict[str, dict[str, Any]],
+    units: list[str],
+    unit_inputs: dict[str, list[str]],
+    unit_outputs: dict[str, list[str]],
+    conn_endpoints: dict[str, tuple[str, str]],
+) -> Figure:
+    """Build a networkx graph and render it to a matplotlib *Figure*.
+
+    This function is separated from the database layer so it can be
+    tested independently with mock data.
+
+    Parameters
+    ----------
+    positioned_nodes
+        ``{node_name: {"lat": float, "lon": float}}``
+    units
+        List of unit entity names.
+    unit_inputs
+        ``{unit_name: [input_node, ...]}``
+    unit_outputs
+        ``{unit_name: [output_node, ...]}``
+    conn_endpoints
+        ``{connection_name: (node1, node2)}``
+    """
+    G = nx.Graph()
+
+    # Add nodes at lat/lon positions
+    pos: dict[str, tuple[float, float]] = {}
+    for name, coords in positioned_nodes.items():
+        G.add_node(name, node_type="node")
+        pos[name] = (coords["lon"], coords["lat"])  # x=lon, y=lat
+
+    # Add units -- position at midpoint of connected nodes
+    for unit in units:
+        connected = unit_inputs.get(unit, []) + unit_outputs.get(unit, [])
+        connected_positioned = [n for n in connected if n in pos]
+        if not connected_positioned:
+            continue  # skip units with no positioned nodes
+        avg_lon = sum(pos[n][0] for n in connected_positioned) / len(
+            connected_positioned
+        )
+        avg_lat = sum(pos[n][1] for n in connected_positioned) / len(
+            connected_positioned
+        )
+        G.add_node(unit, node_type="unit")
+        pos[unit] = (avg_lon, avg_lat)
+
+    # Add connection edges (node -- node arcs)
+    for conn, (n1, n2) in conn_endpoints.items():
+        if n1 in pos and n2 in pos:
+            G.add_edge(n1, n2, edge_type="connection", label=conn)
+
+    # Add unit edges (unit -- node arcs)
+    for unit, input_nodes in unit_inputs.items():
+        for node in input_nodes:
+            if unit in pos and node in pos:
+                G.add_edge(node, unit, edge_type="unit_input")
+
+    for unit, output_nodes in unit_outputs.items():
+        for node in output_nodes:
+            if unit in pos and node in pos:
+                G.add_edge(unit, node, edge_type="unit_output")
+
+    # Render to Figure
+    fig = Figure(figsize=(12, 8))
+    ax = fig.add_subplot(111)
+
+    # Draw nodes by type with different styles
+    node_nodes = [n for n, d in G.nodes(data=True) if d.get("node_type") == "node"]
+    unit_nodes = [n for n, d in G.nodes(data=True) if d.get("node_type") == "unit"]
+
+    nx.draw_networkx_nodes(
+        G, pos, nodelist=node_nodes, ax=ax,
+        node_color="steelblue", node_size=300, node_shape="o",
+    )
+    nx.draw_networkx_nodes(
+        G, pos, nodelist=unit_nodes, ax=ax,
+        node_color="coral", node_size=200, node_shape="s",
+    )
+
+    # Draw edges by type
+    conn_edges = [
+        (u, v)
+        for u, v, d in G.edges(data=True)
+        if d.get("edge_type") == "connection"
+    ]
+    unit_edges = [
+        (u, v)
+        for u, v, d in G.edges(data=True)
+        if d.get("edge_type") in ("unit_input", "unit_output")
+    ]
+
+    nx.draw_networkx_edges(
+        G, pos, edgelist=conn_edges, ax=ax,
+        width=2, edge_color="gray",
+        connectionstyle="arc3,rad=0.1",
+        arrows=True,
+    )
+    nx.draw_networkx_edges(
+        G, pos, edgelist=unit_edges, ax=ax,
+        width=1, edge_color="lightgray", style="dashed",
+    )
+
+    nx.draw_networkx_labels(G, pos, ax=ax, font_size=8)
+
+    ax.set_xlabel("Longitude")
+    ax.set_ylabel("Latitude")
+    ax.set_aspect("equal")
+    fig.tight_layout()
+
+    return fig

flextool/gui/output_actions.py

@@ -0,0 +1,393 @@
+"""Manages output generation actions (plots, Excel, CSV) as background operations."""
+
+from __future__ import annotations
+
+import logging
+import os
+import subprocess
+import sys
+import threading
+from pathlib import Path
+from typing import Callable, TYPE_CHECKING
+
+from flextool.gui.cli_format import format_cmd_for_log
+from flextool.gui.config_parser import parse_plot_configs
+from flextool.gui.data_models import PlotSettings, ProjectSettings
+from flextool.gui.platform_utils import open_file_in_default_app
+from flextool.gui.project_utils import get_projects_dir
+from flextool.gui.scenario_key import resolve_subdir_for_read
+
+if TYPE_CHECKING:
+    from flextool.gui.execution_manager import ExecutionManager
+
+logger = logging.getLogger(__name__)
+
+
+class OutputActionManager:
+    """Manages output generation actions as background operations.
+
+    Each action creates an auxiliary job in the :class:`ExecutionManager` so
+    its output is visible in the ExecutionWindow.  When the command finishes
+    the optional *on_complete* callback is invoked **from the worker thread**
+    -- the caller should use ``root.after()`` to safely update tkinter.
+    """
+
+    def __init__(
+        self,
+        project_path: Path,
+        settings: ProjectSettings,
+        execution_mgr: ExecutionManager | None = None,
+        on_complete: Callable[[str, bool], None] | None = None,
+    ) -> None:
+        self.project_path = project_path
+        self.settings = settings
+        self._execution_mgr = execution_mgr
+        self._on_complete = on_complete
+        self._running_actions: set[str] = set()
+        self._action_gen: dict[str, int] = {}
+        self._lock = threading.Lock()
+
+    # ------------------------------------------------------------------
+    # Public query
+    # ------------------------------------------------------------------
+
+    def is_running(self, action_name: str) -> bool:
+        """Return *True* if the given action is currently running."""
+        with self._lock:
+            return action_name in self._running_actions
+
+    # ------------------------------------------------------------------
+    # Scenario-level actions
+    # ------------------------------------------------------------------
+
+    def run_scenario_plots(self, scenario_ids: list[tuple[int, str]]) -> None:
+        action = "scen_plots"
+        gen = self._mark_running(action)
+        job = self._create_aux_job(action, "Scenario pngs")
+
+        def _work() -> bool:
+            ok = True
+            for source_number, name in scenario_ids:
+                cmd = self._build_write_outputs_cmd(source_number, name, ["plot"])
+                if not self._run_subprocess(cmd, job):
+                    ok = False
+            return ok
+
+        self._start_thread(action, gen, _work, job)
+
+    def run_scenario_excel(self, scenario_ids: list[tuple[int, str]]) -> None:
+        action = "scen_excel"
+        gen = self._mark_running(action)
+        job = self._create_aux_job(action, "Scenario Excels")
+
+        def _work() -> bool:
+            ok = True
+            for source_number, name in scenario_ids:
+                cmd = self._build_write_outputs_cmd(source_number, name, ["excel"])
+                if not self._run_subprocess(cmd, job):
+                    ok = False
+            return ok
+
+        self._start_thread(action, gen, _work, job)
+
+    def run_scenario_csvs(self, scenario_ids: list[tuple[int, str]]) -> None:
+        action = "scen_csvs"
+        gen = self._mark_running(action)
+        job = self._create_aux_job(action, "Scenario csvs")
+
+        def _work() -> bool:
+            ok = True
+            for source_number, name in scenario_ids:
+                cmd = self._build_write_outputs_cmd(source_number, name, ["csv"])
+                if not self._run_subprocess(cmd, job):
+                    ok = False
+            return ok
+
+        self._start_thread(action, gen, _work, job)
+
+    # ------------------------------------------------------------------
+    # Comparison actions
+    # ------------------------------------------------------------------
+
+    def run_comparison_plots(self, scenario_ids: list[tuple[int, str]]) -> None:
+        action = "comp_plots"
+        gen = self._mark_running(action)
+        job = self._create_aux_job(action, "Comparison pngs")
+
+        def _work() -> bool:
+            cmd = self._build_comparison_cmd(scenario_ids, plots=True, excel=False)
+            ok = self._run_subprocess(cmd, job)
+            if ok:
+                comp_dir = self.project_path / "output_plot_comparisons"
+                first_png = self.find_first_plot(comp_dir)
+                if first_png is not None:
+                    try:
+                        open_file_in_default_app(first_png)
+                    except OSError:
+                        logger.warning("Could not open comparison plot: %s", first_png)
+            return ok
+
+        self._start_thread(action, gen, _work, job)
+
+    def run_comparison_excel(self, scenario_ids: list[tuple[int, str]]) -> None:
+        action = "comp_excel"
+        gen = self._mark_running(action)
+        job = self._create_aux_job(action, "Comparison Excel")
+
+        def _work() -> bool:
+            cmd = self._build_comparison_cmd(scenario_ids, plots=False, excel=True)
+            return self._run_subprocess(cmd, job)
+
+        self._start_thread(action, gen, _work, job)
+
+    # ------------------------------------------------------------------
+    # File finders (used by Show/Open buttons)
+    # ------------------------------------------------------------------
+
+    def find_first_plot(self, directory: Path) -> Path | None:
+        if not directory.is_dir():
+            return None
+        pngs = sorted(directory.rglob("*.png"))
+        return pngs[0] if pngs else None
+
+    def find_scenario_excel(self, scenario_name: str) -> Path | None:
+        excel_dir = self.project_path / "output_excel"
+        if not excel_dir.is_dir():
+            return None
+        for f in sorted(excel_dir.iterdir()):
+            if f.suffix.lower() == ".xlsx" and scenario_name in f.stem:
+                return f
+        return None
+
+    def find_comparison_excel(self) -> Path | None:
+        if not self.project_path.is_dir():
+            return None
+        xlsxs = sorted(
+            f for f in self.project_path.iterdir()
+            if f.suffix.lower() == ".xlsx" and f.stem.startswith("compare_")
+        )
+        return xlsxs[0] if xlsxs else None
+
+    # ------------------------------------------------------------------
+    # Config helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _resolve_active_configs(ps: PlotSettings, default_config: str) -> list[str]:
+        if ps.active_configs:
+            return ps.active_configs
+        if ps.config_file:
+            config_path = Path(ps.config_file)
+            if not config_path.is_absolute():
+                config_path = Path.cwd() / config_path
+        else:
+            # Bundled default — ``default_config`` is the historical
+            # repo-relative path (``templates/default_plots.yaml``);
+            # we resolve it against ``schemas/`` in the
+            # package so wheel installs work.
+            from flextool._resources import package_data_path
+            basename = Path(default_config).name
+            config_path = package_data_path(f"schemas/{basename}")
+        return parse_plot_configs(config_path) or ["default"]
+
+    # ------------------------------------------------------------------
+    # Command builders
+    # ------------------------------------------------------------------
+
+    def _build_write_outputs_cmd(
+        self, source_number: int, scenario_name: str, write_methods: list[str]
+    ) -> list[str]:
+        settings = self.settings
+        single = settings.single_plot_settings
+        subdir = resolve_subdir_for_read(
+            settings.bare_output_owners, source_number, scenario_name
+        )
+        parquet_dir = self.project_path / "output_parquet" / subdir
+
+        cmd: list[str] = [
+            sys.executable, "-m", "flextool.cli.cmd_write_outputs",
+            "--scenario-name", scenario_name,
+            "--read-parquet-dir", str(parquet_dir),
+            "--output-location", str(self.project_path),
+            "--subdir", subdir,
+            "--write-methods", *write_methods,
+        ]
+
+        if single.config_file:
+            cmd.extend(["--config-path", single.config_file])
+
+        active = self._resolve_active_configs(single, "templates/default_plots.yaml")
+        cmd.extend(["--active-configs", *active])
+
+        if single.duration > 0:
+            first_row = single.start_time
+            last_row = single.start_time + single.duration - 1
+            cmd.extend(["--plot-rows", str(first_row), str(last_row)])
+
+        if single.only_first_file:
+            cmd.append("--only-first-file-per-plot")
+
+        return cmd
+
+    def _build_comparison_cmd(
+        self,
+        scenario_ids: list[tuple[int, str]],
+        *,
+        plots: bool,
+        excel: bool,
+    ) -> list[str]:
+        settings = self.settings
+        comp = settings.comparison_plot_settings
+        parquet_base = self.project_path / "output_parquet"
+        plot_dir = self.project_path / "output_plot_comparisons"
+
+        subdirs = [
+            resolve_subdir_for_read(settings.bare_output_owners, sn, name)
+            for sn, name in scenario_ids
+        ]
+
+        cmd: list[str] = [
+            sys.executable, "-m", "flextool.cli.cmd_scenario_results",
+            "--parquet-base-dir", str(parquet_base),
+            "--alternatives", *subdirs,
+            "--plot-dir", str(plot_dir),
+        ]
+
+        comparison_parquet_dir = self.project_path / "output_parquet_comparison"
+        cmd.extend(["--comparison-parquet-dir", str(comparison_parquet_dir)])
+
+        if comp.config_file:
+            # Self-heal stale settings still pointing at the removed
+            # default_comparison_plots.yaml.
+            cfg_file = comp.config_file
+            if cfg_file.endswith("default_comparison_plots.yaml"):
+                cfg_file = cfg_file.replace(
+                    "default_comparison_plots.yaml", "default_plots.yaml",
+                )
+            cmd.extend(["--output-config-path", cfg_file])
+
+        active = self._resolve_active_configs(comp, "templates/default_plots.yaml")
+        cmd.extend(["--active-configs", *active])
+
+        if comp.duration > 0:
+            first_row = comp.start_time
+            last_row = comp.start_time + comp.duration - 1
+            cmd.extend(["--plot-rows", str(first_row), str(last_row)])
+
+        if plots and comp.dispatch_plots:
+            cmd.append("--dispatch-plots")
+
+        # When the project debug level is on, generate per-node dispatch
+        # plots too (Stage 4.4).  Reuse the project ``debug_level`` rather
+        # than inventing a new flag; ``off`` stays node-plot-free.
+        if plots and comp.dispatch_plots and settings.debug_level != "off":
+            cmd.append(f"--debug={settings.debug_level}")
+
+        if excel:
+            cmd.extend(["--write-to-xlsx", "--write-dispatch-xlsx"])
+            cmd.extend(["--excel-dir", str(self.project_path)])
+
+        if comp.only_first_file:
+            cmd.append("--only-first-file-per-plot")
+
+        return cmd
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _mark_running(self, action: str) -> int:
+        with self._lock:
+            self._running_actions.add(action)
+            gen = self._action_gen.get(action, 0) + 1
+            self._action_gen[action] = gen
+            return gen
+
+    def _mark_finished(self, action: str) -> None:
+        with self._lock:
+            self._running_actions.discard(action)
+
+    def _create_aux_job(self, action: str, title: str):
+        """Create an auxiliary job in the ExecutionManager (if available)."""
+        if self._execution_mgr is None:
+            return None
+        from flextool.gui.execution_manager import JobType
+        return self._execution_mgr.add_auxiliary_job(
+            JobType.OUTPUT_ACTION,
+            title,
+            f"output:{action}",
+        )
+
+    def _start_thread(self, action: str, gen: int, work, job=None) -> None:
+        def _wrapper() -> None:
+            try:
+                success = work()
+            except Exception:
+                logger.exception("Output action %s failed with exception", action)
+                success = False
+            finally:
+                self._mark_finished(action)
+
+            with self._lock:
+                superseded = self._action_gen.get(action, 0) != gen
+
+            if job is not None and not superseded:
+                self._execution_mgr.finish_job(job.job_id, success)
+
+            if self._on_complete is not None and not superseded:
+                try:
+                    self._on_complete(action, success)
+                except Exception:
+                    logger.exception("on_complete callback failed for %s", action)
+
+        thread = threading.Thread(target=_wrapper, daemon=True)
+        thread.start()
+
+    def _run_subprocess(self, cmd: list[str], job=None) -> bool:
+        """Run a subprocess to completion. Returns True on success."""
+        flextool_root = get_projects_dir().parent
+        cmd_str = format_cmd_for_log(cmd)
+
+        if job is not None:
+            self._execution_mgr.append_stdout(job.job_id, cmd_str)
+            self._execution_mgr.append_stdout(job.job_id, "")
+
+        try:
+            env = {**os.environ, "PYTHONUNBUFFERED": "1"}
+            proc = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                bufsize=1,
+                cwd=str(flextool_root),
+                env=env,
+            )
+
+            if job is not None:
+                with self._execution_mgr._lock:
+                    job.process = proc
+
+            assert proc.stdout is not None
+            for line in proc.stdout:
+                stripped = line.rstrip("\n")
+                if job is not None:
+                    self._execution_mgr.append_stdout(job.job_id, stripped)
+
+            return_code = proc.wait()
+            if return_code != 0:
+                msg = f"Process exited with code {return_code}"
+                logger.error("Output action %s", msg)
+                if job is not None:
+                    self._execution_mgr.append_stdout(job.job_id, f"\n{msg}")
+                return False
+            return True
+        except FileNotFoundError:
+            msg = f"Could not find executable: {cmd[0]}"
+            logger.error(msg)
+            if job is not None:
+                self._execution_mgr.append_stdout(job.job_id, msg)
+            return False
+        except Exception:
+            logger.exception("Subprocess failed: %s", cmd_str)
+            return False