flextool 4.0.0__py3-none-any.whl

flextool/cli/_timing.py

@@ -0,0 +1,147 @@
+"""TimingRecorder — per-CLI-invocation phase timings.
+
+Writes a single structured CSV at
+``<work_folder>/solve_data/timings.csv``.
+
+Schema (CSV header)::
+
+    phase,subphase,solve,roll_index,seconds,started_at_iso,cumulative_s
+
+One row per phase. ``record(...)`` appends immediately so a crash mid-run
+still leaves usable data. ``section(...)`` is a context-manager helper
+that times an arbitrary block. ``finalize(output_dir)`` copies the file
+to its final per-scenario output destination once the run completes.
+
+Always-on; cheap (one stat + one append per phase). Constructed once per
+CLI invocation; tests calling :class:`FlexToolRunner` directly bootstrap
+their own recorder if the CLI did not pass one in.
+"""
+from __future__ import annotations
+
+import csv
+import time
+from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Optional
+
+
+_HEADER = (
+    "phase",
+    "subphase",
+    "solve",
+    "roll_index",
+    "seconds",
+    "started_at_iso",
+    "cumulative_s",
+)
+
+
+class TimingRecorder:
+    """One per CLI invocation; lives on ``FlexToolRunner.state``.
+
+    Captures ``time.perf_counter()`` deltas with absolute UTC timestamps
+    and cumulative-since-start seconds for every phase of a flextool run.
+    """
+
+    def __init__(self, work_folder: Path | str, scenario: Optional[str] = None) -> None:
+        self.t0 = time.perf_counter()
+        self.t0_wall = datetime.now(timezone.utc)
+        self.work_folder = Path(work_folder)
+        self.scenario = scenario  # may be set later if cli didn't have it yet
+        self._path = self.work_folder / "solve_data" / "timings.csv"
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        self._open_for_write()
+
+    def _open_for_write(self) -> None:
+        """Create the file (truncate any pre-existing one) and write the header."""
+        with open(self._path, "w", newline="") as f:
+            writer = csv.writer(f)
+            writer.writerow(_HEADER)
+
+    def set_scenario(self, name: str) -> None:
+        """Set/overwrite the scenario name (CLI may not know it at __init__)."""
+        self.scenario = name
+
+    def record(
+        self,
+        phase: str,
+        *,
+        subphase: str = "",
+        solve: str = "",
+        roll_index: int | str = "",
+        seconds: float,
+        t_start: Optional[float] = None,
+    ) -> None:
+        """Append one row.
+
+        ``t_start`` is the ``perf_counter()`` value at the START of the
+        phase (used to back-compute ``started_at_iso``); if ``None`` we
+        assume the phase ended just now (``started_at_iso = now - seconds``).
+        """
+        now_perf = time.perf_counter()
+        cumulative_s = now_perf - self.t0
+        if t_start is not None:
+            started_perf = t_start
+        else:
+            started_perf = now_perf - max(0.0, float(seconds))
+        # Map perf_counter → wall time via the recorded t0/t0_wall pair.
+        started_wall = self.t0_wall + timedelta(seconds=started_perf - self.t0)
+        started_iso = started_wall.isoformat()
+        row = (
+            str(phase),
+            str(subphase),
+            str(solve),
+            str(roll_index),
+            f"{float(seconds):.6f}",
+            started_iso,
+            f"{cumulative_s:.6f}",
+        )
+        # Atomic append — open/write/close per row so a crash leaves the
+        # file in a parseable state.
+        with open(self._path, "a", newline="") as f:
+            csv.writer(f).writerow(row)
+
+    @contextmanager
+    def section(
+        self,
+        phase: str,
+        *,
+        subphase: str = "",
+        solve: str = "",
+        roll_index: int | str = "",
+    ):
+        """Time a block: ``with recorder.section('write_input'): ...``."""
+        t_start = time.perf_counter()
+        try:
+            yield
+        finally:
+            elapsed = time.perf_counter() - t_start
+            self.record(
+                phase,
+                subphase=subphase,
+                solve=solve,
+                roll_index=roll_index,
+                seconds=elapsed,
+                t_start=t_start,
+            )
+
+    @property
+    def path(self) -> Path:
+        """Current location of the timings.csv (in the work folder)."""
+        return self._path
+
+    def finalize(self, output_dir: Path | str) -> None:
+        """Copy ``timings.csv`` into the final scenario output dir.
+
+        Lives alongside ``summary_solve.csv`` so users have one place to
+        look. Idempotent: if the source file is missing, this is a no-op.
+        """
+        src = self._path
+        if not src.exists():
+            return
+        out = Path(output_dir)
+        out.mkdir(parents=True, exist_ok=True)
+        dst = out / "timings.csv"
+        # Plain byte copy — file is small.
+        dst.write_bytes(src.read_bytes())

flextool/cli/cmd_execute_flextool_workflow.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+"""
+FlexTool Workflow Orchestrator
+
+Runs the FlexTool workflow in two phases:
+1. Input preparation (optional): convert tabular data into a FlexTool input
+   database. Runs only when --tabular-file-path or --csv-directory-path is
+   given; otherwise the existing input database is used as-is.
+2. Model execution + output write: run the optimisation model and write
+   results in the requested formats. Parquet is always produced; --write-methods
+   selects which additional formats (plot, csv, excel) to generate alongside.
+"""
+
+import argparse
+import logging
+import subprocess
+import sys
+from pathlib import Path
+
+from flextool.cli._console import run_tool
+from flextool.update_flextool.ensure_settings_db import ensure_settings_db
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Execute complete FlexTool workflow',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Full workflow with Excel input
+  python execute_flextool_workflow.py input.sqlite results.sqlite my_scenario --tabular-file-path input.xlsx
+
+  # Full workflow with CSV input
+  python execute_flextool_workflow.py input.sqlite results.sqlite my_scenario --csv-directory-path input_data/
+
+  # Use an existing input database (no tabular/csv source -> input prep is skipped)
+  python execute_flextool_workflow.py input.sqlite results.sqlite my_scenario
+
+  # Add plots and CSV alongside the default parquet output
+  python execute_flextool_workflow.py input.sqlite results.sqlite my_scenario --write-methods plot parquet csv
+        """
+    )
+
+    # Required arguments
+    parser.add_argument('input_db_url', help='Input database URL (e.g., sqlite:///input.sqlite or input.sqlite)')
+    parser.add_argument('output_db_url', help='Output database URL for storing result metadata')
+    parser.add_argument('scenario_name', help='Name of the scenario to execute')
+
+    # Input phase arguments. Presence of either flag triggers Phase 1
+    # (input preparation); when neither is given, the existing input
+    # database is used as-is.
+    input_group = parser.add_mutually_exclusive_group()
+    input_group.add_argument('--tabular-file-path', help='Path to Excel/ODS input file')
+    input_group.add_argument('--csv-directory-path', help='Path to directory containing CSV input files')
+
+    # Output controls — forwarded to cmd_run_flextool, which writes
+    # outputs inline using the live in-memory flex_data + solution.
+    parser.add_argument('--write-methods', nargs='+',
+                        default=['plot', 'parquet', 'csv'],
+                        choices=['plot', 'parquet', 'excel', 'csv', 'spinedb'],
+                        help='Output formats to generate (default: plot parquet csv). '
+                             'Parquet is the canonical output and is always produced when this flag is omitted.')
+    parser.add_argument('--output-subdir', help='Subdirectory for output files (default: scenario_name)')
+    parser.add_argument('--output-config', default=None,
+                        help='Path to output configuration YAML file (default: bundled schemas/default_plots.yaml)')
+
+    # Additional options
+    parser.add_argument('--debug',
+                        nargs='?',
+                        const='basic',
+                        default='off',
+                        choices=['off', 'basic', 'full'],
+                        metavar='LEVEL',
+                        help='Diagnostic verbosity level passed through '
+                             'to cmd_run_flextool.  Bare ``--debug`` '
+                             'selects ``basic``.  See cmd_run_flextool '
+                             '--help for level semantics.')
+
+    args = parser.parse_args()
+
+    # Self-heal missing lightweight settings DBs so fresh clones don't
+    # fail opaquely when the user forgot to run `flextool-update`. Only
+    # seeds output_info / output_settings / comparison_settings by
+    # basename; other paths are left untouched.
+    for _candidate in (args.output_db_url,):
+        try:
+            ensure_settings_db(_candidate)
+        except Exception as _exc:
+            logging.warning("Failed to auto-seed %s: %s", _candidate, _exc)
+
+    # Phase 1 runs only when a tabular/csv source is provided; otherwise
+    # the existing input database is used as-is.
+    run_input_prep = bool(args.tabular_file_path or args.csv_directory_path)
+
+    # Set default output subdirectory to scenario name if not specified
+    output_subdir = args.output_subdir if args.output_subdir else args.scenario_name
+
+    # Phase 1: Input Preparation
+    if run_input_prep:
+        print(f"\n{'='*70}")
+        print("PHASE 1: PREPARING INPUT DATA")
+        print(f"{'='*70}")
+        if args.tabular_file_path:
+            print(f"  Input file:  {args.tabular_file_path}")
+        else:
+            print(f"  Input directory: {args.csv_directory_path}")
+        print(f"  Target DB:   {args.input_db_url}")
+        print()
+
+        cmd = [sys.executable, '-m', 'flextool.cli.cmd_read_tabular_input', args.input_db_url]
+        if args.tabular_file_path:
+            cmd.extend(['--tabular-file-path', args.tabular_file_path])
+        else:
+            cmd.extend(['--csv-directory-path', args.csv_directory_path])
+
+        result = subprocess.run(cmd)
+        if result.returncode != 0:
+            print("\n" + "="*70)
+            print("ERROR: Input preparation failed with return code", result.returncode)
+            print("="*70)
+            sys.exit(result.returncode)
+
+        print(f"\n{'='*70}")
+        print("PHASE 1: COMPLETED SUCCESSFULLY")
+        print(f"{'='*70}\n")
+    else:
+        print(f"\n{'='*70}")
+        print("PHASE 1: SKIPPED (using existing input database)")
+        print(f"{'='*70}\n")
+
+    # Phase 2: Model Execution + Output Write
+    # cmd_run_flextool writes outputs inline using the live in-memory
+    # flex_data + solution from the cascade. There is no separate
+    # standalone "write outputs" subprocess — the data is gone once the
+    # solver exits.
+    print(f"\n{'='*70}")
+    print("PHASE 2: RUNNING FLEXTOOL MODEL + WRITING OUTPUTS")
+    print(f"{'='*70}")
+    print(f"  Input DB:    {args.input_db_url}")
+    print(f"  Output DB:   {args.output_db_url}")
+    print(f"  Scenario:    {args.scenario_name}")
+    print(f"  Formats:     {', '.join(args.write_methods)}")
+    print(f"  Subdirectory: {output_subdir}")
+    print()
+
+    # Resolve --output-config: bare basename or unset → bundled
+    # default from the package; absolute path → used verbatim.
+    from flextool._resources import package_data_path
+    output_config = args.output_config
+    if not output_config or Path(output_config).name in {
+        'default_plots.yaml', 'default_comparison_plots.yaml'
+    }:
+        output_config = str(package_data_path("schemas/default_plots.yaml"))
+
+    cmd = [sys.executable, '-m', 'flextool.cli.cmd_run_flextool',
+           args.input_db_url, args.output_db_url,
+           '--scenario-name', args.scenario_name,
+           '--output-config', output_config,
+           '--output-subdir', output_subdir,
+           '--write-methods'] + args.write_methods
+    if args.debug != 'off':
+        cmd.append(f'--debug={args.debug}')
+
+    result = subprocess.run(cmd)
+    if result.returncode != 0:
+        print("\n" + "="*70)
+        print("ERROR: Model execution failed with return code", result.returncode)
+        print("="*70)
+        sys.exit(result.returncode)
+
+    print(f"\n{'='*70}")
+    print("PHASE 2: COMPLETED SUCCESSFULLY")
+    print(f"{'='*70}\n")
+
+    # Success summary
+    print(f"\n{'='*70}")
+    print("SUCCESS: COMPLETE WORKFLOW EXECUTED")
+    print(f"{'='*70}")
+    print(f"  Scenario:     {args.scenario_name}")
+    print(f"  Input DB:     {args.input_db_url}")
+    print(f"  Output DB:    {args.output_db_url}")
+    print(f"  Output dir:   {output_subdir}/")
+    print(f"{'='*70}\n")
+
+
+if __name__ == '__main__':
+    run_tool(main)

flextool/cli/cmd_export_to_tabular.py

@@ -0,0 +1,56 @@
+"""CLI entry point for exporting a FlexTool Spine DB to Excel (.xlsx) format."""
+
+import argparse
+
+from flextool.export_to_tabular import export_to_excel
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Export a FlexTool Spine DB to Excel (.xlsx) format."
+    )
+    parser.add_argument(
+        "db_url",
+        help="URL to FlexTool input database (e.g. sqlite:///input_data.sqlite)",
+    )
+    parser.add_argument(
+        "output_path",
+        help="Output Excel file path (.xlsx)",
+    )
+    parser.add_argument(
+        "--include-advanced",
+        action="store_true",
+        help="Include advanced sheets (solve sequences, stochastic data)",
+    )
+    parser.add_argument(
+        "--old-format",
+        action="store_true",
+        help="Use the old v1 format instead of the self-describing v2 format",
+    )
+    parser.add_argument(
+        "--groups",
+        default=None,
+        help=(
+            "Comma-separated list of parameter_group names to keep "
+            "(e.g. 'basics,model,timeline,solve_basics').  Parameters "
+            "outside these groups are dropped, and sheets with no surviving "
+            "columns are removed unless listed in always_include_with_groups."
+        ),
+    )
+    args = parser.parse_args()
+    include_groups = (
+        [g.strip() for g in args.groups.split(",") if g.strip()]
+        if args.groups
+        else None
+    )
+    export_to_excel(
+        args.db_url,
+        args.output_path,
+        include_advanced=args.include_advanced,
+        use_new_format=not args.old_format,
+        include_groups=include_groups,
+    )
+
+
+if __name__ == "__main__":
+    main()

flextool/cli/cmd_import_sensitivities.py

@@ -0,0 +1,75 @@
+"""CLI entry point for importing FlexTool 2.0 sensitivities into a Spine DB."""
+
+import argparse
+import logging
+import sys
+
+from flextool.cli._console import run_tool
+from flextool.process_inputs.read_old_flextool import (
+    read_old_flextool,
+    read_old_flextool_sensitivities,
+)
+from flextool.process_inputs.write_old_flextool_to_db import write_sensitivities_to_db
+
+logger = logging.getLogger(__name__)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Import FlexTool 2.0 sensitivities into a Spine DB."
+    )
+    parser.add_argument(
+        "master_xlsm",
+        help="Path to old FlexTool master .xlsm file (contains Sensitivity definitions sheet)",
+    )
+    parser.add_argument(
+        "base_xlsm",
+        help="Path to the base data .xlsm file (for unit_type mapping and base data)",
+    )
+    parser.add_argument(
+        "target_db_url",
+        help="Target Spine DB URL (must already contain base import)",
+    )
+    parser.add_argument(
+        "--base-alternative",
+        default="base",
+        help="Name of base alternative in the DB (default: base)",
+    )
+    parser.add_argument(
+        "--verbose", "-v",
+        action="store_true",
+        help="Enable verbose (DEBUG) logging",
+    )
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(levelname)s: %(message)s",
+    )
+
+    print(f"Reading base data from: {args.base_xlsm}")
+    data = read_old_flextool(args.base_xlsm)
+
+    print(f"Reading sensitivities from: {args.master_xlsm}")
+    sensitivities = read_old_flextool_sensitivities(args.master_xlsm)
+
+    if not sensitivities:
+        print("No sensitivities found. Nothing to import.")
+        sys.exit(0)
+
+    print(f"Found {len(sensitivities)} sensitivity scenario(s).")
+    for name, overrides in sensitivities.items():
+        print(f"  - {name}: {len(overrides)} override(s)")
+
+    print(f"Writing sensitivities to: {args.target_db_url}")
+    write_sensitivities_to_db(
+        sensitivities,
+        data,
+        args.target_db_url,
+        base_alternative=args.base_alternative,
+    )
+    print("Sensitivity import complete!")
+
+
+if __name__ == "__main__":
+    run_tool(main)

flextool/cli/cmd_migrate_database.py

@@ -0,0 +1,13 @@
+import argparse
+from flextool.update_flextool.db_migration import migrate_database
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('filepath', help="Filepath of the database, absolute or relative to flextool folder")
+    args = parser.parse_args()
+    migrate_database(args.filepath)
+
+
+if __name__ == '__main__':
+    main()