flextool 4.0.0__py3-none-any.whl

flextool/cli/cmd_scenario_results.py

@@ -0,0 +1,284 @@
+"""CLI entry point for scenario comparison: argparse + settings resolution → orchestrator."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+import matplotlib
+from spinedb_api import DatabaseMapping, from_database, Array
+from spinedb_api.filters.alternative_filter import alternative_filter_config
+from spinedb_api.filters.tools import append_filter_config
+
+from flextool.cli._console import run_tool
+from flextool.scenario_comparison import orchestrator
+from flextool.scenario_comparison.db_reader import build_scenario_folders_from_dir
+from flextool.update_flextool.ensure_settings_db import ensure_settings_db
+
+import logging
+
+
+def main() -> None:
+    matplotlib.use('Agg')
+    parser = argparse.ArgumentParser(
+        description='Read and combine scenario results from multiple folders based on database information')
+    parser.add_argument(
+        'db_url', nargs='?', default=None,
+        help='Database URL containing scenario information (e.g., sqlite:///scenarios.db)')
+    parser.add_argument(
+        '--parquet-base-dir', default=None,
+        help='Base directory containing per-scenario parquet subdirectories '
+             '(alternative to db_url; requires --alternatives)')
+    parser.add_argument(
+        '--parquet-subdir',
+        default='output_parquet',
+        help='Subdirectory containing parquet files (default: output_parquet)')
+    parser.add_argument('--settings-db-url', default=None,
+                        help='Settings database URL (fills in unset params; CLI overrides DB)')
+    parser.add_argument(
+        '--output-config-path', default=None,
+        help='Path to output configuration YAML file (default: templates/default_plots.yaml — '
+             'comparison mode reads scenario_rule + comparison_overrides from each leaf)'
+    )
+    parser.add_argument('--active-configs', type=str, default=None, nargs="+",
+                        help='Which plot configurations from config_path yaml to use (default: default)')
+    parser.add_argument('--plot-rows', type=int, nargs=2, default=None,
+                        help='First and last row to plot in time series (default: 0 167)')
+    parser.add_argument('--write-to-xlsx', action='store_true', default=None,
+                        help='Write combined results to Excel file')
+    parser.add_argument('--write-dispatch-xlsx', action='store_true', default=None,
+                        help='Write dispatch data to Excel file in plot directory')
+    parser.add_argument('--write-to-ods', action='store_true', default=None)
+    parser.add_argument(
+        '--alternatives', metavar='S', type=str, nargs='+',
+        help='Add alternative names manually')
+    parser.add_argument(
+        '--plot-dir', default=None,
+        help='Directory to plot scenario comparison plots '
+             '(default: <project_root>/output_plot_comparisons when the '
+             'compared scenarios share a project, else the CWD-relative '
+             'output_plot_comparisons)'
+    )
+    parser.add_argument(
+        '--dispatch-plots', action='store_true',
+        help='Generate dispatch area plots for nodes and nodeGroups'
+    )
+    parser.add_argument(
+        '--show-plots', action='store_true', default=None,
+        help='Display plots interactively (in addition to saving)'
+    )
+    parser.add_argument(
+        '--plot-file-format', type=str, default=None,
+        choices=['png', 'svg'],
+        help='File format for plot output (default: png)'
+    )
+    parser.add_argument(
+        '--excel-dir', default=None,
+        help='Directory to write comparison Excel files (default: same as --plot-dir)'
+    )
+    parser.add_argument(
+        '--shared-legend', action='store_true', default=None,
+        help='Use shared legend across subplots (default: true)'
+    )
+    parser.add_argument(
+        '--only-first-file-per-plot', action='store_true', default=False,
+        help='Only produce the first file for each plot (quick overview mode)'
+    )
+    parser.add_argument(
+        '--comparison-parquet-dir', default=None,
+        help='Directory to write combined comparison parquet files for viewer integration'
+    )
+    parser.add_argument(
+        '--debug',
+        nargs='?',
+        const='basic',
+        default='off',
+        choices=['off', 'basic', 'full'],
+        metavar='LEVEL',
+        help='Diagnostic verbosity level (default: off).  Any level other '
+             'than ``off`` additionally generates per-node dispatch plots '
+             'for every available node (no node curation).  Mirrors the '
+             '``--debug`` flag of cmd_run_flextool.',
+    )
+
+    args = parser.parse_args()
+    db_url = args.db_url
+    parquet_base_dir = args.parquet_base_dir
+
+    # Self-heal missing lightweight settings DBs so fresh clones don't
+    # fail opaquely when the user forgot to run `flextool-update`. Only
+    # seeds output_info / output_settings / comparison_settings by
+    # basename.
+    for _candidate in (args.settings_db_url,):
+        try:
+            ensure_settings_db(_candidate)
+        except Exception as _exc:
+            logging.warning("Failed to auto-seed %s: %s", _candidate, _exc)
+
+    # Validate: need at least one data source
+    if db_url is None and parquet_base_dir is None:
+        parser.error('Either db_url or --parquet-base-dir must be provided')
+    # Validate: --parquet-base-dir requires --alternatives
+    if parquet_base_dir is not None and not args.alternatives:
+        parser.error('--parquet-base-dir requires --alternatives to specify scenario names')
+
+    # Resolve parameters: CLI args > settings DB > hardcoded defaults
+    output_config_path = args.output_config_path
+    active_configs = args.active_configs
+    plot_rows = args.plot_rows
+    write_to_xlsx = args.write_to_xlsx
+    write_dispatch_xlsx = args.write_dispatch_xlsx
+    write_to_ods = args.write_to_ods
+    show_plots = args.show_plots
+    plot_file_format = args.plot_file_format
+    shared_legend = args.shared_legend
+
+    settings_db_url = args.settings_db_url
+    if settings_db_url and os.path.exists(settings_db_url.replace('sqlite:///', '')):
+        with DatabaseMapping(settings_db_url) as settings_db:
+            settings_entities = settings_db.get_entity_items(entity_class_name="settings")
+            if len(settings_entities) == 1:
+                settings_name = settings_entities[0]["name"]
+                settings_params: dict = {}
+                for pv in settings_db.get_parameter_value_items(entity_class_name="settings"):
+                    if pv["entity_byname"] == (settings_name,):
+                        settings_params[pv["parameter_definition_name"]] = from_database(pv["value"], pv["type"])
+
+                if output_config_path is None and 'output-config-path' in settings_params:
+                    output_config_path = str(settings_params['output-config-path'])
+
+                if active_configs is None and 'active-output-configs' in settings_params:
+                    val = settings_params['active-output-configs']
+                    if isinstance(val, str):
+                        active_configs = [val]
+                    elif isinstance(val, Array):
+                        active_configs = list(val.values)
+                    else:
+                        active_configs = list(val)
+
+                if plot_rows is None:
+                    first = settings_params.get('plot_first_timestep')
+                    duration = settings_params.get('plot_duration')
+                    if first is not None and duration is not None:
+                        plot_rows = [int(first), int(first) + int(duration)]
+
+                if write_to_xlsx is None and 'write-to-excel' in settings_params:
+                    write_to_xlsx = bool(settings_params['write-to-excel'])
+                if write_dispatch_xlsx is None and 'write-dispatch-to-excel' in settings_params:
+                    write_dispatch_xlsx = bool(settings_params['write-dispatch-to-excel'])
+                if write_to_ods is None and 'write-to-ods' in settings_params:
+                    write_to_ods = bool(settings_params['write-to-ods'])
+                if show_plots is None and 'show-plots' in settings_params:
+                    show_plots = bool(settings_params['show-plots'])
+                if plot_file_format is None and 'plot-file-format' in settings_params:
+                    plot_file_format = str(settings_params['plot-file-format'])
+                if shared_legend is None and 'shared-legend' in settings_params:
+                    shared_legend = bool(settings_params['shared-legend'])
+
+    # Apply hardcoded defaults for anything still unset
+    from flextool._resources import package_data_path
+    _default_plots = str(package_data_path("schemas/default_plots.yaml"))
+    if output_config_path is None:
+        output_config_path = _default_plots
+    elif not os.path.isabs(output_config_path):
+        # Legacy: settings DB may carry a repo-relative override like
+        # ``templates/default_plots.yaml``.  Treat such relative paths
+        # as references to the bundled default — the wheel install no
+        # longer has a repo root.
+        if os.path.basename(output_config_path) in {
+            'default_plots.yaml', 'default_comparison_plots.yaml'
+        }:
+            output_config_path = _default_plots
+        else:
+            output_config_path = os.path.abspath(output_config_path)
+
+    # Migrate stale settings: comparison rendering now lives inside
+    # default_plots.yaml (driven by per-leaf ``scenario_rule``).  An
+    # existing settings DB or user override may still point to the
+    # deleted ``default_comparison_plots.yaml`` — silently redirect
+    # to the merged file so the run doesn't blow up.
+    if (os.path.basename(output_config_path) == 'default_comparison_plots.yaml'
+            or not os.path.isfile(output_config_path)):
+        if output_config_path != _default_plots:
+            logging.info(
+                "output-config-path %s not found or superseded — using "
+                "bundled %s instead.", output_config_path, _default_plots,
+            )
+        output_config_path = _default_plots
+    if active_configs is None:
+        active_configs = ['default']
+    if plot_rows is None:
+        plot_rows = [0, 167]
+    if write_to_xlsx is None:
+        write_to_xlsx = False
+    if write_dispatch_xlsx is None:
+        write_dispatch_xlsx = False
+    if write_to_ods is None:
+        write_to_ods = False
+    if show_plots is None:
+        show_plots = False
+    if plot_file_format is None:
+        plot_file_format = 'png'
+    if shared_legend is None:
+        shared_legend = True
+
+    # Build scenario-to-folder mapping from either --parquet-base-dir or db_url
+    alternatives = args.alternatives
+    pre_built_folders: dict[str, str] | None = None
+    parquet_subdir = args.parquet_subdir
+
+    if parquet_base_dir is not None:
+        # Direct parquet directory mode: skip database, build mapping from filesystem
+        pre_built_folders = build_scenario_folders_from_dir(
+            Path(parquet_base_dir), alternatives
+        )
+        if not pre_built_folders:
+            print("Error: no valid scenario directories found under "
+                  f"{parquet_base_dir}", file=sys.stderr)
+            sys.exit(1)
+        # With --parquet-base-dir the parquets sit directly in base_dir/scenario/,
+        # so the extra subdirectory level is not needed.
+        parquet_subdir = ''
+    elif alternatives and db_url is not None:
+        # Database mode with alternative filters
+        alternative_filter = alternative_filter_config(alternatives)
+        db_url = append_filter_config(db_url, alternative_filter)
+
+    # Resolve dispatch_plots flags
+    do_dispatch = args.dispatch_plots
+
+    # ``--plot-dir`` left unset → the orchestrator re-roots the comparison
+    # output dir at ``<project_root>/output_plot_comparisons`` when the
+    # compared scenarios share a sensible project root (GUI/Toolbox parity).
+    # Carry the bare CWD-relative default so an explicit value still wins and
+    # the no-common-root fallback is byte-identical to the old behaviour.
+    plot_dir_is_default = args.plot_dir is None
+    plot_dir = 'output_plot_comparisons' if plot_dir_is_default else args.plot_dir
+
+    orchestrator.run(
+        db_url=db_url,
+        parquet_subdir=parquet_subdir,
+        plot_dir=plot_dir,
+        output_config_path=output_config_path,
+        active_configs=active_configs,
+        plot_rows=plot_rows,
+        write_to_xlsx=write_to_xlsx,
+        write_dispatch_xlsx=write_dispatch_xlsx,
+        write_to_ods=write_to_ods,
+        show_plots=show_plots,
+        dispatch_plots=do_dispatch,
+        plot_file_format=plot_file_format,
+        scenario_folders=pre_built_folders,
+        excel_dir=args.excel_dir,
+        shared_legend=shared_legend,
+        only_first_file=args.only_first_file_per_plot,
+        comparison_parquet_dir=args.comparison_parquet_dir,
+        debug=args.debug != 'off',
+        plot_dir_is_default=plot_dir_is_default,
+    )
+
+
+if __name__ == '__main__':
+    run_tool(main)

flextool/cli/cmd_solve_mps.py

@@ -0,0 +1,169 @@
+"""Standalone HiGHS-from-MPS subprocess solver.
+
+Invoked by FlexTool's ``--save-memory`` path to keep HiGHS' active-solve
+memory footprint isolated from the parent Python process.  Reads an MPS
+file written by ``polar_high.Problem.build_only``, runs HiGHS with the
+caller's options, writes a solution file the parent reads back.
+
+Intentionally imports nothing from the rest of FlexTool so the child
+process's Python footprint stays small.  Only depends on ``highspy``.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from flextool.cli._console import run_tool
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Solve an MPS file with HiGHS in a subprocess.",
+    )
+    parser.add_argument("--mps", type=Path, required=True, help="Input MPS file")
+    parser.add_argument(
+        "--solution", type=Path, required=True,
+        help="Output solution file (write_solution_style=0, pretty/sparse)",
+    )
+    parser.add_argument(
+        "--basis", type=Path, default=None,
+        help="Optional output basis file",
+    )
+    parser.add_argument(
+        "--warm-basis", type=Path, default=None,
+        help="Optional input basis file to warm-start from",
+    )
+    parser.add_argument(
+        "--stats", type=Path, default=None,
+        help="Optional output JSON file with solve measurement fields",
+    )
+    parser.add_argument(
+        "--options", type=Path, default=None,
+        help="Optional HiGHS options file (key=value per line)",
+    )
+    args = parser.parse_args(argv)
+
+    import highspy
+
+    # ``kWarning`` is non-fatal — HiGHS issues it for things like
+    # writeSolution-on-an-LP-without-ranging, options-file lines it
+    # didn't recognise, etc.  Treat the API call as successful in that
+    # case but surface the warning to stderr.
+    def _ok(status: object, what: str) -> bool:
+        if status == highspy.HighsStatus.kOk:
+            return True
+        if status == highspy.HighsStatus.kWarning:
+            print(
+                f"NOTICE: HiGHS {what} returned kWarning (continuing)",
+                file=sys.stderr,
+            )
+            return True
+        return False
+
+    h = highspy.Highs()
+
+    # Apply options BEFORE readModel so presolve / solver / scaling
+    # decisions take effect on the read.  HiGHS' own .opt file format
+    # is ``key=value`` per line; we use that as the IPC mechanism so
+    # FlexTool doesn't need to keep its option-dict translation aligned
+    # with this CLI separately.
+    if args.options is not None:
+        if not _ok(h.readOptions(str(args.options)), "readOptions"):
+            print(
+                f"ERROR: HiGHS rejected options file {args.options}",
+                file=sys.stderr,
+            )
+            return 2
+
+    if not _ok(h.readModel(str(args.mps)), "readModel"):
+        print(f"ERROR: HiGHS failed to read MPS {args.mps}", file=sys.stderr)
+        return 3
+
+    # Warm-start injection.  A rejected / missing / corrupt basis is a
+    # safe silent-cold fallback: HiGHS' run() is correct after a failed
+    # readBasis, so we log and continue rather than abort the solve.
+    warm_basis_used = False
+    if args.warm_basis is not None:
+        try:
+            status = h.readBasis(str(args.warm_basis))
+            if status == highspy.HighsStatus.kOk:
+                warm_basis_used = True
+                print(
+                    f"INFO: warm-basis accepted from {args.warm_basis}",
+                    file=sys.stderr,
+                )
+            else:
+                print(
+                    f"WARNING: warm-basis rejected ({status}) — solving cold",
+                    file=sys.stderr,
+                )
+        except Exception as exc:
+            print(
+                f"WARNING: warm-basis read failed ({exc}) — solving cold",
+                file=sys.stderr,
+            )
+
+    if not _ok(h.run(), "run"):
+        print("ERROR: HiGHS run() returned non-OK status", file=sys.stderr)
+        return 4
+
+    model_status = h.getModelStatus()
+    # Write the solution unconditionally so the parent can inspect
+    # whatever HiGHS produced (useful for debugging non-optimal runs).
+    # Pretty/sparse format (style=0) — matches what the parent's
+    # h.readSolution expects.
+    if not _ok(h.writeSolution(str(args.solution), 0), "writeSolution"):
+        print(
+            f"ERROR: HiGHS failed to write solution {args.solution}",
+            file=sys.stderr,
+        )
+        return 5
+
+    if args.basis is not None:
+        # Best-effort: basis write isn't critical for the parent's
+        # output path, and small models may not have a basis on
+        # interior-point solves.  Warn but don't fail.
+        try:
+            _ok(h.writeBasis(str(args.basis)), "writeBasis")
+        except Exception as exc:
+            print(f"WARNING: writeBasis raised: {exc}", file=sys.stderr)
+
+    if args.stats is not None:
+        # Best-effort measurement sidecar for the parent's A/B gating.
+        # A stats-write failure must never fail the solve.
+        try:
+            info = h.getInfo()
+            # Wall-clock solve time drives the parent's A/B warm-vs-cold
+            # gate.  Best-effort: a highspy build without ``getRunTime``
+            # (or one that returns junk on a tiny model) falls back to
+            # ``-1.0``, which the parent treats as "no timing → skip gate".
+            try:
+                run_time = float(h.getRunTime())
+            except Exception:
+                run_time = -1.0
+            stats = {
+                "simplex_iteration_count": int(info.simplex_iteration_count),
+                "model_status": str(model_status),
+                "objective": float(h.getObjectiveValue()),
+                "warm_basis_used": warm_basis_used,
+                "run_time": run_time,
+            }
+            args.stats.write_text(json.dumps(stats))
+        except Exception as exc:
+            print(f"WARNING: stats write failed: {exc}", file=sys.stderr)
+
+    # Exit code carries the model status so the parent can branch on
+    # optimality without parsing the solution file first.
+    if model_status == highspy.HighsModelStatus.kOptimal:
+        return 0
+    print(
+        f"NOTICE: HiGHS finished with non-optimal status={model_status}",
+        file=sys.stderr,
+    )
+    return 1
+
+
+if __name__ == "__main__":
+    run_tool(main)

flextool/cli/cmd_update_flextool.py

@@ -0,0 +1,17 @@
+import argparse
+from flextool.update_flextool.self_update import update_flextool
+
+
+def main():
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument(
+        "--skip-git",
+        action="store_true",
+        help="skip 'git restore' and 'git pull' steps",
+    )
+    args = arg_parser.parse_args()
+    update_flextool(args.skip_git)
+
+
+if __name__ == "__main__":
+    main()

flextool/cli/cmd_write_outputs.py

@@ -0,0 +1,125 @@
+from flextool.cli._console import run_tool
+from flextool.process_outputs.write_outputs import write_outputs
+from flextool.update_flextool.ensure_settings_db import ensure_settings_db
+from spinedb_api import DatabaseMapping
+from spinedb_api.filters.tools import name_from_dict
+import logging
+import sys
+
+def main():
+
+        import argparse
+
+        parser = argparse.ArgumentParser(description='Write FlexTool outputs to various formats')
+        parser.add_argument('--input-db-url', type=str, help='URL of the input database with scenario filter (used when run_flextool.py calls this from Toolbox)')
+        parser.add_argument('--scenario-name', type=str, help='Name of a scenario that must have raw outputs available (when re-plotting single scenario from terminal)')
+        parser.add_argument('--output-locations-db-url', type=str, help='URL of the database that holds the locations of existing outputs (for re-plotting from Toolbox)')
+        parser.add_argument('--settings-db-url', type=str, default=None,
+                            help='URL of the settings database (fills in unset params)')
+        parser.add_argument('--config-path', type=str, default=None,
+                            help='Path to output configuration YAML file (default: templates/default_plots.yaml)')
+        parser.add_argument('--active-configs', type=str, nargs='+', default=None,
+                            help='Which plot configurations from config_path yaml to use (default: default)')
+        parser.add_argument('--output-location', type=str, default=None,
+                            help='Location for the output directories (default: flextool root), overridden by output-locations-db-url')
+        parser.add_argument('--subdir', type=str, default=None,
+                            help='Subdirectory for outputs (default: scenario name)')
+        parser.add_argument('--read-parquet-dir', type=str, default=False,
+                            help='Directory to read existing parquet files from (default: False, reads from raw CSV files)')
+        parser.add_argument('--write-methods', type=str, nargs='+', default=None,
+                            choices=['plot', 'parquet', 'excel', 'csv', 'spinedb'],
+                            help='Output methods to use (default: plot parquet excel)')
+        parser.add_argument('--results-db-url', type=str, default=None,
+                            help='Target SpineDB URL for the spinedb write-method (default: <output-dir>/results.sqlite)')
+        parser.add_argument('--plot-rows', type=int, nargs=2, default=None,
+                            help='First and last row to plot in time series (default: 0 167)')
+        parser.add_argument('--plot-file-format', type=str, default=None,
+                            choices=['png', 'svg'],
+                            help='File format for plot output (default: png)')
+        parser.add_argument('--debug', action='store_true',
+                            help='Enable debug output')
+        parser.add_argument('--only-first-file-per-plot', action='store_true', default=False,
+                            help='Only produce the first file for each plot (quick overview mode)')
+        parser.add_argument('--single-result', type=str, nargs=6,
+                            metavar=('KEY', 'CSV_NAME', 'PLOT_NAME', 'PLOT_TYPE', 'SUBPLOTS_PER_ROW', 'LEGEND_POSITION'),
+                            help='Process a single result (overrides --config): key csv_name plot_name plot_type subplots_per_row legend_position. Use "null" for None values.')
+
+        args = parser.parse_args()
+        input_db_url = args.input_db_url
+        output_locations_db_url = args.output_locations_db_url
+
+        # Self-heal missing lightweight settings DBs so fresh clones don't
+        # fail opaquely when the user forgot to run `flextool-update`.
+        # Only seeds output_info / output_settings / comparison_settings.
+        for _candidate in (output_locations_db_url, args.settings_db_url):
+            try:
+                ensure_settings_db(_candidate)
+            except Exception as _exc:
+                logging.warning("Failed to auto-seed %s: %s", _candidate, _exc)
+        if args.scenario_name:
+            scenario_names = [args.scenario_name]
+        else:
+            scenario_names = []
+
+        read_parquet_dir=args.read_parquet_dir
+        output_location = args.output_location
+
+        db_map = None
+        try:
+            if input_db_url:
+                with DatabaseMapping(input_db_url) as db_map_tmp:
+                    scenario_names = [name_from_dict(db_map_tmp.get_filter_configs()[0])]
+            elif output_locations_db_url:
+                db_map = DatabaseMapping(output_locations_db_url)
+                filter_configs = db_map.get_filter_configs()
+                if filter_configs:
+                    alternative_names = filter_configs[0]['alternatives']
+                    scenario_names = alternative_names
+                read_parquet_dir = True
+
+            if not scenario_names:
+                logging.critical("No scenario provided through any of the arguments: scenario-name or output-locations-db-url (or input-db-url by run_flextool.py)")
+                sys.exit(1)
+
+            if args.subdir:
+                subdir = args.subdir
+            else:
+                subdir = scenario_names[0]
+
+            for i, scenario_name in enumerate(scenario_names):
+                if not input_db_url and output_locations_db_url:
+                    param_value = db_map.get_parameter_value_item(
+                        entity_class_name='scenario',
+                        entity_byname=(scenario_name,),
+                        parameter_definition_name='output_location',
+                        alternative_name=scenario_name
+                    )
+                    if param_value:
+                        output_location = param_value['parsed_value']
+                    else:
+                        raise FileNotFoundError(f"Could not find output data location directory for scenario {scenario_name} from db {output_locations_db_url}.")
+                    subdir = scenario_name
+
+                write_outputs(
+                    scenario_name=scenario_name,
+                    output_config_path=args.config_path,
+                    active_configs=args.active_configs,
+                    output_funcs=None,
+                    output_location=output_location,
+                    subdir=subdir,
+                    read_parquet_dir=read_parquet_dir,
+                    write_methods=args.write_methods,
+                    plot_rows=tuple(args.plot_rows) if args.plot_rows else None,
+                    debug=args.debug,
+                    single_result=tuple(args.single_result) if args.single_result else None,
+                    settings_db_url=args.settings_db_url,
+                    plot_file_format=args.plot_file_format,
+                    only_first_file=args.only_first_file_per_plot,
+                    results_db_url=args.results_db_url,
+                )
+        finally:
+            if db_map is not None:
+                db_map.close()
+
+if __name__ == '__main__':
+    run_tool(main)

flextool/common_utils/__init__.py

@@ -0,0 +1 @@
+"""Common utility modules shared across flextool subpackages."""

flextool/common_utils/plot_mem_shape.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Render a memory-shape PNG from a flextool memory-sampler log.
+
+Usage:
+    plot_mem_shape.py <mem_log_path> <png_path>
+
+Plots priv_dirty (live set, red filled area) and rss (blue line) against
+wall time. Legend reports the lower (min) and upper (peak) value of each
+series; no on-canvas text annotations.
+"""
+import re
+import sys
+
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+_FIELD = re.compile(r"(\w+)=([-+0-9.eE]+)")
+
+
+def parse(path):
+    t, rss, priv = [], [], []
+    with open(path) as fh:
+        for line in fh:
+            if not line.startswith("ts="):
+                continue
+            f = dict(_FIELD.findall(line))
+            try:
+                t.append(float(f["mono_s"]))
+                rss.append(float(f["rss_gb"]))
+                priv.append(float(f["priv_dirty_gb"]))
+            except (KeyError, ValueError):
+                continue
+    return t, rss, priv
+
+
+def main():
+    if len(sys.argv) != 3:
+        sys.exit(f"usage: {sys.argv[0]} <mem_log_path> <png_path>")
+    mem_path, png_path = sys.argv[1], sys.argv[2]
+
+    t, rss, priv = parse(mem_path)
+    if not t:
+        sys.exit(f"no samples parsed from {mem_path}")
+
+    rss_lo, rss_hi = min(rss), max(rss)
+    priv_lo, priv_hi = min(priv), max(priv)
+
+    fig, ax = plt.subplots(figsize=(14, 6))
+
+    ax.fill_between(t, priv, color="red", alpha=0.12)
+    ax.plot(
+        t, priv, color="red", linewidth=1.0,
+        label=f"priv_dirty (live set)  {priv_lo:.1f}–{priv_hi:.1f} GB",
+    )
+    #ax.plot(
+    #    t, rss, color="tab:blue", linewidth=1.0,
+    #    label=f"rss  {rss_lo:.1f}–{rss_hi:.1f} GB",
+    #)
+
+    ax.set_xlim(min(t), max(t))
+    ax.set_ylim(0, priv_hi * 1.1)
+    ax.set_xlabel("wall time (s)")
+    ax.set_ylabel("GB")
+    ax.margins(x=0)
+    ax.legend(loc="upper right")
+
+    fig.tight_layout()
+    fig.savefig(png_path, dpi=110)
+    print(f"wrote {png_path}  ({len(t)} samples, "
+          f"priv_dirty {priv_lo:.1f}–{priv_hi:.1f} GB, "
+          f"rss {rss_lo:.1f}–{rss_hi:.1f} GB)")
+
+
+if __name__ == "__main__":
+    main()