flextool 4.0.0__py3-none-any.whl

flextool/update_flextool/test_fixtures.py

@@ -0,0 +1,187 @@
+"""Round-trip every tests/fixtures/*.json through the current schema migration.
+
+Mirrors :mod:`flextool.update_flextool.canonical_databases` for the test
+fixture corpus.  The format used here differs from canonical_databases
+(base64-packed values vs. UTF-8 decoded), so this module uses
+``tests.db_utils.json_to_db`` / ``tests.db_utils.db_to_json`` rather than
+the canonical ``initialize_database`` / ``export_database`` pair.
+
+Run after a schema bump alongside ``sync_master_json_template`` and
+``canonical_databases migrate-all``.  See CONTRIBUTING.md.
+
+Usage:
+    python -m flextool.update_flextool.test_fixtures migrate-all
+    python -m flextool.update_flextool.test_fixtures verify
+
+The ``verify`` subcommand is intended for CI: it performs the same
+round-trip in a temp file and exits non-zero (printing a short diff
+summary) when any committed fixture is out of sync with the current
+schema.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import shutil
+import tempfile
+from pathlib import Path
+
+from flextool.update_flextool.db_migration import migrate_database
+
+
+# Resolved at runtime against the repository checkout — these fixtures live
+# under ``tests/fixtures/`` (outside the installed wheel), so we walk up
+# from this module to find the source tree.
+TEST_FIXTURES: tuple[str, ...] = (
+    "tests.json",
+    "stochastics.json",
+    "lh2_three_region.json",
+    "h2_trade_parity.json",
+    "stochastics_pbt_inflow.json",
+    "branch2_parent_period.json",
+    "case14_dc_power_flow.json",
+)
+
+
+def _fixtures_dir() -> Path:
+    """Locate ``tests/fixtures/`` from a source checkout.
+
+    Walks up from this file (``flextool/update_flextool/test_fixtures.py``)
+    to the repo root and joins ``tests/fixtures``.  Fails loudly if the
+    directory is missing — this module is only meaningful in an editable
+    install where the fixtures are present on disk.
+    """
+    here = Path(__file__).resolve()
+    # .../flextool-engine/flextool/update_flextool/test_fixtures.py
+    repo_root = here.parent.parent.parent
+    fixtures = repo_root / "tests" / "fixtures"
+    if not fixtures.is_dir():
+        raise RuntimeError(
+            f"tests/fixtures/ not found at {fixtures}. "
+            "This migration must be run from a source checkout."
+        )
+    return fixtures
+
+
+def _migrate_one(json_path: Path, out_path: Path) -> None:
+    """Round-trip a single fixture: JSON -> SQLite -> migrate -> JSON.
+
+    Uses ``tests.db_utils.json_to_db`` and ``tests.db_utils.db_to_json``
+    so the base64 value-packing convention used by the test fixtures is
+    preserved across the round-trip.
+    """
+    # Imported lazily so this module can be loaded outside a checkout
+    # (e.g. the canonical_databases command in a wheel install) without
+    # tripping over the test-package layering inversion.
+    from tests.db_utils import db_to_json, json_to_db
+
+    with tempfile.TemporaryDirectory() as tmp:
+        sqlite_path = Path(tmp) / "staging.sqlite"
+        url = json_to_db(json_path, sqlite_path)
+        migrate_database(url)
+        db_to_json(sqlite_path, out_path)
+
+
+def migrate_all() -> None:
+    """Migrate every ``tests/fixtures/*.json`` file in place.
+
+    Writes the result back over the source JSON.  Diffs may be large
+    when a fixture's ``parameter_definitions`` are catching up to a new
+    schema; that is expected.
+    """
+    fixtures = _fixtures_dir()
+    for name in TEST_FIXTURES:
+        src = fixtures / name
+        if not src.is_file():
+            print(f"!! test fixture missing: {src}")
+            continue
+        with tempfile.TemporaryDirectory() as tmp:
+            staging_json = Path(tmp) / name
+            _migrate_one(src, staging_json)
+            shutil.copyfile(staging_json, src)
+        print(f"migrated test fixture: {name}")
+
+
+def _summarise_diff(committed: Path, regenerated: Path) -> str:
+    """Return a short human-readable summary of which top-level sections
+    differ between two fixture JSONs.
+
+    Used by ``verify`` to surface drift without flooding the CI log.
+    """
+    with open(committed) as f:
+        a = json.load(f)
+    with open(regenerated) as f:
+        b = json.load(f)
+    keys = sorted(set(a) | set(b))
+    lines = []
+    for key in keys:
+        av = a.get(key)
+        bv = b.get(key)
+        if av == bv:
+            continue
+        la = len(av) if isinstance(av, list) else "?"
+        lb = len(bv) if isinstance(bv, list) else "?"
+        lines.append(f"  {key}: committed={la} regenerated={lb}")
+    return "\n".join(lines) if lines else "  (no top-level section differs — check value-level)"
+
+
+def verify_all() -> int:
+    """Verify that every committed fixture matches its migrated form.
+
+    Returns 0 when every file is byte-stable through the round-trip, or
+    a positive count of mismatched files otherwise.  Intended for CI use
+    via ``python -m flextool.update_flextool.test_fixtures verify``.
+    """
+    fixtures = _fixtures_dir()
+    mismatches = 0
+    for name in TEST_FIXTURES:
+        src = fixtures / name
+        if not src.is_file():
+            print(f"!! test fixture missing: {src}")
+            mismatches += 1
+            continue
+        with tempfile.TemporaryDirectory() as tmp:
+            regenerated = Path(tmp) / name
+            _migrate_one(src, regenerated)
+            with open(src) as f:
+                committed_text = f.read()
+            with open(regenerated) as f:
+                regenerated_text = f.read()
+            if committed_text == regenerated_text:
+                print(f"ok: {name}")
+            else:
+                mismatches += 1
+                print(f"DRIFT: {name}")
+                print(_summarise_diff(src, regenerated))
+    if mismatches:
+        print(
+            f"\n{mismatches} test fixture(s) out of sync with current schema. "
+            "Run `python -m flextool.update_flextool.test_fixtures migrate-all`."
+        )
+    return mismatches
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    sub.add_parser(
+        "migrate-all",
+        help="Round-trip every tests/fixtures/*.json through the current schema",
+    )
+    sub.add_parser(
+        "verify",
+        help="Check every test fixture is byte-stable through the current schema",
+    )
+    args = parser.parse_args(argv)
+
+    if args.cmd == "migrate-all":
+        migrate_all()
+        return 0
+    elif args.cmd == "verify":
+        return verify_all()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

flextool-4.0.0.dist-info/METADATA

@@ -0,0 +1,217 @@
+Metadata-Version: 2.4
+Name: flextool
+Version: 4.0.0
+Summary: IRENA FlexTool is an energy and power systems model for understanding the role of variable power generation in future energy systems.
+Author: IRENA FlexTool contributors
+License: © International Renewable Energy Agency 2018-2022
+        
+        The FlexTool is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License
+        as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+        
+        The FlexTool is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 
+        without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more details.
+        
+        You should have received a copy of the GNU Lesser General Public License along with the FlexTool.  
+        If not, see <https://www.gnu.org/licenses/>.
+        
+        Author: Juha Kiviluoma (2017-2022), VTT Technical Research Centre of Finland
+        
+        
+        
+        
+        -----------------------------------------------
+        IRENA FlexTool uses the HiGHS solver, which has its own license.
+        HiGHS is licensed with MIT license. Please see https://github.com/ERGO-Code/HiGHS for more information.
+Project-URL: Documentation, https://irena-flextool.github.io/flextool/
+Project-URL: Repository, https://github.com/irena-flextool/flextool
+Project-URL: Issues, https://github.com/irena-flextool/flextool/issues
+Keywords: energy-systems,optimization,linear-programming,mixed-integer-programming,power-systems,investment-planning,highs,spine
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: polar-high>=3.7.0
+Requires-Dist: highspy<=1.14.0
+Requires-Dist: polars>=1.40
+Requires-Dist: pandas
+Requires-Dist: spinedb-api
+Requires-Dist: openpyxl
+Requires-Dist: odfpy
+Requires-Dist: python-calamine
+Requires-Dist: xlsxwriter
+Requires-Dist: pyyaml
+Requires-Dist: matplotlib
+Requires-Dist: psutil
+Requires-Dist: sv-ttk
+Requires-Dist: screeninfo
+Requires-Dist: pywin32; sys_platform == "win32"
+Provides-Extra: toolbox
+Requires-Dist: spinetoolbox; extra == "toolbox"
+Provides-Extra: viewer
+Requires-Dist: tsdownsample>=0.1.3; extra == "viewer"
+Requires-Dist: networkx>=3.0; extra == "viewer"
+Provides-Extra: engine-polars
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-timeout>=2.3; extra == "dev"
+Requires-Dist: pytest-xdist>=3.0; extra == "dev"
+Requires-Dist: ruff>=0.5; extra == "dev"
+Dynamic: license-file
+
+[![PyPI version](https://img.shields.io/pypi/v/flextool.svg)](https://pypi.org/project/flextool/)
+[![Documentation Status](https://img.shields.io/badge/Documentation-passing-brightgreen)](https://irena-flextool.github.io/flextool/)
+[![Python](https://img.shields.io/badge/python-3.11%20|%203.12-blue.svg)](https://www.python.org/downloads/release/python-3120/)
+
+![IRENA FlexTool logo](./docs/img/flextool_logo.png)
+
+IRENA FlexTool is an energy and power systems model for understanding the role of variable power generation in future energy systems. It performs capacity expansion planning as well as operational planning.
+
+> [!NOTE]
+> The previous `master` branch is deprecated and no longer updated — `main` is
+> now FlexTool. If you still run FlexTool from `master`, migrate with a fresh
+> parallel install (separate directory and venv); see
+> [Installation](#installation) so your existing setup is left untouched.
+
+## What's new
+
+- **A lot faster.** The model build and pre-processing now run as pure Python
+  instead of the previous GLPSOL-based text pre-processing and AMPL-style model
+  translation. There is no LP file written to disk and re-read between build and
+  solve.
+- **Automatic problem scaling.** The `polar-high` matrix layer automatically
+  scales the optimisation problem before it reaches the solver. Better numerical
+  conditioning often speeds up the HiGHS solve considerably — sometimes
+  dramatically — so total run time usually improves well beyond the faster build
+  alone. The solver is still HiGHS; the gain comes from handing it a
+  better-conditioned problem.
+- **A new, easy-to-use interface.** The **FlexTool GUI** is a standalone
+  application (`python -m flextool.gui`) that manages projects and input
+  sources, runs scenarios, and lets you **browse the results** — a result viewer
+  reads outputs directly and supports single-scenario, scenario-comparison, and
+  network-graph views with keyboard navigation. See the
+  [FlexTool GUI guide](https://irena-flextool.github.io/flextool/flextool_gui_interface/).
+- **Spine Toolbox is still a parallel interface.** Advanced users who build
+  multi-tool workflows or integrate FlexTool with other models can keep using
+  the [Spine Toolbox workflow](https://irena-flextool.github.io/flextool/spine_toolbox/).
+- **The same model and data.** Existing input databases continue to work; the
+  change is in how the matrix is generated and solved, not in the model
+  formulation.
+
+This is IRENA FlexTool v4 (see the current version in CHANGELOG.md). Report any bugs or difficulties in the [issue tracker](https://github.com/irena-flextool/flextool/issues). 
+The previous version of IRENA FlexTool can be found in https://www.irena.org/energytransition/Energy-System-Models-and-Data/IRENA-FlexTool.
+
+## Under the hood: Rust-based matrix generation
+
+FlexTool is a thin Python layer over a Rust engine. It reads its input data and
+builds the optimisation matrix as [polars](https://pola.rs/) DataFrames via
+[`polar-high`](https://github.com/nodal-tools/polar-high), then solves with
+[HiGHS](https://highs.dev/). Variables and parameters are polars frames, so
+multiplications become joins and aggregations become group-bys — the heavy
+coefficient work runs inside polars' Rust core rather than as per-coefficient
+Python objects, and the matrix goes straight to HiGHS through `highspy` with no
+intermediate LP/MPS file. `polar-high` is a general-purpose, domain-free
+modelling layer published separately (Apache-2.0, `pip install polar-high`); it
+also applies the automatic problem scaling noted above, and its
+[benchmark](https://nodal-tools.fi/polar-high/compare/benchmark/) compares it
+against linopy and Pyomo on the same HiGHS solver.
+
+## Installation
+
+### Check your Python (and possibly install it)
+
+FlexTool requires **Python 3.11+**.
+
+Open a terminal (Windows: Command Prompt or PowerShell; macOS: Terminal; Linux: your shell)
+and check which Python you have:
+
+| Platform | Command |
+|---|---|
+| Windows | `py --version` (falls back to `python --version`) |
+| macOS / Linux | `python3 --version` |
+
+The output looks like `Python 3.12.4`. If the command is not found, prints
+something below 3.11, or (on Windows) opens the Microsoft Store, install Python
+from https://www.python.org/downloads/ — on Windows tick
+**"Add python.exe to PATH"** in the installer. Then close and reopen the
+terminal so the new PATH takes effect.
+
+> On Windows, use `py` in place of `python` in the commands below if `python`
+> is not recognised. If several versions are installed, `py -3.11 --version`
+> (or `-3.12`, …) picks a specific one, and `py -0` lists them all.
+> On macOS/Linux, `python` often means Python 2 or does not exist at all, so
+> use `python3`. Inside an activated virtual environment, plain `python` always
+> means that environment's Python on every platform.
+
+
+### Install from PyPI (recommended)
+
+Install FlexTool into a virtual environment to keep its packages separate from
+other Python applications. If you want to install into the base Python (perhaps
+because you do not otherwise use Python), just run the last command from below,
+`pip install flextool`.
+
+```
+python -m venv .venv        # Windows: py -m venv .venv
+                            # macOS/Linux: python3 -m venv .venv
+# activate — Windows PowerShell:  .\.venv\Scripts\Activate.ps1
+#            Windows CMD:         .\.venv\Scripts\activate.bat
+#            Linux / macOS:       . .venv/bin/activate
+python -m pip install --upgrade pip
+pip install flextool
+```
+
+Your prompt should now show `(.venv)`. Verify with `python --version` — it
+should report 3.11 or newer.
+
+(If PowerShell blocks the activate script, run once per user:
+`Set-ExecutionPolicy -Scope CurrentUser RemoteSigned`.)
+
+Launch the FlexTool GUI:
+
+```
+flextool
+```
+
+— equivalently `python -m flextool.gui`. The GUI creates and manages your
+projects and input databases; to seed the bundled template and example
+databases (handy for the CLI or Spine Toolbox workflows) run `flextool-update`.
+
+### Install from source (developers / latest `main`)
+
+To participate in FlexTool development — or to run the not-yet-released `main`
+— clone the repo and install it editable:
+
+```powershell
+git clone https://github.com/irena-flextool/flextool.git
+cd flextool
+python -m venv .venv
+.\.venv\Scripts\Activate.ps1     # Linux / macOS:  . .venv/bin/activate
+python -m pip install --upgrade pip
+pip install -e .
+```
+
+The editable install picks up source changes on `git pull`; re-run
+`pip install -e .` only when a dependency pin in `pyproject.toml` changes.
+
+### Update FlexTool:
+
+```
+# If using virtual environment (venv), then activate venv first:
+#   Windows PowerShell:  .\.venv\Scripts\Activate.ps1
+#   Windows CMD:         .\.venv\Scripts\activate.bat
+#   Linux / macOS:       . .venv/bin/activate
+pip install --upgrade flextool
+```
+
+## [Documentation](https://irena-flextool.github.io/flextool/) and [installation](https://irena-flextool.github.io/flextool/install_toolbox/)
+
+> [!IMPORTANT]
+> Installation, user guide and ***documentation*** can be found at: https://irena-flextool.github.io/flextool/.