flextool 4.0.0__py3-none-any.whl

flextool/update_flextool/generate_canonical.py

@@ -0,0 +1,274 @@
+"""Generate flextool/schemas/canonical_databases/*.json from tests/fixtures/*.json.
+
+Reads recipes from ``flextool/schemas/canonical_databases/_recipes.yaml``.
+
+Format bridge
+-------------
+``tests/fixtures/*.json`` uses **base64-packed** values (via
+:mod:`tests.db_utils`).  ``flextool/schemas/canonical_databases/*.json``
+uses **UTF-8 decoded** values (via
+:mod:`flextool.update_flextool.export_database`).  Rather than translating
+between formats directly, each converter is used in its native direction:
+
+1. Read the source JSON via :func:`tests.db_utils.json_to_db` into a temp
+   SQLite (base64 -> bytes).
+2. Run :func:`flextool.update_flextool.db_migration.migrate_database` —
+   idempotent after Stage 2a of the test_fixtures migration.
+3. Filter: delete excluded scenarios, alternatives, and entities listed
+   in the recipe.  ``spinedb_api`` cascade-removes scenario_alternatives,
+   entity_alternatives, parameter_values, and multi-dim child entities
+   automatically.
+4. Export the SQLite via :func:`flextool.update_flextool.export_database`
+   (bytes -> UTF-8) over the canonical JSON.
+
+CLI
+---
+``python -m flextool.update_flextool.generate_canonical``
+    Generate every recipe.
+
+``python -m flextool.update_flextool.generate_canonical templates_examples``
+    Generate just one recipe.
+
+``python -m flextool.update_flextool.generate_canonical --verify``
+    Regenerate to a temp dir and diff against the committed JSONs; exit
+    non-zero with a short summary on drift.  Intended for CI.
+
+Out of scope (still hand-maintained, follow-up work)
+----------------------------------------------------
+``howto_*.json`` (eight files) and ``templates_time_settings_only.json``
+are not yet covered by a recipe.  Adding them requires per-database
+investigation (each carves a different slice of the test universe); they
+will pick up recipes one at a time in follow-up commits.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import tempfile
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from flextool.update_flextool.db_migration import migrate_database
+from flextool.update_flextool.export_database import export_database
+
+
+# ``Path(__file__).resolve().parents[1]`` -> ``flextool/`` package root
+# inside the source checkout.  Both the recipe file and the destination
+# JSON tree live under ``flextool/schemas/canonical_databases/``.
+RECIPES_FILE = (
+    Path(__file__).resolve().parents[1]
+    / "schemas"
+    / "canonical_databases"
+    / "_recipes.yaml"
+)
+OUTPUT_DIR = (
+    Path(__file__).resolve().parents[1] / "schemas" / "canonical_databases"
+)
+# Source paths in the recipe (``tests/fixtures/...``) are resolved against
+# the repository root — one level above ``flextool/``.
+SOURCE_ROOT = Path(__file__).resolve().parents[2]
+
+
+def _load_recipes() -> dict[str, dict[str, Any]]:
+    """Parse ``_recipes.yaml`` and return its top-level dict.
+
+    Fails loudly if the file is missing — this module is a source-checkout
+    maintenance tool, never invoked from a wheel install.
+    """
+    if not RECIPES_FILE.is_file():
+        raise RuntimeError(f"Recipes file missing: {RECIPES_FILE}")
+    with open(RECIPES_FILE) as f:
+        return yaml.safe_load(f) or {}
+
+
+def _filter_db(
+    sqlite_url: str,
+    exclude_scenarios: list[str],
+    exclude_alternatives: list[str],
+    exclude_entities: list[list[str]],
+) -> None:
+    """Apply the recipe's exclusion lists to a staging SQLite.
+
+    Order matters: scenarios first (cascades scenario_alternatives),
+    then alternatives (cascades entity_alternatives + parameter_values
+    bound to those alternatives), then entities (cascades multi-dim
+    child entities and any remaining parameter_values).
+    """
+    # spinedb_api is the canonical handle for filtering Spine DBs — it
+    # owns the cascade semantics so we don't have to track FK ordering
+    # by hand.
+    from spinedb_api import DatabaseMapping
+
+    if not (exclude_scenarios or exclude_alternatives or exclude_entities):
+        # Recipe wants the source verbatim — skip the DatabaseMapping
+        # open/commit cycle entirely (spinedb_api raises NothingToCommit
+        # if commit_session is called with no pending changes).
+        return
+    with DatabaseMapping(sqlite_url, create=False, upgrade=False) as db:
+        for name in exclude_scenarios:
+            db.remove_scenario(name=name)
+        for name in exclude_alternatives:
+            db.remove_alternative(name=name)
+        for cls_name, ent_name in exclude_entities:
+            db.remove_entity(entity_class_name=cls_name, name=ent_name)
+        db.commit_session("Filtered for canonical export")
+
+
+def generate_one(recipe_name: str, output_path: Path | None = None) -> Path:
+    """Generate a single recipe's output JSON.
+
+    Args:
+        recipe_name: Top-level key in ``_recipes.yaml``.
+        output_path: Override the default ``OUTPUT_DIR / <recipe>.json``
+            destination.  Used by :func:`verify_all` to stage into a
+            temp directory.
+
+    Returns the path written.
+    """
+    recipes = _load_recipes()
+    if recipe_name not in recipes:
+        raise KeyError(
+            f"Unknown recipe {recipe_name!r}. "
+            f"Known: {sorted(recipes)}"
+        )
+    recipe = recipes[recipe_name]
+    source_rel = recipe["source"]
+    source_abs = (SOURCE_ROOT / source_rel).resolve()
+    if not source_abs.is_file():
+        raise FileNotFoundError(f"Recipe source missing: {source_abs}")
+
+    exclude_scenarios = recipe.get("exclude_scenarios") or []
+    exclude_alternatives = recipe.get("exclude_alternatives") or []
+    exclude_entities = recipe.get("exclude_entities") or []
+
+    dest = output_path if output_path is not None else (
+        OUTPUT_DIR / f"{recipe_name}.json"
+    )
+
+    # The base64 round-trip lives in ``tests.db_utils`` — imported lazily
+    # so the module can still be referenced in environments where the
+    # ``tests`` package isn't on ``sys.path`` (this matters only for
+    # accidental wheel-install invocation; generate_canonical itself is
+    # source-checkout only).
+    from tests.db_utils import json_to_db
+
+    with tempfile.TemporaryDirectory() as tmp:
+        staging_sqlite = Path(tmp) / f"{recipe_name}.sqlite"
+        url = json_to_db(source_abs, staging_sqlite)
+        migrate_database(url)
+        _filter_db(url, exclude_scenarios, exclude_alternatives, exclude_entities)
+        export_database(str(staging_sqlite), str(dest))
+
+    return dest
+
+
+def generate_all() -> list[Path]:
+    """Generate every recipe defined in ``_recipes.yaml``.
+
+    Returns the list of destination paths written, in YAML declaration
+    order.
+    """
+    recipes = _load_recipes()
+    written: list[Path] = []
+    for name in recipes:
+        dest = generate_one(name)
+        print(f"generated {dest.relative_to(SOURCE_ROOT)}")
+        written.append(dest)
+    return written
+
+
+def _summarise_diff(committed: Path, regenerated: Path) -> str:
+    """Short top-level-section summary for ``verify``.
+
+    Lists every key whose list length differs (or that's missing on one
+    side).  Value-level drift inside a same-length list is reported as
+    ``length match, value-level drift``.
+    """
+    with open(committed) as f:
+        a = json.load(f)
+    with open(regenerated) as f:
+        b = json.load(f)
+    keys = sorted(set(a) | set(b))
+    lines: list[str] = []
+    for key in keys:
+        av = a.get(key)
+        bv = b.get(key)
+        if av == bv:
+            continue
+        la = len(av) if isinstance(av, list) else "?"
+        lb = len(bv) if isinstance(bv, list) else "?"
+        if la == lb:
+            lines.append(f"  {key}: length match ({la}) — value-level drift")
+        else:
+            lines.append(f"  {key}: committed={la} regenerated={lb}")
+    return "\n".join(lines) if lines else "  (top-level sections match — check whitespace)"
+
+
+def verify_all() -> int:
+    """Regenerate every recipe to a temp dir and diff against committed.
+
+    Returns the count of mismatched recipes (0 on success).  Intended for
+    CI: surfaces drift between ``tests/fixtures/*.json`` and the
+    committed ``canonical_databases/*.json``.
+    """
+    recipes = _load_recipes()
+    mismatches = 0
+    for name in recipes:
+        committed = OUTPUT_DIR / f"{name}.json"
+        if not committed.is_file():
+            print(f"MISSING: {committed}")
+            mismatches += 1
+            continue
+        with tempfile.TemporaryDirectory() as tmp:
+            regenerated = Path(tmp) / f"{name}.json"
+            generate_one(name, output_path=regenerated)
+            with open(committed) as f:
+                a = f.read()
+            with open(regenerated) as f:
+                b = f.read()
+            if a == b:
+                print(f"ok: {name}.json")
+            else:
+                mismatches += 1
+                print(f"DRIFT: {name}.json")
+                print(_summarise_diff(committed, regenerated))
+    if mismatches:
+        print(
+            f"\n{mismatches} canonical database(s) out of sync. "
+            "Run `python -m flextool.update_flextool.generate_canonical`."
+        )
+    return mismatches
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "recipe",
+        nargs="?",
+        default=None,
+        help="Recipe name to generate (default: generate all).",
+    )
+    parser.add_argument(
+        "--verify",
+        action="store_true",
+        help="CI mode: regenerate to a temp dir and diff against committed.",
+    )
+    args = parser.parse_args(argv)
+
+    if args.verify:
+        if args.recipe is not None:
+            parser.error("--verify takes no positional argument")
+        return verify_all()
+    if args.recipe is None:
+        generate_all()
+    else:
+        dest = generate_one(args.recipe)
+        print(f"generated {dest.relative_to(SOURCE_ROOT)}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

flextool/update_flextool/initialize_database.py

@@ -0,0 +1,42 @@
+import json
+import argparse
+from spinedb_api import import_data, DatabaseMapping
+
+from flextool.update_flextool.export_database import keep_serialized_unparse
+
+
+def initialize_database(json_template, database_name="new_database.sqlite"):
+
+    if not database_name.endswith(".sqlite"):
+        print("Give a name with .sqlite file extension")
+        exit(-1)
+
+    with open(json_template) as json_file:
+        template = json.load(json_file)
+
+    with DatabaseMapping('sqlite:///' + database_name, create=True) as new_db:
+        # Remove any default alternatives created by spinedb_api (e.g. 'Base')
+        # so they don't conflict with the template's own alternatives.
+        for alt in new_db.find_alternatives():
+            try:
+                new_db.remove_alternative(name=alt['name'])
+            except Exception:
+                pass
+        # ``keep_serialized_unparse`` handles both author styles:
+        #   * raw scalars (``"no_method"``, ``None``, ``5``) used by the
+        #     hand-maintained ``schemas/spinedb_schema.json`` and
+        #     other static templates.
+        #   * ``[json_str, type_str]`` pairs emitted by ``export_database``
+        #     for canonical databases that carry full parameter values.
+        (num, log) = import_data(new_db, unparse_value=keep_serialized_unparse, **template)
+        print(str(num) + " imports made")
+        print("Initialized " + database_name)
+        new_db.commit_session("Initialized")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('json_filepath',help= "The filepath of the source JSON")
+    parser.add_argument('db_filepath',help= "The filepath of the new database")
+    args = parser.parse_args()
+    initialize_database(args.json_filepath, args.db_filepath)

flextool/update_flextool/install_info.py

@@ -0,0 +1,225 @@
+"""Detect how FlexTool is installed and build its self-upgrade command.
+
+Two supported install shapes:
+
+* an editable **git checkout** (``pip install -e .``) — upgraded with
+  ``git pull`` followed by an editable reinstall;
+* a **PyPI wheel** install — upgraded with ``pip install --upgrade``.
+
+The standalone Tkinter GUI uses this to offer an in-app "Update FlexTool"
+action that works regardless of which shape is in place.  The richer Spine
+Toolbox workflow updater (``git restore`` + project-file rebuild) stays in
+:mod:`flextool.update_flextool.self_update`; that behaviour is intentionally
+*not* used from the GUI.
+"""
+
+from __future__ import annotations
+
+import importlib.metadata as _im
+import json
+import logging
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from urllib.request import Request, urlopen
+
+logger = logging.getLogger(__name__)
+
+
+def git_checkout_root() -> Path | None:
+    """Return the repo root if FlexTool runs from an editable git checkout.
+
+    Relies on the imported package's own location: an editable install imports
+    ``flextool`` straight from the source tree, so its parent directory holds
+    both ``.git`` and ``pyproject.toml``.  A wheel install imports from
+    ``site-packages`` and fails this test (returns ``None``).
+    """
+    import flextool
+
+    pkg_dir = Path(flextool.__file__).resolve().parent  # …/flextool
+    repo_root = pkg_dir.parent
+    if (repo_root / ".git").is_dir() and (repo_root / "pyproject.toml").is_file():
+        return repo_root
+    return None
+
+
+def is_git_install() -> bool:
+    """Whether FlexTool runs from an editable git checkout."""
+    return git_checkout_root() is not None
+
+
+def flextool_version() -> str:
+    """Installed FlexTool version, or ``"unknown"`` if it cannot be read."""
+    try:
+        return _im.version("flextool")
+    except _im.PackageNotFoundError:
+        return "unknown"
+
+
+def toolbox_installed() -> bool:
+    """Whether Spine Toolbox (and thus the Spine DB Editor) is installed."""
+    try:
+        _im.distribution("spinetoolbox")
+        return True
+    except _im.PackageNotFoundError:
+        return shutil.which("spine-db-editor") is not None
+
+
+def describe_install() -> str:
+    """One-line, user-facing description of the current install."""
+    root = git_checkout_root()
+    if root is not None:
+        return f"git checkout at {root}\n(version {flextool_version()})"
+    return f"PyPI install (version {flextool_version()})"
+
+
+def upgrade_steps(include_toolbox: bool) -> tuple[list[list[str]], Path | None]:
+    """Build the command(s) that upgrade FlexTool in place.
+
+    Args:
+        include_toolbox: when ``True`` the upgrade targets the ``[toolbox]``
+            extra, so Spine Toolbox is installed (or kept) alongside FlexTool.
+
+    Returns:
+        ``(steps, cwd)`` where *steps* is a list of ``argv`` lists to run in
+        order (stop at the first failure) and *cwd* is the directory they run
+        in — the repo root for a git checkout, otherwise ``None``.
+    """
+    extra = "[toolbox]" if include_toolbox else ""
+    py = sys.executable
+    root = git_checkout_root()
+    if root is not None:
+        if _has_upstream(root):
+            pull = ["git", "pull", "--ff-only"]
+        else:
+            # The branch has no upstream tracking ref, so a bare `git pull`
+            # fails with "no tracking information"; name the remote and branch
+            # explicitly instead.
+            remote = _default_remote(root)
+            branch = current_branch(root) or "main"
+            pull = ["git", "pull", "--ff-only", remote, branch]
+        steps = [
+            pull,
+            [py, "-m", "pip", "install", "--upgrade", "-e", f".{extra}"],
+        ]
+        return steps, root
+    steps = [[py, "-m", "pip", "install", "--upgrade", f"flextool{extra}"]]
+    return steps, None
+
+
+def _git(root: Path, *args: str, timeout: float = 10.0) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        ["git", *args], cwd=root, timeout=timeout, capture_output=True, text=True
+    )
+
+
+def current_branch(root: Path) -> str | None:
+    """Current branch name, or ``None`` (detached HEAD / error)."""
+    try:
+        result = _git(root, "rev-parse", "--abbrev-ref", "HEAD")
+        name = result.stdout.strip()
+        return name if result.returncode == 0 and name and name != "HEAD" else None
+    except Exception:
+        return None
+
+
+def _has_upstream(root: Path) -> bool:
+    """Whether the current branch has an upstream tracking ref."""
+    try:
+        return _git(
+            root, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"
+        ).returncode == 0
+    except Exception:
+        return False
+
+
+def _default_remote(root: Path) -> str:
+    """Pick a remote to pull from — ``origin`` if present, else the first."""
+    try:
+        remotes = _git(root, "remote").stdout.split()
+    except Exception:
+        remotes = []
+    if "origin" in remotes:
+        return "origin"
+    return remotes[0] if remotes else "origin"
+
+
+def update_available(timeout: float = 5.0) -> bool:
+    """Best-effort check for a newer FlexTool version.
+
+    For a git checkout this fetches and reports whether the tracked upstream
+    branch is ahead; for a PyPI install it compares the installed version with
+    the latest on PyPI. Returns ``True`` only on a confident positive — any
+    error, offline state, or ambiguity returns ``False`` so the UI never nags
+    spuriously. Performs network I/O; call it off the UI thread.
+    """
+    root = git_checkout_root()
+    if root is not None:
+        return _git_behind_upstream(root, timeout)
+    return _pypi_has_newer(timeout)
+
+
+def _git_behind_upstream(root: Path, timeout: float) -> bool:
+    """Whether the checkout's tracked upstream has commits we don't have."""
+    import os
+
+    # Never let fetch block on an interactive credential / host-key prompt;
+    # it must fail fast and silently if auth is needed.
+    env = {
+        **os.environ,
+        "GIT_TERMINAL_PROMPT": "0",
+        "GCM_INTERACTIVE": "never",
+        "GIT_SSH_COMMAND": "ssh -oBatchMode=yes",
+    }
+    try:
+        subprocess.run(
+            ["git", "fetch", "--quiet"],
+            cwd=root, timeout=timeout, check=True, capture_output=True, env=env,
+        )
+        # Prefer the configured upstream; fall back to the remote branch of the
+        # same name when the branch has no tracking ref set.
+        if _has_upstream(root):
+            ref = "@{u}"
+        else:
+            branch = current_branch(root)
+            if branch is None:
+                return False
+            ref = f"{_default_remote(root)}/{branch}"
+        result = subprocess.run(
+            ["git", "rev-list", "--count", f"HEAD..{ref}"],
+            cwd=root, timeout=timeout, check=True, capture_output=True, text=True,
+        )
+        return int(result.stdout.strip() or "0") > 0
+    except Exception:
+        # No upstream, no git, no network, auth needed, detached HEAD, … —
+        # don't nag.
+        logger.debug("git update check inconclusive for %s", root, exc_info=True)
+        return False
+
+
+def _pypi_latest_version(timeout: float) -> str | None:
+    """Latest FlexTool version string on PyPI, or ``None`` on any failure."""
+    request = Request(
+        "https://pypi.org/pypi/flextool/json",
+        headers={"User-Agent": "flextool-gui"},
+    )
+    try:
+        with urlopen(request, timeout=timeout) as response:
+            return json.load(response)["info"]["version"]
+    except Exception:
+        logger.debug("PyPI update check failed", exc_info=True)
+        return None
+
+
+def _pypi_has_newer(timeout: float) -> bool:
+    latest = _pypi_latest_version(timeout)
+    if latest is None:
+        return False
+    try:
+        from packaging.version import Version
+
+        return Version(latest) > Version(flextool_version())
+    except Exception:
+        logger.debug("Version comparison failed (latest=%s)", latest, exc_info=True)
+        return False