squidient 3.4.8__tar.gz → 4.0.0__tar.gz

squidient-4.0.0/PKG-INFO

@@ -0,0 +1,49 @@
+Metadata-Version: 2.4
+Name: squidient
+Version: 4.0.0
+Summary: CI/CD framework for alya
+Project-URL: Homepage, https://alya.gitlab.bsc.es/alya/ci-cd/squidient
+Project-URL: Bug Tracker, https://alya.gitlab.bsc.es/alya/ci-cd/squidient/issues
+Author-email: Damien Dosimont <damien.dosimont@gmail.com>
+License-File: LICENSE.txt
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.7
+Requires-Dist: importlib-resources; python_version < '3.9'
+Requires-Dist: lxml
+Requires-Dist: mariasql>=5
+Requires-Dist: packaging>=21.3
+Description-Content-Type: text/markdown
+
+# Squidient
+
+Squidient is a scheduler for testing, benchmarking and deploying HPC programs on SLURM machines.
+
+## Refreshing Workspace Data
+
+Squidient records a fingerprint of its packaged `data/` in `version/data.json`
+when `squidient init .` completes. A completed full configuration records
+that fingerprint in `configuration.json.dataRevision`. Staging and monitoring stop
+with exit code 1 if either differs from the installed package, including for older
+workspaces without this metadata. No confirmation prompt is displayed. The check
+runs before Terraform provisioning, remote connections, or job submission.
+A package release that leaves `data/` unchanged does not require a refresh.
+
+Refreshing is explicit: back up any customized workspace files and configuration
+first, then run:
+
+```sh
+squidient init .
+squidient configure --all
+```
+
+`init` overwrites packaged workspace files but does not regenerate
+`configuration.json`. `configure --all` regenerates the configuration; reapply
+your local settings as necessary. Partial reconfiguration does not mark an old
+configuration as current. The check never overwrites customizations automatically.
+Help and cleanup commands, including `terraform stop`, remain available.
+The former `fileFormat` counter and `version/version.json` are no longer used;
+legacy configurations must be refreshed before workflow execution when their
+data revision cannot be established.
+

squidient-4.0.0/README.md

@@ -0,0 +1,31 @@
+# Squidient
+
+Squidient is a scheduler for testing, benchmarking and deploying HPC programs on SLURM machines.
+
+## Refreshing Workspace Data
+
+Squidient records a fingerprint of its packaged `data/` in `version/data.json`
+when `squidient init .` completes. A completed full configuration records
+that fingerprint in `configuration.json.dataRevision`. Staging and monitoring stop
+with exit code 1 if either differs from the installed package, including for older
+workspaces without this metadata. No confirmation prompt is displayed. The check
+runs before Terraform provisioning, remote connections, or job submission.
+A package release that leaves `data/` unchanged does not require a refresh.
+
+Refreshing is explicit: back up any customized workspace files and configuration
+first, then run:
+
+```sh
+squidient init .
+squidient configure --all
+```
+
+`init` overwrites packaged workspace files but does not regenerate
+`configuration.json`. `configure --all` regenerates the configuration; reapply
+your local settings as necessary. Partial reconfiguration does not mark an old
+configuration as current. The check never overwrites customizations automatically.
+Help and cleanup commands, including `terraform stop`, remain available.
+The former `fileFormat` counter and `version/version.json` are no longer used;
+legacy configurations must be refreshed before workflow execution when their
+data revision cannot be established.
+

{squidient-3.4.8 → squidient-4.0.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "squidient"
-version = "3.4.8"
+version = "4.0.0"
 authors = [
   { name="Damien Dosimont", email="damien.dosimont@gmail.com" },
 ]

{squidient-3.4.8 → squidient-4.0.0}/src/squidient/cli/monitoring.py

@@ -28,6 +28,8 @@ from ..monitoring.monitoring import Monitoring
 from ..terraform.terraform import Terraform
 from .staging import IgnoredKeyboardInterrupt, darwin_bad_ssh, additional_options, install_interrupt_handlers
 from ..connection.ssh_config_checker import test_darwin_ssh_config, darwin_ssh_config_file
+from ..configuration.data_revision import require_current_data
+from ..exceptions import ConfigurationDataError
 from ..utils.message import console, error
 from ..utils.utils import write_text_file
 
@@ -62,6 +64,9 @@ def _run_cli(argv: Sequence[str]) -> int:
 
         arguments.parse(args)
 
+        if not (len(argv) >= 2 and argv[0] == "terraform" and argv[1] == "stop"):
+            require_current_data()
+
         terraform = Terraform()
         destroy_terraform = False
 
@@ -108,6 +113,14 @@ def _run_cli(argv: Sequence[str]) -> int:
 
         return 0
 
+    except ConfigurationDataError as failure:
+        console.section("Configuration error")
+        message, *instructions = str(failure).splitlines()
+        console.error(message, level=1)
+        for instruction in instructions:
+            console.detail(instruction, level=2)
+        return 1
+
     except KeyboardInterrupt:
         with IgnoredKeyboardInterrupt():
             try:

{squidient-3.4.8 → squidient-4.0.0}/src/squidient/cli/staging.py

@@ -28,6 +28,8 @@ from typing import Optional, Sequence
 from ..utils.arguments import Arguments
 from ..staging.staging import Staging
 from ..connection.ssh_config_checker import test_darwin_ssh_config, darwin_ssh_config_file
+from ..configuration.data_revision import require_current_data
+from ..exceptions import ConfigurationDataError
 from ..terraform.terraform import Terraform
 from ..timers import Timer
 from ..utils.message import console, error
@@ -112,6 +114,12 @@ def _run_cli(argv: Sequence[str]) -> int:
 
             arguments.parse(args)
 
+            is_cleanup = argv[0] == "clean" or (
+                len(argv) >= 2 and argv[0] == "terraform" and argv[1] == "stop"
+            )
+            if not is_cleanup:
+                require_current_data()
+
             terraform = Terraform()
             destroy_terraform = False
 
@@ -183,6 +191,14 @@ def _run_cli(argv: Sequence[str]) -> int:
 
         return 0
 
+    except ConfigurationDataError as failure:
+        console.section("Configuration error")
+        message, *instructions = str(failure).splitlines()
+        console.error(message, level=1)
+        for instruction in instructions:
+            console.detail(instruction, level=2)
+        return 1
+
     except KeyboardInterrupt:
         # Try to mimic the original cleanup, but keep it safe if cicd/terraform is None.
         with IgnoredKeyboardInterrupt():

squidient-4.0.0/src/squidient/configuration/data_revision.py

@@ -0,0 +1,92 @@
+"""Track the packaged data used to initialize and configure a workspace."""
+
+import hashlib
+import json
+import logging
+from pathlib import Path
+
+import squidient
+
+from ..definitions import config_file, data_revision_file
+from ..exceptions import ConfigurationDataError
+
+try:
+    from importlib.resources import files as ir_files
+except ImportError:
+    from importlib_resources import files as ir_files
+
+
+DATA_REVISION_CONFIG_KEY = "dataRevision"
+logger = logging.getLogger("squidient")
+
+
+def packaged_data_revision():
+    root = ir_files(squidient).joinpath("data")
+    pending = [(root, "")]
+    resources = []
+    while pending:
+        node, relative = pending.pop()
+        if node.is_dir():
+            pending.extend(
+                (child, relative + "/" + child.name)
+                for child in node.iterdir()
+                if child.name != "__pycache__" and not child.name.endswith((".pyc", ".pyo"))
+            )
+        else:
+            resources.append((relative, node))
+    digest = hashlib.sha256()
+    for relative, resource in sorted(resources, key=lambda item: item[0]):
+        file_digest = hashlib.sha256()
+        with resource.open("rb") as stream:
+            for block in iter(lambda: stream.read(1024 * 1024), b""):
+                file_digest.update(block)
+        digest.update(relative.encode("utf-8") + b"\0" + file_digest.digest())
+    return "sha256:" + digest.hexdigest()
+
+
+def workspace_data_revision(directory="."):
+    try:
+        with (Path(directory) / data_revision_file).open(encoding="utf-8") as stream:
+            metadata = json.load(stream)
+    except (OSError, ValueError):
+        return None
+    if not isinstance(metadata, dict):
+        return None
+    revision = metadata.get(DATA_REVISION_CONFIG_KEY)
+    if not isinstance(revision, str) or not revision.startswith("sha256:"):
+        return None
+    if len(revision) != 71 or any(character not in "0123456789abcdef" for character in revision[7:]):
+        return None
+    return revision
+
+
+def record_workspace_data_revision(directory, revision):
+    path = Path(directory) / data_revision_file
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(".json.tmp")
+    temporary.write_text(json.dumps({DATA_REVISION_CONFIG_KEY: revision}, indent=4) + "\n", encoding="utf-8")
+    temporary.replace(path)
+
+
+def require_current_data(config=None, directory="."):
+    if config is None:
+        from ..utils.utils import open_critical_json
+        config = open_critical_json(Path(directory) / config_file)
+    installed_revision = packaged_data_revision()
+    initialized_revision = workspace_data_revision(directory)
+    configured_revision = config.get(DATA_REVISION_CONFIG_KEY)
+    logger.debug("Configuration data revisions: installed=%s, initialized=%s, configured=%s",
+                 installed_revision, initialized_revision, configured_revision)
+    if initialized_revision != installed_revision:
+        raise ConfigurationDataError(
+            "Workspace data are outdated or unversioned for the installed squidient. Execution stopped.\n"
+            "Back up customized workspace files and configuration.json before refreshing:\n"
+            "squidient init .\n"
+            "squidient configure --all"
+        )
+    if configured_revision != installed_revision:
+        raise ConfigurationDataError(
+            "configuration.json was generated from outdated or unversioned data. Execution stopped.\n"
+            "Back up configuration.json, then regenerate it with:\n"
+            "squidient configure --all"
+        )

{squidient-3.4.8 → squidient-4.0.0}/src/squidient/configuration/generator.py

@@ -1,4 +1,5 @@
 from .code_coverage import CODE_COVERAGE_CONFIG_KEY
+from .data_revision import DATA_REVISION_CONFIG_KEY
 
 
 def build_default_configuration(
@@ -38,7 +39,8 @@ def build_default_configuration(
     config["staging"] = {}
     config["staging"]["builds"] = build_presets["fast"]["gpp"]
     # various
-    config["fileFormat"] = -1
+    # Record the data revision only after full configuration succeeds.
+    config[DATA_REVISION_CONFIG_KEY] = None
     config["tag"] = "master"
     config["installation"] = False
     config["flex"] = False

{squidient-3.4.8 → squidient-4.0.0}/src/squidient/configure.py

@@ -27,6 +27,7 @@ from typing import Optional, Sequence, Dict, Any
 from .builds.build_definition import BuildDefinition
 from .builds.build_presets import BuildPresets
 from .configuration.code_coverage import get_code_coverage_config, set_code_coverage_config
+from .configuration.data_revision import DATA_REVISION_CONFIG_KEY, workspace_data_revision
 from .configuration.generator import generate_default_configuration
 from .configuration.prompts import (
     ask,
@@ -48,7 +49,7 @@ from .definitions import (
     system_file,
 )
 from .utils.message import console
-from .utils.utils import config_file_format, exist_json, is_option, open_json, save_json
+from .utils.utils import exist_json, is_option, open_json, save_json
 
 from pathlib import Path
 
@@ -332,9 +333,9 @@ def set_benchmarks(provided_value=""):
     save_json(config, config_file)
 
 
-def set_version():
+def complete_configuration():
     config = open_json(config_file)
-    config["fileFormat"] = config_file_format()
+    config[DATA_REVISION_CONFIG_KEY] = workspace_data_revision() or "unknown"
     save_json(config, config_file)
 
 
@@ -476,8 +477,8 @@ def gitlab_config(pipeline, directory, alya_ref, alya_branch, flex=False, system
         console.section("Setting reservation")
         set_reservation(reservation)
 
-    console.section("Setting version")
-    set_version()
+    console.section("Recording configuration data revision")
+    complete_configuration()
 
 
 def _project_benchmark_queues_are_enabled(project_name, build_selection_type):
@@ -612,7 +613,7 @@ def _run_cli(argv: Sequence[str]) -> int:
     base_config = open_json(config_file)
     argument = ""
 
-    if base_config["fileFormat"] == -1:
+    if DATA_REVISION_CONFIG_KEY in base_config and base_config[DATA_REVISION_CONFIG_KEY] is None:
         console.section("First use configuration")
         console.warning("This is your first use of squidient; it requires a full configuration", level=1)
         if len(argv) < 1:
@@ -770,7 +771,7 @@ def _run_cli(argv: Sequence[str]) -> int:
         set_builds()
         set_code_coverage()
         set_benchmarks()
-        set_version()
+        complete_configuration()
 
     elif is_option(option, "-r", "--re"):
         set_alya_git_branch()
@@ -778,7 +779,6 @@ def _run_cli(argv: Sequence[str]) -> int:
         set_builds()
         set_code_coverage()
         set_benchmarks()
-        set_version()
 
     elif is_option(option, "-A", "--ALL"):
         generate_json()
@@ -794,7 +794,7 @@ def _run_cli(argv: Sequence[str]) -> int:
         set_builds()
         set_code_coverage()
         set_benchmarks()
-        set_version()
+        complete_configuration()
 
     elif is_option(option, "--alamak", "--alamak"):
         set_alamak(True)

{squidient-3.4.8 → squidient-4.0.0}/src/squidient/definitions.py

@@ -41,6 +41,7 @@ max_jobs_staging = 100
 max_jobs_monitoring = 50
 check_jobs_every_normal = 30
 config_file = "configuration.json"
+data_revision_file = "version/data.json"
 build_file = "builds/builds.json"
 default_file = "builds/defaults.json"
 cc_file = "builds/cc.json"

{squidient-3.4.8 → squidient-4.0.0}/src/squidient/deployment/initialization.py

@@ -6,6 +6,7 @@ from typing import Union
 
 import squidient
 
+from ..configuration.data_revision import packaged_data_revision, record_workspace_data_revision
 from ..utils.message import console
 
 try:
@@ -75,11 +76,13 @@ def _copy_traversable_tree(src, dst: Path, overwrite: bool = False) -> None:
                 shutil.copyfileobj(src_f, dst_f)
 
 def init_squidient(dist_dir: str = ".") -> None:
+    revision = packaged_data_revision()
     Path(dist_dir + "/squidient").mkdir(parents=True, exist_ok=True)
     copy_packaged_tree(squidient, "data", dist_dir + "/.", overwrite=True)
     copy_packaged_tree(squidient, "reports", dist_dir + "/squidient/reports", overwrite=True)
     copy_packaged_tree(squidient, "compare", dist_dir + "/squidient/compare", overwrite=True)
     copy_packaged_tree(squidient, "utils", dist_dir + "/squidient/utils", overwrite=True)
+    record_workspace_data_revision(dist_dir, revision)
 
 
 if __name__ == "__main__":

{squidient-3.4.8 → squidient-4.0.0}/src/squidient/exceptions.py

@@ -20,3 +20,7 @@
 
 class WorkflowFailure(RuntimeError):
     pass
+
+
+class ConfigurationDataError(RuntimeError):
+    """Workspace data must be refreshed before executing a workflow."""

{squidient-3.4.8 → squidient-4.0.0}/src/squidient/testing/cases/instance.py

@@ -101,7 +101,9 @@ class Instance:
 
         try:
             if self.eval_timeout(self._timeout):
-                raise RuntimeError
+                logger.debug("Clamping test %s timeout from %s to %s",
+                             self._id, self._timeout, test_max_timeout)
+                self._timeout = test_max_timeout
         except (RuntimeError, ValueError):
             self._timeout = test_default_timeout
 

{squidient-3.4.8 → squidient-4.0.0}/src/squidient/utils/utils.py

@@ -298,11 +298,6 @@ def copy_json(source, destination):
     copyfile(source, destination)
 
 
-def config_file_format():
-    data = open_json("version/version.json")
-    return data["fileFormat"]
-
-
 def json2js(variable, json):
     js = json.replace(".json", ".js")
     with open(json) as source, open(js, "w") as destination:

{squidient-3.4.8 → squidient-4.0.0}/src/squidient/workflows/base.py

@@ -26,6 +26,7 @@ import os
 from ..connection.job import Job
 from ..connection.ssh import SSH
 from ..configuration.code_coverage import normalize_code_coverage_config
+from ..configuration.data_revision import require_current_data
 from ..definitions import benchmark_file, config_file
 from ..reports.report import Report
 from ..systems.system_manager import SystemManager
@@ -33,7 +34,7 @@ from ..timers import Timer
 from ..utils.lock import Lock
 from ..utils.log import logging_context
 from ..utils.message import console, print_line, stagingbanner
-from ..utils.utils import config_file_format, open_critical_json
+from ..utils.utils import open_critical_json
 from ..utils.watchdog import Watchdog
 
 
@@ -108,10 +109,7 @@ class Workflow:
 
     def check_configuration_status(self):
         console.section("Checking configuration file")
-        if self._config["fileFormat"] != config_file_format():
-            console.error("Your " + config_file + " version is obsolete; please configure squidient again", level=1)
-            from ..exceptions import WorkflowFailure
-            raise WorkflowFailure("Configuration file version is obsolete")
+        require_current_data(self._config)
         console.success(config_file + " is compatible", level=1)
 
     def clean_locks(self):

squidient-3.4.8/PKG-INFO

@@ -1,22 +0,0 @@
-Metadata-Version: 2.4
-Name: squidient
-Version: 3.4.8
-Summary: CI/CD framework for alya
-Project-URL: Homepage, https://alya.gitlab.bsc.es/alya/ci-cd/squidient
-Project-URL: Bug Tracker, https://alya.gitlab.bsc.es/alya/ci-cd/squidient/issues
-Author-email: Damien Dosimont <damien.dosimont@gmail.com>
-License-File: LICENSE.txt
-Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
-Classifier: Operating System :: POSIX :: Linux
-Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.7
-Requires-Dist: importlib-resources; python_version < '3.9'
-Requires-Dist: lxml
-Requires-Dist: mariasql>=5
-Requires-Dist: packaging>=21.3
-Description-Content-Type: text/markdown
-
-# Squidient
-
-Squidient is a scheduler for testing, benchmarking and deploying HPC programs on SLURM machines.
-

squidient-3.4.8/README.md

@@ -1,4 +0,0 @@
-# Squidient
-
-Squidient is a scheduler for testing, benchmarking and deploying HPC programs on SLURM machines.
-

squidient-3.4.8/src/squidient/data/version/version.json

@@ -1,4 +0,0 @@
-{
-  "fileFormat": 20,
-  "version": "2"
-}