openjd-cli 0.6.1__tar.gz → 0.7.1__tar.gz

{openjd_cli-0.6.1 → openjd_cli-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openjd-cli
-Version: 0.6.1
+Version: 0.7.1
 Summary: Provides a command-line interface for working with Open Job Description templates.
 Project-URL: Homepage, https://github.com/OpenJobDescription/openjd-cli
 Project-URL: Source, https://github.com/OpenJobDescription/openjd-cli
@@ -22,8 +22,8 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.9
-Requires-Dist: openjd-model==0.7.*
-Requires-Dist: openjd-sessions<0.11,>=0.10.1
+Requires-Dist: openjd-model<0.9,>=0.8
+Requires-Dist: openjd-sessions<0.11,>=0.10.3
 Description-Content-Type: text/markdown
 
 ## Open Job Description - CLI
@@ -205,7 +205,7 @@ You can download this package from:
 
 ### Verifying GitHub Releases
 
-See [VERIFYING_PGP_SIGNATURE](VERIFYING_PGP_SIGNATURE.md) for more information.
+See [Verifying GitHub Releases](https://github.com/OpenJobDescription/openjd-cli?tab=security-ov-file#verifying-github-releases) for more information.
 
 ## Security
 

{openjd_cli-0.6.1 → openjd_cli-0.7.1}/README.md

@@ -177,7 +177,7 @@ You can download this package from:
 
 ### Verifying GitHub Releases
 
-See [VERIFYING_PGP_SIGNATURE](VERIFYING_PGP_SIGNATURE.md) for more information.
+See [Verifying GitHub Releases](https://github.com/OpenJobDescription/openjd-cli?tab=security-ov-file#verifying-github-releases) for more information.
 
 ## Security
 

{openjd_cli-0.6.1/openjd/cli → openjd_cli-0.7.1}/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.6.1'
-__version_tuple__ = version_tuple = (0, 6, 1)
+__version__ = version = '0.7.1'
+__version_tuple__ = version_tuple = (0, 7, 1)

openjd_cli-0.7.1/hatch_version_hook.py

@@ -0,0 +1,156 @@
+import logging
+import os
+import shutil
+import sys
+
+from dataclasses import dataclass
+from hatchling.builders.hooks.plugin.interface import BuildHookInterface
+from typing import Any, Optional
+
+
+_logger = logging.Logger(__name__, logging.INFO)
+_stdout_handler = logging.StreamHandler(sys.stdout)
+_stdout_handler.addFilter(lambda record: record.levelno <= logging.INFO)
+_stderr_handler = logging.StreamHandler(sys.stderr)
+_stderr_handler.addFilter(lambda record: record.levelno > logging.INFO)
+_logger.addHandler(_stdout_handler)
+_logger.addHandler(_stderr_handler)
+
+
+@dataclass
+class CopyConfig:
+    sources: list[str]
+    destinations: list[str]
+
+
+class CustomBuildHookException(Exception):
+    pass
+
+
+class CustomBuildHook(BuildHookInterface):
+    """
+    A Hatch build hook that is pulled in automatically by Hatch's "custom" hook support
+    See: https://hatch.pypa.io/1.6/plugins/build-hook/custom/
+    This build hook copies files from one location (sources) to another (destinations).
+    Config options:
+    - `log_level (str)`: The logging level. Any value accepted by logging.Logger.setLevel is allowed. Default is INFO.
+    - `copy_map (list[dict])`: A list of mappings of files to copy and the destinations to copy them into. In TOML files,
+      this is expressed as an array of tables. See https://toml.io/en/v1.0.0#array-of-tables
+    Example TOML config:
+    ```
+    [tool.hatch.build.hooks.custom]
+    path = "hatch_hook.py"
+    log_level = "DEBUG"
+    [[tool.hatch.build.hooks.custom.copy_map]]
+    sources = [
+      "_version.py",
+    ]
+    destinations = [
+      "src/openjd",
+    ]
+    [[tool.hatch.build.hooks.custom.copy_map]]
+    sources = [
+      "something_the_tests_need.py",
+      "something_else_the_tests_need.ini",
+    ]
+    destinations = [
+      "test/openjd",
+    ]
+    ```
+    """
+
+    REQUIRED_OPTS = [
+        "copy_map",
+    ]
+
+    def initialize(self, version: str, build_data: dict[str, Any]) -> None:
+        if not self._prepare():
+            return
+
+        for copy_cfg in self.copy_map:
+            _logger.info(f"Copying {copy_cfg.sources} to {copy_cfg.destinations}")
+            for destination in copy_cfg.destinations:
+                for source in copy_cfg.sources:
+                    copy_func = shutil.copy if os.path.isfile(source) else shutil.copytree
+                    copy_func(
+                        os.path.join(self.root, source),
+                        os.path.join(self.root, destination),
+                    )
+            _logger.info("Copy complete")
+
+    def clean(self, versions: list[str]) -> None:
+        if not self._prepare():
+            return
+
+        for copy_cfg in self.copy_map:
+            _logger.info(f"Cleaning {copy_cfg.sources} from {copy_cfg.destinations}")
+            cleaned_count = 0
+            for destination in copy_cfg.destinations:
+                for source in copy_cfg.sources:
+                    source_path = os.path.join(self.root, destination, source)
+                    remove_func = os.remove if os.path.isfile(source_path) else os.rmdir
+                    try:
+                        remove_func(source_path)
+                    except FileNotFoundError:
+                        _logger.debug(f"Skipping {source_path} because it does not exist...")
+                    else:
+                        cleaned_count += 1
+            _logger.info(f"Cleaned {cleaned_count} items")
+
+    def _prepare(self) -> bool:
+        missing_required_opts = [
+            opt for opt in self.REQUIRED_OPTS if opt not in self.config or not self.config[opt]
+        ]
+        if missing_required_opts:
+            _logger.warn(
+                f"Required options {missing_required_opts} are missing or empty. "
+                "Contining without copying sources to destinations...",
+                file=sys.stderr,
+            )
+            return False
+
+        log_level = self.config.get("log_level")
+        if log_level:
+            _logger.setLevel(log_level)
+
+        return True
+
+    @property
+    def copy_map(self) -> Optional[list[CopyConfig]]:
+        raw_copy_map: list[dict] = self.config.get("copy_map")
+        if not raw_copy_map:
+            return None
+
+        if not (
+            isinstance(raw_copy_map, list)
+            and all(isinstance(copy_cfg, dict) for copy_cfg in raw_copy_map)
+        ):
+            raise CustomBuildHookException(
+                f'"copy_map" config option is a nonvalid type. Expected list[dict], but got {raw_copy_map}'
+            )
+
+        def verify_list_of_file_paths(file_paths: Any, config_name: str):
+            if not (isinstance(file_paths, list) and all(isinstance(fp, str) for fp in file_paths)):
+                raise CustomBuildHookException(
+                    f'"{config_name}" config option is a nonvalid type. Expected list[str], but got {file_paths}'
+                )
+
+            missing_paths = [
+                fp for fp in file_paths if not os.path.exists(os.path.join(self.root, fp))
+            ]
+            if len(missing_paths) > 0:
+                raise CustomBuildHookException(
+                    f'"{config_name}" config option contains some file paths that do not exist: {missing_paths}'
+                )
+
+        copy_map: list[CopyConfig] = []
+        for copy_cfg in raw_copy_map:
+            destinations: list[str] = copy_cfg.get("destinations")
+            verify_list_of_file_paths(destinations, "destinations")
+
+            sources: list[str] = copy_cfg.get("sources")
+            verify_list_of_file_paths(sources, "source")
+
+            copy_map.append(CopyConfig(sources, destinations))
+
+        return copy_map

{openjd_cli-0.6.1 → openjd_cli-0.7.1}/pyproject.toml

@@ -29,8 +29,8 @@ classifiers = [
   "Intended Audience :: End Users/Desktop"
 ]
 dependencies = [
-  "openjd-sessions >= 0.10.1,< 0.11",
-  "openjd-model == 0.7.*"
+  "openjd-sessions >= 0.10.3,< 0.11",
+  "openjd-model >= 0.8,< 0.9"
 ]
 
 [project.urls]
@@ -46,8 +46,6 @@ openjd = "openjd.cli:main"
 artifacts = [
   "*_version.py",
 ]
-only-packages = true
-
 
 [tool.hatch.version]
 source = "vcs"
@@ -70,15 +68,10 @@ destinations = [
 ]
 
 [tool.hatch.build.targets.sdist]
-packages = [
+include = [
   "src/openjd",
+  "hatch_version_hook.py",
 ]
-only-include = [
-  "src/openjd",
-]
-
-[tool.hatch.build.targets.sdist.force-include]
-"src/openjd/__main__.py" = "openjd/__main__.py"
 
 [tool.hatch.build.targets.wheel]
 packages = [
@@ -176,6 +169,7 @@ omit= [
 # Can be removed or set to true once we are v1
 major_on_zero = false
 tag_format = "{version}"
+allow_zero_version = true
 
 [tool.semantic_release.commit_parser_options]
 allowed_tags = [

{openjd_cli-0.6.1 → openjd_cli-0.7.1/src}/openjd/cli/_common/__init__.py

@@ -9,7 +9,7 @@ import json
 import yaml
 import os
 
-from ._extensions import add_extensions_argument, process_extensions_argument
+from ._extensions import add_extensions_argument, process_extensions_argument, SUPPORTED_EXTENSIONS
 from ._job_from_template import (
     job_from_template,
     get_job_params,
@@ -21,7 +21,7 @@ from ._validation_utils import (
     read_job_template,
     read_environment_template,
 )
-from openjd.model import DecodeValidationError, Job
+from openjd.model import DecodeValidationError, Job, JobParameterValues, EnvironmentTemplate
 
 __all__ = [
     "add_extensions_argument",
@@ -33,6 +33,7 @@ __all__ = [
     "read_job_template",
     "read_environment_template",
     "validate_task_parameters",
+    "SUPPORTED_EXTENSIONS",
 ]
 
 
@@ -112,13 +113,20 @@ class SubparserGroup:
         return self.group.add_parser(name, **kwargs)
 
 
-def generate_job(args: Namespace, *, supported_extensions: list[str]) -> Job:
+def generate_job(
+    args: Namespace,
+    environments: list[EnvironmentTemplate] = [],
+    *,
+    supported_extensions: list[str],
+) -> tuple[Job, JobParameterValues]:
     try:
         # Raises: RuntimeError, DecodeValidationError
         template = read_job_template(args.path, supported_extensions=supported_extensions)
+
         # Raises: RuntimeError
         return job_from_template(
             template,
+            environments,
             args.job_params if args.job_params else None,
             Path(os.path.abspath(args.path.parent)),
             Path(os.getcwd()),

{openjd_cli-0.6.1 → openjd_cli-0.7.1/src}/openjd/cli/_common/_extensions.py

@@ -4,7 +4,7 @@ from argparse import ArgumentParser
 from typing import Optional
 
 # This is the list of Open Job Description extensions with implemented support
-SUPPORTED_EXTENSIONS = ["TASK_CHUNKING"]
+SUPPORTED_EXTENSIONS = ["TASK_CHUNKING", "REDACTED_ENV_VARS"]
 
 
 def add_extensions_argument(run_parser: ArgumentParser):

{openjd_cli-0.6.1 → openjd_cli-0.7.1/src}/openjd/cli/_common/_job_from_template.py

@@ -4,14 +4,16 @@ from __future__ import annotations
 import json
 from pathlib import Path
 import re
-from typing import Union
+from typing import Optional, Union
 import yaml
 
 from ._validation_utils import get_doc_type
 from openjd.model import (
     DecodeValidationError,
     DocumentType,
+    EnvironmentTemplate,
     Job,
+    JobParameterValues,
     JobTemplate,
     create_job,
     preprocess_job_parameters,
@@ -54,7 +56,7 @@ def get_params_from_file(parameter_string: str) -> Union[dict, list]:
     return parameters
 
 
-def get_job_params(parameter_args: list[str]) -> dict:
+def get_job_params(parameter_args: Optional[list[str]]) -> dict:
     """
     Resolves Job Parameters from a list of command-line arguments.
     Arguments may be a filepath or a string with format 'Key=Value'.
@@ -62,7 +64,8 @@ def get_job_params(parameter_args: list[str]) -> dict:
     Raises: RuntimeError if the provided Parameters are formatted incorrectly or can't be opened
     """
     parameter_dict: dict = {}
-    for arg in parameter_args:
+
+    for arg in parameter_args or []:
         arg = arg.strip()
         # Case 1: Provided argument is a filepath
         if arg.startswith("file://"):
@@ -104,17 +107,18 @@ def get_job_params(parameter_args: list[str]) -> dict:
 
 def job_from_template(
     template: JobTemplate,
+    environments: list[EnvironmentTemplate],
     parameter_args: list[str] | None,
     job_template_dir: Path,
     current_working_dir: Path,
-) -> Job:
+) -> tuple[Job, JobParameterValues]:
     """
-    Given a decoded Job Template and a user-inputted parameter dictionary,
-    generates a Job object.
+    Given a decoded Job Template and a user-input parameter dictionary,
+    generates a Job object and the parameter values for running the job.
 
     Raises: RuntimeError if parameters are an unsupported type or don't correspond to the template
     """
-    parameter_dict = get_job_params(parameter_args) if parameter_args else {}
+    parameter_dict = get_job_params(parameter_args)
 
     try:
         parameter_values = preprocess_job_parameters(
@@ -122,11 +126,17 @@ def job_from_template(
             job_parameter_values=parameter_dict,
             job_template_dir=job_template_dir,
             current_working_dir=current_working_dir,
+            environment_templates=environments,
         )
     except ValueError as ve:
         raise RuntimeError(str(ve))
 
     try:
-        return create_job(job_template=template, job_parameter_values=parameter_values)
+        job = create_job(
+            job_template=template,
+            job_parameter_values=parameter_values,
+            environment_templates=environments,
+        )
+        return (job, parameter_values)
     except DecodeValidationError as dve:
         raise RuntimeError(f"Could not generate Job from template and parameters: {str(dve)}")

{openjd_cli-0.6.1 → openjd_cli-0.7.1/src}/openjd/cli/_run/_local_session/_session_manager.py

@@ -17,12 +17,14 @@ from ._actions import (
     EnvironmentType,
 )
 from ._logs import LocalSessionLogHandler, LogEntry, LoggingTimestampFormat
+from ..._common import SUPPORTED_EXTENSIONS
+
 from openjd.model import (
     IntRangeExpr,
     Job,
     JobParameterValues,
-    ParameterValue,
-    ParameterValueType,
+    RevisionExtensions,
+    SpecificationRevision,
     Step,
     StepParameterSpaceIterator,
     TaskParameterSet,
@@ -78,12 +80,16 @@ class LocalSession:
         self,
         *,
         job: Job,
+        job_parameter_values: JobParameterValues,
         session_id: str,
         timestamp_format: LoggingTimestampFormat = LoggingTimestampFormat.RELATIVE,
         path_mapping_rules: Optional[list[PathMappingRule]] = None,
         environments: Optional[list[Any]] = None,
         should_print_logs: bool = True,
         retain_working_dir: bool = False,
+        revision_extensions: RevisionExtensions = RevisionExtensions(
+            spec_rev=SpecificationRevision.v2023_09, supported_extensions=SUPPORTED_EXTENSIONS
+        ),
     ):
         self.session_id = session_id
         self._action_ended = Event()
@@ -93,21 +99,13 @@ class LocalSession:
         self._environments = environments
 
         # Create an OpenJD Session
-        job_parameters: JobParameterValues
-        if job.parameters:
-            job_parameters = {
-                name: ParameterValue(type=ParameterValueType(param.type.value), value=param.value)
-                for name, param in job.parameters.items()
-            }
-        else:
-            job_parameters = dict[str, ParameterValue]()
-
         self._openjd_session = Session(
             session_id=self.session_id,
-            job_parameter_values=job_parameters,
+            job_parameter_values=job_parameter_values,
             path_mapping_rules=self._path_mapping_rules,
             callback=self._action_callback,
             retain_working_dir=retain_working_dir,
+            revision_extensions=revision_extensions,
         )
 
         self._should_print_logs = should_print_logs
@@ -131,6 +129,11 @@ class LocalSession:
             signal(SIGTERM, SIG_DFL)
             self._started = False
 
+            # A blank line to separate the job log output from this status message
+            LOG.info(
+                msg="",
+                extra={"session_id": self.session_id},
+            )
             if self.failed:
                 LOG.info(
                     msg=f"Open Job Description CLI: ERROR executing action: '{self.failed_action}' (see Task logs for details)",

{openjd_cli-0.6.1 → openjd_cli-0.7.1/src}/openjd/cli/_run/_run_command.py

@@ -28,11 +28,14 @@ from openjd.model import (
     DecodeValidationError,
     EnvironmentTemplate,
     Job,
+    JobParameterValues,
     Step,
     StepDependencyGraph,
     StepParameterSpaceIterator,
     ParameterValue,
     ParameterValueType,
+    RevisionExtensions,
+    SpecificationRevision,
     TaskParameterSet,
 )
 from openjd.sessions import PathMappingRule, LOG
@@ -317,6 +320,7 @@ def _validate_task_params(step: Step, task_params: list[dict[str, str]]) -> None
 def _run_local_session(
     *,
     job: Job,
+    job_parameter_values: JobParameterValues,
     step_list: list[Step],
     selected_step: Optional[Step],
     timestamp_format: LoggingTimestampFormat,
@@ -326,6 +330,9 @@ def _run_local_session(
     path_mapping_rules: Optional[list[PathMappingRule]],
     should_print_logs: bool = True,
     retain_working_dir: bool = False,
+    revision_extensions: RevisionExtensions = RevisionExtensions(
+        spec_rev=SpecificationRevision.v2023_09, supported_extensions=[]
+    ),
 ) -> OpenJDCliResult:
     """
     Creates a Session object and listens for log messages to synchronously end the session.
@@ -337,12 +344,14 @@ def _run_local_session(
         step_name = ""
         with LocalSession(
             job=job,
+            job_parameter_values=job_parameter_values,
             timestamp_format=timestamp_format,
             session_id="CLI-session",
             path_mapping_rules=path_mapping_rules,
             environments=[env.environment for env in environments] if environments else [],
             should_print_logs=should_print_logs,
             retain_working_dir=retain_working_dir,
+            revision_extensions=revision_extensions,
         ) as session:
             for dep_step in step_list:
                 step_name = dep_step.name
@@ -442,7 +451,9 @@ def do_run(args: Namespace) -> OpenJDCliResult:
 
     try:
         # Raises: RuntimeError
-        the_job = generate_job(args, supported_extensions=extensions)
+        the_job, job_parameter_values = generate_job(
+            args, environments, supported_extensions=extensions
+        )
 
         # Map Step names to Step objects so they can be easily accessed
         step_map = {step.name: step for step in the_job.steps}
@@ -507,8 +518,15 @@ def do_run(args: Namespace) -> OpenJDCliResult:
     except RuntimeError as rte:
         return OpenJDCliResult(status="error", message=str(rte))
 
+    # Create a RevisionExtensions object with the default specification version and enabled extensions
+    # We use the default v2023_09 since that's what we're currently supporting
+    revision_extensions = RevisionExtensions(
+        spec_rev=the_job.revision, supported_extensions=extensions
+    )
+
     return _run_local_session(
         job=the_job,
+        job_parameter_values=job_parameter_values,
         step_list=step_list,
         selected_step=selected_step,
         task_parameter_values=task_parameter_values,
@@ -518,4 +536,5 @@ def do_run(args: Namespace) -> OpenJDCliResult:
         path_mapping_rules=path_mapping_rules,
         should_print_logs=(args.output == "human-readable"),
         retain_working_dir=args.preserve,
+        revision_extensions=revision_extensions,
     )

{openjd_cli-0.6.1 → openjd_cli-0.7.1/src}/openjd/cli/_summary/_summary_command.py

@@ -35,7 +35,7 @@ def do_summary(args: Namespace) -> OpenJDCliResult:
 
     try:
         # Raises: RuntimeError
-        sample_job = generate_job(args, supported_extensions=extensions)
+        sample_job, _ = generate_job(args, supported_extensions=extensions)
     except RuntimeError as rte:
         return OpenJDCliResult(status="error", message=str(rte))
 

openjd_cli-0.7.1/src/openjd/cli/_version.py

@@ -0,0 +1,21 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.7.1'
+__version_tuple__ = version_tuple = (0, 7, 1)