opentf-toolkit-nightly 0.62.0.dev1295__py3-none-any.whl → 0.62.0.dev1298__py3-none-any.whl

opentf/toolkit/channels.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021-2024 Henix, Henix.fr
+# Copyright (c) 2021 Henix, Henix.fr
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -43,7 +43,7 @@ WARNING_COMMAND = re.compile(r'^::warning(\s+(.*)+)?::(.*)$')
 ERROR_COMMAND = re.compile(r'^::error(\s+(.*)+)?::(.*)$')
 STOPCOMMANDS_COMMAND = re.compile(r'^::stop-commands::(\w+)$')
 ADDMASK_COMMAND = re.compile(r'^::add-mask::(.*)$')
-PUT_FILE_COMMAND = re.compile(r'^::put file=(.*?)\s*::(.*?)\s*$')
+PUT_FILE_COMMAND = re.compile(r'^::put\s+file=(.*?)\s*::(.*?)\s*$')
 
 
 ## step sequence IDs
@@ -136,7 +136,8 @@ OPENTF_VARIABLES_TYPE = 'application/vnd.opentestfactory.opentf-variables'
 OPENTF_VARIABLES_REGEX = re.compile(r'^(export|set)\s(\"?.+)$')
 OPENTF_VARIABLES_NAME_REGEX = re.compile(r'^[a-zA-Z0-9_]+$')
 
-## os helpers
+
+## Variables helpers
 
 
 def make_variable_linux(name: str, variable: Union[str, Dict[str, Any]]) -> str:
@@ -187,27 +188,92 @@ def _add_default_variables(
     script.append(VARIABLE_MAKER[runner_os]('CI', 'true'))
 
 
+def _get_opentf_variables_path(metadata: Dict[str, Any]) -> str:
+    return VARIABLES_TEMPLATE[metadata['channel_os']].format(
+        job_id=metadata['job_id'], root=metadata['channel_temp']
+    )
+
+
+def _get_opentf_variables(path: str) -> Dict[str, str]:
+    variables = {}
+    with open(path, 'r') as f:
+        for line in f.readlines():
+            if '=' not in line:
+                continue
+            line = line.strip()
+            if set_export := OPENTF_VARIABLES_REGEX.match(line):
+                line = set_export.group(2)
+                if line.startswith('"'):
+                    line = line[1:-1]
+            key, _, value = line.partition('=')
+            if OPENTF_VARIABLES_NAME_REGEX.match(key):
+                variables[key] = value
+    try:
+        os.remove(path)
+    except FileNotFoundError:
+        pass
+    return variables
+
+
+def process_opentf_variables(result: Dict[str, Any]) -> None:
+    """Process OPENTF_VARIABLES attachment if available.
+
+    If OPENTF_VARIABLES is in the attachments, move its content to
+    the variables field of the result.
+    """
+    if attachments := result['metadata'].get('attachments'):
+        variables = {}
+        for path, data in attachments.copy().items():
+            if data.get('type') == OPENTF_VARIABLES_TYPE:
+                variables = _get_opentf_variables(path)
+                del result['metadata']['attachments'][path]
+                result['attachments'].remove(path)
+                if not result['metadata']['attachments']:
+                    del result['metadata']['attachments']
+                if not result['attachments']:
+                    del result['attachments']
+        result['variables'] = variables
+
+
+# Workflow commands helpers
+
+
 def _make_attachment_url(
     metadata: Dict[str, Any],
     remote: str,
     separator: str,
-    workflow_result: bool = False,
+    is_artifact: bool = False,
     name: Optional[str] = None,
 ) -> Tuple[str, str]:
+    """Prepare attachment URL.
+
+    Attachment URLs are of the form:
+
+    ```
+    /tmp/{workflow_id}-{uuid}_WR_{name}                # artifacts
+    /tmp/{job_id}-{uuid}_{step_sequence_id}_{name}     # attachments
+    ```
+    """
     uuid = make_uuid()
-    prefix = metadata['workflow_id'] if workflow_result else metadata['job_id']
-    suffix = 'WR' if workflow_result else metadata['step_sequence_id']
+    prefix = metadata['workflow_id'] if is_artifact else metadata['job_id']
+    suffix = 'WR' if is_artifact else metadata['step_sequence_id']
     url = f'/tmp/{prefix}-{uuid}_{suffix}_{name or remote.split(separator)[-1]}'
     return url, uuid
 
 
-def _get_cmd_params(args: str, logs: List[str]) -> Union[int, Dict[str, str]]:
+class CommandException(Exception):
+    pass
+
+
+def _get_cmd_params(args: str) -> Dict[str, str]:
+    """Extract workflow command parameters."""
     details = {}
     if args:
         for parameter in args.strip().split(','):
             if '=' not in parameter:
-                logs.append(f'ERROR,Invalid workflow command parameter: {parameter}.')
-                return 2
+                raise CommandException(
+                    f'Invalid workflow command parameter: {parameter}.'
+                )
             key, _, value = parameter.strip().partition('=')
             details[key] = value
     return details
@@ -239,55 +305,22 @@ def _as_log(line: str, jobstate: JobState):
     return mask(line, jobstate).rstrip()
 
 
-def _get_opentf_variables(path: str) -> Dict[str, str]:
-    variables = {}
-    with open(path, 'r') as f:
-        for line in f.readlines():
-            if '=' not in line:
-                continue
-            line = line.strip()
-            if set_export := OPENTF_VARIABLES_REGEX.match(line):
-                line = set_export.group(2)
-                if line.startswith('"'):
-                    line = line[1:-1]
-            key, _, value = line.partition('=')
-            if OPENTF_VARIABLES_NAME_REGEX.match(key):
-                variables[key] = value
-    try:
-        os.remove(path)
-    except FileNotFoundError:
-        pass
-    return variables
-
+def process_upload(result: Dict[str, Any]) -> Dict[str, Any]:
+    """Process ExecutionResult event containing .metadata.upload flag.
 
-def process_opentf_variables(result: Dict[str, Any]) -> None:
-    """Process OPENTF_VARIABLES attachment if available.
+    Remove the artifact-related items from `result` (URLs starting with
+    `/tmp/{workflow_id}_WR_`).
 
-    If OPENTF_VARIABLES is in the attachments, move its content to
-    the variables field of the result.
-    """
-    if attachments := result['metadata'].get('attachments'):
-        variables = {}
-        for path, data in attachments.copy().items():
-            if data.get('type') == OPENTF_VARIABLES_TYPE:
-                variables = _get_opentf_variables(path)
-                del result['metadata']['attachments'][path]
-                result['attachments'].remove(path)
-                if not result['metadata']['attachments']:
-                    del result['metadata']['attachments']
-                if not result['attachments']:
-                    del result['attachments']
-        result['variables'] = variables
+    # Required parameters
 
+    - result: a dictionary (an ExecutionResult command)
 
-def process_upload(result: Dict[str, Any]):
-    """Process ExecutionResult event containing .metadata.upload flag.
+    # Returned value
 
-    Dispatch attachments between ExecutionResult and WorkflowResult events,
-    publish WorkflowResult.
+    A WorkflowResult event.
     """
 
-    def _filter_items(key: str):
+    def _filter_items(key: str) -> Tuple[List[str], Dict[str, Any]]:
         metadata = {}
         items = [item for item in result['attachments'] if item[pos:].startswith(key)]
         if items:
@@ -318,6 +351,53 @@ def process_upload(result: Dict[str, Any]):
     return make_event(WORKFLOWRESULT, metadata=metadata, attachments=upload)
 
 
+def _download_artifacts(
+    artifacts: List[str],
+    targeted_remote_path: str,
+    remote_path: str,
+    params: Dict[str, str],
+    _put: Callable[[str, str], None],
+    is_windows: bool,
+) -> bool:
+    """Download artifacts matching a pattern.
+
+    # Required parameters
+
+    - artifacts: a list of artifact names
+    - targeted_remote_path: a string
+    - remote_path: a string
+    - params: a dictionary
+    - _put: a function copying a local file to a remote environment
+    - is_windows: a boolean
+
+    # Returned value
+
+    A boolean indicating if artifacts were uploaded.
+    """
+    filename, pattern = params.get('file'), params.get('pattern')
+    if filename and pattern:
+        raise CommandException('Cannot specify both "file" and "pattern".')
+    if not filename and not pattern:
+        pattern = '*'
+    found = False
+    for artifact in artifacts:
+        artifact_name = artifact.split('_')[-1]
+        if filename and filename == artifact_name:
+            found = True
+            if not remote_path:
+                targeted_remote_path = core.join_path(
+                    targeted_remote_path, artifact_name, is_windows
+                )
+            _put(targeted_remote_path, artifact)
+        elif pattern and fnmatch.fnmatch(artifact_name, pattern):
+            found = True
+            pattern_path = core.join_path(
+                targeted_remote_path, artifact_name, is_windows
+            )
+            _put(pattern_path, artifact)
+    return found
+
+
 def process_output(
     event: Dict[str, Any],
     resp: int,
@@ -366,95 +446,73 @@ def process_output(
     May raise exceptions.
     """
 
-    def _get_targeted_path(remote_path: str):
+    def _get_targeted_path(remote_path: str) -> str:
         working_directory = core.join_path(
             metadata['job_id'], event.get('working-directory'), is_windows
         )
         return core.join_path(working_directory, remote_path, is_windows)
 
-    def _attach(remote: str, args: str, is_artefact: bool = False):
+    def _attach(remote: str, args: str, is_artifact: bool = False) -> int:
         if is_windows:
             remote = ntpath.normpath(remote)
         try:
-            if (params := _get_cmd_params(args, logs)) == 2:
-                return 2
+            params = _get_cmd_params(args)
             attachment_url, uuid = _make_attachment_url(
-                metadata, remote, separator, is_artefact, name=params.get('name')
+                metadata, remote, separator, is_artifact, name=params.get('name')
             )
             params['uuid'] = uuid
             _get(remote, attachment_url)
             attachments_metadata[attachment_url] = params
             attachments.append(attachment_url)
             return resp
+        except CommandException as err:
+            logs.append(f'ERROR,{err.args[0]}')
         except Exception as err:
             logs.append(f'ERROR,Could not read {remote}: {err}.')
-            return 2
+        return 2
 
-    def _putfile(remote_path: str, data: str):
+    def _putfile(remote_path: str, data: str) -> int:
         targeted_remote_path = _get_targeted_path(remote_path)
         try:
             file_ = f'/tmp/in_{metadata["workflow_id"]}_{data}'
             if not os.path.exists(file_):
-                logs.append(f'ERROR,Invalid resources.files reference {data}.')
+                logs.append(f'ERROR,Invalid "resources.files" reference "{data}".')
                 return 2
             _put(targeted_remote_path, file_)
             return resp
         except Exception as err:
             logs.append(
-                f'ERROR,Could not send file {data} to remote path {remote_path}: {err}.'
+                f'ERROR,Could not send file "{data}" to remote path "{remote_path}": {err}.'
             )
             return 2
 
-    def _download(remote_path: str, args: str):
-        if not (artifacts := event['metadata'].get('artifacts')):
-            logs.append('ERROR,No artifacts available at workflow level.')
-            return 2
+    def _download(remote_path: str, args: str) -> int:
         try:
+            artifacts = metadata.get('artifacts', [])
             targeted_remote_path = _get_targeted_path(remote_path)
-            if (params := _get_cmd_params(args, logs)) == 2:
-                return 2
-            filename, pattern = params.get('file'), params.get('pattern')
-            if not filename and not pattern:
-                pattern = '*'
-            found = False
-            for artifact in artifacts:
-                artifact_name = artifact.split('_')[-1]
-                if filename and filename == artifact_name:
-                    found = True
-                    if not remote_path:
-                        targeted_remote_path = core.join_path(
-                            targeted_remote_path, artifact_name, is_windows
-                        )
-                    _put(targeted_remote_path, artifact)
-                    continue
-                if pattern and fnmatch.fnmatch(artifact_name, pattern):
-                    found = True
-                    pattern_path = core.join_path(
-                        targeted_remote_path, artifact_name, is_windows
-                    )
-                    _put(pattern_path, artifact)
-            if found:
+            params = _get_cmd_params(args)
+            if _download_artifacts(
+                artifacts, targeted_remote_path, remote_path, params, _put, is_windows
+            ):
                 return resp
             logs.append('ERROR,No artifact matching requested name/pattern found.')
-            return 2
+        except CommandException as err:
+            logs.append(f'ERROR,{err.args[0]}')
         except Exception as err:
             logs.append(
-                f'ERROR,Could not send artifacts to remote path {remote_path}: {err}.'
+                f'ERROR,Could not send artifacts to remote path "{remote_path}": {err}.'
             )
-            return 2
+        return 2
 
     metadata: Dict[str, Any] = event['metadata']
     is_windows: bool = metadata['channel_os'] == 'windows'
-    opentfvariables = VARIABLES_TEMPLATE[metadata['channel_os']].format(
-        job_id=metadata['job_id'], root=metadata['channel_temp']
-    )
     separator = '\\' if is_windows else '/'
     outputs = {}
     logs: List[str] = []
     attachments: List[str] = []
     attachments_metadata = {}
 
-    has_artefacts = False
+    has_artifacts = False
     for line in stdout:
         # Parsing stdout for workflow commands
         if jobstate.stop_command:
@@ -465,8 +523,8 @@ def process_output(
         if wcmd := ATTACH_COMMAND.match(line):
             resp = _attach(wcmd.group(2), wcmd.group(1))
         elif wcmd := UPLOAD_COMMAND.match(line):
-            has_artefacts = True
-            resp = _attach(wcmd.group(2), wcmd.group(1), is_artefact=True)
+            has_artifacts = True
+            resp = _attach(wcmd.group(2), wcmd.group(1), is_artifact=True)
         elif wcmd := DOWNLOAD_COMMAND.match(line):
             resp = _download(wcmd.group(2), wcmd.group(1))
         elif wcmd := PUT_FILE_COMMAND.match(line):
@@ -487,10 +545,9 @@ def process_output(
         del metadata['artifacts']
 
     if metadata['step_sequence_id'] != CHANNEL_RELEASE:
-        _attach(opentfvariables, f'type={OPENTF_VARIABLES_TYPE}')
+        _attach(_get_opentf_variables_path(metadata), f'type={OPENTF_VARIABLES_TYPE}')
 
     result = make_event(EXECUTIONRESULT, metadata=metadata, status=resp)
-
     if outputs:
         result['outputs'] = outputs
     if logs:
@@ -498,8 +555,9 @@ def process_output(
     if attachments:
         result['attachments'] = attachments
         result['metadata']['attachments'] = attachments_metadata
-    if has_artefacts:
+    if has_artifacts:
         result['metadata']['upload'] = resp
+
     return result
 
 

{opentf_toolkit_nightly-0.62.0.dev1295.dist-info → opentf_toolkit_nightly-0.62.0.dev1298.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: opentf-toolkit-nightly
-Version: 0.62.0.dev1295
+Version: 0.62.0.dev1298
 Summary: OpenTestFactory Orchestrator Toolkit
 Home-page: https://gitlab.com/henixdevelopment/open-source/opentestfactory/python-toolkit
 Author: Martin Lafaix

{opentf_toolkit_nightly-0.62.0.dev1295.dist-info → opentf_toolkit_nightly-0.62.0.dev1298.dist-info}/RECORD

@@ -56,10 +56,10 @@ opentf/schemas/opentestfactory.org/v1beta2/ServiceConfig.json,sha256=rEvK2YWL5lG
 opentf/scripts/launch_java_service.sh,sha256=S0jAaCuv2sZy0Gf2NGBuPX-eD531rcM-b0fNyhmzSjw,2423
 opentf/scripts/startup.py,sha256=vOGxl7xBcp1-_LsAKiOmeOqFl2iT81A2XRrXBLUrNi4,22785
 opentf/toolkit/__init__.py,sha256=ohrde5mcMY26p64E0Z2XunZAWYOiEkXKTg5E1J4TGGc,23571
-opentf/toolkit/channels.py,sha256=whLfPVT5PksVlprmoeb2ktaZ3KEhqyryUCVWBJq7PeY,24308
+opentf/toolkit/channels.py,sha256=6qKSsAgq_oJpuDRiKqVUz-EAjdfikcCG3SFAGmKZdhQ,25551
 opentf/toolkit/core.py,sha256=fqnGgaYnuVcd4fyeNIwpc0QtyUo7jsKeVgdkBfY3iqo,9443
-opentf_toolkit_nightly-0.62.0.dev1295.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-opentf_toolkit_nightly-0.62.0.dev1295.dist-info/METADATA,sha256=Dftf4ExDEznTennMkNJ7wCZoK1HONVmPhb_jwMqi6W4,2192
-opentf_toolkit_nightly-0.62.0.dev1295.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-opentf_toolkit_nightly-0.62.0.dev1295.dist-info/top_level.txt,sha256=_gPuE6GTT6UNXy1DjtmQSfCcZb_qYA2vWmjg7a30AGk,7
-opentf_toolkit_nightly-0.62.0.dev1295.dist-info/RECORD,,
+opentf_toolkit_nightly-0.62.0.dev1298.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+opentf_toolkit_nightly-0.62.0.dev1298.dist-info/METADATA,sha256=zPOTouTQoLWUCjDEAvzyd6zBD72RgfGlTL9UGDTITuw,2192
+opentf_toolkit_nightly-0.62.0.dev1298.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+opentf_toolkit_nightly-0.62.0.dev1298.dist-info/top_level.txt,sha256=_gPuE6GTT6UNXy1DjtmQSfCcZb_qYA2vWmjg7a30AGk,7
+opentf_toolkit_nightly-0.62.0.dev1298.dist-info/RECORD,,