opentf-toolkit-nightly 0.62.0.dev1291__py3-none-any.whl → 0.62.0.dev1298__py3-none-any.whl

opentf/commons/__init__.py

@@ -181,7 +181,10 @@ def _get_contextparameter_spec(app: Flask, name: str) -> Optional[Dict[str, Any]
     Initialize cache if needed, ignoring context parameters specs from
     other services.
 
-    Adds the `watchdog_polling_delay_seconds` spec.
+    Adds the following specs if not already present:
+
+    - `watchdog_polling_delay_seconds`
+    - `availability_check_delay_seconds`
     """
     if PARAMETERS_KEY not in app.config:
         app.config[PARAMETERS_KEY] = []
@@ -191,14 +194,28 @@ def _get_contextparameter_spec(app: Flask, name: str) -> Optional[Dict[str, Any]
             app.config[PARAMETERS_KEY] += manifest.get('spec', {}).get(
                 'contextParameters', []
             )
-        app.config[PARAMETERS_KEY] += [
-            {
-                'name': 'watchdog_polling_delay_seconds',
-                'descriptiveName': 'polling delay in seconds',
-                'default': 30,
-                'type': 'int',
-            }
-        ]
+        known = {spec['name'] for spec in app.config[PARAMETERS_KEY]}
+        if 'watchdog_polling_delay_seconds' not in known:
+            app.config[PARAMETERS_KEY].append(
+                {
+                    'name': 'watchdog_polling_delay_seconds',
+                    'descriptiveName': 'files watchdog polling delay in seconds',
+                    'default': 30,
+                    'type': 'int',
+                }
+            )
+        if 'availability_check_delay_seconds' not in known:
+            app.config[PARAMETERS_KEY].append(
+                {
+                    'name': 'availability_check_delay_seconds',
+                    'deprecatedNames': ['availability_check_delay'],
+                    'descriptiveName': 'availability check frequency in seconds',
+                    'type': 'int',
+                    'default': 10,
+                    'minValue': 10,
+                }
+            )
+
         app.logger.info('Configuration:')
     parameters = app.config[PARAMETERS_KEY]
     try:

opentf/toolkit/__init__.py

@@ -58,6 +58,7 @@ DISPATCHQUEUE_KEY = '__dispatch queue__'
 
 WATCHDOG_POLLING_DELAY_SECONDS = 30
 WATCHDOG_POLLING_DELAY_KEY = 'watchdog_polling_delay_seconds'
+AVAILABILITY_CHECK_DELAY_SECONDS = 'availability_check_delay_seconds'
 
 Handler = Callable[[Dict[str, Any]], Any]
 
@@ -398,6 +399,10 @@ def watch_file(plugin: Flask, path: str, handler, *args, **kwargs) -> None:
     a file path (a string).  It may take additional parameters.  It will
     be called whenever the file changes.
 
+    The watchdog polls every 30 seconds by default.  This can be
+    adjusted by setting the `watchdog_polling_delay_seconds` context
+    parameter (but it cannot be more frequent).
+
     # Required parameters
 
     - plugin: a Flask application
@@ -460,7 +465,7 @@ def watch_and_notify(
     - items: an iterable
     - notify: a function of no arguments
     """
-    polling_delay = get_context_parameter(plugin, 'availability_check_delay_seconds')
+    polling_delay = get_context_parameter(plugin, AVAILABILITY_CHECK_DELAY_SECONDS)
 
     plugin.logger.debug('Starting watch notifier thread.')
     threading.Thread(

opentf/toolkit/channels.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021-2024 Henix, Henix.fr
+# Copyright (c) 2021 Henix, Henix.fr
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -43,7 +43,7 @@ WARNING_COMMAND = re.compile(r'^::warning(\s+(.*)+)?::(.*)$')
 ERROR_COMMAND = re.compile(r'^::error(\s+(.*)+)?::(.*)$')
 STOPCOMMANDS_COMMAND = re.compile(r'^::stop-commands::(\w+)$')
 ADDMASK_COMMAND = re.compile(r'^::add-mask::(.*)$')
-PUT_FILE_COMMAND = re.compile(r'^::put file=(.*?)\s*::(.*?)\s*$')
+PUT_FILE_COMMAND = re.compile(r'^::put\s+file=(.*?)\s*::(.*?)\s*$')
 
 
 ## step sequence IDs
@@ -136,7 +136,8 @@ OPENTF_VARIABLES_TYPE = 'application/vnd.opentestfactory.opentf-variables'
 OPENTF_VARIABLES_REGEX = re.compile(r'^(export|set)\s(\"?.+)$')
 OPENTF_VARIABLES_NAME_REGEX = re.compile(r'^[a-zA-Z0-9_]+$')
 
-## os helpers
+
+## Variables helpers
 
 
 def make_variable_linux(name: str, variable: Union[str, Dict[str, Any]]) -> str:
@@ -187,27 +188,92 @@ def _add_default_variables(
     script.append(VARIABLE_MAKER[runner_os]('CI', 'true'))
 
 
+def _get_opentf_variables_path(metadata: Dict[str, Any]) -> str:
+    return VARIABLES_TEMPLATE[metadata['channel_os']].format(
+        job_id=metadata['job_id'], root=metadata['channel_temp']
+    )
+
+
+def _get_opentf_variables(path: str) -> Dict[str, str]:
+    variables = {}
+    with open(path, 'r') as f:
+        for line in f.readlines():
+            if '=' not in line:
+                continue
+            line = line.strip()
+            if set_export := OPENTF_VARIABLES_REGEX.match(line):
+                line = set_export.group(2)
+                if line.startswith('"'):
+                    line = line[1:-1]
+            key, _, value = line.partition('=')
+            if OPENTF_VARIABLES_NAME_REGEX.match(key):
+                variables[key] = value
+    try:
+        os.remove(path)
+    except FileNotFoundError:
+        pass
+    return variables
+
+
+def process_opentf_variables(result: Dict[str, Any]) -> None:
+    """Process OPENTF_VARIABLES attachment if available.
+
+    If OPENTF_VARIABLES is in the attachments, move its content to
+    the variables field of the result.
+    """
+    if attachments := result['metadata'].get('attachments'):
+        variables = {}
+        for path, data in attachments.copy().items():
+            if data.get('type') == OPENTF_VARIABLES_TYPE:
+                variables = _get_opentf_variables(path)
+                del result['metadata']['attachments'][path]
+                result['attachments'].remove(path)
+                if not result['metadata']['attachments']:
+                    del result['metadata']['attachments']
+                if not result['attachments']:
+                    del result['attachments']
+        result['variables'] = variables
+
+
+# Workflow commands helpers
+
+
 def _make_attachment_url(
     metadata: Dict[str, Any],
     remote: str,
     separator: str,
-    workflow_result: bool = False,
+    is_artifact: bool = False,
     name: Optional[str] = None,
 ) -> Tuple[str, str]:
+    """Prepare attachment URL.
+
+    Attachment URLs are of the form:
+
+    ```
+    /tmp/{workflow_id}-{uuid}_WR_{name}                # artifacts
+    /tmp/{job_id}-{uuid}_{step_sequence_id}_{name}     # attachments
+    ```
+    """
     uuid = make_uuid()
-    prefix = metadata['workflow_id'] if workflow_result else metadata['job_id']
-    suffix = 'WR' if workflow_result else metadata['step_sequence_id']
+    prefix = metadata['workflow_id'] if is_artifact else metadata['job_id']
+    suffix = 'WR' if is_artifact else metadata['step_sequence_id']
     url = f'/tmp/{prefix}-{uuid}_{suffix}_{name or remote.split(separator)[-1]}'
     return url, uuid
 
 
-def _get_cmd_params(args: str, logs: List[str]) -> Union[int, Dict[str, str]]:
+class CommandException(Exception):
+    pass
+
+
+def _get_cmd_params(args: str) -> Dict[str, str]:
+    """Extract workflow command parameters."""
     details = {}
     if args:
         for parameter in args.strip().split(','):
             if '=' not in parameter:
-                logs.append(f'ERROR,Invalid workflow command parameter: {parameter}.')
-                return 2
+                raise CommandException(
+                    f'Invalid workflow command parameter: {parameter}.'
+                )
             key, _, value = parameter.strip().partition('=')
             details[key] = value
     return details
@@ -239,55 +305,22 @@ def _as_log(line: str, jobstate: JobState):
     return mask(line, jobstate).rstrip()
 
 
-def _get_opentf_variables(path: str) -> Dict[str, str]:
-    variables = {}
-    with open(path, 'r') as f:
-        for line in f.readlines():
-            if '=' not in line:
-                continue
-            line = line.strip()
-            if set_export := OPENTF_VARIABLES_REGEX.match(line):
-                line = set_export.group(2)
-                if line.startswith('"'):
-                    line = line[1:-1]
-            key, _, value = line.partition('=')
-            if OPENTF_VARIABLES_NAME_REGEX.match(key):
-                variables[key] = value
-    try:
-        os.remove(path)
-    except FileNotFoundError:
-        pass
-    return variables
-
+def process_upload(result: Dict[str, Any]) -> Dict[str, Any]:
+    """Process ExecutionResult event containing .metadata.upload flag.
 
-def process_opentf_variables(result: Dict[str, Any]) -> None:
-    """Process OPENTF_VARIABLES attachment if available.
+    Remove the artifact-related items from `result` (URLs starting with
+    `/tmp/{workflow_id}_WR_`).
 
-    If OPENTF_VARIABLES is in the attachments, move its content to
-    the variables field of the result.
-    """
-    if attachments := result['metadata'].get('attachments'):
-        variables = {}
-        for path, data in attachments.copy().items():
-            if data.get('type') == OPENTF_VARIABLES_TYPE:
-                variables = _get_opentf_variables(path)
-                del result['metadata']['attachments'][path]
-                result['attachments'].remove(path)
-                if not result['metadata']['attachments']:
-                    del result['metadata']['attachments']
-                if not result['attachments']:
-                    del result['attachments']
-        result['variables'] = variables
+    # Required parameters
 
+    - result: a dictionary (an ExecutionResult command)
 
-def process_upload(result: Dict[str, Any]):
-    """Process ExecutionResult event containing .metadata.upload flag.
+    # Returned value
 
-    Dispatch attachments between ExecutionResult and WorkflowResult events,
-    publish WorkflowResult.
+    A WorkflowResult event.
     """
 
-    def _filter_items(key: str):
+    def _filter_items(key: str) -> Tuple[List[str], Dict[str, Any]]:
         metadata = {}
         items = [item for item in result['attachments'] if item[pos:].startswith(key)]
         if items:
@@ -318,6 +351,53 @@ def process_upload(result: Dict[str, Any]):
     return make_event(WORKFLOWRESULT, metadata=metadata, attachments=upload)
 
 
+def _download_artifacts(
+    artifacts: List[str],
+    targeted_remote_path: str,
+    remote_path: str,
+    params: Dict[str, str],
+    _put: Callable[[str, str], None],
+    is_windows: bool,
+) -> bool:
+    """Download artifacts matching a pattern.
+
+    # Required parameters
+
+    - artifacts: a list of artifact names
+    - targeted_remote_path: a string
+    - remote_path: a string
+    - params: a dictionary
+    - _put: a function copying a local file to a remote environment
+    - is_windows: a boolean
+
+    # Returned value
+
+    A boolean indicating if artifacts were uploaded.
+    """
+    filename, pattern = params.get('file'), params.get('pattern')
+    if filename and pattern:
+        raise CommandException('Cannot specify both "file" and "pattern".')
+    if not filename and not pattern:
+        pattern = '*'
+    found = False
+    for artifact in artifacts:
+        artifact_name = artifact.split('_')[-1]
+        if filename and filename == artifact_name:
+            found = True
+            if not remote_path:
+                targeted_remote_path = core.join_path(
+                    targeted_remote_path, artifact_name, is_windows
+                )
+            _put(targeted_remote_path, artifact)
+        elif pattern and fnmatch.fnmatch(artifact_name, pattern):
+            found = True
+            pattern_path = core.join_path(
+                targeted_remote_path, artifact_name, is_windows
+            )
+            _put(pattern_path, artifact)
+    return found
+
+
 def process_output(
     event: Dict[str, Any],
     resp: int,
@@ -366,95 +446,73 @@ def process_output(
     May raise exceptions.
     """
 
-    def _get_targeted_path(remote_path: str):
+    def _get_targeted_path(remote_path: str) -> str:
         working_directory = core.join_path(
             metadata['job_id'], event.get('working-directory'), is_windows
         )
         return core.join_path(working_directory, remote_path, is_windows)
 
-    def _attach(remote: str, args: str, is_artefact: bool = False):
+    def _attach(remote: str, args: str, is_artifact: bool = False) -> int:
         if is_windows:
             remote = ntpath.normpath(remote)
         try:
-            if (params := _get_cmd_params(args, logs)) == 2:
-                return 2
+            params = _get_cmd_params(args)
             attachment_url, uuid = _make_attachment_url(
-                metadata, remote, separator, is_artefact, name=params.get('name')
+                metadata, remote, separator, is_artifact, name=params.get('name')
             )
             params['uuid'] = uuid
             _get(remote, attachment_url)
             attachments_metadata[attachment_url] = params
             attachments.append(attachment_url)
             return resp
+        except CommandException as err:
+            logs.append(f'ERROR,{err.args[0]}')
         except Exception as err:
             logs.append(f'ERROR,Could not read {remote}: {err}.')
-            return 2
+        return 2
 
-    def _putfile(remote_path: str, data: str):
+    def _putfile(remote_path: str, data: str) -> int:
         targeted_remote_path = _get_targeted_path(remote_path)
         try:
             file_ = f'/tmp/in_{metadata["workflow_id"]}_{data}'
             if not os.path.exists(file_):
-                logs.append(f'ERROR,Invalid resources.files reference {data}.')
+                logs.append(f'ERROR,Invalid "resources.files" reference "{data}".')
                 return 2
             _put(targeted_remote_path, file_)
             return resp
         except Exception as err:
             logs.append(
-                f'ERROR,Could not send file {data} to remote path {remote_path}: {err}.'
+                f'ERROR,Could not send file "{data}" to remote path "{remote_path}": {err}.'
             )
             return 2
 
-    def _download(remote_path: str, args: str):
-        if not (artifacts := event['metadata'].get('artifacts')):
-            logs.append('ERROR,No artifacts available at workflow level.')
-            return 2
+    def _download(remote_path: str, args: str) -> int:
         try:
+            artifacts = metadata.get('artifacts', [])
             targeted_remote_path = _get_targeted_path(remote_path)
-            if (params := _get_cmd_params(args, logs)) == 2:
-                return 2
-            filename, pattern = params.get('file'), params.get('pattern')
-            if not filename and not pattern:
-                pattern = '*'
-            found = False
-            for artifact in artifacts:
-                artifact_name = artifact.split('_')[-1]
-                if filename and filename == artifact_name:
-                    found = True
-                    if not remote_path:
-                        targeted_remote_path = core.join_path(
-                            targeted_remote_path, artifact_name, is_windows
-                        )
-                    _put(targeted_remote_path, artifact)
-                    continue
-                if pattern and fnmatch.fnmatch(artifact_name, pattern):
-                    found = True
-                    pattern_path = core.join_path(
-                        targeted_remote_path, artifact_name, is_windows
-                    )
-                    _put(pattern_path, artifact)
-            if found:
+            params = _get_cmd_params(args)
+            if _download_artifacts(
+                artifacts, targeted_remote_path, remote_path, params, _put, is_windows
+            ):
                 return resp
             logs.append('ERROR,No artifact matching requested name/pattern found.')
-            return 2
+        except CommandException as err:
+            logs.append(f'ERROR,{err.args[0]}')
         except Exception as err:
             logs.append(
-                f'ERROR,Could not send artifacts to remote path {remote_path}: {err}.'
+                f'ERROR,Could not send artifacts to remote path "{remote_path}": {err}.'
             )
-            return 2
+        return 2
 
     metadata: Dict[str, Any] = event['metadata']
     is_windows: bool = metadata['channel_os'] == 'windows'
-    opentfvariables = VARIABLES_TEMPLATE[metadata['channel_os']].format(
-        job_id=metadata['job_id'], root=metadata['channel_temp']
-    )
     separator = '\\' if is_windows else '/'
     outputs = {}
     logs: List[str] = []
     attachments: List[str] = []
     attachments_metadata = {}
 
-    has_artefacts = False
+    has_artifacts = False
     for line in stdout:
         # Parsing stdout for workflow commands
         if jobstate.stop_command:
@@ -465,8 +523,8 @@ def process_output(
         if wcmd := ATTACH_COMMAND.match(line):
             resp = _attach(wcmd.group(2), wcmd.group(1))
         elif wcmd := UPLOAD_COMMAND.match(line):
-            has_artefacts = True
-            resp = _attach(wcmd.group(2), wcmd.group(1), is_artefact=True)
+            has_artifacts = True
+            resp = _attach(wcmd.group(2), wcmd.group(1), is_artifact=True)
         elif wcmd := DOWNLOAD_COMMAND.match(line):
             resp = _download(wcmd.group(2), wcmd.group(1))
         elif wcmd := PUT_FILE_COMMAND.match(line):
@@ -487,10 +545,9 @@ def process_output(
         del metadata['artifacts']
 
     if metadata['step_sequence_id'] != CHANNEL_RELEASE:
-        _attach(opentfvariables, f'type={OPENTF_VARIABLES_TYPE}')
+        _attach(_get_opentf_variables_path(metadata), f'type={OPENTF_VARIABLES_TYPE}')
 
     result = make_event(EXECUTIONRESULT, metadata=metadata, status=resp)
-
     if outputs:
         result['outputs'] = outputs
     if logs:
@@ -498,8 +555,9 @@ def process_output(
     if attachments:
         result['attachments'] = attachments
         result['metadata']['attachments'] = attachments_metadata
-    if has_artefacts:
+    if has_artifacts:
         result['metadata']['upload'] = resp
+
     return result
 
 

{opentf_toolkit_nightly-0.62.0.dev1291.dist-info → opentf_toolkit_nightly-0.62.0.dev1298.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: opentf-toolkit-nightly
-Version: 0.62.0.dev1291
+Version: 0.62.0.dev1298
 Summary: OpenTestFactory Orchestrator Toolkit
 Home-page: https://gitlab.com/henixdevelopment/open-source/opentestfactory/python-toolkit
 Author: Martin Lafaix

{opentf_toolkit_nightly-0.62.0.dev1291.dist-info → opentf_toolkit_nightly-0.62.0.dev1298.dist-info}/RECORD

@@ -1,4 +1,4 @@
-opentf/commons/__init__.py,sha256=_LTxZRG9KKhmx6T5Tn19un_FBE2WH_1ZV_cG43MYAcI,23441
+opentf/commons/__init__.py,sha256=2hd1g14g9pnZl1uip6Sh90YSmLDWzkH1SGeZweRJ4z8,24180
 opentf/commons/auth.py,sha256=gXRp_0Tf3bfd65F4QiQmh6C6vR9y3ugag_0DSvozJFk,15898
 opentf/commons/config.py,sha256=RVSSdQhMle4oCo_z_AR2EQ4U6sUjSxw-qVBtjKuJVfo,10219
 opentf/commons/exceptions.py,sha256=7dhUXO8iyAbqVwlUKxZhgRzGqVcb7LkG39hFlm-VxIA,2407
@@ -55,11 +55,11 @@ opentf/schemas/opentestfactory.org/v1beta1/Workflow.json,sha256=QZ8mM9PhzsI9gTmw
 opentf/schemas/opentestfactory.org/v1beta2/ServiceConfig.json,sha256=rEvK2YWL5lG94_qYgR_GnLWNsaQhaQ-2kuZdWJr5NnY,3517
 opentf/scripts/launch_java_service.sh,sha256=S0jAaCuv2sZy0Gf2NGBuPX-eD531rcM-b0fNyhmzSjw,2423
 opentf/scripts/startup.py,sha256=vOGxl7xBcp1-_LsAKiOmeOqFl2iT81A2XRrXBLUrNi4,22785
-opentf/toolkit/__init__.py,sha256=_I003tfzaNTpMxS_CgqMIrgud66Rofz4pfFk0Jbjwls,23320
-opentf/toolkit/channels.py,sha256=whLfPVT5PksVlprmoeb2ktaZ3KEhqyryUCVWBJq7PeY,24308
+opentf/toolkit/__init__.py,sha256=ohrde5mcMY26p64E0Z2XunZAWYOiEkXKTg5E1J4TGGc,23571
+opentf/toolkit/channels.py,sha256=6qKSsAgq_oJpuDRiKqVUz-EAjdfikcCG3SFAGmKZdhQ,25551
 opentf/toolkit/core.py,sha256=fqnGgaYnuVcd4fyeNIwpc0QtyUo7jsKeVgdkBfY3iqo,9443
-opentf_toolkit_nightly-0.62.0.dev1291.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-opentf_toolkit_nightly-0.62.0.dev1291.dist-info/METADATA,sha256=jitqIOoKF4yrCa7b44WWH6WzZUdHPKOOpauF1rESCLc,2192
-opentf_toolkit_nightly-0.62.0.dev1291.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-opentf_toolkit_nightly-0.62.0.dev1291.dist-info/top_level.txt,sha256=_gPuE6GTT6UNXy1DjtmQSfCcZb_qYA2vWmjg7a30AGk,7
-opentf_toolkit_nightly-0.62.0.dev1291.dist-info/RECORD,,
+opentf_toolkit_nightly-0.62.0.dev1298.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+opentf_toolkit_nightly-0.62.0.dev1298.dist-info/METADATA,sha256=zPOTouTQoLWUCjDEAvzyd6zBD72RgfGlTL9UGDTITuw,2192
+opentf_toolkit_nightly-0.62.0.dev1298.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+opentf_toolkit_nightly-0.62.0.dev1298.dist-info/top_level.txt,sha256=_gPuE6GTT6UNXy1DjtmQSfCcZb_qYA2vWmjg7a30AGk,7
+opentf_toolkit_nightly-0.62.0.dev1298.dist-info/RECORD,,