opentf-toolkit-nightly 0.57.0.dev1057__py3-none-any.whl → 0.57.0.dev1074__py3-none-any.whl

opentf/commons/datasources.py

@@ -14,7 +14,7 @@
 
 """Datasources (testcases, tags and jobs) retrieval helpers"""
 
-from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
+from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 from datetime import datetime
 
@@ -22,6 +22,7 @@ from datetime import datetime
 from flask import current_app
 
 from opentf.commons.expressions import evaluate_bool
+from opentf.commons.selectors import match_selectors
 
 
 ########################################################################
@@ -99,9 +100,11 @@ def parse_testcase_name(full_name: str) -> Tuple[str, str]:
 ## Datasource: Testcases
 
 
-def in_scope(expr: str, contexts: Dict[str, Any]) -> bool:
+def in_scope(expr: Union[str, bool], contexts: Dict[str, Any]) -> bool:
     """Safely evaluate datasource scope."""
     try:
+        if isinstance(expr, bool):
+            return expr
         return evaluate_bool(expr, contexts)
     except ValueError as err:
         raise ValueError(f'Invalid conditional {expr}: {err}.')
@@ -399,9 +402,7 @@ def _get_testresult_labels(
 
 
 def _make_testcase_from_testresult(
-    item: Dict[str, Any],
-    labels: Dict[str, Any],
-    scope: str,
+    item: Dict[str, Any], labels: Dict[str, Any], scope: Union[str, bool]
 ) -> Dict[str, Any]:
     suite_name, testcase_name = parse_testcase_name(item['name'])
     item_data = {
@@ -434,8 +435,55 @@ def _make_testcase_from_testresult(
     return testcase
 
 
+def _get_max_count(state: Dict[str, Any]) -> int:
+    if state['reset']:
+        return state['per_page'] * state['page']
+    return state['per_page']
+
+
+def _extract_testcases(
+    testresults: List[Dict[str, Any]],
+    state: Dict[str, Any],
+    scope: Union[str, bool],
+    events: List[Dict[str, Any]],
+) -> Dict[str, Dict[str, Any]]:
+    testcases = {}
+    items = 0
+    for i, testresult in enumerate(
+        testresults[state['last_notification_used'] :],
+        start=state['last_notification_used'],
+    ):
+        if i == state['last_notification_used']:
+            last_testresult_used = state['last_testresult_used']
+        else:
+            last_testresult_used = 0
+        execution_id = testresult['metadata']['attachment_origin'][0]
+        labels = _get_testresult_labels(execution_id, events)
+        if not labels:
+            continue
+        for j, item in enumerate(
+            testresult['spec']['testResults'][last_testresult_used:],
+            start=last_testresult_used,
+        ):
+            testcase = _make_testcase_from_testresult(item, labels, scope)
+            if not testcase:
+                continue
+            if not match_selectors(testcase, state['fieldselector']):
+                continue
+            testcases[item['id']] = testcase
+            items += 1
+            if items > _get_max_count(state):
+                state['last_notification_used'] = i
+                state['last_testresult_used'] = j
+                return testcases
+
+    state['last_notification_used'] = i + 1
+    state['last_testresult_used'] = 0
+    return testcases
+
+
 def get_testcases(
-    events: List[Dict[str, Any]], scope: str = 'true'
+    events: List[Dict[str, Any]], scope: Union[str, bool] = True, state=None
 ) -> Dict[str, Dict[str, Any]]:
     """Extract metadata for each test result.
 
@@ -489,6 +537,9 @@ def get_testcases(
     A _ValueError_ exception is raised if there were no test results in
     `events` or some scope errors occured retrieving test results.
     """
+    if not state:
+        raise ValueError('No workflow cache state received from observer.')
+
     if _uses_inception(events):
         testresults = _get_inception_testresults(events)
     else:
@@ -497,17 +548,10 @@ def get_testcases(
     if not testresults:
         raise ValueError('No test results in events.')
 
-    testcases = {}
-    for testresult in testresults:
-        execution_id = testresult['metadata']['attachment_origin'][0]
-        labels = _get_testresult_labels(execution_id, events)
-        if not labels:
-            continue
-        for item in testresult['spec']['testResults']:
-            if testcase := _make_testcase_from_testresult(item, labels, scope):
-                testcases[item['id']] = testcase
+    testcases = _extract_testcases(testresults, state, scope, events)
     if not testcases:
         raise ValueError(f'No test cases matching scope `{scope}`.')
+
     return testcases
 
 
@@ -538,7 +582,9 @@ def _make_tag_datasource(tag: str, parent: Dict[str, Any]) -> Dict[str, Any]:
     }
 
 
-def get_tags(events: List[Dict[str, Any]], scope: str = 'true') -> Dict[str, Any]:
+def get_tags(
+    events: List[Dict[str, Any]], scope: Union[str, bool] = True, state=None
+) -> Dict[str, Any]:
     """Extract metadata for each execution environment tag.
 
     # Required parameters:
@@ -574,7 +620,7 @@ def get_tags(events: List[Dict[str, Any]], scope: str = 'true') -> Dict[str, Any
             'No job events found in workflow. Cannot extract data for tags.'
         )
     try:
-        testcases = get_testcases(events, scope)
+        testcases = get_testcases(events, scope, state)
     except ValueError as err:
         if str(err).startswith('[SCOPE ERROR] '):
             raise ValueError(str(err))
@@ -711,7 +757,9 @@ def _make_job_datasource(
     }
 
 
-def get_jobs(events: List[Dict[str, Any]], scope: str = 'true') -> Dict[str, Any]:
+def get_jobs(
+    events: List[Dict[str, Any]], scope: Union[str, bool] = True, state=None
+) -> Dict[str, Any]:
     """Extract metadata for each job.
 
     # Required parameters:
@@ -764,7 +812,7 @@ def get_jobs(events: List[Dict[str, Any]], scope: str = 'true') -> Dict[str, Any
         )
 
     try:
-        testcases = get_testcases(events, scope)
+        testcases = get_testcases(events, scope, state)
     except ValueError as err:
         if str(err).startswith('[SCOPE ERROR] '):
             raise ValueError(str(err))

opentf/commons/schemas.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2023 Henix, Henix.fr
+# Copyright (c) Henix, Henix.fr
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@ import json
 import logging
 import os
 
-from jsonschema import validate, ValidationError
+from jsonschema import Draft201909Validator, ValidationError
 
 import opentf.schemas
 
@@ -41,10 +41,10 @@ WORKFLOWCANCELED = 'opentestfactory.org/v1alpha1/WorkflowCanceled'
 WORKFLOWRESULT = 'opentestfactory.org/v1alpha1/WorkflowResult'
 
 GENERATORCOMMAND = 'opentestfactory.org/v1alpha1/GeneratorCommand'
-GENERATORRESULT = 'opentestfactory.org/v1beta1/GeneratorResult'
+GENERATORRESULT = 'opentestfactory.org/v1/GeneratorResult'
 
 PROVIDERCOMMAND = 'opentestfactory.org/v1beta1/ProviderCommand'
-PROVIDERRESULT = 'opentestfactory.org/v1beta1/ProviderResult'
+PROVIDERRESULT = 'opentestfactory.org/v1/ProviderResult'
 
 EXECUTIONCOMMAND = 'opentestfactory.org/v1beta1/ExecutionCommand'
 EXECUTIONRESULT = 'opentestfactory.org/v1alpha1/ExecutionResult'
@@ -67,6 +67,7 @@ INSIGHT_COLLECTOR = 'opentestfactory.org/v1alpha1/InsightCollector'
 # JSON Schema Helpers
 
 _schemas = {}
+_validators = {}
 
 SCHEMAS_ROOT_DIRECTORY = list(opentf.schemas.__path__)[0]
 
@@ -101,6 +102,12 @@ def get_schema(name: str) -> Dict[str, Any]:
     return _schemas[name]
 
 
+def _validator(schema: str):
+    if schema not in _validators:
+        _validators[schema] = Draft201909Validator(get_schema(schema))
+    return _validators[schema]
+
+
 def validate_schema(schema, instance) -> Tuple[bool, Any]:
     """Return True if instance validates schema.
 
@@ -120,7 +127,7 @@ def validate_schema(schema, instance) -> Tuple[bool, Any]:
     is logged and raised again.
     """
     try:
-        validate(schema=get_schema(schema), instance=instance)
+        _validator(schema).validate(instance=instance)
     except ValidationError as err:
         return False, err
     return True, None

opentf/schemas/opentestfactory.org/v1/GeneratorResult.json

@@ -0,0 +1,484 @@
+{
+    "$schema": "https://json-schema.org/draft/2019-09/schema#",
+    "title": "JSON SCHEMA for opentestfactory.org/v1 GeneratorResult manifests",
+    "type": "object",
+    "properties": {
+        "apiVersion": {
+            "union": [
+                "opentestfactory.org/v1alpha1",
+                "opentestfactory.org/v1beta1",
+                "opentestfactory.org/v1"
+            ]
+        },
+        "kind": {
+            "const": "GeneratorResult"
+        },
+        "metadata": {
+            "type": "object",
+            "properties": {
+                "name": {
+                    "type": "string"
+                },
+                "namespace": {
+                    "type": "string",
+                    "pattern": "^[a-z0-9][a-z0-9-]*$"
+                },
+                "workflow_id": {
+                    "type": "string"
+                },
+                "job_id": {
+                    "type": "string"
+                },
+                "job_origin": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "labels": {
+                    "type": "object",
+                    "patternProperties": {
+                        "^([a-zA-Z0-9-.]+/)?[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$": {
+                            "type": "string"
+                        }
+                    },
+                    "minProperties": 1,
+                    "additionalProperties": false
+                }
+            },
+            "additionalProperties": true,
+            "required": [
+                "name",
+                "workflow_id",
+                "job_id",
+                "job_origin"
+            ]
+        },
+        "jobs": {
+            "$ref": "#/definitions/jobs"
+        },
+        "outputs": {
+            "$ref": "#/definitions/outputs"
+        }
+    },
+    "required": [
+        "apiVersion",
+        "kind",
+        "metadata",
+        "jobs"
+    ],
+    "additionalProperties": false,
+    "definitions": {
+        "defaults": {
+            "type": "object",
+            "propertyNames": {
+                "pattern": "^[a-zA-Z][a-zA-Z0-9-]*$"
+            }
+        },
+        "job-generator": {
+            "type": "object",
+            "properties": {
+                "name": {
+                    "type": "string"
+                },
+                "if": {
+                    "type": "string"
+                },
+                "concurrency": {
+                    "$ref": "#/definitions/concurrency"
+                },
+                "runs-on": {
+                    "$ref": "#/definitions/runs-on"
+                },
+                "needs": {
+                    "$ref": "#/definitions/needs"
+                },
+                "defaults": {
+                    "$ref": "#/definitions/defaults"
+                },
+                "variables": {
+                    "$ref": "#/definitions/variables"
+                },
+                "timeout-minutes": {
+                    "$ref": "#/definitions/number-expression"
+                },
+                "strategy": {
+                    "$ref": "#/definitions/strategy"
+                },
+                "jobs": {
+                    "$ref": "#/definitions/jobs"
+                },
+                "generator": {
+                    "type": "string"
+                },
+                "uses": {
+                    "type": "string"
+                },
+                "with": {
+                    "$ref": "#/definitions/with"
+                },
+                "outputs": {
+                    "$ref": "#/definitions/outputs"
+                },
+                "continue-on-error": {
+                    "$ref": "#/definitions/boolean-expression"
+                }
+            },
+            "oneOf": [
+                {
+                    "required": [
+                        "generator"
+                    ]
+                },
+                {
+                    "required": [
+                        "uses"
+                    ]
+                },
+                {
+                    "required": [
+                        "jobs"
+                    ]
+                }
+            ],
+            "additionalProperties": false
+        },
+        "job-steps": {
+            "type": "object",
+            "properties": {
+                "name": {
+                    "type": "string"
+                },
+                "if": {
+                    "type": "string"
+                },
+                "concurrency": {
+                    "$ref": "#/definitions/concurrency"
+                },
+                "runs-on": {
+                    "$ref": "#/definitions/runs-on"
+                },
+                "needs": {
+                    "$ref": "#/definitions/needs"
+                },
+                "defaults": {
+                    "$ref": "#/definitions/defaults"
+                },
+                "variables": {
+                    "$ref": "#/definitions/variables"
+                },
+                "timeout-minutes": {
+                    "$ref": "#/definitions/number-expression"
+                },
+                "strategy": {
+                    "$ref": "#/definitions/strategy"
+                },
+                "steps": {
+                    "type": "array",
+                    "minItems": 1,
+                    "items": {
+                        "anyOf": [
+                            {
+                                "type": "object",
+                                "properties": {
+                                    "name": {
+                                        "type": "string"
+                                    },
+                                    "id": {
+                                        "type": "string"
+                                    },
+                                    "if": {
+                                        "type": "string"
+                                    },
+                                    "uses": {
+                                        "type": "string"
+                                    },
+                                    "with": {
+                                        "$ref": "#/definitions/with"
+                                    },
+                                    "variables": {
+                                        "$ref": "#/definitions/variables"
+                                    },
+                                    "timeout-minutes": {
+                                        "$ref": "#/definitions/number-expression"
+                                    },
+                                    "continue-on-error": {
+                                        "$ref": "#/definitions/boolean-expression"
+                                    },
+                                    "working-directory": {
+                                        "type": "string"
+                                    }
+                                },
+                                "required": [
+                                    "uses"
+                                ],
+                                "additionalProperties": false
+                            },
+                            {
+                                "type": "object",
+                                "properties": {
+                                    "name": {
+                                        "type": "string"
+                                    },
+                                    "id": {
+                                        "type": "string"
+                                    },
+                                    "if": {
+                                        "type": "string"
+                                    },
+                                    "run": {
+                                        "type": "string"
+                                    },
+                                    "shell": {
+                                        "type": "string"
+                                    },
+                                    "variables": {
+                                        "$ref": "#/definitions/variables"
+                                    },
+                                    "timeout-minutes": {
+                                        "$ref": "#/definitions/number-expression"
+                                    },
+                                    "continue-on-error": {
+                                        "$ref": "#/definitions/boolean-expression"
+                                    },
+                                    "working-directory": {
+                                        "type": "string"
+                                    }
+                                },
+                                "required": [
+                                    "run"
+                                ],
+                                "additionalProperties": false
+                            }
+                        ]
+                    }
+                },
+                "outputs": {
+                    "$ref": "#/definitions/outputs"
+                },
+                "continue-on-error": {
+                    "$ref": "#/definitions/boolean-expression"
+                }
+            },
+            "required": [
+                "steps"
+            ],
+            "additionalProperties": false
+        },
+        "needs": {
+            "anyOf": [
+                {
+                    "type": "string",
+                    "pattern": "(^[a-zA-Z_][a-zA-Z0-9_-]*$)|(^\\$\\{\\{.*\\}\\}$)"
+                },
+                {
+                    "type": "array",
+                    "minItems": 1,
+                    "items": {
+                        "type": "string",
+                        "pattern": "(^[a-zA-Z_][a-zA-Z0-9_-]*$)|(^\\$\\{\\{.*\\}\\}$)"
+                    }
+                }
+            ]
+        },
+        "number-expression": {
+            "anyOf": [
+                {
+                    "type": "number"
+                },
+                {
+                    "type": "string",
+                    "pattern": "^\\$\\{\\{.*\\}\\}$"
+                }
+            ]
+        },
+        "boolean-expression": {
+            "anyOf": [
+                {
+                    "type": "boolean"
+                },
+                {
+                    "type": "string",
+                    "pattern": "^\\$\\{\\{.*\\}\\}$"
+                }
+            ]
+        },
+        "identifier-expression": {
+            "type": "string",
+            "pattern": "(^[a-zA-Z][a-zA-Z0-9-]*$)|(^\\$\\{\\{.*\\}\\}$)"
+        },
+        "runs-on": {
+            "anyOf": [
+                {
+                    "$ref": "#/definitions/identifier-expression"
+                },
+                {
+                    "type": "array",
+                    "minItems": 1,
+                    "items": {
+                        "$ref": "#/definitions/identifier-expression"
+                    }
+                },
+                {
+                    "type": "object"
+                }
+            ]
+        },
+        "variables": {
+            "type": "object",
+            "patternProperties": {
+                "^[a-zA-Z0-9_]+$": {
+                    "oneOf": [
+                        {
+                            "type": "string"
+                        },
+                        {
+                            "type": "object",
+                            "properties": {
+                                "value": {
+                                    "type": "string"
+                                },
+                                "verbatim": {
+                                    "type": "boolean"
+                                }
+                            },
+                            "required": [
+                                "value"
+                            ],
+                            "additionalProperties": false
+                        }
+                    ]
+                }
+            },
+            "minProperties": 1
+        },
+        "strategy": {
+            "type": "object",
+            "properties": {
+                "max-parallel": {
+                    "$ref": "#/definitions/number-expression"
+                },
+                "fail-fast": {
+                    "$ref": "#/definitions/boolean-expression"
+                },
+                "matrix": {
+                    "type": "object",
+                    "properties": {
+                        "exclude": {
+                            "type": "array",
+                            "minItems": 1,
+                            "items": {
+                                "type": "object",
+                                "patternProperties": {
+                                    "^[a-zA-Z][a-zA-Z0-9-]*$": {
+                                        "type": "string"
+                                    }
+                                }
+                            }
+                        },
+                        "include": {
+                            "type": "array",
+                            "minItems": 1,
+                            "items": {
+                                "type": "object",
+                                "patternProperties": {
+                                    "^[a-zA-Z][a-zA-Z0-9-]*$": {
+                                        "type": "string"
+                                    }
+                                }
+                            }
+                        }
+                    },
+                    "patternProperties": {
+                        "^[a-zA-Z][a-zA-Z0-9-]*$": {
+                            "type": "array",
+                            "minItems": 1,
+                            "items": {
+                                "oneOf": [
+                                    {
+                                        "type": "string"
+                                    },
+                                    {
+                                        "type": "number"
+                                    },
+                                    {
+                                        "type": "object",
+                                        "patternProperties": {
+                                            "^[a-zA-Z][a-zA-Z0-9-]*$": {
+                                                "type": "string"
+                                            }
+                                        }
+                                    }
+                                ]
+                            }
+                        }
+                    }
+                }
+            },
+            "minProperties": 1,
+            "additionalProperties": false
+        },
+        "concurrency": {
+            "type": "object",
+            "properties": {
+                "group": {
+                    "type": "string"
+                },
+                "cancel-in-progress": {
+                    "$ref": "#/definitions/boolean-expression"
+                }
+            },
+            "additionalProperties": false,
+            "required": [
+                "group"
+            ]
+        },
+        "jobs": {
+            "type": "object",
+            "patternProperties": {
+                "^[a-zA-Z_][a-zA-Z0-9_-]*$": {
+                    "oneOf": [
+                        {
+                            "$ref": "#/definitions/job-generator"
+                        },
+                        {
+                            "$ref": "#/definitions/job-steps"
+                        }
+                    ]
+                }
+            },
+            "minProperties": 1
+        },
+        "with": {
+            "type": "object"
+        },
+        "outputs": {
+            "type": "object",
+            "patternProperties": {
+                "^[a-zA-Z_][a-zA-Z0-9_-]*$": {
+                    "oneOf": [
+                        {
+                            "type": "object",
+                            "properties": {
+                                "description": {
+                                    "type": "string"
+                                },
+                                "value": {
+                                    "type": "string"
+                                }
+                            },
+                            "required": [
+                                "value"
+                            ],
+                            "additionalProperties": false
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
+                }
+            },
+            "minProperties": 1
+        }
+    }
+}

opentf/schemas/opentestfactory.org/v1/ProviderResult.json

@@ -0,0 +1,297 @@
+{
+    "$schema": "https://json-schema.org/draft/2019-09/schema#",
+    "title": "JSON SCHEMA for opentestfactory.org/v1 ProviderResult manifests",
+    "type": "object",
+    "properties": {
+        "apiVersion": {
+            "enum": [
+                "opentestfactory.org/v1alpha1",
+                "opentestfactory.org/v1beta1",
+                "opentestfactory.org/v1"
+            ]
+        },
+        "kind": {
+            "const": "ProviderResult"
+        },
+        "metadata": {
+            "type": "object",
+            "properties": {
+                "name": {
+                    "type": "string"
+                },
+                "namespace": {
+                    "type": "string",
+                    "pattern": "^[a-z0-9][a-z0-9-]*$"
+                },
+                "workflow_id": {
+                    "type": "string"
+                },
+                "job_id": {
+                    "type": "string"
+                },
+                "job_origin": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "step_id": {
+                    "type": "string"
+                },
+                "step_origin": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "labels": {
+                    "type": "object",
+                    "patternProperties": {
+                        "^([a-zA-Z0-9-.]+/)?[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$": {
+                            "type": "string"
+                        }
+                    },
+                    "minProperties": 1,
+                    "additionalProperties": false
+                }
+            },
+            "additionalProperties": true,
+            "required": [
+                "name",
+                "workflow_id",
+                "job_id",
+                "job_origin",
+                "step_id",
+                "step_origin"
+            ]
+        },
+        "hooks": {
+            "$ref": "#/definitions/hooks"
+        },
+        "steps": {
+            "type": "array",
+            "minItems": 0,
+            "items": {
+                "anyOf": [
+                    {
+                        "type": "object",
+                        "properties": {
+                            "name": {
+                                "type": "string"
+                            },
+                            "id": {
+                                "type": "string"
+                            },
+                            "if": {
+                                "type": "string"
+                            },
+                            "uses": {
+                                "type": "string"
+                            },
+                            "with": {
+                                "$ref": "#/definitions/with"
+                            },
+                            "variables": {
+                                "$ref": "#/definitions/variables"
+                            },
+                            "timeout-minutes": {
+                                "$ref": "#/definitions/number-expression"
+                            },
+                            "continue-on-error": {
+                                "$ref": "#/definitions/boolean-expression"
+                            },
+                            "working-directory": {
+                                "type": "string"
+                            }
+                        },
+                        "required": [
+                            "uses"
+                        ],
+                        "additionalProperties": false
+                    },
+                    {
+                        "type": "object",
+                        "properties": {
+                            "name": {
+                                "type": "string"
+                            },
+                            "id": {
+                                "type": "string"
+                            },
+                            "if": {
+                                "type": "string"
+                            },
+                            "run": {
+                                "type": "string"
+                            },
+                            "shell": {
+                                "type": "string"
+                            },
+                            "variables": {
+                                "$ref": "#/definitions/variables"
+                            },
+                            "timeout-minutes": {
+                                "$ref": "#/definitions/number-expression"
+                            },
+                            "continue-on-error": {
+                                "$ref": "#/definitions/boolean-expression"
+                            },
+                            "working-directory": {
+                                "type": "string"
+                            }
+                        },
+                        "required": [
+                            "run"
+                        ],
+                        "additionalProperties": false
+                    }
+                ]
+            }
+        },
+        "outputs": {
+            "$ref": "#/definitions/outputs"
+        }
+    },
+    "required": [
+        "apiVersion",
+        "kind",
+        "metadata",
+        "steps"
+    ],
+    "additionalProperties": false,
+    "definitions": {
+        "number-expression": {
+            "anyOf": [
+                {
+                    "type": "number"
+                },
+                {
+                    "type": "string",
+                    "pattern": "^\\$\\{\\{.*\\}\\}$"
+                }
+            ]
+        },
+        "boolean-expression": {
+            "anyOf": [
+                {
+                    "type": "boolean"
+                },
+                {
+                    "type": "string",
+                    "pattern": "^\\$\\{\\{.*\\}\\}$"
+                }
+            ]
+        },
+        "variables": {
+            "type": "object",
+            "patternProperties": {
+                "^[a-zA-Z0-9_]+$": {
+                    "oneOf": [
+                        {
+                            "type": "string"
+                        },
+                        {
+                            "type": "object",
+                            "properties": {
+                                "value": {
+                                    "type": "string"
+                                },
+                                "verbatim": {
+                                    "type": "boolean"
+                                }
+                            },
+                            "required": [
+                                "value"
+                            ],
+                            "additionalProperties": false
+                        }
+                    ]
+                }
+            },
+            "minProperties": 1
+        },
+        "hooks": {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "type": "string"
+                    },
+                    "if": {
+                        "type": "string"
+                    },
+                    "events": {
+                        "type": "array",
+                        "minItems": 1
+                    },
+                    "before": {
+                        "type": "array",
+                        "minItems": 1
+                    },
+                    "after": {
+                        "type": "array",
+                        "minItems": 1
+                    }
+                },
+                "anyOf": [
+                    {
+                        "required": [
+                            "name",
+                            "events",
+                            "before"
+                        ]
+                    },
+                    {
+                        "required": [
+                            "name",
+                            "events",
+                            "after",
+                            "before"
+                        ]
+                    },
+                    {
+                        "required": [
+                            "name",
+                            "events",
+                            "after"
+                        ]
+                    }
+                ],
+                "additionalProperties": false
+            }
+        },
+        "with": {
+            "type": "object"
+        },
+        "outputs": {
+            "type": "object",
+            "patternProperties": {
+                "^[a-zA-Z_][a-zA-Z0-9_-]*$": {
+                    "oneOf": [
+                        {
+                            "type": "object",
+                            "properties": {
+                                "description": {
+                                    "type": "string"
+                                },
+                                "value": {
+                                    "type": "string"
+                                }
+                            },
+                            "required": [
+                                "value"
+                            ],
+                            "additionalProperties": false
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
+                }
+            },
+            "minProperties": 1
+        }
+    }
+}

opentf/toolkit/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021-2023 Henix, Henix.fr
+# Copyright (c) Henix, Henix.fr
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
 
 """A toolkit for creating OpenTestFactory plugins."""
 
-from typing import Any, Callable, Dict, Optional
+from typing import Any, Callable, Dict, Optional, Tuple
 
 import os
 import threading
@@ -50,6 +50,7 @@ from opentf.toolkit import core
 SUBSCRIPTION_KEY = '__subscription uuid__'
 KIND_KEY = '__kind key__'
 INPUTS_KEY = '__inputs key__'
+OUTPUTS_KEY = '__outputs key__'
 WATCHEDFILES_KEY = '__watched files__'
 WATCHEDFILES_EVENT_KEY = '__watched files event__'
 DISPATCHQUEUE_KEY = '__dispatch queue__'
@@ -91,6 +92,33 @@ def _normalize_inputs(inputs: Dict[str, Any]) -> None:
             inputs[normalized] = inputs.pop(key)
 
 
+def _get_pcv(
+    labels: Dict[str, str], default: Optional[str] = None
+) -> Tuple[Optional[str], Optional[str], Optional[str]]:
+    """Extract prefix, category, version from labels."""
+    prefix = labels.get('opentestfactory.org/categoryPrefix', default)
+    category = labels.get('opentestfactory.org/category', default)
+    version = labels.get('opentestfactory.org/categoryVersion', default) or None
+    return prefix, category, version
+
+
+def _maybe_get_item(cache: Dict[Any, Any], labels: Dict[str, str]) -> Optional[Any]:
+    """Get most relevant item from cache if it exists."""
+    prefix, category, version = _get_pcv(labels)
+
+    for keys in (
+        (prefix, category, version),
+        (None, category, version),
+        (prefix, category, None),
+        (None, category, None),
+        (prefix, None, None),
+    ):
+        if (entry := cache.get(keys)) is not None:
+            return entry
+
+    return None
+
+
 def _ensure_inputs_match(
     plugin, labels: Dict[str, str], inputs: Dict[str, Any]
 ) -> None:
@@ -105,20 +133,7 @@ def _ensure_inputs_match(
     or if an unexpected entry is found.
     """
     cache = plugin.config['CONTEXT'][INPUTS_KEY]
-    prefix = labels.get('opentestfactory.org/categoryPrefix')
-    category = labels.get('opentestfactory.org/category')
-    version = labels.get('opentestfactory.org/categoryVersion') or None
-
-    for keys in (
-        (prefix, category, version),
-        (None, category, version),
-        (prefix, category, None),
-        (None, category, None),
-        (prefix, None, None),
-    ):
-        if (entry := cache.get(keys)) is not None:
-            break
-    else:
+    if (entry := _maybe_get_item(cache, labels)) is None:
         return
 
     declaration, additional_inputs = entry
@@ -159,9 +174,7 @@ def _get_target(
 
     `category[@vn]` is more specific than `prefix`.
     """
-    prefix = labels['opentestfactory.org/categoryPrefix']
-    category = labels['opentestfactory.org/category']
-    version = labels.get('opentestfactory.org/categoryVersion')
+    prefix, category, version = _get_pcv(labels)
 
     for template in (f'{prefix}/{category}', category):
         if version:
@@ -282,13 +295,12 @@ def _dispatch_providercommand(plugin, handler: Handler, body: Dict[str, Any]) ->
         plugin.logger.debug(
             'Calling provider function %s (%s/%s@%s).',
             handler.__name__,
-            labels.get('opentestfactory.org/categoryPrefix', '_'),
-            labels.get('opentestfactory.org/category', '_'),
-            labels.get('opentestfactory.org/categoryVersion', '_'),
+            *_get_pcv(labels, default='_'),
         )
         inputs: Dict[str, Any] = body['step'].get('with', {})
         _ensure_inputs_match(plugin, labels, inputs)
-        core.publish_providerresult(handler(inputs))
+        outputs = _maybe_get_item(plugin.config['CONTEXT'][OUTPUTS_KEY], labels)
+        core.publish_providerresult(handler(inputs), outputs)
     except core.ExecutionError as err:
         core.publish_error(str(err))
     except Exception as err:
@@ -310,12 +322,14 @@ def _dispatch_generatorcommand(plugin, handler: Handler, body: Dict[str, Any]):
         plugin.logger.debug(
             'Calling generator %s (%s/%s@%s).',
             handler.__name__,
-            labels.get('opentestfactory.org/categoryPrefix', '_'),
-            labels.get('opentestfactory.org/category', '_'),
-            labels.get('opentestfactory.org/categoryVersion', '_'),
+            *_get_pcv(labels, default='_'),
         )
         inputs: Dict[str, Any] = body.get('with', {})
-        core.publish_generatorresult(handler(inputs))
+        _ensure_inputs_match(plugin, labels, inputs)
+        outputs = _maybe_get_item(plugin.config['CONTEXT'][OUTPUTS_KEY], labels)
+        core.publish_generatorresult(handler(inputs), outputs)
+    except core.ExecutionError as err:
+        core.publish_error(str(err))
     except Exception as err:
         core.publish_error(f'Unexpected execution error: {err}.')
 
@@ -464,6 +478,10 @@ def _subscribe(
         manifest.get('inputs', {}),
         manifest.get('additionalInputs'),
     )
+    context[OUTPUTS_KEY][(cat_prefix, cat, cat_version)] = {
+        k: v['value'] if isinstance(v, dict) else v
+        for k, v in manifest.get('outputs', {}).items()
+    }
     return subscribe(kind=kind, target='inbox', app=plugin, labels=labels)
 
 
@@ -479,7 +497,8 @@ def run_plugin(plugin):
         context = plugin.config['CONTEXT']
         context[SUBSCRIPTION_KEY] = []
         context[INPUTS_KEY] = {}
-        if context[KIND_KEY] == PROVIDERCOMMAND:
+        context[OUTPUTS_KEY] = {}
+        if context[KIND_KEY] in (PROVIDERCOMMAND, GENERATORCOMMAND):
             for manifest in plugin.config['DESCRIPTOR']:
                 metadata = manifest.get('metadata', {})
                 if metadata.get('name', '').lower() != plugin.name.lower():
@@ -502,37 +521,6 @@ def run_plugin(plugin):
             context[SUBSCRIPTION_KEY].append(
                 subscribe(kind=EXECUTIONCOMMAND, target='inbox', app=plugin)
             )
-        elif context[KIND_KEY] == GENERATORCOMMAND:
-            for manifest in plugin.config['DESCRIPTOR']:
-                metadata = manifest.get('metadata', {})
-                if metadata.get('name', '').lower() != plugin.name.lower():
-                    continue
-                if 'action' not in metadata:
-                    continue
-                for event in manifest.get('events', []):
-                    cat_prefix = event.get('categoryPrefix')
-                    cat = event.get('category')
-                    if cat or cat_prefix:
-                        cat_version = event.get('categoryVersion')
-                        labels = {}
-                        if cat is not None:
-                            labels['opentestfactory.org/category'] = cat
-                        if cat_prefix is not None:
-                            labels['opentestfactory.org/categoryPrefix'] = cat_prefix
-                        if cat_version is not None:
-                            labels['opentestfactory.org/categoryVersion'] = cat_version
-                        context[SUBSCRIPTION_KEY].append(
-                            subscribe(
-                                kind=GENERATORCOMMAND,
-                                target='inbox',
-                                labels=labels,
-                                app=plugin,
-                            )
-                        )
-                    else:
-                        plugin.logger.warning(
-                            "At least one of 'category', 'categoryPrefix' required, ignoring."
-                        )
         run_app(plugin)
     finally:
         for subscription_id in plugin.config['CONTEXT'][SUBSCRIPTION_KEY]:
@@ -566,10 +554,10 @@ def make_plugin(
     - Add publication handler
     - Create service (not started)
 
-    Some 'optional' parameters are required for some publin types:
+    Some 'optional' parameters are required for some plugin types:
 
     `args` is required for channel handlers.  It must be a list of one
-    element that implements the `__contains__` interface.
+    element that implements the `__contains__` protocol.
 
     # Required parameters
 
@@ -615,12 +603,11 @@ def make_plugin(
                     f'Not a valid {kind} request: Missing metadata section',
                 )
 
-        if request.remote_addr != '127.0.0.1':
-            valid, extra = validate_schema(kind, body)
-            if not valid:
-                return make_status_response(
-                    'BadRequest', f'Not a valid {kind} request: {extra}.'
-                )
+        valid, extra = validate_schema(kind, body)
+        if not valid:
+            return make_status_response(
+                'BadRequest', f'Not a valid {kind} request: {extra}.'
+            )
 
         if workflow_id := body.get('metadata', {}).get('workflow_id'):
             g.workflow_id = workflow_id
@@ -677,6 +664,8 @@ def make_plugin(
     if kind == PROVIDERCOMMAND:
         _maybe_add_hook_watcher(plugin, schema)
         plugin.config[DISPATCHQUEUE_KEY] = make_dispatchqueue(plugin)
+    elif kind == GENERATORCOMMAND:
+        plugin.config[DISPATCHQUEUE_KEY] = make_dispatchqueue(plugin)
     elif kind == EXECUTIONCOMMAND:
         _maybe_add_hook_watcher(plugin, CHANNEL_HOOKS)
         plugin.config[DISPATCHQUEUE_KEY] = make_dispatchqueue(plugin)

opentf/toolkit/core.py

@@ -126,7 +126,7 @@ def publish_error(error_details) -> None:
     publish_event(error)
 
 
-def publish_providerresult(steps: Iterable) -> None:
+def publish_providerresult(steps: Iterable, outputs: Optional[Dict[str, Any]]) -> None:
     """Publish ProviderResult event."""
     command = make_event(
         PROVIDERRESULT,
@@ -135,18 +135,24 @@ def publish_providerresult(steps: Iterable) -> None:
     )
     if hooks := _getplugin().config['CONFIG'].get('hooks'):
         command['hooks'] = hooks
+    if outputs:
+        command['outputs'] = outputs
     for step in command['steps']:
         step.setdefault('id', make_uuid())
     publish_event(command)
 
 
-def publish_generatorresult(jobs: Dict[str, Any]) -> None:
+def publish_generatorresult(
+    jobs: Dict[str, Any], outputs: Optional[Dict[str, Any]]
+) -> None:
     """Publish GeneratorResult event."""
     command = make_event(
         GENERATORRESULT,
         metadata=_getbody()['metadata'],
         jobs={k: v.copy() for k, v in jobs.items()},
     )
+    if outputs:
+        command['outputs'] = outputs
     publish_event(command)
 
 

{opentf_toolkit_nightly-0.57.0.dev1057.dist-info → opentf_toolkit_nightly-0.57.0.dev1074.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: opentf-toolkit-nightly
-Version: 0.57.0.dev1057
+Version: 0.57.0.dev1074
 Summary: OpenTestFactory Orchestrator Toolkit
 Home-page: https://gitlab.com/henixdevelopment/open-source/opentestfactory/python-toolkit
 Author: Martin Lafaix
@@ -19,8 +19,8 @@ License-File: LICENSE
 Requires-Dist: requests>=2.31
 Requires-Dist: PyJWT[crypto]<2.9,>=2.7
 Requires-Dist: PyYAML>=6
-Requires-Dist: Flask>=2.2.3
-Requires-Dist: jsonschema>=4.17
+Requires-Dist: Flask<3,>=2.2.3
+Requires-Dist: jsonschema>=4.23
 Requires-Dist: toposort>=1.10
 Requires-Dist: waitress>=2.1.2
 Requires-Dist: paste>=3.5.2

{opentf_toolkit_nightly-0.57.0.dev1057.dist-info → opentf_toolkit_nightly-0.57.0.dev1074.dist-info}/RECORD

@@ -1,13 +1,15 @@
 opentf/commons/__init__.py,sha256=Uq-7WvkMoBiF3C1KnhwIL4LCKpT8EvomnuG4MBYpIhs,21994
 opentf/commons/auth.py,sha256=bM2Z3kxm2Wku1lKXaRAIg37LHvXWAXIZIqjplDfN2P8,15899
 opentf/commons/config.py,sha256=dyus4K5Zdmcftc3Y9Z1YRkzA1KwiRLHoeAlg2_A49QM,7876
-opentf/commons/datasources.py,sha256=4ye-TMtaE88O8GVcWx-FtKXOC8aIZLteR6wfIr7Do8U,25232
+opentf/commons/datasources.py,sha256=LjIjZbf08u1VllPN4fDss0OAg-_7gtRqgpIZ2tLuiHo,26807
 opentf/commons/expressions.py,sha256=jM_YKXVOFhvOE2aE2IuacuvxhIsOYTFs2oQkpcbWR6g,19645
 opentf/commons/pubsub.py,sha256=DVrSara5FRfNdPBwXKUkTobqGki0RPDehylTEFcJnFc,7341
-opentf/commons/schemas.py,sha256=YSCvlmqc7satt-OqIoYXnmhOyo9h8wIpNyKaBAY4u9c,4039
+opentf/commons/schemas.py,sha256=P6jyIJR5Zw2L7vH3_jhpTiJAHHwjE6MJUmva0_41B3E,4212
 opentf/commons/selectors.py,sha256=DEpLgRAr5HXSpSYI4liXP2hLUTvOSexFa9Vfa1xIQTk,7134
 opentf/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 opentf/schemas/abac.opentestfactory.org/v1alpha1/Policy.json,sha256=JXsfNAPSEYggeyaDutSQBeG38o4Bmcr70dPLWWeqIh8,2105
+opentf/schemas/opentestfactory.org/v1/GeneratorResult.json,sha256=neoFocJGkVTQQPtnG5xnbqNLX8ivFBJbanbbPweId7s,16608
+opentf/schemas/opentestfactory.org/v1/ProviderResult.json,sha256=Ej4zhCE3rCqCGKcaeAoIHwSJTV_7fw-rAxhJ52qA-Gs,9641
 opentf/schemas/opentestfactory.org/v1/Workflow.json,sha256=mpWxJfP3aV3sYzVxxWOivD55Top4J5WXTTKBzp7gjIw,22486
 opentf/schemas/opentestfactory.org/v1alpha1/AgentRegistration.json,sha256=NQykqU-lKE8LtBhBiFUcpVJq00MRG6dZsoM1xedx6uQ,1230
 opentf/schemas/opentestfactory.org/v1alpha1/AllureCollectorOutput.json,sha256=-L9DDWA0A4x54bPMn4m6Qwi2tf2nHvzIPFOElTjaVck,1366
@@ -46,11 +48,11 @@ opentf/schemas/opentestfactory.org/v1beta1/Workflow.json,sha256=QZ8mM9PhzsI9gTmw
 opentf/schemas/opentestfactory.org/v1beta2/ServiceConfig.json,sha256=rEvK2YWL5lG94_qYgR_GnLWNsaQhaQ-2kuZdWJr5NnY,3517
 opentf/scripts/launch_java_service.sh,sha256=S0jAaCuv2sZy0Gf2NGBuPX-eD531rcM-b0fNyhmzSjw,2423
 opentf/scripts/startup.py,sha256=Da2zo93pBWbdRmj-wgekgLcF94rpNc3ZkbvR8R0w8XY,21279
-opentf/toolkit/__init__.py,sha256=4UbExlqRO8Ew7GYRrMdEDruMIB0zTLSsoVCKfW3vPnQ,23488
+opentf/toolkit/__init__.py,sha256=nbllYXON3Rzd-hU7Cred7r4hPaCPbfSJmoY7cQ6RShc,22589
 opentf/toolkit/channels.py,sha256=6xcVKHUK2FdyVKIQmPQbakngfVuQDzCcD_lInOdKpro,17171
-opentf/toolkit/core.py,sha256=GdmEJ0ikdMdpViEpR4jP-viqfvBUHnpiFCOXwLGThxg,9606
-opentf_toolkit_nightly-0.57.0.dev1057.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-opentf_toolkit_nightly-0.57.0.dev1057.dist-info/METADATA,sha256=Rfyt1TuD1vqvhdTcSAs1G_3Iujzd6P8StHAFXiE9dYo,1943
-opentf_toolkit_nightly-0.57.0.dev1057.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-opentf_toolkit_nightly-0.57.0.dev1057.dist-info/top_level.txt,sha256=_gPuE6GTT6UNXy1DjtmQSfCcZb_qYA2vWmjg7a30AGk,7
-opentf_toolkit_nightly-0.57.0.dev1057.dist-info/RECORD,,
+opentf/toolkit/core.py,sha256=KN6-z8Hmty1thZ-c0mTbcqkF6q1nNrtE8A3F6-wjX4g,9788
+opentf_toolkit_nightly-0.57.0.dev1074.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+opentf_toolkit_nightly-0.57.0.dev1074.dist-info/METADATA,sha256=bE5iD0Xm2FAWCYQHF8Y2Wvwy_4dZm45uEpnNEI7BWLk,1946
+opentf_toolkit_nightly-0.57.0.dev1074.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+opentf_toolkit_nightly-0.57.0.dev1074.dist-info/top_level.txt,sha256=_gPuE6GTT6UNXy1DjtmQSfCcZb_qYA2vWmjg7a30AGk,7
+opentf_toolkit_nightly-0.57.0.dev1074.dist-info/RECORD,,