opentf-toolkit-nightly 0.57.0.dev1051__py3-none-any.whl → 0.57.0.dev1071__py3-none-any.whl

opentf/commons/datasources.py

@@ -14,7 +14,7 @@
 
 """Datasources (testcases, tags and jobs) retrieval helpers"""
 
-from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
+from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 from datetime import datetime
 
@@ -22,6 +22,7 @@ from datetime import datetime
 from flask import current_app
 
 from opentf.commons.expressions import evaluate_bool
+from opentf.commons.selectors import match_selectors
 
 
 ########################################################################
@@ -99,9 +100,11 @@ def parse_testcase_name(full_name: str) -> Tuple[str, str]:
 ## Datasource: Testcases
 
 
-def in_scope(expr: str, contexts: Dict[str, Any]) -> bool:
+def in_scope(expr: Union[str, bool], contexts: Dict[str, Any]) -> bool:
     """Safely evaluate datasource scope."""
     try:
+        if isinstance(expr, bool):
+            return expr
         return evaluate_bool(expr, contexts)
     except ValueError as err:
         raise ValueError(f'Invalid conditional {expr}: {err}.')
@@ -399,9 +402,7 @@ def _get_testresult_labels(
 
 
 def _make_testcase_from_testresult(
-    item: Dict[str, Any],
-    labels: Dict[str, Any],
-    scope: str,
+    item: Dict[str, Any], labels: Dict[str, Any], scope: Union[str, bool]
 ) -> Dict[str, Any]:
     suite_name, testcase_name = parse_testcase_name(item['name'])
     item_data = {
@@ -434,8 +435,55 @@ def _make_testcase_from_testresult(
     return testcase
 
 
+def _get_max_count(state: Dict[str, Any]) -> int:
+    if state['reset']:
+        return state['per_page'] * state['page']
+    return state['per_page']
+
+
+def _extract_testcases(
+    testresults: List[Dict[str, Any]],
+    state: Dict[str, Any],
+    scope: Union[str, bool],
+    events: List[Dict[str, Any]],
+) -> Dict[str, Dict[str, Any]]:
+    testcases = {}
+    items = 0
+    for i, testresult in enumerate(
+        testresults[state['last_notification_used'] :],
+        start=state['last_notification_used'],
+    ):
+        if i == state['last_notification_used']:
+            last_testresult_used = state['last_testresult_used']
+        else:
+            last_testresult_used = 0
+        execution_id = testresult['metadata']['attachment_origin'][0]
+        labels = _get_testresult_labels(execution_id, events)
+        if not labels:
+            continue
+        for j, item in enumerate(
+            testresult['spec']['testResults'][last_testresult_used:],
+            start=last_testresult_used,
+        ):
+            testcase = _make_testcase_from_testresult(item, labels, scope)
+            if not testcase:
+                continue
+            if not match_selectors(testcase, state['fieldselector']):
+                continue
+            testcases[item['id']] = testcase
+            items += 1
+            if items > _get_max_count(state):
+                state['last_notification_used'] = i
+                state['last_testresult_used'] = j
+                return testcases
+
+    state['last_notification_used'] = i + 1
+    state['last_testresult_used'] = 0
+    return testcases
+
+
 def get_testcases(
-    events: List[Dict[str, Any]], scope: str = 'true'
+    events: List[Dict[str, Any]], scope: Union[str, bool] = True, state=None
 ) -> Dict[str, Dict[str, Any]]:
     """Extract metadata for each test result.
 
@@ -489,6 +537,9 @@ def get_testcases(
     A _ValueError_ exception is raised if there were no test results in
     `events` or some scope errors occured retrieving test results.
     """
+    if not state:
+        raise ValueError('No workflow cache state received from observer.')
+
     if _uses_inception(events):
         testresults = _get_inception_testresults(events)
     else:
@@ -497,17 +548,10 @@ def get_testcases(
     if not testresults:
         raise ValueError('No test results in events.')
 
-    testcases = {}
-    for testresult in testresults:
-        execution_id = testresult['metadata']['attachment_origin'][0]
-        labels = _get_testresult_labels(execution_id, events)
-        if not labels:
-            continue
-        for item in testresult['spec']['testResults']:
-            if testcase := _make_testcase_from_testresult(item, labels, scope):
-                testcases[item['id']] = testcase
+    testcases = _extract_testcases(testresults, state, scope, events)
     if not testcases:
         raise ValueError(f'No test cases matching scope `{scope}`.')
+
     return testcases
 
 
@@ -538,7 +582,9 @@ def _make_tag_datasource(tag: str, parent: Dict[str, Any]) -> Dict[str, Any]:
     }
 
 
-def get_tags(events: List[Dict[str, Any]], scope: str = 'true') -> Dict[str, Any]:
+def get_tags(
+    events: List[Dict[str, Any]], scope: Union[str, bool] = True, state=None
+) -> Dict[str, Any]:
     """Extract metadata for each execution environment tag.
 
     # Required parameters:
@@ -574,7 +620,7 @@ def get_tags(events: List[Dict[str, Any]], scope: str = 'true') -> Dict[str, Any
             'No job events found in workflow. Cannot extract data for tags.'
         )
     try:
-        testcases = get_testcases(events, scope)
+        testcases = get_testcases(events, scope, state)
     except ValueError as err:
         if str(err).startswith('[SCOPE ERROR] '):
             raise ValueError(str(err))
@@ -711,7 +757,9 @@ def _make_job_datasource(
     }
 
 
-def get_jobs(events: List[Dict[str, Any]], scope: str = 'true') -> Dict[str, Any]:
+def get_jobs(
+    events: List[Dict[str, Any]], scope: Union[str, bool] = True, state=None
+) -> Dict[str, Any]:
     """Extract metadata for each job.
 
     # Required parameters:
@@ -764,7 +812,7 @@ def get_jobs(events: List[Dict[str, Any]], scope: str = 'true') -> Dict[str, Any
         )
 
     try:
-        testcases = get_testcases(events, scope)
+        testcases = get_testcases(events, scope, state)
     except ValueError as err:
         if str(err).startswith('[SCOPE ERROR] '):
             raise ValueError(str(err))

opentf/commons/pubsub.py

@@ -108,14 +108,14 @@ def _do(req, path: str, eventbus: Dict[str, Any], **kwargs) -> Response:
     )
 
 
-def _dispatch_events(dispatch_queue: Queue, app) -> None:
+def _dispatch_events(dispatch_queue: Queue, fn, app) -> None:
     """Async event dispatch thread handler."""
     delay = 0
     while True:
         try:
             publication = dispatch_queue.get()
             try:
-                publish(publication, app.config['CONTEXT'])
+                fn(publication, app.config['CONTEXT'])
                 delay = 0
             except Exception:
                 dispatch_queue.put(publication)
@@ -125,34 +125,6 @@ def _dispatch_events(dispatch_queue: Queue, app) -> None:
             app.logger.error(f'Internal error while dispatching publication: {err}.')
 
 
-def make_dispatchqueue(app) -> Queue:
-    """Make an asynchronous dispatch queue.
-
-    Handles publication failures by waiting for an increasing delay and
-    re-attempting publication.
-
-    The delay is at most 60 seconds.
-
-    # Required parameters
-
-    - app: a flask app
-
-    # Returned value
-
-    A _queue_.  Events pushed to this queue will be published.
-    """
-    queue = Queue()
-    app.logger.debug('Starting events dispatch thread.')
-    try:
-        threading.Thread(
-            target=_dispatch_events, args=[queue, app], daemon=True
-        ).start()
-        return queue
-    except Exception as err:
-        app.logger.error('Cound not start events dispatch thread: %s.', str(err))
-        sys.exit(2)
-
-
 def subscribe(
     kind: Optional[str],
     target: str,
@@ -262,3 +234,35 @@ def publish(publication: Any, context: Dict[str, Any]) -> Response:
     if isinstance(publication, dict) and 'metadata' in publication:
         publication['metadata']['creationTimestamp'] = datetime.now().isoformat()
     return _do(post, '/publications', context['eventbus'], json=publication)
+
+
+def make_dispatchqueue(app, fn=publish) -> Queue:
+    """Make an asynchronous dispatch queue.
+
+    Handles publication failures by waiting for an increasing delay and
+    re-attempting publication.
+
+    The delay is at most 60 seconds.
+
+    # Required parameters
+
+    - app: a flask app
+
+    # Optional parameters
+
+    - fn: a function (`publish` by default)
+
+    # Returned value
+
+    A _queue_.  Events pushed to this queue will be published.
+    """
+    queue = Queue()
+    app.logger.debug('Starting events dispatch thread.')
+    try:
+        threading.Thread(
+            target=_dispatch_events, args=[queue, fn, app], daemon=True
+        ).start()
+        return queue
+    except Exception as err:
+        app.logger.error('Cound not start events dispatch thread: %s.', str(err))
+        sys.exit(2)

opentf/toolkit/__init__.py

@@ -40,6 +40,7 @@ from opentf.commons import (
     CHANNEL_HOOKS,
     validate_schema,
     make_status_response,
+    make_dispatchqueue,
 )
 from opentf.toolkit import core
 
@@ -51,6 +52,7 @@ KIND_KEY = '__kind key__'
 INPUTS_KEY = '__inputs key__'
 WATCHEDFILES_KEY = '__watched files__'
 WATCHEDFILES_EVENT_KEY = '__watched files event__'
+DISPATCHQUEUE_KEY = '__dispatch queue__'
 
 WATCHDOG_POLLING_DELAY_SECONDS = 30
 WATCHDOG_POLLING_DELAY_KEY = 'watchdog_polling_delay_seconds'
@@ -613,11 +615,12 @@ def make_plugin(
                     f'Not a valid {kind} request: Missing metadata section',
                 )
 
-        valid, extra = validate_schema(kind, body)
-        if not valid:
-            return make_status_response(
-                'BadRequest', f'Not a valid {kind} request: {extra}.'
-            )
+        if request.remote_addr != '127.0.0.1':
+            valid, extra = validate_schema(kind, body)
+            if not valid:
+                return make_status_response(
+                    'BadRequest', f'Not a valid {kind} request: {extra}.'
+                )
 
         if workflow_id := body.get('metadata', {}).get('workflow_id'):
             g.workflow_id = workflow_id
@@ -673,8 +676,10 @@ def make_plugin(
 
     if kind == PROVIDERCOMMAND:
         _maybe_add_hook_watcher(plugin, schema)
+        plugin.config[DISPATCHQUEUE_KEY] = make_dispatchqueue(plugin)
     elif kind == EXECUTIONCOMMAND:
         _maybe_add_hook_watcher(plugin, CHANNEL_HOOKS)
+        plugin.config[DISPATCHQUEUE_KEY] = make_dispatchqueue(plugin)
 
     core.register_defaultplugin(plugin)
 

opentf/toolkit/core.py

@@ -27,7 +27,6 @@ from opentf.commons import (
     GENERATORRESULT,
     PROVIDERRESULT,
     EXECUTIONERROR,
-    publish,
     make_event,
     make_uuid,
 )
@@ -114,7 +113,7 @@ class ExecutionError(Exception):
 
 def publish_event(event) -> None:
     """Publish event."""
-    publish(event, context=_getplugin().config['CONTEXT'])
+    _getplugin().config['__dispatch queue__'].put(event)
 
 
 def publish_error(error_details) -> None:

{opentf_toolkit_nightly-0.57.0.dev1051.dist-info → opentf_toolkit_nightly-0.57.0.dev1071.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: opentf-toolkit-nightly
-Version: 0.57.0.dev1051
+Version: 0.57.0.dev1071
 Summary: OpenTestFactory Orchestrator Toolkit
 Home-page: https://gitlab.com/henixdevelopment/open-source/opentestfactory/python-toolkit
 Author: Martin Lafaix
@@ -16,14 +16,14 @@ Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >= 3.8.0
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: requests >=2.31
-Requires-Dist: PyJWT[crypto] <2.9,>=2.7
-Requires-Dist: PyYAML >=6
-Requires-Dist: Flask >=2.2.3
-Requires-Dist: jsonschema >=4.17
-Requires-Dist: toposort >=1.10
-Requires-Dist: waitress >=2.1.2
-Requires-Dist: paste >=3.5.2
+Requires-Dist: requests>=2.31
+Requires-Dist: PyJWT[crypto]<2.9,>=2.7
+Requires-Dist: PyYAML>=6
+Requires-Dist: Flask>=2.2.3
+Requires-Dist: jsonschema>=4.17
+Requires-Dist: toposort>=1.10
+Requires-Dist: waitress>=2.1.2
+Requires-Dist: paste>=3.5.2
 
 # opentf-toolkit
 

{opentf_toolkit_nightly-0.57.0.dev1051.dist-info → opentf_toolkit_nightly-0.57.0.dev1071.dist-info}/RECORD

@@ -1,9 +1,9 @@
 opentf/commons/__init__.py,sha256=Uq-7WvkMoBiF3C1KnhwIL4LCKpT8EvomnuG4MBYpIhs,21994
 opentf/commons/auth.py,sha256=bM2Z3kxm2Wku1lKXaRAIg37LHvXWAXIZIqjplDfN2P8,15899
 opentf/commons/config.py,sha256=dyus4K5Zdmcftc3Y9Z1YRkzA1KwiRLHoeAlg2_A49QM,7876
-opentf/commons/datasources.py,sha256=4ye-TMtaE88O8GVcWx-FtKXOC8aIZLteR6wfIr7Do8U,25232
+opentf/commons/datasources.py,sha256=LjIjZbf08u1VllPN4fDss0OAg-_7gtRqgpIZ2tLuiHo,26807
 opentf/commons/expressions.py,sha256=jM_YKXVOFhvOE2aE2IuacuvxhIsOYTFs2oQkpcbWR6g,19645
-opentf/commons/pubsub.py,sha256=Y3vOeGNcI4_-uYwBy2grxmn1Oq5r89tyRZZX3mjgiAA,7254
+opentf/commons/pubsub.py,sha256=DVrSara5FRfNdPBwXKUkTobqGki0RPDehylTEFcJnFc,7341
 opentf/commons/schemas.py,sha256=YSCvlmqc7satt-OqIoYXnmhOyo9h8wIpNyKaBAY4u9c,4039
 opentf/commons/selectors.py,sha256=DEpLgRAr5HXSpSYI4liXP2hLUTvOSexFa9Vfa1xIQTk,7134
 opentf/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -46,11 +46,11 @@ opentf/schemas/opentestfactory.org/v1beta1/Workflow.json,sha256=QZ8mM9PhzsI9gTmw
 opentf/schemas/opentestfactory.org/v1beta2/ServiceConfig.json,sha256=rEvK2YWL5lG94_qYgR_GnLWNsaQhaQ-2kuZdWJr5NnY,3517
 opentf/scripts/launch_java_service.sh,sha256=S0jAaCuv2sZy0Gf2NGBuPX-eD531rcM-b0fNyhmzSjw,2423
 opentf/scripts/startup.py,sha256=Da2zo93pBWbdRmj-wgekgLcF94rpNc3ZkbvR8R0w8XY,21279
-opentf/toolkit/__init__.py,sha256=0265hKrPC2trSQcTEIiobFCUzeXGRLbIm-T30NP-Kio,23216
+opentf/toolkit/__init__.py,sha256=4UbExlqRO8Ew7GYRrMdEDruMIB0zTLSsoVCKfW3vPnQ,23488
 opentf/toolkit/channels.py,sha256=6xcVKHUK2FdyVKIQmPQbakngfVuQDzCcD_lInOdKpro,17171
-opentf/toolkit/core.py,sha256=Uc5cRwyi6bs7WVmgvQLTvEa6bXjZ3KfCKWHSdIeUy98,9621
-opentf_toolkit_nightly-0.57.0.dev1051.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-opentf_toolkit_nightly-0.57.0.dev1051.dist-info/METADATA,sha256=dlte0IlbCdEPnhzDEE5pyG-A3PxBkdB0No4Ubh64BB8,1951
-opentf_toolkit_nightly-0.57.0.dev1051.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-opentf_toolkit_nightly-0.57.0.dev1051.dist-info/top_level.txt,sha256=_gPuE6GTT6UNXy1DjtmQSfCcZb_qYA2vWmjg7a30AGk,7
-opentf_toolkit_nightly-0.57.0.dev1051.dist-info/RECORD,,
+opentf/toolkit/core.py,sha256=GdmEJ0ikdMdpViEpR4jP-viqfvBUHnpiFCOXwLGThxg,9606
+opentf_toolkit_nightly-0.57.0.dev1071.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+opentf_toolkit_nightly-0.57.0.dev1071.dist-info/METADATA,sha256=C46QttPC75AZio20HOPCH_uxLwdUoKhC-jNIIFKfgP4,1943
+opentf_toolkit_nightly-0.57.0.dev1071.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+opentf_toolkit_nightly-0.57.0.dev1071.dist-info/top_level.txt,sha256=_gPuE6GTT6UNXy1DjtmQSfCcZb_qYA2vWmjg7a30AGk,7
+opentf_toolkit_nightly-0.57.0.dev1071.dist-info/RECORD,,

{opentf_toolkit_nightly-0.57.0.dev1051.dist-info → opentf_toolkit_nightly-0.57.0.dev1071.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any