taskflow 5.8.0__tar.gz → 5.9.1__tar.gz

{taskflow-5.8.0 → taskflow-5.9.1}/.zuul.yaml

@@ -13,6 +13,14 @@
       tox_environment:
         PIFPAF_DAEMON: redis
 
+- job:
+    name: taskflow-functional-etcd
+    parent: taskflow-functional
+    vars:
+      tox_environment:
+        PIFPAF_DAEMON: etcd
+        SETUP_ENV_SCRIPT: ./setup-etcd-env.sh
+
 - project:
     templates:
       - check-requirements
@@ -25,6 +33,8 @@
     check:
       jobs:
         - taskflow-functional-redis
+        - taskflow-functional-etcd
     gate:
       jobs:
         - taskflow-functional-redis
+        - taskflow-functional-etcd

{taskflow-5.8.0 → taskflow-5.9.1}/ChangeLog

@@ -1,10 +1,22 @@
 CHANGES
 =======
 
+5.9.1
+-----
+
+
+5.9.0
+-----
+
+* Follow-up of "Add Etcd backend for jobboard"
+* Add functional test with etcd
+* Add Etcd backend for jobboard
+
 5.8.0
 -----
 
 * Python3.12: disable process\_executor, which depends on asyncore
+* Fix REVERT\_ALL with Retries in unordered Flows
 * Remove SQLAlchemy tips jobs
 * Remove old excludes
 * Remove assertRaisesRegex

{taskflow-5.8.0 → taskflow-5.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: taskflow
-Version: 5.8.0
+Version: 5.9.1
 Summary: Taskflow structured state management library.
 Home-page: https://docs.openstack.org/taskflow/latest/
 Author: OpenStack
@@ -101,6 +101,7 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: System :: Distributed Computing
 Requires-Python: >=3.8
 Provides-Extra: database
+Provides-Extra: etcd
 Provides-Extra: eventlet
 Provides-Extra: redis
 Provides-Extra: test

{taskflow-5.8.0 → taskflow-5.9.1}/doc/source/user/jobs.rst

@@ -288,6 +288,40 @@ optionally expire after a given amount of time).
     See :py:class:`~taskflow.jobs.backends.impl_redis.RedisJobBoard`
     for implementation details.
 
+Etcd
+----
+
+**Board type**: ``'etcd'``
+
+Uses `etcd`_ to provide the jobboard capabilities by using Etcd key values data
+structures and individual job ownership key (that can optionally expire after a
+given amount of time).
+
+Additional *kwarg* parameters:
+
+* ``persistence``: a class that provides a :doc:`persistence <persistence>`
+  backend interface; it will be used for loading jobs logbooks for usage at
+  runtime or for usage before a job is claimed for introspection.
+
+Additional *configuration* parameters:
+
+* ``path``: the Etcd path to store job information (*defaults* to
+  ``jobboard``)
+* ``host``: the Etcd host to connect to (*defaults* to ``localhost``)
+* ``port``: the port of the Etcd server (*defaults* to ``2379``)
+* ``api_path``: the path of the Etcd API endpoint
+* ``protocol``: the protocol used to communicate with the server (*defaults* to
+  ``http``, choices are ``http`` or ``https``)
+* ``ca_cert``, ``cert_key`` and ``cert_cert``: the certificate information
+  passed to Etcd3gw for ``https`` communications
+* ``timeout``: the timeout used when performing operations with Etcd
+* ``ttl``: the default time-to-live when claiming a job (*defaults* to
+  ``None``)
+
+.. note::
+    See :py:class:`~taskflow.jobs.backends.impl_etcd.EtcdJobBoard`
+    for implementation details.
+
 Considerations
 ==============
 
@@ -356,6 +390,11 @@ Redis
 
 .. automodule:: taskflow.jobs.backends.impl_redis
 
+Etcd
+----
+
+.. automodule:: taskflow.jobs.backends.impl_etcd
+
 Hierarchy
 =========
 
@@ -363,6 +402,7 @@ Hierarchy
     taskflow.jobs.base
     taskflow.jobs.backends.impl_redis
     taskflow.jobs.backends.impl_zookeeper
+    taskflow.jobs.backends.impl_etcd
     :parts: 1
 
 .. _paradigm shift: https://wiki.openstack.org/wiki/TaskFlow/Paradigm_shifts#Workflow_ownership_transfer
@@ -370,3 +410,4 @@ Hierarchy
 .. _kazoo: https://kazoo.readthedocs.io/en/latest/
 .. _stevedore: https://docs.openstack.org/stevedore/latest
 .. _redis: https://redis.io/
+.. _etcd: https://etcd.io/

{taskflow-5.8.0 → taskflow-5.9.1}/playbooks/tests/functional/Debian.yaml

@@ -3,3 +3,4 @@ backend_services_map:
   redis:
     - redis-server
     - redis-sentinel
+  etcd: []

{taskflow-5.8.0 → taskflow-5.9.1}/playbooks/tests/functional/RedHat.yaml

@@ -3,3 +3,4 @@ backend_services_map:
   redis:
     - redis
     - redis-sentinel
+  etcd: []

taskflow-5.9.1/releasenotes/notes/etcd-jobboard-backend-8a9fea2238fb0f12.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Added an Etcd-based backend for jobboard. This backend is similar to the
+    Redis backend, it requires that the consumer extends the expiry of the job
+    that is being running.

taskflow-5.9.1/releasenotes/notes/fix-revert-all-revert-a0310cd7beaa7409.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fixed a bug when using retries with unordered flows, a REVERT_ALL triggered
+    by one of the subflow was overriden by an other subflow running in parallel,
+    leading to an incomplete revert of the flow.

taskflow-5.9.1/setup-etcd-env.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -eux
+if [ -z "$(which etcd)" ]; then
+    ETCD_VERSION=3.3.27
+    case `uname -s` in
+        Darwin)
+            OS=darwin
+            SUFFIX=zip
+            ;;
+        Linux)
+            OS=linux
+            SUFFIX=tar.gz
+            ;;
+        *)
+            echo "Unsupported OS"
+            exit 1
+    esac
+    case `uname -m` in
+         x86_64)
+             MACHINE=amd64
+             ;;
+         *)
+            echo "Unsupported machine"
+            exit 1
+    esac
+    TARBALL_NAME=etcd-v${ETCD_VERSION}-$OS-$MACHINE
+    test ! -d "$TARBALL_NAME" && curl -L https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/${TARBALL_NAME}.${SUFFIX} | tar xz
+    export PATH=$PATH:$TARBALL_NAME
+fi
+
+$*

{taskflow-5.8.0 → taskflow-5.9.1}/setup.cfg

@@ -34,6 +34,7 @@ packages =
 taskflow.jobboards = 
 	zookeeper = taskflow.jobs.backends.impl_zookeeper:ZookeeperJobBoard
 	redis = taskflow.jobs.backends.impl_redis:RedisJobBoard
+	etcd = taskflow.jobs.backends.impl_etcd:EtcdJobBoard
 taskflow.conductors = 
 	blocking = taskflow.conductors.backends.impl_blocking:BlockingConductor
 	nonblocking = taskflow.conductors.backends.impl_nonblocking:NonBlockingConductor
@@ -58,6 +59,8 @@ zookeeper =
 	zake>=0.1.6 # Apache-2.0
 redis = 
 	redis>=4.0.0 # MIT
+etcd = 
+	etcd3gw>=2.0.0 # Apache-2.0
 workers = 
 	kombu>=4.3.0 # BSD
 eventlet = 

{taskflow-5.8.0 → taskflow-5.9.1}/taskflow/engines/action_engine/runtime.py

@@ -291,6 +291,18 @@ class Runtime(object):
         """Resets all the provided atoms to the given state and intention."""
         tweaked = []
         for atom in atoms:
+            cur_intention = self.storage.get_atom_intention(atom.name)
+            # Don't trigger a RETRY if the atom needs to be REVERTED.
+            # This is a workaround for a bug when REVERT_ALL is applied to
+            # unordered flows
+            # (https://bugs.launchpad.net/taskflow/+bug/2043808)
+            # A subflow may trigger a REVERT_ALL, all the atoms of all the
+            # related subflows are marked as REVERT but a task of a related
+            # flow may still be running in another thread. If this task
+            # triggers a RETRY, it overrides the previously set REVERT status,
+            # breaking the revert path of the flow.
+            if cur_intention == st.REVERT and intention == st.RETRY:
+                continue
             if state or intention:
                 tweaked.append((atom, state, intention))
             if state: