bamboo-engine 2.6.0__tar.gz → 2.6.2__tar.gz

{bamboo-engine-2.6.0 → bamboo_engine-2.6.2}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: bamboo-engine
-Version: 2.6.0
+Version: 2.6.2
 Summary: Bamboo-engine is a general-purpose workflow engine
 License: MIT
-Author: homholueng
-Author-email: homholueng@gmail.com
+Author: blueking
+Author-email: blueking@tencent.com
 Requires-Python: >=3.6,<3.8
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3

{bamboo-engine-2.6.0 → bamboo_engine-2.6.2}/bamboo_engine/__version__.py

@@ -11,4 +11,4 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o
 specific language governing permissions and limitations under the License.
 """
 
-__version__ = "2.5.3"
+__version__ = "2.6.2"

{bamboo-engine-2.6.0 → bamboo_engine-2.6.2}/bamboo_engine/engine.py

@@ -78,11 +78,31 @@ class Engine:
     """
 
     PURE_SKIP_ENABLE_NODE_TYPE = {NodeType.ServiceActivity, NodeType.EmptyStartEvent}
+    CALLBACK_LOCK_RETRY_HEADER = "callback_lock_retry_times"
+    CALLBACK_LOCK_RETRY_LIMIT = 3
 
     def __init__(self, runtime: EngineRuntimeInterface):
         self.runtime = runtime
         self._hostname = get_hostname()
 
+    def _schedule_lock_retry_count(self, headers: Optional[dict]) -> int:
+        if not headers:
+            return 0
+
+        try:
+            return int(headers.get(self.CALLBACK_LOCK_RETRY_HEADER, 0))
+        except (TypeError, ValueError):
+            return 0
+
+    def _callback_lock_retry_context(self, node_id: str, callback_data_id: Optional[int]):
+        if not callback_data_id:
+            return None, False
+
+        callback_data = self.runtime.get_callback_data(callback_data_id)
+        node = self.runtime.get_node(node_id)
+        service = self.runtime.get_service(code=node.code, version=node.version)
+        return callback_data, service.callback_lock_retryable(callback_data=callback_data.data)
+
     # api
     def run_pipeline(
         self,
@@ -993,25 +1013,52 @@ class Engine:
             interrupter.check_and_set(ScheduleKeyPoint.APPLY_LOCK_DONE, lock_get=lock_get)
 
             if not lock_get:
-                # only retry at multiple calback type
-                if schedule.type is not ScheduleType.MULTIPLE_CALLBACK:
+                callback_data = None
+                # only retry at multiple callback type
+                should_retry = schedule.type is ScheduleType.MULTIPLE_CALLBACK
+                if not should_retry:
+                    callback_data, should_retry = self._callback_lock_retry_context(node_id, callback_data_id)
+                    retry_count = self._schedule_lock_retry_count(headers)
+                    if should_retry and retry_count >= self.CALLBACK_LOCK_RETRY_LIMIT:
+                        logger.error(
+                            "root pipeline[%s] schedule(%s) %s with version %s callback data(%s) retry exhausted "
+                            "after %s attempts, callback_data=%s",
+                            root_pipeline_id,
+                            schedule_id,
+                            node_id,
+                            schedule.version,
+                            callback_data_id,
+                            retry_count,
+                            callback_data.data if callback_data else None,
+                        )
+                        return
+
+                if not should_retry:
                     logger.info(
-                        "root pipeline[%s] schedule(%s) %s with version %s is not multiple callback type, will not retry to get lock",  # noqa
+                        "root pipeline[%s] schedule(%s) %s with version %s callback data(%s) "
+                        "will not retry to get lock",
                         root_pipeline_id,
                         schedule_id,
                         node_id,
                         schedule.version,
+                        callback_data_id,
                     )
                     return
 
                 try_after = random.randint(1, 5)
+                retry_headers = dict(headers or {})
+                if schedule.type is ScheduleType.CALLBACK:
+                    retry_headers[self.CALLBACK_LOCK_RETRY_HEADER] = self._schedule_lock_retry_count(headers) + 1
                 logger.info(
-                    "root pipeline[%s] schedule(%s) lock %s with data %s fetch fail, try after %s",
+                    "root pipeline[%s] schedule(%s) lock %s with data %s fetch fail, try after %s, "
+                    "retry_count=%s, callback_data=%s",
                     root_pipeline_id,
                     node_id,
                     schedule_id,
                     callback_data_id,
                     try_after,
+                    retry_headers.get(self.CALLBACK_LOCK_RETRY_HEADER, 0),
+                    callback_data.data if callback_data else None,
                 )
                 self.runtime.set_next_schedule(
                     process_id=process_id,
@@ -1019,7 +1066,7 @@ class Engine:
                     schedule_id=schedule_id,
                     callback_data_id=callback_data_id,
                     schedule_after=try_after,
-                    headers=headers,
+                    headers=retry_headers,
                 )
                 return
 

{bamboo-engine-2.6.0 → bamboo_engine-2.6.2}/bamboo_engine/handlers/conditional_parallel_gateway.py

@@ -14,11 +14,12 @@ specific language governing permissions and limitations under the License.
 import json
 import logging
 from typing import Optional
-from bamboo_engine.interrupt import ExecuteKeyPoint
+
+from pyparsing import ParseException
 
 from bamboo_engine.utils.boolrule import BoolRule
 from bamboo_engine.template.template import Template
-
+from bamboo_engine.interrupt import ExecuteKeyPoint
 from bamboo_engine import states, metrics
 from bamboo_engine.eri import NodeType, ProcessInfo, ExecuteInterruptPoint
 from bamboo_engine.context import Context
@@ -116,6 +117,18 @@ class ConditionalParallelGatewayHandler(NodeHandler):
                     resolved_evaluate,
                     result,
                 )
+            except ParseException as e:
+                logger.exception(f"[conditional_parallel_gateway] evaluation parse error: {e}")
+                return self._execute_fail(
+                    ex_data="evaluate[{}] fail with data[{}]："
+                    "please check if some variable not exists or the expression is unsupported, "
+                    "related reference is "
+                    '<a href="https://boolrule.readthedocs.io/en/latest/expressions.html">boolrule</a>'.format(
+                        c.evaluation, json.dumps(hydrated_context)
+                    ),
+                    version=version,
+                    ignore_boring_set=recover_point is not None,
+                )
             except Exception as e:
                 # test failed
                 return self._execute_fail(

{bamboo-engine-2.6.0 → bamboo_engine-2.6.2}/bamboo_engine/handlers/exclusive_gateway.py

@@ -14,6 +14,8 @@ import json
 import logging
 from typing import Optional
 
+from pyparsing import ParseException
+
 from bamboo_engine import states, metrics
 from bamboo_engine.context import Context
 from bamboo_engine.template import Template
@@ -121,6 +123,18 @@ class ExclusiveGatewayHandler(NodeHandler):
                     resolved_evaluate,
                     result,
                 )
+            except ParseException as e:
+                logger.exception(f"[exclusive_gateway] evaluation parse error: {e}")
+                return self._execute_fail(
+                    ex_data="evaluate[{}] fail with data[{}]："
+                    "please check if some variable not exists or the expression is unsupported, "
+                    "related reference is "
+                    '<a href="https://boolrule.readthedocs.io/en/latest/expressions.html">boolrule</a>'.format(
+                        c.evaluation, json.dumps(hydrated_context)
+                    ),
+                    version=version,
+                    ignore_boring_set=recover_point is not None,
+                )
             except Exception as e:
                 # test failed
                 return self._execute_fail(

{bamboo-engine-2.6.0 → bamboo_engine-2.6.2}/pyproject.toml

@@ -1,8 +1,8 @@
 [tool.poetry]
 name = "bamboo-engine"
-version = "2.6.0"
+version = "2.6.2"
 description = "Bamboo-engine is a general-purpose workflow engine"
-authors = ["homholueng <homholueng@gmail.com>"]
+authors = ["blueking <blueking@tencent.com>"]
 license = "MIT"
 
 [tool.poetry.dependencies]
@@ -20,7 +20,7 @@ coverage = {extras = ["toml"], version = "^5.5"}
 flake8 = "^4.0.1"
 
 [build-system]
-requires = ["poetry-core>=1.0.0"]
+requires = ["poetry-core==1.6.1"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.black]

bamboo-engine-2.6.0/setup.py

@@ -1,43 +0,0 @@
-# -*- coding: utf-8 -*-
-from setuptools import setup
-
-packages = \
-['bamboo_engine',
- 'bamboo_engine.builder',
- 'bamboo_engine.builder.flow',
- 'bamboo_engine.eri',
- 'bamboo_engine.eri.models',
- 'bamboo_engine.handlers',
- 'bamboo_engine.template',
- 'bamboo_engine.utils',
- 'bamboo_engine.utils.boolrule',
- 'bamboo_engine.utils.mako_utils',
- 'bamboo_engine.validator']
-
-package_data = \
-{'': ['*']}
-
-install_requires = \
-['Mako>=1.1.4,<2.0.0',
- 'Werkzeug>=1.0.0,<2.0.0',
- 'prometheus-client>=0.9.0,<0.10.0',
- 'pyparsing>=2.2.0,<3.0.0']
-
-setup_kwargs = {
-    'name': 'bamboo-engine',
-    'version': '2.6.0',
-    'description': 'Bamboo-engine is a general-purpose workflow engine',
-    'long_description': 'None',
-    'author': 'homholueng',
-    'author_email': 'homholueng@gmail.com',
-    'maintainer': 'None',
-    'maintainer_email': 'None',
-    'url': 'None',
-    'packages': packages,
-    'package_data': package_data,
-    'install_requires': install_requires,
-    'python_requires': '>=3.6,<3.8',
-}
-
-
-setup(**setup_kwargs)