simpleflow 0.34.0__tar.gz → 0.34.2__tar.gz

{simpleflow-0.34.0 → simpleflow-0.34.2}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: simpleflow
-Version: 0.34.0
+Version: 0.34.2
 Summary: Python library for dataflow programming with Amazon SWF
 Project-URL: documentation, https://botify-labs.github.io/simpleflow
 Project-URL: repository, https://github.com/botify-labs/simpleflow
 Project-URL: changelog, https://github.com/botify-labs/simpleflow/blob/main/CHANGELOG.md
 Author: Greg Leclercq
 Maintainer-email: tech@botify.com
-License: MIT License
+License-Expression: MIT
 License-File: LICENSE
 Keywords: amazon swf,simple workflow,simpleflow
 Classifier: Development Status :: 4 - Beta
@@ -37,31 +37,6 @@ Requires-Dist: psutil
 Requires-Dist: pytz
 Requires-Dist: setproctitle
 Requires-Dist: tabulate<1.0.0,>=0.8.2
-Provides-Extra: dev
-Requires-Dist: boto3-stubs[s3,swf]; extra == 'dev'
-Requires-Dist: cffi==v1.17.0rc1; (python_full_version == '3.13.0b4') and extra == 'dev'
-Requires-Dist: flaky; extra == 'dev'
-Requires-Dist: hatch; extra == 'dev'
-Requires-Dist: invoke; extra == 'dev'
-Requires-Dist: moto<5.0.0,>=4.2.8; extra == 'dev'
-Requires-Dist: packaging; extra == 'dev'
-Requires-Dist: pre-commit; extra == 'dev'
-Requires-Dist: pytest; extra == 'dev'
-Requires-Dist: pytest-xdist>=3.2.0; extra == 'dev'
-Requires-Dist: ruff; extra == 'dev'
-Requires-Dist: setuptools<72; extra == 'dev'
-Requires-Dist: sphinx; extra == 'dev'
-Requires-Dist: sphinx-rtd-theme; extra == 'dev'
-Requires-Dist: sure; extra == 'dev'
-Requires-Dist: twine; extra == 'dev'
-Requires-Dist: typing-extensions; extra == 'dev'
-Requires-Dist: vcrpy; extra == 'dev'
-Provides-Extra: doc
-Requires-Dist: markdown-include; extra == 'doc'
-Requires-Dist: mkdocs; extra == 'doc'
-Requires-Dist: mkdocs-material; extra == 'doc'
-Requires-Dist: pygments; extra == 'doc'
-Requires-Dist: pymdown-extensions; extra == 'doc'
 Description-Content-Type: text/markdown
 
 Simpleflow

{simpleflow-0.34.0 → simpleflow-0.34.2}/pyproject.toml

@@ -7,7 +7,7 @@ name = "simpleflow"
 description = "Python library for dataflow programming with Amazon SWF"
 readme = "README.md"
 requires-python = ">=3.7"
-license = { text = "MIT License" }
+license = "MIT"
 authors = [
     { name = "Greg Leclercq" },
 ]
@@ -48,36 +48,6 @@ dependencies = [
     "pytz",
 ]
 
-[project.optional-dependencies]
-dev = [
-    "boto3-stubs[s3,swf]",
-    "cffi==v1.17.0rc1; python_full_version=='3.13.0b4'", # via cryptography via moto, secretstorage
-    "flaky",
-    "hatch",
-    "invoke",
-    "moto>=4.2.8,<5.0.0",
-    "packaging",
-    "pre-commit",
-    "pytest",
-    "pytest-xdist>=3.2.0",
-    "ruff",
-    "setuptools<72", # See https://github.com/pypa/setuptools/issues/4519
-    "sphinx",
-    "sphinx_rtd_theme",
-    "sure",
-    "typing-extensions",
-    "vcrpy",
-    "twine",
-]
-
-doc = [
-    "mkdocs",
-    "mkdocs-material",
-    "pygments",
-    "pymdown-extensions",
-    "markdown-include",
-]
-
 [project.urls]
 documentation = "https://botify-labs.github.io/simpleflow"
 repository = "https://github.com/botify-labs/simpleflow"
@@ -97,10 +67,11 @@ extend-select = [
     "W",   # pycodestyle warnings
     "RUF", # ruff
 ]
-fixable = ["ALL"]
 allowed-confusables = ["‘", "’"]
 extend-ignore = [
 ]
+ignore = ["E501"]
+
 [tool.ruff.lint.isort]
 required-imports = ["from __future__ import annotations"]
 
@@ -117,3 +88,43 @@ path = "simpleflow/__init__.py"
 [tool.hatch.build]
 include = ["/simpleflow"]
 exclude = ["*~"]
+
+[dependency-groups]
+dev = [
+    "boto3-stubs[s3,swf]",
+    "cffi==v1.17.0rc1; python_full_version=='3.13.0b4'", # via cryptography via moto, secretstorage
+    "flaky",
+    "hatch",
+    "invoke",
+    "moto>=4.2.8,<5.0.0",
+    "packaging",
+    "pre-commit",
+    "pytest",
+    "pytest-xdist>=3.2.0",
+    "ruff",
+    "setuptools<72", # See https://github.com/pypa/setuptools/issues/4519
+    "sphinx",
+    "sphinx_rtd_theme",
+    "sure",
+    "typing-extensions",
+    "vcrpy",
+    "twine",
+]
+
+doc = [
+    "mkdocs",
+    "mkdocs-material",
+    "pygments",
+    "pymdown-extensions",
+    "markdown-include",
+]
+
+[[tool.uv.index]]
+name = "pypi"
+url = "https://pypi.org/simple/"
+publish-url = "https://pypi.org/legacy/"
+
+[[tool.uv.index]]
+name = "testpypi"
+url = "https://test.pypi.org/simple/"
+publish-url = "https://test.pypi.org/legacy/"

{simpleflow-0.34.0 → simpleflow-0.34.2}/simpleflow/__init__.py

@@ -5,6 +5,6 @@ from .runtime import logger  # NOQA
 from .signal import WaitForSignal  # NOQA
 from .workflow import Workflow  # NOQA
 
-__version__ = "0.34.0"
+__version__ = "0.34.2"
 __author__ = "Greg Leclercq"
 __license__ = "MIT"

{simpleflow-0.34.0 → simpleflow-0.34.2}/simpleflow/command.py

@@ -9,6 +9,7 @@ import time
 from contextlib import contextmanager
 from datetime import datetime
 from typing import TYPE_CHECKING
+from urllib.parse import unquote
 from uuid import uuid4
 
 import click
@@ -50,6 +51,18 @@ def comma_separated_list(value: str) -> list[str]:
     return value.split(",")
 
 
+class RunIdParamType(click.ParamType):
+    name = "str"
+
+    def convert(self, value: Any, param: click.Parameter | None, ctx: click.Context | None) -> str:
+        if "%" not in value:
+            return value
+        return unquote(value)
+
+
+RUN_ID = RunIdParamType()
+
+
 CONTEXT_SETTINGS = dict(help_option_names=["--help", "-h"])
 
 
@@ -207,7 +220,7 @@ def start_workflow(
     return execution
 
 
-@click.argument("run_id", required=False)
+@click.argument("run_id", type=RUN_ID, required=False)
 @click.argument("workflow_id")
 @click.argument(
     "domain",
@@ -223,10 +236,13 @@ def terminate_workflow(
     run_id: str | None,
 ):
     ex = helpers.get_workflow_execution(domain, workflow_id, run_id)
+    if not ex:
+        print(f"Execution {workflow_id} {run_id} not found" if run_id else f"Workflow {workflow_id} not found")
+        sys.exit(1)
     ex.terminate()
 
 
-@click.argument("run_id", required=False)
+@click.argument("run_id", type=RUN_ID, required=False)
 @click.argument("workflow_id")
 @click.argument(
     "domain",
@@ -238,6 +254,9 @@ def terminate_workflow(
 )
 def restart_workflow(domain: str, workflow_id: str, run_id: str | None):
     ex = helpers.get_workflow_execution(domain, workflow_id, run_id)
+    if not ex:
+        print(f"Execution {workflow_id} {run_id} not found" if run_id else f"Workflow {workflow_id} not found")
+        sys.exit(1)
     history = ex.history()
     ex.terminate(reason="workflow.restart")
     new_ex = ex.workflow_type.start_execution(
@@ -258,7 +277,7 @@ def with_format(ctx):
     )
 
 
-@click.argument("run_id", required=False)
+@click.argument("run_id", type=RUN_ID, required=False)
 @click.argument("workflow_id")
 @click.argument(
     "domain",
@@ -283,7 +302,7 @@ def workflow_info(ctx, domain: str, workflow_id: str, run_id: str | None):
     type=int,
     help="Maximum number of tasks to display.",
 )
-@click.argument("run_id", required=False)
+@click.argument("run_id", type=RUN_ID, required=False)
 @click.argument("workflow_id")
 @click.argument(
     "domain",
@@ -302,6 +321,7 @@ def profile(ctx, domain, workflow_id, run_id, nb_tasks):
     )
 
 
+# FIXME superseded by history
 @click.option(
     "--nb-tasks",
     "-n",
@@ -309,7 +329,7 @@ def profile(ctx, domain, workflow_id, run_id, nb_tasks):
     type=int,
     help="Maximum number of tasks to display.",
 )
-@click.argument("run_id", required=False)
+@click.argument("run_id", type=RUN_ID, required=False)
 @click.argument("workflow_id")
 @click.argument(
     "domain",
@@ -334,6 +354,7 @@ def workflow_tasks(
     )
 
 
+# FIXME superseded by filter
 @click.argument(
     "domain",
     envvar="SWF_DOMAIN",
@@ -360,18 +381,19 @@ def list_workflows(ctx, domain: str, status: str, started_since: int):
 _NOTSET = object()
 
 
-@click.argument(
-    "domain",
-    envvar="SWF_DOMAIN",
-)
 @cli.command(
     "workflow.history",
     help="Workflow history from workflow WORKFLOW_ID [RUN_ID].",
 )
 @click.argument("workflow_id")
-@click.argument("run_id", required=False)
+@click.argument("run_id", type=RUN_ID, required=False)
+@click.option(
+    "--domain",
+    envvar="SWF_DOMAIN",
+)
 @click.option(
     "--output-format",
+    "--of",
     "-o",
     required=False,
     type=click.Choice(["events", "raw", "cooked"]),
@@ -388,12 +410,15 @@ def workflow_history(
     output_format: str,
     reverse_order: bool = False,
 ) -> None:
-    from simpleflow.swf.mapper.models.history.base import History as BaseHistory
-
     if ctx.parent.params["format"] != "json" or not ctx.parent.params["header"]:
         raise NotImplementedError("Only pretty JSON mode is implemented")
 
+    from simpleflow.swf.mapper.models.history.base import History as BaseHistory
+
     ex = helpers.get_workflow_execution(domain, workflow_id, run_id)
+    if not ex:
+        print(f"Execution {workflow_id} {run_id} not found" if run_id else f"Workflow {workflow_id} not found")
+        sys.exit(1)
     events = ex.history_events(
         callback=get_progression_callback("events"),
         reverse_order=reverse_order,
@@ -418,11 +443,15 @@ def workflow_history(
         elif output_format == "cooked":
             history.parse()
             events = {
+                "workflow": history.workflow,
                 "activities": history.activities,
                 "child_workflows": history.child_workflows,
                 "markers": history.markers,
-                "signals": history.signals,
                 "timers": history.timers,
+                "signals": history.signals,
+                "signal_lists": history.signal_lists,
+                "external_workflows_signaling": history.external_workflows_signaling,
+                "signaled_workflows": history.signaled_workflows,
             }
         else:
             raise NotImplementedError
@@ -740,7 +769,7 @@ def standalone(
         # get the previous execution history, it will serve as "default history"
         # for activities that succeeded in the previous execution
         logger.info(
-            f"retrieving history of previous execution: domain={domain} " f"workflow_id={repair} run_id={repair_run_id}"
+            f"retrieving history of previous execution: domain={domain} workflow_id={repair} run_id={repair_run_id}"
         )
         workflow_execution = get_workflow_execution(domain, repair, run_id=repair_run_id)
         previous_history = History(workflow_execution.history())
@@ -826,6 +855,11 @@ def standalone(
             ex.workflow_id,
             ex.run_id,
         )
+        if not ex:
+            print(
+                f"Execution {workflow_id} {ex.run_id} not found" if ex.run_id else f"Workflow {workflow_id} not found"
+            )
+            sys.exit(1)
         if display_status:
             print(f"status: {ex.status}", file=sys.stderr)
         if ex.status == ex.STATUS_CLOSED:
@@ -841,7 +875,7 @@ def standalone(
 @click.option("--domain", envvar="SWF_DOMAIN", required=False, help="Amazon SWF Domain.")
 @click.option("--workflow-id", required=True, help="ID of the workflow execution.")
 @click.option("--input", "-i", required=False, help="JSON input of the workflow.")
-@click.option("--run-id", required=False, help="Run ID of the workflow execution.")
+@click.option("--run-id", type=RUN_ID, required=False, help="Run ID of the workflow execution.")
 @click.option(
     "--scheduled-id",
     required=False,
@@ -891,7 +925,7 @@ def activity_rerun(domain, workflow_id, run_id, input, scheduled_id, activity_id
     for line in json_dumps(params, pretty=True).split("\n"):
         logger.debug(line)
     if input_override:
-        logger.info("NB: input will be overriden with the passed one!")
+        logger.info("NB: input will be overridden with the passed one!")
     logger.info(f"Will re-run: {task}(*{args}, **{kwargs}) [+meta={meta}]")
 
     # download binaries if needed
@@ -913,8 +947,7 @@ def activity_rerun(domain, workflow_id, run_id, input, scheduled_id, activity_id
 )
 @cli.command(
     "info",
-    help="Display versions, settings, and environment variables. "
-    "Available sections: versions, settings, environment.",
+    help="Display versions, settings, and environment variables. Available sections: versions, settings, environment.",
 )
 def info(sections):
     @contextmanager

{simpleflow-0.34.0 → simpleflow-0.34.2}/simpleflow/dispatch/dynamic_dispatcher.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from simpleflow.activity import Activity
-from simpleflow.utils import import_object_from_module
+from simpleflow.utils import format_exc, import_object_from_module
 
 from .exceptions import DispatchError
 
@@ -25,9 +25,9 @@ class Dispatcher:
         module_name, activity_name = name.rsplit(".", 1)
         try:
             activity = import_object_from_module(module_name, activity_name)
-        except ImportError:
+        except ImportError as e:
             # We were not able to import a function at all.
-            raise DispatchError(f"unable to import '{name}'")
+            raise DispatchError(f"unable to import '{name}': {format_exc(e)}")
         if not isinstance(activity, Activity):
             # We managed to import a function (or callable) but it's not an
             # "Activity". We will transform it into an Activity now. That way

{simpleflow-0.34.0 → simpleflow-0.34.2}/simpleflow/history.py

@@ -43,11 +43,16 @@ class History:
         self.started_decision_id: int | None = None
         self.completed_decision_id: int | None = None
         self.last_event_id: int | None = None
+        self._workflow: dict[str, Any] = {}
 
     @property
     def swf_history(self) -> simpleflow.swf.mapper.models.history.History:
         return self._history
 
+    @property
+    def workflow(self):
+        return self._workflow
+
     @property
     def activities(self) -> dict[str, ActivityTaskEventDict]:
         """
@@ -432,6 +437,118 @@ class History:
         """
         Parse a workflow event.
         """
+        if event.state == "started":
+            self._workflow.update(
+                {
+                    "state": event.state,
+                    f"{event.state}_id": event.id,
+                    f"{event.state}_timestamp": event.timestamp,
+                    "child_policy": getattr(event, "child_policy", None),
+                    "task_list": event.task_list["name"],
+                    "workflow_type": event.workflow_type,
+                    "continued_execution_run_id": getattr(event, "continued_execution_run_id", None),
+                    "execution_start_to_close_timeout": getattr(event, "execution_start_to_close_timeout", None),
+                    "input": getattr(event, "input", None),
+                    "lambda_role": getattr(event, "lambda_role", None),
+                    "parent_initiated_event_id": getattr(event, "parent_initiated_event_id", None),
+                    "parent_workflow_execution": getattr(event, "parent_workflow_execution", None),
+                    "tag_list": getattr(event, "tag_list", None),
+                    "task_priority": getattr(event, "task_priority", None),
+                    "task_start_to_close_timeout": getattr(event, "task_start_to_close_timeout", None),
+                }
+            )
+        elif event.state == "continued_as_new":
+            self._workflow.update(
+                {
+                    "state": event.state,
+                    f"{event.state}_id": event.id,
+                    f"{event.state}_timestamp": event.timestamp,
+                    f"{event.state}_decision_task_completed_event_id": event.decision_task_completed_event_id,
+                    "new_execution_run_id": event.new_execution_run_id,
+                    "task_list": event.task_list["name"],
+                    "workflow_type": event.workflow_type,
+                    "execution_start_to_close_timeout": getattr(event, "execution_start_to_close_timeout", None),
+                    "input": getattr(event, "input", None),
+                    "lambda_role": getattr(event, "lambda_role", None),
+                    "tag_list": getattr(event, "tag_list", None),
+                    "task_priority": getattr(event, "task_priority", None),
+                    "task_start_to_close_timeout": getattr(event, "task_start_to_close_timeout", None),
+                }
+            )
+        elif event.state == "completed":
+            self._workflow.update(
+                {
+                    "state": event.state,
+                    f"{event.state}_id": event.id,
+                    f"{event.state}_timestamp": event.timestamp,
+                    "initiated_event_id": getattr(event, "initiated_event_id", None),
+                    "result": getattr(event, "result", None),
+                }
+            )
+        elif event.state == "cancelled":
+            self._workflow.update(
+                {
+                    "state": event.state,
+                    f"{event.state}_id": event.id,
+                    f"{event.state}_timestamp": event.timestamp,
+                    "initiated_event_id": getattr(event, "initiated_event_id", None),
+                    "decision_task_completed_event_id": event.decision_task_completed_event_id,
+                    "details": getattr(event, "details", None),
+                }
+            )
+        elif event.state == "failed":
+            self._workflow.update(
+                {
+                    "state": event.state,
+                    f"{event.state}_id": event.id,
+                    f"{event.state}_timestamp": event.timestamp,
+                    "initiated_event_id": getattr(event, "initiated_event_id", None),
+                    "decision_task_completed_event_id": event.decision_task_completed_event_id,
+                    "reason": getattr(event, "reason", None),
+                    "details": getattr(event, "details", None),
+                }
+            )
+        elif event.state == "terminated":
+            self._workflow.update(
+                {
+                    "state": event.state,
+                    f"{event.state}_id": event.id,
+                    f"{event.state}_timestamp": event.timestamp,
+                    "initiated_event_id": getattr(event, "initiated_event_id", None),
+                    "cause": getattr(event, "cause", None),
+                    "details": getattr(event, "details", None),
+                }
+            )
+        elif event.state == "timed_out":
+            self._workflow.update(
+                {
+                    "state": event.state,
+                    f"{event.state}_id": event.id,
+                    f"{event.state}_timestamp": event.timestamp,
+                    "initiated_event_id": getattr(event, "initiated_event_id", None),
+                    "timeout_type": event.timeout_type,
+                }
+            )
+        # elif event.state in (
+        #     "cancel_failed",
+        #     "complete_failed",
+        #     "continue_as_new",
+        #     "fail_failed",
+        #     "start_child_failed",
+        #     "start_failed",
+        #     "terminate_failed",
+        # ):
+        #     self._workflow.update(
+        #         {
+        #             "state": event.state,
+        #             f"{event.state}_id": event.id,
+        #             f"{event.state}_cause": getattr(event, "cause", None),
+        #             f"{event.state}_decision_task_completed_event_id": event.decision_task_completed_event_id,
+        #         }
+        #     )
+
+        if event.state == "cancel_requested":
+            self._workflow.update()
         if event.state == "signaled":
             signal = {
                 "type": "signal",

{simpleflow-0.34.0 → simpleflow-0.34.2}/simpleflow/local/executor.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import collections
 import sys
 import traceback
 import uuid
@@ -27,8 +26,8 @@ class Executor(executor.Executor):
         super().__init__(workflow_class)
         self.update_workflow_class()
         self.nb_activities = 0
-        self.signals_sent = set()
-        self._markers = collections.OrderedDict()
+        self.signals_sent: set[str] = set()
+        self._markers: dict[str, list[Marker]] = {}
 
         self.wf_run_id = []
         self.wf_id = []
@@ -206,7 +205,7 @@ class Executor(executor.Executor):
         self.update_workflow_class()
         self.nb_activities = 0
         self.signals_sent = set()
-        self._markers = collections.OrderedDict()
+        self._markers = {}
 
         self.wf_run_id = []
         self.wf_id = []

{simpleflow-0.34.0 → simpleflow-0.34.2}/simpleflow/metrology.py

@@ -6,7 +6,6 @@ import os
 import re
 import time
 from abc import ABC
-from collections import OrderedDict
 from typing import Any
 from urllib.parse import quote_plus
 
@@ -18,31 +17,29 @@ ACTIVITY_KEY_RE = re.compile(r"activity\.(.+)\.json")
 
 
 class StepIO:
-    def __init__(self):
+    def __init__(self) -> None:
         self.bytes = 0
         self.records = 0
         self.sampled = False
 
-    def get_stats(self, time_total):
+    def get_stats(self, time_total: float) -> dict[str, Any]:
         mb_s = None
         rec_s = None
         if self.bytes:
             mb_s = round(float(self.bytes) / (1024 * 1024) / time_total, 2)
         if self.records:
             rec_s = int(self.records / time_total)
-        return OrderedDict(
-            [
-                ("bytes", self.bytes),
-                ("records", self.records),
-                ("mb_s", mb_s),
-                ("rec_s", rec_s),
-                ("sampled", self.sampled),
-            ]
-        )
+        return {
+            "bytes": self.bytes,
+            "records": self.records,
+            "mb_s": mb_s,
+            "rec_s": rec_s,
+            "sampled": self.sampled,
+        }
 
 
 class Step:
-    def __init__(self, name, task):
+    def __init__(self, name: str, task: MetrologyTask) -> None:
         self.name = name
         self.task = task
         self.read = StepIO()
@@ -57,22 +54,17 @@ class Step:
         self.time_total = self.time_finished - self.time_started
 
     def get_stats(self):
-        stats = OrderedDict(
-            [
-                ("name", self.name),
-                ("metadata", self.metadata),
-                ("time_started", self.time_started),
-                ("time_finished", self.time_finished),
-                ("time_total", self.time_total),
-                ("read", self.read.get_stats(self.time_total)),
-                ("write", self.write.get_stats(self.time_total)),
-            ]
-        )
-        return stats
+        stats = {
+            "name": self.name,
+            "metadata": self.metadata,
+            "time_started": self.time_started,
+            "time_finished": self.time_finished,
+            "time_total": self.time_total,
+            "read": self.read.get_stats(self.time_total),
+            "write": self.write.get_stats(self.time_total),
+        }
 
-    def mset_metadata(self, kvs):
-        for k, v in kvs:
-            self.metadata[k] = v
+        return stats
 
 
 class StepExecution:
@@ -90,14 +82,14 @@ class MetrologyTask:
     context: dict[str, Any]
     steps: list[Step]
 
-    def can_upload(self):
+    def can_upload(self) -> bool:
         if not hasattr(self, "context"):
             return False
         return all(c in self.context for c in ("workflow_id", "run_id", "activity_id"))
 
     @property
     def metrology_path(self):
-        path = []
+        path: list[str] = []
         if settings.METROLOGY_PATH_PREFIX is not None:
             path.append(settings.METROLOGY_PATH_PREFIX)
         path.append(self.context["workflow_id"])
@@ -105,7 +97,7 @@ class MetrologyTask:
         path.append(f"activity.{self.context['activity_id']}.json")
         return str(os.path.join(*path))
 
-    def step(self, name):
+    def step(self, name: str) -> StepExecution:
         """
         To be called in a `with` execution
         Ex :

{simpleflow-0.34.0 → simpleflow-0.34.2}/simpleflow/process/_supervisor.py

@@ -208,7 +208,7 @@ class Supervisor(NamedMixin):
         Terminate all worker processes managed by this Supervisor.
         """
         self._terminating = True
-        logger.info("process: will stop workers, this might take up several minutes. " "Please, be patient.")
+        logger.info("process: will stop workers, this might take up several minutes. Please, be patient.")
         self._killall()
 
     def _killall(self):