simpleflow 0.36.0__tar.gz → 0.37.0__tar.gz

{simpleflow-0.36.0 → simpleflow-0.37.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: simpleflow
-Version: 0.36.0
+Version: 0.37.0
 Summary: Python library for dataflow programming with Amazon SWF
 Project-URL: documentation, https://botify-labs.github.io/simpleflow
 Project-URL: repository, https://github.com/botify-labs/simpleflow
@@ -25,7 +25,6 @@ Requires-Dist: attrs
 Requires-Dist: boto3>=1.28.20
 Requires-Dist: click
 Requires-Dist: dill>=0.3.6
-Requires-Dist: diskcache>=4.1.0
 Requires-Dist: jinja2
 Requires-Dist: lazy-object-proxy
 Requires-Dist: lockfile>=0.9.1
@@ -35,6 +34,8 @@ Requires-Dist: psutil
 Requires-Dist: pytz
 Requires-Dist: setproctitle
 Requires-Dist: tabulate<1.0.0,>=0.8.2
+Provides-Extra: diskcache
+Requires-Dist: diskcache>=4.1.0; extra == 'diskcache'
 Description-Content-Type: text/markdown
 
 Simpleflow

{simpleflow-0.36.0 → simpleflow-0.37.0}/pyproject.toml

@@ -35,7 +35,6 @@ dependencies = [
     "attrs",
     "boto3>=1.28.20",
     "dill>=0.3.6",
-    "diskcache>=4.1.0",
     "Jinja2",
     "markupsafe",
     "multiprocess>=0.70.14",
@@ -48,6 +47,11 @@ dependencies = [
     "pytz",
 ]
 
+[project.optional-dependencies]
+# Jumbo fields disk cache, only used when SIMPLEFLOW_ENABLE_DISK_CACHE is set (deprecated).
+# Optional because diskcache unpickles cache entries: https://osv.dev/vulnerability/CVE-2025-69872
+diskcache = ["diskcache>=4.1.0"]
+
 [project.urls]
 documentation = "https://botify-labs.github.io/simpleflow"
 repository = "https://github.com/botify-labs/simpleflow"
@@ -127,7 +131,6 @@ dev = [
     "pytest",
     "pytest-xdist>=3.2.0",
     "ruff",
-    "setuptools<72",                                     # See https://github.com/pypa/setuptools/issues/4519
     "sphinx",
     "sphinx_rtd_theme",
     "typing-extensions",

{simpleflow-0.36.0 → simpleflow-0.37.0}/simpleflow/format.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 
 import os
+import warnings
 from sqlite3 import OperationalError
 from typing import Any, cast
 from uuid import uuid4
 
 import lazy_object_proxy
-from diskcache import Cache
 
 from simpleflow import constants, logger, storage
 from simpleflow.settings import SIMPLEFLOW_ENABLE_DISK_CACHE
@@ -78,6 +78,17 @@ def encode(message: str | None, max_length: int, allow_jumbo_fields: bool = True
     return message
 
 
+DISK_CACHE_DEPRECATION = (
+    "SIMPLEFLOW_ENABLE_DISK_CACHE is deprecated and will be removed: its only backend, diskcache, "
+    "unpickles cache entries (https://osv.dev/vulnerability/CVE-2025-69872), so anyone who can write to the cache "
+    "directory can run code in the process that reads it."
+)
+
+
+def _warn_disk_cache_deprecated() -> None:
+    warnings.warn(DISK_CACHE_DEPRECATION, DeprecationWarning, stacklevel=3)
+
+
 def _get_cached(path: str) -> str | None:
     # 1/ memory cache
     if path in JUMBO_FIELDS_MEMORY_CACHE:
@@ -85,11 +96,15 @@ def _get_cached(path: str) -> str | None:
 
     # 2/ disk cache
     if SIMPLEFLOW_ENABLE_DISK_CACHE:
+        _warn_disk_cache_deprecated()
         try:
             # NB: this cache may also be triggered on activity workers, where it's not that
             # useful. The performance hit should be minimal. To be improved later.
             # NB2: cache has to be lazily instantiated here, cache objects do not survive forks,
             # see DiskCache docs.
+            # Optional extra: simpleflow[diskcache]. Deprecated, see _warn_disk_cache_deprecated.
+            from diskcache import Cache
+
             cache = Cache(constants.CACHE_DIR)
             # generate a dedicated cache key because this cache may be shared with other
             # features of simpleflow at some point
@@ -97,6 +112,8 @@ def _get_cached(path: str) -> str | None:
             if cache_key in cache:
                 logger.debug(f"diskcache: getting key={cache_key} from cache_dir={constants.CACHE_DIR}")
                 return cache[cache_key]
+        except ImportError:
+            logger.warning("diskcache: not installed (pip install simpleflow[diskcache]), skipping cache usage")
         except OperationalError:
             logger.warning("diskcache: got an OperationalError, skipping cache usage")
 
@@ -110,11 +127,17 @@ def _set_cached(path: str, content: str) -> None:
 
     # 2/ disk cache
     if SIMPLEFLOW_ENABLE_DISK_CACHE:
+        _warn_disk_cache_deprecated()
         try:
+            # Optional extra: simpleflow[diskcache]. Deprecated, see _warn_disk_cache_deprecated.
+            from diskcache import Cache
+
             cache = Cache(constants.CACHE_DIR)
             cache_key = "jumbo_fields/" + path.split("/")[-1]
             logger.debug(f"diskcache: setting key={cache_key} on cache_dir={constants.CACHE_DIR}")
             cache.set(cache_key, content, expire=3 * constants.HOUR)
+        except ImportError:
+            logger.warning("diskcache: not installed (pip install simpleflow[diskcache]), skipping cache write")
         except OperationalError:
             logger.warning("diskcache: got an OperationalError on write, skipping cache write")
 

{simpleflow-0.36.0 → simpleflow-0.37.0}/simpleflow/step/constants.py

@@ -25,3 +25,7 @@ STEP_ACTIVITY_PARAMS_DEFAULT: dict[str, Any] = {
 }
 
 UNKNOWN_CONTEXT = {"run_id": "unknown", "workflow_id": "unknown", "version": "unknown"}
+
+# Prefix of the marker memoizing, in the workflow history, the steps done at the
+# beginning of an execution (only used with ``WorkflowStepMixin.steps_in_decider``).
+STEPS_DONE_MARKER_PREFIX = "step.steps_done"

{simpleflow-0.36.0 → simpleflow-0.37.0}/simpleflow/step/submittable.py

@@ -57,6 +57,15 @@ class Step(SubmittableContainer):
         if workflow is None:
             raise ValueError("Executor has no associated workflow")
 
+        steps_in_decider = getattr(workflow, "steps_in_decider", False)
+
+        def mark_step_done_in_decider(marker_done):
+            # The "completed" marker is recorded in the same decision as the S3 push:
+            # once it is in the history, the step is not pushed again on replays.
+            if not any(m.name == "log.step" and m.details == marker_done for m in workflow.list_markers(all=True)):
+                workflow._mark_step_done_in_decider(self.step_name)
+            return workflow.record_marker("log.step", marker_done)
+
         def fn_steps_done(steps_done):
             marker = {
                 "step": self.step_name,
@@ -81,14 +90,19 @@ class Step(SubmittableContainer):
                 chain += (
                     workflow.record_marker("log.step", marker),
                     self.activities,
-                    (
-                        activity.Activity(MarkStepDoneTask, **workflow._get_step_activity_params()),
-                        workflow.get_step_bucket(),
-                        workflow.get_step_path_prefix(),
-                        self.step_name,
-                    ),
-                    workflow.record_marker("log.step", marker_done),
                 )
+                if steps_in_decider:
+                    chain.append(FuncGroup(mark_step_done_in_decider, marker_done))
+                else:
+                    chain += (
+                        (
+                            activity.Activity(MarkStepDoneTask, **workflow._get_step_activity_params()),
+                            workflow.get_step_bucket(),
+                            workflow.get_step_path_prefix(),
+                            self.step_name,
+                        ),
+                        workflow.record_marker("log.step", marker_done),
+                    )
             else:
                 marker["status"] = "skipped"
                 if step_is_skipped_by_force(self.step_name, skipped_steps):
@@ -108,6 +122,8 @@ class Step(SubmittableContainer):
             chain.bubbles_exception_on_failure = self.bubbles_exception_on_failure
             return chain
 
+        if steps_in_decider:
+            return workflow.submit(fn_steps_done(workflow._get_steps_done_in_decider()))
         return workflow.submit(
             Chain(
                 workflow.get_steps_done_activity(),

{simpleflow-0.36.0 → simpleflow-0.37.0}/simpleflow/step/workflow.py

@@ -2,15 +2,17 @@ from __future__ import annotations
 
 import abc
 import copy
+import hashlib
 import os
 from collections import defaultdict
 from typing import TYPE_CHECKING
 
-from simpleflow import activity, settings, task
+from simpleflow import activity, constants, settings, task
+from simpleflow.utils import json_dumps, retry
 
-from .constants import STEP_ACTIVITY_PARAMS_DEFAULT
+from .constants import STEP_ACTIVITY_PARAMS_DEFAULT, STEPS_DONE_MARKER_PREFIX
 from .submittable import Step
-from .tasks import GetStepsDoneTask
+from .tasks import GetStepsDoneTask, MarkStepDoneTask
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
@@ -18,6 +20,11 @@ if TYPE_CHECKING:
 
 
 class WorkflowStepMixin(metaclass=abc.ABCMeta):
+    # Opt-in: list and mark the steps done from the decider instead of
+    # GetStepsDoneTask/MarkStepDoneTask activities. The S3 listing is done once
+    # per execution and memoized in a marker so that replays stay deterministic.
+    steps_in_decider: bool = False
+
     def get_step_bucket(self) -> str:
         """
         Return the S3 bucket where to store the steps files.
@@ -110,3 +117,56 @@ class WorkflowStepMixin(metaclass=abc.ABCMeta):
         Return the list of steps done.
         """
         return self.submit(self.get_steps_done_activity()).result
+
+    def _get_steps_done_in_decider(self) -> list[str]:
+        """
+        Return the list of steps done, as listed on S3 the first time it is
+        needed in this execution.
+
+        The listing is recorded in a marker, then read back from the history on
+        the next replays: the steps marked done later in this execution don't
+        change the decisions already taken.
+        """
+        bucket = self.get_step_bucket()
+        path = self.get_step_path_prefix()
+        location_hash = hashlib.md5(f"{bucket}/{path}".encode()).hexdigest()  # nosec
+        marker_name = f"{STEPS_DONE_MARKER_PREFIX}.{location_hash}"
+
+        # Cache for the current replay only: the workflow instance may be reused
+        # across replays, whereas the executor builds a new history for each one.
+        history = self.executor._history
+        if getattr(self, "_steps_done_snapshots", (None,))[0] is not history:
+            self._steps_done_snapshots: tuple[Any, dict[str, list[str]]] = (history, {})
+        snapshots = self._steps_done_snapshots[1]
+        if marker_name not in snapshots:
+            event = self.get_event_details("marker", marker_name)
+            if event is not None:
+                steps_done = event["details"]["steps_done"]
+            else:
+                steps_done = _execute_with_retry(GetStepsDoneTask(bucket, path))
+                details = {"bucket": bucket, "path": path, "steps_done": steps_done}
+                if len(json_dumps(details)) > constants.MAX_DETAILS_LENGTH:
+                    raise ValueError(
+                        f"Too many steps done in {bucket}/{path} to record them in a marker;"
+                        " disable steps_in_decider for this workflow"
+                    )
+                # Not waited for: the marker is recorded along with the other decisions.
+                self.submit(self.record_marker(marker_name, details))
+            snapshots[marker_name] = steps_done
+        return snapshots[marker_name]
+
+    def _mark_step_done_in_decider(self, step_name: str) -> None:
+        """
+        Push the step file on S3 from the decider.
+        """
+        mark_step_done = MarkStepDoneTask(self.get_step_bucket(), self.get_step_path_prefix(), step_name)
+        run_context = self.get_run_context()
+        if run_context.get("run_id"):
+            mark_step_done.context = run_context
+        _execute_with_retry(mark_step_done)
+
+
+@retry.with_delay(nb_times=3, delay=retry.exponential)
+def _execute_with_retry(a_task: GetStepsDoneTask | MarkStepDoneTask) -> Any:
+    # An exception in the decider fails the workflow: retry transient S3 errors.
+    return a_task.execute()

{simpleflow-0.36.0 → simpleflow-0.37.0}/simpleflow/swf/executor.py

@@ -1260,7 +1260,9 @@ class Executor(executor.Executor):
 
     def list_markers(self, all=False):
         if all:
-            return [Marker(m["name"], format.decode(m["details"])) for ml in self._history.markers.values() for m in ml]
+            return [
+                Marker(m["name"], format.decode(m.get("details"))) for ml in self._history.markers.values() for m in ml
+            ]
         rc = []
         for ml in self._history.markers.values():
             m = ml[-1]