pulse-engine 0.2.8__tar.gz → 0.2.9__tar.gz

{pulse_engine-0.2.8 → pulse_engine-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pulse-engine
-Version: 0.2.8
+Version: 0.2.9
 Summary: Pulse Engine — Hybrid framework for building Pulse products
 Author: Pulse Team
 Requires-Python: >=3.11,<3.13

{pulse_engine-0.2.8 → pulse_engine-0.2.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pulse-engine"
-version = "0.2.8"
+version = "0.2.9"
 description = "Pulse Engine — Hybrid framework for building Pulse products"
 authors = ["Pulse Team"]
 readme = "README.md"

{pulse_engine-0.2.8 → pulse_engine-0.2.9}/src/pulse_engine/runners/prefect_pipeline_flow.py

@@ -1193,8 +1193,60 @@ def _execute_layer(
 
         if step_config.get("for_each"):
             expr = parse_expression(step_config["for_each"])
+            # Fetch S3 refs before fan-out so field access (e.g. .batches) works
+            resolved_for_each = step_results
+            if (
+                expr.source == "steps"
+                and expr.step
+                and _is_s3_ref(step_results.get(expr.step))
+            ):
+                try:
+                    resolved_for_each = {
+                        **step_results,
+                        expr.step: _fetch_s3_ref(step_results[expr.step]),
+                    }
+                except Exception:
+                    logger.warning(
+                        "Could not fetch S3 ref for for_each step '%s'",
+                        expr.step,
+                        exc_info=True,
+                    )
+            elif expr.source == "collect" and collect_from:
+                # $collect.field — resolve S3 refs for all collect_from steps
+                overrides: dict[str, Any] = {}
+                for _cname in collect_from:
+                    _val = step_results.get(_cname)
+                    if _is_s3_ref(_val) and _val is not None:
+                        try:
+                            overrides[_cname] = _fetch_s3_ref(_val)
+                        except Exception:
+                            logger.warning(
+                                "Could not fetch S3 ref for collect step '%s'",
+                                _cname,
+                                exc_info=True,
+                            )
+                    elif isinstance(_val, list):
+                        # fan-out: list of S3 refs — fetch and flatten
+                        fetched: list[Any] = []
+                        for _ref in _val:
+                            if _is_s3_ref(_ref):
+                                try:
+                                    fetched.append(_fetch_s3_ref(_ref))
+                                except Exception:
+                                    logger.warning(
+                                        "Could not fetch S3 ref in collect list"
+                                        " for step '%s'",
+                                        _cname,
+                                        exc_info=True,
+                                    )
+                            else:
+                                fetched.append(_ref)
+                        if fetched:
+                            overrides[_cname] = fetched
+                if overrides:
+                    resolved_for_each = {**step_results, **overrides}
             items = resolve_for_each_items(
-                expr, module_args, step_results, collect_from or None
+                expr, module_args, resolved_for_each, collect_from or None
             )
 
             # Item-level when filter (fan-out only)