pyoco 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

pyoco/cli/main.py

@@ -1,13 +1,16 @@
 import argparse
+import json
 import sys
 import os
 import signal
+import time
 from ..schemas.config import PyocoConfig
 from ..discovery.loader import TaskLoader
 from ..core.models import Flow
 from ..core.engine import Engine
 from ..trace.console import ConsoleTraceBackend
 from ..client import Client
+from ..discovery.plugins import list_available_plugins
 
 def main():
     parser = argparse.ArgumentParser(description="Pyoco Workflow Engine")
@@ -28,6 +31,8 @@ def main():
     check_parser = subparsers.add_parser("check", help="Verify a workflow")
     check_parser.add_argument("--config", required=True, help="Path to flow.yaml")
     check_parser.add_argument("--flow", default="main", help="Flow name to check")
+    check_parser.add_argument("--dry-run", action="store_true", help="Traverse flow without executing tasks")
+    check_parser.add_argument("--json", action="store_true", help="Output report as JSON")
 
     # List tasks command
     list_parser = subparsers.add_parser("list-tasks", help="List available tasks")
@@ -55,6 +60,8 @@ def main():
     runs_list = runs_subparsers.add_parser("list", help="List runs")
     runs_list.add_argument("--server", default="http://localhost:8000", help="Server URL")
     runs_list.add_argument("--status", help="Filter by status")
+    runs_list.add_argument("--flow", help="Filter by flow name")
+    runs_list.add_argument("--limit", type=int, help="Maximum number of runs to show")
     
     runs_show = runs_subparsers.add_parser("show", help="Show run details")
     runs_show.add_argument("run_id", help="Run ID")
@@ -63,6 +70,24 @@ def main():
     runs_cancel = runs_subparsers.add_parser("cancel", help="Cancel a run")
     runs_cancel.add_argument("run_id", help="Run ID")
     runs_cancel.add_argument("--server", default="http://localhost:8000", help="Server URL")
+    
+    runs_inspect = runs_subparsers.add_parser("inspect", help="Inspect run details")
+    runs_inspect.add_argument("run_id", help="Run ID")
+    runs_inspect.add_argument("--server", default="http://localhost:8000", help="Server URL")
+    runs_inspect.add_argument("--json", action="store_true", help="Output JSON payload")
+    
+    runs_logs = runs_subparsers.add_parser("logs", help="Show run logs")
+    runs_logs.add_argument("run_id", help="Run ID")
+    runs_logs.add_argument("--server", default="http://localhost:8000", help="Server URL")
+    runs_logs.add_argument("--task", help="Filter logs by task")
+    runs_logs.add_argument("--tail", type=int, help="Show last N log entries")
+    runs_logs.add_argument("--follow", action="store_true", help="Stream logs until completion")
+    runs_logs.add_argument("--allow-failure", action="store_true", help="Don't exit non-zero when run failed")
+
+    plugins_parser = subparsers.add_parser("plugins", help="Inspect plug-in entry points")
+    plugins_sub = plugins_parser.add_subparsers(dest="plugins_command")
+    plugins_list = plugins_sub.add_parser("list", help="List discovered plug-ins")
+    plugins_list.add_argument("--json", action="store_true", help="Output JSON payload")
 
     args = parser.parse_args()
 
@@ -94,6 +119,21 @@ def main():
             print(f" - {name}")
         return
 
+    if args.command == "plugins":
+        infos = list_available_plugins()
+        if args.plugins_command == "list":
+            if getattr(args, "json", False):
+                print(json.dumps(infos, indent=2))
+            else:
+                if not infos:
+                    print("No plug-ins registered under group 'pyoco.tasks'.")
+                else:
+                    print("Discovered plug-ins:")
+                    for info in infos:
+                        mod = info.get("module") or info.get("value")
+                        print(f" - {info.get('name')} ({mod})")
+        return
+
     if args.command == "server":
         if args.server_command == "start":
             import uvicorn
@@ -113,7 +153,7 @@ def main():
         client = Client(args.server)
         try:
             if args.runs_command == "list":
-                runs = client.list_runs(status=args.status)
+                runs = client.list_runs(status=args.status, flow=args.flow, limit=args.limit)
                 print(f"🐇 Active Runs ({len(runs)}):")
                 print(f"{'ID':<36} | {'Status':<12} | {'Flow':<15}")
                 print("-" * 70)
@@ -134,6 +174,31 @@ def main():
             elif args.runs_command == "cancel":
                 client.cancel_run(args.run_id)
                 print(f"🛑 Cancellation requested for run {args.run_id}")
+            elif args.runs_command == "inspect":
+                run = client.get_run(args.run_id)
+                if args.json:
+                    print(json.dumps(run, indent=2))
+                else:
+                    print(f"🐇 Run: {run['run_id']} ({run.get('flow_name', 'n/a')})")
+                    print(f"Status: {run['status']}")
+                    if run.get("start_time"):
+                        print(f"Started: {run['start_time']}")
+                    if run.get("end_time"):
+                        print(f"Ended: {run['end_time']}")
+                    print("Tasks:")
+                    records = run.get("task_records", {})
+                    for name, info in records.items():
+                        state = info.get("state", run["tasks"].get(name))
+                        duration = info.get("duration_ms")
+                        duration_str = f"{duration:.2f} ms" if duration else "-"
+                        print(f"  - {name}: {state} ({duration_str})")
+                        if info.get("error"):
+                            print(f"      error: {info['error']}")
+                    if not records:
+                        for t_name, t_state in run.get("tasks", {}).items():
+                            print(f"  - {t_name}: {t_state}")
+            elif args.runs_command == "logs":
+                _stream_logs(client, args)
         except Exception as e:
             print(f"Error: {e}")
         return
@@ -164,14 +229,16 @@ def main():
                 sys.exit(1)
             return
         # Build Flow from graph string
-        from ..dsl.syntax import TaskWrapper
+        from ..dsl.syntax import TaskWrapper, switch
         eval_context = {name: TaskWrapper(task) for name, task in loader.tasks.items()}
+        eval_context["switch"] = switch
         
         try:
             # Create Flow and add all loaded tasks
             flow = Flow(name=args.flow)
             for t in loader.tasks.values():
                 flow.add_task(t)
+            eval_context["flow"] = flow
 
             # Evaluate graph to set up dependencies
             exec(flow_conf.graph, {}, eval_context)
@@ -210,36 +277,36 @@ def main():
 
         # 1. Check imports (already done by loader.load(), but we can check for missing tasks in graph)
         # 2. Build flow to check graph
-        from ..dsl.syntax import TaskWrapper
+        from ..dsl.syntax import TaskWrapper, switch
         eval_context = {name: TaskWrapper(task) for name, task in loader.tasks.items()}
+        eval_context["switch"] = switch
         
         try:
             flow = Flow(name=args.flow)
             for t in loader.tasks.values():
                 flow.add_task(t)
+            eval_context["flow"] = flow
             
             eval(flow_conf.graph, {}, eval_context)
             
             # 3. Reachability / Orphans
-            # Nodes with no deps and no dependents (except if single node flow)
             if len(flow.tasks) > 1:
                 for t in flow.tasks:
                     if not t.dependencies and not t.dependents:
                         warnings.append(f"Task '{t.name}' is orphaned (no dependencies or dependents).")
 
             # 4. Cycles
-            # Simple DFS for cycle detection
             visited = set()
             path = set()
+
             def visit(node):
                 if node in path:
-                    return True # Cycle
+                    return True
                 if node in visited:
                     return False
-                
                 visited.add(node)
                 path.add(node)
-                for dep in node.dependencies: # Check upstream
+                for dep in node.dependencies:
                     if visit(dep):
                         return True
                 path.remove(node)
@@ -255,29 +322,75 @@ def main():
             for t in flow.tasks:
                 sig = inspect.signature(t.func)
                 for name, param in sig.parameters.items():
-                    if name == 'ctx': continue
-                    # Check if input provided in task config or defaults
-                    # This is hard because inputs are resolved at runtime.
-                    # But we can check if 'inputs' mapping exists for it.
+                    if name == 'ctx': 
+                        continue
                     if name not in t.inputs and name not in flow_conf.defaults:
-                        # Warning: might be missing input
                         warnings.append(f"Task '{t.name}' argument '{name}' might be missing input (not in inputs or defaults).")
 
         except Exception as e:
             errors.append(f"Graph evaluation failed: {e}")
 
-        # Report
-        print("\n--- Check Report ---")
-        if not errors and not warnings:
-            print("✅ All checks passed!")
+        if args.dry_run:
+            from ..dsl.validator import FlowValidator
+            try:
+                validator = FlowValidator(flow)
+                dr_report = validator.validate()
+                warnings.extend(dr_report.warnings)
+                errors.extend(dr_report.errors)
+            except Exception as exc:
+                print(f"❌ Dry run internal error: {exc}")
+                import traceback
+                traceback.print_exc()
+                sys.exit(3)
+
+        status = "ok"
+        if errors:
+            status = "error"
+        elif warnings:
+            status = "warning"
+
+        report = {"status": status, "warnings": warnings, "errors": errors}
+
+        if args.json:
+            print(json.dumps(report, indent=2))
         else:
-            for w in warnings:
-                print(f"⚠️  {w}")
-            for e in errors:
-                print(f"❌ {e}")
-            
-            if errors:
+            print("\n--- Check Report ---")
+            print(f"Status: {status}")
+            if not errors and not warnings:
+                print("✅ All checks passed!")
+            else:
+                for w in warnings:
+                    print(f"⚠️  {w}")
+                for e in errors:
+                    print(f"❌ {e}")
+        
+        if errors:
+            sys.exit(2 if args.dry_run else 1)
+        return
+
+def _stream_logs(client, args):
+    seen_seq = -1
+    follow = args.follow
+    while True:
+        tail = args.tail if (args.tail and seen_seq == -1 and not follow) else None
+        data = client.get_run_logs(args.run_id, task=args.task, tail=tail)
+        logs = data.get("logs", [])
+        logs.sort(key=lambda entry: entry.get("seq", 0))
+        for entry in logs:
+            seq = entry.get("seq", 0)
+            if seq <= seen_seq:
+                continue
+            line = entry.get("text", "")
+            line = line.rstrip("\n")
+            print(f"[{entry.get('task', 'unknown')}][{entry.get('stream', '')}] {line}")
+            seen_seq = seq
+        status = data.get("run_status", "UNKNOWN")
+        if not follow or status in ("COMPLETED", "FAILED", "CANCELLED"):
+            if status == "FAILED" and not args.allow_failure:
                 sys.exit(1)
+            break
+        time.sleep(1)
+
 
 if __name__ == "__main__":
     main()

pyoco/client.py

@@ -1,6 +1,6 @@
 import httpx
 from typing import Dict, List, Optional, Any
-from .core.models import RunStatus, TaskState
+from .core.models import RunStatus, TaskState, RunContext
 
 class Client:
     def __init__(self, server_url: str, client_id: str = "cli"):
@@ -17,10 +17,19 @@ class Client:
         resp.raise_for_status()
         return resp.json()["run_id"]
 
-    def list_runs(self, status: Optional[str] = None) -> List[Dict]:
+    def list_runs(
+        self,
+        status: Optional[str] = None,
+        flow: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> List[Dict]:
         params = {}
         if status:
             params["status"] = status
+        if flow:
+            params["flow"] = flow
+        if limit:
+            params["limit"] = limit
         resp = self.client.get("/runs", params=params)
         resp.raise_for_status()
         return resp.json()
@@ -49,21 +58,32 @@ class Client:
             # print(f"Poll failed: {e}")
             return None
 
-    def heartbeat(self, run_id: str, task_states: Dict[str, TaskState], run_status: RunStatus) -> bool:
+    def heartbeat(self, run_ctx: RunContext) -> bool:
         """
         Sends heartbeat. Returns True if cancellation is requested.
         """
         try:
-            # Convert Enums to values
-            states_json = {k: v.value if hasattr(v, 'value') else v for k, v in task_states.items()}
-            status_value = run_status.value if hasattr(run_status, 'value') else run_status
-            
-            resp = self.client.post(f"/runs/{run_id}/heartbeat", json={
+            states_json = {k: v.value if hasattr(v, 'value') else v for k, v in run_ctx.tasks.items()}
+            status_value = run_ctx.status.value if hasattr(run_ctx.status, 'value') else run_ctx.status
+            payload = {
                 "task_states": states_json,
+                "task_records": run_ctx.serialize_task_records(),
+                "logs": run_ctx.drain_logs(),
                 "run_status": status_value
-            })
+            }
+            resp = self.client.post(f"/runs/{run_ctx.run_id}/heartbeat", json=payload)
             resp.raise_for_status()
             return resp.json().get("cancel_requested", False)
         except Exception as e:
             print(f"Heartbeat failed: {e}")
             return False
+
+    def get_run_logs(self, run_id: str, task: Optional[str] = None, tail: Optional[int] = None) -> Dict[str, Any]:
+        params = {}
+        if task:
+            params["task"] = task
+        if tail:
+            params["tail"] = tail
+        resp = self.client.get(f"/runs/{run_id}/logs", params=params)
+        resp.raise_for_status()
+        return resp.json()

pyoco/core/context.py

@@ -1,8 +1,46 @@
 import threading
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Sequence
 from dataclasses import dataclass, field
 from .models import RunContext
 
+
+@dataclass
+class LoopFrame:
+    name: str
+    type: str
+    index: Optional[int] = None
+    iteration: Optional[int] = None
+    count: Optional[int] = None
+    item: Any = None
+    condition: Optional[bool] = None
+    path: Optional[str] = None
+
+
+class LoopStack:
+    def __init__(self):
+        self._frames: List[LoopFrame] = []
+
+    def push(self, frame: LoopFrame) -> LoopFrame:
+        parent_path = self._frames[-1].path if self._frames else ""
+        segment = frame.name
+        if frame.index is not None:
+            segment = f"{segment}[{frame.index}]"
+        frame.path = f"{parent_path}.{segment}" if parent_path else segment
+        self._frames.append(frame)
+        return frame
+
+    def pop(self) -> LoopFrame:
+        if not self._frames:
+            raise RuntimeError("Loop stack underflow")
+        return self._frames.pop()
+
+    @property
+    def current(self) -> Optional[LoopFrame]:
+        return self._frames[-1] if self._frames else None
+
+    def snapshot(self) -> Sequence[LoopFrame]:
+        return tuple(self._frames)
+
 @dataclass
 class Context:
     """
@@ -14,11 +52,13 @@ class Context:
     artifacts: Dict[str, Any] = field(default_factory=dict)
     env: Dict[str, str] = field(default_factory=dict)
     artifact_dir: Optional[str] = None
+    _vars: Dict[str, Any] = field(default_factory=dict, repr=False)
     
     # Reference to the parent run context (v0.2.0+)
     run_context: Optional[RunContext] = None
     
     _lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
+    _loop_stack: LoopStack = field(default_factory=LoopStack, repr=False)
 
     @property
     def is_cancelled(self) -> bool:
@@ -27,6 +67,29 @@ class Context:
             return self.run_context.status in [RunStatus.CANCELLING, RunStatus.CANCELLED]
         return False
 
+    @property
+    def loop(self) -> Optional[LoopFrame]:
+        return self._loop_stack.current
+
+    @property
+    def loops(self) -> Sequence[LoopFrame]:
+        return self._loop_stack.snapshot()
+
+    def push_loop(self, frame: LoopFrame) -> LoopFrame:
+        return self._loop_stack.push(frame)
+
+    def pop_loop(self) -> LoopFrame:
+        return self._loop_stack.pop()
+
+    def set_var(self, name: str, value: Any):
+        self._vars[name] = value
+
+    def get_var(self, name: str, default=None):
+        return self._vars.get(name, default)
+
+    def clear_var(self, name: str):
+        self._vars.pop(name, None)
+
     def __post_init__(self):
         # Ensure artifact directory exists
         if self.artifact_dir is None:
@@ -124,3 +187,20 @@ class Context:
 
         return value
 
+    def expression_data(self) -> Dict[str, Any]:
+        data: Dict[str, Any] = {}
+        data.update(self._vars)
+        data["params"] = self.params
+        data["results"] = self.results
+        data["scratch"] = self.scratch
+        data["artifacts"] = self.artifacts
+        data["loop"] = self.loop
+        data["loops"] = list(self.loops)
+        return data
+
+    def env_data(self) -> Dict[str, str]:
+        import os
+
+        env_data = dict(os.environ)
+        env_data.update(self.env)
+        return env_data

pyoco/core/engine.py

@@ -1,9 +1,32 @@
 import time
+import io
+import sys
+import traceback
 from typing import Dict, Any, List, Set, Optional
+import contextlib
 from .models import Flow, Task, RunContext, TaskState, RunStatus
-from .context import Context
+from .context import Context, LoopFrame
+from .exceptions import UntilMaxIterationsExceeded
 from ..trace.backend import TraceBackend
 from ..trace.console import ConsoleTraceBackend
+from ..dsl.nodes import TaskNode, RepeatNode, ForEachNode, UntilNode, SwitchNode, DEFAULT_CASE_VALUE
+from ..dsl.expressions import Expression
+
+class TeeStream:
+    def __init__(self, original):
+        self.original = original
+        self.buffer = io.StringIO()
+
+    def write(self, data):
+        self.original.write(data)
+        self.buffer.write(data)
+        return len(data)
+
+    def flush(self):
+        self.original.flush()
+
+    def getvalue(self):
+        return self.buffer.getvalue()
 
 class Engine:
     """
@@ -44,16 +67,31 @@ class Engine:
             run_context = RunContext()
         
         run_ctx = run_context
+        run_ctx.flow_name = flow.name
+        run_ctx.params = params or {}
         
         # Initialize all tasks as PENDING
         for task in flow.tasks:
             run_ctx.tasks[task.name] = TaskState.PENDING
+            run_ctx.ensure_task_record(task.name)
             
         ctx = Context(params=params or {}, run_context=run_ctx)
         self.trace.on_flow_start(flow.name, run_id=run_ctx.run_id)
         
         # Register active run
         self.active_runs[run_ctx.run_id] = run_ctx
+
+        if flow.has_control_flow():
+            try:
+                program = flow.build_program()
+                self._execute_subflow(program, ctx)
+                run_ctx.status = RunStatus.COMPLETED
+            except Exception:
+                run_ctx.status = RunStatus.FAILED
+                run_ctx.end_time = time.time()
+                raise
+            run_ctx.end_time = time.time()
+            return ctx
         
         try:
             executed: Set[Task] = set()
@@ -264,12 +302,130 @@ class Engine:
         run_ctx.end_time = time.time()
         return ctx
 
+    def _execute_subflow(self, subflow, ctx: Context):
+        for node in subflow.steps:
+            self._execute_node(node, ctx)
+
+    def _execute_node(self, node, ctx: Context):
+        if isinstance(node, TaskNode):
+            self._execute_task(node.task, ctx)
+        elif isinstance(node, RepeatNode):
+            self._execute_repeat(node, ctx)
+        elif isinstance(node, ForEachNode):
+            self._execute_foreach(node, ctx)
+        elif isinstance(node, UntilNode):
+            self._execute_until(node, ctx)
+        elif isinstance(node, SwitchNode):
+            self._execute_switch(node, ctx)
+        else:
+            raise TypeError(f"Unknown node type: {type(node)}")
+
+    def _execute_repeat(self, node: RepeatNode, ctx: Context):
+        count_value = self._resolve_repeat_count(node.count, ctx)
+        for index in range(count_value):
+            frame = LoopFrame(name="repeat", type="repeat", index=index, iteration=index + 1, count=count_value)
+            ctx.push_loop(frame)
+            try:
+                self._execute_subflow(node.body, ctx)
+            finally:
+                ctx.pop_loop()
+
+    def _execute_foreach(self, node: ForEachNode, ctx: Context):
+        sequence = self._eval_expression(node.source, ctx)
+        if not isinstance(sequence, (list, tuple)):
+            raise TypeError("ForEach source must evaluate to a list or tuple.")
+
+        total = len(sequence)
+        label = node.alias or node.source.source
+        for index, item in enumerate(sequence):
+            frame = LoopFrame(
+                name=f"foreach:{label}",
+                type="foreach",
+                index=index,
+                iteration=index + 1,
+                count=total,
+                item=item,
+            )
+            ctx.push_loop(frame)
+            if node.alias:
+                ctx.set_var(node.alias, item)
+            try:
+                self._execute_subflow(node.body, ctx)
+            finally:
+                if node.alias:
+                    ctx.clear_var(node.alias)
+                ctx.pop_loop()
+
+    def _execute_until(self, node: UntilNode, ctx: Context):
+        max_iter = node.max_iter or 1000
+        iteration = 0
+        last_condition = None
+        while True:
+            iteration += 1
+            frame = LoopFrame(
+                name="until",
+                type="until",
+                index=iteration - 1,
+                iteration=iteration,
+                condition=last_condition,
+                count=max_iter,
+            )
+            ctx.push_loop(frame)
+            try:
+                self._execute_subflow(node.body, ctx)
+                condition_result = bool(self._eval_expression(node.condition, ctx))
+            finally:
+                ctx.pop_loop()
+
+            last_condition = condition_result
+            if condition_result:
+                break
+            if iteration >= max_iter:
+                raise UntilMaxIterationsExceeded(node.condition.source, max_iter)
+
+    def _execute_switch(self, node: SwitchNode, ctx: Context):
+        value = self._eval_expression(node.expression, ctx)
+        default_case = None
+        for case in node.cases:
+            if case.value == DEFAULT_CASE_VALUE:
+                if default_case is None:
+                    default_case = case
+                continue
+            if case.value == value:
+                self._execute_subflow(case.target, ctx)
+                return
+        if default_case:
+            self._execute_subflow(default_case.target, ctx)
+    def _resolve_repeat_count(self, count_value, ctx: Context) -> int:
+        if isinstance(count_value, Expression):
+            resolved = self._eval_expression(count_value, ctx)
+        else:
+            resolved = count_value
+        if not isinstance(resolved, int):
+            raise TypeError("Repeat count must evaluate to an integer.")
+        if resolved < 0:
+            raise ValueError("Repeat count cannot be negative.")
+        return resolved
+
+    def _eval_expression(self, expression, ctx: Context):
+        if isinstance(expression, Expression):
+            return expression.evaluate(ctx=ctx.expression_data(), env=ctx.env_data())
+        return expression
+
     def _execute_task(self, task: Task, ctx: Context):
         # Update state to RUNNING
         from .models import TaskState
+        run_ctx = ctx.run_context
         if ctx.run_context:
             ctx.run_context.tasks[task.name] = TaskState.RUNNING
-            
+            record = ctx.run_context.ensure_task_record(task.name)
+            record.state = TaskState.RUNNING
+            record.started_at = time.time()
+            record.error = None
+            record.traceback = None
+        else:
+            record = None
+
         self.trace.on_node_start(task.name)
         start_time = time.time()
         # Retry loop
@@ -301,8 +457,17 @@ class Engine:
                     elif param_name in ctx.results:
                         kwargs[param_name] = ctx.results[param_name]
                 
-                result = task.func(**kwargs)
+                if record:
+                    record.inputs = {k: v for k, v in kwargs.items() if k != "ctx"}
+
+                stdout_capture = TeeStream(sys.stdout)
+                stderr_capture = TeeStream(sys.stderr)
+                with contextlib.redirect_stdout(stdout_capture), contextlib.redirect_stderr(stderr_capture):
+                    result = task.func(**kwargs)
                 ctx.set_result(task.name, result)
+                if run_ctx:
+                    run_ctx.append_log(task.name, "stdout", stdout_capture.getvalue())
+                    run_ctx.append_log(task.name, "stderr", stderr_capture.getvalue())
                 
                 # Handle outputs saving
                 for target_path in task.outputs:
@@ -333,10 +498,24 @@ class Engine:
                 # Update state to SUCCEEDED
                 if ctx.run_context:
                     ctx.run_context.tasks[task.name] = TaskState.SUCCEEDED
+                    if record:
+                        record.state = TaskState.SUCCEEDED
+                        record.ended_at = time.time()
+                        record.duration_ms = (record.ended_at - record.started_at) * 1000
+                        record.output = result
                 
                 return # Success
                 
             except Exception as e:
+                if run_ctx:
+                    run_ctx.append_log(task.name, "stdout", stdout_capture.getvalue() if 'stdout_capture' in locals() else "")
+                    run_ctx.append_log(task.name, "stderr", stderr_capture.getvalue() if 'stderr_capture' in locals() else "")
+                if record:
+                    record.state = TaskState.FAILED
+                    record.ended_at = time.time()
+                    record.duration_ms = (record.ended_at - record.started_at) * 1000
+                    record.error = str(e)
+                    record.traceback = traceback.format_exc()
                 if retries_left > 0:
                     retries_left -= 1
                     # Log retry?