openbox-temporal-sdk-python 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openbox/__init__.py +107 -0
- openbox/activities.py +163 -0
- openbox/activity_interceptor.py +755 -0
- openbox/config.py +274 -0
- openbox/otel_setup.py +969 -0
- openbox/py.typed +0 -0
- openbox/span_processor.py +361 -0
- openbox/tracing.py +228 -0
- openbox/types.py +166 -0
- openbox/worker.py +257 -0
- openbox/workflow_interceptor.py +264 -0
- openbox_temporal_sdk_python-1.0.0.dist-info/METADATA +1214 -0
- openbox_temporal_sdk_python-1.0.0.dist-info/RECORD +15 -0
- openbox_temporal_sdk_python-1.0.0.dist-info/WHEEL +4 -0
- openbox_temporal_sdk_python-1.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,755 @@
|
|
|
1
|
+
# openbox/activity_interceptor.py
|
|
2
|
+
# Handles: ActivityStarted, ActivityCompleted (direct HTTP, WITH spans)
|
|
3
|
+
"""
|
|
4
|
+
Temporal activity interceptor for activity-boundary governance.
|
|
5
|
+
|
|
6
|
+
ActivityGovernanceInterceptor: Factory that creates ActivityInboundInterceptor
|
|
7
|
+
|
|
8
|
+
Captures 2 activity-level events:
|
|
9
|
+
4. ActivityStarted (execute_activity entry)
|
|
10
|
+
5. ActivityCompleted (execute_activity exit)
|
|
11
|
+
|
|
12
|
+
NOTE: Workflow events (WorkflowStarted, WorkflowCompleted, SignalReceived) are
|
|
13
|
+
handled by GovernanceInterceptor in workflow_interceptor.py
|
|
14
|
+
|
|
15
|
+
IMPORTANT: Activities CAN use datetime/time and make HTTP calls directly.
|
|
16
|
+
This is different from workflow interceptors which must maintain determinism.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from typing import Optional, Any, List
|
|
20
|
+
import dataclasses
|
|
21
|
+
from dataclasses import asdict, is_dataclass, fields
|
|
22
|
+
import time
|
|
23
|
+
import json
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _rfc3339_now() -> str:
|
|
27
|
+
"""Return current UTC time in RFC3339 format (UTC+0)."""
|
|
28
|
+
# Lazy import to avoid Temporal sandbox restrictions
|
|
29
|
+
from datetime import datetime, timezone
|
|
30
|
+
return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _deep_update_dataclass(obj: Any, data: dict, _logger=None) -> None:
|
|
34
|
+
"""
|
|
35
|
+
Recursively update a dataclass object's fields from a dict.
|
|
36
|
+
Preserves the original object types while updating values.
|
|
37
|
+
"""
|
|
38
|
+
if not is_dataclass(obj) or isinstance(obj, type):
|
|
39
|
+
return
|
|
40
|
+
|
|
41
|
+
for field in fields(obj):
|
|
42
|
+
if field.name not in data:
|
|
43
|
+
continue
|
|
44
|
+
|
|
45
|
+
new_value = data[field.name]
|
|
46
|
+
current_value = getattr(obj, field.name)
|
|
47
|
+
|
|
48
|
+
# If current field is a dataclass and new value is a dict, recurse
|
|
49
|
+
if is_dataclass(current_value) and not isinstance(current_value, type) and isinstance(new_value, dict):
|
|
50
|
+
_deep_update_dataclass(current_value, new_value, _logger)
|
|
51
|
+
# If current field is a list of dataclasses and new value is a list of dicts
|
|
52
|
+
elif isinstance(current_value, list) and isinstance(new_value, list):
|
|
53
|
+
for i, (curr_item, new_item) in enumerate(zip(current_value, new_value)):
|
|
54
|
+
if is_dataclass(curr_item) and not isinstance(curr_item, type) and isinstance(new_item, dict):
|
|
55
|
+
_deep_update_dataclass(curr_item, new_item, _logger)
|
|
56
|
+
elif i < len(current_value):
|
|
57
|
+
current_value[i] = new_item
|
|
58
|
+
else:
|
|
59
|
+
# Simple value - just update
|
|
60
|
+
if _logger:
|
|
61
|
+
_logger.info(f"_deep_update: Setting {type(obj).__name__}.{field.name} = {new_value}")
|
|
62
|
+
setattr(obj, field.name, new_value)
|
|
63
|
+
|
|
64
|
+
from temporalio import activity
|
|
65
|
+
from temporalio.worker import (
|
|
66
|
+
Interceptor,
|
|
67
|
+
ActivityInboundInterceptor,
|
|
68
|
+
ExecuteActivityInput,
|
|
69
|
+
)
|
|
70
|
+
from opentelemetry import trace
|
|
71
|
+
|
|
72
|
+
from .span_processor import WorkflowSpanProcessor
|
|
73
|
+
from .config import GovernanceConfig
|
|
74
|
+
from .types import WorkflowEventType, WorkflowSpanBuffer, GovernanceVerdictResponse, Verdict
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _serialize_value(value: Any) -> Any:
|
|
78
|
+
"""Convert a value to JSON-serializable format."""
|
|
79
|
+
if value is None:
|
|
80
|
+
return None
|
|
81
|
+
if isinstance(value, (str, int, float, bool, int)):
|
|
82
|
+
return value
|
|
83
|
+
if isinstance(value, bytes):
|
|
84
|
+
# Try to decode bytes as UTF-8, fallback to base64
|
|
85
|
+
try:
|
|
86
|
+
return value.decode('utf-8')
|
|
87
|
+
except Exception:
|
|
88
|
+
import base64
|
|
89
|
+
return base64.b64encode(value).decode('ascii')
|
|
90
|
+
if is_dataclass(value) and not isinstance(value, type):
|
|
91
|
+
return asdict(value)
|
|
92
|
+
if isinstance(value, (list, tuple)):
|
|
93
|
+
return [_serialize_value(v) for v in value]
|
|
94
|
+
if isinstance(value, dict):
|
|
95
|
+
return {k: _serialize_value(v) for k, v in value.items()}
|
|
96
|
+
# Handle Temporal Payload objects
|
|
97
|
+
if hasattr(value, 'data') and hasattr(value, 'metadata'):
|
|
98
|
+
# This is likely a Temporal Payload - try to decode it
|
|
99
|
+
try:
|
|
100
|
+
payload_data = value.data
|
|
101
|
+
if isinstance(payload_data, bytes):
|
|
102
|
+
return json.loads(payload_data.decode('utf-8'))
|
|
103
|
+
return str(payload_data)
|
|
104
|
+
except Exception:
|
|
105
|
+
return f"<Payload: {len(value.data) if hasattr(value, 'data') else '?'} bytes>"
|
|
106
|
+
# Try to convert to string for other types
|
|
107
|
+
try:
|
|
108
|
+
return json.loads(json.dumps(value, default=str))
|
|
109
|
+
except Exception:
|
|
110
|
+
return str(value)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class ActivityGovernanceInterceptor(Interceptor):
|
|
114
|
+
"""Factory for activity interceptor. Events sent directly (activities can do HTTP)."""
|
|
115
|
+
|
|
116
|
+
def __init__(
|
|
117
|
+
self,
|
|
118
|
+
api_url: str,
|
|
119
|
+
api_key: str,
|
|
120
|
+
span_processor: WorkflowSpanProcessor,
|
|
121
|
+
config: Optional[GovernanceConfig] = None,
|
|
122
|
+
):
|
|
123
|
+
self.api_url = api_url.rstrip("/")
|
|
124
|
+
self.api_key = api_key
|
|
125
|
+
self.span_processor = span_processor
|
|
126
|
+
self.config = config or GovernanceConfig()
|
|
127
|
+
|
|
128
|
+
def intercept_activity(
|
|
129
|
+
self, next_interceptor: ActivityInboundInterceptor
|
|
130
|
+
) -> ActivityInboundInterceptor:
|
|
131
|
+
return _ActivityInterceptor(
|
|
132
|
+
next_interceptor,
|
|
133
|
+
self.api_url,
|
|
134
|
+
self.api_key,
|
|
135
|
+
self.span_processor,
|
|
136
|
+
self.config,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class _ActivityInterceptor(ActivityInboundInterceptor):
|
|
141
|
+
def __init__(
|
|
142
|
+
self,
|
|
143
|
+
next_interceptor: ActivityInboundInterceptor,
|
|
144
|
+
api_url: str,
|
|
145
|
+
api_key: str,
|
|
146
|
+
span_processor: WorkflowSpanProcessor,
|
|
147
|
+
config: GovernanceConfig,
|
|
148
|
+
):
|
|
149
|
+
super().__init__(next_interceptor)
|
|
150
|
+
self._api_url = api_url
|
|
151
|
+
self._api_key = api_key
|
|
152
|
+
self._span_processor = span_processor
|
|
153
|
+
self._config = config
|
|
154
|
+
|
|
155
|
+
async def execute_activity(self, input: ExecuteActivityInput) -> Any:
|
|
156
|
+
info = activity.info()
|
|
157
|
+
start_time = time.time()
|
|
158
|
+
|
|
159
|
+
# Skip if configured (e.g., send_governance_event to avoid loops)
|
|
160
|
+
if info.activity_type in self._config.skip_activity_types:
|
|
161
|
+
return await self.next.execute_activity(input)
|
|
162
|
+
|
|
163
|
+
# Check if workflow has a pending "stop" verdict from signal governance
|
|
164
|
+
# This allows signal handlers to block subsequent activities
|
|
165
|
+
buffer = self._span_processor.get_buffer(info.workflow_id)
|
|
166
|
+
|
|
167
|
+
# If buffer exists but run_id doesn't match, it's from a previous workflow run - clear it
|
|
168
|
+
if buffer and buffer.run_id != info.workflow_run_id:
|
|
169
|
+
activity.logger.info(f"Clearing stale buffer for workflow {info.workflow_id} (old run_id={buffer.run_id}, new run_id={info.workflow_run_id})")
|
|
170
|
+
self._span_processor.unregister_workflow(info.workflow_id)
|
|
171
|
+
buffer = None
|
|
172
|
+
|
|
173
|
+
# Check for pending verdict (stored by workflow interceptor for SignalReceived stop)
|
|
174
|
+
# This is checked BEFORE buffer.verdict because buffer may not exist yet
|
|
175
|
+
pending_verdict = self._span_processor.get_verdict(info.workflow_id)
|
|
176
|
+
|
|
177
|
+
# Clear stale verdict from previous workflow run
|
|
178
|
+
if pending_verdict and pending_verdict.get("run_id") != info.workflow_run_id:
|
|
179
|
+
activity.logger.info(f"Clearing stale verdict for workflow {info.workflow_id} (old run_id={pending_verdict.get('run_id')}, new run_id={info.workflow_run_id})")
|
|
180
|
+
self._span_processor.clear_verdict(info.workflow_id)
|
|
181
|
+
pending_verdict = None
|
|
182
|
+
|
|
183
|
+
activity.logger.info(f"Checking verdict for workflow {info.workflow_id}: buffer={buffer is not None}, buffer.verdict={buffer.verdict if buffer else None}, pending_verdict={pending_verdict}")
|
|
184
|
+
|
|
185
|
+
if pending_verdict and pending_verdict.get("verdict") and Verdict.from_string(pending_verdict.get("verdict")).should_stop():
|
|
186
|
+
from temporalio.exceptions import ApplicationError
|
|
187
|
+
reason = pending_verdict.get("reason") or "Workflow blocked by governance"
|
|
188
|
+
activity.logger.info(f"Activity blocked by prior governance verdict (from signal): {reason}")
|
|
189
|
+
raise ApplicationError(
|
|
190
|
+
f"Governance blocked: {reason}",
|
|
191
|
+
type="GovernanceStop",
|
|
192
|
+
non_retryable=True,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
if buffer and buffer.verdict and buffer.verdict.should_stop():
|
|
196
|
+
from temporalio.exceptions import ApplicationError
|
|
197
|
+
reason = buffer.verdict_reason or "Workflow blocked by governance"
|
|
198
|
+
activity.logger.info(f"Activity blocked by prior governance verdict (from buffer): {reason}")
|
|
199
|
+
raise ApplicationError(
|
|
200
|
+
f"Governance blocked: {reason}",
|
|
201
|
+
type="GovernanceStop",
|
|
202
|
+
non_retryable=True,
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
# ═══ Check for pending approval on retry ═══
|
|
206
|
+
# If there's a pending approval, poll OpenBox Core for status
|
|
207
|
+
approval_granted = False
|
|
208
|
+
if self._config.hitl_enabled and info.activity_type not in self._config.skip_hitl_activity_types:
|
|
209
|
+
buffer = self._span_processor.get_buffer(info.workflow_id)
|
|
210
|
+
if buffer and buffer.pending_approval:
|
|
211
|
+
activity.logger.info(f"Polling approval status for workflow_id={info.workflow_id}, activity_id={info.activity_id}")
|
|
212
|
+
|
|
213
|
+
# Poll OpenBox Core for approval status
|
|
214
|
+
approval_response = await self._poll_approval_status(
|
|
215
|
+
workflow_id=info.workflow_id,
|
|
216
|
+
run_id=info.workflow_run_id,
|
|
217
|
+
activity_id=info.activity_id,
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
if approval_response:
|
|
221
|
+
from temporalio.exceptions import ApplicationError
|
|
222
|
+
|
|
223
|
+
activity.logger.info(f"Processing approval response: expired={approval_response.get('expired')}, verdict={approval_response.get('verdict')}")
|
|
224
|
+
|
|
225
|
+
# Check for approval expiration first
|
|
226
|
+
if approval_response.get("expired"):
|
|
227
|
+
buffer.pending_approval = False
|
|
228
|
+
activity.logger.info(f"TERMINATING WORKFLOW - Approval expired for activity {info.activity_type}")
|
|
229
|
+
raise ApplicationError(
|
|
230
|
+
f"Approval expired for activity {info.activity_type} (workflow_id={info.workflow_id}, run_id={info.workflow_run_id}, activity_id={info.activity_id})",
|
|
231
|
+
type="ApprovalExpired",
|
|
232
|
+
non_retryable=True,
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
verdict = Verdict.from_string(approval_response.get("verdict") or approval_response.get("action"))
|
|
236
|
+
|
|
237
|
+
if verdict == Verdict.ALLOW:
|
|
238
|
+
# Approved - clear pending and proceed
|
|
239
|
+
activity.logger.info(f"Approval granted for workflow_id={info.workflow_id}, activity_id={info.activity_id}")
|
|
240
|
+
buffer.pending_approval = False
|
|
241
|
+
approval_granted = True
|
|
242
|
+
elif verdict.should_stop():
|
|
243
|
+
# Rejected - clear pending and fail
|
|
244
|
+
buffer.pending_approval = False
|
|
245
|
+
reason = approval_response.get("reason", "Activity rejected")
|
|
246
|
+
raise ApplicationError(
|
|
247
|
+
f"Activity rejected: {reason}",
|
|
248
|
+
type="ApprovalRejected",
|
|
249
|
+
non_retryable=True,
|
|
250
|
+
)
|
|
251
|
+
else: # REQUIRE_APPROVAL or CONSTRAIN (still pending)
|
|
252
|
+
raise ApplicationError(
|
|
253
|
+
f"Awaiting approval for activity {info.activity_type}",
|
|
254
|
+
type="ApprovalPending",
|
|
255
|
+
non_retryable=False,
|
|
256
|
+
)
|
|
257
|
+
else:
|
|
258
|
+
# Failed to poll - raise retryable error
|
|
259
|
+
from temporalio.exceptions import ApplicationError
|
|
260
|
+
raise ApplicationError(
|
|
261
|
+
f"Failed to check approval status, retrying...",
|
|
262
|
+
type="ApprovalPending",
|
|
263
|
+
non_retryable=False,
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
# Ensure buffer is registered for this workflow (needed for span collection)
|
|
267
|
+
# The span processor's on_end() will add spans to this buffer
|
|
268
|
+
if not self._span_processor.get_buffer(info.workflow_id):
|
|
269
|
+
buffer = WorkflowSpanBuffer(
|
|
270
|
+
workflow_id=info.workflow_id,
|
|
271
|
+
run_id=info.workflow_run_id,
|
|
272
|
+
workflow_type=info.workflow_type,
|
|
273
|
+
task_queue=info.task_queue,
|
|
274
|
+
)
|
|
275
|
+
self._span_processor.register_workflow(info.workflow_id, buffer)
|
|
276
|
+
|
|
277
|
+
tracer = trace.get_tracer(__name__)
|
|
278
|
+
|
|
279
|
+
# Serialize activity input arguments
|
|
280
|
+
# input.args is a Sequence[Any] containing the activity arguments
|
|
281
|
+
# For class methods, self is already bound - args contains only the actual arguments
|
|
282
|
+
activity_input = []
|
|
283
|
+
try:
|
|
284
|
+
# Convert to list and serialize each argument
|
|
285
|
+
args_list = list(input.args) if input.args is not None else []
|
|
286
|
+
if args_list:
|
|
287
|
+
activity_input = _serialize_value(args_list)
|
|
288
|
+
# Debug: log what we're capturing
|
|
289
|
+
activity.logger.info(f"Activity {info.activity_type} input: {len(args_list)} args, types: {[type(a).__name__ for a in args_list]}")
|
|
290
|
+
except Exception as e:
|
|
291
|
+
activity.logger.warning(f"Failed to serialize activity input: {e}")
|
|
292
|
+
try:
|
|
293
|
+
activity_input = [str(arg) for arg in input.args] if input.args else []
|
|
294
|
+
except Exception:
|
|
295
|
+
activity_input = []
|
|
296
|
+
|
|
297
|
+
# Track governance verdict (may include redacted input)
|
|
298
|
+
governance_verdict: Optional[GovernanceVerdictResponse] = None
|
|
299
|
+
|
|
300
|
+
# Optional: Send ActivityStarted event (with input)
|
|
301
|
+
if self._config.send_activity_start_event:
|
|
302
|
+
governance_verdict = await self._send_activity_event(
|
|
303
|
+
info,
|
|
304
|
+
WorkflowEventType.ACTIVITY_STARTED.value,
|
|
305
|
+
activity_input=activity_input,
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
# If governance returned BLOCK/HALT, fail the activity before it runs
|
|
309
|
+
if governance_verdict and governance_verdict.verdict.should_stop():
|
|
310
|
+
from temporalio.exceptions import ApplicationError
|
|
311
|
+
raise ApplicationError(
|
|
312
|
+
f"Governance blocked: {governance_verdict.reason or 'No reason provided'}",
|
|
313
|
+
type="GovernanceStop",
|
|
314
|
+
non_retryable=True,
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
# Check guardrails validation_passed - if False, stop the activity
|
|
318
|
+
if (
|
|
319
|
+
governance_verdict
|
|
320
|
+
and governance_verdict.guardrails_result
|
|
321
|
+
and not governance_verdict.guardrails_result.validation_passed
|
|
322
|
+
):
|
|
323
|
+
from temporalio.exceptions import ApplicationError
|
|
324
|
+
reasons = governance_verdict.guardrails_result.get_reason_strings()
|
|
325
|
+
reason_str = "; ".join(reasons) if reasons else "Guardrails validation failed"
|
|
326
|
+
activity.logger.info(f"Guardrails validation failed: {reason_str}")
|
|
327
|
+
raise ApplicationError(
|
|
328
|
+
f"Guardrails validation failed: {reason_str}",
|
|
329
|
+
type="GuardrailsValidationFailed",
|
|
330
|
+
non_retryable=True,
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
# ═══ Handle REQUIRE_APPROVAL verdict (pre-execution) ═══
|
|
334
|
+
if (
|
|
335
|
+
self._config.hitl_enabled
|
|
336
|
+
and governance_verdict
|
|
337
|
+
and governance_verdict.verdict.requires_approval()
|
|
338
|
+
and info.activity_type not in self._config.skip_hitl_activity_types
|
|
339
|
+
):
|
|
340
|
+
from temporalio.exceptions import ApplicationError
|
|
341
|
+
|
|
342
|
+
# Mark approval as pending in span buffer
|
|
343
|
+
buffer = self._span_processor.get_buffer(info.workflow_id)
|
|
344
|
+
if buffer:
|
|
345
|
+
buffer.pending_approval = True
|
|
346
|
+
activity.logger.info(
|
|
347
|
+
f"Pending approval stored: workflow_id={info.workflow_id}, run_id={info.workflow_run_id}"
|
|
348
|
+
)
|
|
349
|
+
|
|
350
|
+
# Raise retryable error - Temporal will retry the activity
|
|
351
|
+
raise ApplicationError(
|
|
352
|
+
f"Approval required: {governance_verdict.reason or 'Activity requires human approval'}",
|
|
353
|
+
type="ApprovalPending",
|
|
354
|
+
non_retryable=False, # Retryable!
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
# Debug: Log governance verdict details
|
|
358
|
+
if governance_verdict:
|
|
359
|
+
activity.logger.info(
|
|
360
|
+
f"Governance verdict: verdict={governance_verdict.verdict.value}, "
|
|
361
|
+
f"has_guardrails_result={governance_verdict.guardrails_result is not None}"
|
|
362
|
+
)
|
|
363
|
+
if governance_verdict.guardrails_result:
|
|
364
|
+
activity.logger.info(
|
|
365
|
+
f"Guardrails result: input_type={governance_verdict.guardrails_result.input_type}, "
|
|
366
|
+
f"redacted_input_type={type(governance_verdict.guardrails_result.redacted_input).__name__}"
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
# Apply guardrails redaction if present
|
|
370
|
+
if (
|
|
371
|
+
governance_verdict
|
|
372
|
+
and governance_verdict.guardrails_result
|
|
373
|
+
and governance_verdict.guardrails_result.input_type == "activity_input"
|
|
374
|
+
):
|
|
375
|
+
redacted = governance_verdict.guardrails_result.redacted_input
|
|
376
|
+
activity.logger.info(f"Applying guardrails redaction to activity input")
|
|
377
|
+
activity.logger.info(f"Redacted input type: {type(redacted).__name__}, value preview: {str(redacted)[:200]}")
|
|
378
|
+
|
|
379
|
+
# Normalize redacted_input to a list (matching original args structure)
|
|
380
|
+
if isinstance(redacted, dict):
|
|
381
|
+
# API returned a single dict, wrap in list to match args structure
|
|
382
|
+
activity.logger.info("Wrapping dict in list")
|
|
383
|
+
redacted = [redacted]
|
|
384
|
+
|
|
385
|
+
if isinstance(redacted, list):
|
|
386
|
+
original_args = list(input.args) if input.args else []
|
|
387
|
+
activity.logger.info(f"Original args count: {len(original_args)}, redacted count: {len(redacted)}")
|
|
388
|
+
|
|
389
|
+
for i, redacted_item in enumerate(redacted):
|
|
390
|
+
activity.logger.info(f"Processing arg {i}: redacted_item type={type(redacted_item).__name__}")
|
|
391
|
+
if i < len(original_args) and isinstance(redacted_item, dict):
|
|
392
|
+
original_arg = original_args[i]
|
|
393
|
+
activity.logger.info(f"Original arg {i} type: {type(original_arg).__name__}, is_dataclass: {is_dataclass(original_arg)}")
|
|
394
|
+
# If original is a dataclass, update its fields in place (preserves types)
|
|
395
|
+
if is_dataclass(original_arg) and not isinstance(original_arg, type):
|
|
396
|
+
_deep_update_dataclass(original_arg, redacted_item, activity.logger)
|
|
397
|
+
activity.logger.info(f"Updated {type(original_arg).__name__} fields with redacted values")
|
|
398
|
+
# Verify the update
|
|
399
|
+
if hasattr(original_arg, 'prompt'):
|
|
400
|
+
activity.logger.info(f"After update, prompt = {getattr(original_arg, 'prompt', 'N/A')}")
|
|
401
|
+
else:
|
|
402
|
+
# Non-dataclass: replace directly
|
|
403
|
+
original_args[i] = redacted_item
|
|
404
|
+
activity.logger.info(f"Replaced arg {i} directly (non-dataclass)")
|
|
405
|
+
|
|
406
|
+
# Update activity_input for the completed event (shows redacted values)
|
|
407
|
+
activity_input = _serialize_value(original_args)
|
|
408
|
+
activity.logger.info(f"Updated activity_input for completed event")
|
|
409
|
+
else:
|
|
410
|
+
activity.logger.warning(
|
|
411
|
+
f"Unexpected redacted_input type: {type(redacted).__name__}, expected list or dict"
|
|
412
|
+
)
|
|
413
|
+
|
|
414
|
+
# Debug: Log the actual input that will be passed to activity
|
|
415
|
+
if input.args:
|
|
416
|
+
first_arg = input.args[0]
|
|
417
|
+
if hasattr(first_arg, 'prompt'):
|
|
418
|
+
activity.logger.info(f"BEFORE ACTIVITY EXECUTION - input.args[0].prompt = {first_arg.prompt}")
|
|
419
|
+
|
|
420
|
+
status = "completed"
|
|
421
|
+
error = None
|
|
422
|
+
activity_output = None
|
|
423
|
+
|
|
424
|
+
with tracer.start_as_current_span(
|
|
425
|
+
f"activity.{info.activity_type}",
|
|
426
|
+
attributes={
|
|
427
|
+
"temporal.workflow_id": info.workflow_id,
|
|
428
|
+
"temporal.activity_id": info.activity_id,
|
|
429
|
+
},
|
|
430
|
+
) as span:
|
|
431
|
+
# Register trace_id -> workflow_id + activity_id mapping so child spans
|
|
432
|
+
# (HTTP calls) can be associated with this activity even without attributes
|
|
433
|
+
self._span_processor.register_trace(
|
|
434
|
+
span.get_span_context().trace_id,
|
|
435
|
+
info.workflow_id,
|
|
436
|
+
info.activity_id,
|
|
437
|
+
)
|
|
438
|
+
|
|
439
|
+
try:
|
|
440
|
+
result = await self.next.execute_activity(input)
|
|
441
|
+
# Serialize activity output on success
|
|
442
|
+
activity_output = _serialize_value(result)
|
|
443
|
+
except Exception as e:
|
|
444
|
+
status = "failed"
|
|
445
|
+
error = {"type": type(e).__name__, "message": str(e)}
|
|
446
|
+
raise
|
|
447
|
+
finally:
|
|
448
|
+
end_time = time.time()
|
|
449
|
+
|
|
450
|
+
# Get activity spans from buffer
|
|
451
|
+
# Filter by activity_id (stored in span_data by span_processor)
|
|
452
|
+
buffer = self._span_processor.get_buffer(info.workflow_id)
|
|
453
|
+
spans = []
|
|
454
|
+
|
|
455
|
+
# Get pending body data from span processor
|
|
456
|
+
# This data is stored by patched httpx.send but not yet merged to spans
|
|
457
|
+
# (because the activity span hasn't ended yet)
|
|
458
|
+
activity_span_id = span.get_span_context().span_id
|
|
459
|
+
pending_body = self._span_processor.get_pending_body(activity_span_id)
|
|
460
|
+
|
|
461
|
+
if buffer:
|
|
462
|
+
for s in buffer.spans:
|
|
463
|
+
# Check if this span belongs to this activity
|
|
464
|
+
if (s.get("activity_id") == info.activity_id
|
|
465
|
+
or s.get("attributes", {}).get("temporal.activity_id") == info.activity_id):
|
|
466
|
+
spans.append(s)
|
|
467
|
+
|
|
468
|
+
# If we have pending body/header data, propagate to child HTTP spans
|
|
469
|
+
if pending_body:
|
|
470
|
+
for s in spans:
|
|
471
|
+
if "request_body" not in s and pending_body.get("request_body"):
|
|
472
|
+
s["request_body"] = pending_body["request_body"]
|
|
473
|
+
if "response_body" not in s and pending_body.get("response_body"):
|
|
474
|
+
s["response_body"] = pending_body["response_body"]
|
|
475
|
+
if "request_headers" not in s and pending_body.get("request_headers"):
|
|
476
|
+
s["request_headers"] = pending_body["request_headers"]
|
|
477
|
+
if "response_headers" not in s and pending_body.get("response_headers"):
|
|
478
|
+
s["response_headers"] = pending_body["response_headers"]
|
|
479
|
+
|
|
480
|
+
# Send ActivityCompleted event (with input and output)
|
|
481
|
+
# Always send for observability, but skip governance verdict check if already approved
|
|
482
|
+
completed_verdict = await self._send_activity_event(
|
|
483
|
+
info,
|
|
484
|
+
WorkflowEventType.ACTIVITY_COMPLETED.value,
|
|
485
|
+
status=status,
|
|
486
|
+
start_time=start_time,
|
|
487
|
+
end_time=end_time,
|
|
488
|
+
duration_ms=(end_time - start_time) * 1000,
|
|
489
|
+
span_count=len(spans),
|
|
490
|
+
spans=spans,
|
|
491
|
+
activity_input=activity_input,
|
|
492
|
+
activity_output=activity_output,
|
|
493
|
+
error=error,
|
|
494
|
+
)
|
|
495
|
+
|
|
496
|
+
# If governance returned BLOCK/HALT, fail the activity after it completes
|
|
497
|
+
if completed_verdict and completed_verdict.verdict.should_stop():
|
|
498
|
+
from temporalio.exceptions import ApplicationError
|
|
499
|
+
raise ApplicationError(
|
|
500
|
+
f"Governance blocked: {completed_verdict.reason or 'No reason provided'}",
|
|
501
|
+
type="GovernanceStop",
|
|
502
|
+
non_retryable=True,
|
|
503
|
+
)
|
|
504
|
+
|
|
505
|
+
# Check guardrails validation_passed for output - if False, stop
|
|
506
|
+
if (
|
|
507
|
+
completed_verdict
|
|
508
|
+
and completed_verdict.guardrails_result
|
|
509
|
+
and not completed_verdict.guardrails_result.validation_passed
|
|
510
|
+
):
|
|
511
|
+
from temporalio.exceptions import ApplicationError
|
|
512
|
+
reasons = completed_verdict.guardrails_result.get_reason_strings()
|
|
513
|
+
reason_str = "; ".join(reasons) if reasons else "Guardrails output validation failed"
|
|
514
|
+
activity.logger.info(f"Guardrails output validation failed: {reason_str}")
|
|
515
|
+
raise ApplicationError(
|
|
516
|
+
f"Guardrails validation failed: {reason_str}",
|
|
517
|
+
type="GuardrailsValidationFailed",
|
|
518
|
+
non_retryable=True,
|
|
519
|
+
)
|
|
520
|
+
|
|
521
|
+
# ═══ Handle REQUIRE_APPROVAL verdict (post-execution) ═══
|
|
522
|
+
if (
|
|
523
|
+
self._config.hitl_enabled
|
|
524
|
+
and completed_verdict
|
|
525
|
+
and completed_verdict.verdict.requires_approval()
|
|
526
|
+
and info.activity_type not in self._config.skip_hitl_activity_types
|
|
527
|
+
):
|
|
528
|
+
from temporalio.exceptions import ApplicationError
|
|
529
|
+
|
|
530
|
+
# Mark approval as pending in span buffer
|
|
531
|
+
buffer = self._span_processor.get_buffer(info.workflow_id)
|
|
532
|
+
if buffer:
|
|
533
|
+
buffer.pending_approval = True
|
|
534
|
+
activity.logger.info(
|
|
535
|
+
f"Pending approval stored (post-execution): workflow_id={info.workflow_id}, run_id={info.workflow_run_id}"
|
|
536
|
+
)
|
|
537
|
+
|
|
538
|
+
# Raise retryable error - Temporal will retry the activity
|
|
539
|
+
raise ApplicationError(
|
|
540
|
+
f"Approval required for output: {completed_verdict.reason or 'Activity output requires human approval'}",
|
|
541
|
+
type="ApprovalPending",
|
|
542
|
+
non_retryable=False, # Retryable!
|
|
543
|
+
)
|
|
544
|
+
|
|
545
|
+
# Apply output redaction if governance returned guardrails_result for activity_output
|
|
546
|
+
if (
|
|
547
|
+
completed_verdict
|
|
548
|
+
and completed_verdict.guardrails_result
|
|
549
|
+
and completed_verdict.guardrails_result.input_type == "activity_output"
|
|
550
|
+
):
|
|
551
|
+
redacted_output = completed_verdict.guardrails_result.redacted_input
|
|
552
|
+
activity.logger.info(f"Applying guardrails redaction to activity output")
|
|
553
|
+
|
|
554
|
+
if redacted_output is not None:
|
|
555
|
+
# If result is a dataclass, update fields in place
|
|
556
|
+
if is_dataclass(result) and not isinstance(result, type) and isinstance(redacted_output, dict):
|
|
557
|
+
_deep_update_dataclass(result, redacted_output)
|
|
558
|
+
activity.logger.info(f"Updated {type(result).__name__} output fields with redacted values")
|
|
559
|
+
else:
|
|
560
|
+
# Replace result directly (dict, primitive, etc.)
|
|
561
|
+
result = redacted_output
|
|
562
|
+
activity.logger.info(f"Replaced activity output with redacted value")
|
|
563
|
+
|
|
564
|
+
return result
|
|
565
|
+
|
|
566
|
+
async def _send_activity_event(self, info, event_type: str, **extra) -> Optional[GovernanceVerdictResponse]:
|
|
567
|
+
"""Send activity event directly via HTTP (allowed in activity context).
|
|
568
|
+
|
|
569
|
+
Returns:
|
|
570
|
+
GovernanceVerdictResponse with action, reason, and optional guardrails_result
|
|
571
|
+
None if API fails and policy is fail_open
|
|
572
|
+
|
|
573
|
+
NOTE: httpx and datetime are imported lazily here to avoid loading them
|
|
574
|
+
at module level. Module-level httpx import triggers Temporal sandbox
|
|
575
|
+
restrictions because httpx uses os.stat internally.
|
|
576
|
+
"""
|
|
577
|
+
# Lazy imports - only loaded when activity interceptor actually runs
|
|
578
|
+
# (in activity context, not workflow context)
|
|
579
|
+
import httpx
|
|
580
|
+
|
|
581
|
+
# Serialize extra fields to ensure no Payload objects slip through
|
|
582
|
+
serialized_extra = {}
|
|
583
|
+
for key, value in extra.items():
|
|
584
|
+
try:
|
|
585
|
+
serialized_extra[key] = _serialize_value(value)
|
|
586
|
+
except Exception as e:
|
|
587
|
+
activity.logger.warning(f"Failed to serialize {key}: {e}")
|
|
588
|
+
serialized_extra[key] = str(value) if value is not None else None
|
|
589
|
+
|
|
590
|
+
payload = {
|
|
591
|
+
"source": "workflow-telemetry",
|
|
592
|
+
"event_type": event_type,
|
|
593
|
+
"workflow_id": info.workflow_id,
|
|
594
|
+
"run_id": info.workflow_run_id,
|
|
595
|
+
"workflow_type": info.workflow_type,
|
|
596
|
+
"activity_id": info.activity_id,
|
|
597
|
+
"activity_type": info.activity_type,
|
|
598
|
+
"task_queue": info.task_queue,
|
|
599
|
+
"attempt": info.attempt,
|
|
600
|
+
"timestamp": _rfc3339_now(),
|
|
601
|
+
**serialized_extra,
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
# Final safety check - ensure payload is JSON serializable
|
|
605
|
+
try:
|
|
606
|
+
json.dumps(payload)
|
|
607
|
+
except TypeError as e:
|
|
608
|
+
activity.logger.warning(f"Payload not JSON serializable, cleaning: {e}")
|
|
609
|
+
# Fallback: convert entire payload using default=str
|
|
610
|
+
payload = json.loads(json.dumps(payload, default=str))
|
|
611
|
+
|
|
612
|
+
try:
|
|
613
|
+
async with httpx.AsyncClient(timeout=self._config.api_timeout) as client:
|
|
614
|
+
response = await client.post(
|
|
615
|
+
f"{self._api_url}/api/v1/governance/evaluate",
|
|
616
|
+
json=payload,
|
|
617
|
+
headers={"Authorization": f"Bearer {self._api_key}"},
|
|
618
|
+
)
|
|
619
|
+
# Check for HTTP errors
|
|
620
|
+
if response.status_code >= 400:
|
|
621
|
+
error_msg = f"HTTP {response.status_code}"
|
|
622
|
+
activity.logger.warning(f"Governance API error: {error_msg}")
|
|
623
|
+
|
|
624
|
+
# Respect on_api_error policy
|
|
625
|
+
if self._config.on_api_error == "fail_closed":
|
|
626
|
+
return GovernanceVerdictResponse(
|
|
627
|
+
verdict=Verdict.HALT,
|
|
628
|
+
reason=f"Governance API error: {error_msg}",
|
|
629
|
+
)
|
|
630
|
+
|
|
631
|
+
return None # Fail-open: allow activity to proceed
|
|
632
|
+
|
|
633
|
+
# Parse governance response
|
|
634
|
+
if response.status_code == 200:
|
|
635
|
+
try:
|
|
636
|
+
data = response.json()
|
|
637
|
+
# Debug: Log raw response
|
|
638
|
+
activity.logger.info(f"Raw governance response: {data}")
|
|
639
|
+
activity.logger.info(f"guardrails_result in response: {'guardrails_result' in data}, value: {data.get('guardrails_result')}")
|
|
640
|
+
|
|
641
|
+
verdict = GovernanceVerdictResponse.from_dict(data)
|
|
642
|
+
|
|
643
|
+
if verdict.verdict.should_stop():
|
|
644
|
+
activity.logger.info(
|
|
645
|
+
f"Governance blocked activity: {verdict.reason} (policy: {verdict.policy_id})"
|
|
646
|
+
)
|
|
647
|
+
|
|
648
|
+
if verdict.guardrails_result:
|
|
649
|
+
activity.logger.info(
|
|
650
|
+
f"Guardrails redaction applied: input_type={verdict.guardrails_result.input_type}"
|
|
651
|
+
)
|
|
652
|
+
|
|
653
|
+
return verdict
|
|
654
|
+
except Exception as e:
|
|
655
|
+
activity.logger.warning(f"Failed to parse governance response: {e}")
|
|
656
|
+
|
|
657
|
+
return None # Allow activity to proceed
|
|
658
|
+
|
|
659
|
+
except Exception as e:
|
|
660
|
+
error_msg = str(e) if str(e) else repr(e)
|
|
661
|
+
activity.logger.warning(f"Governance API error ({type(e).__name__}): {error_msg}")
|
|
662
|
+
|
|
663
|
+
# Respect on_api_error policy
|
|
664
|
+
if self._config.on_api_error == "fail_closed":
|
|
665
|
+
return GovernanceVerdictResponse(
|
|
666
|
+
verdict=Verdict.HALT,
|
|
667
|
+
reason=f"Governance API error: {error_msg}",
|
|
668
|
+
)
|
|
669
|
+
|
|
670
|
+
return None # Fail-open: allow activity to proceed
|
|
671
|
+
|
|
672
|
+
async def _poll_approval_status(
|
|
673
|
+
self,
|
|
674
|
+
workflow_id: str,
|
|
675
|
+
run_id: str,
|
|
676
|
+
activity_id: str,
|
|
677
|
+
) -> Optional[dict]:
|
|
678
|
+
"""Poll OpenBox Core for approval status.
|
|
679
|
+
|
|
680
|
+
Args:
|
|
681
|
+
workflow_id: The workflow ID
|
|
682
|
+
run_id: The workflow run ID
|
|
683
|
+
activity_id: The activity ID
|
|
684
|
+
|
|
685
|
+
Returns:
|
|
686
|
+
Dict with verdict/action and optional reason. None if API call fails.
|
|
687
|
+
If approval_expiration_time has passed, returns a halt verdict with expired=True.
|
|
688
|
+
"""
|
|
689
|
+
import httpx
|
|
690
|
+
from datetime import datetime, timezone
|
|
691
|
+
|
|
692
|
+
payload = {
|
|
693
|
+
"workflow_id": workflow_id,
|
|
694
|
+
"run_id": run_id,
|
|
695
|
+
"activity_id": activity_id,
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
try:
|
|
699
|
+
async with httpx.AsyncClient(timeout=self._config.api_timeout) as client:
|
|
700
|
+
response = await client.post(
|
|
701
|
+
f"{self._api_url}/api/v1/governance/approval",
|
|
702
|
+
json=payload,
|
|
703
|
+
headers={"Authorization": f"Bearer {self._api_key}"},
|
|
704
|
+
)
|
|
705
|
+
|
|
706
|
+
if response.status_code == 200:
|
|
707
|
+
data = response.json()
|
|
708
|
+
activity.logger.info(f"Approval status response: {data}")
|
|
709
|
+
|
|
710
|
+
# Check for approval expiration (skip if field is missing, null, or empty)
|
|
711
|
+
expiration_time_str = data.get("approval_expiration_time")
|
|
712
|
+
activity.logger.info(f"Checking expiration: approval_expiration_time={expiration_time_str}")
|
|
713
|
+
|
|
714
|
+
if expiration_time_str:
|
|
715
|
+
try:
|
|
716
|
+
# Parse timestamp - handle multiple formats:
|
|
717
|
+
# - "2026-01-11T17:43:39Z" (ISO with Z)
|
|
718
|
+
# - "2026-01-11T17:43:39+00:00" (ISO with offset)
|
|
719
|
+
# - "2026-01-11 17:43:39" (space-separated from DB)
|
|
720
|
+
normalized = expiration_time_str.replace('Z', '+00:00').replace(' ', 'T')
|
|
721
|
+
expiration_time = datetime.fromisoformat(normalized)
|
|
722
|
+
|
|
723
|
+
# If no timezone info (naive), assume UTC
|
|
724
|
+
if expiration_time.tzinfo is None:
|
|
725
|
+
expiration_time = expiration_time.replace(tzinfo=timezone.utc)
|
|
726
|
+
|
|
727
|
+
current_time = datetime.now(timezone.utc)
|
|
728
|
+
|
|
729
|
+
activity.logger.info(
|
|
730
|
+
f"Expiration check: expiration_time={expiration_time.isoformat()}, "
|
|
731
|
+
f"current_time={current_time.isoformat()}, "
|
|
732
|
+
f"is_expired={current_time > expiration_time}"
|
|
733
|
+
)
|
|
734
|
+
|
|
735
|
+
if current_time > expiration_time:
|
|
736
|
+
activity.logger.info(
|
|
737
|
+
f"Approval EXPIRED - terminating workflow"
|
|
738
|
+
)
|
|
739
|
+
# Mark as expired - caller will terminate like HALT verdict
|
|
740
|
+
data["expired"] = True
|
|
741
|
+
return data
|
|
742
|
+
except (ValueError, TypeError) as e:
|
|
743
|
+
activity.logger.warning(
|
|
744
|
+
f"Failed to parse approval_expiration_time '{expiration_time_str}': {e}"
|
|
745
|
+
)
|
|
746
|
+
# Continue with normal processing if parsing fails
|
|
747
|
+
|
|
748
|
+
return data
|
|
749
|
+
else:
|
|
750
|
+
activity.logger.warning(f"Failed to get approval status: HTTP {response.status_code}")
|
|
751
|
+
return None
|
|
752
|
+
|
|
753
|
+
except Exception as e:
|
|
754
|
+
activity.logger.warning(f"Failed to poll approval status: {e}")
|
|
755
|
+
return None
|