uipath-core 0.1.8__tar.gz → 0.1.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {uipath_core-0.1.8 → uipath_core-0.1.10}/PKG-INFO +1 -1
- {uipath_core-0.1.8 → uipath_core-0.1.10}/pyproject.toml +1 -1
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/guardrails/_deterministic_guardrails_service.py +25 -9
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/guardrails/_evaluators.py +74 -37
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/guardrails/guardrails.py +15 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/tracing/trace_manager.py +12 -1
- {uipath_core-0.1.8 → uipath_core-0.1.10}/tests/guardrails/test_deterministic_guardrails_service.py +145 -148
- {uipath_core-0.1.8 → uipath_core-0.1.10}/uv.lock +1 -1
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.cursorrules +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.editorconfig +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.gitattributes +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.github/workflows/cd.yml +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.github/workflows/ci.yml +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.github/workflows/commitlint.yml +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.github/workflows/lint.yml +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.github/workflows/publish-dev.yml +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.github/workflows/test.yml +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.gitignore +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.pre-commit-config.yaml +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.python-version +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.vscode/extensions.json +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.vscode/launch.json +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/.vscode/settings.json +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/CONTRIBUTING.md +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/LICENSE +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/README.md +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/justfile +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/__init__.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/chat/__init__.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/chat/async_stream.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/chat/citation.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/chat/content.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/chat/conversation.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/chat/error.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/chat/event.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/chat/exchange.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/chat/interrupt.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/chat/message.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/chat/meta.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/chat/tool.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/errors/__init__.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/errors/errors.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/guardrails/__init__.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/py.typed +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/tracing/__init__.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/tracing/_utils.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/tracing/decorators.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/tracing/exporters.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/tracing/processors.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/src/uipath/core/tracing/span_utils.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/tests/__init__.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/tests/conftest.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/tests/tracing/test_external_integration.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/tests/tracing/test_serialization.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/tests/tracing/test_span_nesting.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/tests/tracing/test_span_registry.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/tests/tracing/test_trace_manager.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/tests/tracing/test_traced.py +0 -0
- {uipath_core-0.1.8 → uipath_core-0.1.10}/tests/tracing/test_tracing_utils.py +0 -0
|
@@ -40,7 +40,7 @@ class DeterministicGuardrailsService(BaseModel):
|
|
|
40
40
|
if has_output_rule:
|
|
41
41
|
return GuardrailValidationResult(
|
|
42
42
|
result=GuardrailValidationResultType.PASSED,
|
|
43
|
-
reason="
|
|
43
|
+
reason="No rules to apply for input data.",
|
|
44
44
|
)
|
|
45
45
|
return self._evaluate_deterministic_guardrail(
|
|
46
46
|
input_data=input_data,
|
|
@@ -66,7 +66,7 @@ class DeterministicGuardrailsService(BaseModel):
|
|
|
66
66
|
if not has_output_rule:
|
|
67
67
|
return GuardrailValidationResult(
|
|
68
68
|
result=GuardrailValidationResultType.PASSED,
|
|
69
|
-
reason="
|
|
69
|
+
reason="No rules to apply for output data.",
|
|
70
70
|
)
|
|
71
71
|
|
|
72
72
|
return self._evaluate_deterministic_guardrail(
|
|
@@ -117,7 +117,12 @@ class DeterministicGuardrailsService(BaseModel):
|
|
|
117
117
|
output_data: dict[str, Any],
|
|
118
118
|
guardrail: DeterministicGuardrail,
|
|
119
119
|
) -> GuardrailValidationResult:
|
|
120
|
-
"""Evaluate deterministic guardrail rules against input and output data.
|
|
120
|
+
"""Evaluate deterministic guardrail rules against input and output data.
|
|
121
|
+
|
|
122
|
+
Validation fails only if ALL guardrail rules are violated.
|
|
123
|
+
"""
|
|
124
|
+
validated_conditions: list[str] = []
|
|
125
|
+
|
|
121
126
|
for rule in guardrail.rules:
|
|
122
127
|
if isinstance(rule, WordRule):
|
|
123
128
|
passed, reason = evaluate_word_rule(rule, input_data, output_data)
|
|
@@ -132,14 +137,25 @@ class DeterministicGuardrailsService(BaseModel):
|
|
|
132
137
|
result=GuardrailValidationResultType.VALIDATION_FAILED,
|
|
133
138
|
reason=f"Unknown rule type: {type(rule)}",
|
|
134
139
|
)
|
|
135
|
-
|
|
136
|
-
if
|
|
140
|
+
validated_conditions.append(reason)
|
|
141
|
+
if passed:
|
|
137
142
|
return GuardrailValidationResult(
|
|
138
|
-
result=GuardrailValidationResultType.
|
|
139
|
-
reason=reason
|
|
143
|
+
result=GuardrailValidationResultType.PASSED,
|
|
144
|
+
reason=reason,
|
|
140
145
|
)
|
|
141
146
|
|
|
147
|
+
has_always_rule = any(
|
|
148
|
+
condition == "Always rule enforced" for condition in validated_conditions
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
validated_conditions_str = ", ".join(validated_conditions)
|
|
152
|
+
final_reason = (
|
|
153
|
+
"Always rule enforced"
|
|
154
|
+
if has_always_rule
|
|
155
|
+
else f"Data matched all guardrail conditions: [{validated_conditions_str}]"
|
|
156
|
+
)
|
|
157
|
+
|
|
142
158
|
return GuardrailValidationResult(
|
|
143
|
-
result=GuardrailValidationResultType.
|
|
144
|
-
reason=
|
|
159
|
+
result=GuardrailValidationResultType.VALIDATION_FAILED,
|
|
160
|
+
reason=final_reason,
|
|
145
161
|
)
|
|
@@ -159,24 +159,44 @@ def get_fields_from_selector(
|
|
|
159
159
|
return fields
|
|
160
160
|
|
|
161
161
|
|
|
162
|
-
def
|
|
162
|
+
def format_guardrail_passed_validation_result_message(
|
|
163
163
|
field_ref: FieldReference,
|
|
164
|
-
operator: str,
|
|
165
|
-
|
|
164
|
+
operator: str | None,
|
|
165
|
+
rule_description: str | None,
|
|
166
166
|
) -> str:
|
|
167
|
-
"""Format a guardrail
|
|
167
|
+
"""Format a guardrail validation result message following the standard pattern."""
|
|
168
168
|
source = "Input" if field_ref.source == FieldSource.INPUT else "Output"
|
|
169
|
-
|
|
170
|
-
if
|
|
171
|
-
|
|
172
|
-
|
|
169
|
+
|
|
170
|
+
if rule_description:
|
|
171
|
+
return (
|
|
172
|
+
f"{source} data didn't match the guardrail condition for field "
|
|
173
|
+
f"[{field_ref.path}]: {rule_description}"
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
return (
|
|
177
|
+
f"{source} data didn't match the guardrail condition: "
|
|
178
|
+
f"[{field_ref.path}] comparing function [{operator}]"
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def get_validated_conditions_description(
|
|
183
|
+
field_path: str,
|
|
184
|
+
operator: str | None,
|
|
185
|
+
rule_description: str | None,
|
|
186
|
+
) -> str:
|
|
187
|
+
if rule_description:
|
|
188
|
+
return rule_description
|
|
189
|
+
|
|
190
|
+
return f"[{field_path}] comparing function [{operator}]"
|
|
173
191
|
|
|
174
192
|
|
|
175
193
|
def evaluate_word_rule(
|
|
176
194
|
rule: WordRule, input_data: dict[str, Any], output_data: dict[str, Any]
|
|
177
|
-
) -> tuple[bool, str
|
|
195
|
+
) -> tuple[bool, str]:
|
|
178
196
|
"""Evaluate a word rule against input and output data."""
|
|
179
197
|
fields = get_fields_from_selector(rule.field_selector, input_data, output_data)
|
|
198
|
+
operator = _humanize_guardrail_func(rule.detects_violation) or "violation check"
|
|
199
|
+
field_paths = ", ".join({field_ref.path for _, field_ref in fields})
|
|
180
200
|
|
|
181
201
|
for field_value, field_ref in fields:
|
|
182
202
|
if field_value is None:
|
|
@@ -197,22 +217,28 @@ def evaluate_word_rule(
|
|
|
197
217
|
# If function raises an exception, treat as failure
|
|
198
218
|
violation_detected = True
|
|
199
219
|
|
|
200
|
-
if violation_detected:
|
|
201
|
-
|
|
202
|
-
|
|
220
|
+
if not violation_detected:
|
|
221
|
+
reason = format_guardrail_passed_validation_result_message(
|
|
222
|
+
field_ref=field_ref,
|
|
223
|
+
operator=operator,
|
|
224
|
+
rule_description=rule.rule_description,
|
|
203
225
|
)
|
|
204
|
-
|
|
205
|
-
return False, reason
|
|
226
|
+
return True, reason
|
|
206
227
|
|
|
207
|
-
return
|
|
228
|
+
return False, get_validated_conditions_description(
|
|
229
|
+
field_path=field_paths,
|
|
230
|
+
operator=operator,
|
|
231
|
+
rule_description=rule.rule_description,
|
|
232
|
+
)
|
|
208
233
|
|
|
209
234
|
|
|
210
235
|
def evaluate_number_rule(
|
|
211
236
|
rule: NumberRule, input_data: dict[str, Any], output_data: dict[str, Any]
|
|
212
|
-
) -> tuple[bool, str
|
|
237
|
+
) -> tuple[bool, str]:
|
|
213
238
|
"""Evaluate a number rule against input and output data."""
|
|
214
239
|
fields = get_fields_from_selector(rule.field_selector, input_data, output_data)
|
|
215
|
-
|
|
240
|
+
operator = _humanize_guardrail_func(rule.detects_violation) or "violation check"
|
|
241
|
+
field_paths = ", ".join({field_ref.path for _, field_ref in fields})
|
|
216
242
|
for field_value, field_ref in fields:
|
|
217
243
|
if field_value is None:
|
|
218
244
|
continue
|
|
@@ -233,24 +259,30 @@ def evaluate_number_rule(
|
|
|
233
259
|
# If function raises an exception, treat as failure
|
|
234
260
|
violation_detected = True
|
|
235
261
|
|
|
236
|
-
if violation_detected:
|
|
237
|
-
|
|
238
|
-
|
|
262
|
+
if not violation_detected:
|
|
263
|
+
reason = format_guardrail_passed_validation_result_message(
|
|
264
|
+
field_ref=field_ref,
|
|
265
|
+
operator=operator,
|
|
266
|
+
rule_description=rule.rule_description,
|
|
239
267
|
)
|
|
240
|
-
|
|
241
|
-
return False, reason
|
|
268
|
+
return True, reason
|
|
242
269
|
|
|
243
|
-
return
|
|
270
|
+
return False, get_validated_conditions_description(
|
|
271
|
+
field_path=field_paths,
|
|
272
|
+
operator=operator,
|
|
273
|
+
rule_description=rule.rule_description,
|
|
274
|
+
)
|
|
244
275
|
|
|
245
276
|
|
|
246
277
|
def evaluate_boolean_rule(
|
|
247
278
|
rule: BooleanRule,
|
|
248
279
|
input_data: dict[str, Any],
|
|
249
280
|
output_data: dict[str, Any],
|
|
250
|
-
) -> tuple[bool, str
|
|
281
|
+
) -> tuple[bool, str]:
|
|
251
282
|
"""Evaluate a boolean rule against input and output data."""
|
|
252
283
|
fields = get_fields_from_selector(rule.field_selector, input_data, output_data)
|
|
253
|
-
|
|
284
|
+
operator = _humanize_guardrail_func(rule.detects_violation) or "violation check"
|
|
285
|
+
field_paths = ", ".join({field_ref.path for _, field_ref in fields})
|
|
254
286
|
for field_value, field_ref in fields:
|
|
255
287
|
if field_value is None:
|
|
256
288
|
continue
|
|
@@ -270,20 +302,25 @@ def evaluate_boolean_rule(
|
|
|
270
302
|
# If function raises an exception, treat as failure
|
|
271
303
|
violation_detected = True
|
|
272
304
|
|
|
273
|
-
if violation_detected:
|
|
274
|
-
|
|
275
|
-
|
|
305
|
+
if not violation_detected:
|
|
306
|
+
reason = format_guardrail_passed_validation_result_message(
|
|
307
|
+
field_ref=field_ref,
|
|
308
|
+
operator=operator,
|
|
309
|
+
rule_description=rule.rule_description,
|
|
276
310
|
)
|
|
277
|
-
|
|
278
|
-
return False, reason
|
|
311
|
+
return True, reason
|
|
279
312
|
|
|
280
|
-
return
|
|
313
|
+
return False, get_validated_conditions_description(
|
|
314
|
+
field_path=field_paths,
|
|
315
|
+
operator=operator,
|
|
316
|
+
rule_description=rule.rule_description,
|
|
317
|
+
)
|
|
281
318
|
|
|
282
319
|
|
|
283
320
|
def evaluate_universal_rule(
|
|
284
321
|
rule: UniversalRule,
|
|
285
322
|
output_data: dict[str, Any],
|
|
286
|
-
) -> tuple[bool, str
|
|
323
|
+
) -> tuple[bool, str]:
|
|
287
324
|
"""Evaluate a universal rule against input and output data.
|
|
288
325
|
|
|
289
326
|
Universal rules trigger based on the apply_to scope and execution phase:
|
|
@@ -302,18 +339,18 @@ def evaluate_universal_rule(
|
|
|
302
339
|
if rule.apply_to == ApplyTo.INPUT:
|
|
303
340
|
# INPUT: triggers in pre-execution, does not trigger in post-execution
|
|
304
341
|
if is_pre_execution:
|
|
305
|
-
return False, "
|
|
342
|
+
return False, "Always rule enforced"
|
|
306
343
|
else:
|
|
307
|
-
return True, "
|
|
344
|
+
return True, "No rules to apply for output data"
|
|
308
345
|
elif rule.apply_to == ApplyTo.OUTPUT:
|
|
309
346
|
# OUTPUT: does not trigger in pre-execution, triggers in post-execution
|
|
310
347
|
if is_pre_execution:
|
|
311
|
-
return True, "
|
|
348
|
+
return True, "No rules to apply for input data"
|
|
312
349
|
else:
|
|
313
|
-
return False, "
|
|
350
|
+
return False, "Always rule enforced"
|
|
314
351
|
elif rule.apply_to == ApplyTo.INPUT_AND_OUTPUT:
|
|
315
352
|
# INPUT_AND_OUTPUT: triggers in both phases
|
|
316
|
-
return False, "
|
|
353
|
+
return False, "Always rule enforced"
|
|
317
354
|
else:
|
|
318
355
|
return False, f"Unknown apply_to value: {rule.apply_to}"
|
|
319
356
|
|
|
@@ -102,6 +102,11 @@ class WordRule(BaseModel):
|
|
|
102
102
|
|
|
103
103
|
rule_type: Literal["word"] = Field(alias="$ruleType")
|
|
104
104
|
field_selector: FieldSelector = Field(alias="fieldSelector")
|
|
105
|
+
rule_description: str | None = Field(
|
|
106
|
+
default=None,
|
|
107
|
+
exclude=True,
|
|
108
|
+
description="Human-friendly description of the rule condition.",
|
|
109
|
+
)
|
|
105
110
|
detects_violation: Callable[[str], bool] = Field(
|
|
106
111
|
exclude=True,
|
|
107
112
|
description="Function that returns True if the string violates the rule (validation should fail).",
|
|
@@ -124,6 +129,11 @@ class NumberRule(BaseModel):
|
|
|
124
129
|
|
|
125
130
|
rule_type: Literal["number"] = Field(alias="$ruleType")
|
|
126
131
|
field_selector: FieldSelector = Field(alias="fieldSelector")
|
|
132
|
+
rule_description: str | None = Field(
|
|
133
|
+
default=None,
|
|
134
|
+
exclude=True,
|
|
135
|
+
description="Human-friendly description of the rule condition.",
|
|
136
|
+
)
|
|
127
137
|
detects_violation: Callable[[float], bool] = Field(
|
|
128
138
|
exclude=True,
|
|
129
139
|
description="Function that returns True if the number violates the rule (validation should fail).",
|
|
@@ -137,6 +147,11 @@ class BooleanRule(BaseModel):
|
|
|
137
147
|
|
|
138
148
|
rule_type: Literal["boolean"] = Field(alias="$ruleType")
|
|
139
149
|
field_selector: FieldSelector = Field(alias="fieldSelector")
|
|
150
|
+
rule_description: str | None = Field(
|
|
151
|
+
default=None,
|
|
152
|
+
exclude=True,
|
|
153
|
+
description="Human-friendly description of the rule condition.",
|
|
154
|
+
)
|
|
140
155
|
detects_violation: Callable[[bool], bool] = Field(
|
|
141
156
|
exclude=True,
|
|
142
157
|
description="Function that returns True if the boolean violates the rule (validation should fail).",
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"""Tracing manager for handling tracer implementations and function registry."""
|
|
2
2
|
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
3
5
|
import contextlib
|
|
4
6
|
from typing import Any, Generator, Optional
|
|
5
7
|
|
|
@@ -38,7 +40,7 @@ class UiPathTraceManager:
|
|
|
38
40
|
self,
|
|
39
41
|
span_exporter: SpanExporter,
|
|
40
42
|
batch: bool = True,
|
|
41
|
-
) ->
|
|
43
|
+
) -> UiPathTraceManager:
|
|
42
44
|
"""Add a span processor to the tracer provider."""
|
|
43
45
|
span_processor: SpanProcessor
|
|
44
46
|
if batch:
|
|
@@ -49,6 +51,15 @@ class UiPathTraceManager:
|
|
|
49
51
|
self.tracer_provider.add_span_processor(span_processor)
|
|
50
52
|
return self
|
|
51
53
|
|
|
54
|
+
def add_span_processor(
|
|
55
|
+
self,
|
|
56
|
+
span_processor: SpanProcessor,
|
|
57
|
+
) -> UiPathTraceManager:
|
|
58
|
+
"""Add a span processor to the tracer provider."""
|
|
59
|
+
self.tracer_span_processors.append(span_processor)
|
|
60
|
+
self.tracer_provider.add_span_processor(span_processor)
|
|
61
|
+
return self
|
|
62
|
+
|
|
52
63
|
def get_execution_spans(
|
|
53
64
|
self,
|
|
54
65
|
execution_id: str,
|
{uipath_core-0.1.8 → uipath_core-0.1.10}/tests/guardrails/test_deterministic_guardrails_service.py
RENAMED
|
@@ -81,12 +81,9 @@ class TestDeterministicGuardrailsService:
|
|
|
81
81
|
)
|
|
82
82
|
|
|
83
83
|
assert result.result == GuardrailValidationResultType.PASSED
|
|
84
|
-
assert
|
|
85
|
-
result.reason
|
|
86
|
-
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
87
|
-
)
|
|
84
|
+
assert result.reason == "No rules to apply for output data."
|
|
88
85
|
|
|
89
|
-
def
|
|
86
|
+
def test_evaluate_post_deterministic_guardrail_validation_passes_when_input_data_dont_violates_all_the_rules(
|
|
90
87
|
self,
|
|
91
88
|
service: DeterministicGuardrailsService,
|
|
92
89
|
) -> None:
|
|
@@ -146,13 +143,13 @@ class TestDeterministicGuardrailsService:
|
|
|
146
143
|
guardrail=deterministic_guardrail,
|
|
147
144
|
)
|
|
148
145
|
|
|
149
|
-
assert result.result == GuardrailValidationResultType.
|
|
146
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
150
147
|
assert (
|
|
151
148
|
result.reason
|
|
152
|
-
== "Input data didn't match the guardrail condition: [
|
|
149
|
+
== "Input data didn't match the guardrail condition: [isActive] comparing function [(b): b is not True]"
|
|
153
150
|
)
|
|
154
151
|
|
|
155
|
-
def
|
|
152
|
+
def test_evaluate_post_deterministic_guardrail_validation_passes_when_input_and_output_data_dont_violates_all_the_rules(
|
|
156
153
|
self,
|
|
157
154
|
service: DeterministicGuardrailsService,
|
|
158
155
|
) -> None:
|
|
@@ -214,67 +211,22 @@ class TestDeterministicGuardrailsService:
|
|
|
214
211
|
guardrail=deterministic_guardrail,
|
|
215
212
|
)
|
|
216
213
|
|
|
217
|
-
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
218
|
-
assert (
|
|
219
|
-
result.reason
|
|
220
|
-
== "Input data didn't match the guardrail condition: [isActive] comparing function [(b): b is not True]"
|
|
221
|
-
)
|
|
222
|
-
|
|
223
|
-
def test_evaluate_post_deterministic_guardrail_matches_regex_positive(
|
|
224
|
-
self,
|
|
225
|
-
service: DeterministicGuardrailsService,
|
|
226
|
-
) -> None:
|
|
227
|
-
"""Test deterministic guardrail validation passes when regex matches."""
|
|
228
|
-
deterministic_guardrail = DeterministicGuardrail(
|
|
229
|
-
id="test-deterministic-id",
|
|
230
|
-
name="Regex Guardrail",
|
|
231
|
-
description="Test regex guardrail",
|
|
232
|
-
enabled_for_evals=True,
|
|
233
|
-
guardrail_type="custom",
|
|
234
|
-
selector=GuardrailSelector(
|
|
235
|
-
scopes=[GuardrailScope.TOOL], match_names=["test"]
|
|
236
|
-
),
|
|
237
|
-
rules=[
|
|
238
|
-
WordRule(
|
|
239
|
-
rule_type="word",
|
|
240
|
-
field_selector=SpecificFieldsSelector(
|
|
241
|
-
selector_type="specific",
|
|
242
|
-
fields=[
|
|
243
|
-
FieldReference(path="userName", source=FieldSource.INPUT)
|
|
244
|
-
],
|
|
245
|
-
),
|
|
246
|
-
detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)),
|
|
247
|
-
),
|
|
248
|
-
],
|
|
249
|
-
)
|
|
250
|
-
|
|
251
|
-
# Input data with userName that matches the regex pattern
|
|
252
|
-
input_data = {
|
|
253
|
-
"userName": "test123",
|
|
254
|
-
}
|
|
255
|
-
output_data: dict[str, Any] = {}
|
|
256
|
-
|
|
257
|
-
result = service.evaluate_post_deterministic_guardrail(
|
|
258
|
-
input_data=input_data,
|
|
259
|
-
output_data=output_data,
|
|
260
|
-
guardrail=deterministic_guardrail,
|
|
261
|
-
)
|
|
262
|
-
|
|
263
214
|
assert result.result == GuardrailValidationResultType.PASSED
|
|
264
215
|
assert (
|
|
265
216
|
result.reason
|
|
266
|
-
== "
|
|
217
|
+
== "Input data didn't match the guardrail condition: [age] comparing function [(n): n < 21.0]"
|
|
267
218
|
)
|
|
268
219
|
|
|
269
|
-
def
|
|
220
|
+
def test_evaluate_post_deterministic_guardrail_uses_rule_description(
|
|
270
221
|
self,
|
|
271
222
|
service: DeterministicGuardrailsService,
|
|
272
223
|
) -> None:
|
|
273
|
-
"""
|
|
224
|
+
"""Ensure rule_description is returned when a rule fails validation."""
|
|
225
|
+
friendly_description = "Username must include 'te' and a digit"
|
|
274
226
|
deterministic_guardrail = DeterministicGuardrail(
|
|
275
|
-
id="test-
|
|
276
|
-
name="Regex Guardrail",
|
|
277
|
-
description="Test regex guardrail",
|
|
227
|
+
id="test-rule-desc-id",
|
|
228
|
+
name="Regex Guardrail With Description",
|
|
229
|
+
description="Test regex guardrail with description",
|
|
278
230
|
enabled_for_evals=True,
|
|
279
231
|
guardrail_type="custom",
|
|
280
232
|
selector=GuardrailSelector(
|
|
@@ -289,18 +241,9 @@ class TestDeterministicGuardrailsService:
|
|
|
289
241
|
FieldReference(path="userName", source=FieldSource.INPUT)
|
|
290
242
|
],
|
|
291
243
|
),
|
|
244
|
+
rule_description=friendly_description,
|
|
292
245
|
detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)),
|
|
293
246
|
),
|
|
294
|
-
NumberRule(
|
|
295
|
-
rule_type="number",
|
|
296
|
-
field_selector=SpecificFieldsSelector(
|
|
297
|
-
selector_type="specific",
|
|
298
|
-
fields=[
|
|
299
|
-
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
300
|
-
],
|
|
301
|
-
),
|
|
302
|
-
detects_violation=lambda n: n != 200.0,
|
|
303
|
-
),
|
|
304
247
|
],
|
|
305
248
|
)
|
|
306
249
|
|
|
@@ -308,31 +251,27 @@ class TestDeterministicGuardrailsService:
|
|
|
308
251
|
input_data = {
|
|
309
252
|
"userName": "test",
|
|
310
253
|
}
|
|
311
|
-
output_data = {
|
|
312
|
-
"status": 200,
|
|
313
|
-
}
|
|
314
254
|
|
|
315
|
-
result = service.
|
|
255
|
+
result = service.evaluate_pre_deterministic_guardrail(
|
|
316
256
|
input_data=input_data,
|
|
317
|
-
output_data=output_data,
|
|
318
257
|
guardrail=deterministic_guardrail,
|
|
319
258
|
)
|
|
320
259
|
|
|
321
260
|
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
322
261
|
assert (
|
|
323
262
|
result.reason
|
|
324
|
-
==
|
|
263
|
+
== "Data matched all guardrail conditions: [Username must include 'te' and a digit]"
|
|
325
264
|
)
|
|
326
265
|
|
|
327
|
-
def
|
|
266
|
+
def test_evaluate_post_deterministic_guardrail_passes_validation_when_no_output_rules(
|
|
328
267
|
self,
|
|
329
268
|
service: DeterministicGuardrailsService,
|
|
330
269
|
) -> None:
|
|
331
|
-
"""Test deterministic guardrail validation passes when
|
|
270
|
+
"""Test deterministic guardrail validation passes when regex matches."""
|
|
332
271
|
deterministic_guardrail = DeterministicGuardrail(
|
|
333
272
|
id="test-deterministic-id",
|
|
334
|
-
name="
|
|
335
|
-
description="Test
|
|
273
|
+
name="Regex Guardrail",
|
|
274
|
+
description="Test regex guardrail",
|
|
336
275
|
enabled_for_evals=True,
|
|
337
276
|
guardrail_type="custom",
|
|
338
277
|
selector=GuardrailSelector(
|
|
@@ -347,14 +286,14 @@ class TestDeterministicGuardrailsService:
|
|
|
347
286
|
FieldReference(path="userName", source=FieldSource.INPUT)
|
|
348
287
|
],
|
|
349
288
|
),
|
|
350
|
-
detects_violation=lambda s:
|
|
289
|
+
detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)),
|
|
351
290
|
),
|
|
352
291
|
],
|
|
353
292
|
)
|
|
354
293
|
|
|
355
|
-
# Input data with userName that
|
|
294
|
+
# Input data with userName that matches the regex pattern
|
|
356
295
|
input_data = {
|
|
357
|
-
"userName": "
|
|
296
|
+
"userName": "test123",
|
|
358
297
|
}
|
|
359
298
|
output_data: dict[str, Any] = {}
|
|
360
299
|
|
|
@@ -365,20 +304,17 @@ class TestDeterministicGuardrailsService:
|
|
|
365
304
|
)
|
|
366
305
|
|
|
367
306
|
assert result.result == GuardrailValidationResultType.PASSED
|
|
368
|
-
assert
|
|
369
|
-
result.reason
|
|
370
|
-
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
371
|
-
)
|
|
307
|
+
assert result.reason == "No rules to apply for output data."
|
|
372
308
|
|
|
373
|
-
def
|
|
309
|
+
def test_evaluate_post_deterministic_guardrail_failes_validation_when_data_macthes_rules(
|
|
374
310
|
self,
|
|
375
311
|
service: DeterministicGuardrailsService,
|
|
376
312
|
) -> None:
|
|
377
|
-
"""Test deterministic guardrail validation fails when
|
|
313
|
+
"""Test deterministic guardrail validation fails when regex doesn't match."""
|
|
378
314
|
deterministic_guardrail = DeterministicGuardrail(
|
|
379
315
|
id="test-deterministic-id",
|
|
380
|
-
name="
|
|
381
|
-
description="Test
|
|
316
|
+
name="Regex Guardrail",
|
|
317
|
+
description="Test regex guardrail",
|
|
382
318
|
enabled_for_evals=True,
|
|
383
319
|
guardrail_type="custom",
|
|
384
320
|
selector=GuardrailSelector(
|
|
@@ -393,7 +329,7 @@ class TestDeterministicGuardrailsService:
|
|
|
393
329
|
FieldReference(path="userName", source=FieldSource.INPUT)
|
|
394
330
|
],
|
|
395
331
|
),
|
|
396
|
-
detects_violation=lambda s:
|
|
332
|
+
detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)),
|
|
397
333
|
),
|
|
398
334
|
NumberRule(
|
|
399
335
|
rule_type="number",
|
|
@@ -408,12 +344,12 @@ class TestDeterministicGuardrailsService:
|
|
|
408
344
|
],
|
|
409
345
|
)
|
|
410
346
|
|
|
411
|
-
# Input data with userName that
|
|
347
|
+
# Input data with userName that doesn't match the regex pattern
|
|
412
348
|
input_data = {
|
|
413
349
|
"userName": "test",
|
|
414
350
|
}
|
|
415
351
|
output_data = {
|
|
416
|
-
"status":
|
|
352
|
+
"status": 201,
|
|
417
353
|
}
|
|
418
354
|
|
|
419
355
|
result = service.evaluate_post_deterministic_guardrail(
|
|
@@ -422,7 +358,14 @@ class TestDeterministicGuardrailsService:
|
|
|
422
358
|
guardrail=deterministic_guardrail,
|
|
423
359
|
)
|
|
424
360
|
|
|
361
|
+
expected_reason = (
|
|
362
|
+
"Data matched all guardrail conditions: [[userName] comparing function "
|
|
363
|
+
'[(s): not bool(re.search(".*te.*3.*", s))], '
|
|
364
|
+
"[status] comparing function [(n): n != 200.0]]"
|
|
365
|
+
)
|
|
366
|
+
|
|
425
367
|
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
368
|
+
assert result.reason == expected_reason
|
|
426
369
|
|
|
427
370
|
def test_evaluate_post_deterministic_guardrail_word_contains_substring_detects_violation(
|
|
428
371
|
self,
|
|
@@ -467,7 +410,7 @@ class TestDeterministicGuardrailsService:
|
|
|
467
410
|
"userName": "andrei",
|
|
468
411
|
}
|
|
469
412
|
output_data = {
|
|
470
|
-
"status":
|
|
413
|
+
"status": 201,
|
|
471
414
|
}
|
|
472
415
|
|
|
473
416
|
result = service.evaluate_post_deterministic_guardrail(
|
|
@@ -479,10 +422,11 @@ class TestDeterministicGuardrailsService:
|
|
|
479
422
|
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
480
423
|
assert (
|
|
481
424
|
result.reason
|
|
482
|
-
==
|
|
425
|
+
== "Data matched all guardrail conditions: [[userName] comparing function "
|
|
426
|
+
'[(s): "dre" in s], [status] comparing function [(n): n != 200.0]]'
|
|
483
427
|
)
|
|
484
428
|
|
|
485
|
-
def
|
|
429
|
+
def test_evaluate_post_deterministic_guardrail_number_func_passes_when_no_input_rules(
|
|
486
430
|
self,
|
|
487
431
|
service: DeterministicGuardrailsService,
|
|
488
432
|
) -> None:
|
|
@@ -521,10 +465,7 @@ class TestDeterministicGuardrailsService:
|
|
|
521
465
|
)
|
|
522
466
|
|
|
523
467
|
assert result.result == GuardrailValidationResultType.PASSED
|
|
524
|
-
assert
|
|
525
|
-
result.reason
|
|
526
|
-
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
527
|
-
)
|
|
468
|
+
assert result.reason == "No rules to apply for output data."
|
|
528
469
|
|
|
529
470
|
def test_evaluate_post_deterministic_guardrail_number_func_negative(
|
|
530
471
|
self,
|
|
@@ -567,7 +508,7 @@ class TestDeterministicGuardrailsService:
|
|
|
567
508
|
"age": 70,
|
|
568
509
|
}
|
|
569
510
|
output_data = {
|
|
570
|
-
"status":
|
|
511
|
+
"status": 201,
|
|
571
512
|
}
|
|
572
513
|
|
|
573
514
|
result = service.evaluate_post_deterministic_guardrail(
|
|
@@ -578,11 +519,11 @@ class TestDeterministicGuardrailsService:
|
|
|
578
519
|
|
|
579
520
|
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
580
521
|
|
|
581
|
-
def
|
|
522
|
+
def test_evaluate_post_execution_pases_when_only_some_rules_not_met(
|
|
582
523
|
self,
|
|
583
524
|
service: DeterministicGuardrailsService,
|
|
584
525
|
) -> None:
|
|
585
|
-
"""Test
|
|
526
|
+
"""Test post-execution guardrail passes when only some rules are not met."""
|
|
586
527
|
guardrail = self._create_guardrail_for_pre_execution()
|
|
587
528
|
input_data = {
|
|
588
529
|
"userName": "John",
|
|
@@ -599,7 +540,7 @@ class TestDeterministicGuardrailsService:
|
|
|
599
540
|
guardrail=guardrail,
|
|
600
541
|
)
|
|
601
542
|
|
|
602
|
-
assert result.result == GuardrailValidationResultType.
|
|
543
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
603
544
|
|
|
604
545
|
def test_should_ignore_post_execution_guardrail_for_pre_execution_returns_false(
|
|
605
546
|
self,
|
|
@@ -649,11 +590,10 @@ class TestDeterministicGuardrailsService:
|
|
|
649
590
|
# Pre-execution guardrail should still pass in post-execution
|
|
650
591
|
assert result.result == GuardrailValidationResultType.PASSED
|
|
651
592
|
|
|
652
|
-
def
|
|
593
|
+
def test_should_trigger_policy_post_execution_with_output_fields_when_no_violation_then_returns_true(
|
|
653
594
|
self,
|
|
654
595
|
service: DeterministicGuardrailsService,
|
|
655
596
|
) -> None:
|
|
656
|
-
"""Test post-execution guardrail passes when all conditions are met."""
|
|
657
597
|
guardrail = self._create_guardrail_for_post_execution()
|
|
658
598
|
input_data = {
|
|
659
599
|
"userName": "John",
|
|
@@ -674,11 +614,10 @@ class TestDeterministicGuardrailsService:
|
|
|
674
614
|
|
|
675
615
|
assert result.result == GuardrailValidationResultType.PASSED
|
|
676
616
|
|
|
677
|
-
def
|
|
617
|
+
def test_post_execution_with_output_fields_when_only_input_conditions_violated_then_returns_true(
|
|
678
618
|
self,
|
|
679
619
|
service: DeterministicGuardrailsService,
|
|
680
620
|
) -> None:
|
|
681
|
-
"""Test post-execution guardrail fails when input conditions are not met."""
|
|
682
621
|
guardrail = self._create_guardrail_for_post_execution()
|
|
683
622
|
input_data = {
|
|
684
623
|
"userName": "John",
|
|
@@ -697,13 +636,12 @@ class TestDeterministicGuardrailsService:
|
|
|
697
636
|
guardrail=guardrail,
|
|
698
637
|
)
|
|
699
638
|
|
|
700
|
-
assert result.result == GuardrailValidationResultType.
|
|
639
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
701
640
|
|
|
702
|
-
def
|
|
641
|
+
def test_post_execution_with_input_and_output_fields_output_when_only_output_conditions_violated_then_returns_true(
|
|
703
642
|
self,
|
|
704
643
|
service: DeterministicGuardrailsService,
|
|
705
644
|
) -> None:
|
|
706
|
-
"""Test post-execution guardrail fails when output conditions are not met."""
|
|
707
645
|
guardrail = self._create_guardrail_for_post_execution()
|
|
708
646
|
input_data = {
|
|
709
647
|
"userName": "John",
|
|
@@ -722,13 +660,12 @@ class TestDeterministicGuardrailsService:
|
|
|
722
660
|
guardrail=guardrail,
|
|
723
661
|
)
|
|
724
662
|
|
|
725
|
-
assert result.result == GuardrailValidationResultType.
|
|
663
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
726
664
|
|
|
727
|
-
def
|
|
665
|
+
def test_post_execution_multiple_rules_when_all_conditions_when_no_condition_is_violated_then_returns_true(
|
|
728
666
|
self,
|
|
729
667
|
service: DeterministicGuardrailsService,
|
|
730
668
|
) -> None:
|
|
731
|
-
"""Test post-execution guardrail with multiple rules passes when all conditions are met."""
|
|
732
669
|
guardrail = self._create_guardrail_with_multiple_rules()
|
|
733
670
|
input_data = {
|
|
734
671
|
"userName": "John",
|
|
@@ -749,7 +686,7 @@ class TestDeterministicGuardrailsService:
|
|
|
749
686
|
|
|
750
687
|
assert result.result == GuardrailValidationResultType.PASSED
|
|
751
688
|
|
|
752
|
-
def
|
|
689
|
+
def test_post_execution_rule_with_multiple_conditions_when_no_condition_is_violated_then_returns_true(
|
|
753
690
|
self,
|
|
754
691
|
service: DeterministicGuardrailsService,
|
|
755
692
|
) -> None:
|
|
@@ -774,7 +711,7 @@ class TestDeterministicGuardrailsService:
|
|
|
774
711
|
|
|
775
712
|
assert result.result == GuardrailValidationResultType.PASSED
|
|
776
713
|
|
|
777
|
-
def
|
|
714
|
+
def test_post_execution_rule_with_multiple_conditions_when_only_some_conditions_are_violated_then_returns_true(
|
|
778
715
|
self,
|
|
779
716
|
service: DeterministicGuardrailsService,
|
|
780
717
|
) -> None:
|
|
@@ -797,9 +734,34 @@ class TestDeterministicGuardrailsService:
|
|
|
797
734
|
guardrail=guardrail,
|
|
798
735
|
)
|
|
799
736
|
|
|
737
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
738
|
+
|
|
739
|
+
def test_post_execution_rule_with_multiple_conditions_when_all_condition_are_violated_then_returns_false(
|
|
740
|
+
self,
|
|
741
|
+
service: DeterministicGuardrailsService,
|
|
742
|
+
) -> None:
|
|
743
|
+
"""Test guardrail with multiple conditions fails when one condition is not met."""
|
|
744
|
+
guardrail = self._create_guardrail_with_rule_having_multiple_conditions()
|
|
745
|
+
input_data = {
|
|
746
|
+
"userName": "John",
|
|
747
|
+
"age": 15, # < 18
|
|
748
|
+
"isActive": False, # Not True
|
|
749
|
+
}
|
|
750
|
+
output_data = {
|
|
751
|
+
"result": "Success",
|
|
752
|
+
"status": 201, # Not 200
|
|
753
|
+
"success": True,
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
result = service.evaluate_post_deterministic_guardrail(
|
|
757
|
+
input_data=input_data,
|
|
758
|
+
output_data=output_data,
|
|
759
|
+
guardrail=guardrail,
|
|
760
|
+
)
|
|
761
|
+
|
|
800
762
|
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
801
763
|
|
|
802
|
-
def
|
|
764
|
+
def test_post_execution_with_all_fields_selector_when_no_field_violates_condition_then_returns_true(
|
|
803
765
|
self,
|
|
804
766
|
service: DeterministicGuardrailsService,
|
|
805
767
|
) -> None:
|
|
@@ -831,7 +793,7 @@ class TestDeterministicGuardrailsService:
|
|
|
831
793
|
}
|
|
832
794
|
output_data = {
|
|
833
795
|
"result": "Success",
|
|
834
|
-
"status": 25, #
|
|
796
|
+
"status": 25, # Doesn't match the rule value
|
|
835
797
|
"success": True,
|
|
836
798
|
}
|
|
837
799
|
|
|
@@ -843,7 +805,50 @@ class TestDeterministicGuardrailsService:
|
|
|
843
805
|
|
|
844
806
|
assert result.result == GuardrailValidationResultType.PASSED
|
|
845
807
|
|
|
846
|
-
def
|
|
808
|
+
def test_post_execution_with_all_fields_selector_when_all_fields_violate_condition_then_returns_false(
|
|
809
|
+
self,
|
|
810
|
+
service: DeterministicGuardrailsService,
|
|
811
|
+
) -> None:
|
|
812
|
+
guardrail = DeterministicGuardrail(
|
|
813
|
+
id="test-all-fields-id",
|
|
814
|
+
name="Guardrail With All Fields Selector",
|
|
815
|
+
description="Test all fields selector",
|
|
816
|
+
enabled_for_evals=True,
|
|
817
|
+
guardrail_type="custom",
|
|
818
|
+
selector=GuardrailSelector(
|
|
819
|
+
scopes=[GuardrailScope.TOOL], match_names=["test"]
|
|
820
|
+
),
|
|
821
|
+
rules=[
|
|
822
|
+
NumberRule(
|
|
823
|
+
rule_type="number",
|
|
824
|
+
field_selector=AllFieldsSelector(
|
|
825
|
+
selector_type="all", sources=[FieldSource.OUTPUT]
|
|
826
|
+
),
|
|
827
|
+
detects_violation=lambda n: n != 25.0,
|
|
828
|
+
),
|
|
829
|
+
],
|
|
830
|
+
)
|
|
831
|
+
|
|
832
|
+
input_data = {
|
|
833
|
+
"userName": "John",
|
|
834
|
+
"age": 25,
|
|
835
|
+
"isActive": True,
|
|
836
|
+
}
|
|
837
|
+
output_data = {
|
|
838
|
+
"result": "Success",
|
|
839
|
+
"status": 20, # Matches the rule value
|
|
840
|
+
"success": True,
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
result = service.evaluate_post_deterministic_guardrail(
|
|
844
|
+
input_data=input_data,
|
|
845
|
+
output_data=output_data,
|
|
846
|
+
guardrail=guardrail,
|
|
847
|
+
)
|
|
848
|
+
|
|
849
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
850
|
+
|
|
851
|
+
def test_post_execution_with_all_fields_selector_when_empty_output_schema_then_returns_true(
|
|
847
852
|
self,
|
|
848
853
|
service: DeterministicGuardrailsService,
|
|
849
854
|
) -> None:
|
|
@@ -900,11 +905,10 @@ class TestDeterministicGuardrailsService:
|
|
|
900
905
|
guardrail=guardrail,
|
|
901
906
|
)
|
|
902
907
|
|
|
903
|
-
assert
|
|
904
|
-
|
|
905
|
-
) # Should trigger
|
|
908
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
909
|
+
assert result.reason == "Always rule enforced"
|
|
906
910
|
|
|
907
|
-
def
|
|
911
|
+
def test_should_trigger_policy_pre_execution_always_rule_with_output_apply_to_returns_true(
|
|
908
912
|
self,
|
|
909
913
|
service: DeterministicGuardrailsService,
|
|
910
914
|
) -> None:
|
|
@@ -923,9 +927,8 @@ class TestDeterministicGuardrailsService:
|
|
|
923
927
|
guardrail=guardrail,
|
|
924
928
|
)
|
|
925
929
|
|
|
926
|
-
assert
|
|
927
|
-
|
|
928
|
-
) # Should not trigger
|
|
930
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
931
|
+
assert result.reason == "No rules to apply for input data"
|
|
929
932
|
|
|
930
933
|
def test_should_trigger_policy_pre_execution_always_rule_with_input_and_output_apply_to_returns_true(
|
|
931
934
|
self,
|
|
@@ -946,9 +949,8 @@ class TestDeterministicGuardrailsService:
|
|
|
946
949
|
guardrail=guardrail,
|
|
947
950
|
)
|
|
948
951
|
|
|
949
|
-
assert
|
|
950
|
-
|
|
951
|
-
) # Should trigger
|
|
952
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
953
|
+
assert result.reason == "Always rule enforced"
|
|
952
954
|
|
|
953
955
|
def test_should_trigger_policy_post_execution_always_rule_with_input_apply_to_returns_false(
|
|
954
956
|
self,
|
|
@@ -973,9 +975,8 @@ class TestDeterministicGuardrailsService:
|
|
|
973
975
|
guardrail=guardrail,
|
|
974
976
|
)
|
|
975
977
|
|
|
976
|
-
assert
|
|
977
|
-
|
|
978
|
-
) # Should not trigger
|
|
978
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
979
|
+
assert result.reason == "No rules to apply for output data."
|
|
979
980
|
|
|
980
981
|
def test_should_trigger_policy_post_execution_always_rule_with_output_apply_to_returns_true(
|
|
981
982
|
self,
|
|
@@ -1000,9 +1001,8 @@ class TestDeterministicGuardrailsService:
|
|
|
1000
1001
|
guardrail=guardrail,
|
|
1001
1002
|
)
|
|
1002
1003
|
|
|
1003
|
-
assert
|
|
1004
|
-
|
|
1005
|
-
) # Should trigger
|
|
1004
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
1005
|
+
assert result.reason == "Always rule enforced"
|
|
1006
1006
|
|
|
1007
1007
|
def test_should_trigger_policy_post_execution_always_rule_with_input_and_output_apply_to_returns_true(
|
|
1008
1008
|
self,
|
|
@@ -1027,9 +1027,8 @@ class TestDeterministicGuardrailsService:
|
|
|
1027
1027
|
guardrail=guardrail,
|
|
1028
1028
|
)
|
|
1029
1029
|
|
|
1030
|
-
assert
|
|
1031
|
-
|
|
1032
|
-
) # Should trigger
|
|
1030
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
1031
|
+
assert result.reason == "Always rule enforced"
|
|
1033
1032
|
|
|
1034
1033
|
# Helper methods to create guardrails
|
|
1035
1034
|
|
|
@@ -1240,10 +1239,7 @@ class TestDeterministicGuardrailsService:
|
|
|
1240
1239
|
)
|
|
1241
1240
|
|
|
1242
1241
|
assert result.result == GuardrailValidationResultType.PASSED
|
|
1243
|
-
assert
|
|
1244
|
-
result.reason
|
|
1245
|
-
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
1246
|
-
)
|
|
1242
|
+
assert result.reason == "No rules to apply for output data."
|
|
1247
1243
|
|
|
1248
1244
|
def test_evaluate_post_deterministic_guardrail_only_output_rules_passes(
|
|
1249
1245
|
self,
|
|
@@ -1298,7 +1294,10 @@ class TestDeterministicGuardrailsService:
|
|
|
1298
1294
|
)
|
|
1299
1295
|
|
|
1300
1296
|
assert result.result == GuardrailValidationResultType.PASSED
|
|
1301
|
-
assert
|
|
1297
|
+
assert (
|
|
1298
|
+
result.reason
|
|
1299
|
+
== "Output data didn't match the guardrail condition: [status] comparing function [(n): n != 200.0]"
|
|
1300
|
+
)
|
|
1302
1301
|
|
|
1303
1302
|
def test_evaluate_post_deterministic_guardrail_only_always_rule_fails(
|
|
1304
1303
|
self,
|
|
@@ -1336,6 +1335,7 @@ class TestDeterministicGuardrailsService:
|
|
|
1336
1335
|
)
|
|
1337
1336
|
|
|
1338
1337
|
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
1338
|
+
assert result.reason == "Always rule enforced"
|
|
1339
1339
|
|
|
1340
1340
|
def test_evaluate_post_deterministic_guardrail_only_input_rules_passes(
|
|
1341
1341
|
self,
|
|
@@ -1384,10 +1384,7 @@ class TestDeterministicGuardrailsService:
|
|
|
1384
1384
|
)
|
|
1385
1385
|
|
|
1386
1386
|
assert result.result == GuardrailValidationResultType.PASSED
|
|
1387
|
-
assert
|
|
1388
|
-
result.reason
|
|
1389
|
-
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
1390
|
-
)
|
|
1387
|
+
assert result.reason == "No rules to apply for output data."
|
|
1391
1388
|
|
|
1392
1389
|
def test_evaluate_pre_deterministic_guardrail_with_input_and_output_rules_input_true(
|
|
1393
1390
|
self,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|