uipath-core 0.1.4__tar.gz → 0.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {uipath_core-0.1.4 → uipath_core-0.1.6}/PKG-INFO +1 -1
- {uipath_core-0.1.4 → uipath_core-0.1.6}/pyproject.toml +1 -1
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/guardrails/__init__.py +2 -0
- uipath_core-0.1.6/src/uipath/core/guardrails/_deterministic_guardrails_service.py +145 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/guardrails/_evaluators.py +21 -19
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/guardrails/guardrails.py +13 -3
- {uipath_core-0.1.4 → uipath_core-0.1.6}/tests/guardrails/test_deterministic_guardrails_service.py +404 -43
- {uipath_core-0.1.4 → uipath_core-0.1.6}/uv.lock +1 -1
- uipath_core-0.1.4/src/uipath/core/guardrails/_deterministic_guardrails_service.py +0 -79
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.cursorrules +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.editorconfig +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.gitattributes +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.github/workflows/cd.yml +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.github/workflows/ci.yml +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.github/workflows/commitlint.yml +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.github/workflows/lint.yml +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.github/workflows/publish-dev.yml +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.github/workflows/test.yml +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.gitignore +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.pre-commit-config.yaml +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.python-version +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.vscode/extensions.json +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.vscode/launch.json +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/.vscode/settings.json +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/CONTRIBUTING.md +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/LICENSE +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/README.md +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/justfile +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/__init__.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/chat/__init__.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/chat/async_stream.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/chat/citation.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/chat/content.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/chat/conversation.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/chat/error.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/chat/event.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/chat/exchange.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/chat/interrupt.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/chat/message.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/chat/meta.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/chat/tool.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/errors/__init__.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/errors/errors.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/py.typed +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/tracing/__init__.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/tracing/_utils.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/tracing/decorators.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/tracing/exporters.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/tracing/processors.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/tracing/span_utils.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/src/uipath/core/tracing/trace_manager.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/tests/__init__.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/tests/conftest.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/tests/tracing/test_external_integration.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/tests/tracing/test_serialization.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/tests/tracing/test_span_nesting.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/tests/tracing/test_span_registry.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/tests/tracing/test_trace_manager.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/tests/tracing/test_traced.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.6}/tests/tracing/test_tracing_utils.py +0 -0
|
@@ -16,6 +16,7 @@ from .guardrails import (
|
|
|
16
16
|
GuardrailScope,
|
|
17
17
|
GuardrailSelector,
|
|
18
18
|
GuardrailValidationResult,
|
|
19
|
+
GuardrailValidationResultType,
|
|
19
20
|
NumberRule,
|
|
20
21
|
Rule,
|
|
21
22
|
SelectorType,
|
|
@@ -43,4 +44,5 @@ __all__ = [
|
|
|
43
44
|
"GuardrailScope",
|
|
44
45
|
"GuardrailSelector",
|
|
45
46
|
"GuardrailValidationResult",
|
|
47
|
+
"GuardrailValidationResultType",
|
|
46
48
|
]
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel
|
|
4
|
+
|
|
5
|
+
from ..tracing.decorators import traced
|
|
6
|
+
from ._evaluators import (
|
|
7
|
+
evaluate_boolean_rule,
|
|
8
|
+
evaluate_number_rule,
|
|
9
|
+
evaluate_universal_rule,
|
|
10
|
+
evaluate_word_rule,
|
|
11
|
+
)
|
|
12
|
+
from .guardrails import (
|
|
13
|
+
AllFieldsSelector,
|
|
14
|
+
ApplyTo,
|
|
15
|
+
BooleanRule,
|
|
16
|
+
DeterministicGuardrail,
|
|
17
|
+
FieldSource,
|
|
18
|
+
GuardrailValidationResult,
|
|
19
|
+
GuardrailValidationResultType,
|
|
20
|
+
NumberRule,
|
|
21
|
+
SpecificFieldsSelector,
|
|
22
|
+
UniversalRule,
|
|
23
|
+
WordRule,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class DeterministicGuardrailsService(BaseModel):
|
|
28
|
+
@traced("evaluate_pre_deterministic_guardrail", run_type="uipath")
|
|
29
|
+
def evaluate_pre_deterministic_guardrail(
|
|
30
|
+
self,
|
|
31
|
+
input_data: dict[str, Any],
|
|
32
|
+
guardrail: DeterministicGuardrail,
|
|
33
|
+
) -> GuardrailValidationResult:
|
|
34
|
+
"""Evaluate deterministic guardrail rules against input data (pre-execution)."""
|
|
35
|
+
# Check if guardrail contains any output-dependent rules
|
|
36
|
+
has_output_rule = self._has_output_dependent_rule(guardrail, [ApplyTo.OUTPUT])
|
|
37
|
+
|
|
38
|
+
# If guardrail has output-dependent rules, skip evaluation in pre-execution
|
|
39
|
+
# Output rules will be evaluated during post-execution
|
|
40
|
+
if has_output_rule:
|
|
41
|
+
return GuardrailValidationResult(
|
|
42
|
+
result=GuardrailValidationResultType.PASSED,
|
|
43
|
+
reason="Guardrail contains output-dependent rules that will be evaluated during post-execution",
|
|
44
|
+
)
|
|
45
|
+
return self._evaluate_deterministic_guardrail(
|
|
46
|
+
input_data=input_data,
|
|
47
|
+
output_data={},
|
|
48
|
+
guardrail=guardrail,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
@traced("evaluate_post_deterministic_guardrails", run_type="uipath")
|
|
52
|
+
def evaluate_post_deterministic_guardrail(
|
|
53
|
+
self,
|
|
54
|
+
input_data: dict[str, Any],
|
|
55
|
+
output_data: dict[str, Any],
|
|
56
|
+
guardrail: DeterministicGuardrail,
|
|
57
|
+
) -> GuardrailValidationResult:
|
|
58
|
+
"""Evaluate deterministic guardrail rules against input and output data."""
|
|
59
|
+
# Check if guardrail contains any output-dependent rules
|
|
60
|
+
has_output_rule = self._has_output_dependent_rule(
|
|
61
|
+
guardrail, [ApplyTo.OUTPUT, ApplyTo.INPUT_AND_OUTPUT]
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
# If guardrail has no output-dependent rules, skip post-execution evaluation
|
|
65
|
+
# Only input rules exist and they should have been evaluated during pre-execution
|
|
66
|
+
if not has_output_rule:
|
|
67
|
+
return GuardrailValidationResult(
|
|
68
|
+
result=GuardrailValidationResultType.PASSED,
|
|
69
|
+
reason="Guardrail contains only input-dependent rules that were evaluated during pre-execution",
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
return self._evaluate_deterministic_guardrail(
|
|
73
|
+
input_data=input_data,
|
|
74
|
+
output_data=output_data,
|
|
75
|
+
guardrail=guardrail,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
@staticmethod
|
|
79
|
+
def _has_output_dependent_rule(
|
|
80
|
+
guardrail: DeterministicGuardrail,
|
|
81
|
+
universal_rules_apply_to_values: list[ApplyTo],
|
|
82
|
+
) -> bool:
|
|
83
|
+
"""Check if at least one rule EXCLUSIVELY requires output data.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
guardrail: The guardrail to check
|
|
87
|
+
universal_rules_apply_to_values: List of ApplyTo values to consider as output-dependent for UniversalRules.
|
|
88
|
+
|
|
89
|
+
Returns:
|
|
90
|
+
True if at least one rule exclusively depends on output data, False otherwise.
|
|
91
|
+
"""
|
|
92
|
+
for rule in guardrail.rules:
|
|
93
|
+
# UniversalRule: only return True if it applies to values in universal_rules_apply_to_values
|
|
94
|
+
if isinstance(rule, UniversalRule):
|
|
95
|
+
if rule.apply_to in universal_rules_apply_to_values:
|
|
96
|
+
return True
|
|
97
|
+
# Rules with field_selector
|
|
98
|
+
elif isinstance(rule, (WordRule, NumberRule, BooleanRule)):
|
|
99
|
+
field_selector = rule.field_selector
|
|
100
|
+
# AllFieldsSelector applies to both input and output, not exclusively output
|
|
101
|
+
# SpecificFieldsSelector: only return True if at least one field has OUTPUT source
|
|
102
|
+
if isinstance(field_selector, SpecificFieldsSelector):
|
|
103
|
+
if field_selector.fields and any(
|
|
104
|
+
field.source == FieldSource.OUTPUT
|
|
105
|
+
for field in field_selector.fields
|
|
106
|
+
):
|
|
107
|
+
return True
|
|
108
|
+
elif isinstance(field_selector, AllFieldsSelector):
|
|
109
|
+
if FieldSource.OUTPUT in field_selector.sources:
|
|
110
|
+
return True
|
|
111
|
+
|
|
112
|
+
return False
|
|
113
|
+
|
|
114
|
+
@staticmethod
|
|
115
|
+
def _evaluate_deterministic_guardrail(
|
|
116
|
+
input_data: dict[str, Any],
|
|
117
|
+
output_data: dict[str, Any],
|
|
118
|
+
guardrail: DeterministicGuardrail,
|
|
119
|
+
) -> GuardrailValidationResult:
|
|
120
|
+
"""Evaluate deterministic guardrail rules against input and output data."""
|
|
121
|
+
for rule in guardrail.rules:
|
|
122
|
+
if isinstance(rule, WordRule):
|
|
123
|
+
passed, reason = evaluate_word_rule(rule, input_data, output_data)
|
|
124
|
+
elif isinstance(rule, NumberRule):
|
|
125
|
+
passed, reason = evaluate_number_rule(rule, input_data, output_data)
|
|
126
|
+
elif isinstance(rule, BooleanRule):
|
|
127
|
+
passed, reason = evaluate_boolean_rule(rule, input_data, output_data)
|
|
128
|
+
elif isinstance(rule, UniversalRule):
|
|
129
|
+
passed, reason = evaluate_universal_rule(rule, output_data)
|
|
130
|
+
else:
|
|
131
|
+
return GuardrailValidationResult(
|
|
132
|
+
result=GuardrailValidationResultType.VALIDATION_FAILED,
|
|
133
|
+
reason=f"Unknown rule type: {type(rule)}",
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
if not passed:
|
|
137
|
+
return GuardrailValidationResult(
|
|
138
|
+
result=GuardrailValidationResultType.VALIDATION_FAILED,
|
|
139
|
+
reason=reason or "Rule validation failed",
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
return GuardrailValidationResult(
|
|
143
|
+
result=GuardrailValidationResultType.PASSED,
|
|
144
|
+
reason="All deterministic guardrail rules passed",
|
|
145
|
+
)
|
|
@@ -120,23 +120,25 @@ def get_fields_from_selector(
|
|
|
120
120
|
fields: list[tuple[Any, FieldReference]] = []
|
|
121
121
|
|
|
122
122
|
if isinstance(field_selector, AllFieldsSelector):
|
|
123
|
-
# For "all" selector, we need to collect all fields from
|
|
123
|
+
# For "all" selector, we need to collect all fields from the specified sources
|
|
124
124
|
# This is a simplified implementation - in practice, you might want to
|
|
125
125
|
# recursively collect all nested fields
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
(
|
|
129
|
-
|
|
130
|
-
|
|
126
|
+
if FieldSource.INPUT in field_selector.sources:
|
|
127
|
+
for key, value in input_data.items():
|
|
128
|
+
fields.append(
|
|
129
|
+
(
|
|
130
|
+
value,
|
|
131
|
+
FieldReference(path=key, source=FieldSource.INPUT),
|
|
132
|
+
)
|
|
131
133
|
)
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
134
|
+
if FieldSource.OUTPUT in field_selector.sources:
|
|
135
|
+
for key, value in output_data.items():
|
|
136
|
+
fields.append(
|
|
137
|
+
(
|
|
138
|
+
value,
|
|
139
|
+
FieldReference(path=key, source=FieldSource.OUTPUT),
|
|
140
|
+
)
|
|
138
141
|
)
|
|
139
|
-
)
|
|
140
142
|
elif isinstance(field_selector, SpecificFieldsSelector):
|
|
141
143
|
# For specific fields, extract values based on field references
|
|
142
144
|
for field_ref in field_selector.fields:
|
|
@@ -286,13 +288,13 @@ def evaluate_universal_rule(
|
|
|
286
288
|
|
|
287
289
|
Universal rules trigger based on the apply_to scope and execution phase:
|
|
288
290
|
- Pre-execution (empty output_data):
|
|
289
|
-
- INPUT: triggers (
|
|
290
|
-
- OUTPUT: does not trigger (
|
|
291
|
-
- INPUT_AND_OUTPUT: triggers (
|
|
291
|
+
- INPUT: triggers (result = VALIDATION_FAILED)
|
|
292
|
+
- OUTPUT: does not trigger (result = PASSED)
|
|
293
|
+
- INPUT_AND_OUTPUT: triggers (result = VALIDATION_FAILED)
|
|
292
294
|
- Post-execution (output_data has data):
|
|
293
|
-
- INPUT: does not trigger (
|
|
294
|
-
- OUTPUT: triggers (
|
|
295
|
-
- INPUT_AND_OUTPUT: triggers (
|
|
295
|
+
- INPUT: does not trigger (result = PASSED)
|
|
296
|
+
- OUTPUT: triggers (result = VALIDATION_FAILED)
|
|
297
|
+
- INPUT_AND_OUTPUT: triggers (result = VALIDATION_FAILED)
|
|
296
298
|
"""
|
|
297
299
|
# Determine if this is pre-execution (no output data) or post-execution
|
|
298
300
|
is_pre_execution = not output_data or len(output_data) == 0
|
|
@@ -6,18 +6,27 @@ from typing import Annotated, Callable, Literal
|
|
|
6
6
|
from pydantic import BaseModel, ConfigDict, Field
|
|
7
7
|
|
|
8
8
|
|
|
9
|
+
class GuardrailValidationResultType(str, Enum):
|
|
10
|
+
"""Guardrail validation result type enumeration."""
|
|
11
|
+
|
|
12
|
+
PASSED = "passed"
|
|
13
|
+
VALIDATION_FAILED = "validation_failed"
|
|
14
|
+
ENTITLEMENTS_MISSING = "entitlements_missing"
|
|
15
|
+
FEATURE_DISABLED = "feature_disabled"
|
|
16
|
+
|
|
17
|
+
|
|
9
18
|
class GuardrailValidationResult(BaseModel):
|
|
10
19
|
"""Result returned from validating input with a given guardrail.
|
|
11
20
|
|
|
12
21
|
Attributes:
|
|
13
|
-
|
|
22
|
+
result: The validation result type.
|
|
14
23
|
reason: Textual explanation describing why the validation passed or failed.
|
|
15
24
|
"""
|
|
16
25
|
|
|
17
26
|
model_config = ConfigDict(populate_by_name=True)
|
|
18
27
|
|
|
19
|
-
|
|
20
|
-
alias="
|
|
28
|
+
result: GuardrailValidationResultType = Field(
|
|
29
|
+
alias="result", description="Validation result."
|
|
21
30
|
)
|
|
22
31
|
reason: str = Field(
|
|
23
32
|
alias="reason", description="Explanation for the validation result."
|
|
@@ -59,6 +68,7 @@ class AllFieldsSelector(BaseModel):
|
|
|
59
68
|
"""All fields selector."""
|
|
60
69
|
|
|
61
70
|
selector_type: Literal["all"] = Field(alias="$selectorType")
|
|
71
|
+
sources: list[FieldSource]
|
|
62
72
|
|
|
63
73
|
model_config = ConfigDict(populate_by_name=True, extra="allow")
|
|
64
74
|
|
{uipath_core-0.1.4 → uipath_core-0.1.6}/tests/guardrails/test_deterministic_guardrails_service.py
RENAMED
|
@@ -13,6 +13,7 @@ from uipath.core.guardrails import (
|
|
|
13
13
|
FieldSource,
|
|
14
14
|
GuardrailScope,
|
|
15
15
|
GuardrailSelector,
|
|
16
|
+
GuardrailValidationResultType,
|
|
16
17
|
NumberRule,
|
|
17
18
|
SpecificFieldsSelector,
|
|
18
19
|
UniversalRule,
|
|
@@ -79,8 +80,11 @@ class TestDeterministicGuardrailsService:
|
|
|
79
80
|
guardrail=deterministic_guardrail,
|
|
80
81
|
)
|
|
81
82
|
|
|
82
|
-
assert result.
|
|
83
|
-
assert
|
|
83
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
84
|
+
assert (
|
|
85
|
+
result.reason
|
|
86
|
+
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
87
|
+
)
|
|
84
88
|
|
|
85
89
|
def test_evaluate_post_deterministic_guardrail_validation_failed_age(
|
|
86
90
|
self,
|
|
@@ -115,6 +119,16 @@ class TestDeterministicGuardrailsService:
|
|
|
115
119
|
),
|
|
116
120
|
detects_violation=lambda b: b is not True,
|
|
117
121
|
),
|
|
122
|
+
NumberRule(
|
|
123
|
+
rule_type="number",
|
|
124
|
+
field_selector=SpecificFieldsSelector(
|
|
125
|
+
selector_type="specific",
|
|
126
|
+
fields=[
|
|
127
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
128
|
+
],
|
|
129
|
+
),
|
|
130
|
+
detects_violation=lambda n: n != 200.0,
|
|
131
|
+
),
|
|
118
132
|
],
|
|
119
133
|
)
|
|
120
134
|
|
|
@@ -132,7 +146,7 @@ class TestDeterministicGuardrailsService:
|
|
|
132
146
|
guardrail=deterministic_guardrail,
|
|
133
147
|
)
|
|
134
148
|
|
|
135
|
-
assert result.
|
|
149
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
136
150
|
assert (
|
|
137
151
|
result.reason
|
|
138
152
|
== "Input data didn't match the guardrail condition: [age] comparing function [(n): n < 21.0]"
|
|
@@ -171,6 +185,16 @@ class TestDeterministicGuardrailsService:
|
|
|
171
185
|
),
|
|
172
186
|
detects_violation=lambda b: b is not True,
|
|
173
187
|
),
|
|
188
|
+
NumberRule(
|
|
189
|
+
rule_type="number",
|
|
190
|
+
field_selector=SpecificFieldsSelector(
|
|
191
|
+
selector_type="specific",
|
|
192
|
+
fields=[
|
|
193
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
194
|
+
],
|
|
195
|
+
),
|
|
196
|
+
detects_violation=lambda n: n != 200.0,
|
|
197
|
+
),
|
|
174
198
|
],
|
|
175
199
|
)
|
|
176
200
|
|
|
@@ -180,7 +204,9 @@ class TestDeterministicGuardrailsService:
|
|
|
180
204
|
"age": 25,
|
|
181
205
|
"isActive": False,
|
|
182
206
|
}
|
|
183
|
-
output_data
|
|
207
|
+
output_data = {
|
|
208
|
+
"status": 200,
|
|
209
|
+
}
|
|
184
210
|
|
|
185
211
|
result = service.evaluate_post_deterministic_guardrail(
|
|
186
212
|
input_data=input_data,
|
|
@@ -188,7 +214,7 @@ class TestDeterministicGuardrailsService:
|
|
|
188
214
|
guardrail=deterministic_guardrail,
|
|
189
215
|
)
|
|
190
216
|
|
|
191
|
-
assert result.
|
|
217
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
192
218
|
assert (
|
|
193
219
|
result.reason
|
|
194
220
|
== "Input data didn't match the guardrail condition: [isActive] comparing function [(b): b is not True]"
|
|
@@ -234,8 +260,11 @@ class TestDeterministicGuardrailsService:
|
|
|
234
260
|
guardrail=deterministic_guardrail,
|
|
235
261
|
)
|
|
236
262
|
|
|
237
|
-
assert result.
|
|
238
|
-
assert
|
|
263
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
264
|
+
assert (
|
|
265
|
+
result.reason
|
|
266
|
+
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
267
|
+
)
|
|
239
268
|
|
|
240
269
|
def test_evaluate_post_deterministic_guardrail_matches_regex_negative(
|
|
241
270
|
self,
|
|
@@ -262,6 +291,16 @@ class TestDeterministicGuardrailsService:
|
|
|
262
291
|
),
|
|
263
292
|
detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)),
|
|
264
293
|
),
|
|
294
|
+
NumberRule(
|
|
295
|
+
rule_type="number",
|
|
296
|
+
field_selector=SpecificFieldsSelector(
|
|
297
|
+
selector_type="specific",
|
|
298
|
+
fields=[
|
|
299
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
300
|
+
],
|
|
301
|
+
),
|
|
302
|
+
detects_violation=lambda n: n != 200.0,
|
|
303
|
+
),
|
|
265
304
|
],
|
|
266
305
|
)
|
|
267
306
|
|
|
@@ -269,7 +308,9 @@ class TestDeterministicGuardrailsService:
|
|
|
269
308
|
input_data = {
|
|
270
309
|
"userName": "test",
|
|
271
310
|
}
|
|
272
|
-
output_data
|
|
311
|
+
output_data = {
|
|
312
|
+
"status": 200,
|
|
313
|
+
}
|
|
273
314
|
|
|
274
315
|
result = service.evaluate_post_deterministic_guardrail(
|
|
275
316
|
input_data=input_data,
|
|
@@ -277,7 +318,7 @@ class TestDeterministicGuardrailsService:
|
|
|
277
318
|
guardrail=deterministic_guardrail,
|
|
278
319
|
)
|
|
279
320
|
|
|
280
|
-
assert result.
|
|
321
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
281
322
|
assert (
|
|
282
323
|
result.reason
|
|
283
324
|
== 'Input data didn\'t match the guardrail condition: [userName] comparing function [(s): not bool(re.search(".*te.*3.*", s))]'
|
|
@@ -323,8 +364,11 @@ class TestDeterministicGuardrailsService:
|
|
|
323
364
|
guardrail=deterministic_guardrail,
|
|
324
365
|
)
|
|
325
366
|
|
|
326
|
-
assert result.
|
|
327
|
-
assert
|
|
367
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
368
|
+
assert (
|
|
369
|
+
result.reason
|
|
370
|
+
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
371
|
+
)
|
|
328
372
|
|
|
329
373
|
def test_evaluate_post_deterministic_guardrail_word_func_negative(
|
|
330
374
|
self,
|
|
@@ -351,6 +395,16 @@ class TestDeterministicGuardrailsService:
|
|
|
351
395
|
),
|
|
352
396
|
detects_violation=lambda s: len(s) <= 5,
|
|
353
397
|
),
|
|
398
|
+
NumberRule(
|
|
399
|
+
rule_type="number",
|
|
400
|
+
field_selector=SpecificFieldsSelector(
|
|
401
|
+
selector_type="specific",
|
|
402
|
+
fields=[
|
|
403
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
404
|
+
],
|
|
405
|
+
),
|
|
406
|
+
detects_violation=lambda n: n != 200.0,
|
|
407
|
+
),
|
|
354
408
|
],
|
|
355
409
|
)
|
|
356
410
|
|
|
@@ -358,7 +412,9 @@ class TestDeterministicGuardrailsService:
|
|
|
358
412
|
input_data = {
|
|
359
413
|
"userName": "test",
|
|
360
414
|
}
|
|
361
|
-
output_data
|
|
415
|
+
output_data = {
|
|
416
|
+
"status": 200,
|
|
417
|
+
}
|
|
362
418
|
|
|
363
419
|
result = service.evaluate_post_deterministic_guardrail(
|
|
364
420
|
input_data=input_data,
|
|
@@ -366,7 +422,7 @@ class TestDeterministicGuardrailsService:
|
|
|
366
422
|
guardrail=deterministic_guardrail,
|
|
367
423
|
)
|
|
368
424
|
|
|
369
|
-
assert result.
|
|
425
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
370
426
|
|
|
371
427
|
def test_evaluate_post_deterministic_guardrail_word_contains_substring_detects_violation(
|
|
372
428
|
self,
|
|
@@ -393,6 +449,16 @@ class TestDeterministicGuardrailsService:
|
|
|
393
449
|
),
|
|
394
450
|
detects_violation=lambda s: "dre" in s,
|
|
395
451
|
),
|
|
452
|
+
NumberRule(
|
|
453
|
+
rule_type="number",
|
|
454
|
+
field_selector=SpecificFieldsSelector(
|
|
455
|
+
selector_type="specific",
|
|
456
|
+
fields=[
|
|
457
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
458
|
+
],
|
|
459
|
+
),
|
|
460
|
+
detects_violation=lambda n: n != 200.0,
|
|
461
|
+
),
|
|
396
462
|
],
|
|
397
463
|
)
|
|
398
464
|
|
|
@@ -400,7 +466,9 @@ class TestDeterministicGuardrailsService:
|
|
|
400
466
|
input_data = {
|
|
401
467
|
"userName": "andrei",
|
|
402
468
|
}
|
|
403
|
-
output_data
|
|
469
|
+
output_data = {
|
|
470
|
+
"status": 200,
|
|
471
|
+
}
|
|
404
472
|
|
|
405
473
|
result = service.evaluate_post_deterministic_guardrail(
|
|
406
474
|
input_data=input_data,
|
|
@@ -408,7 +476,7 @@ class TestDeterministicGuardrailsService:
|
|
|
408
476
|
guardrail=deterministic_guardrail,
|
|
409
477
|
)
|
|
410
478
|
|
|
411
|
-
assert result.
|
|
479
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
412
480
|
assert (
|
|
413
481
|
result.reason
|
|
414
482
|
== 'Input data didn\'t match the guardrail condition: [userName] comparing function [(s): "dre" in s]'
|
|
@@ -452,8 +520,11 @@ class TestDeterministicGuardrailsService:
|
|
|
452
520
|
guardrail=deterministic_guardrail,
|
|
453
521
|
)
|
|
454
522
|
|
|
455
|
-
assert result.
|
|
456
|
-
assert
|
|
523
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
524
|
+
assert (
|
|
525
|
+
result.reason
|
|
526
|
+
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
527
|
+
)
|
|
457
528
|
|
|
458
529
|
def test_evaluate_post_deterministic_guardrail_number_func_negative(
|
|
459
530
|
self,
|
|
@@ -478,6 +549,16 @@ class TestDeterministicGuardrailsService:
|
|
|
478
549
|
),
|
|
479
550
|
detects_violation=lambda n: n < 18 or n > 65,
|
|
480
551
|
),
|
|
552
|
+
NumberRule(
|
|
553
|
+
rule_type="number",
|
|
554
|
+
field_selector=SpecificFieldsSelector(
|
|
555
|
+
selector_type="specific",
|
|
556
|
+
fields=[
|
|
557
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
558
|
+
],
|
|
559
|
+
),
|
|
560
|
+
detects_violation=lambda n: n != 200.0,
|
|
561
|
+
),
|
|
481
562
|
],
|
|
482
563
|
)
|
|
483
564
|
|
|
@@ -485,7 +566,9 @@ class TestDeterministicGuardrailsService:
|
|
|
485
566
|
input_data = {
|
|
486
567
|
"age": 70,
|
|
487
568
|
}
|
|
488
|
-
output_data
|
|
569
|
+
output_data = {
|
|
570
|
+
"status": 200,
|
|
571
|
+
}
|
|
489
572
|
|
|
490
573
|
result = service.evaluate_post_deterministic_guardrail(
|
|
491
574
|
input_data=input_data,
|
|
@@ -493,7 +576,7 @@ class TestDeterministicGuardrailsService:
|
|
|
493
576
|
guardrail=deterministic_guardrail,
|
|
494
577
|
)
|
|
495
578
|
|
|
496
|
-
assert result.
|
|
579
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
497
580
|
|
|
498
581
|
def test_should_trigger_policy_pre_execution_only_some_rules_not_met_returns_false(
|
|
499
582
|
self,
|
|
@@ -506,7 +589,9 @@ class TestDeterministicGuardrailsService:
|
|
|
506
589
|
"age": 18, # Less than 21
|
|
507
590
|
"isActive": True,
|
|
508
591
|
}
|
|
509
|
-
output_data
|
|
592
|
+
output_data = {
|
|
593
|
+
"status": 200,
|
|
594
|
+
}
|
|
510
595
|
|
|
511
596
|
result = service.evaluate_post_deterministic_guardrail(
|
|
512
597
|
input_data=input_data,
|
|
@@ -514,7 +599,7 @@ class TestDeterministicGuardrailsService:
|
|
|
514
599
|
guardrail=guardrail,
|
|
515
600
|
)
|
|
516
601
|
|
|
517
|
-
assert result.
|
|
602
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
518
603
|
|
|
519
604
|
def test_should_ignore_post_execution_guardrail_for_pre_execution_returns_false(
|
|
520
605
|
self,
|
|
@@ -536,7 +621,7 @@ class TestDeterministicGuardrailsService:
|
|
|
536
621
|
)
|
|
537
622
|
|
|
538
623
|
# Should fail because post-execution guardrail needs output data
|
|
539
|
-
assert result.
|
|
624
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
540
625
|
|
|
541
626
|
def test_should_trigger_policy_post_execution_guardrail_for_pre_execution_returns_false(
|
|
542
627
|
self,
|
|
@@ -562,7 +647,7 @@ class TestDeterministicGuardrailsService:
|
|
|
562
647
|
)
|
|
563
648
|
|
|
564
649
|
# Pre-execution guardrail should still pass in post-execution
|
|
565
|
-
assert result.
|
|
650
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
566
651
|
|
|
567
652
|
def test_should_trigger_policy_post_execution_with_output_fields_all_conditions_met_returns_true(
|
|
568
653
|
self,
|
|
@@ -587,7 +672,7 @@ class TestDeterministicGuardrailsService:
|
|
|
587
672
|
guardrail=guardrail,
|
|
588
673
|
)
|
|
589
674
|
|
|
590
|
-
assert result.
|
|
675
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
591
676
|
|
|
592
677
|
def test_should_trigger_policy_post_execution_with_output_fields_input_conditions_not_met_returns_false(
|
|
593
678
|
self,
|
|
@@ -612,7 +697,7 @@ class TestDeterministicGuardrailsService:
|
|
|
612
697
|
guardrail=guardrail,
|
|
613
698
|
)
|
|
614
699
|
|
|
615
|
-
assert result.
|
|
700
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
616
701
|
|
|
617
702
|
def test_should_trigger_policy_post_execution_with_output_fields_output_conditions_not_met_returns_false(
|
|
618
703
|
self,
|
|
@@ -637,7 +722,7 @@ class TestDeterministicGuardrailsService:
|
|
|
637
722
|
guardrail=guardrail,
|
|
638
723
|
)
|
|
639
724
|
|
|
640
|
-
assert result.
|
|
725
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
641
726
|
|
|
642
727
|
def test_should_trigger_policy_post_execution_multiple_rules_all_conditions_must_be_met_returns_true(
|
|
643
728
|
self,
|
|
@@ -662,7 +747,7 @@ class TestDeterministicGuardrailsService:
|
|
|
662
747
|
guardrail=guardrail,
|
|
663
748
|
)
|
|
664
749
|
|
|
665
|
-
assert result.
|
|
750
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
666
751
|
|
|
667
752
|
def test_should_trigger_policy_post_execution_rule_with_multiple_conditions_all_must_be_met_returns_true(
|
|
668
753
|
self,
|
|
@@ -687,7 +772,7 @@ class TestDeterministicGuardrailsService:
|
|
|
687
772
|
guardrail=guardrail,
|
|
688
773
|
)
|
|
689
774
|
|
|
690
|
-
assert result.
|
|
775
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
691
776
|
|
|
692
777
|
def test_should_trigger_policy_post_execution_rule_with_multiple_conditions_one_condition_not_met_returns_false(
|
|
693
778
|
self,
|
|
@@ -712,7 +797,7 @@ class TestDeterministicGuardrailsService:
|
|
|
712
797
|
guardrail=guardrail,
|
|
713
798
|
)
|
|
714
799
|
|
|
715
|
-
assert result.
|
|
800
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
716
801
|
|
|
717
802
|
def test_should_trigger_policy_post_execution_with_all_fields_selector_output_schema_has_fields_returns_true(
|
|
718
803
|
self,
|
|
@@ -731,7 +816,9 @@ class TestDeterministicGuardrailsService:
|
|
|
731
816
|
rules=[
|
|
732
817
|
NumberRule(
|
|
733
818
|
rule_type="number",
|
|
734
|
-
field_selector=AllFieldsSelector(
|
|
819
|
+
field_selector=AllFieldsSelector(
|
|
820
|
+
selector_type="all", sources=[FieldSource.OUTPUT]
|
|
821
|
+
),
|
|
735
822
|
detects_violation=lambda n: n != 25.0,
|
|
736
823
|
),
|
|
737
824
|
],
|
|
@@ -754,9 +841,9 @@ class TestDeterministicGuardrailsService:
|
|
|
754
841
|
guardrail=guardrail,
|
|
755
842
|
)
|
|
756
843
|
|
|
757
|
-
assert result.
|
|
844
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
758
845
|
|
|
759
|
-
def
|
|
846
|
+
def test_should_trigger_policy_post_execution_with_all_fields_selector_empty_output_schema_returns_true(
|
|
760
847
|
self,
|
|
761
848
|
service: DeterministicGuardrailsService,
|
|
762
849
|
) -> None:
|
|
@@ -773,7 +860,9 @@ class TestDeterministicGuardrailsService:
|
|
|
773
860
|
rules=[
|
|
774
861
|
NumberRule(
|
|
775
862
|
rule_type="number",
|
|
776
|
-
field_selector=AllFieldsSelector(
|
|
863
|
+
field_selector=AllFieldsSelector(
|
|
864
|
+
selector_type="all", sources=[FieldSource.INPUT]
|
|
865
|
+
),
|
|
777
866
|
detects_violation=lambda n: n != 200.0,
|
|
778
867
|
),
|
|
779
868
|
],
|
|
@@ -792,7 +881,7 @@ class TestDeterministicGuardrailsService:
|
|
|
792
881
|
guardrail=guardrail,
|
|
793
882
|
)
|
|
794
883
|
|
|
795
|
-
assert result.
|
|
884
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
796
885
|
|
|
797
886
|
def test_should_trigger_policy_pre_execution_always_rule_with_input_apply_to_returns_true(
|
|
798
887
|
self,
|
|
@@ -805,15 +894,15 @@ class TestDeterministicGuardrailsService:
|
|
|
805
894
|
"age": 25,
|
|
806
895
|
"isActive": True,
|
|
807
896
|
}
|
|
808
|
-
output_data: dict[str, Any] = {}
|
|
809
897
|
|
|
810
|
-
result = service.
|
|
898
|
+
result = service.evaluate_pre_deterministic_guardrail(
|
|
811
899
|
input_data=input_data,
|
|
812
|
-
output_data=output_data,
|
|
813
900
|
guardrail=guardrail,
|
|
814
901
|
)
|
|
815
902
|
|
|
816
|
-
assert
|
|
903
|
+
assert (
|
|
904
|
+
result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
905
|
+
) # Should trigger
|
|
817
906
|
|
|
818
907
|
def test_should_trigger_policy_pre_execution_always_rule_with_output_apply_to_returns_false(
|
|
819
908
|
self,
|
|
@@ -834,7 +923,9 @@ class TestDeterministicGuardrailsService:
|
|
|
834
923
|
guardrail=guardrail,
|
|
835
924
|
)
|
|
836
925
|
|
|
837
|
-
assert
|
|
926
|
+
assert (
|
|
927
|
+
result.result == GuardrailValidationResultType.PASSED
|
|
928
|
+
) # Should not trigger
|
|
838
929
|
|
|
839
930
|
def test_should_trigger_policy_pre_execution_always_rule_with_input_and_output_apply_to_returns_true(
|
|
840
931
|
self,
|
|
@@ -855,7 +946,9 @@ class TestDeterministicGuardrailsService:
|
|
|
855
946
|
guardrail=guardrail,
|
|
856
947
|
)
|
|
857
948
|
|
|
858
|
-
assert
|
|
949
|
+
assert (
|
|
950
|
+
result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
951
|
+
) # Should trigger
|
|
859
952
|
|
|
860
953
|
def test_should_trigger_policy_post_execution_always_rule_with_input_apply_to_returns_false(
|
|
861
954
|
self,
|
|
@@ -880,7 +973,9 @@ class TestDeterministicGuardrailsService:
|
|
|
880
973
|
guardrail=guardrail,
|
|
881
974
|
)
|
|
882
975
|
|
|
883
|
-
assert
|
|
976
|
+
assert (
|
|
977
|
+
result.result == GuardrailValidationResultType.PASSED
|
|
978
|
+
) # Should not trigger
|
|
884
979
|
|
|
885
980
|
def test_should_trigger_policy_post_execution_always_rule_with_output_apply_to_returns_true(
|
|
886
981
|
self,
|
|
@@ -905,7 +1000,9 @@ class TestDeterministicGuardrailsService:
|
|
|
905
1000
|
guardrail=guardrail,
|
|
906
1001
|
)
|
|
907
1002
|
|
|
908
|
-
assert
|
|
1003
|
+
assert (
|
|
1004
|
+
result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
1005
|
+
) # Should trigger
|
|
909
1006
|
|
|
910
1007
|
def test_should_trigger_policy_post_execution_always_rule_with_input_and_output_apply_to_returns_true(
|
|
911
1008
|
self,
|
|
@@ -930,7 +1027,9 @@ class TestDeterministicGuardrailsService:
|
|
|
930
1027
|
guardrail=guardrail,
|
|
931
1028
|
)
|
|
932
1029
|
|
|
933
|
-
assert
|
|
1030
|
+
assert (
|
|
1031
|
+
result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
1032
|
+
) # Should trigger
|
|
934
1033
|
|
|
935
1034
|
# Helper methods to create guardrails
|
|
936
1035
|
|
|
@@ -964,6 +1063,16 @@ class TestDeterministicGuardrailsService:
|
|
|
964
1063
|
),
|
|
965
1064
|
detects_violation=lambda b: b is not True,
|
|
966
1065
|
),
|
|
1066
|
+
NumberRule(
|
|
1067
|
+
rule_type="number",
|
|
1068
|
+
field_selector=SpecificFieldsSelector(
|
|
1069
|
+
selector_type="specific",
|
|
1070
|
+
fields=[
|
|
1071
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
1072
|
+
],
|
|
1073
|
+
),
|
|
1074
|
+
detects_violation=lambda n: n != 200.0,
|
|
1075
|
+
),
|
|
967
1076
|
],
|
|
968
1077
|
)
|
|
969
1078
|
|
|
@@ -1088,6 +1197,258 @@ class TestDeterministicGuardrailsService:
|
|
|
1088
1197
|
],
|
|
1089
1198
|
)
|
|
1090
1199
|
|
|
1200
|
+
def test_evaluate_post_deterministic_guardrail_word_contains_operator_passes(
|
|
1201
|
+
self,
|
|
1202
|
+
service: DeterministicGuardrailsService,
|
|
1203
|
+
) -> None:
|
|
1204
|
+
"""Test deterministic guardrail with word contains operator passes for pre-execution."""
|
|
1205
|
+
deterministic_guardrail = DeterministicGuardrail(
|
|
1206
|
+
id="b4283bd4-5ce0-49de-a918-2604d830460c",
|
|
1207
|
+
name="Before",
|
|
1208
|
+
description="",
|
|
1209
|
+
enabled_for_evals=True,
|
|
1210
|
+
guardrail_type="custom",
|
|
1211
|
+
selector=GuardrailSelector(
|
|
1212
|
+
scopes=[GuardrailScope.TOOL], match_names=["ConverterToStringAgent"]
|
|
1213
|
+
),
|
|
1214
|
+
rules=[
|
|
1215
|
+
WordRule(
|
|
1216
|
+
rule_type="word",
|
|
1217
|
+
field_selector=SpecificFieldsSelector(
|
|
1218
|
+
selector_type="specific",
|
|
1219
|
+
fields=[
|
|
1220
|
+
FieldReference(
|
|
1221
|
+
path="input_string", source=FieldSource.INPUT
|
|
1222
|
+
)
|
|
1223
|
+
],
|
|
1224
|
+
),
|
|
1225
|
+
detects_violation=lambda s: "cti" in s,
|
|
1226
|
+
),
|
|
1227
|
+
],
|
|
1228
|
+
)
|
|
1229
|
+
|
|
1230
|
+
# Input data without "cti" in input_string - should pass
|
|
1231
|
+
input_data = {
|
|
1232
|
+
"input_string": "test value",
|
|
1233
|
+
}
|
|
1234
|
+
output_data: dict[str, Any] = {}
|
|
1235
|
+
|
|
1236
|
+
result = service.evaluate_post_deterministic_guardrail(
|
|
1237
|
+
input_data=input_data,
|
|
1238
|
+
output_data=output_data,
|
|
1239
|
+
guardrail=deterministic_guardrail,
|
|
1240
|
+
)
|
|
1241
|
+
|
|
1242
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
1243
|
+
assert (
|
|
1244
|
+
result.reason
|
|
1245
|
+
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
1246
|
+
)
|
|
1247
|
+
|
|
1248
|
+
def test_evaluate_post_deterministic_guardrail_only_output_rules_passes(
|
|
1249
|
+
self,
|
|
1250
|
+
service: DeterministicGuardrailsService,
|
|
1251
|
+
) -> None:
|
|
1252
|
+
"""Test post guardrail with only output rules passes when conditions are met."""
|
|
1253
|
+
deterministic_guardrail = DeterministicGuardrail(
|
|
1254
|
+
id="test-only-output-id",
|
|
1255
|
+
name="Output Only Guardrail",
|
|
1256
|
+
description="Test guardrail with only output rules",
|
|
1257
|
+
enabled_for_evals=True,
|
|
1258
|
+
guardrail_type="custom",
|
|
1259
|
+
selector=GuardrailSelector(
|
|
1260
|
+
scopes=[GuardrailScope.TOOL], match_names=["test"]
|
|
1261
|
+
),
|
|
1262
|
+
rules=[
|
|
1263
|
+
NumberRule(
|
|
1264
|
+
rule_type="number",
|
|
1265
|
+
field_selector=SpecificFieldsSelector(
|
|
1266
|
+
selector_type="specific",
|
|
1267
|
+
fields=[
|
|
1268
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
1269
|
+
],
|
|
1270
|
+
),
|
|
1271
|
+
detects_violation=lambda n: n != 200.0,
|
|
1272
|
+
),
|
|
1273
|
+
WordRule(
|
|
1274
|
+
rule_type="word",
|
|
1275
|
+
field_selector=SpecificFieldsSelector(
|
|
1276
|
+
selector_type="specific",
|
|
1277
|
+
fields=[
|
|
1278
|
+
FieldReference(path="result", source=FieldSource.OUTPUT)
|
|
1279
|
+
],
|
|
1280
|
+
),
|
|
1281
|
+
detects_violation=lambda s: s != "Success",
|
|
1282
|
+
),
|
|
1283
|
+
],
|
|
1284
|
+
)
|
|
1285
|
+
|
|
1286
|
+
input_data = {
|
|
1287
|
+
"userName": "John",
|
|
1288
|
+
}
|
|
1289
|
+
output_data = {
|
|
1290
|
+
"status": 200,
|
|
1291
|
+
"result": "Success",
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
result = service.evaluate_post_deterministic_guardrail(
|
|
1295
|
+
input_data=input_data,
|
|
1296
|
+
output_data=output_data,
|
|
1297
|
+
guardrail=deterministic_guardrail,
|
|
1298
|
+
)
|
|
1299
|
+
|
|
1300
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
1301
|
+
assert result.reason == "All deterministic guardrail rules passed"
|
|
1302
|
+
|
|
1303
|
+
def test_evaluate_post_deterministic_guardrail_only_always_rule_fails(
|
|
1304
|
+
self,
|
|
1305
|
+
service: DeterministicGuardrailsService,
|
|
1306
|
+
) -> None:
|
|
1307
|
+
"""Test post guardrail with only UniversalRule always fails."""
|
|
1308
|
+
deterministic_guardrail = DeterministicGuardrail(
|
|
1309
|
+
id="test-only-always-id",
|
|
1310
|
+
name="Always Rule Only Guardrail",
|
|
1311
|
+
description="Test guardrail with only always rule",
|
|
1312
|
+
enabled_for_evals=True,
|
|
1313
|
+
guardrail_type="custom",
|
|
1314
|
+
selector=GuardrailSelector(
|
|
1315
|
+
scopes=[GuardrailScope.TOOL], match_names=["test"]
|
|
1316
|
+
),
|
|
1317
|
+
rules=[
|
|
1318
|
+
UniversalRule(
|
|
1319
|
+
rule_type="always",
|
|
1320
|
+
apply_to=ApplyTo.OUTPUT,
|
|
1321
|
+
),
|
|
1322
|
+
],
|
|
1323
|
+
)
|
|
1324
|
+
|
|
1325
|
+
input_data = {
|
|
1326
|
+
"userName": "John",
|
|
1327
|
+
}
|
|
1328
|
+
output_data = {
|
|
1329
|
+
"status": 200,
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
result = service.evaluate_post_deterministic_guardrail(
|
|
1333
|
+
input_data=input_data,
|
|
1334
|
+
output_data=output_data,
|
|
1335
|
+
guardrail=deterministic_guardrail,
|
|
1336
|
+
)
|
|
1337
|
+
|
|
1338
|
+
assert result.result == GuardrailValidationResultType.VALIDATION_FAILED
|
|
1339
|
+
|
|
1340
|
+
def test_evaluate_post_deterministic_guardrail_only_input_rules_passes(
|
|
1341
|
+
self,
|
|
1342
|
+
service: DeterministicGuardrailsService,
|
|
1343
|
+
) -> None:
|
|
1344
|
+
"""Test post guardrail passes when only input rules exist (no output data required)."""
|
|
1345
|
+
deterministic_guardrail = DeterministicGuardrail(
|
|
1346
|
+
id="test-only-input-id",
|
|
1347
|
+
name="Input Only Guardrail",
|
|
1348
|
+
description="Test guardrail with only input rules",
|
|
1349
|
+
enabled_for_evals=True,
|
|
1350
|
+
guardrail_type="custom",
|
|
1351
|
+
selector=GuardrailSelector(
|
|
1352
|
+
scopes=[GuardrailScope.TOOL], match_names=["test"]
|
|
1353
|
+
),
|
|
1354
|
+
rules=[
|
|
1355
|
+
NumberRule(
|
|
1356
|
+
rule_type="number",
|
|
1357
|
+
field_selector=SpecificFieldsSelector(
|
|
1358
|
+
selector_type="specific",
|
|
1359
|
+
fields=[FieldReference(path="age", source=FieldSource.INPUT)],
|
|
1360
|
+
),
|
|
1361
|
+
detects_violation=lambda n: n < 18,
|
|
1362
|
+
),
|
|
1363
|
+
WordRule(
|
|
1364
|
+
rule_type="word",
|
|
1365
|
+
field_selector=SpecificFieldsSelector(
|
|
1366
|
+
selector_type="specific",
|
|
1367
|
+
fields=[FieldReference(path="name", source=FieldSource.INPUT)],
|
|
1368
|
+
),
|
|
1369
|
+
detects_violation=lambda s: len(s) < 2,
|
|
1370
|
+
),
|
|
1371
|
+
],
|
|
1372
|
+
)
|
|
1373
|
+
|
|
1374
|
+
input_data = {
|
|
1375
|
+
"age": 25,
|
|
1376
|
+
"name": "John",
|
|
1377
|
+
}
|
|
1378
|
+
output_data: dict[str, Any] = {}
|
|
1379
|
+
|
|
1380
|
+
result = service.evaluate_post_deterministic_guardrail(
|
|
1381
|
+
input_data=input_data,
|
|
1382
|
+
output_data=output_data,
|
|
1383
|
+
guardrail=deterministic_guardrail,
|
|
1384
|
+
)
|
|
1385
|
+
|
|
1386
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
1387
|
+
assert (
|
|
1388
|
+
result.reason
|
|
1389
|
+
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
1390
|
+
)
|
|
1391
|
+
|
|
1392
|
+
def test_evaluate_pre_deterministic_guardrail_with_input_and_output_rules_input_true(
|
|
1393
|
+
self,
|
|
1394
|
+
service: DeterministicGuardrailsService,
|
|
1395
|
+
) -> None:
|
|
1396
|
+
"""Test pre-execution guardrail with input rule and output rules, should pass because is ignored."""
|
|
1397
|
+
deterministic_guardrail = DeterministicGuardrail(
|
|
1398
|
+
id="test-pre-mixed-rules-id",
|
|
1399
|
+
name="Pre Execution Mixed Rules Guardrail",
|
|
1400
|
+
description="Test pre-execution with both input and output rules",
|
|
1401
|
+
enabled_for_evals=True,
|
|
1402
|
+
guardrail_type="custom",
|
|
1403
|
+
selector=GuardrailSelector(
|
|
1404
|
+
scopes=[GuardrailScope.TOOL], match_names=["test"]
|
|
1405
|
+
),
|
|
1406
|
+
rules=[
|
|
1407
|
+
NumberRule(
|
|
1408
|
+
rule_type="number",
|
|
1409
|
+
field_selector=SpecificFieldsSelector(
|
|
1410
|
+
selector_type="specific",
|
|
1411
|
+
fields=[FieldReference(path="age", source=FieldSource.INPUT)],
|
|
1412
|
+
),
|
|
1413
|
+
detects_violation=lambda n: n < 21.0,
|
|
1414
|
+
),
|
|
1415
|
+
BooleanRule(
|
|
1416
|
+
rule_type="boolean",
|
|
1417
|
+
field_selector=SpecificFieldsSelector(
|
|
1418
|
+
selector_type="specific",
|
|
1419
|
+
fields=[
|
|
1420
|
+
FieldReference(path="isActive", source=FieldSource.INPUT)
|
|
1421
|
+
],
|
|
1422
|
+
),
|
|
1423
|
+
detects_violation=lambda b: b is not True,
|
|
1424
|
+
),
|
|
1425
|
+
# Output rule - should be ignored in pre-execution
|
|
1426
|
+
NumberRule(
|
|
1427
|
+
rule_type="number",
|
|
1428
|
+
field_selector=SpecificFieldsSelector(
|
|
1429
|
+
selector_type="specific",
|
|
1430
|
+
fields=[
|
|
1431
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
1432
|
+
],
|
|
1433
|
+
),
|
|
1434
|
+
detects_violation=lambda n: n != 200.0,
|
|
1435
|
+
),
|
|
1436
|
+
],
|
|
1437
|
+
)
|
|
1438
|
+
|
|
1439
|
+
input_data = {
|
|
1440
|
+
"userName": "John",
|
|
1441
|
+
"age": 18,
|
|
1442
|
+
"isActive": True,
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
result = service.evaluate_pre_deterministic_guardrail(
|
|
1446
|
+
input_data=input_data,
|
|
1447
|
+
guardrail=deterministic_guardrail,
|
|
1448
|
+
)
|
|
1449
|
+
|
|
1450
|
+
assert result.result == GuardrailValidationResultType.PASSED
|
|
1451
|
+
|
|
1091
1452
|
def _create_guardrail_with_always_rule(
|
|
1092
1453
|
self, apply_to: ApplyTo
|
|
1093
1454
|
) -> DeterministicGuardrail:
|
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
from typing import Any
|
|
2
|
-
|
|
3
|
-
from pydantic import BaseModel
|
|
4
|
-
|
|
5
|
-
from ..tracing.decorators import traced
|
|
6
|
-
from ._evaluators import (
|
|
7
|
-
evaluate_boolean_rule,
|
|
8
|
-
evaluate_number_rule,
|
|
9
|
-
evaluate_universal_rule,
|
|
10
|
-
evaluate_word_rule,
|
|
11
|
-
)
|
|
12
|
-
from .guardrails import (
|
|
13
|
-
BooleanRule,
|
|
14
|
-
DeterministicGuardrail,
|
|
15
|
-
GuardrailValidationResult,
|
|
16
|
-
NumberRule,
|
|
17
|
-
UniversalRule,
|
|
18
|
-
WordRule,
|
|
19
|
-
)
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
class DeterministicGuardrailsService(BaseModel):
|
|
23
|
-
@traced("evaluate_pre_deterministic_guardrail", run_type="uipath")
|
|
24
|
-
def evaluate_pre_deterministic_guardrail(
|
|
25
|
-
self,
|
|
26
|
-
input_data: dict[str, Any],
|
|
27
|
-
guardrail: DeterministicGuardrail,
|
|
28
|
-
) -> GuardrailValidationResult:
|
|
29
|
-
"""Evaluate deterministic guardrail rules against input data (pre-execution)."""
|
|
30
|
-
return self._evaluate_deterministic_guardrail(
|
|
31
|
-
input_data=input_data,
|
|
32
|
-
output_data={},
|
|
33
|
-
guardrail=guardrail,
|
|
34
|
-
)
|
|
35
|
-
|
|
36
|
-
@traced("evaluate_post_deterministic_guardrails", run_type="uipath")
|
|
37
|
-
def evaluate_post_deterministic_guardrail(
|
|
38
|
-
self,
|
|
39
|
-
input_data: dict[str, Any],
|
|
40
|
-
output_data: dict[str, Any],
|
|
41
|
-
guardrail: DeterministicGuardrail,
|
|
42
|
-
) -> GuardrailValidationResult:
|
|
43
|
-
"""Evaluate deterministic guardrail rules against input and output data."""
|
|
44
|
-
return self._evaluate_deterministic_guardrail(
|
|
45
|
-
input_data=input_data,
|
|
46
|
-
output_data=output_data,
|
|
47
|
-
guardrail=guardrail,
|
|
48
|
-
)
|
|
49
|
-
|
|
50
|
-
@staticmethod
|
|
51
|
-
def _evaluate_deterministic_guardrail(
|
|
52
|
-
input_data: dict[str, Any],
|
|
53
|
-
output_data: dict[str, Any],
|
|
54
|
-
guardrail: DeterministicGuardrail,
|
|
55
|
-
) -> GuardrailValidationResult:
|
|
56
|
-
"""Evaluate deterministic guardrail rules against input and output data."""
|
|
57
|
-
for rule in guardrail.rules:
|
|
58
|
-
if isinstance(rule, WordRule):
|
|
59
|
-
passed, reason = evaluate_word_rule(rule, input_data, output_data)
|
|
60
|
-
elif isinstance(rule, NumberRule):
|
|
61
|
-
passed, reason = evaluate_number_rule(rule, input_data, output_data)
|
|
62
|
-
elif isinstance(rule, BooleanRule):
|
|
63
|
-
passed, reason = evaluate_boolean_rule(rule, input_data, output_data)
|
|
64
|
-
elif isinstance(rule, UniversalRule):
|
|
65
|
-
passed, reason = evaluate_universal_rule(rule, output_data)
|
|
66
|
-
else:
|
|
67
|
-
return GuardrailValidationResult(
|
|
68
|
-
validation_passed=False,
|
|
69
|
-
reason=f"Unknown rule type: {type(rule)}",
|
|
70
|
-
)
|
|
71
|
-
|
|
72
|
-
if not passed:
|
|
73
|
-
return GuardrailValidationResult(
|
|
74
|
-
validation_passed=False, reason=reason or "Rule validation failed"
|
|
75
|
-
)
|
|
76
|
-
|
|
77
|
-
return GuardrailValidationResult(
|
|
78
|
-
validation_passed=True, reason="All deterministic guardrail rules passed"
|
|
79
|
-
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|