uipath-core 0.1.4__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {uipath_core-0.1.4 → uipath_core-0.1.5}/PKG-INFO +1 -1
- {uipath_core-0.1.4 → uipath_core-0.1.5}/pyproject.toml +1 -1
- uipath_core-0.1.5/src/uipath/core/guardrails/_deterministic_guardrails_service.py +142 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/guardrails/_evaluators.py +15 -13
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/guardrails/guardrails.py +1 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/tests/guardrails/test_deterministic_guardrails_service.py +365 -17
- {uipath_core-0.1.4 → uipath_core-0.1.5}/uv.lock +1 -1
- uipath_core-0.1.4/src/uipath/core/guardrails/_deterministic_guardrails_service.py +0 -79
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.cursorrules +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.editorconfig +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.gitattributes +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.github/workflows/cd.yml +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.github/workflows/ci.yml +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.github/workflows/commitlint.yml +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.github/workflows/lint.yml +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.github/workflows/publish-dev.yml +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.github/workflows/test.yml +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.gitignore +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.pre-commit-config.yaml +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.python-version +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.vscode/extensions.json +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.vscode/launch.json +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/.vscode/settings.json +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/CONTRIBUTING.md +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/LICENSE +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/README.md +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/justfile +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/__init__.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/chat/__init__.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/chat/async_stream.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/chat/citation.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/chat/content.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/chat/conversation.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/chat/error.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/chat/event.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/chat/exchange.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/chat/interrupt.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/chat/message.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/chat/meta.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/chat/tool.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/errors/__init__.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/errors/errors.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/guardrails/__init__.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/py.typed +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/tracing/__init__.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/tracing/_utils.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/tracing/decorators.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/tracing/exporters.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/tracing/processors.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/tracing/span_utils.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/src/uipath/core/tracing/trace_manager.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/tests/__init__.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/tests/conftest.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/tests/tracing/test_external_integration.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/tests/tracing/test_serialization.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/tests/tracing/test_span_nesting.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/tests/tracing/test_span_registry.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/tests/tracing/test_trace_manager.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/tests/tracing/test_traced.py +0 -0
- {uipath_core-0.1.4 → uipath_core-0.1.5}/tests/tracing/test_tracing_utils.py +0 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel
|
|
4
|
+
|
|
5
|
+
from ..tracing.decorators import traced
|
|
6
|
+
from ._evaluators import (
|
|
7
|
+
evaluate_boolean_rule,
|
|
8
|
+
evaluate_number_rule,
|
|
9
|
+
evaluate_universal_rule,
|
|
10
|
+
evaluate_word_rule,
|
|
11
|
+
)
|
|
12
|
+
from .guardrails import (
|
|
13
|
+
AllFieldsSelector,
|
|
14
|
+
ApplyTo,
|
|
15
|
+
BooleanRule,
|
|
16
|
+
DeterministicGuardrail,
|
|
17
|
+
FieldSource,
|
|
18
|
+
GuardrailValidationResult,
|
|
19
|
+
NumberRule,
|
|
20
|
+
SpecificFieldsSelector,
|
|
21
|
+
UniversalRule,
|
|
22
|
+
WordRule,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class DeterministicGuardrailsService(BaseModel):
|
|
27
|
+
@traced("evaluate_pre_deterministic_guardrail", run_type="uipath")
|
|
28
|
+
def evaluate_pre_deterministic_guardrail(
|
|
29
|
+
self,
|
|
30
|
+
input_data: dict[str, Any],
|
|
31
|
+
guardrail: DeterministicGuardrail,
|
|
32
|
+
) -> GuardrailValidationResult:
|
|
33
|
+
"""Evaluate deterministic guardrail rules against input data (pre-execution)."""
|
|
34
|
+
# Check if guardrail contains any output-dependent rules
|
|
35
|
+
has_output_rule = self._has_output_dependent_rule(guardrail, [ApplyTo.OUTPUT])
|
|
36
|
+
|
|
37
|
+
# If guardrail has output-dependent rules, skip evaluation in pre-execution
|
|
38
|
+
# Output rules will be evaluated during post-execution
|
|
39
|
+
if has_output_rule:
|
|
40
|
+
return GuardrailValidationResult(
|
|
41
|
+
validation_passed=True,
|
|
42
|
+
reason="Guardrail contains output-dependent rules that will be evaluated during post-execution",
|
|
43
|
+
)
|
|
44
|
+
return self._evaluate_deterministic_guardrail(
|
|
45
|
+
input_data=input_data,
|
|
46
|
+
output_data={},
|
|
47
|
+
guardrail=guardrail,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
@traced("evaluate_post_deterministic_guardrails", run_type="uipath")
|
|
51
|
+
def evaluate_post_deterministic_guardrail(
|
|
52
|
+
self,
|
|
53
|
+
input_data: dict[str, Any],
|
|
54
|
+
output_data: dict[str, Any],
|
|
55
|
+
guardrail: DeterministicGuardrail,
|
|
56
|
+
) -> GuardrailValidationResult:
|
|
57
|
+
"""Evaluate deterministic guardrail rules against input and output data."""
|
|
58
|
+
# Check if guardrail contains any output-dependent rules
|
|
59
|
+
has_output_rule = self._has_output_dependent_rule(
|
|
60
|
+
guardrail, [ApplyTo.OUTPUT, ApplyTo.INPUT_AND_OUTPUT]
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# If guardrail has no output-dependent rules, skip post-execution evaluation
|
|
64
|
+
# Only input rules exist and they should have been evaluated during pre-execution
|
|
65
|
+
if not has_output_rule:
|
|
66
|
+
return GuardrailValidationResult(
|
|
67
|
+
validation_passed=True,
|
|
68
|
+
reason="Guardrail contains only input-dependent rules that were evaluated during pre-execution",
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
return self._evaluate_deterministic_guardrail(
|
|
72
|
+
input_data=input_data,
|
|
73
|
+
output_data=output_data,
|
|
74
|
+
guardrail=guardrail,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
@staticmethod
|
|
78
|
+
def _has_output_dependent_rule(
|
|
79
|
+
guardrail: DeterministicGuardrail,
|
|
80
|
+
universal_rules_apply_to_values: list[ApplyTo],
|
|
81
|
+
) -> bool:
|
|
82
|
+
"""Check if at least one rule EXCLUSIVELY requires output data.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
guardrail: The guardrail to check
|
|
86
|
+
universal_rules_apply_to_values: List of ApplyTo values to consider as output-dependent for UniversalRules.
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
True if at least one rule exclusively depends on output data, False otherwise.
|
|
90
|
+
"""
|
|
91
|
+
for rule in guardrail.rules:
|
|
92
|
+
# UniversalRule: only return True if it applies to values in universal_rules_apply_to_values
|
|
93
|
+
if isinstance(rule, UniversalRule):
|
|
94
|
+
if rule.apply_to in universal_rules_apply_to_values:
|
|
95
|
+
return True
|
|
96
|
+
# Rules with field_selector
|
|
97
|
+
elif isinstance(rule, (WordRule, NumberRule, BooleanRule)):
|
|
98
|
+
field_selector = rule.field_selector
|
|
99
|
+
# AllFieldsSelector applies to both input and output, not exclusively output
|
|
100
|
+
# SpecificFieldsSelector: only return True if at least one field has OUTPUT source
|
|
101
|
+
if isinstance(field_selector, SpecificFieldsSelector):
|
|
102
|
+
if field_selector.fields and any(
|
|
103
|
+
field.source == FieldSource.OUTPUT
|
|
104
|
+
for field in field_selector.fields
|
|
105
|
+
):
|
|
106
|
+
return True
|
|
107
|
+
elif isinstance(field_selector, AllFieldsSelector):
|
|
108
|
+
if FieldSource.OUTPUT in field_selector.sources:
|
|
109
|
+
return True
|
|
110
|
+
|
|
111
|
+
return False
|
|
112
|
+
|
|
113
|
+
@staticmethod
|
|
114
|
+
def _evaluate_deterministic_guardrail(
|
|
115
|
+
input_data: dict[str, Any],
|
|
116
|
+
output_data: dict[str, Any],
|
|
117
|
+
guardrail: DeterministicGuardrail,
|
|
118
|
+
) -> GuardrailValidationResult:
|
|
119
|
+
"""Evaluate deterministic guardrail rules against input and output data."""
|
|
120
|
+
for rule in guardrail.rules:
|
|
121
|
+
if isinstance(rule, WordRule):
|
|
122
|
+
passed, reason = evaluate_word_rule(rule, input_data, output_data)
|
|
123
|
+
elif isinstance(rule, NumberRule):
|
|
124
|
+
passed, reason = evaluate_number_rule(rule, input_data, output_data)
|
|
125
|
+
elif isinstance(rule, BooleanRule):
|
|
126
|
+
passed, reason = evaluate_boolean_rule(rule, input_data, output_data)
|
|
127
|
+
elif isinstance(rule, UniversalRule):
|
|
128
|
+
passed, reason = evaluate_universal_rule(rule, output_data)
|
|
129
|
+
else:
|
|
130
|
+
return GuardrailValidationResult(
|
|
131
|
+
validation_passed=False,
|
|
132
|
+
reason=f"Unknown rule type: {type(rule)}",
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
if not passed:
|
|
136
|
+
return GuardrailValidationResult(
|
|
137
|
+
validation_passed=False, reason=reason or "Rule validation failed"
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
return GuardrailValidationResult(
|
|
141
|
+
validation_passed=True, reason="All deterministic guardrail rules passed"
|
|
142
|
+
)
|
|
@@ -120,23 +120,25 @@ def get_fields_from_selector(
|
|
|
120
120
|
fields: list[tuple[Any, FieldReference]] = []
|
|
121
121
|
|
|
122
122
|
if isinstance(field_selector, AllFieldsSelector):
|
|
123
|
-
# For "all" selector, we need to collect all fields from
|
|
123
|
+
# For "all" selector, we need to collect all fields from the specified sources
|
|
124
124
|
# This is a simplified implementation - in practice, you might want to
|
|
125
125
|
# recursively collect all nested fields
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
(
|
|
129
|
-
|
|
130
|
-
|
|
126
|
+
if FieldSource.INPUT in field_selector.sources:
|
|
127
|
+
for key, value in input_data.items():
|
|
128
|
+
fields.append(
|
|
129
|
+
(
|
|
130
|
+
value,
|
|
131
|
+
FieldReference(path=key, source=FieldSource.INPUT),
|
|
132
|
+
)
|
|
131
133
|
)
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
134
|
+
if FieldSource.OUTPUT in field_selector.sources:
|
|
135
|
+
for key, value in output_data.items():
|
|
136
|
+
fields.append(
|
|
137
|
+
(
|
|
138
|
+
value,
|
|
139
|
+
FieldReference(path=key, source=FieldSource.OUTPUT),
|
|
140
|
+
)
|
|
138
141
|
)
|
|
139
|
-
)
|
|
140
142
|
elif isinstance(field_selector, SpecificFieldsSelector):
|
|
141
143
|
# For specific fields, extract values based on field references
|
|
142
144
|
for field_ref in field_selector.fields:
|
{uipath_core-0.1.4 → uipath_core-0.1.5}/tests/guardrails/test_deterministic_guardrails_service.py
RENAMED
|
@@ -80,7 +80,10 @@ class TestDeterministicGuardrailsService:
|
|
|
80
80
|
)
|
|
81
81
|
|
|
82
82
|
assert result.validation_passed is True
|
|
83
|
-
assert
|
|
83
|
+
assert (
|
|
84
|
+
result.reason
|
|
85
|
+
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
86
|
+
)
|
|
84
87
|
|
|
85
88
|
def test_evaluate_post_deterministic_guardrail_validation_failed_age(
|
|
86
89
|
self,
|
|
@@ -115,6 +118,16 @@ class TestDeterministicGuardrailsService:
|
|
|
115
118
|
),
|
|
116
119
|
detects_violation=lambda b: b is not True,
|
|
117
120
|
),
|
|
121
|
+
NumberRule(
|
|
122
|
+
rule_type="number",
|
|
123
|
+
field_selector=SpecificFieldsSelector(
|
|
124
|
+
selector_type="specific",
|
|
125
|
+
fields=[
|
|
126
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
127
|
+
],
|
|
128
|
+
),
|
|
129
|
+
detects_violation=lambda n: n != 200.0,
|
|
130
|
+
),
|
|
118
131
|
],
|
|
119
132
|
)
|
|
120
133
|
|
|
@@ -171,6 +184,16 @@ class TestDeterministicGuardrailsService:
|
|
|
171
184
|
),
|
|
172
185
|
detects_violation=lambda b: b is not True,
|
|
173
186
|
),
|
|
187
|
+
NumberRule(
|
|
188
|
+
rule_type="number",
|
|
189
|
+
field_selector=SpecificFieldsSelector(
|
|
190
|
+
selector_type="specific",
|
|
191
|
+
fields=[
|
|
192
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
193
|
+
],
|
|
194
|
+
),
|
|
195
|
+
detects_violation=lambda n: n != 200.0,
|
|
196
|
+
),
|
|
174
197
|
],
|
|
175
198
|
)
|
|
176
199
|
|
|
@@ -180,7 +203,9 @@ class TestDeterministicGuardrailsService:
|
|
|
180
203
|
"age": 25,
|
|
181
204
|
"isActive": False,
|
|
182
205
|
}
|
|
183
|
-
output_data
|
|
206
|
+
output_data = {
|
|
207
|
+
"status": 200,
|
|
208
|
+
}
|
|
184
209
|
|
|
185
210
|
result = service.evaluate_post_deterministic_guardrail(
|
|
186
211
|
input_data=input_data,
|
|
@@ -235,7 +260,10 @@ class TestDeterministicGuardrailsService:
|
|
|
235
260
|
)
|
|
236
261
|
|
|
237
262
|
assert result.validation_passed is True
|
|
238
|
-
assert
|
|
263
|
+
assert (
|
|
264
|
+
result.reason
|
|
265
|
+
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
266
|
+
)
|
|
239
267
|
|
|
240
268
|
def test_evaluate_post_deterministic_guardrail_matches_regex_negative(
|
|
241
269
|
self,
|
|
@@ -262,6 +290,16 @@ class TestDeterministicGuardrailsService:
|
|
|
262
290
|
),
|
|
263
291
|
detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)),
|
|
264
292
|
),
|
|
293
|
+
NumberRule(
|
|
294
|
+
rule_type="number",
|
|
295
|
+
field_selector=SpecificFieldsSelector(
|
|
296
|
+
selector_type="specific",
|
|
297
|
+
fields=[
|
|
298
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
299
|
+
],
|
|
300
|
+
),
|
|
301
|
+
detects_violation=lambda n: n != 200.0,
|
|
302
|
+
),
|
|
265
303
|
],
|
|
266
304
|
)
|
|
267
305
|
|
|
@@ -269,7 +307,9 @@ class TestDeterministicGuardrailsService:
|
|
|
269
307
|
input_data = {
|
|
270
308
|
"userName": "test",
|
|
271
309
|
}
|
|
272
|
-
output_data
|
|
310
|
+
output_data = {
|
|
311
|
+
"status": 200,
|
|
312
|
+
}
|
|
273
313
|
|
|
274
314
|
result = service.evaluate_post_deterministic_guardrail(
|
|
275
315
|
input_data=input_data,
|
|
@@ -324,7 +364,10 @@ class TestDeterministicGuardrailsService:
|
|
|
324
364
|
)
|
|
325
365
|
|
|
326
366
|
assert result.validation_passed is True
|
|
327
|
-
assert
|
|
367
|
+
assert (
|
|
368
|
+
result.reason
|
|
369
|
+
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
370
|
+
)
|
|
328
371
|
|
|
329
372
|
def test_evaluate_post_deterministic_guardrail_word_func_negative(
|
|
330
373
|
self,
|
|
@@ -351,6 +394,16 @@ class TestDeterministicGuardrailsService:
|
|
|
351
394
|
),
|
|
352
395
|
detects_violation=lambda s: len(s) <= 5,
|
|
353
396
|
),
|
|
397
|
+
NumberRule(
|
|
398
|
+
rule_type="number",
|
|
399
|
+
field_selector=SpecificFieldsSelector(
|
|
400
|
+
selector_type="specific",
|
|
401
|
+
fields=[
|
|
402
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
403
|
+
],
|
|
404
|
+
),
|
|
405
|
+
detects_violation=lambda n: n != 200.0,
|
|
406
|
+
),
|
|
354
407
|
],
|
|
355
408
|
)
|
|
356
409
|
|
|
@@ -358,7 +411,9 @@ class TestDeterministicGuardrailsService:
|
|
|
358
411
|
input_data = {
|
|
359
412
|
"userName": "test",
|
|
360
413
|
}
|
|
361
|
-
output_data
|
|
414
|
+
output_data = {
|
|
415
|
+
"status": 200,
|
|
416
|
+
}
|
|
362
417
|
|
|
363
418
|
result = service.evaluate_post_deterministic_guardrail(
|
|
364
419
|
input_data=input_data,
|
|
@@ -393,6 +448,16 @@ class TestDeterministicGuardrailsService:
|
|
|
393
448
|
),
|
|
394
449
|
detects_violation=lambda s: "dre" in s,
|
|
395
450
|
),
|
|
451
|
+
NumberRule(
|
|
452
|
+
rule_type="number",
|
|
453
|
+
field_selector=SpecificFieldsSelector(
|
|
454
|
+
selector_type="specific",
|
|
455
|
+
fields=[
|
|
456
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
457
|
+
],
|
|
458
|
+
),
|
|
459
|
+
detects_violation=lambda n: n != 200.0,
|
|
460
|
+
),
|
|
396
461
|
],
|
|
397
462
|
)
|
|
398
463
|
|
|
@@ -400,7 +465,9 @@ class TestDeterministicGuardrailsService:
|
|
|
400
465
|
input_data = {
|
|
401
466
|
"userName": "andrei",
|
|
402
467
|
}
|
|
403
|
-
output_data
|
|
468
|
+
output_data = {
|
|
469
|
+
"status": 200,
|
|
470
|
+
}
|
|
404
471
|
|
|
405
472
|
result = service.evaluate_post_deterministic_guardrail(
|
|
406
473
|
input_data=input_data,
|
|
@@ -453,7 +520,10 @@ class TestDeterministicGuardrailsService:
|
|
|
453
520
|
)
|
|
454
521
|
|
|
455
522
|
assert result.validation_passed is True
|
|
456
|
-
assert
|
|
523
|
+
assert (
|
|
524
|
+
result.reason
|
|
525
|
+
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
526
|
+
)
|
|
457
527
|
|
|
458
528
|
def test_evaluate_post_deterministic_guardrail_number_func_negative(
|
|
459
529
|
self,
|
|
@@ -478,6 +548,16 @@ class TestDeterministicGuardrailsService:
|
|
|
478
548
|
),
|
|
479
549
|
detects_violation=lambda n: n < 18 or n > 65,
|
|
480
550
|
),
|
|
551
|
+
NumberRule(
|
|
552
|
+
rule_type="number",
|
|
553
|
+
field_selector=SpecificFieldsSelector(
|
|
554
|
+
selector_type="specific",
|
|
555
|
+
fields=[
|
|
556
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
557
|
+
],
|
|
558
|
+
),
|
|
559
|
+
detects_violation=lambda n: n != 200.0,
|
|
560
|
+
),
|
|
481
561
|
],
|
|
482
562
|
)
|
|
483
563
|
|
|
@@ -485,7 +565,9 @@ class TestDeterministicGuardrailsService:
|
|
|
485
565
|
input_data = {
|
|
486
566
|
"age": 70,
|
|
487
567
|
}
|
|
488
|
-
output_data
|
|
568
|
+
output_data = {
|
|
569
|
+
"status": 200,
|
|
570
|
+
}
|
|
489
571
|
|
|
490
572
|
result = service.evaluate_post_deterministic_guardrail(
|
|
491
573
|
input_data=input_data,
|
|
@@ -506,7 +588,9 @@ class TestDeterministicGuardrailsService:
|
|
|
506
588
|
"age": 18, # Less than 21
|
|
507
589
|
"isActive": True,
|
|
508
590
|
}
|
|
509
|
-
output_data
|
|
591
|
+
output_data = {
|
|
592
|
+
"status": 200,
|
|
593
|
+
}
|
|
510
594
|
|
|
511
595
|
result = service.evaluate_post_deterministic_guardrail(
|
|
512
596
|
input_data=input_data,
|
|
@@ -731,7 +815,9 @@ class TestDeterministicGuardrailsService:
|
|
|
731
815
|
rules=[
|
|
732
816
|
NumberRule(
|
|
733
817
|
rule_type="number",
|
|
734
|
-
field_selector=AllFieldsSelector(
|
|
818
|
+
field_selector=AllFieldsSelector(
|
|
819
|
+
selector_type="all", sources=[FieldSource.OUTPUT]
|
|
820
|
+
),
|
|
735
821
|
detects_violation=lambda n: n != 25.0,
|
|
736
822
|
),
|
|
737
823
|
],
|
|
@@ -756,7 +842,7 @@ class TestDeterministicGuardrailsService:
|
|
|
756
842
|
|
|
757
843
|
assert result.validation_passed is True
|
|
758
844
|
|
|
759
|
-
def
|
|
845
|
+
def test_should_trigger_policy_post_execution_with_all_fields_selector_empty_output_schema_returns_true(
|
|
760
846
|
self,
|
|
761
847
|
service: DeterministicGuardrailsService,
|
|
762
848
|
) -> None:
|
|
@@ -773,7 +859,9 @@ class TestDeterministicGuardrailsService:
|
|
|
773
859
|
rules=[
|
|
774
860
|
NumberRule(
|
|
775
861
|
rule_type="number",
|
|
776
|
-
field_selector=AllFieldsSelector(
|
|
862
|
+
field_selector=AllFieldsSelector(
|
|
863
|
+
selector_type="all", sources=[FieldSource.INPUT]
|
|
864
|
+
),
|
|
777
865
|
detects_violation=lambda n: n != 200.0,
|
|
778
866
|
),
|
|
779
867
|
],
|
|
@@ -792,7 +880,7 @@ class TestDeterministicGuardrailsService:
|
|
|
792
880
|
guardrail=guardrail,
|
|
793
881
|
)
|
|
794
882
|
|
|
795
|
-
assert result.validation_passed is
|
|
883
|
+
assert result.validation_passed is True
|
|
796
884
|
|
|
797
885
|
def test_should_trigger_policy_pre_execution_always_rule_with_input_apply_to_returns_true(
|
|
798
886
|
self,
|
|
@@ -805,11 +893,9 @@ class TestDeterministicGuardrailsService:
|
|
|
805
893
|
"age": 25,
|
|
806
894
|
"isActive": True,
|
|
807
895
|
}
|
|
808
|
-
output_data: dict[str, Any] = {}
|
|
809
896
|
|
|
810
|
-
result = service.
|
|
897
|
+
result = service.evaluate_pre_deterministic_guardrail(
|
|
811
898
|
input_data=input_data,
|
|
812
|
-
output_data=output_data,
|
|
813
899
|
guardrail=guardrail,
|
|
814
900
|
)
|
|
815
901
|
|
|
@@ -964,6 +1050,16 @@ class TestDeterministicGuardrailsService:
|
|
|
964
1050
|
),
|
|
965
1051
|
detects_violation=lambda b: b is not True,
|
|
966
1052
|
),
|
|
1053
|
+
NumberRule(
|
|
1054
|
+
rule_type="number",
|
|
1055
|
+
field_selector=SpecificFieldsSelector(
|
|
1056
|
+
selector_type="specific",
|
|
1057
|
+
fields=[
|
|
1058
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
1059
|
+
],
|
|
1060
|
+
),
|
|
1061
|
+
detects_violation=lambda n: n != 200.0,
|
|
1062
|
+
),
|
|
967
1063
|
],
|
|
968
1064
|
)
|
|
969
1065
|
|
|
@@ -1088,6 +1184,258 @@ class TestDeterministicGuardrailsService:
|
|
|
1088
1184
|
],
|
|
1089
1185
|
)
|
|
1090
1186
|
|
|
1187
|
+
def test_evaluate_post_deterministic_guardrail_word_contains_operator_passes(
|
|
1188
|
+
self,
|
|
1189
|
+
service: DeterministicGuardrailsService,
|
|
1190
|
+
) -> None:
|
|
1191
|
+
"""Test deterministic guardrail with word contains operator passes for pre-execution."""
|
|
1192
|
+
deterministic_guardrail = DeterministicGuardrail(
|
|
1193
|
+
id="b4283bd4-5ce0-49de-a918-2604d830460c",
|
|
1194
|
+
name="Before",
|
|
1195
|
+
description="",
|
|
1196
|
+
enabled_for_evals=True,
|
|
1197
|
+
guardrail_type="custom",
|
|
1198
|
+
selector=GuardrailSelector(
|
|
1199
|
+
scopes=[GuardrailScope.TOOL], match_names=["ConverterToStringAgent"]
|
|
1200
|
+
),
|
|
1201
|
+
rules=[
|
|
1202
|
+
WordRule(
|
|
1203
|
+
rule_type="word",
|
|
1204
|
+
field_selector=SpecificFieldsSelector(
|
|
1205
|
+
selector_type="specific",
|
|
1206
|
+
fields=[
|
|
1207
|
+
FieldReference(
|
|
1208
|
+
path="input_string", source=FieldSource.INPUT
|
|
1209
|
+
)
|
|
1210
|
+
],
|
|
1211
|
+
),
|
|
1212
|
+
detects_violation=lambda s: "cti" in s,
|
|
1213
|
+
),
|
|
1214
|
+
],
|
|
1215
|
+
)
|
|
1216
|
+
|
|
1217
|
+
# Input data without "cti" in input_string - should pass
|
|
1218
|
+
input_data = {
|
|
1219
|
+
"input_string": "test value",
|
|
1220
|
+
}
|
|
1221
|
+
output_data: dict[str, Any] = {}
|
|
1222
|
+
|
|
1223
|
+
result = service.evaluate_post_deterministic_guardrail(
|
|
1224
|
+
input_data=input_data,
|
|
1225
|
+
output_data=output_data,
|
|
1226
|
+
guardrail=deterministic_guardrail,
|
|
1227
|
+
)
|
|
1228
|
+
|
|
1229
|
+
assert result.validation_passed is True
|
|
1230
|
+
assert (
|
|
1231
|
+
result.reason
|
|
1232
|
+
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
1233
|
+
)
|
|
1234
|
+
|
|
1235
|
+
def test_evaluate_post_deterministic_guardrail_only_output_rules_passes(
|
|
1236
|
+
self,
|
|
1237
|
+
service: DeterministicGuardrailsService,
|
|
1238
|
+
) -> None:
|
|
1239
|
+
"""Test post guardrail with only output rules passes when conditions are met."""
|
|
1240
|
+
deterministic_guardrail = DeterministicGuardrail(
|
|
1241
|
+
id="test-only-output-id",
|
|
1242
|
+
name="Output Only Guardrail",
|
|
1243
|
+
description="Test guardrail with only output rules",
|
|
1244
|
+
enabled_for_evals=True,
|
|
1245
|
+
guardrail_type="custom",
|
|
1246
|
+
selector=GuardrailSelector(
|
|
1247
|
+
scopes=[GuardrailScope.TOOL], match_names=["test"]
|
|
1248
|
+
),
|
|
1249
|
+
rules=[
|
|
1250
|
+
NumberRule(
|
|
1251
|
+
rule_type="number",
|
|
1252
|
+
field_selector=SpecificFieldsSelector(
|
|
1253
|
+
selector_type="specific",
|
|
1254
|
+
fields=[
|
|
1255
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
1256
|
+
],
|
|
1257
|
+
),
|
|
1258
|
+
detects_violation=lambda n: n != 200.0,
|
|
1259
|
+
),
|
|
1260
|
+
WordRule(
|
|
1261
|
+
rule_type="word",
|
|
1262
|
+
field_selector=SpecificFieldsSelector(
|
|
1263
|
+
selector_type="specific",
|
|
1264
|
+
fields=[
|
|
1265
|
+
FieldReference(path="result", source=FieldSource.OUTPUT)
|
|
1266
|
+
],
|
|
1267
|
+
),
|
|
1268
|
+
detects_violation=lambda s: s != "Success",
|
|
1269
|
+
),
|
|
1270
|
+
],
|
|
1271
|
+
)
|
|
1272
|
+
|
|
1273
|
+
input_data = {
|
|
1274
|
+
"userName": "John",
|
|
1275
|
+
}
|
|
1276
|
+
output_data = {
|
|
1277
|
+
"status": 200,
|
|
1278
|
+
"result": "Success",
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
result = service.evaluate_post_deterministic_guardrail(
|
|
1282
|
+
input_data=input_data,
|
|
1283
|
+
output_data=output_data,
|
|
1284
|
+
guardrail=deterministic_guardrail,
|
|
1285
|
+
)
|
|
1286
|
+
|
|
1287
|
+
assert result.validation_passed is True
|
|
1288
|
+
assert result.reason == "All deterministic guardrail rules passed"
|
|
1289
|
+
|
|
1290
|
+
def test_evaluate_post_deterministic_guardrail_only_always_rule_fails(
|
|
1291
|
+
self,
|
|
1292
|
+
service: DeterministicGuardrailsService,
|
|
1293
|
+
) -> None:
|
|
1294
|
+
"""Test post guardrail with only UniversalRule always fails."""
|
|
1295
|
+
deterministic_guardrail = DeterministicGuardrail(
|
|
1296
|
+
id="test-only-always-id",
|
|
1297
|
+
name="Always Rule Only Guardrail",
|
|
1298
|
+
description="Test guardrail with only always rule",
|
|
1299
|
+
enabled_for_evals=True,
|
|
1300
|
+
guardrail_type="custom",
|
|
1301
|
+
selector=GuardrailSelector(
|
|
1302
|
+
scopes=[GuardrailScope.TOOL], match_names=["test"]
|
|
1303
|
+
),
|
|
1304
|
+
rules=[
|
|
1305
|
+
UniversalRule(
|
|
1306
|
+
rule_type="always",
|
|
1307
|
+
apply_to=ApplyTo.OUTPUT,
|
|
1308
|
+
),
|
|
1309
|
+
],
|
|
1310
|
+
)
|
|
1311
|
+
|
|
1312
|
+
input_data = {
|
|
1313
|
+
"userName": "John",
|
|
1314
|
+
}
|
|
1315
|
+
output_data = {
|
|
1316
|
+
"status": 200,
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
result = service.evaluate_post_deterministic_guardrail(
|
|
1320
|
+
input_data=input_data,
|
|
1321
|
+
output_data=output_data,
|
|
1322
|
+
guardrail=deterministic_guardrail,
|
|
1323
|
+
)
|
|
1324
|
+
|
|
1325
|
+
assert result.validation_passed is False
|
|
1326
|
+
|
|
1327
|
+
def test_evaluate_post_deterministic_guardrail_only_input_rules_passes(
|
|
1328
|
+
self,
|
|
1329
|
+
service: DeterministicGuardrailsService,
|
|
1330
|
+
) -> None:
|
|
1331
|
+
"""Test post guardrail passes when only input rules exist (no output data required)."""
|
|
1332
|
+
deterministic_guardrail = DeterministicGuardrail(
|
|
1333
|
+
id="test-only-input-id",
|
|
1334
|
+
name="Input Only Guardrail",
|
|
1335
|
+
description="Test guardrail with only input rules",
|
|
1336
|
+
enabled_for_evals=True,
|
|
1337
|
+
guardrail_type="custom",
|
|
1338
|
+
selector=GuardrailSelector(
|
|
1339
|
+
scopes=[GuardrailScope.TOOL], match_names=["test"]
|
|
1340
|
+
),
|
|
1341
|
+
rules=[
|
|
1342
|
+
NumberRule(
|
|
1343
|
+
rule_type="number",
|
|
1344
|
+
field_selector=SpecificFieldsSelector(
|
|
1345
|
+
selector_type="specific",
|
|
1346
|
+
fields=[FieldReference(path="age", source=FieldSource.INPUT)],
|
|
1347
|
+
),
|
|
1348
|
+
detects_violation=lambda n: n < 18,
|
|
1349
|
+
),
|
|
1350
|
+
WordRule(
|
|
1351
|
+
rule_type="word",
|
|
1352
|
+
field_selector=SpecificFieldsSelector(
|
|
1353
|
+
selector_type="specific",
|
|
1354
|
+
fields=[FieldReference(path="name", source=FieldSource.INPUT)],
|
|
1355
|
+
),
|
|
1356
|
+
detects_violation=lambda s: len(s) < 2,
|
|
1357
|
+
),
|
|
1358
|
+
],
|
|
1359
|
+
)
|
|
1360
|
+
|
|
1361
|
+
input_data = {
|
|
1362
|
+
"age": 25,
|
|
1363
|
+
"name": "John",
|
|
1364
|
+
}
|
|
1365
|
+
output_data: dict[str, Any] = {}
|
|
1366
|
+
|
|
1367
|
+
result = service.evaluate_post_deterministic_guardrail(
|
|
1368
|
+
input_data=input_data,
|
|
1369
|
+
output_data=output_data,
|
|
1370
|
+
guardrail=deterministic_guardrail,
|
|
1371
|
+
)
|
|
1372
|
+
|
|
1373
|
+
assert result.validation_passed is True
|
|
1374
|
+
assert (
|
|
1375
|
+
result.reason
|
|
1376
|
+
== "Guardrail contains only input-dependent rules that were evaluated during pre-execution"
|
|
1377
|
+
)
|
|
1378
|
+
|
|
1379
|
+
def test_evaluate_pre_deterministic_guardrail_with_input_and_output_rules_input_true(
|
|
1380
|
+
self,
|
|
1381
|
+
service: DeterministicGuardrailsService,
|
|
1382
|
+
) -> None:
|
|
1383
|
+
"""Test pre-execution guardrail with input rule and output rules, should pass because is ignored."""
|
|
1384
|
+
deterministic_guardrail = DeterministicGuardrail(
|
|
1385
|
+
id="test-pre-mixed-rules-id",
|
|
1386
|
+
name="Pre Execution Mixed Rules Guardrail",
|
|
1387
|
+
description="Test pre-execution with both input and output rules",
|
|
1388
|
+
enabled_for_evals=True,
|
|
1389
|
+
guardrail_type="custom",
|
|
1390
|
+
selector=GuardrailSelector(
|
|
1391
|
+
scopes=[GuardrailScope.TOOL], match_names=["test"]
|
|
1392
|
+
),
|
|
1393
|
+
rules=[
|
|
1394
|
+
NumberRule(
|
|
1395
|
+
rule_type="number",
|
|
1396
|
+
field_selector=SpecificFieldsSelector(
|
|
1397
|
+
selector_type="specific",
|
|
1398
|
+
fields=[FieldReference(path="age", source=FieldSource.INPUT)],
|
|
1399
|
+
),
|
|
1400
|
+
detects_violation=lambda n: n < 21.0,
|
|
1401
|
+
),
|
|
1402
|
+
BooleanRule(
|
|
1403
|
+
rule_type="boolean",
|
|
1404
|
+
field_selector=SpecificFieldsSelector(
|
|
1405
|
+
selector_type="specific",
|
|
1406
|
+
fields=[
|
|
1407
|
+
FieldReference(path="isActive", source=FieldSource.INPUT)
|
|
1408
|
+
],
|
|
1409
|
+
),
|
|
1410
|
+
detects_violation=lambda b: b is not True,
|
|
1411
|
+
),
|
|
1412
|
+
# Output rule - should be ignored in pre-execution
|
|
1413
|
+
NumberRule(
|
|
1414
|
+
rule_type="number",
|
|
1415
|
+
field_selector=SpecificFieldsSelector(
|
|
1416
|
+
selector_type="specific",
|
|
1417
|
+
fields=[
|
|
1418
|
+
FieldReference(path="status", source=FieldSource.OUTPUT)
|
|
1419
|
+
],
|
|
1420
|
+
),
|
|
1421
|
+
detects_violation=lambda n: n != 200.0,
|
|
1422
|
+
),
|
|
1423
|
+
],
|
|
1424
|
+
)
|
|
1425
|
+
|
|
1426
|
+
input_data = {
|
|
1427
|
+
"userName": "John",
|
|
1428
|
+
"age": 18,
|
|
1429
|
+
"isActive": True,
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
result = service.evaluate_pre_deterministic_guardrail(
|
|
1433
|
+
input_data=input_data,
|
|
1434
|
+
guardrail=deterministic_guardrail,
|
|
1435
|
+
)
|
|
1436
|
+
|
|
1437
|
+
assert result.validation_passed is True
|
|
1438
|
+
|
|
1091
1439
|
def _create_guardrail_with_always_rule(
|
|
1092
1440
|
self, apply_to: ApplyTo
|
|
1093
1441
|
) -> DeterministicGuardrail:
|
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
from typing import Any
|
|
2
|
-
|
|
3
|
-
from pydantic import BaseModel
|
|
4
|
-
|
|
5
|
-
from ..tracing.decorators import traced
|
|
6
|
-
from ._evaluators import (
|
|
7
|
-
evaluate_boolean_rule,
|
|
8
|
-
evaluate_number_rule,
|
|
9
|
-
evaluate_universal_rule,
|
|
10
|
-
evaluate_word_rule,
|
|
11
|
-
)
|
|
12
|
-
from .guardrails import (
|
|
13
|
-
BooleanRule,
|
|
14
|
-
DeterministicGuardrail,
|
|
15
|
-
GuardrailValidationResult,
|
|
16
|
-
NumberRule,
|
|
17
|
-
UniversalRule,
|
|
18
|
-
WordRule,
|
|
19
|
-
)
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
class DeterministicGuardrailsService(BaseModel):
|
|
23
|
-
@traced("evaluate_pre_deterministic_guardrail", run_type="uipath")
|
|
24
|
-
def evaluate_pre_deterministic_guardrail(
|
|
25
|
-
self,
|
|
26
|
-
input_data: dict[str, Any],
|
|
27
|
-
guardrail: DeterministicGuardrail,
|
|
28
|
-
) -> GuardrailValidationResult:
|
|
29
|
-
"""Evaluate deterministic guardrail rules against input data (pre-execution)."""
|
|
30
|
-
return self._evaluate_deterministic_guardrail(
|
|
31
|
-
input_data=input_data,
|
|
32
|
-
output_data={},
|
|
33
|
-
guardrail=guardrail,
|
|
34
|
-
)
|
|
35
|
-
|
|
36
|
-
@traced("evaluate_post_deterministic_guardrails", run_type="uipath")
|
|
37
|
-
def evaluate_post_deterministic_guardrail(
|
|
38
|
-
self,
|
|
39
|
-
input_data: dict[str, Any],
|
|
40
|
-
output_data: dict[str, Any],
|
|
41
|
-
guardrail: DeterministicGuardrail,
|
|
42
|
-
) -> GuardrailValidationResult:
|
|
43
|
-
"""Evaluate deterministic guardrail rules against input and output data."""
|
|
44
|
-
return self._evaluate_deterministic_guardrail(
|
|
45
|
-
input_data=input_data,
|
|
46
|
-
output_data=output_data,
|
|
47
|
-
guardrail=guardrail,
|
|
48
|
-
)
|
|
49
|
-
|
|
50
|
-
@staticmethod
|
|
51
|
-
def _evaluate_deterministic_guardrail(
|
|
52
|
-
input_data: dict[str, Any],
|
|
53
|
-
output_data: dict[str, Any],
|
|
54
|
-
guardrail: DeterministicGuardrail,
|
|
55
|
-
) -> GuardrailValidationResult:
|
|
56
|
-
"""Evaluate deterministic guardrail rules against input and output data."""
|
|
57
|
-
for rule in guardrail.rules:
|
|
58
|
-
if isinstance(rule, WordRule):
|
|
59
|
-
passed, reason = evaluate_word_rule(rule, input_data, output_data)
|
|
60
|
-
elif isinstance(rule, NumberRule):
|
|
61
|
-
passed, reason = evaluate_number_rule(rule, input_data, output_data)
|
|
62
|
-
elif isinstance(rule, BooleanRule):
|
|
63
|
-
passed, reason = evaluate_boolean_rule(rule, input_data, output_data)
|
|
64
|
-
elif isinstance(rule, UniversalRule):
|
|
65
|
-
passed, reason = evaluate_universal_rule(rule, output_data)
|
|
66
|
-
else:
|
|
67
|
-
return GuardrailValidationResult(
|
|
68
|
-
validation_passed=False,
|
|
69
|
-
reason=f"Unknown rule type: {type(rule)}",
|
|
70
|
-
)
|
|
71
|
-
|
|
72
|
-
if not passed:
|
|
73
|
-
return GuardrailValidationResult(
|
|
74
|
-
validation_passed=False, reason=reason or "Rule validation failed"
|
|
75
|
-
)
|
|
76
|
-
|
|
77
|
-
return GuardrailValidationResult(
|
|
78
|
-
validation_passed=True, reason="All deterministic guardrail rules passed"
|
|
79
|
-
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|