arthur-common 2.1.60__tar.gz → 2.1.66__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of arthur-common might be problematic. Click here for more details.
- {arthur_common-2.1.60 → arthur_common-2.1.66}/PKG-INFO +6 -6
- {arthur_common-2.1.60 → arthur_common-2.1.66}/README.md +4 -5
- {arthur_common-2.1.60 → arthur_common-2.1.66}/pyproject.toml +2 -1
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/agentic_aggregations.py +1 -1
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/confusion_matrix.py +1 -1
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/inference_count_by_class.py +1 -1
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/mean_absolute_error.py +1 -1
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/mean_squared_error.py +1 -1
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/multiclass_confusion_matrix.py +1 -1
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/multiclass_inference_count_by_class.py +1 -1
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/shield_aggregations.py +1 -1
- arthur_common-2.1.66/src/arthur_common/models/common_schemas.py +208 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/models/connectors.py +3 -3
- arthur_common-2.1.66/src/arthur_common/models/constants.py +24 -0
- arthur_common-2.1.66/src/arthur_common/models/datasets.py +14 -0
- arthur_common-2.1.66/src/arthur_common/models/enums.py +144 -0
- arthur_common-2.1.66/src/arthur_common/models/metric_schemas.py +63 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/models/metrics.py +1 -1
- arthur_common-2.1.60/src/arthur_common/models/shield.py → arthur_common-2.1.66/src/arthur_common/models/request_schemas.py +212 -298
- arthur_common-2.1.66/src/arthur_common/models/response_schemas.py +682 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/models/schema_definitions.py +1 -1
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/models/task_job_specs.py +3 -12
- arthur_common-2.1.60/src/arthur_common/models/datasets.py +0 -23
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/__init__.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/__init__.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/aggregator.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/README.md +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/__init__.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/categorical_count.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/inference_count.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/inference_null_count.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/numeric_stats.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/numeric_sum.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/functions/py.typed +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/aggregations/py.typed +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/config/__init__.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/config/config.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/config/settings.yaml +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/models/__init__.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/models/py.typed +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/py.typed +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/tools/__init__.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/tools/aggregation_analyzer.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/tools/aggregation_loader.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/tools/duckdb_data_loader.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/tools/duckdb_utils.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/tools/functions.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/tools/py.typed +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/tools/schema_inferer.py +0 -0
- {arthur_common-2.1.60 → arthur_common-2.1.66}/src/arthur_common/tools/time_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: arthur-common
|
|
3
|
-
Version: 2.1.
|
|
3
|
+
Version: 2.1.66
|
|
4
4
|
Summary: Utility code common to Arthur platform components.
|
|
5
5
|
License: MIT
|
|
6
6
|
Author: Arthur
|
|
@@ -14,6 +14,7 @@ Requires-Dist: datasketches (>=5.1.0)
|
|
|
14
14
|
Requires-Dist: duckdb (>=1.1.3)
|
|
15
15
|
Requires-Dist: fastapi (>=0.115.8)
|
|
16
16
|
Requires-Dist: fsspec (>=2024.10.0)
|
|
17
|
+
Requires-Dist: openinference-semantic-conventions (>=0.1.12,<0.2.0)
|
|
17
18
|
Requires-Dist: pandas (>=2.2.2)
|
|
18
19
|
Requires-Dist: pydantic (>=2)
|
|
19
20
|
Requires-Dist: simple-settings (>=1.2.0)
|
|
@@ -63,12 +64,11 @@ poetry run pytest
|
|
|
63
64
|
```
|
|
64
65
|
|
|
65
66
|
## Release process
|
|
66
|
-
1. Merge changes into
|
|
67
|
+
1. Merge changes into `main` branch
|
|
67
68
|
2. Go to **Actions** -> **Arthur Common Version Bump**
|
|
68
|
-
3.
|
|
69
|
-
4.
|
|
70
|
-
5.
|
|
71
|
-
6. Update package version in your project (arthur-engine)
|
|
69
|
+
3. Click **Run workflow**. The workflow will create a new commit with the version bump, push it back to the same branch it is triggered on (default `main`), and start the release process
|
|
70
|
+
4. Watch in [GitHub Actions](https://github.com/arthur-ai/arthur-common/actions) for Arthur Common Release to run
|
|
71
|
+
5. Update package version in your project (arthur-engine)
|
|
72
72
|
|
|
73
73
|
## License
|
|
74
74
|
|
|
@@ -38,12 +38,11 @@ poetry run pytest
|
|
|
38
38
|
```
|
|
39
39
|
|
|
40
40
|
## Release process
|
|
41
|
-
1. Merge changes into
|
|
41
|
+
1. Merge changes into `main` branch
|
|
42
42
|
2. Go to **Actions** -> **Arthur Common Version Bump**
|
|
43
|
-
3.
|
|
44
|
-
4.
|
|
45
|
-
5.
|
|
46
|
-
6. Update package version in your project (arthur-engine)
|
|
43
|
+
3. Click **Run workflow**. The workflow will create a new commit with the version bump, push it back to the same branch it is triggered on (default `main`), and start the release process
|
|
44
|
+
4. Watch in [GitHub Actions](https://github.com/arthur-ai/arthur-common/actions) for Arthur Common Release to run
|
|
45
|
+
5. Update package version in your project (arthur-engine)
|
|
47
46
|
|
|
48
47
|
## License
|
|
49
48
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "arthur-common"
|
|
3
|
-
version = "2.1.
|
|
3
|
+
version = "2.1.66"
|
|
4
4
|
description = "Utility code common to Arthur platform components."
|
|
5
5
|
authors = ["Arthur <engineering@arthur.ai>"]
|
|
6
6
|
license = "MIT"
|
|
@@ -19,6 +19,7 @@ fsspec = ">=2024.10.0"
|
|
|
19
19
|
tokencost = "0.1.24"
|
|
20
20
|
fastapi = ">=0.115.8"
|
|
21
21
|
simple-settings = ">=1.2.0"
|
|
22
|
+
openinference-semantic-conventions = "^0.1.12"
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
[tool.poetry.group.dev.dependencies]
|
|
@@ -10,7 +10,7 @@ from arthur_common.aggregations.aggregator import (
|
|
|
10
10
|
NumericAggregationFunction,
|
|
11
11
|
SketchAggregationFunction,
|
|
12
12
|
)
|
|
13
|
-
from arthur_common.models.
|
|
13
|
+
from arthur_common.models.enums import ModelProblemType
|
|
14
14
|
from arthur_common.models.metrics import (
|
|
15
15
|
BaseReportedAggregation,
|
|
16
16
|
DatasetReference,
|
|
@@ -4,7 +4,7 @@ from uuid import UUID
|
|
|
4
4
|
from duckdb import DuckDBPyConnection
|
|
5
5
|
|
|
6
6
|
from arthur_common.aggregations.aggregator import NumericAggregationFunction
|
|
7
|
-
from arthur_common.models.
|
|
7
|
+
from arthur_common.models.enums import ModelProblemType
|
|
8
8
|
from arthur_common.models.metrics import (
|
|
9
9
|
BaseReportedAggregation,
|
|
10
10
|
DatasetReference,
|
|
@@ -4,7 +4,7 @@ from uuid import UUID
|
|
|
4
4
|
from duckdb import DuckDBPyConnection
|
|
5
5
|
|
|
6
6
|
from arthur_common.aggregations.aggregator import NumericAggregationFunction
|
|
7
|
-
from arthur_common.models.
|
|
7
|
+
from arthur_common.models.enums import ModelProblemType
|
|
8
8
|
from arthur_common.models.metrics import (
|
|
9
9
|
BaseReportedAggregation,
|
|
10
10
|
DatasetReference,
|
|
@@ -4,7 +4,7 @@ from uuid import UUID
|
|
|
4
4
|
from duckdb import DuckDBPyConnection
|
|
5
5
|
|
|
6
6
|
from arthur_common.aggregations.aggregator import NumericAggregationFunction
|
|
7
|
-
from arthur_common.models.
|
|
7
|
+
from arthur_common.models.enums import ModelProblemType
|
|
8
8
|
from arthur_common.models.metrics import (
|
|
9
9
|
BaseReportedAggregation,
|
|
10
10
|
DatasetReference,
|
|
@@ -4,7 +4,7 @@ from uuid import UUID
|
|
|
4
4
|
from duckdb import DuckDBPyConnection
|
|
5
5
|
|
|
6
6
|
from arthur_common.aggregations.aggregator import NumericAggregationFunction
|
|
7
|
-
from arthur_common.models.
|
|
7
|
+
from arthur_common.models.enums import ModelProblemType
|
|
8
8
|
from arthur_common.models.metrics import (
|
|
9
9
|
BaseReportedAggregation,
|
|
10
10
|
DatasetReference,
|
|
@@ -4,7 +4,7 @@ from uuid import UUID
|
|
|
4
4
|
from duckdb import DuckDBPyConnection
|
|
5
5
|
|
|
6
6
|
from arthur_common.aggregations.aggregator import NumericAggregationFunction
|
|
7
|
-
from arthur_common.models.
|
|
7
|
+
from arthur_common.models.enums import ModelProblemType
|
|
8
8
|
from arthur_common.models.metrics import (
|
|
9
9
|
BaseReportedAggregation,
|
|
10
10
|
DatasetReference,
|
|
@@ -6,7 +6,7 @@ from duckdb import DuckDBPyConnection
|
|
|
6
6
|
from arthur_common.aggregations.functions.inference_count_by_class import (
|
|
7
7
|
BinaryClassifierCountByClassAggregationFunction,
|
|
8
8
|
)
|
|
9
|
-
from arthur_common.models.
|
|
9
|
+
from arthur_common.models.enums import ModelProblemType
|
|
10
10
|
from arthur_common.models.metrics import (
|
|
11
11
|
BaseReportedAggregation,
|
|
12
12
|
DatasetReference,
|
|
@@ -9,7 +9,7 @@ from arthur_common.aggregations.aggregator import (
|
|
|
9
9
|
NumericAggregationFunction,
|
|
10
10
|
SketchAggregationFunction,
|
|
11
11
|
)
|
|
12
|
-
from arthur_common.models.
|
|
12
|
+
from arthur_common.models.enums import ModelProblemType
|
|
13
13
|
from arthur_common.models.metrics import (
|
|
14
14
|
BaseReportedAggregation,
|
|
15
15
|
DatasetReference,
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any, Dict, List, Optional
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
|
6
|
+
|
|
7
|
+
from arthur_common.models.constants import (
|
|
8
|
+
DEFAULT_PII_RULE_CONFIDENCE_SCORE_THRESHOLD,
|
|
9
|
+
DEFAULT_TOXICITY_RULE_THRESHOLD,
|
|
10
|
+
NEGATIVE_BLOOD_EXAMPLE,
|
|
11
|
+
)
|
|
12
|
+
from arthur_common.models.enums import (
|
|
13
|
+
PaginationSortMethod,
|
|
14
|
+
PIIEntityTypes,
|
|
15
|
+
UserPermissionAction,
|
|
16
|
+
UserPermissionResource,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class AuthUserRole(BaseModel):
|
|
21
|
+
id: str | None = None
|
|
22
|
+
name: str
|
|
23
|
+
description: str
|
|
24
|
+
composite: bool
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class ExampleConfig(BaseModel):
|
|
28
|
+
example: str = Field(description="Custom example for the sensitive data")
|
|
29
|
+
result: bool = Field(
|
|
30
|
+
description="Boolean value representing if the example passes or fails the the sensitive "
|
|
31
|
+
"data rule ",
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
model_config = ConfigDict(
|
|
35
|
+
json_schema_extra={
|
|
36
|
+
"example": {"example": NEGATIVE_BLOOD_EXAMPLE, "result": True},
|
|
37
|
+
},
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ExamplesConfig(BaseModel):
|
|
42
|
+
examples: List[ExampleConfig] = Field(
|
|
43
|
+
description="List of all the examples for Sensitive Data Rule",
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
model_config = ConfigDict(
|
|
47
|
+
json_schema_extra={
|
|
48
|
+
"example": {
|
|
49
|
+
"examples": [
|
|
50
|
+
{"example": NEGATIVE_BLOOD_EXAMPLE, "result": True},
|
|
51
|
+
{
|
|
52
|
+
"example": "Most of the people have A positive blood group",
|
|
53
|
+
"result": False,
|
|
54
|
+
},
|
|
55
|
+
],
|
|
56
|
+
"hint": "specific individual's blood type",
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
)
|
|
60
|
+
hint: Optional[str] = Field(
|
|
61
|
+
description="Optional. Hint added to describe what Sensitive Data Rule should be checking for",
|
|
62
|
+
default=None,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
66
|
+
d = self.__dict__
|
|
67
|
+
d["examples"] = [ex.__dict__ for ex in self.examples]
|
|
68
|
+
d["hint"] = self.hint
|
|
69
|
+
return d
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class KeywordsConfig(BaseModel):
|
|
73
|
+
keywords: List[str] = Field(description="List of Keywords")
|
|
74
|
+
|
|
75
|
+
model_config = ConfigDict(
|
|
76
|
+
json_schema_extra={
|
|
77
|
+
"example": {"keywords": ["Blocked_Keyword_1", "Blocked_Keyword_2"]},
|
|
78
|
+
},
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class LLMTokenConsumption(BaseModel):
|
|
83
|
+
prompt_tokens: int
|
|
84
|
+
completion_tokens: int
|
|
85
|
+
|
|
86
|
+
def total_tokens(self) -> int:
|
|
87
|
+
return self.prompt_tokens + self.completion_tokens
|
|
88
|
+
|
|
89
|
+
def add(self, token_consumption: LLMTokenConsumption) -> "LLMTokenConsumption":
|
|
90
|
+
self.prompt_tokens += token_consumption.prompt_tokens
|
|
91
|
+
self.completion_tokens += token_consumption.completion_tokens
|
|
92
|
+
return self
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class PaginationParameters(BaseModel):
|
|
96
|
+
sort: Optional[PaginationSortMethod] = PaginationSortMethod.DESCENDING
|
|
97
|
+
page_size: int = 10
|
|
98
|
+
page: int = 0
|
|
99
|
+
|
|
100
|
+
def calculate_total_pages(self, total_items_count: int) -> int:
|
|
101
|
+
return total_items_count // self.page_size + 1
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class PIIConfig(BaseModel):
|
|
105
|
+
disabled_pii_entities: Optional[list[str]] = Field(
|
|
106
|
+
description=f"Optional. List of PII entities to disable. Valid values are: {PIIEntityTypes.to_string()}",
|
|
107
|
+
default=None,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
confidence_threshold: Optional[float] = Field(
|
|
111
|
+
description=f"Optional. Float (0, 1) indicating the level of tolerable PII to consider the rule passed or failed. Min: 0 (less confident) Max: 1 (very confident). Default: {DEFAULT_PII_RULE_CONFIDENCE_SCORE_THRESHOLD}",
|
|
112
|
+
default=DEFAULT_PII_RULE_CONFIDENCE_SCORE_THRESHOLD,
|
|
113
|
+
json_schema_extra={"deprecated": True},
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
allow_list: Optional[list[str]] = Field(
|
|
117
|
+
description="Optional. List of strings to pass PII validation.",
|
|
118
|
+
default=None,
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
@field_validator("disabled_pii_entities")
|
|
122
|
+
def validate_pii_entities(cls, v: list[str] | None) -> list[str] | None:
|
|
123
|
+
if v:
|
|
124
|
+
entities_passed = set(v)
|
|
125
|
+
entities_supported = set(PIIEntityTypes.values())
|
|
126
|
+
invalid_entities = entities_passed - entities_supported
|
|
127
|
+
if invalid_entities:
|
|
128
|
+
raise ValueError(
|
|
129
|
+
f"The following values are not valid PII entities: {invalid_entities}",
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
# Fail the case where they are trying to disable all PII entity types
|
|
133
|
+
if (not invalid_entities) & (
|
|
134
|
+
len(entities_passed) == len(entities_supported)
|
|
135
|
+
):
|
|
136
|
+
raise ValueError(
|
|
137
|
+
f"Cannot disable all supported PII entities on PIIDataRule",
|
|
138
|
+
)
|
|
139
|
+
return v
|
|
140
|
+
else:
|
|
141
|
+
return v
|
|
142
|
+
|
|
143
|
+
@field_validator("confidence_threshold")
|
|
144
|
+
def validate_confidence_threshold(cls, v: float | None) -> float | None:
|
|
145
|
+
if v:
|
|
146
|
+
if (v < 0) | (v > 1):
|
|
147
|
+
raise ValueError(f'"confidence_threshold" must be between 0 and 1')
|
|
148
|
+
return v
|
|
149
|
+
else:
|
|
150
|
+
return v
|
|
151
|
+
|
|
152
|
+
model_config = ConfigDict(
|
|
153
|
+
json_schema_extra={
|
|
154
|
+
"example": {
|
|
155
|
+
"disabled_pii_entities": ["PERSON", "URL"],
|
|
156
|
+
"confidence_threshold": "0.5",
|
|
157
|
+
"allow_list": ["arthur.ai", "Arthur"],
|
|
158
|
+
},
|
|
159
|
+
},
|
|
160
|
+
extra="forbid",
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class RegexConfig(BaseModel):
|
|
165
|
+
regex_patterns: List[str] = Field(
|
|
166
|
+
description="List of Regex patterns to be used for validation. Be sure to encode requests in JSON and account for escape characters.",
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
model_config = ConfigDict(
|
|
170
|
+
json_schema_extra={
|
|
171
|
+
"example": {
|
|
172
|
+
"regex_patterns": ["\\d{3}-\\d{2}-\\d{4}", "\\d{5}-\\d{6}-\\d{7}"],
|
|
173
|
+
},
|
|
174
|
+
},
|
|
175
|
+
extra="forbid",
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class ToxicityConfig(BaseModel):
|
|
180
|
+
threshold: float = Field(
|
|
181
|
+
default=DEFAULT_TOXICITY_RULE_THRESHOLD,
|
|
182
|
+
description=f"Optional. Float (0, 1) indicating the level of tolerable toxicity to consider the rule passed or failed. Min: 0 (no toxic language) Max: 1 (very toxic language). Default: {DEFAULT_TOXICITY_RULE_THRESHOLD}",
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
model_config = ConfigDict(
|
|
186
|
+
extra="forbid",
|
|
187
|
+
json_schema_extra={"example": {"threshold": DEFAULT_TOXICITY_RULE_THRESHOLD}},
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
@field_validator("threshold", mode="before")
|
|
191
|
+
@classmethod
|
|
192
|
+
def validate_toxicity_threshold(cls, v: float | None) -> float:
|
|
193
|
+
if v is None:
|
|
194
|
+
return float(DEFAULT_TOXICITY_RULE_THRESHOLD)
|
|
195
|
+
if (v < 0) | (v > 1):
|
|
196
|
+
raise ValueError(f'"threshold" must be between 0 and 1')
|
|
197
|
+
return v
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
class UserPermission(BaseModel):
|
|
201
|
+
action: UserPermissionAction
|
|
202
|
+
resource: UserPermissionResource
|
|
203
|
+
|
|
204
|
+
def __hash__(self) -> int:
|
|
205
|
+
return hash((self.action, self.resource))
|
|
206
|
+
|
|
207
|
+
def __eq__(self, other: object) -> bool:
|
|
208
|
+
return isinstance(other, UserPermission) and self.__hash__() == other.__hash__()
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
from pydantic import BaseModel, Field
|
|
1
|
+
from pydantic import BaseModel, ConfigDict, Field, computed_field
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
class ConnectorPaginationOptions(BaseModel):
|
|
5
5
|
page: int = Field(default=1, ge=1)
|
|
6
|
-
page_size: int = Field(default=25,
|
|
6
|
+
page_size: int = Field(default=25, ge=1, le=500)
|
|
7
7
|
|
|
8
8
|
@property
|
|
9
9
|
def page_params(self) -> tuple[int, int]:
|
|
10
10
|
if self.page is not None:
|
|
11
|
-
return self.page, self.page_size
|
|
11
|
+
return (self.page, self.page_size)
|
|
12
12
|
else:
|
|
13
13
|
raise ValueError(
|
|
14
14
|
"Pagination options must be set to return a page and page size",
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# RBAC
|
|
2
|
+
CHAT_USER: str = "CHAT-USER"
|
|
3
|
+
ORG_ADMIN: str = "ORG-ADMIN"
|
|
4
|
+
TASK_ADMIN: str = "TASK-ADMIN"
|
|
5
|
+
DEFAULT_RULE_ADMIN: str = "DEFAULT-RULE-ADMIN"
|
|
6
|
+
VALIDATION_USER: str = "VALIDATION-USER"
|
|
7
|
+
ORG_AUDITOR: str = "ORG-AUDITOR"
|
|
8
|
+
ADMIN_KEY: str = "ADMIN-KEY"
|
|
9
|
+
|
|
10
|
+
LEGACY_KEYCLOAK_ROLES: dict[str, str] = {
|
|
11
|
+
"genai_engine_admin_user": TASK_ADMIN,
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
# Make sure the policy and description match
|
|
15
|
+
GENAI_ENGINE_KEYCLOAK_PASSWORD_LENGTH = 12
|
|
16
|
+
GENAI_ENGINE_KEYCLOAK_PASSWORD_POLICY = f"length({GENAI_ENGINE_KEYCLOAK_PASSWORD_LENGTH}) and specialChars(1) and upperCase(1) and lowerCase(1)"
|
|
17
|
+
ERROR_PASSWORD_POLICY_NOT_MET = f"Password should be at least {GENAI_ENGINE_KEYCLOAK_PASSWORD_LENGTH} characters and contain at least one special character, lowercase character, and uppercase character."
|
|
18
|
+
ERROR_DEFAULT_METRICS_ENGINE = "This metric could not be evaluated"
|
|
19
|
+
|
|
20
|
+
# Miscellaneous
|
|
21
|
+
DEFAULT_TOXICITY_RULE_THRESHOLD = 0.5
|
|
22
|
+
DEFAULT_PII_RULE_CONFIDENCE_SCORE_THRESHOLD = 0
|
|
23
|
+
NEGATIVE_BLOOD_EXAMPLE = "John has O negative blood group"
|
|
24
|
+
HALLUCINATION_RULE_NAME = "Hallucination Rule"
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
from enum import IntEnum, StrEnum
|
|
2
|
+
|
|
3
|
+
from arthur_common.models.constants import (
|
|
4
|
+
DEFAULT_RULE_ADMIN,
|
|
5
|
+
ORG_ADMIN,
|
|
6
|
+
ORG_AUDITOR,
|
|
7
|
+
TASK_ADMIN,
|
|
8
|
+
VALIDATION_USER,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class BaseEnum(StrEnum):
|
|
13
|
+
@classmethod
|
|
14
|
+
def values(self) -> list[str]:
|
|
15
|
+
values: list[str] = [e for e in self]
|
|
16
|
+
return values
|
|
17
|
+
|
|
18
|
+
def __str__(self) -> str:
|
|
19
|
+
return str(self.value)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class APIKeysRolesEnum(BaseEnum):
|
|
23
|
+
DEFAULT_RULE_ADMIN = DEFAULT_RULE_ADMIN
|
|
24
|
+
TASK_ADMIN = TASK_ADMIN
|
|
25
|
+
VALIDATION_USER = VALIDATION_USER
|
|
26
|
+
ORG_AUDITOR = ORG_AUDITOR
|
|
27
|
+
ORG_ADMIN = ORG_ADMIN
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class InferenceFeedbackTarget(BaseEnum):
|
|
31
|
+
CONTEXT = "context"
|
|
32
|
+
RESPONSE_RESULTS = "response_results"
|
|
33
|
+
PROMPT_RESULTS = "prompt_results"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class MetricType(BaseEnum):
|
|
37
|
+
QUERY_RELEVANCE = "QueryRelevance"
|
|
38
|
+
RESPONSE_RELEVANCE = "ResponseRelevance"
|
|
39
|
+
TOOL_SELECTION = "ToolSelection"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class ModelProblemType(BaseEnum):
|
|
43
|
+
REGRESSION = "regression"
|
|
44
|
+
BINARY_CLASSIFICATION = "binary_classification"
|
|
45
|
+
ARTHUR_SHIELD = "arthur_shield"
|
|
46
|
+
CUSTOM = "custom"
|
|
47
|
+
MULTICLASS_CLASSIFICATION = "multiclass_classification"
|
|
48
|
+
AGENTIC_TRACE = "agentic_trace"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# Using version from arthur-engine, which has str and enum type inheritance.
|
|
52
|
+
# Note: These string values are not arbitrary and map to Presidio entity types: https://microsoft.github.io/presidio/supported_entities/
|
|
53
|
+
class PIIEntityTypes(BaseEnum):
|
|
54
|
+
CREDIT_CARD = "CREDIT_CARD"
|
|
55
|
+
CRYPTO = "CRYPTO"
|
|
56
|
+
DATE_TIME = "DATE_TIME"
|
|
57
|
+
EMAIL_ADDRESS = "EMAIL_ADDRESS"
|
|
58
|
+
IBAN_CODE = "IBAN_CODE"
|
|
59
|
+
IP_ADDRESS = "IP_ADDRESS"
|
|
60
|
+
NRP = "NRP"
|
|
61
|
+
LOCATION = "LOCATION"
|
|
62
|
+
PERSON = "PERSON"
|
|
63
|
+
PHONE_NUMBER = "PHONE_NUMBER"
|
|
64
|
+
MEDICAL_LICENSE = "MEDICAL_LICENSE"
|
|
65
|
+
URL = "URL"
|
|
66
|
+
US_BANK_NUMBER = "US_BANK_NUMBER"
|
|
67
|
+
US_DRIVER_LICENSE = "US_DRIVER_LICENSE"
|
|
68
|
+
US_ITIN = "US_ITIN"
|
|
69
|
+
US_PASSPORT = "US_PASSPORT"
|
|
70
|
+
US_SSN = "US_SSN"
|
|
71
|
+
|
|
72
|
+
@classmethod
|
|
73
|
+
def to_string(cls) -> str:
|
|
74
|
+
return ",".join(member.value for member in cls)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class PaginationSortMethod(BaseEnum):
|
|
78
|
+
ASCENDING = "asc"
|
|
79
|
+
DESCENDING = "desc"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class RuleResultEnum(BaseEnum):
|
|
83
|
+
PASS = "Pass"
|
|
84
|
+
FAIL = "Fail"
|
|
85
|
+
SKIPPED = "Skipped"
|
|
86
|
+
UNAVAILABLE = "Unavailable"
|
|
87
|
+
PARTIALLY_UNAVAILABLE = "Partially Unavailable"
|
|
88
|
+
MODEL_NOT_AVAILABLE = "Model Not Available"
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class RuleScope(BaseEnum):
|
|
92
|
+
DEFAULT = "default"
|
|
93
|
+
TASK = "task"
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class RuleType(BaseEnum):
|
|
97
|
+
KEYWORD = "KeywordRule"
|
|
98
|
+
MODEL_HALLUCINATION_V2 = "ModelHallucinationRuleV2"
|
|
99
|
+
MODEL_SENSITIVE_DATA = "ModelSensitiveDataRule"
|
|
100
|
+
PII_DATA = "PIIDataRule"
|
|
101
|
+
PROMPT_INJECTION = "PromptInjectionRule"
|
|
102
|
+
REGEX = "RegexRule"
|
|
103
|
+
TOXICITY = "ToxicityRule"
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class TaskType(BaseEnum):
|
|
107
|
+
TRADITIONAL = "traditional"
|
|
108
|
+
AGENTIC = "agentic"
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class TokenUsageScope(BaseEnum):
|
|
112
|
+
RULE_TYPE = "rule_type"
|
|
113
|
+
TASK = "task"
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class ToolClassEnum(IntEnum):
|
|
117
|
+
WRONG_TOOL_SELECTED = 0
|
|
118
|
+
CORRECT_TOOL_SELECTED = 1
|
|
119
|
+
NO_TOOL_SELECTED = 2
|
|
120
|
+
|
|
121
|
+
def __str__(self) -> str:
|
|
122
|
+
return str(self.value)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class ToxicityViolationType(BaseEnum):
|
|
126
|
+
BENIGN = "benign"
|
|
127
|
+
HARMFUL_REQUEST = "harmful_request"
|
|
128
|
+
TOXIC_CONTENT = "toxic_content"
|
|
129
|
+
PROFANITY = "profanity"
|
|
130
|
+
UNKNOWN = "unknown"
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
# If you added values here, did you update permission_mappings.py in arthur-engine?
|
|
134
|
+
class UserPermissionAction(BaseEnum):
|
|
135
|
+
CREATE = "create"
|
|
136
|
+
READ = "read"
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
# If you added values here, did you update permission_mappings.py in arthur-engine?
|
|
140
|
+
class UserPermissionResource(BaseEnum):
|
|
141
|
+
PROMPTS = "prompts"
|
|
142
|
+
RESPONSES = "responses"
|
|
143
|
+
RULES = "rules"
|
|
144
|
+
TASKS = "tasks"
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
from typing import Any, Dict, List, Optional
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, Field
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class RelevanceMetricConfig(BaseModel):
|
|
7
|
+
"""Configuration for relevance metrics including QueryRelevance and ResponseRelevance"""
|
|
8
|
+
|
|
9
|
+
relevance_threshold: Optional[float] = Field(
|
|
10
|
+
default=None,
|
|
11
|
+
description="Threshold for determining relevance when not using LLM judge",
|
|
12
|
+
)
|
|
13
|
+
use_llm_judge: bool = Field(
|
|
14
|
+
default=True,
|
|
15
|
+
description="Whether to use LLM as a judge for relevance scoring",
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class RelevanceMetric(BaseModel):
|
|
20
|
+
bert_f_score: Optional[float] = None
|
|
21
|
+
reranker_relevance_score: Optional[float] = None
|
|
22
|
+
llm_relevance_score: Optional[float] = None
|
|
23
|
+
reason: Optional[str] = None
|
|
24
|
+
refinement: Optional[str] = None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class QueryRelevanceMetric(RelevanceMetric):
|
|
28
|
+
"""Inherits from RelevanceMetric. This class is left empty so that the openapi response schema remains the same as before, but we have a single source of truth for the relevance metric details."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ResponseRelevanceMetric(RelevanceMetric):
|
|
32
|
+
"""Inherits from RelevanceMetric. This class is left empty so that the openapi response schema remains the same as before, but we have a single source of truth for the relevance metric details."""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class MetricRequest(BaseModel):
|
|
36
|
+
system_prompt: Optional[str] = Field(
|
|
37
|
+
description="System prompt to be used by GenAI Engine for computing metrics.",
|
|
38
|
+
default=None,
|
|
39
|
+
)
|
|
40
|
+
user_query: Optional[str] = Field(
|
|
41
|
+
description="User query to be used by GenAI Engine for computing metrics.",
|
|
42
|
+
default=None,
|
|
43
|
+
)
|
|
44
|
+
context: List[Dict[str, Any]] = Field(
|
|
45
|
+
description="Conversation history and additional context to be used by GenAI Engine for computing metrics.",
|
|
46
|
+
default_factory=list,
|
|
47
|
+
examples=[
|
|
48
|
+
{"role": "user", "value": "What is the weather in Tokyo?"},
|
|
49
|
+
{"role": "assistant", "value": "WeatherTool", "args": {"city": "Tokyo"}},
|
|
50
|
+
{
|
|
51
|
+
"role": "tool",
|
|
52
|
+
"value": '[{"name": "WeatherTool", "result": {"temperature": "20°C", "humidity": "50%", "condition": "sunny"}}]',
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"role": "assistant",
|
|
56
|
+
"value": "The weather in Tokyo is sunny and the temperature is 20°C.",
|
|
57
|
+
},
|
|
58
|
+
],
|
|
59
|
+
)
|
|
60
|
+
response: Optional[str] = Field(
|
|
61
|
+
description="Response to be used by GenAI Engine for computing metrics.",
|
|
62
|
+
default=None,
|
|
63
|
+
)
|
|
@@ -7,7 +7,7 @@ from uuid import UUID
|
|
|
7
7
|
from pydantic import BaseModel, Field, field_validator, model_validator
|
|
8
8
|
from typing_extensions import Self
|
|
9
9
|
|
|
10
|
-
from arthur_common.models.
|
|
10
|
+
from arthur_common.models.enums import ModelProblemType
|
|
11
11
|
from arthur_common.models.schema_definitions import (
|
|
12
12
|
DType,
|
|
13
13
|
SchemaTypeUnion,
|