unique_toolkit 1.11.1__py3-none-any.whl → 1.11.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unique_toolkit/content/functions.py +20 -7
- unique_toolkit/content/schemas.py +16 -3
- unique_toolkit/content/smart_rules.py +301 -0
- unique_toolkit/knowledge_base.py +81 -2
- unique_toolkit/smart_rules/compile.py +56 -301
- {unique_toolkit-1.11.1.dist-info → unique_toolkit-1.11.3.dist-info}/METADATA +8 -1
- {unique_toolkit-1.11.1.dist-info → unique_toolkit-1.11.3.dist-info}/RECORD +9 -8
- {unique_toolkit-1.11.1.dist-info → unique_toolkit-1.11.3.dist-info}/LICENSE +0 -0
- {unique_toolkit-1.11.1.dist-info → unique_toolkit-1.11.3.dist-info}/WHEEL +0 -0
@@ -16,7 +16,8 @@ from unique_toolkit.content.schemas import (
|
|
16
16
|
ContentInfo,
|
17
17
|
ContentRerankerConfig,
|
18
18
|
ContentSearchType,
|
19
|
-
|
19
|
+
FolderInfo,
|
20
|
+
PaginatedContentInfos,
|
20
21
|
)
|
21
22
|
from unique_toolkit.content.utils import map_contents, map_to_content_chunks
|
22
23
|
|
@@ -590,21 +591,33 @@ def get_content_info(
|
|
590
591
|
file_path: str | None = None,
|
591
592
|
):
|
592
593
|
"""Gets the info of a content."""
|
594
|
+
|
593
595
|
get_info_params = unique_sdk.Content.ContentInfoParams(
|
594
|
-
metadataFilter=metadata_filter,
|
595
|
-
skip=skip,
|
596
|
-
take=take,
|
597
|
-
filePath=file_path,
|
596
|
+
metadataFilter=metadata_filter or None, # Dict cannot be empty
|
598
597
|
)
|
598
|
+
if skip:
|
599
|
+
get_info_params["skip"] = skip
|
600
|
+
if take:
|
601
|
+
get_info_params["take"] = take
|
602
|
+
if file_path:
|
603
|
+
get_info_params["filePath"] = file_path
|
599
604
|
|
600
|
-
content_info = unique_sdk.Content.
|
605
|
+
content_info = unique_sdk.Content.get_infos(
|
601
606
|
user_id=user_id, company_id=company_id, **get_info_params
|
602
607
|
)
|
603
|
-
return
|
608
|
+
return PaginatedContentInfos.model_validate(
|
604
609
|
content_info, by_alias=True, by_name=True
|
605
610
|
)
|
606
611
|
|
607
612
|
|
613
|
+
def get_folder_info(user_id: str, company_id: str, scope_id: str) -> FolderInfo:
|
614
|
+
info = unique_sdk.Folder.get_info(
|
615
|
+
user_id=user_id, company_id=company_id, scopeId=scope_id
|
616
|
+
)
|
617
|
+
|
618
|
+
return FolderInfo.model_validate(info, by_alias=True, by_name=True)
|
619
|
+
|
620
|
+
|
608
621
|
def update_content(
|
609
622
|
user_id: str,
|
610
623
|
company_id: str,
|
@@ -154,12 +154,13 @@ class ContentRerankerConfig(BaseModel):
|
|
154
154
|
class ContentInfo(BaseModel):
|
155
155
|
model_config = model_config
|
156
156
|
id: str
|
157
|
+
object: str
|
157
158
|
key: str
|
158
159
|
url: str | None = None
|
159
160
|
title: str | None = None
|
160
161
|
metadata: dict[str, Any] | None = None
|
161
|
-
mime_type: str
|
162
162
|
byte_size: int
|
163
|
+
mime_type: str
|
163
164
|
owner_id: str
|
164
165
|
created_at: datetime
|
165
166
|
updated_at: datetime
|
@@ -168,7 +169,19 @@ class ContentInfo(BaseModel):
|
|
168
169
|
expired_at: datetime | None = None
|
169
170
|
|
170
171
|
|
171
|
-
class
|
172
|
+
class PaginatedContentInfos(BaseModel):
|
172
173
|
model_config = model_config
|
173
|
-
|
174
|
+
object: str
|
175
|
+
content_infos: list[ContentInfo]
|
174
176
|
total_count: int
|
177
|
+
|
178
|
+
|
179
|
+
class FolderInfo(BaseModel):
|
180
|
+
model_config = model_config
|
181
|
+
id: str
|
182
|
+
name: str
|
183
|
+
ingestion_config: dict[str, Any]
|
184
|
+
createdAt: str | None
|
185
|
+
updatedAt: str | None
|
186
|
+
parentId: str | None
|
187
|
+
externalId: str | None
|
@@ -0,0 +1,301 @@
|
|
1
|
+
import re
|
2
|
+
from datetime import datetime, timedelta, timezone
|
3
|
+
from enum import Enum
|
4
|
+
from typing import Any, Dict, List, Mapping, Self, Union
|
5
|
+
|
6
|
+
from pydantic import AliasChoices, BaseModel, Field
|
7
|
+
from pydantic.config import ConfigDict
|
8
|
+
|
9
|
+
|
10
|
+
class Operator(str, Enum):
|
11
|
+
EQUALS = "equals"
|
12
|
+
NOT_EQUALS = "notEquals"
|
13
|
+
GREATER_THAN = "greaterThan"
|
14
|
+
GREATER_THAN_OR_EQUAL = "greaterThanOrEqual"
|
15
|
+
LESS_THAN = "lessThan"
|
16
|
+
LESS_THAN_OR_EQUAL = "lessThanOrEqual"
|
17
|
+
IN = "in"
|
18
|
+
NOT_IN = "notIn"
|
19
|
+
CONTAINS = "contains"
|
20
|
+
NOT_CONTAINS = "notContains"
|
21
|
+
IS_NULL = "isNull"
|
22
|
+
IS_NOT_NULL = "isNotNull"
|
23
|
+
IS_EMPTY = "isEmpty"
|
24
|
+
IS_NOT_EMPTY = "isNotEmpty"
|
25
|
+
NESTED = "nested"
|
26
|
+
|
27
|
+
|
28
|
+
class BaseStatement(BaseModel):
|
29
|
+
model_config = ConfigDict(serialize_by_alias=True)
|
30
|
+
|
31
|
+
def with_variables(
|
32
|
+
self,
|
33
|
+
user_metadata: Mapping[str, Union[str, int, bool]],
|
34
|
+
tool_parameters: Mapping[str, Union[str, int, bool]],
|
35
|
+
) -> Self:
|
36
|
+
return self._fill_in_variables(user_metadata, tool_parameters)
|
37
|
+
|
38
|
+
def is_compiled(self) -> bool:
|
39
|
+
# Serialize the object to json string
|
40
|
+
json_str = self.model_dump_json()
|
41
|
+
# Check if the json string has <T> or <T+> or <T-> or <toolParameters or <userMetadata
|
42
|
+
return (
|
43
|
+
"<T>" in json_str
|
44
|
+
or "<T+" in json_str
|
45
|
+
or "<T-" in json_str
|
46
|
+
or "<toolParameters" in json_str
|
47
|
+
or "<userMetadata" in json_str
|
48
|
+
)
|
49
|
+
|
50
|
+
def _fill_in_variables(
|
51
|
+
self,
|
52
|
+
user_metadata: Mapping[str, Union[str, int, bool]],
|
53
|
+
tool_parameters: Mapping[str, Union[str, int, bool]],
|
54
|
+
) -> Self:
|
55
|
+
return self.model_copy()
|
56
|
+
|
57
|
+
|
58
|
+
class Statement(BaseStatement):
|
59
|
+
operator: Operator
|
60
|
+
value: Union[str, int, bool, list[str], "AndStatement", "OrStatement"]
|
61
|
+
path: List[str] = Field(default_factory=list)
|
62
|
+
|
63
|
+
def _fill_in_variables(
|
64
|
+
self,
|
65
|
+
user_metadata: Mapping[str, Union[str, int, bool]],
|
66
|
+
tool_parameters: Mapping[str, Union[str, int, bool]],
|
67
|
+
) -> Self:
|
68
|
+
new_stmt = self.model_copy()
|
69
|
+
new_stmt.value = eval_operator(self, user_metadata, tool_parameters)
|
70
|
+
return new_stmt
|
71
|
+
|
72
|
+
|
73
|
+
class AndStatement(BaseStatement):
|
74
|
+
and_list: List[Union["Statement", "AndStatement", "OrStatement"]] = Field(
|
75
|
+
validation_alias=AliasChoices("and", "and_list"), serialization_alias="and"
|
76
|
+
)
|
77
|
+
|
78
|
+
def _fill_in_variables(
|
79
|
+
self,
|
80
|
+
user_metadata: Mapping[str, Union[str, int, bool]],
|
81
|
+
tool_parameters: Mapping[str, Union[str, int, bool]],
|
82
|
+
) -> Self:
|
83
|
+
new_stmt = self.model_copy()
|
84
|
+
new_stmt.and_list = [
|
85
|
+
sub_query._fill_in_variables(user_metadata, tool_parameters)
|
86
|
+
for sub_query in self.and_list
|
87
|
+
]
|
88
|
+
return new_stmt
|
89
|
+
|
90
|
+
|
91
|
+
class OrStatement(BaseStatement):
|
92
|
+
or_list: List[Union["Statement", "AndStatement", "OrStatement"]] = Field(
|
93
|
+
validation_alias=AliasChoices("or", "or_list"), serialization_alias="or"
|
94
|
+
)
|
95
|
+
|
96
|
+
def _fill_in_variables(
|
97
|
+
self,
|
98
|
+
user_metadata: Mapping[str, Union[str, int, bool]],
|
99
|
+
tool_parameters: Mapping[str, Union[str, int, bool]],
|
100
|
+
) -> Self:
|
101
|
+
new_stmt = self.model_copy()
|
102
|
+
new_stmt.or_list = [
|
103
|
+
sub_query._fill_in_variables(user_metadata, tool_parameters)
|
104
|
+
for sub_query in self.or_list
|
105
|
+
]
|
106
|
+
return new_stmt
|
107
|
+
|
108
|
+
|
109
|
+
# Update the forward references
|
110
|
+
Statement.model_rebuild()
|
111
|
+
AndStatement.model_rebuild()
|
112
|
+
OrStatement.model_rebuild()
|
113
|
+
|
114
|
+
|
115
|
+
UniqueQL = Union[Statement, AndStatement, OrStatement]
|
116
|
+
|
117
|
+
|
118
|
+
def is_array_of_strings(value: Any) -> bool:
|
119
|
+
return isinstance(value, list) and all(isinstance(item, str) for item in value)
|
120
|
+
|
121
|
+
|
122
|
+
def eval_operator(
|
123
|
+
query: Statement,
|
124
|
+
user_metadata: Mapping[str, Union[str, int, bool]],
|
125
|
+
tool_parameters: Mapping[str, Union[str, int, bool]],
|
126
|
+
) -> Any:
|
127
|
+
if query.operator in [
|
128
|
+
Operator.EQUALS,
|
129
|
+
Operator.NOT_EQUALS,
|
130
|
+
Operator.GREATER_THAN,
|
131
|
+
Operator.GREATER_THAN_OR_EQUAL,
|
132
|
+
Operator.LESS_THAN,
|
133
|
+
Operator.LESS_THAN_OR_EQUAL,
|
134
|
+
Operator.CONTAINS,
|
135
|
+
Operator.NOT_CONTAINS,
|
136
|
+
]:
|
137
|
+
return binary_operator(query.value, user_metadata, tool_parameters)
|
138
|
+
elif query.operator in [Operator.IS_NULL, Operator.IS_NOT_NULL]:
|
139
|
+
return null_operator(query.value, user_metadata, tool_parameters)
|
140
|
+
elif query.operator in [Operator.IS_EMPTY, Operator.IS_NOT_EMPTY]:
|
141
|
+
return empty_operator(query.operator, user_metadata, tool_parameters)
|
142
|
+
elif query.operator == Operator.NESTED:
|
143
|
+
return eval_nested_operator(query.value, user_metadata, tool_parameters)
|
144
|
+
elif query.operator in [Operator.IN, Operator.NOT_IN]:
|
145
|
+
return array_operator(query.value, user_metadata, tool_parameters)
|
146
|
+
else:
|
147
|
+
raise ValueError(f"Operator {query.operator} not supported")
|
148
|
+
|
149
|
+
|
150
|
+
def eval_nested_operator(
|
151
|
+
value: Any,
|
152
|
+
user_metadata: Mapping[str, Union[str, int, bool]],
|
153
|
+
tool_parameters: Mapping[str, Union[str, int, bool]],
|
154
|
+
) -> Union[AndStatement, OrStatement]:
|
155
|
+
if not isinstance(value, (AndStatement, OrStatement)):
|
156
|
+
raise ValueError("Nested operator must be an AndStatement or OrStatement")
|
157
|
+
return value._fill_in_variables(user_metadata, tool_parameters)
|
158
|
+
|
159
|
+
|
160
|
+
def binary_operator(
|
161
|
+
value: Any,
|
162
|
+
user_metadata: Mapping[str, Union[str, int, bool]],
|
163
|
+
tool_parameters: Mapping[str, Union[str, int, bool]],
|
164
|
+
) -> Any:
|
165
|
+
return replace_variables(value, user_metadata, tool_parameters)
|
166
|
+
|
167
|
+
|
168
|
+
def array_operator(
|
169
|
+
value: Any,
|
170
|
+
user_metadata: Mapping[str, Union[str, int, bool]],
|
171
|
+
tool_parameters: Mapping[str, Union[str, int, bool]],
|
172
|
+
) -> Any:
|
173
|
+
if is_array_of_strings(value):
|
174
|
+
return [
|
175
|
+
replace_variables(item, user_metadata, tool_parameters) for item in value
|
176
|
+
]
|
177
|
+
return value
|
178
|
+
|
179
|
+
|
180
|
+
def null_operator(
|
181
|
+
value: Any,
|
182
|
+
user_metadata: Mapping[str, Union[str, int, bool]],
|
183
|
+
tool_parameters: Mapping[str, Union[str, int, bool]],
|
184
|
+
) -> Any:
|
185
|
+
return value # do nothing for now. No variables to replace
|
186
|
+
|
187
|
+
|
188
|
+
def empty_operator(
|
189
|
+
operator: Operator,
|
190
|
+
user_metadata: Mapping[str, Union[str, int, bool]],
|
191
|
+
tool_parameters: Mapping[str, Union[str, int, bool]],
|
192
|
+
) -> Any:
|
193
|
+
"""Handle IS_EMPTY and IS_NOT_EMPTY operators."""
|
194
|
+
if operator == Operator.IS_EMPTY:
|
195
|
+
return ""
|
196
|
+
elif operator == Operator.IS_NOT_EMPTY:
|
197
|
+
return "not_empty"
|
198
|
+
return None
|
199
|
+
|
200
|
+
|
201
|
+
def calculate_current_date() -> str:
|
202
|
+
"""Calculate current date in UTC with seconds precision."""
|
203
|
+
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
204
|
+
|
205
|
+
|
206
|
+
def calculate_earlier_date(input_str: str) -> str:
|
207
|
+
match = re.search(r"<T-(\d+)>", input_str)
|
208
|
+
if not match:
|
209
|
+
return calculate_current_date() # Return current date if no match
|
210
|
+
days = int(match.group(1))
|
211
|
+
return (datetime.now(timezone.utc) - timedelta(days=days)).isoformat(
|
212
|
+
timespec="seconds"
|
213
|
+
)
|
214
|
+
|
215
|
+
|
216
|
+
def calculate_later_date(input_str: str) -> str:
|
217
|
+
match = re.search(r"<T\+(\d+)>", input_str) # Note: escaped + in regex
|
218
|
+
if not match:
|
219
|
+
return calculate_current_date() # Return current date if no match
|
220
|
+
days = int(match.group(1))
|
221
|
+
return (datetime.now(timezone.utc) + timedelta(days=days)).isoformat(
|
222
|
+
timespec="seconds"
|
223
|
+
)
|
224
|
+
|
225
|
+
|
226
|
+
def replace_variables(
|
227
|
+
value: Any,
|
228
|
+
user_metadata: Mapping[str, Union[str, int, bool]],
|
229
|
+
tool_parameters: Mapping[str, Union[str, int, bool]],
|
230
|
+
) -> Any:
|
231
|
+
if isinstance(value, str):
|
232
|
+
if "||" in value:
|
233
|
+
return get_fallback_values(value, user_metadata, tool_parameters)
|
234
|
+
elif value == "<T>":
|
235
|
+
return calculate_current_date()
|
236
|
+
elif "<T-" in value:
|
237
|
+
return calculate_earlier_date(value)
|
238
|
+
elif "<T+" in value:
|
239
|
+
return calculate_later_date(value)
|
240
|
+
|
241
|
+
value = replace_tool_parameters_patterns(value, tool_parameters)
|
242
|
+
value = replace_user_metadata_patterns(value, user_metadata)
|
243
|
+
|
244
|
+
if value == "":
|
245
|
+
return value
|
246
|
+
try:
|
247
|
+
return int(value)
|
248
|
+
except ValueError:
|
249
|
+
if value.lower() in ["true", "false"]:
|
250
|
+
return value.lower() == "true"
|
251
|
+
return value
|
252
|
+
return value
|
253
|
+
|
254
|
+
|
255
|
+
def replace_tool_parameters_patterns(
|
256
|
+
value: str, tool_parameters: Dict[str, Union[str, int, bool]]
|
257
|
+
) -> str:
|
258
|
+
def replace_match(match):
|
259
|
+
param_name = match.group(1)
|
260
|
+
return str(tool_parameters.get(param_name, ""))
|
261
|
+
|
262
|
+
return re.sub(r"<toolParameters\.(\w+)>", replace_match, value)
|
263
|
+
|
264
|
+
|
265
|
+
def replace_user_metadata_patterns(
|
266
|
+
value: str, user_metadata: Dict[str, Union[str, int, bool]]
|
267
|
+
) -> str:
|
268
|
+
def replace_match(match):
|
269
|
+
param_name = match.group(1)
|
270
|
+
return str(user_metadata.get(param_name, ""))
|
271
|
+
|
272
|
+
return re.sub(r"<userMetadata\.(\w+)>", replace_match, value)
|
273
|
+
|
274
|
+
|
275
|
+
def get_fallback_values(
|
276
|
+
value: str,
|
277
|
+
user_metadata: Mapping[str, Union[str, int, bool]],
|
278
|
+
tool_parameters: Mapping[str, Union[str, int, bool]],
|
279
|
+
) -> Any:
|
280
|
+
values = value.split("||")
|
281
|
+
for val in values:
|
282
|
+
data = replace_variables(val, user_metadata, tool_parameters)
|
283
|
+
if data != "":
|
284
|
+
return data
|
285
|
+
return values
|
286
|
+
|
287
|
+
|
288
|
+
# Example usage:
|
289
|
+
def parse_uniqueql(json_data: Dict[str, Any]) -> UniqueQL:
|
290
|
+
if "operator" in json_data:
|
291
|
+
return Statement.model_validate(json_data)
|
292
|
+
elif "or" in json_data:
|
293
|
+
return OrStatement.model_validate(
|
294
|
+
{"or": [parse_uniqueql(item) for item in json_data["or"]]}
|
295
|
+
)
|
296
|
+
elif "and" in json_data:
|
297
|
+
return AndStatement.model_validate(
|
298
|
+
{"and": [parse_uniqueql(item) for item in json_data["and"]]}
|
299
|
+
)
|
300
|
+
else:
|
301
|
+
raise ValueError("Invalid UniqueQL format")
|
unique_toolkit/knowledge_base.py
CHANGED
@@ -14,6 +14,7 @@ from unique_toolkit.content.functions import (
|
|
14
14
|
download_content_to_bytes,
|
15
15
|
download_content_to_file_by_id,
|
16
16
|
get_content_info,
|
17
|
+
get_folder_info,
|
17
18
|
search_content_chunks,
|
18
19
|
search_content_chunks_async,
|
19
20
|
search_contents,
|
@@ -28,7 +29,8 @@ from unique_toolkit.content.schemas import (
|
|
28
29
|
ContentInfo,
|
29
30
|
ContentRerankerConfig,
|
30
31
|
ContentSearchType,
|
31
|
-
|
32
|
+
FolderInfo,
|
33
|
+
PaginatedContentInfos,
|
32
34
|
)
|
33
35
|
|
34
36
|
_LOGGER = logging.getLogger(f"toolkit.knowledge_base.{__name__}")
|
@@ -483,7 +485,7 @@ class KnowledgeBaseService:
|
|
483
485
|
skip: int | None = None,
|
484
486
|
take: int | None = None,
|
485
487
|
file_path: str | None = None,
|
486
|
-
) ->
|
488
|
+
) -> PaginatedContentInfos:
|
487
489
|
return get_content_info(
|
488
490
|
user_id=self._user_id,
|
489
491
|
company_id=self._company_id,
|
@@ -493,6 +495,17 @@ class KnowledgeBaseService:
|
|
493
495
|
file_path=file_path,
|
494
496
|
)
|
495
497
|
|
498
|
+
def get_folder_info(
|
499
|
+
self,
|
500
|
+
*,
|
501
|
+
scope_id: str,
|
502
|
+
) -> FolderInfo:
|
503
|
+
return get_folder_info(
|
504
|
+
user_id=self._user_id,
|
505
|
+
company_id=self._company_id,
|
506
|
+
scope_id=scope_id,
|
507
|
+
)
|
508
|
+
|
496
509
|
def replace_content_metadata(
|
497
510
|
self,
|
498
511
|
*,
|
@@ -506,6 +519,72 @@ class KnowledgeBaseService:
|
|
506
519
|
metadata=metadata,
|
507
520
|
)
|
508
521
|
|
522
|
+
def _resolve_visible_file_tree(self, content_infos: list[ContentInfo]) -> list[str]:
|
523
|
+
# collect all scope ids
|
524
|
+
folder_id_paths: set[str] = set()
|
525
|
+
known_folder_paths: set[str] = set()
|
526
|
+
for content_info in content_infos:
|
527
|
+
if (
|
528
|
+
content_info.metadata
|
529
|
+
and content_info.metadata.get(r"{FullPath}") is not None
|
530
|
+
):
|
531
|
+
known_folder_paths.add(str(content_info.metadata.get(r"{FullPath}")))
|
532
|
+
continue
|
533
|
+
|
534
|
+
if (
|
535
|
+
content_info.metadata
|
536
|
+
and content_info.metadata.get("folderIdPath") is not None
|
537
|
+
):
|
538
|
+
folder_id_paths.add(str(content_info.metadata.get("folderIdPath")))
|
539
|
+
|
540
|
+
scope_ids: set[str] = set()
|
541
|
+
for fp in folder_id_paths:
|
542
|
+
scope_ids_list = set(fp.replace("uniquepathid://", "").split("/"))
|
543
|
+
scope_ids.update(scope_ids_list)
|
544
|
+
|
545
|
+
scope_id_to_folder_name: dict[str, str] = {}
|
546
|
+
for scope_id in scope_ids:
|
547
|
+
folder_info = self.get_folder_info(
|
548
|
+
scope_id=scope_id,
|
549
|
+
)
|
550
|
+
scope_id_to_folder_name[scope_id] = folder_info.name
|
551
|
+
|
552
|
+
folder_paths: set[str] = set()
|
553
|
+
for folder_id_path in folder_id_paths:
|
554
|
+
scope_ids_list = folder_id_path.replace("uniquepathid://", "").split("/")
|
555
|
+
|
556
|
+
if all(scope_id in scope_id_to_folder_name for scope_id in scope_ids_list):
|
557
|
+
folder_path = [
|
558
|
+
scope_id_to_folder_name[scope_id] for scope_id in scope_ids_list
|
559
|
+
]
|
560
|
+
folder_paths.add("/".join(folder_path))
|
561
|
+
|
562
|
+
return [
|
563
|
+
p if p.startswith("/") else f"/{p}"
|
564
|
+
for p in folder_paths.union(known_folder_paths)
|
565
|
+
]
|
566
|
+
|
567
|
+
def resolve_visible_file_tree(
|
568
|
+
self, *, metadata_filter: dict[str, Any] | None = None
|
569
|
+
) -> list[str]:
|
570
|
+
"""
|
571
|
+
Resolves the visible file tree for the knowledge base for the current user.
|
572
|
+
|
573
|
+
Args:
|
574
|
+
metadata_filter (dict[str, Any] | None): The metadata filter to use. Defaults to None.
|
575
|
+
|
576
|
+
Returns:
|
577
|
+
list[str]: The visible file tree.
|
578
|
+
|
579
|
+
|
580
|
+
|
581
|
+
"""
|
582
|
+
info = self.get_paginated_content_infos(
|
583
|
+
metadata_filter=metadata_filter,
|
584
|
+
)
|
585
|
+
|
586
|
+
return self._resolve_visible_file_tree(content_infos=info.content_infos)
|
587
|
+
|
509
588
|
|
510
589
|
if __name__ == "__main__":
|
511
590
|
kb_service = KnowledgeBaseService.from_settings()
|
@@ -1,301 +1,56 @@
|
|
1
|
-
import
|
2
|
-
|
3
|
-
from
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
class Statement(BaseStatement):
|
59
|
-
operator: Operator
|
60
|
-
value: Union[str, int, bool, list[str], "AndStatement", "OrStatement"]
|
61
|
-
path: List[str] = Field(default_factory=list)
|
62
|
-
|
63
|
-
def _fill_in_variables(
|
64
|
-
self,
|
65
|
-
user_metadata: Dict[str, Union[str, int, bool]],
|
66
|
-
tool_parameters: Dict[str, Union[str, int, bool]],
|
67
|
-
) -> Self:
|
68
|
-
new_stmt = self.model_copy()
|
69
|
-
new_stmt.value = eval_operator(self, user_metadata, tool_parameters)
|
70
|
-
return new_stmt
|
71
|
-
|
72
|
-
|
73
|
-
class AndStatement(BaseStatement):
|
74
|
-
and_list: List[Union["Statement", "AndStatement", "OrStatement"]] = Field(
|
75
|
-
alias="and", validation_alias=AliasChoices("and", "and_list")
|
76
|
-
)
|
77
|
-
|
78
|
-
def _fill_in_variables(
|
79
|
-
self,
|
80
|
-
user_metadata: Dict[str, Union[str, int, bool]],
|
81
|
-
tool_parameters: Dict[str, Union[str, int, bool]],
|
82
|
-
) -> Self:
|
83
|
-
new_stmt = self.model_copy()
|
84
|
-
new_stmt.and_list = [
|
85
|
-
sub_query._fill_in_variables(user_metadata, tool_parameters)
|
86
|
-
for sub_query in self.and_list
|
87
|
-
]
|
88
|
-
return new_stmt
|
89
|
-
|
90
|
-
|
91
|
-
class OrStatement(BaseStatement):
|
92
|
-
or_list: List[Union["Statement", "AndStatement", "OrStatement"]] = Field(
|
93
|
-
alias="or", validation_alias=AliasChoices("or", "or_list")
|
94
|
-
)
|
95
|
-
|
96
|
-
def _fill_in_variables(
|
97
|
-
self,
|
98
|
-
user_metadata: Dict[str, Union[str, int, bool]],
|
99
|
-
tool_parameters: Dict[str, Union[str, int, bool]],
|
100
|
-
) -> Self:
|
101
|
-
new_stmt = self.model_copy()
|
102
|
-
new_stmt.or_list = [
|
103
|
-
sub_query._fill_in_variables(user_metadata, tool_parameters)
|
104
|
-
for sub_query in self.or_list
|
105
|
-
]
|
106
|
-
return new_stmt
|
107
|
-
|
108
|
-
|
109
|
-
# Update the forward references
|
110
|
-
Statement.model_rebuild()
|
111
|
-
AndStatement.model_rebuild()
|
112
|
-
OrStatement.model_rebuild()
|
113
|
-
|
114
|
-
|
115
|
-
UniqueQL = Union[Statement, AndStatement, OrStatement]
|
116
|
-
|
117
|
-
|
118
|
-
def is_array_of_strings(value: Any) -> bool:
|
119
|
-
return isinstance(value, list) and all(isinstance(item, str) for item in value)
|
120
|
-
|
121
|
-
|
122
|
-
def eval_operator(
|
123
|
-
query: Statement,
|
124
|
-
user_metadata: Dict[str, Union[str, int, bool]],
|
125
|
-
tool_parameters: Dict[str, Union[str, int, bool]],
|
126
|
-
) -> Any:
|
127
|
-
if query.operator in [
|
128
|
-
Operator.EQUALS,
|
129
|
-
Operator.NOT_EQUALS,
|
130
|
-
Operator.GREATER_THAN,
|
131
|
-
Operator.GREATER_THAN_OR_EQUAL,
|
132
|
-
Operator.LESS_THAN,
|
133
|
-
Operator.LESS_THAN_OR_EQUAL,
|
134
|
-
Operator.CONTAINS,
|
135
|
-
Operator.NOT_CONTAINS,
|
136
|
-
]:
|
137
|
-
return binary_operator(query.value, user_metadata, tool_parameters)
|
138
|
-
elif query.operator in [Operator.IS_NULL, Operator.IS_NOT_NULL]:
|
139
|
-
return null_operator(query.value, user_metadata, tool_parameters)
|
140
|
-
elif query.operator in [Operator.IS_EMPTY, Operator.IS_NOT_EMPTY]:
|
141
|
-
return empty_operator(query.operator, user_metadata, tool_parameters)
|
142
|
-
elif query.operator == Operator.NESTED:
|
143
|
-
return eval_nested_operator(query.value, user_metadata, tool_parameters)
|
144
|
-
elif query.operator in [Operator.IN, Operator.NOT_IN]:
|
145
|
-
return array_operator(query.value, user_metadata, tool_parameters)
|
146
|
-
else:
|
147
|
-
raise ValueError(f"Operator {query.operator} not supported")
|
148
|
-
|
149
|
-
|
150
|
-
def eval_nested_operator(
|
151
|
-
value: Any,
|
152
|
-
user_metadata: Dict[str, Union[str, int, bool]],
|
153
|
-
tool_parameters: Dict[str, Union[str, int, bool]],
|
154
|
-
) -> Union[AndStatement, OrStatement]:
|
155
|
-
if not isinstance(value, (AndStatement, OrStatement)):
|
156
|
-
raise ValueError("Nested operator must be an AndStatement or OrStatement")
|
157
|
-
return value._fill_in_variables(user_metadata, tool_parameters)
|
158
|
-
|
159
|
-
|
160
|
-
def binary_operator(
|
161
|
-
value: Any,
|
162
|
-
user_metadata: Dict[str, Union[str, int, bool]],
|
163
|
-
tool_parameters: Dict[str, Union[str, int, bool]],
|
164
|
-
) -> Any:
|
165
|
-
return replace_variables(value, user_metadata, tool_parameters)
|
166
|
-
|
167
|
-
|
168
|
-
def array_operator(
|
169
|
-
value: Any,
|
170
|
-
user_metadata: Dict[str, Union[str, int, bool]],
|
171
|
-
tool_parameters: Dict[str, Union[str, int, bool]],
|
172
|
-
) -> Any:
|
173
|
-
if is_array_of_strings(value):
|
174
|
-
return [
|
175
|
-
replace_variables(item, user_metadata, tool_parameters) for item in value
|
176
|
-
]
|
177
|
-
return value
|
178
|
-
|
179
|
-
|
180
|
-
def null_operator(
|
181
|
-
value: Any,
|
182
|
-
user_metadata: Dict[str, Union[str, int, bool]],
|
183
|
-
tool_parameters: Dict[str, Union[str, int, bool]],
|
184
|
-
) -> Any:
|
185
|
-
return value # do nothing for now. No variables to replace
|
186
|
-
|
187
|
-
|
188
|
-
def empty_operator(
|
189
|
-
operator: Operator,
|
190
|
-
user_metadata: Dict[str, Union[str, int, bool]],
|
191
|
-
tool_parameters: Dict[str, Union[str, int, bool]],
|
192
|
-
) -> Any:
|
193
|
-
"""Handle IS_EMPTY and IS_NOT_EMPTY operators."""
|
194
|
-
if operator == Operator.IS_EMPTY:
|
195
|
-
return ""
|
196
|
-
elif operator == Operator.IS_NOT_EMPTY:
|
197
|
-
return "not_empty"
|
198
|
-
return None
|
199
|
-
|
200
|
-
|
201
|
-
def calculate_current_date() -> str:
|
202
|
-
"""Calculate current date in UTC with seconds precision."""
|
203
|
-
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
204
|
-
|
205
|
-
|
206
|
-
def calculate_earlier_date(input_str: str) -> str:
|
207
|
-
match = re.search(r"<T-(\d+)>", input_str)
|
208
|
-
if not match:
|
209
|
-
return calculate_current_date() # Return current date if no match
|
210
|
-
days = int(match.group(1))
|
211
|
-
return (datetime.now(timezone.utc) - timedelta(days=days)).isoformat(
|
212
|
-
timespec="seconds"
|
213
|
-
)
|
214
|
-
|
215
|
-
|
216
|
-
def calculate_later_date(input_str: str) -> str:
|
217
|
-
match = re.search(r"<T\+(\d+)>", input_str) # Note: escaped + in regex
|
218
|
-
if not match:
|
219
|
-
return calculate_current_date() # Return current date if no match
|
220
|
-
days = int(match.group(1))
|
221
|
-
return (datetime.now(timezone.utc) + timedelta(days=days)).isoformat(
|
222
|
-
timespec="seconds"
|
223
|
-
)
|
224
|
-
|
225
|
-
|
226
|
-
def replace_variables(
|
227
|
-
value: Any,
|
228
|
-
user_metadata: Dict[str, Union[str, int, bool]],
|
229
|
-
tool_parameters: Dict[str, Union[str, int, bool]],
|
230
|
-
) -> Any:
|
231
|
-
if isinstance(value, str):
|
232
|
-
if "||" in value:
|
233
|
-
return get_fallback_values(value, user_metadata, tool_parameters)
|
234
|
-
elif value == "<T>":
|
235
|
-
return calculate_current_date()
|
236
|
-
elif "<T-" in value:
|
237
|
-
return calculate_earlier_date(value)
|
238
|
-
elif "<T+" in value:
|
239
|
-
return calculate_later_date(value)
|
240
|
-
|
241
|
-
value = replace_tool_parameters_patterns(value, tool_parameters)
|
242
|
-
value = replace_user_metadata_patterns(value, user_metadata)
|
243
|
-
|
244
|
-
if value == "":
|
245
|
-
return value
|
246
|
-
try:
|
247
|
-
return int(value)
|
248
|
-
except ValueError:
|
249
|
-
if value.lower() in ["true", "false"]:
|
250
|
-
return value.lower() == "true"
|
251
|
-
return value
|
252
|
-
return value
|
253
|
-
|
254
|
-
|
255
|
-
def replace_tool_parameters_patterns(
|
256
|
-
value: str, tool_parameters: Dict[str, Union[str, int, bool]]
|
257
|
-
) -> str:
|
258
|
-
def replace_match(match):
|
259
|
-
param_name = match.group(1)
|
260
|
-
return str(tool_parameters.get(param_name, ""))
|
261
|
-
|
262
|
-
return re.sub(r"<toolParameters\.(\w+)>", replace_match, value)
|
263
|
-
|
264
|
-
|
265
|
-
def replace_user_metadata_patterns(
|
266
|
-
value: str, user_metadata: Dict[str, Union[str, int, bool]]
|
267
|
-
) -> str:
|
268
|
-
def replace_match(match):
|
269
|
-
param_name = match.group(1)
|
270
|
-
return str(user_metadata.get(param_name, ""))
|
271
|
-
|
272
|
-
return re.sub(r"<userMetadata\.(\w+)>", replace_match, value)
|
273
|
-
|
274
|
-
|
275
|
-
def get_fallback_values(
|
276
|
-
value: str,
|
277
|
-
user_metadata: Dict[str, Union[str, int, bool]],
|
278
|
-
tool_parameters: Dict[str, Union[str, int, bool]],
|
279
|
-
) -> Any:
|
280
|
-
values = value.split("||")
|
281
|
-
for val in values:
|
282
|
-
data = replace_variables(val, user_metadata, tool_parameters)
|
283
|
-
if data != "":
|
284
|
-
return data
|
285
|
-
return values
|
286
|
-
|
287
|
-
|
288
|
-
# Example usage:
|
289
|
-
def parse_uniqueql(json_data: Dict[str, Any]) -> UniqueQL:
|
290
|
-
if "operator" in json_data:
|
291
|
-
return Statement.model_validate(json_data)
|
292
|
-
elif "or" in json_data:
|
293
|
-
return OrStatement.model_validate(
|
294
|
-
{"or": [parse_uniqueql(item) for item in json_data["or"]]}
|
295
|
-
)
|
296
|
-
elif "and" in json_data:
|
297
|
-
return AndStatement.model_validate(
|
298
|
-
{"and": [parse_uniqueql(item) for item in json_data["and"]]}
|
299
|
-
)
|
300
|
-
else:
|
301
|
-
raise ValueError("Invalid UniqueQL format")
|
1
|
+
import warnings
|
2
|
+
|
3
|
+
from unique_toolkit.content.smart_rules import (
|
4
|
+
AndStatement,
|
5
|
+
BaseStatement,
|
6
|
+
Operator,
|
7
|
+
OrStatement,
|
8
|
+
Statement,
|
9
|
+
UniqueQL,
|
10
|
+
array_operator,
|
11
|
+
binary_operator,
|
12
|
+
calculate_current_date,
|
13
|
+
calculate_earlier_date,
|
14
|
+
calculate_later_date,
|
15
|
+
empty_operator,
|
16
|
+
eval_nested_operator,
|
17
|
+
eval_operator,
|
18
|
+
get_fallback_values,
|
19
|
+
is_array_of_strings,
|
20
|
+
null_operator,
|
21
|
+
parse_uniqueql,
|
22
|
+
replace_tool_parameters_patterns,
|
23
|
+
replace_user_metadata_patterns,
|
24
|
+
replace_variables,
|
25
|
+
)
|
26
|
+
|
27
|
+
warnings.warn(
|
28
|
+
"unique_toolkit.smart_rules.compile is deprecated. "
|
29
|
+
"Please use unique_toolkit.content.smart_rules instead.",
|
30
|
+
DeprecationWarning,
|
31
|
+
stacklevel=2,
|
32
|
+
)
|
33
|
+
|
34
|
+
__all__ = [
|
35
|
+
"AndStatement",
|
36
|
+
"BaseStatement",
|
37
|
+
"Operator",
|
38
|
+
"OrStatement",
|
39
|
+
"Statement",
|
40
|
+
"UniqueQL",
|
41
|
+
"array_operator",
|
42
|
+
"binary_operator",
|
43
|
+
"calculate_current_date",
|
44
|
+
"calculate_earlier_date",
|
45
|
+
"calculate_later_date",
|
46
|
+
"empty_operator",
|
47
|
+
"eval_nested_operator",
|
48
|
+
"eval_operator",
|
49
|
+
"get_fallback_values",
|
50
|
+
"is_array_of_strings",
|
51
|
+
"null_operator",
|
52
|
+
"parse_uniqueql",
|
53
|
+
"replace_tool_parameters_patterns",
|
54
|
+
"replace_user_metadata_patterns",
|
55
|
+
"replace_variables",
|
56
|
+
]
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: unique_toolkit
|
3
|
-
Version: 1.11.
|
3
|
+
Version: 1.11.3
|
4
4
|
Summary:
|
5
5
|
License: Proprietary
|
6
6
|
Author: Cedric Klinkert
|
@@ -117,6 +117,12 @@ All notable changes to this project will be documented in this file.
|
|
117
117
|
|
118
118
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
119
119
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
120
|
+
## [1.11.3] - 2026-10-06
|
121
|
+
- Move smart rules to content
|
122
|
+
- Add to documentation
|
123
|
+
|
124
|
+
## [1.11.2] - 2025-10-07
|
125
|
+
- Fix for empty metadata filter at info retrieval
|
120
126
|
|
121
127
|
## [1.11.1] - 2025-10-07
|
122
128
|
- Fix bug where hallucination check was taking all of the chunks as input instead of only the referenced ones.
|
@@ -132,6 +138,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
132
138
|
## [1.9.1] - 2025-10-06
|
133
139
|
- Switch default model used in evaluation service from `GPT-3.5-turbo (0125)` to `GPT-4o (1120)`
|
134
140
|
|
141
|
+
|
135
142
|
## [1.9.0] - 2026-10-04
|
136
143
|
- Define the RequestContext and add aihttp/httpx requestors
|
137
144
|
|
@@ -112,9 +112,10 @@ unique_toolkit/chat/state.py,sha256=Cjgwv_2vhDFbV69xxsn7SefhaoIAEqLx3ferdVFCnOg,
|
|
112
112
|
unique_toolkit/chat/utils.py,sha256=ihm-wQykBWhB4liR3LnwPVPt_qGW6ETq21Mw4HY0THE,854
|
113
113
|
unique_toolkit/content/__init__.py,sha256=EdJg_A_7loEtCQf4cah3QARQreJx6pdz89Rm96YbMVg,940
|
114
114
|
unique_toolkit/content/constants.py,sha256=1iy4Y67xobl5VTnJB6SxSyuoBWbdLl9244xfVMUZi5o,60
|
115
|
-
unique_toolkit/content/functions.py,sha256=
|
116
|
-
unique_toolkit/content/schemas.py,sha256=
|
115
|
+
unique_toolkit/content/functions.py,sha256=6Z9fOEsngJaT1R5DgoeRl5z06RavYgCxiAAPjkw-NKI,21100
|
116
|
+
unique_toolkit/content/schemas.py,sha256=YmZa6ddjf6_RvDFGPovamLd0YX99am9nzWYrKHo8-Hg,5579
|
117
117
|
unique_toolkit/content/service.py,sha256=i7wN_wYjF8NBZHBcpcA5XRlMRGntuw3mlVoudAoGQRk,23293
|
118
|
+
unique_toolkit/content/smart_rules.py,sha256=z2gHToPrdyj3HqO8Uu-JE5G2ClvJPuhR2XERmmkgoug,9668
|
118
119
|
unique_toolkit/content/utils.py,sha256=qNVmHTuETaPNGqheg7TbgPr1_1jbNHDc09N5RrmUIyo,7901
|
119
120
|
unique_toolkit/embedding/__init__.py,sha256=uUyzjonPvuDCYsvXCIt7ErQXopLggpzX-MEQd3_e2kE,250
|
120
121
|
unique_toolkit/embedding/constants.py,sha256=Lj8-Lcy1FvuC31PM9Exq7vaFuxQV4pEI1huUMFX-J2M,52
|
@@ -129,7 +130,7 @@ unique_toolkit/framework_utilities/openai/__init__.py,sha256=CrHYoC7lv2pBscitLer
|
|
129
130
|
unique_toolkit/framework_utilities/openai/client.py,sha256=ct1cqPcIK1wPl11G9sJV39ZnLJwKr2kDUDSra0FjvtM,2007
|
130
131
|
unique_toolkit/framework_utilities/openai/message_builder.py,sha256=VU6mJm_upLcarJQKFft_t1RlLRncWDxDuLC5LIJ5lQQ,4339
|
131
132
|
unique_toolkit/framework_utilities/utils.py,sha256=JK7g2yMfEx3eMprug26769xqNpS5WJcizf8n2zWMBng,789
|
132
|
-
unique_toolkit/knowledge_base.py,sha256=
|
133
|
+
unique_toolkit/knowledge_base.py,sha256=SHAFs68zDQuHJZdrFcdU7wnkUdfNQlNzpSLNckx3Scg,20167
|
133
134
|
unique_toolkit/language_model/__init__.py,sha256=lRQyLlbwHbNFf4-0foBU13UGb09lwEeodbVsfsSgaCk,1971
|
134
135
|
unique_toolkit/language_model/builder.py,sha256=4OKfwJfj3TrgO1ezc_ewIue6W7BCQ2ZYQXUckWVPPTA,3369
|
135
136
|
unique_toolkit/language_model/constants.py,sha256=B-topqW0r83dkC_25DeQfnPk3n53qzIHUCBS7YJ0-1U,119
|
@@ -147,8 +148,8 @@ unique_toolkit/short_term_memory/functions.py,sha256=3WiK-xatY5nh4Dr5zlDUye1k3E6
|
|
147
148
|
unique_toolkit/short_term_memory/schemas.py,sha256=OhfcXyF6ACdwIXW45sKzjtZX_gkcJs8FEZXcgQTNenw,1406
|
148
149
|
unique_toolkit/short_term_memory/service.py,sha256=5PeVBu1ZCAfyDb2HLVvlmqSbyzBBuE9sI2o9Aajqjxg,8884
|
149
150
|
unique_toolkit/smart_rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
150
|
-
unique_toolkit/smart_rules/compile.py,sha256=
|
151
|
-
unique_toolkit-1.11.
|
152
|
-
unique_toolkit-1.11.
|
153
|
-
unique_toolkit-1.11.
|
154
|
-
unique_toolkit-1.11.
|
151
|
+
unique_toolkit/smart_rules/compile.py,sha256=Ozhh70qCn2yOzRWr9d8WmJeTo7AQurwd3tStgBMPFLA,1246
|
152
|
+
unique_toolkit-1.11.3.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
|
153
|
+
unique_toolkit-1.11.3.dist-info/METADATA,sha256=bc9ld0sdYyiytQL4O7lba5xUSLJb5XtkzoWts984bi4,35749
|
154
|
+
unique_toolkit-1.11.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
155
|
+
unique_toolkit-1.11.3.dist-info/RECORD,,
|
File without changes
|
File without changes
|