unique_toolkit 0.7.7__py3-none-any.whl → 1.23.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of unique_toolkit might be problematic. Click here for more details.

Files changed (166) hide show
  1. unique_toolkit/__init__.py +28 -1
  2. unique_toolkit/_common/api_calling/human_verification_manager.py +343 -0
  3. unique_toolkit/_common/base_model_type_attribute.py +303 -0
  4. unique_toolkit/_common/chunk_relevancy_sorter/config.py +49 -0
  5. unique_toolkit/_common/chunk_relevancy_sorter/exception.py +5 -0
  6. unique_toolkit/_common/chunk_relevancy_sorter/schemas.py +46 -0
  7. unique_toolkit/_common/chunk_relevancy_sorter/service.py +374 -0
  8. unique_toolkit/_common/chunk_relevancy_sorter/tests/test_service.py +275 -0
  9. unique_toolkit/_common/default_language_model.py +12 -0
  10. unique_toolkit/_common/docx_generator/__init__.py +7 -0
  11. unique_toolkit/_common/docx_generator/config.py +12 -0
  12. unique_toolkit/_common/docx_generator/schemas.py +80 -0
  13. unique_toolkit/_common/docx_generator/service.py +252 -0
  14. unique_toolkit/_common/docx_generator/template/Doc Template.docx +0 -0
  15. unique_toolkit/_common/endpoint_builder.py +305 -0
  16. unique_toolkit/_common/endpoint_requestor.py +430 -0
  17. unique_toolkit/_common/exception.py +24 -0
  18. unique_toolkit/_common/feature_flags/schema.py +9 -0
  19. unique_toolkit/_common/pydantic/rjsf_tags.py +936 -0
  20. unique_toolkit/_common/pydantic_helpers.py +154 -0
  21. unique_toolkit/_common/referencing.py +53 -0
  22. unique_toolkit/_common/string_utilities.py +140 -0
  23. unique_toolkit/_common/tests/test_referencing.py +521 -0
  24. unique_toolkit/_common/tests/test_string_utilities.py +506 -0
  25. unique_toolkit/_common/token/image_token_counting.py +67 -0
  26. unique_toolkit/_common/token/token_counting.py +204 -0
  27. unique_toolkit/_common/utils/__init__.py +1 -0
  28. unique_toolkit/_common/utils/files.py +43 -0
  29. unique_toolkit/_common/utils/structured_output/__init__.py +1 -0
  30. unique_toolkit/_common/utils/structured_output/schema.py +5 -0
  31. unique_toolkit/_common/utils/write_configuration.py +51 -0
  32. unique_toolkit/_common/validators.py +101 -4
  33. unique_toolkit/agentic/__init__.py +1 -0
  34. unique_toolkit/agentic/debug_info_manager/debug_info_manager.py +28 -0
  35. unique_toolkit/agentic/debug_info_manager/test/test_debug_info_manager.py +278 -0
  36. unique_toolkit/agentic/evaluation/config.py +36 -0
  37. unique_toolkit/{evaluators → agentic/evaluation}/context_relevancy/prompts.py +25 -0
  38. unique_toolkit/agentic/evaluation/context_relevancy/schema.py +80 -0
  39. unique_toolkit/agentic/evaluation/context_relevancy/service.py +273 -0
  40. unique_toolkit/agentic/evaluation/evaluation_manager.py +218 -0
  41. unique_toolkit/agentic/evaluation/hallucination/constants.py +61 -0
  42. unique_toolkit/agentic/evaluation/hallucination/hallucination_evaluation.py +111 -0
  43. unique_toolkit/{evaluators → agentic/evaluation}/hallucination/prompts.py +1 -1
  44. unique_toolkit/{evaluators → agentic/evaluation}/hallucination/service.py +16 -15
  45. unique_toolkit/{evaluators → agentic/evaluation}/hallucination/utils.py +30 -20
  46. unique_toolkit/{evaluators → agentic/evaluation}/output_parser.py +20 -2
  47. unique_toolkit/{evaluators → agentic/evaluation}/schemas.py +27 -7
  48. unique_toolkit/agentic/evaluation/tests/test_context_relevancy_service.py +253 -0
  49. unique_toolkit/agentic/evaluation/tests/test_output_parser.py +87 -0
  50. unique_toolkit/agentic/history_manager/history_construction_with_contents.py +297 -0
  51. unique_toolkit/agentic/history_manager/history_manager.py +242 -0
  52. unique_toolkit/agentic/history_manager/loop_token_reducer.py +484 -0
  53. unique_toolkit/agentic/history_manager/utils.py +96 -0
  54. unique_toolkit/agentic/postprocessor/postprocessor_manager.py +212 -0
  55. unique_toolkit/agentic/reference_manager/reference_manager.py +103 -0
  56. unique_toolkit/agentic/responses_api/__init__.py +19 -0
  57. unique_toolkit/agentic/responses_api/postprocessors/code_display.py +63 -0
  58. unique_toolkit/agentic/responses_api/postprocessors/generated_files.py +145 -0
  59. unique_toolkit/agentic/responses_api/stream_handler.py +15 -0
  60. unique_toolkit/agentic/short_term_memory_manager/persistent_short_term_memory_manager.py +141 -0
  61. unique_toolkit/agentic/thinking_manager/thinking_manager.py +103 -0
  62. unique_toolkit/agentic/tools/__init__.py +1 -0
  63. unique_toolkit/agentic/tools/a2a/__init__.py +36 -0
  64. unique_toolkit/agentic/tools/a2a/config.py +17 -0
  65. unique_toolkit/agentic/tools/a2a/evaluation/__init__.py +15 -0
  66. unique_toolkit/agentic/tools/a2a/evaluation/_utils.py +66 -0
  67. unique_toolkit/agentic/tools/a2a/evaluation/config.py +55 -0
  68. unique_toolkit/agentic/tools/a2a/evaluation/evaluator.py +260 -0
  69. unique_toolkit/agentic/tools/a2a/evaluation/summarization_user_message.j2 +9 -0
  70. unique_toolkit/agentic/tools/a2a/manager.py +55 -0
  71. unique_toolkit/agentic/tools/a2a/postprocessing/__init__.py +21 -0
  72. unique_toolkit/agentic/tools/a2a/postprocessing/_display_utils.py +185 -0
  73. unique_toolkit/agentic/tools/a2a/postprocessing/_ref_utils.py +73 -0
  74. unique_toolkit/agentic/tools/a2a/postprocessing/config.py +45 -0
  75. unique_toolkit/agentic/tools/a2a/postprocessing/display.py +180 -0
  76. unique_toolkit/agentic/tools/a2a/postprocessing/references.py +101 -0
  77. unique_toolkit/agentic/tools/a2a/postprocessing/test/test_display_utils.py +1335 -0
  78. unique_toolkit/agentic/tools/a2a/postprocessing/test/test_ref_utils.py +603 -0
  79. unique_toolkit/agentic/tools/a2a/prompts.py +46 -0
  80. unique_toolkit/agentic/tools/a2a/response_watcher/__init__.py +6 -0
  81. unique_toolkit/agentic/tools/a2a/response_watcher/service.py +91 -0
  82. unique_toolkit/agentic/tools/a2a/tool/__init__.py +4 -0
  83. unique_toolkit/agentic/tools/a2a/tool/_memory.py +26 -0
  84. unique_toolkit/agentic/tools/a2a/tool/_schema.py +9 -0
  85. unique_toolkit/agentic/tools/a2a/tool/config.py +73 -0
  86. unique_toolkit/agentic/tools/a2a/tool/service.py +306 -0
  87. unique_toolkit/agentic/tools/agent_chunks_hanlder.py +65 -0
  88. unique_toolkit/agentic/tools/config.py +167 -0
  89. unique_toolkit/agentic/tools/factory.py +44 -0
  90. unique_toolkit/agentic/tools/mcp/__init__.py +4 -0
  91. unique_toolkit/agentic/tools/mcp/manager.py +71 -0
  92. unique_toolkit/agentic/tools/mcp/models.py +28 -0
  93. unique_toolkit/agentic/tools/mcp/tool_wrapper.py +234 -0
  94. unique_toolkit/agentic/tools/openai_builtin/__init__.py +11 -0
  95. unique_toolkit/agentic/tools/openai_builtin/base.py +30 -0
  96. unique_toolkit/agentic/tools/openai_builtin/code_interpreter/__init__.py +8 -0
  97. unique_toolkit/agentic/tools/openai_builtin/code_interpreter/config.py +57 -0
  98. unique_toolkit/agentic/tools/openai_builtin/code_interpreter/service.py +230 -0
  99. unique_toolkit/agentic/tools/openai_builtin/manager.py +62 -0
  100. unique_toolkit/agentic/tools/schemas.py +141 -0
  101. unique_toolkit/agentic/tools/test/test_mcp_manager.py +536 -0
  102. unique_toolkit/agentic/tools/test/test_tool_progress_reporter.py +445 -0
  103. unique_toolkit/agentic/tools/tool.py +183 -0
  104. unique_toolkit/agentic/tools/tool_manager.py +523 -0
  105. unique_toolkit/agentic/tools/tool_progress_reporter.py +285 -0
  106. unique_toolkit/agentic/tools/utils/__init__.py +19 -0
  107. unique_toolkit/agentic/tools/utils/execution/__init__.py +1 -0
  108. unique_toolkit/agentic/tools/utils/execution/execution.py +286 -0
  109. unique_toolkit/agentic/tools/utils/source_handling/__init__.py +0 -0
  110. unique_toolkit/agentic/tools/utils/source_handling/schema.py +21 -0
  111. unique_toolkit/agentic/tools/utils/source_handling/source_formatting.py +207 -0
  112. unique_toolkit/agentic/tools/utils/source_handling/tests/test_source_formatting.py +216 -0
  113. unique_toolkit/app/__init__.py +6 -0
  114. unique_toolkit/app/dev_util.py +180 -0
  115. unique_toolkit/app/init_sdk.py +32 -1
  116. unique_toolkit/app/schemas.py +198 -31
  117. unique_toolkit/app/unique_settings.py +367 -0
  118. unique_toolkit/chat/__init__.py +8 -1
  119. unique_toolkit/chat/deprecated/service.py +232 -0
  120. unique_toolkit/chat/functions.py +642 -77
  121. unique_toolkit/chat/rendering.py +34 -0
  122. unique_toolkit/chat/responses_api.py +461 -0
  123. unique_toolkit/chat/schemas.py +133 -2
  124. unique_toolkit/chat/service.py +115 -767
  125. unique_toolkit/content/functions.py +153 -4
  126. unique_toolkit/content/schemas.py +122 -15
  127. unique_toolkit/content/service.py +278 -44
  128. unique_toolkit/content/smart_rules.py +301 -0
  129. unique_toolkit/content/utils.py +8 -3
  130. unique_toolkit/embedding/service.py +102 -11
  131. unique_toolkit/framework_utilities/__init__.py +1 -0
  132. unique_toolkit/framework_utilities/langchain/client.py +71 -0
  133. unique_toolkit/framework_utilities/langchain/history.py +19 -0
  134. unique_toolkit/framework_utilities/openai/__init__.py +6 -0
  135. unique_toolkit/framework_utilities/openai/client.py +83 -0
  136. unique_toolkit/framework_utilities/openai/message_builder.py +229 -0
  137. unique_toolkit/framework_utilities/utils.py +23 -0
  138. unique_toolkit/language_model/__init__.py +3 -0
  139. unique_toolkit/language_model/builder.py +27 -11
  140. unique_toolkit/language_model/default_language_model.py +3 -0
  141. unique_toolkit/language_model/functions.py +327 -43
  142. unique_toolkit/language_model/infos.py +992 -50
  143. unique_toolkit/language_model/reference.py +242 -0
  144. unique_toolkit/language_model/schemas.py +475 -48
  145. unique_toolkit/language_model/service.py +228 -27
  146. unique_toolkit/protocols/support.py +145 -0
  147. unique_toolkit/services/__init__.py +7 -0
  148. unique_toolkit/services/chat_service.py +1630 -0
  149. unique_toolkit/services/knowledge_base.py +861 -0
  150. unique_toolkit/short_term_memory/service.py +178 -41
  151. unique_toolkit/smart_rules/__init__.py +0 -0
  152. unique_toolkit/smart_rules/compile.py +56 -0
  153. unique_toolkit/test_utilities/events.py +197 -0
  154. {unique_toolkit-0.7.7.dist-info → unique_toolkit-1.23.0.dist-info}/METADATA +606 -7
  155. unique_toolkit-1.23.0.dist-info/RECORD +182 -0
  156. unique_toolkit/evaluators/__init__.py +0 -1
  157. unique_toolkit/evaluators/config.py +0 -35
  158. unique_toolkit/evaluators/constants.py +0 -1
  159. unique_toolkit/evaluators/context_relevancy/constants.py +0 -32
  160. unique_toolkit/evaluators/context_relevancy/service.py +0 -53
  161. unique_toolkit/evaluators/context_relevancy/utils.py +0 -142
  162. unique_toolkit/evaluators/hallucination/constants.py +0 -41
  163. unique_toolkit-0.7.7.dist-info/RECORD +0 -64
  164. /unique_toolkit/{evaluators → agentic/evaluation}/exception.py +0 -0
  165. {unique_toolkit-0.7.7.dist-info → unique_toolkit-1.23.0.dist-info}/LICENSE +0 -0
  166. {unique_toolkit-0.7.7.dist-info → unique_toolkit-1.23.0.dist-info}/WHEEL +0 -0
@@ -0,0 +1,861 @@
1
+ import asyncio
2
+ import logging
3
+ from pathlib import Path
4
+ from typing import Any, overload
5
+
6
+ import humps
7
+ import unique_sdk
8
+
9
+ from unique_toolkit._common.validate_required_values import validate_required_values
10
+ from unique_toolkit.app.schemas import BaseEvent, ChatEvent, Event
11
+ from unique_toolkit.app.unique_settings import UniqueSettings
12
+ from unique_toolkit.content.constants import (
13
+ DEFAULT_SEARCH_LANGUAGE,
14
+ )
15
+ from unique_toolkit.content.functions import (
16
+ delete_content,
17
+ delete_content_async,
18
+ download_content_to_bytes,
19
+ download_content_to_file_by_id,
20
+ get_content_info,
21
+ get_folder_info,
22
+ search_content_chunks,
23
+ search_content_chunks_async,
24
+ search_contents,
25
+ search_contents_async,
26
+ update_content,
27
+ upload_content,
28
+ upload_content_from_bytes,
29
+ )
30
+ from unique_toolkit.content.schemas import (
31
+ Content,
32
+ ContentChunk,
33
+ ContentInfo,
34
+ ContentRerankerConfig,
35
+ ContentSearchType,
36
+ DeleteContentResponse,
37
+ FolderInfo,
38
+ PaginatedContentInfos,
39
+ )
40
+
41
+ _LOGGER = logging.getLogger(f"toolkit.knowledge_base.{__name__}")
42
+
43
+ _DEFAULT_SCORE_THRESHOLD: float = 0.5
44
+
45
+
46
+ class KnowledgeBaseService:
47
+ """
48
+ Provides methods for searching, downloading and uploading content in the knowledge base.
49
+ """
50
+
51
+ def __init__(
52
+ self,
53
+ company_id: str,
54
+ user_id: str,
55
+ metadata_filter: dict | None = None,
56
+ ):
57
+ """
58
+ Initialize the ContentService with a company_id, user_id and chat_id.
59
+ """
60
+
61
+ self._metadata_filter = None
62
+ [company_id, user_id] = validate_required_values([company_id, user_id])
63
+ self._company_id = company_id
64
+ self._user_id = user_id
65
+ self._metadata_filter = metadata_filter
66
+
67
+ @classmethod
68
+ def from_event(cls, event: BaseEvent):
69
+ """
70
+ Initialize the ContentService with an event.
71
+ """
72
+ metadata_filter = None
73
+
74
+ if isinstance(event, (ChatEvent | Event)):
75
+ metadata_filter = event.payload.metadata_filter
76
+
77
+ return cls(
78
+ company_id=event.company_id,
79
+ user_id=event.user_id,
80
+ metadata_filter=metadata_filter,
81
+ )
82
+
83
+ @classmethod
84
+ def from_settings(
85
+ cls,
86
+ settings: UniqueSettings | str | None = None,
87
+ metadata_filter: dict | None = None,
88
+ ):
89
+ """
90
+ Initialize the ContentService with a settings object and metadata filter.
91
+ """
92
+
93
+ if settings is None:
94
+ settings = UniqueSettings.from_env_auto_with_sdk_init()
95
+ elif isinstance(settings, str):
96
+ settings = UniqueSettings.from_env_auto_with_sdk_init(filename=settings)
97
+
98
+ return cls(
99
+ company_id=settings.auth.company_id.get_secret_value(),
100
+ user_id=settings.auth.user_id.get_secret_value(),
101
+ metadata_filter=metadata_filter,
102
+ )
103
+
104
+ @overload
105
+ def search_content_chunks(
106
+ self,
107
+ *,
108
+ search_string: str,
109
+ search_type: ContentSearchType,
110
+ limit: int,
111
+ scope_ids: list[str],
112
+ score_threshold: float = _DEFAULT_SCORE_THRESHOLD,
113
+ search_language: str = DEFAULT_SEARCH_LANGUAGE,
114
+ reranker_config: ContentRerankerConfig | None = None,
115
+ ) -> list[ContentChunk]: ...
116
+
117
+ @overload
118
+ def search_content_chunks(
119
+ self,
120
+ *,
121
+ search_string: str,
122
+ search_type: ContentSearchType,
123
+ limit: int,
124
+ metadata_filter: dict,
125
+ scope_ids: list[str] | None = None,
126
+ score_threshold: float = _DEFAULT_SCORE_THRESHOLD,
127
+ search_language: str = DEFAULT_SEARCH_LANGUAGE,
128
+ reranker_config: ContentRerankerConfig | None = None,
129
+ ) -> list[ContentChunk]: ...
130
+
131
+ @overload
132
+ def search_content_chunks(
133
+ self,
134
+ *,
135
+ search_string: str,
136
+ search_type: ContentSearchType,
137
+ limit: int,
138
+ metadata_filter: dict,
139
+ content_ids: list[str],
140
+ score_threshold: float = _DEFAULT_SCORE_THRESHOLD,
141
+ search_language: str = DEFAULT_SEARCH_LANGUAGE,
142
+ reranker_config: ContentRerankerConfig | None = None,
143
+ ) -> list[ContentChunk]: ...
144
+
145
+ def search_content_chunks(
146
+ self,
147
+ *,
148
+ search_string: str,
149
+ search_type: ContentSearchType,
150
+ limit: int,
151
+ search_language: str = DEFAULT_SEARCH_LANGUAGE,
152
+ reranker_config: ContentRerankerConfig | None = None,
153
+ scope_ids: list[str] | None = None,
154
+ metadata_filter: dict | None = None,
155
+ content_ids: list[str] | None = None,
156
+ score_threshold: float | None = None,
157
+ ) -> list[ContentChunk]:
158
+ """
159
+ Performs a synchronous search for content chunks in the knowledge base.
160
+
161
+ Args:
162
+ search_string (str): The search string.
163
+ search_type (ContentSearchType): The type of search to perform.
164
+ limit (int): The maximum number of results to return.
165
+ search_language (str, optional): The language for the full-text search. Defaults to "english".
166
+ reranker_config (ContentRerankerConfig | None, optional): The reranker configuration. Defaults to None.
167
+ scope_ids (list[str] | None, optional): The scope IDs to filter by. Defaults to None.
168
+ metadata_filter (dict | None, optional): UniqueQL metadata filter. If unspecified/None, it tries to use the metadata filter from the event. Defaults to None.
169
+ content_ids (list[str] | None, optional): The content IDs to search within. Defaults to None.
170
+ score_threshold (float | None, optional): Sets the minimum similarity score for search results to be considered. Defaults to 0.
171
+
172
+ Returns:
173
+ list[ContentChunk]: The search results.
174
+
175
+ Raises:
176
+ Exception: If there's an error during the search operation.
177
+ """
178
+
179
+ if metadata_filter is None:
180
+ metadata_filter = self._metadata_filter
181
+
182
+ try:
183
+ searches = search_content_chunks(
184
+ user_id=self._user_id,
185
+ company_id=self._company_id,
186
+ chat_id="",
187
+ search_string=search_string,
188
+ search_type=search_type,
189
+ limit=limit,
190
+ search_language=search_language,
191
+ reranker_config=reranker_config,
192
+ scope_ids=scope_ids,
193
+ chat_only=False,
194
+ metadata_filter=metadata_filter,
195
+ content_ids=content_ids,
196
+ score_threshold=score_threshold,
197
+ )
198
+ return searches
199
+ except Exception as e:
200
+ _LOGGER.error(f"Error while searching content chunks: {e}")
201
+ raise e
202
+
203
+ @overload
204
+ async def search_content_chunks_async(
205
+ self,
206
+ *,
207
+ search_string: str,
208
+ search_type: ContentSearchType,
209
+ limit: int,
210
+ scope_ids: list[str],
211
+ score_threshold: float = _DEFAULT_SCORE_THRESHOLD,
212
+ search_language: str = DEFAULT_SEARCH_LANGUAGE,
213
+ reranker_config: ContentRerankerConfig | None = None,
214
+ ) -> list[ContentChunk]: ...
215
+
216
+ @overload
217
+ async def search_content_chunks_async(
218
+ self,
219
+ *,
220
+ search_string: str,
221
+ search_type: ContentSearchType,
222
+ limit: int,
223
+ metadata_filter: dict,
224
+ scope_ids: list[str] | None = None,
225
+ score_threshold: float = _DEFAULT_SCORE_THRESHOLD,
226
+ search_language: str = DEFAULT_SEARCH_LANGUAGE,
227
+ reranker_config: ContentRerankerConfig | None = None,
228
+ ) -> list[ContentChunk]: ...
229
+
230
+ @overload
231
+ async def search_content_chunks_async(
232
+ self,
233
+ *,
234
+ search_string: str,
235
+ search_type: ContentSearchType,
236
+ limit: int,
237
+ metadata_filter: dict,
238
+ content_ids: list[str],
239
+ score_threshold: float = _DEFAULT_SCORE_THRESHOLD,
240
+ search_language: str = DEFAULT_SEARCH_LANGUAGE,
241
+ reranker_config: ContentRerankerConfig | None = None,
242
+ ) -> list[ContentChunk]: ...
243
+
244
+ async def search_content_chunks_async(
245
+ self,
246
+ *,
247
+ search_string: str,
248
+ search_type: ContentSearchType,
249
+ limit: int,
250
+ search_language: str = DEFAULT_SEARCH_LANGUAGE,
251
+ reranker_config: ContentRerankerConfig | None = None,
252
+ scope_ids: list[str] | None = None,
253
+ metadata_filter: dict | None = None,
254
+ content_ids: list[str] | None = None,
255
+ score_threshold: float | None = None,
256
+ ):
257
+ """
258
+ Performs an asynchronous search for content chunks in the knowledge base.
259
+
260
+ Args:
261
+ search_string (str): The search string.
262
+ search_type (ContentSearchType): The type of search to perform.
263
+ limit (int): The maximum number of results to return.
264
+ search_language (str, optional): The language for the full-text search. Defaults to "english".
265
+ reranker_config (ContentRerankerConfig | None, optional): The reranker configuration. Defaults to None.
266
+ scope_ids (list[str] | None, optional): The scope IDs to filter by. Defaults to None.
267
+ metadata_filter (dict | None, optional): UniqueQL metadata filter. If unspecified/None, it tries to use the metadata filter from the event. Defaults to None.
268
+ content_ids (list[str] | None, optional): The content IDs to search within. Defaults to None.
269
+ score_threshold (float | None, optional): Sets the minimum similarity score for search results to be considered. Defaults to 0.
270
+
271
+ Returns:
272
+ list[ContentChunk]: The search results.
273
+
274
+ Raises:
275
+ Exception: If there's an error during the search operation.
276
+ """
277
+ if metadata_filter is None:
278
+ metadata_filter = self._metadata_filter
279
+
280
+ try:
281
+ searches = await search_content_chunks_async(
282
+ user_id=self._user_id,
283
+ company_id=self._company_id,
284
+ chat_id="",
285
+ search_string=search_string,
286
+ search_type=search_type,
287
+ limit=limit,
288
+ search_language=search_language,
289
+ reranker_config=reranker_config,
290
+ scope_ids=scope_ids,
291
+ chat_only=False,
292
+ metadata_filter=metadata_filter,
293
+ content_ids=content_ids,
294
+ score_threshold=score_threshold,
295
+ )
296
+ return searches
297
+ except Exception as e:
298
+ _LOGGER.error(f"Error while searching content chunks: {e}")
299
+ raise e
300
+
301
+ def search_contents(
302
+ self,
303
+ *,
304
+ where: dict,
305
+ ) -> list[Content]:
306
+ """
307
+ Performs a search in the knowledge base by filter (and not a smilarity search)
308
+ This function loads complete content of the files from the knowledge base in contrast to search_content_chunks.
309
+
310
+ Args:
311
+ where (dict): The search criteria.
312
+
313
+ Returns:
314
+ list[Content]: The search results.
315
+ """
316
+
317
+ return search_contents(
318
+ user_id=self._user_id,
319
+ company_id=self._company_id,
320
+ chat_id="",
321
+ where=where,
322
+ )
323
+
324
+ async def search_contents_async(
325
+ self,
326
+ *,
327
+ where: dict,
328
+ ) -> list[Content]:
329
+ """
330
+ Performs an asynchronous search for content files in the knowledge base by filter.
331
+
332
+ Args:
333
+ where (dict): The search criteria.
334
+
335
+ Returns:
336
+ list[Content]: The search results.
337
+ """
338
+
339
+ return await search_contents_async(
340
+ user_id=self._user_id,
341
+ company_id=self._company_id,
342
+ chat_id="",
343
+ where=where,
344
+ )
345
+
346
+ def upload_content_from_bytes(
347
+ self,
348
+ content: bytes,
349
+ *,
350
+ content_name: str,
351
+ mime_type: str,
352
+ scope_id: str,
353
+ skip_ingestion: bool = False,
354
+ ingestion_config: unique_sdk.Content.IngestionConfig | None = None,
355
+ metadata: dict | None = None,
356
+ ) -> Content:
357
+ """
358
+ Uploads content to the knowledge base.
359
+
360
+ Args:
361
+ content (bytes): The content to upload.
362
+ content_name (str): The name of the content.
363
+ mime_type (str): The MIME type of the content.
364
+ scope_id (str | None): The scope ID. Defaults to None.
365
+ skip_ingestion (bool): Whether to skip ingestion. Defaults to False.
366
+ skip_excel_ingestion (bool): Whether to skip excel ingestion. Defaults to False.
367
+ ingestion_config (unique_sdk.Content.IngestionConfig | None): The ingestion configuration. Defaults to None.
368
+ metadata (dict | None): The metadata to associate with the content. Defaults to None.
369
+
370
+ Returns:
371
+ Content: The uploaded content.
372
+ """
373
+
374
+ return upload_content_from_bytes(
375
+ user_id=self._user_id,
376
+ company_id=self._company_id,
377
+ content=content,
378
+ content_name=content_name,
379
+ mime_type=mime_type,
380
+ scope_id=scope_id,
381
+ chat_id="",
382
+ skip_ingestion=skip_ingestion,
383
+ ingestion_config=ingestion_config,
384
+ metadata=metadata,
385
+ )
386
+
387
+ def upload_content(
388
+ self,
389
+ path_to_content: str,
390
+ content_name: str,
391
+ mime_type: str,
392
+ scope_id: str,
393
+ skip_ingestion: bool = False,
394
+ skip_excel_ingestion: bool = False,
395
+ ingestion_config: unique_sdk.Content.IngestionConfig | None = None,
396
+ metadata: dict[str, Any] | None = None,
397
+ ):
398
+ """
399
+ Uploads content to the knowledge base.
400
+
401
+ Args:
402
+ path_to_content (str): The path to the content to upload.
403
+ content_name (str): The name of the content.
404
+ mime_type (str): The MIME type of the content.
405
+ scope_id (str | None): The scope ID. Defaults to None.
406
+ skip_ingestion (bool): Whether to skip ingestion. Defaults to False.
407
+ skip_excel_ingestion (bool): Whether to skip excel ingestion. Defaults to False.
408
+ ingestion_config (unique_sdk.Content.IngestionConfig | None): The ingestion configuration. Defaults to None.
409
+ metadata (dict[str, Any] | None): The metadata to associate with the content. Defaults to None.
410
+
411
+ Returns:
412
+ Content: The uploaded content.
413
+ """
414
+
415
+ return upload_content(
416
+ user_id=self._user_id,
417
+ company_id=self._company_id,
418
+ path_to_content=path_to_content,
419
+ content_name=content_name,
420
+ mime_type=mime_type,
421
+ scope_id=scope_id,
422
+ chat_id="",
423
+ skip_ingestion=skip_ingestion,
424
+ skip_excel_ingestion=skip_excel_ingestion,
425
+ ingestion_config=ingestion_config,
426
+ metadata=metadata,
427
+ )
428
+
429
+ def download_content_to_file(
430
+ self,
431
+ *,
432
+ content_id: str,
433
+ output_dir_path: Path | None = None,
434
+ output_filename: str | None = None,
435
+ ):
436
+ """
437
+ Downloads content from a chat and saves it to a file.
438
+
439
+ Args:
440
+ content_id (str): The ID of the content to download.
441
+ filename (str | None): The name of the file to save the content as. If not provided, the original filename will be used. Defaults to None.
442
+ tmp_dir_path (str | Path | None): The path to the temporary directory where the content will be saved. Defaults to "/tmp".
443
+
444
+ Returns:
445
+ Path: The path to the downloaded file.
446
+
447
+ Raises:
448
+ Exception: If the download fails or the filename cannot be determined.
449
+ """
450
+
451
+ return download_content_to_file_by_id(
452
+ user_id=self._user_id,
453
+ company_id=self._company_id,
454
+ content_id=content_id,
455
+ chat_id="",
456
+ filename=output_filename,
457
+ tmp_dir_path=output_dir_path,
458
+ )
459
+
460
+ def download_content_to_bytes(
461
+ self,
462
+ *,
463
+ content_id: str,
464
+ ) -> bytes:
465
+ """
466
+ Downloads content to memory
467
+
468
+ Args:
469
+ content_id (str): The id of the uploaded content.
470
+ chat_id (Optional[str]): The chat_id, defaults to None.
471
+
472
+ Returns:
473
+ bytes: The downloaded content.
474
+
475
+ Raises:
476
+ Exception: If the download fails.
477
+ """
478
+
479
+ return download_content_to_bytes(
480
+ user_id=self._user_id,
481
+ company_id=self._company_id,
482
+ content_id=content_id,
483
+ chat_id=None,
484
+ )
485
+
486
+ def get_paginated_content_infos(
487
+ self,
488
+ *,
489
+ metadata_filter: dict[str, Any] | None = None,
490
+ skip: int | None = None,
491
+ take: int | None = None,
492
+ file_path: str | None = None,
493
+ ) -> PaginatedContentInfos:
494
+ return get_content_info(
495
+ user_id=self._user_id,
496
+ company_id=self._company_id,
497
+ metadata_filter=metadata_filter,
498
+ skip=skip,
499
+ take=take,
500
+ file_path=file_path,
501
+ )
502
+
503
+ def get_folder_info(
504
+ self,
505
+ *,
506
+ scope_id: str,
507
+ ) -> FolderInfo:
508
+ return get_folder_info(
509
+ user_id=self._user_id,
510
+ company_id=self._company_id,
511
+ scope_id=scope_id,
512
+ )
513
+
514
+ def replace_content_metadata(
515
+ self,
516
+ *,
517
+ content_id: str,
518
+ metadata: dict[str, Any],
519
+ ) -> ContentInfo:
520
+ return update_content(
521
+ user_id=self._user_id,
522
+ company_id=self._company_id,
523
+ content_id=content_id,
524
+ metadata=metadata,
525
+ )
526
+
527
+ def _resolve_visible_file_tree(self, content_infos: list[ContentInfo]) -> list[str]:
528
+ # collect all scope ids
529
+ folder_id_paths: set[str] = set()
530
+ known_folder_paths: set[str] = set()
531
+ for content_info in content_infos:
532
+ if (
533
+ content_info.metadata
534
+ and content_info.metadata.get(r"{FullPath}") is not None
535
+ ):
536
+ known_folder_paths.add(str(content_info.metadata.get(r"{FullPath}")))
537
+ continue
538
+
539
+ if (
540
+ content_info.metadata
541
+ and content_info.metadata.get("folderIdPath") is not None
542
+ ):
543
+ folder_id_paths.add(str(content_info.metadata.get("folderIdPath")))
544
+
545
+ scope_ids: set[str] = set()
546
+ for fp in folder_id_paths:
547
+ scope_ids_list = set(fp.replace("uniquepathid://", "").split("/"))
548
+ scope_ids.update(scope_ids_list)
549
+
550
+ scope_id_to_folder_name: dict[str, str] = {}
551
+ for scope_id in scope_ids:
552
+ folder_info = self.get_folder_info(
553
+ scope_id=scope_id,
554
+ )
555
+ scope_id_to_folder_name[scope_id] = folder_info.name
556
+
557
+ folder_paths: set[str] = set()
558
+ for folder_id_path in folder_id_paths:
559
+ scope_ids_list = folder_id_path.replace("uniquepathid://", "").split("/")
560
+
561
+ if all(scope_id in scope_id_to_folder_name for scope_id in scope_ids_list):
562
+ folder_path = [
563
+ scope_id_to_folder_name[scope_id] for scope_id in scope_ids_list
564
+ ]
565
+ folder_paths.add("/".join(folder_path))
566
+
567
+ return [
568
+ p if p.startswith("/") else f"/{p}"
569
+ for p in folder_paths.union(known_folder_paths)
570
+ ]
571
+
572
+ def resolve_visible_file_tree(
573
+ self, *, metadata_filter: dict[str, Any] | None = None
574
+ ) -> list[str]:
575
+ """
576
+ Resolves the visible file tree for the knowledge base for the current user.
577
+
578
+ Args:
579
+ metadata_filter (dict[str, Any] | None): The metadata filter to use. Defaults to None.
580
+
581
+ Returns:
582
+ list[str]: The visible file tree.
583
+
584
+
585
+
586
+ """
587
+ info = self.get_paginated_content_infos(
588
+ metadata_filter=metadata_filter,
589
+ )
590
+
591
+ return self._resolve_visible_file_tree(content_infos=info.content_infos)
592
+
593
+ def _pop_forbidden_metadata_keys(self, metadata: dict[str, Any]) -> dict[str, Any]:
594
+ forbidden_keys = [
595
+ "key",
596
+ "url",
597
+ "title",
598
+ "folderId",
599
+ "mimeType",
600
+ "companyId",
601
+ "contentId",
602
+ "folderIdPath",
603
+ "externalFileOwner",
604
+ ]
605
+ for key in forbidden_keys:
606
+ metadata.pop(key, None)
607
+ return metadata
608
+
609
+ def update_content_metadata(
610
+ self,
611
+ *,
612
+ content_info: ContentInfo,
613
+ additional_metadata: dict[str, Any],
614
+ ) -> ContentInfo:
615
+ camelized_additional_metadata = humps.camelize(additional_metadata)
616
+ camelized_additional_metadata = self._pop_forbidden_metadata_keys(
617
+ camelized_additional_metadata
618
+ )
619
+
620
+ if content_info.metadata is not None:
621
+ content_info.metadata.update(camelized_additional_metadata)
622
+ else:
623
+ content_info.metadata = camelized_additional_metadata
624
+
625
+ return update_content(
626
+ user_id=self._user_id,
627
+ company_id=self._company_id,
628
+ content_id=content_info.id,
629
+ metadata=content_info.metadata,
630
+ )
631
+
632
+ def remove_content_metadata(
633
+ self,
634
+ *,
635
+ content_info: ContentInfo,
636
+ keys_to_remove: list[str],
637
+ ) -> ContentInfo:
638
+ """
639
+ Removes the specified keys irreversibly from the content metadata.
640
+
641
+ Note: Keys are camelized before being removed as metadata keys are stored in camelCase.
642
+ """
643
+
644
+ if content_info.metadata is None:
645
+ _LOGGER.warning(f"Content metadata is None for content {content_info.id}")
646
+ return content_info
647
+
648
+ for key in keys_to_remove:
649
+ content_info.metadata[humps.camelize(key)] = None
650
+
651
+ return update_content(
652
+ user_id=self._user_id,
653
+ company_id=self._company_id,
654
+ content_id=content_info.id,
655
+ metadata=content_info.metadata or {},
656
+ )
657
+
658
+ @overload
659
+ def update_contents_metadata(
660
+ self,
661
+ *,
662
+ additional_metadata: dict[str, Any],
663
+ content_infos: list[ContentInfo],
664
+ ) -> list[ContentInfo]: ...
665
+
666
+ @overload
667
+ def update_contents_metadata(
668
+ self, *, additional_metadata: dict[str, Any], metadata_filter: dict[str, Any]
669
+ ) -> list[ContentInfo]: ...
670
+
671
+ def update_contents_metadata(
672
+ self,
673
+ *,
674
+ additional_metadata: dict[str, Any],
675
+ metadata_filter: dict[str, Any] | None = None,
676
+ content_infos: list[ContentInfo] | None = None,
677
+ ) -> list[ContentInfo]:
678
+ """Update the metadata of the contents matching the metadata filter.
679
+
680
+ Note: Keys are camelized before being updated as metadata keys are stored in camelCase.
681
+ """
682
+
683
+ additional_metadata_camelized = humps.camelize(additional_metadata)
684
+ additional_metadata_camelized = self._pop_forbidden_metadata_keys(
685
+ additional_metadata_camelized
686
+ )
687
+
688
+ if content_infos is None:
689
+ content_infos = self.get_paginated_content_infos(
690
+ metadata_filter=metadata_filter,
691
+ ).content_infos
692
+
693
+ for info in content_infos:
694
+ self.update_content_metadata(
695
+ content_info=info, additional_metadata=additional_metadata_camelized
696
+ )
697
+
698
+ return content_infos
699
+
700
+ @overload
701
+ def remove_contents_metadata(
702
+ self,
703
+ *,
704
+ keys_to_remove: list[str],
705
+ content_infos: list[ContentInfo],
706
+ ) -> list[ContentInfo]: ...
707
+
708
+ @overload
709
+ def remove_contents_metadata(
710
+ self, *, keys_to_remove: list[str], metadata_filter: dict[str, Any]
711
+ ) -> list[ContentInfo]: ...
712
+
713
+ def remove_contents_metadata(
714
+ self,
715
+ *,
716
+ keys_to_remove: list[str],
717
+ metadata_filter: dict[str, Any] | None = None,
718
+ content_infos: list[ContentInfo] | None = None,
719
+ ) -> list[ContentInfo]:
720
+ """Remove the specified keys irreversibly from the content metadata.
721
+
722
+ Note: Keys are camelized before being removed as metadata keys are stored in camelCase.
723
+
724
+ """
725
+
726
+ if content_infos is None:
727
+ content_infos = self.get_paginated_content_infos(
728
+ metadata_filter=metadata_filter,
729
+ ).content_infos
730
+
731
+ for info in content_infos:
732
+ self.remove_content_metadata(
733
+ content_info=info, keys_to_remove=keys_to_remove
734
+ )
735
+
736
+ return content_infos
737
+
738
+ @overload
739
+ def delete_content(
740
+ self,
741
+ *,
742
+ content_id: str,
743
+ ) -> DeleteContentResponse: ...
744
+
745
+ """Delete content by id"""
746
+
747
+ @overload
748
+ def delete_content(
749
+ self,
750
+ *,
751
+ file_path: str,
752
+ ) -> DeleteContentResponse: ...
753
+
754
+ """Delete all content matching the file path"""
755
+
756
+ def delete_content(
757
+ self,
758
+ *,
759
+ content_id: str | None = None,
760
+ file_path: str | None = None,
761
+ ) -> DeleteContentResponse:
762
+ """Delete content by id, file path or metadata filter"""
763
+
764
+ return delete_content(
765
+ user_id=self._user_id,
766
+ company_id=self._company_id,
767
+ content_id=content_id,
768
+ file_path=file_path,
769
+ )
770
+
771
+ def delete_contents(
772
+ self,
773
+ *,
774
+ metadata_filter: dict[str, Any],
775
+ ) -> list[DeleteContentResponse]:
776
+ """Delete all content matching the metadata filter"""
777
+ resp: list[DeleteContentResponse] = []
778
+
779
+ if metadata_filter:
780
+ infos = self.get_paginated_content_infos(
781
+ metadata_filter=metadata_filter,
782
+ )
783
+
784
+ for info in infos.content_infos:
785
+ resp.append(
786
+ delete_content(
787
+ user_id=self._user_id,
788
+ company_id=self._company_id,
789
+ content_id=info.id,
790
+ )
791
+ )
792
+
793
+ return resp
794
+
795
+ @overload
796
+ async def delete_content_async(
797
+ self,
798
+ *,
799
+ content_id: str,
800
+ ) -> DeleteContentResponse: ...
801
+
802
+ @overload
803
+ async def delete_content_async(
804
+ self,
805
+ *,
806
+ file_path: str,
807
+ ) -> DeleteContentResponse: ...
808
+
809
+ async def delete_content_async(
810
+ self,
811
+ *,
812
+ content_id: str | None = None,
813
+ file_path: str | None = None,
814
+ ) -> DeleteContentResponse:
815
+ return await delete_content_async(
816
+ user_id=self._user_id,
817
+ company_id=self._company_id,
818
+ content_id=content_id,
819
+ file_path=file_path,
820
+ )
821
+
822
+ async def delete_contents_async(
823
+ self,
824
+ *,
825
+ metadata_filter: dict[str, Any],
826
+ ) -> list[DeleteContentResponse]:
827
+ """Delete all content matching the metadata filter"""
828
+ if not metadata_filter:
829
+ return []
830
+
831
+ infos = self.get_paginated_content_infos(
832
+ metadata_filter=metadata_filter,
833
+ )
834
+
835
+ # Create all delete tasks without awaiting them
836
+ delete_tasks = [
837
+ delete_content_async(
838
+ user_id=self._user_id,
839
+ company_id=self._company_id,
840
+ content_id=info.id,
841
+ )
842
+ for info in infos.content_infos
843
+ ]
844
+
845
+ # Await all delete operations concurrently
846
+ resp = await asyncio.gather(*delete_tasks)
847
+
848
+ return list(resp)
849
+
850
+
851
+ if __name__ == "__main__":
852
+ kb_service = KnowledgeBaseService.from_settings()
853
+
854
+ kb_service.search_contents(where={"metadata.key": "123"})
855
+ kb_service.search_content_chunks(
856
+ search_string="test",
857
+ search_type=ContentSearchType.VECTOR,
858
+ limit=10,
859
+ scope_ids=["123"],
860
+ metadata_filter={"key": "123"},
861
+ )