cwyodmodules 0.3.32__py3-none-any.whl → 0.3.33__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cwyodmodules/api/chat_history.py +14 -7
- cwyodmodules/batch/utilities/chat_history/auth_utils.py +7 -3
- cwyodmodules/batch/utilities/chat_history/cosmosdb.py +17 -1
- cwyodmodules/batch/utilities/chat_history/postgresdbservice.py +239 -254
- cwyodmodules/batch/utilities/common/source_document.py +60 -61
- cwyodmodules/batch/utilities/document_chunking/fixed_size_overlap.py +8 -3
- cwyodmodules/batch/utilities/document_chunking/layout.py +8 -3
- cwyodmodules/batch/utilities/document_chunking/page.py +8 -3
- cwyodmodules/batch/utilities/document_loading/read.py +30 -34
- cwyodmodules/batch/utilities/helpers/azure_computer_vision_client.py +10 -3
- cwyodmodules/batch/utilities/helpers/azure_form_recognizer_helper.py +6 -2
- cwyodmodules/batch/utilities/helpers/azure_postgres_helper.py +14 -2
- cwyodmodules/batch/utilities/helpers/azure_postgres_helper_light_rag.py +14 -2
- cwyodmodules/batch/utilities/helpers/azure_search_helper.py +15 -6
- cwyodmodules/batch/utilities/helpers/config/config_helper.py +24 -2
- cwyodmodules/batch/utilities/helpers/env_helper.py +9 -9
- cwyodmodules/batch/utilities/helpers/lightrag_helper.py +9 -2
- cwyodmodules/batch/utilities/helpers/llm_helper.py +13 -2
- cwyodmodules/batch/utilities/helpers/secret_helper.py +9 -9
- cwyodmodules/batch/utilities/integrated_vectorization/azure_search_index.py +8 -2
- cwyodmodules/batch/utilities/integrated_vectorization/azure_search_indexer.py +9 -2
- cwyodmodules/batch/utilities/integrated_vectorization/azure_search_skillset.py +6 -2
- cwyodmodules/batch/utilities/orchestrator/lang_chain_agent.py +8 -2
- cwyodmodules/batch/utilities/orchestrator/open_ai_functions.py +6 -2
- cwyodmodules/batch/utilities/orchestrator/orchestrator_base.py +9 -3
- cwyodmodules/batch/utilities/orchestrator/prompt_flow.py +8 -2
- cwyodmodules/batch/utilities/orchestrator/semantic_kernel_orchestrator.py +135 -138
- cwyodmodules/batch/utilities/parser/output_parser_tool.py +64 -64
- cwyodmodules/batch/utilities/plugins/outlook_calendar_plugin.py +91 -93
- cwyodmodules/batch/utilities/search/azure_search_handler.py +16 -3
- cwyodmodules/batch/utilities/search/azure_search_handler_light_rag.py +14 -2
- cwyodmodules/batch/utilities/search/integrated_vectorization_search_handler.py +36 -24
- cwyodmodules/batch/utilities/search/lightrag_search_handler.py +14 -2
- cwyodmodules/batch/utilities/search/postgres_search_handler.py +100 -97
- cwyodmodules/batch/utilities/search/postgres_search_handler_light_rag.py +103 -104
- cwyodmodules/batch/utilities/search/search.py +21 -24
- cwyodmodules/batch/utilities/tools/content_safety_checker.py +66 -78
- cwyodmodules/batch/utilities/tools/post_prompt_tool.py +48 -60
- cwyodmodules/batch/utilities/tools/question_answer_tool.py +196 -206
- cwyodmodules/batch/utilities/tools/text_processing_tool.py +36 -39
- cwyodmodules/logging_config.py +15 -0
- {cwyodmodules-0.3.32.dist-info → cwyodmodules-0.3.33.dist-info}/METADATA +2 -1
- {cwyodmodules-0.3.32.dist-info → cwyodmodules-0.3.33.dist-info}/RECORD +46 -45
- {cwyodmodules-0.3.32.dist-info → cwyodmodules-0.3.33.dist-info}/WHEEL +0 -0
- {cwyodmodules-0.3.32.dist-info → cwyodmodules-0.3.33.dist-info}/licenses/LICENSE +0 -0
- {cwyodmodules-0.3.32.dist-info → cwyodmodules-0.3.33.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,3 @@
|
|
1
|
-
import logging
|
2
1
|
from azure.identity import ChainedTokenCredential, DefaultAzureCredential
|
3
2
|
from typing import Union
|
4
3
|
from langchain_community.vectorstores import AzureSearch
|
@@ -27,11 +26,12 @@ from azure.search.documents.indexes.models import (
|
|
27
26
|
)
|
28
27
|
|
29
28
|
from ..helpers.azure_computer_vision_client import AzureComputerVisionClient
|
30
|
-
from .llm_helper import LLMHelper
|
31
|
-
from .env_helper import EnvHelper
|
32
|
-
|
33
|
-
|
34
|
-
|
29
|
+
from ..helpers.llm_helper import LLMHelper
|
30
|
+
from ..helpers.env_helper import EnvHelper
|
31
|
+
from logging_config import logger
|
32
|
+
env_helper: EnvHelper = EnvHelper()
|
33
|
+
log_args = env_helper.LOG_ARGS
|
34
|
+
log_result = env_helper.LOG_RESULT
|
35
35
|
|
36
36
|
class AzureSearchHelper:
|
37
37
|
_search_dimension: int | None = None
|
@@ -47,6 +47,7 @@ class AzureSearchHelper:
|
|
47
47
|
self.search_index_client = self._create_search_index_client(search_credential)
|
48
48
|
self.azure_computer_vision_client = AzureComputerVisionClient(self.env_helper)
|
49
49
|
|
50
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
50
51
|
def _search_credential(self):
|
51
52
|
if self.env_helper.is_auth_type_keys():
|
52
53
|
return AzureKeyCredential(self.env_helper.AZURE_SEARCH_KEY)
|
@@ -54,6 +55,7 @@ class AzureSearchHelper:
|
|
54
55
|
credential = self.azure_identity_helper.get_credential()
|
55
56
|
return credential
|
56
57
|
|
58
|
+
@logger.trace_function(log_args=False, log_result=False)
|
57
59
|
def _create_search_client(
|
58
60
|
self, search_credential: Union[AzureKeyCredential, ChainedTokenCredential, DefaultAzureCredential]
|
59
61
|
) -> SearchClient:
|
@@ -63,6 +65,7 @@ class AzureSearchHelper:
|
|
63
65
|
credential=search_credential,
|
64
66
|
)
|
65
67
|
|
68
|
+
@logger.trace_function(log_args=False, log_result=False)
|
66
69
|
def _create_search_index_client(
|
67
70
|
self, search_credential: Union[AzureKeyCredential, ChainedTokenCredential, DefaultAzureCredential]
|
68
71
|
):
|
@@ -70,11 +73,13 @@ class AzureSearchHelper:
|
|
70
73
|
endpoint=self.env_helper.AZURE_SEARCH_SERVICE, credential=search_credential
|
71
74
|
)
|
72
75
|
|
76
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
73
77
|
def get_search_client(self) -> SearchClient:
|
74
78
|
self.create_index()
|
75
79
|
return self.search_client
|
76
80
|
|
77
81
|
@property
|
82
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
78
83
|
def search_dimensions(self) -> int:
|
79
84
|
if AzureSearchHelper._search_dimension is None:
|
80
85
|
AzureSearchHelper._search_dimension = len(
|
@@ -83,6 +88,7 @@ class AzureSearchHelper:
|
|
83
88
|
return AzureSearchHelper._search_dimension
|
84
89
|
|
85
90
|
@property
|
91
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
86
92
|
def image_search_dimensions(self) -> int:
|
87
93
|
if AzureSearchHelper._image_search_dimension is None:
|
88
94
|
AzureSearchHelper._image_search_dimension = len(
|
@@ -90,6 +96,7 @@ class AzureSearchHelper:
|
|
90
96
|
)
|
91
97
|
return AzureSearchHelper._image_search_dimension
|
92
98
|
|
99
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
93
100
|
def create_index(self):
|
94
101
|
fields = [
|
95
102
|
SimpleField(
|
@@ -204,11 +211,13 @@ class AzureSearchHelper:
|
|
204
211
|
)
|
205
212
|
self.search_index_client.create_index(index)
|
206
213
|
|
214
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
207
215
|
def _index_not_exists(self, index_name: str) -> bool:
|
208
216
|
return index_name not in [
|
209
217
|
name for name in self.search_index_client.list_index_names()
|
210
218
|
]
|
211
219
|
|
220
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
212
221
|
def get_conversation_logger(self):
|
213
222
|
fields = [
|
214
223
|
SimpleField(
|
@@ -1,6 +1,5 @@
|
|
1
1
|
import os
|
2
2
|
import json
|
3
|
-
import logging
|
4
3
|
import functools
|
5
4
|
from string import Template
|
6
5
|
|
@@ -18,7 +17,11 @@ from .database_type import DatabaseType
|
|
18
17
|
CONFIG_CONTAINER_NAME = "config"
|
19
18
|
CONFIG_FILE_NAME = "active.json"
|
20
19
|
ADVANCED_IMAGE_PROCESSING_FILE_TYPES = ["jpeg", "jpg", "png", "tiff", "bmp"]
|
21
|
-
|
20
|
+
|
21
|
+
from logging_config import logger
|
22
|
+
env_helper: EnvHelper = EnvHelper()
|
23
|
+
log_args = env_helper.LOG_ARGS
|
24
|
+
log_result = env_helper.LOG_RESULT
|
22
25
|
|
23
26
|
|
24
27
|
class Config:
|
@@ -56,6 +59,7 @@ class Config:
|
|
56
59
|
"conversational_flow", self.env_helper.CONVERSATION_FLOW
|
57
60
|
)
|
58
61
|
|
62
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
59
63
|
def get_available_document_types(self) -> list[str]:
|
60
64
|
document_types = {
|
61
65
|
"txt",
|
@@ -74,21 +78,27 @@ class Config:
|
|
74
78
|
|
75
79
|
return sorted(document_types)
|
76
80
|
|
81
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
77
82
|
def get_advanced_image_processing_image_types(self):
|
78
83
|
return ADVANCED_IMAGE_PROCESSING_FILE_TYPES
|
79
84
|
|
85
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
80
86
|
def get_available_chunking_strategies(self):
|
81
87
|
return [c.value for c in ChunkingStrategy]
|
82
88
|
|
89
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
83
90
|
def get_available_loading_strategies(self):
|
84
91
|
return [c.value for c in LoadingStrategy]
|
85
92
|
|
93
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
86
94
|
def get_available_orchestration_strategies(self):
|
87
95
|
return [c.value for c in OrchestrationStrategy]
|
88
96
|
|
97
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
89
98
|
def get_available_ai_assistant_types(self):
|
90
99
|
return [c.value for c in AssistantStrategy]
|
91
100
|
|
101
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
92
102
|
def get_available_conversational_flows(self):
|
93
103
|
return [c.value for c in ConversationFlow]
|
94
104
|
|
@@ -139,6 +149,7 @@ class ConfigHelper:
|
|
139
149
|
_default_config = None
|
140
150
|
|
141
151
|
@staticmethod
|
152
|
+
@logger.trace_function(log_args=False, log_result=False)
|
142
153
|
def _set_new_config_properties(config: dict, default_config: dict):
|
143
154
|
"""
|
144
155
|
Function used to set newer properties that will not be present in older configs.
|
@@ -189,6 +200,7 @@ class ConfigHelper:
|
|
189
200
|
|
190
201
|
@staticmethod
|
191
202
|
@functools.cache
|
203
|
+
@logger.trace_function(log_args=False, log_result=False)
|
192
204
|
def get_active_config_or_default():
|
193
205
|
logger.info("Method get_active_config_or_default started")
|
194
206
|
env_helper = EnvHelper()
|
@@ -214,11 +226,13 @@ class ConfigHelper:
|
|
214
226
|
|
215
227
|
@staticmethod
|
216
228
|
@functools.cache
|
229
|
+
@logger.trace_function(log_args=False, log_result=False)
|
217
230
|
def get_default_assistant_prompt():
|
218
231
|
config = ConfigHelper.get_default_config()
|
219
232
|
return config["prompts"]["answering_user_prompt"]
|
220
233
|
|
221
234
|
@staticmethod
|
235
|
+
@logger.trace_function(log_args=False, log_result=log_result)
|
222
236
|
def save_config_as_active(config):
|
223
237
|
ConfigHelper.validate_config(config)
|
224
238
|
blob_client = AzureBlobStorageClient(container_name=CONFIG_CONTAINER_NAME)
|
@@ -230,6 +244,7 @@ class ConfigHelper:
|
|
230
244
|
ConfigHelper.get_active_config_or_default.cache_clear()
|
231
245
|
|
232
246
|
@staticmethod
|
247
|
+
@logger.trace_function(log_args=False, log_result=log_result)
|
233
248
|
def validate_config(config: dict):
|
234
249
|
for document_processor in config.get("document_processors"):
|
235
250
|
document_type = document_processor.get("document_type")
|
@@ -245,6 +260,7 @@ class ConfigHelper:
|
|
245
260
|
)
|
246
261
|
|
247
262
|
@staticmethod
|
263
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
248
264
|
def get_default_config():
|
249
265
|
if ConfigHelper._default_config is None:
|
250
266
|
env_helper = EnvHelper()
|
@@ -276,6 +292,7 @@ class ConfigHelper:
|
|
276
292
|
|
277
293
|
@staticmethod
|
278
294
|
@functools.cache
|
295
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
279
296
|
def get_default_contract_assistant():
|
280
297
|
contract_file_path = os.path.join(
|
281
298
|
os.path.dirname(__file__), "default_contract_assistant_prompt.txt"
|
@@ -288,6 +305,7 @@ class ConfigHelper:
|
|
288
305
|
|
289
306
|
@staticmethod
|
290
307
|
@functools.cache
|
308
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
291
309
|
def get_default_employee_assistant():
|
292
310
|
employee_file_path = os.path.join(
|
293
311
|
os.path.dirname(__file__), "default_employee_assistant_prompt.txt"
|
@@ -299,11 +317,13 @@ class ConfigHelper:
|
|
299
317
|
return "".join([str(elem) for elem in employee_assistant])
|
300
318
|
|
301
319
|
@staticmethod
|
320
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
302
321
|
def clear_config():
|
303
322
|
ConfigHelper._default_config = None
|
304
323
|
ConfigHelper.get_active_config_or_default.cache_clear()
|
305
324
|
|
306
325
|
@staticmethod
|
326
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
307
327
|
def _append_advanced_image_processors():
|
308
328
|
image_file_types = ["jpeg", "jpg", "png", "tiff", "bmp"]
|
309
329
|
ConfigHelper._remove_processors_for_file_types(image_file_types)
|
@@ -315,6 +335,7 @@ class ConfigHelper:
|
|
315
335
|
)
|
316
336
|
|
317
337
|
@staticmethod
|
338
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
318
339
|
def _remove_processors_for_file_types(file_types: list[str]):
|
319
340
|
document_processors = ConfigHelper._default_config["document_processors"]
|
320
341
|
document_processors = [
|
@@ -325,6 +346,7 @@ class ConfigHelper:
|
|
325
346
|
ConfigHelper._default_config["document_processors"] = document_processors
|
326
347
|
|
327
348
|
@staticmethod
|
349
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
328
350
|
def delete_config():
|
329
351
|
blob_client = AzureBlobStorageClient(container_name=CONFIG_CONTAINER_NAME)
|
330
352
|
blob_client.delete_file(CONFIG_FILE_NAME)
|
@@ -9,14 +9,7 @@ from ..helpers.config.database_type import DatabaseType
|
|
9
9
|
from ..helpers.secret_helper import SecretHelper
|
10
10
|
from ..helpers.azure_identity_helper import AzureIdentityHelper
|
11
11
|
|
12
|
-
from
|
13
|
-
from opentelemetry import trace, baggage
|
14
|
-
from opentelemetry.propagate import extract
|
15
|
-
|
16
|
-
# logger = getLogger("__main__" + ".base_package")
|
17
|
-
logger = getLogger("__main__")
|
18
|
-
# tracer = trace.get_tracer("__main__" + ".base_package")
|
19
|
-
tracer = trace.get_tracer("__main__")
|
12
|
+
from logging_config import logger
|
20
13
|
|
21
14
|
|
22
15
|
|
@@ -31,7 +24,8 @@ class EnvHelper:
|
|
31
24
|
instance.__load_config()
|
32
25
|
cls._instance = instance
|
33
26
|
return cls._instance
|
34
|
-
|
27
|
+
|
28
|
+
@logger.trace_function(log_args=False, log_result=False)
|
35
29
|
def __load_config(self, **kwargs) -> None:
|
36
30
|
# load_dotenv()
|
37
31
|
|
@@ -54,6 +48,12 @@ class EnvHelper:
|
|
54
48
|
# )
|
55
49
|
|
56
50
|
self.LOGLEVEL = self.secretHelper.get_secret("logging-level")
|
51
|
+
self.LOG_ARGS = self.get_env_var_bool(
|
52
|
+
"LOG_ARGS", "True"
|
53
|
+
)
|
54
|
+
self.LOG_RESULT = self.get_env_var_bool(
|
55
|
+
"LOG_RESULT", "True"
|
56
|
+
)
|
57
57
|
|
58
58
|
# Azure
|
59
59
|
self.AZURE_SUBSCRIPTION_ID = self.secretHelper.get_secret("subscription-id")
|
@@ -1,9 +1,12 @@
|
|
1
|
-
import logging
|
2
1
|
import psycopg2
|
3
2
|
from psycopg2.extras import execute_values, RealDictCursor
|
4
3
|
from ..helpers.azure_identity_helper import AzureIdentityHelper
|
5
4
|
|
6
|
-
|
5
|
+
from ...utilities.helpers.env_helper import EnvHelper
|
6
|
+
from logging_config import logger
|
7
|
+
env_helper: EnvHelper = EnvHelper()
|
8
|
+
log_args = env_helper.LOG_ARGS
|
9
|
+
log_result = env_helper.LOG_RESULT
|
7
10
|
|
8
11
|
|
9
12
|
class LightRAGHelper:
|
@@ -12,6 +15,7 @@ class LightRAGHelper:
|
|
12
15
|
self.azure_identity_helper = AzureIdentityHelper()
|
13
16
|
self.conn = None
|
14
17
|
|
18
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
15
19
|
def _create_connection(self):
|
16
20
|
"""
|
17
21
|
Establishes a connection to PostgreSQL using AAD authentication.
|
@@ -41,6 +45,7 @@ class LightRAGHelper:
|
|
41
45
|
logger.error(f"Error establishing a connection to PostgreSQL: {e}")
|
42
46
|
raise
|
43
47
|
|
48
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
44
49
|
def get_connection(self):
|
45
50
|
"""
|
46
51
|
Provides a reusable database connection.
|
@@ -49,6 +54,7 @@ class LightRAGHelper:
|
|
49
54
|
self.conn = self._create_connection()
|
50
55
|
return self.conn
|
51
56
|
|
57
|
+
@logger.trace_function(log_args=False, log_result=False)
|
52
58
|
def store_vector_and_text(self, vector, text, metadata):
|
53
59
|
"""
|
54
60
|
Stores a vector and associated text in the PostgreSQL database.
|
@@ -70,6 +76,7 @@ class LightRAGHelper:
|
|
70
76
|
finally:
|
71
77
|
conn.close()
|
72
78
|
|
79
|
+
@logger.trace_function(log_args=False, log_result=False)
|
73
80
|
def retrieve_vectors(self, query_vector, top_k):
|
74
81
|
"""
|
75
82
|
Retrieves the top K vectors similar to the provided query vector.
|
@@ -1,4 +1,3 @@
|
|
1
|
-
import logging
|
2
1
|
from openai import AzureOpenAI
|
3
2
|
from typing import List, Union, cast
|
4
3
|
from langchain_openai import AzureChatOpenAI, AzureOpenAIEmbeddings
|
@@ -11,7 +10,10 @@ from azure.ai.ml import MLClient
|
|
11
10
|
from ..helpers.azure_identity_helper import AzureIdentityHelper
|
12
11
|
from .env_helper import EnvHelper
|
13
12
|
|
14
|
-
|
13
|
+
from logging_config import logger
|
14
|
+
env_helper: EnvHelper = EnvHelper()
|
15
|
+
log_args = env_helper.LOG_ARGS
|
16
|
+
log_result = env_helper.LOG_RESULT
|
15
17
|
|
16
18
|
|
17
19
|
class LLMHelper:
|
@@ -46,6 +48,7 @@ class LLMHelper:
|
|
46
48
|
|
47
49
|
logger.info("Initializing LLMHelper completed")
|
48
50
|
|
51
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
49
52
|
def get_llm(self):
|
50
53
|
if self.auth_type_keys:
|
51
54
|
return AzureChatOpenAI(
|
@@ -67,6 +70,7 @@ class LLMHelper:
|
|
67
70
|
)
|
68
71
|
|
69
72
|
# TODO: This needs to have a custom callback to stream back to the UI
|
73
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
70
74
|
def get_streaming_llm(self):
|
71
75
|
if self.auth_type_keys:
|
72
76
|
return AzureChatOpenAI(
|
@@ -92,6 +96,7 @@ class LLMHelper:
|
|
92
96
|
azure_ad_token_provider=self.token_provider,
|
93
97
|
)
|
94
98
|
|
99
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
95
100
|
def get_embedding_model(self):
|
96
101
|
if self.auth_type_keys:
|
97
102
|
return AzureOpenAIEmbeddings(
|
@@ -108,6 +113,7 @@ class LLMHelper:
|
|
108
113
|
azure_ad_token_provider=self.token_provider,
|
109
114
|
)
|
110
115
|
|
116
|
+
@logger.trace_function(log_args=False, log_result=False)
|
111
117
|
def generate_embeddings(self, input: Union[str, list[int]]) -> List[float]:
|
112
118
|
return (
|
113
119
|
self.openai_client.embeddings.create(
|
@@ -117,6 +123,7 @@ class LLMHelper:
|
|
117
123
|
.embedding
|
118
124
|
)
|
119
125
|
|
126
|
+
@logger.trace_function(log_args=False, log_result=False)
|
120
127
|
def get_chat_completion_with_functions(
|
121
128
|
self, messages: list[dict], functions: list[dict], function_call: str = "auto"
|
122
129
|
):
|
@@ -127,6 +134,7 @@ class LLMHelper:
|
|
127
134
|
function_call=function_call,
|
128
135
|
)
|
129
136
|
|
137
|
+
@logger.trace_function(log_args=False, log_result=False)
|
130
138
|
def get_chat_completion(
|
131
139
|
self, messages: list[dict], model: str | None = None, **kwargs
|
132
140
|
):
|
@@ -137,6 +145,7 @@ class LLMHelper:
|
|
137
145
|
**kwargs,
|
138
146
|
)
|
139
147
|
|
148
|
+
@logger.trace_function(log_args=False, log_result=False)
|
140
149
|
def get_sk_chat_completion_service(self, service_id: str):
|
141
150
|
if self.auth_type_keys:
|
142
151
|
return AzureChatCompletion(
|
@@ -166,6 +175,7 @@ class LLMHelper:
|
|
166
175
|
ad_token_provider=self.token_provider,
|
167
176
|
)
|
168
177
|
|
178
|
+
@logger.trace_function(log_args=False, log_result=False)
|
169
179
|
def get_sk_service_settings(self, service: AzureChatCompletion):
|
170
180
|
return cast(
|
171
181
|
AzureChatPromptExecutionSettings,
|
@@ -176,6 +186,7 @@ class LLMHelper:
|
|
176
186
|
),
|
177
187
|
)
|
178
188
|
|
189
|
+
@logger.trace_function(log_args=False, log_result=False)
|
179
190
|
def get_ml_client(self):
|
180
191
|
if not hasattr(self, "_ml_client"):
|
181
192
|
credential = self.azure_identity_helper.get_credential()
|
@@ -1,8 +1,11 @@
|
|
1
|
-
import logging
|
2
1
|
from ..helpers.azure_identity_helper import AzureIdentityHelper
|
3
2
|
from azure.keyvault.secrets import SecretClient
|
4
3
|
|
5
|
-
|
4
|
+
from ...utilities.helpers.env_helper import EnvHelper
|
5
|
+
from logging_config import logger
|
6
|
+
env_helper: EnvHelper = EnvHelper()
|
7
|
+
log_args = env_helper.LOG_ARGS
|
8
|
+
log_result = env_helper.LOG_RESULT
|
6
9
|
|
7
10
|
|
8
11
|
class SecretHelper:
|
@@ -31,6 +34,7 @@ class SecretHelper:
|
|
31
34
|
connection_verify=True,
|
32
35
|
)
|
33
36
|
|
37
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
34
38
|
def get_secret(self, secret_name: str) -> str:
|
35
39
|
"""
|
36
40
|
Retrieves the value of a secret from the environment variables or Azure Key Vault.
|
@@ -45,11 +49,10 @@ class SecretHelper:
|
|
45
49
|
None
|
46
50
|
|
47
51
|
"""
|
48
|
-
logger.warning(f"Trying to get secret: {secret_name}")
|
49
52
|
secret_value = self.secret_client.get_secret(name=secret_name).value
|
50
|
-
logger.warning(f"Fetched secret value: {secret_value}")
|
51
53
|
return secret_value
|
52
54
|
|
55
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
53
56
|
def set_secret(self, secret_name: str, secret_value: str) -> None:
|
54
57
|
"""
|
55
58
|
Sets the value of a secret in Azure Key Vault only if it doesn't exist or has a different value.
|
@@ -64,12 +67,10 @@ class SecretHelper:
|
|
64
67
|
Raises:
|
65
68
|
None
|
66
69
|
"""
|
67
|
-
logger.warning(f"Trying to set secret: {secret_name}")
|
68
70
|
try:
|
69
71
|
current_secret = self.secret_client.get_secret(name=secret_name)
|
70
72
|
if current_secret.value != secret_value:
|
71
73
|
self.secret_client.set_secret(name=secret_name, value=secret_value)
|
72
|
-
logger.warning(f"Secret {secret_name} has been updated with new value")
|
73
74
|
else:
|
74
75
|
logger.warning(
|
75
76
|
f"Secret {secret_name} already has the same value, skipping update"
|
@@ -78,9 +79,8 @@ class SecretHelper:
|
|
78
79
|
self.secret_client.set_secret(name=secret_name, value=secret_value)
|
79
80
|
logger.warning(f"Secret {secret_name} has been created")
|
80
81
|
|
81
|
-
def get_secret_from_json(self, secret_name: str) -> str:
|
82
82
|
|
83
|
-
|
83
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
84
|
+
def get_secret_from_json(self, secret_name: str) -> str:
|
84
85
|
secret_value = self.secret_client.get_secret(secret_name).value
|
85
|
-
logger.warning(f"Fetched secret value: {secret_value}")
|
86
86
|
return secret_value
|
@@ -1,4 +1,3 @@
|
|
1
|
-
import logging
|
2
1
|
from azure.search.documents.indexes import SearchIndexClient
|
3
2
|
from azure.search.documents.indexes.models import (
|
4
3
|
SearchField,
|
@@ -25,7 +24,10 @@ from ..helpers.azure_identity_helper import AzureIdentityHelper
|
|
25
24
|
from azure.core.credentials import AzureKeyCredential
|
26
25
|
from ..helpers.llm_helper import LLMHelper
|
27
26
|
|
28
|
-
|
27
|
+
from logging_config import logger
|
28
|
+
env_helper: EnvHelper = EnvHelper()
|
29
|
+
log_args = env_helper.LOG_ARGS
|
30
|
+
log_result = env_helper.LOG_RESULT
|
29
31
|
|
30
32
|
|
31
33
|
class AzureSearchIndex:
|
@@ -46,6 +48,7 @@ class AzureSearchIndex:
|
|
46
48
|
)
|
47
49
|
|
48
50
|
@property
|
51
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
49
52
|
def search_dimensions(self) -> int:
|
50
53
|
if AzureSearchIndex._search_dimension is None:
|
51
54
|
AzureSearchIndex._search_dimension = len(
|
@@ -53,6 +56,7 @@ class AzureSearchIndex:
|
|
53
56
|
)
|
54
57
|
return AzureSearchIndex._search_dimension
|
55
58
|
|
59
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
56
60
|
def create_or_update_index(self):
|
57
61
|
# Create a search index
|
58
62
|
fields = [
|
@@ -115,6 +119,7 @@ class AzureSearchIndex:
|
|
115
119
|
logger.info(f"{result.name} index created successfully.")
|
116
120
|
return result
|
117
121
|
|
122
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
118
123
|
def get_vector_search_config(self):
|
119
124
|
if self.env_helper.is_auth_type_keys():
|
120
125
|
azure_open_ai_parameters = AzureOpenAIParameters(
|
@@ -167,6 +172,7 @@ class AzureSearchIndex:
|
|
167
172
|
],
|
168
173
|
)
|
169
174
|
|
175
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
170
176
|
def get_semantic_search_config(self):
|
171
177
|
semantic_config = SemanticConfiguration(
|
172
178
|
name=self.env_helper.AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG,
|
@@ -1,11 +1,15 @@
|
|
1
|
-
import logging
|
2
1
|
from azure.search.documents.indexes.models import SearchIndexer, FieldMapping
|
3
2
|
from azure.search.documents.indexes import SearchIndexerClient
|
4
3
|
from ..helpers.env_helper import EnvHelper
|
5
4
|
from ..helpers.azure_identity_helper import AzureIdentityHelper
|
6
5
|
from azure.core.credentials import AzureKeyCredential
|
7
6
|
|
8
|
-
|
7
|
+
|
8
|
+
from ...utilities.helpers.env_helper import EnvHelper
|
9
|
+
from logging_config import logger
|
10
|
+
env_helper: EnvHelper = EnvHelper()
|
11
|
+
log_args = env_helper.LOG_ARGS
|
12
|
+
log_result = env_helper.LOG_RESULT
|
9
13
|
|
10
14
|
|
11
15
|
class AzureSearchIndexer:
|
@@ -22,6 +26,7 @@ class AzureSearchIndexer:
|
|
22
26
|
),
|
23
27
|
)
|
24
28
|
|
29
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
25
30
|
def create_or_update_indexer(self, indexer_name: str, skillset_name: str):
|
26
31
|
indexer = SearchIndexer(
|
27
32
|
name=indexer_name,
|
@@ -59,6 +64,7 @@ class AzureSearchIndexer:
|
|
59
64
|
)
|
60
65
|
return indexer_result
|
61
66
|
|
67
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
62
68
|
def run_indexer(self, indexer_name: str):
|
63
69
|
self.indexer_client.reset_indexer(indexer_name)
|
64
70
|
self.indexer_client.run_indexer(indexer_name)
|
@@ -66,6 +72,7 @@ class AzureSearchIndexer:
|
|
66
72
|
f" {indexer_name} is created and running. If queries return no results, please wait a bit and try again."
|
67
73
|
)
|
68
74
|
|
75
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
69
76
|
def indexer_exists(self, indexer_name: str):
|
70
77
|
return indexer_name in [
|
71
78
|
name for name in self.indexer_client.get_indexer_names()
|
@@ -1,4 +1,3 @@
|
|
1
|
-
import logging
|
2
1
|
from azure.search.documents.indexes.models import (
|
3
2
|
SplitSkill,
|
4
3
|
InputFieldMappingEntry,
|
@@ -18,7 +17,11 @@ from ..helpers.env_helper import EnvHelper
|
|
18
17
|
from ..helpers.azure_identity_helper import AzureIdentityHelper
|
19
18
|
from azure.core.credentials import AzureKeyCredential
|
20
19
|
|
21
|
-
|
20
|
+
from ...utilities.helpers.env_helper import EnvHelper
|
21
|
+
from logging_config import logger
|
22
|
+
env_helper: EnvHelper = EnvHelper()
|
23
|
+
log_args = env_helper.LOG_ARGS
|
24
|
+
log_result = env_helper.LOG_RESULT
|
22
25
|
|
23
26
|
|
24
27
|
class AzureSearchSkillset:
|
@@ -40,6 +43,7 @@ class AzureSearchSkillset:
|
|
40
43
|
)
|
41
44
|
self.integrated_vectorization_config = integrated_vectorization_config
|
42
45
|
|
46
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
43
47
|
def create_skillset(self):
|
44
48
|
skillset_name = f"{self.env_helper.AZURE_SEARCH_INDEX}-skillset"
|
45
49
|
|
@@ -1,4 +1,3 @@
|
|
1
|
-
import logging
|
2
1
|
from typing import List
|
3
2
|
from langchain.agents import Tool
|
4
3
|
from langchain.memory import ConversationBufferMemory
|
@@ -13,7 +12,11 @@ from ..tools.question_answer_tool import QuestionAnswerTool
|
|
13
12
|
from ..tools.text_processing_tool import TextProcessingTool
|
14
13
|
from ..common.answer import Answer
|
15
14
|
|
16
|
-
|
15
|
+
from ...utilities.helpers.env_helper import EnvHelper
|
16
|
+
from logging_config import logger
|
17
|
+
env_helper: EnvHelper = EnvHelper()
|
18
|
+
log_args = env_helper.LOG_ARGS
|
19
|
+
log_result = env_helper.LOG_RESULT
|
17
20
|
|
18
21
|
|
19
22
|
class LangChainAgent(OrchestratorBase):
|
@@ -48,6 +51,7 @@ class LangChainAgent(OrchestratorBase):
|
|
48
51
|
),
|
49
52
|
]
|
50
53
|
|
54
|
+
@logger.trace_function(log_args=False, log_result=False)
|
51
55
|
def run_tool(self, user_message: str) -> str:
|
52
56
|
"""
|
53
57
|
Executes the question answering tool with the provided user message.
|
@@ -63,6 +67,7 @@ class LangChainAgent(OrchestratorBase):
|
|
63
67
|
)
|
64
68
|
return answer.to_json()
|
65
69
|
|
70
|
+
@logger.trace_function(log_args=False, log_result=False)
|
66
71
|
def run_text_processing_tool(self, user_message: str) -> str:
|
67
72
|
"""
|
68
73
|
Executes the text processing tool with the provided user message.
|
@@ -78,6 +83,7 @@ class LangChainAgent(OrchestratorBase):
|
|
78
83
|
)
|
79
84
|
return answer.to_json()
|
80
85
|
|
86
|
+
@logger.trace_function(log_args=False, log_result=False)
|
81
87
|
async def orchestrate(
|
82
88
|
self, user_message: str, chat_history: List[dict], **kwargs: dict
|
83
89
|
) -> list[dict]:
|
@@ -1,4 +1,3 @@
|
|
1
|
-
import logging
|
2
1
|
from typing import List
|
3
2
|
import json
|
4
3
|
|
@@ -10,7 +9,11 @@ from ..tools.question_answer_tool import QuestionAnswerTool
|
|
10
9
|
from ..tools.text_processing_tool import TextProcessingTool
|
11
10
|
from ..common.answer import Answer
|
12
11
|
|
13
|
-
|
12
|
+
from ...utilities.helpers.env_helper import EnvHelper
|
13
|
+
from logging_config import logger
|
14
|
+
env_helper: EnvHelper = EnvHelper()
|
15
|
+
log_args = env_helper.LOG_ARGS
|
16
|
+
log_result = env_helper.LOG_RESULT
|
14
17
|
|
15
18
|
|
16
19
|
class OpenAIFunctionsOrchestrator(OrchestratorBase):
|
@@ -65,6 +68,7 @@ class OpenAIFunctionsOrchestrator(OrchestratorBase):
|
|
65
68
|
},
|
66
69
|
]
|
67
70
|
|
71
|
+
@logger.trace_function(log_args=False, log_result=False)
|
68
72
|
async def orchestrate(
|
69
73
|
self, user_message: str, chat_history: List[dict], **kwargs: dict
|
70
74
|
) -> list[dict]:
|