cwyodmodules 0.3.31__py3-none-any.whl → 0.3.33__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cwyodmodules/api/chat_history.py +14 -7
- cwyodmodules/batch/utilities/chat_history/auth_utils.py +7 -3
- cwyodmodules/batch/utilities/chat_history/cosmosdb.py +17 -1
- cwyodmodules/batch/utilities/chat_history/postgresdbservice.py +239 -254
- cwyodmodules/batch/utilities/common/source_document.py +60 -61
- cwyodmodules/batch/utilities/document_chunking/fixed_size_overlap.py +8 -3
- cwyodmodules/batch/utilities/document_chunking/layout.py +8 -3
- cwyodmodules/batch/utilities/document_chunking/page.py +8 -3
- cwyodmodules/batch/utilities/document_loading/read.py +30 -34
- cwyodmodules/batch/utilities/helpers/azure_computer_vision_client.py +10 -3
- cwyodmodules/batch/utilities/helpers/azure_form_recognizer_helper.py +6 -2
- cwyodmodules/batch/utilities/helpers/azure_identity_helper.py +3 -34
- cwyodmodules/batch/utilities/helpers/azure_postgres_helper.py +14 -2
- cwyodmodules/batch/utilities/helpers/azure_postgres_helper_light_rag.py +14 -2
- cwyodmodules/batch/utilities/helpers/azure_search_helper.py +15 -6
- cwyodmodules/batch/utilities/helpers/config/config_helper.py +24 -2
- cwyodmodules/batch/utilities/helpers/env_helper.py +9 -9
- cwyodmodules/batch/utilities/helpers/lightrag_helper.py +9 -2
- cwyodmodules/batch/utilities/helpers/llm_helper.py +13 -2
- cwyodmodules/batch/utilities/helpers/secret_helper.py +9 -9
- cwyodmodules/batch/utilities/integrated_vectorization/azure_search_index.py +8 -2
- cwyodmodules/batch/utilities/integrated_vectorization/azure_search_indexer.py +9 -2
- cwyodmodules/batch/utilities/integrated_vectorization/azure_search_skillset.py +6 -2
- cwyodmodules/batch/utilities/orchestrator/lang_chain_agent.py +8 -2
- cwyodmodules/batch/utilities/orchestrator/open_ai_functions.py +6 -2
- cwyodmodules/batch/utilities/orchestrator/orchestrator_base.py +9 -3
- cwyodmodules/batch/utilities/orchestrator/prompt_flow.py +8 -2
- cwyodmodules/batch/utilities/orchestrator/semantic_kernel_orchestrator.py +135 -138
- cwyodmodules/batch/utilities/parser/output_parser_tool.py +64 -64
- cwyodmodules/batch/utilities/plugins/outlook_calendar_plugin.py +91 -93
- cwyodmodules/batch/utilities/search/azure_search_handler.py +16 -3
- cwyodmodules/batch/utilities/search/azure_search_handler_light_rag.py +14 -2
- cwyodmodules/batch/utilities/search/integrated_vectorization_search_handler.py +36 -24
- cwyodmodules/batch/utilities/search/lightrag_search_handler.py +14 -2
- cwyodmodules/batch/utilities/search/postgres_search_handler.py +100 -97
- cwyodmodules/batch/utilities/search/postgres_search_handler_light_rag.py +103 -104
- cwyodmodules/batch/utilities/search/search.py +21 -24
- cwyodmodules/batch/utilities/tools/content_safety_checker.py +66 -78
- cwyodmodules/batch/utilities/tools/post_prompt_tool.py +48 -60
- cwyodmodules/batch/utilities/tools/question_answer_tool.py +196 -206
- cwyodmodules/batch/utilities/tools/text_processing_tool.py +36 -39
- cwyodmodules/logging_config.py +15 -0
- {cwyodmodules-0.3.31.dist-info → cwyodmodules-0.3.33.dist-info}/METADATA +2 -1
- {cwyodmodules-0.3.31.dist-info → cwyodmodules-0.3.33.dist-info}/RECORD +47 -46
- {cwyodmodules-0.3.31.dist-info → cwyodmodules-0.3.33.dist-info}/WHEEL +0 -0
- {cwyodmodules-0.3.31.dist-info → cwyodmodules-0.3.33.dist-info}/licenses/LICENSE +0 -0
- {cwyodmodules-0.3.31.dist-info → cwyodmodules-0.3.33.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,3 @@
|
|
1
|
-
import logging
|
2
1
|
import psycopg2
|
3
2
|
from psycopg2.extras import execute_values, RealDictCursor
|
4
3
|
from .llm_helper import LLMHelper
|
@@ -6,7 +5,10 @@ from .env_helper import EnvHelper
|
|
6
5
|
from .lightrag_helper import LightRAGHelper
|
7
6
|
from ..helpers.azure_identity_helper import AzureIdentityHelper
|
8
7
|
|
9
|
-
|
8
|
+
from logging_config import logger
|
9
|
+
env_helper: EnvHelper = EnvHelper()
|
10
|
+
log_args = env_helper.LOG_ARGS
|
11
|
+
log_result = env_helper.LOG_RESULT
|
10
12
|
|
11
13
|
|
12
14
|
class AzurePostgresHelper:
|
@@ -17,6 +19,7 @@ class AzurePostgresHelper:
|
|
17
19
|
self.azure_identity_helper = AzureIdentityHelper()
|
18
20
|
self.conn = None
|
19
21
|
|
22
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
20
23
|
def _create_search_client(self):
|
21
24
|
"""
|
22
25
|
Establishes a connection to Azure PostgreSQL using AAD authentication.
|
@@ -46,6 +49,7 @@ class AzurePostgresHelper:
|
|
46
49
|
logger.error(f"Error establishing a connection to PostgreSQL: {e}")
|
47
50
|
raise
|
48
51
|
|
52
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
49
53
|
def get_search_client(self):
|
50
54
|
"""
|
51
55
|
Provides a reusable database connection.
|
@@ -54,6 +58,7 @@ class AzurePostgresHelper:
|
|
54
58
|
self.conn = self._create_search_client()
|
55
59
|
return self.conn
|
56
60
|
|
61
|
+
@logger.trace_function(log_args=False, log_result=False)
|
57
62
|
def get_vector_store(self, embedding_array):
|
58
63
|
"""
|
59
64
|
Fetches search indexes from PostgreSQL based on an embedding vector.
|
@@ -82,6 +87,7 @@ class AzurePostgresHelper:
|
|
82
87
|
finally:
|
83
88
|
conn.close()
|
84
89
|
|
90
|
+
@logger.trace_function(log_args=False, log_result=log_result)
|
85
91
|
def create_vector_store(self, documents_to_upload):
|
86
92
|
"""
|
87
93
|
Inserts documents into the `vector_store` table in batch mode.
|
@@ -125,6 +131,7 @@ class AzurePostgresHelper:
|
|
125
131
|
finally:
|
126
132
|
conn.close()
|
127
133
|
|
134
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
128
135
|
def get_files(self):
|
129
136
|
"""
|
130
137
|
Fetches distinct titles from the PostgreSQL database.
|
@@ -157,6 +164,7 @@ class AzurePostgresHelper:
|
|
157
164
|
finally:
|
158
165
|
conn.close()
|
159
166
|
|
167
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
160
168
|
def delete_documents(self, ids_to_delete):
|
161
169
|
"""
|
162
170
|
Deletes documents from the PostgreSQL database based on the provided ids.
|
@@ -204,6 +212,7 @@ class AzurePostgresHelper:
|
|
204
212
|
finally:
|
205
213
|
conn.close()
|
206
214
|
|
215
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
207
216
|
def perform_search(self, title):
|
208
217
|
"""
|
209
218
|
Fetches search results from PostgreSQL based on the title.
|
@@ -230,6 +239,7 @@ class AzurePostgresHelper:
|
|
230
239
|
finally:
|
231
240
|
conn.close()
|
232
241
|
|
242
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
233
243
|
def get_unique_files(self):
|
234
244
|
"""
|
235
245
|
Fetches unique titles from PostgreSQL.
|
@@ -254,6 +264,7 @@ class AzurePostgresHelper:
|
|
254
264
|
finally:
|
255
265
|
conn.close()
|
256
266
|
|
267
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
257
268
|
def search_by_blob_url(self, blob_url):
|
258
269
|
"""
|
259
270
|
Fetches unique titles from PostgreSQL based on a given blob URL.
|
@@ -280,6 +291,7 @@ class AzurePostgresHelper:
|
|
280
291
|
finally:
|
281
292
|
conn.close()
|
282
293
|
|
294
|
+
@logger.trace_function(log_args=False, log_result=False)
|
283
295
|
def store_with_lightrag(self, documents_to_upload):
|
284
296
|
"""
|
285
297
|
Stores documents using LightRAG for enhanced vector and text storage capabilities.
|
@@ -1,4 +1,3 @@
|
|
1
|
-
import logging
|
2
1
|
from azure.identity import ChainedTokenCredential, DefaultAzureCredential
|
3
2
|
from typing import Union
|
4
3
|
from langchain_community.vectorstores import AzureSearch
|
@@ -27,11 +26,12 @@ from azure.search.documents.indexes.models import (
|
|
27
26
|
)
|
28
27
|
|
29
28
|
from ..helpers.azure_computer_vision_client import AzureComputerVisionClient
|
30
|
-
from .llm_helper import LLMHelper
|
31
|
-
from .env_helper import EnvHelper
|
32
|
-
|
33
|
-
|
34
|
-
|
29
|
+
from ..helpers.llm_helper import LLMHelper
|
30
|
+
from ..helpers.env_helper import EnvHelper
|
31
|
+
from logging_config import logger
|
32
|
+
env_helper: EnvHelper = EnvHelper()
|
33
|
+
log_args = env_helper.LOG_ARGS
|
34
|
+
log_result = env_helper.LOG_RESULT
|
35
35
|
|
36
36
|
class AzureSearchHelper:
|
37
37
|
_search_dimension: int | None = None
|
@@ -47,6 +47,7 @@ class AzureSearchHelper:
|
|
47
47
|
self.search_index_client = self._create_search_index_client(search_credential)
|
48
48
|
self.azure_computer_vision_client = AzureComputerVisionClient(self.env_helper)
|
49
49
|
|
50
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
50
51
|
def _search_credential(self):
|
51
52
|
if self.env_helper.is_auth_type_keys():
|
52
53
|
return AzureKeyCredential(self.env_helper.AZURE_SEARCH_KEY)
|
@@ -54,6 +55,7 @@ class AzureSearchHelper:
|
|
54
55
|
credential = self.azure_identity_helper.get_credential()
|
55
56
|
return credential
|
56
57
|
|
58
|
+
@logger.trace_function(log_args=False, log_result=False)
|
57
59
|
def _create_search_client(
|
58
60
|
self, search_credential: Union[AzureKeyCredential, ChainedTokenCredential, DefaultAzureCredential]
|
59
61
|
) -> SearchClient:
|
@@ -63,6 +65,7 @@ class AzureSearchHelper:
|
|
63
65
|
credential=search_credential,
|
64
66
|
)
|
65
67
|
|
68
|
+
@logger.trace_function(log_args=False, log_result=False)
|
66
69
|
def _create_search_index_client(
|
67
70
|
self, search_credential: Union[AzureKeyCredential, ChainedTokenCredential, DefaultAzureCredential]
|
68
71
|
):
|
@@ -70,11 +73,13 @@ class AzureSearchHelper:
|
|
70
73
|
endpoint=self.env_helper.AZURE_SEARCH_SERVICE, credential=search_credential
|
71
74
|
)
|
72
75
|
|
76
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
73
77
|
def get_search_client(self) -> SearchClient:
|
74
78
|
self.create_index()
|
75
79
|
return self.search_client
|
76
80
|
|
77
81
|
@property
|
82
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
78
83
|
def search_dimensions(self) -> int:
|
79
84
|
if AzureSearchHelper._search_dimension is None:
|
80
85
|
AzureSearchHelper._search_dimension = len(
|
@@ -83,6 +88,7 @@ class AzureSearchHelper:
|
|
83
88
|
return AzureSearchHelper._search_dimension
|
84
89
|
|
85
90
|
@property
|
91
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
86
92
|
def image_search_dimensions(self) -> int:
|
87
93
|
if AzureSearchHelper._image_search_dimension is None:
|
88
94
|
AzureSearchHelper._image_search_dimension = len(
|
@@ -90,6 +96,7 @@ class AzureSearchHelper:
|
|
90
96
|
)
|
91
97
|
return AzureSearchHelper._image_search_dimension
|
92
98
|
|
99
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
93
100
|
def create_index(self):
|
94
101
|
fields = [
|
95
102
|
SimpleField(
|
@@ -204,11 +211,13 @@ class AzureSearchHelper:
|
|
204
211
|
)
|
205
212
|
self.search_index_client.create_index(index)
|
206
213
|
|
214
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
207
215
|
def _index_not_exists(self, index_name: str) -> bool:
|
208
216
|
return index_name not in [
|
209
217
|
name for name in self.search_index_client.list_index_names()
|
210
218
|
]
|
211
219
|
|
220
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
212
221
|
def get_conversation_logger(self):
|
213
222
|
fields = [
|
214
223
|
SimpleField(
|
@@ -1,6 +1,5 @@
|
|
1
1
|
import os
|
2
2
|
import json
|
3
|
-
import logging
|
4
3
|
import functools
|
5
4
|
from string import Template
|
6
5
|
|
@@ -18,7 +17,11 @@ from .database_type import DatabaseType
|
|
18
17
|
CONFIG_CONTAINER_NAME = "config"
|
19
18
|
CONFIG_FILE_NAME = "active.json"
|
20
19
|
ADVANCED_IMAGE_PROCESSING_FILE_TYPES = ["jpeg", "jpg", "png", "tiff", "bmp"]
|
21
|
-
|
20
|
+
|
21
|
+
from logging_config import logger
|
22
|
+
env_helper: EnvHelper = EnvHelper()
|
23
|
+
log_args = env_helper.LOG_ARGS
|
24
|
+
log_result = env_helper.LOG_RESULT
|
22
25
|
|
23
26
|
|
24
27
|
class Config:
|
@@ -56,6 +59,7 @@ class Config:
|
|
56
59
|
"conversational_flow", self.env_helper.CONVERSATION_FLOW
|
57
60
|
)
|
58
61
|
|
62
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
59
63
|
def get_available_document_types(self) -> list[str]:
|
60
64
|
document_types = {
|
61
65
|
"txt",
|
@@ -74,21 +78,27 @@ class Config:
|
|
74
78
|
|
75
79
|
return sorted(document_types)
|
76
80
|
|
81
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
77
82
|
def get_advanced_image_processing_image_types(self):
|
78
83
|
return ADVANCED_IMAGE_PROCESSING_FILE_TYPES
|
79
84
|
|
85
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
80
86
|
def get_available_chunking_strategies(self):
|
81
87
|
return [c.value for c in ChunkingStrategy]
|
82
88
|
|
89
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
83
90
|
def get_available_loading_strategies(self):
|
84
91
|
return [c.value for c in LoadingStrategy]
|
85
92
|
|
93
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
86
94
|
def get_available_orchestration_strategies(self):
|
87
95
|
return [c.value for c in OrchestrationStrategy]
|
88
96
|
|
97
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
89
98
|
def get_available_ai_assistant_types(self):
|
90
99
|
return [c.value for c in AssistantStrategy]
|
91
100
|
|
101
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
92
102
|
def get_available_conversational_flows(self):
|
93
103
|
return [c.value for c in ConversationFlow]
|
94
104
|
|
@@ -139,6 +149,7 @@ class ConfigHelper:
|
|
139
149
|
_default_config = None
|
140
150
|
|
141
151
|
@staticmethod
|
152
|
+
@logger.trace_function(log_args=False, log_result=False)
|
142
153
|
def _set_new_config_properties(config: dict, default_config: dict):
|
143
154
|
"""
|
144
155
|
Function used to set newer properties that will not be present in older configs.
|
@@ -189,6 +200,7 @@ class ConfigHelper:
|
|
189
200
|
|
190
201
|
@staticmethod
|
191
202
|
@functools.cache
|
203
|
+
@logger.trace_function(log_args=False, log_result=False)
|
192
204
|
def get_active_config_or_default():
|
193
205
|
logger.info("Method get_active_config_or_default started")
|
194
206
|
env_helper = EnvHelper()
|
@@ -214,11 +226,13 @@ class ConfigHelper:
|
|
214
226
|
|
215
227
|
@staticmethod
|
216
228
|
@functools.cache
|
229
|
+
@logger.trace_function(log_args=False, log_result=False)
|
217
230
|
def get_default_assistant_prompt():
|
218
231
|
config = ConfigHelper.get_default_config()
|
219
232
|
return config["prompts"]["answering_user_prompt"]
|
220
233
|
|
221
234
|
@staticmethod
|
235
|
+
@logger.trace_function(log_args=False, log_result=log_result)
|
222
236
|
def save_config_as_active(config):
|
223
237
|
ConfigHelper.validate_config(config)
|
224
238
|
blob_client = AzureBlobStorageClient(container_name=CONFIG_CONTAINER_NAME)
|
@@ -230,6 +244,7 @@ class ConfigHelper:
|
|
230
244
|
ConfigHelper.get_active_config_or_default.cache_clear()
|
231
245
|
|
232
246
|
@staticmethod
|
247
|
+
@logger.trace_function(log_args=False, log_result=log_result)
|
233
248
|
def validate_config(config: dict):
|
234
249
|
for document_processor in config.get("document_processors"):
|
235
250
|
document_type = document_processor.get("document_type")
|
@@ -245,6 +260,7 @@ class ConfigHelper:
|
|
245
260
|
)
|
246
261
|
|
247
262
|
@staticmethod
|
263
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
248
264
|
def get_default_config():
|
249
265
|
if ConfigHelper._default_config is None:
|
250
266
|
env_helper = EnvHelper()
|
@@ -276,6 +292,7 @@ class ConfigHelper:
|
|
276
292
|
|
277
293
|
@staticmethod
|
278
294
|
@functools.cache
|
295
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
279
296
|
def get_default_contract_assistant():
|
280
297
|
contract_file_path = os.path.join(
|
281
298
|
os.path.dirname(__file__), "default_contract_assistant_prompt.txt"
|
@@ -288,6 +305,7 @@ class ConfigHelper:
|
|
288
305
|
|
289
306
|
@staticmethod
|
290
307
|
@functools.cache
|
308
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
291
309
|
def get_default_employee_assistant():
|
292
310
|
employee_file_path = os.path.join(
|
293
311
|
os.path.dirname(__file__), "default_employee_assistant_prompt.txt"
|
@@ -299,11 +317,13 @@ class ConfigHelper:
|
|
299
317
|
return "".join([str(elem) for elem in employee_assistant])
|
300
318
|
|
301
319
|
@staticmethod
|
320
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
302
321
|
def clear_config():
|
303
322
|
ConfigHelper._default_config = None
|
304
323
|
ConfigHelper.get_active_config_or_default.cache_clear()
|
305
324
|
|
306
325
|
@staticmethod
|
326
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
307
327
|
def _append_advanced_image_processors():
|
308
328
|
image_file_types = ["jpeg", "jpg", "png", "tiff", "bmp"]
|
309
329
|
ConfigHelper._remove_processors_for_file_types(image_file_types)
|
@@ -315,6 +335,7 @@ class ConfigHelper:
|
|
315
335
|
)
|
316
336
|
|
317
337
|
@staticmethod
|
338
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
318
339
|
def _remove_processors_for_file_types(file_types: list[str]):
|
319
340
|
document_processors = ConfigHelper._default_config["document_processors"]
|
320
341
|
document_processors = [
|
@@ -325,6 +346,7 @@ class ConfigHelper:
|
|
325
346
|
ConfigHelper._default_config["document_processors"] = document_processors
|
326
347
|
|
327
348
|
@staticmethod
|
349
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
328
350
|
def delete_config():
|
329
351
|
blob_client = AzureBlobStorageClient(container_name=CONFIG_CONTAINER_NAME)
|
330
352
|
blob_client.delete_file(CONFIG_FILE_NAME)
|
@@ -9,14 +9,7 @@ from ..helpers.config.database_type import DatabaseType
|
|
9
9
|
from ..helpers.secret_helper import SecretHelper
|
10
10
|
from ..helpers.azure_identity_helper import AzureIdentityHelper
|
11
11
|
|
12
|
-
from
|
13
|
-
from opentelemetry import trace, baggage
|
14
|
-
from opentelemetry.propagate import extract
|
15
|
-
|
16
|
-
# logger = getLogger("__main__" + ".base_package")
|
17
|
-
logger = getLogger("__main__")
|
18
|
-
# tracer = trace.get_tracer("__main__" + ".base_package")
|
19
|
-
tracer = trace.get_tracer("__main__")
|
12
|
+
from logging_config import logger
|
20
13
|
|
21
14
|
|
22
15
|
|
@@ -31,7 +24,8 @@ class EnvHelper:
|
|
31
24
|
instance.__load_config()
|
32
25
|
cls._instance = instance
|
33
26
|
return cls._instance
|
34
|
-
|
27
|
+
|
28
|
+
@logger.trace_function(log_args=False, log_result=False)
|
35
29
|
def __load_config(self, **kwargs) -> None:
|
36
30
|
# load_dotenv()
|
37
31
|
|
@@ -54,6 +48,12 @@ class EnvHelper:
|
|
54
48
|
# )
|
55
49
|
|
56
50
|
self.LOGLEVEL = self.secretHelper.get_secret("logging-level")
|
51
|
+
self.LOG_ARGS = self.get_env_var_bool(
|
52
|
+
"LOG_ARGS", "True"
|
53
|
+
)
|
54
|
+
self.LOG_RESULT = self.get_env_var_bool(
|
55
|
+
"LOG_RESULT", "True"
|
56
|
+
)
|
57
57
|
|
58
58
|
# Azure
|
59
59
|
self.AZURE_SUBSCRIPTION_ID = self.secretHelper.get_secret("subscription-id")
|
@@ -1,9 +1,12 @@
|
|
1
|
-
import logging
|
2
1
|
import psycopg2
|
3
2
|
from psycopg2.extras import execute_values, RealDictCursor
|
4
3
|
from ..helpers.azure_identity_helper import AzureIdentityHelper
|
5
4
|
|
6
|
-
|
5
|
+
from ...utilities.helpers.env_helper import EnvHelper
|
6
|
+
from logging_config import logger
|
7
|
+
env_helper: EnvHelper = EnvHelper()
|
8
|
+
log_args = env_helper.LOG_ARGS
|
9
|
+
log_result = env_helper.LOG_RESULT
|
7
10
|
|
8
11
|
|
9
12
|
class LightRAGHelper:
|
@@ -12,6 +15,7 @@ class LightRAGHelper:
|
|
12
15
|
self.azure_identity_helper = AzureIdentityHelper()
|
13
16
|
self.conn = None
|
14
17
|
|
18
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
15
19
|
def _create_connection(self):
|
16
20
|
"""
|
17
21
|
Establishes a connection to PostgreSQL using AAD authentication.
|
@@ -41,6 +45,7 @@ class LightRAGHelper:
|
|
41
45
|
logger.error(f"Error establishing a connection to PostgreSQL: {e}")
|
42
46
|
raise
|
43
47
|
|
48
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
44
49
|
def get_connection(self):
|
45
50
|
"""
|
46
51
|
Provides a reusable database connection.
|
@@ -49,6 +54,7 @@ class LightRAGHelper:
|
|
49
54
|
self.conn = self._create_connection()
|
50
55
|
return self.conn
|
51
56
|
|
57
|
+
@logger.trace_function(log_args=False, log_result=False)
|
52
58
|
def store_vector_and_text(self, vector, text, metadata):
|
53
59
|
"""
|
54
60
|
Stores a vector and associated text in the PostgreSQL database.
|
@@ -70,6 +76,7 @@ class LightRAGHelper:
|
|
70
76
|
finally:
|
71
77
|
conn.close()
|
72
78
|
|
79
|
+
@logger.trace_function(log_args=False, log_result=False)
|
73
80
|
def retrieve_vectors(self, query_vector, top_k):
|
74
81
|
"""
|
75
82
|
Retrieves the top K vectors similar to the provided query vector.
|
@@ -1,4 +1,3 @@
|
|
1
|
-
import logging
|
2
1
|
from openai import AzureOpenAI
|
3
2
|
from typing import List, Union, cast
|
4
3
|
from langchain_openai import AzureChatOpenAI, AzureOpenAIEmbeddings
|
@@ -11,7 +10,10 @@ from azure.ai.ml import MLClient
|
|
11
10
|
from ..helpers.azure_identity_helper import AzureIdentityHelper
|
12
11
|
from .env_helper import EnvHelper
|
13
12
|
|
14
|
-
|
13
|
+
from logging_config import logger
|
14
|
+
env_helper: EnvHelper = EnvHelper()
|
15
|
+
log_args = env_helper.LOG_ARGS
|
16
|
+
log_result = env_helper.LOG_RESULT
|
15
17
|
|
16
18
|
|
17
19
|
class LLMHelper:
|
@@ -46,6 +48,7 @@ class LLMHelper:
|
|
46
48
|
|
47
49
|
logger.info("Initializing LLMHelper completed")
|
48
50
|
|
51
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
49
52
|
def get_llm(self):
|
50
53
|
if self.auth_type_keys:
|
51
54
|
return AzureChatOpenAI(
|
@@ -67,6 +70,7 @@ class LLMHelper:
|
|
67
70
|
)
|
68
71
|
|
69
72
|
# TODO: This needs to have a custom callback to stream back to the UI
|
73
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
70
74
|
def get_streaming_llm(self):
|
71
75
|
if self.auth_type_keys:
|
72
76
|
return AzureChatOpenAI(
|
@@ -92,6 +96,7 @@ class LLMHelper:
|
|
92
96
|
azure_ad_token_provider=self.token_provider,
|
93
97
|
)
|
94
98
|
|
99
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
95
100
|
def get_embedding_model(self):
|
96
101
|
if self.auth_type_keys:
|
97
102
|
return AzureOpenAIEmbeddings(
|
@@ -108,6 +113,7 @@ class LLMHelper:
|
|
108
113
|
azure_ad_token_provider=self.token_provider,
|
109
114
|
)
|
110
115
|
|
116
|
+
@logger.trace_function(log_args=False, log_result=False)
|
111
117
|
def generate_embeddings(self, input: Union[str, list[int]]) -> List[float]:
|
112
118
|
return (
|
113
119
|
self.openai_client.embeddings.create(
|
@@ -117,6 +123,7 @@ class LLMHelper:
|
|
117
123
|
.embedding
|
118
124
|
)
|
119
125
|
|
126
|
+
@logger.trace_function(log_args=False, log_result=False)
|
120
127
|
def get_chat_completion_with_functions(
|
121
128
|
self, messages: list[dict], functions: list[dict], function_call: str = "auto"
|
122
129
|
):
|
@@ -127,6 +134,7 @@ class LLMHelper:
|
|
127
134
|
function_call=function_call,
|
128
135
|
)
|
129
136
|
|
137
|
+
@logger.trace_function(log_args=False, log_result=False)
|
130
138
|
def get_chat_completion(
|
131
139
|
self, messages: list[dict], model: str | None = None, **kwargs
|
132
140
|
):
|
@@ -137,6 +145,7 @@ class LLMHelper:
|
|
137
145
|
**kwargs,
|
138
146
|
)
|
139
147
|
|
148
|
+
@logger.trace_function(log_args=False, log_result=False)
|
140
149
|
def get_sk_chat_completion_service(self, service_id: str):
|
141
150
|
if self.auth_type_keys:
|
142
151
|
return AzureChatCompletion(
|
@@ -166,6 +175,7 @@ class LLMHelper:
|
|
166
175
|
ad_token_provider=self.token_provider,
|
167
176
|
)
|
168
177
|
|
178
|
+
@logger.trace_function(log_args=False, log_result=False)
|
169
179
|
def get_sk_service_settings(self, service: AzureChatCompletion):
|
170
180
|
return cast(
|
171
181
|
AzureChatPromptExecutionSettings,
|
@@ -176,6 +186,7 @@ class LLMHelper:
|
|
176
186
|
),
|
177
187
|
)
|
178
188
|
|
189
|
+
@logger.trace_function(log_args=False, log_result=False)
|
179
190
|
def get_ml_client(self):
|
180
191
|
if not hasattr(self, "_ml_client"):
|
181
192
|
credential = self.azure_identity_helper.get_credential()
|
@@ -1,8 +1,11 @@
|
|
1
|
-
import logging
|
2
1
|
from ..helpers.azure_identity_helper import AzureIdentityHelper
|
3
2
|
from azure.keyvault.secrets import SecretClient
|
4
3
|
|
5
|
-
|
4
|
+
from ...utilities.helpers.env_helper import EnvHelper
|
5
|
+
from logging_config import logger
|
6
|
+
env_helper: EnvHelper = EnvHelper()
|
7
|
+
log_args = env_helper.LOG_ARGS
|
8
|
+
log_result = env_helper.LOG_RESULT
|
6
9
|
|
7
10
|
|
8
11
|
class SecretHelper:
|
@@ -31,6 +34,7 @@ class SecretHelper:
|
|
31
34
|
connection_verify=True,
|
32
35
|
)
|
33
36
|
|
37
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
34
38
|
def get_secret(self, secret_name: str) -> str:
|
35
39
|
"""
|
36
40
|
Retrieves the value of a secret from the environment variables or Azure Key Vault.
|
@@ -45,11 +49,10 @@ class SecretHelper:
|
|
45
49
|
None
|
46
50
|
|
47
51
|
"""
|
48
|
-
logger.warning(f"Trying to get secret: {secret_name}")
|
49
52
|
secret_value = self.secret_client.get_secret(name=secret_name).value
|
50
|
-
logger.warning(f"Fetched secret value: {secret_value}")
|
51
53
|
return secret_value
|
52
54
|
|
55
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
53
56
|
def set_secret(self, secret_name: str, secret_value: str) -> None:
|
54
57
|
"""
|
55
58
|
Sets the value of a secret in Azure Key Vault only if it doesn't exist or has a different value.
|
@@ -64,12 +67,10 @@ class SecretHelper:
|
|
64
67
|
Raises:
|
65
68
|
None
|
66
69
|
"""
|
67
|
-
logger.warning(f"Trying to set secret: {secret_name}")
|
68
70
|
try:
|
69
71
|
current_secret = self.secret_client.get_secret(name=secret_name)
|
70
72
|
if current_secret.value != secret_value:
|
71
73
|
self.secret_client.set_secret(name=secret_name, value=secret_value)
|
72
|
-
logger.warning(f"Secret {secret_name} has been updated with new value")
|
73
74
|
else:
|
74
75
|
logger.warning(
|
75
76
|
f"Secret {secret_name} already has the same value, skipping update"
|
@@ -78,9 +79,8 @@ class SecretHelper:
|
|
78
79
|
self.secret_client.set_secret(name=secret_name, value=secret_value)
|
79
80
|
logger.warning(f"Secret {secret_name} has been created")
|
80
81
|
|
81
|
-
def get_secret_from_json(self, secret_name: str) -> str:
|
82
82
|
|
83
|
-
|
83
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
84
|
+
def get_secret_from_json(self, secret_name: str) -> str:
|
84
85
|
secret_value = self.secret_client.get_secret(secret_name).value
|
85
|
-
logger.warning(f"Fetched secret value: {secret_value}")
|
86
86
|
return secret_value
|
@@ -1,4 +1,3 @@
|
|
1
|
-
import logging
|
2
1
|
from azure.search.documents.indexes import SearchIndexClient
|
3
2
|
from azure.search.documents.indexes.models import (
|
4
3
|
SearchField,
|
@@ -25,7 +24,10 @@ from ..helpers.azure_identity_helper import AzureIdentityHelper
|
|
25
24
|
from azure.core.credentials import AzureKeyCredential
|
26
25
|
from ..helpers.llm_helper import LLMHelper
|
27
26
|
|
28
|
-
|
27
|
+
from logging_config import logger
|
28
|
+
env_helper: EnvHelper = EnvHelper()
|
29
|
+
log_args = env_helper.LOG_ARGS
|
30
|
+
log_result = env_helper.LOG_RESULT
|
29
31
|
|
30
32
|
|
31
33
|
class AzureSearchIndex:
|
@@ -46,6 +48,7 @@ class AzureSearchIndex:
|
|
46
48
|
)
|
47
49
|
|
48
50
|
@property
|
51
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
49
52
|
def search_dimensions(self) -> int:
|
50
53
|
if AzureSearchIndex._search_dimension is None:
|
51
54
|
AzureSearchIndex._search_dimension = len(
|
@@ -53,6 +56,7 @@ class AzureSearchIndex:
|
|
53
56
|
)
|
54
57
|
return AzureSearchIndex._search_dimension
|
55
58
|
|
59
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
56
60
|
def create_or_update_index(self):
|
57
61
|
# Create a search index
|
58
62
|
fields = [
|
@@ -115,6 +119,7 @@ class AzureSearchIndex:
|
|
115
119
|
logger.info(f"{result.name} index created successfully.")
|
116
120
|
return result
|
117
121
|
|
122
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
118
123
|
def get_vector_search_config(self):
|
119
124
|
if self.env_helper.is_auth_type_keys():
|
120
125
|
azure_open_ai_parameters = AzureOpenAIParameters(
|
@@ -167,6 +172,7 @@ class AzureSearchIndex:
|
|
167
172
|
],
|
168
173
|
)
|
169
174
|
|
175
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
170
176
|
def get_semantic_search_config(self):
|
171
177
|
semantic_config = SemanticConfiguration(
|
172
178
|
name=self.env_helper.AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG,
|
@@ -1,11 +1,15 @@
|
|
1
|
-
import logging
|
2
1
|
from azure.search.documents.indexes.models import SearchIndexer, FieldMapping
|
3
2
|
from azure.search.documents.indexes import SearchIndexerClient
|
4
3
|
from ..helpers.env_helper import EnvHelper
|
5
4
|
from ..helpers.azure_identity_helper import AzureIdentityHelper
|
6
5
|
from azure.core.credentials import AzureKeyCredential
|
7
6
|
|
8
|
-
|
7
|
+
|
8
|
+
from ...utilities.helpers.env_helper import EnvHelper
|
9
|
+
from logging_config import logger
|
10
|
+
env_helper: EnvHelper = EnvHelper()
|
11
|
+
log_args = env_helper.LOG_ARGS
|
12
|
+
log_result = env_helper.LOG_RESULT
|
9
13
|
|
10
14
|
|
11
15
|
class AzureSearchIndexer:
|
@@ -22,6 +26,7 @@ class AzureSearchIndexer:
|
|
22
26
|
),
|
23
27
|
)
|
24
28
|
|
29
|
+
@logger.trace_function(log_args=log_args, log_result=False)
|
25
30
|
def create_or_update_indexer(self, indexer_name: str, skillset_name: str):
|
26
31
|
indexer = SearchIndexer(
|
27
32
|
name=indexer_name,
|
@@ -59,6 +64,7 @@ class AzureSearchIndexer:
|
|
59
64
|
)
|
60
65
|
return indexer_result
|
61
66
|
|
67
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
62
68
|
def run_indexer(self, indexer_name: str):
|
63
69
|
self.indexer_client.reset_indexer(indexer_name)
|
64
70
|
self.indexer_client.run_indexer(indexer_name)
|
@@ -66,6 +72,7 @@ class AzureSearchIndexer:
|
|
66
72
|
f" {indexer_name} is created and running. If queries return no results, please wait a bit and try again."
|
67
73
|
)
|
68
74
|
|
75
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
69
76
|
def indexer_exists(self, indexer_name: str):
|
70
77
|
return indexer_name in [
|
71
78
|
name for name in self.indexer_client.get_indexer_names()
|
@@ -1,4 +1,3 @@
|
|
1
|
-
import logging
|
2
1
|
from azure.search.documents.indexes.models import (
|
3
2
|
SplitSkill,
|
4
3
|
InputFieldMappingEntry,
|
@@ -18,7 +17,11 @@ from ..helpers.env_helper import EnvHelper
|
|
18
17
|
from ..helpers.azure_identity_helper import AzureIdentityHelper
|
19
18
|
from azure.core.credentials import AzureKeyCredential
|
20
19
|
|
21
|
-
|
20
|
+
from ...utilities.helpers.env_helper import EnvHelper
|
21
|
+
from logging_config import logger
|
22
|
+
env_helper: EnvHelper = EnvHelper()
|
23
|
+
log_args = env_helper.LOG_ARGS
|
24
|
+
log_result = env_helper.LOG_RESULT
|
22
25
|
|
23
26
|
|
24
27
|
class AzureSearchSkillset:
|
@@ -40,6 +43,7 @@ class AzureSearchSkillset:
|
|
40
43
|
)
|
41
44
|
self.integrated_vectorization_config = integrated_vectorization_config
|
42
45
|
|
46
|
+
@logger.trace_function(log_args=log_args, log_result=log_result)
|
43
47
|
def create_skillset(self):
|
44
48
|
skillset_name = f"{self.env_helper.AZURE_SEARCH_INDEX}-skillset"
|
45
49
|
|