ws-bom-robot-app 0.0.75__py3-none-any.whl → 0.0.76__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ws_bom_robot_app/llm/agent_description.py +123 -123
- ws_bom_robot_app/llm/agent_handler.py +177 -177
- ws_bom_robot_app/llm/agent_lcel.py +50 -50
- ws_bom_robot_app/llm/defaut_prompt.py +15 -15
- ws_bom_robot_app/llm/feedbacks/feedback_manager.py +66 -66
- ws_bom_robot_app/llm/main.py +138 -138
- ws_bom_robot_app/llm/models/feedback.py +30 -30
- ws_bom_robot_app/llm/nebuly_handler.py +181 -181
- ws_bom_robot_app/llm/providers/llm_manager.py +1 -0
- ws_bom_robot_app/llm/settings.py +4 -4
- ws_bom_robot_app/llm/tools/tool_builder.py +65 -65
- ws_bom_robot_app/llm/tools/tool_manager.py +330 -330
- ws_bom_robot_app/llm/tools/utils.py +41 -41
- ws_bom_robot_app/llm/utils/agent.py +34 -34
- ws_bom_robot_app/llm/utils/cms.py +114 -114
- ws_bom_robot_app/llm/utils/download.py +79 -79
- ws_bom_robot_app/llm/utils/print.py +29 -29
- ws_bom_robot_app/llm/vector_store/generator.py +137 -137
- ws_bom_robot_app/llm/vector_store/integration/thron.py +103 -103
- ws_bom_robot_app/llm/vector_store/loader/json_loader.py +25 -25
- {ws_bom_robot_app-0.0.75.dist-info → ws_bom_robot_app-0.0.76.dist-info}/METADATA +1 -1
- {ws_bom_robot_app-0.0.75.dist-info → ws_bom_robot_app-0.0.76.dist-info}/RECORD +24 -24
- {ws_bom_robot_app-0.0.75.dist-info → ws_bom_robot_app-0.0.76.dist-info}/WHEEL +0 -0
- {ws_bom_robot_app-0.0.75.dist-info → ws_bom_robot_app-0.0.76.dist-info}/top_level.txt +0 -0
|
@@ -1,137 +1,137 @@
|
|
|
1
|
-
import os, gc, shutil, logging, traceback
|
|
2
|
-
import asyncio, aiofiles, aiofiles.os
|
|
3
|
-
from fastapi import HTTPException
|
|
4
|
-
from fastapi.responses import StreamingResponse
|
|
5
|
-
from langchain_core.documents import Document
|
|
6
|
-
from ws_bom_robot_app.llm.vector_store.loader.base import Loader
|
|
7
|
-
from ws_bom_robot_app.llm.models.api import RulesRequest, KbRequest, VectorDbResponse
|
|
8
|
-
from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager
|
|
9
|
-
from ws_bom_robot_app.llm.vector_store.db.manager import VectorDbManager
|
|
10
|
-
from ws_bom_robot_app.config import config
|
|
11
|
-
from ws_bom_robot_app.llm.models.kb import load_endpoints
|
|
12
|
-
from ws_bom_robot_app.llm.utils.download import download_files
|
|
13
|
-
|
|
14
|
-
async def _cleanup_directory(directory_path: str):
|
|
15
|
-
if os.path.exists(directory_path):
|
|
16
|
-
await asyncio.to_thread(shutil.rmtree, directory_path)
|
|
17
|
-
|
|
18
|
-
#@timer
|
|
19
|
-
async def rules(rq: RulesRequest) -> VectorDbResponse:
|
|
20
|
-
_config = rq.config()
|
|
21
|
-
db_name = rq.out_name()
|
|
22
|
-
store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
|
|
23
|
-
try:
|
|
24
|
-
await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(),[Document(page_content=rule, metadata={"source": "rules"}) for rule in rq.rules], store_path) #type: ignore
|
|
25
|
-
db_file_path = shutil.make_archive(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
|
|
26
|
-
return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
|
|
27
|
-
except Exception as e:
|
|
28
|
-
try:
|
|
29
|
-
await _cleanup_directory(store_path)
|
|
30
|
-
finally:
|
|
31
|
-
return VectorDbResponse(success = False, error = str(e))
|
|
32
|
-
finally:
|
|
33
|
-
gc.collect()
|
|
34
|
-
|
|
35
|
-
#@atimer
|
|
36
|
-
async def kb(rq: KbRequest) -> VectorDbResponse:
|
|
37
|
-
os.environ['MPLCONFIGDIR'] = './tmp/.matplotlib'
|
|
38
|
-
_config = rq.config()
|
|
39
|
-
db_name = rq.out_name()
|
|
40
|
-
src_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_src)
|
|
41
|
-
working_path = os.path.join(src_path, db_name)
|
|
42
|
-
|
|
43
|
-
if all([not rq.files,not rq.endpoints,not rq.integrations]):
|
|
44
|
-
return VectorDbResponse(success = False, error = "No files, endpoints or integrations provided")
|
|
45
|
-
else:
|
|
46
|
-
await aiofiles.os.makedirs(src_path, exist_ok=True)
|
|
47
|
-
await aiofiles.os.makedirs(working_path, exist_ok=True)
|
|
48
|
-
|
|
49
|
-
documents: list[Document] = []
|
|
50
|
-
# Download/copy all files
|
|
51
|
-
if rq.files:
|
|
52
|
-
try:
|
|
53
|
-
loaders = Loader(working_path)
|
|
54
|
-
filter_file_extensions = loaders.managed_file_extensions()
|
|
55
|
-
files_to_download = [file for file in rq.files if not os.path.exists(os.path.join(src_path, os.path.basename(file)))]
|
|
56
|
-
if files_to_download:
|
|
57
|
-
await download_files(
|
|
58
|
-
[f"{_config.robot_cms_host}/{_config.robot_cms_kb_folder}/{os.path.basename(file)}" for file in files_to_download if any([file.endswith(ext) for ext in filter_file_extensions])],
|
|
59
|
-
src_path, authorization=_config.robot_cms_auth)
|
|
60
|
-
# copy files to working tmp folder
|
|
61
|
-
for file in rq.files:
|
|
62
|
-
async with aiofiles.open(os.path.join(src_path, os.path.basename(file)), 'rb') as src_file:
|
|
63
|
-
async with aiofiles.open(os.path.join(working_path, os.path.basename(file)), 'wb') as dest_file:
|
|
64
|
-
await dest_file.write(await src_file.read())
|
|
65
|
-
#load files
|
|
66
|
-
try:
|
|
67
|
-
documents.extend(await loaders.load())
|
|
68
|
-
except Exception as e:
|
|
69
|
-
tb = traceback.format_exc()
|
|
70
|
-
_error = f"File loader failure: {e} | {tb}"
|
|
71
|
-
logging.warning(_error)
|
|
72
|
-
return VectorDbResponse(success = False, error = _error)
|
|
73
|
-
except Exception as e:
|
|
74
|
-
await _cleanup_directory(working_path)
|
|
75
|
-
return VectorDbResponse(success = False, error = f"Failed to download file {e}")
|
|
76
|
-
|
|
77
|
-
if rq.endpoints:
|
|
78
|
-
try:
|
|
79
|
-
documents.extend(await load_endpoints(rq.endpoints, working_path))
|
|
80
|
-
except Exception as e:
|
|
81
|
-
await _cleanup_directory(working_path)
|
|
82
|
-
tb = traceback.format_exc()
|
|
83
|
-
_error = f"Endpoint failure: {e} | {tb}"
|
|
84
|
-
logging.warning(_error)
|
|
85
|
-
return VectorDbResponse(success = False, error = _error)
|
|
86
|
-
|
|
87
|
-
if rq.integrations:
|
|
88
|
-
tasks = []
|
|
89
|
-
for integration in rq.integrations:
|
|
90
|
-
tasks.append(
|
|
91
|
-
IntegrationManager
|
|
92
|
-
.get_strategy(integration.type.lower(), working_path, integration.__pydantic_extra__) #type: ignore
|
|
93
|
-
.load()
|
|
94
|
-
)
|
|
95
|
-
try:
|
|
96
|
-
integration_documents = await asyncio.gather(*tasks)
|
|
97
|
-
for docs in integration_documents:
|
|
98
|
-
documents.extend(docs)
|
|
99
|
-
except Exception as e:
|
|
100
|
-
await _cleanup_directory(working_path)
|
|
101
|
-
tb = traceback.format_exc()
|
|
102
|
-
_error = f"Integration failure: {e} | {tb}"
|
|
103
|
-
logging.warning(_error)
|
|
104
|
-
return VectorDbResponse(success=False, error=_error)
|
|
105
|
-
|
|
106
|
-
#cleanup
|
|
107
|
-
await _cleanup_directory(working_path)
|
|
108
|
-
|
|
109
|
-
if documents and len(documents) > 0:
|
|
110
|
-
try:
|
|
111
|
-
store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
|
|
112
|
-
db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
|
|
113
|
-
os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
|
|
114
|
-
"zip",
|
|
115
|
-
await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, return_folder_path=True)
|
|
116
|
-
)
|
|
117
|
-
return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
|
|
118
|
-
except Exception as e:
|
|
119
|
-
await _cleanup_directory(store_path)
|
|
120
|
-
return VectorDbResponse(success = False, error = str(e))
|
|
121
|
-
finally:
|
|
122
|
-
del documents
|
|
123
|
-
gc.collect()
|
|
124
|
-
else:
|
|
125
|
-
_error = "No documents found in the knowledgebase folder"
|
|
126
|
-
logging.warning(_error)
|
|
127
|
-
return VectorDbResponse(success = False, error = _error)
|
|
128
|
-
|
|
129
|
-
async def kb_stream_file(filename: str):
|
|
130
|
-
file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)
|
|
131
|
-
if not os.path.isfile(file_path):
|
|
132
|
-
raise HTTPException(status_code=404, detail="File not found")
|
|
133
|
-
def iter_file():
|
|
134
|
-
with open(file_path, mode="rb") as file:
|
|
135
|
-
while chunk := file.read(1024*8):
|
|
136
|
-
yield chunk
|
|
137
|
-
return StreamingResponse(iter_file(), media_type="application/octet-stream", headers={"Content-Disposition": f"attachment; filename={filename}"})
|
|
1
|
+
import os, gc, shutil, logging, traceback
|
|
2
|
+
import asyncio, aiofiles, aiofiles.os
|
|
3
|
+
from fastapi import HTTPException
|
|
4
|
+
from fastapi.responses import StreamingResponse
|
|
5
|
+
from langchain_core.documents import Document
|
|
6
|
+
from ws_bom_robot_app.llm.vector_store.loader.base import Loader
|
|
7
|
+
from ws_bom_robot_app.llm.models.api import RulesRequest, KbRequest, VectorDbResponse
|
|
8
|
+
from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager
|
|
9
|
+
from ws_bom_robot_app.llm.vector_store.db.manager import VectorDbManager
|
|
10
|
+
from ws_bom_robot_app.config import config
|
|
11
|
+
from ws_bom_robot_app.llm.models.kb import load_endpoints
|
|
12
|
+
from ws_bom_robot_app.llm.utils.download import download_files
|
|
13
|
+
|
|
14
|
+
async def _cleanup_directory(directory_path: str):
|
|
15
|
+
if os.path.exists(directory_path):
|
|
16
|
+
await asyncio.to_thread(shutil.rmtree, directory_path)
|
|
17
|
+
|
|
18
|
+
#@timer
|
|
19
|
+
async def rules(rq: RulesRequest) -> VectorDbResponse:
|
|
20
|
+
_config = rq.config()
|
|
21
|
+
db_name = rq.out_name()
|
|
22
|
+
store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
|
|
23
|
+
try:
|
|
24
|
+
await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(),[Document(page_content=rule, metadata={"source": "rules"}) for rule in rq.rules], store_path) #type: ignore
|
|
25
|
+
db_file_path = shutil.make_archive(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
|
|
26
|
+
return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
|
|
27
|
+
except Exception as e:
|
|
28
|
+
try:
|
|
29
|
+
await _cleanup_directory(store_path)
|
|
30
|
+
finally:
|
|
31
|
+
return VectorDbResponse(success = False, error = str(e))
|
|
32
|
+
finally:
|
|
33
|
+
gc.collect()
|
|
34
|
+
|
|
35
|
+
#@atimer
|
|
36
|
+
async def kb(rq: KbRequest) -> VectorDbResponse:
|
|
37
|
+
os.environ['MPLCONFIGDIR'] = './tmp/.matplotlib'
|
|
38
|
+
_config = rq.config()
|
|
39
|
+
db_name = rq.out_name()
|
|
40
|
+
src_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_src)
|
|
41
|
+
working_path = os.path.join(src_path, db_name)
|
|
42
|
+
|
|
43
|
+
if all([not rq.files,not rq.endpoints,not rq.integrations]):
|
|
44
|
+
return VectorDbResponse(success = False, error = "No files, endpoints or integrations provided")
|
|
45
|
+
else:
|
|
46
|
+
await aiofiles.os.makedirs(src_path, exist_ok=True)
|
|
47
|
+
await aiofiles.os.makedirs(working_path, exist_ok=True)
|
|
48
|
+
|
|
49
|
+
documents: list[Document] = []
|
|
50
|
+
# Download/copy all files
|
|
51
|
+
if rq.files:
|
|
52
|
+
try:
|
|
53
|
+
loaders = Loader(working_path)
|
|
54
|
+
filter_file_extensions = loaders.managed_file_extensions()
|
|
55
|
+
files_to_download = [file for file in rq.files if not os.path.exists(os.path.join(src_path, os.path.basename(file)))]
|
|
56
|
+
if files_to_download:
|
|
57
|
+
await download_files(
|
|
58
|
+
[f"{_config.robot_cms_host}/{_config.robot_cms_kb_folder}/{os.path.basename(file)}" for file in files_to_download if any([file.endswith(ext) for ext in filter_file_extensions])],
|
|
59
|
+
src_path, authorization=_config.robot_cms_auth)
|
|
60
|
+
# copy files to working tmp folder
|
|
61
|
+
for file in rq.files:
|
|
62
|
+
async with aiofiles.open(os.path.join(src_path, os.path.basename(file)), 'rb') as src_file:
|
|
63
|
+
async with aiofiles.open(os.path.join(working_path, os.path.basename(file)), 'wb') as dest_file:
|
|
64
|
+
await dest_file.write(await src_file.read())
|
|
65
|
+
#load files
|
|
66
|
+
try:
|
|
67
|
+
documents.extend(await loaders.load())
|
|
68
|
+
except Exception as e:
|
|
69
|
+
tb = traceback.format_exc()
|
|
70
|
+
_error = f"File loader failure: {e} | {tb}"
|
|
71
|
+
logging.warning(_error)
|
|
72
|
+
return VectorDbResponse(success = False, error = _error)
|
|
73
|
+
except Exception as e:
|
|
74
|
+
await _cleanup_directory(working_path)
|
|
75
|
+
return VectorDbResponse(success = False, error = f"Failed to download file {e}")
|
|
76
|
+
|
|
77
|
+
if rq.endpoints:
|
|
78
|
+
try:
|
|
79
|
+
documents.extend(await load_endpoints(rq.endpoints, working_path))
|
|
80
|
+
except Exception as e:
|
|
81
|
+
await _cleanup_directory(working_path)
|
|
82
|
+
tb = traceback.format_exc()
|
|
83
|
+
_error = f"Endpoint failure: {e} | {tb}"
|
|
84
|
+
logging.warning(_error)
|
|
85
|
+
return VectorDbResponse(success = False, error = _error)
|
|
86
|
+
|
|
87
|
+
if rq.integrations:
|
|
88
|
+
tasks = []
|
|
89
|
+
for integration in rq.integrations:
|
|
90
|
+
tasks.append(
|
|
91
|
+
IntegrationManager
|
|
92
|
+
.get_strategy(integration.type.lower(), working_path, integration.__pydantic_extra__) #type: ignore
|
|
93
|
+
.load()
|
|
94
|
+
)
|
|
95
|
+
try:
|
|
96
|
+
integration_documents = await asyncio.gather(*tasks)
|
|
97
|
+
for docs in integration_documents:
|
|
98
|
+
documents.extend(docs)
|
|
99
|
+
except Exception as e:
|
|
100
|
+
await _cleanup_directory(working_path)
|
|
101
|
+
tb = traceback.format_exc()
|
|
102
|
+
_error = f"Integration failure: {e} | {tb}"
|
|
103
|
+
logging.warning(_error)
|
|
104
|
+
return VectorDbResponse(success=False, error=_error)
|
|
105
|
+
|
|
106
|
+
#cleanup
|
|
107
|
+
await _cleanup_directory(working_path)
|
|
108
|
+
|
|
109
|
+
if documents and len(documents) > 0:
|
|
110
|
+
try:
|
|
111
|
+
store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
|
|
112
|
+
db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
|
|
113
|
+
os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
|
|
114
|
+
"zip",
|
|
115
|
+
await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, return_folder_path=True)
|
|
116
|
+
)
|
|
117
|
+
return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
|
|
118
|
+
except Exception as e:
|
|
119
|
+
await _cleanup_directory(store_path)
|
|
120
|
+
return VectorDbResponse(success = False, error = str(e))
|
|
121
|
+
finally:
|
|
122
|
+
del documents
|
|
123
|
+
gc.collect()
|
|
124
|
+
else:
|
|
125
|
+
_error = "No documents found in the knowledgebase folder"
|
|
126
|
+
logging.warning(_error)
|
|
127
|
+
return VectorDbResponse(success = False, error = _error)
|
|
128
|
+
|
|
129
|
+
async def kb_stream_file(filename: str):
|
|
130
|
+
file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)
|
|
131
|
+
if not os.path.isfile(file_path):
|
|
132
|
+
raise HTTPException(status_code=404, detail="File not found")
|
|
133
|
+
def iter_file():
|
|
134
|
+
with open(file_path, mode="rb") as file:
|
|
135
|
+
while chunk := file.read(1024*8):
|
|
136
|
+
yield chunk
|
|
137
|
+
return StreamingResponse(iter_file(), media_type="application/octet-stream", headers={"Content-Disposition": f"attachment; filename={filename}"})
|
|
@@ -1,103 +1,103 @@
|
|
|
1
|
-
import asyncio, logging, aiohttp
|
|
2
|
-
from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
|
|
3
|
-
from unstructured_ingest.v2.processes.connectors.fsspec.sftp import SftpConnectionConfig, SftpAccessConfig, SftpDownloaderConfig, SftpIndexerConfig
|
|
4
|
-
from langchain_core.documents import Document
|
|
5
|
-
from ws_bom_robot_app.llm.vector_store.loader.base import Loader
|
|
6
|
-
from typing import List, Union, Optional
|
|
7
|
-
from pydantic import BaseModel, Field, AliasChoices
|
|
8
|
-
import json
|
|
9
|
-
import os
|
|
10
|
-
|
|
11
|
-
class ThronParams(BaseModel):
|
|
12
|
-
"""
|
|
13
|
-
ThronParams is a model that defines the parameters required for Thron integration.
|
|
14
|
-
|
|
15
|
-
Attributes:
|
|
16
|
-
app_id (str): The application ID for Thron.
|
|
17
|
-
client_id (str): The client ID for Thron.
|
|
18
|
-
client_secret (str): The client secret for Thron.
|
|
19
|
-
"""
|
|
20
|
-
organization_name: str = Field(validation_alias=AliasChoices("organizationName","organization_name"))
|
|
21
|
-
attribute_fields: Optional[List[str]] = Field(default=None, validation_alias=AliasChoices("attributeFields","attribute_fields"))
|
|
22
|
-
client_id: str = Field(validation_alias=AliasChoices("clientId","client_id"))
|
|
23
|
-
client_secret: str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
|
|
24
|
-
|
|
25
|
-
class Thron(IntegrationStrategy):
|
|
26
|
-
def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
|
|
27
|
-
super().__init__(knowledgebase_path, data)
|
|
28
|
-
self.__data = ThronParams.model_validate(self.data)
|
|
29
|
-
|
|
30
|
-
def working_subdirectory(self) -> str:
|
|
31
|
-
return 'thron'
|
|
32
|
-
|
|
33
|
-
async def run(self) -> None:
|
|
34
|
-
_data = await self.__get_data()
|
|
35
|
-
transformed_data = self.__transform_data(_data)
|
|
36
|
-
json_file_path = os.path.join(self.working_directory, 'thron_data.json')
|
|
37
|
-
with open(json_file_path, 'w', encoding='utf-8') as f:
|
|
38
|
-
json.dump(transformed_data, f, indent=2, ensure_ascii=False)
|
|
39
|
-
|
|
40
|
-
async def load(self) -> list[Document]:
|
|
41
|
-
await self.run()
|
|
42
|
-
await asyncio.sleep(1)
|
|
43
|
-
return await Loader(self.working_directory).load()
|
|
44
|
-
|
|
45
|
-
async def __get_auth_token(self) -> str:
|
|
46
|
-
try:
|
|
47
|
-
async with aiohttp.ClientSession() as session:
|
|
48
|
-
auth_data = {
|
|
49
|
-
"grant_type": "client_credentials",
|
|
50
|
-
"client_id": self.__data.client_id,
|
|
51
|
-
"client_secret": self.__data.client_secret
|
|
52
|
-
}
|
|
53
|
-
headers = {
|
|
54
|
-
"accept": "application/json",
|
|
55
|
-
"Content-Type": "application/x-www-form-urlencoded"
|
|
56
|
-
}
|
|
57
|
-
async with session.post("https://websolute.thron.com/api/v1/authentication/oauth2/token", data=auth_data, headers=headers) as response:
|
|
58
|
-
result = await response.json()
|
|
59
|
-
return result.get("access_token", "")
|
|
60
|
-
except Exception as e:
|
|
61
|
-
logging.error(f"Error fetching Thron auth token: {e}")
|
|
62
|
-
return None
|
|
63
|
-
|
|
64
|
-
async def __get_data(self) -> dict:
|
|
65
|
-
try:
|
|
66
|
-
token = await self.__get_auth_token()
|
|
67
|
-
if not token:
|
|
68
|
-
logging.error("Failed to obtain Thron authentication token.")
|
|
69
|
-
return {}
|
|
70
|
-
attribute_fields = ",".join(self.__data.attribute_fields) if self.__data.attribute_fields else ""
|
|
71
|
-
async with aiohttp.ClientSession() as session:
|
|
72
|
-
headers = {
|
|
73
|
-
"accept": "application/json",
|
|
74
|
-
"Authorization": f"Bearer {token}"
|
|
75
|
-
}
|
|
76
|
-
async with session.get(f"https://{self.__data.organization_name}.thron.com/api/v1/product-data/products?attributeFields=product_id,{attribute_fields}", headers=headers) as response:
|
|
77
|
-
result = await response.json()
|
|
78
|
-
return result.get("items", {})
|
|
79
|
-
except Exception as e:
|
|
80
|
-
logging.error(f"Error fetching Thron product data: {e}")
|
|
81
|
-
return {}
|
|
82
|
-
return []
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
def __transform_data(self, data: dict) -> dict:
|
|
87
|
-
_data = []
|
|
88
|
-
for item in data:
|
|
89
|
-
if item.get("hierarchyLevel") == "MASTER":
|
|
90
|
-
# Iterate through variants to find the product_id
|
|
91
|
-
for item_variant in data:
|
|
92
|
-
if item_variant.get("hierarchyLevel") == "VARIANT":
|
|
93
|
-
for attr in item.get("attributes", []):
|
|
94
|
-
if attr.get("code") == "product_id" and attr.get("identifier") == item_variant.get("variation").get("master").split(":")[-1]:
|
|
95
|
-
# Initialize variants list if it doesn't exist
|
|
96
|
-
if "variants" not in item:
|
|
97
|
-
item["variants"] = []
|
|
98
|
-
item["variants"].append(item_variant)
|
|
99
|
-
_data.append(item)
|
|
100
|
-
break
|
|
101
|
-
elif item.get("hierarchyLevel") == "SIMPLE":
|
|
102
|
-
_data.append(item)
|
|
103
|
-
return _data
|
|
1
|
+
import asyncio, logging, aiohttp
|
|
2
|
+
from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
|
|
3
|
+
from unstructured_ingest.v2.processes.connectors.fsspec.sftp import SftpConnectionConfig, SftpAccessConfig, SftpDownloaderConfig, SftpIndexerConfig
|
|
4
|
+
from langchain_core.documents import Document
|
|
5
|
+
from ws_bom_robot_app.llm.vector_store.loader.base import Loader
|
|
6
|
+
from typing import List, Union, Optional
|
|
7
|
+
from pydantic import BaseModel, Field, AliasChoices
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
|
|
11
|
+
class ThronParams(BaseModel):
|
|
12
|
+
"""
|
|
13
|
+
ThronParams is a model that defines the parameters required for Thron integration.
|
|
14
|
+
|
|
15
|
+
Attributes:
|
|
16
|
+
app_id (str): The application ID for Thron.
|
|
17
|
+
client_id (str): The client ID for Thron.
|
|
18
|
+
client_secret (str): The client secret for Thron.
|
|
19
|
+
"""
|
|
20
|
+
organization_name: str = Field(validation_alias=AliasChoices("organizationName","organization_name"))
|
|
21
|
+
attribute_fields: Optional[List[str]] = Field(default=None, validation_alias=AliasChoices("attributeFields","attribute_fields"))
|
|
22
|
+
client_id: str = Field(validation_alias=AliasChoices("clientId","client_id"))
|
|
23
|
+
client_secret: str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
|
|
24
|
+
|
|
25
|
+
class Thron(IntegrationStrategy):
|
|
26
|
+
def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
|
|
27
|
+
super().__init__(knowledgebase_path, data)
|
|
28
|
+
self.__data = ThronParams.model_validate(self.data)
|
|
29
|
+
|
|
30
|
+
def working_subdirectory(self) -> str:
|
|
31
|
+
return 'thron'
|
|
32
|
+
|
|
33
|
+
async def run(self) -> None:
|
|
34
|
+
_data = await self.__get_data()
|
|
35
|
+
transformed_data = self.__transform_data(_data)
|
|
36
|
+
json_file_path = os.path.join(self.working_directory, 'thron_data.json')
|
|
37
|
+
with open(json_file_path, 'w', encoding='utf-8') as f:
|
|
38
|
+
json.dump(transformed_data, f, indent=2, ensure_ascii=False)
|
|
39
|
+
|
|
40
|
+
async def load(self) -> list[Document]:
|
|
41
|
+
await self.run()
|
|
42
|
+
await asyncio.sleep(1)
|
|
43
|
+
return await Loader(self.working_directory).load()
|
|
44
|
+
|
|
45
|
+
async def __get_auth_token(self) -> str:
|
|
46
|
+
try:
|
|
47
|
+
async with aiohttp.ClientSession() as session:
|
|
48
|
+
auth_data = {
|
|
49
|
+
"grant_type": "client_credentials",
|
|
50
|
+
"client_id": self.__data.client_id,
|
|
51
|
+
"client_secret": self.__data.client_secret
|
|
52
|
+
}
|
|
53
|
+
headers = {
|
|
54
|
+
"accept": "application/json",
|
|
55
|
+
"Content-Type": "application/x-www-form-urlencoded"
|
|
56
|
+
}
|
|
57
|
+
async with session.post("https://websolute.thron.com/api/v1/authentication/oauth2/token", data=auth_data, headers=headers) as response:
|
|
58
|
+
result = await response.json()
|
|
59
|
+
return result.get("access_token", "")
|
|
60
|
+
except Exception as e:
|
|
61
|
+
logging.error(f"Error fetching Thron auth token: {e}")
|
|
62
|
+
return None
|
|
63
|
+
|
|
64
|
+
async def __get_data(self) -> dict:
|
|
65
|
+
try:
|
|
66
|
+
token = await self.__get_auth_token()
|
|
67
|
+
if not token:
|
|
68
|
+
logging.error("Failed to obtain Thron authentication token.")
|
|
69
|
+
return {}
|
|
70
|
+
attribute_fields = ",".join(self.__data.attribute_fields) if self.__data.attribute_fields else ""
|
|
71
|
+
async with aiohttp.ClientSession() as session:
|
|
72
|
+
headers = {
|
|
73
|
+
"accept": "application/json",
|
|
74
|
+
"Authorization": f"Bearer {token}"
|
|
75
|
+
}
|
|
76
|
+
async with session.get(f"https://{self.__data.organization_name}.thron.com/api/v1/product-data/products?attributeFields=product_id,{attribute_fields}", headers=headers) as response:
|
|
77
|
+
result = await response.json()
|
|
78
|
+
return result.get("items", {})
|
|
79
|
+
except Exception as e:
|
|
80
|
+
logging.error(f"Error fetching Thron product data: {e}")
|
|
81
|
+
return {}
|
|
82
|
+
return []
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def __transform_data(self, data: dict) -> dict:
|
|
87
|
+
_data = []
|
|
88
|
+
for item in data:
|
|
89
|
+
if item.get("hierarchyLevel") == "MASTER":
|
|
90
|
+
# Iterate through variants to find the product_id
|
|
91
|
+
for item_variant in data:
|
|
92
|
+
if item_variant.get("hierarchyLevel") == "VARIANT":
|
|
93
|
+
for attr in item.get("attributes", []):
|
|
94
|
+
if attr.get("code") == "product_id" and attr.get("identifier") == item_variant.get("variation").get("master").split(":")[-1]:
|
|
95
|
+
# Initialize variants list if it doesn't exist
|
|
96
|
+
if "variants" not in item:
|
|
97
|
+
item["variants"] = []
|
|
98
|
+
item["variants"].append(item_variant)
|
|
99
|
+
_data.append(item)
|
|
100
|
+
break
|
|
101
|
+
elif item.get("hierarchyLevel") == "SIMPLE":
|
|
102
|
+
_data.append(item)
|
|
103
|
+
return _data
|
|
@@ -1,25 +1,25 @@
|
|
|
1
|
-
import json
|
|
2
|
-
from typing import Optional
|
|
3
|
-
from langchain_core.documents import Document
|
|
4
|
-
from langchain_community.document_loaders.base import BaseLoader
|
|
5
|
-
|
|
6
|
-
class JsonLoader(BaseLoader):
|
|
7
|
-
def __init__(self, file_path: str, meta_fields:Optional[list[str]] = [],encoding: Optional[str] = "utf-8"):
|
|
8
|
-
self.file_path = file_path
|
|
9
|
-
self.meta_fields = meta_fields
|
|
10
|
-
self.encoding = encoding
|
|
11
|
-
|
|
12
|
-
def load(self) -> list[Document]:
|
|
13
|
-
with open(self.file_path, "r", encoding=self.encoding) as file:
|
|
14
|
-
data = json.load(file)
|
|
15
|
-
_list = data if isinstance(data, list) else [data]
|
|
16
|
-
return [
|
|
17
|
-
Document(
|
|
18
|
-
page_content=json.dumps(item),
|
|
19
|
-
metadata={
|
|
20
|
-
"source": self.file_path,
|
|
21
|
-
**{field: item.get(field) for field in self.meta_fields if item.get(field)}
|
|
22
|
-
}
|
|
23
|
-
)
|
|
24
|
-
for item in _list
|
|
25
|
-
]
|
|
1
|
+
import json
|
|
2
|
+
from typing import Optional
|
|
3
|
+
from langchain_core.documents import Document
|
|
4
|
+
from langchain_community.document_loaders.base import BaseLoader
|
|
5
|
+
|
|
6
|
+
class JsonLoader(BaseLoader):
|
|
7
|
+
def __init__(self, file_path: str, meta_fields:Optional[list[str]] = [],encoding: Optional[str] = "utf-8"):
|
|
8
|
+
self.file_path = file_path
|
|
9
|
+
self.meta_fields = meta_fields
|
|
10
|
+
self.encoding = encoding
|
|
11
|
+
|
|
12
|
+
def load(self) -> list[Document]:
|
|
13
|
+
with open(self.file_path, "r", encoding=self.encoding) as file:
|
|
14
|
+
data = json.load(file)
|
|
15
|
+
_list = data if isinstance(data, list) else [data]
|
|
16
|
+
return [
|
|
17
|
+
Document(
|
|
18
|
+
page_content=json.dumps(item),
|
|
19
|
+
metadata={
|
|
20
|
+
"source": self.file_path,
|
|
21
|
+
**{field: item.get(field) for field in self.meta_fields if item.get(field)}
|
|
22
|
+
}
|
|
23
|
+
)
|
|
24
|
+
for item in _list
|
|
25
|
+
]
|
|
@@ -7,40 +7,40 @@ ws_bom_robot_app/task_manager.py,sha256=Q3Il2TtkP0FoG9zHEBu48pZGXzimTtvWQsoH6wdv
|
|
|
7
7
|
ws_bom_robot_app/util.py,sha256=RjVD6B9sHje788Lndqq5DHy6TJM0KLs9qx3JYt81Wyk,4834
|
|
8
8
|
ws_bom_robot_app/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
9
|
ws_bom_robot_app/llm/agent_context.py,sha256=uatHJ8wcRly6h0S762BgfzDMpmcwCHwNzwo37aWjeE0,1305
|
|
10
|
-
ws_bom_robot_app/llm/agent_description.py,sha256=
|
|
11
|
-
ws_bom_robot_app/llm/agent_handler.py,sha256
|
|
12
|
-
ws_bom_robot_app/llm/agent_lcel.py,sha256=
|
|
10
|
+
ws_bom_robot_app/llm/agent_description.py,sha256=yK4aVU3RNk1oP4bEneV3QPAi-208JwWk4R6qHlzqYIg,4656
|
|
11
|
+
ws_bom_robot_app/llm/agent_handler.py,sha256=BQ-f--Z5QCJDp-7tzSG_CKrANUCqG65S09psgWVNxa4,7597
|
|
12
|
+
ws_bom_robot_app/llm/agent_lcel.py,sha256=tVa1JJOuL1CG0tXS5AwOB4gli0E2rGqSBD5oEehHvOY,2480
|
|
13
13
|
ws_bom_robot_app/llm/api.py,sha256=2bF-UFczY9LuBqPxKObM0TOWYbZgVztX1RiIz5MSorU,5042
|
|
14
|
-
ws_bom_robot_app/llm/defaut_prompt.py,sha256=
|
|
15
|
-
ws_bom_robot_app/llm/main.py,sha256=
|
|
16
|
-
ws_bom_robot_app/llm/nebuly_handler.py,sha256=
|
|
17
|
-
ws_bom_robot_app/llm/settings.py,sha256=
|
|
14
|
+
ws_bom_robot_app/llm/defaut_prompt.py,sha256=LlCd_nSMkMmHESfiiiQYfnJyB6Pp-LSs4CEKdYW4vFk,1106
|
|
15
|
+
ws_bom_robot_app/llm/main.py,sha256=vzUfaLCRk2SYujD00hnrTiHEVLYgZcbSw6LUea43siU,5235
|
|
16
|
+
ws_bom_robot_app/llm/nebuly_handler.py,sha256=d4TI5XbvIYJLCxHcCUa6QUxsgwKW_4ItCYe4ocn7IJo,7900
|
|
17
|
+
ws_bom_robot_app/llm/settings.py,sha256=EkFGCppORenStH9W4e6_dYvQ-5p6xiEMpmUHBqNqG9M,117
|
|
18
18
|
ws_bom_robot_app/llm/feedbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
-
ws_bom_robot_app/llm/feedbacks/feedback_manager.py,sha256=
|
|
19
|
+
ws_bom_robot_app/llm/feedbacks/feedback_manager.py,sha256=WcKgzlOb8VFG7yqHoIOO_R6LAzdzE4YIRFCVOGBSgfM,2856
|
|
20
20
|
ws_bom_robot_app/llm/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
21
|
ws_bom_robot_app/llm/models/api.py,sha256=8hllPUnPM6Rp6TgcvTQtpUh5Nk36r1qcvIEc2O7LZJE,10968
|
|
22
22
|
ws_bom_robot_app/llm/models/base.py,sha256=1TqxuTK3rjJEALn7lvgoen_1ba3R2brAgGx6EDTtDZo,152
|
|
23
|
-
ws_bom_robot_app/llm/models/feedback.py,sha256=
|
|
23
|
+
ws_bom_robot_app/llm/models/feedback.py,sha256=zh1jLqPRLzNlxInkCMoiJbfSu0-tiOEYHM7FhC46PkM,1692
|
|
24
24
|
ws_bom_robot_app/llm/models/kb.py,sha256=oVSw6_dmNxikAHrPqcfxDXz9M0ezLIYuxpgvzfs_Now,9514
|
|
25
25
|
ws_bom_robot_app/llm/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
26
|
-
ws_bom_robot_app/llm/providers/llm_manager.py,sha256=
|
|
26
|
+
ws_bom_robot_app/llm/providers/llm_manager.py,sha256=j-AnRh5jZ3eSNOZcmVKO63oBtosXA_blBBrjvJtIkWU,11942
|
|
27
27
|
ws_bom_robot_app/llm/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
28
|
-
ws_bom_robot_app/llm/tools/tool_builder.py,sha256=
|
|
29
|
-
ws_bom_robot_app/llm/tools/tool_manager.py,sha256=
|
|
30
|
-
ws_bom_robot_app/llm/tools/utils.py,sha256=
|
|
28
|
+
ws_bom_robot_app/llm/tools/tool_builder.py,sha256=QTRG1c-EnH4APP10IyfZxEkqK9KitUsutXUvDRKeAhU,3224
|
|
29
|
+
ws_bom_robot_app/llm/tools/tool_manager.py,sha256=1IgRXxdB7DU3gbIlfT_aMUWZyWuanFTAFwu3VaYKxfE,14990
|
|
30
|
+
ws_bom_robot_app/llm/tools/utils.py,sha256=tdmOAk8l4HVzw67z3brA9yX-1WLu91paU-WmXHyz4Bg,1883
|
|
31
31
|
ws_bom_robot_app/llm/tools/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
32
32
|
ws_bom_robot_app/llm/tools/models/main.py,sha256=1hICqHs-KS2heenkH7b2eH0N2GrPaaNGBrn64cl_A40,827
|
|
33
33
|
ws_bom_robot_app/llm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
34
|
-
ws_bom_robot_app/llm/utils/agent.py,sha256=
|
|
34
|
+
ws_bom_robot_app/llm/utils/agent.py,sha256=_CY5Dji3UeAIi2iuU7ttz4fml1q8aCFgVWOv970x8Fw,1411
|
|
35
35
|
ws_bom_robot_app/llm/utils/chunker.py,sha256=N7570xBYlObneg-fsvDhPAJ-Pv8C8OaYZOBK6q7LmMI,607
|
|
36
|
-
ws_bom_robot_app/llm/utils/cms.py,sha256=
|
|
37
|
-
ws_bom_robot_app/llm/utils/download.py,sha256=
|
|
36
|
+
ws_bom_robot_app/llm/utils/cms.py,sha256=XhrLQyHQ2JUOInDCCf_uvR4Jiud0YvH2FwwiiuCnnsg,6352
|
|
37
|
+
ws_bom_robot_app/llm/utils/download.py,sha256=iAUxH_NiCpTPtGzhC4hBtxotd2HPFt2MBhttslIxqiI,3194
|
|
38
38
|
ws_bom_robot_app/llm/utils/kb.py,sha256=jja45WCbNI7SGEgqDS99nErlwB5eY8Ga7BMnhdMHZ90,1279
|
|
39
|
-
ws_bom_robot_app/llm/utils/print.py,sha256=
|
|
39
|
+
ws_bom_robot_app/llm/utils/print.py,sha256=IsPYEWRJqu-dqlJA3F9OnnIS4rOq_EYX1Ljp3BvDnww,774
|
|
40
40
|
ws_bom_robot_app/llm/utils/secrets.py,sha256=-HtqLIDVIJrpvGC5YhPAVyLsq8P4ChVM5g3GOfdwqVk,878
|
|
41
41
|
ws_bom_robot_app/llm/utils/webhooks.py,sha256=LAAZqyN6VhV13wu4X-X85TwdDgAV2rNvIwQFIIc0FJM,2114
|
|
42
42
|
ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
43
|
-
ws_bom_robot_app/llm/vector_store/generator.py,sha256=
|
|
43
|
+
ws_bom_robot_app/llm/vector_store/generator.py,sha256=9_xdtCKJhmt1OP0GXDjvFERXMP7ozLZT92KuYEBDgC0,6314
|
|
44
44
|
ws_bom_robot_app/llm/vector_store/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
45
45
|
ws_bom_robot_app/llm/vector_store/db/base.py,sha256=t0Z1VCcg604evEzJENGNqYFBi_AZLTEUzmxA5wgoE_A,8419
|
|
46
46
|
ws_bom_robot_app/llm/vector_store/db/chroma.py,sha256=2riMQvwe2T99X_NtO9yO9lpZ0zj2Nb06l9Hb1lWJ00E,4509
|
|
@@ -62,12 +62,12 @@ ws_bom_robot_app/llm/vector_store/integration/sftp.py,sha256=WNzjjS1EUykgFB-8e7Q
|
|
|
62
62
|
ws_bom_robot_app/llm/vector_store/integration/sharepoint.py,sha256=zqqn-6qPrK50Phch4nZHJTgaPyPkGe7W2InGL_Ru6vE,5376
|
|
63
63
|
ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=g0TIRZ2qIpEuVCZ9Bn0MvvxYZtU3wptnTEjoKNZyBAg,5019
|
|
64
64
|
ws_bom_robot_app/llm/vector_store/integration/slack.py,sha256=FMjESXm2QetFXI6i8epze7Kbbu22fV8CVaxb71AHnJ8,2572
|
|
65
|
-
ws_bom_robot_app/llm/vector_store/integration/thron.py,sha256=
|
|
65
|
+
ws_bom_robot_app/llm/vector_store/integration/thron.py,sha256=AX3Dk6A6TDAEST9VVwhKVB6qr4b1lBXlE-dy_V4Tgu4,4416
|
|
66
66
|
ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
67
67
|
ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=L_ugekNuAq0N9O-24wtlHSNHkqSeD-KsJrfGt_FX9Oc,5340
|
|
68
68
|
ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=yP0zgXLeFAlByaYuj-6cYariuknckrFds0dxdRcnVz8,3456
|
|
69
|
-
ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=
|
|
70
|
-
ws_bom_robot_app-0.0.
|
|
71
|
-
ws_bom_robot_app-0.0.
|
|
72
|
-
ws_bom_robot_app-0.0.
|
|
73
|
-
ws_bom_robot_app-0.0.
|
|
69
|
+
ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=qo9ejRZyKv_k6jnGgXnu1W5uqsMMtgqK_uvPpZQ0p74,833
|
|
70
|
+
ws_bom_robot_app-0.0.76.dist-info/METADATA,sha256=e6QyzVw9spgo-qPmhDmX-O5oYtDvGN_zJLnhy2gHTuk,8609
|
|
71
|
+
ws_bom_robot_app-0.0.76.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
72
|
+
ws_bom_robot_app-0.0.76.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
|
|
73
|
+
ws_bom_robot_app-0.0.76.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|