ws-bom-robot-app 0.0.67__py3-none-any.whl → 0.0.69__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ws_bom_robot_app/config.py +7 -7
- ws_bom_robot_app/llm/agent_description.py +123 -123
- ws_bom_robot_app/llm/agent_handler.py +177 -177
- ws_bom_robot_app/llm/agent_lcel.py +45 -45
- ws_bom_robot_app/llm/defaut_prompt.py +15 -15
- ws_bom_robot_app/llm/feedbacks/feedback_manager.py +66 -66
- ws_bom_robot_app/llm/main.py +138 -138
- ws_bom_robot_app/llm/models/feedback.py +30 -30
- ws_bom_robot_app/llm/nebuly_handler.py +181 -181
- ws_bom_robot_app/llm/providers/llm_manager.py +3 -3
- ws_bom_robot_app/llm/settings.py +4 -4
- ws_bom_robot_app/llm/tools/tool_builder.py +65 -65
- ws_bom_robot_app/llm/tools/tool_manager.py +330 -317
- ws_bom_robot_app/llm/tools/utils.py +41 -41
- ws_bom_robot_app/llm/utils/agent.py +34 -34
- ws_bom_robot_app/llm/utils/cms.py +109 -77
- ws_bom_robot_app/llm/utils/download.py +79 -79
- ws_bom_robot_app/llm/utils/print.py +29 -29
- ws_bom_robot_app/llm/vector_store/generator.py +137 -137
- ws_bom_robot_app/llm/vector_store/integration/manager.py +2 -2
- ws_bom_robot_app/llm/vector_store/integration/thron.py +123 -0
- ws_bom_robot_app/llm/vector_store/loader/json_loader.py +25 -25
- {ws_bom_robot_app-0.0.67.dist-info → ws_bom_robot_app-0.0.69.dist-info}/METADATA +11 -5
- {ws_bom_robot_app-0.0.67.dist-info → ws_bom_robot_app-0.0.69.dist-info}/RECORD +26 -25
- {ws_bom_robot_app-0.0.67.dist-info → ws_bom_robot_app-0.0.69.dist-info}/WHEEL +0 -0
- {ws_bom_robot_app-0.0.67.dist-info → ws_bom_robot_app-0.0.69.dist-info}/top_level.txt +0 -0
|
@@ -1,137 +1,137 @@
|
|
|
1
|
-
import os, gc, shutil, logging, traceback
|
|
2
|
-
import asyncio, aiofiles, aiofiles.os
|
|
3
|
-
from fastapi import HTTPException
|
|
4
|
-
from fastapi.responses import StreamingResponse
|
|
5
|
-
from langchain_core.documents import Document
|
|
6
|
-
from ws_bom_robot_app.llm.vector_store.loader.base import Loader
|
|
7
|
-
from ws_bom_robot_app.llm.models.api import RulesRequest, KbRequest, VectorDbResponse
|
|
8
|
-
from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager
|
|
9
|
-
from ws_bom_robot_app.llm.vector_store.db.manager import VectorDbManager
|
|
10
|
-
from ws_bom_robot_app.config import config
|
|
11
|
-
from ws_bom_robot_app.llm.models.kb import load_endpoints
|
|
12
|
-
from ws_bom_robot_app.llm.utils.download import download_files
|
|
13
|
-
|
|
14
|
-
async def _cleanup_directory(directory_path: str):
|
|
15
|
-
if os.path.exists(directory_path):
|
|
16
|
-
await asyncio.to_thread(shutil.rmtree, directory_path)
|
|
17
|
-
|
|
18
|
-
#@timer
|
|
19
|
-
async def rules(rq: RulesRequest) -> VectorDbResponse:
|
|
20
|
-
_config = rq.config()
|
|
21
|
-
db_name = rq.out_name()
|
|
22
|
-
store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
|
|
23
|
-
try:
|
|
24
|
-
await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(),[Document(page_content=rule, metadata={"source": "rules"}) for rule in rq.rules], store_path) #type: ignore
|
|
25
|
-
db_file_path = shutil.make_archive(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
|
|
26
|
-
return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
|
|
27
|
-
except Exception as e:
|
|
28
|
-
try:
|
|
29
|
-
await _cleanup_directory(store_path)
|
|
30
|
-
finally:
|
|
31
|
-
return VectorDbResponse(success = False, error = str(e))
|
|
32
|
-
finally:
|
|
33
|
-
gc.collect()
|
|
34
|
-
|
|
35
|
-
#@atimer
|
|
36
|
-
async def kb(rq: KbRequest) -> VectorDbResponse:
|
|
37
|
-
os.environ['MPLCONFIGDIR'] = './tmp/.matplotlib'
|
|
38
|
-
_config = rq.config()
|
|
39
|
-
db_name = rq.out_name()
|
|
40
|
-
src_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_src)
|
|
41
|
-
working_path = os.path.join(src_path, db_name)
|
|
42
|
-
|
|
43
|
-
if all([not rq.files,not rq.endpoints,not rq.integrations]):
|
|
44
|
-
return VectorDbResponse(success = False, error = "No files, endpoints or integrations provided")
|
|
45
|
-
else:
|
|
46
|
-
await aiofiles.os.makedirs(src_path, exist_ok=True)
|
|
47
|
-
await aiofiles.os.makedirs(working_path, exist_ok=True)
|
|
48
|
-
|
|
49
|
-
documents: list[Document] = []
|
|
50
|
-
# Download/copy all files
|
|
51
|
-
if rq.files:
|
|
52
|
-
try:
|
|
53
|
-
loaders = Loader(working_path)
|
|
54
|
-
filter_file_extensions = loaders.managed_file_extensions()
|
|
55
|
-
files_to_download = [file for file in rq.files if not os.path.exists(os.path.join(src_path, os.path.basename(file)))]
|
|
56
|
-
if files_to_download:
|
|
57
|
-
await download_files(
|
|
58
|
-
[f"{_config.robot_cms_host}/{_config.robot_cms_kb_folder}/{os.path.basename(file)}" for file in files_to_download if any([file.endswith(ext) for ext in filter_file_extensions])],
|
|
59
|
-
src_path, authorization=_config.robot_cms_auth)
|
|
60
|
-
# copy files to working tmp folder
|
|
61
|
-
for file in rq.files:
|
|
62
|
-
async with aiofiles.open(os.path.join(src_path, os.path.basename(file)), 'rb') as src_file:
|
|
63
|
-
async with aiofiles.open(os.path.join(working_path, os.path.basename(file)), 'wb') as dest_file:
|
|
64
|
-
await dest_file.write(await src_file.read())
|
|
65
|
-
#load files
|
|
66
|
-
try:
|
|
67
|
-
documents.extend(await loaders.load())
|
|
68
|
-
except Exception as e:
|
|
69
|
-
tb = traceback.format_exc()
|
|
70
|
-
_error = f"File loader failure: {e} | {tb}"
|
|
71
|
-
logging.warning(_error)
|
|
72
|
-
return VectorDbResponse(success = False, error = _error)
|
|
73
|
-
except Exception as e:
|
|
74
|
-
await _cleanup_directory(working_path)
|
|
75
|
-
return VectorDbResponse(success = False, error = f"Failed to download file {e}")
|
|
76
|
-
|
|
77
|
-
if rq.endpoints:
|
|
78
|
-
try:
|
|
79
|
-
documents.extend(await load_endpoints(rq.endpoints, working_path))
|
|
80
|
-
except Exception as e:
|
|
81
|
-
await _cleanup_directory(working_path)
|
|
82
|
-
tb = traceback.format_exc()
|
|
83
|
-
_error = f"Endpoint failure: {e} | {tb}"
|
|
84
|
-
logging.warning(_error)
|
|
85
|
-
return VectorDbResponse(success = False, error = _error)
|
|
86
|
-
|
|
87
|
-
if rq.integrations:
|
|
88
|
-
tasks = []
|
|
89
|
-
for integration in rq.integrations:
|
|
90
|
-
tasks.append(
|
|
91
|
-
IntegrationManager
|
|
92
|
-
.get_strategy(integration.type.lower(), working_path, integration.__pydantic_extra__) #type: ignore
|
|
93
|
-
.load()
|
|
94
|
-
)
|
|
95
|
-
try:
|
|
96
|
-
integration_documents = await asyncio.gather(*tasks)
|
|
97
|
-
for docs in integration_documents:
|
|
98
|
-
documents.extend(docs)
|
|
99
|
-
except Exception as e:
|
|
100
|
-
await _cleanup_directory(working_path)
|
|
101
|
-
tb = traceback.format_exc()
|
|
102
|
-
_error = f"Integration failure: {e} | {tb}"
|
|
103
|
-
logging.warning(_error)
|
|
104
|
-
return VectorDbResponse(success=False, error=_error)
|
|
105
|
-
|
|
106
|
-
#cleanup
|
|
107
|
-
await _cleanup_directory(working_path)
|
|
108
|
-
|
|
109
|
-
if documents and len(documents) > 0:
|
|
110
|
-
try:
|
|
111
|
-
store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
|
|
112
|
-
db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
|
|
113
|
-
os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
|
|
114
|
-
"zip",
|
|
115
|
-
await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, return_folder_path=True)
|
|
116
|
-
)
|
|
117
|
-
return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
|
|
118
|
-
except Exception as e:
|
|
119
|
-
await _cleanup_directory(store_path)
|
|
120
|
-
return VectorDbResponse(success = False, error = str(e))
|
|
121
|
-
finally:
|
|
122
|
-
del documents
|
|
123
|
-
gc.collect()
|
|
124
|
-
else:
|
|
125
|
-
_error = "No documents found in the knowledgebase folder"
|
|
126
|
-
logging.warning(_error)
|
|
127
|
-
return VectorDbResponse(success = False, error = _error)
|
|
128
|
-
|
|
129
|
-
async def kb_stream_file(filename: str):
|
|
130
|
-
file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)
|
|
131
|
-
if not os.path.isfile(file_path):
|
|
132
|
-
raise HTTPException(status_code=404, detail="File not found")
|
|
133
|
-
def iter_file():
|
|
134
|
-
with open(file_path, mode="rb") as file:
|
|
135
|
-
while chunk := file.read(1024*8):
|
|
136
|
-
yield chunk
|
|
137
|
-
return StreamingResponse(iter_file(), media_type="application/octet-stream", headers={"Content-Disposition": f"attachment; filename={filename}"})
|
|
1
|
+
import os, gc, shutil, logging, traceback
|
|
2
|
+
import asyncio, aiofiles, aiofiles.os
|
|
3
|
+
from fastapi import HTTPException
|
|
4
|
+
from fastapi.responses import StreamingResponse
|
|
5
|
+
from langchain_core.documents import Document
|
|
6
|
+
from ws_bom_robot_app.llm.vector_store.loader.base import Loader
|
|
7
|
+
from ws_bom_robot_app.llm.models.api import RulesRequest, KbRequest, VectorDbResponse
|
|
8
|
+
from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager
|
|
9
|
+
from ws_bom_robot_app.llm.vector_store.db.manager import VectorDbManager
|
|
10
|
+
from ws_bom_robot_app.config import config
|
|
11
|
+
from ws_bom_robot_app.llm.models.kb import load_endpoints
|
|
12
|
+
from ws_bom_robot_app.llm.utils.download import download_files
|
|
13
|
+
|
|
14
|
+
async def _cleanup_directory(directory_path: str):
|
|
15
|
+
if os.path.exists(directory_path):
|
|
16
|
+
await asyncio.to_thread(shutil.rmtree, directory_path)
|
|
17
|
+
|
|
18
|
+
#@timer
|
|
19
|
+
async def rules(rq: RulesRequest) -> VectorDbResponse:
|
|
20
|
+
_config = rq.config()
|
|
21
|
+
db_name = rq.out_name()
|
|
22
|
+
store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
|
|
23
|
+
try:
|
|
24
|
+
await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(),[Document(page_content=rule, metadata={"source": "rules"}) for rule in rq.rules], store_path) #type: ignore
|
|
25
|
+
db_file_path = shutil.make_archive(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
|
|
26
|
+
return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
|
|
27
|
+
except Exception as e:
|
|
28
|
+
try:
|
|
29
|
+
await _cleanup_directory(store_path)
|
|
30
|
+
finally:
|
|
31
|
+
return VectorDbResponse(success = False, error = str(e))
|
|
32
|
+
finally:
|
|
33
|
+
gc.collect()
|
|
34
|
+
|
|
35
|
+
#@atimer
|
|
36
|
+
async def kb(rq: KbRequest) -> VectorDbResponse:
|
|
37
|
+
os.environ['MPLCONFIGDIR'] = './tmp/.matplotlib'
|
|
38
|
+
_config = rq.config()
|
|
39
|
+
db_name = rq.out_name()
|
|
40
|
+
src_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_src)
|
|
41
|
+
working_path = os.path.join(src_path, db_name)
|
|
42
|
+
|
|
43
|
+
if all([not rq.files,not rq.endpoints,not rq.integrations]):
|
|
44
|
+
return VectorDbResponse(success = False, error = "No files, endpoints or integrations provided")
|
|
45
|
+
else:
|
|
46
|
+
await aiofiles.os.makedirs(src_path, exist_ok=True)
|
|
47
|
+
await aiofiles.os.makedirs(working_path, exist_ok=True)
|
|
48
|
+
|
|
49
|
+
documents: list[Document] = []
|
|
50
|
+
# Download/copy all files
|
|
51
|
+
if rq.files:
|
|
52
|
+
try:
|
|
53
|
+
loaders = Loader(working_path)
|
|
54
|
+
filter_file_extensions = loaders.managed_file_extensions()
|
|
55
|
+
files_to_download = [file for file in rq.files if not os.path.exists(os.path.join(src_path, os.path.basename(file)))]
|
|
56
|
+
if files_to_download:
|
|
57
|
+
await download_files(
|
|
58
|
+
[f"{_config.robot_cms_host}/{_config.robot_cms_kb_folder}/{os.path.basename(file)}" for file in files_to_download if any([file.endswith(ext) for ext in filter_file_extensions])],
|
|
59
|
+
src_path, authorization=_config.robot_cms_auth)
|
|
60
|
+
# copy files to working tmp folder
|
|
61
|
+
for file in rq.files:
|
|
62
|
+
async with aiofiles.open(os.path.join(src_path, os.path.basename(file)), 'rb') as src_file:
|
|
63
|
+
async with aiofiles.open(os.path.join(working_path, os.path.basename(file)), 'wb') as dest_file:
|
|
64
|
+
await dest_file.write(await src_file.read())
|
|
65
|
+
#load files
|
|
66
|
+
try:
|
|
67
|
+
documents.extend(await loaders.load())
|
|
68
|
+
except Exception as e:
|
|
69
|
+
tb = traceback.format_exc()
|
|
70
|
+
_error = f"File loader failure: {e} | {tb}"
|
|
71
|
+
logging.warning(_error)
|
|
72
|
+
return VectorDbResponse(success = False, error = _error)
|
|
73
|
+
except Exception as e:
|
|
74
|
+
await _cleanup_directory(working_path)
|
|
75
|
+
return VectorDbResponse(success = False, error = f"Failed to download file {e}")
|
|
76
|
+
|
|
77
|
+
if rq.endpoints:
|
|
78
|
+
try:
|
|
79
|
+
documents.extend(await load_endpoints(rq.endpoints, working_path))
|
|
80
|
+
except Exception as e:
|
|
81
|
+
await _cleanup_directory(working_path)
|
|
82
|
+
tb = traceback.format_exc()
|
|
83
|
+
_error = f"Endpoint failure: {e} | {tb}"
|
|
84
|
+
logging.warning(_error)
|
|
85
|
+
return VectorDbResponse(success = False, error = _error)
|
|
86
|
+
|
|
87
|
+
if rq.integrations:
|
|
88
|
+
tasks = []
|
|
89
|
+
for integration in rq.integrations:
|
|
90
|
+
tasks.append(
|
|
91
|
+
IntegrationManager
|
|
92
|
+
.get_strategy(integration.type.lower(), working_path, integration.__pydantic_extra__) #type: ignore
|
|
93
|
+
.load()
|
|
94
|
+
)
|
|
95
|
+
try:
|
|
96
|
+
integration_documents = await asyncio.gather(*tasks)
|
|
97
|
+
for docs in integration_documents:
|
|
98
|
+
documents.extend(docs)
|
|
99
|
+
except Exception as e:
|
|
100
|
+
await _cleanup_directory(working_path)
|
|
101
|
+
tb = traceback.format_exc()
|
|
102
|
+
_error = f"Integration failure: {e} | {tb}"
|
|
103
|
+
logging.warning(_error)
|
|
104
|
+
return VectorDbResponse(success=False, error=_error)
|
|
105
|
+
|
|
106
|
+
#cleanup
|
|
107
|
+
await _cleanup_directory(working_path)
|
|
108
|
+
|
|
109
|
+
if documents and len(documents) > 0:
|
|
110
|
+
try:
|
|
111
|
+
store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
|
|
112
|
+
db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
|
|
113
|
+
os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
|
|
114
|
+
"zip",
|
|
115
|
+
await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, return_folder_path=True)
|
|
116
|
+
)
|
|
117
|
+
return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
|
|
118
|
+
except Exception as e:
|
|
119
|
+
await _cleanup_directory(store_path)
|
|
120
|
+
return VectorDbResponse(success = False, error = str(e))
|
|
121
|
+
finally:
|
|
122
|
+
del documents
|
|
123
|
+
gc.collect()
|
|
124
|
+
else:
|
|
125
|
+
_error = "No documents found in the knowledgebase folder"
|
|
126
|
+
logging.warning(_error)
|
|
127
|
+
return VectorDbResponse(success = False, error = _error)
|
|
128
|
+
|
|
129
|
+
async def kb_stream_file(filename: str):
|
|
130
|
+
file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)
|
|
131
|
+
if not os.path.isfile(file_path):
|
|
132
|
+
raise HTTPException(status_code=404, detail="File not found")
|
|
133
|
+
def iter_file():
|
|
134
|
+
with open(file_path, mode="rb") as file:
|
|
135
|
+
while chunk := file.read(1024*8):
|
|
136
|
+
yield chunk
|
|
137
|
+
return StreamingResponse(iter_file(), media_type="application/octet-stream", headers={"Content-Disposition": f"attachment; filename={filename}"})
|
|
@@ -12,7 +12,7 @@ from ws_bom_robot_app.llm.vector_store.integration.sftp import Sftp
|
|
|
12
12
|
from ws_bom_robot_app.llm.vector_store.integration.sharepoint import Sharepoint
|
|
13
13
|
from ws_bom_robot_app.llm.vector_store.integration.sitemap import Sitemap
|
|
14
14
|
from ws_bom_robot_app.llm.vector_store.integration.slack import Slack
|
|
15
|
-
|
|
15
|
+
from ws_bom_robot_app.llm.vector_store.integration.thron import Thron
|
|
16
16
|
class IntegrationManager:
|
|
17
17
|
_list: dict[str, Type[IntegrationStrategy]] = {
|
|
18
18
|
"llmkbazure": Azure,
|
|
@@ -27,7 +27,7 @@ class IntegrationManager:
|
|
|
27
27
|
"llmkbsharepoint": Sharepoint,
|
|
28
28
|
"llmkbsitemap": Sitemap,
|
|
29
29
|
"llmkbslack": Slack,
|
|
30
|
-
|
|
30
|
+
"llmkbthron": Thron,
|
|
31
31
|
}
|
|
32
32
|
@classmethod
|
|
33
33
|
def get_strategy(cls, name: str, knowledgebase_path: str, data: dict[str, str]) -> IntegrationStrategy:
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import asyncio, logging, aiohttp
|
|
2
|
+
from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
|
|
3
|
+
from unstructured_ingest.v2.processes.connectors.fsspec.sftp import SftpConnectionConfig, SftpAccessConfig, SftpDownloaderConfig, SftpIndexerConfig
|
|
4
|
+
from langchain_core.documents import Document
|
|
5
|
+
from ws_bom_robot_app.llm.vector_store.loader.base import Loader
|
|
6
|
+
from typing import List, Union, Optional
|
|
7
|
+
from pydantic import BaseModel, Field, AliasChoices
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
|
|
11
|
+
class ThronParams(BaseModel):
|
|
12
|
+
"""
|
|
13
|
+
ThronParams is a model that defines the parameters required for Thron integration.
|
|
14
|
+
|
|
15
|
+
Attributes:
|
|
16
|
+
app_id (str): The application ID for Thron.
|
|
17
|
+
client_id (str): The client ID for Thron.
|
|
18
|
+
client_secret (str): The client secret for Thron.
|
|
19
|
+
"""
|
|
20
|
+
organization_name: str = Field(validation_alias=AliasChoices("organizationName","organization_name"))
|
|
21
|
+
attribute_fields: Optional[List[str]] = Field(default=None, validation_alias=AliasChoices("attributeFields","attribute_fields"))
|
|
22
|
+
client_id: str = Field(validation_alias=AliasChoices("clientId","client_id"))
|
|
23
|
+
client_secret: str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
|
|
24
|
+
|
|
25
|
+
class Thron(IntegrationStrategy):
|
|
26
|
+
def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
|
|
27
|
+
super().__init__(knowledgebase_path, data)
|
|
28
|
+
self.__data = ThronParams.model_validate(self.data)
|
|
29
|
+
|
|
30
|
+
def working_subdirectory(self) -> str:
|
|
31
|
+
return 'thron'
|
|
32
|
+
|
|
33
|
+
async def run(self) -> None:
|
|
34
|
+
_data = await self.__get_data()
|
|
35
|
+
transformed_data = self.__transform_data(_data)
|
|
36
|
+
json_file_path = os.path.join(self.working_directory, 'thron_data.json')
|
|
37
|
+
with open(json_file_path, 'w', encoding='utf-8') as f:
|
|
38
|
+
json.dump(transformed_data, f, indent=2, ensure_ascii=False)
|
|
39
|
+
|
|
40
|
+
async def load(self) -> list[Document]:
|
|
41
|
+
await self.run()
|
|
42
|
+
await asyncio.sleep(1)
|
|
43
|
+
return await Loader(self.working_directory).load()
|
|
44
|
+
|
|
45
|
+
async def __get_auth_token(self) -> str:
|
|
46
|
+
try:
|
|
47
|
+
async with aiohttp.ClientSession() as session:
|
|
48
|
+
auth_data = {
|
|
49
|
+
"grant_type": "client_credentials",
|
|
50
|
+
"client_id": self.__data.client_id,
|
|
51
|
+
"client_secret": self.__data.client_secret
|
|
52
|
+
}
|
|
53
|
+
headers = {
|
|
54
|
+
"accept": "application/json",
|
|
55
|
+
"Content-Type": "application/x-www-form-urlencoded"
|
|
56
|
+
}
|
|
57
|
+
async with session.post("https://websolute.thron.com/api/v1/authentication/oauth2/token", data=auth_data, headers=headers) as response:
|
|
58
|
+
result = await response.json()
|
|
59
|
+
return result.get("access_token", "")
|
|
60
|
+
except Exception as e:
|
|
61
|
+
logging.error(f"Error fetching Thron auth token: {e}")
|
|
62
|
+
return None
|
|
63
|
+
|
|
64
|
+
async def __get_data(self) -> dict:
|
|
65
|
+
try:
|
|
66
|
+
token = await self.__get_auth_token()
|
|
67
|
+
if not token:
|
|
68
|
+
logging.error("Failed to obtain Thron authentication token.")
|
|
69
|
+
return {}
|
|
70
|
+
attribute_fields = ",".join(self.__data.attribute_fields) if self.__data.attribute_fields else ""
|
|
71
|
+
async with aiohttp.ClientSession() as session:
|
|
72
|
+
headers = {
|
|
73
|
+
"accept": "application/json",
|
|
74
|
+
"Authorization": f"Bearer {token}"
|
|
75
|
+
}
|
|
76
|
+
async with session.get(f"https://{self.__data.organization_name}.thron.com/api/v1/product-data/products?attributeFields=product_id,{attribute_fields}", headers=headers) as response:
|
|
77
|
+
result = await response.json()
|
|
78
|
+
return result.get("items", {})
|
|
79
|
+
except Exception as e:
|
|
80
|
+
logging.error(f"Error fetching Thron product data: {e}")
|
|
81
|
+
return {}
|
|
82
|
+
return []
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def __transform_data(self, data: dict) -> dict:
|
|
87
|
+
transformed_data = []
|
|
88
|
+
|
|
89
|
+
# First pass: collect all MASTER items
|
|
90
|
+
master_items = {item.get("id"): item for item in data if item.get("hierarchyLevel") == "MASTER"}
|
|
91
|
+
|
|
92
|
+
# Second pass: process items
|
|
93
|
+
for item in data:
|
|
94
|
+
hierarchy_level = item.get("hierarchyLevel")
|
|
95
|
+
|
|
96
|
+
if hierarchy_level == "MASTER":
|
|
97
|
+
# Find matching variants for this master
|
|
98
|
+
master_id = item.get("id")
|
|
99
|
+
item_copy = item.copy()
|
|
100
|
+
item_copy["variants"] = []
|
|
101
|
+
|
|
102
|
+
# Look for variants that belong to this master
|
|
103
|
+
for variant_item in data:
|
|
104
|
+
if (variant_item.get("hierarchyLevel") == "VARIANT" and
|
|
105
|
+
variant_item.get("variation", {}).get("master")):
|
|
106
|
+
|
|
107
|
+
variant_master_id = variant_item.get("variation").get("master").split(":")[-1]
|
|
108
|
+
|
|
109
|
+
# Check if this variant belongs to current master by comparing product_ids
|
|
110
|
+
for attr in item.get("attributes", []):
|
|
111
|
+
if (attr.get("code") == "product_id" and
|
|
112
|
+
attr.get("identifier") == variant_master_id):
|
|
113
|
+
item_copy["variants"].append(variant_item)
|
|
114
|
+
break
|
|
115
|
+
|
|
116
|
+
# Only add master items that have variants
|
|
117
|
+
if item_copy["variants"]:
|
|
118
|
+
transformed_data.append(item_copy)
|
|
119
|
+
|
|
120
|
+
elif hierarchy_level == "SIMPLE":
|
|
121
|
+
transformed_data.append(item)
|
|
122
|
+
|
|
123
|
+
return transformed_data
|
|
@@ -1,25 +1,25 @@
|
|
|
1
|
-
import json
|
|
2
|
-
from typing import Optional
|
|
3
|
-
from langchain_core.documents import Document
|
|
4
|
-
from langchain_community.document_loaders.base import BaseLoader
|
|
5
|
-
|
|
6
|
-
class JsonLoader(BaseLoader):
|
|
7
|
-
def __init__(self, file_path: str, meta_fields:Optional[list[str]] = [],encoding: Optional[str] = "utf-8"):
|
|
8
|
-
self.file_path = file_path
|
|
9
|
-
self.meta_fields = meta_fields
|
|
10
|
-
self.encoding = encoding
|
|
11
|
-
|
|
12
|
-
def load(self) -> list[Document]:
|
|
13
|
-
with open(self.file_path, "r", encoding=self.encoding) as file:
|
|
14
|
-
data = json.load(file)
|
|
15
|
-
_list = data if isinstance(data, list) else [data]
|
|
16
|
-
return [
|
|
17
|
-
Document(
|
|
18
|
-
page_content=json.dumps(item),
|
|
19
|
-
metadata={
|
|
20
|
-
"source": self.file_path,
|
|
21
|
-
**{field: item.get(field) for field in self.meta_fields if item.get(field)}
|
|
22
|
-
}
|
|
23
|
-
)
|
|
24
|
-
for item in _list
|
|
25
|
-
]
|
|
1
|
+
import json
|
|
2
|
+
from typing import Optional
|
|
3
|
+
from langchain_core.documents import Document
|
|
4
|
+
from langchain_community.document_loaders.base import BaseLoader
|
|
5
|
+
|
|
6
|
+
class JsonLoader(BaseLoader):
|
|
7
|
+
def __init__(self, file_path: str, meta_fields:Optional[list[str]] = [],encoding: Optional[str] = "utf-8"):
|
|
8
|
+
self.file_path = file_path
|
|
9
|
+
self.meta_fields = meta_fields
|
|
10
|
+
self.encoding = encoding
|
|
11
|
+
|
|
12
|
+
def load(self) -> list[Document]:
|
|
13
|
+
with open(self.file_path, "r", encoding=self.encoding) as file:
|
|
14
|
+
data = json.load(file)
|
|
15
|
+
_list = data if isinstance(data, list) else [data]
|
|
16
|
+
return [
|
|
17
|
+
Document(
|
|
18
|
+
page_content=json.dumps(item),
|
|
19
|
+
metadata={
|
|
20
|
+
"source": self.file_path,
|
|
21
|
+
**{field: item.get(field) for field in self.meta_fields if item.get(field)}
|
|
22
|
+
}
|
|
23
|
+
)
|
|
24
|
+
for item in _list
|
|
25
|
+
]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ws_bom_robot_app
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.69
|
|
4
4
|
Summary: A FastAPI application serving ws bom/robot/llm platform ai.
|
|
5
5
|
Home-page: https://github.com/websolutespa/bom
|
|
6
6
|
Author: Websolute Spa
|
|
@@ -244,7 +244,7 @@ pytest --cov=ws_bom_robot_app --log-cli-level=info
|
|
|
244
244
|
launch debugger
|
|
245
245
|
|
|
246
246
|
```pwsh
|
|
247
|
-
streamlit run debugger.py --server.port
|
|
247
|
+
streamlit run debugger.py --server.port 8051
|
|
248
248
|
```
|
|
249
249
|
|
|
250
250
|
dockerize base image
|
|
@@ -258,11 +258,17 @@ docker build -f Dockerfile-robot-base-gpu -t ghcr.io/websolutespa/ws-bom-robot-b
|
|
|
258
258
|
docker push ghcr.io/websolutespa/ws-bom-robot-base:gpu
|
|
259
259
|
```
|
|
260
260
|
|
|
261
|
-
dockerize app
|
|
261
|
+
dockerize app
|
|
262
262
|
|
|
263
263
|
```pwsh
|
|
264
|
-
docker build -f Dockerfile
|
|
265
|
-
docker run --name ws-bom-robot-app
|
|
264
|
+
docker build -f Dockerfile -t ws-bom-robot-app .
|
|
265
|
+
docker run --rm --name ws-bom-robot-app -d -p 6001:6001 ws-bom-robot-app
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
docker run mounted to src
|
|
269
|
+
|
|
270
|
+
```pwsh
|
|
271
|
+
docker run --rm --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -v "$(pwd)/tmp:/tmp" -p 6001:6001 ws-bom-robot-app
|
|
266
272
|
```
|
|
267
273
|
|
|
268
274
|
### ✈️ publish
|
|
@@ -1,46 +1,46 @@
|
|
|
1
1
|
ws_bom_robot_app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
2
|
ws_bom_robot_app/auth.py,sha256=84nIbmJsMrNs0sxIQGEHbjsjc2P6ZrZZGSn8dkiL6is,895
|
|
3
|
-
ws_bom_robot_app/config.py,sha256=
|
|
3
|
+
ws_bom_robot_app/config.py,sha256=QvoWds1DkBHqd3jAxDQtcmQSN8B6SrpBGERDXhTlswk,4490
|
|
4
4
|
ws_bom_robot_app/cron_manager.py,sha256=pFHV7SZtp6GRmmLD9K1Mb1TE9Ev9n5mIiFScrc7tpCo,9221
|
|
5
5
|
ws_bom_robot_app/main.py,sha256=1vx0k2fEcE53IC5zcE2EUCwQPcUHM4pvuKSun_E0a9I,6501
|
|
6
6
|
ws_bom_robot_app/task_manager.py,sha256=Q3Il2TtkP0FoG9zHEBu48pZGXzimTtvWQsoH6wdvQs0,16077
|
|
7
7
|
ws_bom_robot_app/util.py,sha256=RjVD6B9sHje788Lndqq5DHy6TJM0KLs9qx3JYt81Wyk,4834
|
|
8
8
|
ws_bom_robot_app/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
9
|
ws_bom_robot_app/llm/agent_context.py,sha256=uatHJ8wcRly6h0S762BgfzDMpmcwCHwNzwo37aWjeE0,1305
|
|
10
|
-
ws_bom_robot_app/llm/agent_description.py,sha256=
|
|
11
|
-
ws_bom_robot_app/llm/agent_handler.py,sha256
|
|
12
|
-
ws_bom_robot_app/llm/agent_lcel.py,sha256=
|
|
10
|
+
ws_bom_robot_app/llm/agent_description.py,sha256=5IP0qFSJvaE3zjGS7f0W1DuiegP0RHXRMBoDC5pCofA,4779
|
|
11
|
+
ws_bom_robot_app/llm/agent_handler.py,sha256=-9ia0bpNXgqLGFCSmAiU5ogdoJo30yl-XUNdXONdvbs,7774
|
|
12
|
+
ws_bom_robot_app/llm/agent_lcel.py,sha256=e1ZELfUEpqMXJwLWxyOtgiUmVwoO3Aw-ua4u05XociA,2297
|
|
13
13
|
ws_bom_robot_app/llm/api.py,sha256=2bF-UFczY9LuBqPxKObM0TOWYbZgVztX1RiIz5MSorU,5042
|
|
14
|
-
ws_bom_robot_app/llm/defaut_prompt.py,sha256=
|
|
15
|
-
ws_bom_robot_app/llm/main.py,sha256=
|
|
16
|
-
ws_bom_robot_app/llm/nebuly_handler.py,sha256=
|
|
17
|
-
ws_bom_robot_app/llm/settings.py,sha256=
|
|
14
|
+
ws_bom_robot_app/llm/defaut_prompt.py,sha256=D9dn8yPveu0bVwGM1wQWLYftmBs5O76o0R_caLLll8w,1121
|
|
15
|
+
ws_bom_robot_app/llm/main.py,sha256=HCmozc4mN9LJ6Dp7bcd2S3O-SuvbtBFElYBEfoKcSyc,5373
|
|
16
|
+
ws_bom_robot_app/llm/nebuly_handler.py,sha256=MV4IqFcKv9lrBEAHYZsMkrYH8gpLNpujRcDji_GsnE8,8081
|
|
17
|
+
ws_bom_robot_app/llm/settings.py,sha256=DCLaGZwxlw0xE46LpfUgin_FHD8_XJIthCgI6r2UDlM,121
|
|
18
18
|
ws_bom_robot_app/llm/feedbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
-
ws_bom_robot_app/llm/feedbacks/feedback_manager.py,sha256=
|
|
19
|
+
ws_bom_robot_app/llm/feedbacks/feedback_manager.py,sha256=vNcZLG9IKhurAk7hjBqyFgQTjnh3Cd4GnxeYsX7ZdiA,2922
|
|
20
20
|
ws_bom_robot_app/llm/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
21
|
ws_bom_robot_app/llm/models/api.py,sha256=rsPH1y-hMkwXnvt5344yIDMY_StbbgB5AS3Gz37bOhs,11036
|
|
22
22
|
ws_bom_robot_app/llm/models/base.py,sha256=1TqxuTK3rjJEALn7lvgoen_1ba3R2brAgGx6EDTtDZo,152
|
|
23
|
-
ws_bom_robot_app/llm/models/feedback.py,sha256=
|
|
23
|
+
ws_bom_robot_app/llm/models/feedback.py,sha256=pYNQGxNOBgeAAfdJLI95l7ePLBI5tVdsgnyjp5oMOQU,1722
|
|
24
24
|
ws_bom_robot_app/llm/models/kb.py,sha256=oVSw6_dmNxikAHrPqcfxDXz9M0ezLIYuxpgvzfs_Now,9514
|
|
25
25
|
ws_bom_robot_app/llm/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
26
|
-
ws_bom_robot_app/llm/providers/llm_manager.py,sha256=
|
|
26
|
+
ws_bom_robot_app/llm/providers/llm_manager.py,sha256=DZKb8pKEbR2uECyzsqr_L825zpAIanuljJgsk17aqUw,11422
|
|
27
27
|
ws_bom_robot_app/llm/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
28
|
-
ws_bom_robot_app/llm/tools/tool_builder.py,sha256=
|
|
29
|
-
ws_bom_robot_app/llm/tools/tool_manager.py,sha256=
|
|
30
|
-
ws_bom_robot_app/llm/tools/utils.py,sha256=
|
|
28
|
+
ws_bom_robot_app/llm/tools/tool_builder.py,sha256=RmZFI36766vxCZD7MsayFaoZGW3FccjPX1hYUcAWiB0,3289
|
|
29
|
+
ws_bom_robot_app/llm/tools/tool_manager.py,sha256=Jbs35gvny3_UdSAErYLbOdhMgo_PLfPEtjLvkEVJGUI,15320
|
|
30
|
+
ws_bom_robot_app/llm/tools/utils.py,sha256=Ba7ScFZPVJ3ke8KLO8ik1wyR2f_zC99Bikqx0OGnKoI,1924
|
|
31
31
|
ws_bom_robot_app/llm/tools/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
32
32
|
ws_bom_robot_app/llm/tools/models/main.py,sha256=1hICqHs-KS2heenkH7b2eH0N2GrPaaNGBrn64cl_A40,827
|
|
33
33
|
ws_bom_robot_app/llm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
34
|
-
ws_bom_robot_app/llm/utils/agent.py,sha256=
|
|
34
|
+
ws_bom_robot_app/llm/utils/agent.py,sha256=uFuSfYMfGIE2WCKGNSKL-T2SDFn-tUKvbAYbGTPIw6g,1445
|
|
35
35
|
ws_bom_robot_app/llm/utils/chunker.py,sha256=N7570xBYlObneg-fsvDhPAJ-Pv8C8OaYZOBK6q7LmMI,607
|
|
36
|
-
ws_bom_robot_app/llm/utils/cms.py,sha256=
|
|
37
|
-
ws_bom_robot_app/llm/utils/download.py,sha256=
|
|
36
|
+
ws_bom_robot_app/llm/utils/cms.py,sha256=X2bu37Lf_vM_8YZJE5o4oH9syxJKWdfq_0bV09J9rmY,6160
|
|
37
|
+
ws_bom_robot_app/llm/utils/download.py,sha256=GaRypPgkx16HfYRj-upX9kvmjfAdFFb5TP4P97scWeA,3273
|
|
38
38
|
ws_bom_robot_app/llm/utils/kb.py,sha256=jja45WCbNI7SGEgqDS99nErlwB5eY8Ga7BMnhdMHZ90,1279
|
|
39
|
-
ws_bom_robot_app/llm/utils/print.py,sha256=
|
|
39
|
+
ws_bom_robot_app/llm/utils/print.py,sha256=HK3zhZOd4cEyXZ8QcudLtTIfqqtMOERce_yTofS8NXo,803
|
|
40
40
|
ws_bom_robot_app/llm/utils/secrets.py,sha256=-HtqLIDVIJrpvGC5YhPAVyLsq8P4ChVM5g3GOfdwqVk,878
|
|
41
41
|
ws_bom_robot_app/llm/utils/webhooks.py,sha256=LAAZqyN6VhV13wu4X-X85TwdDgAV2rNvIwQFIIc0FJM,2114
|
|
42
42
|
ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
43
|
-
ws_bom_robot_app/llm/vector_store/generator.py,sha256=
|
|
43
|
+
ws_bom_robot_app/llm/vector_store/generator.py,sha256=W_hi_UOPaSjnEuazhUFIrMAwTvz64Du8_gpiVAxFlVc,6451
|
|
44
44
|
ws_bom_robot_app/llm/vector_store/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
45
45
|
ws_bom_robot_app/llm/vector_store/db/base.py,sha256=rNIYHPDXhVyoP9AJKRbGT5Vh5HzcKYx8MUIhEuCVGW4,6491
|
|
46
46
|
ws_bom_robot_app/llm/vector_store/db/chroma.py,sha256=3UXR7PZidFxgI5jlC0WWPAJ0NGRI2AqSBVlL9VZOJgw,3356
|
|
@@ -56,17 +56,18 @@ ws_bom_robot_app/llm/vector_store/integration/gcs.py,sha256=fFDVDUR6eNB7FVTzDSEp
|
|
|
56
56
|
ws_bom_robot_app/llm/vector_store/integration/github.py,sha256=18PO30AZcgTn6PHhid3MwImVAdmKBNkr0kmAPgOetGw,2663
|
|
57
57
|
ws_bom_robot_app/llm/vector_store/integration/googledrive.py,sha256=R6hr8iEgrR3QMOzIj5jY6w1x8pZ1LGdh4xM_q7g_ttc,3738
|
|
58
58
|
ws_bom_robot_app/llm/vector_store/integration/jira.py,sha256=Sf6nPongo53vG4jhlzFQZj3foNzvUvAZFr7pXAJfZvM,5795
|
|
59
|
-
ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=
|
|
59
|
+
ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=2te3yYjXHW1n5OWF3-Z3uS18RVudJeLwMjQ6d0Rn_pc,1793
|
|
60
60
|
ws_bom_robot_app/llm/vector_store/integration/s3.py,sha256=3kh-VmH84IW7DdSLvOk6td1VBJ9aohlVJsk5F3cYj0U,3320
|
|
61
61
|
ws_bom_robot_app/llm/vector_store/integration/sftp.py,sha256=WNzjjS1EUykgFB-8e7QkecSa1r1jTJqKyGzR25uJCtM,2848
|
|
62
62
|
ws_bom_robot_app/llm/vector_store/integration/sharepoint.py,sha256=zqqn-6qPrK50Phch4nZHJTgaPyPkGe7W2InGL_Ru6vE,5376
|
|
63
63
|
ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=g0TIRZ2qIpEuVCZ9Bn0MvvxYZtU3wptnTEjoKNZyBAg,5019
|
|
64
64
|
ws_bom_robot_app/llm/vector_store/integration/slack.py,sha256=FMjESXm2QetFXI6i8epze7Kbbu22fV8CVaxb71AHnJ8,2572
|
|
65
|
+
ws_bom_robot_app/llm/vector_store/integration/thron.py,sha256=gBDGYYfe7ZkQpYS2ExF5dZP4HlHoBz03nYNYEvrbR0w,4987
|
|
65
66
|
ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
66
67
|
ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=L_ugekNuAq0N9O-24wtlHSNHkqSeD-KsJrfGt_FX9Oc,5340
|
|
67
68
|
ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=yP0zgXLeFAlByaYuj-6cYariuknckrFds0dxdRcnVz8,3456
|
|
68
|
-
ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=
|
|
69
|
-
ws_bom_robot_app-0.0.
|
|
70
|
-
ws_bom_robot_app-0.0.
|
|
71
|
-
ws_bom_robot_app-0.0.
|
|
72
|
-
ws_bom_robot_app-0.0.
|
|
69
|
+
ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=LDppW0ZATo4_1hh-KlsAM3TLawBvwBxva_a7k5Oz1sc,858
|
|
70
|
+
ws_bom_robot_app-0.0.69.dist-info/METADATA,sha256=bgNnIRmeQtbEFOAbISb6P2QONPfS7gwIol9Oam_BMnA,8571
|
|
71
|
+
ws_bom_robot_app-0.0.69.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
72
|
+
ws_bom_robot_app-0.0.69.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
|
|
73
|
+
ws_bom_robot_app-0.0.69.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|