ws-bom-robot-app 0.0.75__py3-none-any.whl → 0.0.77__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,137 +1,137 @@
1
- import os, gc, shutil, logging, traceback
2
- import asyncio, aiofiles, aiofiles.os
3
- from fastapi import HTTPException
4
- from fastapi.responses import StreamingResponse
5
- from langchain_core.documents import Document
6
- from ws_bom_robot_app.llm.vector_store.loader.base import Loader
7
- from ws_bom_robot_app.llm.models.api import RulesRequest, KbRequest, VectorDbResponse
8
- from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager
9
- from ws_bom_robot_app.llm.vector_store.db.manager import VectorDbManager
10
- from ws_bom_robot_app.config import config
11
- from ws_bom_robot_app.llm.models.kb import load_endpoints
12
- from ws_bom_robot_app.llm.utils.download import download_files
13
-
14
- async def _cleanup_directory(directory_path: str):
15
- if os.path.exists(directory_path):
16
- await asyncio.to_thread(shutil.rmtree, directory_path)
17
-
18
- #@timer
19
- async def rules(rq: RulesRequest) -> VectorDbResponse:
20
- _config = rq.config()
21
- db_name = rq.out_name()
22
- store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
23
- try:
24
- await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(),[Document(page_content=rule, metadata={"source": "rules"}) for rule in rq.rules], store_path) #type: ignore
25
- db_file_path = shutil.make_archive(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
26
- return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
27
- except Exception as e:
28
- try:
29
- await _cleanup_directory(store_path)
30
- finally:
31
- return VectorDbResponse(success = False, error = str(e))
32
- finally:
33
- gc.collect()
34
-
35
- #@atimer
36
- async def kb(rq: KbRequest) -> VectorDbResponse:
37
- os.environ['MPLCONFIGDIR'] = './tmp/.matplotlib'
38
- _config = rq.config()
39
- db_name = rq.out_name()
40
- src_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_src)
41
- working_path = os.path.join(src_path, db_name)
42
-
43
- if all([not rq.files,not rq.endpoints,not rq.integrations]):
44
- return VectorDbResponse(success = False, error = "No files, endpoints or integrations provided")
45
- else:
46
- await aiofiles.os.makedirs(src_path, exist_ok=True)
47
- await aiofiles.os.makedirs(working_path, exist_ok=True)
48
-
49
- documents: list[Document] = []
50
- # Download/copy all files
51
- if rq.files:
52
- try:
53
- loaders = Loader(working_path)
54
- filter_file_extensions = loaders.managed_file_extensions()
55
- files_to_download = [file for file in rq.files if not os.path.exists(os.path.join(src_path, os.path.basename(file)))]
56
- if files_to_download:
57
- await download_files(
58
- [f"{_config.robot_cms_host}/{_config.robot_cms_kb_folder}/{os.path.basename(file)}" for file in files_to_download if any([file.endswith(ext) for ext in filter_file_extensions])],
59
- src_path, authorization=_config.robot_cms_auth)
60
- # copy files to working tmp folder
61
- for file in rq.files:
62
- async with aiofiles.open(os.path.join(src_path, os.path.basename(file)), 'rb') as src_file:
63
- async with aiofiles.open(os.path.join(working_path, os.path.basename(file)), 'wb') as dest_file:
64
- await dest_file.write(await src_file.read())
65
- #load files
66
- try:
67
- documents.extend(await loaders.load())
68
- except Exception as e:
69
- tb = traceback.format_exc()
70
- _error = f"File loader failure: {e} | {tb}"
71
- logging.warning(_error)
72
- return VectorDbResponse(success = False, error = _error)
73
- except Exception as e:
74
- await _cleanup_directory(working_path)
75
- return VectorDbResponse(success = False, error = f"Failed to download file {e}")
76
-
77
- if rq.endpoints:
78
- try:
79
- documents.extend(await load_endpoints(rq.endpoints, working_path))
80
- except Exception as e:
81
- await _cleanup_directory(working_path)
82
- tb = traceback.format_exc()
83
- _error = f"Endpoint failure: {e} | {tb}"
84
- logging.warning(_error)
85
- return VectorDbResponse(success = False, error = _error)
86
-
87
- if rq.integrations:
88
- tasks = []
89
- for integration in rq.integrations:
90
- tasks.append(
91
- IntegrationManager
92
- .get_strategy(integration.type.lower(), working_path, integration.__pydantic_extra__) #type: ignore
93
- .load()
94
- )
95
- try:
96
- integration_documents = await asyncio.gather(*tasks)
97
- for docs in integration_documents:
98
- documents.extend(docs)
99
- except Exception as e:
100
- await _cleanup_directory(working_path)
101
- tb = traceback.format_exc()
102
- _error = f"Integration failure: {e} | {tb}"
103
- logging.warning(_error)
104
- return VectorDbResponse(success=False, error=_error)
105
-
106
- #cleanup
107
- await _cleanup_directory(working_path)
108
-
109
- if documents and len(documents) > 0:
110
- try:
111
- store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
112
- db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
113
- os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
114
- "zip",
115
- await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, return_folder_path=True)
116
- )
117
- return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
118
- except Exception as e:
119
- await _cleanup_directory(store_path)
120
- return VectorDbResponse(success = False, error = str(e))
121
- finally:
122
- del documents
123
- gc.collect()
124
- else:
125
- _error = "No documents found in the knowledgebase folder"
126
- logging.warning(_error)
127
- return VectorDbResponse(success = False, error = _error)
128
-
129
- async def kb_stream_file(filename: str):
130
- file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)
131
- if not os.path.isfile(file_path):
132
- raise HTTPException(status_code=404, detail="File not found")
133
- def iter_file():
134
- with open(file_path, mode="rb") as file:
135
- while chunk := file.read(1024*8):
136
- yield chunk
137
- return StreamingResponse(iter_file(), media_type="application/octet-stream", headers={"Content-Disposition": f"attachment; filename={filename}"})
1
+ import os, gc, shutil, logging, traceback
2
+ import asyncio, aiofiles, aiofiles.os
3
+ from fastapi import HTTPException
4
+ from fastapi.responses import StreamingResponse
5
+ from langchain_core.documents import Document
6
+ from ws_bom_robot_app.llm.vector_store.loader.base import Loader
7
+ from ws_bom_robot_app.llm.models.api import RulesRequest, KbRequest, VectorDbResponse
8
+ from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager
9
+ from ws_bom_robot_app.llm.vector_store.db.manager import VectorDbManager
10
+ from ws_bom_robot_app.config import config
11
+ from ws_bom_robot_app.llm.models.kb import load_endpoints
12
+ from ws_bom_robot_app.llm.utils.download import download_files
13
+
14
+ async def _cleanup_directory(directory_path: str):
15
+ if os.path.exists(directory_path):
16
+ await asyncio.to_thread(shutil.rmtree, directory_path)
17
+
18
+ #@timer
19
+ async def rules(rq: RulesRequest) -> VectorDbResponse:
20
+ _config = rq.config()
21
+ db_name = rq.out_name()
22
+ store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
23
+ try:
24
+ await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(),[Document(page_content=rule, metadata={"source": "rules"}) for rule in rq.rules], store_path) #type: ignore
25
+ db_file_path = shutil.make_archive(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
26
+ return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
27
+ except Exception as e:
28
+ try:
29
+ await _cleanup_directory(store_path)
30
+ finally:
31
+ return VectorDbResponse(success = False, error = str(e))
32
+ finally:
33
+ gc.collect()
34
+
35
+ #@atimer
36
+ async def kb(rq: KbRequest) -> VectorDbResponse:
37
+ os.environ['MPLCONFIGDIR'] = './tmp/.matplotlib'
38
+ _config = rq.config()
39
+ db_name = rq.out_name()
40
+ src_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_src)
41
+ working_path = os.path.join(src_path, db_name)
42
+
43
+ if all([not rq.files,not rq.endpoints,not rq.integrations]):
44
+ return VectorDbResponse(success = False, error = "No files, endpoints or integrations provided")
45
+ else:
46
+ await aiofiles.os.makedirs(src_path, exist_ok=True)
47
+ await aiofiles.os.makedirs(working_path, exist_ok=True)
48
+
49
+ documents: list[Document] = []
50
+ # Download/copy all files
51
+ if rq.files:
52
+ try:
53
+ loaders = Loader(working_path)
54
+ filter_file_extensions = loaders.managed_file_extensions()
55
+ files_to_download = [file for file in rq.files if not os.path.exists(os.path.join(src_path, os.path.basename(file)))]
56
+ if files_to_download:
57
+ await download_files(
58
+ [f"{_config.robot_cms_host}/{_config.robot_cms_kb_folder}/{os.path.basename(file)}" for file in files_to_download if any([file.endswith(ext) for ext in filter_file_extensions])],
59
+ src_path, authorization=_config.robot_cms_auth)
60
+ # copy files to working tmp folder
61
+ for file in rq.files:
62
+ async with aiofiles.open(os.path.join(src_path, os.path.basename(file)), 'rb') as src_file:
63
+ async with aiofiles.open(os.path.join(working_path, os.path.basename(file)), 'wb') as dest_file:
64
+ await dest_file.write(await src_file.read())
65
+ #load files
66
+ try:
67
+ documents.extend(await loaders.load())
68
+ except Exception as e:
69
+ tb = traceback.format_exc()
70
+ _error = f"File loader failure: {e} | {tb}"
71
+ logging.warning(_error)
72
+ return VectorDbResponse(success = False, error = _error)
73
+ except Exception as e:
74
+ await _cleanup_directory(working_path)
75
+ return VectorDbResponse(success = False, error = f"Failed to download file {e}")
76
+
77
+ if rq.endpoints:
78
+ try:
79
+ documents.extend(await load_endpoints(rq.endpoints, working_path))
80
+ except Exception as e:
81
+ await _cleanup_directory(working_path)
82
+ tb = traceback.format_exc()
83
+ _error = f"Endpoint failure: {e} | {tb}"
84
+ logging.warning(_error)
85
+ return VectorDbResponse(success = False, error = _error)
86
+
87
+ if rq.integrations:
88
+ tasks = []
89
+ for integration in rq.integrations:
90
+ tasks.append(
91
+ IntegrationManager
92
+ .get_strategy(integration.type.lower(), working_path, integration.__pydantic_extra__) #type: ignore
93
+ .load()
94
+ )
95
+ try:
96
+ integration_documents = await asyncio.gather(*tasks)
97
+ for docs in integration_documents:
98
+ documents.extend(docs)
99
+ except Exception as e:
100
+ await _cleanup_directory(working_path)
101
+ tb = traceback.format_exc()
102
+ _error = f"Integration failure: {e} | {tb}"
103
+ logging.warning(_error)
104
+ return VectorDbResponse(success=False, error=_error)
105
+
106
+ #cleanup
107
+ await _cleanup_directory(working_path)
108
+
109
+ if documents and len(documents) > 0:
110
+ try:
111
+ store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
112
+ db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
113
+ os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
114
+ "zip",
115
+ await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, return_folder_path=True)
116
+ )
117
+ return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
118
+ except Exception as e:
119
+ await _cleanup_directory(store_path)
120
+ return VectorDbResponse(success = False, error = str(e))
121
+ finally:
122
+ del documents
123
+ gc.collect()
124
+ else:
125
+ _error = "No documents found in the knowledgebase folder"
126
+ logging.warning(_error)
127
+ return VectorDbResponse(success = False, error = _error)
128
+
129
+ async def kb_stream_file(filename: str):
130
+ file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)
131
+ if not os.path.isfile(file_path):
132
+ raise HTTPException(status_code=404, detail="File not found")
133
+ def iter_file():
134
+ with open(file_path, mode="rb") as file:
135
+ while chunk := file.read(1024*8):
136
+ yield chunk
137
+ return StreamingResponse(iter_file(), media_type="application/octet-stream", headers={"Content-Disposition": f"attachment; filename={filename}"})
@@ -1,103 +1,103 @@
1
- import asyncio, logging, aiohttp
2
- from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
3
- from unstructured_ingest.v2.processes.connectors.fsspec.sftp import SftpConnectionConfig, SftpAccessConfig, SftpDownloaderConfig, SftpIndexerConfig
4
- from langchain_core.documents import Document
5
- from ws_bom_robot_app.llm.vector_store.loader.base import Loader
6
- from typing import List, Union, Optional
7
- from pydantic import BaseModel, Field, AliasChoices
8
- import json
9
- import os
10
-
11
- class ThronParams(BaseModel):
12
- """
13
- ThronParams is a model that defines the parameters required for Thron integration.
14
-
15
- Attributes:
16
- app_id (str): The application ID for Thron.
17
- client_id (str): The client ID for Thron.
18
- client_secret (str): The client secret for Thron.
19
- """
20
- organization_name: str = Field(validation_alias=AliasChoices("organizationName","organization_name"))
21
- attribute_fields: Optional[List[str]] = Field(default=None, validation_alias=AliasChoices("attributeFields","attribute_fields"))
22
- client_id: str = Field(validation_alias=AliasChoices("clientId","client_id"))
23
- client_secret: str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
24
-
25
- class Thron(IntegrationStrategy):
26
- def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
27
- super().__init__(knowledgebase_path, data)
28
- self.__data = ThronParams.model_validate(self.data)
29
-
30
- def working_subdirectory(self) -> str:
31
- return 'thron'
32
-
33
- async def run(self) -> None:
34
- _data = await self.__get_data()
35
- transformed_data = self.__transform_data(_data)
36
- json_file_path = os.path.join(self.working_directory, 'thron_data.json')
37
- with open(json_file_path, 'w', encoding='utf-8') as f:
38
- json.dump(transformed_data, f, indent=2, ensure_ascii=False)
39
-
40
- async def load(self) -> list[Document]:
41
- await self.run()
42
- await asyncio.sleep(1)
43
- return await Loader(self.working_directory).load()
44
-
45
- async def __get_auth_token(self) -> str:
46
- try:
47
- async with aiohttp.ClientSession() as session:
48
- auth_data = {
49
- "grant_type": "client_credentials",
50
- "client_id": self.__data.client_id,
51
- "client_secret": self.__data.client_secret
52
- }
53
- headers = {
54
- "accept": "application/json",
55
- "Content-Type": "application/x-www-form-urlencoded"
56
- }
57
- async with session.post("https://websolute.thron.com/api/v1/authentication/oauth2/token", data=auth_data, headers=headers) as response:
58
- result = await response.json()
59
- return result.get("access_token", "")
60
- except Exception as e:
61
- logging.error(f"Error fetching Thron auth token: {e}")
62
- return None
63
-
64
- async def __get_data(self) -> dict:
65
- try:
66
- token = await self.__get_auth_token()
67
- if not token:
68
- logging.error("Failed to obtain Thron authentication token.")
69
- return {}
70
- attribute_fields = ",".join(self.__data.attribute_fields) if self.__data.attribute_fields else ""
71
- async with aiohttp.ClientSession() as session:
72
- headers = {
73
- "accept": "application/json",
74
- "Authorization": f"Bearer {token}"
75
- }
76
- async with session.get(f"https://{self.__data.organization_name}.thron.com/api/v1/product-data/products?attributeFields=product_id,{attribute_fields}", headers=headers) as response:
77
- result = await response.json()
78
- return result.get("items", {})
79
- except Exception as e:
80
- logging.error(f"Error fetching Thron product data: {e}")
81
- return {}
82
- return []
83
-
84
-
85
-
86
- def __transform_data(self, data: dict) -> dict:
87
- _data = []
88
- for item in data:
89
- if item.get("hierarchyLevel") == "MASTER":
90
- # Iterate through variants to find the product_id
91
- for item_variant in data:
92
- if item_variant.get("hierarchyLevel") == "VARIANT":
93
- for attr in item.get("attributes", []):
94
- if attr.get("code") == "product_id" and attr.get("identifier") == item_variant.get("variation").get("master").split(":")[-1]:
95
- # Initialize variants list if it doesn't exist
96
- if "variants" not in item:
97
- item["variants"] = []
98
- item["variants"].append(item_variant)
99
- _data.append(item)
100
- break
101
- elif item.get("hierarchyLevel") == "SIMPLE":
102
- _data.append(item)
103
- return _data
1
+ import asyncio, logging, aiohttp
2
+ from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
3
+ from unstructured_ingest.v2.processes.connectors.fsspec.sftp import SftpConnectionConfig, SftpAccessConfig, SftpDownloaderConfig, SftpIndexerConfig
4
+ from langchain_core.documents import Document
5
+ from ws_bom_robot_app.llm.vector_store.loader.base import Loader
6
+ from typing import List, Union, Optional
7
+ from pydantic import BaseModel, Field, AliasChoices
8
+ import json
9
+ import os
10
+
11
+ class ThronParams(BaseModel):
12
+ """
13
+ ThronParams is a model that defines the parameters required for Thron integration.
14
+
15
+ Attributes:
16
+ app_id (str): The application ID for Thron.
17
+ client_id (str): The client ID for Thron.
18
+ client_secret (str): The client secret for Thron.
19
+ """
20
+ organization_name: str = Field(validation_alias=AliasChoices("organizationName","organization_name"))
21
+ attribute_fields: Optional[List[str]] = Field(default=None, validation_alias=AliasChoices("attributeFields","attribute_fields"))
22
+ client_id: str = Field(validation_alias=AliasChoices("clientId","client_id"))
23
+ client_secret: str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
24
+
25
+ class Thron(IntegrationStrategy):
26
+ def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
27
+ super().__init__(knowledgebase_path, data)
28
+ self.__data = ThronParams.model_validate(self.data)
29
+
30
+ def working_subdirectory(self) -> str:
31
+ return 'thron'
32
+
33
+ async def run(self) -> None:
34
+ _data = await self.__get_data()
35
+ transformed_data = self.__transform_data(_data)
36
+ json_file_path = os.path.join(self.working_directory, 'thron_data.json')
37
+ with open(json_file_path, 'w', encoding='utf-8') as f:
38
+ json.dump(transformed_data, f, indent=2, ensure_ascii=False)
39
+
40
+ async def load(self) -> list[Document]:
41
+ await self.run()
42
+ await asyncio.sleep(1)
43
+ return await Loader(self.working_directory).load()
44
+
45
+ async def __get_auth_token(self) -> str:
46
+ try:
47
+ async with aiohttp.ClientSession() as session:
48
+ auth_data = {
49
+ "grant_type": "client_credentials",
50
+ "client_id": self.__data.client_id,
51
+ "client_secret": self.__data.client_secret
52
+ }
53
+ headers = {
54
+ "accept": "application/json",
55
+ "Content-Type": "application/x-www-form-urlencoded"
56
+ }
57
+ async with session.post("https://websolute.thron.com/api/v1/authentication/oauth2/token", data=auth_data, headers=headers) as response:
58
+ result = await response.json()
59
+ return result.get("access_token", "")
60
+ except Exception as e:
61
+ logging.error(f"Error fetching Thron auth token: {e}")
62
+ return None
63
+
64
+ async def __get_data(self) -> dict:
65
+ try:
66
+ token = await self.__get_auth_token()
67
+ if not token:
68
+ logging.error("Failed to obtain Thron authentication token.")
69
+ return {}
70
+ attribute_fields = ",".join(self.__data.attribute_fields) if self.__data.attribute_fields else ""
71
+ async with aiohttp.ClientSession() as session:
72
+ headers = {
73
+ "accept": "application/json",
74
+ "Authorization": f"Bearer {token}"
75
+ }
76
+ async with session.get(f"https://{self.__data.organization_name}.thron.com/api/v1/product-data/products?attributeFields=product_id,{attribute_fields}", headers=headers) as response:
77
+ result = await response.json()
78
+ return result.get("items", {})
79
+ except Exception as e:
80
+ logging.error(f"Error fetching Thron product data: {e}")
81
+ return {}
82
+ return []
83
+
84
+
85
+
86
+ def __transform_data(self, data: dict) -> dict:
87
+ _data = []
88
+ for item in data:
89
+ if item.get("hierarchyLevel") == "MASTER":
90
+ # Iterate through variants to find the product_id
91
+ for item_variant in data:
92
+ if item_variant.get("hierarchyLevel") == "VARIANT":
93
+ for attr in item.get("attributes", []):
94
+ if attr.get("code") == "product_id" and attr.get("identifier") == item_variant.get("variation").get("master").split(":")[-1]:
95
+ # Initialize variants list if it doesn't exist
96
+ if "variants" not in item:
97
+ item["variants"] = []
98
+ item["variants"].append(item_variant)
99
+ _data.append(item)
100
+ break
101
+ elif item.get("hierarchyLevel") == "SIMPLE":
102
+ _data.append(item)
103
+ return _data
@@ -1,25 +1,25 @@
1
- import json
2
- from typing import Optional
3
- from langchain_core.documents import Document
4
- from langchain_community.document_loaders.base import BaseLoader
5
-
6
- class JsonLoader(BaseLoader):
7
- def __init__(self, file_path: str, meta_fields:Optional[list[str]] = [],encoding: Optional[str] = "utf-8"):
8
- self.file_path = file_path
9
- self.meta_fields = meta_fields
10
- self.encoding = encoding
11
-
12
- def load(self) -> list[Document]:
13
- with open(self.file_path, "r", encoding=self.encoding) as file:
14
- data = json.load(file)
15
- _list = data if isinstance(data, list) else [data]
16
- return [
17
- Document(
18
- page_content=json.dumps(item),
19
- metadata={
20
- "source": self.file_path,
21
- **{field: item.get(field) for field in self.meta_fields if item.get(field)}
22
- }
23
- )
24
- for item in _list
25
- ]
1
+ import json
2
+ from typing import Optional
3
+ from langchain_core.documents import Document
4
+ from langchain_community.document_loaders.base import BaseLoader
5
+
6
+ class JsonLoader(BaseLoader):
7
+ def __init__(self, file_path: str, meta_fields:Optional[list[str]] = [],encoding: Optional[str] = "utf-8"):
8
+ self.file_path = file_path
9
+ self.meta_fields = meta_fields
10
+ self.encoding = encoding
11
+
12
+ def load(self) -> list[Document]:
13
+ with open(self.file_path, "r", encoding=self.encoding) as file:
14
+ data = json.load(file)
15
+ _list = data if isinstance(data, list) else [data]
16
+ return [
17
+ Document(
18
+ page_content=json.dumps(item),
19
+ metadata={
20
+ "source": self.file_path,
21
+ **{field: item.get(field) for field in self.meta_fields if item.get(field)}
22
+ }
23
+ )
24
+ for item in _list
25
+ ]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ws_bom_robot_app
3
- Version: 0.0.75
3
+ Version: 0.0.77
4
4
  Summary: A FastAPI application serving ws bom/robot/llm platform ai.
5
5
  Home-page: https://github.com/websolutespa/bom
6
6
  Author: Websolute Spa
@@ -25,7 +25,7 @@ Requires-Dist: langchain-anthropic==0.3.6
25
25
  Requires-Dist: langchain-ibm==0.3.14
26
26
  Requires-Dist: langchain-google-genai==2.0.7
27
27
  Requires-Dist: langchain-google-vertexai==2.0.27
28
- Requires-Dist: langchain-groq==0.3.5
28
+ Requires-Dist: langchain-groq==0.3.6
29
29
  Requires-Dist: langchain-ollama==0.3.3
30
30
  Requires-Dist: faiss-cpu==1.11.0
31
31
  Requires-Dist: chromadb==1.0.15
@@ -7,40 +7,39 @@ ws_bom_robot_app/task_manager.py,sha256=Q3Il2TtkP0FoG9zHEBu48pZGXzimTtvWQsoH6wdv
7
7
  ws_bom_robot_app/util.py,sha256=RjVD6B9sHje788Lndqq5DHy6TJM0KLs9qx3JYt81Wyk,4834
8
8
  ws_bom_robot_app/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
9
9
  ws_bom_robot_app/llm/agent_context.py,sha256=uatHJ8wcRly6h0S762BgfzDMpmcwCHwNzwo37aWjeE0,1305
10
- ws_bom_robot_app/llm/agent_description.py,sha256=5IP0qFSJvaE3zjGS7f0W1DuiegP0RHXRMBoDC5pCofA,4779
11
- ws_bom_robot_app/llm/agent_handler.py,sha256=-9ia0bpNXgqLGFCSmAiU5ogdoJo30yl-XUNdXONdvbs,7774
12
- ws_bom_robot_app/llm/agent_lcel.py,sha256=QRgGkdVXCwDXWjJj8R8qaYeLqUfpaYjtRnl3GrZCwVM,2530
13
- ws_bom_robot_app/llm/api.py,sha256=2bF-UFczY9LuBqPxKObM0TOWYbZgVztX1RiIz5MSorU,5042
14
- ws_bom_robot_app/llm/defaut_prompt.py,sha256=D9dn8yPveu0bVwGM1wQWLYftmBs5O76o0R_caLLll8w,1121
15
- ws_bom_robot_app/llm/main.py,sha256=HCmozc4mN9LJ6Dp7bcd2S3O-SuvbtBFElYBEfoKcSyc,5373
16
- ws_bom_robot_app/llm/nebuly_handler.py,sha256=MV4IqFcKv9lrBEAHYZsMkrYH8gpLNpujRcDji_GsnE8,8081
17
- ws_bom_robot_app/llm/settings.py,sha256=DCLaGZwxlw0xE46LpfUgin_FHD8_XJIthCgI6r2UDlM,121
10
+ ws_bom_robot_app/llm/agent_description.py,sha256=yK4aVU3RNk1oP4bEneV3QPAi-208JwWk4R6qHlzqYIg,4656
11
+ ws_bom_robot_app/llm/agent_handler.py,sha256=TnpfChHLWVQ-gCEHNQPW3UXiuS8AmiP8JYwRz9pqbCg,7203
12
+ ws_bom_robot_app/llm/agent_lcel.py,sha256=tVa1JJOuL1CG0tXS5AwOB4gli0E2rGqSBD5oEehHvOY,2480
13
+ ws_bom_robot_app/llm/api.py,sha256=EUllZaJvtm1CQPxJ6QfufpBhZG_-ew8gSK-vxVg5r58,5054
14
+ ws_bom_robot_app/llm/defaut_prompt.py,sha256=LlCd_nSMkMmHESfiiiQYfnJyB6Pp-LSs4CEKdYW4vFk,1106
15
+ ws_bom_robot_app/llm/main.py,sha256=BXTIfVc9Ck7izZ893qry7C_uz1A8ZupbcHivrZrjpxY,5372
16
+ ws_bom_robot_app/llm/nebuly_handler.py,sha256=GeHcHUBeHy8-rrpCFhQn8WWBKDwMzJ6KtabWb6MYMS0,7916
18
17
  ws_bom_robot_app/llm/feedbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
19
- ws_bom_robot_app/llm/feedbacks/feedback_manager.py,sha256=vNcZLG9IKhurAk7hjBqyFgQTjnh3Cd4GnxeYsX7ZdiA,2922
18
+ ws_bom_robot_app/llm/feedbacks/feedback_manager.py,sha256=WcKgzlOb8VFG7yqHoIOO_R6LAzdzE4YIRFCVOGBSgfM,2856
20
19
  ws_bom_robot_app/llm/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
21
- ws_bom_robot_app/llm/models/api.py,sha256=8hllPUnPM6Rp6TgcvTQtpUh5Nk36r1qcvIEc2O7LZJE,10968
20
+ ws_bom_robot_app/llm/models/api.py,sha256=qr9BO3NqQ4juYcJGARPiKas5Mj2gIg6gUH9MpmDkMLE,11010
22
21
  ws_bom_robot_app/llm/models/base.py,sha256=1TqxuTK3rjJEALn7lvgoen_1ba3R2brAgGx6EDTtDZo,152
23
- ws_bom_robot_app/llm/models/feedback.py,sha256=pYNQGxNOBgeAAfdJLI95l7ePLBI5tVdsgnyjp5oMOQU,1722
22
+ ws_bom_robot_app/llm/models/feedback.py,sha256=zh1jLqPRLzNlxInkCMoiJbfSu0-tiOEYHM7FhC46PkM,1692
24
23
  ws_bom_robot_app/llm/models/kb.py,sha256=oVSw6_dmNxikAHrPqcfxDXz9M0ezLIYuxpgvzfs_Now,9514
25
24
  ws_bom_robot_app/llm/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
26
- ws_bom_robot_app/llm/providers/llm_manager.py,sha256=kxqzYtgpdmvwA6Ku_GVIkRkLGR2DsLlSnjZGM3_V8zQ,11912
25
+ ws_bom_robot_app/llm/providers/llm_manager.py,sha256=j-AnRh5jZ3eSNOZcmVKO63oBtosXA_blBBrjvJtIkWU,11942
27
26
  ws_bom_robot_app/llm/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
28
- ws_bom_robot_app/llm/tools/tool_builder.py,sha256=RmZFI36766vxCZD7MsayFaoZGW3FccjPX1hYUcAWiB0,3289
29
- ws_bom_robot_app/llm/tools/tool_manager.py,sha256=Jbs35gvny3_UdSAErYLbOdhMgo_PLfPEtjLvkEVJGUI,15320
30
- ws_bom_robot_app/llm/tools/utils.py,sha256=Ba7ScFZPVJ3ke8KLO8ik1wyR2f_zC99Bikqx0OGnKoI,1924
27
+ ws_bom_robot_app/llm/tools/tool_builder.py,sha256=QTRG1c-EnH4APP10IyfZxEkqK9KitUsutXUvDRKeAhU,3224
28
+ ws_bom_robot_app/llm/tools/tool_manager.py,sha256=1IgRXxdB7DU3gbIlfT_aMUWZyWuanFTAFwu3VaYKxfE,14990
29
+ ws_bom_robot_app/llm/tools/utils.py,sha256=tdmOAk8l4HVzw67z3brA9yX-1WLu91paU-WmXHyz4Bg,1883
31
30
  ws_bom_robot_app/llm/tools/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
32
31
  ws_bom_robot_app/llm/tools/models/main.py,sha256=1hICqHs-KS2heenkH7b2eH0N2GrPaaNGBrn64cl_A40,827
33
32
  ws_bom_robot_app/llm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
34
- ws_bom_robot_app/llm/utils/agent.py,sha256=uFuSfYMfGIE2WCKGNSKL-T2SDFn-tUKvbAYbGTPIw6g,1445
33
+ ws_bom_robot_app/llm/utils/agent.py,sha256=_CY5Dji3UeAIi2iuU7ttz4fml1q8aCFgVWOv970x8Fw,1411
35
34
  ws_bom_robot_app/llm/utils/chunker.py,sha256=N7570xBYlObneg-fsvDhPAJ-Pv8C8OaYZOBK6q7LmMI,607
36
- ws_bom_robot_app/llm/utils/cms.py,sha256=5TBDDlTsE4O8_bGvlqFOkkK13WFEoOvYRp_FOEXUuKY,6466
37
- ws_bom_robot_app/llm/utils/download.py,sha256=GaRypPgkx16HfYRj-upX9kvmjfAdFFb5TP4P97scWeA,3273
35
+ ws_bom_robot_app/llm/utils/cms.py,sha256=XhrLQyHQ2JUOInDCCf_uvR4Jiud0YvH2FwwiiuCnnsg,6352
36
+ ws_bom_robot_app/llm/utils/download.py,sha256=iAUxH_NiCpTPtGzhC4hBtxotd2HPFt2MBhttslIxqiI,3194
38
37
  ws_bom_robot_app/llm/utils/kb.py,sha256=jja45WCbNI7SGEgqDS99nErlwB5eY8Ga7BMnhdMHZ90,1279
39
- ws_bom_robot_app/llm/utils/print.py,sha256=HK3zhZOd4cEyXZ8QcudLtTIfqqtMOERce_yTofS8NXo,803
38
+ ws_bom_robot_app/llm/utils/print.py,sha256=IsPYEWRJqu-dqlJA3F9OnnIS4rOq_EYX1Ljp3BvDnww,774
40
39
  ws_bom_robot_app/llm/utils/secrets.py,sha256=-HtqLIDVIJrpvGC5YhPAVyLsq8P4ChVM5g3GOfdwqVk,878
41
40
  ws_bom_robot_app/llm/utils/webhooks.py,sha256=LAAZqyN6VhV13wu4X-X85TwdDgAV2rNvIwQFIIc0FJM,2114
42
41
  ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
43
- ws_bom_robot_app/llm/vector_store/generator.py,sha256=W_hi_UOPaSjnEuazhUFIrMAwTvz64Du8_gpiVAxFlVc,6451
42
+ ws_bom_robot_app/llm/vector_store/generator.py,sha256=9_xdtCKJhmt1OP0GXDjvFERXMP7ozLZT92KuYEBDgC0,6314
44
43
  ws_bom_robot_app/llm/vector_store/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
45
44
  ws_bom_robot_app/llm/vector_store/db/base.py,sha256=t0Z1VCcg604evEzJENGNqYFBi_AZLTEUzmxA5wgoE_A,8419
46
45
  ws_bom_robot_app/llm/vector_store/db/chroma.py,sha256=2riMQvwe2T99X_NtO9yO9lpZ0zj2Nb06l9Hb1lWJ00E,4509
@@ -62,12 +61,12 @@ ws_bom_robot_app/llm/vector_store/integration/sftp.py,sha256=WNzjjS1EUykgFB-8e7Q
62
61
  ws_bom_robot_app/llm/vector_store/integration/sharepoint.py,sha256=zqqn-6qPrK50Phch4nZHJTgaPyPkGe7W2InGL_Ru6vE,5376
63
62
  ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=g0TIRZ2qIpEuVCZ9Bn0MvvxYZtU3wptnTEjoKNZyBAg,5019
64
63
  ws_bom_robot_app/llm/vector_store/integration/slack.py,sha256=FMjESXm2QetFXI6i8epze7Kbbu22fV8CVaxb71AHnJ8,2572
65
- ws_bom_robot_app/llm/vector_store/integration/thron.py,sha256=y_VWuOfOcepXzbWAw3idEuUDuI68E_2_7sy6cBWjiq0,4519
64
+ ws_bom_robot_app/llm/vector_store/integration/thron.py,sha256=AX3Dk6A6TDAEST9VVwhKVB6qr4b1lBXlE-dy_V4Tgu4,4416
66
65
  ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
67
66
  ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=L_ugekNuAq0N9O-24wtlHSNHkqSeD-KsJrfGt_FX9Oc,5340
68
67
  ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=yP0zgXLeFAlByaYuj-6cYariuknckrFds0dxdRcnVz8,3456
69
- ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=LDppW0ZATo4_1hh-KlsAM3TLawBvwBxva_a7k5Oz1sc,858
70
- ws_bom_robot_app-0.0.75.dist-info/METADATA,sha256=Q_CGN6Doa1Go2ya4L9_jfinE26I1bipVxLqSdKG0Eck,8609
71
- ws_bom_robot_app-0.0.75.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
72
- ws_bom_robot_app-0.0.75.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
73
- ws_bom_robot_app-0.0.75.dist-info/RECORD,,
68
+ ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=qo9ejRZyKv_k6jnGgXnu1W5uqsMMtgqK_uvPpZQ0p74,833
69
+ ws_bom_robot_app-0.0.77.dist-info/METADATA,sha256=IrMnkVY_PiCFBG6EVSG2bV8VIdvvuflAiBfR3lOYo4Q,8609
70
+ ws_bom_robot_app-0.0.77.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
71
+ ws_bom_robot_app-0.0.77.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
72
+ ws_bom_robot_app-0.0.77.dist-info/RECORD,,
@@ -1,4 +0,0 @@
1
- def init():
2
- """Initialize the chat history list as a global var"""
3
- global chat_history
4
- chat_history = []