ws-bom-robot-app 0.0.75__py3-none-any.whl → 0.0.76__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,137 +1,137 @@
1
- import os, gc, shutil, logging, traceback
2
- import asyncio, aiofiles, aiofiles.os
3
- from fastapi import HTTPException
4
- from fastapi.responses import StreamingResponse
5
- from langchain_core.documents import Document
6
- from ws_bom_robot_app.llm.vector_store.loader.base import Loader
7
- from ws_bom_robot_app.llm.models.api import RulesRequest, KbRequest, VectorDbResponse
8
- from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager
9
- from ws_bom_robot_app.llm.vector_store.db.manager import VectorDbManager
10
- from ws_bom_robot_app.config import config
11
- from ws_bom_robot_app.llm.models.kb import load_endpoints
12
- from ws_bom_robot_app.llm.utils.download import download_files
13
-
14
- async def _cleanup_directory(directory_path: str):
15
- if os.path.exists(directory_path):
16
- await asyncio.to_thread(shutil.rmtree, directory_path)
17
-
18
- #@timer
19
- async def rules(rq: RulesRequest) -> VectorDbResponse:
20
- _config = rq.config()
21
- db_name = rq.out_name()
22
- store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
23
- try:
24
- await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(),[Document(page_content=rule, metadata={"source": "rules"}) for rule in rq.rules], store_path) #type: ignore
25
- db_file_path = shutil.make_archive(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
26
- return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
27
- except Exception as e:
28
- try:
29
- await _cleanup_directory(store_path)
30
- finally:
31
- return VectorDbResponse(success = False, error = str(e))
32
- finally:
33
- gc.collect()
34
-
35
- #@atimer
36
- async def kb(rq: KbRequest) -> VectorDbResponse:
37
- os.environ['MPLCONFIGDIR'] = './tmp/.matplotlib'
38
- _config = rq.config()
39
- db_name = rq.out_name()
40
- src_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_src)
41
- working_path = os.path.join(src_path, db_name)
42
-
43
- if all([not rq.files,not rq.endpoints,not rq.integrations]):
44
- return VectorDbResponse(success = False, error = "No files, endpoints or integrations provided")
45
- else:
46
- await aiofiles.os.makedirs(src_path, exist_ok=True)
47
- await aiofiles.os.makedirs(working_path, exist_ok=True)
48
-
49
- documents: list[Document] = []
50
- # Download/copy all files
51
- if rq.files:
52
- try:
53
- loaders = Loader(working_path)
54
- filter_file_extensions = loaders.managed_file_extensions()
55
- files_to_download = [file for file in rq.files if not os.path.exists(os.path.join(src_path, os.path.basename(file)))]
56
- if files_to_download:
57
- await download_files(
58
- [f"{_config.robot_cms_host}/{_config.robot_cms_kb_folder}/{os.path.basename(file)}" for file in files_to_download if any([file.endswith(ext) for ext in filter_file_extensions])],
59
- src_path, authorization=_config.robot_cms_auth)
60
- # copy files to working tmp folder
61
- for file in rq.files:
62
- async with aiofiles.open(os.path.join(src_path, os.path.basename(file)), 'rb') as src_file:
63
- async with aiofiles.open(os.path.join(working_path, os.path.basename(file)), 'wb') as dest_file:
64
- await dest_file.write(await src_file.read())
65
- #load files
66
- try:
67
- documents.extend(await loaders.load())
68
- except Exception as e:
69
- tb = traceback.format_exc()
70
- _error = f"File loader failure: {e} | {tb}"
71
- logging.warning(_error)
72
- return VectorDbResponse(success = False, error = _error)
73
- except Exception as e:
74
- await _cleanup_directory(working_path)
75
- return VectorDbResponse(success = False, error = f"Failed to download file {e}")
76
-
77
- if rq.endpoints:
78
- try:
79
- documents.extend(await load_endpoints(rq.endpoints, working_path))
80
- except Exception as e:
81
- await _cleanup_directory(working_path)
82
- tb = traceback.format_exc()
83
- _error = f"Endpoint failure: {e} | {tb}"
84
- logging.warning(_error)
85
- return VectorDbResponse(success = False, error = _error)
86
-
87
- if rq.integrations:
88
- tasks = []
89
- for integration in rq.integrations:
90
- tasks.append(
91
- IntegrationManager
92
- .get_strategy(integration.type.lower(), working_path, integration.__pydantic_extra__) #type: ignore
93
- .load()
94
- )
95
- try:
96
- integration_documents = await asyncio.gather(*tasks)
97
- for docs in integration_documents:
98
- documents.extend(docs)
99
- except Exception as e:
100
- await _cleanup_directory(working_path)
101
- tb = traceback.format_exc()
102
- _error = f"Integration failure: {e} | {tb}"
103
- logging.warning(_error)
104
- return VectorDbResponse(success=False, error=_error)
105
-
106
- #cleanup
107
- await _cleanup_directory(working_path)
108
-
109
- if documents and len(documents) > 0:
110
- try:
111
- store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
112
- db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
113
- os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
114
- "zip",
115
- await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, return_folder_path=True)
116
- )
117
- return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
118
- except Exception as e:
119
- await _cleanup_directory(store_path)
120
- return VectorDbResponse(success = False, error = str(e))
121
- finally:
122
- del documents
123
- gc.collect()
124
- else:
125
- _error = "No documents found in the knowledgebase folder"
126
- logging.warning(_error)
127
- return VectorDbResponse(success = False, error = _error)
128
-
129
- async def kb_stream_file(filename: str):
130
- file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)
131
- if not os.path.isfile(file_path):
132
- raise HTTPException(status_code=404, detail="File not found")
133
- def iter_file():
134
- with open(file_path, mode="rb") as file:
135
- while chunk := file.read(1024*8):
136
- yield chunk
137
- return StreamingResponse(iter_file(), media_type="application/octet-stream", headers={"Content-Disposition": f"attachment; filename={filename}"})
1
+ import os, gc, shutil, logging, traceback
2
+ import asyncio, aiofiles, aiofiles.os
3
+ from fastapi import HTTPException
4
+ from fastapi.responses import StreamingResponse
5
+ from langchain_core.documents import Document
6
+ from ws_bom_robot_app.llm.vector_store.loader.base import Loader
7
+ from ws_bom_robot_app.llm.models.api import RulesRequest, KbRequest, VectorDbResponse
8
+ from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager
9
+ from ws_bom_robot_app.llm.vector_store.db.manager import VectorDbManager
10
+ from ws_bom_robot_app.config import config
11
+ from ws_bom_robot_app.llm.models.kb import load_endpoints
12
+ from ws_bom_robot_app.llm.utils.download import download_files
13
+
14
+ async def _cleanup_directory(directory_path: str):
15
+ if os.path.exists(directory_path):
16
+ await asyncio.to_thread(shutil.rmtree, directory_path)
17
+
18
+ #@timer
19
+ async def rules(rq: RulesRequest) -> VectorDbResponse:
20
+ _config = rq.config()
21
+ db_name = rq.out_name()
22
+ store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
23
+ try:
24
+ await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(),[Document(page_content=rule, metadata={"source": "rules"}) for rule in rq.rules], store_path) #type: ignore
25
+ db_file_path = shutil.make_archive(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
26
+ return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
27
+ except Exception as e:
28
+ try:
29
+ await _cleanup_directory(store_path)
30
+ finally:
31
+ return VectorDbResponse(success = False, error = str(e))
32
+ finally:
33
+ gc.collect()
34
+
35
+ #@atimer
36
+ async def kb(rq: KbRequest) -> VectorDbResponse:
37
+ os.environ['MPLCONFIGDIR'] = './tmp/.matplotlib'
38
+ _config = rq.config()
39
+ db_name = rq.out_name()
40
+ src_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_src)
41
+ working_path = os.path.join(src_path, db_name)
42
+
43
+ if all([not rq.files,not rq.endpoints,not rq.integrations]):
44
+ return VectorDbResponse(success = False, error = "No files, endpoints or integrations provided")
45
+ else:
46
+ await aiofiles.os.makedirs(src_path, exist_ok=True)
47
+ await aiofiles.os.makedirs(working_path, exist_ok=True)
48
+
49
+ documents: list[Document] = []
50
+ # Download/copy all files
51
+ if rq.files:
52
+ try:
53
+ loaders = Loader(working_path)
54
+ filter_file_extensions = loaders.managed_file_extensions()
55
+ files_to_download = [file for file in rq.files if not os.path.exists(os.path.join(src_path, os.path.basename(file)))]
56
+ if files_to_download:
57
+ await download_files(
58
+ [f"{_config.robot_cms_host}/{_config.robot_cms_kb_folder}/{os.path.basename(file)}" for file in files_to_download if any([file.endswith(ext) for ext in filter_file_extensions])],
59
+ src_path, authorization=_config.robot_cms_auth)
60
+ # copy files to working tmp folder
61
+ for file in rq.files:
62
+ async with aiofiles.open(os.path.join(src_path, os.path.basename(file)), 'rb') as src_file:
63
+ async with aiofiles.open(os.path.join(working_path, os.path.basename(file)), 'wb') as dest_file:
64
+ await dest_file.write(await src_file.read())
65
+ #load files
66
+ try:
67
+ documents.extend(await loaders.load())
68
+ except Exception as e:
69
+ tb = traceback.format_exc()
70
+ _error = f"File loader failure: {e} | {tb}"
71
+ logging.warning(_error)
72
+ return VectorDbResponse(success = False, error = _error)
73
+ except Exception as e:
74
+ await _cleanup_directory(working_path)
75
+ return VectorDbResponse(success = False, error = f"Failed to download file {e}")
76
+
77
+ if rq.endpoints:
78
+ try:
79
+ documents.extend(await load_endpoints(rq.endpoints, working_path))
80
+ except Exception as e:
81
+ await _cleanup_directory(working_path)
82
+ tb = traceback.format_exc()
83
+ _error = f"Endpoint failure: {e} | {tb}"
84
+ logging.warning(_error)
85
+ return VectorDbResponse(success = False, error = _error)
86
+
87
+ if rq.integrations:
88
+ tasks = []
89
+ for integration in rq.integrations:
90
+ tasks.append(
91
+ IntegrationManager
92
+ .get_strategy(integration.type.lower(), working_path, integration.__pydantic_extra__) #type: ignore
93
+ .load()
94
+ )
95
+ try:
96
+ integration_documents = await asyncio.gather(*tasks)
97
+ for docs in integration_documents:
98
+ documents.extend(docs)
99
+ except Exception as e:
100
+ await _cleanup_directory(working_path)
101
+ tb = traceback.format_exc()
102
+ _error = f"Integration failure: {e} | {tb}"
103
+ logging.warning(_error)
104
+ return VectorDbResponse(success=False, error=_error)
105
+
106
+ #cleanup
107
+ await _cleanup_directory(working_path)
108
+
109
+ if documents and len(documents) > 0:
110
+ try:
111
+ store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
112
+ db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
113
+ os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
114
+ "zip",
115
+ await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, return_folder_path=True)
116
+ )
117
+ return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
118
+ except Exception as e:
119
+ await _cleanup_directory(store_path)
120
+ return VectorDbResponse(success = False, error = str(e))
121
+ finally:
122
+ del documents
123
+ gc.collect()
124
+ else:
125
+ _error = "No documents found in the knowledgebase folder"
126
+ logging.warning(_error)
127
+ return VectorDbResponse(success = False, error = _error)
128
+
129
+ async def kb_stream_file(filename: str):
130
+ file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)
131
+ if not os.path.isfile(file_path):
132
+ raise HTTPException(status_code=404, detail="File not found")
133
+ def iter_file():
134
+ with open(file_path, mode="rb") as file:
135
+ while chunk := file.read(1024*8):
136
+ yield chunk
137
+ return StreamingResponse(iter_file(), media_type="application/octet-stream", headers={"Content-Disposition": f"attachment; filename={filename}"})
@@ -1,103 +1,103 @@
1
- import asyncio, logging, aiohttp
2
- from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
3
- from unstructured_ingest.v2.processes.connectors.fsspec.sftp import SftpConnectionConfig, SftpAccessConfig, SftpDownloaderConfig, SftpIndexerConfig
4
- from langchain_core.documents import Document
5
- from ws_bom_robot_app.llm.vector_store.loader.base import Loader
6
- from typing import List, Union, Optional
7
- from pydantic import BaseModel, Field, AliasChoices
8
- import json
9
- import os
10
-
11
- class ThronParams(BaseModel):
12
- """
13
- ThronParams is a model that defines the parameters required for Thron integration.
14
-
15
- Attributes:
16
- app_id (str): The application ID for Thron.
17
- client_id (str): The client ID for Thron.
18
- client_secret (str): The client secret for Thron.
19
- """
20
- organization_name: str = Field(validation_alias=AliasChoices("organizationName","organization_name"))
21
- attribute_fields: Optional[List[str]] = Field(default=None, validation_alias=AliasChoices("attributeFields","attribute_fields"))
22
- client_id: str = Field(validation_alias=AliasChoices("clientId","client_id"))
23
- client_secret: str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
24
-
25
- class Thron(IntegrationStrategy):
26
- def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
27
- super().__init__(knowledgebase_path, data)
28
- self.__data = ThronParams.model_validate(self.data)
29
-
30
- def working_subdirectory(self) -> str:
31
- return 'thron'
32
-
33
- async def run(self) -> None:
34
- _data = await self.__get_data()
35
- transformed_data = self.__transform_data(_data)
36
- json_file_path = os.path.join(self.working_directory, 'thron_data.json')
37
- with open(json_file_path, 'w', encoding='utf-8') as f:
38
- json.dump(transformed_data, f, indent=2, ensure_ascii=False)
39
-
40
- async def load(self) -> list[Document]:
41
- await self.run()
42
- await asyncio.sleep(1)
43
- return await Loader(self.working_directory).load()
44
-
45
- async def __get_auth_token(self) -> str:
46
- try:
47
- async with aiohttp.ClientSession() as session:
48
- auth_data = {
49
- "grant_type": "client_credentials",
50
- "client_id": self.__data.client_id,
51
- "client_secret": self.__data.client_secret
52
- }
53
- headers = {
54
- "accept": "application/json",
55
- "Content-Type": "application/x-www-form-urlencoded"
56
- }
57
- async with session.post("https://websolute.thron.com/api/v1/authentication/oauth2/token", data=auth_data, headers=headers) as response:
58
- result = await response.json()
59
- return result.get("access_token", "")
60
- except Exception as e:
61
- logging.error(f"Error fetching Thron auth token: {e}")
62
- return None
63
-
64
- async def __get_data(self) -> dict:
65
- try:
66
- token = await self.__get_auth_token()
67
- if not token:
68
- logging.error("Failed to obtain Thron authentication token.")
69
- return {}
70
- attribute_fields = ",".join(self.__data.attribute_fields) if self.__data.attribute_fields else ""
71
- async with aiohttp.ClientSession() as session:
72
- headers = {
73
- "accept": "application/json",
74
- "Authorization": f"Bearer {token}"
75
- }
76
- async with session.get(f"https://{self.__data.organization_name}.thron.com/api/v1/product-data/products?attributeFields=product_id,{attribute_fields}", headers=headers) as response:
77
- result = await response.json()
78
- return result.get("items", {})
79
- except Exception as e:
80
- logging.error(f"Error fetching Thron product data: {e}")
81
- return {}
82
- return []
83
-
84
-
85
-
86
- def __transform_data(self, data: dict) -> dict:
87
- _data = []
88
- for item in data:
89
- if item.get("hierarchyLevel") == "MASTER":
90
- # Iterate through variants to find the product_id
91
- for item_variant in data:
92
- if item_variant.get("hierarchyLevel") == "VARIANT":
93
- for attr in item.get("attributes", []):
94
- if attr.get("code") == "product_id" and attr.get("identifier") == item_variant.get("variation").get("master").split(":")[-1]:
95
- # Initialize variants list if it doesn't exist
96
- if "variants" not in item:
97
- item["variants"] = []
98
- item["variants"].append(item_variant)
99
- _data.append(item)
100
- break
101
- elif item.get("hierarchyLevel") == "SIMPLE":
102
- _data.append(item)
103
- return _data
1
+ import asyncio, logging, aiohttp
2
+ from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
3
+ from unstructured_ingest.v2.processes.connectors.fsspec.sftp import SftpConnectionConfig, SftpAccessConfig, SftpDownloaderConfig, SftpIndexerConfig
4
+ from langchain_core.documents import Document
5
+ from ws_bom_robot_app.llm.vector_store.loader.base import Loader
6
+ from typing import List, Union, Optional
7
+ from pydantic import BaseModel, Field, AliasChoices
8
+ import json
9
+ import os
10
+
11
+ class ThronParams(BaseModel):
12
+ """
13
+ ThronParams is a model that defines the parameters required for Thron integration.
14
+
15
+ Attributes:
16
+ app_id (str): The application ID for Thron.
17
+ client_id (str): The client ID for Thron.
18
+ client_secret (str): The client secret for Thron.
19
+ """
20
+ organization_name: str = Field(validation_alias=AliasChoices("organizationName","organization_name"))
21
+ attribute_fields: Optional[List[str]] = Field(default=None, validation_alias=AliasChoices("attributeFields","attribute_fields"))
22
+ client_id: str = Field(validation_alias=AliasChoices("clientId","client_id"))
23
+ client_secret: str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
24
+
25
+ class Thron(IntegrationStrategy):
26
+ def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
27
+ super().__init__(knowledgebase_path, data)
28
+ self.__data = ThronParams.model_validate(self.data)
29
+
30
+ def working_subdirectory(self) -> str:
31
+ return 'thron'
32
+
33
+ async def run(self) -> None:
34
+ _data = await self.__get_data()
35
+ transformed_data = self.__transform_data(_data)
36
+ json_file_path = os.path.join(self.working_directory, 'thron_data.json')
37
+ with open(json_file_path, 'w', encoding='utf-8') as f:
38
+ json.dump(transformed_data, f, indent=2, ensure_ascii=False)
39
+
40
+ async def load(self) -> list[Document]:
41
+ await self.run()
42
+ await asyncio.sleep(1)
43
+ return await Loader(self.working_directory).load()
44
+
45
+ async def __get_auth_token(self) -> str:
46
+ try:
47
+ async with aiohttp.ClientSession() as session:
48
+ auth_data = {
49
+ "grant_type": "client_credentials",
50
+ "client_id": self.__data.client_id,
51
+ "client_secret": self.__data.client_secret
52
+ }
53
+ headers = {
54
+ "accept": "application/json",
55
+ "Content-Type": "application/x-www-form-urlencoded"
56
+ }
57
+ async with session.post("https://websolute.thron.com/api/v1/authentication/oauth2/token", data=auth_data, headers=headers) as response:
58
+ result = await response.json()
59
+ return result.get("access_token", "")
60
+ except Exception as e:
61
+ logging.error(f"Error fetching Thron auth token: {e}")
62
+ return None
63
+
64
+ async def __get_data(self) -> dict:
65
+ try:
66
+ token = await self.__get_auth_token()
67
+ if not token:
68
+ logging.error("Failed to obtain Thron authentication token.")
69
+ return {}
70
+ attribute_fields = ",".join(self.__data.attribute_fields) if self.__data.attribute_fields else ""
71
+ async with aiohttp.ClientSession() as session:
72
+ headers = {
73
+ "accept": "application/json",
74
+ "Authorization": f"Bearer {token}"
75
+ }
76
+ async with session.get(f"https://{self.__data.organization_name}.thron.com/api/v1/product-data/products?attributeFields=product_id,{attribute_fields}", headers=headers) as response:
77
+ result = await response.json()
78
+ return result.get("items", {})
79
+ except Exception as e:
80
+ logging.error(f"Error fetching Thron product data: {e}")
81
+ return {}
82
+ return []
83
+
84
+
85
+
86
+ def __transform_data(self, data: dict) -> dict:
87
+ _data = []
88
+ for item in data:
89
+ if item.get("hierarchyLevel") == "MASTER":
90
+ # Iterate through variants to find the product_id
91
+ for item_variant in data:
92
+ if item_variant.get("hierarchyLevel") == "VARIANT":
93
+ for attr in item.get("attributes", []):
94
+ if attr.get("code") == "product_id" and attr.get("identifier") == item_variant.get("variation").get("master").split(":")[-1]:
95
+ # Initialize variants list if it doesn't exist
96
+ if "variants" not in item:
97
+ item["variants"] = []
98
+ item["variants"].append(item_variant)
99
+ _data.append(item)
100
+ break
101
+ elif item.get("hierarchyLevel") == "SIMPLE":
102
+ _data.append(item)
103
+ return _data
@@ -1,25 +1,25 @@
1
- import json
2
- from typing import Optional
3
- from langchain_core.documents import Document
4
- from langchain_community.document_loaders.base import BaseLoader
5
-
6
- class JsonLoader(BaseLoader):
7
- def __init__(self, file_path: str, meta_fields:Optional[list[str]] = [],encoding: Optional[str] = "utf-8"):
8
- self.file_path = file_path
9
- self.meta_fields = meta_fields
10
- self.encoding = encoding
11
-
12
- def load(self) -> list[Document]:
13
- with open(self.file_path, "r", encoding=self.encoding) as file:
14
- data = json.load(file)
15
- _list = data if isinstance(data, list) else [data]
16
- return [
17
- Document(
18
- page_content=json.dumps(item),
19
- metadata={
20
- "source": self.file_path,
21
- **{field: item.get(field) for field in self.meta_fields if item.get(field)}
22
- }
23
- )
24
- for item in _list
25
- ]
1
+ import json
2
+ from typing import Optional
3
+ from langchain_core.documents import Document
4
+ from langchain_community.document_loaders.base import BaseLoader
5
+
6
+ class JsonLoader(BaseLoader):
7
+ def __init__(self, file_path: str, meta_fields:Optional[list[str]] = [],encoding: Optional[str] = "utf-8"):
8
+ self.file_path = file_path
9
+ self.meta_fields = meta_fields
10
+ self.encoding = encoding
11
+
12
+ def load(self) -> list[Document]:
13
+ with open(self.file_path, "r", encoding=self.encoding) as file:
14
+ data = json.load(file)
15
+ _list = data if isinstance(data, list) else [data]
16
+ return [
17
+ Document(
18
+ page_content=json.dumps(item),
19
+ metadata={
20
+ "source": self.file_path,
21
+ **{field: item.get(field) for field in self.meta_fields if item.get(field)}
22
+ }
23
+ )
24
+ for item in _list
25
+ ]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ws_bom_robot_app
3
- Version: 0.0.75
3
+ Version: 0.0.76
4
4
  Summary: A FastAPI application serving ws bom/robot/llm platform ai.
5
5
  Home-page: https://github.com/websolutespa/bom
6
6
  Author: Websolute Spa
@@ -7,40 +7,40 @@ ws_bom_robot_app/task_manager.py,sha256=Q3Il2TtkP0FoG9zHEBu48pZGXzimTtvWQsoH6wdv
7
7
  ws_bom_robot_app/util.py,sha256=RjVD6B9sHje788Lndqq5DHy6TJM0KLs9qx3JYt81Wyk,4834
8
8
  ws_bom_robot_app/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
9
9
  ws_bom_robot_app/llm/agent_context.py,sha256=uatHJ8wcRly6h0S762BgfzDMpmcwCHwNzwo37aWjeE0,1305
10
- ws_bom_robot_app/llm/agent_description.py,sha256=5IP0qFSJvaE3zjGS7f0W1DuiegP0RHXRMBoDC5pCofA,4779
11
- ws_bom_robot_app/llm/agent_handler.py,sha256=-9ia0bpNXgqLGFCSmAiU5ogdoJo30yl-XUNdXONdvbs,7774
12
- ws_bom_robot_app/llm/agent_lcel.py,sha256=QRgGkdVXCwDXWjJj8R8qaYeLqUfpaYjtRnl3GrZCwVM,2530
10
+ ws_bom_robot_app/llm/agent_description.py,sha256=yK4aVU3RNk1oP4bEneV3QPAi-208JwWk4R6qHlzqYIg,4656
11
+ ws_bom_robot_app/llm/agent_handler.py,sha256=BQ-f--Z5QCJDp-7tzSG_CKrANUCqG65S09psgWVNxa4,7597
12
+ ws_bom_robot_app/llm/agent_lcel.py,sha256=tVa1JJOuL1CG0tXS5AwOB4gli0E2rGqSBD5oEehHvOY,2480
13
13
  ws_bom_robot_app/llm/api.py,sha256=2bF-UFczY9LuBqPxKObM0TOWYbZgVztX1RiIz5MSorU,5042
14
- ws_bom_robot_app/llm/defaut_prompt.py,sha256=D9dn8yPveu0bVwGM1wQWLYftmBs5O76o0R_caLLll8w,1121
15
- ws_bom_robot_app/llm/main.py,sha256=HCmozc4mN9LJ6Dp7bcd2S3O-SuvbtBFElYBEfoKcSyc,5373
16
- ws_bom_robot_app/llm/nebuly_handler.py,sha256=MV4IqFcKv9lrBEAHYZsMkrYH8gpLNpujRcDji_GsnE8,8081
17
- ws_bom_robot_app/llm/settings.py,sha256=DCLaGZwxlw0xE46LpfUgin_FHD8_XJIthCgI6r2UDlM,121
14
+ ws_bom_robot_app/llm/defaut_prompt.py,sha256=LlCd_nSMkMmHESfiiiQYfnJyB6Pp-LSs4CEKdYW4vFk,1106
15
+ ws_bom_robot_app/llm/main.py,sha256=vzUfaLCRk2SYujD00hnrTiHEVLYgZcbSw6LUea43siU,5235
16
+ ws_bom_robot_app/llm/nebuly_handler.py,sha256=d4TI5XbvIYJLCxHcCUa6QUxsgwKW_4ItCYe4ocn7IJo,7900
17
+ ws_bom_robot_app/llm/settings.py,sha256=EkFGCppORenStH9W4e6_dYvQ-5p6xiEMpmUHBqNqG9M,117
18
18
  ws_bom_robot_app/llm/feedbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
19
- ws_bom_robot_app/llm/feedbacks/feedback_manager.py,sha256=vNcZLG9IKhurAk7hjBqyFgQTjnh3Cd4GnxeYsX7ZdiA,2922
19
+ ws_bom_robot_app/llm/feedbacks/feedback_manager.py,sha256=WcKgzlOb8VFG7yqHoIOO_R6LAzdzE4YIRFCVOGBSgfM,2856
20
20
  ws_bom_robot_app/llm/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
21
21
  ws_bom_robot_app/llm/models/api.py,sha256=8hllPUnPM6Rp6TgcvTQtpUh5Nk36r1qcvIEc2O7LZJE,10968
22
22
  ws_bom_robot_app/llm/models/base.py,sha256=1TqxuTK3rjJEALn7lvgoen_1ba3R2brAgGx6EDTtDZo,152
23
- ws_bom_robot_app/llm/models/feedback.py,sha256=pYNQGxNOBgeAAfdJLI95l7ePLBI5tVdsgnyjp5oMOQU,1722
23
+ ws_bom_robot_app/llm/models/feedback.py,sha256=zh1jLqPRLzNlxInkCMoiJbfSu0-tiOEYHM7FhC46PkM,1692
24
24
  ws_bom_robot_app/llm/models/kb.py,sha256=oVSw6_dmNxikAHrPqcfxDXz9M0ezLIYuxpgvzfs_Now,9514
25
25
  ws_bom_robot_app/llm/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
26
- ws_bom_robot_app/llm/providers/llm_manager.py,sha256=kxqzYtgpdmvwA6Ku_GVIkRkLGR2DsLlSnjZGM3_V8zQ,11912
26
+ ws_bom_robot_app/llm/providers/llm_manager.py,sha256=j-AnRh5jZ3eSNOZcmVKO63oBtosXA_blBBrjvJtIkWU,11942
27
27
  ws_bom_robot_app/llm/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
28
- ws_bom_robot_app/llm/tools/tool_builder.py,sha256=RmZFI36766vxCZD7MsayFaoZGW3FccjPX1hYUcAWiB0,3289
29
- ws_bom_robot_app/llm/tools/tool_manager.py,sha256=Jbs35gvny3_UdSAErYLbOdhMgo_PLfPEtjLvkEVJGUI,15320
30
- ws_bom_robot_app/llm/tools/utils.py,sha256=Ba7ScFZPVJ3ke8KLO8ik1wyR2f_zC99Bikqx0OGnKoI,1924
28
+ ws_bom_robot_app/llm/tools/tool_builder.py,sha256=QTRG1c-EnH4APP10IyfZxEkqK9KitUsutXUvDRKeAhU,3224
29
+ ws_bom_robot_app/llm/tools/tool_manager.py,sha256=1IgRXxdB7DU3gbIlfT_aMUWZyWuanFTAFwu3VaYKxfE,14990
30
+ ws_bom_robot_app/llm/tools/utils.py,sha256=tdmOAk8l4HVzw67z3brA9yX-1WLu91paU-WmXHyz4Bg,1883
31
31
  ws_bom_robot_app/llm/tools/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
32
32
  ws_bom_robot_app/llm/tools/models/main.py,sha256=1hICqHs-KS2heenkH7b2eH0N2GrPaaNGBrn64cl_A40,827
33
33
  ws_bom_robot_app/llm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
34
- ws_bom_robot_app/llm/utils/agent.py,sha256=uFuSfYMfGIE2WCKGNSKL-T2SDFn-tUKvbAYbGTPIw6g,1445
34
+ ws_bom_robot_app/llm/utils/agent.py,sha256=_CY5Dji3UeAIi2iuU7ttz4fml1q8aCFgVWOv970x8Fw,1411
35
35
  ws_bom_robot_app/llm/utils/chunker.py,sha256=N7570xBYlObneg-fsvDhPAJ-Pv8C8OaYZOBK6q7LmMI,607
36
- ws_bom_robot_app/llm/utils/cms.py,sha256=5TBDDlTsE4O8_bGvlqFOkkK13WFEoOvYRp_FOEXUuKY,6466
37
- ws_bom_robot_app/llm/utils/download.py,sha256=GaRypPgkx16HfYRj-upX9kvmjfAdFFb5TP4P97scWeA,3273
36
+ ws_bom_robot_app/llm/utils/cms.py,sha256=XhrLQyHQ2JUOInDCCf_uvR4Jiud0YvH2FwwiiuCnnsg,6352
37
+ ws_bom_robot_app/llm/utils/download.py,sha256=iAUxH_NiCpTPtGzhC4hBtxotd2HPFt2MBhttslIxqiI,3194
38
38
  ws_bom_robot_app/llm/utils/kb.py,sha256=jja45WCbNI7SGEgqDS99nErlwB5eY8Ga7BMnhdMHZ90,1279
39
- ws_bom_robot_app/llm/utils/print.py,sha256=HK3zhZOd4cEyXZ8QcudLtTIfqqtMOERce_yTofS8NXo,803
39
+ ws_bom_robot_app/llm/utils/print.py,sha256=IsPYEWRJqu-dqlJA3F9OnnIS4rOq_EYX1Ljp3BvDnww,774
40
40
  ws_bom_robot_app/llm/utils/secrets.py,sha256=-HtqLIDVIJrpvGC5YhPAVyLsq8P4ChVM5g3GOfdwqVk,878
41
41
  ws_bom_robot_app/llm/utils/webhooks.py,sha256=LAAZqyN6VhV13wu4X-X85TwdDgAV2rNvIwQFIIc0FJM,2114
42
42
  ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
43
- ws_bom_robot_app/llm/vector_store/generator.py,sha256=W_hi_UOPaSjnEuazhUFIrMAwTvz64Du8_gpiVAxFlVc,6451
43
+ ws_bom_robot_app/llm/vector_store/generator.py,sha256=9_xdtCKJhmt1OP0GXDjvFERXMP7ozLZT92KuYEBDgC0,6314
44
44
  ws_bom_robot_app/llm/vector_store/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
45
45
  ws_bom_robot_app/llm/vector_store/db/base.py,sha256=t0Z1VCcg604evEzJENGNqYFBi_AZLTEUzmxA5wgoE_A,8419
46
46
  ws_bom_robot_app/llm/vector_store/db/chroma.py,sha256=2riMQvwe2T99X_NtO9yO9lpZ0zj2Nb06l9Hb1lWJ00E,4509
@@ -62,12 +62,12 @@ ws_bom_robot_app/llm/vector_store/integration/sftp.py,sha256=WNzjjS1EUykgFB-8e7Q
62
62
  ws_bom_robot_app/llm/vector_store/integration/sharepoint.py,sha256=zqqn-6qPrK50Phch4nZHJTgaPyPkGe7W2InGL_Ru6vE,5376
63
63
  ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=g0TIRZ2qIpEuVCZ9Bn0MvvxYZtU3wptnTEjoKNZyBAg,5019
64
64
  ws_bom_robot_app/llm/vector_store/integration/slack.py,sha256=FMjESXm2QetFXI6i8epze7Kbbu22fV8CVaxb71AHnJ8,2572
65
- ws_bom_robot_app/llm/vector_store/integration/thron.py,sha256=y_VWuOfOcepXzbWAw3idEuUDuI68E_2_7sy6cBWjiq0,4519
65
+ ws_bom_robot_app/llm/vector_store/integration/thron.py,sha256=AX3Dk6A6TDAEST9VVwhKVB6qr4b1lBXlE-dy_V4Tgu4,4416
66
66
  ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
67
67
  ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=L_ugekNuAq0N9O-24wtlHSNHkqSeD-KsJrfGt_FX9Oc,5340
68
68
  ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=yP0zgXLeFAlByaYuj-6cYariuknckrFds0dxdRcnVz8,3456
69
- ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=LDppW0ZATo4_1hh-KlsAM3TLawBvwBxva_a7k5Oz1sc,858
70
- ws_bom_robot_app-0.0.75.dist-info/METADATA,sha256=Q_CGN6Doa1Go2ya4L9_jfinE26I1bipVxLqSdKG0Eck,8609
71
- ws_bom_robot_app-0.0.75.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
72
- ws_bom_robot_app-0.0.75.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
73
- ws_bom_robot_app-0.0.75.dist-info/RECORD,,
69
+ ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=qo9ejRZyKv_k6jnGgXnu1W5uqsMMtgqK_uvPpZQ0p74,833
70
+ ws_bom_robot_app-0.0.76.dist-info/METADATA,sha256=e6QyzVw9spgo-qPmhDmX-O5oYtDvGN_zJLnhy2gHTuk,8609
71
+ ws_bom_robot_app-0.0.76.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
72
+ ws_bom_robot_app-0.0.76.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
73
+ ws_bom_robot_app-0.0.76.dist-info/RECORD,,