ws-bom-robot-app 0.0.105__tar.gz → 0.0.106__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {ws_bom_robot_app-0.0.105/ws_bom_robot_app.egg-info → ws_bom_robot_app-0.0.106}/PKG-INFO +1 -1
  2. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/requirements.txt +55 -55
  3. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/setup.py +1 -1
  4. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/models/api.py +1 -0
  5. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/tools/tool_manager.py +77 -1
  6. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/api.py +216 -216
  7. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106/ws_bom_robot_app.egg-info}/PKG-INFO +1 -1
  8. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/MANIFEST.in +0 -0
  9. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/README.md +0 -0
  10. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/pyproject.toml +0 -0
  11. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/setup.cfg +0 -0
  12. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/__init__.py +0 -0
  13. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/auth.py +0 -0
  14. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/config.py +0 -0
  15. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/cron_manager.py +0 -0
  16. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/__init__.py +0 -0
  17. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/agent_context.py +0 -0
  18. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/agent_description.py +0 -0
  19. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/agent_handler.py +0 -0
  20. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/agent_lcel.py +0 -0
  21. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/api.py +0 -0
  22. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/defaut_prompt.py +0 -0
  23. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/evaluator.py +0 -0
  24. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/feedbacks/__init__.py +0 -0
  25. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/feedbacks/feedback_manager.py +0 -0
  26. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/main.py +0 -0
  27. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/models/__init__.py +0 -0
  28. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/models/base.py +0 -0
  29. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/models/feedback.py +0 -0
  30. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/models/kb.py +0 -0
  31. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/nebuly_handler.py +0 -0
  32. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/providers/__init__.py +0 -0
  33. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/providers/llm_manager.py +0 -0
  34. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/tools/__init__.py +0 -0
  35. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/tools/models/__init__.py +0 -0
  36. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/tools/models/main.py +0 -0
  37. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/tools/tool_builder.py +0 -0
  38. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/tools/utils.py +0 -0
  39. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/utils/__init__.py +0 -0
  40. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/utils/agent.py +0 -0
  41. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/utils/chunker.py +0 -0
  42. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/utils/cleanup.py +0 -0
  43. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/utils/cms.py +0 -0
  44. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/utils/download.py +0 -0
  45. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/utils/print.py +0 -0
  46. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/utils/secrets.py +0 -0
  47. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/utils/webhooks.py +0 -0
  48. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/__init__.py +0 -0
  49. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/db/__init__.py +0 -0
  50. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/db/base.py +0 -0
  51. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/db/chroma.py +0 -0
  52. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/db/faiss.py +0 -0
  53. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/db/manager.py +0 -0
  54. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/db/qdrant.py +0 -0
  55. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/generator.py +0 -0
  56. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/__init__.py +0 -0
  57. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/azure.py +0 -0
  58. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/base.py +0 -0
  59. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/confluence.py +0 -0
  60. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/dropbox.py +0 -0
  61. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/gcs.py +0 -0
  62. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/github.py +0 -0
  63. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/googledrive.py +0 -0
  64. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/jira.py +0 -0
  65. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/manager.py +0 -0
  66. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/s3.py +0 -0
  67. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/sftp.py +0 -0
  68. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/sharepoint.py +0 -0
  69. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/shopify.py +0 -0
  70. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/sitemap.py +0 -0
  71. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/slack.py +0 -0
  72. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/integration/thron.py +0 -0
  73. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/loader/__init__.py +0 -0
  74. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/loader/base.py +0 -0
  75. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/loader/docling.py +0 -0
  76. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/llm/vector_store/loader/json_loader.py +0 -0
  77. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/main.py +0 -0
  78. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/subprocess_runner.py +0 -0
  79. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/task_manager.py +0 -0
  80. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app/util.py +0 -0
  81. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app.egg-info/SOURCES.txt +0 -0
  82. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app.egg-info/dependency_links.txt +0 -0
  83. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app.egg-info/requires.txt +0 -0
  84. {ws_bom_robot_app-0.0.105 → ws_bom_robot_app-0.0.106}/ws_bom_robot_app.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ws_bom_robot_app
3
- Version: 0.0.105
3
+ Version: 0.0.106
4
4
  Summary: A FastAPI application serving ws bom/robot/llm platform ai.
5
5
  Home-page: https://github.com/websolutespa/bom
6
6
  Author: Websolute Spa
@@ -1,55 +1,55 @@
1
- #app
2
- standardwebhooks==1.0.0
3
- apscheduler==3.11.1
4
- aiofiles==25.1.0
5
- pydantic==2.12.4
6
- pydantic-settings==2.12.0
7
- fastapi[standard]==0.121.1
8
- chevron==0.14.0
9
- msoffcrypto-tool==5.4.2
10
-
11
- #framework
12
- langchain==0.3.27
13
- langchain-community==0.3.30
14
- langchain-core==0.3.76
15
- langchain-openai==0.3.33
16
- langchain-anthropic==0.3.21
17
- langchain-ibm==0.3.18
18
- langchain-google-genai==2.1.12
19
- langchain-google-vertexai==2.1.2
20
- langchain-groq==0.3.8
21
- langchain-ollama==0.3.8
22
- openevals==0.1.2 #langsmith evaluation
23
-
24
- #vector DB
25
- faiss-cpu==1.12.0
26
- chromadb==1.3.4
27
- langchain-chroma==0.2.6
28
- langchain-qdrant==0.2.1
29
- qdrant-client[fastembed]==1.15.1
30
- lark==1.3.1 #self-query retriever
31
-
32
- #loaders
33
- unstructured==0.18.18
34
- unstructured[image]
35
- unstructured-ingest==1.2.21
36
- unstructured-ingest[azure]
37
- unstructured-ingest[confluence]
38
- unstructured-ingest[dropbox]
39
- unstructured-ingest[gcs]
40
- unstructured-ingest[github]
41
- unstructured-ingest[google_drive]
42
- unstructured-ingest[jira]
43
- unstructured-ingest[s3]
44
- unstructured-ingest[sftp]
45
- unstructured-ingest[sharepoint]
46
- unstructured-ingest[slack]
47
- html5lib==1.1 #beautifulsoup4 parser
48
-
49
- #integrations
50
- markdownify==1.2.0 #sitemap
51
-
52
- #tools
53
- duckduckgo-search==8.1.1
54
- langchain-google-community==2.0.10
55
- trafilatura==2.0.0
1
+ #app
2
+ standardwebhooks==1.0.0
3
+ apscheduler==3.11.1
4
+ aiofiles==25.1.0
5
+ pydantic==2.12.4
6
+ pydantic-settings==2.12.0
7
+ fastapi[standard]==0.121.1
8
+ chevron==0.14.0
9
+ msoffcrypto-tool==5.4.2
10
+
11
+ #framework
12
+ langchain==0.3.27
13
+ langchain-community==0.3.30
14
+ langchain-core==0.3.76
15
+ langchain-openai==0.3.33
16
+ langchain-anthropic==0.3.21
17
+ langchain-ibm==0.3.18
18
+ langchain-google-genai==2.1.12
19
+ langchain-google-vertexai==2.1.2
20
+ langchain-groq==0.3.8
21
+ langchain-ollama==0.3.8
22
+ openevals==0.1.2 #langsmith evaluation
23
+
24
+ #vector DB
25
+ faiss-cpu==1.12.0
26
+ chromadb==1.3.4
27
+ langchain-chroma==0.2.6
28
+ langchain-qdrant==0.2.1
29
+ qdrant-client[fastembed]==1.15.1
30
+ lark==1.3.1 #self-query retriever
31
+
32
+ #loaders
33
+ unstructured==0.18.18
34
+ unstructured[image]
35
+ unstructured-ingest==1.2.21
36
+ unstructured-ingest[azure]
37
+ unstructured-ingest[confluence]
38
+ unstructured-ingest[dropbox]
39
+ unstructured-ingest[gcs]
40
+ unstructured-ingest[github]
41
+ unstructured-ingest[google_drive]
42
+ unstructured-ingest[jira]
43
+ unstructured-ingest[s3]
44
+ unstructured-ingest[sftp]
45
+ unstructured-ingest[sharepoint]
46
+ unstructured-ingest[slack]
47
+ html5lib==1.1 #beautifulsoup4 parser
48
+
49
+ #integrations
50
+ markdownify==1.2.0 #sitemap
51
+
52
+ #tools
53
+ duckduckgo-search==8.1.1
54
+ langchain-google-community==2.0.10
55
+ trafilatura==2.0.0
@@ -4,7 +4,7 @@ _requirements = [line.split('#')[0].strip() for line in open("requirements.txt")
4
4
 
5
5
  setup(
6
6
  name="ws_bom_robot_app",
7
- version="0.0.105",
7
+ version="0.0.106",
8
8
  description="A FastAPI application serving ws bom/robot/llm platform ai.",
9
9
  long_description=open("README.md", encoding='utf-8').read(),
10
10
  long_description_content_type="text/markdown",
@@ -66,6 +66,7 @@ class LlmAppToolChainSettings(BaseModel):
66
66
 
67
67
  class LlmAppToolDbSettings(BaseModel):
68
68
  connection_string: Optional[str] = Field(None, validation_alias=AliasChoices("connectionString","connection_string"))
69
+ additionalPrompt: Optional[str] = Field(None, validation_alias=AliasChoices("additionalPrompt","additional_prompt"))
69
70
 
70
71
  class LlmAppTool(BaseModel):
71
72
  id: Optional[str] = None
@@ -78,6 +78,80 @@ class ToolManager:
78
78
  source=app_tool.function_id,
79
79
  )
80
80
 
81
+ async def __download_sqlite_file(self, db_uri: str) -> str:
82
+ """
83
+ Scarica il file SQLite dalla CMS se necessario e restituisce il percorso locale.
84
+ Usa la stessa logica dell'integrazione Sitemap.
85
+
86
+ Args:
87
+ db_uri: URI del database o nome del file SQLite
88
+
89
+ Returns:
90
+ str: URI del database locale (sqlite:///path/to/file.db)
91
+ """
92
+ import os
93
+ from ws_bom_robot_app.config import config
94
+ from ws_bom_robot_app.llm.utils.download import download_file
95
+
96
+ if not db_uri.endswith('.db') and not db_uri.endswith('.sqlite') and not db_uri.endswith('.sqlite3'):
97
+ return db_uri
98
+
99
+ if db_uri.startswith('sqlite:///'):
100
+ file_path = db_uri.replace('sqlite:///', '')
101
+ if os.path.isabs(file_path) and os.path.exists(file_path):
102
+ return db_uri
103
+ filename = os.path.basename(file_path)
104
+ else:
105
+ filename = db_uri
106
+
107
+ db_folder = os.path.join(config.robot_data_folder, 'db')
108
+ os.makedirs(db_folder, exist_ok=True)
109
+
110
+ local_db_path = os.path.join(db_folder, filename)
111
+
112
+ if os.path.exists(local_db_path):
113
+ return f"sqlite:///{local_db_path}"
114
+
115
+ cms_file_url = f"{config.robot_cms_host}/{config.robot_cms_kb_folder}/{filename}"
116
+ auth = config.robot_cms_auth
117
+
118
+ try:
119
+ result = await download_file(cms_file_url, local_db_path, authorization=auth)
120
+ if result:
121
+ return f"sqlite:///{local_db_path}"
122
+ else:
123
+ raise ValueError(f"File SQLite {filename} non trovato nella CMS")
124
+ except Exception as e:
125
+ raise ValueError(f"Errore durante il download del file SQLite {filename}: {str(e)}")
126
+
127
+ async def __query_database(self, query: str, app_tool: LlmAppTool):
128
+ from langchain_community.agent_toolkits.sql.base import create_sql_agent
129
+ from langchain_community.utilities import SQLDatabase
130
+
131
+ secrets = app_tool.secrets_to_dict()
132
+
133
+ db_uri = app_tool.db_settings.connection_string
134
+ additional_prompt = app_tool.db_settings.additionalPrompt
135
+ if not db_uri:
136
+ raise ValueError("Database URI not found in tool secrets")
137
+
138
+ db_uri = await self.__download_sqlite_file(db_uri)
139
+
140
+ db = SQLDatabase.from_uri(db_uri)
141
+ llm = self.llm.get_llm()
142
+
143
+ agent = create_sql_agent(
144
+ llm=llm,
145
+ db=db,
146
+ agent_type="tool-calling",
147
+ suffix=additional_prompt if additional_prompt else None,
148
+ )
149
+
150
+ result = await agent.ainvoke({"input": query}, config={"callbacks": []})
151
+ if result and "output" in result:
152
+ return result["output"]
153
+ return None
154
+
81
155
  #region functions
82
156
  async def document_retriever(self, query: str) -> list:
83
157
  """
@@ -99,9 +173,11 @@ class ToolManager:
99
173
  """
100
174
  if (
101
175
  self.app_tool.type == "function" and self.app_tool.vector_db
102
- #and self.settings.get("dataSource") == "knowledgebase"
176
+ and self.app_tool.data_source == "knowledgebase"
103
177
  ):
104
178
  return await self.__extract_documents(query, self.app_tool)
179
+ elif self.app_tool.type == "function" and self.app_tool.data_source == "database":
180
+ return await self.__query_database(query, self.app_tool)
105
181
 
106
182
  async def image_generator(self, query: str, language: str = "it"):
107
183
  """
@@ -1,216 +1,216 @@
1
- import asyncio, logging, aiohttp
2
- from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
3
- from langchain_core.documents import Document
4
- from ws_bom_robot_app.llm.vector_store.loader.base import Loader
5
- from typing import List, Union, Optional, Dict, Any, Literal
6
- from pydantic import BaseModel, Field, AliasChoices, field_validator
7
- import json
8
- import os
9
-
10
-
11
- class AuthConfig(BaseModel):
12
- """
13
- Configuration for API authentication.
14
-
15
- Attributes:
16
- type: Type of authentication (bearer, basic, api_key, custom, none)
17
- token: Bearer token or API key value
18
- username: Username for basic auth
19
- password: Password for basic auth
20
- header_name: Custom header name for API key
21
- prefix: Prefix for the auth value (e.g., 'Bearer', 'Token')
22
- """
23
- type: Literal["bearer", "basic", "api_key", "custom", "none"] = Field(default="none")
24
- token: Optional[str] = Field(default=None)
25
- username: Optional[str] = Field(default=None)
26
- password: Optional[str] = Field(default=None)
27
- header_name: Optional[str] = Field(default=None, validation_alias=AliasChoices("headerName", "header_name"))
28
- prefix: Optional[str] = Field(default=None)
29
-
30
-
31
- class ApiParams(BaseModel):
32
- """
33
- Generic API Integration Parameters.
34
-
35
- Attributes:
36
- url: The base URL of the API endpoint
37
- method: HTTP method (GET, POST, PUT, DELETE, PATCH)
38
- headers: Custom headers to include in the request
39
- params: Query parameters for the request
40
- body: Request body for POST/PUT/PATCH requests
41
- auth: Authentication configuration
42
- response_data_path: JSON path to extract data from response (e.g., 'data.items', 'results')
43
- max_retries: Maximum number of retry attempts for failed requests
44
- retry_delay: Base delay in seconds between retries (uses exponential backoff)
45
- timeout: Request timeout in seconds
46
- """
47
- url: str = Field(validation_alias=AliasChoices("url", "endpoint"))
48
- method: Literal["GET", "POST", "PUT", "DELETE", "PATCH"] = Field(default="GET")
49
- headers: Optional[Dict[str, str]] = Field(default_factory=dict)
50
- params: Optional[Dict[str, Any]] = Field(default_factory=dict)
51
- body: Optional[Union[Dict[str, Any], str]] = Field(default=None)
52
- auth: Optional[AuthConfig] = Field(default_factory=lambda: AuthConfig())
53
- response_data_path: Optional[str] = Field(default=None, validation_alias=AliasChoices("responseDataPath", "response_data_path"))
54
- max_retries: int = Field(default=5, validation_alias=AliasChoices("maxRetries", "max_retries"))
55
- retry_delay: float = Field(default=1.0, validation_alias=AliasChoices("retryDelay", "retry_delay"))
56
- timeout: int = Field(default=30)
57
-
58
- @field_validator('auth', mode='before')
59
- @classmethod
60
- def parse_auth(cls, v):
61
- """Parse auth config from dict if needed"""
62
- if isinstance(v, dict):
63
- return AuthConfig(**v)
64
- return v or AuthConfig()
65
-
66
-
67
- class Api(IntegrationStrategy):
68
- """
69
- Generic API Integration that supports:
70
- - Multiple HTTP methods (GET, POST, PUT, DELETE, PATCH)
71
- - Various authentication types (Bearer, Basic, API Key, Custom)
72
- - Custom headers and parameters
73
- - Automatic retry with exponential backoff
74
- - Flexible response data extraction
75
- """
76
-
77
- def __init__(self, knowledgebase_path: str, data: dict[str, Union[str, int, list]]):
78
- super().__init__(knowledgebase_path, data)
79
- self.__data = ApiParams.model_validate(self.data)
80
-
81
- def working_subdirectory(self) -> str:
82
- return 'api_integration'
83
-
84
- async def run(self) -> None:
85
- """Fetch data from the API and save to JSON file"""
86
- _data = await self.__fetch_data()
87
- json_file_path = os.path.join(self.working_directory, 'api_data.json')
88
- with open(json_file_path, 'w', encoding='utf-8') as f:
89
- json.dump(_data, f, ensure_ascii=False, indent=2)
90
- logging.info(f"Saved {len(_data) if isinstance(_data, list) else 1} items to {json_file_path}")
91
-
92
- async def load(self) -> list[Document]:
93
- """Load data from API and convert to documents"""
94
- await self.run()
95
- await asyncio.sleep(1)
96
- return await Loader(self.working_directory).load()
97
-
98
- def __prepare_headers(self) -> Dict[str, str]:
99
- """Prepare request headers with authentication"""
100
- headers = self.__data.headers.copy() if self.__data.headers else {}
101
-
102
- # Add Content-Type if not present
103
- if 'Content-Type' not in headers and self.__data.method in ["POST", "PUT", "PATCH"]:
104
- headers['Content-Type'] = 'application/json'
105
-
106
- # Add authentication
107
- auth = self.__data.auth
108
- if auth.type == "bearer":
109
- prefix = auth.prefix or "Bearer"
110
- headers['Authorization'] = f"{prefix} {auth.token}"
111
- elif auth.type == "basic":
112
- import base64
113
- credentials = f"{auth.username}:{auth.password}"
114
- encoded = base64.b64encode(credentials.encode()).decode()
115
- headers['Authorization'] = f"Basic {encoded}"
116
- elif auth.type == "api_key" and auth.header_name:
117
- prefix = f"{auth.prefix} " if auth.prefix else ""
118
- headers[auth.header_name] = f"{prefix}{auth.token}"
119
-
120
- return headers
121
-
122
- def __get_nested_value(self, data: Any, path: Optional[str]) -> Any:
123
- """Extract nested value from data using dot notation path"""
124
- if not path:
125
- return data
126
-
127
- keys = path.split('.')
128
- current = data
129
- for key in keys:
130
- if isinstance(current, dict):
131
- current = current.get(key)
132
- elif isinstance(current, list) and key.isdigit():
133
- current = current[int(key)]
134
- else:
135
- return None
136
-
137
- if current is None:
138
- return None
139
-
140
- return current
141
-
142
- async def __make_request(
143
- self,
144
- url: str,
145
- headers: Dict[str, str],
146
- params: Optional[Dict[str, Any]] = None
147
- ) -> Dict[str, Any]:
148
- """Make HTTP request with retry logic"""
149
- retry_count = 0
150
-
151
- while retry_count <= self.__data.max_retries:
152
- try:
153
- timeout = aiohttp.ClientTimeout(total=self.__data.timeout)
154
-
155
- async with aiohttp.ClientSession(timeout=timeout) as session:
156
- request_kwargs = {
157
- "headers": headers,
158
- "params": params or self.__data.params
159
- }
160
-
161
- # Add body for POST/PUT/PATCH
162
- if self.__data.method in ["POST", "PUT", "PATCH"] and self.__data.body:
163
- if isinstance(self.__data.body, dict):
164
- request_kwargs["json"] = self.__data.body
165
- else:
166
- request_kwargs["data"] = self.__data.body
167
-
168
- async with session.request(
169
- self.__data.method,
170
- url,
171
- **request_kwargs
172
- ) as response:
173
- # Check response status
174
- if response.status == 429: # Rate limit
175
- retry_count += 1
176
- if retry_count > self.__data.max_retries:
177
- raise Exception("Rate limit exceeded. Maximum retries reached.")
178
-
179
- wait_time = self.__data.retry_delay * (2 ** retry_count)
180
- logging.warning(f"Rate limited. Waiting {wait_time}s (Attempt {retry_count}/{self.__data.max_retries})")
181
- await asyncio.sleep(wait_time)
182
- continue
183
-
184
- response.raise_for_status()
185
-
186
- # Parse response
187
- try:
188
- data = await response.json()
189
- return data
190
- except aiohttp.ContentTypeError:
191
- text = await response.text()
192
- logging.warning(f"Non-JSON response received: {text[:200]}")
193
- return {"text": text}
194
-
195
- except aiohttp.ClientError as e:
196
- retry_count += 1
197
- if retry_count > self.__data.max_retries:
198
- raise Exception(f"Request failed after {self.__data.max_retries} retries: {e}")
199
-
200
- wait_time = self.__data.retry_delay * (2 ** retry_count)
201
- logging.warning(f"Request error: {e}. Retrying in {wait_time}s...")
202
- await asyncio.sleep(wait_time)
203
- continue
204
-
205
- raise Exception("Maximum retries exceeded")
206
-
207
- async def __fetch_data(self) -> Any:
208
- """Fetch data from API"""
209
- headers = self.__prepare_headers()
210
- response = await self.__make_request(self.__data.url, headers)
211
-
212
- # Extract data from response using path if specified
213
- data = self.__get_nested_value(response, self.__data.response_data_path)
214
- result = data if data is not None else response
215
-
216
- return result
1
+ import asyncio, logging, aiohttp
2
+ from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
3
+ from langchain_core.documents import Document
4
+ from ws_bom_robot_app.llm.vector_store.loader.base import Loader
5
+ from typing import List, Union, Optional, Dict, Any, Literal
6
+ from pydantic import BaseModel, Field, AliasChoices, field_validator
7
+ import json
8
+ import os
9
+
10
+
11
+ class AuthConfig(BaseModel):
12
+ """
13
+ Configuration for API authentication.
14
+
15
+ Attributes:
16
+ type: Type of authentication (bearer, basic, api_key, custom, none)
17
+ token: Bearer token or API key value
18
+ username: Username for basic auth
19
+ password: Password for basic auth
20
+ header_name: Custom header name for API key
21
+ prefix: Prefix for the auth value (e.g., 'Bearer', 'Token')
22
+ """
23
+ type: Literal["bearer", "basic", "api_key", "custom", "none"] = Field(default="none")
24
+ token: Optional[str] = Field(default=None)
25
+ username: Optional[str] = Field(default=None)
26
+ password: Optional[str] = Field(default=None)
27
+ header_name: Optional[str] = Field(default=None, validation_alias=AliasChoices("headerName", "header_name"))
28
+ prefix: Optional[str] = Field(default=None)
29
+
30
+
31
+ class ApiParams(BaseModel):
32
+ """
33
+ Generic API Integration Parameters.
34
+
35
+ Attributes:
36
+ url: The base URL of the API endpoint
37
+ method: HTTP method (GET, POST, PUT, DELETE, PATCH)
38
+ headers: Custom headers to include in the request
39
+ params: Query parameters for the request
40
+ body: Request body for POST/PUT/PATCH requests
41
+ auth: Authentication configuration
42
+ response_data_path: JSON path to extract data from response (e.g., 'data.items', 'results')
43
+ max_retries: Maximum number of retry attempts for failed requests
44
+ retry_delay: Base delay in seconds between retries (uses exponential backoff)
45
+ timeout: Request timeout in seconds
46
+ """
47
+ url: str = Field(validation_alias=AliasChoices("url", "endpoint"))
48
+ method: Literal["GET", "POST", "PUT", "DELETE", "PATCH"] = Field(default="GET")
49
+ headers: Optional[Dict[str, str]] = Field(default_factory=dict)
50
+ params: Optional[Dict[str, Any]] = Field(default_factory=dict)
51
+ body: Optional[Union[Dict[str, Any], str]] = Field(default=None)
52
+ auth: Optional[AuthConfig] = Field(default_factory=lambda: AuthConfig())
53
+ response_data_path: Optional[str] = Field(default=None, validation_alias=AliasChoices("responseDataPath", "response_data_path"))
54
+ max_retries: int = Field(default=5, validation_alias=AliasChoices("maxRetries", "max_retries"))
55
+ retry_delay: float = Field(default=1.0, validation_alias=AliasChoices("retryDelay", "retry_delay"))
56
+ timeout: int = Field(default=30)
57
+
58
+ @field_validator('auth', mode='before')
59
+ @classmethod
60
+ def parse_auth(cls, v):
61
+ """Parse auth config from dict if needed"""
62
+ if isinstance(v, dict):
63
+ return AuthConfig(**v)
64
+ return v or AuthConfig()
65
+
66
+
67
+ class Api(IntegrationStrategy):
68
+ """
69
+ Generic API Integration that supports:
70
+ - Multiple HTTP methods (GET, POST, PUT, DELETE, PATCH)
71
+ - Various authentication types (Bearer, Basic, API Key, Custom)
72
+ - Custom headers and parameters
73
+ - Automatic retry with exponential backoff
74
+ - Flexible response data extraction
75
+ """
76
+
77
+ def __init__(self, knowledgebase_path: str, data: dict[str, Union[str, int, list]]):
78
+ super().__init__(knowledgebase_path, data)
79
+ self.__data = ApiParams.model_validate(self.data)
80
+
81
+ def working_subdirectory(self) -> str:
82
+ return 'api_integration'
83
+
84
+ async def run(self) -> None:
85
+ """Fetch data from the API and save to JSON file"""
86
+ _data = await self.__fetch_data()
87
+ json_file_path = os.path.join(self.working_directory, 'api_data.json')
88
+ with open(json_file_path, 'w', encoding='utf-8') as f:
89
+ json.dump(_data, f, ensure_ascii=False, indent=2)
90
+ logging.info(f"Saved {len(_data) if isinstance(_data, list) else 1} items to {json_file_path}")
91
+
92
+ async def load(self) -> list[Document]:
93
+ """Load data from API and convert to documents"""
94
+ await self.run()
95
+ await asyncio.sleep(1)
96
+ return await Loader(self.working_directory).load()
97
+
98
+ def __prepare_headers(self) -> Dict[str, str]:
99
+ """Prepare request headers with authentication"""
100
+ headers = self.__data.headers.copy() if self.__data.headers else {}
101
+
102
+ # Add Content-Type if not present
103
+ if 'Content-Type' not in headers and self.__data.method in ["POST", "PUT", "PATCH"]:
104
+ headers['Content-Type'] = 'application/json'
105
+
106
+ # Add authentication
107
+ auth = self.__data.auth
108
+ if auth.type == "bearer":
109
+ prefix = auth.prefix or "Bearer"
110
+ headers['Authorization'] = f"{prefix} {auth.token}"
111
+ elif auth.type == "basic":
112
+ import base64
113
+ credentials = f"{auth.username}:{auth.password}"
114
+ encoded = base64.b64encode(credentials.encode()).decode()
115
+ headers['Authorization'] = f"Basic {encoded}"
116
+ elif auth.type == "api_key" and auth.header_name:
117
+ prefix = f"{auth.prefix} " if auth.prefix else ""
118
+ headers[auth.header_name] = f"{prefix}{auth.token}"
119
+
120
+ return headers
121
+
122
+ def __get_nested_value(self, data: Any, path: Optional[str]) -> Any:
123
+ """Extract nested value from data using dot notation path"""
124
+ if not path:
125
+ return data
126
+
127
+ keys = path.split('.')
128
+ current = data
129
+ for key in keys:
130
+ if isinstance(current, dict):
131
+ current = current.get(key)
132
+ elif isinstance(current, list) and key.isdigit():
133
+ current = current[int(key)]
134
+ else:
135
+ return None
136
+
137
+ if current is None:
138
+ return None
139
+
140
+ return current
141
+
142
+ async def __make_request(
143
+ self,
144
+ url: str,
145
+ headers: Dict[str, str],
146
+ params: Optional[Dict[str, Any]] = None
147
+ ) -> Dict[str, Any]:
148
+ """Make HTTP request with retry logic"""
149
+ retry_count = 0
150
+
151
+ while retry_count <= self.__data.max_retries:
152
+ try:
153
+ timeout = aiohttp.ClientTimeout(total=self.__data.timeout)
154
+
155
+ async with aiohttp.ClientSession(timeout=timeout) as session:
156
+ request_kwargs = {
157
+ "headers": headers,
158
+ "params": params or self.__data.params
159
+ }
160
+
161
+ # Add body for POST/PUT/PATCH
162
+ if self.__data.method in ["POST", "PUT", "PATCH"] and self.__data.body:
163
+ if isinstance(self.__data.body, dict):
164
+ request_kwargs["json"] = self.__data.body
165
+ else:
166
+ request_kwargs["data"] = self.__data.body
167
+
168
+ async with session.request(
169
+ self.__data.method,
170
+ url,
171
+ **request_kwargs
172
+ ) as response:
173
+ # Check response status
174
+ if response.status == 429: # Rate limit
175
+ retry_count += 1
176
+ if retry_count > self.__data.max_retries:
177
+ raise Exception("Rate limit exceeded. Maximum retries reached.")
178
+
179
+ wait_time = self.__data.retry_delay * (2 ** retry_count)
180
+ logging.warning(f"Rate limited. Waiting {wait_time}s (Attempt {retry_count}/{self.__data.max_retries})")
181
+ await asyncio.sleep(wait_time)
182
+ continue
183
+
184
+ response.raise_for_status()
185
+
186
+ # Parse response
187
+ try:
188
+ data = await response.json()
189
+ return data
190
+ except aiohttp.ContentTypeError:
191
+ text = await response.text()
192
+ logging.warning(f"Non-JSON response received: {text[:200]}")
193
+ return {"text": text}
194
+
195
+ except aiohttp.ClientError as e:
196
+ retry_count += 1
197
+ if retry_count > self.__data.max_retries:
198
+ raise Exception(f"Request failed after {self.__data.max_retries} retries: {e}")
199
+
200
+ wait_time = self.__data.retry_delay * (2 ** retry_count)
201
+ logging.warning(f"Request error: {e}. Retrying in {wait_time}s...")
202
+ await asyncio.sleep(wait_time)
203
+ continue
204
+
205
+ raise Exception("Maximum retries exceeded")
206
+
207
+ async def __fetch_data(self) -> Any:
208
+ """Fetch data from API"""
209
+ headers = self.__prepare_headers()
210
+ response = await self.__make_request(self.__data.url, headers)
211
+
212
+ # Extract data from response using path if specified
213
+ data = self.__get_nested_value(response, self.__data.response_data_path)
214
+ result = data if data is not None else response
215
+
216
+ return result
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ws_bom_robot_app
3
- Version: 0.0.105
3
+ Version: 0.0.106
4
4
  Summary: A FastAPI application serving ws bom/robot/llm platform ai.
5
5
  Home-page: https://github.com/websolutespa/bom
6
6
  Author: Websolute Spa