ws-bom-robot-app 0.0.15__tar.gz → 0.0.17__tar.gz

{ws_bom_robot_app-0.0.15 → ws_bom_robot_app-0.0.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.15
+Version: 0.0.17
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -13,8 +13,8 @@ Description-Content-Type: text/markdown
 Requires-Dist: standardwebhooks==1.0.0
 Requires-Dist: apscheduler==3.11.0
 Requires-Dist: aiofiles==24.1.0
-Requires-Dist: pydantic==2.9.2
-Requires-Dist: pydantic-settings==2.6.0
+Requires-Dist: pydantic==2.10.3
+Requires-Dist: pydantic-settings==2.6.1
 Requires-Dist: fastapi[standard]==0.115.5
 Requires-Dist: langchain==0.3.9
 Requires-Dist: langchain-openai==0.2.10
@@ -23,7 +23,6 @@ Requires-Dist: langchain-core==0.3.21
 Requires-Dist: faiss-cpu==1.9.0
 Requires-Dist: python-magic==0.4.27
 Requires-Dist: opencv-python-headless==4.10.0.84
-Requires-Dist: jmespath==1.0.1
 Requires-Dist: unstructured[all-docs]==0.15.14
 Requires-Dist: langchain_unstructured==0.1.5
 Requires-Dist: html5lib==1.1
@@ -174,7 +173,31 @@ py -m pip install --upgrade -e .
 py -m pip show ws-bom-robot-app
 ```
 
-launch the debugger
+code quality tools
+  
+```pwsh
+# .\src\robot
+!py -m pip install -U scanreq prospector[with_everything]
+## unused requirements
+scanreq -r requirements.txt -p ./ws_bom_robot_app
+## style/linting
+prospector ./ws_bom_robot_app -t pylint -t pydocstyle
+## code quality/complexity
+prospector ./ws_bom_robot_app -t vulture -t mccabe -t mypy 
+## security
+prospector ./ws_bom_robot_app -t dodgy -t bandit
+## package
+prospector ./ws_bom_robot_app -t pyroma
+```
+
+lauch pytest
+
+```pwsh
+!py -m pip install -U pytest pytest-asyncio pytest-mock pytest-cov
+pytest --cov=ws_bom_robot_app --log-cli-level=info
+```
+
+launch debugger
 
 ```pwsh
 streamlit run debugger.py --server.port 6002

{ws_bom_robot_app-0.0.15 → ws_bom_robot_app-0.0.17}/README.md

@@ -142,7 +142,31 @@ py -m pip install --upgrade -e .
 py -m pip show ws-bom-robot-app
 ```
 
-launch the debugger
+code quality tools
+  
+```pwsh
+# .\src\robot
+!py -m pip install -U scanreq prospector[with_everything]
+## unused requirements
+scanreq -r requirements.txt -p ./ws_bom_robot_app
+## style/linting
+prospector ./ws_bom_robot_app -t pylint -t pydocstyle
+## code quality/complexity
+prospector ./ws_bom_robot_app -t vulture -t mccabe -t mypy 
+## security
+prospector ./ws_bom_robot_app -t dodgy -t bandit
+## package
+prospector ./ws_bom_robot_app -t pyroma
+```
+
+lauch pytest
+
+```pwsh
+!py -m pip install -U pytest pytest-asyncio pytest-mock pytest-cov
+pytest --cov=ws_bom_robot_app --log-cli-level=info
+```
+
+launch debugger
 
 ```pwsh
 streamlit run debugger.py --server.port 6002

{ws_bom_robot_app-0.0.15 → ws_bom_robot_app-0.0.17}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.15",
+    version="0.0.17",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

ws_bom_robot_app-0.0.17/ws_bom_robot_app/llm/vector_store/integration/base.py

@@ -0,0 +1,44 @@
+import os
+from langchain_core.documents import Document
+from abc import ABC, abstractmethod
+from unstructured_ingest.interfaces import PartitionConfig,  ProcessorConfig, ReadConfig, RetryStrategyConfig
+from typing import Union
+
+class IntegrationStrategy(ABC):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    self.knowledgebase_path = knowledgebase_path
+    self.data = data
+    self.working_directory = os.path.join(self.knowledgebase_path,self.working_subdirectory())
+    os.makedirs(self.working_directory, exist_ok=True)
+  @property
+  @abstractmethod
+  def working_subdirectory(self) -> str:
+    pass
+  @abstractmethod
+  #@timer
+  def load(self) -> list[Document]:
+    pass
+
+class UnstructuredIngest():
+  def __init__(self, working_directory: str):
+    self.working_directory = working_directory
+  def processor_config(self) -> ProcessorConfig:
+    return ProcessorConfig(
+      reprocess=False,
+      verbose=False,
+      #work_dir=os.path.join(self.working_directory,'.__work_dir'),
+      #output_dir=self.working_directory,
+      num_processes=1,
+      raise_on_error=False
+    )
+  def read_config(self) -> ReadConfig:
+    return ReadConfig(
+      download_dir=self.working_directory,
+      re_download=True,
+      preserve_downloads=False,
+      download_only=True
+    )
+  def partition_config(self) -> PartitionConfig:
+    return None
+  def retry_strategy_config(self) -> RetryStrategyConfig:
+    return None

ws_bom_robot_app-0.0.17/ws_bom_robot_app/llm/vector_store/integration/confluence.py

@@ -0,0 +1,47 @@
+import asyncio
+import os
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.connector.confluence import SimpleConfluenceConfig, ConfluenceAccessConfig
+from unstructured_ingest.runner import ConfluenceRunner
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import Optional, Union
+from pydantic import BaseModel, Field, AliasChoices
+
+class ConfluenceParams(BaseModel):
+  url: str
+  access_token: str = Field(validation_alias=AliasChoices("accessToken","access_token"))
+  user_email: str = Field(validation_alias=AliasChoices("userEmail","user_email"))
+  spaces: list[str] = []
+class Confluence(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = ConfluenceParams.model_validate(self.data)
+    self.__loader = Loader(self.working_directory)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+  def working_subdirectory(self) -> str:
+    return 'confluence'
+  def run(self) -> None:
+    access_config = ConfluenceAccessConfig(
+      api_token=self.__data.access_token
+    )
+    config = SimpleConfluenceConfig(
+      user_email=self.__data.user_email,
+      url = self.__data.url,
+      access_config=access_config,
+      #max_num_of_spaces=self.data.get('max_num_of_spaces',500),
+      #max_num_of_docs_from_each_space=self.data.get('max_num_of_docs_from_each_space',100),
+      spaces=self.__data.spaces
+    )
+    runner = ConfluenceRunner(
+      connector_config=config,
+      processor_config=self.__unstructured_ingest.processor_config(),
+      read_config=self.__unstructured_ingest.read_config(),
+      partition_config=self.__unstructured_ingest.partition_config(),
+      retry_strategy_config=self.__unstructured_ingest.retry_strategy_config()
+      )
+    runner.run()
+  async def load(self) -> list[Document]:
+      self.run()
+      await asyncio.sleep(1)
+      return await self.__loader.load()

ws_bom_robot_app-0.0.17/ws_bom_robot_app/llm/vector_store/integration/github.py

@@ -0,0 +1,48 @@
+import asyncio
+import os
+from typing import Optional, Union
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.connector.git import GitAccessConfig
+from unstructured_ingest.connector.github import SimpleGitHubConfig
+from unstructured_ingest.runner import GithubRunner
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from pydantic import BaseModel, Field, AliasChoices
+
+class GithubParams(BaseModel):
+  repo: str
+  access_token: Optional[str] | None = Field(None,validation_alias=AliasChoices("accessToken","access_token"))
+  branch: Optional[str] = 'main'
+  file_ext: Optional[list[str]] = Field(default_factory=list, validation_alias=AliasChoices("fileExt","file_ext"))
+class Github(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = GithubParams.model_validate(self.data)
+    self.__loader = Loader(self.working_directory)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+  def working_subdirectory(self) -> str:
+    return 'github'
+  def run(self) -> None:
+    access_config = GitAccessConfig(
+      access_token=self.__data.access_token
+    )
+    file_ext = self.__data.file_ext or None
+    file_glob = [f"**/*{ext}" for ext in file_ext] if file_ext else None
+    config = SimpleGitHubConfig(
+      url = self.__data.repo,
+      access_config=access_config,
+      branch=self.__data.branch,
+      file_glob=file_glob
+    )
+    runner = GithubRunner(
+      connector_config=config,
+      processor_config=self.__unstructured_ingest.processor_config(),
+      read_config=self.__unstructured_ingest.read_config(),
+      partition_config=self.__unstructured_ingest.partition_config(),
+      retry_strategy_config=self.__unstructured_ingest.retry_strategy_config()
+      )
+    runner.run()
+  async def load(self) -> list[Document]:
+      self.run()
+      await asyncio.sleep(1)
+      return await self.__loader.load()

ws_bom_robot_app-0.0.17/ws_bom_robot_app/llm/vector_store/integration/jira.py

@@ -0,0 +1,49 @@
+import asyncio
+import os
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.connector.jira import SimpleJiraConfig, JiraAccessConfig
+from unstructured_ingest.runner import JiraRunner
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from pydantic import BaseModel, Field, AliasChoices
+from typing import Optional, Union
+
+class JiraParams(BaseModel):
+  url: str
+  access_token: str = Field(validation_alias=AliasChoices("accessToken","access_token"))
+  user_email: str = Field(validation_alias=AliasChoices("userEmail","user_email"))
+  projects: list[str]
+  boards: Optional[list[str]] | None = None
+  issues: Optional[list[str]] | None = None
+class Jira(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = JiraParams.model_validate(self.data)
+    self.__loader = Loader(self.working_directory)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+  def working_subdirectory(self) -> str:
+    return 'jira'
+  def run(self) -> None:
+    access_config = JiraAccessConfig(
+      api_token=self.__data.access_token
+    )
+    config = SimpleJiraConfig(
+      user_email=self.__data.user_email,
+      url = self.__data.url,
+      access_config=access_config,
+      projects=self.__data.projects,
+      boards=self.__data.boards,
+      issues=self.__data.issues
+    )
+    runner = JiraRunner(
+      connector_config=config,
+      processor_config=self.__unstructured_ingest.processor_config(),
+      read_config=self.__unstructured_ingest.read_config(),
+      partition_config=self.__unstructured_ingest.partition_config(),
+      retry_strategy_config=self.__unstructured_ingest.retry_strategy_config()
+      )
+    runner.run()
+  async def load(self) -> list[Document]:
+      self.run()
+      await asyncio.sleep(1)
+      return await self.__loader.load()

{ws_bom_robot_app-0.0.15 → ws_bom_robot_app-0.0.17}/ws_bom_robot_app/llm/vector_store/integration/manager.py

@@ -1,10 +1,16 @@
 from typing import Type
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
+from ws_bom_robot_app.llm.vector_store.integration.confluence import Confluence
+from ws_bom_robot_app.llm.vector_store.integration.github import Github
+from ws_bom_robot_app.llm.vector_store.integration.jira import Jira
 from ws_bom_robot_app.llm.vector_store.integration.sitemap import Sitemap
 
 class IntegrationManager:
   _list: dict[str, Type[IntegrationStrategy]] = {
     "llmkbsitemap": Sitemap,
+    "llmkbgithub": Github,
+    "llmkbjira": Jira,
+    "llmkbconfluence": Confluence,
   }
   @classmethod
   def get_strategy(cls, name: str, knowledgebase_path: str, data: dict[str, str]) -> IntegrationStrategy:

{ws_bom_robot_app-0.0.15 → ws_bom_robot_app-0.0.17}/ws_bom_robot_app/requirements.txt

@@ -2,8 +2,8 @@
 standardwebhooks==1.0.0
 apscheduler==3.11.0
 aiofiles==24.1.0
-pydantic==2.9.2
-pydantic-settings==2.6.0
+pydantic==2.10.3
+pydantic-settings==2.6.1
 fastapi[standard]==0.115.5
 
 #framework
@@ -20,7 +20,6 @@ faiss-cpu==1.9.0
 #loaders
 python-magic==0.4.27
 opencv-python-headless==4.10.0.84 #docker specs
-jmespath==1.0.1 #tmp fix to pre-extract from native pkg due to issues with wheel
 unstructured[all-docs]==0.15.14
 langchain_unstructured==0.1.5
 html5lib==1.1 #beautifulsoup4 parser

{ws_bom_robot_app-0.0.15 → ws_bom_robot_app-0.0.17}/ws_bom_robot_app.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.15
+Version: 0.0.17
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -13,8 +13,8 @@ Description-Content-Type: text/markdown
 Requires-Dist: standardwebhooks==1.0.0
 Requires-Dist: apscheduler==3.11.0
 Requires-Dist: aiofiles==24.1.0
-Requires-Dist: pydantic==2.9.2
-Requires-Dist: pydantic-settings==2.6.0
+Requires-Dist: pydantic==2.10.3
+Requires-Dist: pydantic-settings==2.6.1
 Requires-Dist: fastapi[standard]==0.115.5
 Requires-Dist: langchain==0.3.9
 Requires-Dist: langchain-openai==0.2.10
@@ -23,7 +23,6 @@ Requires-Dist: langchain-core==0.3.21
 Requires-Dist: faiss-cpu==1.9.0
 Requires-Dist: python-magic==0.4.27
 Requires-Dist: opencv-python-headless==4.10.0.84
-Requires-Dist: jmespath==1.0.1
 Requires-Dist: unstructured[all-docs]==0.15.14
 Requires-Dist: langchain_unstructured==0.1.5
 Requires-Dist: html5lib==1.1
@@ -174,7 +173,31 @@ py -m pip install --upgrade -e .
 py -m pip show ws-bom-robot-app
 ```
 
-launch the debugger
+code quality tools
+  
+```pwsh
+# .\src\robot
+!py -m pip install -U scanreq prospector[with_everything]
+## unused requirements
+scanreq -r requirements.txt -p ./ws_bom_robot_app
+## style/linting
+prospector ./ws_bom_robot_app -t pylint -t pydocstyle
+## code quality/complexity
+prospector ./ws_bom_robot_app -t vulture -t mccabe -t mypy 
+## security
+prospector ./ws_bom_robot_app -t dodgy -t bandit
+## package
+prospector ./ws_bom_robot_app -t pyroma
+```
+
+lauch pytest
+
+```pwsh
+!py -m pip install -U pytest pytest-asyncio pytest-mock pytest-cov
+pytest --cov=ws_bom_robot_app --log-cli-level=info
+```
+
+launch debugger
 
 ```pwsh
 streamlit run debugger.py --server.port 6002

{ws_bom_robot_app-0.0.15 → ws_bom_robot_app-0.0.17}/ws_bom_robot_app.egg-info/SOURCES.txt

@@ -43,6 +43,9 @@ ws_bom_robot_app/llm/vector_store/__init__.py
 ws_bom_robot_app/llm/vector_store/generator.py
 ws_bom_robot_app/llm/vector_store/integration/__init__.py
 ws_bom_robot_app/llm/vector_store/integration/base.py
+ws_bom_robot_app/llm/vector_store/integration/confluence.py
+ws_bom_robot_app/llm/vector_store/integration/github.py
+ws_bom_robot_app/llm/vector_store/integration/jira.py
 ws_bom_robot_app/llm/vector_store/integration/manager.py
 ws_bom_robot_app/llm/vector_store/integration/sitemap.py
 ws_bom_robot_app/llm/vector_store/loader/__init__.py

{ws_bom_robot_app-0.0.15 → ws_bom_robot_app-0.0.17}/ws_bom_robot_app.egg-info/requires.txt

@@ -1,8 +1,8 @@
 standardwebhooks==1.0.0
 apscheduler==3.11.0
 aiofiles==24.1.0
-pydantic==2.9.2
-pydantic-settings==2.6.0
+pydantic==2.10.3
+pydantic-settings==2.6.1
 fastapi[standard]==0.115.5
 langchain==0.3.9
 langchain-openai==0.2.10
@@ -11,7 +11,6 @@ langchain-core==0.3.21
 faiss-cpu==1.9.0
 python-magic==0.4.27
 opencv-python-headless==4.10.0.84
-jmespath==1.0.1
 unstructured[all-docs]==0.15.14
 langchain_unstructured==0.1.5
 html5lib==1.1

ws_bom_robot_app-0.0.15/ws_bom_robot_app/llm/vector_store/integration/base.py

@@ -1,19 +0,0 @@
-import os
-from langchain_core.documents import Document
-from abc import ABC, abstractmethod
-from ws_bom_robot_app.util import timer
-
-class IntegrationStrategy(ABC):
-  def __init__(self, knowledgebase_path: str, data: dict[str, str]):
-    self.knowledgebase_path = knowledgebase_path
-    self.data = data
-    self.working_directory = os.path.join(self.knowledgebase_path,self.working_subdirectory())
-    os.makedirs(self.working_directory, exist_ok=True)
-  @property
-  @abstractmethod
-  def working_subdirectory(self) -> str:
-    pass
-  @abstractmethod
-  #@timer
-  def load(self) -> list[Document]:
-    pass