ws-bom-robot-app 0.0.40__tar.gz → 0.0.42__tar.gz

{ws_bom_robot_app-0.0.40/ws_bom_robot_app.egg-info → ws_bom_robot_app-0.0.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ws_bom_robot_app
-Version: 0.0.40
+Version: 0.0.42
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa

{ws_bom_robot_app-0.0.40 → ws_bom_robot_app-0.0.42}/setup.py

@@ -4,7 +4,7 @@ _requirements = [line.split('#')[0].strip() for line in open("requirements.txt")
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.40",
+    version="0.0.42",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

{ws_bom_robot_app-0.0.40 → ws_bom_robot_app-0.0.42}/ws_bom_robot_app/llm/main.py

@@ -1,5 +1,5 @@
 from asyncio import Queue
-import asyncio, json, logging, os, traceback
+import asyncio, json, logging, os, traceback, re
 from fastapi import Request
 from langchain.callbacks.tracers import LangChainTracer
 from langchain_core.callbacks.base import AsyncCallbackHandler
@@ -28,6 +28,18 @@ async def invoke(rq: InvokeRequest) -> str:
   result: AIMessage = await processor.run_agent(_msg)
   return {"result": result.content}
 
+def _parse_formatted_message(message: str) -> str:
+  try:
+    text_fragments = []
+    quoted_strings = re.findall(r'"([^"\\]*(?:\\.[^"\\]*)*)"', message)
+    for string in quoted_strings:
+      if not string.startswith(('threadId', 'type')) and len(string) > 1:
+          text_fragments.append(string)
+    result = ''.join(text_fragments)
+    result = result.replace('\\n', '\n')
+  except:
+    result = message
+  return result
 async def __stream(rq: StreamRequest, ctx: Request, queue: Queue,formatted: bool = True) -> None:
   await rq.initialize()
   #os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
@@ -41,21 +53,33 @@ async def __stream(rq: StreamRequest, ctx: Request, queue: Queue,formatted: bool
 
   #CREATION OF CHAT HISTORY FOR AGENT
   for message in rq.messages:
-      if message.role == "user":
-          settings.chat_history.append(HumanMessage(content=message.content))
-      elif message.role == "assistant":
-          message_content = ""
-          if formatted and '{\"type\":\"text\"' in message.content:
-            try:
-              json_msg = json.loads('[' + message.content[:-1] + ']')
-              for msg in json_msg:
-                if msg.get("content"):
-                  message_content += msg["content"]
-            except:
-              message_content = message.content
-          else:
-            message_content = message.content
-          settings.chat_history.append(AIMessage(content=message_content))
+    if message.role in ["human","user"]:
+      settings.chat_history.append(HumanMessage(content=message.content))
+    elif message.role in ["ai","assistant"]:
+      message_content = ""
+      if formatted:
+        if '{\"type\":\"string\"' in message.content:
+          try:
+            json_msg = json.loads('[' + message.content[:-1] + ']')
+            for msg in json_msg:
+              if msg.get("content"):
+                message_content += msg["content"]
+          except:
+            message_content = _parse_formatted_message(message.content)
+        elif '{\"type\":\"text\"' in message.content:
+          try:
+            json_msg = json.loads('[' + message.content[:-1] + ']')
+            for msg in json_msg:
+              if msg.get("text"):
+                message_content += msg["text"]
+          except:
+            message_content = _parse_formatted_message(message.content)
+        else:
+          message_content = _parse_formatted_message(message.content)
+      else:
+        message_content = message.content
+      if message_content:
+        settings.chat_history.append(AIMessage(content=message_content))
 
   if rq.lang_chain_tracing:
     client = LangSmithClient(

{ws_bom_robot_app-0.0.40 → ws_bom_robot_app-0.0.42}/ws_bom_robot_app/llm/providers/llm_manager.py

@@ -44,7 +44,9 @@ class OpenAI(LlmInterface):
 
     def get_llm(self):
         from langchain_openai import ChatOpenAI
-        chat = ChatOpenAI(api_key=self.config.api_key, model=self.config.model)
+        chat = ChatOpenAI(
+            api_key=self.config.api_key or os.getenv("OPENAI_API_KEY"),
+            model=self.config.model)
         if not any(self.config.model.startswith(prefix) for prefix in ["o1", "o3"]):
             chat.temperature = self.config.temperature
             chat.streaming = True
@@ -60,7 +62,7 @@ class DeepSeek(LlmInterface):
     def get_llm(self):
         from langchain_openai import ChatOpenAI
         return ChatOpenAI(
-            api_key=self.config.api_key,
+            api_key=self.config.api_key or os.getenv("DEEPSEEK_API_KEY"),
             model=self.config.model,
             base_url="https://api.deepseek.com/v1",
             max_tokens=8192,
@@ -79,7 +81,7 @@ class Google(LlmInterface):
     from langchain_google_genai.chat_models import ChatGoogleGenerativeAI
     return ChatGoogleGenerativeAI(
       name="chat",
-      api_key=self.config.api_key,
+      api_key=self.config.api_key or os.getenv("GOOGLE_API_KEY"),
       model=self.config.model,
       temperature=self.config.temperature,
       disable_streaming=False
@@ -131,7 +133,7 @@ class Anthropic(LlmInterface):
     def get_llm(self):
         from langchain_anthropic import ChatAnthropic
         return ChatAnthropic(
-            api_key=self.config.api_key,
+            api_key=self.config.api_key or os.getenv("ANTHROPIC_API_KEY"),
             model=self.config.model,
             temperature=self.config.temperature,
             streaming=True,
@@ -156,7 +158,7 @@ class Groq(LlmInterface):
     def get_llm(self):
         from langchain_groq import ChatGroq
         return ChatGroq(
-            api_key=self.config.api_key,
+            api_key=self.config.api_key or os.getenv("GROQ_API_KEY"),
             model=self.config.model,
             #max_tokens=8192,
             temperature=self.config.temperature,

ws_bom_robot_app-0.0.42/ws_bom_robot_app/llm/vector_store/integration/jira.py

@@ -0,0 +1,118 @@
+import asyncio, os
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from pydantic import BaseModel, Field, AliasChoices
+from typing import Any, Optional, Union
+from unstructured_ingest.interfaces import  ProcessorConfig, ReadConfig
+from unstructured_ingest.connector.jira import SimpleJiraConfig, JiraAccessConfig, JiraSourceConnector, JiraIngestDoc, nested_object_to_field_getter, _get_id_fields_for_issue, _get_project_fields_for_issue
+from unstructured_ingest.runner import JiraRunner
+
+
+class JiraParams(BaseModel):
+  """
+  JiraParams is a Pydantic model that represents the parameters required to interact with a Jira instance.
+
+  Attributes:
+    url (str): The URL of the Jira instance, e.g., 'https://example.atlassian.net'.
+    access_token (str): The access token for authenticating with the Jira API.
+    user_email (str): The email address of the Jira user.
+    projects (list[str]): A list of project keys or IDs to interact with, e.g., ['SCRUM', 'PROJ1'].
+    boards (Optional[list[str]]): An optional list of board IDs to interact with. Defaults to None, e.g., ['1', '2'].
+    issues (Optional[list[str]]): An optional list of issue keys or IDs to interact with. Defaults to None, e.g., ['SCRUM-1', 'PROJ1-1'].
+  """
+  url: str = Field(..., pattern=r'^https?:\/\/.+')
+  access_token: str = Field(..., validation_alias=AliasChoices("accessToken","access_token"), min_length=1)
+  user_email: str = Field(validation_alias=AliasChoices("userEmail","user_email"), min_length=1)
+  projects: list[str]
+  boards: Optional[list[str]] | None = None
+  issues: Optional[list[str]] | None = None
+
+class Jira(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = JiraParams.model_validate(self.data)
+  def working_subdirectory(self) -> str:
+    return 'jira'
+  def run(self) -> None:
+    access_config = JiraAccessConfig(
+      api_token=self.__data.access_token
+    )
+    config = SimpleJiraConfig(
+      user_email=self.__data.user_email,
+      url = self.__data.url,
+      access_config=access_config,
+      projects=self.__data.projects,
+      boards=self.__data.boards,
+      issues=self.__data.issues
+    )
+    # runner override: waiting for v2 migration https://github.com/Unstructured-IO/unstructured-ingest/issues/106
+    runner = _JiraRunner(
+      connector_config=config,
+      processor_config=ProcessorConfig(reprocess=False,verbose=False,num_processes=2,raise_on_error=False),
+      read_config=ReadConfig(download_dir=self.working_directory,re_download=True,preserve_downloads=True,download_only=True),
+      partition_config=None,
+      retry_strategy_config=None
+      )
+    runner.run()
+  async def load(self) -> list[Document]:
+      await asyncio.to_thread(self.run)
+      await asyncio.sleep(1)
+      return await Loader(self.working_directory).load()
+
+
+# region override
+class _JiraIngestDoc(JiraIngestDoc):
+  def _get_dropdown_custom_fields_for_issue(issue: dict, c_sep=" " * 5, r_sep="\n") -> str:
+      def _parse_value(value: Any) -> Any:
+          if isinstance(value, dict):
+            _candidate = ["displayName", "name", "value"]
+            for item in _candidate:
+                if item in value:
+                    return value[item]
+          return value
+      def _remap_custom_fields(fields: dict):
+        remapped_fields = {}
+        for field_key, field_value in fields.items():
+          new_key = next((map_item["name"] for map_item in _JiraSourceConnector.CUSTOM_FIELDS if field_key == map_item["id"]), field_key)
+          if new_key != field_value:
+            remapped_fields[new_key] = field_value
+        return remapped_fields
+      filtered_fields = {key: _parse_value(value) for key, value in issue.items() if value is not None and type(value) not in [list]}
+      custom_fields =_remap_custom_fields(filtered_fields)
+      return (r_sep + c_sep ).join([f"{key}: {value}{r_sep}" for key, value in custom_fields.items()])
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    _issue = self.issue
+    _nested: dict = nested_object_to_field_getter(_issue["fields"])
+    document = "\n\n\n".join(
+              [
+                  _get_id_fields_for_issue(_issue),
+                  _get_project_fields_for_issue(_nested),
+                  _JiraIngestDoc._get_dropdown_custom_fields_for_issue(_nested)
+              ],
+          )
+    _full_filename = str(self.filename)
+    _file_extension  = _full_filename.split(".")[-1]
+    _file_without_extension = _full_filename.replace(f".{_file_extension}","")
+    os.makedirs(os.path.dirname(_file_without_extension), exist_ok=True)
+    with open(f"{_file_without_extension}_extra.{_file_extension}", "w", encoding="utf8") as f:
+      f.write(document)
+
+class _JiraSourceConnector(JiraSourceConnector):
+  CUSTOM_FIELDS: list | None = None
+  def __set_custom_fields(self) -> None:
+    _custom_fields = self.jira.get_all_custom_fields()
+    _JiraSourceConnector.CUSTOM_FIELDS = [{"id":item["id"],"name":item["name"]} for item in _custom_fields]
+    self._jira = None # fix serialization
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    if not _JiraSourceConnector.CUSTOM_FIELDS:
+      self.__set_custom_fields()
+  def get_ingest_docs(self) -> list[_JiraIngestDoc]:
+     return [_JiraIngestDoc(**item.__dict__) for item in super().get_ingest_docs()]
+
+class _JiraRunner(JiraRunner):
+  def get_source_connector_cls(self):
+    return _JiraSourceConnector
+# endregion

{ws_bom_robot_app-0.0.40 → ws_bom_robot_app-0.0.42/ws_bom_robot_app.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ws_bom_robot_app
-Version: 0.0.40
+Version: 0.0.42
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa

ws_bom_robot_app-0.0.40/ws_bom_robot_app/llm/vector_store/integration/jira.py

@@ -1,114 +0,0 @@
-import asyncio
-from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
-from unstructured_ingest.interfaces import  ProcessorConfig, ReadConfig
-from unstructured_ingest.connector.jira import SimpleJiraConfig, JiraAccessConfig
-from unstructured_ingest.runner import JiraRunner
-from langchain_core.documents import Document
-from ws_bom_robot_app.llm.vector_store.loader.base import Loader
-from pydantic import BaseModel, Field, AliasChoices
-from typing import Optional, Union
-import requests
-import unstructured_ingest.connector.jira
-
-class JiraParams(BaseModel):
-  """
-  JiraParams is a Pydantic model that represents the parameters required to interact with a Jira instance.
-
-  Attributes:
-    url (str): The URL of the Jira instance, e.g., 'https://example.atlassian.net'.
-    access_token (str): The access token for authenticating with the Jira API.
-    user_email (str): The email address of the Jira user.
-    projects (list[str]): A list of project keys or IDs to interact with, e.g., ['SCRUM', 'PROJ1'].
-    boards (Optional[list[str]]): An optional list of board IDs to interact with. Defaults to None, e.g., ['1', '2'].
-    issues (Optional[list[str]]): An optional list of issue keys or IDs to interact with. Defaults to None, e.g., ['SCRUM-1', 'PROJ1-1'].
-  """
-  url: str
-  access_token: str = Field(validation_alias=AliasChoices("accessToken","access_token"))
-  user_email: str = Field(validation_alias=AliasChoices("userEmail","user_email"))
-  projects: list[str]
-  boards: Optional[list[str]] | None = None
-  issues: Optional[list[str]] | None = None
-  fieldsMappingUrl: Optional[str] | None = None
-
-class Jira(IntegrationStrategy):
-  DEFAULT_C_SEP = " " * 5
-  DEFAULT_R_SEP = "\n"
-  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
-    super().__init__(knowledgebase_path, data)
-    self.__data = JiraParams.model_validate(self.data)
-  def working_subdirectory(self) -> str:
-    return 'jira'
-  def run(self) -> None:
-    unstructured_ingest.connector.jira._get_dropdown_fields_for_issue = self._get_dropdown_fields_for_issue
-    access_config = JiraAccessConfig(
-      api_token=self.__data.access_token
-    )
-    config = SimpleJiraConfig(
-      user_email=self.__data.user_email,
-      url = self.__data.url,
-      access_config=access_config,
-      projects=self.__data.projects,
-      boards=self.__data.boards,
-      issues=self.__data.issues
-    )
-    runner = JiraRunner(
-      connector_config=config,
-      processor_config=ProcessorConfig(reprocess=False,verbose=False,num_processes=2,raise_on_error=False),
-      read_config=ReadConfig(download_dir=self.working_directory,re_download=True,preserve_downloads=True,download_only=True),
-      partition_config=None,
-      retry_strategy_config=None
-      )
-    runner.run()
-  async def load(self) -> list[Document]:
-      await asyncio.to_thread(self.run)
-      await asyncio.sleep(1)
-      return await Loader(self.working_directory).load()
-
-  def _remap_custom_fields(self, field_list):
-    auth = (self.__data.user_email, self.__data.access_token)
-    response = requests.get(self.__data.fieldsMappingUrl, auth=auth)
-
-    if response.status_code == 200:
-      mapper: dict = response.json()
-    remapped_field_list = {}
-    for field_key, field_value in field_list.items():
-        new_key = None
-        for map_item in mapper:
-            if field_key == map_item["id"]:
-                # Usa il nome mappato come nuova chiave
-                new_key = map_item["name"]
-                break
-
-        if new_key is None:
-            new_key = field_key
-
-        remapped_field_list[new_key] = field_value
-
-    return remapped_field_list
-
-  def _get_dropdown_fields_for_issue(self, issue, c_sep=DEFAULT_C_SEP, r_sep=DEFAULT_R_SEP):
-      all_fields = {}
-      for key, value in issue.items():
-          if value is not None:
-              if isinstance(value, list) and (len(value) > 0):
-                  all_fields[key] = value
-              else:
-                  all_fields[key] = value
-      mapped_fields = self._remap_custom_fields(all_fields)
-      return f"""
-      IssueType:{issue["issuetype"]["name"]}
-      {r_sep}
-      Status:{issue["status"]["name"]}
-      {r_sep}
-      Priority:{issue["priority"]}
-      {r_sep}
-      AssigneeID_Name:{issue["assignee"]["accountId"]}{c_sep}{issue["assignee"]["displayName"]}
-      {r_sep}
-      ReporterAdr_Name:{issue["reporter"]["emailAddress"]}{c_sep}{issue["reporter"]["displayName"]}
-      {r_sep}
-      Labels:{c_sep.join(issue["labels"])}
-      {r_sep}
-      Components:{c_sep.join([component["name"] for component in issue["components"]])}
-      {r_sep}
-      {(r_sep + c_sep ).join([f"{key}:{value}{r_sep}" for key, value in mapped_fields.items()])}
-      """