ws-bom-robot-app 0.0.40__py3-none-any.whl → 0.0.42__py3-none-any.whl

ws_bom_robot_app/llm/main.py

@@ -1,5 +1,5 @@
 from asyncio import Queue
-import asyncio, json, logging, os, traceback
+import asyncio, json, logging, os, traceback, re
 from fastapi import Request
 from langchain.callbacks.tracers import LangChainTracer
 from langchain_core.callbacks.base import AsyncCallbackHandler
@@ -28,6 +28,18 @@ async def invoke(rq: InvokeRequest) -> str:
   result: AIMessage = await processor.run_agent(_msg)
   return {"result": result.content}
 
+def _parse_formatted_message(message: str) -> str:
+  try:
+    text_fragments = []
+    quoted_strings = re.findall(r'"([^"\\]*(?:\\.[^"\\]*)*)"', message)
+    for string in quoted_strings:
+      if not string.startswith(('threadId', 'type')) and len(string) > 1:
+          text_fragments.append(string)
+    result = ''.join(text_fragments)
+    result = result.replace('\\n', '\n')
+  except:
+    result = message
+  return result
 async def __stream(rq: StreamRequest, ctx: Request, queue: Queue,formatted: bool = True) -> None:
   await rq.initialize()
   #os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
@@ -41,21 +53,33 @@ async def __stream(rq: StreamRequest, ctx: Request, queue: Queue,formatted: bool
 
   #CREATION OF CHAT HISTORY FOR AGENT
   for message in rq.messages:
-      if message.role == "user":
-          settings.chat_history.append(HumanMessage(content=message.content))
-      elif message.role == "assistant":
-          message_content = ""
-          if formatted and '{\"type\":\"text\"' in message.content:
-            try:
-              json_msg = json.loads('[' + message.content[:-1] + ']')
-              for msg in json_msg:
-                if msg.get("content"):
-                  message_content += msg["content"]
-            except:
-              message_content = message.content
-          else:
-            message_content = message.content
-          settings.chat_history.append(AIMessage(content=message_content))
+    if message.role in ["human","user"]:
+      settings.chat_history.append(HumanMessage(content=message.content))
+    elif message.role in ["ai","assistant"]:
+      message_content = ""
+      if formatted:
+        if '{\"type\":\"string\"' in message.content:
+          try:
+            json_msg = json.loads('[' + message.content[:-1] + ']')
+            for msg in json_msg:
+              if msg.get("content"):
+                message_content += msg["content"]
+          except:
+            message_content = _parse_formatted_message(message.content)
+        elif '{\"type\":\"text\"' in message.content:
+          try:
+            json_msg = json.loads('[' + message.content[:-1] + ']')
+            for msg in json_msg:
+              if msg.get("text"):
+                message_content += msg["text"]
+          except:
+            message_content = _parse_formatted_message(message.content)
+        else:
+          message_content = _parse_formatted_message(message.content)
+      else:
+        message_content = message.content
+      if message_content:
+        settings.chat_history.append(AIMessage(content=message_content))
 
   if rq.lang_chain_tracing:
     client = LangSmithClient(

ws_bom_robot_app/llm/providers/llm_manager.py

@@ -44,7 +44,9 @@ class OpenAI(LlmInterface):
 
     def get_llm(self):
         from langchain_openai import ChatOpenAI
-        chat = ChatOpenAI(api_key=self.config.api_key, model=self.config.model)
+        chat = ChatOpenAI(
+            api_key=self.config.api_key or os.getenv("OPENAI_API_KEY"),
+            model=self.config.model)
         if not any(self.config.model.startswith(prefix) for prefix in ["o1", "o3"]):
             chat.temperature = self.config.temperature
             chat.streaming = True
@@ -60,7 +62,7 @@ class DeepSeek(LlmInterface):
     def get_llm(self):
         from langchain_openai import ChatOpenAI
         return ChatOpenAI(
-            api_key=self.config.api_key,
+            api_key=self.config.api_key or os.getenv("DEEPSEEK_API_KEY"),
             model=self.config.model,
             base_url="https://api.deepseek.com/v1",
             max_tokens=8192,
@@ -79,7 +81,7 @@ class Google(LlmInterface):
     from langchain_google_genai.chat_models import ChatGoogleGenerativeAI
     return ChatGoogleGenerativeAI(
       name="chat",
-      api_key=self.config.api_key,
+      api_key=self.config.api_key or os.getenv("GOOGLE_API_KEY"),
       model=self.config.model,
       temperature=self.config.temperature,
       disable_streaming=False
@@ -131,7 +133,7 @@ class Anthropic(LlmInterface):
     def get_llm(self):
         from langchain_anthropic import ChatAnthropic
         return ChatAnthropic(
-            api_key=self.config.api_key,
+            api_key=self.config.api_key or os.getenv("ANTHROPIC_API_KEY"),
             model=self.config.model,
             temperature=self.config.temperature,
             streaming=True,
@@ -156,7 +158,7 @@ class Groq(LlmInterface):
     def get_llm(self):
         from langchain_groq import ChatGroq
         return ChatGroq(
-            api_key=self.config.api_key,
+            api_key=self.config.api_key or os.getenv("GROQ_API_KEY"),
             model=self.config.model,
             #max_tokens=8192,
             temperature=self.config.temperature,

ws_bom_robot_app/llm/vector_store/integration/jira.py

@@ -1,14 +1,13 @@
-import asyncio
+import asyncio, os
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
-from unstructured_ingest.interfaces import  ProcessorConfig, ReadConfig
-from unstructured_ingest.connector.jira import SimpleJiraConfig, JiraAccessConfig
-from unstructured_ingest.runner import JiraRunner
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from pydantic import BaseModel, Field, AliasChoices
-from typing import Optional, Union
-import requests
-import unstructured_ingest.connector.jira
+from typing import Any, Optional, Union
+from unstructured_ingest.interfaces import  ProcessorConfig, ReadConfig
+from unstructured_ingest.connector.jira import SimpleJiraConfig, JiraAccessConfig, JiraSourceConnector, JiraIngestDoc, nested_object_to_field_getter, _get_id_fields_for_issue, _get_project_fields_for_issue
+from unstructured_ingest.runner import JiraRunner
+
 
 class JiraParams(BaseModel):
   """
@@ -22,24 +21,20 @@ class JiraParams(BaseModel):
     boards (Optional[list[str]]): An optional list of board IDs to interact with. Defaults to None, e.g., ['1', '2'].
     issues (Optional[list[str]]): An optional list of issue keys or IDs to interact with. Defaults to None, e.g., ['SCRUM-1', 'PROJ1-1'].
   """
-  url: str
-  access_token: str = Field(validation_alias=AliasChoices("accessToken","access_token"))
-  user_email: str = Field(validation_alias=AliasChoices("userEmail","user_email"))
+  url: str = Field(..., pattern=r'^https?:\/\/.+')
+  access_token: str = Field(..., validation_alias=AliasChoices("accessToken","access_token"), min_length=1)
+  user_email: str = Field(validation_alias=AliasChoices("userEmail","user_email"), min_length=1)
   projects: list[str]
   boards: Optional[list[str]] | None = None
   issues: Optional[list[str]] | None = None
-  fieldsMappingUrl: Optional[str] | None = None
 
 class Jira(IntegrationStrategy):
-  DEFAULT_C_SEP = " " * 5
-  DEFAULT_R_SEP = "\n"
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
     super().__init__(knowledgebase_path, data)
     self.__data = JiraParams.model_validate(self.data)
   def working_subdirectory(self) -> str:
     return 'jira'
   def run(self) -> None:
-    unstructured_ingest.connector.jira._get_dropdown_fields_for_issue = self._get_dropdown_fields_for_issue
     access_config = JiraAccessConfig(
       api_token=self.__data.access_token
     )
@@ -51,7 +46,8 @@ class Jira(IntegrationStrategy):
       boards=self.__data.boards,
       issues=self.__data.issues
     )
-    runner = JiraRunner(
+    # runner override: waiting for v2 migration https://github.com/Unstructured-IO/unstructured-ingest/issues/106
+    runner = _JiraRunner(
       connector_config=config,
       processor_config=ProcessorConfig(reprocess=False,verbose=False,num_processes=2,raise_on_error=False),
       read_config=ReadConfig(download_dir=self.working_directory,re_download=True,preserve_downloads=True,download_only=True),
@@ -64,51 +60,59 @@ class Jira(IntegrationStrategy):
       await asyncio.sleep(1)
       return await Loader(self.working_directory).load()
 
-  def _remap_custom_fields(self, field_list):
-    auth = (self.__data.user_email, self.__data.access_token)
-    response = requests.get(self.__data.fieldsMappingUrl, auth=auth)
-
-    if response.status_code == 200:
-      mapper: dict = response.json()
-    remapped_field_list = {}
-    for field_key, field_value in field_list.items():
-        new_key = None
-        for map_item in mapper:
-            if field_key == map_item["id"]:
-                # Usa il nome mappato come nuova chiave
-                new_key = map_item["name"]
-                break
-
-        if new_key is None:
-            new_key = field_key
 
-        remapped_field_list[new_key] = field_value
+# region override
+class _JiraIngestDoc(JiraIngestDoc):
+  def _get_dropdown_custom_fields_for_issue(issue: dict, c_sep=" " * 5, r_sep="\n") -> str:
+      def _parse_value(value: Any) -> Any:
+          if isinstance(value, dict):
+            _candidate = ["displayName", "name", "value"]
+            for item in _candidate:
+                if item in value:
+                    return value[item]
+          return value
+      def _remap_custom_fields(fields: dict):
+        remapped_fields = {}
+        for field_key, field_value in fields.items():
+          new_key = next((map_item["name"] for map_item in _JiraSourceConnector.CUSTOM_FIELDS if field_key == map_item["id"]), field_key)
+          if new_key != field_value:
+            remapped_fields[new_key] = field_value
+        return remapped_fields
+      filtered_fields = {key: _parse_value(value) for key, value in issue.items() if value is not None and type(value) not in [list]}
+      custom_fields =_remap_custom_fields(filtered_fields)
+      return (r_sep + c_sep ).join([f"{key}: {value}{r_sep}" for key, value in custom_fields.items()])
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    _issue = self.issue
+    _nested: dict = nested_object_to_field_getter(_issue["fields"])
+    document = "\n\n\n".join(
+              [
+                  _get_id_fields_for_issue(_issue),
+                  _get_project_fields_for_issue(_nested),
+                  _JiraIngestDoc._get_dropdown_custom_fields_for_issue(_nested)
+              ],
+          )
+    _full_filename = str(self.filename)
+    _file_extension  = _full_filename.split(".")[-1]
+    _file_without_extension = _full_filename.replace(f".{_file_extension}","")
+    os.makedirs(os.path.dirname(_file_without_extension), exist_ok=True)
+    with open(f"{_file_without_extension}_extra.{_file_extension}", "w", encoding="utf8") as f:
+      f.write(document)
 
-    return remapped_field_list
+class _JiraSourceConnector(JiraSourceConnector):
+  CUSTOM_FIELDS: list | None = None
+  def __set_custom_fields(self) -> None:
+    _custom_fields = self.jira.get_all_custom_fields()
+    _JiraSourceConnector.CUSTOM_FIELDS = [{"id":item["id"],"name":item["name"]} for item in _custom_fields]
+    self._jira = None # fix serialization
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    if not _JiraSourceConnector.CUSTOM_FIELDS:
+      self.__set_custom_fields()
+  def get_ingest_docs(self) -> list[_JiraIngestDoc]:
+     return [_JiraIngestDoc(**item.__dict__) for item in super().get_ingest_docs()]
 
-  def _get_dropdown_fields_for_issue(self, issue, c_sep=DEFAULT_C_SEP, r_sep=DEFAULT_R_SEP):
-      all_fields = {}
-      for key, value in issue.items():
-          if value is not None:
-              if isinstance(value, list) and (len(value) > 0):
-                  all_fields[key] = value
-              else:
-                  all_fields[key] = value
-      mapped_fields = self._remap_custom_fields(all_fields)
-      return f"""
-      IssueType:{issue["issuetype"]["name"]}
-      {r_sep}
-      Status:{issue["status"]["name"]}
-      {r_sep}
-      Priority:{issue["priority"]}
-      {r_sep}
-      AssigneeID_Name:{issue["assignee"]["accountId"]}{c_sep}{issue["assignee"]["displayName"]}
-      {r_sep}
-      ReporterAdr_Name:{issue["reporter"]["emailAddress"]}{c_sep}{issue["reporter"]["displayName"]}
-      {r_sep}
-      Labels:{c_sep.join(issue["labels"])}
-      {r_sep}
-      Components:{c_sep.join([component["name"] for component in issue["components"]])}
-      {r_sep}
-      {(r_sep + c_sep ).join([f"{key}:{value}{r_sep}" for key, value in mapped_fields.items()])}
-      """
+class _JiraRunner(JiraRunner):
+  def get_source_connector_cls(self):
+    return _JiraSourceConnector
+# endregion

{ws_bom_robot_app-0.0.40.dist-info → ws_bom_robot_app-0.0.42.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ws_bom_robot_app
-Version: 0.0.40
+Version: 0.0.42
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa

{ws_bom_robot_app-0.0.40.dist-info → ws_bom_robot_app-0.0.42.dist-info}/RECORD

@@ -11,14 +11,14 @@ ws_bom_robot_app/llm/agent_handler.py,sha256=4zdpSf5iVLxMZ90c_vUl_k-O9SF6u_h7GOB
 ws_bom_robot_app/llm/agent_lcel.py,sha256=Yt-hbKarLktv5BBiTp9OHYRdwOTRu5ogyTrYapFdiTU,2389
 ws_bom_robot_app/llm/api.py,sha256=UaD1oJyAOe7ASoXxPNJcth3kDuWcjk1xqUNEjuPWbR4,3759
 ws_bom_robot_app/llm/defaut_prompt.py,sha256=LlCd_nSMkMmHESfiiiQYfnJyB6Pp-LSs4CEKdYW4vFk,1106
-ws_bom_robot_app/llm/main.py,sha256=fVXyS9TOu22ZC7M8o2mRCya9vTmMFf5jRgs9V0K_4cw,4189
+ws_bom_robot_app/llm/main.py,sha256=YcfenQkBFpXGgFdU2NgHutBs6saPzJzcFMmJ6nwdg0Q,5084
 ws_bom_robot_app/llm/settings.py,sha256=EkFGCppORenStH9W4e6_dYvQ-5p6xiEMpmUHBqNqG9M,117
 ws_bom_robot_app/llm/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/models/api.py,sha256=mLbPG7jHh1EjgQG-xpBhEgiTIHpK35HZ51obgqQSfq4,8890
 ws_bom_robot_app/llm/models/base.py,sha256=1TqxuTK3rjJEALn7lvgoen_1ba3R2brAgGx6EDTtDZo,152
 ws_bom_robot_app/llm/models/kb.py,sha256=oVSw6_dmNxikAHrPqcfxDXz9M0ezLIYuxpgvzfs_Now,9514
 ws_bom_robot_app/llm/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/providers/llm_manager.py,sha256=GUi5aHQX_4A4pRgv-XEBH42wGPBYW2HUeQeiU_j044E,7924
+ws_bom_robot_app/llm/providers/llm_manager.py,sha256=XQpTmJ-ChituFyKMv_UOPwNr1aJgihAlsuq0kw2YLVo,8109
 ws_bom_robot_app/llm/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/tools/tool_builder.py,sha256=OaA0jReNUpjfe7c8TVLM86acQ4w0cQaR3NE22hGKJb0,1165
 ws_bom_robot_app/llm/tools/tool_manager.py,sha256=RZcJVPyWT9D3HUxSO1d5kSfTQtJB2CG5hocuFa01AzY,5816
@@ -49,7 +49,7 @@ ws_bom_robot_app/llm/vector_store/integration/dropbox.py,sha256=yhGvHTN0TEpUfhdv
 ws_bom_robot_app/llm/vector_store/integration/gcs.py,sha256=fFDVDUR6eNB7FVTzDSEpMHFEWMgG16GLnpSf_mqGDdE,3184
 ws_bom_robot_app/llm/vector_store/integration/github.py,sha256=18PO30AZcgTn6PHhid3MwImVAdmKBNkr0kmAPgOetGw,2663
 ws_bom_robot_app/llm/vector_store/integration/googledrive.py,sha256=R6hr8iEgrR3QMOzIj5jY6w1x8pZ1LGdh4xM_q7g_ttc,3738
-ws_bom_robot_app/llm/vector_store/integration/jira.py,sha256=uiNkCOSqxOgm3AoO6k5TN_bsYXiO60I_wF0i_z8Yp8Y,4874
+ws_bom_robot_app/llm/vector_store/integration/jira.py,sha256=Sf6nPongo53vG4jhlzFQZj3foNzvUvAZFr7pXAJfZvM,5795
 ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=5Fl3XML6f1wmgraigpUwIFIXh7QFPX0RI0YFgFxBAvg,1700
 ws_bom_robot_app/llm/vector_store/integration/s3.py,sha256=3kh-VmH84IW7DdSLvOk6td1VBJ9aohlVJsk5F3cYj0U,3320
 ws_bom_robot_app/llm/vector_store/integration/sftp.py,sha256=WNzjjS1EUykgFB-8e7QkecSa1r1jTJqKyGzR25uJCtM,2848
@@ -60,7 +60,7 @@ ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
 ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=L_ugekNuAq0N9O-24wtlHSNHkqSeD-KsJrfGt_FX9Oc,5340
 ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=yP0zgXLeFAlByaYuj-6cYariuknckrFds0dxdRcnVz8,3456
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=qo9ejRZyKv_k6jnGgXnu1W5uqsMMtgqK_uvPpZQ0p74,833
-ws_bom_robot_app-0.0.40.dist-info/METADATA,sha256=0kGu7SHsL99RrZAA_ZBAYoa8Ao_CSy4Np9CPZdGGSyk,8348
-ws_bom_robot_app-0.0.40.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ws_bom_robot_app-0.0.40.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.40.dist-info/RECORD,,
+ws_bom_robot_app-0.0.42.dist-info/METADATA,sha256=oqyzo0YlOL6buOYsrDYoRSj0-SkyD3hJU0uwJ5gtcQA,8348
+ws_bom_robot_app-0.0.42.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+ws_bom_robot_app-0.0.42.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.42.dist-info/RECORD,,

{ws_bom_robot_app-0.0.40.dist-info → ws_bom_robot_app-0.0.42.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (75.8.2)
 Root-Is-Purelib: true
 Tag: py3-none-any