ws-bom-robot-app 0.0.40__py3-none-any.whl → 0.0.41__py3-none-any.whl

ws_bom_robot_app/llm/vector_store/integration/jira.py

@@ -1,14 +1,13 @@
-import asyncio
+import asyncio, os
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
-from unstructured_ingest.interfaces import  ProcessorConfig, ReadConfig
-from unstructured_ingest.connector.jira import SimpleJiraConfig, JiraAccessConfig
-from unstructured_ingest.runner import JiraRunner
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from pydantic import BaseModel, Field, AliasChoices
-from typing import Optional, Union
-import requests
-import unstructured_ingest.connector.jira
+from typing import Any, Optional, Union
+from unstructured_ingest.interfaces import  ProcessorConfig, ReadConfig
+from unstructured_ingest.connector.jira import SimpleJiraConfig, JiraAccessConfig, JiraSourceConnector, JiraIngestDoc, nested_object_to_field_getter, _get_id_fields_for_issue, _get_project_fields_for_issue
+from unstructured_ingest.runner import JiraRunner
+
 
 class JiraParams(BaseModel):
   """
@@ -22,24 +21,20 @@ class JiraParams(BaseModel):
     boards (Optional[list[str]]): An optional list of board IDs to interact with. Defaults to None, e.g., ['1', '2'].
     issues (Optional[list[str]]): An optional list of issue keys or IDs to interact with. Defaults to None, e.g., ['SCRUM-1', 'PROJ1-1'].
   """
-  url: str
-  access_token: str = Field(validation_alias=AliasChoices("accessToken","access_token"))
-  user_email: str = Field(validation_alias=AliasChoices("userEmail","user_email"))
+  url: str = Field(..., pattern=r'^https?:\/\/.+')
+  access_token: str = Field(..., validation_alias=AliasChoices("accessToken","access_token"), min_length=1)
+  user_email: str = Field(validation_alias=AliasChoices("userEmail","user_email"), min_length=1)
   projects: list[str]
   boards: Optional[list[str]] | None = None
   issues: Optional[list[str]] | None = None
-  fieldsMappingUrl: Optional[str] | None = None
 
 class Jira(IntegrationStrategy):
-  DEFAULT_C_SEP = " " * 5
-  DEFAULT_R_SEP = "\n"
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
     super().__init__(knowledgebase_path, data)
     self.__data = JiraParams.model_validate(self.data)
   def working_subdirectory(self) -> str:
     return 'jira'
   def run(self) -> None:
-    unstructured_ingest.connector.jira._get_dropdown_fields_for_issue = self._get_dropdown_fields_for_issue
     access_config = JiraAccessConfig(
       api_token=self.__data.access_token
     )
@@ -51,7 +46,8 @@ class Jira(IntegrationStrategy):
       boards=self.__data.boards,
       issues=self.__data.issues
     )
-    runner = JiraRunner(
+    # runner override: waiting for v2 migration https://github.com/Unstructured-IO/unstructured-ingest/issues/106
+    runner = _JiraRunner(
       connector_config=config,
       processor_config=ProcessorConfig(reprocess=False,verbose=False,num_processes=2,raise_on_error=False),
       read_config=ReadConfig(download_dir=self.working_directory,re_download=True,preserve_downloads=True,download_only=True),
@@ -64,51 +60,59 @@ class Jira(IntegrationStrategy):
       await asyncio.sleep(1)
       return await Loader(self.working_directory).load()
 
-  def _remap_custom_fields(self, field_list):
-    auth = (self.__data.user_email, self.__data.access_token)
-    response = requests.get(self.__data.fieldsMappingUrl, auth=auth)
-
-    if response.status_code == 200:
-      mapper: dict = response.json()
-    remapped_field_list = {}
-    for field_key, field_value in field_list.items():
-        new_key = None
-        for map_item in mapper:
-            if field_key == map_item["id"]:
-                # Usa il nome mappato come nuova chiave
-                new_key = map_item["name"]
-                break
-
-        if new_key is None:
-            new_key = field_key
 
-        remapped_field_list[new_key] = field_value
+# region override
+class _JiraIngestDoc(JiraIngestDoc):
+  def _get_dropdown_custom_fields_for_issue(issue: dict, c_sep=" " * 5, r_sep="\n") -> str:
+      def _parse_value(value: Any) -> Any:
+          if isinstance(value, dict):
+            _candidate = ["displayName", "name", "value"]
+            for item in _candidate:
+                if item in value:
+                    return value[item]
+          return value
+      def _remap_custom_fields(fields: dict):
+        remapped_fields = {}
+        for field_key, field_value in fields.items():
+          new_key = next((map_item["name"] for map_item in _JiraSourceConnector.CUSTOM_FIELDS if field_key == map_item["id"]), field_key)
+          if new_key != field_value:
+            remapped_fields[new_key] = field_value
+        return remapped_fields
+      filtered_fields = {key: _parse_value(value) for key, value in issue.items() if value is not None and type(value) not in [list]}
+      custom_fields =_remap_custom_fields(filtered_fields)
+      return (r_sep + c_sep ).join([f"{key}: {value}{r_sep}" for key, value in custom_fields.items()])
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    _issue = self.issue
+    _nested: dict = nested_object_to_field_getter(_issue["fields"])
+    document = "\n\n\n".join(
+              [
+                  _get_id_fields_for_issue(_issue),
+                  _get_project_fields_for_issue(_nested),
+                  _JiraIngestDoc._get_dropdown_custom_fields_for_issue(_nested)
+              ],
+          )
+    _full_filename = str(self.filename)
+    _file_extension  = _full_filename.split(".")[-1]
+    _file_without_extension = _full_filename.replace(f".{_file_extension}","")
+    os.makedirs(os.path.dirname(_file_without_extension), exist_ok=True)
+    with open(f"{_file_without_extension}_extra.{_file_extension}", "w", encoding="utf8") as f:
+      f.write(document)
 
-    return remapped_field_list
+class _JiraSourceConnector(JiraSourceConnector):
+  CUSTOM_FIELDS: list | None = None
+  def __set_custom_fields(self) -> None:
+    _custom_fields = self.jira.get_all_custom_fields()
+    _JiraSourceConnector.CUSTOM_FIELDS = [{"id":item["id"],"name":item["name"]} for item in _custom_fields]
+    self._jira = None # fix serialization
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    if not _JiraSourceConnector.CUSTOM_FIELDS:
+      self.__set_custom_fields()
+  def get_ingest_docs(self) -> list[_JiraIngestDoc]:
+     return [_JiraIngestDoc(**item.__dict__) for item in super().get_ingest_docs()]
 
-  def _get_dropdown_fields_for_issue(self, issue, c_sep=DEFAULT_C_SEP, r_sep=DEFAULT_R_SEP):
-      all_fields = {}
-      for key, value in issue.items():
-          if value is not None:
-              if isinstance(value, list) and (len(value) > 0):
-                  all_fields[key] = value
-              else:
-                  all_fields[key] = value
-      mapped_fields = self._remap_custom_fields(all_fields)
-      return f"""
-      IssueType:{issue["issuetype"]["name"]}
-      {r_sep}
-      Status:{issue["status"]["name"]}
-      {r_sep}
-      Priority:{issue["priority"]}
-      {r_sep}
-      AssigneeID_Name:{issue["assignee"]["accountId"]}{c_sep}{issue["assignee"]["displayName"]}
-      {r_sep}
-      ReporterAdr_Name:{issue["reporter"]["emailAddress"]}{c_sep}{issue["reporter"]["displayName"]}
-      {r_sep}
-      Labels:{c_sep.join(issue["labels"])}
-      {r_sep}
-      Components:{c_sep.join([component["name"] for component in issue["components"]])}
-      {r_sep}
-      {(r_sep + c_sep ).join([f"{key}:{value}{r_sep}" for key, value in mapped_fields.items()])}
-      """
+class _JiraRunner(JiraRunner):
+  def get_source_connector_cls(self):
+    return _JiraSourceConnector
+# endregion

{ws_bom_robot_app-0.0.40.dist-info → ws_bom_robot_app-0.0.41.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ws_bom_robot_app
-Version: 0.0.40
+Version: 0.0.41
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa

{ws_bom_robot_app-0.0.40.dist-info → ws_bom_robot_app-0.0.41.dist-info}/RECORD

@@ -49,7 +49,7 @@ ws_bom_robot_app/llm/vector_store/integration/dropbox.py,sha256=yhGvHTN0TEpUfhdv
 ws_bom_robot_app/llm/vector_store/integration/gcs.py,sha256=fFDVDUR6eNB7FVTzDSEpMHFEWMgG16GLnpSf_mqGDdE,3184
 ws_bom_robot_app/llm/vector_store/integration/github.py,sha256=18PO30AZcgTn6PHhid3MwImVAdmKBNkr0kmAPgOetGw,2663
 ws_bom_robot_app/llm/vector_store/integration/googledrive.py,sha256=R6hr8iEgrR3QMOzIj5jY6w1x8pZ1LGdh4xM_q7g_ttc,3738
-ws_bom_robot_app/llm/vector_store/integration/jira.py,sha256=uiNkCOSqxOgm3AoO6k5TN_bsYXiO60I_wF0i_z8Yp8Y,4874
+ws_bom_robot_app/llm/vector_store/integration/jira.py,sha256=Sf6nPongo53vG4jhlzFQZj3foNzvUvAZFr7pXAJfZvM,5795
 ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=5Fl3XML6f1wmgraigpUwIFIXh7QFPX0RI0YFgFxBAvg,1700
 ws_bom_robot_app/llm/vector_store/integration/s3.py,sha256=3kh-VmH84IW7DdSLvOk6td1VBJ9aohlVJsk5F3cYj0U,3320
 ws_bom_robot_app/llm/vector_store/integration/sftp.py,sha256=WNzjjS1EUykgFB-8e7QkecSa1r1jTJqKyGzR25uJCtM,2848
@@ -60,7 +60,7 @@ ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
 ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=L_ugekNuAq0N9O-24wtlHSNHkqSeD-KsJrfGt_FX9Oc,5340
 ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=yP0zgXLeFAlByaYuj-6cYariuknckrFds0dxdRcnVz8,3456
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=qo9ejRZyKv_k6jnGgXnu1W5uqsMMtgqK_uvPpZQ0p74,833
-ws_bom_robot_app-0.0.40.dist-info/METADATA,sha256=0kGu7SHsL99RrZAA_ZBAYoa8Ao_CSy4Np9CPZdGGSyk,8348
-ws_bom_robot_app-0.0.40.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ws_bom_robot_app-0.0.40.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.40.dist-info/RECORD,,
+ws_bom_robot_app-0.0.41.dist-info/METADATA,sha256=wfGSdQemB5CByJc9Hujsud2LmLXzKOddHTy_Z_pUPII,8348
+ws_bom_robot_app-0.0.41.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ws_bom_robot_app-0.0.41.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.41.dist-info/RECORD,,