ws-bom-robot-app 0.0.84__tar.gz → 0.0.86__tar.gz

{ws_bom_robot_app-0.0.84/ws_bom_robot_app.egg-info → ws_bom_robot_app-0.0.86}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.84
+Version: 0.0.86
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -17,26 +17,26 @@ Requires-Dist: pydantic==2.11.7
 Requires-Dist: pydantic-settings==2.10.1
 Requires-Dist: fastapi[standard]==0.116.1
 Requires-Dist: chevron==0.14.0
-Requires-Dist: langchain==0.3.26
-Requires-Dist: langchain-community==0.3.26
-Requires-Dist: langchain-core==0.3.72
-Requires-Dist: langchain-openai==0.3.27
-Requires-Dist: langchain-anthropic==0.3.6
-Requires-Dist: langchain-ibm==0.3.14
-Requires-Dist: langchain-google-genai==2.0.7
-Requires-Dist: langchain-google-vertexai==2.0.27
-Requires-Dist: langchain-groq==0.3.6
-Requires-Dist: langchain-ollama==0.3.3
-Requires-Dist: faiss-cpu==1.11.0
-Requires-Dist: chromadb==1.0.15
-Requires-Dist: langchain_chroma==0.2.5
-Requires-Dist: fastembed==0.7.1
+Requires-Dist: langchain==0.3.27
+Requires-Dist: langchain-community==0.3.29
+Requires-Dist: langchain-core==0.3.75
+Requires-Dist: langchain-openai==0.3.32
+Requires-Dist: langchain-anthropic==0.3.19
+Requires-Dist: langchain-ibm==0.3.17
+Requires-Dist: langchain-google-genai==2.1.10
+Requires-Dist: langchain-google-vertexai==2.0.28
+Requires-Dist: langchain-groq==0.3.7
+Requires-Dist: langchain-ollama==0.3.7
+Requires-Dist: openevals==0.1.0
+Requires-Dist: faiss-cpu==1.12.0
+Requires-Dist: chromadb==1.0.20
+Requires-Dist: langchain-chroma==0.2.5
 Requires-Dist: langchain-qdrant==0.2.0
-Requires-Dist: qdrant-client==1.15.0
+Requires-Dist: qdrant-client[fastembed]==1.15.1
 Requires-Dist: lark==1.2.2
-Requires-Dist: unstructured==0.18.11
+Requires-Dist: unstructured==0.18.14
 Requires-Dist: unstructured[image]
-Requires-Dist: unstructured-ingest==1.2.6
+Requires-Dist: unstructured-ingest==1.2.11
 Requires-Dist: unstructured-ingest[azure]
 Requires-Dist: unstructured-ingest[confluence]
 Requires-Dist: unstructured-ingest[dropbox]
@@ -49,9 +49,9 @@ Requires-Dist: unstructured-ingest[sftp]
 Requires-Dist: unstructured-ingest[sharepoint]
 Requires-Dist: unstructured-ingest[slack]
 Requires-Dist: html5lib==1.1
-Requires-Dist: markdownify==1.1.0
+Requires-Dist: markdownify==1.2.0
 Requires-Dist: duckduckgo-search==8.0.4
-Requires-Dist: langchain_google_community==2.0.7
+Requires-Dist: langchain-google-community==2.0.7
 Requires-Dist: trafilatura==2.0.0
 Dynamic: author
 Dynamic: author-email

{ws_bom_robot_app-0.0.84 → ws_bom_robot_app-0.0.86}/requirements.txt

@@ -8,30 +8,30 @@ fastapi[standard]==0.116.1
 chevron==0.14.0
 
 #framework
-langchain==0.3.26
-langchain-community==0.3.26
-langchain-core==0.3.72
-langchain-openai==0.3.27
-langchain-anthropic==0.3.6 #issue get_models() from 0.3.7
-langchain-ibm==0.3.14
-langchain-google-genai==2.0.7 #waiting for new release: https://github.com/langchain-ai/langchain-google/issues/711
-langchain-google-vertexai==2.0.27
-langchain-groq==0.3.6
-langchain-ollama==0.3.3
+langchain==0.3.27
+langchain-community==0.3.29
+langchain-core==0.3.75
+langchain-openai==0.3.32
+langchain-anthropic==0.3.19
+langchain-ibm==0.3.17
+langchain-google-genai==2.1.10
+langchain-google-vertexai==2.0.28
+langchain-groq==0.3.7
+langchain-ollama==0.3.7
+openevals==0.1.0 #langsmith evaluation
 
 #vector DB
-faiss-cpu==1.11.0
-chromadb==1.0.15
-langchain_chroma==0.2.5
-fastembed==0.7.1 #qdrant sparse embedding
+faiss-cpu==1.12.0
+chromadb==1.0.20
+langchain-chroma==0.2.5
 langchain-qdrant==0.2.0
-qdrant-client==1.15.0
+qdrant-client[fastembed]==1.15.1
 lark==1.2.2 #self-query retriever
 
 #loaders
-unstructured==0.18.11
+unstructured==0.18.14
 unstructured[image]
-unstructured-ingest==1.2.6
+unstructured-ingest==1.2.11
 unstructured-ingest[azure]
 unstructured-ingest[confluence]
 unstructured-ingest[dropbox]
@@ -46,9 +46,9 @@ unstructured-ingest[slack]
 html5lib==1.1 #beautifulsoup4 parser
 
 #integrations
-markdownify==1.1.0 #sitemap
+markdownify==1.2.0 #sitemap
 
-##tools
+#tools
 duckduckgo-search==8.0.4
-langchain_google_community==2.0.7
+langchain-google-community==2.0.7
 trafilatura==2.0.0

{ws_bom_robot_app-0.0.84 → ws_bom_robot_app-0.0.86}/setup.py

@@ -4,7 +4,7 @@ _requirements = [line.split('#')[0].strip() for line in open("requirements.txt")
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.84",
+    version="0.0.86",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

{ws_bom_robot_app-0.0.84 → ws_bom_robot_app-0.0.86}/ws_bom_robot_app/config.py

@@ -36,11 +36,12 @@ class Settings(BaseSettings):
     OLLAMA_API_URL: str = 'http://localhost:11434'
     GROQ_API_KEY: str = ''
     GOOGLE_API_KEY: str = ''
+    GOOGLE_APPLICATION_CREDENTIALS: str = '' # path to google credentials iam file, e.d. ./.secrets/google-credentials.json
     WATSONX_URL: str = ''
     WATSONX_APIKEY: str = ''
     WATSONX_PROJECTID: str = ''
     NEBULY_API_URL: str ='https://backend.nebuly.com/'
-    GOOGLE_APPLICATION_CREDENTIALS: str = '' # path to google credentials iam file, e.d. ./.secrets/google-credentials.json
+    LANGSMITH_API_KEY: str = '' # app-wide api key to run evaluation
     model_config = ConfigDict(
         env_file='./.env',
         extra='ignore',
@@ -61,6 +62,7 @@ class Settings(BaseSettings):
         os.environ["WATSONX_APIKEY"] = self.WATSONX_APIKEY
         os.environ["WATSONX_PROJECTID"] = self.WATSONX_PROJECTID
         os.environ["NEBULY_API_URL"] = self.NEBULY_API_URL
+        os.environ["LANGSMITH_API_KEY"] = self.LANGSMITH_API_KEY
         # dir
         os.makedirs(self.robot_data_folder, exist_ok=True)
         for subfolder in [self.robot_data_db_folder, self.robot_data_attachment_folder, 'db']:

{ws_bom_robot_app-0.0.84 → ws_bom_robot_app-0.0.86}/ws_bom_robot_app/cron_manager.py

@@ -56,9 +56,9 @@ class Job:
 
 class CronManager:
     _list_default = [
-            Job('cleanup-task-history',task_cleanup_history, interval=5 * 60),
-            Job('cleanup-kb-data',kb_cleanup_data_file, interval=180 * 60),
-            Job('cleanup-chat-attachment',chat_cleanup_attachment, interval=120 * 60),
+            Job('cleanup-task-history',task_cleanup_history, interval=4 * 60 * 60),
+            Job('cleanup-kb-data',kb_cleanup_data_file, interval=8 * 60 * 60),
+            Job('cleanup-chat-attachment',chat_cleanup_attachment, interval=6 * 60 * 60),
         ]
     def __get_jobstore_strategy(self) -> JobstoreStrategy:
         if config.robot_cron_strategy == 'memory':

{ws_bom_robot_app-0.0.84 → ws_bom_robot_app-0.0.86}/ws_bom_robot_app/llm/api.py

@@ -1,7 +1,8 @@
-from typing import Annotated, Any, Mapping
+from typing import Annotated, Any, Mapping, Union
 from fastapi import APIRouter, HTTPException, Request, Header, Body
 from fastapi.responses import StreamingResponse
 from ws_bom_robot_app.llm.agent_description import AgentDescriptor
+from ws_bom_robot_app.llm.evaluator import EvaluatorRunRequest
 from ws_bom_robot_app.llm.models.api import InvokeRequest, StreamRequest, RulesRequest, KbRequest, VectorDbResponse
 from ws_bom_robot_app.llm.main import invoke, stream
 from ws_bom_robot_app.llm.models.base import IdentifiableEntity
@@ -52,7 +53,7 @@ async def _kb(rq: KbRequest) -> VectorDbResponse:
 
 @router.post("/kb/task")
 async def _kb_task(rq: KbRequest, headers: Annotated[TaskHeader, Header()]) -> IdentifiableEntity:
-    return task_manager.create_task(lambda: kb(rq),headers)
+    return task_manager.create_task(lambda: kb(rq),headers, queue="slow")
 
 @router.post("/rules")
 async def _rules(rq: RulesRequest) -> VectorDbResponse:
@@ -60,7 +61,7 @@ async def _rules(rq: RulesRequest) -> VectorDbResponse:
 
 @router.post("/rules/task")
 async def _rules_task(rq: RulesRequest, headers: Annotated[TaskHeader, Header()]) -> IdentifiableEntity:
-    return task_manager.create_task(lambda: rules(rq), headers)
+    return task_manager.create_task(lambda: rules(rq), headers, queue="fast")
 
 @router.get("/kb/file/{filename}")
 async def _kb_get_file(filename: str) -> StreamingResponse:
@@ -115,3 +116,64 @@ async def _send_feedback(feedback: FeedbackConfig):
     strategy: FeedbackInterface = strategy_cls(feedback)
     result = strategy.send_feedback()
     return {"result": result}
+
+#region evaluate
+@router.get("/evaluation/datasets", tags=["evaluation"])
+async def _evaluation_datasets():
+    from ws_bom_robot_app.llm.evaluator import EvaluatorDataSets
+    return [ds for ds in EvaluatorDataSets.all()]
+
+@router.post("/evaluation/datasets/find", tags=["evaluation"])
+async def _evaluation_find_datasets(project: str):
+    from ws_bom_robot_app.llm.evaluator import EvaluatorDataSets
+    return [ds for ds in EvaluatorDataSets.find(project)]
+
+@router.get("/evaluation/datasets/{id}", tags=["evaluation"])
+async def _evaluation_datasets_by_id(id: str):
+    from ws_bom_robot_app.llm.evaluator import EvaluatorDataSets
+    return EvaluatorDataSets.example(id)
+
+@router.get("/evaluation/evaluators", tags=["evaluation"])
+async def _evaluation_evaluators() -> list:
+    from ws_bom_robot_app.llm.evaluator import EvaluatorType
+    return EvaluatorType.all()
+
+@router.post("/evaluation/run", tags=["evaluation"])
+async def _evaluate(rq: EvaluatorRunRequest):
+    from ws_bom_robot_app.llm.evaluator import Evaluator, EvaluatorType
+    from langsmith.schemas import Dataset, Example
+
+    _data: Union[Dataset, list[Example]] = None
+    if rq.example and any(rq.example):
+        _examples: list[Example] = filter(lambda ex: str(ex.id) in [str(e.get("id")) for e in rq.example],
+                   await _evaluation_datasets_by_id(rq.example[0].get("dataset_id"))
+                   )
+
+        _data = list(_examples)
+    else:
+        _data = Dataset(**rq.dataset)
+    evaluator = Evaluator(
+        rq=rq.rq,
+        data=_data,
+        judge_model=rq.judge
+    )
+
+    if not rq.evaluators is None and any(rq.evaluators):
+        def __convert_evaluator_type(evaluator: str) -> EvaluatorType:
+            try:
+                return EvaluatorType[evaluator.upper()]
+            except KeyError:
+                pass
+        _evaluators = []
+        _evaluators.extend(__convert_evaluator_type(evaluator) for evaluator in rq.evaluators)
+        if not any(_evaluators):
+            _evaluators = None
+    else:
+        _evaluators = None
+    result = await evaluator.run(evaluators=_evaluators)
+    return result
+
+@router.post("/evaluation/run/task", tags=["evaluation"])
+async def _evaluate_task(rq: EvaluatorRunRequest, headers: Annotated[TaskHeader, Header()]) -> IdentifiableEntity:
+    return task_manager.create_task(lambda: _evaluate(rq), headers, queue="fast")
+#endregion evaluate

ws_bom_robot_app-0.0.86/ws_bom_robot_app/llm/evaluator.py

@@ -0,0 +1,319 @@
+from uuid import UUID
+import requests, base64
+from typing import Iterator, Optional, List, Union
+from enum import Enum
+from ws_bom_robot_app.config import config
+from ws_bom_robot_app.llm.models.api import LlmMessage, StreamRequest
+from langsmith import Client, traceable
+from langsmith.schemas import Dataset, Example, Feedback, Run
+from openevals.llm import create_llm_as_judge
+from openevals.prompts import CORRECTNESS_PROMPT, RAG_HELPFULNESS_PROMPT, CONCISENESS_PROMPT, RAG_GROUNDEDNESS_PROMPT, HALLUCINATION_PROMPT
+from pydantic import BaseModel
+
+ls_client = Client()
+
+class EvaluatorType(Enum):
+    """Available evaluator types"""
+    CORRECTNESS = "correctness"
+    HELPFULNESS = "helpfulness"
+    CONCISENESS = "conciseness"
+    RAG_GROUNDEDNESS = "rag_groundedness"
+    RAG_HALLUCINATION = "rag_hallucination"
+
+    @classmethod
+    def all(cls) -> List['EvaluatorType']:
+        """Get all available evaluator types"""
+        return list(cls)
+
+    @classmethod
+    def default(cls) -> List['EvaluatorType']:
+        """Get default evaluator types"""
+        return [cls.CORRECTNESS]
+
+class EvaluatorDataSets:
+
+    @classmethod
+    def all(cls) -> List[Dataset]:
+        return list(ls_client.list_datasets())
+    @classmethod
+    def find(cls, name: str) -> List[Dataset]:
+        return [d for d in cls.all() if d.name.lower().__contains__(name.lower())]
+    @classmethod
+    def get(cls, id: Union[str, UUID]) -> Optional[Dataset]:
+        return next((d for d in cls.all() if str(d.id) == str(id)), None)
+    @classmethod
+    def create(cls, name: str) -> Dataset:
+        return ls_client.create_dataset(name=name)
+    @classmethod
+    def delete(cls, id: str) -> None:
+        ls_client.delete_dataset(id=id)
+    @classmethod
+    def example(cls, id: str) -> List[Example]:
+        return list(ls_client.list_examples(dataset_id=id, include_attachments=True))
+    @classmethod
+    def add_example(cls, dataset_id: str, inputs: dict, outputs: dict) -> Example:
+        """Add an example to the dataset.
+        Args:
+            inputs (dict): The input data for the example.
+            outputs (dict): The output data for the example.
+        Sample:
+            - inputs: {"question": "What is the capital of France?"}
+              outputs: {"answer": "Paris"}
+        """
+        return ls_client.create_example(dataset_id=dataset_id, inputs=inputs, outputs=outputs)
+    @classmethod
+    def feedback(cls, experiment_name: str) -> Iterator[Feedback]:
+        return ls_client.list_feedback(
+            run_ids=[r.id for r in ls_client.list_runs(project_name=experiment_name)]
+        )
+
+class Evaluator:
+    def __init__(self, rq: StreamRequest, data: Union[Dataset,List[Example]], judge_model: Optional[str] = None):
+        """Evaluator class for assessing model performance.
+
+        Args:
+            rq (StreamRequest): The request object containing input data.
+            data (Union[Dataset, List[Example]]): The dataset to use for evaluation or a list of examples.
+            judge_model (Optional[str], optional): The model to use for evaluation, defaults to "openai:o4-mini".
+              For a list of available models, see the LangChain documentation:
+              https://python.langchain.com/api_reference/langchain/chat_models/langchain.chat_models.base.init_chat_model.html
+        """
+        self.judge_model: str = judge_model or "openai:o4-mini"
+        self.data = data
+        self.rq: StreamRequest = rq
+
+    #region evaluators
+
+    def _get_evaluator_function(self, evaluator_type: EvaluatorType):
+        """Get the evaluator function for a given type"""
+        evaluator_map = {
+            EvaluatorType.CORRECTNESS: self.correctness_evaluator,
+            EvaluatorType.HELPFULNESS: self.helpfulness_evaluator,
+            EvaluatorType.CONCISENESS: self.conciseness_evaluator,
+            EvaluatorType.RAG_GROUNDEDNESS: self.rag_groundedness_evaluator,
+            EvaluatorType.RAG_HALLUCINATION: self.rag_hallucination_evaluator,
+        }
+        return evaluator_map.get(evaluator_type)
+
+    def correctness_evaluator(self, inputs: dict, outputs: dict, reference_outputs: dict):
+        evaluator = create_llm_as_judge(
+          prompt=CORRECTNESS_PROMPT,
+          feedback_key="correctness",
+          model=self.judge_model,
+          continuous=True,
+          choices=[i/10 for i in range(11)]
+        )
+        return evaluator(
+            inputs=inputs,
+            outputs=outputs,
+            reference_outputs=reference_outputs
+        )
+
+    def helpfulness_evaluator(self, inputs: dict, outputs: dict):
+        evaluator = create_llm_as_judge(
+            prompt=RAG_HELPFULNESS_PROMPT,
+            feedback_key="helpfulness",
+            model=self.judge_model,
+            continuous=True,
+            choices=[i/10 for i in range(11)]
+        )
+        return evaluator(
+            inputs=inputs,
+            outputs=outputs,
+        )
+
+    def conciseness_evaluator(self, inputs: dict, outputs: dict, reference_outputs: dict):
+        evaluator = create_llm_as_judge(
+            prompt=CONCISENESS_PROMPT,
+            feedback_key="conciseness",
+            model=self.judge_model,
+            continuous=True,
+            choices=[i/10 for i in range(11)]
+        )
+        return evaluator(
+            inputs=inputs,
+            outputs=outputs,
+            reference_outputs=reference_outputs
+        )
+
+    def _find_retrievers(self, run: Run) -> List[Run]:
+      retrievers = []
+      for child in getattr(run, "child_runs", []):
+        if child.run_type == "retriever":
+          retrievers.append(child)
+        retrievers.extend(self._find_retrievers(child))
+      return retrievers
+
+    def _retriever_documents(self, retrievers_run: List[Run]) -> str:
+      unique_contents = set()
+      for r in retrievers_run:
+        for doc in r.outputs.get("documents", []):
+          unique_contents.add(doc.page_content)
+      return "\n\n".join(unique_contents)
+
+    def rag_groundedness_evaluator(self, run: Run):
+        evaluator = create_llm_as_judge(
+            prompt=RAG_GROUNDEDNESS_PROMPT,
+            feedback_key="rag_groundedness",
+            model=self.judge_model,
+            continuous=True,
+            choices=[i/10 for i in range(11)]
+        )
+        retrievers_run = self._find_retrievers(run)
+        if retrievers_run:
+            try:
+                return evaluator(
+                    outputs=run.outputs["answer"],
+                    context=self._retriever_documents(retrievers_run)
+                )
+            except Exception as e:
+                return 0.0
+        else:
+            return 0.0
+
+    def rag_hallucination_evaluator(self, inputs: dict, outputs: dict, reference_outputs: dict, run: Run):
+        evaluator = create_llm_as_judge(
+            prompt=HALLUCINATION_PROMPT,
+            feedback_key="rag_hallucination",
+            model=self.judge_model,
+            continuous=True,
+            choices=[i/10 for i in range(11)]
+        )
+        retrievers_run = self._find_retrievers(run)
+        if retrievers_run:
+            try:
+                return evaluator(
+                    inputs=inputs['question'],
+                    outputs=outputs['answer'],
+                    reference_outputs=reference_outputs['answer'],
+                    context=self._retriever_documents(retrievers_run)
+                )
+            except Exception as e:
+                return 0.0
+        else:
+            return 0.0
+
+    #endregion evaluators
+
+    #region target
+    def _parse_rq(self, inputs: dict, attachments: dict) -> StreamRequest:
+        _rq = self.rq.__deepcopy__()
+        if not attachments is None and len(attachments) > 0:
+            _content = []
+            _content.append({"type": "text", "text": inputs["question"]})
+            for k,v in attachments.items():
+                if isinstance(v, dict):
+                    _content.append({"type": ("image" if "image" in v.get("mime_type","") else "file"), "url": v.get("presigned_url","")})
+            _rq.messages = [LlmMessage(role="user", content=_content)]
+        else:
+            _rq.messages = [LlmMessage(role="user", content=inputs["question"])]
+        return _rq
+
+    @traceable(run_type="chain",name="stream_internal")
+    async def target_internal(self,inputs: dict, attachments: dict) -> dict:
+      from ws_bom_robot_app.llm.main import stream
+      from unittest.mock import Mock
+      from fastapi import Request
+      _ctx = Mock(spec=Request)
+      _ctx.base_url.return_value = "http://evaluator"
+      _rq = self._parse_rq(inputs, attachments)
+      _chunks = []
+      async for chunk in stream(rq=_rq, ctx=_ctx, formatted=False):
+          _chunks.append(chunk)
+      _content = ''.join(_chunks) if _chunks else ""
+      del _rq, _chunks
+      return { "answer": _content.strip() }
+
+    @traceable(run_type="chain",name="stream_http")
+    async def target_http(self,inputs: dict, attachments: dict) -> dict:
+      _rq = self._parse_rq(inputs, attachments)
+      _host= "http://localhost:6001"
+      _endpoint = f"{_host}/api/llm/stream/raw"
+      _robot_auth =f"Basic {base64.b64encode((config.robot_user + ':' + config.robot_password).encode('utf-8')).decode('utf-8')}"
+      _rs = requests.post(_endpoint, data=_rq.model_dump_json(), stream=True, headers={"Authorization": _robot_auth}, verify=True)
+      _content = ''.join([chunk.decode('utf-8') for chunk in _rs.iter_content(chunk_size=1024, decode_unicode=False)])
+      del _rq, _rs
+      return { "answer": _content.strip() }
+    #endregion target
+
+    async def run(self,
+                  evaluators: Optional[List[EvaluatorType]] = None,
+                  target_method: str = "target_internal") -> dict:
+        """Run evaluation with specified evaluators
+
+        Args:
+            evaluators: List of evaluator types to use. If None, uses default (correctness only)
+            target_method: Method to use for target evaluation ("target_internal" or "target")
+
+        Returns:
+            dict: Evaluation results with scores
+
+        Usage:
+          ```
+          await evaluator.run()  # Uses default (correctness only)
+          await evaluator.run([EvaluatorType.CORRECTNESS, EvaluatorType.HELPFULNESS])
+          await evaluator.run(EvaluatorType.all())  # Uses all available evaluators
+          ```
+        """
+        try:
+          # evaluator functions
+          evaluator_functions = []
+          if evaluators is None:
+              evaluators = EvaluatorType.default()
+          for eval_type in evaluators:
+              func = self._get_evaluator_function(eval_type)
+              if func:
+                  evaluator_functions.append(func)
+              else:
+                  print(f"Warning: Unknown evaluator type: {eval_type}")
+          if not evaluator_functions:
+              print("No valid evaluators provided, using default (correctness)")
+              evaluator_functions = [self.correctness_evaluator]
+
+          # target method
+          target_func = getattr(self, target_method, self.target_internal)
+
+          # run
+          _dataset: Dataset = self.data if isinstance(self.data, Dataset) else EvaluatorDataSets.get(self.data[0].dataset_id)
+          experiment = await ls_client.aevaluate(
+              target_func,
+              data=_dataset.name if isinstance(self.data, Dataset) else self.data,
+              evaluators=evaluator_functions,
+              experiment_prefix=_dataset.name,
+              upload_results=True,
+              max_concurrency=4,
+              metadata={
+                  "app": _dataset.name,
+                  "model": f"{self.rq.provider}:{self.rq.model}",
+                  "judge": self.judge_model,
+                  "evaluators": [e.value for e in evaluators]
+              }
+          )
+          feedback = list(EvaluatorDataSets.feedback(experiment.experiment_name))
+          scores = [f.score for f in feedback]
+          url = f"{ls_client._host_url}/o/{ls_client._tenant_id}/datasets/{_dataset.id}/compare?selectedSessions={feedback[0].session_id}"
+
+          # group scores by evaluator type
+          evaluator_scores = {}
+          for i, eval_type in enumerate(evaluators):
+              eval_scores = [f.score for f in feedback if f.key.lower() == eval_type.value.lower()]
+              if eval_scores:
+                  evaluator_scores[eval_type.value] = sum(eval_scores) / len(eval_scores)
+
+          return {
+              "experiment": {"name": experiment.experiment_name, "url": url},
+              "overall_score": sum(scores) / len(scores) if scores else 0,
+              "evaluator_scores": evaluator_scores
+          }
+        except Exception as e:
+            from traceback import print_exc
+            print(f"Error occurred during evaluation: {e}")
+            print_exc()
+            return {"error": str(e)}
+
+class EvaluatorRunRequest(BaseModel):
+    dataset: dict
+    rq: StreamRequest
+    example: Optional[List[dict]] = None
+    evaluators: Optional[List[str]] = None
+    judge: Optional[str] = None

{ws_bom_robot_app-0.0.84 → ws_bom_robot_app-0.0.86}/ws_bom_robot_app/llm/models/api.py

@@ -163,7 +163,7 @@ class LlmApp(BaseModel):
       return list(set(
           os.path.basename(db) for db in [self.vector_db] +
           ([self.rules.vector_db] if self.rules and self.rules.vector_db else []) +
-          [db for tool in (self.app_tools or []) for db in [tool.vector_db]]
+          [db for tool in (self.app_tools or []) for db in [tool.vector_db] if tool.is_active]
           if db is not None
       ))
   def __decompress_zip(self,zip_file_path, extract_to):