divi 0.0.1.dev23__tar.gz → 0.0.1.dev28__tar.gz

{divi-0.0.1.dev23 → divi-0.0.1.dev28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: divi
-Version: 0.0.1.dev23
+Version: 0.0.1.dev28
 Summary: The Agent Platform for Observability & Evaluation
 License-File: LICENSE
 Requires-Python: >=3.11

{divi-0.0.1.dev23 → divi-0.0.1.dev28}/apps/graphql/package.json

@@ -13,7 +13,7 @@
     "@apollo/datasource-rest": "^6.4.1",
     "@apollo/server": "^4.12.0",
     "@apollo/utils.keyvaluecache": "^3.1.0",
-    "graphql": "^16.10.0",
+    "graphql": "^16.11.0",
     "graphql-tag": "^2.12.6"
   },
   "devDependencies": {
@@ -21,7 +21,7 @@
     "@graphql-codegen/typescript": "^4.1.6",
     "@graphql-codegen/typescript-resolvers": "^4.5.0",
     "@workspace/typescript-config": "workspace:*",
-    "nodemon": "^3.1.9",
+    "nodemon": "^3.1.10",
     "ts-node-dev": "^2.0.0",
     "typescript": "catalog:"
   }

{divi-0.0.1.dev23 → divi-0.0.1.dev28}/apps/web/package.json

@@ -2,6 +2,7 @@
   "name": "@app/web",
   "private": true,
   "type": "module",
+  "packageManager": "pnpm@10.6.5",
   "scripts": {
     "dev": "next dev --turbopack --port 4001",
     "build": "next build",
@@ -9,18 +10,19 @@
     "analyze": "ANALYZE=true pnpm build"
   },
   "dependencies": {
-    "@apollo/client": "^3.13.7",
+    "@apollo/client": "^3.13.8",
     "@apollo/experimental-nextjs-app-support": "^0.11.11",
     "@dnd-kit/core": "^6.3.1",
     "@dnd-kit/modifiers": "^9.0.0",
     "@dnd-kit/sortable": "^10.0.0",
     "@dnd-kit/utilities": "^3.2.2",
     "@hookform/resolvers": "^4.1.3",
-    "@icons-pack/react-simple-icons": "^12.6.0",
+    "@icons-pack/react-simple-icons": "^12.7.0",
     "@tabler/icons-react": "^3.31.0",
-    "@tanstack/react-table": "^8.21.2",
+    "@tanstack/react-table": "^8.21.3",
     "@workspace/graphql-client": "workspace:*",
     "@workspace/ui": "workspace:*",
+    "date-fns": "catalog:",
     "highlight.js": "^11.11.1",
     "lucide-react": "catalog:",
     "next": "catalog:",
@@ -28,12 +30,11 @@
     "react": "catalog:",
     "react-day-picker": "catalog:",
     "react-dom": "catalog:",
-    "react-hook-form": "^7.55.0",
-    "recharts": "^2.15.2",
+    "react-hook-form": "^7.56.1",
+    "recharts": "^2.15.3",
     "server-only": "^0.0.1",
     "sonner": "catalog:",
-    "zod": "^3.24.2",
-    "date-fns": "catalog:"
+    "zod": "^3.24.3"
   },
   "devDependencies": {
     "@types/highlightjs": "^9.12.6",
@@ -41,7 +42,7 @@
     "@types/react": "catalog:",
     "@types/react-dom": "catalog:",
     "@workspace/typescript-config": "workspace:*",
-    "openai": "^4.93.0",
+    "openai": "^4.96.0",
     "typescript": "catalog:"
   }
 }

{divi-0.0.1.dev23 → divi-0.0.1.dev28}/divi/__init__.py

@@ -2,6 +2,7 @@ from typing import Optional
 
 from . import proto
 from .decorators import obs_openai, observable
+from .evaluation import Evaluator, Score
 from .services import Auth, Core, DataPark
 from .session import Session
 
@@ -11,6 +12,7 @@ _session: Optional[Session] = None
 _core: Optional[Core] = None
 _auth: Optional[Auth] = None
 _datapark: Optional[DataPark] = None
+_evaluator: Optional[Evaluator] = None
 
-__version__ = "0.0.1.dev23"
-__all__ = ["proto", "obs_openai", "observable"]
+__version__ = "0.0.1.dev28"
+__all__ = ["proto", "obs_openai", "observable", "Score"]

{divi-0.0.1.dev23 → divi-0.0.1.dev28}/divi/decorators/obs_openai.py

@@ -2,7 +2,10 @@ import functools
 from collections.abc import Callable
 from typing import TYPE_CHECKING, TypeVar, Union
 
+from typing_extensions import Optional
+
 from divi.decorators.observable import observable
+from divi.evaluation.scores import Score
 from divi.utils import is_async
 
 if TYPE_CHECKING:
@@ -11,22 +14,34 @@ if TYPE_CHECKING:
 C = TypeVar("C", bound=Union["OpenAI", "AsyncOpenAI"])
 
 
-def _get_observable_create(create: Callable) -> Callable:
+def _get_observable_create(
+    create: Callable,
+    name: Optional[str] = None,
+    scores: Optional[list[Score]] = None,
+) -> Callable:
     @functools.wraps(create)
     def observable_create(*args, stream: bool = False, **kwargs):
-        decorator = observable(kind="llm")
+        decorator = observable(kind="llm", name=name, scores=scores)
         return decorator(create)(*args, stream=stream, **kwargs)
 
     # TODO Async Observable Create
     return observable_create if not is_async(create) else create
 
 
-def obs_openai(client: C) -> C:
+def obs_openai(
+    client: C,
+    name: Optional[str] = "Chat Bot",
+    scores: Optional[list[Score]] = None,
+) -> C:
     """Make OpenAI client observable."""
     client.chat.completions.create = _get_observable_create(
-        client.chat.completions.create
+        client.chat.completions.create,
+        name=name,
+        scores=scores,
     )
     client.completions.create = _get_observable_create(
-        client.completions.create
+        client.completions.create,
+        name=name,
+        scores=scores,
     )
     return client

{divi-0.0.1.dev23 → divi-0.0.1.dev28}/divi/decorators/observable.py

@@ -1,11 +1,9 @@
 import contextvars
 import functools
-import inspect
 from typing import (
     Any,
     Callable,
     Generic,
-    List,
     Mapping,
     Optional,
     ParamSpec,
@@ -19,6 +17,8 @@ from typing import (
 from openai.types.chat import ChatCompletion
 
 import divi
+from divi.evaluation.evaluate import evaluate_scores
+from divi.evaluation.scores import Score
 from divi.proto.trace.v1.trace_pb2 import ScopeSpans
 from divi.session import SessionExtra
 from divi.session.setup import setup
@@ -54,6 +54,7 @@ def observable(
     kind: str = "function",
     *,
     name: Optional[str] = None,
+    scores: Optional[list[Score]] = None,
     metadata: Optional[Mapping[str, Any]] = None,
 ) -> Callable[[Callable[P, R]], WithSessionExtra[P, R]]: ...
 
@@ -66,6 +67,7 @@ def observable(
     kind = kwargs.pop("kind", "function")
     name = kwargs.pop("name", None)
     metadata = kwargs.pop("metadata", None)
+    scores: list[Score] = kwargs.pop("scores", None)
 
     def decorator(func):
         @functools.wraps(func)
@@ -88,7 +90,7 @@ def observable(
             # get the trace to collect data
             trace = session_extra.get("trace")
             if not trace:
-                raise ValueError("Trace not found in session_extra")
+                raise ValueError("Trace not found in session context.")
             # TODO: collect inputs and outputs for SPAN_KIND_FUNCTION
             inputs = extract_flattened_inputs(func, *args, **kwargs)
             # create the span if it is the root span
@@ -107,6 +109,9 @@ def observable(
                     inputs=inputs,
                     completion=result,
                 )
+                # evaluate the scores if they are provided
+                if scores is not None and scores.__len__() > 0:
+                    evaluate_scores(inputs, outputs=result, scores=scores)
 
             return result
 

divi-0.0.1.dev28/divi/evaluation/__init__.py

@@ -0,0 +1,4 @@
+from .evaluator import Evaluator
+from .scores import Score
+
+__all__ = ["Evaluator", "Score"]

divi-0.0.1.dev28/divi/evaluation/evaluate.py

@@ -0,0 +1,57 @@
+import copy
+import os
+from typing import Any, Dict, Optional
+
+from openai.types.chat import ChatCompletion
+from typing_extensions import List
+
+import divi
+from divi.evaluation import Evaluator
+from divi.evaluation.evaluator import EvaluatorConfig
+from divi.evaluation.scores import Score
+
+OPENAI_API_KEY = "OPENAI_API_KEY"
+OPENAI_BASE_URL = "OPENAI_BASE_URL"
+
+
+def init_evaluator(config: Optional[EvaluatorConfig] = None):
+    _config = config or EvaluatorConfig()
+    api_key = _config.api_key if _config.api_key else os.getenv(OPENAI_API_KEY)
+    base_url = (
+        _config.base_url if _config.base_url else os.getenv(OPENAI_BASE_URL)
+    )
+    if api_key is None:
+        raise ValueError("API key is required for evaluator")
+    _config.api_key = api_key
+    _config.base_url = base_url
+    evaluator = Evaluator(_config)
+    return evaluator
+
+
+def evaluate_scores(
+    inputs: Dict[str, Any],
+    outputs: ChatCompletion,
+    scores: List[Score],
+    config: Optional[EvaluatorConfig] = None,
+):
+    if not divi._evaluator:
+        divi._evaluator = init_evaluator(config)
+
+    # create conversation with result and inputs
+    input_messages = inputs.get("messages", None)
+    if input_messages is None:
+        raise ValueError("No messages found in inputs")
+    output_message = outputs.choices[0].message
+    if output_message is None:
+        raise ValueError("No message found in outputs")
+
+    conversations = copy.deepcopy(input_messages)
+    conversations.append(
+        {"role": output_message.role, "content": output_message.content}
+    )
+    evaluation_scores = divi._evaluator.evaluate(
+        "\n".join(f"{m['role']}: {m['content']}" for m in conversations), scores
+    )
+
+    # TODO: collect all evaluation scores and link them to span
+    print(evaluation_scores)

divi-0.0.1.dev28/divi/evaluation/evaluator.py

@@ -0,0 +1,161 @@
+import asyncio
+import concurrent.futures
+import random
+from typing import List, Literal, Optional
+
+import openai
+from pydantic import BaseModel
+
+from divi.evaluation.prompts import PRESET_PROMPT, PROMPT_TEMPLATE
+from divi.evaluation.scores import Score
+
+
+class EvaluatorConfig:
+    def __init__(
+        self,
+        model: str = "gpt-4.1-nano",
+        temperature: float = 0.5,
+        max_concurrency: int = 10,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+    ):
+        self.model = model
+        self.api_key = api_key
+        self.base_url = base_url
+        self.temperature = temperature
+        self.max_concurrency = max_concurrency
+
+
+class EvaluationResult(BaseModel):
+    name: Score
+    judgment: bool
+    reasoning: str
+
+
+class EvaluationScore(BaseModel):
+    name: Score
+    score: float
+    representative_reasoning: str
+    all_evaluations: List[EvaluationResult]
+
+
+class Evaluator:
+    def __init__(self, config: Optional[EvaluatorConfig] = None):
+        self.config = config or EvaluatorConfig()
+        self.async_client = openai.AsyncOpenAI(
+            api_key=self.config.api_key, base_url=self.config.base_url
+        )
+        self.sync_client = openai.OpenAI(
+            api_key=self.config.api_key, base_url=self.config.base_url
+        )
+
+    @staticmethod
+    def generate_prompt(conversation: str, score: Score) -> str:
+        return PROMPT_TEMPLATE.format(
+            requirements=PRESET_PROMPT[score.value], conversation=conversation
+        )
+
+    def _sync_evaluate_once(
+        self, conversation: str, score: Score
+    ) -> Optional[EvaluationResult]:
+        prompt = self.generate_prompt(conversation, score)
+        response = self.sync_client.beta.chat.completions.parse(
+            model=self.config.model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=self.config.temperature,
+            response_format=EvaluationResult,
+        )
+        return response.choices[0].message.parsed
+
+    async def _async_evaluate_once(
+        self, conversation: str, score: Score
+    ) -> Optional[EvaluationResult]:
+        prompt = self.generate_prompt(conversation, score)
+        response = await self.async_client.beta.chat.completions.parse(
+            model=self.config.model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=self.config.temperature,
+            response_format=EvaluationResult,
+        )
+        return response.choices[0].message.parsed
+
+    def _aggregate_result(
+        self, name: Score, evaluations: List[EvaluationResult]
+    ) -> EvaluationScore:
+        n = len(evaluations)
+        true_count = sum(1 for e in evaluations if e.judgment is True)
+        score = true_count / n
+        majority_judgment = True if true_count >= (n / 2) else False
+        majority_reasons = [
+            e.reasoning for e in evaluations if e.judgment == majority_judgment
+        ]
+        representative_reasoning = (
+            random.choice(majority_reasons) if majority_reasons else ""
+        )
+        return EvaluationScore(
+            name=name,
+            score=score,
+            representative_reasoning=representative_reasoning,
+            all_evaluations=evaluations,
+        )
+
+    def _aggregate_results(
+        self, evaluations: List[EvaluationResult]
+    ) -> List[EvaluationScore]:
+        scores = {}
+        for evaluation in evaluations:
+            if evaluation.name not in scores:
+                scores[evaluation.name] = []
+            scores[evaluation.name].append(evaluation)
+
+        aggregated_results = [
+            self._aggregate_result(name, evals)
+            for name, evals in scores.items()
+        ]
+        return aggregated_results
+
+    def evaluate_sync(
+        self, conversation: str, scores: list[Score], n_rounds: int
+    ) -> List[EvaluationScore]:
+        with concurrent.futures.ThreadPoolExecutor(
+            max_workers=self.config.max_concurrency
+        ) as executor:
+            futures = [
+                executor.submit(self._sync_evaluate_once, conversation, score)
+                for _ in range(n_rounds)
+                for score in scores
+            ]
+            evaluations = [
+                f.result() for f in concurrent.futures.as_completed(futures)
+            ]
+        return self._aggregate_results(
+            [e for e in evaluations if e is not None]
+        )
+
+    async def evaluate_async(
+        self, conversation: str, scores: list[Score], n_rounds: int
+    ) -> List[EvaluationScore]:
+        semaphore = asyncio.Semaphore(self.config.max_concurrency)
+
+        async def sem_task(score):
+            async with semaphore:
+                return await self._async_evaluate_once(conversation, score)
+
+        tasks = [sem_task(score) for _ in range(n_rounds) for score in scores]
+        evaluations = await asyncio.gather(*tasks)
+        return self._aggregate_results(
+            [e for e in evaluations if e is not None]
+        )
+
+    def evaluate(
+        self,
+        conversation: str,
+        scores: list[Score],
+        n_rounds: int = 5,
+        mode: Literal["sync", "async"] = "sync",
+    ) -> List[EvaluationScore]:
+        if mode == "async":
+            return asyncio.run(
+                self.evaluate_async(conversation, scores, n_rounds)
+            )
+        return self.evaluate_sync(conversation, scores, n_rounds)

divi-0.0.1.dev28/divi/evaluation/prompts.py

@@ -0,0 +1,14 @@
+PROMPT_TEMPLATE = (
+    "Your evaluation task is: {requirements}\n\n"
+    "Please perform step-by-step reasoning to reach your judgment.\n\n"
+    "Strictly output your answer in the following JSON format:\n"
+    '{{"judgment": bool, "reasoning": "string"}}\n\n'
+    "Do not output anything else.\n\n"
+    "Here is the conversation to evaluate:\n"
+    "{conversation}"
+)
+
+PRESET_PROMPT = {
+    "task_completion": "Evaluate whether the model's output completely fulfills the user's task requirements.",
+    "instruction_adherence": "Evaluate whether the model's output strictly follows the user's instructions without omissions, deviations, or hallucinations.",
+}

divi-0.0.1.dev28/divi/evaluation/scores.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class Score(str, Enum):
+    """Enum for score types."""
+
+    task_completion = "task_completion"
+    instruction_adherence = "instruction_adherence"

{divi-0.0.1.dev23 → divi-0.0.1.dev28}/divi/services/init.py

@@ -1,6 +1,7 @@
 import divi
 from divi.services.auth import init as init_auth
-from divi.services.core import init as init_core
+
+# from divi.services.core import init as init_core
 from divi.services.datapark import init as init_datapark
 
 

{divi-0.0.1.dev23 → divi-0.0.1.dev28}/divi/session/setup.py

@@ -1,4 +1,5 @@
 from typing_extensions import Optional
+
 import divi
 from divi.services import init as init_services
 from divi.session import Session, SessionExtra
@@ -6,7 +7,7 @@ from divi.signals.trace import Span
 from divi.signals.trace.trace import Trace
 
 
-def init_session(name: Optional[str]=None) -> Session:
+def init_session(name: Optional[str] = None) -> Session:
     """init initializes the services and the Run"""
     init_services()
     session = Session(name=name)
@@ -29,7 +30,9 @@ def setup(
 
     # init the session if not already initialized
     if not divi._session:
-        divi._session = init_session(name=session_extra.get('session_name') or span.name)
+        divi._session = init_session(
+            name=session_extra.get("session_name") or span.name
+        )
 
     # setup trace
     trace = session_extra.get("trace") or Trace(divi._session.id, span.name)

{divi-0.0.1.dev23 → divi-0.0.1.dev28}/package.json

@@ -13,7 +13,7 @@
   "devDependencies": {
     "@biomejs/biome": "1.9.4",
     "@workspace/typescript-config": "workspace:*",
-    "turbo": "^2.5.0",
+    "turbo": "^2.5.2",
     "typescript": "5.7.3",
     "ultracite": "^4.2.1"
   },