divi 0.0.1b0__py3-none-any.whl → 0.0.1b1__py3-none-any.whl

divi/README.md

@@ -0,0 +1,3 @@
+# DIVI SDK
+
+> this is the Python SDK for the divine-agent project.

divi/__init__.py

@@ -1,10 +1,9 @@
 from typing import Optional
 
 from .decorators import obs_openai, observable
-from .evaluation import Evaluator, Score
+from .evaluation import Evaluator
 from .services import Auth, Core, DataPark
 from .session import Session
-from .signals import Kind
 
 name: str = "divi"
 
@@ -14,5 +13,5 @@ _auth: Optional[Auth] = None
 _datapark: Optional[DataPark] = None
 _evaluator: Optional[Evaluator] = None
 
-__version__ = "0.0.1b0"
-__all__ = ["obs_openai", "observable", "Score", "Kind"]
+__version__ = "0.0.1b1"
+__all__ = ["obs_openai", "observable"]

divi/decorators/obs_openai.py

@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, TypeVar, Union
 from typing_extensions import Optional
 
 from divi.decorators.observable import observable
+from divi.evaluation.evaluator import EvaluatorConfig
 from divi.evaluation.scores import Score
 from divi.signals.span import Kind
 from divi.utils import is_async
@@ -19,10 +20,13 @@ def _get_observable_create(
     create: Callable,
     name: Optional[str] = None,
     scores: Optional[list[Score]] = None,
+    eval: Optional[EvaluatorConfig] = None,
 ) -> Callable:
     @functools.wraps(create)
     def observable_create(*args, stream: bool = False, **kwargs):
-        decorator = observable(kind=Kind.llm, name=name, scores=scores)
+        decorator = observable(
+            kind=Kind.llm, name=name, scores=scores, eval=eval
+        )
         return decorator(create)(*args, stream=stream, **kwargs)
 
     # TODO Async Observable Create
@@ -33,16 +37,13 @@ def obs_openai(
     client: C,
     name: Optional[str] = "Agent",
     scores: Optional[list[Score]] = None,
+    eval: Optional[EvaluatorConfig] = None,
 ) -> C:
     """Make OpenAI client observable."""
     client.chat.completions.create = _get_observable_create(
         client.chat.completions.create,
         name=name,
         scores=scores,
-    )
-    client.completions.create = _get_observable_create(
-        client.completions.create,
-        name=name,
-        scores=scores,
+        eval=eval,
     )
     return client

divi/decorators/observable.py

@@ -15,6 +15,7 @@ from typing import (
 
 from divi.decorators.observe import observe
 from divi.evaluation.evaluate import evaluate_scores
+from divi.evaluation.evaluator import EvaluatorConfig
 from divi.evaluation.scores import Score
 from divi.session import SessionExtra
 from divi.signals.span import Kind, Span
@@ -43,6 +44,7 @@ def observable(
     *,
     name: Optional[str] = None,
     scores: Optional[list[Score]] = None,
+    eval: Optional[EvaluatorConfig] = None,
     metadata: Optional[Mapping[str, Any]] = None,
 ) -> Callable[[Callable[P, R]], WithSessionExtra[P, R]]: ...
 
@@ -56,6 +58,7 @@ def observable(
     name = kwargs.pop("name", None)
     metadata = kwargs.pop("metadata", None)
     scores: list[Score] = kwargs.pop("scores", None)
+    eval: EvaluatorConfig = kwargs.pop("eval", None)
 
     def decorator(func):
         @functools.wraps(func)
@@ -78,7 +81,9 @@ def observable(
 
             # 3. evaluate the scores if they are provided
             messages = kwargs.get("messages", [])
-            evaluate_scores(messages, outputs=result, scores=scores)
+            evaluate_scores(
+                messages, outputs=result, scores=scores, config=eval
+            )
 
             return result
 

divi/evaluation/__init__.py

@@ -1,4 +1,4 @@
-from .evaluator import Evaluator
+from .evaluator import Evaluator, EvaluatorConfig
 from .scores import Score
 
-__all__ = ["Evaluator", "Score"]
+__all__ = ["Evaluator", "Score", "EvaluatorConfig"]

divi/evaluation/evaluator.py

@@ -13,8 +13,9 @@ from divi.evaluation.scores import Score
 class EvaluatorConfig:
     def __init__(
         self,
-        model: str = "gpt-4.1-nano",
+        model: str = "gpt-4o",
         temperature: float = 0.5,
+        n_rounds: int = 5,
         max_concurrency: int = 10,
         api_key: Optional[str] = None,
         base_url: Optional[str] = None,
@@ -23,6 +24,7 @@ class EvaluatorConfig:
         self.api_key = api_key
         self.base_url = base_url
         self.temperature = temperature
+        self.n_rounds = n_rounds
         self.max_concurrency = max_concurrency
 
 
@@ -123,7 +125,11 @@ class Evaluator:
         return aggregated_results
 
     def evaluate_sync(
-        self, target: str, conversation: str, scores: list[Score], n_rounds: int
+        self,
+        target: str,
+        conversation: str,
+        scores: list[Score],
+        n_rounds: Optional[int] = None,
     ) -> List[EvaluationScore]:
         with concurrent.futures.ThreadPoolExecutor(
             max_workers=self.config.max_concurrency
@@ -132,7 +138,7 @@ class Evaluator:
                 executor.submit(
                     self._sync_evaluate_once, target, conversation, score
                 )
-                for _ in range(n_rounds)
+                for _ in range(n_rounds if n_rounds else self.config.n_rounds)
                 for score in scores
             ]
             evaluations = [
@@ -143,7 +149,11 @@ class Evaluator:
         )
 
     async def evaluate_async(
-        self, target: str, conversation: str, scores: list[Score], n_rounds: int
+        self,
+        target: str,
+        conversation: str,
+        scores: list[Score],
+        n_rounds: Optional[int] = None,
     ) -> List[EvaluationScore]:
         semaphore = asyncio.Semaphore(self.config.max_concurrency)
 
@@ -153,7 +163,11 @@ class Evaluator:
                     target, conversation, score
                 )
 
-        tasks = [sem_task(score) for _ in range(n_rounds) for score in scores]
+        tasks = [
+            sem_task(score)
+            for _ in range(n_rounds if n_rounds else self.config.n_rounds)
+            for score in scores
+        ]
         evaluations = await asyncio.gather(*tasks)
         return self._aggregate_results(
             [e for e in evaluations if e is not None]
@@ -164,7 +178,7 @@ class Evaluator:
         target: str,
         conversation: str,
         scores: list[Score],
-        n_rounds: int = 5,
+        n_rounds: Optional[int] = None,
         mode: Literal["sync", "async"] = "sync",
     ) -> List[EvaluationScore]:
         if mode == "async":

{divi-0.0.1b0.dist-info → divi-0.0.1b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: divi
-Version: 0.0.1b0
+Version: 0.0.1b1
 Summary: The Agent Platform for Observability & Evaluation
 License-File: LICENSE
 Requires-Python: >=3.11

{divi-0.0.1b0.dist-info → divi-0.0.1b1.dist-info}/RECORD

@@ -1,13 +1,14 @@
-divi/__init__.py,sha256=Mxu0UYIKgegwUU7gTe5jMHYZ0wf3rzwtiHfHyLcTysA,486
+divi/README.md,sha256=hdw10Mkebd1SGMeA3tPJtlOwbWZzuQ8J-N9YPEHGuXg,67
+divi/__init__.py,sha256=XTB0By729PUQY9TYgoQXrzfQ2fiNEXRsx2mrA4A-83Y,436
 divi/utils.py,sha256=fXkjoyo_Lh8AZliKICOP460m0czUcNQjcEcceJbaOVA,1439
 divi/decorators/__init__.py,sha256=HkyWdC1ctTsVFucCWCkj57JB4NmwONus1d2S2dUbvs4,110
 divi/decorators/collect.py,sha256=5iUxAnbHYx4ISkFg64IK_4miGdrWgbOXLJxKz8lGIv8,1074
-divi/decorators/obs_openai.py,sha256=ouw3GYDFg6S27tcUzY0dIqz8JX_JM8IOXttzo7HK7nk,1359
-divi/decorators/observable.py,sha256=isUS3P_07wbZBj2UcRAoYNDceQTIn6zdein3-PWVsi8,2289
+divi/decorators/obs_openai.py,sha256=eJkPL7T9ribA8Utl8RLvVsp1dRKbS3-Q4vwDfPz16Sg,1414
+divi/decorators/observable.py,sha256=orDNCP1_uOTwgg71lg2c0qXOv6DtcxufRzuQZqXOrH4,2483
 divi/decorators/observe.py,sha256=I2RVsp2WQep6iTLSxkAlMP8wiRsSYiiYrxR2hJzPxcI,1211
-divi/evaluation/__init__.py,sha256=3qMHWu_zBh6FJa6-1dZZEWiAblQZurn5doa0OjGvDGs,93
+divi/evaluation/__init__.py,sha256=SVu4tYZLgx4juEa1JtkXGcwB_Xuwvh60lbVhYKMseQo,129
 divi/evaluation/evaluate.py,sha256=lVMCw5vHGa5sJvUyhVDZ9m3Sgl4baCjWhw2OKazhvgM,1861
-divi/evaluation/evaluator.py,sha256=ulTyfSg2JXxzCCL7hRsn-EBb9UKcpQFA6rVT42mouVQ,5819
+divi/evaluation/evaluator.py,sha256=D1s6upIOiGTjf24WAQfygYYUI8Lcc65vRgJ2wW2_3Uo,6108
 divi/evaluation/prompts.py,sha256=4oL8K8X2-zdmTDuAetc53o7Ys_vph-eWYhPmvRKYAng,960
 divi/evaluation/scores.py,sha256=ZgSxfve-ZivX3WU4TGcgPOSpUQVMbG5a15IQNPeq_bQ,173
 divi/proto/common/v1/common.proto,sha256=Rx8wr0_tOtQ1NseTMnsav4ApD1MDALzQDBA2IvLRTU0,1775
@@ -45,7 +46,7 @@ divi/session/teardown.py,sha256=YiBz_3yCiljMFEofZ60VmRL5sb8WA5GT7EYF8nFznZ4,133
 divi/signals/__init__.py,sha256=wfSkkCwkRsFP4aLj8aGHk_k6Y50P5yN44WWlO3XyW18,43
 divi/signals/span.py,sha256=FQWql6ivAeXGk1HPZCsCjL5mXW6S6Nn9SmOiKH4aXik,2629
 divi/signals/trace.py,sha256=IoYeTfd6x_Xmxcp4HbFSEne0d48hol4ng2Mb_AO8hZw,2144
-divi-0.0.1b0.dist-info/METADATA,sha256=BW1GBcbxovWLvVJGoDylvomnBVaApGtJkTSgHIRBtuw,493
-divi-0.0.1b0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-divi-0.0.1b0.dist-info/licenses/LICENSE,sha256=5OJuZ4wMMEV0DgF0tofhAlS_KLkaUsZwwwDS2U_GwQ0,1063
-divi-0.0.1b0.dist-info/RECORD,,
+divi-0.0.1b1.dist-info/METADATA,sha256=-qCsxelGi6SeeA6x3aodNtiI_Z4GWcLUagFb5LWrfNQ,493
+divi-0.0.1b1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+divi-0.0.1b1.dist-info/licenses/LICENSE,sha256=5OJuZ4wMMEV0DgF0tofhAlS_KLkaUsZwwwDS2U_GwQ0,1063
+divi-0.0.1b1.dist-info/RECORD,,