divi 0.0.1b0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- divi/__init__.py +18 -0
- divi/decorators/__init__.py +4 -0
- divi/decorators/collect.py +34 -0
- divi/decorators/obs_openai.py +48 -0
- divi/decorators/observable.py +91 -0
- divi/decorators/observe.py +47 -0
- divi/evaluation/__init__.py +4 -0
- divi/evaluation/evaluate.py +61 -0
- divi/evaluation/evaluator.py +174 -0
- divi/evaluation/prompts.py +19 -0
- divi/evaluation/scores.py +8 -0
- divi/proto/common/v1/common.proto +49 -0
- divi/proto/common/v1/common_pb2.py +43 -0
- divi/proto/common/v1/common_pb2.pyi +44 -0
- divi/proto/core/health/v1/health_service.proto +17 -0
- divi/proto/core/health/v1/health_service_pb2.py +41 -0
- divi/proto/core/health/v1/health_service_pb2.pyi +19 -0
- divi/proto/core/health/v1/health_service_pb2_grpc.py +100 -0
- divi/proto/metric/v1/metric.proto +26 -0
- divi/proto/metric/v1/metric_pb2.py +40 -0
- divi/proto/metric/v1/metric_pb2.pyi +25 -0
- divi/proto/trace/v1/trace.proto +50 -0
- divi/proto/trace/v1/trace_pb2.py +42 -0
- divi/proto/trace/v1/trace_pb2.pyi +42 -0
- divi/services/__init__.py +7 -0
- divi/services/auth/__init__.py +4 -0
- divi/services/auth/auth.py +13 -0
- divi/services/auth/init.py +22 -0
- divi/services/auth/tokman.py +42 -0
- divi/services/core/__init__.py +5 -0
- divi/services/core/core.py +35 -0
- divi/services/core/finish.py +15 -0
- divi/services/core/init.py +70 -0
- divi/services/datapark/__init__.py +4 -0
- divi/services/datapark/datapark.py +91 -0
- divi/services/datapark/init.py +21 -0
- divi/services/finish.py +9 -0
- divi/services/init.py +10 -0
- divi/services/service.py +54 -0
- divi/session/__init__.py +3 -0
- divi/session/session.py +40 -0
- divi/session/setup.py +48 -0
- divi/session/teardown.py +7 -0
- divi/signals/__init__.py +3 -0
- divi/signals/span.py +83 -0
- divi/signals/trace.py +79 -0
- divi/utils.py +49 -0
- divi-0.0.1b0.dist-info/METADATA +18 -0
- divi-0.0.1b0.dist-info/RECORD +51 -0
- divi-0.0.1b0.dist-info/WHEEL +4 -0
- divi-0.0.1b0.dist-info/licenses/LICENSE +21 -0
divi/__init__.py
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
from typing import Optional
|
2
|
+
|
3
|
+
from .decorators import obs_openai, observable
|
4
|
+
from .evaluation import Evaluator, Score
|
5
|
+
from .services import Auth, Core, DataPark
|
6
|
+
from .session import Session
|
7
|
+
from .signals import Kind
|
8
|
+
|
9
|
+
name: str = "divi"
|
10
|
+
|
11
|
+
_session: Optional[Session] = None
|
12
|
+
_core: Optional[Core] = None
|
13
|
+
_auth: Optional[Auth] = None
|
14
|
+
_datapark: Optional[DataPark] = None
|
15
|
+
_evaluator: Optional[Evaluator] = None
|
16
|
+
|
17
|
+
__version__ = "0.0.1b0"
|
18
|
+
__all__ = ["obs_openai", "observable", "Score", "Kind"]
|
@@ -0,0 +1,34 @@
|
|
1
|
+
from typing import Any
|
2
|
+
|
3
|
+
from google.protobuf.message import Error
|
4
|
+
from openai.types.chat import ChatCompletion
|
5
|
+
from typing_extensions import Dict
|
6
|
+
|
7
|
+
import divi
|
8
|
+
from divi.evaluation.evaluator import EvaluationScore
|
9
|
+
from divi.signals.span import Span
|
10
|
+
|
11
|
+
|
12
|
+
def collect(span: Span, input: Dict[str, Any], result: Any):
|
13
|
+
if not divi._datapark or span.trace_id is None:
|
14
|
+
raise Error("divi._datapark or span.trace_id is None")
|
15
|
+
# TODO: collect inputs and outputs for SPAN_KIND_FUNCTION
|
16
|
+
|
17
|
+
# collect inputs and outputs for SPAN_KIND_LLM
|
18
|
+
if isinstance(result, ChatCompletion):
|
19
|
+
divi._datapark.create_chat_completion(
|
20
|
+
span_id=span.span_id,
|
21
|
+
trace_id=span.trace_id,
|
22
|
+
inputs=input,
|
23
|
+
completion=result,
|
24
|
+
)
|
25
|
+
|
26
|
+
# collect inputs and outputs for SPAN_KIND_EVALUATION
|
27
|
+
if isinstance(result, list) and all(
|
28
|
+
isinstance(x, EvaluationScore) for x in result
|
29
|
+
):
|
30
|
+
divi._datapark.create_scores(
|
31
|
+
span_id=span.span_id,
|
32
|
+
trace_id=span.trace_id,
|
33
|
+
scores=result,
|
34
|
+
)
|
@@ -0,0 +1,48 @@
|
|
1
|
+
import functools
|
2
|
+
from collections.abc import Callable
|
3
|
+
from typing import TYPE_CHECKING, TypeVar, Union
|
4
|
+
|
5
|
+
from typing_extensions import Optional
|
6
|
+
|
7
|
+
from divi.decorators.observable import observable
|
8
|
+
from divi.evaluation.scores import Score
|
9
|
+
from divi.signals.span import Kind
|
10
|
+
from divi.utils import is_async
|
11
|
+
|
12
|
+
if TYPE_CHECKING:
|
13
|
+
from openai import AsyncOpenAI, OpenAI
|
14
|
+
|
15
|
+
C = TypeVar("C", bound=Union["OpenAI", "AsyncOpenAI"])
|
16
|
+
|
17
|
+
|
18
|
+
def _get_observable_create(
|
19
|
+
create: Callable,
|
20
|
+
name: Optional[str] = None,
|
21
|
+
scores: Optional[list[Score]] = None,
|
22
|
+
) -> Callable:
|
23
|
+
@functools.wraps(create)
|
24
|
+
def observable_create(*args, stream: bool = False, **kwargs):
|
25
|
+
decorator = observable(kind=Kind.llm, name=name, scores=scores)
|
26
|
+
return decorator(create)(*args, stream=stream, **kwargs)
|
27
|
+
|
28
|
+
# TODO Async Observable Create
|
29
|
+
return observable_create if not is_async(create) else create
|
30
|
+
|
31
|
+
|
32
|
+
def obs_openai(
|
33
|
+
client: C,
|
34
|
+
name: Optional[str] = "Agent",
|
35
|
+
scores: Optional[list[Score]] = None,
|
36
|
+
) -> C:
|
37
|
+
"""Make OpenAI client observable."""
|
38
|
+
client.chat.completions.create = _get_observable_create(
|
39
|
+
client.chat.completions.create,
|
40
|
+
name=name,
|
41
|
+
scores=scores,
|
42
|
+
)
|
43
|
+
client.completions.create = _get_observable_create(
|
44
|
+
client.completions.create,
|
45
|
+
name=name,
|
46
|
+
scores=scores,
|
47
|
+
)
|
48
|
+
return client
|
@@ -0,0 +1,91 @@
|
|
1
|
+
import functools
|
2
|
+
from typing import (
|
3
|
+
Any,
|
4
|
+
Callable,
|
5
|
+
Generic,
|
6
|
+
Mapping,
|
7
|
+
Optional,
|
8
|
+
ParamSpec,
|
9
|
+
Protocol,
|
10
|
+
TypeVar,
|
11
|
+
Union,
|
12
|
+
overload,
|
13
|
+
runtime_checkable,
|
14
|
+
)
|
15
|
+
|
16
|
+
from divi.decorators.observe import observe
|
17
|
+
from divi.evaluation.evaluate import evaluate_scores
|
18
|
+
from divi.evaluation.scores import Score
|
19
|
+
from divi.session import SessionExtra
|
20
|
+
from divi.signals.span import Kind, Span
|
21
|
+
|
22
|
+
R = TypeVar("R", covariant=True)
|
23
|
+
P = ParamSpec("P")
|
24
|
+
|
25
|
+
|
26
|
+
@runtime_checkable
|
27
|
+
class WithSessionExtra(Protocol, Generic[P, R]):
|
28
|
+
def __call__(
|
29
|
+
self,
|
30
|
+
*args: P.args,
|
31
|
+
session_extra: Optional[SessionExtra] = None, # type: ignore[valid-type]
|
32
|
+
**kwargs: P.kwargs,
|
33
|
+
) -> R: ...
|
34
|
+
|
35
|
+
|
36
|
+
@overload
|
37
|
+
def observable(func: Callable[P, R]) -> WithSessionExtra[P, R]: ...
|
38
|
+
|
39
|
+
|
40
|
+
@overload
|
41
|
+
def observable(
|
42
|
+
kind: Kind = Kind.function,
|
43
|
+
*,
|
44
|
+
name: Optional[str] = None,
|
45
|
+
scores: Optional[list[Score]] = None,
|
46
|
+
metadata: Optional[Mapping[str, Any]] = None,
|
47
|
+
) -> Callable[[Callable[P, R]], WithSessionExtra[P, R]]: ...
|
48
|
+
|
49
|
+
|
50
|
+
def observable(
|
51
|
+
*args, **kwargs
|
52
|
+
) -> Union[Callable, Callable[[Callable], Callable]]:
|
53
|
+
"""Observable decorator factory."""
|
54
|
+
|
55
|
+
kind = kwargs.pop("kind", Kind.function)
|
56
|
+
name = kwargs.pop("name", None)
|
57
|
+
metadata = kwargs.pop("metadata", None)
|
58
|
+
scores: list[Score] = kwargs.pop("scores", None)
|
59
|
+
|
60
|
+
def decorator(func):
|
61
|
+
@functools.wraps(func)
|
62
|
+
def wrapper(
|
63
|
+
*args, session_extra: Optional[SessionExtra] = None, **kwargs
|
64
|
+
):
|
65
|
+
# 1. init the span
|
66
|
+
span = Span(
|
67
|
+
kind=kind, name=name or func.__name__, metadata=metadata
|
68
|
+
)
|
69
|
+
|
70
|
+
# 2. observe the function
|
71
|
+
result = observe(
|
72
|
+
*args,
|
73
|
+
func=func,
|
74
|
+
span=span,
|
75
|
+
session_extra=session_extra,
|
76
|
+
**kwargs,
|
77
|
+
)
|
78
|
+
|
79
|
+
# 3. evaluate the scores if they are provided
|
80
|
+
messages = kwargs.get("messages", [])
|
81
|
+
evaluate_scores(messages, outputs=result, scores=scores)
|
82
|
+
|
83
|
+
return result
|
84
|
+
|
85
|
+
return wrapper
|
86
|
+
|
87
|
+
# Function Decorator
|
88
|
+
if len(args) == 1 and callable(args[0]) and not kwargs:
|
89
|
+
return decorator(args[0])
|
90
|
+
# Factory Decorator
|
91
|
+
return decorator
|
@@ -0,0 +1,47 @@
|
|
1
|
+
import contextvars
|
2
|
+
from typing import (
|
3
|
+
Callable,
|
4
|
+
Optional,
|
5
|
+
)
|
6
|
+
|
7
|
+
from divi.decorators.collect import collect
|
8
|
+
from divi.session import SessionExtra
|
9
|
+
from divi.session.setup import setup
|
10
|
+
from divi.signals.span import Span
|
11
|
+
from divi.utils import extract_flattened_inputs
|
12
|
+
|
13
|
+
# ContextVar to store the extra information
|
14
|
+
# from the Session and parent Span
|
15
|
+
_SESSION_EXTRA = contextvars.ContextVar[Optional[SessionExtra]](
|
16
|
+
"_SESSION_EXTRA", default=None
|
17
|
+
)
|
18
|
+
|
19
|
+
|
20
|
+
def observe(
|
21
|
+
*args,
|
22
|
+
func: Callable,
|
23
|
+
span: Span,
|
24
|
+
session_extra: Optional[SessionExtra] = None,
|
25
|
+
**kwargs,
|
26
|
+
):
|
27
|
+
session_extra = setup(span, _SESSION_EXTRA.get() or session_extra)
|
28
|
+
# set current context
|
29
|
+
token = _SESSION_EXTRA.set(session_extra)
|
30
|
+
# execute the function
|
31
|
+
span.start()
|
32
|
+
result = func(*args, **kwargs)
|
33
|
+
span.end()
|
34
|
+
# recover parent context
|
35
|
+
_SESSION_EXTRA.reset(token)
|
36
|
+
|
37
|
+
# get the trace to collect data
|
38
|
+
trace = session_extra.get("trace")
|
39
|
+
# end the trace if it is the root span
|
40
|
+
if trace and not span.parent_span_id:
|
41
|
+
trace.end()
|
42
|
+
|
43
|
+
# collect inputs and outputs
|
44
|
+
inputs = extract_flattened_inputs(func, *args, **kwargs)
|
45
|
+
collect(span, inputs, result)
|
46
|
+
|
47
|
+
return result
|
@@ -0,0 +1,61 @@
|
|
1
|
+
import os
|
2
|
+
from typing import Optional
|
3
|
+
|
4
|
+
from openai.types.chat import (
|
5
|
+
ChatCompletion,
|
6
|
+
ChatCompletionMessageParam,
|
7
|
+
)
|
8
|
+
from typing_extensions import List
|
9
|
+
|
10
|
+
import divi
|
11
|
+
from divi.decorators.observe import observe
|
12
|
+
from divi.evaluation import Evaluator
|
13
|
+
from divi.evaluation.evaluator import EvaluatorConfig
|
14
|
+
from divi.evaluation.scores import Score
|
15
|
+
from divi.signals.span import Kind, Span
|
16
|
+
|
17
|
+
OPENAI_API_KEY = "OPENAI_API_KEY"
|
18
|
+
OPENAI_BASE_URL = "OPENAI_BASE_URL"
|
19
|
+
|
20
|
+
|
21
|
+
def init_evaluator(config: Optional[EvaluatorConfig] = None):
|
22
|
+
_config = config or EvaluatorConfig()
|
23
|
+
api_key = _config.api_key if _config.api_key else os.getenv(OPENAI_API_KEY)
|
24
|
+
base_url = (
|
25
|
+
_config.base_url if _config.base_url else os.getenv(OPENAI_BASE_URL)
|
26
|
+
)
|
27
|
+
if api_key is None:
|
28
|
+
raise ValueError("API key is required for evaluator")
|
29
|
+
_config.api_key = api_key
|
30
|
+
_config.base_url = base_url
|
31
|
+
evaluator = Evaluator(_config)
|
32
|
+
return evaluator
|
33
|
+
|
34
|
+
|
35
|
+
def evaluate_scores(
|
36
|
+
messages: Optional[List[ChatCompletionMessageParam]],
|
37
|
+
outputs: Optional[ChatCompletion],
|
38
|
+
scores: Optional[List[Score]],
|
39
|
+
config: Optional[EvaluatorConfig] = None,
|
40
|
+
):
|
41
|
+
if messages is None or scores is None or scores.__len__() == 0:
|
42
|
+
return
|
43
|
+
if not divi._evaluator:
|
44
|
+
divi._evaluator = init_evaluator(config)
|
45
|
+
|
46
|
+
if isinstance(outputs, ChatCompletion):
|
47
|
+
output_message = outputs.choices[0].message.content
|
48
|
+
if not output_message:
|
49
|
+
return
|
50
|
+
|
51
|
+
evaluation_span = Span(kind=Kind.evaluation, name="Evaluation")
|
52
|
+
observe(
|
53
|
+
func=divi._evaluator.evaluate,
|
54
|
+
span=evaluation_span,
|
55
|
+
target=output_message,
|
56
|
+
conversation="\n".join(
|
57
|
+
f"{m.get('role', 'unknown')}: {m.get('content')}"
|
58
|
+
for m in messages
|
59
|
+
),
|
60
|
+
scores=scores,
|
61
|
+
)
|
@@ -0,0 +1,174 @@
|
|
1
|
+
import asyncio
|
2
|
+
import concurrent.futures
|
3
|
+
import random
|
4
|
+
from typing import List, Literal, Optional
|
5
|
+
|
6
|
+
import openai
|
7
|
+
from pydantic import BaseModel
|
8
|
+
|
9
|
+
from divi.evaluation.prompts import PRESET_PROMPT, PROMPT_TEMPLATE
|
10
|
+
from divi.evaluation.scores import Score
|
11
|
+
|
12
|
+
|
13
|
+
class EvaluatorConfig:
|
14
|
+
def __init__(
|
15
|
+
self,
|
16
|
+
model: str = "gpt-4.1-nano",
|
17
|
+
temperature: float = 0.5,
|
18
|
+
max_concurrency: int = 10,
|
19
|
+
api_key: Optional[str] = None,
|
20
|
+
base_url: Optional[str] = None,
|
21
|
+
):
|
22
|
+
self.model = model
|
23
|
+
self.api_key = api_key
|
24
|
+
self.base_url = base_url
|
25
|
+
self.temperature = temperature
|
26
|
+
self.max_concurrency = max_concurrency
|
27
|
+
|
28
|
+
|
29
|
+
class EvaluationResult(BaseModel):
|
30
|
+
name: Score
|
31
|
+
judgment: bool
|
32
|
+
reasoning: str
|
33
|
+
|
34
|
+
|
35
|
+
class EvaluationScore(BaseModel):
|
36
|
+
name: Score
|
37
|
+
score: float
|
38
|
+
representative_reasoning: str
|
39
|
+
all_evaluations: List[EvaluationResult]
|
40
|
+
|
41
|
+
|
42
|
+
class Evaluator:
|
43
|
+
def __init__(self, config: Optional[EvaluatorConfig] = None):
|
44
|
+
self.config = config or EvaluatorConfig()
|
45
|
+
self.async_client = openai.AsyncOpenAI(
|
46
|
+
api_key=self.config.api_key, base_url=self.config.base_url
|
47
|
+
)
|
48
|
+
self.sync_client = openai.OpenAI(
|
49
|
+
api_key=self.config.api_key, base_url=self.config.base_url
|
50
|
+
)
|
51
|
+
|
52
|
+
@staticmethod
|
53
|
+
def generate_prompt(target: str, conversation: str, score: Score) -> str:
|
54
|
+
return PROMPT_TEMPLATE.format(
|
55
|
+
requirements=PRESET_PROMPT[score.value],
|
56
|
+
target=target,
|
57
|
+
conversation=conversation,
|
58
|
+
)
|
59
|
+
|
60
|
+
def _sync_evaluate_once(
|
61
|
+
self, target: str, conversation: str, score: Score
|
62
|
+
) -> Optional[EvaluationResult]:
|
63
|
+
prompt = self.generate_prompt(target, conversation, score)
|
64
|
+
response = self.sync_client.beta.chat.completions.parse(
|
65
|
+
model=self.config.model,
|
66
|
+
messages=[{"role": "user", "content": prompt}],
|
67
|
+
temperature=self.config.temperature,
|
68
|
+
response_format=EvaluationResult,
|
69
|
+
)
|
70
|
+
result = response.choices[0].message.parsed
|
71
|
+
if result is not None:
|
72
|
+
result.name = score
|
73
|
+
return result
|
74
|
+
|
75
|
+
async def _async_evaluate_once(
|
76
|
+
self, target: str, conversation: str, score: Score
|
77
|
+
) -> Optional[EvaluationResult]:
|
78
|
+
prompt = self.generate_prompt(target, conversation, score)
|
79
|
+
response = await self.async_client.beta.chat.completions.parse(
|
80
|
+
model=self.config.model,
|
81
|
+
messages=[{"role": "user", "content": prompt}],
|
82
|
+
temperature=self.config.temperature,
|
83
|
+
response_format=EvaluationResult,
|
84
|
+
)
|
85
|
+
result = response.choices[0].message.parsed
|
86
|
+
if result is not None:
|
87
|
+
result.name = score
|
88
|
+
return result
|
89
|
+
|
90
|
+
def _aggregate_result(
|
91
|
+
self, name: Score, evaluations: List[EvaluationResult]
|
92
|
+
) -> EvaluationScore:
|
93
|
+
n = len(evaluations)
|
94
|
+
true_count = sum(1 for e in evaluations if e.judgment is True)
|
95
|
+
score = true_count / n
|
96
|
+
majority_judgment = True if true_count >= (n / 2) else False
|
97
|
+
majority_reasons = [
|
98
|
+
e.reasoning for e in evaluations if e.judgment == majority_judgment
|
99
|
+
]
|
100
|
+
representative_reasoning = (
|
101
|
+
random.choice(majority_reasons) if majority_reasons else ""
|
102
|
+
)
|
103
|
+
return EvaluationScore(
|
104
|
+
name=name,
|
105
|
+
score=score,
|
106
|
+
representative_reasoning=representative_reasoning,
|
107
|
+
all_evaluations=evaluations,
|
108
|
+
)
|
109
|
+
|
110
|
+
def _aggregate_results(
|
111
|
+
self, evaluations: List[EvaluationResult]
|
112
|
+
) -> List[EvaluationScore]:
|
113
|
+
scores = {}
|
114
|
+
for evaluation in evaluations:
|
115
|
+
if evaluation.name not in scores:
|
116
|
+
scores[evaluation.name] = []
|
117
|
+
scores[evaluation.name].append(evaluation)
|
118
|
+
|
119
|
+
aggregated_results = [
|
120
|
+
self._aggregate_result(name, evals)
|
121
|
+
for name, evals in scores.items()
|
122
|
+
]
|
123
|
+
return aggregated_results
|
124
|
+
|
125
|
+
def evaluate_sync(
|
126
|
+
self, target: str, conversation: str, scores: list[Score], n_rounds: int
|
127
|
+
) -> List[EvaluationScore]:
|
128
|
+
with concurrent.futures.ThreadPoolExecutor(
|
129
|
+
max_workers=self.config.max_concurrency
|
130
|
+
) as executor:
|
131
|
+
futures = [
|
132
|
+
executor.submit(
|
133
|
+
self._sync_evaluate_once, target, conversation, score
|
134
|
+
)
|
135
|
+
for _ in range(n_rounds)
|
136
|
+
for score in scores
|
137
|
+
]
|
138
|
+
evaluations = [
|
139
|
+
f.result() for f in concurrent.futures.as_completed(futures)
|
140
|
+
]
|
141
|
+
return self._aggregate_results(
|
142
|
+
[e for e in evaluations if e is not None]
|
143
|
+
)
|
144
|
+
|
145
|
+
async def evaluate_async(
|
146
|
+
self, target: str, conversation: str, scores: list[Score], n_rounds: int
|
147
|
+
) -> List[EvaluationScore]:
|
148
|
+
semaphore = asyncio.Semaphore(self.config.max_concurrency)
|
149
|
+
|
150
|
+
async def sem_task(score):
|
151
|
+
async with semaphore:
|
152
|
+
return await self._async_evaluate_once(
|
153
|
+
target, conversation, score
|
154
|
+
)
|
155
|
+
|
156
|
+
tasks = [sem_task(score) for _ in range(n_rounds) for score in scores]
|
157
|
+
evaluations = await asyncio.gather(*tasks)
|
158
|
+
return self._aggregate_results(
|
159
|
+
[e for e in evaluations if e is not None]
|
160
|
+
)
|
161
|
+
|
162
|
+
def evaluate(
|
163
|
+
self,
|
164
|
+
target: str,
|
165
|
+
conversation: str,
|
166
|
+
scores: list[Score],
|
167
|
+
n_rounds: int = 5,
|
168
|
+
mode: Literal["sync", "async"] = "sync",
|
169
|
+
) -> List[EvaluationScore]:
|
170
|
+
if mode == "async":
|
171
|
+
return asyncio.run(
|
172
|
+
self.evaluate_async(target, conversation, scores, n_rounds)
|
173
|
+
)
|
174
|
+
return self.evaluate_sync(target, conversation, scores, n_rounds)
|
@@ -0,0 +1,19 @@
|
|
1
|
+
PROMPT_TEMPLATE = (
|
2
|
+
"The *requirements* of the evaluation task are: {requirements}\n\n"
|
3
|
+
"Below is the *context* of the conversation (for reference only):\n"
|
4
|
+
"{conversation}\n\n"
|
5
|
+
"Now, in view of both the requirements and the context, evaluate the assistant’s response:\n"
|
6
|
+
"{target}\n\n"
|
7
|
+
"Please reason step by step to reach your judgment.\n\n"
|
8
|
+
"Strictly output your answer in the following JSON format:\n"
|
9
|
+
"{{\n"
|
10
|
+
' "judgment": bool, # true if the response meets all requirements\n'
|
11
|
+
' "reasoning": "string" # concise explanation, hitting only the key points\n'
|
12
|
+
"}}\n"
|
13
|
+
"Do not output anything else."
|
14
|
+
)
|
15
|
+
|
16
|
+
PRESET_PROMPT = {
|
17
|
+
"task_completion": "Assess whether the assistant response fulfills the user's task requirements.",
|
18
|
+
"instruction_adherence": "Assess whether the assistant response strictly follows every instruction given by the user, without omissions, deviations, or hallucinations.",
|
19
|
+
}
|
@@ -0,0 +1,49 @@
|
|
1
|
+
syntax = "proto3";
|
2
|
+
|
3
|
+
package divi.proto.common.v1;
|
4
|
+
|
5
|
+
option go_package = "services/pb";
|
6
|
+
|
7
|
+
// AnyValue is used to represent any type of attribute value. AnyValue may contain a
|
8
|
+
// primitive value such as a string or integer or it may contain an arbitrary nested
|
9
|
+
// object containing arrays, key-value lists and primitives.
|
10
|
+
message AnyValue {
|
11
|
+
// The value is one of the listed fields. It is valid for all values to be unspecified
|
12
|
+
// in which case this AnyValue is considered to be "empty".
|
13
|
+
oneof value {
|
14
|
+
string string_value = 1;
|
15
|
+
bool bool_value = 2;
|
16
|
+
int64 int_value = 3;
|
17
|
+
double double_value = 4;
|
18
|
+
ArrayValue array_value = 5;
|
19
|
+
KeyValueList kvlist_value = 6;
|
20
|
+
bytes bytes_value = 7;
|
21
|
+
}
|
22
|
+
}
|
23
|
+
|
24
|
+
// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message
|
25
|
+
// since oneof in AnyValue does not allow repeated fields.
|
26
|
+
message ArrayValue {
|
27
|
+
// Array of values. The array may be empty (contain 0 elements).
|
28
|
+
repeated AnyValue values = 1;
|
29
|
+
}
|
30
|
+
|
31
|
+
// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message
|
32
|
+
// since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need
|
33
|
+
// a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to
|
34
|
+
// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches
|
35
|
+
// are semantically equivalent.
|
36
|
+
message KeyValueList {
|
37
|
+
// A collection of key/value pairs of key-value pairs. The list may be empty (may
|
38
|
+
// contain 0 elements).
|
39
|
+
// The keys MUST be unique (it is not allowed to have more than one
|
40
|
+
// value with the same key).
|
41
|
+
repeated KeyValue values = 1;
|
42
|
+
}
|
43
|
+
|
44
|
+
// KeyValue is a key-value pair that is used to store Span attributes, Link
|
45
|
+
// attributes, etc.
|
46
|
+
message KeyValue {
|
47
|
+
string key = 1;
|
48
|
+
AnyValue value = 2;
|
49
|
+
}
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
3
|
+
# NO CHECKED-IN PROTOBUF GENCODE
|
4
|
+
# source: divi/proto/common/v1/common.proto
|
5
|
+
# Protobuf Python Version: 5.29.0
|
6
|
+
"""Generated protocol buffer code."""
|
7
|
+
from google.protobuf import descriptor as _descriptor
|
8
|
+
from google.protobuf import descriptor_pool as _descriptor_pool
|
9
|
+
from google.protobuf import runtime_version as _runtime_version
|
10
|
+
from google.protobuf import symbol_database as _symbol_database
|
11
|
+
from google.protobuf.internal import builder as _builder
|
12
|
+
_runtime_version.ValidateProtobufRuntimeVersion(
|
13
|
+
_runtime_version.Domain.PUBLIC,
|
14
|
+
5,
|
15
|
+
29,
|
16
|
+
0,
|
17
|
+
'',
|
18
|
+
'divi/proto/common/v1/common.proto'
|
19
|
+
)
|
20
|
+
# @@protoc_insertion_point(imports)
|
21
|
+
|
22
|
+
_sym_db = _symbol_database.Default()
|
23
|
+
|
24
|
+
|
25
|
+
|
26
|
+
|
27
|
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n!divi/proto/common/v1/common.proto\x12\x14\x64ivi.proto.common.v1\"\xfa\x01\n\x08\x41nyValue\x12\x16\n\x0cstring_value\x18\x01 \x01(\tH\x00\x12\x14\n\nbool_value\x18\x02 \x01(\x08H\x00\x12\x13\n\tint_value\x18\x03 \x01(\x03H\x00\x12\x16\n\x0c\x64ouble_value\x18\x04 \x01(\x01H\x00\x12\x37\n\x0b\x61rray_value\x18\x05 \x01(\x0b\x32 .divi.proto.common.v1.ArrayValueH\x00\x12:\n\x0ckvlist_value\x18\x06 \x01(\x0b\x32\".divi.proto.common.v1.KeyValueListH\x00\x12\x15\n\x0b\x62ytes_value\x18\x07 \x01(\x0cH\x00\x42\x07\n\x05value\"<\n\nArrayValue\x12.\n\x06values\x18\x01 \x03(\x0b\x32\x1e.divi.proto.common.v1.AnyValue\">\n\x0cKeyValueList\x12.\n\x06values\x18\x01 \x03(\x0b\x32\x1e.divi.proto.common.v1.KeyValue\"F\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x1e.divi.proto.common.v1.AnyValueB\rZ\x0bservices/pbb\x06proto3')
|
28
|
+
|
29
|
+
_globals = globals()
|
30
|
+
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
31
|
+
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'divi.proto.common.v1.common_pb2', _globals)
|
32
|
+
if not _descriptor._USE_C_DESCRIPTORS:
|
33
|
+
_globals['DESCRIPTOR']._loaded_options = None
|
34
|
+
_globals['DESCRIPTOR']._serialized_options = b'Z\013services/pb'
|
35
|
+
_globals['_ANYVALUE']._serialized_start=60
|
36
|
+
_globals['_ANYVALUE']._serialized_end=310
|
37
|
+
_globals['_ARRAYVALUE']._serialized_start=312
|
38
|
+
_globals['_ARRAYVALUE']._serialized_end=372
|
39
|
+
_globals['_KEYVALUELIST']._serialized_start=374
|
40
|
+
_globals['_KEYVALUELIST']._serialized_end=436
|
41
|
+
_globals['_KEYVALUE']._serialized_start=438
|
42
|
+
_globals['_KEYVALUE']._serialized_end=508
|
43
|
+
# @@protoc_insertion_point(module_scope)
|
@@ -0,0 +1,44 @@
|
|
1
|
+
from google.protobuf.internal import containers as _containers
|
2
|
+
from google.protobuf import descriptor as _descriptor
|
3
|
+
from google.protobuf import message as _message
|
4
|
+
from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union
|
5
|
+
|
6
|
+
DESCRIPTOR: _descriptor.FileDescriptor
|
7
|
+
|
8
|
+
class AnyValue(_message.Message):
|
9
|
+
__slots__ = ("string_value", "bool_value", "int_value", "double_value", "array_value", "kvlist_value", "bytes_value")
|
10
|
+
STRING_VALUE_FIELD_NUMBER: _ClassVar[int]
|
11
|
+
BOOL_VALUE_FIELD_NUMBER: _ClassVar[int]
|
12
|
+
INT_VALUE_FIELD_NUMBER: _ClassVar[int]
|
13
|
+
DOUBLE_VALUE_FIELD_NUMBER: _ClassVar[int]
|
14
|
+
ARRAY_VALUE_FIELD_NUMBER: _ClassVar[int]
|
15
|
+
KVLIST_VALUE_FIELD_NUMBER: _ClassVar[int]
|
16
|
+
BYTES_VALUE_FIELD_NUMBER: _ClassVar[int]
|
17
|
+
string_value: str
|
18
|
+
bool_value: bool
|
19
|
+
int_value: int
|
20
|
+
double_value: float
|
21
|
+
array_value: ArrayValue
|
22
|
+
kvlist_value: KeyValueList
|
23
|
+
bytes_value: bytes
|
24
|
+
def __init__(self, string_value: _Optional[str] = ..., bool_value: bool = ..., int_value: _Optional[int] = ..., double_value: _Optional[float] = ..., array_value: _Optional[_Union[ArrayValue, _Mapping]] = ..., kvlist_value: _Optional[_Union[KeyValueList, _Mapping]] = ..., bytes_value: _Optional[bytes] = ...) -> None: ...
|
25
|
+
|
26
|
+
class ArrayValue(_message.Message):
|
27
|
+
__slots__ = ("values",)
|
28
|
+
VALUES_FIELD_NUMBER: _ClassVar[int]
|
29
|
+
values: _containers.RepeatedCompositeFieldContainer[AnyValue]
|
30
|
+
def __init__(self, values: _Optional[_Iterable[_Union[AnyValue, _Mapping]]] = ...) -> None: ...
|
31
|
+
|
32
|
+
class KeyValueList(_message.Message):
|
33
|
+
__slots__ = ("values",)
|
34
|
+
VALUES_FIELD_NUMBER: _ClassVar[int]
|
35
|
+
values: _containers.RepeatedCompositeFieldContainer[KeyValue]
|
36
|
+
def __init__(self, values: _Optional[_Iterable[_Union[KeyValue, _Mapping]]] = ...) -> None: ...
|
37
|
+
|
38
|
+
class KeyValue(_message.Message):
|
39
|
+
__slots__ = ("key", "value")
|
40
|
+
KEY_FIELD_NUMBER: _ClassVar[int]
|
41
|
+
VALUE_FIELD_NUMBER: _ClassVar[int]
|
42
|
+
key: str
|
43
|
+
value: AnyValue
|
44
|
+
def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[AnyValue, _Mapping]] = ...) -> None: ...
|
@@ -0,0 +1,17 @@
|
|
1
|
+
syntax = "proto3";
|
2
|
+
|
3
|
+
package divi.proto.core.health.v1;
|
4
|
+
|
5
|
+
option go_package = "services/pb";
|
6
|
+
|
7
|
+
// HealthService is a service that implements health check.
|
8
|
+
service HealthService {
|
9
|
+
rpc Check(HealthCheckRequest) returns (HealthCheckResponse) {}
|
10
|
+
}
|
11
|
+
|
12
|
+
message HealthCheckRequest { string version = 1; }
|
13
|
+
|
14
|
+
message HealthCheckResponse {
|
15
|
+
bool status = 1;
|
16
|
+
string message = 2;
|
17
|
+
}
|