divi 0.0.1.dev21__py3-none-any.whl → 0.0.1.dev28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- divi/__init__.py +4 -2
- divi/decorators/obs_openai.py +20 -6
- divi/decorators/observable.py +18 -34
- divi/evaluation/__init__.py +4 -0
- divi/evaluation/evaluate.py +57 -0
- divi/evaluation/evaluator.py +161 -0
- divi/evaluation/prompts.py +14 -0
- divi/evaluation/scores.py +8 -0
- divi/services/datapark/datapark.py +10 -4
- divi/services/init.py +2 -1
- divi/session/session.py +3 -3
- divi/session/setup.py +12 -26
- divi/signals/trace/trace.py +18 -18
- {divi-0.0.1.dev21.dist-info → divi-0.0.1.dev28.dist-info}/METADATA +1 -1
- {divi-0.0.1.dev21.dist-info → divi-0.0.1.dev28.dist-info}/RECORD +17 -13
- divi/config/config.py +0 -0
- {divi-0.0.1.dev21.dist-info → divi-0.0.1.dev28.dist-info}/WHEEL +0 -0
- {divi-0.0.1.dev21.dist-info → divi-0.0.1.dev28.dist-info}/licenses/LICENSE +0 -0
divi/__init__.py
CHANGED
@@ -2,6 +2,7 @@ from typing import Optional
|
|
2
2
|
|
3
3
|
from . import proto
|
4
4
|
from .decorators import obs_openai, observable
|
5
|
+
from .evaluation import Evaluator, Score
|
5
6
|
from .services import Auth, Core, DataPark
|
6
7
|
from .session import Session
|
7
8
|
|
@@ -11,6 +12,7 @@ _session: Optional[Session] = None
|
|
11
12
|
_core: Optional[Core] = None
|
12
13
|
_auth: Optional[Auth] = None
|
13
14
|
_datapark: Optional[DataPark] = None
|
15
|
+
_evaluator: Optional[Evaluator] = None
|
14
16
|
|
15
|
-
__version__ = "0.0.1.
|
16
|
-
__all__ = ["proto", "obs_openai", "observable"]
|
17
|
+
__version__ = "0.0.1.dev28"
|
18
|
+
__all__ = ["proto", "obs_openai", "observable", "Score"]
|
divi/decorators/obs_openai.py
CHANGED
@@ -2,7 +2,10 @@ import functools
|
|
2
2
|
from collections.abc import Callable
|
3
3
|
from typing import TYPE_CHECKING, TypeVar, Union
|
4
4
|
|
5
|
+
from typing_extensions import Optional
|
6
|
+
|
5
7
|
from divi.decorators.observable import observable
|
8
|
+
from divi.evaluation.scores import Score
|
6
9
|
from divi.utils import is_async
|
7
10
|
|
8
11
|
if TYPE_CHECKING:
|
@@ -11,23 +14,34 @@ if TYPE_CHECKING:
|
|
11
14
|
C = TypeVar("C", bound=Union["OpenAI", "AsyncOpenAI"])
|
12
15
|
|
13
16
|
|
14
|
-
def _get_observable_create(
|
17
|
+
def _get_observable_create(
|
18
|
+
create: Callable,
|
19
|
+
name: Optional[str] = None,
|
20
|
+
scores: Optional[list[Score]] = None,
|
21
|
+
) -> Callable:
|
15
22
|
@functools.wraps(create)
|
16
23
|
def observable_create(*args, stream: bool = False, **kwargs):
|
17
|
-
decorator = observable(kind="llm")
|
24
|
+
decorator = observable(kind="llm", name=name, scores=scores)
|
18
25
|
return decorator(create)(*args, stream=stream, **kwargs)
|
19
26
|
|
20
27
|
# TODO Async Observable Create
|
21
|
-
print("Is async", is_async(create))
|
22
28
|
return observable_create if not is_async(create) else create
|
23
29
|
|
24
30
|
|
25
|
-
def obs_openai(
|
31
|
+
def obs_openai(
|
32
|
+
client: C,
|
33
|
+
name: Optional[str] = "Chat Bot",
|
34
|
+
scores: Optional[list[Score]] = None,
|
35
|
+
) -> C:
|
26
36
|
"""Make OpenAI client observable."""
|
27
37
|
client.chat.completions.create = _get_observable_create(
|
28
|
-
client.chat.completions.create
|
38
|
+
client.chat.completions.create,
|
39
|
+
name=name,
|
40
|
+
scores=scores,
|
29
41
|
)
|
30
42
|
client.completions.create = _get_observable_create(
|
31
|
-
client.completions.create
|
43
|
+
client.completions.create,
|
44
|
+
name=name,
|
45
|
+
scores=scores,
|
32
46
|
)
|
33
47
|
return client
|
divi/decorators/observable.py
CHANGED
@@ -1,11 +1,9 @@
|
|
1
1
|
import contextvars
|
2
2
|
import functools
|
3
|
-
import inspect
|
4
3
|
from typing import (
|
5
4
|
Any,
|
6
5
|
Callable,
|
7
6
|
Generic,
|
8
|
-
List,
|
9
7
|
Mapping,
|
10
8
|
Optional,
|
11
9
|
ParamSpec,
|
@@ -19,6 +17,8 @@ from typing import (
|
|
19
17
|
from openai.types.chat import ChatCompletion
|
20
18
|
|
21
19
|
import divi
|
20
|
+
from divi.evaluation.evaluate import evaluate_scores
|
21
|
+
from divi.evaluation.scores import Score
|
22
22
|
from divi.proto.trace.v1.trace_pb2 import ScopeSpans
|
23
23
|
from divi.session import SessionExtra
|
24
24
|
from divi.session.setup import setup
|
@@ -54,6 +54,7 @@ def observable(
|
|
54
54
|
kind: str = "function",
|
55
55
|
*,
|
56
56
|
name: Optional[str] = None,
|
57
|
+
scores: Optional[list[Score]] = None,
|
57
58
|
metadata: Optional[Mapping[str, Any]] = None,
|
58
59
|
) -> Callable[[Callable[P, R]], WithSessionExtra[P, R]]: ...
|
59
60
|
|
@@ -66,6 +67,7 @@ def observable(
|
|
66
67
|
kind = kwargs.pop("kind", "function")
|
67
68
|
name = kwargs.pop("name", None)
|
68
69
|
metadata = kwargs.pop("metadata", None)
|
70
|
+
scores: list[Score] = kwargs.pop("scores", None)
|
69
71
|
|
70
72
|
def decorator(func):
|
71
73
|
@functools.wraps(func)
|
@@ -85,7 +87,11 @@ def observable(
|
|
85
87
|
# recover parent context
|
86
88
|
_SESSION_EXTRA.reset(token)
|
87
89
|
|
88
|
-
#
|
90
|
+
# get the trace to collect data
|
91
|
+
trace = session_extra.get("trace")
|
92
|
+
if not trace:
|
93
|
+
raise ValueError("Trace not found in session context.")
|
94
|
+
# TODO: collect inputs and outputs for SPAN_KIND_FUNCTION
|
89
95
|
inputs = extract_flattened_inputs(func, *args, **kwargs)
|
90
96
|
# create the span if it is the root span
|
91
97
|
if divi._datapark and span.trace_id:
|
@@ -94,43 +100,21 @@ def observable(
|
|
94
100
|
)
|
95
101
|
# end the trace if it is the root span
|
96
102
|
if divi._datapark and not span.parent_span_id:
|
97
|
-
trace
|
98
|
-
if trace:
|
99
|
-
trace.end()
|
100
|
-
divi._datapark.upsert_traces(
|
101
|
-
session_id=trace.session_id, traces=[trace.signal]
|
102
|
-
)
|
103
|
+
trace.end()
|
103
104
|
# create the chat completion if it is a chat completion
|
104
105
|
if divi._datapark and isinstance(result, ChatCompletion):
|
105
106
|
divi._datapark.create_chat_completion(
|
106
|
-
span_id=span.span_id,
|
107
|
+
span_id=span.span_id,
|
108
|
+
trace_id=trace.trace_id,
|
109
|
+
inputs=inputs,
|
110
|
+
completion=result,
|
107
111
|
)
|
108
|
-
|
112
|
+
# evaluate the scores if they are provided
|
113
|
+
if scores is not None and scores.__len__() > 0:
|
114
|
+
evaluate_scores(inputs, outputs=result, scores=scores)
|
109
115
|
|
110
|
-
|
111
|
-
def generator_wrapper(
|
112
|
-
*args, session_extra: Optional[SessionExtra] = None, **kwargs
|
113
|
-
):
|
114
|
-
span = Span(
|
115
|
-
kind=kind, name=name or func.__name__, metadata=metadata
|
116
|
-
)
|
117
|
-
session_extra = setup(span, _SESSION_EXTRA.get() or session_extra)
|
118
|
-
# set current context
|
119
|
-
token = _SESSION_EXTRA.set(session_extra)
|
120
|
-
# execute the function
|
121
|
-
results: List[Any] = []
|
122
|
-
span.start()
|
123
|
-
for item in func(*args, **kwargs):
|
124
|
-
results.append(item)
|
125
|
-
yield item
|
126
|
-
span.end()
|
127
|
-
|
128
|
-
# recover parent context
|
129
|
-
_SESSION_EXTRA.reset(token)
|
130
|
-
# TODO: collect results
|
116
|
+
return result
|
131
117
|
|
132
|
-
if inspect.isgeneratorfunction(func):
|
133
|
-
return generator_wrapper
|
134
118
|
return wrapper
|
135
119
|
|
136
120
|
# Function Decorator
|
@@ -0,0 +1,57 @@
|
|
1
|
+
import copy
|
2
|
+
import os
|
3
|
+
from typing import Any, Dict, Optional
|
4
|
+
|
5
|
+
from openai.types.chat import ChatCompletion
|
6
|
+
from typing_extensions import List
|
7
|
+
|
8
|
+
import divi
|
9
|
+
from divi.evaluation import Evaluator
|
10
|
+
from divi.evaluation.evaluator import EvaluatorConfig
|
11
|
+
from divi.evaluation.scores import Score
|
12
|
+
|
13
|
+
OPENAI_API_KEY = "OPENAI_API_KEY"
|
14
|
+
OPENAI_BASE_URL = "OPENAI_BASE_URL"
|
15
|
+
|
16
|
+
|
17
|
+
def init_evaluator(config: Optional[EvaluatorConfig] = None):
|
18
|
+
_config = config or EvaluatorConfig()
|
19
|
+
api_key = _config.api_key if _config.api_key else os.getenv(OPENAI_API_KEY)
|
20
|
+
base_url = (
|
21
|
+
_config.base_url if _config.base_url else os.getenv(OPENAI_BASE_URL)
|
22
|
+
)
|
23
|
+
if api_key is None:
|
24
|
+
raise ValueError("API key is required for evaluator")
|
25
|
+
_config.api_key = api_key
|
26
|
+
_config.base_url = base_url
|
27
|
+
evaluator = Evaluator(_config)
|
28
|
+
return evaluator
|
29
|
+
|
30
|
+
|
31
|
+
def evaluate_scores(
|
32
|
+
inputs: Dict[str, Any],
|
33
|
+
outputs: ChatCompletion,
|
34
|
+
scores: List[Score],
|
35
|
+
config: Optional[EvaluatorConfig] = None,
|
36
|
+
):
|
37
|
+
if not divi._evaluator:
|
38
|
+
divi._evaluator = init_evaluator(config)
|
39
|
+
|
40
|
+
# create conversation with result and inputs
|
41
|
+
input_messages = inputs.get("messages", None)
|
42
|
+
if input_messages is None:
|
43
|
+
raise ValueError("No messages found in inputs")
|
44
|
+
output_message = outputs.choices[0].message
|
45
|
+
if output_message is None:
|
46
|
+
raise ValueError("No message found in outputs")
|
47
|
+
|
48
|
+
conversations = copy.deepcopy(input_messages)
|
49
|
+
conversations.append(
|
50
|
+
{"role": output_message.role, "content": output_message.content}
|
51
|
+
)
|
52
|
+
evaluation_scores = divi._evaluator.evaluate(
|
53
|
+
"\n".join(f"{m['role']}: {m['content']}" for m in conversations), scores
|
54
|
+
)
|
55
|
+
|
56
|
+
# TODO: collect all evaluation scores and link them to span
|
57
|
+
print(evaluation_scores)
|
@@ -0,0 +1,161 @@
|
|
1
|
+
import asyncio
|
2
|
+
import concurrent.futures
|
3
|
+
import random
|
4
|
+
from typing import List, Literal, Optional
|
5
|
+
|
6
|
+
import openai
|
7
|
+
from pydantic import BaseModel
|
8
|
+
|
9
|
+
from divi.evaluation.prompts import PRESET_PROMPT, PROMPT_TEMPLATE
|
10
|
+
from divi.evaluation.scores import Score
|
11
|
+
|
12
|
+
|
13
|
+
class EvaluatorConfig:
|
14
|
+
def __init__(
|
15
|
+
self,
|
16
|
+
model: str = "gpt-4.1-nano",
|
17
|
+
temperature: float = 0.5,
|
18
|
+
max_concurrency: int = 10,
|
19
|
+
api_key: Optional[str] = None,
|
20
|
+
base_url: Optional[str] = None,
|
21
|
+
):
|
22
|
+
self.model = model
|
23
|
+
self.api_key = api_key
|
24
|
+
self.base_url = base_url
|
25
|
+
self.temperature = temperature
|
26
|
+
self.max_concurrency = max_concurrency
|
27
|
+
|
28
|
+
|
29
|
+
class EvaluationResult(BaseModel):
|
30
|
+
name: Score
|
31
|
+
judgment: bool
|
32
|
+
reasoning: str
|
33
|
+
|
34
|
+
|
35
|
+
class EvaluationScore(BaseModel):
|
36
|
+
name: Score
|
37
|
+
score: float
|
38
|
+
representative_reasoning: str
|
39
|
+
all_evaluations: List[EvaluationResult]
|
40
|
+
|
41
|
+
|
42
|
+
class Evaluator:
|
43
|
+
def __init__(self, config: Optional[EvaluatorConfig] = None):
|
44
|
+
self.config = config or EvaluatorConfig()
|
45
|
+
self.async_client = openai.AsyncOpenAI(
|
46
|
+
api_key=self.config.api_key, base_url=self.config.base_url
|
47
|
+
)
|
48
|
+
self.sync_client = openai.OpenAI(
|
49
|
+
api_key=self.config.api_key, base_url=self.config.base_url
|
50
|
+
)
|
51
|
+
|
52
|
+
@staticmethod
|
53
|
+
def generate_prompt(conversation: str, score: Score) -> str:
|
54
|
+
return PROMPT_TEMPLATE.format(
|
55
|
+
requirements=PRESET_PROMPT[score.value], conversation=conversation
|
56
|
+
)
|
57
|
+
|
58
|
+
def _sync_evaluate_once(
|
59
|
+
self, conversation: str, score: Score
|
60
|
+
) -> Optional[EvaluationResult]:
|
61
|
+
prompt = self.generate_prompt(conversation, score)
|
62
|
+
response = self.sync_client.beta.chat.completions.parse(
|
63
|
+
model=self.config.model,
|
64
|
+
messages=[{"role": "user", "content": prompt}],
|
65
|
+
temperature=self.config.temperature,
|
66
|
+
response_format=EvaluationResult,
|
67
|
+
)
|
68
|
+
return response.choices[0].message.parsed
|
69
|
+
|
70
|
+
async def _async_evaluate_once(
|
71
|
+
self, conversation: str, score: Score
|
72
|
+
) -> Optional[EvaluationResult]:
|
73
|
+
prompt = self.generate_prompt(conversation, score)
|
74
|
+
response = await self.async_client.beta.chat.completions.parse(
|
75
|
+
model=self.config.model,
|
76
|
+
messages=[{"role": "user", "content": prompt}],
|
77
|
+
temperature=self.config.temperature,
|
78
|
+
response_format=EvaluationResult,
|
79
|
+
)
|
80
|
+
return response.choices[0].message.parsed
|
81
|
+
|
82
|
+
def _aggregate_result(
|
83
|
+
self, name: Score, evaluations: List[EvaluationResult]
|
84
|
+
) -> EvaluationScore:
|
85
|
+
n = len(evaluations)
|
86
|
+
true_count = sum(1 for e in evaluations if e.judgment is True)
|
87
|
+
score = true_count / n
|
88
|
+
majority_judgment = True if true_count >= (n / 2) else False
|
89
|
+
majority_reasons = [
|
90
|
+
e.reasoning for e in evaluations if e.judgment == majority_judgment
|
91
|
+
]
|
92
|
+
representative_reasoning = (
|
93
|
+
random.choice(majority_reasons) if majority_reasons else ""
|
94
|
+
)
|
95
|
+
return EvaluationScore(
|
96
|
+
name=name,
|
97
|
+
score=score,
|
98
|
+
representative_reasoning=representative_reasoning,
|
99
|
+
all_evaluations=evaluations,
|
100
|
+
)
|
101
|
+
|
102
|
+
def _aggregate_results(
|
103
|
+
self, evaluations: List[EvaluationResult]
|
104
|
+
) -> List[EvaluationScore]:
|
105
|
+
scores = {}
|
106
|
+
for evaluation in evaluations:
|
107
|
+
if evaluation.name not in scores:
|
108
|
+
scores[evaluation.name] = []
|
109
|
+
scores[evaluation.name].append(evaluation)
|
110
|
+
|
111
|
+
aggregated_results = [
|
112
|
+
self._aggregate_result(name, evals)
|
113
|
+
for name, evals in scores.items()
|
114
|
+
]
|
115
|
+
return aggregated_results
|
116
|
+
|
117
|
+
def evaluate_sync(
|
118
|
+
self, conversation: str, scores: list[Score], n_rounds: int
|
119
|
+
) -> List[EvaluationScore]:
|
120
|
+
with concurrent.futures.ThreadPoolExecutor(
|
121
|
+
max_workers=self.config.max_concurrency
|
122
|
+
) as executor:
|
123
|
+
futures = [
|
124
|
+
executor.submit(self._sync_evaluate_once, conversation, score)
|
125
|
+
for _ in range(n_rounds)
|
126
|
+
for score in scores
|
127
|
+
]
|
128
|
+
evaluations = [
|
129
|
+
f.result() for f in concurrent.futures.as_completed(futures)
|
130
|
+
]
|
131
|
+
return self._aggregate_results(
|
132
|
+
[e for e in evaluations if e is not None]
|
133
|
+
)
|
134
|
+
|
135
|
+
async def evaluate_async(
|
136
|
+
self, conversation: str, scores: list[Score], n_rounds: int
|
137
|
+
) -> List[EvaluationScore]:
|
138
|
+
semaphore = asyncio.Semaphore(self.config.max_concurrency)
|
139
|
+
|
140
|
+
async def sem_task(score):
|
141
|
+
async with semaphore:
|
142
|
+
return await self._async_evaluate_once(conversation, score)
|
143
|
+
|
144
|
+
tasks = [sem_task(score) for _ in range(n_rounds) for score in scores]
|
145
|
+
evaluations = await asyncio.gather(*tasks)
|
146
|
+
return self._aggregate_results(
|
147
|
+
[e for e in evaluations if e is not None]
|
148
|
+
)
|
149
|
+
|
150
|
+
def evaluate(
|
151
|
+
self,
|
152
|
+
conversation: str,
|
153
|
+
scores: list[Score],
|
154
|
+
n_rounds: int = 5,
|
155
|
+
mode: Literal["sync", "async"] = "sync",
|
156
|
+
) -> List[EvaluationScore]:
|
157
|
+
if mode == "async":
|
158
|
+
return asyncio.run(
|
159
|
+
self.evaluate_async(conversation, scores, n_rounds)
|
160
|
+
)
|
161
|
+
return self.evaluate_sync(conversation, scores, n_rounds)
|
@@ -0,0 +1,14 @@
|
|
1
|
+
PROMPT_TEMPLATE = (
|
2
|
+
"Your evaluation task is: {requirements}\n\n"
|
3
|
+
"Please perform step-by-step reasoning to reach your judgment.\n\n"
|
4
|
+
"Strictly output your answer in the following JSON format:\n"
|
5
|
+
'{{"judgment": bool, "reasoning": "string"}}\n\n'
|
6
|
+
"Do not output anything else.\n\n"
|
7
|
+
"Here is the conversation to evaluate:\n"
|
8
|
+
"{conversation}"
|
9
|
+
)
|
10
|
+
|
11
|
+
PRESET_PROMPT = {
|
12
|
+
"task_completion": "Evaluate whether the model's output completely fulfills the user's task requirements.",
|
13
|
+
"instruction_adherence": "Evaluate whether the model's output strictly follows the user's instructions without omissions, deviations, or hallucinations.",
|
14
|
+
}
|
@@ -1,16 +1,16 @@
|
|
1
|
-
from typing import Any, Dict
|
2
|
-
from typing_extensions import Mapping
|
1
|
+
from typing import Any, Dict
|
3
2
|
|
4
3
|
from google.protobuf.json_format import MessageToDict
|
4
|
+
from openai import NotGiven
|
5
5
|
from openai.types.chat import ChatCompletion
|
6
6
|
from pydantic import UUID4
|
7
|
+
from typing_extensions import Mapping
|
7
8
|
|
8
9
|
import divi
|
9
10
|
from divi.proto.trace.v1.trace_pb2 import ScopeSpans
|
10
11
|
from divi.services.service import Service
|
11
12
|
from divi.session.session import SessionSignal
|
12
13
|
from divi.signals.trace.trace import TraceSignal
|
13
|
-
from openai import NotGiven
|
14
14
|
|
15
15
|
|
16
16
|
class DataPark(Service):
|
@@ -33,7 +33,11 @@ class DataPark(Service):
|
|
33
33
|
if not isinstance(obj, Mapping):
|
34
34
|
return obj
|
35
35
|
|
36
|
-
return {
|
36
|
+
return {
|
37
|
+
key: value
|
38
|
+
for key, value in obj.items()
|
39
|
+
if not isinstance(value, NotGiven)
|
40
|
+
}
|
37
41
|
|
38
42
|
def create_session(self, session: SessionSignal) -> None:
|
39
43
|
self.post("/api/session/", payload=session)
|
@@ -49,6 +53,7 @@ class DataPark(Service):
|
|
49
53
|
def create_chat_completion(
|
50
54
|
self,
|
51
55
|
span_id: bytes,
|
56
|
+
trace_id: UUID4,
|
52
57
|
inputs: Dict[str, Any],
|
53
58
|
completion: ChatCompletion,
|
54
59
|
) -> None:
|
@@ -61,6 +66,7 @@ class DataPark(Service):
|
|
61
66
|
},
|
62
67
|
"/api/v1/chat/completions": {
|
63
68
|
"span_id": hex_span_id,
|
69
|
+
"trace_id": str(trace_id),
|
64
70
|
"data": completion.model_dump(),
|
65
71
|
},
|
66
72
|
}
|
divi/services/init.py
CHANGED
divi/session/session.py
CHANGED
@@ -9,13 +9,13 @@ class SessionExtra(TypedDict, total=False):
|
|
9
9
|
|
10
10
|
session_name: Optional[str]
|
11
11
|
"""Name of the session"""
|
12
|
-
trace:
|
13
|
-
"""Trace
|
12
|
+
trace: Trace
|
13
|
+
"""Trace in session"""
|
14
14
|
parent_span_id: Optional[bytes]
|
15
15
|
"""Parent Span ID fixed string(8)"""
|
16
16
|
|
17
17
|
|
18
|
-
class SessionSignal(TypedDict
|
18
|
+
class SessionSignal(TypedDict):
|
19
19
|
"""Session request"""
|
20
20
|
|
21
21
|
id: str
|
divi/session/setup.py
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
from typing_extensions import Optional
|
2
|
+
|
1
3
|
import divi
|
2
4
|
from divi.services import init as init_services
|
3
5
|
from divi.session import Session, SessionExtra
|
@@ -5,10 +7,10 @@ from divi.signals.trace import Span
|
|
5
7
|
from divi.signals.trace.trace import Trace
|
6
8
|
|
7
9
|
|
8
|
-
def
|
10
|
+
def init_session(name: Optional[str] = None) -> Session:
|
9
11
|
"""init initializes the services and the Run"""
|
10
12
|
init_services()
|
11
|
-
session = Session(name=
|
13
|
+
session = Session(name=name)
|
12
14
|
if divi._datapark:
|
13
15
|
divi._datapark.create_session(session.signal)
|
14
16
|
return session
|
@@ -24,39 +26,23 @@ def setup(
|
|
24
26
|
span (Span): Span instance
|
25
27
|
session_extra (SessionExtra | None): Extra information from user input
|
26
28
|
"""
|
27
|
-
# TOOD: merge run_extra input by user with the one from the context
|
28
|
-
# temp solution: Priority: run_extra_context.get() > run_extra
|
29
29
|
session_extra = session_extra or SessionExtra()
|
30
30
|
|
31
31
|
# init the session if not already initialized
|
32
32
|
if not divi._session:
|
33
|
-
divi._session =
|
33
|
+
divi._session = init_session(
|
34
|
+
name=session_extra.get("session_name") or span.name
|
35
|
+
)
|
34
36
|
|
35
37
|
# setup trace
|
36
|
-
|
37
|
-
trace = session_extra.get("trace")
|
38
|
+
trace = session_extra.get("trace") or Trace(divi._session.id, span.name)
|
38
39
|
parent_span_id = session_extra.get("parent_span_id")
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
trace.start()
|
44
|
-
span._as_root(trace.trace_id)
|
45
|
-
# create the trace
|
46
|
-
if divi._datapark:
|
47
|
-
divi._datapark.upsert_traces(
|
48
|
-
session_id=divi._session.id, traces=[trace.signal]
|
49
|
-
)
|
50
|
-
|
51
|
-
# update the session_extra with the current span
|
52
|
-
# session_extra["trace_id"] = span.trace_id
|
53
|
-
# session_extra["parent_span_id"] = span.span_id
|
54
|
-
session_extra = SessionExtra(
|
40
|
+
span._add_node(trace.trace_id, parent_span_id)
|
41
|
+
|
42
|
+
# update the session_extra with the current trace and span
|
43
|
+
return SessionExtra(
|
55
44
|
session_name=divi._session.name,
|
56
45
|
trace=trace,
|
57
46
|
# set the parent_span_id to the current span_id
|
58
47
|
parent_span_id=span.span_id,
|
59
48
|
)
|
60
|
-
|
61
|
-
# offer end hook to collect data at whe end of the span ?
|
62
|
-
return session_extra
|
divi/signals/trace/trace.py
CHANGED
@@ -7,6 +7,7 @@ from uuid import uuid4
|
|
7
7
|
from pydantic import UUID4
|
8
8
|
from typing_extensions import TypedDict
|
9
9
|
|
10
|
+
import divi
|
10
11
|
from divi.proto.common.v1.common_pb2 import KeyValue
|
11
12
|
from divi.proto.trace.v1.trace_pb2 import Span as SpanProto
|
12
13
|
|
@@ -29,15 +30,19 @@ class TraceSignal(TypedDict, total=False):
|
|
29
30
|
"""Start time in iso format"""
|
30
31
|
end_time: NullTime
|
31
32
|
"""End time in iso format"""
|
33
|
+
name: Optional[str]
|
32
34
|
|
33
35
|
|
34
36
|
class Trace:
|
35
|
-
def __init__(self, session_id: UUID4):
|
37
|
+
def __init__(self, session_id: UUID4, name: Optional[str] = None):
|
36
38
|
self.trace_id: UUID4 = uuid4()
|
37
39
|
self.start_time: str | None = None
|
38
40
|
self.end_time: str | None = None
|
41
|
+
self.name: Optional[str] = name
|
39
42
|
self.session_id: UUID4 = session_id
|
40
43
|
|
44
|
+
self.start()
|
45
|
+
|
41
46
|
@property
|
42
47
|
def signal(self) -> TraceSignal:
|
43
48
|
if self.start_time is None:
|
@@ -45,6 +50,7 @@ class Trace:
|
|
45
50
|
signal = TraceSignal(
|
46
51
|
id=str(self.trace_id),
|
47
52
|
start_time=self.start_time,
|
53
|
+
name=self.name,
|
48
54
|
)
|
49
55
|
if self.end_time is not None:
|
50
56
|
signal["end_time"] = NullTime(
|
@@ -60,12 +66,21 @@ class Trace:
|
|
60
66
|
def start(self):
|
61
67
|
"""Start the trace by recording the current time in nanoseconds."""
|
62
68
|
self.start_time = datetime.now(UTC).isoformat()
|
69
|
+
self.upsert_trace()
|
63
70
|
|
64
71
|
def end(self):
|
65
72
|
"""End the trace by recording the end time in nanoseconds."""
|
66
73
|
if self.start_time is None:
|
67
74
|
raise ValueError("Span must be started before ending.")
|
68
75
|
self.end_time = datetime.now(UTC).isoformat()
|
76
|
+
self.upsert_trace()
|
77
|
+
|
78
|
+
def upsert_trace(self):
|
79
|
+
"""Upsert trace with datapark."""
|
80
|
+
if divi._datapark:
|
81
|
+
divi._datapark.upsert_traces(
|
82
|
+
session_id=self.session_id, traces=[self.signal]
|
83
|
+
)
|
69
84
|
|
70
85
|
|
71
86
|
class Span:
|
@@ -102,7 +117,6 @@ class Span:
|
|
102
117
|
trace_id=self.trace_id.bytes if self.trace_id else None,
|
103
118
|
parent_span_id=self.parent_span_id,
|
104
119
|
)
|
105
|
-
print(self.kind)
|
106
120
|
signal.metadata.extend(
|
107
121
|
KeyValue(key=k, value=v)
|
108
122
|
for k, v in (self.metadata or dict()).items()
|
@@ -131,21 +145,7 @@ class Span:
|
|
131
145
|
raise ValueError("Span must be started before ending.")
|
132
146
|
self.end_time_unix_nano = time.time_ns()
|
133
147
|
|
134
|
-
def
|
135
|
-
"""
|
136
|
-
self.trace_id = trace_id
|
137
|
-
print("as root")
|
138
|
-
print(f"name: {self.name}")
|
139
|
-
print(f"trace_id: {self.trace_id}")
|
140
|
-
print(f"span_id: {self.span_id}")
|
141
|
-
|
142
|
-
def _add_parent(self, trace_id: UUID4, parent_id: bytes):
|
143
|
-
"""Set the parent span ID."""
|
148
|
+
def _add_node(self, trace_id: UUID4, parent_id: Optional[bytes] = None):
|
149
|
+
"""Add node for obs tree."""
|
144
150
|
self.trace_id = trace_id
|
145
151
|
self.parent_span_id = parent_id
|
146
|
-
|
147
|
-
print("add parent")
|
148
|
-
print(f"name: {self.name}")
|
149
|
-
print(f"trace_id: {trace_id}")
|
150
|
-
print(f"span_id: {self.span_id}")
|
151
|
-
print(f"parent_span_id: {parent_id}")
|
@@ -1,9 +1,13 @@
|
|
1
|
-
divi/__init__.py,sha256
|
1
|
+
divi/__init__.py,sha256=IcRE_HzskERpnkps5ABPTGxtsLGcRN8eRTjozDU5OMU,485
|
2
2
|
divi/utils.py,sha256=fXkjoyo_Lh8AZliKICOP460m0czUcNQjcEcceJbaOVA,1439
|
3
|
-
divi/config/config.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
4
3
|
divi/decorators/__init__.py,sha256=HkyWdC1ctTsVFucCWCkj57JB4NmwONus1d2S2dUbvs4,110
|
5
|
-
divi/decorators/obs_openai.py,sha256=
|
6
|
-
divi/decorators/observable.py,sha256=
|
4
|
+
divi/decorators/obs_openai.py,sha256=CGX60JgOKsN3E06yCq2L8baN9ZLiJJm3ukBsaN3IBBg,1324
|
5
|
+
divi/decorators/observable.py,sha256=Ko_Pmw8l_T9mNp0corXkWqawwNEmQN7gf7rcAq-DB9M,3829
|
6
|
+
divi/evaluation/__init__.py,sha256=3qMHWu_zBh6FJa6-1dZZEWiAblQZurn5doa0OjGvDGs,93
|
7
|
+
divi/evaluation/evaluate.py,sha256=4Li8NciQXcgmuAYRGFaB5C3EzMZR79n8IW0ubJBVrbQ,1789
|
8
|
+
divi/evaluation/evaluator.py,sha256=22URc2pqg6WSYT-xUavxI8eT5w6t-mdVyTDZy-Qe4LU,5395
|
9
|
+
divi/evaluation/prompts.py,sha256=yRFP3QjtcMflZ-jsFqXSu9RHlQA1cEVgDXo5D4VC6lo,662
|
10
|
+
divi/evaluation/scores.py,sha256=ZgSxfve-ZivX3WU4TGcgPOSpUQVMbG5a15IQNPeq_bQ,173
|
7
11
|
divi/proto/common/v1/common.proto,sha256=Rx8wr0_tOtQ1NseTMnsav4ApD1MDALzQDBA2IvLRTU0,1775
|
8
12
|
divi/proto/common/v1/common_pb2.py,sha256=br61OHQVAi6SI3baFcb5xJv2Xd-AZ04A19xeSjLNMXo,2442
|
9
13
|
divi/proto/common/v1/common_pb2.pyi,sha256=LmTpFFLxHg2a_qPIdNXXwGEMkbiDcTJdarR9eC-6Fq8,2133
|
@@ -19,7 +23,7 @@ divi/proto/trace/v1/trace_pb2.py,sha256=CuTkSSvhxCa1bk3Ku7tgLqRSovp_Gi52CZ0zLcLP
|
|
19
23
|
divi/proto/trace/v1/trace_pb2.pyi,sha256=rPo2Oa3NWrINE_dyOVU9HUYHo5LY82Bm5TMenj5dnK8,2136
|
20
24
|
divi/services/__init__.py,sha256=TcVJ_gKxyPIcwhT9GgttqHeyk0icW44uE285KmUiyh4,185
|
21
25
|
divi/services/finish.py,sha256=XKPKGJ5cWd5H95G_VpIOlOZOLrcf9StoTs7ayRic2jY,173
|
22
|
-
divi/services/init.py,sha256=
|
26
|
+
divi/services/init.py,sha256=BFSsW6UzeujfHba8pvVPnEYmW0Y8VVFQLuIXI5Z2Mu4,436
|
23
27
|
divi/services/service.py,sha256=539MhcYfMvsVGjDdu0UtYSZnL2cloaPeYeOSMl2eUy8,1532
|
24
28
|
divi/services/auth/__init__.py,sha256=PIQ9rQ0jcRqcy03a3BOY7wbzwluIRG_4kI_H4J4mRFk,74
|
25
29
|
divi/services/auth/auth.py,sha256=eRcE6Kq8jbBr6YL93HCGDIoga90SoZf3ogOAKeza9WY,445
|
@@ -30,16 +34,16 @@ divi/services/core/core.py,sha256=PRwPtLgrgmCrejUfKf7HJNrAhGS0paFNZ7JwDToEUAk,12
|
|
30
34
|
divi/services/core/finish.py,sha256=dIGQpVXcJY4-tKe7A1_VV3yoSHNCDPfOlUltvzvk6VI,231
|
31
35
|
divi/services/core/init.py,sha256=e7-fgpOPglBXyEoPkgOAnpJk2ApdFbo7LPupxOb8N-w,1966
|
32
36
|
divi/services/datapark/__init__.py,sha256=GbV1mwHE07yutgOlCIYHykSEL5KJ-ApgLutGMzu2eUE,86
|
33
|
-
divi/services/datapark/datapark.py,sha256=
|
37
|
+
divi/services/datapark/datapark.py,sha256=d2pbrzVJtR3mNW1eQpbm-Wca-SvcfJqT7IuaQy7yHT0,2285
|
34
38
|
divi/services/datapark/init.py,sha256=C32f9t3eLsxcYNqEyheh6nW455G2oR0YhhdqBcbN3ec,92
|
35
39
|
divi/session/__init__.py,sha256=6lYemv21VQCIHx-xIdi7BxXcPNxVdvE60--8ArReUew,82
|
36
|
-
divi/session/session.py,sha256=
|
37
|
-
divi/session/setup.py,sha256=
|
40
|
+
divi/session/session.py,sha256=LlB2W2qGo0Vf-0L0CTQoXfzg_gCGpf0MTFsXQW7E6i4,817
|
41
|
+
divi/session/setup.py,sha256=XU4_wXqdIg91dZSttjmjlC6u3hIyV0Nn_xqRw8db7os,1405
|
38
42
|
divi/session/teardown.py,sha256=YiBz_3yCiljMFEofZ60VmRL5sb8WA5GT7EYF8nFznZ4,133
|
39
43
|
divi/signals/__init__.py,sha256=K1PaTAMwyBDsK6jJUg4QWy0xVJ_5MA6dlWiUyJeiSQA,44
|
40
44
|
divi/signals/trace/__init__.py,sha256=K1PaTAMwyBDsK6jJUg4QWy0xVJ_5MA6dlWiUyJeiSQA,44
|
41
|
-
divi/signals/trace/trace.py,sha256=
|
42
|
-
divi-0.0.1.
|
43
|
-
divi-0.0.1.
|
44
|
-
divi-0.0.1.
|
45
|
-
divi-0.0.1.
|
45
|
+
divi/signals/trace/trace.py,sha256=OsfrZPHp241_NN8W79U4O69HsHQajez_d3rz6yJRN9s,4508
|
46
|
+
divi-0.0.1.dev28.dist-info/METADATA,sha256=I2KiV8mzi0AnMFpUaWYHGKE0ETSDKvS6wcMMx08vLVw,497
|
47
|
+
divi-0.0.1.dev28.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
48
|
+
divi-0.0.1.dev28.dist-info/licenses/LICENSE,sha256=5OJuZ4wMMEV0DgF0tofhAlS_KLkaUsZwwwDS2U_GwQ0,1063
|
49
|
+
divi-0.0.1.dev28.dist-info/RECORD,,
|
divi/config/config.py
DELETED
File without changes
|
File without changes
|
File without changes
|