divi 0.0.1.dev23__tar.gz → 0.0.1.dev47__tar.gz

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: divi
-Version: 0.0.1.dev23
+Version: 0.0.1.dev47
 Summary: The Agent Platform for Observability & Evaluation
 License-File: LICENSE
 Requires-Python: >=3.11

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/apps/graphql/package.json

@@ -13,7 +13,7 @@
     "@apollo/datasource-rest": "^6.4.1",
     "@apollo/server": "^4.12.0",
     "@apollo/utils.keyvaluecache": "^3.1.0",
-    "graphql": "^16.10.0",
+    "graphql": "^16.11.0",
     "graphql-tag": "^2.12.6"
   },
   "devDependencies": {
@@ -21,7 +21,7 @@
     "@graphql-codegen/typescript": "^4.1.6",
     "@graphql-codegen/typescript-resolvers": "^4.5.0",
     "@workspace/typescript-config": "workspace:*",
-    "nodemon": "^3.1.9",
+    "nodemon": "^3.1.10",
     "ts-node-dev": "^2.0.0",
     "typescript": "catalog:"
   }

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/apps/graphql/src/datasources/datapark-api.ts

@@ -1,5 +1,5 @@
 import type { FetchResponse } from '@/types/response';
-import type { ChatInput, Span, Trace, UsageResult } from '@/types/types';
+import type { ChatInput, Score, Span, Trace, UsageResult } from '@/types/types';
 import { type AugmentedRequest, RESTDataSource } from '@apollo/datasource-rest';
 import type { KeyValueCache } from '@apollo/utils.keyvaluecache';
 
@@ -30,6 +30,12 @@ export class DataParkAPI extends RESTDataSource {
     return await this.get<FetchResponse<Span[]>>(`/api/trace/${traceId}/spans`);
   }
 
+  async getScores(traceId: string) {
+    return await this.get<FetchResponse<Score[]>>(
+      `/api/trace/${traceId}/scores`
+    );
+  }
+
   async getChatInput(spanId: string) {
     return await this.get<FetchResponse<ChatInput>>(
       `/api/v1/chat/completions/${spanId}/input`

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/apps/graphql/src/resolvers.ts

@@ -26,6 +26,9 @@ export const resolvers: Resolvers = {
     spans: async (_, { trace_id }, { dataSources }) => {
       return (await dataSources.dataparkAPI.getSpans(trace_id)).data;
     },
+    scores: async (_, { trace_id }, { dataSources }) => {
+      return (await dataSources.dataparkAPI.getScores(trace_id)).data;
+    },
     chat_input: async (_, { span_id }, { dataSources }) => {
       return (await dataSources.dataparkAPI.getChatInput(span_id)).data;
     },

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/apps/graphql/src/schema.graphql

@@ -20,6 +20,8 @@ type Query {
     end_time: Int
     group_by: GroupingKey
   ): [UsageResult!]
+  "Fetch all scores by trace id"
+  scores(trace_id: ID!): [Score!]
 }
 
 "GroupingKey is an enum that represents the key used to group usage results"
@@ -70,6 +72,7 @@ type Trace {
 enum Kind {
   SPAN_KIND_FUNCTION
   SPAN_KIND_LLM
+  SPAN_KIND_EVALUATION
 }
 
 "Span is a record of a single unit of work within a trace"
@@ -79,8 +82,8 @@ type Span {
   trace_id: ID!
   name: String!
   start_time: String!
-  end_time: String!
-  duration: Float!
+  end_time: NullTime!
+  duration: Float
   kind: Kind!
   metadata: [KeyValue!]
 }
@@ -113,6 +116,14 @@ type UsageResult {
   date: Int
 }
 
+"Score is a record of a score for a trace"
+type Score {
+  span_id: ID!
+  name: String!
+  score: Float!
+  representative_reasoning: String!
+}
+
 "Mutation is a collection of mutations that can be made to the API"
 type Mutation {
   "Auth Mutations"

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/apps/graphql/src/types/types.d.ts

@@ -92,6 +92,7 @@ export type KeyValue = {
 
 /** Kind is an enum that represents the type of span */
 export enum Kind {
+  SpanKindEvaluation = 'SPAN_KIND_EVALUATION',
   SpanKindFunction = 'SPAN_KIND_FUNCTION',
   SpanKindLlm = 'SPAN_KIND_LLM'
 }
@@ -201,6 +202,8 @@ export type Query = {
   completion_usage?: Maybe<Array<UsageResult>>;
   /** Fetch current user */
   me: User;
+  /** Fetch all scores by trace id */
+  scores?: Maybe<Array<Score>>;
   /** Fetch all spans by trace id */
   spans?: Maybe<Array<Span>>;
   /** Fetch traces by session id */
@@ -224,6 +227,12 @@ export type QueryCompletion_UsageArgs = {
 };
 
 
+/** Query is a collection of queries that can be made to the API */
+export type QueryScoresArgs = {
+  trace_id: Scalars['ID']['input'];
+};
+
+
 /** Query is a collection of queries that can be made to the API */
 export type QuerySpansArgs = {
   trace_id: Scalars['ID']['input'];
@@ -249,11 +258,20 @@ export type RevokeApiKeyResponse = MutationResponse & {
   success: Scalars['Boolean']['output'];
 };
 
+/** Score is a record of a score for a trace */
+export type Score = {
+  __typename?: 'Score';
+  name: Scalars['String']['output'];
+  representative_reasoning: Scalars['String']['output'];
+  score: Scalars['Float']['output'];
+  span_id: Scalars['ID']['output'];
+};
+
 /** Span is a record of a single unit of work within a trace */
 export type Span = {
   __typename?: 'Span';
-  duration: Scalars['Float']['output'];
-  end_time: Scalars['String']['output'];
+  duration?: Maybe<Scalars['Float']['output']>;
+  end_time: NullTime;
   id: Scalars['ID']['output'];
   kind: Kind;
   metadata?: Maybe<Array<KeyValue>>;
@@ -405,6 +423,7 @@ export type ResolversTypes = {
   NullTime: ResolverTypeWrapper<NullTime>;
   Query: ResolverTypeWrapper<{}>;
   RevokeAPIKeyResponse: ResolverTypeWrapper<RevokeApiKeyResponse>;
+  Score: ResolverTypeWrapper<Score>;
   Span: ResolverTypeWrapper<Span>;
   String: ResolverTypeWrapper<Scalars['String']['output']>;
   Trace: ResolverTypeWrapper<Trace>;
@@ -433,6 +452,7 @@ export type ResolversParentTypes = {
   NullTime: NullTime;
   Query: {};
   RevokeAPIKeyResponse: RevokeApiKeyResponse;
+  Score: Score;
   Span: Span;
   String: Scalars['String']['output'];
   Trace: Trace;
@@ -536,6 +556,7 @@ export type QueryResolvers<ContextType = DataSourceContext, ParentType extends R
   chat_input?: Resolver<Maybe<ResolversTypes['ChatInput']>, ParentType, ContextType, RequireFields<QueryChat_InputArgs, 'span_id'>>;
   completion_usage?: Resolver<Maybe<Array<ResolversTypes['UsageResult']>>, ParentType, ContextType, RequireFields<QueryCompletion_UsageArgs, 'start_time'>>;
   me?: Resolver<ResolversTypes['User'], ParentType, ContextType>;
+  scores?: Resolver<Maybe<Array<ResolversTypes['Score']>>, ParentType, ContextType, RequireFields<QueryScoresArgs, 'trace_id'>>;
   spans?: Resolver<Maybe<Array<ResolversTypes['Span']>>, ParentType, ContextType, RequireFields<QuerySpansArgs, 'trace_id'>>;
   traces?: Resolver<Maybe<Array<ResolversTypes['Trace']>>, ParentType, ContextType, RequireFields<QueryTracesArgs, 'session_id'>>;
   user?: Resolver<ResolversTypes['User'], ParentType, ContextType, RequireFields<QueryUserArgs, 'id'>>;
@@ -548,9 +569,17 @@ export type RevokeApiKeyResponseResolvers<ContextType = DataSourceContext, Paren
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
+export type ScoreResolvers<ContextType = DataSourceContext, ParentType extends ResolversParentTypes['Score'] = ResolversParentTypes['Score']> = {
+  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  representative_reasoning?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  score?: Resolver<ResolversTypes['Float'], ParentType, ContextType>;
+  span_id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
 export type SpanResolvers<ContextType = DataSourceContext, ParentType extends ResolversParentTypes['Span'] = ResolversParentTypes['Span']> = {
-  duration?: Resolver<ResolversTypes['Float'], ParentType, ContextType>;
-  end_time?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  duration?: Resolver<Maybe<ResolversTypes['Float']>, ParentType, ContextType>;
+  end_time?: Resolver<ResolversTypes['NullTime'], ParentType, ContextType>;
   id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
   kind?: Resolver<ResolversTypes['Kind'], ParentType, ContextType>;
   metadata?: Resolver<Maybe<Array<ResolversTypes['KeyValue']>>, ParentType, ContextType>;
@@ -618,6 +647,7 @@ export type Resolvers<ContextType = DataSourceContext> = {
   NullTime?: NullTimeResolvers<ContextType>;
   Query?: QueryResolvers<ContextType>;
   RevokeAPIKeyResponse?: RevokeApiKeyResponseResolvers<ContextType>;
+  Score?: ScoreResolvers<ContextType>;
   Span?: SpanResolvers<ContextType>;
   Trace?: TraceResolvers<ContextType>;
   UpdateAPIKeyResponse?: UpdateApiKeyResponseResolvers<ContextType>;

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/apps/web/app/dashboard/traces/(children)/[id]/components/Span.tsx

@@ -1,5 +1,10 @@
+import Highlighter from '@/components/Highter';
 import type { ExtendedSpan } from '@/lib/types/span';
 import { formatDate } from '@/lib/utils';
+import {
+  Kind,
+  type Score,
+} from '@workspace/graphql-client/src/types.generated';
 import {
   Accordion,
   AccordionContent,
@@ -16,7 +21,6 @@ import {
 } from '@workspace/ui/components/card';
 import { Timer } from 'lucide-react';
 import type * as React from 'react';
-import Highlighter from '@/components/Highter';
 
 interface SpanProps {
   span: ExtendedSpan;
@@ -30,11 +34,18 @@ export function Span({ span }: SpanProps) {
       <GeneralInfo span={span} />
       <Accordion
         type="multiple"
-        defaultValue={['properties', 'Input', 'Output']}
+        defaultValue={['properties', 'Input', 'Output', 'scores']}
       >
         <AccordionProperties span={span} />
-        <AccordionJsonCards name="Input" datas={messages} />
-        <AccordionJsonCards name="Output" datas={choices} />
+        {span.kind === Kind.SpanKindLlm && (
+          <>
+            <AccordionJsonCards name="Input" datas={messages} />
+            <AccordionJsonCards name="Output" datas={choices} />
+          </>
+        )}
+        {span.kind === Kind.SpanKindEvaluation && (
+          <AccordionScores scores={span.scores ?? []} />
+        )}
       </Accordion>
     </>
   );
@@ -57,10 +68,35 @@ function GeneralInfo({ span }: SpanProps) {
   );
 }
 
+function AccordionScores({ scores }: { scores: Score[] }) {
+  return (
+    <AccordionItem value="scores" className="px-4">
+      <AccordionTrigger className="hover:no-underline">Scores</AccordionTrigger>
+      <AccordionContent className="flex flex-col gap-3">
+        {scores.map((score) => (
+          <PropertyCard
+            title={score.name
+              .split('_')
+              .map((n) => n.charAt(0).toUpperCase() + n.slice(1).toLowerCase())
+              .join(' ')}
+            key={score.name}
+          >
+            <PropertyItem label="score" value={score.score} />
+            <PropertyItem
+              label="explanation"
+              value={score.representative_reasoning}
+            />
+          </PropertyCard>
+        ))}
+      </AccordionContent>
+    </AccordionItem>
+  );
+}
+
 function AccordionProperties({ span }: SpanProps) {
   // TODO select the properties to show
   const startTime = formatDate(span.start_time);
-  const endTime = formatDate(span.end_time);
+  const endTime = span.end_time.Valid ? formatDate(span.end_time.Time) : 'N/A';
   const usage = span.completion?.usage;
   // input should pop messages
   const { messages, ...inputs } = span.input ?? {};
@@ -128,9 +164,9 @@ function PropertyItem({
   value: string | number | boolean | null;
 }) {
   return (
-    <div className="flex items-center justify-between">
+    <div className="flex items-start justify-between gap-4">
       <span className="text-sm">{label}</span>
-      <span className="truncate font-light text-sm">{String(value)}</span>
+      <span className="break-words font-light text-sm">{String(value)}</span>
     </div>
   );
 }

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/apps/web/app/dashboard/traces/(children)/[id]/components/trace-board.tsx

@@ -16,6 +16,11 @@ export function TraceBoard({ spans, direction }: TraceBoardProps) {
   const selectAction = (_data: ExtendedSpan, index: number) => {
     setIndex(index);
   };
+  if (!spans || spans.length === 0) {
+    return (
+      <div className="flex h-full items-center justify-center">No data</div>
+    );
+  }
 
   return (
     <ResponsiveResizable

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/apps/web/app/dashboard/traces/(children)/[id]/components/trace-chart.tsx

@@ -32,6 +32,9 @@ const chartConfig = {
   SPAN_KIND_LLM: {
     color: 'var(--chart-2)',
   },
+  SPAN_KIND_EVALUATION: {
+    color: 'var(--chart-3)',
+  },
 } satisfies ChartConfig;
 
 interface TraceWaterfallChartProps {
@@ -107,7 +110,7 @@ export function TraceWaterfallChart({
           <LabelList
             dataKey="name"
             position="insideLeft"
-            className="fill-background"
+            className="fill-card"
           />
         </Bar>
       </BarChart>

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/apps/web/app/dashboard/usages/actions.ts

@@ -8,7 +8,7 @@ import type {
   GroupingKey,
   UsageResult,
 } from '@workspace/graphql-client/src/types.generated';
-import { eachDayOfInterval, getUnixTime } from 'date-fns';
+import { addDays, eachDayOfInterval, getUnixTime, startOfDay } from 'date-fns';
 import { cache } from 'react';
 
 export const getCompletionUsage = cache(
@@ -25,7 +25,9 @@ export const getCompletionUsage = cache(
       query: GetCompletionUsageDocument,
       variables: {
         startTime: getUnixTime(startTime),
-        endTime: getUnixTime(endTime ?? new Date()),
+        endTime: getUnixTime(
+          endTime ? addDays(startOfDay(endTime), 1) : new Date()
+        ),
         groupBy,
       },
       context,

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/apps/web/app/home/components/home-hero.tsx

@@ -9,7 +9,7 @@ export default function HomeHero() {
         <div className="flex flex-col items-center justify-center gap-8 py-20 lg:py-40">
           <div>
             <Button variant="secondary" size="sm" className="gap-4" asChild>
-              <Link href="https://docs.divine-agent.com">
+              <Link href="https://docs.divine-agent.com" target="_blank">
                 Read our launch article <MoveRight className="h-4 w-4" />
               </Link>
             </Button>

divi-0.0.1.dev47/apps/web/lib/server/evaluation.ts

@@ -0,0 +1,17 @@
+import { query } from '@/hooks/apolloClient';
+import { GetScoresDocument } from '@workspace/graphql-client/src/datapark/evaluation.generated';
+import { cache } from 'react';
+import { getAuthContext } from './auth';
+
+export const getScores = cache(async (traceId: string) => {
+  const context = await getAuthContext();
+  if (!context) {
+    return null;
+  }
+  const { data } = await query({
+    query: GetScoresDocument,
+    variables: { traceId },
+    context,
+  });
+  return data?.scores;
+});

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/apps/web/lib/server/span.ts

@@ -2,9 +2,14 @@ import 'server-only';
 import { query } from '@/hooks/apolloClient';
 import type { Chat, ExtendedSpan } from '@/lib/types/span';
 import { GetSpansDocument } from '@workspace/graphql-client/src/datapark/traces.generated';
-import { Kind, type Span } from '@workspace/graphql-client/src/types.generated';
+import {
+  Kind,
+  type Score,
+  type Span,
+} from '@workspace/graphql-client/src/types.generated';
 import { cache } from 'react';
 import { getAuthContext } from './auth';
+import { getScores } from './evaluation';
 import { getChat } from './openai';
 
 /**
@@ -31,7 +36,10 @@ export const getSpans = cache(async (traceId: string) => {
 export const getTraceChartData = cache(
   async (traceId: string): Promise<ExtendedSpan[]> => {
     // get spans for a trace
-    let spans = await getSpans(traceId);
+    let [spans, scores] = await Promise.all([
+      getSpans(traceId),
+      getScores(traceId),
+    ]);
     if (!spans) {
       return [];
     }
@@ -39,12 +47,23 @@ export const getTraceChartData = cache(
     spans = sortSpans(spans);
     // get chat input and completion for LLM span
     const llmSpanIds = spans
-      .filter((span) => span.kind === Kind.SpanKindLlm)
+      .filter((span) => span.kind === Kind.SpanKindLlm && span.end_time.Valid)
       .map((s) => s.id);
     const chats = await Promise.all(llmSpanIds.map((id) => getChat(id)));
     const chatsMap = new Map<string, Chat>(
       chats.map((chat) => [chat.span_id, chat])
     );
+    const scoresMap = new Map<string, Score[]>();
+    // group scores by span_id
+    if (scores) {
+      for (const score of scores) {
+        const spanId = score.span_id;
+        if (!scoresMap.has(spanId)) {
+          scoresMap.set(spanId, []);
+        }
+        scoresMap.get(spanId)?.push(score);
+      }
+    }
     // calculate relative start_time with milliseconds
     const startTime = new Date(spans[0]?.start_time).getTime();
     return spans.map((span) => {
@@ -54,6 +73,11 @@ export const getTraceChartData = cache(
         relative_start_time: new Date(span.start_time).getTime() - startTime,
         input: chat?.input,
         completion: chat?.completion,
+        duration: span.end_time.Valid
+          ? span.duration
+          : Date.now() - new Date(span.start_time).getTime(),
+        scores:
+          span.kind === Kind.SpanKindEvaluation ? scoresMap.get(span.id) : [],
       };
     });
   }

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/apps/web/lib/types/span.d.ts

@@ -1,4 +1,7 @@
-import type { Span } from '@workspace/graphql-client/src/types.generated';
+import type {
+  Score,
+  Span,
+} from '@workspace/graphql-client/src/types.generated';
 import type { ChatInput } from '@workspace/graphql-client/src/types.generated';
 import type { ChatCompletion } from 'openai/resources/index.mjs';
 
@@ -10,6 +13,7 @@ export interface ExtendedSpan extends Span {
   relative_start_time: number;
   input?: ChatInput;
   completion?: ChatCompletion;
+  scores?: Score[];
 }
 
 /**

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/apps/web/package.json

@@ -2,6 +2,7 @@
   "name": "@app/web",
   "private": true,
   "type": "module",
+  "packageManager": "pnpm@10.6.5",
   "scripts": {
     "dev": "next dev --turbopack --port 4001",
     "build": "next build",
@@ -9,18 +10,19 @@
     "analyze": "ANALYZE=true pnpm build"
   },
   "dependencies": {
-    "@apollo/client": "^3.13.7",
+    "@apollo/client": "^3.13.8",
     "@apollo/experimental-nextjs-app-support": "^0.11.11",
     "@dnd-kit/core": "^6.3.1",
     "@dnd-kit/modifiers": "^9.0.0",
     "@dnd-kit/sortable": "^10.0.0",
     "@dnd-kit/utilities": "^3.2.2",
     "@hookform/resolvers": "^4.1.3",
-    "@icons-pack/react-simple-icons": "^12.6.0",
+    "@icons-pack/react-simple-icons": "^12.7.0",
     "@tabler/icons-react": "^3.31.0",
-    "@tanstack/react-table": "^8.21.2",
+    "@tanstack/react-table": "^8.21.3",
     "@workspace/graphql-client": "workspace:*",
     "@workspace/ui": "workspace:*",
+    "date-fns": "catalog:",
     "highlight.js": "^11.11.1",
     "lucide-react": "catalog:",
     "next": "catalog:",
@@ -28,12 +30,11 @@
     "react": "catalog:",
     "react-day-picker": "catalog:",
     "react-dom": "catalog:",
-    "react-hook-form": "^7.55.0",
-    "recharts": "^2.15.2",
+    "react-hook-form": "^7.56.1",
+    "recharts": "^2.15.3",
     "server-only": "^0.0.1",
     "sonner": "catalog:",
-    "zod": "^3.24.2",
-    "date-fns": "catalog:"
+    "zod": "^3.24.3"
   },
   "devDependencies": {
     "@types/highlightjs": "^9.12.6",
@@ -41,7 +42,7 @@
     "@types/react": "catalog:",
     "@types/react-dom": "catalog:",
     "@workspace/typescript-config": "workspace:*",
-    "openai": "^4.93.0",
+    "openai": "^4.96.0",
     "typescript": "catalog:"
   }
 }

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/divi/__init__.py

@@ -2,8 +2,10 @@ from typing import Optional
 
 from . import proto
 from .decorators import obs_openai, observable
+from .evaluation import Evaluator, Score
 from .services import Auth, Core, DataPark
 from .session import Session
+from .signals import Kind
 
 name: str = "divi"
 
@@ -11,6 +13,7 @@ _session: Optional[Session] = None
 _core: Optional[Core] = None
 _auth: Optional[Auth] = None
 _datapark: Optional[DataPark] = None
+_evaluator: Optional[Evaluator] = None
 
-__version__ = "0.0.1.dev23"
-__all__ = ["proto", "obs_openai", "observable"]
+__version__ = "0.0.1.dev47"
+__all__ = ["proto", "obs_openai", "observable", "Score", "Kind"]

divi-0.0.1.dev47/divi/decorators/collect.py

@@ -0,0 +1,34 @@
+from typing import Any
+
+from google.protobuf.message import Error
+from openai.types.chat import ChatCompletion
+from typing_extensions import Dict
+
+import divi
+from divi.evaluation.evaluator import EvaluationScore
+from divi.signals.span import Span
+
+
+def collect(span: Span, input: Dict[str, Any], result: Any):
+    if not divi._datapark or span.trace_id is None:
+        raise Error("divi._datapark or span.trace_id is None")
+    # TODO: collect inputs and outputs for SPAN_KIND_FUNCTION
+
+    # collect inputs and outputs for SPAN_KIND_LLM
+    if isinstance(result, ChatCompletion):
+        divi._datapark.create_chat_completion(
+            span_id=span.span_id,
+            trace_id=span.trace_id,
+            inputs=input,
+            completion=result,
+        )
+
+    # collect inputs and outputs for SPAN_KIND_EVALUATION
+    if isinstance(result, list) and all(
+        isinstance(x, EvaluationScore) for x in result
+    ):
+        divi._datapark.create_scores(
+            span_id=span.span_id,
+            trace_id=span.trace_id,
+            scores=result,
+        )

{divi-0.0.1.dev23 → divi-0.0.1.dev47}/divi/decorators/obs_openai.py

@@ -2,7 +2,11 @@ import functools
 from collections.abc import Callable
 from typing import TYPE_CHECKING, TypeVar, Union
 
+from typing_extensions import Optional
+
 from divi.decorators.observable import observable
+from divi.evaluation.scores import Score
+from divi.signals.span import Kind
 from divi.utils import is_async
 
 if TYPE_CHECKING:
@@ -11,22 +15,34 @@ if TYPE_CHECKING:
 C = TypeVar("C", bound=Union["OpenAI", "AsyncOpenAI"])
 
 
-def _get_observable_create(create: Callable) -> Callable:
+def _get_observable_create(
+    create: Callable,
+    name: Optional[str] = None,
+    scores: Optional[list[Score]] = None,
+) -> Callable:
     @functools.wraps(create)
     def observable_create(*args, stream: bool = False, **kwargs):
-        decorator = observable(kind="llm")
+        decorator = observable(kind=Kind.llm, name=name, scores=scores)
         return decorator(create)(*args, stream=stream, **kwargs)
 
     # TODO Async Observable Create
     return observable_create if not is_async(create) else create
 
 
-def obs_openai(client: C) -> C:
+def obs_openai(
+    client: C,
+    name: Optional[str] = "Agent",
+    scores: Optional[list[Score]] = None,
+) -> C:
     """Make OpenAI client observable."""
     client.chat.completions.create = _get_observable_create(
-        client.chat.completions.create
+        client.chat.completions.create,
+        name=name,
+        scores=scores,
     )
     client.completions.create = _get_observable_create(
-        client.completions.create
+        client.completions.create,
+        name=name,
+        scores=scores,
     )
     return client