scorecard-ai 2.2.0 → 2.4.0

package/src/lib/wrapAISDK.ts

@@ -0,0 +1,232 @@
+import { NodeTracerProvider, BatchSpanProcessor } from '@opentelemetry/sdk-trace-node';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import { defaultResource, resourceFromAttributes } from '@opentelemetry/resources';
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
+import { Tracer } from '@opentelemetry/api';
+import { readEnv } from '../internal/utils';
+import { ScorecardError } from '../error';
+import { Scorecard } from '../client';
+
+/**
+ * Configuration for the Scorecard AI SDK wrapper
+ */
+interface ScorecardConfig {
+  /**
+   * ID of the Scorecard project that traces should be associated with.
+   * Defaults to SCORECARD_PROJECT_ID environment variable.
+   */
+  projectId?: string;
+
+  /**
+   * Metrics to score the traces against.
+   * It can be a list of metric IDs, descriptions of what you want the metric to do, or a mix of both.
+   * For example: ['123', 'Check if the response is concise'].
+   * If it's a description, the metric will be created if it doesn't exist.
+   * Defaults to empty array.
+   */
+  metrics?: string[];
+
+  /**
+   * Scorecard API key for authentication.
+   * Defaults to SCORECARD_API_KEY environment variable.
+   */
+  apiKey?: string;
+
+  /**
+   * Service name for telemetry.
+   * Defaults to "ai-sdk-app".
+   */
+  serviceName?: string;
+
+  /**
+   * Service version for telemetry.
+   * Defaults to undefined.
+   */
+  serviceVersion?: string;
+
+  /**
+   * Max export batch size for the batch span processor.
+   * Defaults to 1 so the traces are exported immediately.
+   */
+  maxExportBatchSize?: number;
+}
+
+const DEFAULT_SERVICE_NAME = 'ai-sdk-app';
+const AI_SDK_FUNCTIONS_WITH_TELEMETRY = new Set([
+  'embed',
+  'embedMany',
+  'generateObject',
+  'generateText',
+  'streamObject',
+  'streamText',
+]);
+/**
+ * Initialize the OpenTelemetry tracer with Scorecard configuration.
+ */
+function initializeTracer(config: ScorecardConfig = {}): Tracer {
+  const {
+    projectId = readEnv('SCORECARD_PROJECT_ID'),
+    apiKey = readEnv('SCORECARD_API_KEY'),
+    serviceName = DEFAULT_SERVICE_NAME,
+    serviceVersion,
+    maxExportBatchSize = 1,
+  } = config;
+
+  // Create resource with service information
+  const resource = defaultResource().merge(
+    resourceFromAttributes({
+      [ATTR_SERVICE_NAME]: serviceName,
+      ...(serviceVersion != null ? { [ATTR_SERVICE_VERSION]: serviceVersion } : null),
+      ...(projectId != null ? { 'scorecard.project_id': projectId } : null),
+    }),
+  );
+
+  // Create span processors
+  const spanProcessors = [];
+
+  try {
+    const tracingUrl = readEnv('SCORECARD_TRACING_URL') || 'https://tracing.scorecard.io/otel/v1/traces';
+    // Create OTLP exporter for Scorecard
+    const otlpExporter = new OTLPTraceExporter({
+      url: tracingUrl,
+      headers:
+        apiKey ?
+          {
+            Authorization: `Bearer ${apiKey}`,
+          }
+        : {},
+    });
+
+    // Add batch span processor with OTLP exporter
+    spanProcessors.push(
+      new BatchSpanProcessor(otlpExporter, {
+        maxExportBatchSize,
+      }),
+    );
+  } catch (error) {
+    // Handle initialization errors
+    console.error(error);
+    throw new ScorecardError('Failed to initialize Scorecard telemetry');
+  }
+
+  // Create tracer provider with span processors
+  const tracerProvider = new NodeTracerProvider({
+    resource,
+    spanProcessors,
+  });
+
+  // Register the provider
+  tracerProvider.register();
+
+  // Return the tracer instance
+  return tracerProvider.getTracer('ai-sdk-wrapper');
+}
+
+/**
+ * Wraps the AI SDK module to automatically inject telemetry configuration.
+ *
+ * @param aiSDKModule - The AI SDK module (e.g., import('ai'))
+ * @param config - Optional Scorecard configuration
+ * @returns Proxied AI SDK module with automatic telemetry injection
+ *
+ * @example
+ * ```typescript
+ * import ai from 'ai';
+ * import { wrapAISDK } from 'scorecard-ai';
+ *
+ * const aiSDK = await wrapAISDK(ai);
+ *
+ * // Now all AI SDK calls will automatically send traces to Scorecard.
+ * const { text } = await aiSDK.generateText({
+ *   model: openai('gpt-4o-mini'),
+ *   prompt: 'What is the capital of France? Answer in one sentence.',
+ * });
+ * ```
+ */
+export function wrapAISDK<T extends Record<string, unknown>>(
+  aiSDKModule: T,
+  config: ScorecardConfig = {},
+): T {
+  const projectId = config.projectId || readEnv('SCORECARD_PROJECT_ID');
+  const apiKey = config.apiKey || readEnv('SCORECARD_API_KEY');
+  const serviceName = config.serviceName || DEFAULT_SERVICE_NAME;
+  if (!apiKey) {
+    throw new ScorecardError(
+      'The SCORECARD_API_KEY environment variable is missing or empty; either provide it, or instantiate the AI SDK wrapper client with an apiKey option, like wrapAISDK(ai, { apiKey: "My API Key" }).',
+    );
+  }
+  const client = new Scorecard({ apiKey });
+
+  if (config.metrics && config.metrics.length > 0 && !projectId) {
+    throw new ScorecardError(
+      "The SCORECARD_PROJECT_ID environment variable is missing or empty; either provide it, or instantiate the AI SDK wrapper with a projectId option, like wrapAISDK(ai, { projectId: '123' }).",
+    );
+  }
+
+  // Create metrics and monitor if needed
+  if (config.metrics && config.metrics.length > 0 && projectId) {
+    client
+      .put(`/api/v2/projects/${projectId}/monitors`, {
+        body: {
+          description: serviceName,
+          metrics: config.metrics,
+          filter: {
+            serviceName,
+          },
+        },
+      })
+      // we use .catch instead of try/catch to avoid needing async/await in the wrapper
+      // otherwise the code would await here before giving back the wrapped AI SDK module
+      .catch((error) => {
+        console.error(
+          'Failed to create a Scorecard monitor for your AI SDK traces. The most common case is that the projectId is incorrect.',
+          error,
+        );
+      });
+  }
+  // Initialize tracer
+  const tracerInstance = initializeTracer(config);
+
+  // Create telemetry configuration to inject
+  const telemetryConfig = {
+    isEnabled: true,
+    recordInputs: true,
+    recordOutputs: true,
+    tracer: tracerInstance,
+  };
+
+  // Create a proxy to intercept function calls
+  return new Proxy(aiSDKModule, {
+    get(target, prop: string | symbol) {
+      const originalValue = target[prop as keyof T];
+
+      // Only wrap functions that support telemetry
+      if (
+        typeof prop === 'string' &&
+        AI_SDK_FUNCTIONS_WITH_TELEMETRY.has(prop) &&
+        typeof originalValue === 'function'
+      ) {
+        return function (this: any, ...args: any[]) {
+          try {
+            // The first argument is typically the options object
+            if (args.length > 0 && args[0] && typeof args[0] === 'object') {
+              args[0].experimental_telemetry = {
+                ...telemetryConfig,
+                ...(args[0].experimental_telemetry || {}),
+              };
+            }
+          } catch (error) {
+            // Silently handle any errors during injection
+            console.error('Failed to inject telemetry config:', error);
+          }
+
+          // Call the original function
+          return originalValue.apply(this, args);
+        };
+      }
+
+      // Return original value for non-function properties or non-telemetry functions
+      return originalValue;
+    },
+  });
+}

package/src/resources/index.ts

@@ -20,6 +20,7 @@ export {
   Records,
   type Record,
   type RecordListResponse,
+  type RecordDeleteResponse,
   type RecordCreateParams,
   type RecordListParams,
   type RecordListResponsesPaginatedResponse,

package/src/resources/records.ts

@@ -51,6 +51,18 @@ export class Records extends APIResource {
       ...options,
     });
   }
+
+  /**
+   * Delete a specific Record by ID.
+   *
+   * @example
+   * ```ts
+   * const record = await client.records.delete('777');
+   * ```
+   */
+  delete(recordID: string, options?: RequestOptions): APIPromise<RecordDeleteResponse> {
+    return this._client.delete(path`/records/${recordID}`, options);
+  }
 }
 
 export type RecordListResponsesPaginatedResponse = PaginatedResponse<RecordListResponse>;
@@ -101,6 +113,13 @@ export interface RecordListResponse extends Record {
   scores: Array<ScoresAPI.Score>;
 }
 
+export interface RecordDeleteResponse {
+  /**
+   * Whether the deletion was successful.
+   */
+  success: boolean;
+}
+
 export interface RecordCreateParams {
   /**
    * The expected outputs for the Testcase.
@@ -130,6 +149,7 @@ export declare namespace Records {
   export {
     type Record as Record,
     type RecordListResponse as RecordListResponse,
+    type RecordDeleteResponse as RecordDeleteResponse,
     type RecordListResponsesPaginatedResponse as RecordListResponsesPaginatedResponse,
     type RecordCreateParams as RecordCreateParams,
     type RecordListParams as RecordListParams,

package/src/resources/scores.ts

@@ -81,7 +81,7 @@ export namespace Score {
 
 export interface ScoreUpsertParams {
   /**
-   * Path param: The ID of the Record.
+   * Path param: The ID of the Record to upsert the Score for.
    */
   recordId: string;
 

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = '2.2.0'; // x-release-please-version
+export const VERSION = '2.4.0'; // x-release-please-version

package/version.d.mts

@@ -1,2 +1,2 @@
-export declare const VERSION = "2.2.0";
+export declare const VERSION = "2.4.0";
 //# sourceMappingURL=version.d.mts.map

package/version.d.ts

@@ -1,2 +1,2 @@
-export declare const VERSION = "2.2.0";
+export declare const VERSION = "2.4.0";
 //# sourceMappingURL=version.d.ts.map

package/version.js

@@ -1,5 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '2.2.0'; // x-release-please-version
+exports.VERSION = '2.4.0'; // x-release-please-version
 //# sourceMappingURL=version.js.map

package/version.mjs

@@ -1,2 +1,2 @@
-export const VERSION = '2.2.0'; // x-release-please-version
+export const VERSION = '2.4.0'; // x-release-please-version
 //# sourceMappingURL=version.mjs.map