qualifire 1.2.1 → 1.2.2

package/lib/index.d.ts

@@ -1,4 +1,5 @@
-import type { EvaluationResponse } from './types';
+import { type EvaluationRequest, type EvaluationResponse } from './types';
+export { EvaluationRequest, EvaluationResponse } from './types';
 /**
  * Represents the Qualifire SDK.
  */
@@ -16,20 +17,11 @@ export declare class Qualifire {
         apiKey?: string;
         baseUrl?: string;
     });
+    init(): void;
     /**
      * Evaluates the output of a model against a set of criteria.
      *
-     * @param input - The input to the model.
-     * @param output - The output of the model.
-     * @param assertions - An array of assertions to check.
-     * @param consistencyCheck - Whether to check for consistency.
-     * @param dangerousContentCheck - Whether to check for dangerous content.
-     * @param hallucinationsCheck - Whether to check for hallucinations.
-     * @param harassmentCheck - Whether to check for harassment.
-     * @param hateSpeechCheck - Whether to check for hate speech.
-     * @param piiCheck - Whether to check for personally identifiable information.
-     * @param promptInjections - Whether to check for prompt injections.
-     * @param sexualContentCheck - Whether to check for sexual content.
+     * @param request - The EvaluationRequest to send.
      * @returns An object containing the evaluation results.
      *
      * @example
@@ -50,19 +42,7 @@ export declare class Qualifire {
      * });
      * ```
      */
-    evaluate: ({ input, output, assertions, consistencyCheck, dangerousContentCheck, hallucinationsCheck, harassmentCheck, hateSpeechCheck, piiCheck, promptInjections, sexualContentCheck, }: {
-        input: string;
-        output: string;
-        assertions?: string[] | undefined;
-        consistencyCheck?: boolean | undefined;
-        dangerousContentCheck?: boolean | undefined;
-        hallucinationsCheck?: boolean | undefined;
-        harassmentCheck?: boolean | undefined;
-        hateSpeechCheck?: boolean | undefined;
-        piiCheck?: boolean | undefined;
-        promptInjections?: boolean | undefined;
-        sexualContentCheck?: boolean | undefined;
-    }) => Promise<EvaluationResponse | undefined>;
+    evaluate: (request: EvaluationRequest) => Promise<EvaluationResponse | undefined>;
     /**
      * Invokes an evaluation for a given evaluation ID.
      *

package/lib/index.js

@@ -1,6 +1,41 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Qualifire = void 0;
+const types_1 = require("./types");
+const traceloop = __importStar(require("@traceloop/node-server-sdk"));
 /**
  * Represents the Qualifire SDK.
  */
@@ -16,17 +51,7 @@ class Qualifire {
         /**
          * Evaluates the output of a model against a set of criteria.
          *
-         * @param input - The input to the model.
-         * @param output - The output of the model.
-         * @param assertions - An array of assertions to check.
-         * @param consistencyCheck - Whether to check for consistency.
-         * @param dangerousContentCheck - Whether to check for dangerous content.
-         * @param hallucinationsCheck - Whether to check for hallucinations.
-         * @param harassmentCheck - Whether to check for harassment.
-         * @param hateSpeechCheck - Whether to check for hate speech.
-         * @param piiCheck - Whether to check for personally identifiable information.
-         * @param promptInjections - Whether to check for prompt injections.
-         * @param sexualContentCheck - Whether to check for sexual content.
+         * @param request - The EvaluationRequest to send.
          * @returns An object containing the evaluation results.
          *
          * @example
@@ -47,21 +72,10 @@ class Qualifire {
          * });
          * ```
          */
-        this.evaluate = async ({ input, output, assertions = [], consistencyCheck = false, dangerousContentCheck = false, hallucinationsCheck = false, harassmentCheck = false, hateSpeechCheck = false, piiCheck = false, promptInjections = false, sexualContentCheck = false, }) => {
+        this.evaluate = async (request) => {
+            const parsedRequest = types_1.EvaluationRequestSchema.parse(request);
             const url = `${this.baseUrl}/api/evaluation/evaluate`;
-            const body = JSON.stringify({
-                input,
-                output,
-                assertions,
-                consistency_check: consistencyCheck,
-                dangerous_content_check: dangerousContentCheck,
-                hallucinations_check: hallucinationsCheck,
-                harassment_check: harassmentCheck,
-                hate_speech_check: hateSpeechCheck,
-                pii_check: piiCheck,
-                prompt_injections: promptInjections,
-                sexual_content_check: sexualContentCheck,
-            });
+            const body = JSON.stringify(parsedRequest);
             const headers = {
                 'Content-Type': 'application/json',
                 'X-Qualifire-API-Key': this.sdkKey,
@@ -125,5 +139,17 @@ class Qualifire {
         this.sdkKey = key;
         this.baseUrl = qualifireBaseUrl;
     }
+    init() {
+        process.env.TRACELOOP_TELEMETRY = 'false';
+        traceloop.initialize({
+            baseUrl: `${this.baseUrl}/api/telemetry`,
+            headers: {
+                'X-Qualifire-API-Key': this.sdkKey,
+            },
+            traceloopSyncEnabled: false,
+            silenceInitializationMessage: true,
+            disableBatch: true,
+        });
+    }
 }
 exports.Qualifire = Qualifire;

package/lib/types.d.ts

@@ -163,59 +163,196 @@ export declare const outputSchema: z.ZodObject<{
     system_fingerprint?: string | undefined;
 }>;
 export type Output = z.infer<typeof outputSchema> | string;
-declare const EvaluationRequestSchema: z.ZodObject<{
-    input: z.ZodString;
-    output: z.ZodString;
-    consistency_check: z.ZodBoolean;
-    dangerous_content_check: z.ZodBoolean;
-    hallucinations_check: z.ZodBoolean;
-    harassment_check: z.ZodBoolean;
-    hate_speech_check: z.ZodBoolean;
-    pii_check: z.ZodBoolean;
-    prompt_injections: z.ZodBoolean;
-    sexual_content_check: z.ZodBoolean;
+export declare const EvaluationRequestSchema: z.ZodEffects<z.ZodObject<{
+    input: z.ZodOptional<z.ZodString>;
+    output: z.ZodOptional<z.ZodString>;
     messages: z.ZodOptional<z.ZodArray<z.ZodObject<{
-        content: z.ZodString;
         role: z.ZodString;
+        content: z.ZodOptional<z.ZodString>;
+        tool_calls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            arguments: z.ZodRecord<z.ZodString, z.ZodAny>;
+            id: z.ZodOptional<z.ZodString>;
+        }, "strip", z.ZodTypeAny, {
+            name: string;
+            arguments: Record<string, any>;
+            id?: string | undefined;
+        }, {
+            name: string;
+            arguments: Record<string, any>;
+            id?: string | undefined;
+        }>, "many">>;
     }, "strip", z.ZodTypeAny, {
         role: string;
-        content: string;
+        content?: string | undefined;
+        tool_calls?: {
+            name: string;
+            arguments: Record<string, any>;
+            id?: string | undefined;
+        }[] | undefined;
     }, {
         role: string;
-        content: string;
+        content?: string | undefined;
+        tool_calls?: {
+            name: string;
+            arguments: Record<string, any>;
+            id?: string | undefined;
+        }[] | undefined;
     }>, "many">>;
+    available_tools: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        description: z.ZodString;
+        parameters: z.ZodRecord<z.ZodString, z.ZodAny>;
+    }, "strip", z.ZodTypeAny, {
+        name: string;
+        description: string;
+        parameters: Record<string, any>;
+    }, {
+        name: string;
+        description: string;
+        parameters: Record<string, any>;
+    }>, "many">>;
+    dangerous_content_check: z.ZodDefault<z.ZodBoolean>;
+    grounding_check: z.ZodDefault<z.ZodBoolean>;
+    hallucinations_check: z.ZodDefault<z.ZodBoolean>;
+    harassment_check: z.ZodDefault<z.ZodBoolean>;
+    hate_speech_check: z.ZodDefault<z.ZodBoolean>;
+    instructions_following_check: z.ZodDefault<z.ZodBoolean>;
+    pii_check: z.ZodDefault<z.ZodBoolean>;
+    prompt_injections: z.ZodDefault<z.ZodBoolean>;
+    sexual_content_check: z.ZodDefault<z.ZodBoolean>;
+    syntax_checks: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
+        args: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        args: string;
+    }, {
+        args: string;
+    }>>>;
+    tool_selection_quality_check: z.ZodDefault<z.ZodBoolean>;
     assertions: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
 }, "strip", z.ZodTypeAny, {
-    input: string;
-    output: string;
-    consistency_check: boolean;
     dangerous_content_check: boolean;
+    grounding_check: boolean;
     hallucinations_check: boolean;
     harassment_check: boolean;
     hate_speech_check: boolean;
+    instructions_following_check: boolean;
     pii_check: boolean;
     prompt_injections: boolean;
     sexual_content_check: boolean;
+    tool_selection_quality_check: boolean;
     messages?: {
         role: string;
-        content: string;
+        content?: string | undefined;
+        tool_calls?: {
+            name: string;
+            arguments: Record<string, any>;
+            id?: string | undefined;
+        }[] | undefined;
+    }[] | undefined;
+    input?: string | undefined;
+    output?: string | undefined;
+    available_tools?: {
+        name: string;
+        description: string;
+        parameters: Record<string, any>;
     }[] | undefined;
+    syntax_checks?: Record<string, {
+        args: string;
+    }> | undefined;
     assertions?: string[] | undefined;
 }, {
-    input: string;
-    output: string;
-    consistency_check: boolean;
+    messages?: {
+        role: string;
+        content?: string | undefined;
+        tool_calls?: {
+            name: string;
+            arguments: Record<string, any>;
+            id?: string | undefined;
+        }[] | undefined;
+    }[] | undefined;
+    input?: string | undefined;
+    output?: string | undefined;
+    available_tools?: {
+        name: string;
+        description: string;
+        parameters: Record<string, any>;
+    }[] | undefined;
+    dangerous_content_check?: boolean | undefined;
+    grounding_check?: boolean | undefined;
+    hallucinations_check?: boolean | undefined;
+    harassment_check?: boolean | undefined;
+    hate_speech_check?: boolean | undefined;
+    instructions_following_check?: boolean | undefined;
+    pii_check?: boolean | undefined;
+    prompt_injections?: boolean | undefined;
+    sexual_content_check?: boolean | undefined;
+    syntax_checks?: Record<string, {
+        args: string;
+    }> | undefined;
+    tool_selection_quality_check?: boolean | undefined;
+    assertions?: string[] | undefined;
+}>, {
     dangerous_content_check: boolean;
+    grounding_check: boolean;
     hallucinations_check: boolean;
     harassment_check: boolean;
     hate_speech_check: boolean;
+    instructions_following_check: boolean;
     pii_check: boolean;
     prompt_injections: boolean;
     sexual_content_check: boolean;
+    tool_selection_quality_check: boolean;
     messages?: {
         role: string;
-        content: string;
+        content?: string | undefined;
+        tool_calls?: {
+            name: string;
+            arguments: Record<string, any>;
+            id?: string | undefined;
+        }[] | undefined;
     }[] | undefined;
+    input?: string | undefined;
+    output?: string | undefined;
+    available_tools?: {
+        name: string;
+        description: string;
+        parameters: Record<string, any>;
+    }[] | undefined;
+    syntax_checks?: Record<string, {
+        args: string;
+    }> | undefined;
+    assertions?: string[] | undefined;
+}, {
+    messages?: {
+        role: string;
+        content?: string | undefined;
+        tool_calls?: {
+            name: string;
+            arguments: Record<string, any>;
+            id?: string | undefined;
+        }[] | undefined;
+    }[] | undefined;
+    input?: string | undefined;
+    output?: string | undefined;
+    available_tools?: {
+        name: string;
+        description: string;
+        parameters: Record<string, any>;
+    }[] | undefined;
+    dangerous_content_check?: boolean | undefined;
+    grounding_check?: boolean | undefined;
+    hallucinations_check?: boolean | undefined;
+    harassment_check?: boolean | undefined;
+    hate_speech_check?: boolean | undefined;
+    instructions_following_check?: boolean | undefined;
+    pii_check?: boolean | undefined;
+    prompt_injections?: boolean | undefined;
+    sexual_content_check?: boolean | undefined;
+    syntax_checks?: Record<string, {
+        args: string;
+    }> | undefined;
+    tool_selection_quality_check?: boolean | undefined;
     assertions?: string[] | undefined;
 }>;
 declare const EvaluationResponseSchema: z.ZodObject<{
@@ -229,18 +366,18 @@ declare const EvaluationResponseSchema: z.ZodObject<{
             reason: z.ZodString;
             score: z.ZodNumber;
         }, "strip", z.ZodTypeAny, {
+            name: string;
             claim: string;
             confidence_score: number;
             label: string;
-            name: string;
             quote: string;
             reason: string;
             score: number;
         }, {
+            name: string;
             claim: string;
             confidence_score: number;
             label: string;
-            name: string;
             quote: string;
             reason: string;
             score: number;
@@ -249,10 +386,10 @@ declare const EvaluationResponseSchema: z.ZodObject<{
     }, "strip", z.ZodTypeAny, {
         type: string;
         results: {
+            name: string;
             claim: string;
             confidence_score: number;
             label: string;
-            name: string;
             quote: string;
             reason: string;
             score: number;
@@ -260,10 +397,10 @@ declare const EvaluationResponseSchema: z.ZodObject<{
     }, {
         type: string;
         results: {
+            name: string;
             claim: string;
             confidence_score: number;
             label: string;
-            name: string;
             quote: string;
             reason: string;
             score: number;
@@ -277,10 +414,10 @@ declare const EvaluationResponseSchema: z.ZodObject<{
     evaluationResults: {
         type: string;
         results: {
+            name: string;
             claim: string;
             confidence_score: number;
             label: string;
-            name: string;
             quote: string;
             reason: string;
             score: number;
@@ -292,16 +429,16 @@ declare const EvaluationResponseSchema: z.ZodObject<{
     evaluationResults: {
         type: string;
         results: {
+            name: string;
             claim: string;
             confidence_score: number;
             label: string;
-            name: string;
             quote: string;
             reason: string;
             score: number;
         }[];
     }[];
 }>;
+export type EvaluationRequest = z.input<typeof EvaluationRequestSchema>;
 export type EvaluationResponse = z.infer<typeof EvaluationResponseSchema>;
-export type EvaluationRequestSchema = z.infer<typeof EvaluationRequestSchema>;
 export {};

package/lib/types.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.outputSchema = exports.inputSchema = exports.usageSchema = exports.choiceSchema = exports.messageSchema = void 0;
+exports.EvaluationRequestSchema = exports.outputSchema = exports.inputSchema = exports.usageSchema = exports.choiceSchema = exports.messageSchema = void 0;
 const zod_1 = require("zod");
 exports.messageSchema = zod_1.z.object({
     role: zod_1.z.string(),
@@ -29,23 +29,73 @@ exports.outputSchema = zod_1.z.object({
     usage: exports.usageSchema.optional(),
     system_fingerprint: zod_1.z.string().optional(),
 });
+const LLMToolDefinitionSchema = zod_1.z.object({
+    name: zod_1.z.string(),
+    description: zod_1.z.string(),
+    parameters: zod_1.z.record(zod_1.z.string(), zod_1.z.any()),
+});
+const LLMToolCallSchema = zod_1.z.object({
+    name: zod_1.z.string(),
+    arguments: zod_1.z.record(zod_1.z.string(), zod_1.z.any()),
+    id: zod_1.z.string().optional(),
+});
 const LLMMessageSchema = zod_1.z.object({
-    content: zod_1.z.string(),
     role: zod_1.z.string(),
+    content: zod_1.z.string().optional(),
+    tool_calls: zod_1.z.array(LLMToolCallSchema).optional(),
+});
+const SyntaxCheckArgsSchema = zod_1.z.object({
+    args: zod_1.z.string(),
 });
-const EvaluationRequestSchema = zod_1.z.object({
-    input: zod_1.z.string(),
-    output: zod_1.z.string(),
-    consistency_check: zod_1.z.boolean(),
-    dangerous_content_check: zod_1.z.boolean(),
-    hallucinations_check: zod_1.z.boolean(),
-    harassment_check: zod_1.z.boolean(),
-    hate_speech_check: zod_1.z.boolean(),
-    pii_check: zod_1.z.boolean(),
-    prompt_injections: zod_1.z.boolean(),
-    sexual_content_check: zod_1.z.boolean(),
+exports.EvaluationRequestSchema = zod_1.z
+    .object({
+    input: zod_1.z.string().optional(),
+    output: zod_1.z.string().optional(),
     messages: zod_1.z.array(LLMMessageSchema).optional(),
+    available_tools: zod_1.z.array(LLMToolDefinitionSchema).optional(),
+    dangerous_content_check: zod_1.z.boolean().default(false),
+    grounding_check: zod_1.z.boolean().default(false),
+    hallucinations_check: zod_1.z.boolean().default(false),
+    harassment_check: zod_1.z.boolean().default(false),
+    hate_speech_check: zod_1.z.boolean().default(false),
+    instructions_following_check: zod_1.z.boolean().default(false),
+    pii_check: zod_1.z.boolean().default(false),
+    prompt_injections: zod_1.z.boolean().default(false),
+    sexual_content_check: zod_1.z.boolean().default(false),
+    syntax_checks: zod_1.z.record(zod_1.z.string(), SyntaxCheckArgsSchema).optional(),
+    tool_selection_quality_check: zod_1.z.boolean().default(false),
     assertions: zod_1.z.array(zod_1.z.string()).optional(),
+})
+    .superRefine((data, ctx) => {
+    const hasMessages = Array.isArray(data.messages) && data.messages.length > 0;
+    const hasInput = typeof data.input === 'string' && data.input.trim() !== '';
+    const hasOutput = typeof data.output === 'string' && data.output.trim() !== '';
+    // Validation: At least one of messages, input, or output
+    if (!hasMessages && !hasInput && !hasOutput) {
+        ctx.addIssue({
+            code: zod_1.z.ZodIssueCode.custom,
+            message: 'At least one of messages, input, or output must be set',
+            path: [], // Top level
+        });
+    }
+    // Validation: tool_selection_quality_check requires messages and available_tools
+    if (data.tool_selection_quality_check) {
+        const hasAvailableTools = Array.isArray(data.available_tools) && data.available_tools.length > 0;
+        if (!hasMessages) {
+            ctx.addIssue({
+                code: zod_1.z.ZodIssueCode.custom,
+                message: 'messages must be provided when tool_selection_quality_check is true',
+                path: ['messages'],
+            });
+        }
+        if (!hasAvailableTools) {
+            ctx.addIssue({
+                code: zod_1.z.ZodIssueCode.custom,
+                message: 'available_tools must be provided when tool_selection_quality_check is true',
+                path: ['available_tools'],
+            });
+        }
+    }
 });
 const EvaluationResultSchema = zod_1.z.object({
     claim: zod_1.z.string(),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qualifire",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "description": "Qualifire client SDK",
   "main": "./lib/index.js",
   "files": [
@@ -114,7 +114,9 @@
     ]
   },
   "dependencies": {
+    "@traceloop/node-server-sdk": "^0.13.3",
     "openai": "^4.28.4",
     "zod": "^3.22.4"
-  }
+  },
+  "packageManager": "pnpm@9.6.0+sha512.38dc6fba8dba35b39340b9700112c2fe1e12f10b17134715a4aa98ccf7bb035e76fd981cf0bb384dfa98f8d6af5481c2bef2f4266a24bfa20c34eb7147ce0b5e"
 }