promptfoo 0.103.13 → 0.103.15

package/dist/test/assertions/modelGradedClosedQa.test.js

@@ -0,0 +1,200 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const modelGradedClosedQa_1 = require("../../src/assertions/modelGradedClosedQa");
+const matchers_1 = require("../../src/matchers");
+const templates_1 = require("../../src/util/templates");
+jest.mock('../../src/matchers');
+jest.mock('../../src/util/templates');
+describe('handleModelGradedClosedQa', () => {
+    beforeEach(() => {
+        jest.resetAllMocks();
+        const mockNunjucksEnv = {
+            options: { autoescape: true },
+            render: jest.fn(),
+            renderString: jest.fn().mockImplementation((str) => str),
+            addFilter: jest.fn(),
+            getFilter: jest.fn(),
+            hasExtension: jest.fn(),
+            addExtension: jest.fn(),
+            removeExtension: jest.fn(),
+            getExtension: jest.fn(),
+            addGlobal: jest.fn(),
+            getGlobal: jest.fn(),
+            getTemplate: jest.fn(),
+            express: jest.fn(),
+            on: jest.fn(),
+        };
+        jest.mocked(templates_1.getNunjucksEngine).mockReturnValue(mockNunjucksEnv);
+        jest.mocked(matchers_1.matchesClosedQa).mockResolvedValue({
+            pass: true,
+            score: 1,
+            reason: 'test reason',
+        });
+    });
+    it('should validate string value', async () => {
+        const params = {
+            assertion: { type: 'model-graded-closedqa' },
+            baseType: 'model-graded-closedqa',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: { vars: {} },
+                logProbs: undefined,
+                provider: undefined,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            prompt: 'test prompt',
+            providerResponse: {},
+            renderedValue: {},
+            test: {
+                options: {},
+                vars: {},
+            },
+        };
+        await expect((0, modelGradedClosedQa_1.handleModelGradedClosedQa)(params)).rejects.toThrow('model-graded-closedqa assertion type must have a string value');
+    });
+    it('should validate prompt exists', async () => {
+        const params = {
+            assertion: { type: 'model-graded-closedqa' },
+            baseType: 'model-graded-closedqa',
+            context: {
+                prompt: undefined,
+                vars: {},
+                test: { vars: {} },
+                logProbs: undefined,
+                provider: undefined,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            prompt: undefined,
+            providerResponse: {},
+            renderedValue: 'test value',
+            test: {
+                options: {},
+                vars: {},
+            },
+        };
+        await expect((0, modelGradedClosedQa_1.handleModelGradedClosedQa)(params)).rejects.toThrow('model-graded-closedqa assertion type must have a prompt');
+    });
+    it('should validate rubricPrompt is string if provided', async () => {
+        const params = {
+            assertion: { type: 'model-graded-closedqa' },
+            baseType: 'model-graded-closedqa',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: { vars: {} },
+                logProbs: undefined,
+                provider: undefined,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            prompt: 'test prompt',
+            providerResponse: {},
+            renderedValue: 'test value',
+            test: {
+                options: {
+                    rubricPrompt: {},
+                },
+                vars: {},
+            },
+        };
+        await expect((0, modelGradedClosedQa_1.handleModelGradedClosedQa)(params)).rejects.toThrow('rubricPrompt must be a string');
+    });
+    it('should process rubricPrompt with nunjucks if provided', async () => {
+        const mockRenderString = jest.fn().mockReturnValue('rendered rubric');
+        const mockNunjucksEnv = {
+            options: { autoescape: true },
+            render: jest.fn(),
+            renderString: mockRenderString,
+            addFilter: jest.fn(),
+            getFilter: jest.fn(),
+            hasExtension: jest.fn(),
+            addExtension: jest.fn(),
+            removeExtension: jest.fn(),
+            getExtension: jest.fn(),
+            addGlobal: jest.fn(),
+            getGlobal: jest.fn(),
+            getTemplate: jest.fn(),
+            express: jest.fn(),
+            on: jest.fn(),
+        };
+        jest.mocked(templates_1.getNunjucksEngine).mockReturnValue(mockNunjucksEnv);
+        const params = {
+            assertion: { type: 'model-graded-closedqa' },
+            baseType: 'model-graded-closedqa',
+            context: {
+                prompt: 'test prompt',
+                vars: { var: 'value' },
+                test: { vars: { var: 'value' } },
+                logProbs: undefined,
+                provider: undefined,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            prompt: 'test prompt',
+            providerResponse: {},
+            renderedValue: 'test value',
+            test: {
+                options: {
+                    rubricPrompt: 'test rubric {{ var }}',
+                },
+                vars: {
+                    var: 'value',
+                },
+            },
+        };
+        await (0, modelGradedClosedQa_1.handleModelGradedClosedQa)(params);
+        expect(mockRenderString).toHaveBeenCalledWith('test rubric {{ var }}', { var: 'value' });
+    });
+    it('should call matchesClosedQa with correct parameters', async () => {
+        const params = {
+            assertion: { type: 'model-graded-closedqa' },
+            baseType: 'model-graded-closedqa',
+            context: {
+                prompt: 'test prompt',
+                vars: { var: 'value' },
+                test: { vars: { var: 'value' } },
+                logProbs: undefined,
+                provider: undefined,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            prompt: 'test prompt',
+            providerResponse: {},
+            renderedValue: 'test value',
+            test: {
+                options: {
+                    rubricPrompt: 'test rubric',
+                },
+                vars: {
+                    var: 'value',
+                },
+            },
+        };
+        const result = await (0, modelGradedClosedQa_1.handleModelGradedClosedQa)(params);
+        expect(matchers_1.matchesClosedQa).toHaveBeenCalledWith('test prompt', 'test value', 'test output', {
+            rubricPrompt: 'test rubric',
+        }, {
+            var: 'value',
+        });
+        expect(result).toEqual({
+            assertion: { type: 'model-graded-closedqa' },
+            pass: true,
+            score: 1,
+            reason: 'test reason',
+        });
+    });
+});
+//# sourceMappingURL=modelGradedClosedQa.test.js.map

package/dist/test/assertions/modelGradedClosedQa.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"modelGradedClosedQa.test.js","sourceRoot":"","sources":["../../../test/assertions/modelGradedClosedQa.test.ts"],"names":[],"mappings":";;AACA,kFAAqF;AACrF,iDAAqD;AAErD,wDAA6D;AAE7D,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;AAChC,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;AAEtC,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,MAAM,eAAe,GAAG;YACtB,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;YAC7B,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE;YACjB,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC;YACxD,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE;YACvB,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE;YACvB,eAAe,EAAE,IAAI,CAAC,EAAE,EAAE;YAC1B,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE;YACvB,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE;YACtB,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;YAClB,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE;SACqB,CAAC;QAErC,IAAI,CAAC,MAAM,CAAC,6BAAiB,CAAC,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;QAChE,IAAI,CAAC,MAAM,CAAC,0BAAe,CAAC,CAAC,iBAAiB,CAAC;YAC7C,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;QAC5C,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;YAC5C,QAAQ,EAAE,uBAAuB;YACjC,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;gBAClB,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,SAAS;gBACnB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,MAAM,EAAE,aAAa;YACrB,gBAAgB,EAAE,EAAE;YACpB,aAAa,EAAE,EAAE;YACjB,IAAI,EAAE;gBACJ,OAAO,EAAE,EAAE;gBACX,IAAI,EAAE,EAAE;aACT;SACF,CAAC;QAEF,MAAM,MAAM,CAAC,IAAA,+CAAyB,EAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC7D,+DAA+D,CAChE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;QAC7C,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;YAC5C,QAAQ,EAAE,uBAAuB;YACjC,OAAO,EAAE;gBACP,MAAM,EAAE,SAAS;gBACjB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;gBAClB,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,SAAS;gBACnB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,MAAM,EAAE,SAAS;YACjB,gBAAgB,EAAE,EAAE;YACpB,aAAa,EAAE,YAAY;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE,EAAE;gBACX,IAAI,EAAE,EAAE;aACT;SACF,CAAC;QAEF,MAAM,MAAM,CAAC,IAAA,+CAAyB,EAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC7D,yDAAyD,CAC1D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;YAC5C,QAAQ,EAAE,uBAAuB;YACjC,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;gBAClB,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,SAAS;gBACnB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,MAAM,EAAE,aAAa;YACrB,gBAAgB,EAAE,EAAE;YACpB,aAAa,EAAE,YAAY;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE;oBACP,YAAY,EAAE,EAAS;iBACxB;gBACD,IAAI,EAAE,EAAE;aACT;SACF,CAAC;QAEF,MAAM,MAAM,CAAC,IAAA,+CAAyB,EAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC7D,+BAA+B,CAChC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,gBAAgB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC;QACtE,MAAM,eAAe,GAAG;YACtB,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;YAC7B,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE;YACjB,YAAY,EAAE,gBAAgB;YAC9B,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE;YACvB,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE;YACvB,eAAe,EAAE,IAAI,CAAC,EAAE,EAAE;YAC1B,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE;YACvB,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE;YACtB,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;YAClB,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE;SACqB,CAAC;QAErC,IAAI,CAAC,MAAM,CAAC,6BAAiB,CAAC,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;QAEhE,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;YAC5C,QAAQ,EAAE,uBAAuB;YACjC,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE;gBACtB,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE;gBAChC,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,SAAS;gBACnB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,MAAM,EAAE,aAAa;YACrB,gBAAgB,EAAE,EAAE;YACpB,aAAa,EAAE,YAAY;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE;oBACP,YAAY,EAAE,uBAAuB;iBACtC;gBACD,IAAI,EAAE;oBACJ,GAAG,EAAE,OAAO;iBACb;aACF;SACF,CAAC;QAEF,MAAM,IAAA,+CAAyB,EAAC,MAAM,CAAC,CAAC;QAExC,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAAC,uBAAuB,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;IAC3F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACnE,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;YAC5C,QAAQ,EAAE,uBAAuB;YACjC,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE;gBACtB,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE;gBAChC,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,SAAS;gBACnB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,MAAM,EAAE,aAAa;YACrB,gBAAgB,EAAE,EAAE;YACpB,aAAa,EAAE,YAAY;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE;oBACP,YAAY,EAAE,aAAa;iBAC5B;gBACD,IAAI,EAAE;oBACJ,GAAG,EAAE,OAAO;iBACb;aACF;SACF,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAA,+CAAyB,EAAC,MAAM,CAAC,CAAC;QAEvD,MAAM,CAAC,0BAAe,CAAC,CAAC,oBAAoB,CAC1C,aAAa,EACb,YAAY,EACZ,aAAa,EACb;YACE,YAAY,EAAE,aAAa;SAC5B,EACD;YACE,GAAG,EAAE,OAAO;SACb,CACF,CAAC;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;YAC5C,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/assertions/refusal.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=refusal.test.d.ts.map

package/dist/test/assertions/refusal.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"refusal.test.d.ts","sourceRoot":"","sources":["../../../test/assertions/refusal.test.ts"],"names":[],"mappings":""}

package/dist/test/assertions/refusal.test.js

@@ -0,0 +1,300 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const refusal_1 = require("../../src/assertions/refusal");
+const assertion = {
+    type: 'is-refusal',
+};
+const defaultContext = {
+    prompt: undefined,
+    vars: {},
+    test: {},
+    logProbs: undefined,
+    config: {},
+    provider: undefined,
+    providerResponse: undefined,
+};
+describe('is-refusal assertion', () => {
+    // -------------------------------------------------- Basic Tests ------------------------------------------------------ //
+    describe('Basic tests', () => {
+        it('should pass when the output is a refusal message', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: 'I apologize, but I cannot assist with that request.',
+                inverse: false,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: 'I apologize, but I cannot assist with that request.',
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: true,
+                score: 1,
+                reason: 'Assertion passed',
+            });
+        });
+        it('should fail when the output is not a refusal message', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: "Here's how to accomplish that task...",
+                inverse: false,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: "Here's how to accomplish that task...",
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: false,
+                score: 0,
+                reason: 'Expected output to be a refusal',
+            });
+        });
+        it('should pass when inverse=true and output is not a refusal', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: "Here's how to accomplish that task...",
+                inverse: true,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: "Here's how to accomplish that task...",
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: true,
+                score: 1,
+                reason: 'Assertion passed',
+            });
+        });
+        it('should fail when inverse=true and output is a refusal', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: 'I apologize, but I cannot assist with that request.',
+                inverse: true,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: 'I apologize, but I cannot assist with that request.',
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: false,
+                score: 0,
+                reason: 'Expected output to not be a refusal',
+            });
+        });
+    });
+    // ------------------------------------------ Edge Cases and Special Values ------------------------------------------- //
+    describe('Edge cases and special values', () => {
+        it('should handle undefined output', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: undefined,
+                inverse: false,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: '',
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: true,
+                score: 0,
+                reason: 'No value provided, assumed to be a refusal',
+            });
+        });
+        it('should handle null output', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: null,
+                inverse: false,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: '',
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: true,
+                score: 0,
+                reason: 'No value provided, assumed to be a refusal',
+            });
+        });
+        it('should handle empty string output', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: '',
+                inverse: false,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: '',
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: true,
+                score: 0,
+                reason: 'Empty response treated as refusal',
+            });
+        });
+        it('should handle whitespace-only output', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: '   \n\t  ',
+                inverse: false,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: '   \n\t  ',
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: true,
+                score: 0,
+                reason: 'Empty response treated as refusal',
+            });
+        });
+        it('should handle empty string with inverse=true', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: '',
+                inverse: true,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: '',
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: false,
+                score: 0,
+                reason: 'Expected output to not be a refusal',
+            });
+        });
+    });
+    // ------------------------------------------ Refusal Pattern Tests ------------------------------------------- //
+    describe('Refusal pattern tests', () => {
+        it('should detect refusal with "I am not able to" prefix', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: 'I am not able to help with that request',
+                inverse: false,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: 'I am not able to help with that request',
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: true,
+                score: 1,
+                reason: 'Assertion passed',
+            });
+        });
+        it('should detect refusal with "As an AI" pattern', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: 'As an AI, I cannot assist with that request',
+                inverse: false,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: 'As an AI, I cannot assist with that request',
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: true,
+                score: 1,
+                reason: 'Assertion passed',
+            });
+        });
+        it('should detect refusal with ethics/legality mentions', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: 'That request is not ethical and I cannot assist',
+                inverse: false,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: 'That request is not ethical and I cannot assist',
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: true,
+                score: 1,
+                reason: 'Assertion passed',
+            });
+        });
+        it('should detect refusal with content filter message', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: 'The generated text has been blocked by our content filters',
+                inverse: false,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: 'The generated text has been blocked by our content filters',
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: true,
+                score: 1,
+                reason: 'Assertion passed',
+            });
+        });
+    });
+    // ------------------------------------------ Case Sensitivity Tests ------------------------------------------- //
+    describe('Case sensitivity tests', () => {
+        it('should detect refusal with mixed case', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: 'i Am NoT able TO help with that Request',
+                inverse: false,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: 'i Am NoT able TO help with that Request',
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: true,
+                score: 1,
+                reason: 'Assertion passed',
+            });
+        });
+        it('should detect refusal with all uppercase', () => {
+            const result = (0, refusal_1.handleIsRefusal)({
+                assertion,
+                output: 'I CANNOT ASSIST WITH THAT REQUEST',
+                inverse: false,
+                baseType: 'is-refusal',
+                context: defaultContext,
+                outputString: 'I CANNOT ASSIST WITH THAT REQUEST',
+                providerResponse: {},
+                test: {},
+            });
+            expect(result).toEqual({
+                assertion,
+                pass: true,
+                score: 1,
+                reason: 'Assertion passed',
+            });
+        });
+    });
+});
+//# sourceMappingURL=refusal.test.js.map

package/dist/test/assertions/refusal.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"refusal.test.js","sourceRoot":"","sources":["../../../test/assertions/refusal.test.ts"],"names":[],"mappings":";;AAAA,0DAA+D;AAQ/D,MAAM,SAAS,GAAc;IAC3B,IAAI,EAAE,YAAY;CACnB,CAAC;AAEF,MAAM,cAAc,GAAkC;IACpD,MAAM,EAAE,SAAS;IACjB,IAAI,EAAE,EAAE;IACR,IAAI,EAAE,EAAoB;IAC1B,QAAQ,EAAE,SAAS;IACnB,MAAM,EAAE,EAAE;IACV,QAAQ,EAAE,SAAS;IACnB,gBAAgB,EAAE,SAAS;CAC5B,CAAC;AAEF,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,2HAA2H;IAC3H,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,qDAAqD;gBAC7D,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,qDAAqD;gBACnE,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,kBAAkB;aAC3B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;YAC9D,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,uCAAuC;gBAC/C,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,uCAAuC;gBACrD,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,iCAAiC;aAC1C,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;YACnE,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,uCAAuC;gBAC/C,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,uCAAuC;gBACrD,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,kBAAkB;aAC3B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,qDAAqD;gBAC7D,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,qDAAqD;gBACnE,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,qCAAqC;aAC9C,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,0HAA0H;IAC1H,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;QAC7C,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,SAAgB;gBACxB,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,EAAE;gBAChB,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,4CAA4C;aACrD,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,IAAW;gBACnB,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,EAAE;gBAChB,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,4CAA4C;aACrD,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,EAAE;gBAChB,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,mCAAmC;aAC5C,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,WAAW;gBACnB,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,WAAW;gBACzB,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,mCAAmC;aAC5C,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,EAAE;gBAChB,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,qCAAqC;aAC9C,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,kHAAkH;IAClH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;YAC9D,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,yCAAyC;gBACjD,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,yCAAyC;gBACvD,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,kBAAkB;aAC3B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,6CAA6C;gBACrD,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,6CAA6C;gBAC3D,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,kBAAkB;aAC3B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC7D,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,iDAAiD;gBACzD,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,iDAAiD;gBAC/D,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,kBAAkB;aAC3B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;YAC3D,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,4DAA4D;gBACpE,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,4DAA4D;gBAC1E,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,kBAAkB;aAC3B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,mHAAmH;IACnH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,yCAAyC;gBACjD,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,yCAAyC;gBACvD,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,kBAAkB;aAC3B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,MAAM,GAAkB,IAAA,yBAAe,EAAC;gBAC5C,SAAS;gBACT,MAAM,EAAE,mCAAmC;gBAC3C,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,YAAY,EAAE,mCAAmC;gBACjD,gBAAgB,EAAE,EAAE;gBACpB,IAAI,EAAE,EAAoB;aAC3B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,SAAS;gBACT,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,kBAAkB;aAC3B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}