promptfoo 0.115.1 → 0.115.3

package/dist/src/assertions/synthesis.js

@@ -0,0 +1,504 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateNewQuestionsPrompt = generateNewQuestionsPrompt;
+exports.convertQuestionToPythonPrompt = convertQuestionToPythonPrompt;
+exports.synthesize = synthesize;
+exports.synthesizeFromTestSuite = synthesizeFromTestSuite;
+const dedent_1 = __importDefault(require("dedent"));
+const logger_1 = __importDefault(require("../logger"));
+const providers_1 = require("../providers");
+const defaults_1 = require("../providers/defaults");
+const generation_1 = require("../util/generation");
+const invariant_1 = __importDefault(require("../util/invariant"));
+const json_1 = require("../util/json");
+function generateNewQuestionsPrompt(prompts, testCases, numQuestions) {
+    const allAssertions = testCases.flatMap((c) => c.assert || []);
+    return (0, dedent_1.default) `
+      Role: You are a senior data scientist specializing in metric design for stochastic AI systems. You will be given 
+      an series of system prompts and existing assertions being tested in an evaluation, your task is to create objective evaluation questions that assess 
+      individual AI responses—not the application holistically—based on input-output pairs. 
+      
+      Make sure to generate questions that are different from ones that already exist.
+      
+      Clarification: Some applications (like scam detection, content moderation, or classification tasks) ask the AI to evaluate an input artifact. 
+      Your task is **NOT** to evaluate the artifact (input) directly, but to assess the AI's response — i.e., how well the assistant performed the requested evaluation.
+      For example, don’t ask: “Does the message contain suspicious links?”
+      Instead, ask: “Did the response correctly identify suspicious links in the message?” or “Are the ratings in the output aligned with the rubric?”
+      
+      Core Requirements
+      1. Question Types:
+      Questions may use one of the following scoring formats: binary (Yes/No), 5-point Likert scale, or 0–1 continuous scale.
+      Design each question to naturally align with its scale—for example, use binary for clear-cut presence/absence traits, Likert for subjective gradations, and continuous for measurable properties.
+      Binary questions can still be scored on a Likert scale by mapping “Yes = 5” and “No = 1” if needed.
+      
+      IMPORTANT: Questions should be phrased so that a 'Yes' answer or higher score **always** indicates compliance with the desired metric or requirement.
+      
+      2. Focus:
+      Questions can evaluate:
+        i. Input-output relationships (e.g., "Does the output address all parts of the input query?").
+        ii. Response attributes (e.g., structure, clarity, safety).
+      Avoid holistic/system-level judgments (e.g., "Is the AI helpful?").
+      
+      3. Objectivity:
+      Be as objective as possible. Replace ambiguous terms (e.g., "inspiring," "too long") with quantifiable criteria (e.g., "Is the output > 100 words?").
+      Allowed subjectivity: Verbs/adjectives are fine if they describe inherent properties of language (e.g., "Does the response contain abusive language?").
+        Rationale: "Abusive" is a property of language, even if borderline cases exist. 
+      Avoid unbounded subjectivity (e.g., "Is the output extremely concise?" → replace with "Is the output ≤ 50 words?").
+      In general, think of ways to replace subjective ideas with objective ones.
+      
+      4. Atomicity:
+      Each question should test one attribute or relationship (e.g., split "Is the response clear and concise?" into two questions).
+      
+      5. Independence:
+      Questions should avoid overlap to prevent double-counting issues in evaluation. They should not overlap with any assertions either.
+      
+      6. Self-Containment:
+      Permitted: Derive answers from the input/output text (e.g., "Does the output cite a verbatim quote from the input?").
+      Forbidden: Reliance on external knowledge (e.g., "Is the cited source reputable?" → replace with "Does the citation include a DOI?").
+      
+      7. Special Cases:
+      For creative tasks: Focus on technical execution (e.g., "Does each stanza have 4 lines?").
+      For list outputs: Evaluate per item (e.g., "Does each bullet point contain a complete sentence?").
+      
+      Each question must be preceded by a label in Title Case, no longer than three words, that serves as a concise and descriptive title for the question.
+      
+      After writing each question, **always** set 'is_lower_score_desirable' to false because if the answer to the question is “Yes” (or higher score in case of likert/0-1 scales), 
+      it always indicates a good response. You are only generating such type of questions.
+      
+      Each question should have a question_source. If the question is implied in the input application_description, use
+      IMPLIED_IN_INSTRUCTIONS; otherwise if you are generating it from scratch, use FULLY_NEWLY_GENERATED.
+      
+      Each question should have a question_type. If the question is core for this specific application, use 
+      CORE_FOR_APPLICATION. If the question is a generic check which applies to many other applications like check for
+      abusive content or toxic language, use HORIZONTAL. If the question is regarding output format or some structure
+      in the response of the application, use FORMAT_CHECK.
+      
+      Anti-Patterns to Avoid
+      1. Reasoning Dependencies:
+      Bad: "Is the argument persuasive?"
+      Fixed: "Does the response list at least 2 supporting facts?"
+      
+      2. World Knowledge:
+      Bad: "Is the cited author an expert?"
+      Fixed: "Does the citation include the author’s institutional affiliation?"
+      
+      3. Unbounded Subjectivity:
+      Bad: "Is the output extremely concise?"
+      Fixed: "Is the output ≤ 3 sentences?"
+      
+      Process
+      1. Classify the Application:
+      First classify the application into appropriate categories such as information extraction, information summarization, creative task, analysis task.
+      Note that an application can belong to multiple categories.
+      Define key attributes (e.g., accuracy, structure, safety).
+      
+      2. Extract Implied Questions (Mandatory):
+      Scan the application_description for any *implied requirements*—expectations stated or suggested in the instructions.
+      For each implied requirement, generate an evaluation question marked with:
+        - 'question_source = implied_in_instructions'
+      These must be generated **before** any newly inferred or generic questions.
+      
+      3. Generate Deep Criteria (for new questions):
+      For each key attribute not already covered by an implied question:
+        - Identify subtle failure modes
+        - Design objectively measurable, atomic, and independent evaluation criteria
+        - Use quantifiable standards and avoid vague constructs
+        - Generate questions with 'question_source = fully_newly_generated'
+      
+      4. Generate Questions:
+      ${numQuestions > 0
+        ? `Create total ${numQuestions} questions with:`
+        : `Create a comprehensive set of evaluation questions with:`}
+      Binary (if absolute criteria exist) or Likert/continuous scales.
+      Concrete thresholds for quantifiable traits (e.g., word/line counts).
+      
+      **IMPORTANT**: You must prioritize and fully exhaust all questions implied by the application description before generating any new questions.
+      ${numQuestions > 0
+        ? `Do not generate any 'fully_newly_generated' questions if the implied questions alone fulfill the requested ${numQuestions}.`
+        : ``}
+      
+      
+      # OUTPUT FORMAT
+      
+      Only respond in JSON with no extra content.
+      
+      # EXAMPLES
+      
+      <application>
+      Describe a recipe for an input dish in bulleted list format.
+      </application>
+      <existing_assertions>
+      [
+        {
+          "type" : "llm-rubric",
+          "value": "Does the output list all necessary ingredients for the dish?",
+          "metric": "Ingredient Inclusion"
+        },
+        {
+          "type" : "g-eval",
+          "value": "Does each step in the recipe provide clear and complete instructions for preparation?"
+        }
+      ]
+      </existing_assertions>
+      ${'```'}json
+      
+      {
+        "questions": [
+          {
+            "label": "Sequential Order",
+            "question": "Are the preparation steps listed in a logical and sequential order?",
+            "question_source": "implied_in_instructions",
+            "question_type": "core_for_application"
+          },
+          {
+            "label": "Bullet Format",
+            "question": "Is each item in the recipe presented as a distinct bullet point?",
+            "question_source": "implied_in_instructions",
+            "question_type": "format_check"
+          },
+          {
+            "label": "Cooking Times",
+            "question": "Are the cooking and preparation times mentioned in the recipe?",
+            "question_source": "fully_newly_generated",
+            "question_type": "core_for_application"
+          },
+          {
+            "label": "Ingredient Quantities",
+            "question": "Are the quantities for each ingredient specified in the recipe?",
+            "question_source": "fully_newly_generated",
+            "question_type": "core_for_application"
+          },
+          {
+            "label": "Serving Size",
+            "question": "Does the recipe specify the number of servings it makes?",
+            "question_source": "fully_newly_generated",
+            "question_type": "core_for_application"
+          },
+          {
+            "label": "Filler Words",
+            "question": "Does the recipe avoid including unnecessary details?",
+            "question_source": "fully_newly_generated",
+            "question_type": "horizontal"
+          }
+        ]
+      }
+    
+    
+      Consider the following prompt${prompts.length > 1 ? 's' : ''} and assertions for an LLM application:
+      
+      <Prompts>
+  ${prompts
+        .map((prompt, i) => `      ${i > 0 ? '  ' : ''}<Prompt>
+          ${prompt}
+        </Prompt>`)
+        .join('\n')}
+      </Prompts>
+     
+      <existing_assertions>
+        ${JSON.stringify(allAssertions, null, 2)
+        .split('\n')
+        .map((line, i) => (i > 0 ? `        ${line}` : line))
+        .join('\n')}
+      </existing_assertions>`;
+}
+function convertQuestionToPythonPrompt(prompts, question) {
+    return (0, dedent_1.default) `
+    You are a specialized system that analyzes an LLM evaluation question and generates a Python function to automatically check LLM responses against the specific criterion. 
+    Your task is to determine if the given evaluation question can be reliably answered using a deterministic Python function.
+    
+    ## Input Format
+    You will be provided with:
+    1. A description of the LLM application (string)
+    2. A single evaluation question used to assess LLM responses (string)
+    
+    ## Output Format
+    For the evaluation question, you must:
+    
+    - Determine if the question can be reliably answered with a deterministic Python function using ONLY the LLM response
+    - If YES: Return only the Python function body (without the function signature) that:
+      - Assumes the LLM's response text is available as a string variable named \`output\`
+      - Returns a dictionary with two keys:
+        - \`'pass'\`: boolean value (True if criterion is met, False if not)
+        - \`'score'\`: float value (1.0 if criterion is met, 0.0 if not)
+      - The Answer "Yes" to the question should correspond to \`{'pass': True, 'score': 1.0}\`
+      - The answer "No" to the question should correspond to \`{'pass': False, 'score': 0.0}\`
+      - Includes clear comments
+      - Handles edge cases gracefully (e.g., empty responses, invalid formats)
+      - Performs any necessary parsing of the response string (JSON parsing, text extraction, etc.)
+    - If NO: Return the string "None" (when the question requires semantic understanding, subjective judgment, domain expertise, or requires examining the original prompt/input)
+    
+    ## Critical Requirements
+    - The function must evaluate ONLY the LLM response itself, which will always be provided as a string
+    - The evaluation question might refer to the LLM output by domain-specific terms (e.g., "story", "recipe", "code", "answer") based on the application description, rather than generic terms like "response" or "output"
+    - Regardless of terminology used in the question, the variable name in your code must be "output".                                                               
+    - If evaluation requires comparing the response to the original prompt/input, return "None"
+    - If evaluation requires external knowledge, context, or resources, return "None"
+    - When in doubt, return "None" rather than an unreliable function
+    - Any required parsing (JSON, XML, etc.) must be handled within the function
+                                          
+    ## IMPORTANT
+    - Return "None" for any evaluation that requires semantic understanding or could have multiple valid expressions
+    - For questions about greetings, politeness, tone, style, or other subjective language features, return "None"
+    - Avoid creating functions that rely on hardcoded lists of phrases, expressions, or patterns when the concept being evaluated could be expressed in many different ways
+    - Only create functions for criteria that can be evaluated through standardized, unambiguous patterns or clear structural properties
+                                                                          
+    ## Guidelines for Domain-Specific References
+    - When the question refers to the output by a domain-specific term (e.g., "Is the story less than 2 lines long?", "Does the recipe include four or more spices?"), understand that it's referring to the same content that will be available as the \`output\` variable
+    - The application description often provides context for what type of output to expect (story, recipe, etc.)
+    
+    ## Guidelines for Function Generation
+    
+    ### Questions Suitable for Functions (return a function):
+    - Counting elements (words, sentences, lines, items)
+    - Checking for presence of specific strings, patterns, or structures within the response
+    - Validating formats (JSON, dates, emails, etc.)
+    - Measuring response length in characters/bytes etc
+    - Checking for code syntax, structure, or presence of specific elements
+    - Verifying mathematical properties or numerical ranges
+    
+    ### Questions NOT Suitable for Functions (return "None"):
+    - Any evaluation requiring comparison to the original prompt
+    - Evaluating relevance, accuracy, or helpfulness
+    - Assessing tone, intent, style, sentiment or semantics
+    - Checking factual correctness
+    - Determining completeness of explanations
+    - Evaluating creativity or originality
+    - Assessing logical coherence or reasoning quality
+    - Any judgment requiring domain expertise
+    - Any evaluation that would require an exhaustive list of possible expressions (like apologies, call-to-action etc.)
+                                          
+    Please provide only the Python function body without markdown formatting or function signature. 
+    The function body should assume the LLM's response is available as a variable named \`output\`.
+    Also include the necessary import statements within the function body itself.
+    
+    ## Example Input/Output Pairs
+                                          
+    ### Example 1:
+    **Application Description:** A JSON API documentation system
+    **Evaluation Question:** "Does the response contain valid JSON?"
+    
+    **Output:**
+    \`\`\`python
+    import json
+    import re
+    
+    # Try to find JSON blocks in the output
+    # Look for content within code blocks with \`\`\`json
+    json_block_pattern = r'\`\`\`(?:json)?\\s*([\\s\\S]*?)\\s*\`\`\`'
+    json_blocks = re.findall(json_block_pattern, output)
+    
+    # Also look for content within curly braces that might be JSON
+    potential_json = re.findall(r'(\\{[\\s\\S]*?\\})', output)
+    
+    # Combine all potential JSON content
+    all_potential_json = json_blocks + potential_json
+    
+    # If we don't find any potential JSON patterns, return False
+    if not all_potential_json:
+        return {'pass': False, 'score': 0.0}
+    
+    # Try to parse each potential JSON block
+    for json_str in all_potential_json:
+        try:
+            json.loads(json_str)
+            return {'pass': True, 'score': 1.0}  # Valid JSON found
+        except json.JSONDecodeError:
+            continue
+    
+    return {'pass': False, 'score': 0.0}  # No valid JSON found
+    \`\`\`
+    
+    ### Example 2:
+    **Application Description:** A customer service chatbot
+    **Evaluation Question:** "Does the response address the customer's initial query?"
+    
+    **Output:**
+    None
+                                          
+    ### Example 3:
+    **Application Description:** A code assistant that generates SQL queries.
+    **Evaluation Question:** "Does the SQL query use a JOIN statement?"
+    
+    **Output:**
+    \`\`\`python
+    import re
+    
+    # Convert to lowercase for case-insensitive matching
+    output_lower = output.lower()
+    
+    # Extract code blocks if present
+    code_blocks = re.findall(r'\`\`\`(?:sql)?([^\`]+)\`\`\`', output_lower)
+    
+    # If code blocks are found, check them first
+    if code_blocks:
+        for block in code_blocks:
+            # Check for JOIN keyword with word boundaries
+            if re.search(r'\\b(join|inner\\s+join|left\\s+join|right\\s+join|full\\s+join|cross\\s+join)\\b', block):
+                return {'pass': True, 'score': 1.0}
+    
+    # If no code blocks or no JOIN found in code blocks, check the entire output
+    join_patterns = [
+        r'\\b(join)\\b',
+        r'\\b(inner\\s+join)\\b',
+        r'\\b(left\\s+join)\\b',
+        r'\\b(right\\s+join)\\b',
+        r'\\b(full\\s+join)\\b',
+        r'\\b(cross\\s+join)\\b'
+    ]
+    
+    for pattern in join_patterns:
+        if re.search(pattern, output_lower):
+            return {'pass': True, 'score': 1.0}
+    
+    return {'pass': False, 'score': 0.0}
+    \`\`\`
+    
+    ### Example 4:
+    **Application Description:** An eval agent that can plan weekend trips.
+    **Evaluation Question:** "Does the response exceed 1500 words?"
+    
+    **Output:**
+    \`\`\`python
+    # Split the output into words
+    words = output.split()
+    
+    # Count the number of words
+    word_count = len(words)
+    
+    # Check if the word count exceeds 1500
+    if word_count > 1500:
+        return {'pass': True, 'score': 1.0}
+    return {'pass': False, 'score': 0.0}
+    \`\`\`
+    
+    ### Example 5:
+    **Application Description:** A customer service chatbot
+    **Evaluation Question:** "Does the response start with a greeting?"
+    
+    **Output:**
+    None
+    
+    Remember: When in doubt, return "None". It's better to use some other evaluation mechanism than to generate an unreliable function.
+      
+    <application_description>
+      <Prompts>
+  ${prompts
+        .map((prompt, i) => `      ${i > 0 ? '  ' : ''}<Prompt>
+          ${prompt}
+        </Prompt>`)
+        .join('\n')}
+      </Prompts>
+    </application_description>
+    <question>
+    ${question}
+    </question>
+`;
+}
+async function synthesize({ prompts, instructions, numQuestions = 5, tests, provider, type = 'pi', }) {
+    if (prompts.length < 1) {
+        throw new Error('Assertion synthesis requires at least one prompt.');
+    }
+    let progressBar;
+    if (logger_1.default.level !== 'debug') {
+        const cliProgress = await Promise.resolve().then(() => __importStar(require('cli-progress')));
+        progressBar = new cliProgress.SingleBar({ gracefulExit: true }, cliProgress.Presets.shades_classic);
+        progressBar.start(1 + numQuestions, 0);
+    }
+    logger_1.default.debug(`Starting assertion synthesis. We'll begin by generating a set of questions`);
+    logger_1.default.debug(`Generating user personas from ${prompts.length} prompt${prompts.length > 1 ? 's' : ''}...`);
+    let providerModel;
+    if (typeof provider === 'undefined') {
+        providerModel = (await (0, defaults_1.getDefaultProviders)()).synthesizeProvider;
+    }
+    else {
+        providerModel = await (0, providers_1.loadApiProvider)(provider);
+    }
+    let newQuestionsPrompt = generateNewQuestionsPrompt(prompts, tests, numQuestions);
+    if (instructions) {
+        newQuestionsPrompt = `${newQuestionsPrompt}\n${instructions}`;
+    }
+    logger_1.default.debug(`Generated questions prompt:\n${newQuestionsPrompt}`);
+    const resp = await providerModel.callApi(newQuestionsPrompt);
+    logger_1.default.debug(`Received questions response:\n${resp.output}`);
+    (0, invariant_1.default)(typeof resp.output !== 'undefined', 'resp.output must be defined');
+    const output = typeof resp.output === 'string' ? resp.output : JSON.stringify(resp.output);
+    const respObjects = (0, json_1.extractJsonObjects)(output);
+    (0, invariant_1.default)(respObjects.length >= 1, `Expected at least one JSON object in the response for questions, got ${respObjects.length}`);
+    const questionsWrapper = respObjects[0];
+    const questions = (0, generation_1.sampleArray)(questionsWrapper.questions, numQuestions);
+    logger_1.default.debug(`Generated ${questions.length} question${questions.length === 1 ? '' : 's'}:\n${questions.map((p) => `  - ${p.question}`).join('\n')}`);
+    if (progressBar) {
+        progressBar.increment();
+    }
+    // Determine which dimensions should be objectively answered with python. If so, generate the python.
+    providerModel.config = {
+        maxTokens: 3000,
+    };
+    const assertions = await Promise.all(questions.map(async (q) => {
+        const pythonConvertPrompt = convertQuestionToPythonPrompt(prompts, q.question);
+        const resp = await providerModel.callApi(pythonConvertPrompt);
+        const output = resp.output;
+        if (progressBar) {
+            progressBar.increment();
+        }
+        if (output.toLowerCase().trim() == 'none') {
+            return { type, metric: q.label, value: q.question };
+        }
+        else {
+            return {
+                type: 'python',
+                metric: q.label,
+                value: output,
+            };
+        }
+    }));
+    logger_1.default.debug(`Generated ${assertions.length} new assertions`);
+    if (progressBar) {
+        progressBar.stop();
+    }
+    return assertions;
+}
+async function synthesizeFromTestSuite(testSuite, options) {
+    return synthesize({
+        ...options,
+        prompts: testSuite.prompts.map((prompt) => prompt.raw),
+        tests: testSuite.tests || [],
+    });
+}
+//# sourceMappingURL=synthesis.js.map

package/dist/src/assertions/synthesis.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"synthesis.js","sourceRoot":"","sources":["../../../src/assertions/synthesis.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmBA,gEAyMC;AAED,sEAmMC;AASD,gCAuFC;AAED,0DASC;AA3gBD,oDAA4B;AAC5B,uDAA+B;AAC/B,4CAA+C;AAC/C,oDAA4D;AAE5D,mDAAiD;AACjD,kEAA0C;AAC1C,uCAAkD;AAWlD,SAAgB,0BAA0B,CACxC,OAAiB,EACjB,SAAqB,EACrB,YAAoB;IAEpB,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IAE/D,OAAO,IAAA,gBAAM,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QA8FP,YAAY,GAAG,CAAC;QACd,CAAC,CAAC,gBAAgB,YAAY,kBAAkB;QAChD,CAAC,CAAC,0DACN;;;;;QAME,YAAY,GAAG,CAAC;QACd,CAAC,CAAC,8GAA8G,YAAY,GAAG;QAC/H,CAAC,CAAC,EACN;;;;;;;;;;;;;;;;;;;;;;;;;QAyBE,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qCA4CwB,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;;;IAG9D,OAAO;SACN,GAAG,CACF,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CACZ,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE;YACtB,MAAM;kBACA,CACb;SACA,IAAI,CAAC,IAAI,CAAC;;;;UAIL,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;SACrC,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SACpD,IAAI,CAAC,IAAI,CAAC;6BACQ,CAAC;AAC9B,CAAC;AAED,SAAgB,6BAA6B,CAAC,OAAiB,EAAE,QAAgB;IAC/E,OAAO,IAAA,gBAAM,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAoLX,OAAO;SACN,GAAG,CACF,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CACZ,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE;YACtB,MAAM;kBACA,CACb;SACA,IAAI,CAAC,IAAI,CAAC;;;;MAIT,QAAQ;;CAEb,CAAC;AACF,CAAC;AASM,KAAK,UAAU,UAAU,CAAC,EAC/B,OAAO,EACP,YAAY,EACZ,YAAY,GAAG,CAAC,EAChB,KAAK,EACL,QAAQ,EACR,IAAI,GAAG,IAAI,GACO;IAClB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IAED,IAAI,WAAkC,CAAC;IACvC,IAAI,gBAAM,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;QAC7B,MAAM,WAAW,GAAG,wDAAa,cAAc,GAAC,CAAC;QACjD,WAAW,GAAG,IAAI,WAAW,CAAC,SAAS,CACrC,EAAE,YAAY,EAAE,IAAI,EAAE,EACtB,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;QACF,WAAW,CAAC,KAAK,CAAC,CAAC,GAAG,YAAY,EAAE,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,gBAAM,CAAC,KAAK,CAAC,4EAA4E,CAAC,CAAC;IAE3F,gBAAM,CAAC,KAAK,CACV,iCAAiC,OAAO,CAAC,MAAM,UAAU,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAC5F,CAAC;IAEF,IAAI,aAA0B,CAAC;IAC/B,IAAI,OAAO,QAAQ,KAAK,WAAW,EAAE,CAAC;QACpC,aAAa,GAAG,CAAC,MAAM,IAAA,8BAAmB,GAAE,CAAC,CAAC,kBAAkB,CAAC;IACnE,CAAC;SAAM,CAAC;QACN,aAAa,GAAG,MAAM,IAAA,2BAAe,EAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,kBAAkB,GAAG,0BAA0B,CAAC,OAAO,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;IAClF,IAAI,YAAY,EAAE,CAAC;QACjB,kBAAkB,GAAG,GAAG,kBAAkB,KAAK,YAAY,EAAE,CAAC;IAChE,CAAC;IACD,gBAAM,CAAC,KAAK,CAAC,gCAAgC,kBAAkB,EAAE,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC7D,gBAAM,CAAC,KAAK,CAAC,iCAAiC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7D,IAAA,mBAAS,EAAC,OAAO,IAAI,CAAC,MAAM,KAAK,WAAW,EAAE,6BAA6B,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3F,MAAM,WAAW,GAAG,IAAA,yBAAkB,EAAC,MAAM,CAAC,CAAC;IAC/C,IAAA,mBAAS,EACP,WAAW,CAAC,MAAM,IAAI,CAAC,EACvB,wEAAwE,WAAW,CAAC,MAAM,EAAE,CAC7F,CAAC;IACF,MAAM,gBAAgB,GAAG,WAAW,CAAC,CAAC,CAAuC,CAAC;IAC9E,MAAM,SAAS,GAAG,IAAA,wBAAW,EAAC,gBAAgB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAExE,gBAAM,CAAC,KAAK,CACV,aAAa,SAAS,CAAC,MAAM,YAAY,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACvI,CAAC;IAEF,IAAI,WAAW,EAAE,CAAC;QAChB,WAAW,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IACD,qGAAqG;IACrG,aAAa,CAAC,MAAM,GAAG;QACrB,SAAS,EAAE,IAAI;KAChB,CAAC;IACF,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACxB,MAAM,mBAAmB,GAAG,6BAA6B,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC/E,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAW,IAAI,CAAC,MAAM,CAAC;QACnC,IAAI,WAAW,EAAE,CAAC;YAChB,WAAW,CAAC,SAAS,EAAE,CAAC;QAC1B,CAAC;QAED,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;YAC1C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,OAAO;gBACL,IAAI,EAAE,QAA6B;gBACnC,MAAM,EAAE,CAAC,CAAC,KAAK;gBACf,KAAK,EAAE,MAAM;aACd,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IACF,gBAAM,CAAC,KAAK,CAAC,aAAa,UAAU,CAAC,MAAM,iBAAiB,CAAC,CAAC;IAC9D,IAAI,WAAW,EAAE,CAAC;QAChB,WAAW,CAAC,IAAI,EAAE,CAAC;IACrB,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAEM,KAAK,UAAU,uBAAuB,CAC3C,SAAoB,EACpB,OAAmC;IAEnC,OAAO,UAAU,CAAC;QAChB,GAAG,OAAO;QACV,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC;QACtD,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,EAAE;KAC7B,CAAC,CAAC;AACL,CAAC"}

package/dist/src/commands/delete.d.ts

@@ -1,3 +1,5 @@
 import type { Command } from 'commander';
+export declare function handleEvalDelete(evalId: string, envPath?: string): Promise<void>;
+export declare function handleEvalDeleteAll(): Promise<void>;
 export declare function deleteCommand(program: Command): void;
 //# sourceMappingURL=delete.d.ts.map

package/dist/src/commands/delete.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../../src/commands/delete.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA6BzC,wBAAgB,aAAa,CAAC,OAAO,EAAE,OAAO,QA+C7C"}
+{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../../src/commands/delete.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOzC,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,iBAQtE;AAED,wBAAsB,mBAAmB,kBAUxC;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,OAAO,QAgD7C"}

package/dist/src/commands/delete.js

@@ -3,6 +3,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleEvalDelete = handleEvalDelete;
+exports.handleEvalDeleteAll = handleEvalDeleteAll;
 exports.deleteCommand = deleteCommand;
 const confirm_1 = __importDefault(require("@inquirer/confirm"));
 const logger_1 = __importDefault(require("../logger"));
@@ -45,6 +47,7 @@ function deleteCommand(program) {
             return handleEvalDelete(id, cmdObj.envPath);
         }
         logger_1.default.error(`No resource found with ID ${id}`);
+        process.exitCode = 1;
     });
     deleteCommand
         .command('eval <id>')

package/dist/src/commands/delete.js.map

@@ -1 +1 @@
-{"version":3,"file":"delete.js","sourceRoot":"","sources":["../../../src/commands/delete.ts"],"names":[],"mappings":";;;;;AA8BA,sCA+CC;AA7ED,gEAAwC;AAExC,uDAA+B;AAC/B,0DAAkC;AAClC,6DAAqC;AACrC,kCAAmC;AACnC,+CAA6E;AAE7E,KAAK,UAAU,gBAAgB,CAAC,MAAc,EAAE,OAAgB;IAC9D,IAAI,CAAC;QACH,MAAM,IAAA,qBAAU,EAAC,MAAM,CAAC,CAAC;QACzB,gBAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,iCAAiC,CAAC,CAAC;IAC7E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,gBAAM,CAAC,KAAK,CAAC,uCAAuC,MAAM,MAAM,KAAK,EAAE,CAAC,CAAC;QACzE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB;IAChC,MAAM,SAAS,GAAG,MAAM,IAAA,iBAAO,EAAC;QAC9B,OAAO,EACL,uFAAuF;KAC1F,CAAC,CAAC;IACH,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO;IACT,CAAC;IACD,MAAM,IAAA,yBAAc,GAAE,CAAC;IACvB,gBAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;AACpD,CAAC;AAED,SAAgB,aAAa,CAAC,OAAgB;IAC5C,MAAM,aAAa,GAAG,OAAO;SAC1B,OAAO,CAAC,aAAa,CAAC;SACtB,WAAW,CAAC,0BAA0B,CAAC;SACvC,MAAM,CAAC,+BAA+B,EAAE,mBAAmB,CAAC;SAC5D,MAAM,CAAC,KAAK,EAAE,EAAU,EAAE,MAA4B,EAAE,EAAE;QACzD,IAAA,eAAQ,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzB,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,QAAQ;SACf,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,IAAA,wBAAa,EAAC,EAAE,CAAC,CAAC;QACpC,IAAI,GAAG,EAAE,CAAC;YACR,OAAO,gBAAgB,CAAC,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9C,CAAC;QAED,gBAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEL,aAAa;SACV,OAAO,CAAC,WAAW,CAAC;SACpB,WAAW,CACV,oHAAoH,CACrH;SACA,MAAM,CAAC,+BAA+B,EAAE,mBAAmB,CAAC;SAC5D,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;QAC/B,IAAA,eAAQ,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzB,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,aAAa;YACnB,MAAM;SACP,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxB,MAAM,aAAa,GAAG,MAAM,cAAI,CAAC,MAAM,EAAE,CAAC;YAC1C,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,gBAAgB,CAAC,aAAa,CAAC,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;YAC3D,CAAC;iBAAM,CAAC;gBACN,gBAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;gBAC/B,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;aAAM,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YAC5B,MAAM,mBAAmB,EAAE,CAAC;QAC9B,CAAC;aAAM,CAAC;YACN,MAAM,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACjD,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
+{"version":3,"file":"delete.js","sourceRoot":"","sources":["../../../src/commands/delete.ts"],"names":[],"mappings":";;;;;AAQA,4CAQC;AAED,kDAUC;AAED,sCAgDC;AA9ED,gEAAwC;AAExC,uDAA+B;AAC/B,0DAAkC;AAClC,6DAAqC;AACrC,kCAAmC;AACnC,+CAA6E;AAEtE,KAAK,UAAU,gBAAgB,CAAC,MAAc,EAAE,OAAgB;IACrE,IAAI,CAAC;QACH,MAAM,IAAA,qBAAU,EAAC,MAAM,CAAC,CAAC;QACzB,gBAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,iCAAiC,CAAC,CAAC;IAC7E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,gBAAM,CAAC,KAAK,CAAC,uCAAuC,MAAM,MAAM,KAAK,EAAE,CAAC,CAAC;QACzE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,mBAAmB;IACvC,MAAM,SAAS,GAAG,MAAM,IAAA,iBAAO,EAAC;QAC9B,OAAO,EACL,uFAAuF;KAC1F,CAAC,CAAC;IACH,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO;IACT,CAAC;IACD,MAAM,IAAA,yBAAc,GAAE,CAAC;IACvB,gBAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;AACpD,CAAC;AAED,SAAgB,aAAa,CAAC,OAAgB;IAC5C,MAAM,aAAa,GAAG,OAAO;SAC1B,OAAO,CAAC,aAAa,CAAC;SACtB,WAAW,CAAC,0BAA0B,CAAC;SACvC,MAAM,CAAC,+BAA+B,EAAE,mBAAmB,CAAC;SAC5D,MAAM,CAAC,KAAK,EAAE,EAAU,EAAE,MAA4B,EAAE,EAAE;QACzD,IAAA,eAAQ,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzB,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,QAAQ;SACf,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,IAAA,wBAAa,EAAC,EAAE,CAAC,CAAC;QACpC,IAAI,GAAG,EAAE,CAAC;YACR,OAAO,gBAAgB,CAAC,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9C,CAAC;QAED,gBAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,CAAC,CAAC;QAChD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;IAEL,aAAa;SACV,OAAO,CAAC,WAAW,CAAC;SACpB,WAAW,CACV,oHAAoH,CACrH;SACA,MAAM,CAAC,+BAA+B,EAAE,mBAAmB,CAAC;SAC5D,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;QAC/B,IAAA,eAAQ,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzB,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,aAAa;YACnB,MAAM;SACP,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxB,MAAM,aAAa,GAAG,MAAM,cAAI,CAAC,MAAM,EAAE,CAAC;YAC1C,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,gBAAgB,CAAC,aAAa,CAAC,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;YAC3D,CAAC;iBAAM,CAAC;gBACN,gBAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;gBAC/B,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;aAAM,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YAC5B,MAAM,mBAAmB,EAAE,CAAC;QAC9B,CAAC;aAAM,CAAC;YACN,MAAM,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACjD,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/src/commands/eval.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../../src/commands/eval.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAczC,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAKlC,OAAO,KAAK,EACV,kBAAkB,EAClB,eAAe,EAEf,SAAS,EACT,UAAU,EACV,aAAa,EACd,MAAM,UAAU,CAAC;AAyBlB,wBAAgB,sCAAsC,CAAC,SAAS,EAAE,SAAS,QAa1E;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,GAAG,MAAM,CAwBnE;AAED,wBAAsB,MAAM,CAC1B,MAAM,EAAE,OAAO,CAAC,kBAAkB,GAAG,OAAO,CAAC,EAC7C,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EACrC,iBAAiB,EAAE,MAAM,GAAG,SAAS,EACrC,eAAe,EAAE,eAAe,GAC/B,OAAO,CAAC,IAAI,CAAC,CAmjBf;AAED,wBAAgB,WAAW,CACzB,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EACrC,iBAAiB,EAAE,MAAM,GAAG,SAAS,WAwMtC"}
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../../src/commands/eval.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAczC,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAKlC,OAAO,KAAK,EACV,kBAAkB,EAClB,eAAe,EAEf,SAAS,EACT,UAAU,EACV,aAAa,EACd,MAAM,UAAU,CAAC;AAyBlB,wBAAgB,sCAAsC,CAAC,SAAS,EAAE,SAAS,QAa1E;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,GAAG,MAAM,CAwBnE;AAED,wBAAsB,MAAM,CAC1B,MAAM,EAAE,OAAO,CAAC,kBAAkB,GAAG,OAAO,CAAC,EAC7C,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EACrC,iBAAiB,EAAE,MAAM,GAAG,SAAS,EACrC,eAAe,EAAE,eAAe,GAC/B,OAAO,CAAC,IAAI,CAAC,CAojBf;AAED,wBAAgB,WAAW,CACzB,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EACrC,iBAAiB,EAAE,MAAM,GAAG,SAAS,WAwMtC"}

package/dist/src/commands/eval.js

@@ -359,7 +359,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
                 else {
                     logger_1.default.info(`» Do you want to share this with your team? Sign up for free at ${chalk_1.default.greenBright.bold('https://promptfoo.app')}`);
                 }
-                logger_1.default.info(`» This project needs your feedback. What's one thing we can improve? ${chalk_1.default.greenBright.bold('https://forms.gle/YFLgTe1dKJKNSCsU7')}`);
+                logger_1.default.info(`» This project needs your feedback. What's one thing we can improve? ${chalk_1.default.greenBright.bold('https://promptfoo.dev/feedback')}`);
             }
         }
         else {
@@ -370,22 +370,13 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
         const duration = Math.round((Date.now() - startTime) / 1000);
         const durationDisplay = (0, formatDuration_1.formatDuration)(duration);
         const isRedteam = Boolean(config.redteam);
-        logger_1.default.info(chalk_1.default.green.bold(`Successes: ${successes}`));
-        logger_1.default.info(chalk_1.default.red.bold(`Failures: ${failures}`));
-        if (!Number.isNaN(errors)) {
-            logger_1.default.info(chalk_1.default.red.bold(`Errors: ${errors}`));
-        }
-        if (!Number.isNaN(passRate)) {
-            logger_1.default.info(chalk_1.default.blue.bold(`Pass Rate: ${passRate.toFixed(2)}%`));
-        }
-        logger_1.default.info(chalk_1.default.blue.bold(`Duration: ${durationDisplay} (concurrency: ${maxConcurrency})`));
         // Handle token usage display
         if (tokenUsage.total > 0 || (tokenUsage.prompt || 0) + (tokenUsage.completion || 0) > 0) {
             const combinedTotal = (tokenUsage.prompt || 0) + (tokenUsage.completion || 0);
             const evalTokens = {
                 prompt: tokenUsage.prompt || 0,
                 completion: tokenUsage.completion || 0,
-                total: combinedTotal,
+                total: tokenUsage.total || combinedTotal,
                 cached: tokenUsage.cached || 0,
                 completionDetails: tokenUsage.completionDetails || {
                     reasoning: 0,
@@ -393,7 +384,6 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
                     rejectedPrediction: 0,
                 },
             };
-            (0, util_2.printBorder)();
             logger_1.default.info(chalk_1.default.bold('Token Usage Summary:'));
             if (isRedteam) {
                 logger_1.default.info(`  ${chalk_1.default.cyan('Probes:')} ${chalk_1.default.white.bold(tokenUsage.numRequests.toLocaleString())}`);
@@ -412,7 +402,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
             // Provider breakdown
             const tracker = tokenUsage_1.TokenUsageTracker.getInstance();
             const providerIds = tracker.getProviderIds();
-            if (providerIds.length > 0) {
+            if (providerIds.length > 1) {
                 logger_1.default.info(`\n  ${chalk_1.default.cyan.bold('Provider Breakdown:')}`);
                 // Sort providers by total token usage (descending)
                 const sortedProviders = providerIds
@@ -460,9 +450,20 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
                 }
             }
             // Grand total
-            const grandTotal = evalTokens.total + tokenUsage.assertions.total;
+            const grandTotal = evalTokens.total + (tokenUsage.assertions.total || 0);
             logger_1.default.info(`\n  ${chalk_1.default.blue.bold('Grand Total:')} ${chalk_1.default.white.bold(grandTotal.toLocaleString())} tokens`);
+            (0, util_2.printBorder)();
         }
+        logger_1.default.info(chalk_1.default.gray(`Duration: ${durationDisplay} (concurrency: ${maxConcurrency})`));
+        logger_1.default.info(chalk_1.default.green.bold(`Successes: ${successes}`));
+        logger_1.default.info(chalk_1.default.red.bold(`Failures: ${failures}`));
+        if (!Number.isNaN(errors)) {
+            logger_1.default.info(chalk_1.default.red.bold(`Errors: ${errors}`));
+        }
+        if (!Number.isNaN(passRate)) {
+            logger_1.default.info(chalk_1.default.blue.bold(`Pass Rate: ${passRate.toFixed(2)}%`));
+        }
+        (0, util_2.printBorder)();
         telemetry_1.default.record('command_used', {
             name: 'eval',
             watch: Boolean(cmdObj.watch),