rippletide 1.0.17 → 1.0.20

package/.claude/settings.local.json

@@ -0,0 +1,13 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(npm run:*)",
+      "Bash(npx tsc:*)",
+      "Bash(npx:*)",
+      "Bash(npm install:*)",
+      "Bash(npm version:*)",
+      "Bash(npm publish:*)",
+      "Bash(npm whoami:*)"
+    ]
+  }
+}

package/dist/App.js

@@ -13,7 +13,7 @@ import { BaseError, ValidationError } from './errors/types.js';
 import { logger } from './utils/logger.js';
 import { analytics } from './utils/analytics.js';
 const knowledgeSources = [
-    { label: 'Local Files (qanda.json)', value: 'files', description: 'Use qanda.json from current directory' },
+    { label: 'Local Files (qanda.json)', value: 'files', description: 'Use a question-answer file (qanda.json) from the current directory' },
     { label: 'PDF Document', value: 'pdf', description: 'Upload and extract knowledge from a PDF file' },
     { label: 'Pinecone', value: 'pinecone', description: 'Fetch Q&A from Pinecone database' },
     { label: 'PostgreSQL Database', value: 'postgresql', description: 'Connect to PostgreSQL database' },
@@ -190,7 +190,7 @@ export const App = ({ backendUrl, dashboardUrl, nonInteractive, agentEndpoint: i
                         passed: 0,
                         failed: 0,
                         duration: 'Failed',
-                        evaluationUrl: dashboardUrl || 'https://eval.rippletide.com',
+                        evaluationUrl: dashboardUrl || 'https://app.rippletide.com/eval',
                         error: errorMessage,
                     });
                     setStep('complete');
@@ -235,7 +235,7 @@ export const App = ({ backendUrl, dashboardUrl, nonInteractive, agentEndpoint: i
                         passed: 0,
                         failed: 0,
                         duration: 'Failed',
-                        evaluationUrl: dashboardUrl || 'https://eval.rippletide.com',
+                        evaluationUrl: dashboardUrl || 'https://app.rippletide.com/eval',
                         error: errorMessage,
                     });
                     setStep('complete');
@@ -273,7 +273,7 @@ export const App = ({ backendUrl, dashboardUrl, nonInteractive, agentEndpoint: i
                         passed: 0,
                         failed: 0,
                         duration: 'Failed',
-                        evaluationUrl: dashboardUrl || 'https://eval.rippletide.com',
+                        evaluationUrl: dashboardUrl || 'https://app.rippletide.com/eval',
                         error: errorMessage,
                     });
                     setStep('complete');
@@ -299,8 +299,9 @@ export const App = ({ backendUrl, dashboardUrl, nonInteractive, agentEndpoint: i
                         setCurrentAgentId(agentId);
                     }
                     setEvaluationProgress(30);
+                    // Import knowledge based on source
+                    let knowledgeData = null;
                     if (knowledgeSource === 'files') {
-                        let knowledgeData = null;
                         if (templatePath) {
                             try {
                                 if (isRemoteTemplate) {
@@ -334,17 +335,29 @@ export const App = ({ backendUrl, dashboardUrl, nonInteractive, agentEndpoint: i
                                 }
                             }
                         }
-                        if (knowledgeData) {
-                            setEvaluationProgress(35);
-                            try {
-                                const importResult = await api.importKnowledge(agentId, knowledgeData);
-                                logger.debug('Knowledge import result:', importResult);
-                                await new Promise(resolve => setTimeout(resolve, 1000));
-                            }
-                            catch (error) {
-                                logger.error('Failed to import knowledge:', error?.message || error);
-                                logger.debug('Import error details:', error?.response?.data);
-                            }
+                    }
+                    else if (knowledgeSource === 'pinecone' && pineconeQAndA.length > 0) {
+                        // Import Pinecone Q&A as knowledge
+                        knowledgeData = pineconeQAndA;
+                    }
+                    else if (knowledgeSource === 'postgresql' && postgresqlQAndA.length > 0) {
+                        // Import PostgreSQL Q&A as knowledge
+                        knowledgeData = postgresqlQAndA;
+                    }
+                    else if (knowledgeSource === 'pdf' && pdfQAndA.length > 0) {
+                        // Import PDF Q&A as knowledge
+                        knowledgeData = pdfQAndA;
+                    }
+                    if (knowledgeData && Array.isArray(knowledgeData) && knowledgeData.length > 0) {
+                        setEvaluationProgress(35);
+                        try {
+                            const importResult = await api.importKnowledge(agentId, knowledgeData);
+                            logger.debug('Knowledge import result:', importResult);
+                            await new Promise(resolve => setTimeout(resolve, 1000));
+                        }
+                        catch (error) {
+                            logger.error('Failed to import knowledge:', error?.message || error);
+                            logger.debug('Import error details:', error?.response?.data);
                         }
                     }
                     setEvaluationProgress(40);
@@ -458,7 +471,7 @@ export const App = ({ backendUrl, dashboardUrl, nonInteractive, agentEndpoint: i
                         passed,
                         failed,
                         duration: durationStr,
-                        evaluationUrl: `${dashboardUrl || 'https://eval.rippletide.com'}/eval/${agentId}`,
+                        evaluationUrl: `${dashboardUrl || 'https://app.rippletide.com/eval'}/eval/${agentId}`,
                         agentId,
                     };
                     analytics.track('evaluation_completed', {
@@ -480,7 +493,7 @@ export const App = ({ backendUrl, dashboardUrl, nonInteractive, agentEndpoint: i
                         passed: 0,
                         failed: 0,
                         duration: 'Failed',
-                        evaluationUrl: dashboardUrl || 'https://eval.rippletide.com',
+                        evaluationUrl: dashboardUrl || 'https://app.rippletide.com/eval',
                         error: errorMessage,
                     });
                     setStep('complete');
@@ -674,7 +687,11 @@ export const App = ({ backendUrl, dashboardUrl, nonInteractive, agentEndpoint: i
                 React.createElement(Text, { bold: true, color: "#eba1b5" }, "Choose your data source:")),
             knowledgeFound && (React.createElement(Box, { marginBottom: 1 },
                 React.createElement(Text, { color: "white" }, "qanda.json found in current directory"))),
-            React.createElement(SelectMenu, { title: "Data Source", options: knowledgeSources, onSelect: handleSourceSelect }))),
+            React.createElement(SelectMenu, { title: "Data Source", options: knowledgeSources, onSelect: handleSourceSelect }),
+            React.createElement(Box, { marginTop: 1, flexDirection: "column" },
+                React.createElement(Text, { dimColor: true }, "Example qanda.json format:"),
+                React.createElement(Box, { paddingLeft: 2, flexDirection: "column" },
+                    React.createElement(Text, { dimColor: true }, `[ { "question": "What are your hours?", "answer": "Mon-Fri 9am-6pm" } ]`))))),
         step === 'pinecone-url' && (React.createElement(Box, { flexDirection: "column" },
             React.createElement(TextInput, { label: "Pinecone database URL", placeholder: "https://sample-movies-02j22s8.svc.aped-4627-b74a.pinecone.io", onSubmit: handlePineconeUrlSubmit }))),
         step === 'pinecone-api-key' && (React.createElement(Box, { flexDirection: "column" },

package/dist/api/evaluation.js

@@ -193,12 +193,13 @@ export async function runPromptEvaluation(agentId, promptId, promptText, agentEn
             if (hallucinationResult.hallucinationLabel && hallucinationResult.hallucinationLabel !== '') {
                 payload.hallucinationLabel = hallucinationResult.hallucinationLabel;
             }
-            if (hallucinationResult.hallucinationFindings && hallucinationResult.hallucinationFindings.length > 0) {
-                payload.hallucinationFindings = hallucinationResult.hallucinationFindings;
-                logger.debug(`Including ${hallucinationResult.hallucinationFindings.length} hallucination findings`);
+            // Always include hallucinationFindings, even if empty
+            payload.hallucinationFindings = hallucinationResult.hallucinationFindings || [];
+            if (payload.hallucinationFindings.length > 0) {
+                logger.debug(`Including ${payload.hallucinationFindings.length} hallucination findings`);
             }
             else {
-                logger.debug('No hallucination findings to include');
+                logger.debug('No hallucination findings to include (empty array)');
             }
             await client.post(`/api/agents/${agentId}/test-results/${promptId}`, payload);
             logger.debug(`Stored test result for prompt ${promptId} with status ${status}`);
@@ -214,6 +215,8 @@ export async function runPromptEvaluation(agentId, promptId, promptText, agentEn
             if (hallucinationResult.hallucinationLabel) {
                 minimalPayload.hallucinationLabel = hallucinationResult.hallucinationLabel;
             }
+            // Always include hallucinationFindings, even if empty
+            minimalPayload.hallucinationFindings = hallucinationResult.hallucinationFindings || [];
             await client.post(`/api/agents/${agentId}/test-results/${promptId}`, minimalPayload);
             logger.debug(`Stored minimal test result for prompt ${promptId}`);
         }
@@ -231,7 +234,8 @@ export async function runPromptEvaluation(agentId, promptId, promptText, agentEn
                 await client.post(`/api/agents/${agentId}/test-results/${promptId}`, {
                     status: 'passed',
                     response: llmResponse,
-                    expectedAnswer: expectedAnswer || null
+                    expectedAnswer: expectedAnswer || null,
+                    hallucinationFindings: []
                 });
             }
             catch (storeError) {
@@ -265,7 +269,8 @@ export async function runPromptEvaluation(agentId, promptId, promptText, agentEn
             await client.post(`/api/agents/${agentId}/test-results/${promptId}`, {
                 status: 'failed',
                 response: errorResponse,
-                expectedAnswer: expectedAnswer || null
+                expectedAnswer: expectedAnswer || null,
+                hallucinationFindings: []
             });
             logger.debug(`Stored failed result for prompt ${promptId}`);
         }
@@ -274,7 +279,8 @@ export async function runPromptEvaluation(agentId, promptId, promptText, agentEn
             logger.debug('Error details:', e?.response?.data);
             try {
                 await client.post(`/api/agents/${agentId}/test-results/${promptId}`, {
-                    status: 'failed'
+                    status: 'failed',
+                    hallucinationFindings: []
                 });
                 logger.debug(`Stored minimal failed result for prompt ${promptId}`);
             }

package/dist/components/Header.js

@@ -3,5 +3,7 @@ import { Box, Text } from 'ink';
 export const Header = () => {
     return (React.createElement(Box, { flexDirection: "column", marginBottom: 2 },
         React.createElement(Text, { bold: true, color: "#eba1b5" }, "Rippletide Evaluation"),
+        React.createElement(Text, { bold: true, color: "#eba1b5" }, "How It Works"),
+        React.createElement(Text, { dimColor: true }, "1. Connect your endpoint  2. Add your knowledge source  3. Run the evaluation"),
         React.createElement(Text, { color: "gray" }, "\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501")));
 };

package/dist/index.js

@@ -31,7 +31,7 @@ const parseArgs = async () => {
     }
     const options = {
         backendUrl: 'https://agent-evalserver-production.up.railway.app',
-        dashboardUrl: 'https://eval.rippletide.com',
+        dashboardUrl: 'https://app.rippletide.com/eval',
         debug: false,
         nonInteractive: false
     };
@@ -127,7 +127,7 @@ Options:
   -a, --agent <url>           Agent endpoint URL (e.g., localhost:8000)
   -k, --knowledge <source>    Knowledge source: files, pinecone, postgresql, or pdf (default: files)
   -b, --backend-url <url>     Backend API URL (default: https://agent-evalserver-production.up.railway.app)
-  -d, --dashboard-url <url>   Dashboard URL (default: https://eval.rippletide.com)
+  -d, --dashboard-url <url>   Dashboard URL (default: https://app.rippletide.com/eval)
   
   Pinecone options:
   -pu, --pinecone-url <url>   Pinecone database URL

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rippletide",
-  "version": "1.0.17",
+  "version": "1.0.20",
   "description": "Rippletide Evaluation CLI",
   "main": "dist/index.js",
   "bin": {

package/src/App.tsx

@@ -34,7 +34,7 @@ type Step =
   | 'complete';
 
 const knowledgeSources = [
-  { label: 'Local Files (qanda.json)', value: 'files', description: 'Use qanda.json from current directory' },
+  { label: 'Local Files (qanda.json)', value: 'files', description: 'Use a question-answer file (qanda.json) from the current directory' },
   { label: 'PDF Document', value: 'pdf', description: 'Upload and extract knowledge from a PDF file' },
   { label: 'Pinecone', value: 'pinecone', description: 'Fetch Q&A from Pinecone database' },
   { label: 'PostgreSQL Database', value: 'postgresql', description: 'Connect to PostgreSQL database' },
@@ -239,7 +239,7 @@ export const App: React.FC<AppProps> = ({
             passed: 0,
             failed: 0,
             duration: 'Failed',
-            evaluationUrl: dashboardUrl || 'https://eval.rippletide.com',
+            evaluationUrl: dashboardUrl || 'https://app.rippletide.com/eval',
             error: errorMessage,
           });
           setStep('complete');
@@ -293,7 +293,7 @@ export const App: React.FC<AppProps> = ({
             passed: 0,
             failed: 0,
             duration: 'Failed',
-            evaluationUrl: dashboardUrl || 'https://eval.rippletide.com',
+            evaluationUrl: dashboardUrl || 'https://app.rippletide.com/eval',
             error: errorMessage,
           });
           setStep('complete');
@@ -339,7 +339,7 @@ export const App: React.FC<AppProps> = ({
             passed: 0,
             failed: 0,
             duration: 'Failed',
-            evaluationUrl: dashboardUrl || 'https://eval.rippletide.com',
+            evaluationUrl: dashboardUrl || 'https://app.rippletide.com/eval',
             error: errorMessage,
           });
           setStep('complete');
@@ -370,8 +370,10 @@ export const App: React.FC<AppProps> = ({
           
           setEvaluationProgress(30);
           
+          // Import knowledge based on source
+          let knowledgeData: any = null;
+
           if (knowledgeSource === 'files') {
-            let knowledgeData: any = null;
             if (templatePath) {
               try {
                 if (isRemoteTemplate) {
@@ -401,17 +403,26 @@ export const App: React.FC<AppProps> = ({
                 }
               }
             }
-            
-            if (knowledgeData) {
-              setEvaluationProgress(35);
-              try {
-                const importResult = await api.importKnowledge(agentId, knowledgeData);
-                logger.debug('Knowledge import result:', importResult);
-                await new Promise(resolve => setTimeout(resolve, 1000));
-              } catch (error: any) {
-                logger.error('Failed to import knowledge:', error?.message || error);
-                logger.debug('Import error details:', error?.response?.data);
-              }
+          } else if (knowledgeSource === 'pinecone' && pineconeQAndA.length > 0) {
+            // Import Pinecone Q&A as knowledge
+            knowledgeData = pineconeQAndA;
+          } else if (knowledgeSource === 'postgresql' && postgresqlQAndA.length > 0) {
+            // Import PostgreSQL Q&A as knowledge
+            knowledgeData = postgresqlQAndA;
+          } else if (knowledgeSource === 'pdf' && pdfQAndA.length > 0) {
+            // Import PDF Q&A as knowledge
+            knowledgeData = pdfQAndA;
+          }
+
+          if (knowledgeData && Array.isArray(knowledgeData) && knowledgeData.length > 0) {
+            setEvaluationProgress(35);
+            try {
+              const importResult = await api.importKnowledge(agentId, knowledgeData);
+              logger.debug('Knowledge import result:', importResult);
+              await new Promise(resolve => setTimeout(resolve, 1000));
+            } catch (error: any) {
+              logger.error('Failed to import knowledge:', error?.message || error);
+              logger.debug('Import error details:', error?.response?.data);
             }
           }
           
@@ -532,7 +543,7 @@ export const App: React.FC<AppProps> = ({
             passed,
             failed,
             duration: durationStr,
-            evaluationUrl: `${dashboardUrl || 'https://eval.rippletide.com'}/eval/${agentId}`,
+            evaluationUrl: `${dashboardUrl || 'https://app.rippletide.com/eval'}/eval/${agentId}`,
             agentId,
           };
           
@@ -555,7 +566,7 @@ export const App: React.FC<AppProps> = ({
             passed: 0,
             failed: 0,
             duration: 'Failed',
-            evaluationUrl: dashboardUrl || 'https://eval.rippletide.com',
+            evaluationUrl: dashboardUrl || 'https://app.rippletide.com/eval',
             error: errorMessage,
           });
           setStep('complete');
@@ -833,6 +844,12 @@ export const App: React.FC<AppProps> = ({
             options={knowledgeSources}
             onSelect={handleSourceSelect}
           />
+          <Box marginTop={1} flexDirection="column">
+            <Text dimColor>Example qanda.json format:</Text>
+            <Box paddingLeft={2} flexDirection="column">
+              <Text dimColor>{`[ { "question": "What are your hours?", "answer": "Mon-Fri 9am-6pm" } ]`}</Text>
+            </Box>
+          </Box>
         </Box>
       )}
 

package/src/api/evaluation.ts

@@ -260,12 +260,13 @@ export async function runPromptEvaluation(
       if (hallucinationResult.hallucinationLabel && hallucinationResult.hallucinationLabel !== '') {
         payload.hallucinationLabel = hallucinationResult.hallucinationLabel;
       }
-      
-      if (hallucinationResult.hallucinationFindings && hallucinationResult.hallucinationFindings.length > 0) {
-        payload.hallucinationFindings = hallucinationResult.hallucinationFindings;
-        logger.debug(`Including ${hallucinationResult.hallucinationFindings.length} hallucination findings`);
+
+      // Always include hallucinationFindings, even if empty
+      payload.hallucinationFindings = hallucinationResult.hallucinationFindings || [];
+      if (payload.hallucinationFindings.length > 0) {
+        logger.debug(`Including ${payload.hallucinationFindings.length} hallucination findings`);
       } else {
-        logger.debug('No hallucination findings to include');
+        logger.debug('No hallucination findings to include (empty array)');
       }
       
       await client.post(`/api/agents/${agentId}/test-results/${promptId}`, payload);
@@ -279,11 +280,14 @@ export async function runPromptEvaluation(
         response: llmResponse,
         expectedAnswer: expectedAnswer || null
       };
-      
+
       if (hallucinationResult.hallucinationLabel) {
         minimalPayload.hallucinationLabel = hallucinationResult.hallucinationLabel;
       }
-      
+
+      // Always include hallucinationFindings, even if empty
+      minimalPayload.hallucinationFindings = hallucinationResult.hallucinationFindings || [];
+
       await client.post(`/api/agents/${agentId}/test-results/${promptId}`, minimalPayload);
       logger.debug(`Stored minimal test result for prompt ${promptId}`);
     }
@@ -301,7 +305,8 @@ export async function runPromptEvaluation(
         await client.post(`/api/agents/${agentId}/test-results/${promptId}`, {
           status: 'passed',
           response: llmResponse,
-          expectedAnswer: expectedAnswer || null
+          expectedAnswer: expectedAnswer || null,
+          hallucinationFindings: []
         });
       } catch (storeError) {
         logger.error('Could not store passed result:', storeError);
@@ -339,16 +344,18 @@ export async function runPromptEvaluation(
       await client.post(`/api/agents/${agentId}/test-results/${promptId}`, {
         status: 'failed',
         response: errorResponse,
-        expectedAnswer: expectedAnswer || null
+        expectedAnswer: expectedAnswer || null,
+        hallucinationFindings: []
       });
       logger.debug(`Stored failed result for prompt ${promptId}`);
     } catch (e: any) {
       logger.error('Failed to store failed result:', e?.message || e);
       logger.debug('Error details:', e?.response?.data);
-      
+
       try {
         await client.post(`/api/agents/${agentId}/test-results/${promptId}`, {
-          status: 'failed'
+          status: 'failed',
+          hallucinationFindings: []
         });
         logger.debug(`Stored minimal failed result for prompt ${promptId}`);
       } catch (fallbackError) {

package/src/components/Header.tsx

@@ -5,6 +5,8 @@ export const Header: React.FC = () => {
   return (
     <Box flexDirection="column" marginBottom={2}>
       <Text bold color="#eba1b5">Rippletide Evaluation</Text>
+      <Text bold color="#eba1b5">How It Works</Text>
+      <Text dimColor>1. Connect your endpoint  2. Add your knowledge source  3. Run the evaluation</Text>
       <Text color="gray">━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━</Text>
     </Box>
   );

package/src/index.tsx

@@ -33,7 +33,7 @@ const parseArgs = async () => {
   
   const options: any = {
     backendUrl: 'https://agent-evalserver-production.up.railway.app',
-    dashboardUrl: 'https://eval.rippletide.com',
+    dashboardUrl: 'https://app.rippletide.com/eval',
     debug: false,
     nonInteractive: false
   };
@@ -115,7 +115,7 @@ Options:
   -a, --agent <url>           Agent endpoint URL (e.g., localhost:8000)
   -k, --knowledge <source>    Knowledge source: files, pinecone, postgresql, or pdf (default: files)
   -b, --backend-url <url>     Backend API URL (default: https://agent-evalserver-production.up.railway.app)
-  -d, --dashboard-url <url>   Dashboard URL (default: https://eval.rippletide.com)
+  -d, --dashboard-url <url>   Dashboard URL (default: https://app.rippletide.com/eval)
   
   Pinecone options:
   -pu, --pinecone-url <url>   Pinecone database URL