quickdistill 0.1.6__py3-none-any.whl → 0.1.7__py3-none-any.whl

quickdistill/__init__.py

@@ -8,7 +8,7 @@ This package provides tools to:
 - Export datasets for model evaluation
 """
 
-__version__ = "0.1.6"
+__version__ = "0.1.7"
 __author__ = "Brett Young"
 __email__ = "bdytx5@umsystem.edu"
 

quickdistill/server.py

@@ -3,7 +3,6 @@ import json
 import openai
 import weave
 import shutil
-import threading
 from flask import Flask, request, jsonify, send_from_directory
 from flask_cors import CORS
 from llmasajudge import LLMAsAJudge
@@ -34,8 +33,25 @@ CORS(app)
 # Progress tracking for long-running operations
 progress_state = {}
 
-# Configuration
-PROJECT = "wandb_inference"
+# Load settings
+SETTINGS_FILE = DATA_DIR / 'settings.json'
+DEFAULT_SETTINGS = {
+    'inference_project': 'wandb_fc/quickstart_playground',
+    'evaluation_project': 'wandb_inference'
+}
+
+def load_settings():
+    if SETTINGS_FILE.exists():
+        with open(SETTINGS_FILE, 'r') as f:
+            return {**DEFAULT_SETTINGS, **json.load(f)}
+    return DEFAULT_SETTINGS.copy()
+
+def save_settings(settings):
+    with open(SETTINGS_FILE, 'w') as f:
+        json.dump(settings, f, indent=2)
+
+settings = load_settings()
+PROJECT = settings['evaluation_project']
 
 weave.init(PROJECT)
 
@@ -46,7 +62,7 @@ def create_client():
         api_key=os.getenv("WANDB_API_KEY"),
         project=PROJECT,
         default_headers={
-            "OpenAI-Project": "wandb_fc/quickstart_playground"  # replace with your team/project
+            "OpenAI-Project": settings['inference_project']
         }
     )
 
@@ -175,8 +191,9 @@ def run_inference_endpoint():
     if not traces:
         return jsonify({'error': 'No traces in export file'}), 400
 
-    # Limit traces to num_examples
+    # Limit traces to num_examples (convert to int if needed)
     if num_examples:
+        num_examples = int(num_examples)
         traces = traces[:num_examples]
 
     output_files = []
@@ -284,6 +301,241 @@ def get_progress(task_id):
     return jsonify({'error': 'Task not found'}), 404
 
 
+@app.route('/settings', methods=['GET'])
+def get_settings():
+    """Get current settings"""
+    return jsonify(settings)
+
+
+@app.route('/settings', methods=['POST'])
+def update_settings():
+    """Update settings"""
+    global settings
+    data = request.json
+    settings.update(data)
+    save_settings(settings)
+    return jsonify({'status': 'success', 'settings': settings})
+
+
+@app.route('/test_judge', methods=['POST'])
+def test_judge():
+    """Test a judge on sample data to see raw inputs/outputs"""
+    data = request.json
+    judge = data.get('judge')
+    weak_model_file = data.get('weak_model_file')
+    num_samples = data.get('num_samples', 5)
+
+    if not judge or not weak_model_file:
+        return jsonify({'error': 'Missing judge or weak_model_file'}), 400
+
+    # Load weak model results
+    model_path = DATA_DIR / weak_model_file
+    with open(model_path, 'r') as f:
+        file_data = json.load(f)
+
+    # Handle both formats
+    if isinstance(file_data, dict) and 'results' in file_data:
+        results = file_data['results']
+    else:
+        results = file_data
+
+    # Limit to num_samples
+    samples_to_test = results[:min(num_samples, len(results))]
+
+    test_results = []
+
+    for example in samples_to_test:
+        # Skip examples with errors
+        if example.get('error') or not example.get('output'):
+            continue
+
+        strong_output = example.get('strong_model_output', '')
+        weak_output = example.get('output', '')
+
+        # Extract question
+        question = ""
+        messages = example.get('messages', [])
+        if messages and len(messages) > 0:
+            question = messages[0].get('content', '')
+
+        # Build the prompt
+        prompt = judge['prompt']
+        if '{question}' in prompt:
+            prompt = prompt.replace('{question}', question or '')
+        if '{strong_output}' in prompt:
+            prompt = prompt.replace('{strong_output}', strong_output or '')
+        if '{weak_output}' in prompt:
+            prompt = prompt.replace('{weak_output}', weak_output or '')
+
+        # Run the judge and capture raw response
+        if judge['type'] == 'llm':
+            return_type = judge.get('returnType', 'scalar')
+
+            # Use a list to capture the raw response (mutable so we can access from closure)
+            captured_raw = []
+
+            def score_parser(response: str):
+                """Parse the judge response based on return type"""
+                # Capture the raw response before any processing
+                captured_raw.append(response)
+
+                response = response.strip()
+
+                # Remove markdown code blocks if present
+                if response.startswith('```'):
+                    # Remove ```json or ``` at start
+                    response = response.split('\n', 1)[1] if '\n' in response else response[3:]
+                    # Remove ``` at end
+                    if response.endswith('```'):
+                        response = response.rsplit('\n', 1)[0] if '\n' in response else response[:-3]
+                    response = response.strip()
+
+                try:
+                    # Parse JSON response
+                    parsed = json.loads(response)
+
+                    if return_type == 'boolean':
+                        # Extract boolean value - return just the bool
+                        val = parsed.get('correct', parsed.get('result', parsed.get('value', False)))
+                        return bool(val)
+                    elif return_type == 'scalar':
+                        # Extract numeric score - return just the number
+                        val = parsed.get('score', parsed.get('scores', parsed.get('value', 0)))
+                        return float(val) if isinstance(val, (int, float)) else 0
+                    else:
+                        # Unsupported return type
+                        print(f"Unsupported return type: {return_type}")
+                        return 0
+                except:
+                    print(f"Failed to parse judge response as JSON: {response}")
+                    if return_type == 'scalar':
+                        return 0
+                    elif return_type == 'boolean':
+                        return False
+                    else:
+                        return 0
+
+            # Use LLMAsAJudge exactly like the evaluation code
+            try:
+                # Initialize LLMAsAJudge with custom prompt
+                judge_instance = LLMAsAJudge(
+                    models=[judge['model']],
+                    use_fully_custom_prompt=True,
+                    output_parser=score_parser,
+                    return_type=return_type if return_type else None
+                )
+
+                # Get judgment
+                result = judge_instance.judge(prompt=prompt)
+
+                # Extract the raw response that was captured
+                raw_text = captured_raw[0] if captured_raw else "No response captured"
+
+                # Extract parsed scores from result
+                if return_type == 'scalar':
+                    score_val = result.get('scores', result.get('correct', 0))
+                    parsed_scores = {'score': score_val}
+                elif return_type == 'boolean':
+                    bool_val = result.get('correct', False)
+                    parsed_scores = {'correct': bool_val}
+                else:
+                    # Unsupported return type - default to scalar
+                    score_val = result.get('scores', result.get('correct', 0))
+                    parsed_scores = {'score': score_val}
+
+            except Exception as e:
+                raw_text = f"Error: {str(e)}"
+                parsed_scores = {'error': str(e)}
+
+            test_results.append({
+                'question': question,
+                'strong_output': strong_output,
+                'weak_output': weak_output,
+                'judge_prompt': prompt,
+                'raw_response': raw_text,
+                'parsed_scores': parsed_scores
+            })
+
+    return jsonify({
+        'status': 'success',
+        'judge_name': judge['name'],
+        'num_samples': len(test_results),
+        'samples': test_results
+    })
+
+
+@app.route('/generate_judge_prompt', methods=['POST'])
+def generate_judge_prompt():
+    """Generate a judge prompt using AI based on sample data"""
+    data = request.json
+    weak_model_file = data.get('weak_model_file')
+    num_samples = data.get('num_samples', 3)
+    model = data.get('model', 'openai/gpt-5')
+    meta_prompt = data.get('meta_prompt')
+
+    if not weak_model_file or not meta_prompt:
+        return jsonify({'error': 'Missing weak_model_file or meta_prompt'}), 400
+
+    # Load weak model results
+    model_path = DATA_DIR / weak_model_file
+    with open(model_path, 'r') as f:
+        file_data = json.load(f)
+
+    # Handle both formats
+    if isinstance(file_data, dict) and 'results' in file_data:
+        results = file_data['results']
+    else:
+        results = file_data
+
+    # Limit to num_samples
+    samples_to_use = results[:min(num_samples, len(results))]
+
+    # Format samples for meta-prompt
+    samples_text = []
+    for i, example in enumerate(samples_to_use):
+        # Skip examples with errors
+        if example.get('error') or not example.get('output'):
+            continue
+
+        strong_output = example.get('strong_model_output', '')
+        weak_output = example.get('output', '')
+
+        # Extract question
+        question = ""
+        messages = example.get('messages', [])
+        if messages and len(messages) > 0:
+            question = messages[0].get('content', '')
+
+        samples_text.append(f"""Sample {i+1}:
+Question: {question}
+Strong Model Output: {strong_output}
+Weak Model Output: {weak_output}
+---""")
+
+    samples_formatted = "\n\n".join(samples_text)
+
+    # Replace {SAMPLES} placeholder in meta-prompt
+    final_prompt = meta_prompt.replace('{SAMPLES}', samples_formatted)
+
+    # Call OpenRouter to generate the prompt
+    try:
+        client = create_openrouter_client()
+        response = client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": final_prompt}]
+        )
+        generated_prompt = response.choices[0].message.content.strip()
+
+        return jsonify({
+            'status': 'success',
+            'generated_prompt': generated_prompt,
+            'num_samples_used': len(samples_text)
+        })
+
+    except Exception as e:
+        return jsonify({'error': f'Failed to generate prompt: {str(e)}'}), 500
+
+
 @app.route('/list_weak_models', methods=['GET'])
 def list_weak_models():
     """List available weak model result files with metadata"""

quickdistill/static/judge_manager.html

@@ -162,6 +162,13 @@
     <div class="container">
         <h1>Judge Manager</h1>
 
+        <!-- Prompt Generator Button -->
+        <div style="margin-bottom: 20px; display: none;">
+            <button onclick="openPromptGenerator()" style="padding: 10px 20px; background: #6a4a7e; color: white; border: none; border-radius: 4px; cursor: pointer; font-size: 14px;">
+                ✨ Generate Judge Prompt with AI
+            </button>
+        </div>
+
         <!-- Create/Edit Judge Section -->
         <div class="section">
             <h2 id="form-title">Create New Judge</h2>
@@ -217,6 +224,73 @@
         </div>
     </div>
 
+    <!-- Prompt Generator Panel -->
+    <div id="prompt-generator-panel" style="display: none; position: fixed; top: 0; left: 0; width: 100%; height: 100%; background: rgba(0,0,0,0.9); z-index: 1000; padding: 40px; overflow-y: auto;">
+        <div style="max-width: 1200px; margin: 0 auto; background: #1a1a1a; border-radius: 8px; padding: 30px; border: 1px solid #3a2a4a;">
+            <h2 style="color: #fff; margin-bottom: 10px;">AI-Powered Judge Prompt Generator</h2>
+            <p style="color: #888; font-size: 13px; margin-bottom: 25px;">
+                Generate specialized judge prompts by showing sample data to an AI model
+            </p>
+
+            <!-- Configuration -->
+            <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 20px; margin-bottom: 20px;">
+                <div>
+                    <label style="display: block; color: #aaa; margin-bottom: 8px; font-size: 14px;">Weak Model Dataset:</label>
+                    <select id="gen-weak-model-select" style="width: 100%; padding: 10px; background: #2a2a2a; color: #fff; border: 1px solid #3a3a3a; border-radius: 4px; font-size: 14px;">
+                        <option value="">Loading weak model files...</option>
+                    </select>
+                </div>
+
+                <div>
+                    <label style="display: block; color: #aaa; margin-bottom: 8px; font-size: 14px;">Number of Samples:</label>
+                    <input type="number" id="gen-num-samples" value="3" min="1" max="10"
+                        style="width: 100%; padding: 10px; background: #2a2a2a; color: #fff; border: 1px solid #3a3a3a; border-radius: 4px; font-size: 14px;">
+                    <div style="color: #666; font-size: 12px; margin-top: 5px;">Max: 10 (for context limits)</div>
+                </div>
+            </div>
+
+            <div style="margin-bottom: 20px;">
+                <label style="display: block; color: #aaa; margin-bottom: 8px; font-size: 14px;">Generation Model:</label>
+                <input type="text" id="gen-model" value="openai/gpt-5"
+                    style="width: 100%; padding: 10px; background: #2a2a2a; color: #fff; border: 1px solid #3a3a3a; border-radius: 4px; font-size: 14px;"
+                    placeholder="e.g., openai/gpt-5, anthropic/claude-3.5-sonnet">
+                <div style="color: #666; font-size: 12px; margin-top: 5px;">OpenRouter model to use for generating the prompt</div>
+            </div>
+
+            <!-- Meta-Prompt -->
+            <div style="margin-bottom: 25px;">
+                <label style="display: block; color: #aaa; margin-bottom: 8px; font-size: 14px;">Meta-Prompt (edit as needed):</label>
+                <textarea id="gen-meta-prompt"
+                    style="width: 100%; min-height: 250px; padding: 10px; background: #2a2a2a; color: #fff; border: 1px solid #3a3a3a; border-radius: 4px; font-size: 13px; font-family: 'Courier New', monospace; resize: vertical;"></textarea>
+                <div style="color: #666; font-size: 12px; margin-top: 5px;">
+                    This prompt will be sent to the generation model along with sample data
+                </div>
+            </div>
+
+            <!-- Actions -->
+            <div style="display: flex; gap: 10px; margin-bottom: 25px;">
+                <button onclick="generatePrompt()" style="padding: 10px 20px; background: #6a4a7e; color: white; border: none; border-radius: 4px; cursor: pointer; font-weight: 500;">
+                    Generate Prompt
+                </button>
+                <button onclick="closePromptGenerator()" style="padding: 10px 20px; background: #5a2a2a; color: white; border: none; border-radius: 4px; cursor: pointer;">
+                    Close
+                </button>
+            </div>
+
+            <!-- Generated Output -->
+            <div id="gen-output-section" style="display: none;">
+                <h3 style="color: #4a9eff; margin-bottom: 15px;">Generated Judge Prompt</h3>
+                <textarea id="gen-output" readonly
+                    style="width: 100%; min-height: 300px; padding: 15px; background: #0f0f0f; color: #4a9eff; border: 1px solid #4a9eff; border-radius: 4px; font-size: 13px; font-family: 'Courier New', monospace; resize: vertical;"></textarea>
+                <div style="margin-top: 10px;">
+                    <button onclick="copyGeneratedPrompt()" style="padding: 8px 16px; background: #2a7c4a; color: white; border: none; border-radius: 4px; cursor: pointer;">
+                        Copy to Clipboard
+                    </button>
+                </div>
+            </div>
+        </div>
+    </div>
+
     <script>
         let judges = [];
         let editingIndex = null;
@@ -489,6 +563,113 @@ Respond in JSON format: {'correct': true} or {'correct': false}`
                 console.log('Not changing prompt - user has edited it');
             }
         });
+
+        // === PROMPT GENERATOR ===
+
+        const DEFAULT_META_PROMPT = `You are an expert at creating evaluation prompts for judging AI model outputs. I'm building a specialized judge prompt to evaluate the quality/similarity of weak model outputs compared to strong reference model outputs.
+
+I will show you some sample data below. Each sample contains:
+- A question/input
+- The strong reference model's output (ground truth)
+- The weak model's output (what we're evaluating)
+
+Your task: Create a specialized, detailed judge prompt that can be used to systematically evaluate the delta/difference between these outputs. The prompt should:
+1. Be specific to the patterns you see in this data
+2. Include clear evaluation criteria
+3. Be written in second-person ("You are...")
+4. Include the placeholders {question}, {strong_output}, and {weak_output}
+5. Specify the exact JSON format to return (either {'score': number} for scalar or {'correct': boolean} for boolean)
+
+Sample Data:
+{SAMPLES}
+
+Based on these samples, create a specialized judge prompt that would effectively evaluate this type of data. Return ONLY the judge prompt text, nothing else.`;
+
+        async function openPromptGenerator() {
+            // Load weak model files
+            try {
+                const response = await fetch('/list_weak_models');
+                const data = await response.json();
+                const select = document.getElementById('gen-weak-model-select');
+
+                if (data.files && data.files.length > 0) {
+                    select.innerHTML = data.files.map(f =>
+                        `<option value="${f.filename}">${f.weak_model || f.filename}</option>`
+                    ).join('');
+                } else {
+                    select.innerHTML = '<option value="">No weak model files available</option>';
+                }
+            } catch (error) {
+                console.error('Error loading weak models:', error);
+            }
+
+            // Set default meta-prompt
+            document.getElementById('gen-meta-prompt').value = DEFAULT_META_PROMPT;
+
+            // Show panel
+            document.getElementById('prompt-generator-panel').style.display = 'block';
+            document.getElementById('gen-output-section').style.display = 'none';
+        }
+
+        function closePromptGenerator() {
+            document.getElementById('prompt-generator-panel').style.display = 'none';
+        }
+
+        async function generatePrompt() {
+            const weakModelFile = document.getElementById('gen-weak-model-select').value;
+            const numSamples = parseInt(document.getElementById('gen-num-samples').value) || 3;
+            const model = document.getElementById('gen-model').value.trim();
+            const metaPrompt = document.getElementById('gen-meta-prompt').value.trim();
+
+            if (!weakModelFile) {
+                alert('Please select a weak model dataset');
+                return;
+            }
+
+            if (!model) {
+                alert('Please enter a generation model');
+                return;
+            }
+
+            if (!metaPrompt) {
+                alert('Please enter a meta-prompt');
+                return;
+            }
+
+            try {
+                const response = await fetch('/generate_judge_prompt', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({
+                        weak_model_file: weakModelFile,
+                        num_samples: numSamples,
+                        model: model,
+                        meta_prompt: metaPrompt
+                    })
+                });
+
+                if (!response.ok) {
+                    throw new Error('Failed to generate prompt');
+                }
+
+                const result = await response.json();
+
+                // Display generated prompt
+                document.getElementById('gen-output').value = result.generated_prompt;
+                document.getElementById('gen-output-section').style.display = 'block';
+
+            } catch (error) {
+                alert('Error generating prompt: ' + error.message);
+                console.error('Generation error:', error);
+            }
+        }
+
+        function copyGeneratedPrompt() {
+            const output = document.getElementById('gen-output');
+            output.select();
+            document.execCommand('copy');
+            alert('Prompt copied to clipboard!');
+        }
     </script>
 </body>
 </html>

quickdistill/static/trace_viewer.html

@@ -316,6 +316,22 @@
                 Manage Judges
             </a>
 
+            <button id="open-test-judge-btn" style="padding: 8px 16px; background: #6a4a7e; color: white; border: none; border-radius: 4px; cursor: pointer;">
+                Test Judges
+            </button>
+
+            <button id="open-settings-btn" style="padding: 8px 16px; background: #5a5a5a; color: white; border: none; border-radius: 4px; cursor: pointer;">
+                Settings
+            </button>
+
+            <div style="margin: 20px 0; padding: 15px; background: #2a1a2a; border-radius: 8px; border: 1px solid #4a2a4a;">
+                <div style="color: #aaa; font-size: 13px; margin-bottom: 10px;">Automatic Workflow:</div>
+                <button id="open-e2e-btn" style="padding: 10px 20px; background: #7a4a9e; color: white; border: none; border-radius: 4px; cursor: pointer; font-weight: 500;">
+                    ⚡ Run End-to-End Test
+                </button>
+                <div style="color: #666; font-size: 11px; margin-top: 8px;">Export → Generate → Evaluate (all in one)</div>
+            </div>
+
             <div class="stats">
                 <div>Total: <span id="total-count">0</span></div>
                 <div>Shown: <span id="shown-count">0</span></div>
@@ -427,6 +443,193 @@
                 </div>
             </div>
         </div>
+
+        <!-- Settings Panel -->
+        <div id="settings-panel" style="display: none; position: fixed; top: 0; left: 0; width: 100%; height: 100%; background: rgba(0,0,0,0.8); z-index: 1000; padding: 40px;">
+            <div style="max-width: 600px; margin: 0 auto; background: #1a1a1a; border-radius: 8px; padding: 30px; border: 1px solid #2a2a2a;">
+                <h2 style="color: #fff; margin-bottom: 20px;">Settings</h2>
+
+                <div style="margin-bottom: 20px;">
+                    <label style="display: block; color: #aaa; margin-bottom: 8px; font-size: 14px;">W&B Inference Project</label>
+                    <input type="text" id="settings-inference-project" placeholder="e.g., wandb_fc/quickstart_playground"
+                        style="width: 100%; padding: 10px; background: #2a2a2a; color: #fff; border: 1px solid #3a3a3a; border-radius: 4px; font-size: 14px;">
+                    <div style="color: #666; font-size: 12px; margin-top: 5px;">Used for running weak model inference</div>
+                </div>
+
+                <div style="margin-bottom: 30px;">
+                    <label style="display: block; color: #aaa; margin-bottom: 8px; font-size: 14px;">W&B Evaluation Project</label>
+                    <input type="text" id="settings-evaluation-project" placeholder="e.g., wandb_inference"
+                        style="width: 100%; padding: 10px; background: #2a2a2a; color: #fff; border: 1px solid #3a3a3a; border-radius: 4px; font-size: 14px;">
+                    <div style="color: #666; font-size: 12px; margin-top: 5px;">Used for logging evaluation results with Weave</div>
+                </div>
+
+                <div style="display: flex; gap: 10px; justify-content: flex-end;">
+                    <button id="close-settings-btn" style="padding: 10px 20px; background: #5a2a2a; color: white; border: none; border-radius: 4px; cursor: pointer;">
+                        Cancel
+                    </button>
+                    <button id="save-settings-btn" style="padding: 10px 20px; background: #2a7c4a; color: white; border: none; border-radius: 4px; cursor: pointer;">
+                        Save Settings
+                    </button>
+                </div>
+            </div>
+        </div>
+
+        <!-- Test Judges Panel -->
+        <div id="test-judge-panel" style="display: none; position: fixed; top: 0; left: 0; width: 100%; height: 100%; background: rgba(0,0,0,0.8); z-index: 1000; padding: 40px; overflow-y: auto;">
+            <div style="max-width: 1000px; margin: 0 auto; background: #1a1a1a; border-radius: 8px; padding: 30px; border: 1px solid #3a2a4a;">
+                <h2 style="color: #fff; margin-bottom: 10px;">Test Judge</h2>
+                <p style="color: #888; font-size: 13px; margin-bottom: 25px;">
+                    Test your judge on sample data to see exactly what inputs/outputs it receives
+                </p>
+
+                <!-- Configuration -->
+                <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 20px; margin-bottom: 25px;">
+                    <div>
+                        <label style="display: block; color: #aaa; margin-bottom: 8px; font-size: 14px;">Select Judge:</label>
+                        <select id="test-judge-select" style="width: 100%; padding: 10px; background: #2a2a2a; color: #fff; border: 1px solid #3a3a3a; border-radius: 4px; font-size: 14px;">
+                            <option value="">Loading judges...</option>
+                        </select>
+                    </div>
+
+                    <div>
+                        <label style="display: block; color: #aaa; margin-bottom: 8px; font-size: 14px;">Weak Model Data:</label>
+                        <select id="test-weak-model-select" style="width: 100%; padding: 10px; background: #2a2a2a; color: #fff; border: 1px solid #3a3a3a; border-radius: 4px; font-size: 14px;">
+                            <option value="">Loading weak model files...</option>
+                        </select>
+                    </div>
+                </div>
+
+                <div style="margin-bottom: 20px;">
+                    <label style="display: block; color: #aaa; margin-bottom: 8px; font-size: 14px;">Number of Samples:</label>
+                    <input type="number" id="test-num-samples" value="5" min="1" max="50"
+                        style="width: 150px; padding: 10px; background: #2a2a2a; color: #fff; border: 1px solid #3a3a3a; border-radius: 4px; font-size: 14px;">
+                    <span style="color: #666; font-size: 12px; margin-left: 10px;">Max: 50</span>
+                </div>
+
+                <!-- Judge Model -->
+                <div style="margin-bottom: 20px;">
+                    <label style="display: block; color: #aaa; margin-bottom: 8px; font-size: 14px;">Judge Model:</label>
+                    <input type="text" id="test-judge-model"
+                        style="width: 100%; padding: 10px; background: #2a2a2a; color: #fff; border: 1px solid #3a3a3a; border-radius: 4px; font-size: 14px;"
+                        placeholder="e.g., gpt-4o, claude-3-5-sonnet-20241022">
+                    <div style="color: #666; font-size: 12px; margin-top: 5px;">
+                        Override the judge's model for this test
+                    </div>
+                </div>
+
+                <!-- Judge Prompt -->
+                <div style="margin-bottom: 30px;">
+                    <label style="display: block; color: #aaa; margin-bottom: 8px; font-size: 14px;">Judge Prompt:</label>
+                    <textarea id="test-judge-prompt"
+                        style="width: 100%; min-height: 200px; padding: 10px; background: #2a2a2a; color: #fff; border: 1px solid #3a3a3a; border-radius: 4px; font-size: 13px; font-family: 'Courier New', monospace; resize: vertical;"
+                        placeholder="Select a judge to load its prompt..."></textarea>
+                    <div style="color: #666; font-size: 12px; margin-top: 5px;">
+                        Edit the prompt and test changes, or save to update the judge permanently
+                    </div>
+                </div>
+
+                <!-- Actions -->
+                <div style="display: flex; gap: 10px; margin-bottom: 30px;">
+                    <button id="run-test-judge-btn" style="padding: 10px 20px; background: #6a4a7e; color: white; border: none; border-radius: 4px; cursor: pointer; font-weight: 500;">
+                        Run Test
+                    </button>
+                    <button id="save-test-judge-prompt-btn" style="padding: 10px 20px; background: #2a7c4a; color: white; border: none; border-radius: 4px; cursor: pointer;">
+                        Save Prompt to Judge
+                    </button>
+                    <button id="close-test-judge-btn" style="padding: 10px 20px; background: #5a2a2a; color: white; border: none; border-radius: 4px; cursor: pointer;">
+                        Close
+                    </button>
+                </div>
+
+                <!-- Results -->
+                <div id="test-judge-results" style="display: none;">
+                    <h3 style="color: #4a9eff; margin-bottom: 15px;">Test Results</h3>
+                    <div id="test-judge-results-content" style="max-height: 600px; overflow-y: auto;">
+                        <!-- Results populated here -->
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <!-- End-to-End Test Panel -->
+        <div id="e2e-panel" style="display: none; position: fixed; top: 0; left: 0; width: 100%; height: 100%; background: rgba(0,0,0,0.8); z-index: 1000; padding: 40px; overflow-y: auto;">
+            <div style="max-width: 800px; margin: 0 auto; background: #1a1a1a; border-radius: 8px; padding: 30px; border: 1px solid #4a2a4a;">
+                <h2 style="color: #fff; margin-bottom: 10px;">⚡ Run End-to-End Test</h2>
+                <p style="color: #888; font-size: 13px; margin-bottom: 25px;">
+                    This will automatically: Export selected traces → Run weak models → Evaluate with judge
+                </p>
+
+                <!-- Weak Model Selection -->
+                <div style="margin-bottom: 25px;">
+                    <h3 style="color: #fff; font-size: 16px; margin-bottom: 15px;">1. Select Weak Models</h3>
+
+                    <div style="margin-bottom: 15px;">
+                        <label style="display: block; color: #aaa; margin-bottom: 8px; font-size: 13px;">W&B Models:</label>
+                        <div id="e2e-wandb-models" style="max-height: 150px; overflow-y: auto; background: #0f0f0f; padding: 10px; border-radius: 4px; border: 1px solid #2a2a2a;">
+                            <!-- Populated dynamically -->
+                        </div>
+                    </div>
+
+                    <div style="margin-bottom: 15px;">
+                        <label style="display: block; color: #aaa; margin-bottom: 8px; font-size: 13px;">OpenRouter Models (optional):</label>
+                        <textarea id="e2e-openrouter-models" placeholder="Enter OpenRouter models (one per line)&#10;e.g.,&#10;meta-llama/llama-3.3-70b-instruct&#10;anthropic/claude-3.5-sonnet"
+                            style="width: 100%; padding: 8px; background: #2a2a2a; color: #fff; border: 1px solid #3a3a3a; border-radius: 4px; font-size: 13px; min-height: 80px; font-family: monospace;"></textarea>
+                        <div style="color: #666; font-size: 11px; margin-top: 5px;">One model per line</div>
+                    </div>
+
+                    <div>
+                        <label style="display: block; color: #aaa; margin-bottom: 8px; font-size: 13px;">Max Examples (optional):</label>
+                        <input type="number" id="e2e-num-examples" placeholder="Leave empty to use all selected traces"
+                            style="width: 200px; padding: 8px; background: #2a2a2a; color: #fff; border: 1px solid #3a3a3a; border-radius: 4px; font-size: 13px;">
+                    </div>
+                </div>
+
+                <!-- Judge Selection -->
+                <div style="margin-bottom: 30px;">
+                    <h3 style="color: #fff; font-size: 16px; margin-bottom: 15px;">2. Select Judge</h3>
+                    <select id="e2e-judge" style="width: 100%; padding: 10px; background: #2a2a2a; color: #fff; border: 1px solid #3a3a3a; border-radius: 4px; font-size: 14px;">
+                        <option value="">Loading judges...</option>
+                    </select>
+                </div>
+
+                <!-- Actions -->
+                <div style="display: flex; gap: 10px; justify-content: flex-end;">
+                    <button id="close-e2e-btn" style="padding: 10px 20px; background: #5a2a2a; color: white; border: none; border-radius: 4px; cursor: pointer;">
+                        Cancel
+                    </button>
+                    <button id="run-e2e-btn" style="padding: 10px 20px; background: #7a4a9e; color: white; border: none; border-radius: 4px; cursor: pointer; font-weight: 500;">
+                        ⚡ Run Test
+                    </button>
+                </div>
+            </div>
+        </div>
+
+        <!-- End-to-End Progress Panel -->
+        <div id="e2e-progress-panel" style="display: none; position: fixed; top: 0; left: 0; width: 100%; height: 100%; background: rgba(0,0,0,0.9); z-index: 1100; padding: 40px; overflow-y: auto;">
+            <div style="max-width: 800px; margin: 0 auto; background: #1a1a1a; border-radius: 8px; padding: 30px; border: 1px solid #4a2a4a;">
+                <h2 style="color: #fff; margin-bottom: 20px;">Running End-to-End Test</h2>
+
+                <!-- Overall Progress -->
+                <div style="margin-bottom: 30px;">
+                    <div style="color: #4a9eff; font-size: 14px; margin-bottom: 10px;" id="e2e-step-label">Step 1/3: Exporting traces...</div>
+                    <div style="width: 100%; height: 8px; background: #2a2a2a; border-radius: 4px; overflow: hidden;">
+                        <div id="e2e-overall-progress" style="height: 100%; background: #7a4a9e; width: 0%; transition: width 0.3s;"></div>
+                    </div>
+                </div>
+
+                <!-- Detailed Progress -->
+                <div id="e2e-progress-text" style="color: #888; font-family: monospace; font-size: 12px; white-space: pre-wrap; background: #0f0f0f; padding: 15px; border-radius: 4px; max-height: 400px; overflow-y: auto;"></div>
+
+                <!-- Results -->
+                <div id="e2e-results" style="display: none; margin-top: 20px;">
+                    <h3 style="color: #4a9eff; margin-bottom: 15px;">✓ Test Complete!</h3>
+                    <div id="e2e-results-content" style="background: #0f0f0f; padding: 15px; border-radius: 4px;"></div>
+                    <button id="close-e2e-progress-btn" style="margin-top: 20px; padding: 10px 20px; background: #2a7c4a; color: white; border: none; border-radius: 4px; cursor: pointer;">
+                        Close
+                    </button>
+                </div>
+            </div>
+        </div>
     </div>
 
     <script>
@@ -1287,6 +1490,517 @@
                 console.error('Delete error:', error);
             }
         }
+
+        // === SETTINGS ===
+
+        // Load and display settings
+        async function loadSettings() {
+            try {
+                const response = await fetch('/settings');
+                const settings = await response.json();
+                document.getElementById('settings-inference-project').value = settings.inference_project || '';
+                document.getElementById('settings-evaluation-project').value = settings.evaluation_project || '';
+            } catch (error) {
+                console.error('Error loading settings:', error);
+            }
+        }
+
+        // Open settings panel
+        document.getElementById('open-settings-btn').addEventListener('click', async () => {
+            await loadSettings();
+            document.getElementById('settings-panel').style.display = 'block';
+        });
+
+        // Close settings panel
+        document.getElementById('close-settings-btn').addEventListener('click', () => {
+            document.getElementById('settings-panel').style.display = 'none';
+        });
+
+        // Save settings
+        document.getElementById('save-settings-btn').addEventListener('click', async () => {
+            const settings = {
+                inference_project: document.getElementById('settings-inference-project').value.trim(),
+                evaluation_project: document.getElementById('settings-evaluation-project').value.trim()
+            };
+
+            if (!settings.inference_project || !settings.evaluation_project) {
+                alert('Both project fields are required');
+                return;
+            }
+
+            try {
+                const response = await fetch('/settings', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify(settings)
+                });
+
+                const result = await response.json();
+                if (result.status === 'success') {
+                    alert('Settings saved! Please restart the server for changes to take effect.');
+                    document.getElementById('settings-panel').style.display = 'none';
+                } else {
+                    alert('Error saving settings');
+                }
+            } catch (error) {
+                alert('Error saving settings: ' + error.message);
+            }
+        });
+
+        // === TEST JUDGES ===
+
+        let testJudgesData = []; // Store judges globally for test panel
+
+        // Open test judge panel
+        document.getElementById('open-test-judge-btn').addEventListener('click', async () => {
+            // Load judges
+            try {
+                const response = await fetch('/list_judges');
+                const data = await response.json();
+                testJudgesData = data.judges || []; // Store globally
+                const judgeSelect = document.getElementById('test-judge-select');
+
+                if (testJudgesData.length > 0) {
+                    judgeSelect.innerHTML = testJudgesData.map((judge, idx) =>
+                        `<option value="${idx}">${judge.name} (${judge.type})</option>`
+                    ).join('');
+
+                    // Load first judge's prompt and model
+                    if (testJudgesData[0]) {
+                        document.getElementById('test-judge-prompt').value = testJudgesData[0].prompt || '';
+                        document.getElementById('test-judge-model').value = testJudgesData[0].model || '';
+                    }
+                } else {
+                    judgeSelect.innerHTML = '<option value="">No judges available</option>';
+                }
+            } catch (error) {
+                console.error('Error loading judges:', error);
+            }
+
+            // Load weak model files
+            try {
+                const response = await fetch('/list_weak_models');
+                const data = await response.json();
+                const weakModelSelect = document.getElementById('test-weak-model-select');
+
+                if (data.files && data.files.length > 0) {
+                    weakModelSelect.innerHTML = data.files.map(f =>
+                        `<option value="${f.filename}">${f.weak_model || f.filename}</option>`
+                    ).join('');
+                } else {
+                    weakModelSelect.innerHTML = '<option value="">No weak model files available</option>';
+                }
+            } catch (error) {
+                console.error('Error loading weak models:', error);
+            }
+
+            document.getElementById('test-judge-panel').style.display = 'block';
+            document.getElementById('test-judge-results').style.display = 'none';
+        });
+
+        // When judge selection changes, update the prompt and model
+        document.getElementById('test-judge-select').addEventListener('change', (e) => {
+            const judgeIndex = parseInt(e.target.value);
+            if (!isNaN(judgeIndex) && testJudgesData[judgeIndex]) {
+                const judge = testJudgesData[judgeIndex];
+                document.getElementById('test-judge-prompt').value = judge.prompt || '';
+                document.getElementById('test-judge-model').value = judge.model || '';
+            }
+        });
+
+        // Close test judge panel
+        document.getElementById('close-test-judge-btn').addEventListener('click', () => {
+            document.getElementById('test-judge-panel').style.display = 'none';
+        });
+
+        // Run test judge
+        document.getElementById('run-test-judge-btn').addEventListener('click', async () => {
+            const judgeIndex = document.getElementById('test-judge-select').value;
+            const weakModelFile = document.getElementById('test-weak-model-select').value;
+            const numSamples = parseInt(document.getElementById('test-num-samples').value) || 5;
+            const editedPrompt = document.getElementById('test-judge-prompt').value;
+            const editedModel = document.getElementById('test-judge-model').value;
+
+            if (!judgeIndex) {
+                alert('Please select a judge');
+                return;
+            }
+
+            if (!weakModelFile) {
+                alert('Please select a weak model file');
+                return;
+            }
+
+            if (!editedPrompt.trim()) {
+                alert('Please enter a judge prompt');
+                return;
+            }
+
+            if (!editedModel.trim()) {
+                alert('Please enter a judge model');
+                return;
+            }
+
+            // Get judge data and override with edited prompt and model
+            const judge = { ...testJudgesData[parseInt(judgeIndex)] };
+            judge.prompt = editedPrompt; // Use the edited prompt from textarea
+            judge.model = editedModel; // Use the edited model from input
+
+            // Call test endpoint
+            try {
+                const response = await fetch('/test_judge', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({
+                        judge: judge,
+                        weak_model_file: weakModelFile,
+                        num_samples: numSamples
+                    })
+                });
+
+                if (!response.ok) {
+                    throw new Error('Failed to test judge');
+                }
+
+                const result = await response.json();
+
+                // Display results
+                const resultsDiv = document.getElementById('test-judge-results-content');
+                resultsDiv.innerHTML = result.samples.map((sample, idx) => `
+                    <div style="margin-bottom: 20px; padding: 20px; background: #0f0f0f; border-radius: 8px; border: 1px solid #2a2a2a;">
+                        <h4 style="color: #4a9eff; margin-bottom: 15px;">Sample ${idx + 1} of ${result.samples.length}</h4>
+
+                        <div style="margin-bottom: 15px;">
+                            <div style="color: #888; font-size: 12px; margin-bottom: 5px;">Question:</div>
+                            <div style="color: #ccc; font-size: 13px; background: #1a1a1a; padding: 10px; border-radius: 4px; max-height: 100px; overflow-y: auto;">${sample.question || 'N/A'}</div>
+                        </div>
+
+                        <div style="margin-bottom: 15px;">
+                            <div style="color: #888; font-size: 12px; margin-bottom: 5px;">Strong Model Output:</div>
+                            <div style="color: #ccc; font-size: 13px; background: #1a1a1a; padding: 10px; border-radius: 4px; max-height: 100px; overflow-y: auto;">${sample.strong_output || 'N/A'}</div>
+                        </div>
+
+                        <div style="margin-bottom: 15px;">
+                            <div style="color: #888; font-size: 12px; margin-bottom: 5px;">Weak Model Output:</div>
+                            <div style="color: #ccc; font-size: 13px; background: #1a1a1a; padding: 10px; border-radius: 4px; max-height: 100px; overflow-y: auto;">${sample.weak_output || 'N/A'}</div>
+                        </div>
+
+                        <div style="margin-bottom: 15px;">
+                            <div style="color: #888; font-size: 12px; margin-bottom: 5px;">Judge Prompt (filled):</div>
+                            <pre style="color: #aaa; font-size: 11px; background: #1a1a1a; padding: 10px; border-radius: 4px; max-height: 150px; overflow-y: auto; white-space: pre-wrap; font-family: monospace;">${sample.judge_prompt}</pre>
+                        </div>
+
+                        <div style="margin-bottom: 15px;">
+                            <div style="color: #888; font-size: 12px; margin-bottom: 5px;">Raw Judge Response:</div>
+                            <pre style="color: #f4d03f; font-size: 11px; background: #1a1a1a; padding: 10px; border-radius: 4px; max-height: 150px; overflow-y: auto; white-space: pre-wrap; font-family: monospace;">${sample.raw_response}</pre>
+                        </div>
+
+                        <div>
+                            <div style="color: #888; font-size: 12px; margin-bottom: 5px;">Parsed Scores:</div>
+                            <div style="color: #4a9eff; font-size: 13px; background: #1a1a1a; padding: 10px; border-radius: 4px; font-family: monospace;">${JSON.stringify(sample.parsed_scores, null, 2)}</div>
+                        </div>
+                    </div>
+                `).join('');
+
+                document.getElementById('test-judge-results').style.display = 'block';
+
+            } catch (error) {
+                alert('Error testing judge: ' + error.message);
+                console.error('Test error:', error);
+            }
+        });
+
+        // Save prompt to judge
+        document.getElementById('save-test-judge-prompt-btn').addEventListener('click', async () => {
+            const judgeIndex = document.getElementById('test-judge-select').value;
+            const editedPrompt = document.getElementById('test-judge-prompt').value;
+
+            if (!judgeIndex) {
+                alert('Please select a judge');
+                return;
+            }
+
+            if (!editedPrompt.trim()) {
+                alert('Please enter a judge prompt');
+                return;
+            }
+
+            // Get judge data and update prompt
+            const judge = { ...testJudgesData[parseInt(judgeIndex)] };
+            judge.prompt = editedPrompt;
+
+            // Confirm with user
+            if (!confirm(`Save this prompt to judge "${judge.name}"? This will permanently update the judge.`)) {
+                return;
+            }
+
+            // Call save endpoint
+            try {
+                const response = await fetch('/save_judge', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({ judge: judge })
+                });
+
+                if (!response.ok) {
+                    throw new Error('Failed to save judge');
+                }
+
+                const result = await response.json();
+
+                // Update local judges data
+                testJudgesData = result.judges || [];
+
+                alert('Judge prompt saved successfully!');
+            } catch (error) {
+                alert('Error saving judge: ' + error.message);
+                console.error('Save error:', error);
+            }
+        });
+
+        // === END-TO-END TEST ===
+
+        // Open E2E panel
+        document.getElementById('open-e2e-btn').addEventListener('click', async () => {
+            if (selectedTraces.size === 0) {
+                alert('Please select at least one trace first!');
+                return;
+            }
+
+            // Populate W&B models
+            const wandbModelsDiv = document.getElementById('e2e-wandb-models');
+            wandbModelsDiv.innerHTML = AVAILABLE_MODELS.map(model => `
+                <label style="display: block; padding: 5px 0; color: #ccc; cursor: pointer;">
+                    <input type="checkbox" class="e2e-model-checkbox" value="${model}" style="margin-right: 8px;">
+                    ${model}
+                </label>
+            `).join('');
+
+            // Load judges
+            try {
+                const response = await fetch('/list_judges');
+                const data = await response.json();
+                const judgeSelect = document.getElementById('e2e-judge');
+
+                if (data.judges && data.judges.length > 0) {
+                    judgeSelect.innerHTML = data.judges.map((judge, idx) =>
+                        `<option value="${idx}">${judge.name} (${judge.type})</option>`
+                    ).join('');
+                } else {
+                    judgeSelect.innerHTML = '<option value="">No judges available - create one first</option>';
+                }
+            } catch (error) {
+                console.error('Error loading judges:', error);
+            }
+
+            document.getElementById('e2e-panel').style.display = 'block';
+        });
+
+        // Close E2E panel
+        document.getElementById('close-e2e-btn').addEventListener('click', () => {
+            document.getElementById('e2e-panel').style.display = 'none';
+        });
+
+        // Close E2E progress
+        document.getElementById('close-e2e-progress-btn').addEventListener('click', () => {
+            document.getElementById('e2e-progress-panel').style.display = 'none';
+            document.getElementById('e2e-results').style.display = 'none';
+        });
+
+        // Run end-to-end test
+        document.getElementById('run-e2e-btn').addEventListener('click', async () => {
+            // Gather selected models
+            const selectedWanbModels = Array.from(document.querySelectorAll('.e2e-model-checkbox:checked')).map(cb => cb.value);
+            const openRouterModelsText = document.getElementById('e2e-openrouter-models').value.trim();
+            const openRouterModels = openRouterModelsText
+                .split('\n')
+                .map(m => m.trim())
+                .filter(m => m.length > 0);
+            const allModels = [...selectedWanbModels, ...openRouterModels];
+
+            if (allModels.length === 0) {
+                alert('Please select at least one model!');
+                return;
+            }
+
+            const judgeIndex = document.getElementById('e2e-judge').value;
+            if (!judgeIndex) {
+                alert('Please select a judge!');
+                return;
+            }
+
+            const numExamples = document.getElementById('e2e-num-examples').value ? parseInt(document.getElementById('e2e-num-examples').value) : null;
+
+            // Load judge data
+            const judgesResponse = await fetch('/list_judges');
+            const judgesData = await judgesResponse.json();
+            const judge = judgesData.judges[parseInt(judgeIndex)];
+
+            // Hide config panel, show progress panel
+            document.getElementById('e2e-panel').style.display = 'none';
+            document.getElementById('e2e-progress-panel').style.display = 'block';
+
+            const progressText = document.getElementById('e2e-progress-text');
+            const stepLabel = document.getElementById('e2e-step-label');
+            const overallProgress = document.getElementById('e2e-overall-progress');
+
+            progressText.textContent = '';
+
+            try {
+                // === STEP 1: Export Selected Traces ===
+                stepLabel.textContent = 'Step 1/3: Exporting selected traces...';
+                overallProgress.style.width = '10%';
+                progressText.textContent += '📦 Exporting selected traces...\n';
+
+                // Get full trace objects for selected IDs
+                const selectedTraceObjects = allTraces.filter(t => selectedTraces.has(t.id));
+
+                const exportResponse = await fetch('/export_strong_traces', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({
+                        traces: selectedTraceObjects,
+                        nickname: `e2e_export_${Date.now()}`
+                    })
+                });
+
+                if (!exportResponse.ok) {
+                    throw new Error('Failed to export traces');
+                }
+
+                const exportResult = await exportResponse.json();
+                const exportFilename = exportResult.filename;
+                progressText.textContent += `✓ Exported ${exportResult.count} traces to ${exportFilename}\n\n`;
+                overallProgress.style.width = '20%';
+
+                // === STEP 2: Run Weak Model Inference ===
+                stepLabel.textContent = 'Step 2/3: Running weak model inference...';
+                progressText.textContent += `⚙️  Running inference with ${allModels.length} model(s)...\n`;
+
+                const taskId = `inference_${Date.now()}`;
+                let pollInterval = null;
+
+                const pollProgress = async () => {
+                    try {
+                        const resp = await fetch(`/progress/${taskId}`);
+                        if (resp.ok) {
+                            const progress = await resp.json();
+                            const percent = (progress.current / progress.total) * 100;
+                            // Map inference progress to 20-60% of overall
+                            const overallPercent = 20 + (percent * 0.4);
+                            overallProgress.style.width = `${overallPercent}%`;
+                        }
+                    } catch (e) {
+                        console.error('Error polling progress:', e);
+                    }
+                };
+
+                pollInterval = setInterval(pollProgress, 300);
+
+                const inferenceResponse = await fetch('/run_inference', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({
+                        models: allModels,
+                        strong_export_file: exportFilename,
+                        num_examples: numExamples,
+                        task_id: taskId
+                    })
+                });
+
+                if (pollInterval) clearInterval(pollInterval);
+
+                if (!inferenceResponse.ok) {
+                    throw new Error('Failed to run inference');
+                }
+
+                const inferenceResult = await inferenceResponse.json();
+                progressText.textContent += `✓ Generated outputs for ${allModels.length} model(s)\n\n`;
+                overallProgress.style.width = '60%';
+
+                // === STEP 3: Run Evaluations ===
+                stepLabel.textContent = 'Step 3/3: Running evaluations...';
+                progressText.textContent += `📊 Running evaluations with judge: ${judge.name}...\n`;
+
+                const evaluationResults = [];
+
+                // Get list of weak model files that were just generated
+                const weakModelsResponse = await fetch('/list_weak_models');
+                const weakModelsData = await weakModelsResponse.json();
+
+                // Filter to only the models we just ran
+                const weakModelFiles = weakModelsData.files
+                    .filter(f => allModels.some(m => f.filename.includes(m.replace('/', '_'))))
+                    .map(f => f.filename);
+
+                for (let i = 0; i < weakModelFiles.length; i++) {
+                    const modelFile = weakModelFiles[i];
+                    const evalTaskId = `eval_${Date.now()}_${i}`;
+
+                    progressText.textContent += `\n[${i+1}/${weakModelFiles.length}] Evaluating ${modelFile}...\n`;
+
+                    let evalPollInterval = null;
+                    const pollEvalProgress = async () => {
+                        try {
+                            const resp = await fetch(`/progress/${evalTaskId}`);
+                            if (resp.ok) {
+                                const progress = await resp.json();
+                                const percent = (progress.current / progress.total) * 100;
+                                // Map eval progress to 60-100% of overall
+                                const basePercent = 60 + (i / weakModelFiles.length) * 40;
+                                const stepPercent = (percent / 100) * (40 / weakModelFiles.length);
+                                overallProgress.style.width = `${basePercent + stepPercent}%`;
+                            }
+                        } catch (e) {
+                            console.error('Error polling eval progress:', e);
+                        }
+                    };
+
+                    evalPollInterval = setInterval(pollEvalProgress, 300);
+
+                    const evalResponse = await fetch('/run_evaluation', {
+                        method: 'POST',
+                        headers: { 'Content-Type': 'application/json' },
+                        body: JSON.stringify({
+                            model_file: modelFile,
+                            judge: judge,
+                            task_id: evalTaskId
+                        })
+                    });
+
+                    if (evalPollInterval) clearInterval(evalPollInterval);
+
+                    if (evalResponse.ok) {
+                        const evalResult = await evalResponse.json();
+                        progressText.textContent += `  ✓ Complete: ${evalResult.examples_evaluated} examples evaluated\n`;
+                        evaluationResults.push(evalResult);
+                    } else {
+                        progressText.textContent += `  ✗ Error evaluating ${modelFile}\n`;
+                    }
+                }
+
+                overallProgress.style.width = '100%';
+                stepLabel.textContent = 'Complete!';
+                progressText.textContent += `\n✅ All evaluations complete!\n`;
+
+                // Show results
+                document.getElementById('e2e-results').style.display = 'block';
+                const resultsContent = document.getElementById('e2e-results-content');
+                resultsContent.innerHTML = evaluationResults.map(r => `
+                    <div style="margin-bottom: 15px; padding: 15px; background: #1a1a1a; border-radius: 4px; border: 1px solid #2a2a2a;">
+                        <div style="font-weight: bold; color: #fff; margin-bottom: 8px;">${r.evaluation_name}</div>
+                        <div style="font-size: 12px; color: #888; margin-bottom: 8px;">
+                            ${r.examples_evaluated} examples evaluated
+                        </div>
+                        <a href="${r.weave_url}" target="_blank" style="color: #4a9eff; font-size: 13px;">View in Weave →</a>
+                    </div>
+                `).join('');
+
+            } catch (error) {
+                progressText.textContent += `\n\n❌ Error: ${error.message}\n`;
+                stepLabel.textContent = 'Error occurred';
+            }
+        });
     </script>
 </body>
 </html>

{quickdistill-0.1.6.dist-info → quickdistill-0.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quickdistill
-Version: 0.1.6
+Version: 0.1.7
 Summary: Fast and easy toolkit for distilling AI models
 Author-email: Brett Young <bdytx5@umsystem.edu>
 License: MIT

quickdistill-0.1.7.dist-info/RECORD

@@ -0,0 +1,17 @@
+quickdistill/__init__.py,sha256=U8mvMbfYKLFegcEA4D-P6AFHvSiHQPXoFn0KKd-xh0A,397
+quickdistill/cli.py,sha256=A8d5GN9NdBS299WyAsJ6-p8ynW3DJnDRHZ-UGH7TXLM,2212
+quickdistill/default_judges.json,sha256=w0TkIniELPPG-Mi3hm7zPW06eq46W1BI_ufWXnkDDDM,1432
+quickdistill/get_traces.py,sha256=mfy9fMiK-CZQN1noZ4DfOwdwP45ntthVDLgh4-u2iNk,4896
+quickdistill/server.py,sha256=0Y0XG-8oYoNZgmo10LPZgtwlHuGqrq0urxE-KabyIvI,36789
+quickdistill/__pycache__/__init__.cpython-310.pyc,sha256=Tbov274p3OjaOuOsQwcW-meATEfkz0mHKmpytksuDJI,603
+quickdistill/__pycache__/cli.cpython-310.pyc,sha256=xtVgJTayQLKS4gE_te7U1Wo8LmkDtPkaa2rnzu8h9fY,2443
+quickdistill/__pycache__/get_traces.cpython-310.pyc,sha256=T7Suxp9vpqYDQJ_3uJvXWemqoLf5tnRC2I0BfHrSiNM,2956
+quickdistill/__pycache__/server.cpython-310.pyc,sha256=_taKWofMtdgfMZzfVsd7PoC4jnuKxEOGzW82YBxqPPc,22051
+quickdistill/default_projects/byyoung3_arena-detailed/traces_data.json,sha256=iz-cBmXBYj0bC3Vn754QTnGuDh6sRvlE_RzSyGXaxbY,15496950
+quickdistill/static/judge_manager.html,sha256=fXteyx_ry4gY166WypBkVGGCqieE88MigqLRLVCKnG8,26887
+quickdistill/static/trace_viewer.html,sha256=kPC4GnxeDPq7jxClRhZBOuS6xmA3RaY-loJDZmKDADE,94426
+quickdistill-0.1.7.dist-info/METADATA,sha256=1pE5fDep0l0kAxhHuT1C_H4CYHIiPLP4n9QraAqI9bM,5084
+quickdistill-0.1.7.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+quickdistill-0.1.7.dist-info/entry_points.txt,sha256=AUUTxnwdD9gRnsOEcTXQTAZIZ_F0aRU7JGstIJ3Xk_o,55
+quickdistill-0.1.7.dist-info/top_level.txt,sha256=ysiMvurJYsE1IhkxmObe-0G8A-GIav40kTh2z6axjxg,13
+quickdistill-0.1.7.dist-info/RECORD,,

quickdistill-0.1.6.dist-info/RECORD

@@ -1,17 +0,0 @@
-quickdistill/__init__.py,sha256=DquS7slegbCcNr33DA4WEy4RnHFUPHbl3tGhOkw8Yzo,397
-quickdistill/cli.py,sha256=A8d5GN9NdBS299WyAsJ6-p8ynW3DJnDRHZ-UGH7TXLM,2212
-quickdistill/default_judges.json,sha256=w0TkIniELPPG-Mi3hm7zPW06eq46W1BI_ufWXnkDDDM,1432
-quickdistill/get_traces.py,sha256=mfy9fMiK-CZQN1noZ4DfOwdwP45ntthVDLgh4-u2iNk,4896
-quickdistill/server.py,sha256=EXifo8rF8wU_5mhX7ZnpYTi3iRus9XL9nuBdR7FFBRg,27761
-quickdistill/__pycache__/__init__.cpython-310.pyc,sha256=jC6GheK56FqSP9ZP_kHookaiqaKcfY82xOlo2Qn8sag,603
-quickdistill/__pycache__/cli.cpython-310.pyc,sha256=xtVgJTayQLKS4gE_te7U1Wo8LmkDtPkaa2rnzu8h9fY,2443
-quickdistill/__pycache__/get_traces.cpython-310.pyc,sha256=T7Suxp9vpqYDQJ_3uJvXWemqoLf5tnRC2I0BfHrSiNM,2956
-quickdistill/__pycache__/server.cpython-310.pyc,sha256=DXP_J3nP4lPDWwB4fiXbRTSTzM-A1ymWqX593-gDMuA,17503
-quickdistill/default_projects/byyoung3_arena-detailed/traces_data.json,sha256=iz-cBmXBYj0bC3Vn754QTnGuDh6sRvlE_RzSyGXaxbY,15496950
-quickdistill/static/judge_manager.html,sha256=YzzMWpNHyVc7Lyu8Cn55hWzAvYJ1WscXEufQLQ4jR18,17410
-quickdistill/static/trace_viewer.html,sha256=MoXxp_FroAbs8PLzFV7qNkxRI-IY3GGkQWDOHnZM_j8,56257
-quickdistill-0.1.6.dist-info/METADATA,sha256=-Ku5g1GGf8jo7rCZb_vcwy5AB968Zv8INGymUPZDNAg,5084
-quickdistill-0.1.6.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-quickdistill-0.1.6.dist-info/entry_points.txt,sha256=AUUTxnwdD9gRnsOEcTXQTAZIZ_F0aRU7JGstIJ3Xk_o,55
-quickdistill-0.1.6.dist-info/top_level.txt,sha256=ysiMvurJYsE1IhkxmObe-0G8A-GIav40kTh2z6axjxg,13
-quickdistill-0.1.6.dist-info/RECORD,,