egogym 0.1.0__py3-none-any.whl

egogym/scripts/plot_failure_hardcoded.py

@@ -0,0 +1,195 @@
+
+import numpy as np
+import pandas as pd
+import altair as alt
+
+# Register custom font for export
+alt.themes.register('custom_theme', lambda: {
+    'config': {
+        'title': {'font': 'Produkt'},
+        'axis': {'labelFont': 'Produkt', 'titleFont': 'Produkt'},
+        'legend': {'labelFont': 'Produkt', 'titleFont': 'Produkt'},
+        'mark': {'font': 'Produkt'},
+        'text': {'font': 'Produkt'},
+    }
+})
+alt.themes.enable('custom_theme')
+
+# Hardcoded data from experiments
+CHECKPOINT_DATA = {
+    "CAP-A": {
+        "total": 5000,
+        "outcomes": {
+            "Success": 1001,
+            "Did not lift enough": 677,
+            "Object touched but not grasped": 1295,
+            "Picked wrong object": 337,
+            "Empty Grasp": 1690,
+            "Did not grasp": 0,
+        }
+    },
+    "CAP-B": {
+        "total": 5000,
+        "outcomes": {
+            "Success": 1847,
+            "Did not lift enough": 995,
+            "Object touched but not grasped": 1213,
+            "Picked wrong object": 208,
+            "Empty Grasp": 737,
+            "Did not grasp": 0,
+        }
+    },
+    "CAP-C": {
+        "total": 5000,
+        "outcomes": {
+            "Success": 3137,
+            "Did not lift enough": 263,
+            "Object touched but not grasped": 597,
+            "Picked wrong object": 110,
+            "Empty Grasp": 893,
+            "Did not grasp": 0,
+        }
+    },
+    "CAP-D": {
+        "total": 5000,
+        "outcomes": {
+            "Success": 3994,
+            "Did not lift enough": 139,
+            "Object touched but not grasped": 362,
+            "Picked wrong object": 59,
+            "Empty Grasp": 446,
+            "Did not grasp": 0,
+        }
+    }
+}
+
+
+def plot_failure_modes():
+    checkpoint_order = ["CAP-A", "CAP-B", "CAP-C", "CAP-D"]
+    
+    # Print results to terminal
+    print("\n" + "="*80)
+    print("FAILURE MODE ANALYSIS BY CHECKPOINT")
+    print("="*80)
+    for checkpoint in checkpoint_order:
+        data = CHECKPOINT_DATA[checkpoint]
+        total = data["total"]
+        print(f"\n{checkpoint} (n={total}):")
+        print("-" * 60)
+        for outcome, count in data["outcomes"].items():
+            percentage = (count / total * 100) if total > 0 else 0
+            print(f"  {outcome:40s}: {count:3d} ({percentage:5.1f}%)")
+    print("\n" + "="*80 + "\n")
+    
+    outcome_order = [
+        "Success",
+        "Did not lift enough",
+        "Object touched but not grasped",
+        "Picked wrong object",
+        "Empty Grasp",
+        "Did not grasp",
+    ]
+    
+    colors = {
+        "Success": "#388038",  # Green
+        "Did not lift enough": "#F7D45B",  # Yellow
+        "Object touched but not grasped": "#66ACF7",  # Blue
+        "Picked wrong object": "#F0529C",  # Pink
+        "Empty Grasp": "#9B66BB",  # Purple
+        "Did not grasp": "#870927",  # Dark red
+    }
+    
+    # Calculate average percentage for each outcome to sort by size
+    outcome_totals = {}
+    for checkpoint in checkpoint_order:
+        data = CHECKPOINT_DATA[checkpoint]
+        total = data["total"] or 1
+        for outcome in outcome_order:
+            count = data["outcomes"].get(outcome, 0)
+            percentage = count / total * 100
+            if outcome not in outcome_totals:
+                outcome_totals[outcome] = 0
+            outcome_totals[outcome] += percentage
+    
+    # Sort outcomes by total percentage (largest first), excluding Success
+    failure_modes = [o for o in outcome_order if o != "Success"]
+    sorted_failures = sorted(failure_modes, key=lambda x: outcome_totals.get(x, 0), reverse=True)
+    
+    # Build stacking order: Success at bottom, then failures from largest to smallest going up
+    stacking_order = ["Success"] + sorted_failures
+    
+    chart_data = []
+    for checkpoint in checkpoint_order:
+        data = CHECKPOINT_DATA[checkpoint]
+        total = data["total"] or 1
+        for outcome in stacking_order:
+            count = data["outcomes"].get(outcome, 0)
+            percentage = count / total * 100
+            if percentage > 0:  # Only include non-zero values
+                chart_data.append({
+                    'Policy': checkpoint,
+                    'Outcome': outcome,
+                    'Percentage': percentage,
+                    'Color': colors.get(outcome, "#999999")
+                })
+    
+    df = pd.DataFrame(chart_data)
+    
+    # Get only outcomes that actually appear in the data
+    outcomes_in_data = df['Outcome'].unique().tolist()
+    
+    # Filter stacking_order to only include outcomes present in data
+    filtered_stacking_order = [o for o in stacking_order if o in outcomes_in_data]
+    
+    # Add sort index to control stacking order
+    outcome_to_index = {outcome: i for i, outcome in enumerate(filtered_stacking_order)}
+    df['sort_index'] = df['Outcome'].map(outcome_to_index)
+    
+    # Create color scale only for outcomes in data
+    color_scale = alt.Scale(
+        domain=filtered_stacking_order,
+        range=[colors[o] for o in filtered_stacking_order]
+    )
+    
+    # Create stacked bar chart
+    chart = alt.Chart(df).mark_bar(
+        stroke='white',
+        strokeWidth=1
+    ).encode(
+        x=alt.X('Policy:N', title=None, axis=alt.Axis(labelFontSize=18, labelAngle=0)),
+        y=alt.Y('Percentage:Q', title='Share of Episodes (%)', axis=alt.Axis(labelFontSize=18, titleFontSize=20)).scale(domain=[0, 100]),
+        color=alt.Color('Outcome:N', scale=color_scale, sort=filtered_stacking_order, legend=alt.Legend(
+            title=None,
+            labelFontSize=16,
+            symbolSize=200,
+            orient="bottom",
+            direction="horizontal",
+            labelLimit=0,
+            columns=3
+        )),
+        order=alt.Order('sort_index:Q'),
+        tooltip=['Policy', 'Outcome', alt.Tooltip('Percentage:Q', format='.1f')]
+    ).properties(
+        width=400,
+        height=400,
+        title={
+            'text': '       EgoGym-Pick Failure Modes by Policy',
+            'fontSize': 22,
+            'anchor': 'start',
+            'dx': 60,
+            'dy': -20
+        },
+        padding={"left": 5, "right": 5, "top": 20, "bottom": 40}
+    ).configure_view(
+        strokeWidth=0
+    )
+    
+    # Save chart
+    chart.save("failure_modes_by_checkpoint.html")
+    chart.save("failure_modes_by_checkpoint.pdf", scale_factor=3)
+    chart.save("failure_modes_by_checkpoint.png", scale_factor=3)
+    print("\nPlot saved to: failure_modes_by_checkpoint.html and failure_modes_by_checkpoint.png")
+
+
+if __name__ == "__main__":
+    plot_failure_modes()

egogym/scripts/plot_failure_vlm.py

@@ -0,0 +1,257 @@
+
+import os
+import numpy as np
+import pandas as pd
+import altair as alt
+
+# Register custom font for export
+alt.themes.register('custom_theme', lambda: {
+    'config': {
+        'title': {'font': 'Produkt'},
+        'axis': {'labelFont': 'Produkt', 'titleFont': 'Produkt'},
+        'legend': {'labelFont': 'Produkt', 'titleFont': 'Produkt'},
+        'mark': {'font': 'Produkt'},
+        'text': {'font': 'Produkt'},
+    }
+})
+alt.themes.enable('custom_theme')
+
+BASE_DIR = "logs"
+REWARD_THRESHOLD = 0.03
+PARTIAL_LIFT_THRESHOLD = 0.01
+
+MODELS = {
+    "CAP + Oracle": "rum/oracle",
+    "CAP + Gemini": "rum/gemini",
+    "CAP + Molmo": "rum/molmo",
+    "CAP + Moondream": "rum/moondream",
+}
+
+
+def compute_outcomes_from_csv(csv_path):
+    if not os.path.exists(csv_path):
+        return None, None
+    df = pd.read_csv(csv_path, sep="\t")
+    total_episodes = len(df)
+    
+    # Count successes
+    successes = (df["max_reward"] > REWARD_THRESHOLD).sum()
+    
+    def get_failure_mode(row):
+        if row["max_reward"] > REWARD_THRESHOLD:
+            return "Success"
+        
+        bodies_contacted = str(row.get("grasped_bodies", ""))
+        object_name = str(row.get("object_name", ""))
+        grasping_object = row.get("grasping_object", False)
+        is_grasping = row.get("is_grasping", False)  # Final gripper state only
+        
+        # Extract body names from grasped_bodies list
+        has_target_contact = object_name in bodies_contacted
+        has_gripper_contact = "left" in bodies_contacted or "right" in bodies_contacted
+        has_any_object_contact = "object" in bodies_contacted
+        has_wrong_object_contact = has_any_object_contact and not has_target_contact
+        
+        # Decision tree (most specific to least specific)
+        # Note: is_grasping is only final state, grasping_object tracks if target was ever grasped
+        
+        # 1. Successfully grasped target but didn't lift high enough
+        if grasping_object and row["max_reward"] >= PARTIAL_LIFT_THRESHOLD:
+            return "Did not lift enough"
+        
+        # 2. Grasped target but barely lifted (or dropped immediately)
+        if grasping_object:
+            return "Object touched but not grasped"
+        
+        # 3. Made contact with target but never achieved grasp
+        if has_target_contact:
+            return "Object touched but not grasped"
+        
+        # 4. Grasped wrong object with significant lift
+        if has_wrong_object_contact and row["max_reward"] >= PARTIAL_LIFT_THRESHOLD:
+            return "Picked wrong object"
+        
+        # 5. Contacted wrong object (with or without final grasp state)
+        if has_wrong_object_contact:
+            return "Picked wrong object"
+        
+        # 6. Gripper closed at end or had gripper contact but no object identification
+        if is_grasping or has_gripper_contact:
+            return "Empty Grasp"
+        
+        # 7. Never made meaningful contact with anything
+        return "Did not grasp"
+    
+    df["outcome"] = df.apply(get_failure_mode, axis=1)
+    
+    outcomes = {
+        "Success": 0,
+        "Did not lift enough": 0,
+        "Object touched but not grasped": 0,
+        "Picked wrong object": 0,
+        "Empty Grasp": 0,
+        "Did not grasp": 0,
+    }
+    
+    for _, row in df.iterrows():
+        mode = row["outcome"]
+        if mode in outcomes:
+            outcomes[mode] += 1
+    
+    return outcomes, total_episodes
+
+
+def plot_failure_modes():
+    outcome_data = {}
+    episode_totals = {}
+    
+    # Get data for 5 objects (4 distractors) for each model
+    for model_name, model_folder in MODELS.items():
+        # Try multiple folder name variations and nested evaluation folders
+        possible_paths = [
+            os.path.join(BASE_DIR, model_folder, "5_objects", "log.csv"),
+            os.path.join(BASE_DIR, model_folder, "5_object", "log.csv"),
+        ]
+        
+        csv_path = None
+        for path in possible_paths:
+            if os.path.exists(path):
+                csv_path = path
+                break
+        
+        # Check for nested evaluation folders
+        if csv_path is None:
+            for folder_name in ["5_objects", "5_object"]:
+                folder_path = os.path.join(BASE_DIR, model_folder, folder_name)
+                if os.path.exists(folder_path) and os.path.isdir(folder_path):
+                    for subdir in os.listdir(folder_path):
+                        subdir_path = os.path.join(folder_path, subdir)
+                        if os.path.isdir(subdir_path):
+                            candidate = os.path.join(subdir_path, "log.csv")
+                            if os.path.exists(candidate):
+                                csv_path = candidate
+                                break
+                    if csv_path:
+                        break
+        
+        if csv_path is None:
+            print(f"Warning: Could not find CSV data for {model_name}")
+            continue
+            
+        outcomes, total_episodes = compute_outcomes_from_csv(csv_path)
+        if outcomes is None:
+            print(f"Warning: Could not find CSV data for {model_name}")
+            continue
+        outcome_data[model_name] = outcomes
+        episode_totals[model_name] = total_episodes
+    
+    if not outcome_data:
+        print("No data found!")
+        return
+    
+    model_names = list(outcome_data.keys())
+    
+    # Print results to terminal
+    print("\n" + "="*80)
+    print("WRONG OBJECT SELECTION RATE BY VLM (4 Distractors)")
+    print("="*80)
+    for model in model_names:
+        total = episode_totals.get(model, 0)
+        print(f"\n{model} (n={total}):")
+        print("-" * 60)
+        for outcome, count in outcome_data[model].items():
+            percentage = (count / total * 100) if total > 0 else 0
+            print(f"  {outcome:40s}: {count:3d} ({percentage:5.1f}%)")
+    print("\n" + "="*80 + "\n")
+    
+    # Only show "Picked wrong object" percentage
+    chart_data = []
+    for model in model_names:
+        total = episode_totals.get(model, 0) or 1
+        count = outcome_data[model].get("Picked wrong object", 0)
+        proportion = count / total
+        percentage = proportion * 100
+        
+        # Calculate standard error for binomial proportion
+        se = np.sqrt(proportion * (1 - proportion) / total) * 100
+        
+        chart_data.append({
+            'Model': model,
+            'Percentage': percentage,
+            'SE': se,
+        })
+    
+    df = pd.DataFrame(chart_data)
+    
+    # Sort by percentage (lowest to highest)
+    df = df.sort_values('Percentage')
+    
+    # Create ordered list for x-axis
+    model_order = df['Model'].tolist()
+    
+    # Assign colors from purple family (lightest for best, darkest for worst)
+    purple_shades = ["#D6BAE2", "#9B66BB", "#7B4FA2", "#4B136D"]  # Lightest to darkest
+    color_mapping = {model: purple_shades[i] for i, model in enumerate(model_order)}
+    df['Color'] = df['Model'].map(color_mapping)
+    
+    # Create simple bar chart with gradient colors
+    bars = alt.Chart(df).mark_bar(
+        stroke='white',
+        strokeWidth=1,
+        cornerRadiusTopLeft=8,
+        cornerRadiusTopRight=8
+    ).encode(
+        x=alt.X('Model:N', title=None, sort=model_order, axis=alt.Axis(labelFontSize=18, labelAngle=0)),
+        y=alt.Y('Percentage:Q', title=None, axis=alt.Axis(labelFontSize=18)),
+        color=alt.Color('Model:N', scale=alt.Scale(domain=model_order, range=[color_mapping[m] for m in model_order]), legend=None),
+        tooltip=['Model', alt.Tooltip('Percentage:Q', format='.1f')]
+    )
+    
+    # Add error bars
+    error_bars = alt.Chart(df).mark_errorbar(
+        color='black',
+        thickness=2,
+        ticks=alt.MarkConfig(width=10)
+    ).encode(
+        x=alt.X('Model:N', sort=model_order),
+        y=alt.Y('Percentage:Q'),
+        yError=alt.YError('SE:Q')
+    )
+    
+    # Add percentage labels on top of bars
+    text = alt.Chart(df).mark_text(
+        align='center',
+        baseline='bottom',
+        dy=-15,
+        fontSize=16,
+        fontWeight='bold'
+    ).encode(
+        x=alt.X('Model:N', sort=model_order),
+        y=alt.Y('Percentage:Q'),
+        text=alt.Text('Percentage:Q', format='.1f')
+    )
+    
+    chart = (bars + error_bars + text).properties(
+        width=600,
+        height=400,
+        title={
+            'text': '       Wrong Object Selection Rate by VLM (4 Distractors)',
+            'fontSize': 22,
+            'anchor': 'start',
+            'dx': 60,
+            'dy': -20
+        },
+        padding={"left": 5, "right": 5, "top": 20, "bottom": 0}
+    ).configure_view(
+        strokeWidth=0
+    )
+    
+    # Save chart
+    chart.save("failure_modes_by_model.html")
+    chart.save("failure_modes_by_model.png", scale_factor=3)
+    chart.save("failure_modes_by_model.pdf")
+    print("Saved: failure_modes_by_model.html, .png, .pdf")
+
+
+if __name__ == "__main__":
+    plot_failure_modes()

egogym/scripts/plot_failure_vlm_hardcoded.py

@@ -0,0 +1,177 @@
+
+import numpy as np
+import pandas as pd
+import altair as alt
+
+# Register custom font for export
+alt.themes.register('custom_theme', lambda: {
+    'config': {
+        'title': {'font': 'Produkt'},
+        'axis': {'labelFont': 'Produkt', 'titleFont': 'Produkt'},
+        'legend': {'labelFont': 'Produkt', 'titleFont': 'Produkt'},
+        'mark': {'font': 'Produkt'},
+        'text': {'font': 'Produkt'},
+    }
+})
+alt.themes.enable('custom_theme')
+
+# Hardcoded data from experiments
+MODEL_DATA = {
+    "CAP + Oracle": {
+        "total": 5000,
+        "outcomes": {
+            "Success": 3994,
+            "Did not lift enough": 89,
+            "Object touched but not grasped": 412,
+            "Picked wrong object": 59,
+            "Empty Grasp": 446,
+            "Did not grasp": 0,
+        }
+    },
+    "CAP + Gemini": {
+        "total": 5000,
+        "outcomes": {
+            "Success": 3441,
+            "Did not lift enough": 88,
+            "Object touched but not grasped": 389,
+            "Picked wrong object": 448,
+            "Empty Grasp": 634,
+            "Did not grasp": 0,
+        }
+    },
+    "CAP + Molmo": {
+        "total": 5000,
+        "outcomes": {
+            "Success": 3008,
+            "Did not lift enough": 94,
+            "Object touched but not grasped": 467,
+            "Picked wrong object": 877,
+            "Empty Grasp": 554,
+            "Did not grasp": 0,
+        }
+    },
+    "CAP + Moondream": {
+        "total": 5000,
+        "outcomes": {
+            "Success": 3033,
+            "Did not lift enough": 83,
+            "Object touched but not grasped": 342,
+            "Picked wrong object": 746,
+            "Empty Grasp": 796,
+            "Did not grasp": 0,
+        }
+    }
+}
+
+
+def plot_failure_modes():
+    model_names = list(MODEL_DATA.keys())
+    
+    # Print results to terminal
+    print("\n" + "="*80)
+    print("WRONG OBJECT SELECTION RATE BY VLM (4 Distractors)")
+    print("="*80)
+    for model in model_names:
+        data = MODEL_DATA[model]
+        total = data["total"]
+        print(f"\n{model} (n={total}):")
+        print("-" * 60)
+        for outcome, count in data["outcomes"].items():
+            percentage = (count / total * 100) if total > 0 else 0
+            print(f"  {outcome:40s}: {count:3d} ({percentage:5.1f}%)")
+    print("\n" + "="*80 + "\n")
+    
+    # Only show "Picked wrong object" percentage
+    chart_data = []
+    for model in model_names:
+        data = MODEL_DATA[model]
+        total = data["total"] or 1
+        count = data["outcomes"].get("Picked wrong object", 0)
+        proportion = count / total
+        percentage = proportion * 100
+        
+        # Calculate standard error for binomial proportion
+        se = np.sqrt(proportion * (1 - proportion) / total) * 100
+        
+        chart_data.append({
+            'Model': model,
+            'Percentage': percentage,
+            'SE': se,
+        })
+    
+    df = pd.DataFrame(chart_data)
+    
+    # Sort by percentage (lowest to highest)
+    df = df.sort_values('Percentage')
+    
+    # Create ordered list for x-axis
+    model_order = df['Model'].tolist()
+    
+    # Assign colors from purple family (lightest for best, darkest for worst)
+    purple_shades = ["#D6BAE2", "#9B66BB", "#7B4FA2", "#4B136D"]  # Lightest to darkest
+    color_mapping = {model: purple_shades[i] for i, model in enumerate(model_order)}
+    df['Color'] = df['Model'].map(color_mapping)
+    
+    # Create simple bar chart with gradient colors
+    bars = alt.Chart(df).mark_bar(
+        stroke='white',
+        strokeWidth=1,
+        cornerRadiusTopLeft=8,
+        cornerRadiusTopRight=8
+    ).encode(
+        x=alt.X('Model:N', title=None, sort=model_order, axis=alt.Axis(labelFontSize=18, labelAngle=0)),
+        y=alt.Y('Percentage:Q', title=None, axis=alt.Axis(labelFontSize=18)),
+        color=alt.Color('Model:N', scale=alt.Scale(domain=model_order, range=[color_mapping[m] for m in model_order]), legend=None),
+        tooltip=['Model', alt.Tooltip('Percentage:Q', format='.1f')]
+    )
+    
+    # Add error bars
+    error_bars = alt.Chart(df).mark_errorbar(
+        color='black',
+        thickness=2,
+        ticks=alt.MarkConfig(width=10)
+    ).encode(
+        x=alt.X('Model:N', sort=model_order),
+        y=alt.Y('Percentage:Q'),
+        yError=alt.YError('SE:Q')
+    )
+    
+    # Add percentage labels on top of bars
+    text = alt.Chart(df).mark_text(
+        align='center',
+        baseline='bottom',
+        dy=-15,
+        fontSize=16,
+        fontWeight='bold'
+    ).encode(
+        x=alt.X('Model:N', sort=model_order),
+        y=alt.Y('Percentage:Q'),
+        text=alt.Text('Percentage:Q', format='.0f')
+    ).transform_calculate(
+        label='datum.Percentage + "%"'
+    )
+    
+    chart = (bars + error_bars + text).properties(
+        width=600,
+        height=400,
+        title={
+            'text': '       Wrong Object Selection Rate by VLM (4 Distractors)',
+            'fontSize': 22,
+            'anchor': 'start',
+            'dx': 60,
+            'dy': -20
+        },
+        padding={"left": 5, "right": 5, "top": 20, "bottom": 0}
+    ).configure_view(
+        strokeWidth=0
+    )
+    
+    # Save chart
+    chart.save("failure_modes_by_model.html")
+    chart.save("failure_modes_by_model.png", scale_factor=3)
+    chart.save("failure_modes_by_model.pdf")
+    print("Saved: failure_modes_by_model.html, .png, .pdf")
+
+
+if __name__ == "__main__":
+    plot_failure_modes()