eval-vla 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,94 @@
1
+ Metadata-Version: 2.4
2
+ Name: eval-vla
3
+ Version: 0.1.0
4
+ Summary: Add your description here
5
+ Requires-Python: >=3.13
6
+ Description-Content-Type: text/markdown
7
+ Requires-Dist: clip==0.2.0
8
+ Requires-Dist: ftfy==6.3.1
9
+ Requires-Dist: mediapipe==1.2.2
10
+ Requires-Dist: ollama==0.6.3
11
+ Requires-Dist: pin==4.1.0
12
+ Requires-Dist: regex==2026.9.29
13
+ Requires-Dist: rerun-sdk==0.43.2
14
+ Requires-Dist: timm==1.0.30
15
+ Requires-Dist: trossen-arm==1.11.0
16
+ Requires-Dist: ultralytics==8.4.183
17
+
18
+ # pequi-eval
19
+
20
+ Métricas reprodutíveis para avaliar modelos Vision-Language-Action (VLA).
21
+
22
+ ## Instalação
23
+
24
+ ```bash
25
+ pip install git+https://github.com/<org>/Eval_VLA.git # só numpy
26
+ pip install "pequi-eval[scipy] @ git+https://github.com/<org>/Eval_VLA.git" # + Clopper-Pearson
27
+ pip install -e ".[dev]" # desenvolvimento
28
+ ```
29
+
30
+ ## Uso
31
+
32
+ ```python
33
+ from pequi_eval import SetupConfig, Statistical
34
+
35
+ config = SetupConfig(
36
+ robot_embodiment={"name": "ur5e", "gripper": "robotiq_2f85"},
37
+ camera_configuration={"wrist": {"fps": 30}, "front": {"fps": 30}},
38
+ action_space={"type": "ee_delta", "dim": 7},
39
+ control_frequency=10.0,
40
+ object_set={"cube": "red 5cm"},
41
+ scene_layout={"table_height_m": 0.75},
42
+ reset_procedure="manual, object at marked spot",
43
+ success_criteria="cube inside the bowl",
44
+ )
45
+ ev = Statistical(config, metadata={"model": "openvla-7b", "seed": 0})
46
+
47
+ for episode in range(50):
48
+ ev.start_trial("pick_cube")
49
+ while not done:
50
+ with ev.measure_latency():
51
+ action = policy(obs)
52
+ ...
53
+ ev.end_trial(success=ok)
54
+
55
+ ev.trials_per_task() # {"pick_cube": 50}; avisa se < 50
56
+ ev.success_rate("pick_cube") # rate + IC 95% (method="wilson" | "clopper_pearson")
57
+ ev.completion_time("pick_cube") # mean_s, std_s
58
+ ev.inference_latency() # mean_ms, p95_ms
59
+ ev.report() # todas as métricas, por tarefa e agregadas ("all")
60
+ ev.save("results/run.json") # config + trials brutos + versão/ambiente
61
+ ev = Statistical.load("results/run.json")
62
+ ```
63
+
64
+ Se o tempo for medido por outro sistema, use `ev.record_trial(task, success, duration_s, latencies_ms)`.
65
+
66
+ ## Várias famílias de métricas
67
+
68
+ Toda família herda de `EvalVLA` e pode compartilhar o mesmo registro de trials:
69
+
70
+ ```python
71
+ robust = Robustness.from_eval(ev) # mesma config, mesmos trials
72
+ ```
73
+
74
+ ## Adicionando uma nova família
75
+
76
+ 1. Criar `src/pequi_eval/metrics/<nome>.py`:
77
+
78
+ ```python
79
+ from ..base import EvalVLA, metric
80
+
81
+ class Robustness(EvalVLA):
82
+ @metric
83
+ def minha_metrica(self, task: str | None = None) -> dict:
84
+ trials = self.log.filter(task)
85
+ ...
86
+ ```
87
+
88
+ Não adicione parâmetros obrigatórios ao `__init__`; dados extras por trial vão em
89
+ `record_trial(..., **metadata)` / `start_trial(..., **metadata)`.
90
+ 2. Exportar em `metrics/__init__.py` e `pequi_eval/__init__.py`.
91
+ 3. Testes em `tests/`.
92
+
93
+ Métodos com `@metric` entram automaticamente em `report()`; se aceitarem `task`, são
94
+ calculados por tarefa e agregados.
@@ -0,0 +1,77 @@
1
+ # pequi-eval
2
+
3
+ Métricas reprodutíveis para avaliar modelos Vision-Language-Action (VLA).
4
+
5
+ ## Instalação
6
+
7
+ ```bash
8
+ pip install git+https://github.com/<org>/Eval_VLA.git # só numpy
9
+ pip install "pequi-eval[scipy] @ git+https://github.com/<org>/Eval_VLA.git" # + Clopper-Pearson
10
+ pip install -e ".[dev]" # desenvolvimento
11
+ ```
12
+
13
+ ## Uso
14
+
15
+ ```python
16
+ from pequi_eval import SetupConfig, Statistical
17
+
18
+ config = SetupConfig(
19
+ robot_embodiment={"name": "ur5e", "gripper": "robotiq_2f85"},
20
+ camera_configuration={"wrist": {"fps": 30}, "front": {"fps": 30}},
21
+ action_space={"type": "ee_delta", "dim": 7},
22
+ control_frequency=10.0,
23
+ object_set={"cube": "red 5cm"},
24
+ scene_layout={"table_height_m": 0.75},
25
+ reset_procedure="manual, object at marked spot",
26
+ success_criteria="cube inside the bowl",
27
+ )
28
+ ev = Statistical(config, metadata={"model": "openvla-7b", "seed": 0})
29
+
30
+ for episode in range(50):
31
+ ev.start_trial("pick_cube")
32
+ while not done:
33
+ with ev.measure_latency():
34
+ action = policy(obs)
35
+ ...
36
+ ev.end_trial(success=ok)
37
+
38
+ ev.trials_per_task() # {"pick_cube": 50}; avisa se < 50
39
+ ev.success_rate("pick_cube") # rate + IC 95% (method="wilson" | "clopper_pearson")
40
+ ev.completion_time("pick_cube") # mean_s, std_s
41
+ ev.inference_latency() # mean_ms, p95_ms
42
+ ev.report() # todas as métricas, por tarefa e agregadas ("all")
43
+ ev.save("results/run.json") # config + trials brutos + versão/ambiente
44
+ ev = Statistical.load("results/run.json")
45
+ ```
46
+
47
+ Se o tempo for medido por outro sistema, use `ev.record_trial(task, success, duration_s, latencies_ms)`.
48
+
49
+ ## Várias famílias de métricas
50
+
51
+ Toda família herda de `EvalVLA` e pode compartilhar o mesmo registro de trials:
52
+
53
+ ```python
54
+ robust = Robustness.from_eval(ev) # mesma config, mesmos trials
55
+ ```
56
+
57
+ ## Adicionando uma nova família
58
+
59
+ 1. Criar `src/pequi_eval/metrics/<nome>.py`:
60
+
61
+ ```python
62
+ from ..base import EvalVLA, metric
63
+
64
+ class Robustness(EvalVLA):
65
+ @metric
66
+ def minha_metrica(self, task: str | None = None) -> dict:
67
+ trials = self.log.filter(task)
68
+ ...
69
+ ```
70
+
71
+ Não adicione parâmetros obrigatórios ao `__init__`; dados extras por trial vão em
72
+ `record_trial(..., **metadata)` / `start_trial(..., **metadata)`.
73
+ 2. Exportar em `metrics/__init__.py` e `pequi_eval/__init__.py`.
74
+ 3. Testes em `tests/`.
75
+
76
+ Métodos com `@metric` entram automaticamente em `report()`; se aceitarem `task`, são
77
+ calculados por tarefa e agregados.
@@ -0,0 +1,18 @@
1
+ [project]
2
+ name = "eval-vla"
3
+ version = "0.1.0"
4
+ description = "Add your description here"
5
+ readme = "README.md"
6
+ requires-python = ">=3.13"
7
+ dependencies = [
8
+ "clip==0.2.0",
9
+ "ftfy==6.3.1",
10
+ "mediapipe==1.2.2",
11
+ "ollama==0.6.3",
12
+ "pin==4.1.0",
13
+ "regex==2026.9.29",
14
+ "rerun-sdk==0.43.2",
15
+ "timm==1.0.30",
16
+ "trossen-arm==1.11.0",
17
+ "ultralytics==8.4.183",
18
+ ]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,94 @@
1
+ Metadata-Version: 2.4
2
+ Name: eval-vla
3
+ Version: 0.1.0
4
+ Summary: Add your description here
5
+ Requires-Python: >=3.13
6
+ Description-Content-Type: text/markdown
7
+ Requires-Dist: clip==0.2.0
8
+ Requires-Dist: ftfy==6.3.1
9
+ Requires-Dist: mediapipe==1.2.2
10
+ Requires-Dist: ollama==0.6.3
11
+ Requires-Dist: pin==4.1.0
12
+ Requires-Dist: regex==2026.9.29
13
+ Requires-Dist: rerun-sdk==0.43.2
14
+ Requires-Dist: timm==1.0.30
15
+ Requires-Dist: trossen-arm==1.11.0
16
+ Requires-Dist: ultralytics==8.4.183
17
+
18
+ # pequi-eval
19
+
20
+ Métricas reprodutíveis para avaliar modelos Vision-Language-Action (VLA).
21
+
22
+ ## Instalação
23
+
24
+ ```bash
25
+ pip install git+https://github.com/<org>/Eval_VLA.git # só numpy
26
+ pip install "pequi-eval[scipy] @ git+https://github.com/<org>/Eval_VLA.git" # + Clopper-Pearson
27
+ pip install -e ".[dev]" # desenvolvimento
28
+ ```
29
+
30
+ ## Uso
31
+
32
+ ```python
33
+ from pequi_eval import SetupConfig, Statistical
34
+
35
+ config = SetupConfig(
36
+ robot_embodiment={"name": "ur5e", "gripper": "robotiq_2f85"},
37
+ camera_configuration={"wrist": {"fps": 30}, "front": {"fps": 30}},
38
+ action_space={"type": "ee_delta", "dim": 7},
39
+ control_frequency=10.0,
40
+ object_set={"cube": "red 5cm"},
41
+ scene_layout={"table_height_m": 0.75},
42
+ reset_procedure="manual, object at marked spot",
43
+ success_criteria="cube inside the bowl",
44
+ )
45
+ ev = Statistical(config, metadata={"model": "openvla-7b", "seed": 0})
46
+
47
+ for episode in range(50):
48
+ ev.start_trial("pick_cube")
49
+ while not done:
50
+ with ev.measure_latency():
51
+ action = policy(obs)
52
+ ...
53
+ ev.end_trial(success=ok)
54
+
55
+ ev.trials_per_task() # {"pick_cube": 50}; avisa se < 50
56
+ ev.success_rate("pick_cube") # rate + IC 95% (method="wilson" | "clopper_pearson")
57
+ ev.completion_time("pick_cube") # mean_s, std_s
58
+ ev.inference_latency() # mean_ms, p95_ms
59
+ ev.report() # todas as métricas, por tarefa e agregadas ("all")
60
+ ev.save("results/run.json") # config + trials brutos + versão/ambiente
61
+ ev = Statistical.load("results/run.json")
62
+ ```
63
+
64
+ Se o tempo for medido por outro sistema, use `ev.record_trial(task, success, duration_s, latencies_ms)`.
65
+
66
+ ## Várias famílias de métricas
67
+
68
+ Toda família herda de `EvalVLA` e pode compartilhar o mesmo registro de trials:
69
+
70
+ ```python
71
+ robust = Robustness.from_eval(ev) # mesma config, mesmos trials
72
+ ```
73
+
74
+ ## Adicionando uma nova família
75
+
76
+ 1. Criar `src/pequi_eval/metrics/<nome>.py`:
77
+
78
+ ```python
79
+ from ..base import EvalVLA, metric
80
+
81
+ class Robustness(EvalVLA):
82
+ @metric
83
+ def minha_metrica(self, task: str | None = None) -> dict:
84
+ trials = self.log.filter(task)
85
+ ...
86
+ ```
87
+
88
+ Não adicione parâmetros obrigatórios ao `__init__`; dados extras por trial vão em
89
+ `record_trial(..., **metadata)` / `start_trial(..., **metadata)`.
90
+ 2. Exportar em `metrics/__init__.py` e `pequi_eval/__init__.py`.
91
+ 3. Testes em `tests/`.
92
+
93
+ Métodos com `@metric` entram automaticamente em `report()`; se aceitarem `task`, são
94
+ calculados por tarefa e agregados.
@@ -0,0 +1,21 @@
1
+ README.md
2
+ pyproject.toml
3
+ src/eval_vla.egg-info/PKG-INFO
4
+ src/eval_vla.egg-info/SOURCES.txt
5
+ src/eval_vla.egg-info/dependency_links.txt
6
+ src/eval_vla.egg-info/requires.txt
7
+ src/eval_vla.egg-info/top_level.txt
8
+ src/vla_eval/base.py
9
+ src/vla_eval/config.py
10
+ src/vla_eval/plots.py
11
+ src/vla_eval/metrics/failure_analysis.py
12
+ src/vla_eval/metrics/long_horizon.py
13
+ src/vla_eval/metrics/operation.py
14
+ src/vla_eval/metrics/statistical.py
15
+ src/vla_eval/success_detection/background_subtraction.py
16
+ src/vla_eval/success_detection/gripper_tracking.py
17
+ src/vla_eval/success_detection/sam_inference.py
18
+ src/vla_eval/success_detection/vlm_failure.py
19
+ src/vla_eval/vision/camera.py
20
+ src/vla_eval/vision/roi_selector.py
21
+ src/vla_eval/vision/viewer.py
@@ -0,0 +1,10 @@
1
+ clip==0.2.0
2
+ ftfy==6.3.1
3
+ mediapipe==1.2.2
4
+ ollama==0.6.3
5
+ pin==4.1.0
6
+ regex==2026.9.29
7
+ rerun-sdk==0.43.2
8
+ timm==1.0.30
9
+ trossen-arm==1.11.0
10
+ ultralytics==8.4.183
@@ -0,0 +1 @@
1
+ vla_eval
@@ -0,0 +1,77 @@
1
+ from .config import SetupConfig
2
+ from pathlib import Path
3
+ import json
4
+ import functools
5
+ import dataclasses
6
+ import csv
7
+ from datetime import datetime
8
+
9
+ def metric(fn):
10
+ """Marks metódo from base class as a métric, and stores its result to use in the report"""
11
+ @functools.wraps(fn)
12
+ def wrapper(self, *args, **kwargs):
13
+ result = fn(self, *args, **kwargs)
14
+ self._results[fn.__name__] = result
15
+ return result
16
+
17
+ wrapper.__metric__ = True
18
+ return wrapper
19
+
20
+ class EvalVLA:
21
+ """
22
+ Base classe for VLA Evaluation, which holds the default variables to be used.
23
+ """
24
+
25
+ def __init__(self, config: SetupConfig, output_dir: str | Path = "eval_results",
26
+ run_dir: str | Path | None = None, name: str | None = None) -> None:
27
+ """`run_dir` makes several families share one folder; `name` then prefixes each one's results/CSV files."""
28
+ self.config = config
29
+ self.output_dir = Path(run_dir) if run_dir else Path(output_dir) / f"{config.experiment_name.replace('/', '-')}_{datetime.now():%d-%m_%H:%M:%S}"
30
+ self.output_dir.mkdir(parents=True, exist_ok=True)
31
+ self.json_path = self.output_dir / (f"{name}_results.json" if name else "results.json")
32
+ self.csv_path = self.output_dir / (f"{name}_episodes.csv" if name else "episodes.csv")
33
+ self._results: dict[str, dict] = {}
34
+ self._episode = 0
35
+
36
+
37
+ def _save_json(self) -> None:
38
+ report = {
39
+ "config": dataclasses.asdict(self.config),
40
+ "episodes_completed": self._episode,
41
+ "updated_at": datetime.now().isoformat(),
42
+ "metrics": self._results,
43
+ }
44
+ tmp = self.json_path.with_suffix(".tmp")
45
+ tmp.write_text(json.dumps(report, indent=2))
46
+ tmp.replace(self.json_path)
47
+
48
+
49
+ def _append_csv(self, extra: dict) -> None:
50
+ row = {"episode": self._episode, **extra}
51
+ for name, result in self._results.items():
52
+ if result:
53
+ row.update(result)
54
+ new_file = not self.csv_path.exists()
55
+
56
+ with self.csv_path.open("a", newline="") as f:
57
+ writer = csv.DictWriter(f, fieldnames=list(row.keys()), extrasaction="ignore")
58
+ if new_file:
59
+ writer.writeheader()
60
+ writer.writerow(row)
61
+
62
+
63
+
64
+ def _save_graphs(self) -> None:
65
+ """Base implementation for graph analisys for each class."""
66
+ pass
67
+
68
+
69
+ def end_episode(self, **extra) -> None:
70
+ """Call once per episode: overwrites the JSON, appends to the CSV and refreshes the graphs."""
71
+ self._episode += 1
72
+ self._save_json()
73
+ self._append_csv(extra)
74
+ try:
75
+ self._save_graphs()
76
+ except Exception as exc:
77
+ print(f"Could not save graphs: {exc!r}")
@@ -0,0 +1,53 @@
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Any, Mapping
5
+
6
+
7
+ @dataclass(frozen=True)
8
+ class SetupConfig:
9
+ """
10
+ Environment and robot parameters that must be reported with every result.
11
+
12
+ Robot embodiment (manufacturer, model, DoF (Degrees of Freedom)).
13
+
14
+ Camera configuration (number, mounting positions, resolution).
15
+
16
+ Action space definition (joint position, joint velocity, end-effector pose, or hybrid).
17
+
18
+ Control frequency (Hz) on the deployed hardware.
19
+
20
+ Object set (number of objects, material categories, whether objects are seen or unseen during training).
21
+
22
+ Scene layout (fixture positions, lighting conditions).
23
+
24
+ Reset procedure (manual, semi-automatic, or automatic; degree of randomization per trial).
25
+
26
+ Success criteria (explicit definition, including tolerance thresholds).
27
+
28
+ Model Name (base_model_name-finetune_name)
29
+
30
+ Experiment Name (task_experiment-date (d/m/year))
31
+
32
+ Task Name (choose one of the tasks defined in "Statistical" class or create a new one)
33
+
34
+ Perturbation type (robustness evaluation): list with the perturbations applied during the trials, any of
35
+ "object", "layout", "camera", "lighting", "background", "initial_state", "sensor_noise".
36
+ None means a nominal (unperturbed) run.
37
+
38
+ """
39
+
40
+ #TODO:Deixar as possíveis variáveis dentro mapping mais rigidas no formato.
41
+
42
+ robot_embodiment: Mapping[str, Any]
43
+ camera_configuration: Mapping[str, Any]
44
+ action_space: Mapping[str, Any]
45
+ control_frequency: float #
46
+ object_set: Mapping[str, Any]
47
+ scene_layout: Mapping[str, Any]
48
+ reset_procedure: str
49
+ success_criteria: str
50
+ model_name: str
51
+ experiment_name: str
52
+ task_name: str
53
+ perturbation_type: list[str] | None = None
@@ -0,0 +1,97 @@
1
+ from __future__ import annotations
2
+
3
+ from enum import Enum
4
+ from pathlib import Path
5
+
6
+ from ..base import EvalVLA, metric
7
+ from ..config import SetupConfig
8
+ from ..plots import plot_failure_analysis
9
+ from ..success_detection.gripper_tracking import track_gripper
10
+ from ..success_detection.vlm_failure import classify_failure
11
+
12
+
13
+ class FailureType(str, Enum):
14
+ PERCEPTION = "perception"
15
+ LANGUAGE_UNDERSTANDING = "language_understanding"
16
+ PLANNING = "planning"
17
+ GRASPING = "grasping"
18
+ CONTROL = "control"
19
+ RECOVERY = "recovery"
20
+ HARDWARE = "hardware"
21
+
22
+
23
+ class FailureAnalysis(EvalVLA):
24
+ """Failure analysis.
25
+
26
+ Perception errors: the policy misidentifies or fails to locate the target object.
27
+
28
+ Language-understanding errors: the executed behavior does not match the instruction semantics.
29
+
30
+ Planning errors: the policy selects an incorrect or suboptimal action sequence.
31
+
32
+ Grasping errors: the end-effector fails to acquire or maintain a stable grasp.
33
+
34
+ Control errors: the low-level controller produces jerky, oscillatory, or divergent motions.
35
+
36
+ Recovery failures: the policy fails to recover after an intermediate error.
37
+
38
+ Hardware or calibration failures: the trial fails due to mechanical, sensor, or calibration issues unrelated to the policy.
39
+ """
40
+
41
+ def __init__(self, config: SetupConfig, video_path: str | Path, output_dir: str | Path = "eval_results",
42
+ window_s: float = 30.0, min_detected_ratio: float = 0.5, vlm_model: str = "qwen3.5:latest",
43
+ vlm_host: str = "http://localhost:11434", vlm_frames: int = 30,
44
+ debug_video: bool = False, vlm_timeout_s: float = 300.0,
45
+ vlm_num_ctx: int = 16384, run_dir: str | Path | None = None, name: str | None = None):
46
+ super().__init__(config, output_dir, run_dir, name)
47
+
48
+ self.vlm_num_ctx = vlm_num_ctx
49
+ self.vlm_timeout_s = vlm_timeout_s
50
+ self.debug_video = debug_video
51
+ self.video_path = video_path
52
+ self.window_s = window_s
53
+ self.failure_counts: dict[FailureType, int] = {t: 0 for t in FailureType}
54
+ self.episodes_evaluated: int = 0
55
+ self.failure_type: FailureType | None = None
56
+ self.vlm_model = vlm_model
57
+ self.vlm_host = vlm_host
58
+ self.vlm_frames = vlm_frames
59
+
60
+ self.main()
61
+
62
+
63
+ def _save_graphs(self) -> None:
64
+ plot_failure_analysis(self.csv_path, self.output_dir, self.config.experiment_name)
65
+
66
+
67
+ def main(self) -> FailureType | None:
68
+ """Asks the VLM which failure type the episode video shows and sets `failure_type`."""
69
+ self.episodes_evaluated += 1
70
+ self.failure_type = None
71
+
72
+ name = self.vlm_failure_type()["vlm_failure_type"]
73
+
74
+ if name is not None:
75
+ self.failure_type = FailureType(name)
76
+ self.failure_counts[self.failure_type] += 1
77
+
78
+ return self.failure_type
79
+
80
+
81
+ @metric
82
+ def vlm_failure_type(self) -> dict[str, str | None]:
83
+ """Samples frames from the last `window_s` seconds of the video and asks the Ollama VLM for the failure type."""
84
+ name, reply = classify_failure(
85
+ self.video_path,
86
+ task=self.config.task_name,
87
+ success_criteria=self.config.success_criteria,
88
+ window_s=self.window_s,
89
+ n_frames=self.vlm_frames,
90
+ model=self.vlm_model,
91
+ host=self.vlm_host,
92
+ timeout_s=self.vlm_timeout_s,
93
+ num_ctx=self.vlm_num_ctx,
94
+ debug_video_path=self.output_dir / f"vlm_frames_ep{self._episode}.mp4" if self.debug_video else None,
95
+ )
96
+ return {"vlm_failure_type": name, "vlm_model": self.vlm_model, "vlm_raw_reply": reply.strip()}
97
+
@@ -0,0 +1,44 @@
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+
5
+ from ..base import EvalVLA, metric
6
+ from ..config import SetupConfig
7
+ from ..plots import plot_long_horizon
8
+
9
+
10
+ class LongHorizon(EvalVLA):
11
+ """Long-horizon tasks (C).
12
+
13
+ Subtask completion rate (fraction of individual subtasks completed, independent of full-sequence success).
14
+
15
+ Average completed sequence length (e.g., CALVIN-style metric).
16
+
17
+ Distribution of failure positions along the subtask sequence (e.g., histogram showing at which step failures concentrate).
18
+ """
19
+
20
+ def __init__(self, config: SetupConfig, output_dir: str | Path = "eval_results",
21
+ run_dir: str | Path | None = None, name: str | None = None):
22
+ super().__init__(config, output_dir, run_dir, name)
23
+ self.completed_lengths: list[int] = []
24
+ self.failure_positions: list[int] = []
25
+
26
+
27
+ def _save_graphs(self) -> None:
28
+ plot_long_horizon(self.csv_path, self.output_dir, self.config.experiment_name)
29
+
30
+
31
+ @metric
32
+ def subtask_completion_rate(self, subtasks_completed: list[bool]) -> dict[str, float]:
33
+ pass
34
+
35
+
36
+ @metric
37
+ def average_completed_sequence_length(self, subtasks_completed: list[bool]) -> dict[str, float]:
38
+ pass
39
+
40
+
41
+ @metric
42
+ def failure_position_distribution(self, subtasks_completed: list[bool]) -> dict[str, float]:
43
+ """Histogram of the subtask index (1-based) at which each trial first failed."""
44
+ pass