eval-vla 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eval_vla-0.1.0/PKG-INFO +94 -0
- eval_vla-0.1.0/README.md +77 -0
- eval_vla-0.1.0/pyproject.toml +18 -0
- eval_vla-0.1.0/setup.cfg +4 -0
- eval_vla-0.1.0/src/eval_vla.egg-info/PKG-INFO +94 -0
- eval_vla-0.1.0/src/eval_vla.egg-info/SOURCES.txt +21 -0
- eval_vla-0.1.0/src/eval_vla.egg-info/dependency_links.txt +1 -0
- eval_vla-0.1.0/src/eval_vla.egg-info/requires.txt +10 -0
- eval_vla-0.1.0/src/eval_vla.egg-info/top_level.txt +1 -0
- eval_vla-0.1.0/src/vla_eval/base.py +77 -0
- eval_vla-0.1.0/src/vla_eval/config.py +53 -0
- eval_vla-0.1.0/src/vla_eval/metrics/failure_analysis.py +97 -0
- eval_vla-0.1.0/src/vla_eval/metrics/long_horizon.py +44 -0
- eval_vla-0.1.0/src/vla_eval/metrics/operation.py +207 -0
- eval_vla-0.1.0/src/vla_eval/metrics/statistical.py +110 -0
- eval_vla-0.1.0/src/vla_eval/plots.py +278 -0
- eval_vla-0.1.0/src/vla_eval/success_detection/background_subtraction.py +152 -0
- eval_vla-0.1.0/src/vla_eval/success_detection/gripper_tracking.py +16 -0
- eval_vla-0.1.0/src/vla_eval/success_detection/sam_inference.py +118 -0
- eval_vla-0.1.0/src/vla_eval/success_detection/vlm_failure.py +118 -0
- eval_vla-0.1.0/src/vla_eval/vision/camera.py +206 -0
- eval_vla-0.1.0/src/vla_eval/vision/roi_selector.py +180 -0
- eval_vla-0.1.0/src/vla_eval/vision/viewer.py +68 -0
eval_vla-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: eval-vla
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Add your description here
|
|
5
|
+
Requires-Python: >=3.13
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: clip==0.2.0
|
|
8
|
+
Requires-Dist: ftfy==6.3.1
|
|
9
|
+
Requires-Dist: mediapipe==1.2.2
|
|
10
|
+
Requires-Dist: ollama==0.6.3
|
|
11
|
+
Requires-Dist: pin==4.1.0
|
|
12
|
+
Requires-Dist: regex==2026.9.29
|
|
13
|
+
Requires-Dist: rerun-sdk==0.43.2
|
|
14
|
+
Requires-Dist: timm==1.0.30
|
|
15
|
+
Requires-Dist: trossen-arm==1.11.0
|
|
16
|
+
Requires-Dist: ultralytics==8.4.183
|
|
17
|
+
|
|
18
|
+
# pequi-eval
|
|
19
|
+
|
|
20
|
+
Métricas reprodutíveis para avaliar modelos Vision-Language-Action (VLA).
|
|
21
|
+
|
|
22
|
+
## Instalação
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install git+https://github.com/<org>/Eval_VLA.git # só numpy
|
|
26
|
+
pip install "pequi-eval[scipy] @ git+https://github.com/<org>/Eval_VLA.git" # + Clopper-Pearson
|
|
27
|
+
pip install -e ".[dev]" # desenvolvimento
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Uso
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from pequi_eval import SetupConfig, Statistical
|
|
34
|
+
|
|
35
|
+
config = SetupConfig(
|
|
36
|
+
robot_embodiment={"name": "ur5e", "gripper": "robotiq_2f85"},
|
|
37
|
+
camera_configuration={"wrist": {"fps": 30}, "front": {"fps": 30}},
|
|
38
|
+
action_space={"type": "ee_delta", "dim": 7},
|
|
39
|
+
control_frequency=10.0,
|
|
40
|
+
object_set={"cube": "red 5cm"},
|
|
41
|
+
scene_layout={"table_height_m": 0.75},
|
|
42
|
+
reset_procedure="manual, object at marked spot",
|
|
43
|
+
success_criteria="cube inside the bowl",
|
|
44
|
+
)
|
|
45
|
+
ev = Statistical(config, metadata={"model": "openvla-7b", "seed": 0})
|
|
46
|
+
|
|
47
|
+
for episode in range(50):
|
|
48
|
+
ev.start_trial("pick_cube")
|
|
49
|
+
while not done:
|
|
50
|
+
with ev.measure_latency():
|
|
51
|
+
action = policy(obs)
|
|
52
|
+
...
|
|
53
|
+
ev.end_trial(success=ok)
|
|
54
|
+
|
|
55
|
+
ev.trials_per_task() # {"pick_cube": 50}; avisa se < 50
|
|
56
|
+
ev.success_rate("pick_cube") # rate + IC 95% (method="wilson" | "clopper_pearson")
|
|
57
|
+
ev.completion_time("pick_cube") # mean_s, std_s
|
|
58
|
+
ev.inference_latency() # mean_ms, p95_ms
|
|
59
|
+
ev.report() # todas as métricas, por tarefa e agregadas ("all")
|
|
60
|
+
ev.save("results/run.json") # config + trials brutos + versão/ambiente
|
|
61
|
+
ev = Statistical.load("results/run.json")
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Se o tempo for medido por outro sistema, use `ev.record_trial(task, success, duration_s, latencies_ms)`.
|
|
65
|
+
|
|
66
|
+
## Várias famílias de métricas
|
|
67
|
+
|
|
68
|
+
Toda família herda de `EvalVLA` e pode compartilhar o mesmo registro de trials:
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
robust = Robustness.from_eval(ev) # mesma config, mesmos trials
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Adicionando uma nova família
|
|
75
|
+
|
|
76
|
+
1. Criar `src/pequi_eval/metrics/<nome>.py`:
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from ..base import EvalVLA, metric
|
|
80
|
+
|
|
81
|
+
class Robustness(EvalVLA):
|
|
82
|
+
@metric
|
|
83
|
+
def minha_metrica(self, task: str | None = None) -> dict:
|
|
84
|
+
trials = self.log.filter(task)
|
|
85
|
+
...
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Não adicione parâmetros obrigatórios ao `__init__`; dados extras por trial vão em
|
|
89
|
+
`record_trial(..., **metadata)` / `start_trial(..., **metadata)`.
|
|
90
|
+
2. Exportar em `metrics/__init__.py` e `pequi_eval/__init__.py`.
|
|
91
|
+
3. Testes em `tests/`.
|
|
92
|
+
|
|
93
|
+
Métodos com `@metric` entram automaticamente em `report()`; se aceitarem `task`, são
|
|
94
|
+
calculados por tarefa e agregados.
|
eval_vla-0.1.0/README.md
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# pequi-eval
|
|
2
|
+
|
|
3
|
+
Métricas reprodutíveis para avaliar modelos Vision-Language-Action (VLA).
|
|
4
|
+
|
|
5
|
+
## Instalação
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install git+https://github.com/<org>/Eval_VLA.git # só numpy
|
|
9
|
+
pip install "pequi-eval[scipy] @ git+https://github.com/<org>/Eval_VLA.git" # + Clopper-Pearson
|
|
10
|
+
pip install -e ".[dev]" # desenvolvimento
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Uso
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
from pequi_eval import SetupConfig, Statistical
|
|
17
|
+
|
|
18
|
+
config = SetupConfig(
|
|
19
|
+
robot_embodiment={"name": "ur5e", "gripper": "robotiq_2f85"},
|
|
20
|
+
camera_configuration={"wrist": {"fps": 30}, "front": {"fps": 30}},
|
|
21
|
+
action_space={"type": "ee_delta", "dim": 7},
|
|
22
|
+
control_frequency=10.0,
|
|
23
|
+
object_set={"cube": "red 5cm"},
|
|
24
|
+
scene_layout={"table_height_m": 0.75},
|
|
25
|
+
reset_procedure="manual, object at marked spot",
|
|
26
|
+
success_criteria="cube inside the bowl",
|
|
27
|
+
)
|
|
28
|
+
ev = Statistical(config, metadata={"model": "openvla-7b", "seed": 0})
|
|
29
|
+
|
|
30
|
+
for episode in range(50):
|
|
31
|
+
ev.start_trial("pick_cube")
|
|
32
|
+
while not done:
|
|
33
|
+
with ev.measure_latency():
|
|
34
|
+
action = policy(obs)
|
|
35
|
+
...
|
|
36
|
+
ev.end_trial(success=ok)
|
|
37
|
+
|
|
38
|
+
ev.trials_per_task() # {"pick_cube": 50}; avisa se < 50
|
|
39
|
+
ev.success_rate("pick_cube") # rate + IC 95% (method="wilson" | "clopper_pearson")
|
|
40
|
+
ev.completion_time("pick_cube") # mean_s, std_s
|
|
41
|
+
ev.inference_latency() # mean_ms, p95_ms
|
|
42
|
+
ev.report() # todas as métricas, por tarefa e agregadas ("all")
|
|
43
|
+
ev.save("results/run.json") # config + trials brutos + versão/ambiente
|
|
44
|
+
ev = Statistical.load("results/run.json")
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Se o tempo for medido por outro sistema, use `ev.record_trial(task, success, duration_s, latencies_ms)`.
|
|
48
|
+
|
|
49
|
+
## Várias famílias de métricas
|
|
50
|
+
|
|
51
|
+
Toda família herda de `EvalVLA` e pode compartilhar o mesmo registro de trials:
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
robust = Robustness.from_eval(ev) # mesma config, mesmos trials
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Adicionando uma nova família
|
|
58
|
+
|
|
59
|
+
1. Criar `src/pequi_eval/metrics/<nome>.py`:
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from ..base import EvalVLA, metric
|
|
63
|
+
|
|
64
|
+
class Robustness(EvalVLA):
|
|
65
|
+
@metric
|
|
66
|
+
def minha_metrica(self, task: str | None = None) -> dict:
|
|
67
|
+
trials = self.log.filter(task)
|
|
68
|
+
...
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Não adicione parâmetros obrigatórios ao `__init__`; dados extras por trial vão em
|
|
72
|
+
`record_trial(..., **metadata)` / `start_trial(..., **metadata)`.
|
|
73
|
+
2. Exportar em `metrics/__init__.py` e `pequi_eval/__init__.py`.
|
|
74
|
+
3. Testes em `tests/`.
|
|
75
|
+
|
|
76
|
+
Métodos com `@metric` entram automaticamente em `report()`; se aceitarem `task`, são
|
|
77
|
+
calculados por tarefa e agregados.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "eval-vla"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Add your description here"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.13"
|
|
7
|
+
dependencies = [
|
|
8
|
+
"clip==0.2.0",
|
|
9
|
+
"ftfy==6.3.1",
|
|
10
|
+
"mediapipe==1.2.2",
|
|
11
|
+
"ollama==0.6.3",
|
|
12
|
+
"pin==4.1.0",
|
|
13
|
+
"regex==2026.9.29",
|
|
14
|
+
"rerun-sdk==0.43.2",
|
|
15
|
+
"timm==1.0.30",
|
|
16
|
+
"trossen-arm==1.11.0",
|
|
17
|
+
"ultralytics==8.4.183",
|
|
18
|
+
]
|
eval_vla-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: eval-vla
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Add your description here
|
|
5
|
+
Requires-Python: >=3.13
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: clip==0.2.0
|
|
8
|
+
Requires-Dist: ftfy==6.3.1
|
|
9
|
+
Requires-Dist: mediapipe==1.2.2
|
|
10
|
+
Requires-Dist: ollama==0.6.3
|
|
11
|
+
Requires-Dist: pin==4.1.0
|
|
12
|
+
Requires-Dist: regex==2026.9.29
|
|
13
|
+
Requires-Dist: rerun-sdk==0.43.2
|
|
14
|
+
Requires-Dist: timm==1.0.30
|
|
15
|
+
Requires-Dist: trossen-arm==1.11.0
|
|
16
|
+
Requires-Dist: ultralytics==8.4.183
|
|
17
|
+
|
|
18
|
+
# pequi-eval
|
|
19
|
+
|
|
20
|
+
Métricas reprodutíveis para avaliar modelos Vision-Language-Action (VLA).
|
|
21
|
+
|
|
22
|
+
## Instalação
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install git+https://github.com/<org>/Eval_VLA.git # só numpy
|
|
26
|
+
pip install "pequi-eval[scipy] @ git+https://github.com/<org>/Eval_VLA.git" # + Clopper-Pearson
|
|
27
|
+
pip install -e ".[dev]" # desenvolvimento
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Uso
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from pequi_eval import SetupConfig, Statistical
|
|
34
|
+
|
|
35
|
+
config = SetupConfig(
|
|
36
|
+
robot_embodiment={"name": "ur5e", "gripper": "robotiq_2f85"},
|
|
37
|
+
camera_configuration={"wrist": {"fps": 30}, "front": {"fps": 30}},
|
|
38
|
+
action_space={"type": "ee_delta", "dim": 7},
|
|
39
|
+
control_frequency=10.0,
|
|
40
|
+
object_set={"cube": "red 5cm"},
|
|
41
|
+
scene_layout={"table_height_m": 0.75},
|
|
42
|
+
reset_procedure="manual, object at marked spot",
|
|
43
|
+
success_criteria="cube inside the bowl",
|
|
44
|
+
)
|
|
45
|
+
ev = Statistical(config, metadata={"model": "openvla-7b", "seed": 0})
|
|
46
|
+
|
|
47
|
+
for episode in range(50):
|
|
48
|
+
ev.start_trial("pick_cube")
|
|
49
|
+
while not done:
|
|
50
|
+
with ev.measure_latency():
|
|
51
|
+
action = policy(obs)
|
|
52
|
+
...
|
|
53
|
+
ev.end_trial(success=ok)
|
|
54
|
+
|
|
55
|
+
ev.trials_per_task() # {"pick_cube": 50}; avisa se < 50
|
|
56
|
+
ev.success_rate("pick_cube") # rate + IC 95% (method="wilson" | "clopper_pearson")
|
|
57
|
+
ev.completion_time("pick_cube") # mean_s, std_s
|
|
58
|
+
ev.inference_latency() # mean_ms, p95_ms
|
|
59
|
+
ev.report() # todas as métricas, por tarefa e agregadas ("all")
|
|
60
|
+
ev.save("results/run.json") # config + trials brutos + versão/ambiente
|
|
61
|
+
ev = Statistical.load("results/run.json")
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Se o tempo for medido por outro sistema, use `ev.record_trial(task, success, duration_s, latencies_ms)`.
|
|
65
|
+
|
|
66
|
+
## Várias famílias de métricas
|
|
67
|
+
|
|
68
|
+
Toda família herda de `EvalVLA` e pode compartilhar o mesmo registro de trials:
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
robust = Robustness.from_eval(ev) # mesma config, mesmos trials
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Adicionando uma nova família
|
|
75
|
+
|
|
76
|
+
1. Criar `src/pequi_eval/metrics/<nome>.py`:
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from ..base import EvalVLA, metric
|
|
80
|
+
|
|
81
|
+
class Robustness(EvalVLA):
|
|
82
|
+
@metric
|
|
83
|
+
def minha_metrica(self, task: str | None = None) -> dict:
|
|
84
|
+
trials = self.log.filter(task)
|
|
85
|
+
...
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Não adicione parâmetros obrigatórios ao `__init__`; dados extras por trial vão em
|
|
89
|
+
`record_trial(..., **metadata)` / `start_trial(..., **metadata)`.
|
|
90
|
+
2. Exportar em `metrics/__init__.py` e `pequi_eval/__init__.py`.
|
|
91
|
+
3. Testes em `tests/`.
|
|
92
|
+
|
|
93
|
+
Métodos com `@metric` entram automaticamente em `report()`; se aceitarem `task`, são
|
|
94
|
+
calculados por tarefa e agregados.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
src/eval_vla.egg-info/PKG-INFO
|
|
4
|
+
src/eval_vla.egg-info/SOURCES.txt
|
|
5
|
+
src/eval_vla.egg-info/dependency_links.txt
|
|
6
|
+
src/eval_vla.egg-info/requires.txt
|
|
7
|
+
src/eval_vla.egg-info/top_level.txt
|
|
8
|
+
src/vla_eval/base.py
|
|
9
|
+
src/vla_eval/config.py
|
|
10
|
+
src/vla_eval/plots.py
|
|
11
|
+
src/vla_eval/metrics/failure_analysis.py
|
|
12
|
+
src/vla_eval/metrics/long_horizon.py
|
|
13
|
+
src/vla_eval/metrics/operation.py
|
|
14
|
+
src/vla_eval/metrics/statistical.py
|
|
15
|
+
src/vla_eval/success_detection/background_subtraction.py
|
|
16
|
+
src/vla_eval/success_detection/gripper_tracking.py
|
|
17
|
+
src/vla_eval/success_detection/sam_inference.py
|
|
18
|
+
src/vla_eval/success_detection/vlm_failure.py
|
|
19
|
+
src/vla_eval/vision/camera.py
|
|
20
|
+
src/vla_eval/vision/roi_selector.py
|
|
21
|
+
src/vla_eval/vision/viewer.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
vla_eval
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
from .config import SetupConfig
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
import json
|
|
4
|
+
import functools
|
|
5
|
+
import dataclasses
|
|
6
|
+
import csv
|
|
7
|
+
from datetime import datetime
|
|
8
|
+
|
|
9
|
+
def metric(fn):
|
|
10
|
+
"""Marks metódo from base class as a métric, and stores its result to use in the report"""
|
|
11
|
+
@functools.wraps(fn)
|
|
12
|
+
def wrapper(self, *args, **kwargs):
|
|
13
|
+
result = fn(self, *args, **kwargs)
|
|
14
|
+
self._results[fn.__name__] = result
|
|
15
|
+
return result
|
|
16
|
+
|
|
17
|
+
wrapper.__metric__ = True
|
|
18
|
+
return wrapper
|
|
19
|
+
|
|
20
|
+
class EvalVLA:
|
|
21
|
+
"""
|
|
22
|
+
Base classe for VLA Evaluation, which holds the default variables to be used.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self, config: SetupConfig, output_dir: str | Path = "eval_results",
|
|
26
|
+
run_dir: str | Path | None = None, name: str | None = None) -> None:
|
|
27
|
+
"""`run_dir` makes several families share one folder; `name` then prefixes each one's results/CSV files."""
|
|
28
|
+
self.config = config
|
|
29
|
+
self.output_dir = Path(run_dir) if run_dir else Path(output_dir) / f"{config.experiment_name.replace('/', '-')}_{datetime.now():%d-%m_%H:%M:%S}"
|
|
30
|
+
self.output_dir.mkdir(parents=True, exist_ok=True)
|
|
31
|
+
self.json_path = self.output_dir / (f"{name}_results.json" if name else "results.json")
|
|
32
|
+
self.csv_path = self.output_dir / (f"{name}_episodes.csv" if name else "episodes.csv")
|
|
33
|
+
self._results: dict[str, dict] = {}
|
|
34
|
+
self._episode = 0
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _save_json(self) -> None:
|
|
38
|
+
report = {
|
|
39
|
+
"config": dataclasses.asdict(self.config),
|
|
40
|
+
"episodes_completed": self._episode,
|
|
41
|
+
"updated_at": datetime.now().isoformat(),
|
|
42
|
+
"metrics": self._results,
|
|
43
|
+
}
|
|
44
|
+
tmp = self.json_path.with_suffix(".tmp")
|
|
45
|
+
tmp.write_text(json.dumps(report, indent=2))
|
|
46
|
+
tmp.replace(self.json_path)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _append_csv(self, extra: dict) -> None:
|
|
50
|
+
row = {"episode": self._episode, **extra}
|
|
51
|
+
for name, result in self._results.items():
|
|
52
|
+
if result:
|
|
53
|
+
row.update(result)
|
|
54
|
+
new_file = not self.csv_path.exists()
|
|
55
|
+
|
|
56
|
+
with self.csv_path.open("a", newline="") as f:
|
|
57
|
+
writer = csv.DictWriter(f, fieldnames=list(row.keys()), extrasaction="ignore")
|
|
58
|
+
if new_file:
|
|
59
|
+
writer.writeheader()
|
|
60
|
+
writer.writerow(row)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _save_graphs(self) -> None:
|
|
65
|
+
"""Base implementation for graph analisys for each class."""
|
|
66
|
+
pass
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def end_episode(self, **extra) -> None:
|
|
70
|
+
"""Call once per episode: overwrites the JSON, appends to the CSV and refreshes the graphs."""
|
|
71
|
+
self._episode += 1
|
|
72
|
+
self._save_json()
|
|
73
|
+
self._append_csv(extra)
|
|
74
|
+
try:
|
|
75
|
+
self._save_graphs()
|
|
76
|
+
except Exception as exc:
|
|
77
|
+
print(f"Could not save graphs: {exc!r}")
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any, Mapping
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen=True)
|
|
8
|
+
class SetupConfig:
|
|
9
|
+
"""
|
|
10
|
+
Environment and robot parameters that must be reported with every result.
|
|
11
|
+
|
|
12
|
+
Robot embodiment (manufacturer, model, DoF (Degrees of Freedom)).
|
|
13
|
+
|
|
14
|
+
Camera configuration (number, mounting positions, resolution).
|
|
15
|
+
|
|
16
|
+
Action space definition (joint position, joint velocity, end-effector pose, or hybrid).
|
|
17
|
+
|
|
18
|
+
Control frequency (Hz) on the deployed hardware.
|
|
19
|
+
|
|
20
|
+
Object set (number of objects, material categories, whether objects are seen or unseen during training).
|
|
21
|
+
|
|
22
|
+
Scene layout (fixture positions, lighting conditions).
|
|
23
|
+
|
|
24
|
+
Reset procedure (manual, semi-automatic, or automatic; degree of randomization per trial).
|
|
25
|
+
|
|
26
|
+
Success criteria (explicit definition, including tolerance thresholds).
|
|
27
|
+
|
|
28
|
+
Model Name (base_model_name-finetune_name)
|
|
29
|
+
|
|
30
|
+
Experiment Name (task_experiment-date (d/m/year))
|
|
31
|
+
|
|
32
|
+
Task Name (choose one of the tasks defined in "Statistical" class or create a new one)
|
|
33
|
+
|
|
34
|
+
Perturbation type (robustness evaluation): list with the perturbations applied during the trials, any of
|
|
35
|
+
"object", "layout", "camera", "lighting", "background", "initial_state", "sensor_noise".
|
|
36
|
+
None means a nominal (unperturbed) run.
|
|
37
|
+
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
#TODO:Deixar as possíveis variáveis dentro mapping mais rigidas no formato.
|
|
41
|
+
|
|
42
|
+
robot_embodiment: Mapping[str, Any]
|
|
43
|
+
camera_configuration: Mapping[str, Any]
|
|
44
|
+
action_space: Mapping[str, Any]
|
|
45
|
+
control_frequency: float #
|
|
46
|
+
object_set: Mapping[str, Any]
|
|
47
|
+
scene_layout: Mapping[str, Any]
|
|
48
|
+
reset_procedure: str
|
|
49
|
+
success_criteria: str
|
|
50
|
+
model_name: str
|
|
51
|
+
experiment_name: str
|
|
52
|
+
task_name: str
|
|
53
|
+
perturbation_type: list[str] | None = None
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from enum import Enum
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from ..base import EvalVLA, metric
|
|
7
|
+
from ..config import SetupConfig
|
|
8
|
+
from ..plots import plot_failure_analysis
|
|
9
|
+
from ..success_detection.gripper_tracking import track_gripper
|
|
10
|
+
from ..success_detection.vlm_failure import classify_failure
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class FailureType(str, Enum):
|
|
14
|
+
PERCEPTION = "perception"
|
|
15
|
+
LANGUAGE_UNDERSTANDING = "language_understanding"
|
|
16
|
+
PLANNING = "planning"
|
|
17
|
+
GRASPING = "grasping"
|
|
18
|
+
CONTROL = "control"
|
|
19
|
+
RECOVERY = "recovery"
|
|
20
|
+
HARDWARE = "hardware"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class FailureAnalysis(EvalVLA):
|
|
24
|
+
"""Failure analysis.
|
|
25
|
+
|
|
26
|
+
Perception errors: the policy misidentifies or fails to locate the target object.
|
|
27
|
+
|
|
28
|
+
Language-understanding errors: the executed behavior does not match the instruction semantics.
|
|
29
|
+
|
|
30
|
+
Planning errors: the policy selects an incorrect or suboptimal action sequence.
|
|
31
|
+
|
|
32
|
+
Grasping errors: the end-effector fails to acquire or maintain a stable grasp.
|
|
33
|
+
|
|
34
|
+
Control errors: the low-level controller produces jerky, oscillatory, or divergent motions.
|
|
35
|
+
|
|
36
|
+
Recovery failures: the policy fails to recover after an intermediate error.
|
|
37
|
+
|
|
38
|
+
Hardware or calibration failures: the trial fails due to mechanical, sensor, or calibration issues unrelated to the policy.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(self, config: SetupConfig, video_path: str | Path, output_dir: str | Path = "eval_results",
|
|
42
|
+
window_s: float = 30.0, min_detected_ratio: float = 0.5, vlm_model: str = "qwen3.5:latest",
|
|
43
|
+
vlm_host: str = "http://localhost:11434", vlm_frames: int = 30,
|
|
44
|
+
debug_video: bool = False, vlm_timeout_s: float = 300.0,
|
|
45
|
+
vlm_num_ctx: int = 16384, run_dir: str | Path | None = None, name: str | None = None):
|
|
46
|
+
super().__init__(config, output_dir, run_dir, name)
|
|
47
|
+
|
|
48
|
+
self.vlm_num_ctx = vlm_num_ctx
|
|
49
|
+
self.vlm_timeout_s = vlm_timeout_s
|
|
50
|
+
self.debug_video = debug_video
|
|
51
|
+
self.video_path = video_path
|
|
52
|
+
self.window_s = window_s
|
|
53
|
+
self.failure_counts: dict[FailureType, int] = {t: 0 for t in FailureType}
|
|
54
|
+
self.episodes_evaluated: int = 0
|
|
55
|
+
self.failure_type: FailureType | None = None
|
|
56
|
+
self.vlm_model = vlm_model
|
|
57
|
+
self.vlm_host = vlm_host
|
|
58
|
+
self.vlm_frames = vlm_frames
|
|
59
|
+
|
|
60
|
+
self.main()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _save_graphs(self) -> None:
|
|
64
|
+
plot_failure_analysis(self.csv_path, self.output_dir, self.config.experiment_name)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def main(self) -> FailureType | None:
|
|
68
|
+
"""Asks the VLM which failure type the episode video shows and sets `failure_type`."""
|
|
69
|
+
self.episodes_evaluated += 1
|
|
70
|
+
self.failure_type = None
|
|
71
|
+
|
|
72
|
+
name = self.vlm_failure_type()["vlm_failure_type"]
|
|
73
|
+
|
|
74
|
+
if name is not None:
|
|
75
|
+
self.failure_type = FailureType(name)
|
|
76
|
+
self.failure_counts[self.failure_type] += 1
|
|
77
|
+
|
|
78
|
+
return self.failure_type
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@metric
|
|
82
|
+
def vlm_failure_type(self) -> dict[str, str | None]:
|
|
83
|
+
"""Samples frames from the last `window_s` seconds of the video and asks the Ollama VLM for the failure type."""
|
|
84
|
+
name, reply = classify_failure(
|
|
85
|
+
self.video_path,
|
|
86
|
+
task=self.config.task_name,
|
|
87
|
+
success_criteria=self.config.success_criteria,
|
|
88
|
+
window_s=self.window_s,
|
|
89
|
+
n_frames=self.vlm_frames,
|
|
90
|
+
model=self.vlm_model,
|
|
91
|
+
host=self.vlm_host,
|
|
92
|
+
timeout_s=self.vlm_timeout_s,
|
|
93
|
+
num_ctx=self.vlm_num_ctx,
|
|
94
|
+
debug_video_path=self.output_dir / f"vlm_frames_ep{self._episode}.mp4" if self.debug_video else None,
|
|
95
|
+
)
|
|
96
|
+
return {"vlm_failure_type": name, "vlm_model": self.vlm_model, "vlm_raw_reply": reply.strip()}
|
|
97
|
+
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from ..base import EvalVLA, metric
|
|
6
|
+
from ..config import SetupConfig
|
|
7
|
+
from ..plots import plot_long_horizon
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class LongHorizon(EvalVLA):
|
|
11
|
+
"""Long-horizon tasks (C).
|
|
12
|
+
|
|
13
|
+
Subtask completion rate (fraction of individual subtasks completed, independent of full-sequence success).
|
|
14
|
+
|
|
15
|
+
Average completed sequence length (e.g., CALVIN-style metric).
|
|
16
|
+
|
|
17
|
+
Distribution of failure positions along the subtask sequence (e.g., histogram showing at which step failures concentrate).
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
def __init__(self, config: SetupConfig, output_dir: str | Path = "eval_results",
|
|
21
|
+
run_dir: str | Path | None = None, name: str | None = None):
|
|
22
|
+
super().__init__(config, output_dir, run_dir, name)
|
|
23
|
+
self.completed_lengths: list[int] = []
|
|
24
|
+
self.failure_positions: list[int] = []
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _save_graphs(self) -> None:
|
|
28
|
+
plot_long_horizon(self.csv_path, self.output_dir, self.config.experiment_name)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@metric
|
|
32
|
+
def subtask_completion_rate(self, subtasks_completed: list[bool]) -> dict[str, float]:
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@metric
|
|
37
|
+
def average_completed_sequence_length(self, subtasks_completed: list[bool]) -> dict[str, float]:
|
|
38
|
+
pass
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@metric
|
|
42
|
+
def failure_position_distribution(self, subtasks_completed: list[bool]) -> dict[str, float]:
|
|
43
|
+
"""Histogram of the subtask index (1-based) at which each trial first failed."""
|
|
44
|
+
pass
|