positronic-evals 0.1.0__tar.gz

positronic_evals-0.1.0/.gitignore

@@ -0,0 +1,167 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+.venv-typecheck
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be added to the global gitignore or merged into this project gitignore.  For a PyCharm
+#  project, it is generally recommended to exclude comment-based line additions.
+.idea/
+
+# VSCode
+.vscode/
+*.code-workspace
+
+# Environment variables
+.env*

positronic_evals-0.1.0/PKG-INFO

@@ -0,0 +1,108 @@
+Metadata-Version: 2.5
+Name: positronic-evals
+Version: 0.1.0
+Summary: The public evals of the Positronic platform: their definitions, labelers, scoring rules and episode lists
+License-Expression: Apache-2.0
+Requires-Python: >=3.11
+Requires-Dist: positronic-eval-vocabulary==0.4.0
+Requires-Dist: pyarrow
+Requires-Dist: pydantic>=2.11
+Description-Content-Type: text/markdown
+
+# positronic-evals
+
+The definitions of the public evals, the labelers that read a recorded episode, the rules that score
+a run, and MolmoSpaces' committed episode lists.
+
+`positronic` runs these evals on your machine, `positronic-platform-client` names them on the wire,
+and a service that scores a run reads them. Each of them installs this distribution. It depends on
+`pydantic`, `pyarrow` and `positronic-eval-vocabulary`, and on no robotics stack and no client, so a
+service installs it alone:
+
+```bash
+uv add "positronic-evals @ git+https://github.com/Positronic-Robotics/positronic@<tag or commit>#subdirectory=evals"
+```
+
+`positronic_evals.slug` carries an `IntEnum` on the wire as its lowercase name.
+
+## Public evals
+
+`positronic_evals.evals` carries the definitions of the public evals. A definition pins what the
+platform runs under the eval's name: the positronic eval config and its arguments, the trials, the
+time limit of one run, its scoring, and the positronic commit that the platform runs the config at.
+`public_eval(name)` returns one definition, and `PUBLIC_EVALS` names them all. The platform also
+offers evals that it does not publish, and `public_eval` refuses their names.
+
+`positronic eval run` with `--policy` takes a public eval's name as `--eval`, and runs that eval on
+your machine. The run uses the positronic you installed. It warns when that positronic is not the
+commit the definition names, because the two runs can then differ. A local run does not stop at the
+time limit.
+
+## Labels and scores
+
+`positronic_evals.labels` reads what happened in a recorded episode. A labeler reads one episode
+directory and gives its `Label`: the episode's id and task, its end, and a list of timed events. An
+event is a stage that the arm reached, a count of items delivered, a count of subtasks achieved, or
+one of the three end events: `Success`, `Fail` and `Safety`. The earliest end event gives the
+`outcome`. A label with no end event ran out of time. The label's other views, such as
+`stage_seconds` and `placements`, are properties computed from its events. `Label.until(seconds)`
+gives the label as it stood at that time.
+
+`positronic_evals.scoring` states what a label is worth. An eval's `Scoring` names its labelers in
+order of precedence, and its primary metric. `label(episode, scoring.labelers)` gives a pair: the
+first labeler that labels the episode, and its label. It gives None when no labeler labels the
+episode. `score(scoring, labels)` takes the label of each episode, or None. For the episode
+directories `episodes` of one run:
+
+```python
+from positronic_evals.evals import EvalRef, public_eval
+from positronic_evals.labels import label
+from positronic_evals.scoring import score
+
+scoring = public_eval(EvalRef('molmo.franka_pick_mini')).scoring
+picked = [label(episode, scoring.labelers) for episode in episodes]
+scores = score(scoring, [None if given is None else given[1] for given in picked])
+```
+
+`score` gives `Scores`, which holds three numbers. `primary` is the mean episode score, or the mean
+of the task means where the metric sets `per_task`, and a board ranks on it. `episodes` counts every
+episode, and `unscored` counts the episodes that no labeler labelled. `primary` does not count those
+as failures. No file holds a score: a score is a function of the labels.
+
+`PUBLIC_LABELERS` holds the labelers that this package publishes, `molmo` and `robolab`. A caller with
+labelers of its own passes `label` a table that holds them beside `PUBLIC_LABELERS`. The MolmoSpaces
+labeler takes the end of a trial from the episode's signal files: the time from the first sample to
+the last, on the world clock. That time can exceed the trial by up to one control step.
+
+The RoboLab labeler reads the end of a trial in the same way. Its task is the task id that RoboLab
+names, and not the instruction. It also reads the `subtask` signal, which RoboLab records at each
+step, and gives a `SubtaskAchieved` event, with RoboLab's score, for each change of the subtasks
+completed. A change of the score alone is not an event. A RoboLab eval scores a success 1.0, and any
+other episode the score at its last change of the count, or 0 where it completed no subtask. Where
+RoboLab grades progress within a subtask, this score does not count that progress:
+
+```python
+from positronic_evals.labels import ROBOLAB_LABELER
+from positronic_evals.scoring import EpisodeScore, MeanEpisodeScore, Scoring
+
+robolab = Scoring(
+    labelers=(ROBOLAB_LABELER,), primary=MeanEpisodeScore(episode_score=EpisodeScore(use_subtask_score=True))
+)
+```
+
+## MolmoSpaces episode lists
+
+`positronic_evals.molmo_spaces` holds the episode lists that an offline sweep builds, by benchmark
+path below the asset packs' `benchmarks` directory. `RIGHT_VIEW_BLOCKED_PATH` lists the episodes where
+scene geometry hides the start grasp site from the right shoulder camera. The MolmoSpaces sim server
+leaves those episodes out of an open selection and refuses a pinned one. No public eval pins one.
+
+```python
+from positronic_evals.molmo_spaces import RIGHT_VIEW_BLOCKED_PATH, read_episode_list
+
+blocked = read_episode_list(RIGHT_VIEW_BLOCKED_PATH)
+```
+
+A benchmark that a list does not name is unswept. Rebuild a list when the pinned MolmoSpaces revision
+or the asset packs change. A new list goes beside `right_view_blocked.json`, with its path constant in
+`positronic_evals.molmo_spaces`, and `read_episode_list` reads it.

positronic_evals-0.1.0/README.md

@@ -0,0 +1,97 @@
+# positronic-evals
+
+The definitions of the public evals, the labelers that read a recorded episode, the rules that score
+a run, and MolmoSpaces' committed episode lists.
+
+`positronic` runs these evals on your machine, `positronic-platform-client` names them on the wire,
+and a service that scores a run reads them. Each of them installs this distribution. It depends on
+`pydantic`, `pyarrow` and `positronic-eval-vocabulary`, and on no robotics stack and no client, so a
+service installs it alone:
+
+```bash
+uv add "positronic-evals @ git+https://github.com/Positronic-Robotics/positronic@<tag or commit>#subdirectory=evals"
+```
+
+`positronic_evals.slug` carries an `IntEnum` on the wire as its lowercase name.
+
+## Public evals
+
+`positronic_evals.evals` carries the definitions of the public evals. A definition pins what the
+platform runs under the eval's name: the positronic eval config and its arguments, the trials, the
+time limit of one run, its scoring, and the positronic commit that the platform runs the config at.
+`public_eval(name)` returns one definition, and `PUBLIC_EVALS` names them all. The platform also
+offers evals that it does not publish, and `public_eval` refuses their names.
+
+`positronic eval run` with `--policy` takes a public eval's name as `--eval`, and runs that eval on
+your machine. The run uses the positronic you installed. It warns when that positronic is not the
+commit the definition names, because the two runs can then differ. A local run does not stop at the
+time limit.
+
+## Labels and scores
+
+`positronic_evals.labels` reads what happened in a recorded episode. A labeler reads one episode
+directory and gives its `Label`: the episode's id and task, its end, and a list of timed events. An
+event is a stage that the arm reached, a count of items delivered, a count of subtasks achieved, or
+one of the three end events: `Success`, `Fail` and `Safety`. The earliest end event gives the
+`outcome`. A label with no end event ran out of time. The label's other views, such as
+`stage_seconds` and `placements`, are properties computed from its events. `Label.until(seconds)`
+gives the label as it stood at that time.
+
+`positronic_evals.scoring` states what a label is worth. An eval's `Scoring` names its labelers in
+order of precedence, and its primary metric. `label(episode, scoring.labelers)` gives a pair: the
+first labeler that labels the episode, and its label. It gives None when no labeler labels the
+episode. `score(scoring, labels)` takes the label of each episode, or None. For the episode
+directories `episodes` of one run:
+
+```python
+from positronic_evals.evals import EvalRef, public_eval
+from positronic_evals.labels import label
+from positronic_evals.scoring import score
+
+scoring = public_eval(EvalRef('molmo.franka_pick_mini')).scoring
+picked = [label(episode, scoring.labelers) for episode in episodes]
+scores = score(scoring, [None if given is None else given[1] for given in picked])
+```
+
+`score` gives `Scores`, which holds three numbers. `primary` is the mean episode score, or the mean
+of the task means where the metric sets `per_task`, and a board ranks on it. `episodes` counts every
+episode, and `unscored` counts the episodes that no labeler labelled. `primary` does not count those
+as failures. No file holds a score: a score is a function of the labels.
+
+`PUBLIC_LABELERS` holds the labelers that this package publishes, `molmo` and `robolab`. A caller with
+labelers of its own passes `label` a table that holds them beside `PUBLIC_LABELERS`. The MolmoSpaces
+labeler takes the end of a trial from the episode's signal files: the time from the first sample to
+the last, on the world clock. That time can exceed the trial by up to one control step.
+
+The RoboLab labeler reads the end of a trial in the same way. Its task is the task id that RoboLab
+names, and not the instruction. It also reads the `subtask` signal, which RoboLab records at each
+step, and gives a `SubtaskAchieved` event, with RoboLab's score, for each change of the subtasks
+completed. A change of the score alone is not an event. A RoboLab eval scores a success 1.0, and any
+other episode the score at its last change of the count, or 0 where it completed no subtask. Where
+RoboLab grades progress within a subtask, this score does not count that progress:
+
+```python
+from positronic_evals.labels import ROBOLAB_LABELER
+from positronic_evals.scoring import EpisodeScore, MeanEpisodeScore, Scoring
+
+robolab = Scoring(
+    labelers=(ROBOLAB_LABELER,), primary=MeanEpisodeScore(episode_score=EpisodeScore(use_subtask_score=True))
+)
+```
+
+## MolmoSpaces episode lists
+
+`positronic_evals.molmo_spaces` holds the episode lists that an offline sweep builds, by benchmark
+path below the asset packs' `benchmarks` directory. `RIGHT_VIEW_BLOCKED_PATH` lists the episodes where
+scene geometry hides the start grasp site from the right shoulder camera. The MolmoSpaces sim server
+leaves those episodes out of an open selection and refuses a pinned one. No public eval pins one.
+
+```python
+from positronic_evals.molmo_spaces import RIGHT_VIEW_BLOCKED_PATH, read_episode_list
+
+blocked = read_episode_list(RIGHT_VIEW_BLOCKED_PATH)
+```
+
+A benchmark that a list does not name is unswept. Rebuild a list when the pinned MolmoSpaces revision
+or the asset packs change. A new list goes beside `right_view_blocked.json`, with its path constant in
+`positronic_evals.molmo_spaces`, and `read_episode_list` reads it.

positronic_evals-0.1.0/positronic_evals/evals.py

@@ -0,0 +1,107 @@
+"""The one name a caller picks a run by, and the definitions of the public evals.
+
+A name is a task suite AND the embodiment that runs it, so there is no second axis to get wrong.
+The platform owns the set, so a name that this package does not define still reaches the platform.
+
+The platform holds the definitions of the evals it does not publish.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict, Field, GetCoreSchemaHandler, JsonValue, PositiveInt
+from pydantic_core import core_schema
+
+from positronic_evals.labels import MOLMO_LABELER
+from positronic_evals.scoring import MeanEpisodeScore, Scoring
+
+
+class EvalRef(str):
+    """The name of an eval the platform offers, as a validated `str`."""
+
+    __slots__ = ()
+
+    def __new__(cls, value: str) -> EvalRef:
+        if not value or any(c.isspace() for c in value):
+            raise ValueError(f'not an eval name: {value!r}')
+        return super().__new__(cls, value)
+
+    @classmethod
+    def __get_pydantic_core_schema__(cls, source: type, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema:
+        return core_schema.no_info_after_validator_function(cls, core_schema.str_schema())
+
+
+_DEFINITION_CONFIG = ConfigDict(extra='forbid', frozen=True)
+
+
+class EvalTask(BaseModel):
+    """One task of an eval, and its trials in run order."""
+
+    model_config = _DEFINITION_CONFIG
+
+    name: str = Field(min_length=1)
+    # Each trial's initial condition, in the keys of the config that runs it.
+    trials: list[dict[str, JsonValue]] = Field(min_length=1)
+
+
+class EvalDefinition(BaseModel):
+    """What a run of a named eval runs."""
+
+    model_config = _DEFINITION_CONFIG
+
+    # The positronic eval config, as `positronic eval run --eval=` takes it.
+    config: str = Field(pattern=r'^\.[A-Za-z_][A-Za-z0-9_.]*$')
+    # The config's `--eval.<name>=` overrides, which expand to `tasks`.
+    args: dict[str, JsonValue]
+    tasks: list[EvalTask] = Field(min_length=1)
+    # The wall-clock limit of one platform run. A local run does not stop at it.
+    time_limit_s: PositiveInt
+    scoring: Scoring
+    # The positronic commit the platform runs the config at.
+    positronic_revision: str = Field(pattern=r'^[0-9a-f]{40}$')
+
+
+# The MolmoSpaces evals: the config they name, its task and trial key as positronic spells them
+# (`positronic/simulator/molmo_spaces/keys.py`), their scoring, and the positronic commit the platform's sim
+# image holds. A run ranks on its success rate.
+MOLMO_CONFIG = '.sim.molmo.benchmarks'
+MOLMO_TASK = 'franka_pick_droid_mini'
+MOLMO_EPISODE_INDEX_KEY = 'molmo.episode_index'
+MOLMO_SCORING = Scoring(labelers=(MOLMO_LABELER,), primary=MeanEpisodeScore())
+MOLMO_POSITRONIC_REVISION = 'a9e13e8a1fbe624ff62b49e25628b39859619986'
+
+
+def _franka_pick_mini(episodes: list[int]) -> EvalDefinition:
+    """`episodes` of the one benchmark the MolmoSpaces evals run, once each."""
+    return EvalDefinition(
+        config=MOLMO_CONFIG,
+        # Each key must name a parameter of positronic's `benchmarks` config.
+        args={
+            'suite': 'molmospaces-bench-v2',
+            'scene_dataset': 'procthor-10k',
+            'task_config': 'FrankaPickDroidMiniBench',
+            'benchmark': 'FrankaPickDroidMiniBench_json_benchmark_20251231',
+            'episodes': list(episodes),
+            'trial_count': 1,
+        },
+        tasks=[EvalTask(name=MOLMO_TASK, trials=[{MOLMO_EPISODE_INDEX_KEY: i} for i in episodes])],
+        time_limit_s=7200,
+        scoring=MOLMO_SCORING,
+        positronic_revision=MOLMO_POSITRONIC_REVISION,
+    )
+
+
+_PUBLIC_EVALS: dict[EvalRef, EvalDefinition] = {
+    # Episode 6 hides the start grasp site from the right shoulder camera; the sim server refuses it.
+    EvalRef('molmo.franka_pick_mini'): _franka_pick_mini([*range(6), *range(7, 21)]),
+    EvalRef('molmo.franka_pick_mini_smoke'): _franka_pick_mini([0, 1, 2, 3, 4]),
+}
+
+PUBLIC_EVALS: tuple[EvalRef, ...] = tuple(_PUBLIC_EVALS)
+
+
+def public_eval(name: EvalRef) -> EvalDefinition:
+    """The definition of the public eval `name`, as a copy, or LookupError naming the public evals."""
+    definition = _PUBLIC_EVALS.get(name)
+    if definition is None:
+        raise LookupError(f'{name!r} is not a public eval; the public evals are {", ".join(PUBLIC_EVALS)}')
+    return definition.model_copy(deep=True)