forkscope 0.1.0__tar.gz

forkscope-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,52 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build tools
+        run: python -m pip install --upgrade build
+
+      - name: Build distributions
+        run: python -m build
+
+      - name: Store distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/forkscope
+    permissions:
+      id-token: write
+    steps:
+      - name: Download distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      - name: Publish distributions
+        uses: pypa/gh-action-pypi-publish@release/v1

forkscope-0.1.0/.github/workflows/test.yml

@@ -0,0 +1,32 @@
+name: Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Install package and development tools
+        run: python -m pip install -e ".[dev]"
+
+      - name: Check formatting and lint
+        run: ruff check .
+
+      - name: Run tests
+        run: pytest
+
+      - name: Build wheel
+        run: python -m build

forkscope-0.1.0/.gitignore

@@ -0,0 +1,29 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+venv/
+
+# Tool caches
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+
+# Local environment and editor files
+.env
+.idea/
+.vscode/
+.DS_Store
+
+# Forkscope outputs and local model artifacts
+outputs/
+*.ckpt
+*.pt
+*.pth
+*.safetensors
+

forkscope-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,8 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.9
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+

forkscope-0.1.0/.python-version

@@ -0,0 +1,2 @@
+3.12
+

forkscope-0.1.0/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 Forkscope contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

forkscope-0.1.0/PKG-INFO

@@ -0,0 +1,51 @@
+Metadata-Version: 2.5
+Name: forkscope
+Version: 0.1.0
+Summary: A local workbench for evaluating test-time planning rollouts.
+Author: Forkscope contributors
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: numpy>=1.26
+Requires-Dist: pydantic>=2.8
+Requires-Dist: typer>=0.12
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == 'dev'
+Requires-Dist: mypy>=1.11; extra == 'dev'
+Requires-Dist: pytest>=8.3; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: gymnasium
+Requires-Dist: gymnasium>=1.0; extra == 'gymnasium'
+Provides-Extra: jax
+Requires-Dist: jax>=0.4.30; extra == 'jax'
+Provides-Extra: torch
+Requires-Dist: torch>=2.4; extra == 'torch'
+Description-Content-Type: text/markdown
+
+# ForkScope
+
+a tool for finding the optimum hyperparameters for world model rollouts at inference.
+
+works for setups that use test-time, action-conditioned planning rollouts (planner generates a bunch
+of actions, world model scores those actions based on outcome, planner evaluates and selects the best
+one to execute).
+
+## Runnable example
+
+The [point-mass example](examples/point_mass/README.md) includes a small environment, a planner,
+zero-argument factories for the CLI, a reference YAML config, and a programmatic example. Install
+ForkScope in editable mode from the repository root with `python -m pip install -e .`, then run:
+
+```sh
+PYTHONPATH=. forkscope run \
+  --env-factory examples.point_mass.example:make_environment \
+  --planner-factory examples.point_mass.example:make_planner \
+  --seeds 0,1,2 \
+  --horizons 1,2,4 \
+  --max-steps 10 \
+  --output outputs/point_mass/results.jsonl
+```
+
+The same factories can be passed to `RolloutBenchmarker` from Python. See the example README for
+that usage and details about the JSON Lines output. The YAML file documents the matching settings;
+the CLI currently accepts options directly and does not load YAML configs.

forkscope-0.1.0/README.md

@@ -0,0 +1,27 @@
+# ForkScope
+
+a tool for finding the optimum hyperparameters for world model rollouts at inference.
+
+works for setups that use test-time, action-conditioned planning rollouts (planner generates a bunch
+of actions, world model scores those actions based on outcome, planner evaluates and selects the best
+one to execute).
+
+## Runnable example
+
+The [point-mass example](examples/point_mass/README.md) includes a small environment, a planner,
+zero-argument factories for the CLI, a reference YAML config, and a programmatic example. Install
+ForkScope in editable mode from the repository root with `python -m pip install -e .`, then run:
+
+```sh
+PYTHONPATH=. forkscope run \
+  --env-factory examples.point_mass.example:make_environment \
+  --planner-factory examples.point_mass.example:make_planner \
+  --seeds 0,1,2 \
+  --horizons 1,2,4 \
+  --max-steps 10 \
+  --output outputs/point_mass/results.jsonl
+```
+
+The same factories can be passed to `RolloutBenchmarker` from Python. See the example README for
+that usage and details about the JSON Lines output. The YAML file documents the matching settings;
+the CLI currently accepts options directly and does not load YAML configs.

forkscope-0.1.0/RELEASING.md

@@ -0,0 +1,22 @@
+# Releasing ForkScope
+
+ForkScope releases are published to PyPI from GitHub Actions using PyPI trusted publishing. No PyPI API token is stored in GitHub.
+
+## One-time setup
+
+1. Confirm the project name `forkscope` is available on PyPI and create the project with its first trusted publisher. For the first release, PyPI's pending publisher flow can be used before the project exists.
+2. Configure the trusted publisher with:
+   - Owner: `alonso130r`
+   - Repository: `forkscope`
+   - Workflow: `publish.yml`
+   - Environment: `pypi`
+3. In the GitHub repository, create the `pypi` deployment environment under **Settings > Environments**. Configure any desired approval rules before publishing.
+
+## Each release
+
+1. Update `version` in `pyproject.toml` and commit the change to `main`.
+2. Create and push a tag that matches that version with a `v` prefix, for example `v0.1.0` for version `0.1.0`.
+3. Create a GitHub Release from that tag and publish it. The `Publish to PyPI` workflow builds the sdist and wheel, then publishes both to PyPI.
+4. Confirm the workflow succeeds and the new version appears on the [ForkScope PyPI project page](https://pypi.org/project/forkscope/).
+
+PyPI does not allow re-uploading an existing version. If a release workflow fails after a version has already been published, fix the issue and release a new patch version.

forkscope-0.1.0/examples/__init__.py

@@ -0,0 +1 @@
+"""Runnable ForkScope examples."""

forkscope-0.1.0/examples/point_mass/README.md

@@ -0,0 +1,52 @@
+# Point-mass example
+
+This example compares planning horizons on a deterministic one-dimensional task. The environment starts at `-seed`, has a goal at position `5`, and moves by the action selected by the planner. The planner moves toward the goal by at most the current horizon. This makes it easy to see how the same environment and seed behave across horizon settings.
+
+Run commands from the repository root after installing ForkScope and its NumPy dependency:
+
+```sh
+python -m pip install -e .
+```
+
+## CLI
+
+The example exposes zero-argument factories for the environment and planner. Set `PYTHONPATH=.` so the CLI can import the example package from the repository root, then pass the settings from [`config.yaml`](config.yaml):
+
+```sh
+PYTHONPATH=. forkscope run \
+  --env-factory examples.point_mass.example:make_environment \
+  --planner-factory examples.point_mass.example:make_planner \
+  --seeds 0,1,2 \
+  --horizons 1,2,4 \
+  --max-steps 10 \
+  --output outputs/point_mass/results.jsonl
+```
+
+The CLI accepts factory paths and individual options; it does not read YAML files directly. `config.yaml` records the same settings for reference and for use in your own launcher or workflow.
+
+## Python API
+
+The script runs the same sweep using `RolloutBenchmarker`:
+
+```sh
+PYTHONPATH=. python -m examples.point_mass.example
+```
+
+It writes JSON Lines records to `outputs/point_mass/programmatic-results.jsonl`. You can also import its factories into your own program:
+
+```python
+from examples.point_mass.example import make_environment, make_planner
+from forkscope import RolloutBenchmarker
+
+benchmarker = RolloutBenchmarker(
+    make_environment,
+    make_planner,
+    seeds=[0, 1, 2],
+    horizons=[1, 2, 4],
+    max_steps=10,
+)
+records = benchmarker.run()
+benchmarker.write_results("outputs/point_mass/results.jsonl")
+```
+
+Each output line is one episode record, including its seed, horizon, total reward, termination status, and step-level observations, actions, rewards, and environment info.

forkscope-0.1.0/examples/point_mass/__init__.py

@@ -0,0 +1 @@
+"""Point-mass horizon-sweep example."""

forkscope-0.1.0/examples/point_mass/config.yaml

@@ -0,0 +1,14 @@
+# Reference settings for the CLI and Python API examples.
+# ForkScope's CLI currently takes these as command-line options; it does not
+# load YAML files directly.
+environment:
+  factory: examples.point_mass.example:make_environment
+
+planner:
+  factory: examples.point_mass.example:make_planner
+
+evaluation:
+  seeds: [0, 1, 2]
+  horizons: [1, 2, 4]
+  max_steps: 10
+  output_path: outputs/point_mass/results.jsonl

forkscope-0.1.0/examples/point_mass/example.py

@@ -0,0 +1,81 @@
+"""Small deterministic environment and planner for a horizon sweep."""
+
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from forkscope import RolloutBenchmarker
+
+
+class PointMassEnvironment:
+    """One-dimensional point mass with a fixed goal at position 5."""
+
+    observation_space = None
+    action_space = None
+
+    def __init__(self, goal: float = 5.0) -> None:
+        self.goal = goal
+        self.position = 0.0
+
+    def reset(
+        self, *, seed: int | None = None, options: dict[str, Any] | None = None
+    ) -> tuple[dict[str, np.ndarray], dict[str, Any]]:
+        self.position = -float(seed or 0)
+        return self._observation(), {"seed": seed}
+
+    def step(
+        self, action: float
+    ) -> tuple[dict[str, np.ndarray], float, bool, bool, dict[str, Any]]:
+        self.position += float(action)
+        distance = abs(self.goal - self.position)
+        terminated = distance == 0.0
+        return self._observation(), -distance, terminated, False, {"distance": distance}
+
+    def close(self) -> None:
+        """Release resources (none are used by this example)."""
+
+    def _observation(self) -> dict[str, np.ndarray]:
+        return {
+            "position": np.array([self.position], dtype=np.float64),
+            "goal": np.array([self.goal], dtype=np.float64),
+        }
+
+
+class BoundedGreedyPlanner:
+    """Move toward the goal by at most the configured planning horizon."""
+
+    def reset(self, *, seed: int) -> None:
+        self.seed = seed
+
+    def plan(self, observation: dict[str, np.ndarray], *, horizon: int) -> float:
+        distance = float(observation["goal"][0] - observation["position"][0])
+        return float(np.clip(distance, -horizon, horizon))
+
+
+def make_environment() -> PointMassEnvironment:
+    """Factory used by the CLI and programmatic benchmarker."""
+    return PointMassEnvironment()
+
+
+def make_planner() -> BoundedGreedyPlanner:
+    """Factory used by the CLI and programmatic benchmarker."""
+    return BoundedGreedyPlanner()
+
+
+def main() -> None:
+    benchmarker = RolloutBenchmarker(
+        make_environment,
+        make_planner,
+        seeds=[0, 1, 2],
+        horizons=[1, 2, 4],
+        max_steps=10,
+    )
+    records = benchmarker.run()
+    output = Path("outputs/point_mass/programmatic-results.jsonl")
+    benchmarker.write_results(output)
+    print(f"Wrote {len(records)} episode records to {output}")
+
+
+if __name__ == "__main__":
+    main()

forkscope-0.1.0/pyproject.toml

@@ -0,0 +1,53 @@
+[build-system]
+requires = ["hatchling>=1.26"]
+build-backend = "hatchling.build"
+
+[project]
+name = "forkscope"
+version = "0.1.0"
+description = "A local workbench for evaluating test-time planning rollouts."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Forkscope contributors" }]
+dependencies = [
+  "numpy>=1.26",
+  "pydantic>=2.8",
+  "typer>=0.12",
+]
+
+[project.optional-dependencies]
+torch = ["torch>=2.4"]
+jax = ["jax>=0.4.30"]
+gymnasium = ["gymnasium>=1.0"]
+dev = [
+  "build>=1.2",
+  "mypy>=1.11",
+  "pytest>=8.3",
+  "ruff>=0.6",
+]
+
+[project.scripts]
+forkscope = "forkscope.cli:app"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/forkscope"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+addopts = "-ra"
+
+[tool.mypy]
+python_version = "3.10"
+strict = true
+mypy_path = "src"

forkscope-0.1.0/src/forkscope/__init__.py

@@ -0,0 +1,7 @@
+"""Forkscope package."""
+
+from forkscope.benchmarker import RolloutBenchmarker
+
+__version__ = "0.1.0"
+
+__all__ = ["RolloutBenchmarker", "__version__"]

forkscope-0.1.0/src/forkscope/benchmarker.py

@@ -0,0 +1,57 @@
+"""Public programmatic API for running rollout horizon benchmarks."""
+
+from collections.abc import Callable, Sequence
+from pathlib import Path
+
+from forkscope.evaluation.sweep import run_sweep
+from forkscope.interface.interfaces import InterfaceEnvironment
+from forkscope.planning.planner import Planner
+from forkscope.records.records import EpisodeRecord
+from forkscope.records.writer import write_jsonl
+
+
+class RolloutBenchmarker:
+    """Run matched-seed horizon sweeps and retain their episode records."""
+
+    def __init__(
+        self,
+        env_factory: Callable[[], InterfaceEnvironment],
+        planner_factory: Callable[[], Planner],
+        *,
+        seeds: Sequence[int],
+        horizons: Sequence[int],
+        max_steps: int,
+    ) -> None:
+        seed_values = tuple(seeds)
+        horizon_values = tuple(horizons)
+        if not seed_values:
+            raise ValueError("seeds must contain at least one value.")
+        if any(seed < 0 for seed in seed_values):
+            raise ValueError("seeds must be non-negative.")
+        if not horizon_values or any(horizon <= 0 for horizon in horizon_values):
+            raise ValueError("horizons must contain positive values.")
+        if max_steps <= 0:
+            raise ValueError("max_steps must be greater than zero.")
+
+        self.env_factory = env_factory
+        self.planner_factory = planner_factory
+        self.seeds = seed_values
+        self.horizons = horizon_values
+        self.max_steps = max_steps
+        self.results: list[EpisodeRecord] = []
+
+    def run(self) -> list[EpisodeRecord]:
+        """Execute the sweep, save results on this instance, and return them."""
+        self.results = []
+        self.results = run_sweep(
+            self.env_factory,
+            self.planner_factory,
+            seeds=self.seeds,
+            horizons=self.horizons,
+            max_steps=self.max_steps,
+        )
+        return self.results
+
+    def write_results(self, path: str | Path) -> None:
+        """Write the current episode results as JSON Lines."""
+        write_jsonl(self.results, path)

forkscope-0.1.0/src/forkscope/cli.py

@@ -0,0 +1,70 @@
+"""Command-line entry point for Forkscope."""
+
+import importlib
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+import typer
+
+from forkscope import RolloutBenchmarker
+
+app = typer.Typer(help="Evaluate test-time planning rollouts on your scenarios.")
+OUTPUT_OPTION = typer.Option(Path("results.jsonl"), help="JSONL output path.")
+
+
+@app.callback()
+def main() -> None:
+    """Run ForkScope commands."""
+
+
+def _load_factory(spec: str) -> Callable[[], Any]:
+    """Load a zero-argument factory from a ``module:attribute`` path."""
+    module_name, separator, attribute_name = spec.partition(":")
+    if not separator or not module_name or not attribute_name:
+        raise typer.BadParameter("Use an import path in the form 'package.module:factory'.")
+
+    try:
+        factory = getattr(importlib.import_module(module_name), attribute_name)
+    except (ImportError, AttributeError) as exc:
+        raise typer.BadParameter(f"Could not load factory {spec!r}: {exc}") from exc
+    if not callable(factory):
+        raise typer.BadParameter(f"Configured factory {spec!r} is not callable.")
+    return factory
+
+
+def _parse_ints(value: str, option: str, *, minimum: int) -> list[int]:
+    try:
+        values = [int(item.strip()) for item in value.split(",")]
+    except ValueError as exc:
+        raise typer.BadParameter(f"{option} must be comma-separated integers.") from exc
+    if not values or any(item < minimum for item in values):
+        comparison = "non-negative" if minimum == 0 else "greater than zero"
+        raise typer.BadParameter(f"{option} values must be {comparison}.")
+    return values
+
+
+@app.command()
+def run(
+    env_factory: str = typer.Option(..., help="Environment factory as package.module:callable."),
+    planner_factory: str = typer.Option(..., help="Planner factory as package.module:callable."),
+    seeds: str = typer.Option("1,2,3,4,5", help="Comma-separated episode seeds."),
+    horizons: str = typer.Option("1,2,4,8,16", help="Comma-separated planning horizons."),
+    max_steps: int = typer.Option(500, min=1, help="Maximum environment steps per episode."),
+    output: Path = OUTPUT_OPTION,
+) -> None:
+    """Run a matched-seed horizon sweep and write its episode records."""
+    benchmarker = RolloutBenchmarker(
+        _load_factory(env_factory),
+        _load_factory(planner_factory),
+        seeds=_parse_ints(seeds, "--seeds", minimum=0),
+        horizons=_parse_ints(horizons, "--horizons", minimum=1),
+        max_steps=max_steps,
+    )
+    benchmarker.run()
+    benchmarker.write_results(output)
+    typer.echo(f"Wrote {len(benchmarker.results)} episode records to {output}")
+
+
+if __name__ == "__main__":
+    app()

forkscope-0.1.0/src/forkscope/evaluation/__init__.py

@@ -0,0 +1 @@
+"""Episode and sweep evaluation helpers."""

forkscope-0.1.0/src/forkscope/evaluation/episode.py

@@ -0,0 +1,57 @@
+from forkscope.interface.interfaces import InterfaceEnvironment
+from forkscope.planning.planner import Planner
+from forkscope.records.records import EpisodeRecord, StepRecord
+
+
+def run_episode(
+    env: InterfaceEnvironment,
+    planner: Planner,
+    *,
+    seed: int,
+    horizon: int,
+    max_steps: int,
+) -> EpisodeRecord:
+    if max_steps <= 0:
+        raise ValueError("max_steps must be greater than zero.")
+    if horizon <= 0:
+        raise ValueError("horizon must be greater than zero.")
+
+    planner.reset(seed=seed)
+    observation, _ = env.reset(seed=seed)
+    steps: list[StepRecord] = []
+    total_reward = 0.0
+    terminated = truncated = False
+
+    for step_index in range(max_steps):
+        action = planner.plan(
+            observation,
+            horizon=horizon,
+        )
+        next_observation, reward, terminated, truncated, info = env.step(action)
+        steps.append(
+            StepRecord(
+                index=step_index,
+                observation=observation,
+                action=action,
+                reward=reward,
+                terminated=terminated,
+                truncated=truncated,
+                info=info,
+            )
+        )
+        total_reward += reward
+        observation = next_observation
+
+        if terminated or truncated:
+            break
+
+    step_limit_reached = not terminated and not truncated and len(steps) == max_steps
+    return EpisodeRecord(
+        seed=seed,
+        horizon=horizon,
+        total_reward=total_reward,
+        steps=steps,
+        terminated=terminated,
+        truncated=truncated,
+        step_limit_reached=step_limit_reached,
+    )