tolokaforge 0.2.0__tar.gz

tolokaforge-0.2.0/.cursor/rules/cursor.mdc

@@ -0,0 +1,19 @@
+---
+description: Cursor settings — all project rules are in AGENTS.md
+alwaysApply: true
+---
+# Cursor Configuration
+
+All project rules, conventions, and commands are defined in `AGENTS.md` at the repository root.
+Read it first — it is the single source of truth for all AI agents.
+
+## Session Startup
+
+1. Read `README.md` and `.vscode/tasks.json` before writing any code
+2. Do not ask permission — these are essential context
+
+## Cursor-Specific
+
+- Use Context7 MCP (if available) to look up library/framework documentation before guessing at APIs
+- Available VSCode tasks are in `.vscode/tasks.json`
+- Plans and scratch files go in `plans/` directory — it is gitignored

tolokaforge-0.2.0/.dockerignore

@@ -0,0 +1,63 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+dist/
+build/
+*.egg
+
+# Virtual environments
+venv/
+env/
+ENV/
+.venv
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Git
+.git/
+.gitignore
+.gitattributes
+
+# Documentation
+*.md
+!README.md
+docs/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+
+# Results and outputs
+results/
+*.log
+*.json
+*.csv
+*.html
+
+# CI/CD
+.github/
+.gitlab-ci.yml
+
+# Docker
+docker-compose.override.yml
+.dockerignore
+
+# Temporary files
+tmp/
+temp/
+*.tmp

tolokaforge-0.2.0/.env.example

@@ -0,0 +1,24 @@
+# Environment variables for Tolokaforge
+# Copy this file to .env and fill in your API keys:
+#   cp .env.example .env
+
+# At least one LLM provider key is required.
+# Most examples use OpenRouter by default.
+OPENROUTER_API_KEY=your-openrouter-api-key-here
+
+# Optional: direct provider keys (used when provider is set to "anthropic", "openai", etc.)
+# ANTHROPIC_API_KEY=your-anthropic-api-key-here
+# OPENAI_API_KEY=your-openai-api-key-here
+# GOOGLE_API_KEY=your-google-api-key-here
+
+# Gemini direct (Google AI Studio): use when provider=gemini in ModelConfig.
+# Create at https://aistudio.google.com (no GCP IAM required; key is a bearer token).
+# Bypasses OpenRouter — useful for discriminating model behavior from provider transport.
+# GEMINI_API_KEY=your-gemini-studio-api-key-here
+
+# Vertex AI (GCP): use when provider=vertex_ai in ModelConfig.
+# Service account needs role roles/aiplatform.user (or aiplatform.endpoints.predict).
+# Download a JSON key, point GOOGLE_APPLICATION_CREDENTIALS at it.
+# GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json
+# VERTEXAI_PROJECT=your-gcp-project-id
+# VERTEXAI_LOCATION=us-central1

tolokaforge-0.2.0/.gitattributes

@@ -0,0 +1,6 @@
+# Trajectory output files (large JSON/YAML from benchmark runs)
+tests/data/projects/*/output/**/*.json filter=lfs diff=lfs merge=lfs -text
+tests/data/projects/*/output/**/*.yaml filter=lfs diff=lfs merge=lfs -text
+
+# Mobile task screenshots (binary PNGs)
+tasks/mobile/_assets/images/*.png filter=lfs diff=lfs merge=lfs -text

tolokaforge-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,309 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'tolokaforge/**'
+      - 'tests/**'
+      - 'scripts/**'
+      - 'tools/**'
+      - 'pyproject.toml'
+      - 'uv.lock'
+      - '.github/workflows/ci.yml'
+      - '.pre-commit-config.yaml'
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+    paths:
+      - 'tolokaforge/**'
+      - 'tests/**'
+      - 'scripts/**'
+      - 'tools/**'
+      - 'pyproject.toml'
+      - 'uv.lock'
+      - '.github/workflows/ci.yml'
+      - '.pre-commit-config.yaml'
+  schedule:
+    - cron: '0 6 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  # ============================================================================
+  # Lint job - Fast feedback on code quality
+  # ============================================================================
+  lint:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v4
+      with:
+        version: "latest"
+        enable-cache: true
+        cache-dependency-glob: "uv.lock"
+
+    - name: Set up Python 3.12
+      run: uv python install 3.12
+
+
+    - name: Install dependencies
+      run: uv sync --dev
+
+    - name: Run pre-commit checks
+      run: uv run pre-commit run --all-files --show-diff-on-failure
+
+  # ============================================================================
+  # PR smoke tests - fast required gate
+  # ============================================================================
+  test-smoke:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    needs: lint  # Only run tests if linting passes
+    timeout-minutes: 45
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        token: ${{ github.token }}
+        lfs: true
+
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v4
+      with:
+        version: "latest"
+        enable-cache: true
+        cache-dependency-glob: "uv.lock"
+
+    - name: Set up Python 3.12
+      run: uv python install 3.12
+
+
+    - name: Install dependencies
+      run: |
+        uv sync --dev
+
+    - name: Install Playwright browsers
+      run: |
+        uv run python -m playwright install chromium
+
+    - name: Create .env file
+      run: |
+        echo "OPENROUTER_API_KEY=${{ secrets.OPENROUTER_TESTING_TOKEN }}" > .env
+
+    - name: Run unit + canonical tests with coverage
+      run: |
+        uv run pytest tests/unit/ tests/canonical/ -v --tb=short \
+          --cov=tolokaforge --cov-report=term-missing --cov-fail-under=60
+
+    - name: Run tool tests
+      run: |
+        for tool_dir in tools/*/; do
+          if [ -d "${tool_dir}tests" ]; then
+            echo "=== Testing ${tool_dir} ==="
+            (cd "${tool_dir}" && uv run pytest tests/ -v --tb=short -p no:cacheprovider) || exit 1
+          fi
+        done
+
+    - name: Validate public example suites
+      run: |
+        uv run tolokaforge validate --tasks "examples/native/*/dataset/**/task.yaml"
+
+
+    - name: Docker task-pack mount smoke
+      run: |
+        scripts/tests/task_pack_docker_smoke.sh
+
+    # TODO: Re-enable once in-process runtime or Docker-less orchestrator is available.
+    # The smoke script requires `tolokaforge run` which now needs a Docker runtime.
+    # - name: Run public examples end-to-end (mock smoke)
+    #   run: |
+    #     scripts/tests/run_public_examples_smoke.sh
+    #
+    # - name: Summarize public examples
+    #   run: |
+    #     uv run python scripts/tests/summarize_public_examples.py --min-pass-rate 0.0 --min-completion-rate 1.0
+    #
+    # - name: Upload public example summary artifact
+    #   uses: actions/upload-artifact@v4
+    #   with:
+    #     name: public-example-summary-pr
+    #     path: |
+    #       output/public_examples_summary.json
+    #       output/public_examples_summary.md
+
+  # ============================================================================
+  # Full test suite - push + nightly confidence
+  # ============================================================================
+  test-full:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    needs: [lint]
+    timeout-minutes: 120
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        token: ${{ github.token }}
+        lfs: true
+
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v4
+      with:
+        version: "latest"
+        enable-cache: true
+        cache-dependency-glob: "uv.lock"
+
+    - name: Set up Python 3.12
+      run: uv python install 3.12
+
+
+    - name: Install dependencies
+      run: |
+        uv sync --dev
+
+    - name: Install Playwright browsers
+      run: |
+        uv run python -m playwright install chromium
+
+    - name: Create .env file
+      run: |
+        echo "OPENROUTER_API_KEY=${{ secrets.OPENROUTER_TESTING_TOKEN }}" > .env
+
+    - name: Validate public example suites
+      run: |
+        uv run tolokaforge validate --tasks "examples/native/*/dataset/**/task.yaml"
+
+
+    - name: Docker task-pack mount smoke
+      run: |
+        scripts/tests/task_pack_docker_smoke.sh
+
+    # TODO: Re-enable once in-process runtime or Docker-less orchestrator is available.
+    # - name: Run public examples end-to-end (mock smoke)
+    #   run: |
+    #     scripts/tests/run_public_examples_smoke.sh
+    #
+    # - name: Summarize public examples
+    #   run: |
+    #     uv run python scripts/tests/summarize_public_examples.py --min-pass-rate 0.0 --min-completion-rate 1.0
+
+    - name: Run unit + canonical tests with coverage
+      run: |
+        uv run pytest tests/unit/ tests/canonical/ -v --tb=short \
+          --cov=tolokaforge --cov-report=term-missing --cov-fail-under=60
+
+    - name: Run tool tests
+      run: |
+        for tool_dir in tools/*/; do
+          if [ -d "${tool_dir}tests" ]; then
+            echo "=== Testing ${tool_dir} ==="
+            (cd "${tool_dir}" && uv run pytest tests/ -v --tb=short -p no:cacheprovider) || exit 1
+          fi
+        done
+
+    - name: Build Docker images for integration tests
+      if: github.event_name == 'schedule' || github.event_name == 'push'
+      id: docker-build-full
+      run: |
+        uv run tolokaforge docker build --core
+
+    - name: Run integration tests
+      if: (github.event_name == 'schedule' || github.event_name == 'push') && steps.docker-build-full.outcome == 'success'
+      run: |
+        uv run pytest tests/integration/ -v --tb=short
+
+    - name: Generate coverage XML for Codecov
+      if: github.event_name == 'schedule'
+      run: |
+        uv run coverage xml
+
+    - name: Upload coverage to Codecov
+      if: github.event_name == 'schedule'
+      uses: codecov/codecov-action@v3
+      with:
+        files: ./coverage.xml
+        flags: unittests
+        name: codecov-umbrella
+      continue-on-error: true
+
+  # ============================================================================
+  # Label-triggered gate - full suite + coverage (add 'ready-to-merge' label)
+  # ============================================================================
+  test-gate:
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name == 'pull_request' &&
+      contains(github.event.pull_request.labels.*.name, 'ready-to-merge')
+    needs: [lint]
+    timeout-minutes: 120
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        token: ${{ github.token }}
+        lfs: true
+
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v4
+      with:
+        version: "latest"
+        enable-cache: true
+        cache-dependency-glob: "uv.lock"
+
+    - name: Set up Python 3.12
+      run: uv python install 3.12
+
+
+    - name: Install dependencies
+      run: |
+        uv sync --dev
+
+    - name: Install Playwright browsers
+      run: |
+        uv run python -m playwright install chromium
+
+    - name: Create .env file
+      run: |
+        echo "OPENROUTER_API_KEY=${{ secrets.OPENROUTER_TESTING_TOKEN }}" > .env
+
+    - name: Validate public example suites
+      run: |
+        uv run tolokaforge validate --tasks "examples/native/*/dataset/**/task.yaml"
+
+
+    - name: Docker task-pack mount smoke
+      run: |
+        scripts/tests/task_pack_docker_smoke.sh
+
+    - name: Run unit + canonical tests with coverage
+      run: |
+        uv run pytest tests/unit/ tests/canonical/ -v --tb=short \
+          --cov=tolokaforge --cov-report=xml --cov-report=term-missing \
+          --cov-fail-under=60
+
+    - name: Build Docker images for integration tests
+      id: docker-build
+      run: |
+        uv run tolokaforge docker build --core
+
+    - name: Run integration tests
+      if: steps.docker-build.outcome == 'success'
+      run: |
+        uv run pytest tests/integration/ -v --tb=short
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        files: ./coverage.xml
+        flags: merge-gate
+        name: codecov-merge-gate
+      continue-on-error: true

tolokaforge-0.2.0/.github/workflows/claude-review.yml

@@ -0,0 +1,100 @@
+name: Claude Code PR Review
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+
+jobs:
+  hygiene-review:
+    runs-on: ubuntu-latest
+    if: >
+      github.event_name == 'pull_request' ||
+      (github.event_name == 'issue_comment' &&
+       contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review_comment' &&
+       contains(github.event.comment.body, '@claude'))
+    permissions:
+      contents: read
+      pull-requests: write
+      issues: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: anthropics/claude-code-action@beta
+        with:
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          direct_prompt: |
+            Review this PR against the project's AGENTS.md rules. Flag any
+            violation with file path, line number, and a concrete suggested
+            fix. If everything checks out, say so explicitly.
+
+            1. CODE QUALITY: Errors are surfaced explicitly. No silent
+               fallbacks, no swallowed exceptions (`except Exception: pass`),
+               no returning `None` instead of raising. Functions under 100
+               lines, nesting depth < 3.
+
+            2. SECRETS — single abstraction (HARD RULE): API keys, DB
+               credentials, tokens, signing keys, and any other credential
+               are accessed *only* via `SecretManager`
+               (`tolokaforge.secrets`). Flag any new `os.environ.get` /
+               `os.getenv` for credentials, any `load_dotenv()` call, any
+               `from dotenv import` outside `tolokaforge.secrets`, any
+               direct `.env` / `.netrc` / `.aws/credentials` file read,
+               any one-off env-var helper, any secret baked into a Docker
+               image / build-arg / mount / image-tag. New secret backends
+               must ship as new `SecretProvider` subclasses, never as
+               ad-hoc call sites. The
+               `tests/unit/secrets/test_no_raw_secret_access.py`
+               enforcement test must remain green.
+
+            3. TESTING: Every new test file has a `pytestmark` marker
+               (`pytest.mark.unit`, `.canonical`, or `.integration`).
+               Zero `xfail`, zero bare `@skip` — use conditional markers
+               (`requires_api`, `requires_docker`). MockAsyncClient comes
+               from `tests.utils.mock_clients` only — no local copies.
+
+            4. ARCHITECTURE: Harness logic stays generic. Task-specific
+               logic lives in task packs only. Clean abstraction
+               boundaries. Backward compatibility for task contracts.
+
+            5. REPOSITORY HYGIENE: No scripts, data, temp files, or logs
+               in the repo root. Documentation updated when user-facing
+               behaviour changes.
+
+            6. CODE STANDARDS: DRY — no duplicated logic. Self-describing
+               names. Early returns to minimise nesting. No warnings
+               suppressed.
+
+            7. ROOT CLEANLINESS: New files in the repo root must be on
+               this allow-list — README.md, LICENSE, CHANGELOG.md,
+               CONTRIBUTING.md, CONTRIBUTORS.md, CITATION.*, CLAUDE.md,
+               AGENTS.md, pyproject.toml, uv.lock, Makefile,
+               docker-compose.yaml, and dotfiles (.gitignore,
+               .pre-commit-config.yaml, etc.).
+
+            8. NO TEMP ARTIFACTS: No temporary plans, log files, JSON
+               data dumps, or build outputs.
+
+            9. SCRIPT LOCATION: Bash scripts only in scripts/. Exceptions:
+               tests/ for test helpers, tasks/ for benchmark data,
+               .devcontainer/ for container setup, Docker entrypoints
+               alongside Dockerfiles.
+
+            10. SCRIPTS ORGANIZATION: New scripts placed in the correct
+                scripts/ subdirectory (benchmark/, setup/, lint/, tests/,
+                release/, analysis/).
+
+            11. PYTHON TOOLS: Complex Python tools in tools/ as uv
+                workspace members, linked in
+                `pyproject.toml [tool.uv.workspace]`.
+
+            12. NO PROJECT-SPECIFIC CONTENT ON MAIN: No domain-specific
+                configs or runner scripts on the `main` branch.

tolokaforge-0.2.0/.github/workflows/publish-adapter-terminal-bench.yml

@@ -0,0 +1,132 @@
+name: Publish tolokaforge-adapter-terminal-bench to PyPI
+
+on:
+  push:
+    tags:
+      - "adapter-terminal-bench-v*"
+  workflow_dispatch:
+    inputs:
+      target:
+        description: "Publish target"
+        required: true
+        type: choice
+        options:
+          - testpypi
+          - pypi
+
+# Cancel in-progress runs for the same tag
+concurrency:
+  group: publish-adapter-terminal-bench-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Install Python
+        run: uv python install 3.12
+
+      - name: Build package
+        run: uv build --package tolokaforge-adapter-terminal-bench
+
+      - name: Verify package contents
+        run: |
+          echo "=== Built artifacts ==="
+          ls -lh dist/
+          echo ""
+          echo "=== Wheel contents ==="
+          uv run python -c "
+          import zipfile, sys
+          for f in __import__('pathlib').Path('dist').glob('*.whl'):
+              print(f'--- {f.name} ---')
+              with zipfile.ZipFile(f) as zf:
+                  for name in sorted(zf.namelist()):
+                      print(f'  {name}')
+          "
+
+      - name: Upload distribution artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: adapter-terminal-bench-dist
+          path: dist/
+          if-no-files-found: error
+
+  publish-testpypi:
+    name: Publish to TestPyPI
+    needs: build
+    if: github.event_name == 'workflow_dispatch' && github.event.inputs.target == 'testpypi'
+    runs-on: ubuntu-latest
+    environment: testpypi
+    permissions:
+      id-token: write
+    steps:
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Download distribution artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: adapter-terminal-bench-dist
+          path: dist/
+
+      - name: List downloaded files
+        run: find dist/ -type f
+
+      - name: Publish to TestPyPI
+        run: uv publish dist/* --publish-url https://test.pypi.org/legacy/ --trusted-publishing always
+
+  publish-pypi:
+    name: Publish to PyPI
+    needs: build
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/adapter-terminal-bench-v') || (github.event_name == 'workflow_dispatch' && github.event.inputs.target == 'pypi')
+    runs-on: ubuntu-latest
+    environment: release
+    permissions:
+      id-token: write
+    steps:
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Download distribution artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: adapter-terminal-bench-dist
+          path: dist/
+
+      - name: List downloaded files
+        run: find dist/ -type f
+
+      - name: Publish to PyPI
+        run: uv publish dist/* --trusted-publishing always
+
+  github-release:
+    name: Create GitHub Release
+    needs: publish-pypi
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/adapter-terminal-bench-v')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download distribution artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: adapter-terminal-bench-dist
+          path: dist/
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          generate_release_notes: true
+          files: dist/*