rigorloop 0.1.0__tar.gz

rigorloop-0.1.0/.claude/settings.json

@@ -0,0 +1,9 @@
+{
+  "sandbox": {
+    "enabled": true,
+    "allowUnsandboxedCommands": false,
+    "filesystem": {
+      "allowWrite": ["."]
+    }
+  }
+}

rigorloop-0.1.0/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,22 @@
+---
+name: Bug report
+about: Something misbehaved
+labels: bug
+---
+
+## What happened
+
+<!-- What did you run, what did you expect, what happened instead? -->
+
+## Reproduction
+
+- `rigorloop --version`:
+- OS:
+- `rigorloop.toml` (redact anything sensitive):
+- Command and full output:
+
+## Run artifacts
+
+If the failure happened mid-run, the contents of `runs/<run_id>/` (especially
+`state.json` and the relevant `loops/<n>/` directory) are the most useful
+thing you can attach — minus anything confidential in your examples.

rigorloop-0.1.0/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,18 @@
+---
+name: Feature request
+about: Propose an improvement
+labels: enhancement
+---
+
+## The problem
+
+<!-- What are you trying to do that RigorLoop makes hard? -->
+
+## Proposed behavior
+
+<!-- What would you like to happen? Config sketch welcome. -->
+
+## Statistical implications (if any)
+
+<!-- RigorLoop's core promise is honest evaluation. If your proposal touches
+splits, peeks, selection, or reporting, say how it keeps that promise. -->

rigorloop-0.1.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,10 @@
+# Summary
+
+<!-- What does this PR change, and why? -->
+
+## Checklist
+
+- [ ] `just check` passes locally (ruff, mypy --strict, pytest with coverage gates)
+- [ ] New logic lives in the functional core where possible (see `CODING_STYLE.md`)
+- [ ] Leakage guarantees untouched, or their tests extended
+- [ ] `CHANGELOG.md` updated if `src/` changed

rigorloop-0.1.0/.github/dependabot.yml

@@ -0,0 +1,17 @@
+version: 2
+updates:
+  - package-ecosystem: "uv"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      python-dev:
+        patterns: ["*"]
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      actions:
+        patterns: ["*"]

rigorloop-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,105 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.12"
+          enable-cache: true
+      - run: uv sync --locked
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.12"
+          enable-cache: true
+      - run: uv sync --locked
+      - run: uv run mypy
+
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.12", "3.13", "3.14"]
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python }}
+          enable-cache: true
+      - run: uv sync --locked
+      - run: uv run pytest --cov --cov-report=term >> "$GITHUB_STEP_SUMMARY"
+      - run: uv run coverage report --include='*/rigorloop/core/*' --fail-under=95
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.12"
+          enable-cache: true
+      - run: uv build
+      - run: uvx twine check --strict dist/*
+      - name: Smoke-test the built wheel in a fresh venv
+        run: |
+          uv venv /tmp/wheel-venv
+          VIRTUAL_ENV=/tmp/wheel-venv uv pip install dist/*.whl
+          /tmp/wheel-venv/bin/rigorloop --version
+          /tmp/wheel-venv/bin/python -c "import rigorloop"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  changelog-reminder:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Warn when src/ changes without a CHANGELOG entry (advisory)
+        run: |
+          git fetch origin "${{ github.base_ref }}" --depth=1
+          changed=$(git diff --name-only "origin/${{ github.base_ref }}"...HEAD)
+          if echo "$changed" | grep -q '^src/' && ! echo "$changed" | grep -q '^CHANGELOG.md'; then
+            echo "::warning::This PR touches src/ without updating CHANGELOG.md"
+          fi
+
+  ci-ok:
+    # The single required status check for branch protection (matrix-proof).
+    needs: [lint, typecheck, test, build]
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "all green"

rigorloop-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,60 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+*"
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    # Rebuild from the tag; never reuse CI artifacts.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.12"
+      - run: uv build
+      - run: uvx twine check --strict dist/*
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish-pypi:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write # OIDC Trusted Publishing; no API token exists
+    steps:
+      # Download-and-publish only: no project code runs with the OIDC token.
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    needs: publish-pypi
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Create GitHub Release with notes and artifacts
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release create "$GITHUB_REF_NAME" dist/* \
+            --title "$GITHUB_REF_NAME" \
+            --generate-notes

rigorloop-0.1.0/.gitignore

@@ -0,0 +1,223 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+# Temporary file for partial code execution
+tempCodeRunnerFile.py
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml
+
+# RigorLoop additions
+src/rigorloop/_version.py
+runs/
+.uv-cache/

rigorloop-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-toml
+      - id: check-yaml
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.4
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: local
+    hooks:
+      - id: mypy
+        name: mypy (strict)
+        entry: uv run mypy
+        language: system
+        types: [python]
+        pass_filenames: false

rigorloop-0.1.0/CHANGELOG.md

@@ -0,0 +1,30 @@
+# Changelog
+
+All notable changes to this project are documented in this file, following
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and
+[SemVer](https://semver.org/) (`0.x`: minor = features/breaking, patch = fixes).
+
+## [Unreleased]
+
+## [0.1.0] - 2026-07-08
+
+### Added
+
+- Initial implementation of the full RigorLoop protocol:
+  - `rigorloop init | check | run | report` CLI.
+  - Deterministic dev/validation/test splitting with content-hash manifests,
+    exact-duplicate collapsing, and statistical power warnings.
+  - Strategy agent / executor agent loop over the `claude` CLI (headless,
+    tool-less), with strict output contracts, reformat retries, and a
+    fallback path for malformed strategy replies.
+  - Checks: exact/normalized/JSON/regex/numeric matching, custom Python
+    checkers, and n-sample majority-vote LLM judges.
+  - Statistics: Wilson intervals, seeded bootstrap CIs, exact McNemar paired
+    tests; CI-band-gated champion selection, budgeted validation peeks, and
+    a one-shot final test evaluation with a selection-bias caveat in the
+    report.
+  - Solution kinds: executable Python scripts, agent skills (SKILL.md), and
+    guidance markdown (AGENTS.md/CLAUDE.md style).
+  - Resumable run directories (`runs/<run_id>/`) with split-drift protection.
+  - Type-level and test-level leakage guards keeping holdout data out of all
+    agent-context prompts.

rigorloop-0.1.0/CLAUDE.md

@@ -0,0 +1,39 @@
+# CLAUDE.md for RigorLoop
+
+## Project overview
+
+This a is statistically-sound agentic build framework that employs agentic loops to create code artifacts (whether a script, a skill markdown file, etc). Crucially, it splits verification data into the classic data science-like dev, validation, and final test sets to avoid overfitting. 
+
+This framework IS NOT meant to be used when someone wants to create an agentic engineering loop to produce a simple, deterministic script that will pass a series of simple unit tests. Instead, it is meant to be used for data science-like efforts like when the user needs to create a coding solution to take a set of inputs (structured or unstructured text data) and convert it into a set of structured outputs. The user would provide this frame with a (hopefully) large set of gold-standard, correct example inputs and outputs, and this agentic framework attempts to create a coding solution through statistically-rigorous loops and verifications. 
+
+The "coding solution" can take a number of forms:
+
+- a set of executable scripts
+- a "skill" for an agentic coding harness (e.g. Claude Skills)
+- a markdown file to guide agents (e.g. AGENTS.md, CLAUDE.md, etc.)
+  
+
+## Requirements
+
+* A strict splitting of a "development", "validation", and "final test" set must be made out of the input data the user provides.
+* The user must provide structured input text with the expected output too. The user must provide a set of either deterministic or probablistic verifications/checks of the proposed solutions. This framework must flexibly allow for whatever the user provides (within realistic bounds and with guidelines). The examples the user provides should be highly-representative of the universe of inputs and outputs they might encounter.
+* The framework will generate a series of solutions that iteratively improve through a series of loops. 
+* The framework will employ a strategy agent (with its own prompt) and a series of execution agents (with their own prompts). The strategy agent will review each iterations results on the "development" set, and periodically spin up an agent to test the best approach on the "validation" set. Each execution agent will only be able to see the current loop's strategy and not know about prior loops. The one sanctioned exception: the strategy agent may embed the current champion solution's *content* in a directive as a refinement starting point — solution content only, never scores, mistakes, or per-example failures from prior loops. 
+* Protection must be taken to prevent data leakage among the agents: e.g. an execution agent sees all mistakes from prior loops. Again, the idea is that the strategy agent understands what works and doesn't work, loop-to-loop, keeps a log for itself, and farms out the pure execution to a set of concurrent executor agents. This leakage guarantee applies to *agent-context* prompts (strategy and executor). *Evaluation* prompts — running a solution-under-test or an LLM judge on a single example — necessarily embed that example (whatever its split) and are a separate, sanctioned channel whose outputs return only to the harness as scores. 
+* The final loop will produce the best performing coding solution, which should be able to be copied by the user, used outside of this framework, and produce good results for them on new input data. 
+* Claude will be called in headless model and without tools (`claude -p --tools ""`) to spin up the above agents.
+
+
+## Comprehensive implementation plan
+
+See the plan in `PLAN.md`
+
+
+## Coding style
+
+See the rules and guidelines described in `CODING_STYLE.md`
+
+
+## Packaging as an open source release
+
+See the `PACKAGING_PLAN.md` file for instructions on how to setup the packaging around this framework. Includes details on dependencies, the `pyproject.toml` file, linting, testing, and deploying to PyPi.