promptcheckup 1.0.0__tar.gz

promptcheckup-1.0.0/.github/workflows/ci.yml

@@ -0,0 +1,24 @@
+name: ci
+
+on:
+  push:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install pytest==9.1.1 ruff==0.17.1
+      - run: ruff check .
+      - run: ruff format --check .
+      - run: pytest

promptcheckup-1.0.0/.github/workflows/release.yml

@@ -0,0 +1,45 @@
+name: release
+
+# Publishes to PyPI when a GitHub release is published, using trusted publishing:
+# no API token is stored anywhere. One-time setup on pypi.org: add this repository,
+# workflow file `release.yml` and environment `pypi` as a trusted publisher.
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
+        with:
+          python-version: "3.12"
+      - run: python -m pip install build==1.6.1 pytest==9.1.1
+      - name: Run the tests on the exact commit being released
+        run: python -m pytest -q
+      - name: Require the release tag to match the package version
+        run: |
+          version=$(python -c 'import re, pathlib; print(re.search(r"__version__ = \"(.+)\"", pathlib.Path("src/promptcheckup/__init__.py").read_text())[1])')
+          test "v$version" = "$GITHUB_REF_NAME" || { echo "tag $GITHUB_REF_NAME does not match version $version"; exit 1; }
+      - run: python -m build
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write # trusted publishing; no token secret exists
+    steps:
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3

promptcheckup-1.0.0/.gitignore

@@ -0,0 +1,11 @@
+.venv/
+__pycache__/
+*.egg-info/
+dist/
+.pytest_cache/
+.ruff_cache/
+.promptcheckup/
+
+# Outputs from running the examples
+examples/**/report.md
+examples/**/result.json

promptcheckup-1.0.0/CHANGELOG.md

@@ -0,0 +1,33 @@
+# Changelog
+
+All notable changes are listed here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and versions follow [Semantic Versioning](https://semver.org/).
+
+## [1.0.0] - 2026-11-01
+
+The first public release.
+
+### Added
+
+- **Statistical gate.** A paired 95% t confidence interval with an exact Student t quantile, at least 20 cases, and three verdicts: better, worse, or not sure yet. A skew guard refuses "better" when a few large losses hide among small gains.
+- **Paired replay.** Every case runs through both prompts back to back, so shared noise cancels. Outputs are cached, calls are capped, and errors are reported, never dropped.
+- **Agents in any language.** A Python `module:function`, or any command speaking JSON over stdin and stdout. `@record` captures real cases from production, with redaction.
+- **LLM judge.** Pairwise, order-swapped comparisons with a self-check that compares the old prompt with itself; an unreliable judge downgrades the verdict. Untrusted data cannot close its prompt block. A judge may reason before its JSON verdict; no temperature is sent unless configured.
+- **`promptcheckup gate`** compares the prompts in a pull request with its base branch, with a workflow template that comments on the pull request.
+- **`promptcheckup checkup`,** the weekly check-up:
+  - drift against last week's snapshot, including after a prompt change;
+  - starter cases when none exist;
+  - failure patterns;
+  - one small proposed edit that must pass the gate;
+  - a workflow template that posts to one issue and opens a pull request. It never merges.
+- **`promptcheckup evolve`:** repeated proposals, learned on fixed-rule learning cases and gated on held-out cases at a stricter confidence for every extra round. It never merges.
+- **Trial ledger:** reports how many prompt variants were tried on the same cases and how many false "better" verdicts that many trials would produce by chance. `run.confidence` sets the base confidence.
+- **`promptcheckup doctor`:** checks a project's wiring with two agent calls and at most one judge call, including whether the agent really uses the prompts it is given.
+- **`promptcheckup.testing`:** `gate`, `assert_better`, `assert_not_worse` and `assert_no_lookahead`, for use in an existing test suite.
+- **Examples:**
+  - a support-bot quickstart with real reports from three hosted judges;
+  - a trading agent on a seeded synthetic market, with recorded replies, a look-ahead test and outcome-blind cases;
+  - a planted example comparing naive and paired evaluation.
+- **Benchmarks:** false-positive and power simulations of the gate, and a simulated year of weekly check-ups with planted incidents.
+- **Documentation:** a guide for connecting an agent, and the full design in `DESIGN.md`.
+
+[1.0.0]: https://github.com/gael55x/promptcheckup/releases/tag/v1.0.0

promptcheckup-1.0.0/CITATION.cff

@@ -0,0 +1,79 @@
+cff-version: 1.2.0
+message: "If you use promptcheckup in your work, please cite it as below."
+type: software
+title: "promptcheckup: paired replay for deciding whether a prompt change made an AI agent better"
+version: 1.0.0
+license: Apache-2.0
+repository-code: "https://github.com/gael55x/promptcheckup"
+authors:
+  - family-names: Amolong
+    given-names: Gaille
+keywords:
+  - large language models
+  - agents
+  - prompt engineering
+  - evaluation
+  - paired comparison
+references:
+  - type: article
+    title: "MemTrial: Learning When to Trust Memory in LLM Portfolio Agents"
+    authors:
+      - family-names: Wu
+        given-names: Guanghao
+      - family-names: Cai
+        given-names: Zhuo
+      - family-names: Wang
+        given-names: Shoujin
+    year: 2026
+    url: "https://arxiv.org/abs/2610.11732"
+    notes: "The paired-replay gate: compare decisions on the same inputs so shared noise cancels."
+  - type: article
+    title: "EvolveTrade: Experience-Driven Policy Refinement for Self-Evolving LLM Trading Agents"
+    authors:
+      - family-names: Kim
+        given-names: Sehee
+      - family-names: Choi
+        given-names: Yumin
+      - family-names: Kang
+        given-names: Minki
+      - family-names: Hwang
+        given-names: Sung Ju
+    year: 2026
+    url: "https://arxiv.org/abs/2609.17632"
+    notes: "The proposer: an LLM rewrites the agent's prompt from decision traces and outcomes."
+  - type: article
+    title: "SHARP: A Self-Evolving Human-Auditable Rubric Policy for Financial Trading Agents"
+    authors:
+      - family-names: Chen
+        given-names: Xiwen
+      - family-names: Zhu
+        given-names: Wenhui
+      - family-names: Zheng
+        given-names: Songzhu
+      - family-names: Rasul
+        given-names: Kashif
+      - family-names: Deng
+        given-names: Yueyue
+      - family-names: Li
+        given-names: Huayu
+    year: 2026
+    url: "https://arxiv.org/abs/2605.06822"
+    notes: "Small, attributable rule edits with validation, and an audit trail."
+  - type: article
+    title: "ATLAS: Adaptive Trading with LLM AgentS Through Dynamic Prompt Optimization and Multi-Agent Coordination"
+    authors:
+      - family-names: Papadakis
+        given-names: Charidimos
+      - family-names: Dimitriou
+        given-names: Angeliki
+      - family-names: Filandrianos
+        given-names: Giorgos
+      - family-names: Lymperaiou
+        given-names: Maria
+      - family-names: Thomas
+        given-names: Konstantinos
+      - family-names: Stamou
+        given-names: Giorgos
+    year: 2025
+    url: "https://arxiv.org/abs/2510.15949"
+    notes: "Prior art: feedback-driven prompt optimization for a trading agent."