promptline-opt 0.1.0__tar.gz

promptline_opt-0.1.0/.claude/settings.local.json

@@ -0,0 +1,10 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(mkdir -p /tmp/pl-demo)",
+      "Read(//private/tmp/pl-demo/**)",
+      "Bash(rm -rf ./* .promptline)",
+      "Bash(uv run *)"
+    ]
+  }
+}

promptline_opt-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,45 @@
+name: Bug report
+description: Report something that isn't working as expected
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: Thanks for taking the time to file a bug. Please fill in the sections below.
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: A clear description of the bug, and what you expected instead.
+      placeholder: When I run `promptline optimize ...`, it ...
+    validations:
+      required: true
+  - type: textarea
+    id: repro
+    attributes:
+      label: Steps to reproduce
+      description: Ideally the exact commands. A `PROMPTLINE_FAKE_SCRIPT` repro that needs no API key is ideal.
+      placeholder: |
+        1. promptline init
+        2. promptline optimize --optimizer gepa ...
+        3. ...
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant output / traceback
+      render: shell
+  - type: input
+    id: version
+    attributes:
+      label: Promptline version
+      description: Output of `promptline --version`
+    validations:
+      required: true
+  - type: input
+    id: environment
+    attributes:
+      label: Environment
+      description: OS, Python version, install method (pip / editable / clone)
+    validations:
+      required: true

promptline_opt-0.1.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Question / discussion
+    url: https://github.com/sanskarpan/promptline/discussions
+    about: Ask a question or discuss an idea before filing an issue.

promptline_opt-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,27 @@
+name: Feature request
+description: Suggest an idea or enhancement
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem / motivation
+      description: What are you trying to do that Promptline doesn't support today?
+    validations:
+      required: true
+  - type: textarea
+    id: proposal
+    attributes:
+      label: Proposed solution
+      description: What would you like to see? (A new optimizer, a config option, an API endpoint, etc.)
+    validations:
+      required: true
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives considered
+  - type: textarea
+    id: context
+    attributes:
+      label: Additional context
+      description: Links to papers, related tools, or prior art.

promptline_opt-0.1.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,23 @@
+<!-- Thanks for contributing to Promptline! -->
+
+## Summary
+
+<!-- What does this PR change and why? -->
+
+## Related issues
+
+<!-- e.g. Closes #123 -->
+
+## Changes
+
+<!-- Bullet the notable changes. -->
+-
+
+## Checklist
+
+- [ ] `uv run pytest -q` passes
+- [ ] `uv run ruff check .` and `uv run ruff format --check .` pass
+- [ ] `uv run pyright promptline` is clean
+- [ ] Tests added/updated for the change (TDD; `FakeLLMClient` — no live LLM calls in unit tests)
+- [ ] Docs updated if behavior/CLI/API changed (`README.md`, `docs/concepts/`)
+- [ ] For web changes: `cd web && npm test && npm run build` pass

promptline_opt-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,103 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  python:
+    name: python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Sync dependencies
+        run: uv sync --all-extras --dev
+
+      - name: Ruff lint
+        run: uv run ruff check .
+
+      - name: Ruff format check
+        run: uv run ruff format --check .
+
+      - name: Pyright
+        run: uv run pyright promptline
+
+      - name: Pytest (offline suite)
+        run: uv run pytest -q
+
+  web:
+    name: web (vitest + build)
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: web
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: npm
+          cache-dependency-path: web/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Unit tests
+        run: npm test
+
+      - name: Production build
+        run: npm run build
+
+  dashboard-e2e:
+    name: dashboard e2e (playwright)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Sync Python dependencies
+        run: uv sync --all-extras --dev
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: npm
+          cache-dependency-path: web/package-lock.json
+
+      - name: Install web dependencies and build
+        working-directory: web
+        run: |
+          npm ci
+          npm run build
+
+      - name: Install Playwright browser
+        working-directory: web
+        run: npx playwright install --with-deps chromium
+
+      - name: Run dashboard e2e
+        working-directory: web
+        run: npm run e2e

promptline_opt-0.1.0/.gitignore

@@ -0,0 +1,18 @@
+.venv/
+runs/
+.promptline/
+node_modules/
+web/dist/
+__pycache__/
+.pytest_cache/
+*.pyc
+*.egg-info/
+dist/
+build/
+.DS_Store
+promptline.yaml
+
+# Playwright
+web/test-results/
+web/playwright-report/
+web/.playwright/

promptline_opt-0.1.0/CHANGELOG.md

@@ -0,0 +1,47 @@
+# Changelog
+
+All notable changes to this project are documented here. The format is based on
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
+adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.0] — 2026-07-06
+
+Initial release. The full `calibrate → optimize → gate → serve` pipeline.
+
+### Added
+
+- **Core library**: declarative signatures/programs with per-module trace
+  recording, an `LLMClient` protocol with an OpenRouter adapter (BYO key, pooled
+  connections, retries with backoff, per-call cost tracking), a SQLite call
+  cache, and an eval harness with hard rollout/cost budget walls.
+- **Optimizers (from scratch, one contract)**: GEPA (per-instance Pareto
+  selection, reflective mutation, system-aware merge, resumable checkpoints),
+  MIPRO (bootstrap demos → grounded instruction proposals → TPE search),
+  BootstrapFewShot (+ random search), ProTeGi (textual gradients + CAPO-style
+  racing), and OPRO.
+- **Judge subsystem**: pointwise and pairwise rubric judges (CoT-before-verdict,
+  anti-verbosity, position-swap debiasing, k-sampling), agreement metrics
+  (Cohen's κ, Spearman, pairwise accuracy), a calibrator that certifies the judge
+  against human labels, and meta-optimization of the judge prompt. The calibrated
+  judge is the default optimization/gate metric, and a passing certificate is
+  required to optimize.
+- **Statistical deploy gate**: paired bootstrap CIs, Holm correction across
+  candidates, independent validation-split confirmation, verbosity tripwires, and
+  refusals for uncalibrated judges, undersized splits, and dev/val contamination.
+- **Registry & serving**: SQLite-backed versioned prompt registry with lineage,
+  an active pointer only the gate advances, rollback, and a FastAPI serving plane
+  (`GET /prompts/{program}/active` with ETag) plus a control plane with SSE run
+  streaming.
+- **Interfaces**: a Typer CLI, a Textual TUI cockpit, and a React + Vite
+  dashboard (Runs, Lineage, Judge, Gate, Registry) served statically by FastAPI.
+- **Data**: JSONL schema + adapters, and loaders for HelpSteer2, Bitext, and
+  MT-Bench human judgments.
+- **Demo**: `promptline demo setup` (online and `--offline`) for the
+  support-assistant walkthrough.
+- **Tests**: 444 offline unit/integration tests, an offline end-to-end suite,
+  Playwright dashboard e2e, and an opt-in live smoke test.
+
+[Unreleased]: https://github.com/sanskarpan/promptline/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/sanskarpan/promptline/releases/tag/v0.1.0

promptline_opt-0.1.0/CODE_OF_CONDUCT.md

@@ -0,0 +1,47 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment:
+
+- Demonstrating empathy and kindness toward other people
+- Being respectful of differing opinions, viewpoints, and experiences
+- Giving and gracefully accepting constructive feedback
+- Accepting responsibility and apologizing to those affected by our mistakes
+- Focusing on what is best for the overall community
+
+Examples of unacceptable behavior:
+
+- The use of sexualized language or imagery, and sexual attention or advances
+- Trolling, insulting or derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information without explicit permission
+- Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the project maintainer. All complaints will be reviewed and
+investigated promptly and fairly. The maintainer is obligated to respect the
+privacy and security of the reporter of any incident.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+https://www.contributor-covenant.org/version/2/1/code_of_conduct.html.
+
+[homepage]: https://www.contributor-covenant.org

promptline_opt-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,42 @@
+# Contributing to Promptline
+
+## Setup
+
+```sh
+uv sync                 # Python 3.11+, installs promptline + dev deps
+cd web && npm install   # only needed for dashboard work
+```
+
+## Test tiers
+
+| Tier | Command | Needs | When |
+|---|---|---|---|
+| Offline suite (unit + e2e) | `make test` (`uv run pytest -q`) | nothing | every change |
+| Dashboard e2e | `make e2e` | `npm run build` in `web/` + `npx playwright install chromium` | dashboard changes |
+| Live smoke | `make live-smoke` | `OPENROUTER_API_KEY` (< $0.50) | before releases / adapter changes |
+
+The offline suite is the gate for merging; the live tier is opt-in and marked
+`@pytest.mark.live` (auto-skipped without a key). Lint with `make lint`.
+
+## The fake-script pattern
+
+Tests never hit a real LLM. Two mechanisms:
+
+1. **In Python** — `FakeLLMClient(script=...)` accepts a list of canned
+   responses or a callable `(LLMCall) -> str`. The canonical e2e trick
+   (see `tests/e2e/conftest.py`): the client answers with a marker string
+   iff the system prompt already contains it, while scripted
+   reflection/proposal calls return an instruction that *adds* the marker —
+   so a metric that rewards the marker makes optimizers genuinely "improve".
+2. **Through the CLI** — set `PROMPTLINE_FAKE_SCRIPT=/path/to/script.json`.
+   The JSON holds a cycling `"responses": [...]` list and/or first-match
+   `"keyed": [{"contains": ..., "response": ...}]` rules matched against the
+   full prompt text. See `tests/e2e/test_cli_e2e.py` for a complete
+   init → calibrate → optimize → gate → serve chain driven this way.
+
+## Conventions
+
+- Conventional commits (`feat:`, `fix:`, `test:`, `docs:` …).
+- `uv run ruff check .` must pass; keep new tests deterministic (seeded RNG,
+  scripted clients, tmp_path workspaces).
+- Prefer composing real components in tests and faking only the LLM.

promptline_opt-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sanskar Pandey
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

promptline_opt-0.1.0/Makefile

@@ -0,0 +1,39 @@
+.PHONY: help install test e2e live-smoke lint format format-check typecheck check web-test web-build build
+
+help:  ## Show this help
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-14s\033[0m %s\n", $$1, $$2}'
+
+install:  ## Sync all dependencies (runtime + data extra + dev)
+	uv sync --all-extras --dev
+
+test:  ## Offline test suite (fast, no network, no API key)
+	uv run pytest -q
+
+lint:  ## Ruff lint
+	uv run ruff check .
+
+format:  ## Apply Ruff formatting
+	uv run ruff format .
+
+format-check:  ## Check Ruff formatting without writing
+	uv run ruff format --check .
+
+typecheck:  ## Pyright type check
+	uv run pyright promptline
+
+check: lint format-check typecheck test  ## Run the full local gate (matches CI python job)
+
+web-test:  ## Dashboard unit tests (vitest)
+	cd web && npm test
+
+web-build:  ## Build the dashboard (writes web/dist)
+	cd web && npm run build
+
+e2e:  ## Dashboard Playwright e2e (needs web-build + chromium; see web/README.md)
+	cd web && npm run e2e
+
+live-smoke:  ## Opt-in live smoke against OpenRouter (needs OPENROUTER_API_KEY; ~<$0.50)
+	uv run pytest -m live tests/integration -q
+
+build:  ## Build the sdist + wheel
+	uv build