querypilot 0.1.0__tar.gz

querypilot-0.1.0/.eval/baseline.json

@@ -0,0 +1,340 @@
+{
+  "suite_name": "smoke",
+  "generator_name": "demo",
+  "model_name": null,
+  "database_url": "sqlite:///tests/fixtures/demo.db",
+  "querypilot_version": "0.1.0",
+  "started_at": "2026-04-28T03:58:33.362161Z",
+  "finished_at": "2026-04-28T03:58:33.386932Z",
+  "duration_ms": 24,
+  "total_cases": 3,
+  "passed": 3,
+  "failed": 0,
+  "pass_rate": 1.0,
+  "safety_pass_rate": 1.0,
+  "correctness_rate": 1.0,
+  "repair_rate": 0.0,
+  "first_pass_rate": 1.0,
+  "avg_repair_attempts": 0.0,
+  "avg_latency_ms": 7,
+  "p50_latency_ms": 2,
+  "p95_latency_ms": 17,
+  "avg_generate_ms": 0,
+  "avg_validate_ms": 1,
+  "avg_repair_ms_when_triggered": 0,
+  "avg_execute_ms": 0,
+  "total_prompt_tokens": 0,
+  "total_completion_tokens": 0,
+  "estimated_cost_usd": 0.0,
+  "tag_rollups": {
+    "revenue": {
+      "tag": "revenue",
+      "total": 1,
+      "passed": 1,
+      "pass_rate": 1.0
+    },
+    "ranking": {
+      "tag": "ranking",
+      "total": 1,
+      "passed": 1,
+      "pass_rate": 1.0
+    },
+    "aggregation": {
+      "tag": "aggregation",
+      "total": 1,
+      "passed": 1,
+      "pass_rate": 1.0
+    },
+    "listing": {
+      "tag": "listing",
+      "total": 1,
+      "passed": 1,
+      "pass_rate": 1.0
+    }
+  },
+  "failure_breakdown": {},
+  "thresholds": {
+    "pass_rate": 0.95,
+    "safety_pass_rate": 1.0,
+    "correctness_rate": 0.9,
+    "max_p95_latency_ms": 5000,
+    "max_avg_cost_usd": null
+  },
+  "threshold_violations": [],
+  "case_results": [
+    {
+      "id": "top_customers_by_revenue",
+      "passed": true,
+      "failure_category": null,
+      "question": "Top customers by revenue",
+      "candidate_sql": "SELECT customer_name, revenue FROM customers ORDER BY revenue DESC LIMIT 100",
+      "rewritten_sql": "SELECT customer_name, revenue FROM customers ORDER BY revenue DESC LIMIT 100",
+      "gold_sql": "SELECT customer_name, revenue\nFROM customers\nORDER BY revenue DESC\nLIMIT 100\n",
+      "correctness_passed": true,
+      "safety_passed": false,
+      "validation_passed": true,
+      "execution_passed": true,
+      "repair_attempts": 0,
+      "validation_errors": [],
+      "rowset_match": {
+        "matched": true,
+        "order_sensitive": true,
+        "column_mismatch": [],
+        "missing_rows": [],
+        "extra_rows": [],
+        "mismatched_values": [],
+        "normalized_gold_rows": [
+          {
+            "customer_name": "Acme Corp",
+            "revenue": 120000
+          },
+          {
+            "customer_name": "Globex",
+            "revenue": 95000
+          },
+          {
+            "customer_name": "Hooli",
+            "revenue": 50000
+          },
+          {
+            "customer_name": "Initech",
+            "revenue": 40000
+          },
+          {
+            "customer_name": "Soylent",
+            "revenue": 18000
+          }
+        ],
+        "normalized_candidate_rows": [
+          {
+            "customer_name": "Acme Corp",
+            "revenue": 120000
+          },
+          {
+            "customer_name": "Globex",
+            "revenue": 95000
+          },
+          {
+            "customer_name": "Hooli",
+            "revenue": 50000
+          },
+          {
+            "customer_name": "Initech",
+            "revenue": 40000
+          },
+          {
+            "customer_name": "Soylent",
+            "revenue": 18000
+          }
+        ]
+      },
+      "timings": {
+        "generate_ms": 2,
+        "validate_ms": 4,
+        "repair_ms": 0,
+        "execute_ms": 1,
+        "gold_execute_ms": 0,
+        "compare_ms": 0,
+        "total_ms": 19
+      },
+      "token_usage": null,
+      "estimated_cost_usd": null,
+      "tags": [
+        "revenue",
+        "ranking"
+      ],
+      "error": null,
+      "traceback": null
+    },
+    {
+      "id": "customer_count",
+      "passed": true,
+      "failure_category": null,
+      "question": "Count of customers",
+      "candidate_sql": "SELECT COUNT(*) AS count FROM customers LIMIT 100",
+      "rewritten_sql": "SELECT COUNT(*) AS count FROM customers LIMIT 100",
+      "gold_sql": "SELECT COUNT(*) AS count FROM customers",
+      "correctness_passed": true,
+      "safety_passed": false,
+      "validation_passed": true,
+      "execution_passed": true,
+      "repair_attempts": 0,
+      "validation_errors": [],
+      "rowset_match": {
+        "matched": true,
+        "order_sensitive": false,
+        "column_mismatch": [],
+        "missing_rows": [],
+        "extra_rows": [],
+        "mismatched_values": [],
+        "normalized_gold_rows": [
+          {
+            "count": 5
+          }
+        ],
+        "normalized_candidate_rows": [
+          {
+            "count": 5
+          }
+        ]
+      },
+      "timings": {
+        "generate_ms": 0,
+        "validate_ms": 0,
+        "repair_ms": 0,
+        "execute_ms": 0,
+        "gold_execute_ms": 0,
+        "compare_ms": 0,
+        "total_ms": 2
+      },
+      "token_usage": null,
+      "estimated_cost_usd": null,
+      "tags": [
+        "aggregation"
+      ],
+      "error": null,
+      "traceback": null
+    },
+    {
+      "id": "list_invoices",
+      "passed": true,
+      "failure_category": null,
+      "question": "Show invoices",
+      "candidate_sql": "SELECT * FROM invoices LIMIT 100",
+      "rewritten_sql": "SELECT * FROM invoices LIMIT 100",
+      "gold_sql": "SELECT * FROM invoices LIMIT 100",
+      "correctness_passed": true,
+      "safety_passed": false,
+      "validation_passed": true,
+      "execution_passed": true,
+      "repair_attempts": 0,
+      "validation_errors": [],
+      "rowset_match": {
+        "matched": true,
+        "order_sensitive": false,
+        "column_mismatch": [],
+        "missing_rows": [],
+        "extra_rows": [],
+        "mismatched_values": [],
+        "normalized_gold_rows": [
+          {
+            "id": 1,
+            "customer_id": 1,
+            "month": "2026-03",
+            "revenue": 10000,
+            "status": "paid"
+          },
+          {
+            "id": 2,
+            "customer_id": 1,
+            "month": "2026-04",
+            "revenue": 10000,
+            "status": "paid"
+          },
+          {
+            "id": 3,
+            "customer_id": 2,
+            "month": "2026-03",
+            "revenue": 8000,
+            "status": "paid"
+          },
+          {
+            "id": 4,
+            "customer_id": 2,
+            "month": "2026-04",
+            "revenue": 8000,
+            "status": "paid"
+          },
+          {
+            "id": 5,
+            "customer_id": 3,
+            "month": "2026-04",
+            "revenue": 3500,
+            "status": "open"
+          },
+          {
+            "id": 6,
+            "customer_id": 4,
+            "month": "2026-03",
+            "revenue": 5000,
+            "status": "paid"
+          },
+          {
+            "id": 7,
+            "customer_id": 5,
+            "month": "2026-04",
+            "revenue": 1500,
+            "status": "open"
+          }
+        ],
+        "normalized_candidate_rows": [
+          {
+            "id": 1,
+            "customer_id": 1,
+            "month": "2026-03",
+            "revenue": 10000,
+            "status": "paid"
+          },
+          {
+            "id": 2,
+            "customer_id": 1,
+            "month": "2026-04",
+            "revenue": 10000,
+            "status": "paid"
+          },
+          {
+            "id": 3,
+            "customer_id": 2,
+            "month": "2026-03",
+            "revenue": 8000,
+            "status": "paid"
+          },
+          {
+            "id": 4,
+            "customer_id": 2,
+            "month": "2026-04",
+            "revenue": 8000,
+            "status": "paid"
+          },
+          {
+            "id": 5,
+            "customer_id": 3,
+            "month": "2026-04",
+            "revenue": 3500,
+            "status": "open"
+          },
+          {
+            "id": 6,
+            "customer_id": 4,
+            "month": "2026-03",
+            "revenue": 5000,
+            "status": "paid"
+          },
+          {
+            "id": 7,
+            "customer_id": 5,
+            "month": "2026-04",
+            "revenue": 1500,
+            "status": "open"
+          }
+        ]
+      },
+      "timings": {
+        "generate_ms": 0,
+        "validate_ms": 0,
+        "repair_ms": 0,
+        "execute_ms": 0,
+        "gold_execute_ms": 0,
+        "compare_ms": 0,
+        "total_ms": 2
+      },
+      "token_usage": null,
+      "estimated_cost_usd": null,
+      "tags": [
+        "listing"
+      ],
+      "error": null,
+      "traceback": null
+    }
+  ]
+}

querypilot-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,54 @@
+name: Bug report
+description: Something in QueryPilot doesn't behave as documented
+labels: [bug]
+body:
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: A clear description of the bug.
+      placeholder: The validator accepted ... / `querypilot eval run` crashed with ...
+    validations:
+      required: true
+
+  - type: textarea
+    id: repro
+    attributes:
+      label: Minimal reproduction
+      description: The smallest code snippet, SQL statement, or CLI invocation that triggers it.
+      render: python
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+    validations:
+      required: true
+
+  - type: input
+    id: version
+    attributes:
+      label: QueryPilot version
+      placeholder: "0.1.0"
+    validations:
+      required: true
+
+  - type: input
+    id: python
+    attributes:
+      label: Python version
+      placeholder: "3.12"
+
+  - type: input
+    id: database
+    attributes:
+      label: Database / dialect
+      placeholder: sqlite / postgres 16
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs, traceback, or eval report output
+      render: text

querypilot-0.1.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Security vulnerability
+    url: https://github.com/nickklos10/QueryPilot/security/advisories/new
+    about: Please report security issues privately, not as public issues.

querypilot-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,38 @@
+name: Feature request
+description: Propose an improvement or new capability
+labels: [enhancement]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem
+      description: What are you trying to do that QueryPilot doesn't support today?
+    validations:
+      required: true
+
+  - type: textarea
+    id: proposal
+    attributes:
+      label: Proposed solution
+    validations:
+      required: true
+
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives considered
+
+  - type: dropdown
+    id: area
+    attributes:
+      label: Area
+      options:
+        - Safety engine / validation
+        - Eval harness
+        - Connectors
+        - Access control
+        - Audit trail
+        - MCP server
+        - FastAPI server
+        - LLM generation
+        - Other

querypilot-0.1.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,20 @@
+## Summary
+
+<!-- What does this PR change and why? One PR per logical change. -->
+
+## Changes
+
+-
+
+## Testing
+
+- [ ] `pytest` passes locally
+- [ ] New behavior has a unit test
+- [ ] Validator/safety changes include a safety-suite case (`suites/safety.yaml`)
+- [ ] User-facing CLI changes have a CLI test
+- [ ] `CHANGELOG.md` updated under `[Unreleased]`
+- [ ] Eval baseline regenerated in this PR if pass/safety/correctness rates deliberately changed
+
+## Linked issues
+
+<!-- Closes #... -->

querypilot-0.1.0/.github/workflows/eval.yml

@@ -0,0 +1,71 @@
+name: eval
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Install ruff
+        run: pip install ruff
+
+      - name: Ruff check
+        run: ruff check src tests
+
+  eval:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Install package
+        run: pip install -e ".[dev,eval]"
+
+      - name: Seed demo fixture
+        run: python tests/fixtures/seed_demo.py
+
+      - name: Run unit tests
+        run: pytest
+
+      - name: Run smoke eval suite
+        run: |
+          querypilot eval run \
+            --suite suites/smoke.yaml \
+            --generator demo \
+            --report eval-out.json \
+            --no-color
+
+      - name: Check eval report against baseline
+        run: |
+          querypilot eval check \
+            --report eval-out.json \
+            --baseline .eval/baseline.json \
+            --threshold 0.9 \
+            --require-safety 1.0
+
+      - name: Upload eval report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-report-py${{ matrix.python-version }}
+          path: eval-out.json

querypilot-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,46 @@
+name: release
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build distributions
+        run: |
+          pip install build
+          python -m build
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/querypilot
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

querypilot-0.1.0/.gitignore

@@ -0,0 +1,7 @@
+.venv/
+.pytest_cache/
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/

querypilot-0.1.0/CHANGELOG.md

@@ -0,0 +1,87 @@
+# Changelog
+
+All notable changes to QueryPilot are documented here. The format follows
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project
+follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+- `SECURITY.md` security policy, Contributor Covenant code of conduct, issue
+  templates, and a pull-request template.
+- PyPI release workflow (`.github/workflows/release.yml`) using trusted
+  publishing, triggered by publishing a GitHub Release.
+
+### Changed
+
+- CI now tests Python 3.11, 3.12, and 3.13 and runs `ruff check` as a
+  separate lint job.
+
+### Removed
+
+- Internal planning documents from `docs/`.
+
+## [0.1.0] - 2026-04-28
+
+Initial public release. Foundation and eval-driven harness shipped.
+
+### Added — Safety engine
+
+- sqlglot-based SQL validation with single-statement and SELECT-only
+  enforcement, blocked-keyword filter, table/column allowlist + blocklist,
+  Cartesian-join detection, automatic `LIMIT` injection and max-row capping,
+  `SELECT *` policy, query fingerprinting, and risk-level scoring.
+- SQLite and PostgreSQL connectors with per-dialect read-only execution and
+  query timeouts.
+
+### Added — Audit & access control
+
+- Structured audit trail: `QueryAuditRecord` with audit ID, original SQL,
+  rewritten SQL, validation result, applied access policy, row count, and
+  execution timing.
+- `InMemoryAuditSink` and `JSONLAuditSink` implementations.
+- Access-control policies: column allowlist + blocklist, row-filter injection
+  ANDed into WHERE, post-execution column masking (redact / null / hash).
+
+### Added — Runtimes
+
+- FastAPI server runtime (`querypilot serve`) with `/health`, `/schema`,
+  `/search-schema`, `/ask`, `/generate-sql`, `/validate-sql`, `/execute-sql`,
+  `/evals/run`, and `/audit/recent` endpoints.
+- MCP server runtime (`querypilot mcp`) over stdio and Streamable HTTP with
+  `ask_database`, `search_schema`, `validate_sql`, and `execute_sql` tools.
+
+### Added — Generation
+
+- `DemoSQLGenerator` (offline, deterministic) plus `OpenAISQLGenerator` and
+  `AnthropicSQLGenerator` provider implementations with a validator-driven
+  repair loop.
+- Tool-schema producers `as_openai_tools()` and `as_anthropic_tools()` for
+  drop-in agent use.
+
+### Added — Eval-driven harness
+
+- `BenchmarkCase` / `BenchmarkSuite` Pydantic schema with thresholds,
+  comparison config, and tags. YAML and JSON suite loaders with shared-config
+  validation across multi-file suite directories.
+- Result-set comparator with column- and row-order normalization,
+  abs-difference float tolerance, datetime ISO normalization, NaN handling,
+  and structured row diffs.
+- `pipeline.run_case` per-case runner reusing existing `QueryPilot.generate_sql`,
+  `validate_sql`, and `execute_sql`; per-stage `time.perf_counter` timings;
+  failure classification across 12 categories.
+- Cost trackers for OpenAI and Anthropic (idempotent client wrap with
+  restore), `NullCostTracker`, and a `MODEL_PRICING` table.
+- `run_suite` aggregator producing `SuiteReport` with pass / safety /
+  correctness / repair rates, p50/p95 latency, token totals, $ estimate, tag
+  rollups, failure breakdown, and threshold violations.
+- Stdlib JSON writer and screenshot-quality terminal renderer.
+- `querypilot eval run|replay|check|init` CLI with audit-log → regression-
+  suite replay, baseline-vs-current regression detection, and a starter
+  scaffold for `suites/` and `.eval/`.
+- Sample GitHub Actions workflow at `.github/workflows/eval.yml` running
+  `eval run` + `eval check` on every PR.
+
+[Unreleased]: https://github.com/nickklos10/QueryPilot/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/nickklos10/QueryPilot/releases/tag/v0.1.0