isitright 0.1.0.dev0__tar.gz

isitright-0.1.0.dev0/.github/workflows/ci.yml

@@ -0,0 +1,33 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+      - run: uv sync --extra dev
+      - run: uv run ruff check .
+      - run: uv run mypy src
+      - run: uv run pytest
+      - run: uv build
+      - name: Verify wheel imports without network
+        shell: bash
+        run: |
+          uv pip install --reinstall --no-deps dist/*.whl
+          .venv/bin/python -c "import isitright; print(isitright.__version__)"
+

isitright-0.1.0.dev0/.github/workflows/publish.yml

@@ -0,0 +1,97 @@
+name: Publish distributions
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.11"
+          enable-cache: true
+      - run: uv sync --extra dev
+      - run: uv run ruff check .
+      - run: uv run mypy src
+      - run: uv run pytest
+      - run: uv build
+      - run: uv run python -m twine check dist/*
+      - uses: actions/upload-artifact@v4
+        with:
+          name: python-distributions
+          path: dist/
+          if-no-files-found: error
+          retention-days: 7
+
+  publish-testpypi:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: testpypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: python-distributions
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  verify-testpypi:
+    needs: publish-testpypi
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - uses: actions/download-artifact@v4
+        with:
+          name: python-distributions
+          path: dist/
+      - name: Read the package version from the built wheel
+        shell: bash
+        run: |
+          wheel=$(find dist -name 'isitright-*.whl' -print -quit)
+          version=${wheel##*/isitright-}
+          version=${version%-py3-none-any.whl}
+          echo "PACKAGE_VERSION=$version" >> "$GITHUB_ENV"
+      - name: Install runtime dependencies from PyPI
+        run: >-
+          python -m pip install
+          "pydantic>=2.10,<3"
+          "pyyaml>=6.0.2,<7"
+          "sqlglot>=27,<29"
+          "typer>=0.15,<1"
+      - name: Install the exact TestPyPI release
+        shell: bash
+        run: |
+          for attempt in 1 2 3 4 5; do
+            python -m pip install \
+              --index-url https://test.pypi.org/simple/ \
+              --no-deps "isitright==$PACKAGE_VERSION" && break
+            if [ "$attempt" = 5 ]; then exit 1; fi
+            sleep 15
+          done
+      - run: python -c "import isitright, os; assert isitright.__version__ == os.environ['PACKAGE_VERSION']"
+      - run: isitright --version
+
+  publish-pypi:
+    needs: verify-testpypi
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: python-distributions
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+

isitright-0.1.0.dev0/.gitignore

@@ -0,0 +1,13 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.coverage
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+.venv/
+.tools/
+build/
+dist/
+htmlcov/
+

isitright-0.1.0.dev0/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 isitright contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

isitright-0.1.0.dev0/PKG-INFO

@@ -0,0 +1,105 @@
+Metadata-Version: 2.4
+Name: isitright
+Version: 0.1.0.dev0
+Summary: Evidence-first stress testing for text-to-SQL evaluations
+Project-URL: Repository, https://github.com/isitright-ai/isitright
+Author: isitright contributors
+License: MIT License
+        
+        Copyright (c) 2026 isitright contributors
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+License-File: LICENSE
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Requires-Dist: pydantic<3,>=2.10
+Requires-Dist: pyyaml<7,>=6.0.2
+Requires-Dist: sqlglot<29,>=27
+Requires-Dist: typer<1,>=0.15
+Provides-Extra: dev
+Requires-Dist: build<2,>=1.2.2; extra == 'dev'
+Requires-Dist: mypy<2,>=1.14; extra == 'dev'
+Requires-Dist: pytest-cov<8,>=6; extra == 'dev'
+Requires-Dist: pytest<9,>=8.3; extra == 'dev'
+Requires-Dist: ruff<1,>=0.9; extra == 'dev'
+Requires-Dist: twine<7,>=6.1; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# isitright
+
+`isitright` is an evidence-first tool for stress-testing text-to-SQL evaluations.
+Given reference and candidate SQL, the eventual v0.1 engine will search for a valid
+database state on which their outputs differ and emit a minimized, replayable
+counterexample. A disagreement is evidence, not automatic attribution of fault.
+
+## Current status: Stage 0 review gate
+
+This repository currently contains package scaffolding and specification contracts
+only. SQL execution, database mutation, constraint validation, search, comparison,
+and minimization are deliberately not implemented. The proposed fixtures under
+`tests/fixtures/proposals` require owner approval before Stage 1 begins.
+
+## Development
+
+Python 3.11 or newer is required. With `uv` installed:
+
+```console
+uv sync --extra dev
+uv run ruff check .
+uv run mypy src
+uv run pytest
+uv build
+```
+
+The package exposes the future command surface now:
+
+```console
+isitright --version
+isitright check --help
+isitright suite --help
+```
+
+Calling `check` or `suite` during Stage 0 exits with the internal/not-implemented
+code because no analysis engine exists yet.
+
+The Stage 0 preview is versioned `0.1.0.dev0`. It is intentionally a development
+release and must not be presented as an implemented analysis engine.
+
+## Contract highlights
+
+- Result statuses keep observed equivalence, state validity, and attribution separate.
+- Unordered query results will be compared as multisets, preserving duplicates.
+- Automatic ordering will honor a top-level `ORDER BY`; otherwise results are unordered.
+- SQLite values are represented with explicit storage classes.
+- Reports use schema version `0.1`; package versioning remains pre-release until an
+  engine is implemented.
+
+See [the contract reference](docs/contracts.md) and
+[the fixture proposal](tests/fixtures/proposals/README.md).
+
+## License
+
+MIT
+

isitright-0.1.0.dev0/README.md

@@ -0,0 +1,56 @@
+# isitright
+
+`isitright` is an evidence-first tool for stress-testing text-to-SQL evaluations.
+Given reference and candidate SQL, the eventual v0.1 engine will search for a valid
+database state on which their outputs differ and emit a minimized, replayable
+counterexample. A disagreement is evidence, not automatic attribution of fault.
+
+## Current status: Stage 0 review gate
+
+This repository currently contains package scaffolding and specification contracts
+only. SQL execution, database mutation, constraint validation, search, comparison,
+and minimization are deliberately not implemented. The proposed fixtures under
+`tests/fixtures/proposals` require owner approval before Stage 1 begins.
+
+## Development
+
+Python 3.11 or newer is required. With `uv` installed:
+
+```console
+uv sync --extra dev
+uv run ruff check .
+uv run mypy src
+uv run pytest
+uv build
+```
+
+The package exposes the future command surface now:
+
+```console
+isitright --version
+isitright check --help
+isitright suite --help
+```
+
+Calling `check` or `suite` during Stage 0 exits with the internal/not-implemented
+code because no analysis engine exists yet.
+
+The Stage 0 preview is versioned `0.1.0.dev0`. It is intentionally a development
+release and must not be presented as an implemented analysis engine.
+
+## Contract highlights
+
+- Result statuses keep observed equivalence, state validity, and attribution separate.
+- Unordered query results will be compared as multisets, preserving duplicates.
+- Automatic ordering will honor a top-level `ORDER BY`; otherwise results are unordered.
+- SQLite values are represented with explicit storage classes.
+- Reports use schema version `0.1`; package versioning remains pre-release until an
+  engine is implemented.
+
+See [the contract reference](docs/contracts.md) and
+[the fixture proposal](tests/fixtures/proposals/README.md).
+
+## License
+
+MIT
+

isitright-0.1.0.dev0/docs/contracts.md

@@ -0,0 +1,78 @@
+# Stage 0 contract reference
+
+This document freezes the interfaces proposed for schema version `0.1`. It does
+not claim that the analysis engine is implemented.
+
+## Public API
+
+`check(...)` accepts a concrete SQLite database, two SQL strings, an optional
+question, an optional constraints file, a `CheckConfig`, and an optional trusted
+validator callback. `check_suite(...)` accepts the same shared inputs plus a JSONL
+case file. During Stage 0 both functions raise `StageNotImplementedError`.
+
+`CheckConfig` defaults to seed `0`, 100 attempts, a 30-second query budget,
+positional column comparison, automatic ordering, and a `1e-9` relative float
+tolerance.
+
+## Status fields
+
+The three result axes are deliberately independent:
+
+- `equivalence_status`: `COUNTEREXAMPLE_FOUND`, `NOT_FOUND_WITHIN_BUDGET`,
+  `INCONCLUSIVE`, or `UNSUPPORTED`.
+- `validity_status`: `DOMAIN_VALIDATED`, `SCHEMA_VALID_ONLY`, `INVALID`, or
+  `UNKNOWN`.
+- `attribution_status`: `UNDETERMINED` in core v0.1. Values reserved for future
+  adjudication are `AGENT_DEFECT`, `REFERENCE_DEFECT`, `BOTH_DEFECTIVE`, and
+  `AMBIGUOUS_TASK`.
+
+Headlines summarize evidence and may not contain the standalone words “pass,”
+“fail,” “correct,” or “wrong.”
+
+## Comparison contract
+
+- Unordered rows are a multiset. Duplicate multiplicity is significant.
+- `auto` ordering treats a top-level `ORDER BY` as ordered and all other queries
+  as unordered. `ordered` and `unordered` override this.
+- Column comparison is `positional`, `by_name`, or `strict`.
+- Values are tagged as SQLite `null`, `integer`, `real`, `text`, or `blob`.
+  Blobs are base64 strings in JSON. Types are significant; only real values use
+  the configured relative tolerance.
+
+## Suite input
+
+Each nonblank JSONL line requires `case_id`, `reference_sql`, and
+`candidate_sql`. `question` is optional. All other properties are moved into the
+record's `metadata` mapping. Case IDs must be unique within a suite.
+
+## Reports and reproducer
+
+A case report contains the version, status axes, evidence-only headline, inputs
+and SHA-256 hashes, active comparison policy, original-state result, search
+accounting, checked rules, assumptions, evidence, errors, timing, and carried
+metadata. Suite reports contain all cases, status totals, mutator-family totals,
+timing, and metadata.
+
+When later stages find a counterexample, `counterexample.sql` will contain only
+deterministic DDL and minimized inserts needed to rebuild the state. The report
+retains both queries, both typed outputs, the original injected change, and the
+exact replay command. This separation keeps the SQL fixture independently
+inspectable without embedding shell-specific behavior.
+
+Suite ranking is stable within four priority groups:
+
+1. Counterexamples on domain-validated states.
+2. Counterexamples on schema-valid-only states.
+3. Inconclusive or unsupported cases, plus counterexamples of unknown validity.
+4. Cases with no divergence found within budget.
+
+## CLI exit codes
+
+- `0`: analysis completed, including unsupported and not-found outcomes.
+- `1`: internal failure or a Stage 0 unimplemented command.
+- `2`: invalid invocation or input.
+- `3`: an explicit `--fail-on` policy matched.
+
+`--fail-on` accepts `none`, `domain-validated`, or `any-counterexample` and never
+changes attribution.
+

isitright-0.1.0.dev0/docs/releasing.md

@@ -0,0 +1,44 @@
+# Releasing
+
+`isitright` uses Hatchling as its PEP 517 build backend and `uv build` as the
+build frontend. The version has one source of truth in
+`src/isitright/__about__.py`.
+
+Hatchling is temporarily capped below 1.32 so distributions use Core Metadata
+2.4, which the current Twine release can validate. Remove the cap when the
+publishing toolchain supports Hatchling's newer default metadata version.
+
+## Release invariants
+
+- Publish only from the `main` branch after CI passes.
+- Build the source distribution and wheel exactly once.
+- Run `twine check` before either upload.
+- Publish to TestPyPI and smoke-test the indexed wheel before approving PyPI.
+- Promote the exact same downloaded GitHub artifact to PyPI; never rebuild it.
+- Never reuse a version. PyPI files are immutable even if a release is deleted.
+- Configure separate `testpypi` and `pypi` GitHub environments with required
+  reviewers for the production environment.
+
+## One-time registry setup
+
+Create pending Trusted Publishers on both registries with:
+
+- Owner: `isitright-ai`
+- Repository: `isitright`
+- Workflow: `publish.yml`
+- Environment: `testpypi` on TestPyPI and `pypi` on PyPI
+
+The repository must exist on GitHub before the workflow can run. No long-lived
+API token is stored in GitHub; the publish jobs request short-lived OIDC
+credentials.
+
+## Release
+
+1. Confirm the version in `src/isitright/__about__.py` is new.
+2. Push the reviewed commit to `main`.
+3. Run the **Publish distributions** workflow manually.
+4. Approve the `testpypi` environment deployment.
+5. Wait for the TestPyPI installation smoke test to pass.
+6. Approve the `pypi` environment deployment.
+7. Verify the project page and install the exact version in a clean environment.
+

isitright-0.1.0.dev0/pyproject.toml

@@ -0,0 +1,73 @@
+[build-system]
+requires = ["hatchling>=1.27,<1.32"]
+build-backend = "hatchling.build"
+
+[project]
+name = "isitright"
+dynamic = ["version"]
+description = "Evidence-first stress testing for text-to-SQL evaluations"
+readme = "README.md"
+requires-python = ">=3.11"
+license = { file = "LICENSE" }
+authors = [{ name = "isitright contributors" }]
+classifiers = [
+  "Development Status :: 2 - Pre-Alpha",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+]
+dependencies = [
+  "pydantic>=2.10,<3",
+  "pyyaml>=6.0.2,<7",
+  "sqlglot>=27,<29",
+  "typer>=0.15,<1",
+]
+
+[project.optional-dependencies]
+dev = [
+  "build>=1.2.2,<2",
+  "mypy>=1.14,<2",
+  "pytest>=8.3,<9",
+  "pytest-cov>=6,<8",
+  "ruff>=0.9,<1",
+  "twine>=6.1,<7",
+]
+
+[project.scripts]
+isitright = "isitright.cli:app"
+
+[project.urls]
+Repository = "https://github.com/isitright-ai/isitright"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/isitright"]
+
+[tool.hatch.version]
+path = "src/isitright/__about__.py"
+
+[tool.pytest.ini_options]
+addopts = "--strict-config --strict-markers --cov=isitright --cov-report=term-missing"
+testpaths = ["tests"]
+
+[tool.coverage.run]
+branch = true
+source = ["isitright"]
+
+[tool.coverage.report]
+fail_under = 90
+show_missing = true
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM", "RUF"]
+
+[tool.mypy]
+python_version = "3.11"
+strict = true
+packages = ["isitright"]
+

isitright-0.1.0.dev0/src/isitright/__about__.py

@@ -0,0 +1,4 @@
+"""Package metadata kept independent from runtime imports."""
+
+__version__ = "0.1.0.dev0"
+

isitright-0.1.0.dev0/src/isitright/__init__.py

@@ -0,0 +1,34 @@
+"""Public package interface for isitright."""
+
+from isitright.__about__ import __version__
+from isitright.api import StageNotImplementedError, check, check_suite
+from isitright.models import (
+    AttributionStatus,
+    CaseResult,
+    CheckConfig,
+    ColumnComparisonPolicy,
+    EquivalenceStatus,
+    FailOnPolicy,
+    OrderingPolicy,
+    SuiteCase,
+    SuiteResult,
+    ValidityStatus,
+)
+
+__all__ = [
+    "AttributionStatus",
+    "CaseResult",
+    "CheckConfig",
+    "ColumnComparisonPolicy",
+    "EquivalenceStatus",
+    "FailOnPolicy",
+    "OrderingPolicy",
+    "StageNotImplementedError",
+    "SuiteCase",
+    "SuiteResult",
+    "ValidityStatus",
+    "__version__",
+    "check",
+    "check_suite",
+]
+

isitright-0.1.0.dev0/src/isitright/api.py

@@ -0,0 +1,46 @@
+"""Reserved public API for the future analysis engine."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from os import PathLike
+from pathlib import Path
+
+from isitright.models import CaseResult, CheckConfig, SuiteResult
+
+
+class StageNotImplementedError(RuntimeError):
+    """Raised when a post-Stage-0 capability is invoked."""
+
+
+def check(
+    *,
+    database: str | PathLike[str],
+    reference_sql: str,
+    candidate_sql: str,
+    question: str | None = None,
+    constraints_file: str | PathLike[str] | None = None,
+    config: CheckConfig | None = None,
+    validator: Callable[[Path], bool] | None = None,
+) -> CaseResult:
+    """Compare one SQL pair once the Stage 1+ engine is implemented."""
+    del database, reference_sql, candidate_sql, question, constraints_file, config, validator
+    raise StageNotImplementedError(
+        "check() is a Stage 1+ capability; Stage 0 defines its contract only"
+    )
+
+
+def check_suite(
+    *,
+    database: str | PathLike[str],
+    cases: str | PathLike[str],
+    constraints_file: str | PathLike[str] | None = None,
+    config: CheckConfig | None = None,
+    validator: Callable[[Path], bool] | None = None,
+) -> SuiteResult:
+    """Compare a JSONL suite once the Stage 3 engine is implemented."""
+    del database, cases, constraints_file, config, validator
+    raise StageNotImplementedError(
+        "check_suite() is a Stage 3 capability; Stage 0 defines its contract only"
+    )
+