evalforge-ci 0.3.1__tar.gz

evalforge_ci-0.3.1/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,22 @@
+name: Bug report
+description: Report a reproducible EvalForge problem
+title: "[Bug]: "
+labels: [bug]
+body:
+  - type: textarea
+    attributes:
+      label: What happened?
+      description: Include expected and actual behavior.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Reproduction
+      description: Provide the smallest dataset, config, and command that reproduce the issue.
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: EvalForge version
+    validations:
+      required: true

evalforge_ci-0.3.1/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,21 @@
+name: Feature request
+description: Propose an evidence-backed EvalForge improvement
+title: "[Feature]: "
+labels: [enhancement]
+body:
+  - type: textarea
+    attributes:
+      label: Maintainer or user problem
+      description: Describe the workflow that is currently difficult or impossible.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Proposed behavior
+      description: Include example inputs, outputs, or policy where useful.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Compatibility impact
+      description: Note any artifact, policy, metric, API, or CLI behavior that could change.

evalforge_ci-0.3.1/.github/dependabot.yml

@@ -0,0 +1,22 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: weekly
+    groups:
+      python-runtime:
+        dependency-type: production
+      python-development:
+        dependency-type: development
+    # mypy 2.x and Twine 7.x require Python >=3.10. Keep development tooling
+    # installable in the maintained Python 3.9 CI job.
+    ignore:
+      - dependency-name: mypy
+        update-types: [version-update:semver-major]
+      - dependency-name: twine
+        update-types: [version-update:semver-major]
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly

evalforge_ci-0.3.1/.github/workflows/ci.yml

@@ -0,0 +1,84 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.12"]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+      - run: python -m pip install -e ".[rag,dashboard,dev]"
+      - run: ruff check .
+      - run: mypy src/evalforge
+      - run: pytest --cov=evalforge --cov-report=term-missing --cov-fail-under=85
+      - if: matrix.python-version == '3.12'
+        run: make gate-demo
+      - name: Run the reproducible RAG release gate
+        if: matrix.python-version == '3.12'
+        run: |
+          EVALFORGE_DATABASE_URL=sqlite:///./ci_evalforge.db evalforge seed
+          EVALFORGE_DATABASE_URL=sqlite:///./ci_evalforge.db evalforge check hybrid_top3 --report-dir artifacts
+      - name: Run the promptfoo interoperability gate
+        if: matrix.python-version == '3.12'
+        run: |
+          evalforge import promptfoo tests/fixtures/promptfoo/results-v3.json \
+            --output artifacts/promptfoo-artifact.json \
+            --source-revision "$GITHUB_SHA"
+          evalforge gate artifacts/promptfoo-artifact.json \
+            --policy examples/promptfoo_policy.json \
+            --json artifacts/promptfoo-report.json \
+            --junit artifacts/promptfoo-junit.xml \
+            --sarif artifacts/promptfoo.sarif
+      - name: Validate release-gate reports
+        if: matrix.python-version == '3.12'
+        run: |
+          python -m json.tool artifacts/evalforge-report.json >/dev/null
+          python -m json.tool artifacts/evalforge.sarif >/dev/null
+          python -m json.tool artifacts/promptfoo-artifact.json >/dev/null
+          python -m json.tool artifacts/promptfoo-report.json >/dev/null
+          python -m json.tool artifacts/promptfoo.sarif >/dev/null
+          python -c "from xml.etree import ElementTree; ElementTree.parse('artifacts/evalforge-junit.xml')"
+          python -c "from xml.etree import ElementTree; ElementTree.parse('artifacts/promptfoo-junit.xml')"
+      - name: Build and check distributions
+        if: matrix.python-version == '3.12'
+        run: make release-check
+      - name: Test the built wheel in a fresh environment
+        if: matrix.python-version == '3.12'
+        run: |
+          python -m venv "$RUNNER_TEMP/evalforge-wheel"
+          "$RUNNER_TEMP/evalforge-wheel/bin/python" -m pip install dist/*.whl
+          "$RUNNER_TEMP/evalforge-wheel/bin/evalforge" gate examples/candidate_summary.json \
+            --policy examples/quality_policy.json \
+            --baseline examples/baseline_summary.json \
+            --json "$RUNNER_TEMP/evalforge-wheel/report.json" \
+            --junit "$RUNNER_TEMP/evalforge-wheel/junit.xml" \
+            --sarif "$RUNNER_TEMP/evalforge-wheel/report.sarif"
+
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+      - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
+      - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
+        with:
+          context: .
+          push: false
+          tags: evalforge:ci

evalforge_ci-0.3.1/.github/workflows/codeql.yml

@@ -0,0 +1,35 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "17 4 * * 1"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: codeql-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: Analyze (python)
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      security-events: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
+        with:
+          languages: python
+          build-mode: none
+      - name: Analyze
+        uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
+        with:
+          category: /language:python

evalforge_ci-0.3.1/.github/workflows/release.yml

@@ -0,0 +1,93 @@
+name: Release assets
+
+on:
+  release:
+    types: [published]
+
+permissions: {}
+
+jobs:
+  build:
+    if: startsWith(github.event.release.tag_name, 'v') && github.event.release.prerelease == false
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          ref: ${{ github.event.release.tag_name }}
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Verify checked-out tag target
+        env:
+          RELEASE_TAG: ${{ github.event.release.tag_name }}
+        run: |
+          release_tag_commit=$(git rev-list -n 1 "$RELEASE_TAG")
+          checked_out_commit=$(git rev-parse HEAD)
+          test "$release_tag_commit" = "$checked_out_commit"
+          package_version=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")
+          test "$RELEASE_TAG" = "v$package_version"
+      - run: python -m pip install ".[dev]"
+      - run: make release-check
+      - name: Assemble verified release files
+        run: |
+          mkdir -p release-files/packages
+          cp dist/*.whl dist/*.tar.gz release-files/packages/
+          sha256sum release-files/packages/* | sed 's#release-files/packages/##' > release-files/SHA256SUMS
+          cd release-files/packages
+          sha256sum --check ../SHA256SUMS
+      - name: Store verified distributions
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: verified-release-files
+          path: release-files/
+          if-no-files-found: error
+          retention-days: 7
+
+  github-release:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: write
+    steps:
+      - name: Retrieve verified release files
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: verified-release-files
+          path: release-files
+      - name: Verify downloaded distributions
+        working-directory: release-files/packages
+        run: sha256sum --check ../SHA256SUMS
+      - name: Attach distributions and checksums
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release upload "${{ github.event.release.tag_name }}" \
+            release-files/packages/* release-files/SHA256SUMS --clobber \
+            --repo "${{ github.repository }}"
+
+  pypi-publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/evalforge-ci
+    permissions:
+      actions: read
+      id-token: write
+    steps:
+      - name: Retrieve verified distributions
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: verified-release-files
+          path: release-files
+      - name: Verify downloaded distributions
+        working-directory: release-files/packages
+        run: sha256sum --check ../SHA256SUMS
+      - name: Publish distributions with OIDC
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
+        with:
+          packages-dir: release-files/packages/

evalforge_ci-0.3.1/ADOPTERS.md

@@ -0,0 +1,16 @@
+# Adopters
+
+EvalForge records downstream use only when it can be verified publicly and the project's maintainer has agreed to be listed.
+
+There are no confirmed downstream adopters yet. GitHub stars and automated trend indexes show early interest, but they are not evidence of an integration or production use.
+
+## Add your project
+
+Open an issue or pull request with:
+
+- the public repository and a link to the EvalForge Action or artifact-schema integration;
+- the EvalForge version or commit in use;
+- a short description of what the quality gate protects; and
+- confirmation that the project may be named here.
+
+The maintainers verify the link before adding an entry. Private users may share feedback without being listed.

evalforge_ci-0.3.1/CHANGELOG.md

@@ -0,0 +1,67 @@
+# Changelog
+
+All notable changes are documented here. The project follows semantic versioning while the artifact and policy formats carry independent schema versions.
+
+## [0.3.1] - 2026-08-31
+
+### Added
+
+- An explicit promptfoo JSON results-schema-v3 adapter that emits sanitized aggregate evidence and preserves producer provenance.
+- Upstream-format synthetic promptfoo fixtures and compatibility tests covering unknown fields, missing provenance, non-finite evidence, inconsistent counts, and the CLI import path.
+- CodeQL analysis for Python pull requests, main-branch pushes, and scheduled scans with minimal permissions.
+- PyPI Trusted Publishing through GitHub OIDC, with the official publish Action pinned to a reviewed commit.
+- Workflow invariants that reject floating third-party Action references and keep the OIDC permission scoped to the publish job.
+
+### Changed
+
+- Build release distributions once, verify them with Twine and SHA-256, then reuse the same workflow artifact for GitHub Release assets and PyPI publication.
+- Document the promptfoo metric mapping, source-version boundary, privacy exclusions, and first-release Trusted Publisher checklist.
+- Keep Dependabot from proposing mypy 2.x or Twine 7.x while the project maintains Python 3.9 compatibility.
+- Include conformance fixtures, examples, workflow definitions, and release metadata in the source distribution so its bundled test suite is self-contained.
+
+## [0.3.0] - 2026-08-31
+
+### Added
+
+- Baseline-versus-candidate comparisons with direction-aware metric verdicts.
+- Dataset fingerprints, complete configuration snapshots, and metric-version metadata.
+- Stored-experiment release gates in the API and Streamlit dashboard.
+- An end-to-end `evalforge check` command that runs the built-in RAG evaluator and emits portable JSON, JUnit, and SARIF evidence.
+- A seeded RAG release gate in GitHub Actions.
+- An engineering case study and resume-ready project narrative.
+- An expanded test suite with an enforced 85% branch-coverage floor.
+
+## [0.2.1] - 2026-08-30
+
+### Fixed
+
+- Reject non-finite metric values and policy thresholds before they can enter JSON, JUnit, or SARIF evidence.
+- Fail baseline-delta checks when candidate and baseline units or metric directions do not match.
+- Reject duplicate policy check IDs so report identifiers remain unambiguous.
+- Isolate SQLite databases per pytest session so concurrent test runs cannot corrupt each other.
+
+### Changed
+
+- Update maintained GitHub Actions and development dependency ranges after successful Dependabot CI runs; pin Actions to full commit SHAs.
+- Cancel superseded CI runs on the same ref.
+- Build and attach checked Python distributions plus SHA-256 checksums when a GitHub Release is published.
+- Mark the installed Python package as typed for PEP 561-aware consumers.
+
+## [0.2.0] - 2026-08-24
+
+### Added
+
+- Evaluator-neutral evaluation artifact schema 1.0.
+- Policy schema 1 with absolute and baseline-delta checks.
+- JSON, JUnit XML, and SARIF 2.1.0 gate reports.
+- CI-safe gate exit codes and a reusable GitHub composite Action.
+- Public governance, maintainer responsibilities, and ecosystem success measures.
+
+### Changed
+
+- The unique Python distribution name is now `evalforge-ci`; the import package and CLI remain `evalforge`.
+- `evalforge run` can write a portable artifact with `--output`.
+
+## [0.1.0] - 2026-08-24
+
+- Initial RAG quality and security evaluation MVP.

evalforge_ci-0.3.1/CITATION.cff

@@ -0,0 +1,11 @@
+cff-version: 1.2.0
+message: If EvalForge supports your research or engineering work, please cite the software.
+title: EvalForge
+type: software
+version: 0.3.1
+date-released: 2026-08-31
+license: MIT
+repository-code: https://github.com/jsdhwfmax/EvalForge
+url: https://github.com/jsdhwfmax/EvalForge
+authors:
+  - alias: jsdhwfmax

evalforge_ci-0.3.1/CODE_OF_CONDUCT.md

@@ -0,0 +1,15 @@
+# Contributor Covenant Code of Conduct
+
+## Our pledge
+
+We pledge to make participation in this project a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, experience level, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation.
+
+## Our standards
+
+Positive behavior includes empathy, constructive feedback, respect for differing viewpoints, and responsibility for mistakes. Unacceptable behavior includes harassment, discriminatory language, personal attacks, publishing private information without permission, or conduct inappropriate in a professional setting.
+
+## Enforcement
+
+Project maintainers may remove, edit, or reject contributions and interactions that violate this Code of Conduct. Reports may be made through the repository's private security/reporting channels. Maintainers will respect the privacy and safety of reporters.
+
+This text is adapted from Contributor Covenant 2.1.

evalforge_ci-0.3.1/CONTRIBUTING.md

@@ -0,0 +1,25 @@
+# Contributing
+
+Thanks for helping make RAG quality measurable.
+
+1. Open an issue for substantial behavior or architecture changes.
+2. Create a focused branch and add tests for new behavior.
+3. Run `make lint` and `make test` locally.
+4. Keep metrics deterministic or document nondeterminism and confidence intervals.
+5. Never include customer data, real credentials, or private prompts in fixtures.
+
+Pull requests should explain the user problem, implementation, verification evidence, and any metric-compatibility impact. Changes to a metric definition should update `docs/METRICS.md` and include before/after fixtures.
+
+Changes to the portable artifact or gate policy must also update the matching JSON Schema, `docs/INTEROPERABILITY.md`, and compatibility tests. New evaluator integrations should be backed by a sanitized upstream fixture rather than an invented output shape.
+
+## Development setup
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -e ".[rag,dashboard,dev]"
+evalforge seed
+pytest
+```
+
+The project targets Python 3.9 and 3.12. Please keep new code compatible with both.

evalforge_ci-0.3.1/GOVERNANCE.md

@@ -0,0 +1,20 @@
+# Governance
+
+EvalForge uses a maintainer-led, contribution-driven model while the community is small.
+
+## Decisions
+
+- Routine fixes are decided through pull-request review.
+- Behavior, metric, artifact, policy, or compatibility changes require an issue and documented migration impact.
+- Security fixes may be developed privately and disclosed after a patch is available.
+- The primary maintainer has final responsibility for releases, security response, and resolving deadlock.
+
+Material decisions should be recorded in public issues or pull requests. Compatibility and user safety take priority over release speed.
+
+## Becoming a maintainer
+
+Contributors may be invited as reviewers after sustained, constructive work across code, documentation, issue triage, or integrations. Commit access follows demonstrated responsibility rather than a fixed contribution count. Maintainer additions and departures are recorded in `MAINTAINERS.md`.
+
+## Project assets
+
+Repository, package, and release credentials must use least privilege and multi-factor authentication where supported. No single contributor should copy user datasets, provider keys, or private vulnerability details into public project systems.

evalforge_ci-0.3.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 EvalForge Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evalforge_ci-0.3.1/MAINTAINERS.md

@@ -0,0 +1,15 @@
+# Maintainers
+
+## Primary maintainer
+
+- [`@jsdhwfmax`](https://github.com/jsdhwfmax) — architecture, issue and pull-request triage, releases, security response, and compatibility policy.
+
+The primary maintainer is currently the only person with release authority. This file will be updated when responsibility is shared; contributors are not presented as maintainers without their consent.
+
+## Maintenance expectations
+
+- Triage reproducible bugs and security reports.
+- Review dependency and CI updates.
+- Keep artifact and policy schemas backward compatible within a schema version.
+- Publish release notes and migration guidance for behavior changes.
+- Keep public adoption and ecosystem claims tied to verifiable evidence.

evalforge_ci-0.3.1/MANIFEST.in

@@ -0,0 +1,11 @@
+include LICENSE
+include README.md
+include *.md
+include CITATION.cff
+include Makefile
+include action.yml
+recursive-include .github *.yml
+recursive-include docs *.md *.svg
+recursive-include examples *.json
+recursive-include schemas *.json
+recursive-include tests *.py *.json *.md

evalforge_ci-0.3.1/Makefile

@@ -0,0 +1,43 @@
+.PHONY: install test lint gate-demo package release-check run dashboard demo docker
+
+PYTHON ?= python3
+
+install:
+	$(PYTHON) -m pip install -e ".[rag,dashboard,dev]"
+
+test:
+	pytest --cov=evalforge --cov-report=term-missing --cov-fail-under=85
+
+lint:
+	ruff check .
+	mypy src/evalforge
+
+gate-demo:
+	mkdir -p build
+	$(PYTHON) -m evalforge.cli gate examples/candidate_summary.json \
+		--policy examples/quality_policy.json \
+		--baseline examples/baseline_summary.json \
+		--json build/evalforge-report.json \
+		--junit build/evalforge-junit.xml \
+		--sarif build/evalforge.sarif
+
+package:
+	$(PYTHON) -m build
+
+release-check: package
+	$(PYTHON) -m twine check dist/*.whl dist/*.tar.gz
+
+run:
+	uvicorn evalforge.api:app --reload
+
+dashboard:
+	streamlit run dashboard/app.py
+
+demo:
+	evalforge seed
+	evalforge run baseline_top1 --name "Baseline"
+	evalforge run hybrid_top3 --name "Candidate"
+	evalforge check hybrid_top3 --name "Release candidate" --report-dir artifacts
+
+docker:
+	docker compose up --build