ragfence 0.1.0__tar.gz

ragfence-0.1.0/.env.example

@@ -0,0 +1,21 @@
+# RAGFence environment configuration template.
+# Copy this file to `.env` and fill in real values for your environment.
+# NEVER commit real credentials. Every value below is a placeholder.
+
+# PostgreSQL DSN for the reference implementation (see docs/TRD.md, Section 11).
+RAGFENCE_DATABASE_DSN=postgresql+psycopg://ragfence:CHANGEME@localhost:5432/ragfence
+
+# Embedding dimension for pgvector columns (TRD default: 1536).
+RAGFENCE_EMBEDDING_DIM=1536
+
+# Default evaluation pass threshold (0-100).
+RAGFENCE_EVALUATION_THRESHOLD=80
+
+# Optional BYOK provider keys (see docs/TRD.md, Section 10).
+# Leave unset to use the fake default providers with zero external dependencies.
+RAGFENCE_LLM_API_KEY=CHANGEME
+RAGFENCE_EMBEDDING_API_KEY=CHANGEME
+
+# Optional OIDC identity provider settings for `auth` (synthetic source otherwise).
+RAGFENCE_OIDC_ISSUER=CHANGEME
+RAGFENCE_OIDC_CLIENT_ID=CHANGEME

ragfence-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,65 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  ci:
+    name: Lint, type, and test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_USER: ragfence
+          POSTGRES_PASSWORD: ragfence
+          POSTGRES_DB: ragfence
+        ports:
+          - 5434:5432
+        options: >-
+          --health-cmd "pg_isready -U ragfence -d ragfence"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    env:
+      RAGFENCE_TEST_DATABASE_DSN: postgresql+psycopg://ragfence:ragfence@localhost:5434/ragfence
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+
+      - name: Migrate database schema
+        run: alembic upgrade head
+
+      - name: Seed reference database
+        run: |
+          python -c "from sqlalchemy import create_engine; from sqlalchemy.orm import Session; from ragfence.datasets import seed_reference_corp; import os; engine = create_engine(os.environ['RAGFENCE_TEST_DATABASE_DSN']); sess = Session(engine); seed_reference_corp(sess); sess.commit(); engine.dispose()"
+
+      - name: Lint (ruff check)
+        run: ruff check .
+
+      - name: Format (ruff format --check)
+        run: ruff format --check .
+
+      - name: Type check (mypy)
+        run: mypy
+
+      - name: Tests (pytest)
+        run: python -m pytest

ragfence-0.1.0/.github/workflows/publish-pypi.yml

@@ -0,0 +1,35 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  publish:
+    name: Build and publish distribution
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/ragfence
+    steps:
+      - name: Check out release source
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build tooling
+        run: python -m pip install --upgrade build
+
+      - name: Build wheel and source distribution
+        run: python -m build
+
+      - name: Publish distributions with PyPI Trusted Publishing
+        uses: pypa/gh-action-pypi-publish@release/v1

ragfence-0.1.0/.github/workflows/ragfence-test.yml

@@ -0,0 +1,287 @@
+name: RAGFence test
+
+on:
+  workflow_call:
+    inputs:
+      adapter:
+        description: Adapter passed to ragfence test
+        type: string
+        required: false
+        default: reference
+      target:
+        description: Target service name
+        type: string
+        required: false
+        default: target
+      base-url:
+        description: Target base URL
+        type: string
+        required: false
+        default: ""
+      threshold:
+        description: Inclusive score threshold from 0 to 100
+        type: number
+        required: false
+        default: 80
+      start-target-command:
+        description: Quoted command used to start the target service
+        type: string
+        required: false
+        default: ""
+      comment:
+        description: Enable the trusted same-repository PR comment path
+        type: boolean
+        required: false
+        default: true
+      fake-providers:
+        description: Use deterministic local fake providers
+        type: boolean
+        required: false
+        default: true
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  ragfence-test:
+    name: RAGFence evaluation (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_USER: ragfence
+          POSTGRES_PASSWORD: ragfence
+          POSTGRES_DB: ragfence
+        ports:
+          - 5434:5432
+        options: >-
+          --health-cmd "pg_isready -U ragfence -d ragfence"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    env:
+      REPORT_PATH: .ragfence/reports/ci.json
+      ARTIFACT_NAME: ragfence-report-${{ matrix.python-version }}
+      ADAPTER: ${{ inputs.adapter }}
+      TARGET: ${{ inputs.target }}
+      BASE_URL: ${{ inputs.base-url }}
+      THRESHOLD: ${{ inputs.threshold }}
+      START_TARGET_COMMAND: ${{ inputs.start-target-command }}
+      FAKE_PROVIDERS: ${{ inputs.fake-providers }}
+      RAGFENCE_TEST_DATABASE_DSN: postgresql+psycopg://ragfence:ragfence@localhost:5434/ragfence
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+
+      - name: Validate inputs
+        env:
+          INPUT_ADAPTER: ${{ inputs.adapter }}
+          INPUT_TARGET: ${{ inputs.target }}
+          INPUT_BASE_URL: ${{ inputs.base-url }}
+          INPUT_THRESHOLD: ${{ inputs.threshold }}
+          INPUT_START_COMMAND: ${{ inputs.start-target-command }}
+        run: |
+          python - <<'PY'
+          import math
+          import os
+          from urllib.parse import urlparse
+
+          adapter = os.environ["INPUT_ADAPTER"].strip()
+          target = os.environ["INPUT_TARGET"].strip()
+          base_url = os.environ["INPUT_BASE_URL"].strip()
+          start_command = os.environ["INPUT_START_COMMAND"].strip()
+          try:
+              threshold = float(os.environ["INPUT_THRESHOLD"])
+          except ValueError as error:
+              raise SystemExit("Invalid threshold configuration.") from error
+          if not adapter or not target:
+              raise SystemExit("Invalid RAGFence CI input configuration.")
+          if adapter == "generic_http":
+              parsed = urlparse(base_url)
+              if not start_command or parsed.scheme not in {"http", "https"} or not parsed.netloc:
+                  raise SystemExit("Invalid RAGFence CI input configuration.")
+          if not math.isfinite(threshold) or not 0 <= threshold <= 100:
+              raise SystemExit("threshold must satisfy 0 <= threshold <= 100")
+          PY
+
+      - name: Initialize RAGFence
+        run: |
+          mkdir -p .ragfence/reports
+          ragfence init
+
+      - name: Migrate and seed reference database
+        run: |
+          alembic upgrade head
+          python -c "from sqlalchemy import create_engine; from sqlalchemy.orm import Session; from ragfence.datasets import seed_reference_corp; import os; engine = create_engine(os.environ['RAGFENCE_TEST_DATABASE_DSN']); sess = Session(engine); seed_reference_corp(sess); sess.commit(); engine.dispose()"
+
+      - name: Start target
+        if: ${{ inputs.start-target-command != '' }}
+        run: |
+          bash -c -- "$START_TARGET_COMMAND" >.ragfence/target-start.log 2>&1 &
+          printf '%s\n' "$!" > .ragfence/target.pid
+
+      - name: Wait for target readiness
+        if: ${{ inputs.start-target-command != '' && inputs.base-url != '' }}
+        env:
+          READINESS_ATTEMPTS: 30
+          READINESS_DELAY_SECONDS: 2
+        run: |
+          python - <<'PY'
+          import os
+          import sys
+          import time
+          import urllib.error
+          import urllib.request
+
+          url = os.environ["BASE_URL"]
+          attempts = int(os.environ["READINESS_ATTEMPTS"])
+          delay = int(os.environ["READINESS_DELAY_SECONDS"])
+          for attempt in range(attempts):
+              try:
+                  with urllib.request.urlopen(url, timeout=2) as response:
+                      if response.status < 500:
+                          raise SystemExit(0)
+              except (OSError, urllib.error.URLError):
+                  pass
+              if attempt + 1 < attempts:
+                  time.sleep(delay)
+          print("Target readiness check failed after bounded retries.", file=sys.stderr)
+          raise SystemExit(2)
+          PY
+
+      - name: Evaluate target
+        id: evaluate
+        env:
+          CLI_THRESHOLD: ${{ inputs.threshold }}
+        run: |
+          set +e
+          if [ "$FAKE_PROVIDERS" = "true" ]; then
+            export RAGFENCE_LLM_PROVIDER=fake
+            export RAGFENCE_EMBEDDING_PROVIDER=fake
+          fi
+          LEGACY_THRESHOLD=$(python -c "print(float('$CLI_THRESHOLD') / 100.0)")
+          if [ "$ADAPTER" = "generic_http" ]; then
+            ragfence test --adapter "$ADAPTER" --base-url "$BASE_URL" --threshold "$LEGACY_THRESHOLD" --json --output "$REPORT_PATH"
+          else
+            ragfence test --adapter "$ADAPTER" --threshold "$LEGACY_THRESHOLD" --json --output "$REPORT_PATH"
+          fi
+          cli_status=$?
+          set -e
+          case "$cli_status" in
+            0|1|2) ;;
+            *) cli_status=2 ;;
+          esac
+          if [ -s "$REPORT_PATH" ]; then report_exists=true; else report_exists=false; fi
+          {
+            printf 'status=%s\n' "$cli_status"
+            printf 'report_exists=%s\n' "$report_exists"
+          } >> "$GITHUB_OUTPUT"
+          exit "$cli_status"
+
+      - name: Upload JSON report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ${{ env.REPORT_PATH }}
+          if-no-files-found: error
+          retention-days: 14
+
+      - name: Write job summary
+        if: always()
+        env:
+          CLI_STATUS: ${{ steps.evaluate.outputs.status || '2' }}
+          REPORT_EXISTS: ${{ steps.evaluate.outputs.report_exists || 'false' }}
+        run: |
+          : "${GITHUB_STEP_SUMMARY:?summary file is required}"
+          python - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          path = Path(os.environ["REPORT_PATH"])
+          status = int(os.environ["CLI_STATUS"])
+          exists = os.environ["REPORT_EXISTS"] == "true" and path.is_file()
+          score = "unavailable"
+          outcome = "execution/configuration error"
+          passed = "false"
+          if exists:
+              try:
+                  report = json.loads(path.read_text(encoding="utf-8"))
+                  score_value = report.get("score")
+                  if isinstance(score_value, (int, float)):
+                      score = str(score_value)[:32]
+                  outcome_value = report.get("outcome")
+                  if isinstance(outcome_value, str):
+                      outcome = outcome_value[:80]
+                  passed_value = report.get("gate_passed")
+                  if passed_value is None:
+                      passed_value = report.get("passed_threshold")
+                  if isinstance(passed_value, bool):
+                      passed = str(passed_value).lower()
+              except (OSError, ValueError, TypeError):
+                  outcome = "execution/configuration error"
+          if status == 1 and outcome == "execution/configuration error":
+              outcome = "below threshold"
+          summary = [
+              "## RAGFence evaluation",
+              "",
+              f"- Score: `{score}`",
+              f"- Threshold: `{os.environ['THRESHOLD'][:32]}`",
+              f"- Outcome: `{outcome}`",
+              f"- Passed threshold: `{passed}`",
+              f"- CLI exit status: `{status}`",
+              f"- Report: `{os.environ['REPORT_PATH']}`",
+              f"- Artifact: `{os.environ['ARTIFACT_NAME']}`",
+          ]
+          with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as handle:
+              handle.write("\n".join(summary) + "\n")
+          PY
+
+      - name: Comment on trusted same-repository PR
+        if: ${{ inputs.comment && github.event_name == 'pull_request' && github.event_name != 'pull_request_target' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository }}
+        uses: actions/github-script@v7
+        env:
+          CLI_STATUS: ${{ steps.evaluate.outputs.status || '2' }}
+          REPORT_PATH: ${{ env.REPORT_PATH }}
+        with:
+          script: |
+            const marker = '<!-- ragfence-ci-summary -->';
+            const body = `${marker}\n## RAGFence evaluation\n- CLI exit status: ${process.env.CLI_STATUS}\n- Report artifact: ${process.env.REPORT_PATH}`;
+            const { owner, repo } = context.repo;
+            const issue_number = context.issue.number;
+            const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number, per_page: 100 });
+            const existing = comments.find((comment) => comment.body && comment.body.includes(marker));
+            if (existing) {
+              await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
+            } else {
+              await github.rest.issues.createComment({ owner, repo, issue_number, body });
+            }
+
+      - name: Final gate
+        if: always()
+        env:
+          CLI_STATUS: ${{ steps.evaluate.outputs.status || '2' }}
+          REPORT_EXISTS: ${{ steps.evaluate.outputs.report_exists || 'false' }}
+        run: |
+          status="$CLI_STATUS"
+          if [ "$REPORT_EXISTS" != "true" ]; then status=2; fi
+          case "$status" in
+            0|1|2) exit "$status" ;;
+            *) exit 2 ;;
+          esac

ragfence-0.1.0/.gitignore

@@ -0,0 +1,33 @@
+# Python bytecode and build artifacts
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# Tooling caches
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+
+# Local agent tooling state (never ship or commit)
+.atl/
+.codegraph/
+.hermes/
+
+# Environment files — keep the committed template, never the real one
+.env
+.env.*
+!.env.example
+
+# OS noise
+.DS_Store
+Thumbs.db

ragfence-0.1.0/CHANGELOG.md

@@ -0,0 +1,38 @@
+# Changelog
+
+All notable changes to RAGFence are documented here. This project follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) conventions.
+
+## [0.1.0] - 2026-08-21
+
+### Added
+
+- Production evaluation path: declarative control matrix (8 required controls),
+  fail-closed `EvaluationGate` with PASS / FAIL / INCONCLUSIVE / SKIPPED statuses,
+  and security / utility / overall scores.
+- ReferenceAdapter backed by PostgreSQL 16 + pgvector with DB-resolved identities:
+  Acme + Globex tenants, tenant isolation enforced before vector search,
+  soft-delete and clearance blocking, MUST_ALLOW / MUST_BLOCK semantics.
+- Identity strategies for `generic_http`: header templates with allowlisted actor
+  placeholders and bearer-token resolution from environment configuration.
+- Secret redaction across reports, findings, reasons, and errors.
+- Self-contained wheel: `ragfence init --up` bootstraps the bundled pgvector
+  reference environment without a repository checkout.
+- Idempotent bootstrap: a reachable reference database is migrated and reseeded
+  in place without invoking Docker Compose.
+- Reusable GitHub Actions evaluation workflow with mandatory pgvector service,
+  healthchecks, real Alembic migrations, seeded data, and JSON report artifacts.
+
+### Changed
+
+- Reports with a `.json` destination are always machine-parseable, with or
+  without `--json`.
+- Explicit `--config` paths that do not exist fail closed with an actionable
+  error instead of silently falling back to defaults.
+- Unwritable report destinations fail closed with exit code 2 and an actionable
+  message.
+
+### Security
+
+- Least-privilege permissions on both CI workflows.
+- Dependency audit clean (`pip-audit`); no secrets in package artifacts;
+  sdist excludes local agent tooling state.

ragfence-0.1.0/CODE_OF_CONDUCT.md

@@ -0,0 +1,20 @@
+# Code of Conduct
+
+RAGFence is committed to a welcoming, respectful, and harassment-free community for everyone, regardless of identity, background, experience, or viewpoint.
+
+## Expected behavior
+
+- Be kind, constructive, and respectful.
+- Assume good intent while addressing impact.
+- Focus feedback on the work, not the person.
+- Respect privacy and do not publish private information.
+
+## Unacceptable behavior
+
+Harassment, discrimination, threats, sexualized conduct, personal attacks, doxxing, and disruptive or abusive behavior are not tolerated.
+
+## Reporting
+
+Report a concern privately to the project maintainers through the repository's issue/contact channel. Do not include sensitive information in a public issue. Reports will be reviewed promptly and handled confidentially where possible.
+
+Maintainers may remove content, restrict participation, or take other action they consider appropriate. Retaliation against anyone who reports a concern in good faith is not acceptable.

ragfence-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,40 @@
+# Contributing to RAGFence
+
+Thanks for contributing. This project follows a spec-driven development (SDD)
+workflow: significant changes start from a proposal in `openspec/changes/` and
+flow through spec, design, tasks, implementation, and verification.
+
+## Development setup
+
+Requires Python >= 3.12.
+
+```console
+python -m venv .venv
+.venv\Scripts\python -m pip install -e ".[dev]"   # Windows
+# source .venv/bin/activate && pip install -e ".[dev]"   # POSIX
+```
+
+## Quality gates
+
+All gates must pass before a change is reviewable:
+
+```console
+python -m pytest          # tests
+ruff check .              # lint
+ruff format --check .     # formatting
+mypy                      # strict type checking
+```
+
+Tests must pass **without Docker and without API keys**. Database-backed tests are
+marked `@pytest.mark.db` and skip when PostgreSQL is unavailable.
+
+## Committing
+
+- Use Conventional Commits (`feat:`, `fix:`, `chore:`, `docs:`, `ci:`, ...).
+- Tests ship with the behavior they verify — one work unit per commit.
+- Do not add AI/Co-Authored-By attribution lines.
+- Keep PRs under ~400 changed lines; larger changes are delivered as stacked PRs.
+
+## Reporting issues
+
+Open an issue with reproduction steps and expected vs. actual behavior.