ragops 1.0.0__tar.gz

ragops-1.0.0/.dockerignore

@@ -0,0 +1,9 @@
+.git
+.github
+.venv
+__pycache__
+.pytest_cache
+.ruff_cache
+reports
+tests
+scenarios

ragops-1.0.0/.github/ISSUE_TEMPLATE/bug.yml

@@ -0,0 +1,41 @@
+name: Bug report
+description: Report a reproducible defect without private data
+title: "[Bug]: "
+labels: [bug]
+body:
+  - type: markdown
+    attributes:
+      value: Never include secrets, customer-confidential data, or private traces.
+  - type: input
+    id: version
+    attributes:
+      label: RAGOps version or commit
+      placeholder: v1.0 or commit SHA
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Minimal reproduction
+      description: Include the command and a synthetic or redacted fixture.
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+    validations:
+      required: true
+  - type: textarea
+    id: actual
+    attributes:
+      label: Actual behavior
+    validations:
+      required: true
+  - type: input
+    id: environment
+    attributes:
+      label: Environment
+      placeholder: Python 3.12, macOS 15
+    validations:
+      required: true

ragops-1.0.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Security vulnerability
+    url: https://github.com/thangldw/ragops/security/policy
+    about: Report vulnerabilities privately through the security policy.
+  - name: Support guidance
+    url: https://github.com/thangldw/ragops/blob/main/SUPPORT.md
+    about: Read what to include before opening an issue.

ragops-1.0.0/.github/ISSUE_TEMPLATE/design-partner.yml

@@ -0,0 +1,48 @@
+name: Design-partner interest
+description: Explore an offline RAGOps evaluation pilot using synthetic or redacted data.
+title: "[Design partner]: "
+labels: [design-partner]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Do not include production secrets, personal data, customer-confidential
+        content, API keys, or proprietary traces in this public issue.
+  - type: textarea
+    id: workflow
+    attributes:
+      label: AI workflow
+      description: What RAG or agent workflow are you evaluating?
+    validations:
+      required: true
+  - type: textarea
+    id: decision
+    attributes:
+      label: Release decision today
+      description: How do you currently decide whether a candidate is safe and useful enough to release?
+    validations:
+      required: true
+  - type: textarea
+    id: failure
+    attributes:
+      label: Representative synthetic failure
+      description: Describe a redacted or synthetic regression without sharing sensitive data.
+    validations:
+      required: true
+  - type: dropdown
+    id: data
+    attributes:
+      label: Safe fixture availability
+      options:
+        - Synthetic fixtures available
+        - Redacted fixtures available
+        - Need help creating safe fixtures
+    validations:
+      required: true
+  - type: checkboxes
+    id: consent
+    attributes:
+      label: Public issue safety
+      options:
+        - label: I confirm this issue contains no secrets, personal data, or customer-confidential content.
+          required: true

ragops-1.0.0/.github/ISSUE_TEMPLATE/evaluator.yml

@@ -0,0 +1,40 @@
+name: Evaluator proposal
+description: Propose a measurable evaluator with calibration and failure modes.
+title: "[Evaluator]: "
+labels: [evaluation]
+body:
+  - type: textarea
+    id: construct
+    attributes:
+      label: Construct being measured
+      description: Define precisely what the score means and does not mean.
+    validations:
+      required: true
+  - type: textarea
+    id: method
+    attributes:
+      label: Method and inputs
+      description: Explain deterministic/provider behavior, required evidence, and output range.
+    validations:
+      required: true
+  - type: textarea
+    id: calibration
+    attributes:
+      label: Calibration plan
+      description: Labeled cases, human rubric, agreement analysis, and thresholds.
+    validations:
+      required: true
+  - type: textarea
+    id: failures
+    attributes:
+      label: Known failure modes
+    validations:
+      required: true
+  - type: checkboxes
+    id: boundary
+    attributes:
+      label: Product boundary
+      options:
+        - label: The core remains useful offline or this is an explicitly optional plugin.
+          required: true
+

ragops-1.0.0/.github/ISSUE_TEMPLATE/feature.yml

@@ -0,0 +1,38 @@
+name: Product feature
+description: Propose a capability tied to a user outcome and release contract.
+title: "[Feature]: "
+labels: [enhancement]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: User problem
+      description: Who has the problem, in what workflow, and why does it matter?
+    validations:
+      required: true
+  - type: textarea
+    id: outcome
+    attributes:
+      label: Desired outcome
+      description: Describe observable success without prescribing implementation.
+    validations:
+      required: true
+  - type: textarea
+    id: contract
+    attributes:
+      label: Contract and acceptance criteria
+      description: Include input, output, release behavior, compatibility, and tests.
+    validations:
+      required: true
+  - type: textarea
+    id: non_goals
+    attributes:
+      label: Non-goals and boundaries
+    validations:
+      required: true
+  - type: textarea
+    id: evidence
+    attributes:
+      label: Supporting evidence
+      description: Customer observation, benchmark case, incident, or design-partner request.
+

ragops-1.0.0/.github/pull_request_template.md

@@ -0,0 +1,27 @@
+## Outcome
+
+- User problem:
+- Decision enabled:
+- Requirement or issue:
+
+## Contract
+
+- Public schema/API impact:
+- Metric or PASS/BLOCK impact:
+- Architecture/open-core impact:
+- Explicit non-goals:
+
+## Evidence
+
+- [ ] Expected and failure tests
+- [ ] `ruff check .`
+- [ ] `PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest -q`
+- [ ] Affected CLI/API path exercised
+- [ ] Canonical documentation and changelog updated
+- [ ] Superseded content removed
+
+## Owner review
+
+- Known limitations:
+- Compatibility recommendation:
+- Publication requested: no / yes (specify)

ragops-1.0.0/.github/workflows/ci.yml

@@ -0,0 +1,73 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install -e '.[dev]'
+      - run: ruff check .
+      - run: pytest
+      - run: ragops evaluate --scenario scenarios/japanese_troubleshooting/scenario.json --responses scenarios/japanese_troubleshooting/sample_responses.json
+      - name: Validate reference benchmark coverage
+        run: ragops inspect --scenario scenarios/japanese_troubleshooting/benchmark-v0.2.json
+      - name: Evaluate reference benchmark baseline
+        run: >-
+          ragops evaluate
+          --scenario scenarios/japanese_troubleshooting/benchmark-v0.2.json
+          --responses scenarios/japanese_troubleshooting/benchmark-baseline.json
+          --evaluator citation_correctness
+          --evaluator claim_support
+      - name: Confirm regressed benchmark is blocked
+        shell: bash
+        run: |
+          set +e
+          ragops compare \
+            --scenario scenarios/japanese_troubleshooting/benchmark-v0.2.json \
+            --baseline scenarios/japanese_troubleshooting/benchmark-baseline.json \
+            --candidate scenarios/japanese_troubleshooting/benchmark-regressed.json \
+            --format markdown
+          code=$?
+          set -e
+          test "$code" -eq 2
+      - name: Confirm adversarial benchmark is blocked
+        shell: bash
+        run: |
+          set +e
+          ragops evaluate \
+            --scenario scenarios/japanese_troubleshooting/benchmark-v0.2.json \
+            --responses scenarios/japanese_troubleshooting/benchmark-adversarial.json
+          code=$?
+          set -e
+          test "$code" -eq 2
+      - name: Render regression report
+        run: >-
+          ragops compare
+          --scenario scenarios/japanese_troubleshooting/scenario.json
+          --baseline scenarios/japanese_troubleshooting/sample_responses.json
+          --candidate scenarios/japanese_troubleshooting/sample_responses.json
+          --format markdown
+          --output reports/regression.md
+      - name: Run reference GraphRAG agent
+        run: >-
+          python -m examples.japanese_troubleshooting_agent.cli
+          --suite examples/japanese_troubleshooting_agent/suite.json
+          --retriever graph
+          --output /tmp/reference-traces.jsonl
+      - name: Evaluate reference deployment traces
+        run: >-
+          ragops evaluate
+          --scenario examples/japanese_troubleshooting_agent/scenario.json
+          --traces /tmp/reference-traces.jsonl
+          --evaluator citation_correctness
+          --evaluator claim_support

ragops-1.0.0/.github/workflows/nightly.yml

@@ -0,0 +1,39 @@
+name: Nightly benchmark
+
+on:
+  schedule:
+    - cron: "17 18 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install -e '.[dev]'
+      - name: Evaluate 30-case benchmark
+        run: >-
+          ragops evaluate
+          --scenario scenarios/japanese_troubleshooting/benchmark-v0.2.json
+          --responses scenarios/japanese_troubleshooting/benchmark-baseline.json
+          --evaluator citation_correctness
+          --evaluator claim_support
+      - name: Run reference deployment
+        run: >-
+          python -m examples.japanese_troubleshooting_agent.cli
+          --suite examples/japanese_troubleshooting_agent/suite.json
+          --retriever graph
+          --output /tmp/reference-traces.jsonl
+      - name: Gate reference traces
+        run: >-
+          ragops evaluate
+          --scenario examples/japanese_troubleshooting_agent/scenario.json
+          --traces /tmp/reference-traces.jsonl
+          --evaluator citation_correctness
+          --evaluator claim_support

ragops-1.0.0/.github/workflows/pages.yml

@@ -0,0 +1,31 @@
+name: Pages
+
+on:
+  push:
+    branches: [main]
+    paths: ["site/**", ".github/workflows/pages.yml"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: true
+
+jobs:
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
+      - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
+        with:
+          path: site
+      - id: deployment
+        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4

ragops-1.0.0/.github/workflows/publish-pypi.yml

@@ -0,0 +1,65 @@
+name: Publish package to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: Existing RAGOps release tag to publish
+        required: true
+        type: string
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          ref: ${{ inputs.tag }}
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.12"
+      - name: Verify tag and package version
+        env:
+          RELEASE_TAG: ${{ inputs.tag }}
+        run: |
+          python - "$RELEASE_TAG" <<'PY'
+          import sys
+          import tomllib
+          from pathlib import Path
+
+          tag = sys.argv[1]
+          version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
+          parts = version.split(".")
+          if len(parts) != 3 or not all(part.isdigit() for part in parts):
+              raise SystemExit("package version must use numeric MAJOR.MINOR.PATCH")
+          major, minor, patch = parts
+          expected_tag = f"v{major}.{minor}"
+          if patch != "0" or tag != expected_tag:
+              raise SystemExit(
+                  f"tag {tag!r} does not match milestone {expected_tag!r} "
+                  f"for package version {version!r}"
+              )
+          print(f"publishing ragops {version}")
+          PY
+      - name: Download promoted GitHub Release artifacts
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_TAG: ${{ inputs.tag }}
+        run: |
+          mkdir release-assets dist
+          gh release download "$RELEASE_TAG" \
+            --pattern 'ragops-*.whl' \
+            --pattern 'ragops-*.tar.gz' \
+            --pattern 'ragops-*.cdx.json' \
+            --pattern 'SHA256SUMS' \
+            --dir release-assets
+          (cd release-assets && sha256sum --check SHA256SUMS)
+          test "$(find release-assets -maxdepth 1 -name 'ragops-*.whl' | wc -l)" -eq 1
+          test "$(find release-assets -maxdepth 1 -name 'ragops-*.tar.gz' | wc -l)" -eq 1
+          cp release-assets/ragops-*.whl release-assets/ragops-*.tar.gz dist/
+      - name: Publish distributions with Trusted Publishing
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0

ragops-1.0.0/.github/workflows/ragops-gate-smoke.yml

@@ -0,0 +1,24 @@
+name: RAGOps reusable gate smoke
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - ".github/workflows/ragops-gate.yml"
+      - ".github/workflows/ragops-gate-smoke.yml"
+      - "tests/test_github_workflows.py"
+  pull_request:
+    paths:
+      - ".github/workflows/ragops-gate.yml"
+      - ".github/workflows/ragops-gate-smoke.yml"
+      - "tests/test_github_workflows.py"
+  workflow_dispatch:
+
+jobs:
+  passing-reference:
+    uses: ./.github/workflows/ragops-gate.yml
+    with:
+      ragops-version: ${{ github.event.pull_request.head.sha || github.sha }}
+      scenario: scenarios/japanese_troubleshooting/benchmark-v0.2.json
+      baseline: scenarios/japanese_troubleshooting/benchmark-baseline.json
+      candidate: scenarios/japanese_troubleshooting/benchmark-baseline.json

ragops-1.0.0/.github/workflows/ragops-gate.yml

@@ -0,0 +1,133 @@
+name: RAGOps reusable release gate
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        description: Python version used to run RAGOps
+        required: false
+        type: string
+        default: "3.12"
+      ragops-version:
+        description: RAGOps Git revision or tag to install
+        required: false
+        type: string
+        default: "v1.0"
+      scenario:
+        description: Caller-relative scenario JSON path
+        required: true
+        type: string
+      baseline:
+        description: Caller-relative accepted baseline response path
+        required: true
+        type: string
+      candidate:
+        description: Caller-relative candidate response path
+        required: true
+        type: string
+      policy:
+        description: Optional caller-relative regression policy path
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  release-gate:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: ${{ inputs.python-version }}
+      - name: Install RAGOps
+        env:
+          RAGOPS_VERSION: ${{ inputs.ragops-version }}
+        run: >-
+          python -m pip install
+          "ragops @ git+https://github.com/thangldw/ragops.git@${RAGOPS_VERSION}"
+      - id: gate
+        name: Compare candidate with accepted baseline
+        shell: bash
+        env:
+          RAGOPS_SCENARIO: ${{ inputs.scenario }}
+          RAGOPS_BASELINE: ${{ inputs.baseline }}
+          RAGOPS_CANDIDATE: ${{ inputs.candidate }}
+          RAGOPS_POLICY: ${{ inputs.policy }}
+        run: |
+          set +e
+          command=(
+            ragops compare
+            --scenario "$RAGOPS_SCENARIO"
+            --baseline "$RAGOPS_BASELINE"
+            --candidate "$RAGOPS_CANDIDATE"
+            --format markdown
+            --output ragops-report.md
+          )
+          if [[ -n "$RAGOPS_POLICY" ]]; then
+            command+=(--policy "$RAGOPS_POLICY")
+          fi
+          "${command[@]}" >ragops-command.log 2>&1
+          gate_exit=$?
+          set -e
+          if [[ ! -f ragops-report.md ]]; then
+            {
+              echo "# RAGOps release gate error"
+              echo
+              echo '```text'
+              cat ragops-command.log
+              echo '```'
+            } >ragops-report.md
+          fi
+          echo "exit_code=$gate_exit" >>"$GITHUB_OUTPUT"
+      - name: Write bounded evidence manifest
+        if: always()
+        env:
+          RAGOPS_EVENT_NAME: ${{ github.event_name }}
+          RAGOPS_GATE_EXIT: ${{ steps.gate.outputs.exit_code }}
+          RAGOPS_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+          RAGOPS_PR_NUMBER: ${{ github.event.pull_request.number }}
+          RAGOPS_WORKFLOW: ${{ github.workflow }}
+        run: |
+          python - <<'PY'
+          import json
+          import os
+
+          gate_exit = os.environ.get("RAGOPS_GATE_EXIT", "")
+          manifest = {
+              "schema_version": "ragops-github-evidence-0.1",
+              "repository": os.environ["GITHUB_REPOSITORY"],
+              "event_name": os.environ["RAGOPS_EVENT_NAME"],
+              "workflow": os.environ["RAGOPS_WORKFLOW"],
+              "run_id": int(os.environ["GITHUB_RUN_ID"]),
+              "head_sha": os.environ["RAGOPS_HEAD_SHA"],
+              "pull_request_number": int(os.environ["RAGOPS_PR_NUMBER"] or 0),
+              "gate_exit_code": int(gate_exit) if gate_exit else -1,
+          }
+          with open("ragops-evidence.json", "w", encoding="utf-8") as output:
+              json.dump(manifest, output, ensure_ascii=False, indent=2)
+              output.write("\n")
+          PY
+      - name: Publish evidence to job summary
+        if: always()
+        run: cat ragops-report.md >>"$GITHUB_STEP_SUMMARY"
+      - name: Upload release evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: ragops-release-evidence
+          path: |
+            ragops-report.md
+            ragops-command.log
+            ragops-evidence.json
+      - name: Enforce release decision
+        if: always()
+        env:
+          RAGOPS_GATE_EXIT: ${{ steps.gate.outputs.exit_code }}
+        run: |
+          if [[ -z "$RAGOPS_GATE_EXIT" ]]; then
+            echo "RAGOps gate did not produce an exit code." >&2
+            exit 1
+          fi
+          exit "$RAGOPS_GATE_EXIT"

ragops-1.0.0/.github/workflows/ragops-pr-comment.yml

@@ -0,0 +1,27 @@
+name: RAGOps PR comment publisher
+
+on:
+  workflow_run:
+    workflows: ["RAGOps reusable gate smoke", "RAGOps statistical gate smoke"]
+    types: [completed]
+
+permissions:
+  actions: read
+  contents: read
+  pull-requests: write
+
+jobs:
+  publish:
+    if: github.event.workflow_run.event == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout trusted default-branch publisher
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          persist-credentials: false
+      - name: Validate evidence and publish one idempotent comment
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          RAGOPS_SOURCE_WORKFLOW: ${{ github.event.workflow_run.name }}
+        run: python apps/github_pr_comment.py

ragops-1.0.0/.github/workflows/ragops-statistical-gate-smoke.yml

@@ -0,0 +1,28 @@
+name: RAGOps statistical gate smoke
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - ".github/workflows/ragops-statistical-gate.yml"
+      - ".github/workflows/ragops-statistical-gate-smoke.yml"
+      - "src/ragops/**"
+      - "scenarios/statistical_gate/**"
+      - "tests/test_github_workflows.py"
+  pull_request:
+    paths:
+      - ".github/workflows/ragops-statistical-gate.yml"
+      - ".github/workflows/ragops-statistical-gate-smoke.yml"
+      - "src/ragops/**"
+      - "scenarios/statistical_gate/**"
+      - "tests/test_github_workflows.py"
+  workflow_dispatch:
+
+jobs:
+  passing-statistical-reference:
+    uses: ./.github/workflows/ragops-statistical-gate.yml
+    with:
+      ragops-version: ${{ github.event.pull_request.head.sha || github.sha }}
+      baseline-bundle: scenarios/statistical_gate/baseline.json
+      candidate-bundle: scenarios/statistical_gate/candidate-pass.json
+      policy: scenarios/statistical_gate/policy.toml