ragops 1.6.0__tar.gz

ragops-1.6.0/.dockerignore

@@ -0,0 +1,9 @@
+.git
+.github
+.venv
+__pycache__
+.pytest_cache
+.ruff_cache
+reports
+tests
+scenarios

ragops-1.6.0/.github/ISSUE_TEMPLATE/bug.yml

@@ -0,0 +1,41 @@
+name: Bug report
+description: Report a reproducible defect without private data
+title: "[Bug]: "
+labels: [bug]
+body:
+  - type: markdown
+    attributes:
+      value: Never include secrets, customer-confidential data, or private traces.
+  - type: input
+    id: version
+    attributes:
+      label: RAGOps version or commit
+      placeholder: v1.6.0 or commit SHA
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Minimal reproduction
+      description: Include the command and a synthetic or redacted fixture.
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+    validations:
+      required: true
+  - type: textarea
+    id: actual
+    attributes:
+      label: Actual behavior
+    validations:
+      required: true
+  - type: input
+    id: environment
+    attributes:
+      label: Environment
+      placeholder: Python 3.12, macOS 15
+    validations:
+      required: true

ragops-1.6.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Security vulnerability
+    url: https://github.com/thangldw/ragops/security/policy
+    about: Report vulnerabilities privately through the security policy.
+  - name: Support guidance
+    url: https://github.com/thangldw/ragops/blob/main/SUPPORT.md
+    about: Read what to include before opening an issue.

ragops-1.6.0/.github/ISSUE_TEMPLATE/design-partner.yml

@@ -0,0 +1,48 @@
+name: Design-partner interest
+description: Explore an offline RAGOps evaluation pilot using synthetic or redacted data.
+title: "[Design partner]: "
+labels: [design-partner]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Do not include production secrets, personal data, customer-confidential
+        content, API keys, or proprietary traces in this public issue.
+  - type: textarea
+    id: workflow
+    attributes:
+      label: AI workflow
+      description: What RAG or agent workflow are you evaluating?
+    validations:
+      required: true
+  - type: textarea
+    id: decision
+    attributes:
+      label: Release decision today
+      description: How do you currently decide whether a candidate is safe and useful enough to release?
+    validations:
+      required: true
+  - type: textarea
+    id: failure
+    attributes:
+      label: Representative synthetic failure
+      description: Describe a redacted or synthetic regression without sharing sensitive data.
+    validations:
+      required: true
+  - type: dropdown
+    id: data
+    attributes:
+      label: Safe fixture availability
+      options:
+        - Synthetic fixtures available
+        - Redacted fixtures available
+        - Need help creating safe fixtures
+    validations:
+      required: true
+  - type: checkboxes
+    id: consent
+    attributes:
+      label: Public issue safety
+      options:
+        - label: I confirm this issue contains no secrets, personal data, or customer-confidential content.
+          required: true

ragops-1.6.0/.github/ISSUE_TEMPLATE/evaluator.yml

@@ -0,0 +1,40 @@
+name: Evaluator proposal
+description: Propose a measurable evaluator with calibration and failure modes.
+title: "[Evaluator]: "
+labels: [evaluation]
+body:
+  - type: textarea
+    id: construct
+    attributes:
+      label: Construct being measured
+      description: Define precisely what the score means and does not mean.
+    validations:
+      required: true
+  - type: textarea
+    id: method
+    attributes:
+      label: Method and inputs
+      description: Explain deterministic/provider behavior, required evidence, and output range.
+    validations:
+      required: true
+  - type: textarea
+    id: calibration
+    attributes:
+      label: Calibration plan
+      description: Labeled cases, human rubric, agreement analysis, and thresholds.
+    validations:
+      required: true
+  - type: textarea
+    id: failures
+    attributes:
+      label: Known failure modes
+    validations:
+      required: true
+  - type: checkboxes
+    id: boundary
+    attributes:
+      label: Product boundary
+      options:
+        - label: The core remains useful offline or this is an explicitly optional plugin.
+          required: true
+

ragops-1.6.0/.github/ISSUE_TEMPLATE/feature.yml

@@ -0,0 +1,38 @@
+name: Product feature
+description: Propose a capability tied to a user outcome and release contract.
+title: "[Feature]: "
+labels: [enhancement]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: User problem
+      description: Who has the problem, in what workflow, and why does it matter?
+    validations:
+      required: true
+  - type: textarea
+    id: outcome
+    attributes:
+      label: Desired outcome
+      description: Describe observable success without prescribing implementation.
+    validations:
+      required: true
+  - type: textarea
+    id: contract
+    attributes:
+      label: Contract and acceptance criteria
+      description: Include input, output, release behavior, compatibility, and tests.
+    validations:
+      required: true
+  - type: textarea
+    id: non_goals
+    attributes:
+      label: Non-goals and boundaries
+    validations:
+      required: true
+  - type: textarea
+    id: evidence
+    attributes:
+      label: Supporting evidence
+      description: Customer observation, benchmark case, incident, or design-partner request.
+

ragops-1.6.0/.github/pull_request_template.md

@@ -0,0 +1,26 @@
+## Product contract
+
+- Requirement / issue:
+- User outcome:
+- Explicit non-goals:
+
+## Change
+
+- Public API or schema impact:
+- Metric or release-decision impact:
+- Architecture / ADR impact:
+
+## Verification
+
+- [ ] Tests added or updated
+- [ ] `ruff check .`
+- [ ] `PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest -q`
+- [ ] Affected CLI/API workflow exercised
+- [ ] Documentation and changelog updated
+
+## Owner acceptance
+
+- Evidence or demo:
+- Known limitations:
+- Decision requested from product owner:
+

ragops-1.6.0/.github/workflows/ci.yml

@@ -0,0 +1,73 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install -e '.[dev]'
+      - run: ruff check .
+      - run: pytest
+      - run: ragops evaluate --scenario scenarios/japanese_troubleshooting/scenario.json --responses scenarios/japanese_troubleshooting/sample_responses.json
+      - name: Validate reference benchmark coverage
+        run: ragops inspect --scenario scenarios/japanese_troubleshooting/benchmark-v0.2.json
+      - name: Evaluate reference benchmark baseline
+        run: >-
+          ragops evaluate
+          --scenario scenarios/japanese_troubleshooting/benchmark-v0.2.json
+          --responses scenarios/japanese_troubleshooting/benchmark-baseline.json
+          --evaluator citation_correctness
+          --evaluator claim_support
+      - name: Confirm regressed benchmark is blocked
+        shell: bash
+        run: |
+          set +e
+          ragops compare \
+            --scenario scenarios/japanese_troubleshooting/benchmark-v0.2.json \
+            --baseline scenarios/japanese_troubleshooting/benchmark-baseline.json \
+            --candidate scenarios/japanese_troubleshooting/benchmark-regressed.json \
+            --format markdown
+          code=$?
+          set -e
+          test "$code" -eq 2
+      - name: Confirm adversarial benchmark is blocked
+        shell: bash
+        run: |
+          set +e
+          ragops evaluate \
+            --scenario scenarios/japanese_troubleshooting/benchmark-v0.2.json \
+            --responses scenarios/japanese_troubleshooting/benchmark-adversarial.json
+          code=$?
+          set -e
+          test "$code" -eq 2
+      - name: Render regression report
+        run: >-
+          ragops compare
+          --scenario scenarios/japanese_troubleshooting/scenario.json
+          --baseline scenarios/japanese_troubleshooting/sample_responses.json
+          --candidate scenarios/japanese_troubleshooting/sample_responses.json
+          --format markdown
+          --output reports/regression.md
+      - name: Run reference GraphRAG agent
+        run: >-
+          python -m examples.japanese_troubleshooting_agent.cli
+          --suite examples/japanese_troubleshooting_agent/suite.json
+          --retriever graph
+          --output /tmp/reference-traces.jsonl
+      - name: Evaluate reference deployment traces
+        run: >-
+          ragops evaluate
+          --scenario examples/japanese_troubleshooting_agent/scenario.json
+          --traces /tmp/reference-traces.jsonl
+          --evaluator citation_correctness
+          --evaluator claim_support

ragops-1.6.0/.github/workflows/nightly.yml

@@ -0,0 +1,39 @@
+name: Nightly benchmark
+
+on:
+  schedule:
+    - cron: "17 18 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install -e '.[dev]'
+      - name: Evaluate 30-case benchmark
+        run: >-
+          ragops evaluate
+          --scenario scenarios/japanese_troubleshooting/benchmark-v0.2.json
+          --responses scenarios/japanese_troubleshooting/benchmark-baseline.json
+          --evaluator citation_correctness
+          --evaluator claim_support
+      - name: Run reference deployment
+        run: >-
+          python -m examples.japanese_troubleshooting_agent.cli
+          --suite examples/japanese_troubleshooting_agent/suite.json
+          --retriever graph
+          --output /tmp/reference-traces.jsonl
+      - name: Gate reference traces
+        run: >-
+          ragops evaluate
+          --scenario examples/japanese_troubleshooting_agent/scenario.json
+          --traces /tmp/reference-traces.jsonl
+          --evaluator citation_correctness
+          --evaluator claim_support

ragops-1.6.0/.github/workflows/pages.yml

@@ -0,0 +1,31 @@
+name: Pages
+
+on:
+  push:
+    branches: [main]
+    paths: ["site/**", ".github/workflows/pages.yml"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: true
+
+jobs:
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/configure-pages@v5
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: site
+      - id: deployment
+        uses: actions/deploy-pages@v4

ragops-1.6.0/.github/workflows/publish-pypi.yml

@@ -0,0 +1,43 @@
+name: Publish package to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: Existing RAGOps release tag to publish
+        required: true
+        type: string
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.tag }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Verify tag and package version
+        env:
+          RELEASE_TAG: ${{ inputs.tag }}
+        run: |
+          python - "$RELEASE_TAG" <<'PY'
+          import sys
+          import tomllib
+          from pathlib import Path
+
+          tag = sys.argv[1]
+          version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
+          if tag != f"v{version}":
+              raise SystemExit(f"tag {tag!r} does not match package version v{version}")
+          print(f"publishing ragops {version}")
+          PY
+      - run: python -m pip install --upgrade build
+      - run: python -m build
+      - name: Publish distributions with Trusted Publishing
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0

ragops-1.6.0/.github/workflows/ragops-gate-smoke.yml

@@ -0,0 +1,24 @@
+name: RAGOps reusable gate smoke
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - ".github/workflows/ragops-gate.yml"
+      - ".github/workflows/ragops-gate-smoke.yml"
+      - "tests/test_github_workflows.py"
+  pull_request:
+    paths:
+      - ".github/workflows/ragops-gate.yml"
+      - ".github/workflows/ragops-gate-smoke.yml"
+      - "tests/test_github_workflows.py"
+  workflow_dispatch:
+
+jobs:
+  passing-reference:
+    uses: ./.github/workflows/ragops-gate.yml
+    with:
+      ragops-version: v1.6.0
+      scenario: scenarios/japanese_troubleshooting/benchmark-v0.2.json
+      baseline: scenarios/japanese_troubleshooting/benchmark-baseline.json
+      candidate: scenarios/japanese_troubleshooting/benchmark-baseline.json

ragops-1.6.0/.github/workflows/ragops-gate.yml

@@ -0,0 +1,104 @@
+name: RAGOps reusable release gate
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        description: Python version used to run RAGOps
+        required: false
+        type: string
+        default: "3.12"
+      ragops-version:
+        description: RAGOps Git revision or tag to install
+        required: false
+        type: string
+        default: "v1.6.0"
+      scenario:
+        description: Caller-relative scenario JSON path
+        required: true
+        type: string
+      baseline:
+        description: Caller-relative accepted baseline response path
+        required: true
+        type: string
+      candidate:
+        description: Caller-relative candidate response path
+        required: true
+        type: string
+      policy:
+        description: Optional caller-relative regression policy path
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  release-gate:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ inputs.python-version }}
+      - name: Install RAGOps
+        env:
+          RAGOPS_VERSION: ${{ inputs.ragops-version }}
+        run: >-
+          python -m pip install
+          "ragops @ git+https://github.com/thangldw/ragops.git@${RAGOPS_VERSION}"
+      - id: gate
+        name: Compare candidate with accepted baseline
+        shell: bash
+        env:
+          RAGOPS_SCENARIO: ${{ inputs.scenario }}
+          RAGOPS_BASELINE: ${{ inputs.baseline }}
+          RAGOPS_CANDIDATE: ${{ inputs.candidate }}
+          RAGOPS_POLICY: ${{ inputs.policy }}
+        run: |
+          set +e
+          command=(
+            ragops compare
+            --scenario "$RAGOPS_SCENARIO"
+            --baseline "$RAGOPS_BASELINE"
+            --candidate "$RAGOPS_CANDIDATE"
+            --format markdown
+            --output ragops-report.md
+          )
+          if [[ -n "$RAGOPS_POLICY" ]]; then
+            command+=(--policy "$RAGOPS_POLICY")
+          fi
+          "${command[@]}" >ragops-command.log 2>&1
+          gate_exit=$?
+          set -e
+          if [[ ! -f ragops-report.md ]]; then
+            {
+              echo "# RAGOps release gate error"
+              echo
+              echo '```text'
+              cat ragops-command.log
+              echo '```'
+            } >ragops-report.md
+          fi
+          echo "exit_code=$gate_exit" >>"$GITHUB_OUTPUT"
+      - name: Publish evidence to job summary
+        if: always()
+        run: cat ragops-report.md >>"$GITHUB_STEP_SUMMARY"
+      - name: Upload release evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ragops-release-evidence
+          path: |
+            ragops-report.md
+            ragops-command.log
+      - name: Enforce release decision
+        if: always()
+        env:
+          RAGOPS_GATE_EXIT: ${{ steps.gate.outputs.exit_code }}
+        run: |
+          if [[ -z "$RAGOPS_GATE_EXIT" ]]; then
+            echo "RAGOps gate did not produce an exit code." >&2
+            exit 1
+          fi
+          exit "$RAGOPS_GATE_EXIT"

ragops-1.6.0/.github/workflows/release.yml

@@ -0,0 +1,38 @@
+name: Release
+
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  contents: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: python -m pip install --upgrade build
+      - run: pip install -e '.[dev]'
+      - run: pytest
+      - run: ruff check .
+      - run: python -m build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: python-package
+          path: dist/
+      - name: Publish GitHub Release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then
+            gh release upload "$GITHUB_REF_NAME" dist/* --clobber
+          else
+            gh release create "$GITHUB_REF_NAME" dist/* \
+              --verify-tag \
+              --title "RAGOps $GITHUB_REF_NAME" \
+              --notes-file "docs/releases/$GITHUB_REF_NAME.md"
+          fi

ragops-1.6.0/.gitignore

@@ -0,0 +1,13 @@
+.venv/
+.release-venv/
+.release-check-venv/
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.ruff_cache/
+dist/
+build/
+*.egg-info/
+reports/
+.env
+.DS_Store

ragops-1.6.0/AGENTS.md

@@ -0,0 +1,36 @@
+# RAGOps collaboration contract
+
+This repository is product-first. Product intent, contracts, and acceptance
+criteria are authoritative; implementation must not silently redefine them.
+
+## Ownership
+
+- Thang owns product vision, architecture acceptance, review, and release.
+- Codex and Claude Code may implement, test, refactor, and propose ADRs.
+- Any change to a public schema, release gate, metric meaning, or open-core
+  boundary requires an ADR and explicit owner review.
+
+## Required workflow
+
+1. Read the relevant product requirement and architecture decision.
+2. State the contract being changed and its compatibility impact.
+3. Implement the smallest vertical slice, including tests and documentation.
+4. Run `ruff check .` and `PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest -q`.
+5. Update the changelog for user-visible behavior.
+6. Leave the work ready for owner acceptance; do not publish or release unless
+   explicitly requested.
+
+## Architectural boundaries
+
+- `src/ragops/` is the dependency-free evaluation core.
+- `apps/` contains adapters and user interfaces, not evaluation semantics.
+- `scenarios/` contains portable fixtures, policies, and expected evidence.
+- Provider integrations are optional adapters or plugins.
+- The open-source core must remain valuable without a hosted service.
+
+## Repository skills
+
+- Use `skills/ragops-feature/SKILL.md` for feature implementation.
+- Use `skills/ragops-release/SKILL.md` for validation and release readiness.
+- Use `skills/ragops-presentation/SKILL.md` for demos and presentation assets.
+