sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/data/rag-rerank-v1.json

@@ -0,0 +1,507 @@
+{
+  "schema_version": "rag-evals.dataset.v2",
+  "dataset_id": "help-center-evidence-mini",
+  "version": "2.0.0",
+  "default_relevance_threshold": 2,
+  "tasks": [
+    {
+      "task_id": "core-onboarding-security",
+      "query_id": "q-onboarding-security",
+      "cluster_id": "cluster-onboarding-security",
+      "split_group_id": "split-onboarding-security",
+      "split": "dev",
+      "suite_type": "core",
+      "domain": "help-center",
+      "query": "How do onboarding and the security review work?",
+      "query_variants": ["What happens during setup and compliance review?"],
+      "answerability": "answerable",
+      "query_categories": ["multi_part", "multi_document", "straightforward_factual"],
+      "business_severity": "high",
+      "versions": {
+        "corpus": "fixture-corpus-v2",
+        "index": "fixture-index-v2",
+        "chunking": "section-v2",
+        "embedding": "fixture-embed-v2"
+      },
+      "candidates": [
+        {
+          "candidate_id": "onboarding-flow",
+          "document_id": "doc-onboarding",
+          "url": "https://example.invalid/onboarding",
+          "source_id": "help",
+          "chunk_id": "onboarding-1",
+          "input": "Start onboarding by connecting the workspace, inviting an owner, and completing the setup checklist.",
+          "dense_rank": 1,
+          "dense_score": 0.94,
+          "relevance_grade": 3,
+          "evidence_group_ids": ["onboarding-workflow"],
+          "evidence_spans": [{ "start": 0, "end": 16, "text": "Start onboarding" }],
+          "annotations": [
+            { "annotator_id": "labeler-a", "relevance_grade": 3, "rationale": "Direct workflow evidence" },
+            { "annotator_id": "labeler-b", "relevance_grade": 2, "rationale": "Useful but concise" }
+          ],
+          "adjudication": {
+            "adjudicator_id": "sme-a",
+            "relevance_grade": 3,
+            "rationale": "Directly answers the onboarding facet"
+          }
+        },
+        {
+          "candidate_id": "login-help",
+          "document_id": "doc-login",
+          "url": "https://example.invalid/login",
+          "source_id": "help",
+          "chunk_id": "login-1",
+          "input": "Reset a forgotten password from the login page.",
+          "dense_rank": 2,
+          "dense_score": 0.82,
+          "relevance_grade": 0,
+          "evidence_group_ids": [],
+          "known_hard_negative": true
+        },
+        {
+          "candidate_id": "pricing-help",
+          "document_id": "doc-pricing",
+          "url": "https://example.invalid/pricing",
+          "source_id": "help",
+          "chunk_id": "pricing-1",
+          "input": "Annual plans are billed at the beginning of each term.",
+          "dense_rank": 3,
+          "dense_score": 0.76,
+          "relevance_grade": 0,
+          "evidence_group_ids": []
+        },
+        {
+          "candidate_id": "export-help",
+          "document_id": "doc-export",
+          "url": "https://example.invalid/export",
+          "source_id": "help",
+          "chunk_id": "export-1",
+          "input": "Export reports as CSV from the Reports page.",
+          "dense_rank": 4,
+          "dense_score": 0.72,
+          "relevance_grade": 0,
+          "evidence_group_ids": []
+        },
+        {
+          "candidate_id": "audit-overview",
+          "document_id": "doc-audit",
+          "url": "https://example.invalid/audit",
+          "source_id": "help",
+          "chunk_id": "audit-1",
+          "input": "The audit log records administrative changes.",
+          "dense_rank": 5,
+          "dense_score": 0.66,
+          "relevance_grade": 1,
+          "evidence_group_ids": [],
+          "known_hard_negative": true
+        },
+        {
+          "candidate_id": "security-review",
+          "document_id": "doc-security",
+          "url": "https://example.invalid/security",
+          "source_id": "trust",
+          "chunk_id": "security-4",
+          "input": "The security review covers data handling, access controls, compliance evidence, and approval before launch.",
+          "dense_rank": 6,
+          "dense_score": 0.61,
+          "relevance_grade": 3,
+          "evidence_group_ids": ["security-review"]
+        }
+      ],
+      "evidence_groups": [
+        {
+          "evidence_group_id": "onboarding-workflow",
+          "description": "The onboarding workflow",
+          "required": true,
+          "acceptable_candidate_ids": ["onboarding-flow"]
+        },
+        {
+          "evidence_group_id": "security-review",
+          "description": "The security/compliance review",
+          "required": true,
+          "acceptable_candidate_ids": ["security-review"]
+        }
+      ]
+    },
+    {
+      "task_id": "core-onboarding-security-paraphrase",
+      "query_id": "q-onboarding-security-paraphrase",
+      "cluster_id": "cluster-onboarding-security",
+      "split_group_id": "split-onboarding-security",
+      "split": "dev",
+      "suite_type": "core",
+      "domain": "help-center",
+      "query": "What happens during setup and compliance review?",
+      "answerability": "answerable",
+      "query_categories": ["semantic_paraphrase", "multi_part", "multi_document"],
+      "business_severity": "high",
+      "versions": {
+        "corpus": "fixture-corpus-v2",
+        "index": "fixture-index-v2",
+        "chunking": "section-v2",
+        "embedding": "fixture-embed-v2"
+      },
+      "candidates": [
+        {
+          "candidate_id": "onboarding-flow",
+          "document_id": "doc-onboarding",
+          "url": "https://example.invalid/onboarding",
+          "source_id": "help",
+          "chunk_id": "onboarding-1",
+          "input": "Start onboarding by connecting the workspace, inviting an owner, and completing the setup checklist.",
+          "dense_rank": 1,
+          "dense_score": 0.9,
+          "relevance_grade": 3,
+          "evidence_group_ids": ["onboarding-workflow"]
+        },
+        {
+          "candidate_id": "login-help",
+          "document_id": "doc-login",
+          "url": "https://example.invalid/login",
+          "source_id": "help",
+          "chunk_id": "login-1",
+          "input": "Reset a forgotten password from the login page.",
+          "dense_rank": 2,
+          "dense_score": 0.8,
+          "relevance_grade": 0,
+          "evidence_group_ids": []
+        },
+        {
+          "candidate_id": "pricing-help",
+          "document_id": "doc-pricing",
+          "url": "https://example.invalid/pricing",
+          "source_id": "help",
+          "chunk_id": "pricing-1",
+          "input": "Annual plans are billed at the beginning of each term.",
+          "dense_rank": 3,
+          "dense_score": 0.75,
+          "relevance_grade": 0,
+          "evidence_group_ids": []
+        },
+        {
+          "candidate_id": "export-help",
+          "document_id": "doc-export",
+          "url": "https://example.invalid/export",
+          "source_id": "help",
+          "chunk_id": "export-1",
+          "input": "Export reports as CSV from the Reports page.",
+          "dense_rank": 4,
+          "dense_score": 0.7,
+          "relevance_grade": 0,
+          "evidence_group_ids": []
+        },
+        {
+          "candidate_id": "audit-overview",
+          "document_id": "doc-audit",
+          "url": "https://example.invalid/audit",
+          "source_id": "help",
+          "chunk_id": "audit-1",
+          "input": "The audit log records administrative changes.",
+          "dense_rank": 5,
+          "dense_score": 0.65,
+          "relevance_grade": 1,
+          "evidence_group_ids": []
+        },
+        {
+          "candidate_id": "security-review",
+          "document_id": "doc-security",
+          "url": "https://example.invalid/security",
+          "source_id": "trust",
+          "chunk_id": "security-4",
+          "input": "The security review covers data handling, access controls, compliance evidence, and approval before launch.",
+          "dense_rank": 6,
+          "dense_score": 0.6,
+          "relevance_grade": 3,
+          "evidence_group_ids": ["security-review"]
+        }
+      ],
+      "evidence_groups": [
+        {
+          "evidence_group_id": "onboarding-workflow",
+          "description": "The onboarding workflow",
+          "required": true,
+          "acceptable_candidate_ids": ["onboarding-flow"]
+        },
+        {
+          "evidence_group_id": "security-review",
+          "description": "The security/compliance review",
+          "required": true,
+          "acceptable_candidate_ids": ["security-review"]
+        }
+      ]
+    },
+    {
+      "task_id": "customer-seat-transfer",
+      "query_id": "q-customer-seat-transfer",
+      "cluster_id": "cluster-seat-transfer",
+      "split_group_id": "split-seat-transfer",
+      "split": "test",
+      "suite_type": "customer_acceptance",
+      "customer": "customer-fixture-a",
+      "domain": "account-admin",
+      "query": "Move a named seat to the new workspace steward.",
+      "answerability": "answerable",
+      "query_categories": ["customer_specific_terminology", "postsales_language", "exact_product_feature_name"],
+      "business_severity": "critical",
+      "versions": {
+        "corpus": "fixture-corpus-v2",
+        "index": "fixture-index-v2",
+        "chunking": "section-v2",
+        "embedding": "fixture-embed-v2"
+      },
+      "candidates": [
+        {
+          "candidate_id": "seat-transfer",
+          "document_id": "doc-seats",
+          "url": "https://example.invalid/seats",
+          "source_id": "help",
+          "chunk_id": "seats-2",
+          "input": "An administrator can transfer an assigned seat to another teammate from Workspace members.",
+          "dense_rank": 1,
+          "dense_score": 0.93,
+          "relevance_grade": 3,
+          "evidence_group_ids": ["seat-transfer"]
+        },
+        {
+          "candidate_id": "seat-cancel",
+          "document_id": "doc-seats",
+          "url": "https://example.invalid/seats",
+          "source_id": "help",
+          "chunk_id": "seats-3",
+          "input": "Canceling a seat removes access at the end of the term.",
+          "dense_rank": 2,
+          "dense_score": 0.8,
+          "relevance_grade": 1,
+          "evidence_group_ids": [],
+          "known_hard_negative": true
+        },
+        {
+          "candidate_id": "invite-user",
+          "document_id": "doc-users",
+          "url": "https://example.invalid/users",
+          "source_id": "help",
+          "chunk_id": "users-1",
+          "input": "Invite a new user by email.",
+          "dense_rank": 3,
+          "dense_score": 0.7,
+          "relevance_grade": 0,
+          "evidence_group_ids": []
+        },
+        {
+          "candidate_id": "change-owner",
+          "document_id": "doc-owner",
+          "url": "https://example.invalid/owner",
+          "source_id": "help",
+          "chunk_id": "owner-1",
+          "input": "Workspace ownership can be changed by an existing owner.",
+          "dense_rank": 4,
+          "dense_score": 0.65,
+          "relevance_grade": 1,
+          "evidence_group_ids": []
+        },
+        {
+          "candidate_id": "billing-owner",
+          "document_id": "doc-billing",
+          "url": "https://example.invalid/billing",
+          "source_id": "help",
+          "chunk_id": "billing-2",
+          "input": "The billing contact receives invoices.",
+          "dense_rank": 5,
+          "dense_score": 0.6,
+          "relevance_grade": 0,
+          "evidence_group_ids": []
+        },
+        {
+          "candidate_id": "delete-user",
+          "document_id": "doc-users",
+          "url": "https://example.invalid/users",
+          "source_id": "help",
+          "chunk_id": "users-4",
+          "input": "Deleting a user permanently removes the profile.",
+          "dense_rank": 6,
+          "dense_score": 0.55,
+          "relevance_grade": 0,
+          "evidence_group_ids": []
+        }
+      ],
+      "evidence_groups": [
+        {
+          "evidence_group_id": "seat-transfer",
+          "description": "Transfer an existing seat without canceling it",
+          "required": true,
+          "acceptable_candidate_ids": ["seat-transfer"]
+        }
+      ]
+    },
+    {
+      "task_id": "production-passage-not-url",
+      "query_id": "q-production-passage-not-url",
+      "cluster_id": "cluster-production-passage",
+      "split_group_id": "split-production-passage",
+      "split": "test",
+      "suite_type": "production_failure",
+      "domain": "api",
+      "query": "How do I rotate an API key?",
+      "answerability": "answerable",
+      "query_categories": ["exact_product_feature_name", "postsales_language"],
+      "business_severity": "high",
+      "versions": {
+        "corpus": "fixture-corpus-v2",
+        "index": "fixture-index-v2",
+        "chunking": "section-v2",
+        "embedding": "fixture-embed-v2"
+      },
+      "candidates": [
+        {
+          "candidate_id": "api-overview",
+          "document_id": "doc-api",
+          "url": "https://example.invalid/api",
+          "source_id": "help",
+          "chunk_id": "api-1",
+          "input": "API keys authenticate requests to the service.",
+          "dense_rank": 1,
+          "dense_score": 0.9,
+          "relevance_grade": 1,
+          "evidence_group_ids": [],
+          "known_hard_negative": true
+        },
+        {
+          "candidate_id": "api-rotate",
+          "document_id": "doc-api",
+          "url": "https://example.invalid/api",
+          "source_id": "help",
+          "chunk_id": "api-7",
+          "input": "Create a replacement key, update clients, verify traffic, and revoke the previous key.",
+          "dense_rank": 2,
+          "dense_score": 0.86,
+          "relevance_grade": 3,
+          "evidence_group_ids": ["key-rotation"]
+        }
+      ],
+      "evidence_groups": [
+        {
+          "evidence_group_id": "key-rotation",
+          "description": "Safe API-key rotation procedure",
+          "required": true,
+          "acceptable_candidate_ids": ["api-rotate"]
+        }
+      ],
+      "source_case_id": "anonymized-regression-key-rotation"
+    },
+    {
+      "task_id": "adversarial-unsupported-sso",
+      "query_id": "q-adversarial-unsupported-sso",
+      "cluster_id": "cluster-unsupported-sso",
+      "split_group_id": "split-unsupported-sso",
+      "split": "test",
+      "suite_type": "adversarial_robustness",
+      "domain": "security",
+      "query": "Can this product configure SAML single sign-on?",
+      "answerability": "unsupported",
+      "query_categories": ["unsupported_not_in_kb", "false_premise", "competitor_named_entity"],
+      "business_severity": "critical",
+      "versions": {
+        "corpus": "fixture-corpus-v2",
+        "index": "fixture-index-v2",
+        "chunking": "section-v2",
+        "embedding": "fixture-embed-v2"
+      },
+      "candidates": [
+        {
+          "candidate_id": "two-factor",
+          "document_id": "doc-security",
+          "url": "https://example.invalid/security",
+          "source_id": "help",
+          "chunk_id": "security-2",
+          "input": "Two-factor authentication can be enabled in account security.",
+          "dense_rank": 1,
+          "dense_score": 0.79,
+          "relevance_grade": 0,
+          "evidence_group_ids": [],
+          "known_hard_negative": true
+        },
+        {
+          "candidate_id": "password-reset",
+          "document_id": "doc-password",
+          "url": "https://example.invalid/password",
+          "source_id": "help",
+          "chunk_id": "password-1",
+          "input": "Send a password reset link from the login screen.",
+          "dense_rank": 2,
+          "dense_score": 0.61,
+          "relevance_grade": 0,
+          "evidence_group_ids": []
+        }
+      ],
+      "evidence_groups": [],
+      "expected_unsupported_behavior": "abstain"
+    },
+    {
+      "task_id": "adversarial-duplicate-guides",
+      "query_id": "q-adversarial-duplicate-guides",
+      "cluster_id": "cluster-duplicate-guides",
+      "split_group_id": "split-duplicate-guides",
+      "split": "test",
+      "suite_type": "adversarial_robustness",
+      "domain": "billing",
+      "query": "Where can I download an invoice?",
+      "answerability": "answerable",
+      "query_categories": ["duplicate_documentation", "adjacent_chunk_dependency"],
+      "business_severity": "medium",
+      "versions": {
+        "corpus": "fixture-corpus-v2",
+        "index": "fixture-index-v2",
+        "chunking": "section-v2",
+        "embedding": "fixture-embed-v2"
+      },
+      "candidates": [
+        {
+          "candidate_id": "invoice-primary",
+          "document_id": "doc-invoice",
+          "url": "https://example.invalid/invoices",
+          "source_id": "help",
+          "chunk_id": "invoice-3",
+          "input": "Open Billing, choose an invoice, then select Download PDF.",
+          "dense_rank": 1,
+          "dense_score": 0.95,
+          "relevance_grade": 3,
+          "evidence_group_ids": ["invoice-download"]
+        },
+        {
+          "candidate_id": "invoice-copy",
+          "document_id": "doc-invoice-copy",
+          "url": "https://example.invalid/invoice-copy",
+          "source_id": "help-copy",
+          "chunk_id": "invoice-copy-3",
+          "input": "Open Billing, choose an invoice, then select Download PDF.",
+          "dense_rank": 2,
+          "dense_score": 0.94,
+          "relevance_grade": 3,
+          "evidence_group_ids": ["invoice-download"],
+          "duplicate_of_candidate_id": "invoice-primary"
+        },
+        {
+          "candidate_id": "invoice-email",
+          "document_id": "doc-invoice",
+          "url": "https://example.invalid/invoices",
+          "source_id": "help",
+          "chunk_id": "invoice-4",
+          "input": "Invoices are emailed to the billing contact after payment.",
+          "dense_rank": 3,
+          "dense_score": 0.62,
+          "relevance_grade": 1,
+          "evidence_group_ids": []
+        }
+      ],
+      "evidence_groups": [
+        {
+          "evidence_group_id": "invoice-download",
+          "description": "Invoice PDF download workflow",
+          "required": true,
+          "acceptable_candidate_ids": ["invoice-primary", "invoice-copy"]
+        }
+      ]
+    }
+  ]
+}