ragvidence 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. ragvidence-0.1.0/.env.example +12 -0
  2. ragvidence-0.1.0/.gitignore +14 -0
  3. ragvidence-0.1.0/.python-version +1 -0
  4. ragvidence-0.1.0/CHANGELOG.md +24 -0
  5. ragvidence-0.1.0/LICENSE.md +74 -0
  6. ragvidence-0.1.0/PKG-INFO +213 -0
  7. ragvidence-0.1.0/README.md +111 -0
  8. ragvidence-0.1.0/SECURITY.md +31 -0
  9. ragvidence-0.1.0/SUPPORT.md +24 -0
  10. ragvidence-0.1.0/THIRD_PARTY_LICENSES.md +51 -0
  11. ragvidence-0.1.0/THIRD_PARTY_NOTICES.md +19 -0
  12. ragvidence-0.1.0/docs/bring-your-own-data.md +46 -0
  13. ragvidence-0.1.0/docs/command-reference.md +32 -0
  14. ragvidence-0.1.0/docs/getting-started.md +102 -0
  15. ragvidence-0.1.0/docs/interpreting-results.md +30 -0
  16. ragvidence-0.1.0/docs/plans.md +36 -0
  17. ragvidence-0.1.0/docs/privacy-and-providers.md +13 -0
  18. ragvidence-0.1.0/docs/troubleshooting.md +56 -0
  19. ragvidence-0.1.0/docs/user-guide.md +191 -0
  20. ragvidence-0.1.0/pyproject.toml +83 -0
  21. ragvidence-0.1.0/src/ragvidence/__init__.py +4 -0
  22. ragvidence-0.1.0/src/ragvidence/cli.py +373 -0
  23. ragvidence-0.1.0/src/ragvidence/comparison.py +35 -0
  24. ragvidence-0.1.0/src/ragvidence/config.py +16 -0
  25. ragvidence-0.1.0/src/ragvidence/data.py +72 -0
  26. ragvidence-0.1.0/src/ragvidence/diagnosis.py +40 -0
  27. ragvidence-0.1.0/src/ragvidence/evaluation/__init__.py +19 -0
  28. ragvidence-0.1.0/src/ragvidence/evaluation/evaluator.py +54 -0
  29. ragvidence-0.1.0/src/ragvidence/evaluation/metrics.py +131 -0
  30. ragvidence-0.1.0/src/ragvidence/exercises/__init__.py +1 -0
  31. ragvidence-0.1.0/src/ragvidence/exercises/heading_context/README.md +5 -0
  32. ragvidence-0.1.0/src/ragvidence/exercises/heading_context/integrity.json +4 -0
  33. ragvidence-0.1.0/src/ragvidence/exercises/heading_context/questions.jsonl +12 -0
  34. ragvidence-0.1.0/src/ragvidence/exercises/heading_context/sources.jsonl +8 -0
  35. ragvidence-0.1.0/src/ragvidence/generation.py +60 -0
  36. ragvidence-0.1.0/src/ragvidence/health.py +67 -0
  37. ragvidence-0.1.0/src/ragvidence/importer.py +132 -0
  38. ragvidence-0.1.0/src/ragvidence/models.py +101 -0
  39. ragvidence-0.1.0/src/ragvidence/providers/__init__.py +12 -0
  40. ragvidence-0.1.0/src/ragvidence/providers/base.py +12 -0
  41. ragvidence-0.1.0/src/ragvidence/providers/fastembed.py +19 -0
  42. ragvidence-0.1.0/src/ragvidence/providers/remote.py +114 -0
  43. ragvidence-0.1.0/src/ragvidence/reporting/__init__.py +6 -0
  44. ragvidence-0.1.0/src/ragvidence/reporting/html.py +15 -0
  45. ragvidence-0.1.0/src/ragvidence/reporting/markdown.py +72 -0
  46. ragvidence-0.1.0/src/ragvidence/reporting/payload.py +32 -0
  47. ragvidence-0.1.0/src/ragvidence/reporting/templates/report.html.j2 +689 -0
  48. ragvidence-0.1.0/src/ragvidence/representation.py +25 -0
  49. ragvidence-0.1.0/src/ragvidence/runner.py +83 -0
  50. ragvidence-0.1.0/src/ragvidence/strategies/__init__.py +6 -0
  51. ragvidence-0.1.0/src/ragvidence/strategies/base.py +12 -0
  52. ragvidence-0.1.0/src/ragvidence/strategies/bm25.py +49 -0
  53. ragvidence-0.1.0/src/ragvidence/strategies/semantic.py +86 -0
  54. ragvidence-0.1.0/src/ragvidence/verification.py +115 -0
  55. ragvidence-0.1.0/src/ragvidence/workspace.py +33 -0
  56. ragvidence-0.1.0/tests/unit/test_cli.py +51 -0
  57. ragvidence-0.1.0/tests/unit/test_cli_errors.py +23 -0
  58. ragvidence-0.1.0/tests/unit/test_comparison.py +23 -0
  59. ragvidence-0.1.0/tests/unit/test_data_and_retrieval.py +42 -0
  60. ragvidence-0.1.0/tests/unit/test_evaluator_contract.py +51 -0
  61. ragvidence-0.1.0/tests/unit/test_importer.py +47 -0
  62. ragvidence-0.1.0/tests/unit/test_metrics.py +95 -0
  63. ragvidence-0.1.0/tests/unit/test_models.py +25 -0
  64. ragvidence-0.1.0/tests/unit/test_remote_providers.py +54 -0
  65. ragvidence-0.1.0/tests/unit/test_reporting.py +118 -0
  66. ragvidence-0.1.0/tests/unit/test_semantic.py +51 -0
  67. ragvidence-0.1.0/tests/unit/test_verification.py +9 -0
  68. ragvidence-0.1.0/uv.lock +818 -0
@@ -0,0 +1,12 @@
1
+ # Optional. Core defaults are fastembed + BAAI/bge-small-en-v1.5.
2
+ # Other values: ollama, openai_compatible.
3
+ RAGVIDENCE_EMBEDDING_PROVIDER=fastembed
4
+ RAGVIDENCE_EMBEDDING_MODEL=BAAI/bge-small-en-v1.5
5
+ RAGVIDENCE_EMBEDDING_BASE_URL=
6
+ RAGVIDENCE_EMBEDDING_API_KEY=
7
+
8
+ # Optional answer inspection: ollama or openai_compatible.
9
+ RAGVIDENCE_GENERATION_PROVIDER=
10
+ RAGVIDENCE_GENERATION_MODEL=
11
+ RAGVIDENCE_GENERATION_BASE_URL=
12
+ RAGVIDENCE_GENERATION_API_KEY=
@@ -0,0 +1,14 @@
1
+ .venv/
2
+ .env
3
+ .pytest_cache/
4
+ __pycache__/
5
+ *.py[cod]
6
+ workspaces/
7
+ .ragvidence-cache/
8
+ dist/
9
+ build/
10
+ .ruff_cache/
11
+ *.egg-info/
12
+ .coverage
13
+ htmlcov/
14
+ *.log
@@ -0,0 +1 @@
1
+ 3.12
@@ -0,0 +1,24 @@
1
+ # Changelog
2
+
3
+ All notable user-facing changes are documented here.
4
+
5
+ ## 0.1.0 — 2026-10-08
6
+
7
+ ### Added
8
+
9
+ - Local BM25 body and heading-context experiments.
10
+ - Optional semantic retrieval with FastEmbed, Ollama, and OpenAI-compatible endpoints.
11
+ - Import path for existing retrievers.
12
+ - Evidence sufficiency, coverage, rank, reciprocal-rank, and precision metrics.
13
+ - Question-level regression classification and bounded diagnosis.
14
+ - JSON, Markdown, and self-contained HTML reports.
15
+ - Guided `ragvi tutorial` workflow.
16
+ - Workspace backup and restore.
17
+ - Health and deterministic verification commands.
18
+ - Customer documentation, privacy boundaries, and Free/Plus license terms.
19
+
20
+ ### Known limits
21
+
22
+ - Unanswerable questions are excluded from retrieval-success scoring; abstention is not evaluated.
23
+ - The bundled teaching dataset is small, fictional, and English-focused.
24
+ - Retrieval evaluation does not establish generated-answer correctness.
@@ -0,0 +1,74 @@
1
+ # RAGvidence Software License
2
+
3
+ Version 1.0 — 8 October 2026
4
+
5
+ Copyright © 2026 Medo. All rights reserved.
6
+
7
+ This is proprietary software, not open-source software. “Licensor” means the seller identified on the product page or transaction receipt. “You” means the individual who obtained the software, or the organization identified on a valid order if the order expressly grants an organizational license.
8
+
9
+ By downloading, installing, copying, or using RAGvidence, you agree to this license. If you do not agree, do not use the software.
10
+
11
+ ## 1. Free license grant
12
+
13
+ For a build designated **Free**, Licensor grants one individual a revocable, non-exclusive, non-transferable license to install and use the software on devices controlled by that individual for personal, educational, evaluation, and internal professional work.
14
+
15
+ You may use reports and other output you generate, including in commercial work, subject to third-party rights and your responsibility to review confidential content.
16
+
17
+ No account or paid activation is required for the Free edition.
18
+
19
+ ## 2. Plus license grant
20
+
21
+ For a build or entitlement designated **Plus**, and while the applicable purchase entitlement remains valid, Licensor grants the purchasing individual the Free rights plus access to the Plus features and updates expressly listed for that purchase.
22
+
23
+ Unless the order expressly states otherwise, Plus is a single-user license. It is not a team, organization-wide, redistribution, resale, or hosted-service license. Price, billing period, included update period, support level, taxes, and refund eligibility are controlled by the checkout terms and receipt.
24
+
25
+ ## 3. Restrictions
26
+
27
+ Under either plan, you may not:
28
+
29
+ - redistribute, publish, sublicense, sell, rent, lease, or share the software or a license credential;
30
+ - remove copyright, license, attribution, or entitlement notices;
31
+ - provide the software itself as a hosted or managed service to third parties;
32
+ - bypass, tamper with, forge, or defeat an entitlement or plan check;
33
+ - use the software or documentation to create a substantially identical competing product for redistribution;
34
+ - use the software unlawfully or in violation of third-party rights.
35
+
36
+ You may make reasonable backup copies for your own licensed use. You may inspect and modify source files delivered to you solely for your own licensed use, but modifications do not grant redistribution rights or transfer support obligations to Licensor.
37
+
38
+ ## 4. Data, providers, and third-party components
39
+
40
+ The software processes data locally by default. Features you configure to use external providers transmit selected source text, questions, or prompts to those providers. You are responsible for having authority to process that data and for reviewing each provider’s terms, security, retention, and training policies.
41
+
42
+ Third-party components and downloaded models remain governed by their own licenses. See `THIRD_PARTY_NOTICES.md`. This license does not grant rights to third-party content, datasets, models, or services.
43
+
44
+ ## 5. High-risk use and professional advice
45
+
46
+ The software is a retrieval evaluation and debugging aid. It does not provide legal, financial, medical, audit, compliance, or other professional advice. It does not guarantee that retrieved evidence, generated answers, labels, or reports are accurate or complete. You are responsible for human review, testing, security controls, and decisions made using the software.
47
+
48
+ ## 6. Confidentiality and exported reports
49
+
50
+ Reports can contain source identifiers, metadata, and text previews. You are responsible for inspecting and redacting exported artifacts before sharing them and for complying with confidentiality, privacy, employment, and contractual obligations.
51
+
52
+ ## 7. Updates, support, and availability
53
+
54
+ Licensor may change or discontinue editions and future features. Only capabilities present in the delivered build and expressly included in the applicable order are licensed. Planned or roadmap items are not commitments. Support, if included, is limited to the scope stated at checkout or on the receipt.
55
+
56
+ ## 8. Termination
57
+
58
+ This license ends automatically if you materially violate it. On termination, you must stop using and delete the software and license credentials, except where applicable law requires otherwise. Sections intended by their nature to survive—including restrictions, disclaimers, liability limits, and third-party terms—remain effective.
59
+
60
+ ## 9. Disclaimer of warranties
61
+
62
+ To the maximum extent permitted by applicable law, the software is provided “as is” and “as available,” without warranties of merchantability, fitness for a particular purpose, non-infringement, uninterrupted operation, or error-free results. Some jurisdictions do not allow certain exclusions, so these exclusions apply only to the extent permitted.
63
+
64
+ ## 10. Limitation of liability
65
+
66
+ To the maximum extent permitted by applicable law, Licensor is not liable for indirect, incidental, special, consequential, exemplary, or punitive damages, or for loss of data, profit, revenue, business, goodwill, or opportunity arising from the software. Licensor’s aggregate liability relating to a paid license will not exceed the amount you paid for that license during the twelve months before the event giving rise to the claim. Nothing here excludes liability that cannot legally be excluded or limited.
67
+
68
+ ## 11. Consumer rights and controlling terms
69
+
70
+ Mandatory consumer rights remain unaffected. If checkout terms or a written order conflict with this license, the more specific checkout terms or order control only for that transaction. If any provision is unenforceable, the remaining provisions continue to apply.
71
+
72
+ ## 12. Contact
73
+
74
+ Use the contact or support channel identified on the product page or transaction receipt for license questions.
@@ -0,0 +1,213 @@
1
+ Metadata-Version: 2.5
2
+ Name: ragvidence
3
+ Version: 0.1.0
4
+ Summary: Evidence-first RAG retrieval diagnostics, built to run locally.
5
+ Project-URL: Homepage, https://ragvidence.medo-id.com
6
+ Project-URL: Documentation, https://ragvidence.medo-id.com/docs
7
+ Author: Medo
8
+ License: # RAGvidence Software License
9
+
10
+ Version 1.0 — 8 October 2026
11
+
12
+ Copyright © 2026 Medo. All rights reserved.
13
+
14
+ This is proprietary software, not open-source software. “Licensor” means the seller identified on the product page or transaction receipt. “You” means the individual who obtained the software, or the organization identified on a valid order if the order expressly grants an organizational license.
15
+
16
+ By downloading, installing, copying, or using RAGvidence, you agree to this license. If you do not agree, do not use the software.
17
+
18
+ ## 1. Free license grant
19
+
20
+ For a build designated **Free**, Licensor grants one individual a revocable, non-exclusive, non-transferable license to install and use the software on devices controlled by that individual for personal, educational, evaluation, and internal professional work.
21
+
22
+ You may use reports and other output you generate, including in commercial work, subject to third-party rights and your responsibility to review confidential content.
23
+
24
+ No account or paid activation is required for the Free edition.
25
+
26
+ ## 2. Plus license grant
27
+
28
+ For a build or entitlement designated **Plus**, and while the applicable purchase entitlement remains valid, Licensor grants the purchasing individual the Free rights plus access to the Plus features and updates expressly listed for that purchase.
29
+
30
+ Unless the order expressly states otherwise, Plus is a single-user license. It is not a team, organization-wide, redistribution, resale, or hosted-service license. Price, billing period, included update period, support level, taxes, and refund eligibility are controlled by the checkout terms and receipt.
31
+
32
+ ## 3. Restrictions
33
+
34
+ Under either plan, you may not:
35
+
36
+ - redistribute, publish, sublicense, sell, rent, lease, or share the software or a license credential;
37
+ - remove copyright, license, attribution, or entitlement notices;
38
+ - provide the software itself as a hosted or managed service to third parties;
39
+ - bypass, tamper with, forge, or defeat an entitlement or plan check;
40
+ - use the software or documentation to create a substantially identical competing product for redistribution;
41
+ - use the software unlawfully or in violation of third-party rights.
42
+
43
+ You may make reasonable backup copies for your own licensed use. You may inspect and modify source files delivered to you solely for your own licensed use, but modifications do not grant redistribution rights or transfer support obligations to Licensor.
44
+
45
+ ## 4. Data, providers, and third-party components
46
+
47
+ The software processes data locally by default. Features you configure to use external providers transmit selected source text, questions, or prompts to those providers. You are responsible for having authority to process that data and for reviewing each provider’s terms, security, retention, and training policies.
48
+
49
+ Third-party components and downloaded models remain governed by their own licenses. See `THIRD_PARTY_NOTICES.md`. This license does not grant rights to third-party content, datasets, models, or services.
50
+
51
+ ## 5. High-risk use and professional advice
52
+
53
+ The software is a retrieval evaluation and debugging aid. It does not provide legal, financial, medical, audit, compliance, or other professional advice. It does not guarantee that retrieved evidence, generated answers, labels, or reports are accurate or complete. You are responsible for human review, testing, security controls, and decisions made using the software.
54
+
55
+ ## 6. Confidentiality and exported reports
56
+
57
+ Reports can contain source identifiers, metadata, and text previews. You are responsible for inspecting and redacting exported artifacts before sharing them and for complying with confidentiality, privacy, employment, and contractual obligations.
58
+
59
+ ## 7. Updates, support, and availability
60
+
61
+ Licensor may change or discontinue editions and future features. Only capabilities present in the delivered build and expressly included in the applicable order are licensed. Planned or roadmap items are not commitments. Support, if included, is limited to the scope stated at checkout or on the receipt.
62
+
63
+ ## 8. Termination
64
+
65
+ This license ends automatically if you materially violate it. On termination, you must stop using and delete the software and license credentials, except where applicable law requires otherwise. Sections intended by their nature to survive—including restrictions, disclaimers, liability limits, and third-party terms—remain effective.
66
+
67
+ ## 9. Disclaimer of warranties
68
+
69
+ To the maximum extent permitted by applicable law, the software is provided “as is” and “as available,” without warranties of merchantability, fitness for a particular purpose, non-infringement, uninterrupted operation, or error-free results. Some jurisdictions do not allow certain exclusions, so these exclusions apply only to the extent permitted.
70
+
71
+ ## 10. Limitation of liability
72
+
73
+ To the maximum extent permitted by applicable law, Licensor is not liable for indirect, incidental, special, consequential, exemplary, or punitive damages, or for loss of data, profit, revenue, business, goodwill, or opportunity arising from the software. Licensor’s aggregate liability relating to a paid license will not exceed the amount you paid for that license during the twelve months before the event giving rise to the claim. Nothing here excludes liability that cannot legally be excluded or limited.
74
+
75
+ ## 11. Consumer rights and controlling terms
76
+
77
+ Mandatory consumer rights remain unaffected. If checkout terms or a written order conflict with this license, the more specific checkout terms or order control only for that transaction. If any provision is unenforceable, the remaining provisions continue to apply.
78
+
79
+ ## 12. Contact
80
+
81
+ Use the contact or support channel identified on the product page or transaction receipt for license questions.
82
+ License-File: LICENSE.md
83
+ Keywords: bm25,debugging,embeddings,evaluation,rag,retrieval
84
+ Classifier: Development Status :: 3 - Alpha
85
+ Classifier: Environment :: Console
86
+ Classifier: Intended Audience :: Developers
87
+ Classifier: License :: Other/Proprietary License
88
+ Classifier: Operating System :: OS Independent
89
+ Classifier: Programming Language :: Python :: 3.12
90
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
91
+ Requires-Python: <3.13,>=3.12
92
+ Requires-Dist: bm25s<0.3,>=0.2.14
93
+ Requires-Dist: jinja2<4,>=3.1
94
+ Requires-Dist: numpy<3,>=2
95
+ Requires-Dist: pydantic-settings<3,>=2.10
96
+ Requires-Dist: pydantic<3,>=2.11
97
+ Requires-Dist: rich<15,>=14
98
+ Requires-Dist: typer<1,>=0.16
99
+ Provides-Extra: semantic
100
+ Requires-Dist: fastembed<1,>=0.7; extra == 'semantic'
101
+ Description-Content-Type: text/markdown
102
+
103
+ # RAGvidence
104
+
105
+ **Evidence-first RAG diagnostics, built to run locally.**
106
+
107
+ RAGvidence is a local, CLI-first toolkit for testing retrieval changes with reviewed evidence instead of judging generated answers by intuition.
108
+
109
+ It helps you answer questions such as:
110
+
111
+ - Did the retriever return **all** evidence required for the answer?
112
+ - Did a new chunk representation fix failures or introduce regressions?
113
+ - Is the first relevant result early enough at the real context depth `k`?
114
+ - Does lexical, semantic, or an existing application retriever work better on your data?
115
+
116
+ No account, telemetry, hosted database, or vendor-provided API key is required. BM25 works completely offline. Semantic retrieval can run locally or through a provider chosen by the user.
117
+
118
+ > RAGvidence evaluates retrieval evidence. It does not prove that a generated answer is correct, safe, or legally compliant.
119
+
120
+ ## Install
121
+
122
+ ### Customer release archive
123
+
124
+ Extract the release, open a terminal in its directory, and run:
125
+
126
+ ```bash
127
+ uv sync --frozen
128
+ uv run ragvi health
129
+ uv run ragvi tutorial
130
+ ```
131
+
132
+ During source-based use, prefix commands with `uv run`. After installing the CLI as a tool, use `ragvi` directly.
133
+
134
+ ### Install a built wheel
135
+
136
+ ```bash
137
+ uv tool install ./ragvidence-0.1.0-py3-none-any.whl
138
+ ragvi health
139
+ ragvi tutorial
140
+ ```
141
+
142
+ Python 3.12 is pinned in `.python-version`; `uv` can install the required Python runtime automatically.
143
+
144
+ ## First report
145
+
146
+ The tutorial writes a complete comparison to:
147
+
148
+ ```text
149
+ workspaces/first-run/reports/tutorial.html
150
+ ```
151
+
152
+ Open it in any browser. It contains the baseline, intervention, changed questions, diagnosis, ranked evidence, limitations, and reproducibility metadata.
153
+
154
+ Then follow [Getting started](docs/getting-started.md) to understand the experiment rather than only running it.
155
+
156
+ ## Supported workflows
157
+
158
+ | Goal | Start here |
159
+ | --- | --- |
160
+ | Learn the workflow on bundled data | `ragvi tutorial` |
161
+ | Evaluate your own labeled dataset | [Bring your own data](docs/bring-your-own-data.md) |
162
+ | Compare BM25 and semantic retrieval | [User guide](docs/user-guide.md#semantic-retrieval) |
163
+ | Test an existing RAG retriever | [User guide](docs/user-guide.md#import-an-existing-retriever) |
164
+ | Generate optional answers for inspection | [User guide](docs/user-guide.md#optional-answer-inspection) |
165
+ | Understand each metric | [Interpreting results](docs/interpreting-results.md) |
166
+ | Recover a broken workspace | [Troubleshooting](docs/troubleshooting.md) |
167
+ | Review data and provider boundaries | [Privacy and providers](docs/privacy-and-providers.md) |
168
+
169
+ The complete CLI reference is in [Command reference](docs/command-reference.md).
170
+
171
+ ## What is included
172
+
173
+ - Controlled body versus heading-context experiments
174
+ - BM25 lexical retrieval
175
+ - Optional local semantic retrieval with FastEmbed
176
+ - Ollama and OpenAI-compatible embedding adapters
177
+ - Import of ranked results from another RAG stack
178
+ - Evidence sufficiency, coverage, first relevant rank, reciprocal rank, and precision@k
179
+ - Explicit improvement and regression classification
180
+ - Evidence-backed diagnostic suggestions
181
+ - JSON, Markdown, and self-contained HTML reports
182
+ - Optional answer inspection using a user-selected model
183
+ - Recoverable local workspaces and deterministic installation verification
184
+
185
+ ## Free and Plus
186
+
187
+ The current release contains the complete Free feature set. Plus is a separate paid entitlement for advanced features released in Plus builds; features are never advertised as available until they ship. See [Plans and editions](docs/plans.md) and [LICENSE.md](LICENSE.md).
188
+
189
+ The Free edition does not require activation. Plus activation will be issued after verified checkout and must never require placing payment credentials inside this CLI.
190
+
191
+ ## Privacy and trust boundaries
192
+
193
+ - Workspaces, caches, `.env`, and generated reports are ignored by Git.
194
+ - BM25 performs no network requests.
195
+ - FastEmbed downloads a model, then performs inference locally.
196
+ - Remote providers receive source text and questions; the CLI displays a disclosure first.
197
+ - API keys are loaded from `.env` and are not included in run manifests or reports.
198
+ - HTML reports escape dataset content, contain no scripts, and use no remote assets.
199
+ - Reports include short source previews and may still contain confidential information. Review before sharing.
200
+
201
+ ## Development and release verification
202
+
203
+ ```bash
204
+ uv sync --frozen --dev
205
+ uv run ruff check src tests
206
+ uv run pytest
207
+ uv run ragvi verify
208
+ uv build
209
+ ```
210
+
211
+ See [Security](SECURITY.md), [Support](SUPPORT.md), [Changelog](CHANGELOG.md), [Third-party notices](THIRD_PARTY_NOTICES.md), and the [resolved dependency inventory](THIRD_PARTY_LICENSES.md).
212
+
213
+ Copyright © 2026 Medo. Distributed under the proprietary Free or Plus license in [LICENSE.md](LICENSE.md).
@@ -0,0 +1,111 @@
1
+ # RAGvidence
2
+
3
+ **Evidence-first RAG diagnostics, built to run locally.**
4
+
5
+ RAGvidence is a local, CLI-first toolkit for testing retrieval changes with reviewed evidence instead of judging generated answers by intuition.
6
+
7
+ It helps you answer questions such as:
8
+
9
+ - Did the retriever return **all** evidence required for the answer?
10
+ - Did a new chunk representation fix failures or introduce regressions?
11
+ - Is the first relevant result early enough at the real context depth `k`?
12
+ - Does lexical, semantic, or an existing application retriever work better on your data?
13
+
14
+ No account, telemetry, hosted database, or vendor-provided API key is required. BM25 works completely offline. Semantic retrieval can run locally or through a provider chosen by the user.
15
+
16
+ > RAGvidence evaluates retrieval evidence. It does not prove that a generated answer is correct, safe, or legally compliant.
17
+
18
+ ## Install
19
+
20
+ ### Customer release archive
21
+
22
+ Extract the release, open a terminal in its directory, and run:
23
+
24
+ ```bash
25
+ uv sync --frozen
26
+ uv run ragvi health
27
+ uv run ragvi tutorial
28
+ ```
29
+
30
+ During source-based use, prefix commands with `uv run`. After installing the CLI as a tool, use `ragvi` directly.
31
+
32
+ ### Install a built wheel
33
+
34
+ ```bash
35
+ uv tool install ./ragvidence-0.1.0-py3-none-any.whl
36
+ ragvi health
37
+ ragvi tutorial
38
+ ```
39
+
40
+ Python 3.12 is pinned in `.python-version`; `uv` can install the required Python runtime automatically.
41
+
42
+ ## First report
43
+
44
+ The tutorial writes a complete comparison to:
45
+
46
+ ```text
47
+ workspaces/first-run/reports/tutorial.html
48
+ ```
49
+
50
+ Open it in any browser. It contains the baseline, intervention, changed questions, diagnosis, ranked evidence, limitations, and reproducibility metadata.
51
+
52
+ Then follow [Getting started](docs/getting-started.md) to understand the experiment rather than only running it.
53
+
54
+ ## Supported workflows
55
+
56
+ | Goal | Start here |
57
+ | --- | --- |
58
+ | Learn the workflow on bundled data | `ragvi tutorial` |
59
+ | Evaluate your own labeled dataset | [Bring your own data](docs/bring-your-own-data.md) |
60
+ | Compare BM25 and semantic retrieval | [User guide](docs/user-guide.md#semantic-retrieval) |
61
+ | Test an existing RAG retriever | [User guide](docs/user-guide.md#import-an-existing-retriever) |
62
+ | Generate optional answers for inspection | [User guide](docs/user-guide.md#optional-answer-inspection) |
63
+ | Understand each metric | [Interpreting results](docs/interpreting-results.md) |
64
+ | Recover a broken workspace | [Troubleshooting](docs/troubleshooting.md) |
65
+ | Review data and provider boundaries | [Privacy and providers](docs/privacy-and-providers.md) |
66
+
67
+ The complete CLI reference is in [Command reference](docs/command-reference.md).
68
+
69
+ ## What is included
70
+
71
+ - Controlled body versus heading-context experiments
72
+ - BM25 lexical retrieval
73
+ - Optional local semantic retrieval with FastEmbed
74
+ - Ollama and OpenAI-compatible embedding adapters
75
+ - Import of ranked results from another RAG stack
76
+ - Evidence sufficiency, coverage, first relevant rank, reciprocal rank, and precision@k
77
+ - Explicit improvement and regression classification
78
+ - Evidence-backed diagnostic suggestions
79
+ - JSON, Markdown, and self-contained HTML reports
80
+ - Optional answer inspection using a user-selected model
81
+ - Recoverable local workspaces and deterministic installation verification
82
+
83
+ ## Free and Plus
84
+
85
+ The current release contains the complete Free feature set. Plus is a separate paid entitlement for advanced features released in Plus builds; features are never advertised as available until they ship. See [Plans and editions](docs/plans.md) and [LICENSE.md](LICENSE.md).
86
+
87
+ The Free edition does not require activation. Plus activation will be issued after verified checkout and must never require placing payment credentials inside this CLI.
88
+
89
+ ## Privacy and trust boundaries
90
+
91
+ - Workspaces, caches, `.env`, and generated reports are ignored by Git.
92
+ - BM25 performs no network requests.
93
+ - FastEmbed downloads a model, then performs inference locally.
94
+ - Remote providers receive source text and questions; the CLI displays a disclosure first.
95
+ - API keys are loaded from `.env` and are not included in run manifests or reports.
96
+ - HTML reports escape dataset content, contain no scripts, and use no remote assets.
97
+ - Reports include short source previews and may still contain confidential information. Review before sharing.
98
+
99
+ ## Development and release verification
100
+
101
+ ```bash
102
+ uv sync --frozen --dev
103
+ uv run ruff check src tests
104
+ uv run pytest
105
+ uv run ragvi verify
106
+ uv build
107
+ ```
108
+
109
+ See [Security](SECURITY.md), [Support](SUPPORT.md), [Changelog](CHANGELOG.md), [Third-party notices](THIRD_PARTY_NOTICES.md), and the [resolved dependency inventory](THIRD_PARTY_LICENSES.md).
110
+
111
+ Copyright © 2026 Medo. Distributed under the proprietary Free or Plus license in [LICENSE.md](LICENSE.md).
@@ -0,0 +1,31 @@
1
+ # Security policy
2
+
3
+ ## Supported versions
4
+
5
+ Security fixes are provided for the latest released Free build and actively supported Plus builds. Upgrade before reporting behavior from an older release.
6
+
7
+ ## Report a vulnerability privately
8
+
9
+ Do not publish exploitable details in a public issue or discussion. Contact the seller through the support channel shown on the product page or purchase receipt and include:
10
+
11
+ - affected version and operating system;
12
+ - minimal reproduction steps;
13
+ - expected and observed behavior;
14
+ - potential impact;
15
+ - whether real credentials or private data were exposed.
16
+
17
+ Do not include live API keys, customer documents, or unrelated personal information. Use synthetic data whenever possible.
18
+
19
+ ## Security boundaries
20
+
21
+ - BM25 is offline and does not transmit data.
22
+ - FastEmbed downloads model files and then runs locally.
23
+ - Ollama and OpenAI-compatible endpoints are selected by the user.
24
+ - Remote providers receive the text required for the requested operation.
25
+ - `.env` values are not written to manifests or reports.
26
+ - Exported reports may contain source identifiers and previews and must be reviewed before sharing.
27
+ - The CLI is not a sandbox. Only run trusted releases and trusted custom templates.
28
+
29
+ ## Release integrity
30
+
31
+ Official releases should publish SHA-256 checksums. Verify the downloaded artifact before installation and never install a package from an unofficial mirror or unexpected URL.
@@ -0,0 +1,24 @@
1
+ # Support
2
+
3
+ ## Before requesting help
4
+
5
+ ```bash
6
+ ragvi --version
7
+ ragvi health
8
+ ragvi verify
9
+ ```
10
+
11
+ Then check [Getting started](docs/getting-started.md) and [Troubleshooting](docs/troubleshooting.md).
12
+
13
+ ## What to include
14
+
15
+ - product version;
16
+ - operating system and Python version;
17
+ - installation method;
18
+ - exact command;
19
+ - complete error text with credentials and private document content removed;
20
+ - whether the bundled tutorial succeeds.
21
+
22
+ Use the support channel listed on the product page or purchase receipt. Support scope and response targets, if any, are determined by the plan and checkout terms.
23
+
24
+ Support does not include reviewing confidential corpora, designing a production architecture, operating third-party providers, or guaranteeing model/retrieval quality unless separately agreed in writing.
@@ -0,0 +1,51 @@
1
+ # Resolved dependency license inventory
2
+
3
+ Generated from the locked Free and semantic runtime environment for version 0.1.0. Package metadata is informational; upstream license files remain authoritative.
4
+
5
+ | Name | Version | License | URL |
6
+ |--------------------|-----------|----------------------------------------------------|----------------------------------------------------------------------|
7
+ | Jinja2 | 3.1.6 | BSD License | https://github.com/pallets/jinja/ |
8
+ | MarkupSafe | 3.0.3 | BSD-3-Clause | https://github.com/pallets/markupsafe/ |
9
+ | PyYAML | 6.0.3 | MIT License | https://pyyaml.org/ |
10
+ | Pygments | 2.21.0 | BSD-2-Clause | https://pygments.org |
11
+ | annotated-doc | 0.0.5 | MIT | https://github.com/fastapi/annotated-doc |
12
+ | annotated-types | 0.8.0 | MIT | https://github.com/annotated-types/annotated-types |
13
+ | anyio | 4.15.1 | MIT | https://anyio.readthedocs.io/en/stable/versionhistory.html |
14
+ | bm25s | 0.2.14 | MIT License | https://github.com/xhluca/bm25s |
15
+ | certifi | 2026.7.22 | Mozilla Public License 2.0 (MPL 2.0) | https://github.com/certifi/python-certifi |
16
+ | charset-normalizer | 3.5.1 | MIT | https://github.com/jawah/charset_normalizer/blob/master/CHANGELOG.md |
17
+ | click | 8.5.0 | BSD-3-Clause | https://github.com/pallets/click/ |
18
+ | fastembed | 0.8.1 | Apache Software License | https://github.com/qdrant/fastembed |
19
+ | filelock | 4.0.4 | MIT | https://github.com/tox-dev/py-filelock |
20
+ | flatbuffers | 25.12.19 | Apache Software License | https://google.github.io/flatbuffers/ |
21
+ | fsspec | 2026.9.0 | BSD-3-Clause | https://github.com/fsspec/filesystem_spec |
22
+ | h11 | 0.16.0 | MIT License | https://github.com/python-hyper/h11 |
23
+ | hf-xet | 1.6.0 | Apache-2.0 | https://github.com/huggingface/xet-core |
24
+ | httpcore | 1.0.9 | BSD-3-Clause | https://www.encode.io/httpcore/ |
25
+ | httpx | 0.28.1 | BSD License | https://github.com/encode/httpx |
26
+ | huggingface_hub | 1.33.0 | Apache Software License | https://github.com/huggingface/huggingface_hub |
27
+ | idna | 3.20 | BSD-3-Clause | https://github.com/kjd/idna |
28
+ | loguru | 0.7.3 | MIT License | https://github.com/Delgan/loguru |
29
+ | markdown-it-py | 4.2.0 | MIT License | https://github.com/executablebooks/markdown-it-py |
30
+ | mdurl | 0.1.2 | MIT License | https://github.com/executablebooks/mdurl |
31
+ | mmh3 | 5.3.0 | MIT License | https://pypi.org/project/mmh3/ |
32
+ | numpy | 2.5.3 | BSD-3-Clause AND 0BSD AND MIT AND Zlib AND CC0-1.0 | https://numpy.org |
33
+ | onnxruntime | 1.30.0 | MIT License | https://onnxruntime.ai |
34
+ | packaging | 26.3 | Apache-2.0 OR BSD-2-Clause | https://github.com/pypa/packaging |
35
+ | pillow | 12.3.0 | MIT-CMU | https://python-pillow.github.io |
36
+ | protobuf | 7.36.2 | 3-Clause BSD License | https://developers.google.com/protocol-buffers/ |
37
+ | py_rust_stemmers | 0.1.8 | UNKNOWN | UNKNOWN |
38
+ | pydantic | 2.13.5 | MIT | https://github.com/pydantic/pydantic |
39
+ | pydantic-settings | 2.15.0 | MIT | https://github.com/pydantic/pydantic-settings |
40
+ | pydantic_core | 2.46.5 | MIT | https://github.com/pydantic |
41
+ | python-dotenv | 1.2.3 | BSD-3-Clause | https://github.com/theskumar/python-dotenv |
42
+ | requests | 2.34.2 | Apache Software License | https://github.com/psf/requests |
43
+ | rich | 14.3.4 | MIT License | https://github.com/Textualize/rich |
44
+ | scipy | 1.18.1 | BSD License | https://scipy.org/ |
45
+ | shellingham | 1.5.4 | ISC License (ISCL) | https://github.com/sarugaku/shellingham |
46
+ | tokenizers | 0.23.2 | Apache Software License | https://github.com/huggingface/tokenizers |
47
+ | tqdm | 4.70.1 | MPL-2.0 AND MIT | https://tqdm.github.io |
48
+ | typer | 0.27.2 | MIT | https://github.com/fastapi/typer |
49
+ | typing-inspection | 0.4.4 | MIT | https://github.com/pydantic/typing-inspection |
50
+ | typing_extensions | 4.16.0 | PSF-2.0 | https://github.com/python/typing_extensions |
51
+ | urllib3 | 2.8.0 | MIT | https://github.com/urllib3/urllib3/blob/main/CHANGES.rst |
@@ -0,0 +1,19 @@
1
+ # Third-party notices
2
+
3
+ This inventory covers direct runtime dependencies selected for RAGvidence v0.1.0. Exact resolved versions are recorded in `uv.lock`. Each dependency remains governed by its own license; consult the installed distribution and upstream project for the complete text.
4
+
5
+ | Component | Role | License identified by upstream/package metadata |
6
+ | --- | --- | --- |
7
+ | BM25S | Local lexical retrieval | MIT |
8
+ | Jinja2 | Escaped HTML templating | BSD-3-Clause |
9
+ | NumPy | Vector operations | BSD-3-Clause and bundled component licenses |
10
+ | Pydantic | Data validation | MIT |
11
+ | Pydantic Settings | Reserved configuration boundary | MIT |
12
+ | Rich | Terminal presentation | MIT |
13
+ | Typer | CLI framework | MIT |
14
+ | FastEmbed (optional) | Local ONNX embedding runtime | Apache-2.0 |
15
+ | BAAI/bge-small-en-v1.5 (downloaded optionally) | Default English embedding model | MIT |
16
+
17
+ FastEmbed installs transitive runtime dependencies whose versions and licenses may change when `uv.lock` is updated. The resolved inventory for this release is recorded in `THIRD_PARTY_LICENSES.md`. The `py-rust-stemmers` 0.1.8 package metadata does not declare a machine-readable license; its upstream repository license file must be checked manually before each commercial release.
18
+
19
+ The default model is downloaded separately on first semantic use and is not redistributed inside the core archive.
@@ -0,0 +1,46 @@
1
+ # Bring your own data
2
+
3
+ Start with a small reviewed slice—roughly 10–50 source units and 10–30 questions. The value comes from trustworthy labels, not corpus size.
4
+
5
+ ## `sources.jsonl`
6
+
7
+ One JSON object per retrievable unit:
8
+
9
+ ```json
10
+ {"source_unit_id":"S-17","document_id":"manual-a","heading_path":["Device A","Reset"],"body":"Hold the button for eight seconds.","evidence_ids":["E-RESET-17"],"metadata":{"language":"en"}}
11
+ ```
12
+
13
+ - `source_unit_id`: stable unique identifier for the retrievable unit.
14
+ - `document_id`: stable source-document identifier.
15
+ - `heading_path`: authentic source headings; never generated gold-answer text.
16
+ - `body`: original text used in the body-only baseline.
17
+ - `evidence_ids`: reviewed facts/passages represented by this unit.
18
+ - `metadata`: optional information; not searched in v1.
19
+
20
+ ## `questions.jsonl`
21
+
22
+ ```json
23
+ {"question_id":"Q-001","question":"How long is the reset?","split":"dev","answerable":true,"required_evidence_ids":["E-RESET-17"],"reference_answer":"Eight seconds.","tags":["procedure"]}
24
+ ```
25
+
26
+ - Answerable questions require at least one evidence ID.
27
+ - Unanswerable questions must use an empty evidence-ID list.
28
+ - Use `dev` for diagnosis and intervention design.
29
+ - Keep `heldout` questions untouched until the intervention is frozen.
30
+ - A multi-evidence answer lists every required evidence ID.
31
+
32
+ ## Avoid label leakage
33
+
34
+ Do not place reference answers, required-evidence IDs, grader notes or invented contextual hints into searchable text. Heading context must come from the source itself. If the intervention changes source boundaries, labels or `k`, it is no longer the controlled comparison used by the guided exercise.
35
+
36
+ ## Validation
37
+
38
+ ```bash
39
+ uv run ragvi validate workspaces/my-experiment
40
+ ```
41
+
42
+ Validation rejects malformed JSON, unknown fields, duplicate IDs, answerability contradictions and required evidence that does not exist in the sources.
43
+
44
+ ## Choosing `k`
45
+
46
+ Use the depth your application would actually pass downstream. A large `k` can hide retrieval failures while increasing noise and context cost. Keep `k` frozen across a baseline/intervention comparison.