bioai-evidence-validator 0.4.1__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/.github/workflows/ci.yml +31 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/CHANGELOG.md +12 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/CITATION.cff +1 -1
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/PKG-INFO +66 -8
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/README.md +65 -7
- bioai_evidence_validator-0.5.0/action.yml +57 -0
- bioai_evidence_validator-0.5.0/docs/DRAFTS.md +99 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/docs/ENGINEERING.md +24 -10
- bioai_evidence_validator-0.5.0/examples/drafts/llm_claim.yaml +28 -0
- bioai_evidence_validator-0.5.0/examples/drafts/reviewed_claim.yaml +35 -0
- bioai_evidence_validator-0.5.0/examples/drafts/synthetic_paper.txt +2 -0
- bioai_evidence_validator-0.5.0/examples/quickstart.ipynb +255 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/vbo_canine/results/summary.json +1 -1
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/pyproject.toml +1 -1
- bioai_evidence_validator-0.5.0/src/bioevidence_validator/__init__.py +8 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/src/bioevidence_validator/cli.py +22 -0
- bioai_evidence_validator-0.5.0/src/bioevidence_validator/draft.py +250 -0
- bioai_evidence_validator-0.5.0/tests/test_draft.py +174 -0
- bioai_evidence_validator-0.5.0/tests/test_github_action.py +43 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/tools/check_distribution.py +9 -2
- bioai_evidence_validator-0.5.0/tools/github_action.py +91 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/uv.lock +1 -1
- bioai_evidence_validator-0.4.1/src/bioevidence_validator/__init__.py +0 -3
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/.gitattributes +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/.github/workflows/release.yml +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/.gitignore +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/LICENSE +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/docs/ADR-001-canine-breed-first.md +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/docs/ADR-002-domain-neutral-main.md +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/docs/CASE_STUDY.md +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/docs/GOLD_STANDARD.md +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/docs/MIGRATION-0.4.md +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/docs/PROFILES.md +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/docs/assets/vbo_canine_benchmark.svg +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/evaluation/gold_standard/README.md +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/evaluation/gold_standard/adjudications.template.csv +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/evaluation/gold_standard/annotations.template.csv +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/evaluation/gold_standard/manifest.template.json +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/custom_profile/assay.yaml +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/custom_profile/assay_record.json +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/dataset_label/curated_sample_label.json +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/dataset_label/missing_sample_link.json +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/general/curated_assertion.json +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/literature_claim/curated_association.json +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/literature_claim/llm_only.json +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/vbo_canine/README.md +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/vbo_canine/pipeline.py +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/vbo_canine/prepare_source.py +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/vbo_canine/profile.yaml +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/vbo_canine/reference_cases.json +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/vbo_canine/results/decisions.jsonl +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/vbo_canine/results/review_queue.csv +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/vbo_canine/results/summary.md +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/vbo_canine/run.py +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/vbo_canine/sources/README.md +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/vbo_canine/sources/manifest.json +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/examples/vbo_canine/sources/vbo-dogs.json +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/src/bioevidence_validator/config.py +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/src/bioevidence_validator/engine.py +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/src/bioevidence_validator/profiles/dataset-label.yaml +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/src/bioevidence_validator/profiles/general.yaml +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/src/bioevidence_validator/profiles/literature-claim.yaml +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/src/bioevidence_validator/schema/bioevidence_core.yaml +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/tests/test_cli.py +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/tests/test_engine.py +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/tests/test_evidence_quality.py +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/tests/test_fail_closed.py +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/tests/test_profiles.py +0 -0
- {bioai_evidence_validator-0.4.1 → bioai_evidence_validator-0.5.0}/tests/test_vbo_case.py +0 -0
|
@@ -36,3 +36,34 @@ jobs:
|
|
|
36
36
|
- name: Verify installed wheel and CLI outside editable source
|
|
37
37
|
shell: bash
|
|
38
38
|
run: uv run --isolated --no-project --python ${{ matrix.python }} --with ./dist/*.whl python tools/check_distribution.py
|
|
39
|
+
|
|
40
|
+
action:
|
|
41
|
+
timeout-minutes: 10
|
|
42
|
+
strategy:
|
|
43
|
+
matrix:
|
|
44
|
+
os: [ubuntu-latest, windows-latest]
|
|
45
|
+
runs-on: ${{ matrix.os }}
|
|
46
|
+
steps:
|
|
47
|
+
- uses: actions/checkout@v4
|
|
48
|
+
- name: Admitted and review-required drafts pass with fail-on rejected
|
|
49
|
+
id: drafts
|
|
50
|
+
uses: ./
|
|
51
|
+
with:
|
|
52
|
+
files: examples/drafts/*.yaml
|
|
53
|
+
profile: literature-claim
|
|
54
|
+
format: draft
|
|
55
|
+
fail-on: rejected
|
|
56
|
+
- name: A rejected record fails the action
|
|
57
|
+
id: rejected
|
|
58
|
+
continue-on-error: true
|
|
59
|
+
uses: ./
|
|
60
|
+
with:
|
|
61
|
+
files: examples/dataset_label/*.json
|
|
62
|
+
profile: dataset-label
|
|
63
|
+
- name: Check action outcomes and outputs
|
|
64
|
+
shell: bash
|
|
65
|
+
run: |
|
|
66
|
+
test "${{ steps.drafts.outputs.admitted }}" = 1
|
|
67
|
+
test "${{ steps.drafts.outputs.review_required }}" = 1
|
|
68
|
+
test "${{ steps.rejected.outcome }}" = failure
|
|
69
|
+
test "${{ steps.rejected.outputs.rejected }}" = 1
|
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.5.0 — Drafts, LLM draft schema and GitHub Action
|
|
4
|
+
|
|
5
|
+
- Add compact YAML/JSON drafts: `bioevidence build` and `build_record()` derive identifiers,
|
|
6
|
+
group evidence lines and hash named local files, without supplying scope, method, times
|
|
7
|
+
or review decisions. Strict parsing rejects unknown, missing and out-of-choice fields.
|
|
8
|
+
- Add `bioevidence draft-schema` and `draft_json_schema()`: a per-profile JSON Schema for
|
|
9
|
+
drafts, e.g. for LLM structured output.
|
|
10
|
+
- Add a composite GitHub Action that validates records or drafts in pull requests, with a
|
|
11
|
+
job summary, file annotations and count outputs.
|
|
12
|
+
- Add a Colab quickstart notebook and draft examples.
|
|
13
|
+
- Export `build_record`, `load_draft`, `draft_json_schema` and `validate_record` from the package root.
|
|
14
|
+
|
|
3
15
|
## 0.4.1 — Required-evidence quality and real-source evaluation
|
|
4
16
|
|
|
5
17
|
- Apply extraction-method quality gates to each required evidence type independently.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: bioai-evidence-validator
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Standards-aligned evidence policy validation for AI-assisted biological curation
|
|
5
5
|
Project-URL: Homepage, https://github.com/NingyuSUN/bioai-evidence-validator
|
|
6
6
|
Project-URL: Documentation, https://github.com/NingyuSUN/bioai-evidence-validator/tree/main/docs
|
|
@@ -34,6 +34,7 @@ Description-Content-Type: text/markdown
|
|
|
34
34
|
[](https://pypi.org/project/bioai-evidence-validator/)
|
|
35
35
|
[](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/pyproject.toml)
|
|
36
36
|
[](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/LICENSE)
|
|
37
|
+
[](https://colab.research.google.com/github/NingyuSUN/bioai-evidence-validator/blob/main/examples/quickstart.ipynb)
|
|
37
38
|
|
|
38
39
|
**Stop AI-extracted biological claims from entering your knowledge base or
|
|
39
40
|
training set before their evidence is good enough for that use.**
|
|
@@ -49,6 +50,9 @@ each requested use.
|
|
|
49
50
|
pip install bioai-evidence-validator
|
|
50
51
|
```
|
|
51
52
|
|
|
53
|
+
Or try it in the browser, nothing to install:
|
|
54
|
+
[quickstart notebook on Colab](https://colab.research.google.com/github/NingyuSUN/bioai-evidence-validator/blob/main/examples/quickstart.ipynb).
|
|
55
|
+
|
|
52
56
|
## 30-second example
|
|
53
57
|
|
|
54
58
|
The two records below are identical except for one field: how the supporting
|
|
@@ -105,13 +109,45 @@ injected faults; the full validator admitted **0/160**.
|
|
|
105
109
|
|
|
106
110
|
## Use it
|
|
107
111
|
|
|
112
|
+
### Write a draft, not a full record
|
|
113
|
+
|
|
114
|
+
A full record spells out identifiers, evidence lines and hashes. A draft states each fact
|
|
115
|
+
once; `bioevidence build` derives the rest and hashes local source files:
|
|
116
|
+
|
|
117
|
+
```yaml
|
|
118
|
+
profile: literature-claim
|
|
119
|
+
uses: [research_summary]
|
|
120
|
+
statement:
|
|
121
|
+
subject: {id: "SYN:GENE_A", label: Synthetic gene A, type: gene}
|
|
122
|
+
predicate: associated_with
|
|
123
|
+
object: {id: "SYN:PHENOTYPE_A", label: Synthetic phenotype A, type: phenotype}
|
|
124
|
+
scope: ["taxon:synthetic"]
|
|
125
|
+
sources:
|
|
126
|
+
- {id: paper, title: Synthetic paper, type: publication, version: v1,
|
|
127
|
+
retrieved_at: "2026-09-21T00:00:00Z", file: synthetic_paper.txt}
|
|
128
|
+
evidence:
|
|
129
|
+
- {source: paper, locator: Table 2, type: publication_result,
|
|
130
|
+
method: llm_extraction, scope: ["taxon:synthetic"]}
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
bioevidence build examples/drafts/llm_claim.yaml --output record.json
|
|
135
|
+
bioevidence validate record.json --profile literature-claim # exit 2: review required
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Building never fills in scope, extraction method, retrieval time or review decisions for
|
|
139
|
+
you. For LLM pipelines, `bioevidence draft-schema --profile literature-claim` prints a JSON
|
|
140
|
+
Schema for structured output. See the [draft format](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/docs/DRAFTS.md).
|
|
141
|
+
|
|
108
142
|
### Command line
|
|
109
143
|
|
|
110
144
|
```bash
|
|
111
145
|
bioevidence profiles # list built-in profiles and their use contracts
|
|
146
|
+
bioevidence build draft.yaml --output record.json # expand a compact draft
|
|
112
147
|
bioevidence validate record.json --profile literature-claim
|
|
113
148
|
bioevidence validate record.json --profile my_profile.yaml --output report.json
|
|
114
|
-
bioevidence
|
|
149
|
+
bioevidence draft-schema --profile literature-claim # JSON Schema for drafts (e.g. LLM output)
|
|
150
|
+
bioevidence generate-schema --output record.schema.json # JSON Schema for full records
|
|
115
151
|
```
|
|
116
152
|
|
|
117
153
|
Exit codes: **0** admitted, **1** rejected, **2** review required, **3** input or configuration error.
|
|
@@ -119,12 +155,9 @@ Exit codes: **0** admitted, **1** rejected, **2** review required, **3** input o
|
|
|
119
155
|
### Python
|
|
120
156
|
|
|
121
157
|
```python
|
|
122
|
-
import
|
|
123
|
-
from pathlib import Path
|
|
124
|
-
|
|
125
|
-
from bioevidence_validator.engine import validate_record
|
|
158
|
+
from bioevidence_validator import build_record, load_draft, validate_record
|
|
126
159
|
|
|
127
|
-
record =
|
|
160
|
+
record = build_record(load_draft("draft.yaml"), base_dir=".") # or load a full record JSON
|
|
128
161
|
report = validate_record(record, profile="literature-claim")
|
|
129
162
|
|
|
130
163
|
for decision in report["use_decisions"]:
|
|
@@ -133,6 +166,30 @@ for decision in report["use_decisions"]:
|
|
|
133
166
|
|
|
134
167
|
`profile` accepts a built-in name or a path to your own YAML profile.
|
|
135
168
|
|
|
169
|
+
### Check records in CI
|
|
170
|
+
|
|
171
|
+
Validate every record or draft in a pull request, with a summary table and inline annotations:
|
|
172
|
+
|
|
173
|
+
```yaml
|
|
174
|
+
# .github/workflows/evidence.yml
|
|
175
|
+
on: pull_request
|
|
176
|
+
jobs:
|
|
177
|
+
evidence:
|
|
178
|
+
runs-on: ubuntu-latest
|
|
179
|
+
steps:
|
|
180
|
+
- uses: actions/checkout@v4
|
|
181
|
+
- uses: NingyuSUN/bioai-evidence-validator@v0.5.0
|
|
182
|
+
with:
|
|
183
|
+
files: records/**/*.yaml # whitespace-separated globs
|
|
184
|
+
format: draft # or: record (default)
|
|
185
|
+
profile: literature-claim # built-in name or path to your profile YAML
|
|
186
|
+
fail-on: review # or: rejected
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
With `fail-on: review` (default) the job fails unless every file is admitted; with
|
|
190
|
+
`fail-on: rejected` it fails only on rejected files or files that cannot be read. The
|
|
191
|
+
action's outputs `admitted`, `review_required`, `rejected` and `error` hold the counts.
|
|
192
|
+
|
|
136
193
|
## How it works
|
|
137
194
|
|
|
138
195
|
```mermaid
|
|
@@ -222,7 +279,7 @@ projection. External source truth and cohort independence require upstream verif
|
|
|
222
279
|
|
|
223
280
|
## Versions and branches
|
|
224
281
|
|
|
225
|
-
`main` is the domain-neutral framework (0.
|
|
282
|
+
`main` is the domain-neutral framework (0.5.0). The complete canine implementation
|
|
226
283
|
and SQLite adapter from 0.3 live on the
|
|
227
284
|
[`canine-breed` branch](https://github.com/NingyuSUN/bioai-evidence-validator/tree/canine-breed);
|
|
228
285
|
see the [0.4 migration guide](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/docs/MIGRATION-0.4.md) and
|
|
@@ -235,6 +292,7 @@ If you use this toolkit in research, please cite it using the metadata in
|
|
|
235
292
|
(GitHub's "Cite this repository" button generates APA and BibTeX).
|
|
236
293
|
|
|
237
294
|
[Create a profile](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/docs/PROFILES.md) ·
|
|
295
|
+
[Draft format](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/docs/DRAFTS.md) ·
|
|
238
296
|
[Engineering contract](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/docs/ENGINEERING.md) ·
|
|
239
297
|
[Design case study](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/docs/CASE_STUDY.md) ·
|
|
240
298
|
[Architecture decision](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/docs/ADR-002-domain-neutral-main.md) ·
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
[](https://pypi.org/project/bioai-evidence-validator/)
|
|
5
5
|
[](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/pyproject.toml)
|
|
6
6
|
[](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/LICENSE)
|
|
7
|
+
[](https://colab.research.google.com/github/NingyuSUN/bioai-evidence-validator/blob/main/examples/quickstart.ipynb)
|
|
7
8
|
|
|
8
9
|
**Stop AI-extracted biological claims from entering your knowledge base or
|
|
9
10
|
training set before their evidence is good enough for that use.**
|
|
@@ -19,6 +20,9 @@ each requested use.
|
|
|
19
20
|
pip install bioai-evidence-validator
|
|
20
21
|
```
|
|
21
22
|
|
|
23
|
+
Or try it in the browser, nothing to install:
|
|
24
|
+
[quickstart notebook on Colab](https://colab.research.google.com/github/NingyuSUN/bioai-evidence-validator/blob/main/examples/quickstart.ipynb).
|
|
25
|
+
|
|
22
26
|
## 30-second example
|
|
23
27
|
|
|
24
28
|
The two records below are identical except for one field: how the supporting
|
|
@@ -75,13 +79,45 @@ injected faults; the full validator admitted **0/160**.
|
|
|
75
79
|
|
|
76
80
|
## Use it
|
|
77
81
|
|
|
82
|
+
### Write a draft, not a full record
|
|
83
|
+
|
|
84
|
+
A full record spells out identifiers, evidence lines and hashes. A draft states each fact
|
|
85
|
+
once; `bioevidence build` derives the rest and hashes local source files:
|
|
86
|
+
|
|
87
|
+
```yaml
|
|
88
|
+
profile: literature-claim
|
|
89
|
+
uses: [research_summary]
|
|
90
|
+
statement:
|
|
91
|
+
subject: {id: "SYN:GENE_A", label: Synthetic gene A, type: gene}
|
|
92
|
+
predicate: associated_with
|
|
93
|
+
object: {id: "SYN:PHENOTYPE_A", label: Synthetic phenotype A, type: phenotype}
|
|
94
|
+
scope: ["taxon:synthetic"]
|
|
95
|
+
sources:
|
|
96
|
+
- {id: paper, title: Synthetic paper, type: publication, version: v1,
|
|
97
|
+
retrieved_at: "2026-09-21T00:00:00Z", file: synthetic_paper.txt}
|
|
98
|
+
evidence:
|
|
99
|
+
- {source: paper, locator: Table 2, type: publication_result,
|
|
100
|
+
method: llm_extraction, scope: ["taxon:synthetic"]}
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
bioevidence build examples/drafts/llm_claim.yaml --output record.json
|
|
105
|
+
bioevidence validate record.json --profile literature-claim # exit 2: review required
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Building never fills in scope, extraction method, retrieval time or review decisions for
|
|
109
|
+
you. For LLM pipelines, `bioevidence draft-schema --profile literature-claim` prints a JSON
|
|
110
|
+
Schema for structured output. See the [draft format](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/docs/DRAFTS.md).
|
|
111
|
+
|
|
78
112
|
### Command line
|
|
79
113
|
|
|
80
114
|
```bash
|
|
81
115
|
bioevidence profiles # list built-in profiles and their use contracts
|
|
116
|
+
bioevidence build draft.yaml --output record.json # expand a compact draft
|
|
82
117
|
bioevidence validate record.json --profile literature-claim
|
|
83
118
|
bioevidence validate record.json --profile my_profile.yaml --output report.json
|
|
84
|
-
bioevidence
|
|
119
|
+
bioevidence draft-schema --profile literature-claim # JSON Schema for drafts (e.g. LLM output)
|
|
120
|
+
bioevidence generate-schema --output record.schema.json # JSON Schema for full records
|
|
85
121
|
```
|
|
86
122
|
|
|
87
123
|
Exit codes: **0** admitted, **1** rejected, **2** review required, **3** input or configuration error.
|
|
@@ -89,12 +125,9 @@ Exit codes: **0** admitted, **1** rejected, **2** review required, **3** input o
|
|
|
89
125
|
### Python
|
|
90
126
|
|
|
91
127
|
```python
|
|
92
|
-
import
|
|
93
|
-
from pathlib import Path
|
|
94
|
-
|
|
95
|
-
from bioevidence_validator.engine import validate_record
|
|
128
|
+
from bioevidence_validator import build_record, load_draft, validate_record
|
|
96
129
|
|
|
97
|
-
record =
|
|
130
|
+
record = build_record(load_draft("draft.yaml"), base_dir=".") # or load a full record JSON
|
|
98
131
|
report = validate_record(record, profile="literature-claim")
|
|
99
132
|
|
|
100
133
|
for decision in report["use_decisions"]:
|
|
@@ -103,6 +136,30 @@ for decision in report["use_decisions"]:
|
|
|
103
136
|
|
|
104
137
|
`profile` accepts a built-in name or a path to your own YAML profile.
|
|
105
138
|
|
|
139
|
+
### Check records in CI
|
|
140
|
+
|
|
141
|
+
Validate every record or draft in a pull request, with a summary table and inline annotations:
|
|
142
|
+
|
|
143
|
+
```yaml
|
|
144
|
+
# .github/workflows/evidence.yml
|
|
145
|
+
on: pull_request
|
|
146
|
+
jobs:
|
|
147
|
+
evidence:
|
|
148
|
+
runs-on: ubuntu-latest
|
|
149
|
+
steps:
|
|
150
|
+
- uses: actions/checkout@v4
|
|
151
|
+
- uses: NingyuSUN/bioai-evidence-validator@v0.5.0
|
|
152
|
+
with:
|
|
153
|
+
files: records/**/*.yaml # whitespace-separated globs
|
|
154
|
+
format: draft # or: record (default)
|
|
155
|
+
profile: literature-claim # built-in name or path to your profile YAML
|
|
156
|
+
fail-on: review # or: rejected
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
With `fail-on: review` (default) the job fails unless every file is admitted; with
|
|
160
|
+
`fail-on: rejected` it fails only on rejected files or files that cannot be read. The
|
|
161
|
+
action's outputs `admitted`, `review_required`, `rejected` and `error` hold the counts.
|
|
162
|
+
|
|
106
163
|
## How it works
|
|
107
164
|
|
|
108
165
|
```mermaid
|
|
@@ -192,7 +249,7 @@ projection. External source truth and cohort independence require upstream verif
|
|
|
192
249
|
|
|
193
250
|
## Versions and branches
|
|
194
251
|
|
|
195
|
-
`main` is the domain-neutral framework (0.
|
|
252
|
+
`main` is the domain-neutral framework (0.5.0). The complete canine implementation
|
|
196
253
|
and SQLite adapter from 0.3 live on the
|
|
197
254
|
[`canine-breed` branch](https://github.com/NingyuSUN/bioai-evidence-validator/tree/canine-breed);
|
|
198
255
|
see the [0.4 migration guide](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/docs/MIGRATION-0.4.md) and
|
|
@@ -205,6 +262,7 @@ If you use this toolkit in research, please cite it using the metadata in
|
|
|
205
262
|
(GitHub's "Cite this repository" button generates APA and BibTeX).
|
|
206
263
|
|
|
207
264
|
[Create a profile](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/docs/PROFILES.md) ·
|
|
265
|
+
[Draft format](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/docs/DRAFTS.md) ·
|
|
208
266
|
[Engineering contract](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/docs/ENGINEERING.md) ·
|
|
209
267
|
[Design case study](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/docs/CASE_STUDY.md) ·
|
|
210
268
|
[Architecture decision](https://github.com/NingyuSUN/bioai-evidence-validator/blob/main/docs/ADR-002-domain-neutral-main.md) ·
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
name: BioAI Evidence Validator
|
|
2
|
+
description: Check that biological evidence records meet a profile's use contract before they are merged.
|
|
3
|
+
author: Ningyu Sun
|
|
4
|
+
branding:
|
|
5
|
+
icon: check-circle
|
|
6
|
+
color: green
|
|
7
|
+
inputs:
|
|
8
|
+
files:
|
|
9
|
+
description: Whitespace-separated glob patterns of record (JSON) or draft (YAML/JSON) files, e.g. "records/**/*.json".
|
|
10
|
+
required: true
|
|
11
|
+
profile:
|
|
12
|
+
description: Built-in profile name (general, literature-claim, dataset-label) or a path to a profile YAML.
|
|
13
|
+
default: general
|
|
14
|
+
format:
|
|
15
|
+
description: "record: full evidence records. draft: compact drafts, built into records before validation."
|
|
16
|
+
default: record
|
|
17
|
+
fail-on:
|
|
18
|
+
description: "review: fail unless every file is admitted. rejected: fail only on rejected files or errors."
|
|
19
|
+
default: review
|
|
20
|
+
python-version:
|
|
21
|
+
description: Python version used to run the validator.
|
|
22
|
+
default: '3.12'
|
|
23
|
+
outputs:
|
|
24
|
+
admitted:
|
|
25
|
+
description: Number of admitted files.
|
|
26
|
+
value: ${{ steps.validate.outputs.admitted }}
|
|
27
|
+
review_required:
|
|
28
|
+
description: Number of files that need human review.
|
|
29
|
+
value: ${{ steps.validate.outputs.review_required }}
|
|
30
|
+
rejected:
|
|
31
|
+
description: Number of rejected files.
|
|
32
|
+
value: ${{ steps.validate.outputs.rejected }}
|
|
33
|
+
error:
|
|
34
|
+
description: Number of files that could not be read or built.
|
|
35
|
+
value: ${{ steps.validate.outputs.error }}
|
|
36
|
+
runs:
|
|
37
|
+
using: composite
|
|
38
|
+
steps:
|
|
39
|
+
- uses: actions/setup-python@v5
|
|
40
|
+
with:
|
|
41
|
+
python-version: ${{ inputs.python-version }}
|
|
42
|
+
- name: Install the validator from this action's revision
|
|
43
|
+
shell: bash
|
|
44
|
+
run: python -m pip install --quiet --disable-pip-version-check "$GITHUB_ACTION_PATH"
|
|
45
|
+
- id: validate
|
|
46
|
+
name: Validate evidence files
|
|
47
|
+
shell: bash
|
|
48
|
+
env:
|
|
49
|
+
PYTHONIOENCODING: utf-8
|
|
50
|
+
INPUT_FILES: ${{ inputs.files }}
|
|
51
|
+
INPUT_PROFILE: ${{ inputs.profile }}
|
|
52
|
+
INPUT_FORMAT: ${{ inputs.format }}
|
|
53
|
+
INPUT_FAIL_ON: ${{ inputs.fail-on }}
|
|
54
|
+
run: >-
|
|
55
|
+
python "$GITHUB_ACTION_PATH/tools/github_action.py"
|
|
56
|
+
--files "$INPUT_FILES" --profile "$INPUT_PROFILE"
|
|
57
|
+
--format "$INPUT_FORMAT" --fail-on "$INPUT_FAIL_ON"
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# Drafts: write a claim in a few lines
|
|
2
|
+
|
|
3
|
+
A full evidence record spells out identifiers, evidence lines and hashes. A **draft** is a
|
|
4
|
+
compact YAML or JSON form of the same claim. `bioevidence build` (or `build_record` in
|
|
5
|
+
Python) expands it into a full record, which is then validated as usual.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
bioevidence build examples/drafts/llm_claim.yaml --output record.json
|
|
9
|
+
bioevidence validate record.json --profile literature-claim
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## What building does, and what it does not
|
|
13
|
+
|
|
14
|
+
Building only removes repetition. It:
|
|
15
|
+
|
|
16
|
+
- derives identifiers from the record `id` (or, when omitted, from a hash of the draft content);
|
|
17
|
+
- groups evidence items into one evidence line per `direction`;
|
|
18
|
+
- computes SHA-256 hashes of local source files named in `file`.
|
|
19
|
+
|
|
20
|
+
It never supplies facts you did not state. Scope, extraction method, retrieval time, source
|
|
21
|
+
version and reviewer decisions must be written explicitly, because each one can change an
|
|
22
|
+
admission decision. Unknown fields, missing fields, blank values, repeated list entries and
|
|
23
|
+
values outside the allowed choices are errors, not warnings.
|
|
24
|
+
|
|
25
|
+
## Format
|
|
26
|
+
|
|
27
|
+
```yaml
|
|
28
|
+
id: lab:claim-7 # optional record id
|
|
29
|
+
profile: literature-claim # must match the profile used for validation
|
|
30
|
+
uses: [research_summary, knowledge_base]
|
|
31
|
+
status: proposed # optional: proposed (default), accepted, rejected, superseded
|
|
32
|
+
|
|
33
|
+
statement:
|
|
34
|
+
subject: {id: "HGNC:1100", label: BRCA1, type: gene}
|
|
35
|
+
predicate: associated_with
|
|
36
|
+
object: {id: "MONDO:0007254", label: breast cancer, type: disease}
|
|
37
|
+
scope: ["taxon:9606"]
|
|
38
|
+
|
|
39
|
+
sources:
|
|
40
|
+
- id: paper # local name, referenced by evidence[].source
|
|
41
|
+
title: Example paper
|
|
42
|
+
type: publication # ontology_snapshot, registry_snapshot, dataset_snapshot,
|
|
43
|
+
# publication, web_page, local_file
|
|
44
|
+
uri: https://doi.org/10.0000/example # optional
|
|
45
|
+
version: "2024-05"
|
|
46
|
+
retrieved_at: "2026-09-21T00:00:00Z"
|
|
47
|
+
file: sources/paper.pdf # hashed at build time, relative to the draft
|
|
48
|
+
# sha256: <64 hex> # frozen hash; see below
|
|
49
|
+
|
|
50
|
+
evidence:
|
|
51
|
+
- source: paper
|
|
52
|
+
locator: Table 2
|
|
53
|
+
text: BRCA1 variants were associated with ... # optional
|
|
54
|
+
type: publication_result
|
|
55
|
+
method: llm_extraction # deterministic_parser, manual_curation,
|
|
56
|
+
# normalized_string_match, llm_extraction
|
|
57
|
+
scope: ["taxon:9606"]
|
|
58
|
+
direction: supports # optional: supports (default), contradicts, neutral
|
|
59
|
+
|
|
60
|
+
reviews: # optional
|
|
61
|
+
- reviewer: {id: "orcid:0000-0000-0000-0000", name: A. Curator, type: human}
|
|
62
|
+
decision: accept # accept, reject, defer
|
|
63
|
+
uses: [knowledge_base]
|
|
64
|
+
rationale: Checked Table 2 against the source.
|
|
65
|
+
decided_at: "2026-09-22T00:00:00Z"
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Quote identifiers containing `:`, versions and timestamps so YAML keeps them as strings.
|
|
69
|
+
Unquoted timestamps are converted to ISO 8601 strings.
|
|
70
|
+
|
|
71
|
+
## Source hashes
|
|
72
|
+
|
|
73
|
+
| You provide | Record gets | Meaning |
|
|
74
|
+
|---|---|---|
|
|
75
|
+
| `file` | `sha256` = hash of the file | The file you have is the reference snapshot. |
|
|
76
|
+
| `sha256` | `sha256` as stated | A frozen reference hash, e.g. from a registry or manifest. |
|
|
77
|
+
| both | `sha256` as stated, `observed_sha256` = hash of the file | Validation rejects the record (`BEV002`) if the file changed. |
|
|
78
|
+
|
|
79
|
+
Hashes identify bytes; they do not prove that a source is authentic or that the file is the
|
|
80
|
+
one a URI points to.
|
|
81
|
+
|
|
82
|
+
## Drafts from an LLM
|
|
83
|
+
|
|
84
|
+
`bioevidence draft-schema --profile literature-claim` prints a JSON Schema for drafts under
|
|
85
|
+
that profile: its allowed predicates, entity types and uses become enums. Use it for
|
|
86
|
+
structured output, then let your pipeline, not the model, set `method` and any `reviews`:
|
|
87
|
+
a model should not declare how its own output was produced or that a human accepted it.
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
from bioevidence_validator import build_record, draft_json_schema, validate_record
|
|
91
|
+
|
|
92
|
+
schema = draft_json_schema("literature-claim") # give this to your model
|
|
93
|
+
draft = ... # the model's structured output
|
|
94
|
+
for item in draft["evidence"]:
|
|
95
|
+
item["method"] = "llm_extraction" # set by your pipeline
|
|
96
|
+
report = validate_record(build_record(draft, base_dir="."), profile="literature-claim")
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
The schema guides authoring only. `build_record` and validation remain authoritative.
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
# Engineering contract — 0.
|
|
1
|
+
# Engineering contract — 0.5.0
|
|
2
2
|
|
|
3
3
|
## Validation stages
|
|
4
4
|
|
|
5
5
|
1. Parse one JSON record. The CLI rejects duplicate keys, nonfinite numbers, malformed
|
|
6
|
-
UTF-8/JSON, and
|
|
6
|
+
UTF-8/JSON, and nesting deeper than 100 levels as operational errors.
|
|
7
7
|
2. Validate against the packaged LinkML core compiled to JSON Schema, plus any
|
|
8
8
|
explicitly selected extension. Structural failures stop semantic evaluation.
|
|
9
9
|
3. Check profile binding, supported/distinct uses, IDs and references, and review targets.
|
|
@@ -48,8 +48,9 @@ between runs. Hashes identify inputs/configuration; they do not sign records or
|
|
|
48
48
|
that a source, source hash, label, or reviewer identity is authentic.
|
|
49
49
|
|
|
50
50
|
Source artifacts require a declared version, retrieval time, and frozen hash. The optional
|
|
51
|
-
observed hash is compared to that hash
|
|
52
|
-
|
|
51
|
+
observed hash is compared to that hash. Validation never retrieves source bytes or
|
|
52
|
+
calculates their hashes. Only `build` hashes local files explicitly named in a draft; it
|
|
53
|
+
does not fetch URIs. A profile can require a declared `independent_cohort_review`
|
|
53
54
|
evidence item, but the engine cannot establish independence or evaluation validity.
|
|
54
55
|
|
|
55
56
|
Report writes use a temporary file and atomic replacement. Input, selected profile,
|
|
@@ -63,20 +64,33 @@ Argparse usage errors also exit 2, with usage text rather than a validation repo
|
|
|
63
64
|
exports the compiled core schema. Custom schema export exports that selected schema;
|
|
64
65
|
validation still applies the baseline independently.
|
|
65
66
|
|
|
67
|
+
`build draft.yaml [--output record.json]` expands a [draft](DRAFTS.md) into a full record:
|
|
68
|
+
it derives identifiers, groups evidence lines by direction and hashes named local files,
|
|
69
|
+
but never supplies scope, extraction method, retrieval time, versions or review decisions.
|
|
70
|
+
Unknown, missing, blank, repeated or out-of-choice draft fields exit 3. The output cannot
|
|
71
|
+
overwrite the draft. `draft-schema --profile NAME` prints a JSON Schema for drafts in which
|
|
72
|
+
the profile's allowlists are enums; it guides authoring and never replaces validation.
|
|
73
|
+
|
|
74
|
+
The repository's composite GitHub Action (`action.yml`) installs the package from the
|
|
75
|
+
action's own revision and runs `build`/`validate` through the CLI for each matched file.
|
|
76
|
+
It fails on rejected or unreadable files, and on review-required files unless
|
|
77
|
+
`fail-on: rejected` is set.
|
|
78
|
+
|
|
66
79
|
## Reproduction
|
|
67
80
|
|
|
68
81
|
```bash
|
|
69
82
|
uv sync --frozen --extra dev
|
|
70
83
|
uv run --frozen pytest
|
|
71
84
|
uv build
|
|
72
|
-
uv run --isolated --no-project --with ./dist
|
|
85
|
+
uv run --isolated --no-project --with ./dist/*.whl python tools/check_distribution.py
|
|
73
86
|
```
|
|
74
87
|
|
|
75
|
-
CI runs on Linux/Python 3.11 and Windows/Python 3.13
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
88
|
+
CI runs on Linux/Python 3.11–3.13 and Windows/Python 3.13, and runs the GitHub Action on
|
|
89
|
+
Linux and Windows. Tests cover multi-domain acceptance, negative evidence cases,
|
|
90
|
+
use-specific human review, strict configuration/JSON/draft parsing, baseline enforcement,
|
|
91
|
+
context snapshots, audit digests, and CLI report behavior. The wheel smoke test imports
|
|
92
|
+
outside editable source, checks packaged profiles/schema, and exercises accepted, rejected,
|
|
93
|
+
review-required, custom-domain and draft-built records. It does not assess
|
|
80
94
|
biological correctness, model calibration, or predictive performance.
|
|
81
95
|
|
|
82
96
|
The [VBO case](../examples/vbo_canine/README.md) adds offline source verification and a
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# A compact draft of an LLM-extracted literature claim. Synthetic data only.
|
|
2
|
+
# Build: bioevidence build examples/drafts/llm_claim.yaml --output record.json
|
|
3
|
+
# Validate: bioevidence validate record.json --profile literature-claim (exit 2: review required)
|
|
4
|
+
profile: literature-claim
|
|
5
|
+
uses: [research_summary]
|
|
6
|
+
|
|
7
|
+
statement:
|
|
8
|
+
subject: {id: "SYN:GENE_A", label: Synthetic gene A, type: gene}
|
|
9
|
+
predicate: associated_with
|
|
10
|
+
object: {id: "SYN:PHENOTYPE_A", label: Synthetic phenotype A, type: phenotype}
|
|
11
|
+
scope: ["taxon:synthetic"]
|
|
12
|
+
|
|
13
|
+
sources:
|
|
14
|
+
- id: paper
|
|
15
|
+
title: Synthetic source; not real biological evidence
|
|
16
|
+
type: publication
|
|
17
|
+
uri: https://example.org/synthetic-evidence
|
|
18
|
+
version: synthetic-v1
|
|
19
|
+
retrieved_at: "2026-09-21T00:00:00Z"
|
|
20
|
+
file: synthetic_paper.txt # hashed at build time
|
|
21
|
+
|
|
22
|
+
evidence:
|
|
23
|
+
- source: paper
|
|
24
|
+
locator: Table 2
|
|
25
|
+
text: Synthetic gene A is associated with synthetic phenotype A.
|
|
26
|
+
type: publication_result
|
|
27
|
+
method: llm_extraction # set by your pipeline, not by the model
|
|
28
|
+
scope: ["taxon:synthetic"]
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# The same claim after manual curation and human review, requested for two uses.
|
|
2
|
+
# Build: bioevidence build examples/drafts/reviewed_claim.yaml --output record.json
|
|
3
|
+
# Validate: bioevidence validate record.json --profile literature-claim (exit 0: admitted)
|
|
4
|
+
profile: literature-claim
|
|
5
|
+
uses: [research_summary, knowledge_base]
|
|
6
|
+
|
|
7
|
+
statement:
|
|
8
|
+
subject: {id: "SYN:GENE_A", label: Synthetic gene A, type: gene}
|
|
9
|
+
predicate: associated_with
|
|
10
|
+
object: {id: "SYN:PHENOTYPE_A", label: Synthetic phenotype A, type: phenotype}
|
|
11
|
+
scope: ["taxon:synthetic"]
|
|
12
|
+
|
|
13
|
+
sources:
|
|
14
|
+
- id: paper
|
|
15
|
+
title: Synthetic source; not real biological evidence
|
|
16
|
+
type: publication
|
|
17
|
+
uri: https://example.org/synthetic-evidence
|
|
18
|
+
version: synthetic-v1
|
|
19
|
+
retrieved_at: "2026-09-21T00:00:00Z"
|
|
20
|
+
file: synthetic_paper.txt
|
|
21
|
+
|
|
22
|
+
evidence:
|
|
23
|
+
- source: paper
|
|
24
|
+
locator: Table 2
|
|
25
|
+
text: Synthetic gene A is associated with synthetic phenotype A.
|
|
26
|
+
type: publication_result
|
|
27
|
+
method: manual_curation
|
|
28
|
+
scope: ["taxon:synthetic"]
|
|
29
|
+
|
|
30
|
+
reviews:
|
|
31
|
+
- reviewer: {id: "orcid:0000-0000-0000-0000", name: Synthetic curator, type: human}
|
|
32
|
+
decision: accept
|
|
33
|
+
uses: [knowledge_base]
|
|
34
|
+
rationale: Synthetic example of a recorded human acceptance.
|
|
35
|
+
decided_at: "2026-09-22T00:00:00Z"
|