scientific-reasoning-auditor 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scientific_reasoning_auditor-0.5.0/LICENSE +21 -0
- scientific_reasoning_auditor-0.5.0/MANIFEST.in +2 -0
- scientific_reasoning_auditor-0.5.0/PKG-INFO +158 -0
- scientific_reasoning_auditor-0.5.0/README.md +130 -0
- scientific_reasoning_auditor-0.5.0/examples/logic_review_demo.md +29 -0
- scientific_reasoning_auditor-0.5.0/examples/sample_manuscript.md +20 -0
- scientific_reasoning_auditor-0.5.0/examples/welch_t_example.json +19 -0
- scientific_reasoning_auditor-0.5.0/pyproject.toml +41 -0
- scientific_reasoning_auditor-0.5.0/schemas/semantic-review.schema.json +226 -0
- scientific_reasoning_auditor-0.5.0/setup.cfg +4 -0
- scientific_reasoning_auditor-0.5.0/src/scientific_reasoning_auditor.egg-info/PKG-INFO +158 -0
- scientific_reasoning_auditor-0.5.0/src/scientific_reasoning_auditor.egg-info/SOURCES.txt +27 -0
- scientific_reasoning_auditor-0.5.0/src/scientific_reasoning_auditor.egg-info/dependency_links.txt +1 -0
- scientific_reasoning_auditor-0.5.0/src/scientific_reasoning_auditor.egg-info/entry_points.txt +2 -0
- scientific_reasoning_auditor-0.5.0/src/scientific_reasoning_auditor.egg-info/requires.txt +6 -0
- scientific_reasoning_auditor-0.5.0/src/scientific_reasoning_auditor.egg-info/top_level.txt +1 -0
- scientific_reasoning_auditor-0.5.0/src/sra/__init__.py +1 -0
- scientific_reasoning_auditor-0.5.0/src/sra/audit.py +46 -0
- scientific_reasoning_auditor-0.5.0/src/sra/checks.py +394 -0
- scientific_reasoning_auditor-0.5.0/src/sra/cli.py +113 -0
- scientific_reasoning_auditor-0.5.0/src/sra/document.py +160 -0
- scientific_reasoning_auditor-0.5.0/src/sra/graph.py +431 -0
- scientific_reasoning_auditor-0.5.0/src/sra/model_client.py +80 -0
- scientific_reasoning_auditor-0.5.0/src/sra/models.py +62 -0
- scientific_reasoning_auditor-0.5.0/src/sra/provenance.py +164 -0
- scientific_reasoning_auditor-0.5.0/src/sra/report.py +88 -0
- scientific_reasoning_auditor-0.5.0/src/sra/semantic_review.py +421 -0
- scientific_reasoning_auditor-0.5.0/src/sra/stats.py +197 -0
- scientific_reasoning_auditor-0.5.0/tests/test_audit.py +850 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Furox-Art
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scientific-reasoning-auditor
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: Evidence-traceable screening and reasoning review for scientific manuscripts
|
|
5
|
+
Author: Furox-Art
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Furox-Art/scientific-reasoning-auditor
|
|
8
|
+
Project-URL: Repository, https://github.com/Furox-Art/scientific-reasoning-auditor
|
|
9
|
+
Project-URL: Issues, https://github.com/Furox-Art/scientific-reasoning-auditor/issues
|
|
10
|
+
Keywords: scientific-reasoning,reproducibility,manuscript-review,statistics,research-software
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Provides-Extra: pdf
|
|
24
|
+
Requires-Dist: pypdf>=5.0; extra == "pdf"
|
|
25
|
+
Provides-Extra: statistics
|
|
26
|
+
Requires-Dist: scipy>=1.11; extra == "statistics"
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+
# Scientific Reasoning Auditor
|
|
30
|
+
|
|
31
|
+
A research-software project for auditing the reasoning chain in scientific manuscripts:
|
|
32
|
+
|
|
33
|
+
**Hypothesis → assumptions → methods → data → analysis → inference → conclusion**
|
|
34
|
+
|
|
35
|
+
The software exposes reviewable reasoning risks with evidence locations and reproducibility metadata. It does not declare a manuscript scientifically true or false.
|
|
36
|
+
|
|
37
|
+
## Current status: v0.5.0
|
|
38
|
+
|
|
39
|
+
This release advances the original v0.1 screening baseline with a traceable reasoning-review layer and an independently recalculable statistical summary check.
|
|
40
|
+
|
|
41
|
+
### Implemented
|
|
42
|
+
|
|
43
|
+
- Read Markdown and UTF-8 text manuscripts; optional PDF text extraction.
|
|
44
|
+
- Preserve repeated section headings, ignore fenced code blocks during section/heuristic extraction, and conservatively detect standard standalone headings in plain text/PDF extraction.
|
|
45
|
+
- Apply negation-aware causal/directional screening and avoid treating a cited prior randomized trial as proof that the current study was randomized.
|
|
46
|
+
- Record manuscript SHA-256, exact source/config tree SHA-256 (including uncommitted changes), file size, parser details, ruleset/configuration hashes, Python/runtime details and an environment fingerprint.
|
|
47
|
+
- Attach line and section references to extracted Discussion/Conclusion claims and candidate result passages.
|
|
48
|
+
- Identify conclusion claims for which no lexical result candidate is found.
|
|
49
|
+
- Flag possible opposite-direction statements between Results and Conclusion (for example, increased versus decreased for overlapping topics).
|
|
50
|
+
- Flag causal conclusion language when Methods text describes an observational or non-randomized design.
|
|
51
|
+
- Flag selected absolute/conclusive wording.
|
|
52
|
+
- Recalculate a two-independent-group Welch t-test from supplied group means, sample standard deviations and sample sizes; compare reported mean difference, t statistic, degrees of freedom and two-sided p-value within explicit tolerances.
|
|
53
|
+
- Produce an explicit reasoning graph with claim, evidence, method, assumption, limitation, inference and grouped document-context nodes, source locations, relation edges and a graph fingerprint. Shared context nodes avoid method/limitation-to-inference Cartesian edge growth.
|
|
54
|
+
- Validate graph structure for dangling references, duplicate IDs and dependency cycles; report inferences without lexical evidence.
|
|
55
|
+
- Create source-bound semantic-review templates for external LLMs/agents and validate returned source hashes, graph fingerprints, inference-linked support evidence, explicitly reviewer-selected additional evidence, confidence fields and meaningful source quotations.
|
|
56
|
+
- Require explicit causal-assumption review even when a lexically similar assumption sentence was extracted; lexical overlap no longer clears the causal assumption gap.
|
|
57
|
+
- Produce JSON and Markdown audit reports plus a machine-readable statistical verification result.
|
|
58
|
+
- Run tests using Python's standard library plus the optional SciPy statistics extra.
|
|
59
|
+
|
|
60
|
+
### Interpretation rules
|
|
61
|
+
|
|
62
|
+
- A finding is a review signal, not an automatic verdict.
|
|
63
|
+
- Lexical overlap is not semantic entailment and does not prove that evidence supports a claim.
|
|
64
|
+
- A missing lexical match is not proof that a conclusion is unsupported; relevant evidence may be paraphrased or located elsewhere.
|
|
65
|
+
- A possible direction conflict may be caused by subgroup definitions, recoding, different endpoints or sentence-extraction errors.
|
|
66
|
+
- Observational wording does not by itself invalidate a causal inference; causal identification may be justified by design and assumptions not detected by these rules.
|
|
67
|
+
- Welch verification checks arithmetic conditional on supplied summary statistics. It does not validate the raw data, study design, assumptions or choice of statistical test.
|
|
68
|
+
- No finding does not mean that the paper is logically valid.
|
|
69
|
+
|
|
70
|
+
## Install
|
|
71
|
+
|
|
72
|
+
Install the published release from PyPI:
|
|
73
|
+
|
|
74
|
+
python -m pip install "scientific-reasoning-auditor[statistics]"
|
|
75
|
+
|
|
76
|
+
For PDF input:
|
|
77
|
+
|
|
78
|
+
python -m pip install "scientific-reasoning-auditor[pdf,statistics]"
|
|
79
|
+
|
|
80
|
+
To work from source:
|
|
81
|
+
|
|
82
|
+
git clone https://github.com/Furox-Art/scientific-reasoning-auditor.git
|
|
83
|
+
cd scientific-reasoning-auditor
|
|
84
|
+
python3 -m venv .venv
|
|
85
|
+
source .venv/bin/activate
|
|
86
|
+
python -m pip install -e ".[statistics]"
|
|
87
|
+
|
|
88
|
+
For PDF input:
|
|
89
|
+
|
|
90
|
+
python -m pip install -e ".[pdf,statistics]"
|
|
91
|
+
|
|
92
|
+
## Run manuscript audit
|
|
93
|
+
|
|
94
|
+
sra audit examples/sample_manuscript.md --out reports/sample
|
|
95
|
+
|
|
96
|
+
The command writes reports/sample.json and reports/sample.md. JSON includes the reproducibility manifest; Markdown is the human-readable review.
|
|
97
|
+
|
|
98
|
+
A synthetic document with deliberately planted inference and direction-conflict signals is provided:
|
|
99
|
+
|
|
100
|
+
sra audit examples/logic_review_demo.md --out reports/logic-review-demo
|
|
101
|
+
|
|
102
|
+
## Build a reasoning graph and request semantic review
|
|
103
|
+
|
|
104
|
+
Run an audit to include a graph in the JSON and Markdown reports:
|
|
105
|
+
|
|
106
|
+
sra audit examples/logic_review_demo.md --out reports/logic-review
|
|
107
|
+
|
|
108
|
+
The output contract is documented in the [semantic-review JSON Schema](https://github.com/Furox-Art/scientific-reasoning-auditor/blob/main/schemas/semantic-review.schema.json).
|
|
109
|
+
|
|
110
|
+
Create a JSON template and choose the review mode:
|
|
111
|
+
|
|
112
|
+
sra review-template examples/logic_review_demo.md --mode single --out reports/semantic-review-single.json
|
|
113
|
+
sra review-template examples/logic_review_demo.md --mode parallel --out reports/semantic-review-parallel.json
|
|
114
|
+
|
|
115
|
+
The single mode (default) prepares the whole review for one model. It can now call an OpenAI-compatible Chat Completions endpoint directly. The parallel mode remains an external agent-orchestrator workflow; the user selects the mode for each run. The Python CLI does not pretend to launch ChatGPT parallel agents.
|
|
116
|
+
|
|
117
|
+
For a direct single-model API review, set the API key in the environment and run:
|
|
118
|
+
|
|
119
|
+
export OPENAI_API_KEY="your-api-key"
|
|
120
|
+
sra review-model examples/logic_review_demo.md --model "your-model-id" --base-url "https://api.openai.com/v1" --out reports/semantic-review-completed.json
|
|
121
|
+
|
|
122
|
+
The command sends the complete manuscript and source-bound template to one model, parses its JSON response, then validates the returned review against the manuscript and reasoning graph. Any OpenAI-compatible Chat Completions endpoint can be configured with --base-url and --api-key-env. The API key is read from the environment and is not written to reports. Model/API charges and provider data-retention policies are controlled by the provider.
|
|
123
|
+
|
|
124
|
+
For parallel mode, export the template and pass it to the connected external parallel-agent orchestrator. Each agent must receive the full manuscript and its assigned assessment subset; merge all results by inference ID before importing. The current CLI does not launch the ChatGPT UI-based parallel-agent tool.
|
|
125
|
+
|
|
126
|
+
Give the template and full manuscript to the selected model or external orchestration layer for passage-grounded assessment. Fill every assessment with a relation (entailed, partially_supported, contradicted, not_supported, or unclear), confidence, rationale and exact source quotations where applicable. Then import the completed file:
|
|
127
|
+
|
|
128
|
+
sra audit examples/logic_review_demo.md --semantic-review reports/semantic-review-completed.json --out reports/semantic-reviewed
|
|
129
|
+
|
|
130
|
+
The template includes a complete source-bound catalog of Results evidence-node IDs, passages and locations. The importer checks the manuscript hash and reasoning-graph fingerprint, requires graph-linked support IDs or explicit reviewer-selected Results evidence IDs, and validates the existence, type and source quotation provenance of selected nodes. Reviewer-selected evidence provenance is checked, but its semantic relevance is not proven. Current-schema quotations must be at least 20 characters and match both the manuscript and selected node text. The importer also enforces the review contract and UTC timestamps. It does not prove the external reviewer’s semantic judgment; expert adjudication remains necessary. The application can call one OpenAI-compatible model in single mode. Parallel-agent execution remains an external orchestration step.
|
|
131
|
+
|
|
132
|
+
## Independently verify a Welch t-test
|
|
133
|
+
|
|
134
|
+
Create a JSON file with two groups' mean, sample SD and n, plus the reported statistics to compare. Example:
|
|
135
|
+
|
|
136
|
+
sra verify-stats examples/welch_t_example.json --out reports/welch-verification.json
|
|
137
|
+
|
|
138
|
+
Each reported value is compared with the independently calculated value using the configured absolute tolerance. Without an explicit tolerance, the verifier uses half the last displayed decimal place; field-specific or default overrides can be supplied in the JSON input. A mismatch returns a non-zero exit code. Report p-values as positive values in (0, 1], not as zero. The command supports only independent two-group Welch t-tests in this version.
|
|
139
|
+
|
|
140
|
+
## Scientific validation roadmap
|
|
141
|
+
|
|
142
|
+
The project is split into separate assurance layers. A rule-based text scan must not be presented as full scientific validation.
|
|
143
|
+
|
|
144
|
+
1. **Reasoning graph** — extend the current graph from rule-extracted candidates to validated, semantic claim/evidence/assumption/method/inference relations with provenance.
|
|
145
|
+
2. **Quantitative verification** — extend beyond Welch tests to supported tests, effect sizes, confidence intervals, model-fit statistics and data-level recalculation.
|
|
146
|
+
3. **Mathematical verification** — symbolic/numeric checks of equations, dimensions, boundary conditions and derivation steps where machine-checkable.
|
|
147
|
+
4. **Code and data reproducibility** — controlled execution of supplied analysis code, dataset fingerprints, environment lockfiles, logs and output comparison.
|
|
148
|
+
5. **Literature and citation verification** — resolve references and verify whether cited sources support the exact claims.
|
|
149
|
+
6. **Specialist review** — task-specific expert analysis, disagreement tracking and human adjudication.
|
|
150
|
+
7. **Benchmarking** — curated positive/negative examples, false-positive/false-negative rates, calibration and regression tests.
|
|
151
|
+
|
|
152
|
+
## Development
|
|
153
|
+
|
|
154
|
+
python -m unittest discover -s tests -v
|
|
155
|
+
|
|
156
|
+
## Safety and scope
|
|
157
|
+
|
|
158
|
+
Never run untrusted supplementary code automatically. Any future execution layer must be sandboxed, resource-limited and explicit about network, file and process permissions. Preserve original source files; reports are written separately.
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# Scientific Reasoning Auditor
|
|
2
|
+
|
|
3
|
+
A research-software project for auditing the reasoning chain in scientific manuscripts:
|
|
4
|
+
|
|
5
|
+
**Hypothesis → assumptions → methods → data → analysis → inference → conclusion**
|
|
6
|
+
|
|
7
|
+
The software exposes reviewable reasoning risks with evidence locations and reproducibility metadata. It does not declare a manuscript scientifically true or false.
|
|
8
|
+
|
|
9
|
+
## Current status: v0.5.0
|
|
10
|
+
|
|
11
|
+
This release advances the original v0.1 screening baseline with a traceable reasoning-review layer and an independently recalculable statistical summary check.
|
|
12
|
+
|
|
13
|
+
### Implemented
|
|
14
|
+
|
|
15
|
+
- Read Markdown and UTF-8 text manuscripts; optional PDF text extraction.
|
|
16
|
+
- Preserve repeated section headings, ignore fenced code blocks during section/heuristic extraction, and conservatively detect standard standalone headings in plain text/PDF extraction.
|
|
17
|
+
- Apply negation-aware causal/directional screening and avoid treating a cited prior randomized trial as proof that the current study was randomized.
|
|
18
|
+
- Record manuscript SHA-256, exact source/config tree SHA-256 (including uncommitted changes), file size, parser details, ruleset/configuration hashes, Python/runtime details and an environment fingerprint.
|
|
19
|
+
- Attach line and section references to extracted Discussion/Conclusion claims and candidate result passages.
|
|
20
|
+
- Identify conclusion claims for which no lexical result candidate is found.
|
|
21
|
+
- Flag possible opposite-direction statements between Results and Conclusion (for example, increased versus decreased for overlapping topics).
|
|
22
|
+
- Flag causal conclusion language when Methods text describes an observational or non-randomized design.
|
|
23
|
+
- Flag selected absolute/conclusive wording.
|
|
24
|
+
- Recalculate a two-independent-group Welch t-test from supplied group means, sample standard deviations and sample sizes; compare reported mean difference, t statistic, degrees of freedom and two-sided p-value within explicit tolerances.
|
|
25
|
+
- Produce an explicit reasoning graph with claim, evidence, method, assumption, limitation, inference and grouped document-context nodes, source locations, relation edges and a graph fingerprint. Shared context nodes avoid method/limitation-to-inference Cartesian edge growth.
|
|
26
|
+
- Validate graph structure for dangling references, duplicate IDs and dependency cycles; report inferences without lexical evidence.
|
|
27
|
+
- Create source-bound semantic-review templates for external LLMs/agents and validate returned source hashes, graph fingerprints, inference-linked support evidence, explicitly reviewer-selected additional evidence, confidence fields and meaningful source quotations.
|
|
28
|
+
- Require explicit causal-assumption review even when a lexically similar assumption sentence was extracted; lexical overlap no longer clears the causal assumption gap.
|
|
29
|
+
- Produce JSON and Markdown audit reports plus a machine-readable statistical verification result.
|
|
30
|
+
- Run tests using Python's standard library plus the optional SciPy statistics extra.
|
|
31
|
+
|
|
32
|
+
### Interpretation rules
|
|
33
|
+
|
|
34
|
+
- A finding is a review signal, not an automatic verdict.
|
|
35
|
+
- Lexical overlap is not semantic entailment and does not prove that evidence supports a claim.
|
|
36
|
+
- A missing lexical match is not proof that a conclusion is unsupported; relevant evidence may be paraphrased or located elsewhere.
|
|
37
|
+
- A possible direction conflict may be caused by subgroup definitions, recoding, different endpoints or sentence-extraction errors.
|
|
38
|
+
- Observational wording does not by itself invalidate a causal inference; causal identification may be justified by design and assumptions not detected by these rules.
|
|
39
|
+
- Welch verification checks arithmetic conditional on supplied summary statistics. It does not validate the raw data, study design, assumptions or choice of statistical test.
|
|
40
|
+
- No finding does not mean that the paper is logically valid.
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
|
|
44
|
+
Install the published release from PyPI:
|
|
45
|
+
|
|
46
|
+
python -m pip install "scientific-reasoning-auditor[statistics]"
|
|
47
|
+
|
|
48
|
+
For PDF input:
|
|
49
|
+
|
|
50
|
+
python -m pip install "scientific-reasoning-auditor[pdf,statistics]"
|
|
51
|
+
|
|
52
|
+
To work from source:
|
|
53
|
+
|
|
54
|
+
git clone https://github.com/Furox-Art/scientific-reasoning-auditor.git
|
|
55
|
+
cd scientific-reasoning-auditor
|
|
56
|
+
python3 -m venv .venv
|
|
57
|
+
source .venv/bin/activate
|
|
58
|
+
python -m pip install -e ".[statistics]"
|
|
59
|
+
|
|
60
|
+
For PDF input:
|
|
61
|
+
|
|
62
|
+
python -m pip install -e ".[pdf,statistics]"
|
|
63
|
+
|
|
64
|
+
## Run manuscript audit
|
|
65
|
+
|
|
66
|
+
sra audit examples/sample_manuscript.md --out reports/sample
|
|
67
|
+
|
|
68
|
+
The command writes reports/sample.json and reports/sample.md. JSON includes the reproducibility manifest; Markdown is the human-readable review.
|
|
69
|
+
|
|
70
|
+
A synthetic document with deliberately planted inference and direction-conflict signals is provided:
|
|
71
|
+
|
|
72
|
+
sra audit examples/logic_review_demo.md --out reports/logic-review-demo
|
|
73
|
+
|
|
74
|
+
## Build a reasoning graph and request semantic review
|
|
75
|
+
|
|
76
|
+
Run an audit to include a graph in the JSON and Markdown reports:
|
|
77
|
+
|
|
78
|
+
sra audit examples/logic_review_demo.md --out reports/logic-review
|
|
79
|
+
|
|
80
|
+
The output contract is documented in the [semantic-review JSON Schema](https://github.com/Furox-Art/scientific-reasoning-auditor/blob/main/schemas/semantic-review.schema.json).
|
|
81
|
+
|
|
82
|
+
Create a JSON template and choose the review mode:
|
|
83
|
+
|
|
84
|
+
sra review-template examples/logic_review_demo.md --mode single --out reports/semantic-review-single.json
|
|
85
|
+
sra review-template examples/logic_review_demo.md --mode parallel --out reports/semantic-review-parallel.json
|
|
86
|
+
|
|
87
|
+
The single mode (default) prepares the whole review for one model. It can now call an OpenAI-compatible Chat Completions endpoint directly. The parallel mode remains an external agent-orchestrator workflow; the user selects the mode for each run. The Python CLI does not pretend to launch ChatGPT parallel agents.
|
|
88
|
+
|
|
89
|
+
For a direct single-model API review, set the API key in the environment and run:
|
|
90
|
+
|
|
91
|
+
export OPENAI_API_KEY="your-api-key"
|
|
92
|
+
sra review-model examples/logic_review_demo.md --model "your-model-id" --base-url "https://api.openai.com/v1" --out reports/semantic-review-completed.json
|
|
93
|
+
|
|
94
|
+
The command sends the complete manuscript and source-bound template to one model, parses its JSON response, then validates the returned review against the manuscript and reasoning graph. Any OpenAI-compatible Chat Completions endpoint can be configured with --base-url and --api-key-env. The API key is read from the environment and is not written to reports. Model/API charges and provider data-retention policies are controlled by the provider.
|
|
95
|
+
|
|
96
|
+
For parallel mode, export the template and pass it to the connected external parallel-agent orchestrator. Each agent must receive the full manuscript and its assigned assessment subset; merge all results by inference ID before importing. The current CLI does not launch the ChatGPT UI-based parallel-agent tool.
|
|
97
|
+
|
|
98
|
+
Give the template and full manuscript to the selected model or external orchestration layer for passage-grounded assessment. Fill every assessment with a relation (entailed, partially_supported, contradicted, not_supported, or unclear), confidence, rationale and exact source quotations where applicable. Then import the completed file:
|
|
99
|
+
|
|
100
|
+
sra audit examples/logic_review_demo.md --semantic-review reports/semantic-review-completed.json --out reports/semantic-reviewed
|
|
101
|
+
|
|
102
|
+
The template includes a complete source-bound catalog of Results evidence-node IDs, passages and locations. The importer checks the manuscript hash and reasoning-graph fingerprint, requires graph-linked support IDs or explicit reviewer-selected Results evidence IDs, and validates the existence, type and source quotation provenance of selected nodes. Reviewer-selected evidence provenance is checked, but its semantic relevance is not proven. Current-schema quotations must be at least 20 characters and match both the manuscript and selected node text. The importer also enforces the review contract and UTC timestamps. It does not prove the external reviewer’s semantic judgment; expert adjudication remains necessary. The application can call one OpenAI-compatible model in single mode. Parallel-agent execution remains an external orchestration step.
|
|
103
|
+
|
|
104
|
+
## Independently verify a Welch t-test
|
|
105
|
+
|
|
106
|
+
Create a JSON file with two groups' mean, sample SD and n, plus the reported statistics to compare. Example:
|
|
107
|
+
|
|
108
|
+
sra verify-stats examples/welch_t_example.json --out reports/welch-verification.json
|
|
109
|
+
|
|
110
|
+
Each reported value is compared with the independently calculated value using the configured absolute tolerance. Without an explicit tolerance, the verifier uses half the last displayed decimal place; field-specific or default overrides can be supplied in the JSON input. A mismatch returns a non-zero exit code. Report p-values as positive values in (0, 1], not as zero. The command supports only independent two-group Welch t-tests in this version.
|
|
111
|
+
|
|
112
|
+
## Scientific validation roadmap
|
|
113
|
+
|
|
114
|
+
The project is split into separate assurance layers. A rule-based text scan must not be presented as full scientific validation.
|
|
115
|
+
|
|
116
|
+
1. **Reasoning graph** — extend the current graph from rule-extracted candidates to validated, semantic claim/evidence/assumption/method/inference relations with provenance.
|
|
117
|
+
2. **Quantitative verification** — extend beyond Welch tests to supported tests, effect sizes, confidence intervals, model-fit statistics and data-level recalculation.
|
|
118
|
+
3. **Mathematical verification** — symbolic/numeric checks of equations, dimensions, boundary conditions and derivation steps where machine-checkable.
|
|
119
|
+
4. **Code and data reproducibility** — controlled execution of supplied analysis code, dataset fingerprints, environment lockfiles, logs and output comparison.
|
|
120
|
+
5. **Literature and citation verification** — resolve references and verify whether cited sources support the exact claims.
|
|
121
|
+
6. **Specialist review** — task-specific expert analysis, disagreement tracking and human adjudication.
|
|
122
|
+
7. **Benchmarking** — curated positive/negative examples, false-positive/false-negative rates, calibration and regression tests.
|
|
123
|
+
|
|
124
|
+
## Development
|
|
125
|
+
|
|
126
|
+
python -m unittest discover -s tests -v
|
|
127
|
+
|
|
128
|
+
## Safety and scope
|
|
129
|
+
|
|
130
|
+
Never run untrusted supplementary code automatically. Any future execution layer must be sandboxed, resource-limited and explicit about network, file and process permissions. Preserve original source files; reports are written separately.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Synthetic logic-review demonstration
|
|
2
|
+
|
|
3
|
+
This file is intentionally constructed to contain review signals. It is not a real study or real scientific evidence.
|
|
4
|
+
|
|
5
|
+
# Methods
|
|
6
|
+
|
|
7
|
+
This was a cross-sectional observational study. Participants were not randomly assigned to exposure groups.
|
|
8
|
+
|
|
9
|
+
# Results
|
|
10
|
+
|
|
11
|
+
Exposure X was associated with a higher measured outcome.
|
|
12
|
+
In subgroup A, the treatment increased the measured score.
|
|
13
|
+
|
|
14
|
+
# Discussion
|
|
15
|
+
|
|
16
|
+
The association may be influenced by residual confounding and selection effects.
|
|
17
|
+
|
|
18
|
+
# Conclusion
|
|
19
|
+
|
|
20
|
+
Exposure X caused a higher measured outcome.
|
|
21
|
+
In subgroup A, the treatment decreased the measured score.
|
|
22
|
+
|
|
23
|
+
# Limitations
|
|
24
|
+
|
|
25
|
+
The design is observational, residual confounding cannot be excluded, and subgroup estimates are imprecise.
|
|
26
|
+
|
|
27
|
+
# References
|
|
28
|
+
|
|
29
|
+
Synthetic placeholder only. No real source is cited.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Abstract
|
|
2
|
+
This example tests the first-pass screening workflow.
|
|
3
|
+
|
|
4
|
+
# Methods
|
|
5
|
+
We evaluated a hypothetical intervention using a controlled design.
|
|
6
|
+
|
|
7
|
+
# Results
|
|
8
|
+
The intervention increased the measured score compared with the baseline group.
|
|
9
|
+
|
|
10
|
+
# Discussion
|
|
11
|
+
The result is associated with the intervention, but unmeasured confounding may remain.
|
|
12
|
+
|
|
13
|
+
# Conclusion
|
|
14
|
+
The findings suggest that the intervention increased the measured score.
|
|
15
|
+
|
|
16
|
+
# Limitations
|
|
17
|
+
The sample is small and the observation period is short.
|
|
18
|
+
|
|
19
|
+
# References
|
|
20
|
+
Example reference placeholder. Replace with real references before use.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
{
|
|
2
|
+
"test": "welch_t",
|
|
3
|
+
"group_a": {
|
|
4
|
+
"mean": 5.0,
|
|
5
|
+
"sd": 1.0,
|
|
6
|
+
"n": 25
|
|
7
|
+
},
|
|
8
|
+
"group_b": {
|
|
9
|
+
"mean": 4.0,
|
|
10
|
+
"sd": 1.0,
|
|
11
|
+
"n": 25
|
|
12
|
+
},
|
|
13
|
+
"reported": {
|
|
14
|
+
"mean_difference": 1.0,
|
|
15
|
+
"t": 3.536,
|
|
16
|
+
"df": 48.0,
|
|
17
|
+
"p_two_sided": 0.001
|
|
18
|
+
}
|
|
19
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "scientific-reasoning-auditor"
|
|
7
|
+
version = "0.5.0"
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
requires-python = ">=3.11"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "Furox-Art" }]
|
|
13
|
+
description = "Evidence-traceable screening and reasoning review for scientific manuscripts"
|
|
14
|
+
keywords = ["scientific-reasoning", "reproducibility", "manuscript-review", "statistics", "research-software"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Topic :: Scientific/Engineering",
|
|
24
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
25
|
+
]
|
|
26
|
+
dependencies = []
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://github.com/Furox-Art/scientific-reasoning-auditor"
|
|
30
|
+
Repository = "https://github.com/Furox-Art/scientific-reasoning-auditor"
|
|
31
|
+
Issues = "https://github.com/Furox-Art/scientific-reasoning-auditor/issues"
|
|
32
|
+
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
pdf = ["pypdf>=5.0"]
|
|
35
|
+
statistics = ["scipy>=1.11"]
|
|
36
|
+
|
|
37
|
+
[project.scripts]
|
|
38
|
+
sra = "sra.cli:main"
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.packages.find]
|
|
41
|
+
where = ["src"]
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/Furox-Art/scientific-reasoning-auditor/blob/main/schemas/semantic-review.schema.json",
|
|
4
|
+
"title": "Scientific Reasoning Auditor external semantic review",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": [
|
|
7
|
+
"schema_version",
|
|
8
|
+
"review_type",
|
|
9
|
+
"review_mode",
|
|
10
|
+
"source_sha256",
|
|
11
|
+
"graph_fingerprint",
|
|
12
|
+
"evidence_catalog",
|
|
13
|
+
"reviewer",
|
|
14
|
+
"instructions",
|
|
15
|
+
"relation_labels",
|
|
16
|
+
"assessments"
|
|
17
|
+
],
|
|
18
|
+
"properties": {
|
|
19
|
+
"schema_version": {
|
|
20
|
+
"const": "1.3"
|
|
21
|
+
},
|
|
22
|
+
"review_type": {
|
|
23
|
+
"const": "external_semantic_reasoning_review"
|
|
24
|
+
},
|
|
25
|
+
"review_mode": {
|
|
26
|
+
"enum": [
|
|
27
|
+
"single",
|
|
28
|
+
"parallel"
|
|
29
|
+
],
|
|
30
|
+
"description": "User-selected review orchestration preference; this field does not execute a model or start agents."
|
|
31
|
+
},
|
|
32
|
+
"source_sha256": {
|
|
33
|
+
"type": "string",
|
|
34
|
+
"pattern": "^[a-fA-F0-9]{64}$"
|
|
35
|
+
},
|
|
36
|
+
"graph_fingerprint": {
|
|
37
|
+
"type": "string",
|
|
38
|
+
"pattern": "^[a-fA-F0-9]{64}$"
|
|
39
|
+
},
|
|
40
|
+
"reviewer": {
|
|
41
|
+
"type": "object",
|
|
42
|
+
"required": [
|
|
43
|
+
"name",
|
|
44
|
+
"model_or_agent",
|
|
45
|
+
"reviewed_at_utc"
|
|
46
|
+
],
|
|
47
|
+
"properties": {
|
|
48
|
+
"name": {
|
|
49
|
+
"type": "string"
|
|
50
|
+
},
|
|
51
|
+
"model_or_agent": {
|
|
52
|
+
"type": "string"
|
|
53
|
+
},
|
|
54
|
+
"reviewed_at_utc": {
|
|
55
|
+
"anyOf": [
|
|
56
|
+
{
|
|
57
|
+
"const": ""
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"type": "string",
|
|
61
|
+
"format": "date-time",
|
|
62
|
+
"pattern": "(Z|\\+00:00)$"
|
|
63
|
+
}
|
|
64
|
+
],
|
|
65
|
+
"description": "Blank in an unfilled template; completed reviews must use an ISO-8601 UTC timestamp."
|
|
66
|
+
}
|
|
67
|
+
},
|
|
68
|
+
"additionalProperties": false
|
|
69
|
+
},
|
|
70
|
+
"instructions": {
|
|
71
|
+
"type": "array",
|
|
72
|
+
"items": {
|
|
73
|
+
"type": "string"
|
|
74
|
+
}
|
|
75
|
+
},
|
|
76
|
+
"relation_labels": {
|
|
77
|
+
"type": "array",
|
|
78
|
+
"items": {
|
|
79
|
+
"enum": [
|
|
80
|
+
"entailed",
|
|
81
|
+
"partially_supported",
|
|
82
|
+
"contradicted",
|
|
83
|
+
"not_supported",
|
|
84
|
+
"unclear"
|
|
85
|
+
]
|
|
86
|
+
}
|
|
87
|
+
},
|
|
88
|
+
"assessments": {
|
|
89
|
+
"type": "array",
|
|
90
|
+
"items": {
|
|
91
|
+
"type": "object",
|
|
92
|
+
"required": [
|
|
93
|
+
"inference_node_id",
|
|
94
|
+
"claim_node_id",
|
|
95
|
+
"evidence_node_ids",
|
|
96
|
+
"relation",
|
|
97
|
+
"confidence",
|
|
98
|
+
"rationale",
|
|
99
|
+
"source_quotes",
|
|
100
|
+
"missing_assumptions",
|
|
101
|
+
"counter_evidence_node_ids",
|
|
102
|
+
"reviewer_selected_evidence_node_ids"
|
|
103
|
+
],
|
|
104
|
+
"properties": {
|
|
105
|
+
"inference_node_id": {
|
|
106
|
+
"type": "string",
|
|
107
|
+
"pattern": "^inference:[a-f0-9]{16}$"
|
|
108
|
+
},
|
|
109
|
+
"claim_node_id": {
|
|
110
|
+
"type": "string",
|
|
111
|
+
"pattern": "^claim:[a-f0-9]{16}$"
|
|
112
|
+
},
|
|
113
|
+
"evidence_node_ids": {
|
|
114
|
+
"type": "array",
|
|
115
|
+
"items": {
|
|
116
|
+
"type": "string",
|
|
117
|
+
"pattern": "^evidence:[a-f0-9]{16}$"
|
|
118
|
+
},
|
|
119
|
+
"uniqueItems": true
|
|
120
|
+
},
|
|
121
|
+
"relation": {
|
|
122
|
+
"enum": [
|
|
123
|
+
"",
|
|
124
|
+
"entailed",
|
|
125
|
+
"partially_supported",
|
|
126
|
+
"contradicted",
|
|
127
|
+
"not_supported",
|
|
128
|
+
"unclear"
|
|
129
|
+
]
|
|
130
|
+
},
|
|
131
|
+
"confidence": {
|
|
132
|
+
"oneOf": [
|
|
133
|
+
{
|
|
134
|
+
"type": "number",
|
|
135
|
+
"minimum": 0,
|
|
136
|
+
"maximum": 1
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
"type": "null"
|
|
140
|
+
}
|
|
141
|
+
]
|
|
142
|
+
},
|
|
143
|
+
"rationale": {
|
|
144
|
+
"type": "string"
|
|
145
|
+
},
|
|
146
|
+
"source_quotes": {
|
|
147
|
+
"type": "array",
|
|
148
|
+
"items": {
|
|
149
|
+
"type": "string",
|
|
150
|
+
"minLength": 20
|
|
151
|
+
}
|
|
152
|
+
},
|
|
153
|
+
"missing_assumptions": {
|
|
154
|
+
"type": "array",
|
|
155
|
+
"items": {
|
|
156
|
+
"type": "string"
|
|
157
|
+
}
|
|
158
|
+
},
|
|
159
|
+
"counter_evidence_node_ids": {
|
|
160
|
+
"type": "array",
|
|
161
|
+
"items": {
|
|
162
|
+
"type": "string",
|
|
163
|
+
"pattern": "^evidence:[a-f0-9]{16}$"
|
|
164
|
+
},
|
|
165
|
+
"uniqueItems": true
|
|
166
|
+
},
|
|
167
|
+
"reviewer_selected_evidence_node_ids": {
|
|
168
|
+
"type": "array",
|
|
169
|
+
"items": {
|
|
170
|
+
"type": "string",
|
|
171
|
+
"pattern": "^evidence:[a-f0-9]{16}$"
|
|
172
|
+
},
|
|
173
|
+
"uniqueItems": true
|
|
174
|
+
}
|
|
175
|
+
},
|
|
176
|
+
"additionalProperties": false
|
|
177
|
+
}
|
|
178
|
+
},
|
|
179
|
+
"evidence_catalog": {
|
|
180
|
+
"type": "array",
|
|
181
|
+
"items": {
|
|
182
|
+
"type": "object",
|
|
183
|
+
"required": [
|
|
184
|
+
"node_id",
|
|
185
|
+
"text",
|
|
186
|
+
"source_location"
|
|
187
|
+
],
|
|
188
|
+
"properties": {
|
|
189
|
+
"node_id": {
|
|
190
|
+
"type": "string",
|
|
191
|
+
"pattern": "^evidence:[a-f0-9]{16}$"
|
|
192
|
+
},
|
|
193
|
+
"text": {
|
|
194
|
+
"type": "string",
|
|
195
|
+
"minLength": 1
|
|
196
|
+
},
|
|
197
|
+
"source_location": {
|
|
198
|
+
"type": "object",
|
|
199
|
+
"required": [
|
|
200
|
+
"line",
|
|
201
|
+
"section"
|
|
202
|
+
],
|
|
203
|
+
"properties": {
|
|
204
|
+
"line": {
|
|
205
|
+
"type": [
|
|
206
|
+
"integer",
|
|
207
|
+
"null"
|
|
208
|
+
],
|
|
209
|
+
"minimum": 1
|
|
210
|
+
},
|
|
211
|
+
"section": {
|
|
212
|
+
"type": [
|
|
213
|
+
"string",
|
|
214
|
+
"null"
|
|
215
|
+
]
|
|
216
|
+
}
|
|
217
|
+
},
|
|
218
|
+
"additionalProperties": false
|
|
219
|
+
}
|
|
220
|
+
},
|
|
221
|
+
"additionalProperties": false
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
},
|
|
225
|
+
"additionalProperties": false
|
|
226
|
+
}
|