article-q 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {article_q-0.2.1 → article_q-0.2.2}/.gitignore +3 -0
- {article_q-0.2.1 → article_q-0.2.2}/PKG-INFO +60 -6
- {article_q-0.2.1 → article_q-0.2.2}/README.md +59 -5
- {article_q-0.2.1 → article_q-0.2.2}/articleq.example.toml +3 -1
- {article_q-0.2.1 → article_q-0.2.2}/pyproject.toml +12 -1
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/agents/consensus.py +3 -1
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/agents/prompts.py +38 -0
- article_q-0.2.2/src/articleq/agents/synthesis.py +47 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/cli.py +59 -3
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/config.py +8 -4
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/models/config.py +3 -1
- article_q-0.2.2/src/articleq/models/synthesis.py +52 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/output/tabular_output.py +39 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/parsing/questions.py +51 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/pipeline/orchestrator.py +1 -0
- article_q-0.2.2/src/articleq/pipeline/synthesis.py +146 -0
- article_q-0.2.1/.claude/settings.local.json +0 -8
- article_q-0.2.1/PLAN.md +0 -80
- article_q-0.2.1/docs/assets/article-q-logo-old.png +0 -0
- article_q-0.2.1/docs/assets/article-q-logo-old2.png +0 -0
- article_q-0.2.1/docs/assets/article-q-logo.png +0 -0
- article_q-0.2.1/docs/assets/extra.css +0 -6
- article_q-0.2.1/docs/getting-started.md +0 -152
- article_q-0.2.1/docs/index.md +0 -66
- article_q-0.2.1/docs/reference/api.md +0 -19
- article_q-0.2.1/docs/reference/cli.md +0 -112
- article_q-0.2.1/docs/user-guide/configuration.md +0 -79
- article_q-0.2.1/docs/user-guide/evaluation.md +0 -58
- article_q-0.2.1/docs/user-guide/questions-file.md +0 -54
- article_q-0.2.1/docs/user-guide/visualization.md +0 -77
- article_q-0.2.1/mkdocs.yml +0 -83
- article_q-0.2.1/overrides/main.html +0 -6
- article_q-0.2.1/tests/__init__.py +0 -0
- article_q-0.2.1/tests/conftest.py +0 -117
- article_q-0.2.1/tests/fixtures/questions.csv +0 -5
- article_q-0.2.1/tests/test_dependencies.py +0 -345
- article_q-0.2.1/uv.lock +0 -4030
- {article_q-0.2.1 → article_q-0.2.2}/LICENSE +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/__init__.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/__main__.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/agents/__init__.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/agents/extraction.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/agents/provider.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/agents/validation.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/eval.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/models/__init__.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/models/answer.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/models/paper.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/models/question.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/models/review.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/output/__init__.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/output/json_output.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/output/visualizer.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/parsing/__init__.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/parsing/base.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/parsing/marker_parser.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/parsing/pymupdf_parser.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/pipeline/__init__.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/pipeline/progress.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/pipeline/scheduler.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/utils/__init__.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/utils/chunking.py +0 -0
- {article_q-0.2.1 → article_q-0.2.2}/src/articleq/utils/logging.py +0 -0
|
@@ -188,6 +188,9 @@ cython_debug/
|
|
|
188
188
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
189
189
|
# .idea/
|
|
190
190
|
|
|
191
|
+
# Claude Code
|
|
192
|
+
.claude/
|
|
193
|
+
|
|
191
194
|
# Abstra
|
|
192
195
|
# Abstra is an AI-powered process automation framework.
|
|
193
196
|
# Ignore directories containing user credentials, local state, and settings.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: article-q
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Agent/LLM-enabled narrative reviews of academic manuscripts
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -24,10 +24,22 @@ Agent/LLM-enabled narrative reviews of academic manuscripts. Parses PDFs, extrac
|
|
|
24
24
|
|
|
25
25
|
## Installation
|
|
26
26
|
|
|
27
|
-
Requires Python 3.11
|
|
27
|
+
Requires Python 3.11+ and [uv](https://docs.astral.sh/uv/getting-started/installation/) (recommended).
|
|
28
28
|
|
|
29
29
|
```bash
|
|
30
|
-
pip install -
|
|
30
|
+
uv pip install article-q
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Or with pip:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install article-q
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
For development, clone the repository and install in editable mode:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
uv pip install -e .
|
|
31
43
|
```
|
|
32
44
|
|
|
33
45
|
## Step-by-step workflow
|
|
@@ -151,6 +163,44 @@ articleq visualize -r output/results.json
|
|
|
151
163
|
|
|
152
164
|
The viewer shows each paper's content alongside extracted answers, with evidence passages highlighted in the text. Pass `-q questions.csv` to include question text in the results panel.
|
|
153
165
|
|
|
166
|
+
### Step 7: Synthesize findings across papers (optional)
|
|
167
|
+
|
|
168
|
+
After extraction, you can synthesize findings across all papers to answer higher-level research questions. Create a synthesis questions CSV:
|
|
169
|
+
|
|
170
|
+
```csv
|
|
171
|
+
id,question,source_questions,description
|
|
172
|
+
overall_findings,"What are the overall findings regarding primary outcomes across the included studies?","primary_outcome,sample_size",
|
|
173
|
+
study_designs,"Summarize the study designs used across the included studies.","study_design",
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
| Column | Required | Description |
|
|
177
|
+
|---|---|---|
|
|
178
|
+
| `id` | yes | Unique synthesis question identifier |
|
|
179
|
+
| `question` | yes | The synthesis question text |
|
|
180
|
+
| `source_questions` | no | Comma-separated extraction question IDs to include (all if empty) |
|
|
181
|
+
| `description` | no | Additional guidance for the LLM |
|
|
182
|
+
|
|
183
|
+
Set `synthesis_questions_file` in your config:
|
|
184
|
+
|
|
185
|
+
```toml
|
|
186
|
+
[project]
|
|
187
|
+
synthesis_questions_file = "./synthesis_questions.csv"
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
Then run synthesis:
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
articleq synthesize -c articleq.toml
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
This reads the extraction results from `output/results.json`, runs the synthesis agent for each question, and writes `output/synthesis.json`. Export to CSV:
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
articleq export output/synthesis.json --type synthesis --format csv
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
The synthesis agent sees only the tabulated extraction data (values, confidence scores, evidence quotes) — not full paper text. It produces a narrative synthesis with key findings, citing specific studies.
|
|
203
|
+
|
|
154
204
|
## How it works
|
|
155
205
|
|
|
156
206
|
Each question for each paper goes through a three-agent workflow:
|
|
@@ -205,6 +255,7 @@ papers_dir = "./papers" # Directory containing PDF files
|
|
|
205
255
|
questions_file = "./questions.csv" # Path to questions CSV/Excel
|
|
206
256
|
output_dir = "./output" # Where results are written
|
|
207
257
|
# context_file = "./context.md" # Optional: additional instructions for the LLM
|
|
258
|
+
# synthesis_questions_file = "./synthesis_questions.csv" # Optional: cross-study synthesis questions
|
|
208
259
|
|
|
209
260
|
[parsing]
|
|
210
261
|
backend = "pymupdf" # "pymupdf" or "marker"
|
|
@@ -214,6 +265,7 @@ reparse = false # Force re-parsing even if cached results exist
|
|
|
214
265
|
extraction_model = "openai:gpt-4o" # Model for primary extraction
|
|
215
266
|
validation_model = "openai:gpt-4o" # Model for validation pass
|
|
216
267
|
consensus_model = "openai:gpt-4o" # Model for arbitration
|
|
268
|
+
synthesis_model = "openai:gpt-4o" # Model for cross-study synthesis
|
|
217
269
|
# temperature = 0.0 # LLM sampling temperature (omit to use provider default)
|
|
218
270
|
|
|
219
271
|
[llm.api_keys]
|
|
@@ -228,7 +280,7 @@ chunk_max_tokens = 8000 # Max tokens per chunk for large PDFs
|
|
|
228
280
|
|
|
229
281
|
[validation]
|
|
230
282
|
auto_accept_threshold = 0.9 # Min confidence to auto-accept agreement
|
|
231
|
-
|
|
283
|
+
always_validate_questions = ["primary_outcome"] # Always run full 3-agent flow for these
|
|
232
284
|
```
|
|
233
285
|
|
|
234
286
|
## Additional topics
|
|
@@ -258,7 +310,8 @@ output/
|
|
|
258
310
|
│ ├── study_smith_2020_img1.png
|
|
259
311
|
│ ├── study_smith_2020_img2.png
|
|
260
312
|
│ └── study_jones_2021_img1.png
|
|
261
|
-
|
|
313
|
+
├── results.json
|
|
314
|
+
└── synthesis.json # (after running synthesize)
|
|
262
315
|
```
|
|
263
316
|
|
|
264
317
|
### Context file
|
|
@@ -387,7 +440,8 @@ articleq run -c CONFIG [--reparse] Run the full pipeline (parse + extract)
|
|
|
387
440
|
articleq parse -c CONFIG [--reparse] Parse PDFs and save as markdown (no LLM calls)
|
|
388
441
|
articleq rebuild -c CONFIG Rebuild markdown and JSON from edited blocks
|
|
389
442
|
articleq extract -c CONFIG Run LLM extraction on pre-parsed papers
|
|
390
|
-
articleq
|
|
443
|
+
articleq synthesize -c CONFIG Synthesize findings across papers
|
|
444
|
+
articleq export RESULTS [--format csv|excel] [-o PATH] [--type extraction|synthesis] Export to CSV/Excel
|
|
391
445
|
articleq eval RESULTS EXPECTED [-q QUESTIONS] [--judge-model MODEL] [-c CONFIG] Evaluate against ground truth
|
|
392
446
|
articleq visualize -r RESULTS [-o PATH] [-q QUESTIONS] Generate HTML evidence viewer
|
|
393
447
|
articleq visualize --parsed-dir DIR [-o PATH] Preview parsed papers (no results)
|
|
@@ -4,10 +4,22 @@ Agent/LLM-enabled narrative reviews of academic manuscripts. Parses PDFs, extrac
|
|
|
4
4
|
|
|
5
5
|
## Installation
|
|
6
6
|
|
|
7
|
-
Requires Python 3.11
|
|
7
|
+
Requires Python 3.11+ and [uv](https://docs.astral.sh/uv/getting-started/installation/) (recommended).
|
|
8
8
|
|
|
9
9
|
```bash
|
|
10
|
-
pip install -
|
|
10
|
+
uv pip install article-q
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Or with pip:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install article-q
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
For development, clone the repository and install in editable mode:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
uv pip install -e .
|
|
11
23
|
```
|
|
12
24
|
|
|
13
25
|
## Step-by-step workflow
|
|
@@ -131,6 +143,44 @@ articleq visualize -r output/results.json
|
|
|
131
143
|
|
|
132
144
|
The viewer shows each paper's content alongside extracted answers, with evidence passages highlighted in the text. Pass `-q questions.csv` to include question text in the results panel.
|
|
133
145
|
|
|
146
|
+
### Step 7: Synthesize findings across papers (optional)
|
|
147
|
+
|
|
148
|
+
After extraction, you can synthesize findings across all papers to answer higher-level research questions. Create a synthesis questions CSV:
|
|
149
|
+
|
|
150
|
+
```csv
|
|
151
|
+
id,question,source_questions,description
|
|
152
|
+
overall_findings,"What are the overall findings regarding primary outcomes across the included studies?","primary_outcome,sample_size",
|
|
153
|
+
study_designs,"Summarize the study designs used across the included studies.","study_design",
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
| Column | Required | Description |
|
|
157
|
+
|---|---|---|
|
|
158
|
+
| `id` | yes | Unique synthesis question identifier |
|
|
159
|
+
| `question` | yes | The synthesis question text |
|
|
160
|
+
| `source_questions` | no | Comma-separated extraction question IDs to include (all if empty) |
|
|
161
|
+
| `description` | no | Additional guidance for the LLM |
|
|
162
|
+
|
|
163
|
+
Set `synthesis_questions_file` in your config:
|
|
164
|
+
|
|
165
|
+
```toml
|
|
166
|
+
[project]
|
|
167
|
+
synthesis_questions_file = "./synthesis_questions.csv"
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Then run synthesis:
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
articleq synthesize -c articleq.toml
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
This reads the extraction results from `output/results.json`, runs the synthesis agent for each question, and writes `output/synthesis.json`. Export to CSV:
|
|
177
|
+
|
|
178
|
+
```bash
|
|
179
|
+
articleq export output/synthesis.json --type synthesis --format csv
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
The synthesis agent sees only the tabulated extraction data (values, confidence scores, evidence quotes) — not full paper text. It produces a narrative synthesis with key findings, citing specific studies.
|
|
183
|
+
|
|
134
184
|
## How it works
|
|
135
185
|
|
|
136
186
|
Each question for each paper goes through a three-agent workflow:
|
|
@@ -185,6 +235,7 @@ papers_dir = "./papers" # Directory containing PDF files
|
|
|
185
235
|
questions_file = "./questions.csv" # Path to questions CSV/Excel
|
|
186
236
|
output_dir = "./output" # Where results are written
|
|
187
237
|
# context_file = "./context.md" # Optional: additional instructions for the LLM
|
|
238
|
+
# synthesis_questions_file = "./synthesis_questions.csv" # Optional: cross-study synthesis questions
|
|
188
239
|
|
|
189
240
|
[parsing]
|
|
190
241
|
backend = "pymupdf" # "pymupdf" or "marker"
|
|
@@ -194,6 +245,7 @@ reparse = false # Force re-parsing even if cached results exist
|
|
|
194
245
|
extraction_model = "openai:gpt-4o" # Model for primary extraction
|
|
195
246
|
validation_model = "openai:gpt-4o" # Model for validation pass
|
|
196
247
|
consensus_model = "openai:gpt-4o" # Model for arbitration
|
|
248
|
+
synthesis_model = "openai:gpt-4o" # Model for cross-study synthesis
|
|
197
249
|
# temperature = 0.0 # LLM sampling temperature (omit to use provider default)
|
|
198
250
|
|
|
199
251
|
[llm.api_keys]
|
|
@@ -208,7 +260,7 @@ chunk_max_tokens = 8000 # Max tokens per chunk for large PDFs
|
|
|
208
260
|
|
|
209
261
|
[validation]
|
|
210
262
|
auto_accept_threshold = 0.9 # Min confidence to auto-accept agreement
|
|
211
|
-
|
|
263
|
+
always_validate_questions = ["primary_outcome"] # Always run full 3-agent flow for these
|
|
212
264
|
```
|
|
213
265
|
|
|
214
266
|
## Additional topics
|
|
@@ -238,7 +290,8 @@ output/
|
|
|
238
290
|
│ ├── study_smith_2020_img1.png
|
|
239
291
|
│ ├── study_smith_2020_img2.png
|
|
240
292
|
│ └── study_jones_2021_img1.png
|
|
241
|
-
|
|
293
|
+
├── results.json
|
|
294
|
+
└── synthesis.json # (after running synthesize)
|
|
242
295
|
```
|
|
243
296
|
|
|
244
297
|
### Context file
|
|
@@ -367,7 +420,8 @@ articleq run -c CONFIG [--reparse] Run the full pipeline (parse + extract)
|
|
|
367
420
|
articleq parse -c CONFIG [--reparse] Parse PDFs and save as markdown (no LLM calls)
|
|
368
421
|
articleq rebuild -c CONFIG Rebuild markdown and JSON from edited blocks
|
|
369
422
|
articleq extract -c CONFIG Run LLM extraction on pre-parsed papers
|
|
370
|
-
articleq
|
|
423
|
+
articleq synthesize -c CONFIG Synthesize findings across papers
|
|
424
|
+
articleq export RESULTS [--format csv|excel] [-o PATH] [--type extraction|synthesis] Export to CSV/Excel
|
|
371
425
|
articleq eval RESULTS EXPECTED [-q QUESTIONS] [--judge-model MODEL] [-c CONFIG] Evaluate against ground truth
|
|
372
426
|
articleq visualize -r RESULTS [-o PATH] [-q QUESTIONS] Generate HTML evidence viewer
|
|
373
427
|
articleq visualize --parsed-dir DIR [-o PATH] Preview parsed papers (no results)
|
|
@@ -4,6 +4,7 @@ papers_dir = "./papers"
|
|
|
4
4
|
questions_file = "./questions.csv"
|
|
5
5
|
output_dir = "./output"
|
|
6
6
|
context_file = "./context.md"
|
|
7
|
+
# synthesis_questions_file = "./synthesis_questions.csv"
|
|
7
8
|
|
|
8
9
|
[parsing]
|
|
9
10
|
backend = "pymupdf" # or "marker"
|
|
@@ -12,6 +13,7 @@ backend = "pymupdf" # or "marker"
|
|
|
12
13
|
extraction_model = "openai:gpt-4o"
|
|
13
14
|
validation_model = "openai:gpt-4o"
|
|
14
15
|
consensus_model = "openai:gpt-4o"
|
|
16
|
+
synthesis_model = "openai:gpt-4o"
|
|
15
17
|
|
|
16
18
|
[llm.api_keys]
|
|
17
19
|
openai = "${OPENAI_API_KEY}"
|
|
@@ -25,4 +27,4 @@ chunk_max_tokens = 8000
|
|
|
25
27
|
|
|
26
28
|
[validation]
|
|
27
29
|
auto_accept_threshold = 0.9
|
|
28
|
-
|
|
30
|
+
always_validate_questions = ["primary_outcome"]
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "article-q"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "Agent/LLM-enabled narrative reviews of academic manuscripts"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -44,5 +44,16 @@ docs = [
|
|
|
44
44
|
[project.scripts]
|
|
45
45
|
articleq = "articleq.cli:app"
|
|
46
46
|
|
|
47
|
+
[tool.hatch.build.targets.sdist]
|
|
48
|
+
exclude = [
|
|
49
|
+
"benchmarks",
|
|
50
|
+
".github",
|
|
51
|
+
"docs",
|
|
52
|
+
"overrides",
|
|
53
|
+
"mkdocs.yml",
|
|
54
|
+
"tests",
|
|
55
|
+
"uv.lock",
|
|
56
|
+
]
|
|
57
|
+
|
|
47
58
|
[tool.hatch.build.targets.wheel]
|
|
48
59
|
packages = ["src/articleq"]
|
|
@@ -101,6 +101,7 @@ async def resolve_consensus(
|
|
|
101
101
|
context: str | None = None,
|
|
102
102
|
model_settings: ModelSettings | None = None,
|
|
103
103
|
usage: RunUsage | None = None,
|
|
104
|
+
force_consensus_ids: set[str] | None = None,
|
|
104
105
|
) -> ConsensusResult:
|
|
105
106
|
"""Run the full consensus workflow for a question.
|
|
106
107
|
|
|
@@ -111,7 +112,8 @@ async def resolve_consensus(
|
|
|
111
112
|
|
|
112
113
|
agreed = _answers_agree(answer_a, answer_b, question)
|
|
113
114
|
|
|
114
|
-
|
|
115
|
+
force = force_consensus_ids and question.id in force_consensus_ids
|
|
116
|
+
if agreed and not force and min(answer_a.confidence, answer_b.confidence) >= auto_accept_threshold:
|
|
115
117
|
return ConsensusResult(
|
|
116
118
|
question_id=question.id,
|
|
117
119
|
final_value=answer_a.value,
|
|
@@ -60,6 +60,44 @@ regardless of which agent provided it.
|
|
|
60
60
|
"""
|
|
61
61
|
|
|
62
62
|
|
|
63
|
+
SYNTHESIS_SYSTEM_PROMPT = """\
|
|
64
|
+
You are an experienced systematic review methodologist performing evidence \
|
|
65
|
+
synthesis across multiple studies. Your task is to synthesize extracted data \
|
|
66
|
+
from individual papers to answer a cross-study research question.
|
|
67
|
+
|
|
68
|
+
Guidelines:
|
|
69
|
+
- Cite specific studies by filename when making claims
|
|
70
|
+
- Report counts, ranges, and proportions where appropriate (e.g. "3 of 5 studies \
|
|
71
|
+
reported...", "sample sizes ranged from 50 to 500")
|
|
72
|
+
- Note heterogeneity across studies when present
|
|
73
|
+
- Flag limitations such as small sample sizes, inconsistent reporting, or missing data
|
|
74
|
+
- Distinguish between what the evidence shows versus what it does not address
|
|
75
|
+
- Organize your narrative logically with clear topic sentences
|
|
76
|
+
- Provide bullet-point key findings that summarize the most important points
|
|
77
|
+
- Rate your confidence based on the consistency, quality, and completeness of the \
|
|
78
|
+
extracted data
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def build_synthesis_user_prompt(
|
|
83
|
+
question: str,
|
|
84
|
+
description: str,
|
|
85
|
+
extracted_data_table: str,
|
|
86
|
+
) -> str:
|
|
87
|
+
"""Build the user prompt for the synthesis agent.
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
question: The synthesis question text.
|
|
91
|
+
description: Additional guidance for the synthesis.
|
|
92
|
+
extracted_data_table: Markdown table of extracted data across papers.
|
|
93
|
+
"""
|
|
94
|
+
parts = [f"## Synthesis Question\n{question}"]
|
|
95
|
+
if description:
|
|
96
|
+
parts.append(f"\n## Additional Guidance\n{description}")
|
|
97
|
+
parts.append(f"\n## Extracted Data\n{extracted_data_table}")
|
|
98
|
+
return "\n".join(parts)
|
|
99
|
+
|
|
100
|
+
|
|
63
101
|
def _append_images(
|
|
64
102
|
text: str,
|
|
65
103
|
images: list[tuple[str, bytes]] | None,
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Synthesis agent for cross-study evidence synthesis."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pydantic_ai import Agent
|
|
6
|
+
from pydantic_ai.settings import ModelSettings
|
|
7
|
+
from pydantic_ai.usage import RunUsage, UsageLimits
|
|
8
|
+
|
|
9
|
+
from articleq.agents.prompts import SYNTHESIS_SYSTEM_PROMPT, build_synthesis_user_prompt
|
|
10
|
+
from articleq.models.synthesis import SynthesisResult
|
|
11
|
+
|
|
12
|
+
synthesis_agent = Agent(
|
|
13
|
+
system_prompt=SYNTHESIS_SYSTEM_PROMPT,
|
|
14
|
+
output_type=SynthesisResult,
|
|
15
|
+
defer_model_check=True,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
async def synthesize(
|
|
20
|
+
question_id: str,
|
|
21
|
+
question_text: str,
|
|
22
|
+
description: str,
|
|
23
|
+
extracted_data_table: str,
|
|
24
|
+
model: str,
|
|
25
|
+
context: str | None = None,
|
|
26
|
+
model_settings: ModelSettings | None = None,
|
|
27
|
+
usage: RunUsage | None = None,
|
|
28
|
+
) -> SynthesisResult:
|
|
29
|
+
"""Run the synthesis agent for a single synthesis question."""
|
|
30
|
+
user_prompt = build_synthesis_user_prompt(
|
|
31
|
+
question=question_text,
|
|
32
|
+
description=description,
|
|
33
|
+
extracted_data_table=extracted_data_table,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
result = await synthesis_agent.run(
|
|
37
|
+
user_prompt,
|
|
38
|
+
model=model,
|
|
39
|
+
instructions=context,
|
|
40
|
+
model_settings=model_settings,
|
|
41
|
+
usage=usage,
|
|
42
|
+
usage_limits=UsageLimits(request_limit=None),
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
answer = result.output
|
|
46
|
+
answer.question_id = question_id
|
|
47
|
+
return answer
|
|
@@ -154,6 +154,53 @@ def extract(
|
|
|
154
154
|
asyncio.run(run_extraction(cfg, papers))
|
|
155
155
|
|
|
156
156
|
|
|
157
|
+
@app.command()
|
|
158
|
+
def synthesize(
|
|
159
|
+
config: Annotated[
|
|
160
|
+
Path, typer.Option("--config", "-c", help="Path to config TOML file")
|
|
161
|
+
] = Path("articleq.toml"),
|
|
162
|
+
) -> None:
|
|
163
|
+
"""Synthesize findings across papers from extraction results."""
|
|
164
|
+
from articleq.config import load_config
|
|
165
|
+
from articleq.output.json_output import load_results
|
|
166
|
+
from articleq.parsing.questions import load_synthesis_questions
|
|
167
|
+
from articleq.pipeline.synthesis import run_synthesis
|
|
168
|
+
|
|
169
|
+
if not config.exists():
|
|
170
|
+
console.print(f"[red]Config file not found: {config}[/red]")
|
|
171
|
+
console.print("Run 'articleq init' to create one.")
|
|
172
|
+
raise typer.Exit(1)
|
|
173
|
+
|
|
174
|
+
cfg = load_config(config)
|
|
175
|
+
|
|
176
|
+
if not cfg.project.synthesis_questions_file:
|
|
177
|
+
console.print("[red]No synthesis_questions_file set in config.[/red]")
|
|
178
|
+
console.print("Add 'synthesis_questions_file' to [project] in your config file.")
|
|
179
|
+
raise typer.Exit(1)
|
|
180
|
+
|
|
181
|
+
if not cfg.project.synthesis_questions_file.exists():
|
|
182
|
+
console.print(f"[red]Synthesis questions file not found: {cfg.project.synthesis_questions_file}[/red]")
|
|
183
|
+
raise typer.Exit(1)
|
|
184
|
+
|
|
185
|
+
results_path = cfg.project.output_dir / "results.json"
|
|
186
|
+
if not results_path.exists():
|
|
187
|
+
console.print(f"[red]Results file not found: {results_path}[/red]")
|
|
188
|
+
console.print("Run 'articleq extract' or 'articleq run' first.")
|
|
189
|
+
raise typer.Exit(1)
|
|
190
|
+
|
|
191
|
+
batch = load_results(results_path)
|
|
192
|
+
synthesis_questions = load_synthesis_questions(cfg.project.synthesis_questions_file)
|
|
193
|
+
console.print(f"Loaded {len(synthesis_questions)} synthesis questions")
|
|
194
|
+
|
|
195
|
+
synthesis_batch = asyncio.run(run_synthesis(cfg, batch, synthesis_questions))
|
|
196
|
+
|
|
197
|
+
# Write synthesis results
|
|
198
|
+
output_path = cfg.project.output_dir / "synthesis.json"
|
|
199
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
200
|
+
output_path.write_text(synthesis_batch.model_dump_json(indent=2))
|
|
201
|
+
console.print(f"[green]Synthesis results written to {output_path}[/green]")
|
|
202
|
+
|
|
203
|
+
|
|
157
204
|
@app.command()
|
|
158
205
|
def export(
|
|
159
206
|
results_file: Annotated[Path, typer.Argument(help="Path to results JSON file")],
|
|
@@ -163,15 +210,24 @@ def export(
|
|
|
163
210
|
output: Annotated[
|
|
164
211
|
Optional[Path], typer.Option("--output", "-o", help="Output file path")
|
|
165
212
|
] = None,
|
|
213
|
+
type: Annotated[
|
|
214
|
+
str, typer.Option("--type", "-t", help="Result type: extraction or synthesis")
|
|
215
|
+
] = "extraction",
|
|
166
216
|
) -> None:
|
|
167
217
|
"""Export results JSON to CSV or Excel."""
|
|
168
|
-
from articleq.output.tabular_output import export_results
|
|
169
|
-
|
|
170
218
|
if not results_file.exists():
|
|
171
219
|
console.print(f"[red]Results file not found: {results_file}[/red]")
|
|
172
220
|
raise typer.Exit(1)
|
|
173
221
|
|
|
174
|
-
|
|
222
|
+
if type == "synthesis":
|
|
223
|
+
from articleq.output.tabular_output import export_synthesis
|
|
224
|
+
|
|
225
|
+
out_path = export_synthesis(results_file, format=format, output_path=output)
|
|
226
|
+
else:
|
|
227
|
+
from articleq.output.tabular_output import export_results
|
|
228
|
+
|
|
229
|
+
out_path = export_results(results_file, format=format, output_path=output)
|
|
230
|
+
|
|
175
231
|
console.print(f"[green]Exported to {out_path}[/green]")
|
|
176
232
|
|
|
177
233
|
|
|
@@ -44,6 +44,8 @@ def _resolve_paths(config: AppConfig, config_dir: Path) -> None:
|
|
|
44
44
|
setattr(project, attr, (config_dir / p).resolve())
|
|
45
45
|
if project.context_file and not project.context_file.is_absolute():
|
|
46
46
|
project.context_file = (config_dir / project.context_file).resolve()
|
|
47
|
+
if project.synthesis_questions_file and not project.synthesis_questions_file.is_absolute():
|
|
48
|
+
project.synthesis_questions_file = (config_dir / project.synthesis_questions_file).resolve()
|
|
47
49
|
|
|
48
50
|
|
|
49
51
|
def load_config(path: Path) -> AppConfig:
|
|
@@ -72,6 +74,7 @@ def write_example_config(path: Path) -> None:
|
|
|
72
74
|
"extraction_model": config.llm.extraction_model,
|
|
73
75
|
"validation_model": config.llm.validation_model,
|
|
74
76
|
"consensus_model": config.llm.consensus_model,
|
|
77
|
+
"synthesis_model": config.llm.synthesis_model,
|
|
75
78
|
"api_keys": {
|
|
76
79
|
"openai": "${OPENAI_API_KEY}",
|
|
77
80
|
"google": "${GEMINI_API_KEY}",
|
|
@@ -85,7 +88,7 @@ def write_example_config(path: Path) -> None:
|
|
|
85
88
|
},
|
|
86
89
|
"validation": {
|
|
87
90
|
"auto_accept_threshold": config.validation.auto_accept_threshold,
|
|
88
|
-
"
|
|
91
|
+
"always_validate_questions": config.validation.always_validate_questions,
|
|
89
92
|
},
|
|
90
93
|
}
|
|
91
94
|
with open(path, "wb") as f:
|
|
@@ -93,8 +96,9 @@ def write_example_config(path: Path) -> None:
|
|
|
93
96
|
# Insert commented-out context_file after output_dir (tomli_w doesn't support comments)
|
|
94
97
|
text = path.read_text()
|
|
95
98
|
text = text.replace(
|
|
96
|
-
'output_dir = "
|
|
97
|
-
'output_dir = "
|
|
98
|
-
'# context_file = "./context.md" # optional: additional instructions for the LLM'
|
|
99
|
+
'output_dir = "output"',
|
|
100
|
+
'output_dir = "output"\n'
|
|
101
|
+
'# context_file = "./context.md" # optional: additional instructions for the LLM\n'
|
|
102
|
+
'# synthesis_questions_file = "./synthesis_questions.csv" # optional: cross-study synthesis questions',
|
|
99
103
|
)
|
|
100
104
|
path.write_text(text)
|
|
@@ -13,6 +13,7 @@ class ProjectConfig(BaseModel):
|
|
|
13
13
|
questions_file: Path = Path("./questions.csv")
|
|
14
14
|
output_dir: Path = Path("./output")
|
|
15
15
|
context_file: Path | None = None
|
|
16
|
+
synthesis_questions_file: Path | None = None
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
class ParsingConfig(BaseModel):
|
|
@@ -29,6 +30,7 @@ class LLMConfig(BaseModel):
|
|
|
29
30
|
extraction_model: str = "openai:gpt-4o"
|
|
30
31
|
validation_model: str = "openai:gpt-4o"
|
|
31
32
|
consensus_model: str = "openai:gpt-4o"
|
|
33
|
+
synthesis_model: str = "openai:gpt-4o"
|
|
32
34
|
temperature: float | None = None
|
|
33
35
|
api_keys: ApiKeysConfig = Field(default_factory=ApiKeysConfig)
|
|
34
36
|
|
|
@@ -42,7 +44,7 @@ class PipelineConfig(BaseModel):
|
|
|
42
44
|
|
|
43
45
|
class ValidationConfig(BaseModel):
|
|
44
46
|
auto_accept_threshold: float = 0.9
|
|
45
|
-
|
|
47
|
+
always_validate_questions: list[str] = Field(default_factory=list)
|
|
46
48
|
|
|
47
49
|
|
|
48
50
|
class AppConfig(BaseModel):
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Models for cross-study synthesis results."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel, Field
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class SynthesisQuestion(BaseModel):
|
|
11
|
+
"""A question for cross-study evidence synthesis."""
|
|
12
|
+
|
|
13
|
+
id: str
|
|
14
|
+
question: str
|
|
15
|
+
source_questions: list[str] = Field(default_factory=list)
|
|
16
|
+
description: str = ""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class SourceReference(BaseModel):
|
|
20
|
+
"""A reference to an extracted value from a specific paper."""
|
|
21
|
+
|
|
22
|
+
paper: str
|
|
23
|
+
question_id: str
|
|
24
|
+
value: str
|
|
25
|
+
confidence: float = Field(ge=0.0, le=1.0)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class SynthesisResult(BaseModel):
|
|
29
|
+
"""Result of synthesizing evidence across papers for a single question."""
|
|
30
|
+
|
|
31
|
+
question_id: str
|
|
32
|
+
narrative: str = Field(description="The main synthesis text")
|
|
33
|
+
key_findings: list[str] = Field(
|
|
34
|
+
default_factory=list, description="Bullet-point findings"
|
|
35
|
+
)
|
|
36
|
+
evidence_summary: list[SourceReference] = Field(
|
|
37
|
+
default_factory=list, description="Papers and values that contributed"
|
|
38
|
+
)
|
|
39
|
+
confidence: float = Field(
|
|
40
|
+
ge=0.0, le=1.0, description="Overall synthesis confidence"
|
|
41
|
+
)
|
|
42
|
+
reasoning: str = Field(default="", description="Reasoning behind the synthesis")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class SynthesisBatch(BaseModel):
|
|
46
|
+
"""Results from synthesizing across a batch of papers."""
|
|
47
|
+
|
|
48
|
+
project_name: str
|
|
49
|
+
results: list[SynthesisResult] = Field(default_factory=list)
|
|
50
|
+
papers_included: int = 0
|
|
51
|
+
started_at: datetime = Field(default_factory=datetime.now)
|
|
52
|
+
completed_at: datetime | None = None
|
|
@@ -7,6 +7,7 @@ from pathlib import Path
|
|
|
7
7
|
import pandas as pd
|
|
8
8
|
|
|
9
9
|
from articleq.output.json_output import load_results
|
|
10
|
+
from articleq.models.synthesis import SynthesisBatch
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
def export_results(
|
|
@@ -53,3 +54,41 @@ def export_results(
|
|
|
53
54
|
df.to_csv(output_path, index=False)
|
|
54
55
|
|
|
55
56
|
return output_path
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def export_synthesis(
|
|
60
|
+
synthesis_path: Path,
|
|
61
|
+
format: str = "csv",
|
|
62
|
+
output_path: Path | None = None,
|
|
63
|
+
) -> Path:
|
|
64
|
+
"""Export synthesis results JSON to a tabular format (CSV or Excel)."""
|
|
65
|
+
synthesis_batch = SynthesisBatch.model_validate_json(synthesis_path.read_text())
|
|
66
|
+
|
|
67
|
+
rows: list[dict] = []
|
|
68
|
+
for result in synthesis_batch.results:
|
|
69
|
+
papers_cited = ", ".join(
|
|
70
|
+
sorted({ref.paper for ref in result.evidence_summary})
|
|
71
|
+
)
|
|
72
|
+
rows.append(
|
|
73
|
+
{
|
|
74
|
+
"question_id": result.question_id,
|
|
75
|
+
"narrative": result.narrative,
|
|
76
|
+
"key_findings": "; ".join(result.key_findings),
|
|
77
|
+
"papers_cited": papers_cited,
|
|
78
|
+
"confidence": result.confidence,
|
|
79
|
+
}
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
df = pd.DataFrame(rows)
|
|
83
|
+
|
|
84
|
+
if output_path is None:
|
|
85
|
+
stem = synthesis_path.stem
|
|
86
|
+
suffix = ".xlsx" if format == "excel" else ".csv"
|
|
87
|
+
output_path = synthesis_path.parent / f"{stem}{suffix}"
|
|
88
|
+
|
|
89
|
+
if format == "excel":
|
|
90
|
+
df.to_excel(output_path, index=False)
|
|
91
|
+
else:
|
|
92
|
+
df.to_csv(output_path, index=False)
|
|
93
|
+
|
|
94
|
+
return output_path
|