fastslr 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fastslr/__init__.py +6 -0
- fastslr/__main__.py +5 -0
- fastslr/app/__init__.py +1 -0
- fastslr/app/cli.py +593 -0
- fastslr/app/controller.py +805 -0
- fastslr/app/profiles.py +128 -0
- fastslr/app/tui.py +1035 -0
- fastslr/core/__init__.py +17 -0
- fastslr/core/config.py +551 -0
- fastslr/core/constants.py +64 -0
- fastslr/core/coverage.py +252 -0
- fastslr/core/default_config.json +64 -0
- fastslr/core/engine.py +467 -0
- fastslr/core/io.py +724 -0
- fastslr/core/models.py +115 -0
- fastslr/core/normalization.py +208 -0
- fastslr/core/patterns.py +336 -0
- fastslr/core/presets.py +123 -0
- fastslr/core/scoring.py +492 -0
- fastslr/i18n/__init__.py +160 -0
- fastslr/i18n/locales/en.json +73 -0
- fastslr/i18n/locales/es.json +73 -0
- fastslr/i18n/locales/pt_BR.json +73 -0
- fastslr/py.typed +0 -0
- fastslr-3.0.0.dist-info/METADATA +707 -0
- fastslr-3.0.0.dist-info/RECORD +30 -0
- fastslr-3.0.0.dist-info/WHEEL +5 -0
- fastslr-3.0.0.dist-info/entry_points.txt +2 -0
- fastslr-3.0.0.dist-info/licenses/LICENSE +21 -0
- fastslr-3.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,805 @@
|
|
|
1
|
+
"""FastSLR App Controller — shared orchestration for CLI and TUI.
|
|
2
|
+
|
|
3
|
+
This is the single point of contact between the application layer (CLI/TUI)
|
|
4
|
+
and the core engine. Neither CLI nor TUI should import from core directly.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import logging
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
import pandas as pd
|
|
16
|
+
|
|
17
|
+
from ..core.config import (
|
|
18
|
+
get_domain_blocks,
|
|
19
|
+
load_config,
|
|
20
|
+
load_global_params,
|
|
21
|
+
parse_terms_csv,
|
|
22
|
+
)
|
|
23
|
+
from ..core.constants import VERSION
|
|
24
|
+
from ..core.coverage import (
|
|
25
|
+
TermCoverageReport,
|
|
26
|
+
analyze_term_coverage,
|
|
27
|
+
export_coverage_csv,
|
|
28
|
+
format_coverage_report,
|
|
29
|
+
)
|
|
30
|
+
from ..core.engine import process_articles, resolve_field_columns, sample_articles
|
|
31
|
+
from ..core.io import (
|
|
32
|
+
build_protocol_snapshot,
|
|
33
|
+
compute_config_hash,
|
|
34
|
+
compute_file_hash,
|
|
35
|
+
export_appendix_pack,
|
|
36
|
+
export_config_audit,
|
|
37
|
+
export_protocol_snapshot,
|
|
38
|
+
export_results,
|
|
39
|
+
generate_academic_report,
|
|
40
|
+
generate_report,
|
|
41
|
+
get_export_opts,
|
|
42
|
+
load_table_safe,
|
|
43
|
+
read_result_table,
|
|
44
|
+
)
|
|
45
|
+
from ..core.normalization import NormalizationEngine
|
|
46
|
+
from ..core.patterns import precompile_patterns
|
|
47
|
+
from ..core.presets import generate_config
|
|
48
|
+
|
|
49
|
+
logger = logging.getLogger(__name__)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def get_version() -> str:
|
|
53
|
+
"""Return the application version."""
|
|
54
|
+
return VERSION
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def format_coverage(report: TermCoverageReport) -> str:
|
|
58
|
+
"""Format a coverage report for CLI/TUI display."""
|
|
59
|
+
return format_coverage_report(report)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def read_results_table(path: Path) -> pd.DataFrame:
|
|
63
|
+
"""Read a result table using the same rules as diff/export workflows."""
|
|
64
|
+
return read_result_table(path)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# ── Data classes ─────────────────────────────────────────────────────────────
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass
|
|
71
|
+
class ValidationIssue:
|
|
72
|
+
"""A configuration validation issue."""
|
|
73
|
+
|
|
74
|
+
level: str # "error", "warning"
|
|
75
|
+
message: str
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass
|
|
79
|
+
class TriageResult:
|
|
80
|
+
"""Result of a triage run."""
|
|
81
|
+
|
|
82
|
+
result_df: pd.DataFrame
|
|
83
|
+
stats: dict
|
|
84
|
+
config: dict
|
|
85
|
+
output_dir: Path
|
|
86
|
+
result_path: Path | None = None
|
|
87
|
+
artifact_paths: dict[str, Path] = field(default_factory=dict)
|
|
88
|
+
academic_package_path: Path | None = None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclass
|
|
92
|
+
class PreviewResult:
|
|
93
|
+
"""Result of a preview/dry-run."""
|
|
94
|
+
|
|
95
|
+
result_df: pd.DataFrame
|
|
96
|
+
stats: dict
|
|
97
|
+
sample_size: int
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass
|
|
101
|
+
class DiffEntry:
|
|
102
|
+
"""A single article that changed between two runs."""
|
|
103
|
+
|
|
104
|
+
article_id: str
|
|
105
|
+
old_decision: str
|
|
106
|
+
new_decision: str
|
|
107
|
+
old_score: float | None = None
|
|
108
|
+
new_score: float | None = None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@dataclass
|
|
112
|
+
class DiffReport:
|
|
113
|
+
"""Comparison between two triage runs."""
|
|
114
|
+
|
|
115
|
+
changed: list[DiffEntry] = field(default_factory=list)
|
|
116
|
+
total_a: int = 0
|
|
117
|
+
total_b: int = 0
|
|
118
|
+
summary: dict[str, int] = field(default_factory=dict)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@dataclass
|
|
122
|
+
class ProfileInfo:
|
|
123
|
+
"""Metadata about a saved profile."""
|
|
124
|
+
|
|
125
|
+
name: str
|
|
126
|
+
path: Path
|
|
127
|
+
description: str = ""
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@dataclass
|
|
131
|
+
class SetupInspection:
|
|
132
|
+
"""Human-readable setup inspection for a planned triage run."""
|
|
133
|
+
|
|
134
|
+
ok: bool
|
|
135
|
+
errors: list[str] = field(default_factory=list)
|
|
136
|
+
warnings: list[str] = field(default_factory=list)
|
|
137
|
+
messages: list[str] = field(default_factory=list)
|
|
138
|
+
input_rows: int | None = None
|
|
139
|
+
input_columns: list[str] = field(default_factory=list)
|
|
140
|
+
field_mapping: dict[str, str | None] = field(default_factory=dict)
|
|
141
|
+
domain_blocks: list[str] = field(default_factory=list)
|
|
142
|
+
terms_count: int | None = None
|
|
143
|
+
output_dir: Path | None = None
|
|
144
|
+
run_command: str | None = None
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
# ── Configuration & Validation ───────────────────────────────────────────────
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def validate_config(config: dict) -> list[ValidationIssue]:
|
|
151
|
+
"""Check configuration for common issues."""
|
|
152
|
+
issues: list[ValidationIssue] = []
|
|
153
|
+
|
|
154
|
+
if "global" not in config:
|
|
155
|
+
issues.append(ValidationIssue("error", "Missing 'global' section"))
|
|
156
|
+
|
|
157
|
+
domain_blocks = get_domain_blocks(config)
|
|
158
|
+
if not domain_blocks:
|
|
159
|
+
issues.append(
|
|
160
|
+
ValidationIssue(
|
|
161
|
+
"error",
|
|
162
|
+
"No domain blocks defined. Add BLOCK_ORDER in config.json or provide a terms file "
|
|
163
|
+
"with at least one non-GLOBAL block.",
|
|
164
|
+
)
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
for block_name in domain_blocks:
|
|
168
|
+
if block_name not in config:
|
|
169
|
+
issues.append(ValidationIssue("error", f"Block '{block_name}' listed but not defined"))
|
|
170
|
+
continue
|
|
171
|
+
block = config[block_name]
|
|
172
|
+
positives = block.get("positives", [])
|
|
173
|
+
if not positives:
|
|
174
|
+
issues.append(ValidationIssue("warning", f"Block '{block_name}' has no positive terms"))
|
|
175
|
+
|
|
176
|
+
global_cfg = config.get("global", {})
|
|
177
|
+
policy = global_cfg.get("DECISION_POLICY", "special")
|
|
178
|
+
if policy not in ("special", "strict", "k_of_n"):
|
|
179
|
+
issues.append(ValidationIssue("error", f"Unknown decision policy: '{policy}'"))
|
|
180
|
+
|
|
181
|
+
# Warn about nonsensical threshold values
|
|
182
|
+
for key in ("LIMITES_APROVADO", "LIMITES_SINALIZADO"):
|
|
183
|
+
thresholds = global_cfg.get(key, {})
|
|
184
|
+
for level, value in thresholds.items():
|
|
185
|
+
if value is not None and isinstance(value, (int, float)) and value < 0:
|
|
186
|
+
issues.append(
|
|
187
|
+
ValidationIssue(
|
|
188
|
+
"warning",
|
|
189
|
+
f"Negative threshold in {key}[{level}]: {value}",
|
|
190
|
+
)
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
# Surface parse errors from terms table (logic contradictions — block run)
|
|
194
|
+
for msg in config.get("_parse_errors", []):
|
|
195
|
+
issues.append(ValidationIssue("error", msg))
|
|
196
|
+
|
|
197
|
+
# Surface parse warnings from terms table
|
|
198
|
+
for msg in config.get("_parse_warnings", []):
|
|
199
|
+
issues.append(ValidationIssue("warning", msg))
|
|
200
|
+
|
|
201
|
+
return issues
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _prepare_config(config_path: Path, terms_path: Path | None = None) -> dict:
|
|
205
|
+
"""Load and prepare a full configuration (config + terms + patterns)."""
|
|
206
|
+
config = load_config(config_path)
|
|
207
|
+
|
|
208
|
+
if terms_path is not None:
|
|
209
|
+
config = parse_terms_csv(terms_path, config)
|
|
210
|
+
|
|
211
|
+
norm_rules = config.get("normalization_rules", {})
|
|
212
|
+
norm_engine = NormalizationEngine(norm_rules)
|
|
213
|
+
global_params = load_global_params(config.get("global", {}))
|
|
214
|
+
|
|
215
|
+
compile_warnings: list[str] = config.get("_parse_warnings", [])
|
|
216
|
+
|
|
217
|
+
for block_name in get_domain_blocks(config):
|
|
218
|
+
if block_name in config:
|
|
219
|
+
config[block_name] = precompile_patterns(
|
|
220
|
+
config[block_name],
|
|
221
|
+
norm_engine,
|
|
222
|
+
global_params,
|
|
223
|
+
block_name=block_name,
|
|
224
|
+
warnings=compile_warnings,
|
|
225
|
+
)
|
|
226
|
+
config[block_name]["normalization_engine"] = norm_engine
|
|
227
|
+
|
|
228
|
+
if "T0" in config:
|
|
229
|
+
config["T0"] = precompile_patterns(
|
|
230
|
+
config["T0"],
|
|
231
|
+
norm_engine,
|
|
232
|
+
global_params,
|
|
233
|
+
block_name="T0",
|
|
234
|
+
warnings=compile_warnings,
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
config["_parse_warnings"] = compile_warnings
|
|
238
|
+
|
|
239
|
+
return config
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _cmd_path(path: Path) -> str:
|
|
243
|
+
text = str(path)
|
|
244
|
+
if " " in text:
|
|
245
|
+
return f'"{text}"'
|
|
246
|
+
return text
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def inspect_run_setup(
|
|
250
|
+
input_path: Path | None = None,
|
|
251
|
+
config_path: Path | None = None,
|
|
252
|
+
terms_path: Path | None = None,
|
|
253
|
+
output_dir: Path | None = None,
|
|
254
|
+
) -> SetupInspection:
|
|
255
|
+
"""Inspect files and configuration before a run and return actionable guidance."""
|
|
256
|
+
inspection = SetupInspection(ok=True, output_dir=output_dir)
|
|
257
|
+
|
|
258
|
+
if input_path is None and config_path is None and terms_path is None:
|
|
259
|
+
inspection.messages.extend(
|
|
260
|
+
[
|
|
261
|
+
"Create a project: fastslr new-project my-review --blocks CTX,TECH,SCM",
|
|
262
|
+
"Fill terms.xlsx with block, kind, term, level, section_scope and is_regex.",
|
|
263
|
+
"Check files: fastslr doctor --input articles.csv -c config.json -t terms.xlsx",
|
|
264
|
+
"Run triage: fastslr run articles.csv -c config.json -t terms.xlsx",
|
|
265
|
+
]
|
|
266
|
+
)
|
|
267
|
+
return inspection
|
|
268
|
+
|
|
269
|
+
config: dict = {}
|
|
270
|
+
if config_path is not None:
|
|
271
|
+
if not config_path.exists():
|
|
272
|
+
inspection.errors.append(f"Config file not found: {config_path}")
|
|
273
|
+
else:
|
|
274
|
+
try:
|
|
275
|
+
config = _prepare_config(config_path, terms_path)
|
|
276
|
+
for issue in validate_config(config):
|
|
277
|
+
if issue.level == "error":
|
|
278
|
+
inspection.errors.append(issue.message)
|
|
279
|
+
else:
|
|
280
|
+
inspection.warnings.append(issue.message)
|
|
281
|
+
inspection.domain_blocks = get_domain_blocks(config)
|
|
282
|
+
inspection.terms_count = config.get("_valid_terms_count")
|
|
283
|
+
except Exception as exc:
|
|
284
|
+
inspection.errors.append(f"Could not load config/terms: {exc}")
|
|
285
|
+
|
|
286
|
+
if input_path is not None:
|
|
287
|
+
if not input_path.exists():
|
|
288
|
+
inspection.errors.append(f"Input file not found: {input_path}")
|
|
289
|
+
else:
|
|
290
|
+
try:
|
|
291
|
+
df = load_table_safe(input_path)
|
|
292
|
+
inspection.input_rows = len(df)
|
|
293
|
+
inspection.input_columns = [str(c) for c in df.columns]
|
|
294
|
+
fields = config.get("fields", {}) if config else {}
|
|
295
|
+
inspection.field_mapping = resolve_field_columns(df, fields)
|
|
296
|
+
|
|
297
|
+
for required in ("id", "title", "abstract"):
|
|
298
|
+
if inspection.field_mapping.get(required) is None:
|
|
299
|
+
inspection.warnings.append(
|
|
300
|
+
f"Could not identify the '{required}' column. "
|
|
301
|
+
f"Set fields.{required} in config.json."
|
|
302
|
+
)
|
|
303
|
+
except Exception as exc:
|
|
304
|
+
inspection.errors.append(f"Could not load input table: {exc}")
|
|
305
|
+
|
|
306
|
+
if terms_path is not None and not terms_path.exists():
|
|
307
|
+
inspection.errors.append(f"Terms file not found: {terms_path}")
|
|
308
|
+
|
|
309
|
+
if output_dir is None and input_path is not None:
|
|
310
|
+
inspection.output_dir = input_path.parent / "output"
|
|
311
|
+
|
|
312
|
+
if input_path is not None and config_path is not None:
|
|
313
|
+
parts = ["fastslr", "run", _cmd_path(input_path), "-c", _cmd_path(config_path)]
|
|
314
|
+
if terms_path is not None:
|
|
315
|
+
parts.extend(["-t", _cmd_path(terms_path)])
|
|
316
|
+
if output_dir is not None:
|
|
317
|
+
parts.extend(["-o", _cmd_path(output_dir)])
|
|
318
|
+
inspection.run_command = " ".join(parts)
|
|
319
|
+
|
|
320
|
+
inspection.ok = not inspection.errors
|
|
321
|
+
return inspection
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
# ── Triage ───────────────────────────────────────────────────────────────────
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def run_triage(
|
|
328
|
+
input_path: Path,
|
|
329
|
+
config_path: Path,
|
|
330
|
+
terms_path: Path | None = None,
|
|
331
|
+
output_dir: Path | None = None,
|
|
332
|
+
on_progress: Callable[[int, int], None] | None = None,
|
|
333
|
+
) -> TriageResult:
|
|
334
|
+
"""Execute a full triage run.
|
|
335
|
+
|
|
336
|
+
Raises ``ValueError`` when the input corpus has zero data rows (e.g. a
|
|
337
|
+
header-only CSV). Producing an empty academic package / report with
|
|
338
|
+
``TOTAL ARTICLES: 0`` while reporting success silently misleads a user who
|
|
339
|
+
exported the wrong corpus, so the run is aborted with a clear message
|
|
340
|
+
before any artifact is written.
|
|
341
|
+
"""
|
|
342
|
+
config = _prepare_config(config_path, terms_path)
|
|
343
|
+
df = load_table_safe(input_path)
|
|
344
|
+
|
|
345
|
+
if len(df) == 0:
|
|
346
|
+
raise ValueError(
|
|
347
|
+
f"O corpus de entrada nao contem nenhum artigo (0 linhas): "
|
|
348
|
+
f"{input_path}. Verifique se o arquivo exportado esta correto."
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
if output_dir is None:
|
|
352
|
+
output_dir = input_path.parent / "output"
|
|
353
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
354
|
+
|
|
355
|
+
result_df, stats = process_articles(df, config, on_progress=on_progress)
|
|
356
|
+
|
|
357
|
+
# Export results
|
|
358
|
+
exported_results = export_results(result_df, output_dir / "triage_results", config)
|
|
359
|
+
result_path = exported_results.get("xlsx") or exported_results.get("csv")
|
|
360
|
+
if result_path is None:
|
|
361
|
+
raise RuntimeError("No result file was exported.")
|
|
362
|
+
|
|
363
|
+
# Export report
|
|
364
|
+
report_path = output_dir / "triage_report.txt"
|
|
365
|
+
generate_report(result_df, stats, config, report_path)
|
|
366
|
+
|
|
367
|
+
# Export config audit
|
|
368
|
+
audit_path = output_dir / "config_audit.json"
|
|
369
|
+
export_config_audit(config, audit_path)
|
|
370
|
+
|
|
371
|
+
# Protocol snapshot
|
|
372
|
+
config_hash = compute_config_hash(config)
|
|
373
|
+
input_hash = compute_file_hash(str(input_path))
|
|
374
|
+
terms_hash = compute_file_hash(str(terms_path)) if terms_path else "n/a"
|
|
375
|
+
|
|
376
|
+
snapshot = build_protocol_snapshot(
|
|
377
|
+
config=config,
|
|
378
|
+
stats=stats,
|
|
379
|
+
input_path=input_path,
|
|
380
|
+
terms_path=terms_path or Path("n/a"),
|
|
381
|
+
result_path=result_path,
|
|
382
|
+
input_hash=input_hash,
|
|
383
|
+
terms_hash=terms_hash,
|
|
384
|
+
config_hash=config_hash,
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
protocol_path = output_dir / "protocol.json"
|
|
388
|
+
export_protocol_snapshot(snapshot, protocol_path)
|
|
389
|
+
|
|
390
|
+
# Academic report
|
|
391
|
+
academic_path = output_dir / "academic_report.md"
|
|
392
|
+
generate_academic_report(snapshot, academic_path)
|
|
393
|
+
|
|
394
|
+
artifact_paths: dict[str, Path] = {
|
|
395
|
+
**{f"results_{fmt}": path for fmt, path in exported_results.items()},
|
|
396
|
+
"report": report_path,
|
|
397
|
+
"audit": audit_path,
|
|
398
|
+
"protocol": protocol_path,
|
|
399
|
+
"academic_report": academic_path,
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
academic_package_path: Path | None = None
|
|
403
|
+
if get_export_opts(config)["academic_package"]:
|
|
404
|
+
academic_package_path = output_dir / "academic_package.zip"
|
|
405
|
+
export_appendix_pack(artifact_paths, academic_package_path, snapshot["execution_id"])
|
|
406
|
+
artifact_paths["academic_package"] = academic_package_path
|
|
407
|
+
|
|
408
|
+
return TriageResult(
|
|
409
|
+
result_df=result_df,
|
|
410
|
+
stats=stats,
|
|
411
|
+
config=config,
|
|
412
|
+
output_dir=output_dir,
|
|
413
|
+
result_path=result_path,
|
|
414
|
+
artifact_paths=artifact_paths,
|
|
415
|
+
academic_package_path=academic_package_path,
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def preview_triage(
|
|
420
|
+
input_path: Path,
|
|
421
|
+
config_path: Path,
|
|
422
|
+
terms_path: Path | None = None,
|
|
423
|
+
sample_size: int = 50,
|
|
424
|
+
seed: int | None = 42,
|
|
425
|
+
) -> PreviewResult:
|
|
426
|
+
"""Run triage on a sample for validation."""
|
|
427
|
+
config = _prepare_config(config_path, terms_path)
|
|
428
|
+
df = load_table_safe(input_path)
|
|
429
|
+
sample_df = sample_articles(df, sample_size, seed=seed)
|
|
430
|
+
result_df, stats = process_articles(sample_df, config)
|
|
431
|
+
return PreviewResult(
|
|
432
|
+
result_df=result_df,
|
|
433
|
+
stats=stats,
|
|
434
|
+
sample_size=len(sample_df),
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
# ── Coverage ─────────────────────────────────────────────────────────────────
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def analyze_coverage(
|
|
442
|
+
input_path: Path,
|
|
443
|
+
config_path: Path,
|
|
444
|
+
terms_path: Path | None = None,
|
|
445
|
+
output_path: Path | None = None,
|
|
446
|
+
) -> TermCoverageReport:
|
|
447
|
+
"""Run triage and analyze term coverage."""
|
|
448
|
+
config = _prepare_config(config_path, terms_path)
|
|
449
|
+
df = load_table_safe(input_path)
|
|
450
|
+
result_df, _ = process_articles(df, config)
|
|
451
|
+
report = analyze_term_coverage(result_df, config)
|
|
452
|
+
|
|
453
|
+
if output_path:
|
|
454
|
+
export_coverage_csv(report, output_path)
|
|
455
|
+
|
|
456
|
+
return report
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
# ── Diff ─────────────────────────────────────────────────────────────────────
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def diff_results(path_a: Path, path_b: Path) -> DiffReport:
|
|
463
|
+
"""Compare two triage result files and find changed decisions."""
|
|
464
|
+
df_a = read_result_table(path_a)
|
|
465
|
+
df_b = read_result_table(path_b)
|
|
466
|
+
|
|
467
|
+
# Validate that both files have Final_Decision
|
|
468
|
+
if "Final_Decision" not in df_a.columns:
|
|
469
|
+
raise ValueError(f"File A is missing 'Final_Decision' column: {path_a}")
|
|
470
|
+
if "Final_Decision" not in df_b.columns:
|
|
471
|
+
raise ValueError(f"File B is missing 'Final_Decision' column: {path_b}")
|
|
472
|
+
|
|
473
|
+
# Find the ID column
|
|
474
|
+
id_col = None
|
|
475
|
+
for candidate in ("ID", "id", "Key", "key"):
|
|
476
|
+
if candidate in df_a.columns and candidate in df_b.columns:
|
|
477
|
+
id_col = candidate
|
|
478
|
+
break
|
|
479
|
+
|
|
480
|
+
if id_col is None:
|
|
481
|
+
# No shared canonical ID column. Fall back to the first column of A,
|
|
482
|
+
# but only if it also exists in B; otherwise the merge would raise a
|
|
483
|
+
# raw KeyError on a column missing from B.
|
|
484
|
+
fallback = df_a.columns[0] if len(df_a.columns) else None
|
|
485
|
+
if fallback is not None and fallback in df_b.columns:
|
|
486
|
+
id_col = fallback
|
|
487
|
+
else:
|
|
488
|
+
raise ValueError(
|
|
489
|
+
"Nenhuma coluna de ID comum encontrada entre os dois arquivos "
|
|
490
|
+
"(esperado uma de: ID, id, Key, key). "
|
|
491
|
+
f"Colunas de A: {list(df_a.columns)}; colunas de B: {list(df_b.columns)}."
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
report = DiffReport(total_a=len(df_a), total_b=len(df_b))
|
|
495
|
+
|
|
496
|
+
merged = pd.merge(
|
|
497
|
+
df_a[[id_col, "Final_Decision"]],
|
|
498
|
+
df_b[[id_col, "Final_Decision"]],
|
|
499
|
+
on=id_col,
|
|
500
|
+
suffixes=("_a", "_b"),
|
|
501
|
+
how="outer",
|
|
502
|
+
)
|
|
503
|
+
|
|
504
|
+
# An outer merge introduces NaN for IDs that exist in only one file. The
|
|
505
|
+
# decision columns always exist after the merge, so `.get(..., "MISSING")`
|
|
506
|
+
# never triggers its default; fill the NaNs explicitly so exclusive rows
|
|
507
|
+
# report "MISSING" instead of the string "nan".
|
|
508
|
+
merged = merged.fillna("MISSING")
|
|
509
|
+
|
|
510
|
+
for _, row in merged.iterrows():
|
|
511
|
+
old = str(row.get("Final_Decision_a", "MISSING"))
|
|
512
|
+
new = str(row.get("Final_Decision_b", "MISSING"))
|
|
513
|
+
if old != new:
|
|
514
|
+
report.changed.append(
|
|
515
|
+
DiffEntry(
|
|
516
|
+
article_id=str(row[id_col]),
|
|
517
|
+
old_decision=old,
|
|
518
|
+
new_decision=new,
|
|
519
|
+
)
|
|
520
|
+
)
|
|
521
|
+
|
|
522
|
+
# Summarize transitions
|
|
523
|
+
transitions: dict[str, int] = {}
|
|
524
|
+
for entry in report.changed:
|
|
525
|
+
key = f"{entry.old_decision} -> {entry.new_decision}"
|
|
526
|
+
transitions[key] = transitions.get(key, 0) + 1
|
|
527
|
+
report.summary = transitions
|
|
528
|
+
|
|
529
|
+
return report
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
# ── Project ──────────────────────────────────────────────────────────────────
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def create_project(
|
|
536
|
+
name: str,
|
|
537
|
+
blocks: list[dict],
|
|
538
|
+
preset: str = "standard",
|
|
539
|
+
output_dir: Path | None = None,
|
|
540
|
+
force: bool = False,
|
|
541
|
+
) -> Path:
|
|
542
|
+
"""Create a new project with config.json and terms template.
|
|
543
|
+
|
|
544
|
+
Refuses to overwrite an existing project (a directory already containing
|
|
545
|
+
``config.json`` or a ``terms`` template) unless ``force=True`` is passed.
|
|
546
|
+
"""
|
|
547
|
+
if output_dir is None:
|
|
548
|
+
output_dir = Path.cwd() / name
|
|
549
|
+
|
|
550
|
+
# Guard against silently clobbering an existing project. The previous
|
|
551
|
+
# behavior (``mkdir(exist_ok=True)`` + unconditional writes) overwrote any
|
|
552
|
+
# config/terms files already present in the target directory.
|
|
553
|
+
if not force:
|
|
554
|
+
existing = [
|
|
555
|
+
candidate
|
|
556
|
+
for candidate in (
|
|
557
|
+
output_dir / "config.json",
|
|
558
|
+
output_dir / "terms.xlsx",
|
|
559
|
+
output_dir / "terms.csv",
|
|
560
|
+
)
|
|
561
|
+
if candidate.exists()
|
|
562
|
+
]
|
|
563
|
+
if existing:
|
|
564
|
+
names = ", ".join(sorted(p.name for p in existing))
|
|
565
|
+
raise FileExistsError(
|
|
566
|
+
f"O diretorio de projeto '{output_dir}' ja contem arquivos de "
|
|
567
|
+
f"projeto ({names}). Use force=True para sobrescrever."
|
|
568
|
+
)
|
|
569
|
+
|
|
570
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
571
|
+
|
|
572
|
+
config = generate_config(preset_name=preset, blocks=blocks)
|
|
573
|
+
|
|
574
|
+
config_path = output_dir / "config.json"
|
|
575
|
+
config_path.write_text(
|
|
576
|
+
json.dumps(config, indent=2, ensure_ascii=False),
|
|
577
|
+
encoding="utf-8",
|
|
578
|
+
)
|
|
579
|
+
|
|
580
|
+
# Generate terms templates. XLSX is the primary editing format; CSV is kept
|
|
581
|
+
# as a lightweight fallback for scripts and version control.
|
|
582
|
+
terms_rows: list[dict] = []
|
|
583
|
+
for block in blocks:
|
|
584
|
+
block_name = block["name"]
|
|
585
|
+
terms_rows.append(
|
|
586
|
+
{
|
|
587
|
+
"block": block_name,
|
|
588
|
+
"kind": "pos",
|
|
589
|
+
"term": f"example term for {block_name}",
|
|
590
|
+
"level": "1",
|
|
591
|
+
"section_scope": "any",
|
|
592
|
+
"is_regex": "0",
|
|
593
|
+
"normalization_type": "",
|
|
594
|
+
"normalization_target": "",
|
|
595
|
+
}
|
|
596
|
+
)
|
|
597
|
+
|
|
598
|
+
# Add example with abbreviation normalization
|
|
599
|
+
if blocks:
|
|
600
|
+
terms_rows.append(
|
|
601
|
+
{
|
|
602
|
+
"block": blocks[0]["name"],
|
|
603
|
+
"kind": "pos",
|
|
604
|
+
"term": "example abbreviation",
|
|
605
|
+
"level": "2",
|
|
606
|
+
"section_scope": "any",
|
|
607
|
+
"is_regex": "0",
|
|
608
|
+
"normalization_type": "abbreviation",
|
|
609
|
+
"normalization_target": "example expanded form",
|
|
610
|
+
}
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
# Add a GLOBAL anti-term example
|
|
614
|
+
terms_rows.append(
|
|
615
|
+
{
|
|
616
|
+
"block": "GLOBAL",
|
|
617
|
+
"kind": "anti",
|
|
618
|
+
"term": "systematic review",
|
|
619
|
+
"level": "",
|
|
620
|
+
"section_scope": "any",
|
|
621
|
+
"is_regex": "0",
|
|
622
|
+
"normalization_type": "",
|
|
623
|
+
"normalization_target": "",
|
|
624
|
+
}
|
|
625
|
+
)
|
|
626
|
+
|
|
627
|
+
terms_df = pd.DataFrame(terms_rows)
|
|
628
|
+
terms_xlsx_path = output_dir / "terms.xlsx"
|
|
629
|
+
terms_csv_path = output_dir / "terms.csv"
|
|
630
|
+
terms_df.to_excel(terms_xlsx_path, index=False, engine="openpyxl")
|
|
631
|
+
terms_df.to_csv(terms_csv_path, sep=";", index=False, encoding="utf-8-sig")
|
|
632
|
+
|
|
633
|
+
return output_dir
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def save_profile_config(name: str, config_path: Path, description: str = "") -> Path:
|
|
637
|
+
"""Save a config file as a named profile."""
|
|
638
|
+
from . import profiles
|
|
639
|
+
|
|
640
|
+
cfg = load_config(config_path)
|
|
641
|
+
return profiles.save_profile(name, cfg, description)
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
# ── Export ───────────────────────────────────────────────────────────────────
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
def export_academic_package(
|
|
648
|
+
result_path: Path,
|
|
649
|
+
output_dir: Path,
|
|
650
|
+
config_path: Path | None = None,
|
|
651
|
+
) -> Path:
|
|
652
|
+
"""Create an academic package ZIP from existing results.
|
|
653
|
+
|
|
654
|
+
Raises ``ValueError`` when ``result_path`` is not a FastSLR result table
|
|
655
|
+
(i.e. it lacks the ``Final_Decision`` column). Without this guard, pointing
|
|
656
|
+
at any arbitrary CSV produced a ZIP "successfully", yielding an invalid
|
|
657
|
+
academic package that contains raw data instead of triage decisions.
|
|
658
|
+
"""
|
|
659
|
+
result_df = read_result_table(result_path)
|
|
660
|
+
if "Final_Decision" not in result_df.columns:
|
|
661
|
+
raise ValueError(
|
|
662
|
+
f"O arquivo '{result_path}' nao parece ser um resultado do FastSLR: "
|
|
663
|
+
f"falta a coluna 'Final_Decision'. Rode 'fastslr run' primeiro para "
|
|
664
|
+
f"gerar um resultado de triagem antes de exportar o pacote academico."
|
|
665
|
+
)
|
|
666
|
+
|
|
667
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
668
|
+
|
|
669
|
+
artifacts: dict[str, Path] = {"results": result_path}
|
|
670
|
+
|
|
671
|
+
# Collect protocol and report if they exist alongside results
|
|
672
|
+
result_dir = result_path.parent
|
|
673
|
+
for name, filename in (
|
|
674
|
+
("protocol", "protocol.json"),
|
|
675
|
+
("report", "triage_report.txt"),
|
|
676
|
+
("academic", "academic_report.md"),
|
|
677
|
+
("audit", "config_audit.json"),
|
|
678
|
+
):
|
|
679
|
+
candidate = result_dir / filename
|
|
680
|
+
if candidate.exists():
|
|
681
|
+
artifacts[name] = candidate
|
|
682
|
+
|
|
683
|
+
if config_path and config_path.exists():
|
|
684
|
+
artifacts["config"] = config_path
|
|
685
|
+
|
|
686
|
+
zip_path = output_dir / "academic_package.zip"
|
|
687
|
+
execution_id = f"fastslr_v{VERSION}"
|
|
688
|
+
|
|
689
|
+
export_appendix_pack(artifacts, zip_path, execution_id)
|
|
690
|
+
|
|
691
|
+
return zip_path
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
# ── Browse Terms ─────────────────────────────────────────────────────────────
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
@dataclass
|
|
698
|
+
class TermsView:
|
|
699
|
+
"""Structured view of all configured terms."""
|
|
700
|
+
|
|
701
|
+
terms: list[dict] = field(default_factory=list)
|
|
702
|
+
total: int = 0
|
|
703
|
+
blocks: list[str] = field(default_factory=list)
|
|
704
|
+
parse_warnings: list[str] = field(default_factory=list)
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def browse_terms(
|
|
708
|
+
config_path: Path,
|
|
709
|
+
terms_path: Path | None = None,
|
|
710
|
+
block_filter: str | None = None,
|
|
711
|
+
kind_filter: str | None = None,
|
|
712
|
+
) -> TermsView:
|
|
713
|
+
"""Load and return a structured view of all configured terms."""
|
|
714
|
+
config = load_config(config_path)
|
|
715
|
+
if terms_path:
|
|
716
|
+
config = parse_terms_csv(terms_path, config)
|
|
717
|
+
|
|
718
|
+
domain_blocks = get_domain_blocks(config)
|
|
719
|
+
all_terms: list[dict] = []
|
|
720
|
+
|
|
721
|
+
target_blocks = [block_filter] if block_filter else domain_blocks
|
|
722
|
+
if not block_filter and "T0" in config:
|
|
723
|
+
target_blocks.append("T0")
|
|
724
|
+
|
|
725
|
+
for block_name in target_blocks:
|
|
726
|
+
if block_name not in config:
|
|
727
|
+
continue
|
|
728
|
+
block = config[block_name]
|
|
729
|
+
|
|
730
|
+
for entry in block.get("positives", []):
|
|
731
|
+
if kind_filter and kind_filter != "pos":
|
|
732
|
+
continue
|
|
733
|
+
all_terms.append(
|
|
734
|
+
{
|
|
735
|
+
"block": block_name,
|
|
736
|
+
"kind": "pos",
|
|
737
|
+
"term": entry.get("term", ""),
|
|
738
|
+
"level": entry.get("level"),
|
|
739
|
+
"scope": entry.get("scope", "any"),
|
|
740
|
+
}
|
|
741
|
+
)
|
|
742
|
+
|
|
743
|
+
anti = block.get("anti", {})
|
|
744
|
+
for entry in anti.get("exclude", []):
|
|
745
|
+
if kind_filter and kind_filter != "anti":
|
|
746
|
+
continue
|
|
747
|
+
all_terms.append(
|
|
748
|
+
{
|
|
749
|
+
"block": block_name,
|
|
750
|
+
"kind": "anti",
|
|
751
|
+
"term": entry.get("term", ""),
|
|
752
|
+
"level": None,
|
|
753
|
+
"scope": entry.get("scope", "any"),
|
|
754
|
+
}
|
|
755
|
+
)
|
|
756
|
+
|
|
757
|
+
for entry in anti.get("flag", []):
|
|
758
|
+
if kind_filter and kind_filter != "flag":
|
|
759
|
+
continue
|
|
760
|
+
all_terms.append(
|
|
761
|
+
{
|
|
762
|
+
"block": block_name,
|
|
763
|
+
"kind": "flag",
|
|
764
|
+
"term": entry.get("term", ""),
|
|
765
|
+
"level": None,
|
|
766
|
+
"scope": entry.get("scope", "any"),
|
|
767
|
+
}
|
|
768
|
+
)
|
|
769
|
+
|
|
770
|
+
# Propagate parse warnings (invalid kind, empty term, out-of-range level,
|
|
771
|
+
# skipped rows) so callers can surface which CSV rows were not accepted.
|
|
772
|
+
# Without this, the user assumes every row was loaded.
|
|
773
|
+
parse_warnings: list[str] = list(config.get("_parse_warnings", []))
|
|
774
|
+
|
|
775
|
+
return TermsView(
|
|
776
|
+
terms=all_terms,
|
|
777
|
+
total=len(all_terms),
|
|
778
|
+
blocks=domain_blocks,
|
|
779
|
+
parse_warnings=parse_warnings,
|
|
780
|
+
)
|
|
781
|
+
|
|
782
|
+
|
|
783
|
+
__all__ = [
|
|
784
|
+
"ValidationIssue",
|
|
785
|
+
"TriageResult",
|
|
786
|
+
"PreviewResult",
|
|
787
|
+
"DiffEntry",
|
|
788
|
+
"DiffReport",
|
|
789
|
+
"ProfileInfo",
|
|
790
|
+
"SetupInspection",
|
|
791
|
+
"TermsView",
|
|
792
|
+
"get_version",
|
|
793
|
+
"format_coverage",
|
|
794
|
+
"read_results_table",
|
|
795
|
+
"validate_config",
|
|
796
|
+
"inspect_run_setup",
|
|
797
|
+
"run_triage",
|
|
798
|
+
"preview_triage",
|
|
799
|
+
"analyze_coverage",
|
|
800
|
+
"diff_results",
|
|
801
|
+
"create_project",
|
|
802
|
+
"save_profile_config",
|
|
803
|
+
"export_academic_package",
|
|
804
|
+
"browse_terms",
|
|
805
|
+
]
|