reprovane 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. reprovane/__init__.py +1 -0
  2. reprovane/__main__.py +105 -0
  3. reprovane/_version.py +3 -0
  4. reprovane/common.py +202 -0
  5. reprovane/config.py +115 -0
  6. reprovane/core/__init__.py +29 -0
  7. reprovane/core/artifacts.py +100 -0
  8. reprovane/core/db.py +78 -0
  9. reprovane/core/foundation.py +228 -0
  10. reprovane/core/ids.py +11 -0
  11. reprovane/core/migrations/0001_persistence_foundation.sql +222 -0
  12. reprovane/core/migrations/0002_typed_observations.sql +322 -0
  13. reprovane/core/migrations/0003_publication_authorship_observation.sql +53 -0
  14. reprovane/core/migrations/0004_publication_identity_resolution.sql +272 -0
  15. reprovane/core/migrations/0005_venue_resolution_scimago.sql +300 -0
  16. reprovane/core/migrations/0006_venue_provider_attributes.sql +33 -0
  17. reprovane/core/migrations/0007_openalex_acquisition.sql +52 -0
  18. reprovane/core/migrations/0008_discovery_provenance.sql +138 -0
  19. reprovane/core/migrations/0009_frontier_search_validation.sql +93 -0
  20. reprovane/core/migrations/0010_screening_separation.sql +127 -0
  21. reprovane/core/migrations/0011_immutable_corpus_release.sql +52 -0
  22. reprovane/core/migrations/0012_fulltext_acquisition.sql +98 -0
  23. reprovane/core/migrations/0013_legacy_v4_migration.sql +38 -0
  24. reprovane/core/migrations.py +79 -0
  25. reprovane/core/provenance.py +52 -0
  26. reprovane/corpus_release/__init__.py +3 -0
  27. reprovane/corpus_release/__main__.py +39 -0
  28. reprovane/corpus_release/model.py +50 -0
  29. reprovane/corpus_release/plan.py +33 -0
  30. reprovane/corpus_release/portable.py +72 -0
  31. reprovane/corpus_release/store.py +172 -0
  32. reprovane/discovery/__init__.py +2 -0
  33. reprovane/discovery/__main__.py +43 -0
  34. reprovane/discovery/expansion.py +145 -0
  35. reprovane/discovery/store.py +243 -0
  36. reprovane/frontier/__init__.py +2 -0
  37. reprovane/frontier/__main__.py +35 -0
  38. reprovane/frontier/policy.py +39 -0
  39. reprovane/frontier/store.py +179 -0
  40. reprovane/fulltext.py +728 -0
  41. reprovane/fulltext_acquisition/__init__.py +6 -0
  42. reprovane/fulltext_acquisition/__main__.py +47 -0
  43. reprovane/fulltext_acquisition/candidates.py +61 -0
  44. reprovane/fulltext_acquisition/policy.py +77 -0
  45. reprovane/fulltext_acquisition/portable.py +67 -0
  46. reprovane/fulltext_acquisition/safety.py +82 -0
  47. reprovane/fulltext_acquisition/store.py +372 -0
  48. reprovane/fulltext_acquisition/transport.py +120 -0
  49. reprovane/legacy_migration/__init__.py +2 -0
  50. reprovane/legacy_migration/inventory.py +112 -0
  51. reprovane/legacy_migration/normalize.py +27 -0
  52. reprovane/legacy_migration/store.py +127 -0
  53. reprovane/mirror/__init__.py +6 -0
  54. reprovane/mirror/cache.py +144 -0
  55. reprovane/mirror/github.py +103 -0
  56. reprovane/mirror/models.py +83 -0
  57. reprovane/mirror/pinned_files.py +81 -0
  58. reprovane/mirror/scimago_sync.py +156 -0
  59. reprovane/observations/__init__.py +5 -0
  60. reprovane/observations/store.py +217 -0
  61. reprovane/observations/validation.py +117 -0
  62. reprovane/openalex.py +553 -0
  63. reprovane/openalex_acquisition/__init__.py +4 -0
  64. reprovane/openalex_acquisition/__main__.py +38 -0
  65. reprovane/openalex_acquisition/credentials.py +24 -0
  66. reprovane/openalex_acquisition/normalize.py +61 -0
  67. reprovane/openalex_acquisition/observation_store.py +23 -0
  68. reprovane/openalex_acquisition/spec.py +58 -0
  69. reprovane/openalex_acquisition/store.py +181 -0
  70. reprovane/openalex_http.py +82 -0
  71. reprovane/operations/__init__.py +1 -0
  72. reprovane/operations/cli.py +28 -0
  73. reprovane/operations/lock.py +52 -0
  74. reprovane/operations/workspace.py +149 -0
  75. reprovane/pipeline.py +642 -0
  76. reprovane/providers/__init__.py +1 -0
  77. reprovane/providers/scimago.py +274 -0
  78. reprovane/resolution/__init__.py +31 -0
  79. reprovane/resolution/evidence.py +55 -0
  80. reprovane/resolution/normalization.py +38 -0
  81. reprovane/resolution/policy.py +72 -0
  82. reprovane/resolution/resolver.py +95 -0
  83. reprovane/resolution/store.py +360 -0
  84. reprovane/scimago.py +825 -0
  85. reprovane/scimago_repo.py +303 -0
  86. reprovane/screening/__init__.py +2 -0
  87. reprovane/screening/__main__.py +40 -0
  88. reprovane/screening/evidence.py +52 -0
  89. reprovane/screening/plan.py +83 -0
  90. reprovane/screening/store.py +232 -0
  91. reprovane/search_validation/__init__.py +1 -0
  92. reprovane/search_validation/__main__.py +36 -0
  93. reprovane/search_validation/store.py +154 -0
  94. reprovane/stage09.py +29 -0
  95. reprovane/venue_intelligence/__init__.py +3 -0
  96. reprovane/venue_intelligence/__main__.py +40 -0
  97. reprovane/venue_intelligence/parsers.py +65 -0
  98. reprovane/venue_intelligence/projection.py +73 -0
  99. reprovane/venue_intelligence/store.py +177 -0
  100. reprovane/venue_resolution/__init__.py +10 -0
  101. reprovane/venue_resolution/evidence.py +51 -0
  102. reprovane/venue_resolution/normalization.py +24 -0
  103. reprovane/venue_resolution/policy.py +40 -0
  104. reprovane/venue_resolution/projection.py +71 -0
  105. reprovane/venue_resolution/resolver.py +90 -0
  106. reprovane/venue_resolution/store.py +413 -0
  107. reprovane/venues.py +193 -0
  108. reprovane-2.0.0.data/data/share/doc/reprovane/ARCHITECTURE.md +11 -0
  109. reprovane-2.0.0.data/data/share/doc/reprovane/CHANGELOG.md +36 -0
  110. reprovane-2.0.0.data/data/share/doc/reprovane/LICENSE +21 -0
  111. reprovane-2.0.0.data/data/share/doc/reprovane/MIGRATION_FROM_SCHOLARFLOW_1.md +40 -0
  112. reprovane-2.0.0.data/data/share/doc/reprovane/MIGRATION_TO_SCHOLARFLOW.md +31 -0
  113. reprovane-2.0.0.data/data/share/doc/reprovane/OPERATIONS.md +71 -0
  114. reprovane-2.0.0.data/data/share/doc/reprovane/PROVENANCE.md +27 -0
  115. reprovane-2.0.0.data/data/share/doc/reprovane/QUICKSTART_vi.md +137 -0
  116. reprovane-2.0.0.data/data/share/doc/reprovane/README.md +70 -0
  117. reprovane-2.0.0.data/data/share/doc/reprovane/SECURITY.md +17 -0
  118. reprovane-2.0.0.data/data/share/doc/reprovane/THIRD_PARTY_DATA.md +9 -0
  119. reprovane-2.0.0.data/data/share/doc/reprovane/USER_GUIDE_vi.md +183 -0
  120. reprovane-2.0.0.data/data/share/doc/reprovane/architecture/discovery_provenance_citation_expansion.md +137 -0
  121. reprovane-2.0.0.data/data/share/doc/reprovane/architecture/frontier_policy_search_validation.md +205 -0
  122. reprovane-2.0.0.data/data/share/doc/reprovane/architecture/fulltext_acquisition.md +65 -0
  123. reprovane-2.0.0.data/data/share/doc/reprovane/architecture/immutable_corpus_release.md +175 -0
  124. reprovane-2.0.0.data/data/share/doc/reprovane/architecture/openalex_acquisition_migration.md +185 -0
  125. reprovane-2.0.0.data/data/share/doc/reprovane/architecture/persistence_foundation.md +184 -0
  126. reprovane-2.0.0.data/data/share/doc/reprovane/architecture/pinned_github_mirror_sync.md +168 -0
  127. reprovane-2.0.0.data/data/share/doc/reprovane/architecture/publication_identity_resolution.md +201 -0
  128. reprovane-2.0.0.data/data/share/doc/reprovane/architecture/scopus_wos_venue_intelligence.md +156 -0
  129. reprovane-2.0.0.data/data/share/doc/reprovane/architecture/screening_separation.md +183 -0
  130. reprovane-2.0.0.data/data/share/doc/reprovane/architecture/typed_observations.md +172 -0
  131. reprovane-2.0.0.data/data/share/doc/reprovane/architecture/venue_resolution_scimago.md +186 -0
  132. reprovane-2.0.0.dist-info/METADATA +86 -0
  133. reprovane-2.0.0.dist-info/RECORD +137 -0
  134. reprovane-2.0.0.dist-info/WHEEL +5 -0
  135. reprovane-2.0.0.dist-info/entry_points.txt +2 -0
  136. reprovane-2.0.0.dist-info/licenses/LICENSE +21 -0
  137. reprovane-2.0.0.dist-info/top_level.txt +1 -0
reprovane/__init__.py ADDED
@@ -0,0 +1 @@
1
+ from ._version import __version__
reprovane/__main__.py ADDED
@@ -0,0 +1,105 @@
1
+ """Thin discoverable router; scientific operations stay in their accepted modules."""
2
+ import argparse,importlib,json,sys
3
+ from pathlib import Path
4
+ from . import __version__
5
+ from .core import ValidationError,ArtifactIntegrityError,MigrationDriftError,FoundationError
6
+
7
+ ROUTES={'acquisition':'openalex_acquisition','venue-data':'venue_intelligence','discovery':'discovery','frontier':'frontier','search-validation':'search_validation','screening':'screening','corpus':'corpus_release','fulltext':'fulltext_acquisition'}
8
+
9
+
10
+ def main(argv=None):
11
+ args=list(sys.argv[1:] if argv is None else argv)
12
+ if args and args[0] in ROUTES:return importlib.import_module('reprovane.'+ROUTES[args[0]]+'.__main__').main(args[1:])
13
+ parser=argparse.ArgumentParser(prog='reprovane',description='Reprovane — reproducible literature discovery, screening, evidence management, corpus release, and full-text acquisition.')
14
+ parser.add_argument('--version',action='version',version=__version__);sub=parser.add_subparsers(dest='family',required=True)
15
+ for family,module in ROUTES.items():sub.add_parser(family,help='Existing '+module+' CLI (use '+family+' --help)')
16
+ for name in ('doctor','verify','inspect'):
17
+ p=sub.add_parser(name);p.add_argument('--offline',action='store_true',default=True);p.add_argument('--workspace',type=Path);p.add_argument('--full',action='store_true')
18
+ p=sub.add_parser('workspace');s=p.add_subparsers(dest='command',required=True)
19
+ i=s.add_parser('init');i.add_argument('--workspace',type=Path,required=True)
20
+ i=s.add_parser('upgrade');i.add_argument('--workspace',type=Path,required=True);i.add_argument('--target',type=Path,required=True);i.add_argument('--backup',type=Path,required=True)
21
+ p=sub.add_parser('legacy');s=p.add_subparsers(dest='command',required=True)
22
+ for cmd in ('audit','plan','migrate','verify','inspect'):
23
+ i=s.add_parser(cmd)
24
+ if cmd in ('audit','plan','migrate'):i.add_argument('--source',type=Path,required=True)
25
+ if cmd=='plan':i.add_argument('--project-key',default='legacy-v4-import');i.add_argument('--revision-number',type=int,default=1)
26
+ if cmd=='migrate':i.add_argument('--target',type=Path);i.add_argument('--plan',type=Path);i.add_argument('--dry-run',action='store_true')
27
+ if cmd in ('verify','inspect'):i.add_argument('--workspace',type=Path,required=True);i.add_argument('--migration-id',required=True);i.add_argument('--source',type=Path)
28
+ p=sub.add_parser('project');s=p.add_subparsers(dest='command',required=True)
29
+ for cmd in ('create','revision','inspect'):
30
+ i=s.add_parser(cmd);i.add_argument('--workspace',type=Path,required=True)
31
+ if cmd=='create':i.add_argument('--key',required=True);i.add_argument('--display-name',required=True)
32
+ else:i.add_argument('--project-id',required=True)
33
+ if cmd=='revision':i.add_argument('--number',type=int,required=True);i.add_argument('--manifest',type=Path,required=True)
34
+ for family in ('resolution','venue-resolution'):
35
+ p=sub.add_parser(family);s=p.add_subparsers(dest='command',required=True)
36
+ for cmd in ('create','add','generate','evaluate','materialize','seal','verify','inspect'):
37
+ i=s.add_parser(cmd);i.add_argument('--workspace',type=Path,required=True)
38
+ if cmd=='create':i.add_argument('--project-revision',required=True);i.add_argument('--selector',type=Path,required=True)
39
+ else:i.add_argument('--run-id',required=True)
40
+ if cmd=='add':i.add_argument('--record-id',action='append',required=True)
41
+ p=sub.add_parser('scimago');p.add_argument('--workspace',type=Path,required=True);p.add_argument('--ref',default='control');p.add_argument('--offline',action='store_true');p.add_argument('--import-snapshot',action='store_true')
42
+ a=parser.parse_args(args)
43
+ try:
44
+ result=dispatch(a)
45
+ print(json.dumps(result,ensure_ascii=False,sort_keys=True));return 0
46
+ except (ArtifactIntegrityError,MigrationDriftError):print('Integrity verification failed; private inputs are not printed.',file=sys.stderr);return 3
47
+ except (ValidationError,FoundationError,ValueError,OSError):print('Validation/configuration failed; inspect the offline doctor and immutable inputs.',file=sys.stderr);return 5 if a.family=='legacy' and a.command=='migrate' else 2
48
+ except Exception:
49
+ print('Operation failed safely; no private exception payload is printed.',file=sys.stderr)
50
+ return 4 if a.family=='scimago' else 5 if a.family=='legacy' and a.command=='migrate' else 2
51
+
52
+
53
+ def dispatch(a):
54
+ from .operations.workspace import doctor,initialize,upgrade,read_database
55
+ from .operations.lock import workspace_lock
56
+ from .core.db import Database
57
+ if a.family in ('doctor','verify','inspect'):return doctor(a.workspace,full=a.full or a.family=='verify')
58
+ if a.family=='workspace':return initialize(a.workspace) if a.command=='init' else upgrade(a.workspace,a.target,a.backup)
59
+ if a.family=='legacy':
60
+ from .legacy_migration import audit,plan,execute,LegacyMigrationStore
61
+ if a.command=='audit':return audit(a.source)
62
+ if a.command=='plan':return plan(a.source,a.project_key,a.revision_number)
63
+ if a.command=='migrate':
64
+ if a.dry_run:return plan(a.source)
65
+ if a.target is None or a.plan is None:raise ValidationError('Execution requires saved plan and separate target')
66
+ return execute(a.source,a.target,json.loads(a.plan.read_text(encoding='utf-8-sig')))
67
+ with read_database(a.workspace) as db:
68
+ store=LegacyMigrationStore(db)
69
+ return store.verify(a.migration_id,a.source) if a.command=='verify' else store.get(a.migration_id)
70
+ if a.family=='scimago':
71
+ from .mirror import sync_scimago_from_github,import_synced_scimago
72
+ with workspace_lock(a.workspace):
73
+ result=sync_scimago_from_github(a.workspace,requested_ref=a.ref,offline=a.offline)
74
+ if a.import_snapshot:
75
+ with Database(a.workspace) as db:return import_synced_scimago(db,result)
76
+ from dataclasses import asdict
77
+ return asdict(result)
78
+ readonly=a.command in ('verify','inspect')
79
+ from contextlib import nullcontext
80
+ with (nullcontext() if readonly else workspace_lock(a.workspace)):
81
+ with (read_database(a.workspace) if readonly else Database(a.workspace)) as db:
82
+ if a.family=='project':
83
+ from .core.foundation import Foundation
84
+ f=Foundation(db)
85
+ if a.command=='create':return f.create_project(a.key,a.display_name)
86
+ if a.command=='revision':return f.create_project_revision(a.project_id,a.number,'json',a.manifest.read_text(encoding='utf-8-sig'))
87
+ return dict(project=f.get_project(a.project_id),revisions=f.list_project_revisions(a.project_id))
88
+ venue=a.family=='venue-resolution'
89
+ if venue:
90
+ from .venue_resolution import VenueResolutionStore
91
+ store=VenueResolutionStore(db);prefix='venue_resolution'
92
+ else:
93
+ from .resolution import ResolutionStore
94
+ store=ResolutionStore(db);prefix='resolution'
95
+ if a.command=='create':return getattr(store,'create_'+prefix+'_run')(a.project_revision,json.loads(a.selector.read_text(encoding='utf-8-sig')))
96
+ if a.command=='add':return getattr(store,'add_'+prefix+'_inputs')(a.run_id,a.record_id)
97
+ if a.command=='generate':
98
+ if venue:store.generate_provider_nodes(a.run_id)
99
+ return store.generate_candidates(a.run_id)
100
+ if a.command=='evaluate':return store.evaluate_candidates_automatically(a.run_id)
101
+ method={'materialize':'materialize_','seal':'seal_','verify':'verify_','inspect':'get_'}[a.command]+prefix+'_run'
102
+ return getattr(store,method)(a.run_id)
103
+
104
+
105
+ if __name__=='__main__':raise SystemExit(main())
reprovane/_version.py ADDED
@@ -0,0 +1,3 @@
1
+ """Reprovane product version; predecessor version is provenance only."""
2
+ __version__ = "2.0.0"
3
+ LEGACY_PREDECESSOR_VERSION = "4.1.0"
reprovane/common.py ADDED
@@ -0,0 +1,202 @@
1
+ from __future__ import annotations
2
+
3
+ import csv
4
+ import hashlib
5
+ import json
6
+ import re
7
+ import unicodedata
8
+ from pathlib import Path
9
+ from typing import Any, Dict, Iterable, List, Optional
10
+
11
+
12
+ def safe_str(value: Any) -> str:
13
+ return "" if value is None else str(value)
14
+
15
+
16
+ def normalize_doi(doi: Optional[str]) -> str:
17
+ if not doi:
18
+ return ""
19
+ value = doi.strip().lower()
20
+ for prefix in (
21
+ "https://doi.org/",
22
+ "http://doi.org/",
23
+ "https://dx.doi.org/",
24
+ "http://dx.doi.org/",
25
+ "doi:",
26
+ ):
27
+ if value.startswith(prefix):
28
+ value = value[len(prefix):]
29
+ break
30
+ return value.strip()
31
+
32
+
33
+ def short_openalex_id(value: Optional[str]) -> str:
34
+ if not value:
35
+ return ""
36
+ return value.rstrip("/").split("/")[-1]
37
+
38
+
39
+ def normalize_issn(value: Optional[str]) -> str:
40
+ if not value:
41
+ return ""
42
+ return re.sub(r"[^0-9Xx]", "", value).upper()
43
+
44
+
45
+ def split_issns(value: Optional[str]) -> List[str]:
46
+ if not value:
47
+ return []
48
+ parts = re.split(r"[;,|]", value)
49
+ result: List[str] = []
50
+ seen = set()
51
+ for part in parts:
52
+ normalized = normalize_issn(part)
53
+ if normalized and normalized not in seen:
54
+ seen.add(normalized)
55
+ result.append(normalized)
56
+ return result
57
+
58
+
59
+ def title_exact_key(value: Optional[str]) -> str:
60
+ text = safe_str(value).casefold().strip()
61
+ return re.sub(r"\s+", " ", text)
62
+
63
+
64
+ _NON_ALNUM_RE = re.compile(r"[^a-z0-9]+")
65
+
66
+
67
+ def title_normalized_key(value: Optional[str]) -> str:
68
+ # "&" means "and" (OpenAlex "Computers & Security" vs SCImago "Computers and Security"); it was
69
+ # previously deleted like other punctuation, so such titles never matched without an ISSN.
70
+ text = safe_str(value).replace("&", " and ")
71
+ if not text.isascii():
72
+ # NFKD + combining-mark removal is a no-op for ASCII, so it only runs when needed.
73
+ text = unicodedata.normalize("NFKD", text)
74
+ text = "".join(ch for ch in text if not unicodedata.combining(ch))
75
+ text = text.casefold()
76
+ return _NON_ALNUM_RE.sub("", text)
77
+
78
+
79
+ STATUS_SUFFIX_RE = re.compile(r"\s*\((?:discontinued|ceased)\)\s*$", re.IGNORECASE)
80
+
81
+
82
+ def strip_status_suffix(value: Optional[str]) -> str:
83
+ """Remove SCImago status labels such as ' (discontinued)' from the end of a title."""
84
+ return STATUS_SUFFIX_RE.sub("", safe_str(value))
85
+
86
+
87
+ def has_status_suffix(value: Optional[str]) -> bool:
88
+ return bool(STATUS_SUFFIX_RE.search(safe_str(value)))
89
+
90
+
91
+ def title_status_normalized_key(value: Optional[str]) -> str:
92
+ return title_normalized_key(strip_status_suffix(value))
93
+
94
+
95
+ VALID_QUARTILES = {"Q1", "Q2", "Q3", "Q4"}
96
+
97
+
98
+ def normalize_quartile(value: Any) -> str:
99
+ """Return Q1-Q4, or "" for SCImago placeholders such as "-" (no quartile assigned)."""
100
+ text = safe_str(value).strip().upper()
101
+ return text if text in VALID_QUARTILES else ""
102
+
103
+
104
+ _SECRET_PARAM_RE = re.compile(r"(?i)(api_key|apikey|access_token|token)=([^&\s'\"<>]+)")
105
+
106
+
107
+ def redact_secrets(text: Any, *secrets: str) -> str:
108
+ """Mask credential query parameters and any literal secret values in a message."""
109
+ result = _SECRET_PARAM_RE.sub(lambda m: f"{m.group(1)}=REDACTED", safe_str(text))
110
+ for secret in secrets:
111
+ if secret:
112
+ result = result.replace(secret, "REDACTED")
113
+ return result
114
+
115
+
116
+ def locale_float(value: Any) -> Optional[float]:
117
+ text = safe_str(value).strip()
118
+ if not text:
119
+ return None
120
+ text = text.replace(" ", "").replace(",", ".")
121
+ try:
122
+ return float(text)
123
+ except ValueError:
124
+ return None
125
+
126
+
127
+ def int_or_none(value: Any) -> Optional[int]:
128
+ text = safe_str(value).strip().replace(" ", "")
129
+ if not text:
130
+ return None
131
+ try:
132
+ return int(float(text.replace(",", ".")))
133
+ except ValueError:
134
+ return None
135
+
136
+
137
+ def sha256_file(path: Path, chunk_size: int = 1024 * 1024) -> str:
138
+ digest = hashlib.sha256()
139
+ with path.open("rb") as handle:
140
+ for chunk in iter(lambda: handle.read(chunk_size), b""):
141
+ digest.update(chunk)
142
+ return digest.hexdigest()
143
+
144
+
145
+ def read_json(path: Path) -> Dict[str, Any]:
146
+ with path.open("r", encoding="utf-8") as handle:
147
+ return json.load(handle)
148
+
149
+
150
+ def write_json(path: Path, data: Any) -> None:
151
+ path.parent.mkdir(parents=True, exist_ok=True)
152
+ with path.open("w", encoding="utf-8") as handle:
153
+ json.dump(data, handle, ensure_ascii=False, indent=2)
154
+ handle.write("\n")
155
+
156
+
157
+ def write_jsonl(path: Path, rows: Iterable[Dict[str, Any]]) -> None:
158
+ path.parent.mkdir(parents=True, exist_ok=True)
159
+ with path.open("w", encoding="utf-8") as handle:
160
+ for row in rows:
161
+ handle.write(json.dumps(row, ensure_ascii=False))
162
+ handle.write("\n")
163
+
164
+
165
+ def read_jsonl(path: Path) -> List[Dict[str, Any]]:
166
+ rows: List[Dict[str, Any]] = []
167
+ if not path.exists():
168
+ return rows
169
+ with path.open("r", encoding="utf-8") as handle:
170
+ for line in handle:
171
+ line = line.strip()
172
+ if not line:
173
+ continue
174
+ rows.append(json.loads(line))
175
+ return rows
176
+
177
+
178
+ def union_fieldnames(rows: Iterable[Dict[str, Any]]) -> List[str]:
179
+ ordered: List[str] = []
180
+ seen = set()
181
+ for row in rows:
182
+ for key in row.keys():
183
+ if key not in seen:
184
+ seen.add(key)
185
+ ordered.append(key)
186
+ return ordered
187
+
188
+
189
+ def write_csv(path: Path, rows: List[Dict[str, Any]], fieldnames: Optional[List[str]] = None) -> None:
190
+ path.parent.mkdir(parents=True, exist_ok=True)
191
+ if fieldnames is None:
192
+ fieldnames = union_fieldnames(rows)
193
+ with path.open("w", encoding="utf-8-sig", newline="") as handle:
194
+ writer = csv.DictWriter(handle, fieldnames=fieldnames, extrasaction="ignore")
195
+ writer.writeheader()
196
+ for row in rows:
197
+ writer.writerow(row)
198
+
199
+
200
+ def read_csv(path: Path) -> List[Dict[str, str]]:
201
+ with path.open("r", encoding="utf-8-sig", newline="") as handle:
202
+ return list(csv.DictReader(handle))
reprovane/config.py ADDED
@@ -0,0 +1,115 @@
1
+ """Optional private configuration file (config.json).
2
+
3
+ Precedence for every setting: CLI argument > config.json > code default. For the OpenAlex API key
4
+ only, the legacy OPENALEX_API_KEY environment variable is still honoured after config.json
5
+ (deprecated, kept so existing v3.0.x setups do not break).
6
+
7
+ config.json holds the API key, so it is validated strictly (unknown sections/keys, wrong types and
8
+ values outside a fixed list are errors) and no error message ever echoes a value, or an unknown key
9
+ name, from the file.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import json
14
+ from dataclasses import dataclass, field
15
+ from pathlib import Path
16
+ from typing import Any, Dict, Optional, Tuple
17
+
18
+ from .openalex import SEARCH_FIELDS
19
+
20
+ CONFIG_FILENAME = "config.json"
21
+
22
+ # section -> key -> allowed JSON types
23
+ SCHEMA: Dict[str, Dict[str, Tuple[type, ...]]] = {
24
+ "openalex": {"api_key": (str,), "email": (str,)},
25
+ "scimago": {"manifest_url": (str,), "cache_dir": (str,)},
26
+ "run": {"from_year": (int,), "to_year": (int,), "require_abstract": (bool,), "max_per_query": (int, type(None)),
27
+ "search_field": (str,)},
28
+ # defaults for run and enrich: the venue list (--venue-tiers) and the tiers it admits (--admit-tier)
29
+ "screening": {"venue_tiers": (str,), "admit_tiers": (list,)},
30
+ }
31
+ # (section, key) -> allowed string values; an empty string still means "use the default"
32
+ CHOICES: Dict[Tuple[str, str], Tuple[str, ...]] = {("run", "search_field"): SEARCH_FIELDS}
33
+ _TYPE_NAMES = {str: "a string", int: "an integer", bool: "true/false", type(None): "null", list: "a list of strings"}
34
+
35
+
36
+ class ConfigError(ValueError):
37
+ pass
38
+
39
+
40
+ @dataclass
41
+ class Config:
42
+ path: Optional[Path] = None
43
+ data: Dict[str, Dict[str, Any]] = field(default_factory=dict)
44
+
45
+ def get(self, section: str, key: str) -> Any:
46
+ """Configured value, or None when the file, section or key is absent or the string is empty."""
47
+ value = (self.data.get(section) or {}).get(key)
48
+ return None if value == "" else value
49
+
50
+ @property
51
+ def label(self) -> str:
52
+ """Name used in metadata/console as the source of a setting (never the value)."""
53
+ return self.path.name if self.path else ""
54
+
55
+
56
+ def _type_ok(value: Any, allowed: Tuple[type, ...]) -> bool:
57
+ if isinstance(value, bool): # bool is an int subclass: only accept it where bool is allowed
58
+ return bool in allowed
59
+ return isinstance(value, allowed)
60
+
61
+
62
+ def load_config(explicit_path: Optional[Path], default_dir: Path) -> Config:
63
+ """Load `explicit_path`, else `<default_dir>/config.json` if it exists, else an empty Config."""
64
+ if explicit_path is not None:
65
+ path = Path(explicit_path)
66
+ if not path.is_file():
67
+ raise ConfigError(f"Config file not found: {path}")
68
+ else:
69
+ path = Path(default_dir) / CONFIG_FILENAME
70
+ if not path.is_file():
71
+ return Config()
72
+ try:
73
+ raw = json.loads(path.read_text(encoding="utf-8-sig"))
74
+ except json.JSONDecodeError as exc:
75
+ raise ConfigError(f"{path}: invalid JSON ({exc.msg} at line {exc.lineno}, column {exc.colno})") from None
76
+ except UnicodeDecodeError:
77
+ raise ConfigError(f"{path}: file is not UTF-8 text") from None
78
+ if not isinstance(raw, dict):
79
+ raise ConfigError(f"{path}: the top level must be a JSON object")
80
+
81
+ data: Dict[str, Dict[str, Any]] = {}
82
+ for section, values in raw.items():
83
+ if section.startswith("_"): # comment entries such as "_comment"
84
+ continue
85
+ if section not in SCHEMA:
86
+ raise ConfigError(f"{path}: unknown top-level section (allowed: {', '.join(SCHEMA)}, or names starting with '_')")
87
+ if not isinstance(values, dict):
88
+ raise ConfigError(f"{path}: section '{section}' must be a JSON object")
89
+ clean: Dict[str, Any] = {}
90
+ for key, value in values.items():
91
+ if key.startswith("_"):
92
+ continue
93
+ if key not in SCHEMA[section]:
94
+ raise ConfigError(f"{path}: unknown key in section '{section}' (allowed: {', '.join(SCHEMA[section])})")
95
+ allowed = SCHEMA[section][key]
96
+ if not _type_ok(value, allowed):
97
+ expected = " or ".join(_TYPE_NAMES[t] for t in allowed)
98
+ raise ConfigError(f"{path}: '{section}.{key}' must be {expected}")
99
+ if isinstance(value, list):
100
+ if not all(isinstance(item, str) and item.strip() for item in value):
101
+ raise ConfigError(f"{path}: '{section}.{key}' must be a list of non-empty strings")
102
+ value = [item.strip() for item in value]
103
+ choices = CHOICES.get((section, key))
104
+ if choices and value.strip() and value.strip() not in choices:
105
+ raise ConfigError(f"{path}: '{section}.{key}' must be one of: {', '.join(choices)}")
106
+ clean[key] = value.strip() if isinstance(value, str) else value
107
+ data[section] = clean
108
+
109
+ # Relative paths are relative to the config file, not to the current directory.
110
+ for section, key in (("scimago", "cache_dir"), ("screening", "venue_tiers")):
111
+ value = (data.get(section) or {}).get(key)
112
+ if value:
113
+ resolved = Path(value).expanduser()
114
+ data[section][key] = str(resolved if resolved.is_absolute() else path.parent / resolved)
115
+ return Config(path=path, data=data)
@@ -0,0 +1,29 @@
1
+ """Isolated standard-library persistence foundation; unused by the legacy CLI."""
2
+
3
+
4
+ class FoundationError(Exception):
5
+ """Base error for foundation operations (messages contain no input values)."""
6
+
7
+
8
+ class MigrationError(FoundationError):
9
+ pass
10
+
11
+
12
+ class MigrationDriftError(MigrationError):
13
+ pass
14
+
15
+
16
+ class ImmutabilityError(FoundationError):
17
+ pass
18
+
19
+
20
+ class SnapshotSealedError(ImmutabilityError):
21
+ pass
22
+
23
+
24
+ class ArtifactIntegrityError(FoundationError):
25
+ pass
26
+
27
+
28
+ class ValidationError(FoundationError):
29
+ pass
@@ -0,0 +1,100 @@
1
+ """Immutable exact-byte SHA-256 objects. Placement precedes database registration."""
2
+ import hashlib
3
+ import os
4
+ import re
5
+ import tempfile
6
+ from dataclasses import dataclass
7
+ from pathlib import Path
8
+ from typing import BinaryIO
9
+
10
+ from . import ArtifactIntegrityError, ValidationError
11
+
12
+
13
+ @dataclass(frozen=True)
14
+ class StoredArtifact:
15
+ artifact_sha256: str
16
+ byte_size: int
17
+ relative_path: str
18
+
19
+
20
+ class ContentAddressedArtifactStore:
21
+ def __init__(self, workspace: Path):
22
+ self.workspace = Path(workspace).resolve()
23
+ self.root = self.workspace / "raw" / "sha256"
24
+
25
+ def path_for_hash(self, digest: str) -> Path:
26
+ if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest):
27
+ raise ValidationError("Invalid artifact SHA-256")
28
+ path = self.root / digest[:2] / digest
29
+ if not path.resolve().is_relative_to(self.workspace):
30
+ raise ValidationError("Artifact path escapes workspace")
31
+ return path
32
+
33
+ def verify(self, digest: str) -> StoredArtifact:
34
+ path = self.path_for_hash(digest)
35
+ try:
36
+ hasher = hashlib.sha256()
37
+ size = 0
38
+ with path.open("rb") as handle:
39
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
40
+ hasher.update(chunk)
41
+ size += len(chunk)
42
+ if hasher.hexdigest() != digest:
43
+ raise ArtifactIntegrityError("Artifact hash mismatch")
44
+ return StoredArtifact(digest, size, path.relative_to(self.workspace).as_posix())
45
+ except OSError:
46
+ raise ArtifactIntegrityError("Artifact missing or unreadable") from None
47
+
48
+ @staticmethod
49
+ def _sync_directory(path: Path) -> None:
50
+ # Windows does not expose a portable directory fsync; file fsync is still mandatory.
51
+ if os.name == "posix":
52
+ fd = os.open(path, os.O_RDONLY)
53
+ try:
54
+ os.fsync(fd)
55
+ finally:
56
+ os.close(fd)
57
+
58
+ def _put_stream(self, stream: BinaryIO) -> StoredArtifact:
59
+ # Staging shares the destination filesystem. No user locator enters a filename.
60
+ if not self.root.resolve().is_relative_to(self.workspace):
61
+ raise ValidationError("Artifact directory escapes workspace")
62
+ self.root.mkdir(parents=True, exist_ok=True)
63
+ tmp: Path | None = None
64
+ try:
65
+ with tempfile.NamedTemporaryFile(dir=self.root, prefix=".put-", delete=False) as handle:
66
+ tmp = Path(handle.name)
67
+ digest = hashlib.sha256()
68
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
69
+ digest.update(chunk)
70
+ handle.write(chunk)
71
+ handle.flush()
72
+ os.fsync(handle.fileno())
73
+ key = digest.hexdigest()
74
+ target = self.path_for_hash(key)
75
+ target.parent.mkdir(parents=True, exist_ok=True)
76
+ if target.exists():
77
+ return self.verify(key)
78
+ os.replace(tmp, target)
79
+ # Persist the new file entry AND newly created shard/root directory entries.
80
+ for folder in (target.parent, self.root, self.root.parent, self.workspace):
81
+ self._sync_directory(folder)
82
+ return self.verify(key)
83
+ except OSError:
84
+ raise ArtifactIntegrityError("Artifact placement failed") from None
85
+ finally:
86
+ if tmp is not None:
87
+ tmp.unlink(missing_ok=True)
88
+
89
+ def put_bytes(self, data: bytes) -> StoredArtifact:
90
+ from io import BytesIO
91
+ if not isinstance(data, bytes):
92
+ raise ValidationError("Artifact input must be bytes")
93
+ return self._put_stream(BytesIO(data))
94
+
95
+ def put_file(self, path: Path) -> StoredArtifact:
96
+ try:
97
+ with Path(path).open("rb") as handle:
98
+ return self._put_stream(handle)
99
+ except OSError:
100
+ raise ArtifactIntegrityError("Artifact source missing or unreadable") from None
reprovane/core/db.py ADDED
@@ -0,0 +1,78 @@
1
+ """Explicit SQLite durability/transaction boundary; one connection per Database."""
2
+ import hashlib
3
+ import sqlite3
4
+ from contextlib import contextmanager
5
+ from pathlib import Path
6
+ from typing import Any, Iterator
7
+
8
+ from . import FoundationError, ImmutabilityError, SnapshotSealedError, ValidationError
9
+ from .migrations import MIGRATION_DIR, apply_migrations
10
+ from .provenance import sanitize_locator
11
+
12
+
13
+ class Database:
14
+ def __init__(self, workspace: Path, *, migration_dir: Path = MIGRATION_DIR, target_schema_version: int = 1):
15
+ self.workspace = Path(workspace).resolve()
16
+ folder = self.workspace / "db"
17
+ folder.mkdir(parents=True, exist_ok=True)
18
+ if not folder.resolve().is_relative_to(self.workspace):
19
+ raise ValidationError("Database directory escapes workspace")
20
+ self.path = folder / "litintel.sqlite"
21
+ if not self.path.resolve().is_relative_to(self.workspace):
22
+ raise ValidationError("Database path escapes workspace")
23
+ self._sealing_snapshot: str | None = None
24
+ self.connection = sqlite3.connect(self.path, isolation_level=None, timeout=5.0)
25
+ self.connection.row_factory = sqlite3.Row
26
+ self.connection.create_function("foundation_seal_authorized", 1,
27
+ lambda sid: int(sid == self._sealing_snapshot))
28
+ self.connection.create_function("foundation_sha256_text", 1,
29
+ lambda text: hashlib.sha256(text.encode("utf-8")).hexdigest(), deterministic=True)
30
+ self.connection.create_function("foundation_sanitize_locator", 1, sanitize_locator, deterministic=True)
31
+ try:
32
+ self.connection.execute("PRAGMA foreign_keys = ON")
33
+ mode = self.connection.execute("PRAGMA journal_mode = WAL").fetchone()[0]
34
+ self.connection.execute("PRAGMA synchronous = FULL")
35
+ self.connection.execute("PRAGMA busy_timeout = 5000")
36
+ # REPLACE must fire delete guards too; otherwise it could bypass immutable-row triggers.
37
+ self.connection.execute("PRAGMA recursive_triggers = ON")
38
+ if mode.lower() != "wal" or self.connection.execute("PRAGMA foreign_keys").fetchone()[0] != 1:
39
+ raise FoundationError("Required SQLite durability settings unavailable")
40
+ apply_migrations(self.connection, migration_dir, target_version=target_schema_version)
41
+ except Exception:
42
+ self.connection.close()
43
+ raise
44
+
45
+ def execute(self, sql: str, parameters: Any = ()) -> sqlite3.Cursor:
46
+ try:
47
+ return self.connection.execute(sql, parameters)
48
+ except sqlite3.IntegrityError as exc:
49
+ message = str(exc)
50
+ if "snapshot sealed" in message:
51
+ raise SnapshotSealedError("Snapshot is sealed") from None
52
+ if "immutable" in message or "seal transition" in message:
53
+ raise ImmutabilityError("Immutable object or unauthorized seal transition") from None
54
+ raise ValidationError("Database constraint rejected the operation") from None
55
+ except sqlite3.DatabaseError:
56
+ raise FoundationError("Database operation failed") from None
57
+
58
+ @contextmanager
59
+ def transaction(self) -> Iterator[None]:
60
+ if self.connection.in_transaction:
61
+ raise FoundationError("Nested transactions are unsupported; reuse the active transaction")
62
+ self.execute("BEGIN IMMEDIATE")
63
+ try:
64
+ yield
65
+ self.execute("COMMIT")
66
+ except BaseException:
67
+ if self.connection.in_transaction:
68
+ self.connection.execute("ROLLBACK")
69
+ raise
70
+
71
+ def close(self) -> None:
72
+ self.connection.close()
73
+
74
+ def __enter__(self):
75
+ return self
76
+
77
+ def __exit__(self, *exc):
78
+ self.close()