RadGraph-IT 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- radgraph_it-1.0.0.dist-info/METADATA +159 -0
- radgraph_it-1.0.0.dist-info/RECORD +39 -0
- radgraph_it-1.0.0.dist-info/WHEEL +4 -0
- radgraph_it-1.0.0.dist-info/entry_points.txt +2 -0
- radgraph_it-1.0.0.dist-info/licenses/LICENSE +23 -0
- radgraph_it-1.0.0.dist-info/licenses/LICENSES/Third-Party.md +41 -0
- radgraphit/__init__.py +47 -0
- radgraphit/api.py +168 -0
- radgraphit/artifacts/__init__.py +5 -0
- radgraphit/artifacts/manifest.py +290 -0
- radgraphit/artifacts/registry.py +24 -0
- radgraphit/artifacts/resolver.py +200 -0
- radgraphit/cli.py +131 -0
- radgraphit/core/__init__.py +1 -0
- radgraphit/core/errors.py +46 -0
- radgraphit/core/models.py +116 -0
- radgraphit/inference/__init__.py +5 -0
- radgraphit/inference/backends/__init__.py +1 -0
- radgraphit/inference/backends/base.py +13 -0
- radgraphit/inference/backends/dygie_v2/__init__.py +11 -0
- radgraphit/inference/backends/dygie_v2/backend.py +32 -0
- radgraphit/inference/backends/dygie_v2/device.py +46 -0
- radgraphit/inference/backends/dygie_v2/feedforward.py +25 -0
- radgraphit/inference/backends/dygie_v2/loader.py +165 -0
- radgraphit/inference/backends/dygie_v2/model.py +101 -0
- radgraphit/inference/backends/dygie_v2/ner_head.py +75 -0
- radgraphit/inference/backends/dygie_v2/pruner.py +27 -0
- radgraphit/inference/backends/dygie_v2/relation_head.py +147 -0
- radgraphit/inference/backends/dygie_v2/span_extractor.py +51 -0
- radgraphit/inference/backends/dygie_v2/spans.py +20 -0
- radgraphit/inference/backends/dygie_v2/tokenizer_embedder.py +123 -0
- radgraphit/inference/backends/dygie_v2/transformer_block.py +62 -0
- radgraphit/inference/backends/dygie_v2/vocab.py +123 -0
- radgraphit/inference/decoder.py +53 -0
- radgraphit/inference/service.py +80 -0
- radgraphit/inference/tokenizer.py +60 -0
- radgraphit/py.typed +0 -0
- radgraphit/serialization/__init__.py +15 -0
- radgraphit/serialization/radgraph_xl.py +135 -0
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
"""``model_manifest.json``: schema, parsing, and SHA-256 verification.
|
|
2
|
+
|
|
3
|
+
The manifest is versioned *inside the Hugging Face model repository*, alongside ``config.json``,
|
|
4
|
+
``vocab.json`` and the weights. It records what package/schema/backend the bundle targets, which
|
|
5
|
+
encoder it needs, the pinned revision, and a SHA-256 for every required file. The resolver verifies
|
|
6
|
+
it against the downloaded bytes **before** the model is built, so a corrupted or tampered download
|
|
7
|
+
fails loudly and early rather than as an obscure state-dict error.
|
|
8
|
+
|
|
9
|
+
Example ``model_manifest.json`` (see also ``scripts/publish_model.py``, which generates it):
|
|
10
|
+
|
|
11
|
+
.. code-block:: json
|
|
12
|
+
|
|
13
|
+
{
|
|
14
|
+
"manifest_version": 1,
|
|
15
|
+
"package": "radgraphit",
|
|
16
|
+
"min_package_version": "1.0.0",
|
|
17
|
+
"schema": "radgraph-xl",
|
|
18
|
+
"backend": "dygie_v2",
|
|
19
|
+
"encoder": {
|
|
20
|
+
"model_name": "IVN-RIN/medBIT-r3-plus",
|
|
21
|
+
"revision": "<immutable encoder commit sha>",
|
|
22
|
+
"max_length": 512
|
|
23
|
+
},
|
|
24
|
+
"revision": "<git-sha-or-tag of this bundle>",
|
|
25
|
+
"weights_file": "best.pt",
|
|
26
|
+
"files": {
|
|
27
|
+
"config.json": {"sha256": "…"},
|
|
28
|
+
"vocab.json": {"sha256": "…"},
|
|
29
|
+
"best.pt": {"sha256": "…"}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import hashlib
|
|
37
|
+
import json
|
|
38
|
+
import re
|
|
39
|
+
from dataclasses import dataclass
|
|
40
|
+
from pathlib import Path, PurePosixPath, PureWindowsPath
|
|
41
|
+
|
|
42
|
+
from packaging.version import InvalidVersion, Version
|
|
43
|
+
|
|
44
|
+
from ..core.errors import ManifestError
|
|
45
|
+
from . import registry
|
|
46
|
+
|
|
47
|
+
MANIFEST_VERSION = 1
|
|
48
|
+
_CHUNK = 1 << 20 # 1 MiB streaming reads, so large weight files don't load into memory to hash
|
|
49
|
+
_SHA256 = re.compile(r"^[0-9a-fA-F]{64}$")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def sha256_file(path: Path) -> str:
|
|
53
|
+
"""Return the hex SHA-256 of a file, read in streaming chunks."""
|
|
54
|
+
digest = hashlib.sha256()
|
|
55
|
+
with path.open("rb") as handle:
|
|
56
|
+
for chunk in iter(lambda: handle.read(_CHUNK), b""):
|
|
57
|
+
digest.update(chunk)
|
|
58
|
+
return digest.hexdigest()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _safe_filename(value: object, *, field: str) -> str:
|
|
62
|
+
if not isinstance(value, str) or not value:
|
|
63
|
+
raise ManifestError(f"Manifest field {field} must be a non-empty filename.")
|
|
64
|
+
posix = PurePosixPath(value)
|
|
65
|
+
windows = PureWindowsPath(value)
|
|
66
|
+
if (
|
|
67
|
+
posix.is_absolute()
|
|
68
|
+
or windows.is_absolute()
|
|
69
|
+
or len(posix.parts) != 1
|
|
70
|
+
or len(windows.parts) != 1
|
|
71
|
+
or value in {".", ".."}
|
|
72
|
+
):
|
|
73
|
+
raise ManifestError(
|
|
74
|
+
f"Manifest field {field} must be a plain filename inside the bundle, got {value!r}."
|
|
75
|
+
)
|
|
76
|
+
return value
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _positive_int(value: object, *, field: str) -> int:
|
|
80
|
+
if not isinstance(value, int) or isinstance(value, bool):
|
|
81
|
+
raise ManifestError(f"Manifest field {field} must be a positive integer.")
|
|
82
|
+
if value <= 0:
|
|
83
|
+
raise ManifestError(f"Manifest field {field} must be a positive integer.")
|
|
84
|
+
return value
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _non_empty_string(value: object, *, field: str) -> str:
|
|
88
|
+
if not isinstance(value, str) or not value.strip():
|
|
89
|
+
raise ManifestError(f"Manifest field {field} must be a non-empty string.")
|
|
90
|
+
return value
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@dataclass(frozen=True, slots=True)
|
|
94
|
+
class ModelManifest:
|
|
95
|
+
"""Parsed, validated model manifest."""
|
|
96
|
+
|
|
97
|
+
manifest_version: int
|
|
98
|
+
package: str
|
|
99
|
+
min_package_version: str
|
|
100
|
+
schema: str
|
|
101
|
+
backend: str
|
|
102
|
+
encoder_model_name: str
|
|
103
|
+
encoder_revision: str
|
|
104
|
+
encoder_max_length: int
|
|
105
|
+
revision: str
|
|
106
|
+
weights_file: str
|
|
107
|
+
files: dict[str, str] # filename -> expected sha256
|
|
108
|
+
|
|
109
|
+
@classmethod
|
|
110
|
+
def from_path(cls, path: Path) -> ModelManifest:
|
|
111
|
+
"""Parse and structurally validate a ``model_manifest.json`` file."""
|
|
112
|
+
if not path.is_file():
|
|
113
|
+
raise ManifestError(
|
|
114
|
+
f"Model manifest not found at {path}. A valid RadGraphIT bundle must contain "
|
|
115
|
+
f"{registry.MANIFEST_FILENAME} alongside its config, vocabulary, and weights."
|
|
116
|
+
)
|
|
117
|
+
try:
|
|
118
|
+
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
119
|
+
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
120
|
+
raise ManifestError(f"Could not read a valid JSON manifest at {path}: {exc}") from exc
|
|
121
|
+
if not isinstance(raw, dict):
|
|
122
|
+
raise ManifestError(f"Model manifest at {path} must be a JSON object.")
|
|
123
|
+
|
|
124
|
+
try:
|
|
125
|
+
encoder = raw["encoder"]
|
|
126
|
+
files_raw = raw["files"]
|
|
127
|
+
if not isinstance(encoder, dict):
|
|
128
|
+
raise TypeError("'encoder' must be an object")
|
|
129
|
+
if not isinstance(files_raw, dict) or not files_raw:
|
|
130
|
+
raise TypeError("'files' must be a non-empty object")
|
|
131
|
+
files: dict[str, str] = {}
|
|
132
|
+
for name, metadata in files_raw.items():
|
|
133
|
+
filename = _safe_filename(name, field="files key")
|
|
134
|
+
if not isinstance(metadata, dict):
|
|
135
|
+
raise TypeError(f"metadata for {filename!r} must be an object")
|
|
136
|
+
expected = metadata["sha256"]
|
|
137
|
+
if not isinstance(expected, str) or not _SHA256.fullmatch(expected):
|
|
138
|
+
raise ValueError(f"sha256 for {filename!r} must contain exactly 64 hex digits")
|
|
139
|
+
files[filename] = expected.lower()
|
|
140
|
+
|
|
141
|
+
weights_file = _safe_filename(raw["weights_file"], field="weights_file")
|
|
142
|
+
if weights_file not in registry.WEIGHT_FILENAMES:
|
|
143
|
+
raise ValueError(
|
|
144
|
+
f"weights_file must be one of {', '.join(registry.WEIGHT_FILENAMES)}"
|
|
145
|
+
)
|
|
146
|
+
encoder_model_name = encoder["model_name"]
|
|
147
|
+
if not isinstance(encoder_model_name, str) or not encoder_model_name.strip():
|
|
148
|
+
raise ValueError("encoder.model_name must be a non-empty string")
|
|
149
|
+
if "revision" in encoder:
|
|
150
|
+
encoder_revision = encoder["revision"]
|
|
151
|
+
else:
|
|
152
|
+
encoder_revision = registry.ENCODER_REVISIONS.get(encoder_model_name)
|
|
153
|
+
if not isinstance(encoder_revision, str) or not encoder_revision.strip():
|
|
154
|
+
raise ValueError(
|
|
155
|
+
"encoder.revision must contain an immutable commit SHA or release tag"
|
|
156
|
+
)
|
|
157
|
+
manifest_version = raw["manifest_version"]
|
|
158
|
+
if not isinstance(manifest_version, int) or isinstance(manifest_version, bool):
|
|
159
|
+
raise ValueError("manifest_version must be an integer")
|
|
160
|
+
|
|
161
|
+
manifest = cls(
|
|
162
|
+
manifest_version=manifest_version,
|
|
163
|
+
package=_non_empty_string(raw["package"], field="package"),
|
|
164
|
+
min_package_version=_non_empty_string(
|
|
165
|
+
raw["min_package_version"], field="min_package_version"
|
|
166
|
+
),
|
|
167
|
+
schema=_non_empty_string(raw["schema"], field="schema"),
|
|
168
|
+
backend=_non_empty_string(raw["backend"], field="backend"),
|
|
169
|
+
encoder_model_name=encoder_model_name,
|
|
170
|
+
encoder_revision=encoder_revision,
|
|
171
|
+
encoder_max_length=_positive_int(encoder["max_length"], field="encoder.max_length"),
|
|
172
|
+
revision=_non_empty_string(raw["revision"], field="revision"),
|
|
173
|
+
weights_file=weights_file,
|
|
174
|
+
files=files,
|
|
175
|
+
)
|
|
176
|
+
except ManifestError:
|
|
177
|
+
raise
|
|
178
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
179
|
+
raise ManifestError(
|
|
180
|
+
f"Model manifest at {path} is missing or has a malformed field: {exc}"
|
|
181
|
+
) from exc
|
|
182
|
+
required = {
|
|
183
|
+
registry.CONFIG_FILENAME,
|
|
184
|
+
registry.VOCAB_FILENAME,
|
|
185
|
+
manifest.weights_file,
|
|
186
|
+
}
|
|
187
|
+
missing_checksums = required.difference(manifest.files)
|
|
188
|
+
if missing_checksums:
|
|
189
|
+
raise ManifestError(
|
|
190
|
+
"Model manifest must provide SHA-256 checksums for every required bundle file; "
|
|
191
|
+
f"missing: {', '.join(sorted(missing_checksums))}."
|
|
192
|
+
)
|
|
193
|
+
return manifest
|
|
194
|
+
|
|
195
|
+
def check_compatibility(self, package_version: str) -> None:
|
|
196
|
+
"""Validate manifest/backend/schema/version compatibility with this package.
|
|
197
|
+
|
|
198
|
+
Raises :class:`ManifestError` on any incompatibility, with a message naming the mismatch.
|
|
199
|
+
"""
|
|
200
|
+
if self.manifest_version != MANIFEST_VERSION:
|
|
201
|
+
if self.manifest_version > MANIFEST_VERSION:
|
|
202
|
+
raise ManifestError(
|
|
203
|
+
f"Manifest version {self.manifest_version} is newer than this package supports "
|
|
204
|
+
f"({MANIFEST_VERSION}). Upgrade radgraphit."
|
|
205
|
+
)
|
|
206
|
+
raise ManifestError(
|
|
207
|
+
f"Manifest version {self.manifest_version} is unsupported; expected "
|
|
208
|
+
f"{MANIFEST_VERSION}."
|
|
209
|
+
)
|
|
210
|
+
if self.package != "radgraphit":
|
|
211
|
+
raise ManifestError(f"Manifest targets package {self.package!r}, not 'radgraphit'.")
|
|
212
|
+
if self.backend not in registry.SUPPORTED_BACKENDS:
|
|
213
|
+
raise ManifestError(
|
|
214
|
+
f"Manifest backend {self.backend!r} is not supported by this package "
|
|
215
|
+
f"(supported: {sorted(registry.SUPPORTED_BACKENDS)})."
|
|
216
|
+
)
|
|
217
|
+
if self.schema not in registry.SUPPORTED_SCHEMAS:
|
|
218
|
+
raise ManifestError(
|
|
219
|
+
f"Manifest schema {self.schema!r} is not supported by this package "
|
|
220
|
+
f"(supported: {sorted(registry.SUPPORTED_SCHEMAS)})."
|
|
221
|
+
)
|
|
222
|
+
try:
|
|
223
|
+
installed = Version(package_version)
|
|
224
|
+
required = Version(self.min_package_version)
|
|
225
|
+
except InvalidVersion as exc:
|
|
226
|
+
raise ManifestError(f"Manifest or package contains an invalid version: {exc}") from exc
|
|
227
|
+
if installed < required:
|
|
228
|
+
raise ManifestError(
|
|
229
|
+
f"This model requires radgraphit >= {self.min_package_version}, but "
|
|
230
|
+
f"{package_version} is installed. Upgrade radgraphit."
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
def verify_files(self, directory: Path) -> None:
|
|
234
|
+
"""Verify every file listed in the manifest exists in ``directory`` with the right SHA-256.
|
|
235
|
+
|
|
236
|
+
Also checks that the manifest's encoder metadata agrees with ``config.json``.
|
|
237
|
+
Raises :class:`ManifestError` naming the first inconsistency.
|
|
238
|
+
"""
|
|
239
|
+
for filename, expected in self.files.items():
|
|
240
|
+
file_path = directory / filename
|
|
241
|
+
if not file_path.is_file():
|
|
242
|
+
raise ManifestError(
|
|
243
|
+
f"Manifest lists required file {filename!r} but it is missing from {directory}."
|
|
244
|
+
)
|
|
245
|
+
try:
|
|
246
|
+
actual = sha256_file(file_path)
|
|
247
|
+
except OSError as exc:
|
|
248
|
+
raise ManifestError(f"Could not hash manifest file {file_path}: {exc}") from exc
|
|
249
|
+
if actual != expected:
|
|
250
|
+
raise ManifestError(
|
|
251
|
+
f"Checksum mismatch for {filename!r} in {directory}: manifest expects "
|
|
252
|
+
f"{expected}, downloaded file is {actual}. The bundle is corrupted or has been "
|
|
253
|
+
"tampered with; delete the cache directory and re-download."
|
|
254
|
+
)
|
|
255
|
+
config_path = directory / registry.CONFIG_FILENAME
|
|
256
|
+
try:
|
|
257
|
+
config = json.loads(config_path.read_text(encoding="utf-8"))
|
|
258
|
+
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
259
|
+
raise ManifestError(
|
|
260
|
+
f"Could not read a valid JSON config at {config_path}: {exc}"
|
|
261
|
+
) from exc
|
|
262
|
+
if not isinstance(config, dict) or not isinstance(config.get("encoder"), dict):
|
|
263
|
+
raise ManifestError(f"{config_path} must contain an 'encoder' object.")
|
|
264
|
+
encoder = config["encoder"]
|
|
265
|
+
if encoder.get("model_name") != self.encoder_model_name:
|
|
266
|
+
raise ManifestError(
|
|
267
|
+
"Manifest encoder.model_name does not match config.json: "
|
|
268
|
+
f"{self.encoder_model_name!r} != {encoder.get('model_name')!r}."
|
|
269
|
+
)
|
|
270
|
+
try:
|
|
271
|
+
config_max_length = _positive_int(
|
|
272
|
+
encoder.get("max_length"), field="config encoder.max_length"
|
|
273
|
+
)
|
|
274
|
+
except ManifestError as exc:
|
|
275
|
+
raise ManifestError(f"Invalid encoder configuration in {config_path}: {exc}") from exc
|
|
276
|
+
if config_max_length != self.encoder_max_length:
|
|
277
|
+
raise ManifestError(
|
|
278
|
+
"Manifest encoder.max_length does not match config.json: "
|
|
279
|
+
f"{self.encoder_max_length} != {config_max_length}."
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
def required_filenames(self) -> tuple[str, ...]:
|
|
283
|
+
"""All filenames the manifest declares (used to drive selective downloads)."""
|
|
284
|
+
names = set(self.files) | {
|
|
285
|
+
registry.MANIFEST_FILENAME,
|
|
286
|
+
registry.CONFIG_FILENAME,
|
|
287
|
+
registry.VOCAB_FILENAME,
|
|
288
|
+
self.weights_file,
|
|
289
|
+
}
|
|
290
|
+
return tuple(sorted(names))
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""The single source of truth for the default model location and supported bundle schema."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
DEFAULT_MODEL_ID = "radgraphIT/Radgraph-IT-v1"
|
|
6
|
+
DEFAULT_REVISION = "368c01baddbe1bdfb1b439243ae0c224f0e6624d"
|
|
7
|
+
ENCODER_REVISIONS = {
|
|
8
|
+
"IVN-RIN/medBIT-r3-plus": "0f1c91e551551869a72935962b54c1f5247333a4",
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
#: Files the model bundle must contain on the Hub / in a local directory.
|
|
12
|
+
MANIFEST_FILENAME = "model_manifest.json"
|
|
13
|
+
CONFIG_FILENAME = "config.json"
|
|
14
|
+
VOCAB_FILENAME = "vocab.json"
|
|
15
|
+
|
|
16
|
+
#: Weight file candidates, in preference order (safetensors preferred once published; the current
|
|
17
|
+
#: v2 artifact ships ``best.pt``).
|
|
18
|
+
WEIGHT_FILENAMES = ("model.safetensors", "best.pt")
|
|
19
|
+
|
|
20
|
+
#: Backend identifiers this package version knows how to build.
|
|
21
|
+
SUPPORTED_BACKENDS = frozenset({"dygie_v2"})
|
|
22
|
+
|
|
23
|
+
#: Manifest ``schema`` values this package version can serialize to.
|
|
24
|
+
SUPPORTED_SCHEMAS = frozenset({"radgraph-xl"})
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"""Resolve a model bundle to a verified local directory.
|
|
2
|
+
|
|
3
|
+
Two entry points:
|
|
4
|
+
|
|
5
|
+
* :func:`resolve_pretrained` — the standard path. Downloads only the files declared by the bundle's
|
|
6
|
+
manifest from an explicitly pinned Hugging Face Hub revision, caches them, and verifies
|
|
7
|
+
compatibility + SHA-256 before returning.
|
|
8
|
+
* :func:`resolve_local` — for development, offline testing, and diagnostics. Points at an existing
|
|
9
|
+
local directory and requires the exact same manifest verification.
|
|
10
|
+
|
|
11
|
+
Both return a :class:`ResolvedModel` (a verified directory + parsed manifest). Neither ever falls
|
|
12
|
+
back to unverified loading: a missing manifest, an unreachable repository, or a bad checksum raises
|
|
13
|
+
a specific, actionable error.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
import logging
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
22
|
+
from pathlib import Path, PurePosixPath
|
|
23
|
+
|
|
24
|
+
from huggingface_hub import hf_hub_download, snapshot_download
|
|
25
|
+
|
|
26
|
+
from ..core.errors import ModelNotConfiguredError, ModelNotFoundError
|
|
27
|
+
from . import registry
|
|
28
|
+
from .manifest import ModelManifest
|
|
29
|
+
|
|
30
|
+
logger = logging.getLogger("radgraphit")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True, slots=True)
|
|
34
|
+
class ResolvedModel:
|
|
35
|
+
"""A verified, on-disk model bundle."""
|
|
36
|
+
|
|
37
|
+
directory: Path
|
|
38
|
+
manifest: ModelManifest
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def config_path(self) -> Path:
|
|
42
|
+
return self.directory / registry.CONFIG_FILENAME
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def vocab_path(self) -> Path:
|
|
46
|
+
return self.directory / registry.VOCAB_FILENAME
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def weights_path(self) -> Path:
|
|
50
|
+
return self.directory / self.manifest.weights_file
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _package_version() -> str:
|
|
54
|
+
try:
|
|
55
|
+
return version("radgraphit")
|
|
56
|
+
except PackageNotFoundError: # not installed (e.g. running from a source checkout)
|
|
57
|
+
from .. import __version__
|
|
58
|
+
|
|
59
|
+
return __version__
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
_BUNDLE_SUBDIRS = ("", "model") # search the snapshot root first, then a conventional model/ subdir
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _looks_like_bundle(directory: Path) -> bool:
|
|
66
|
+
"""Whether ``directory`` holds a dygie training bundle (config + vocab + weights).
|
|
67
|
+
|
|
68
|
+
The ``config.json`` must carry an ``"encoder"`` block, which distinguishes the training config
|
|
69
|
+
from an unrelated Hugging Face ``config.json`` that may sit at the repo root next to custom
|
|
70
|
+
modeling code (as in the published RadGraphIT repo, whose bundle lives under ``model/``).
|
|
71
|
+
"""
|
|
72
|
+
if not directory.is_dir():
|
|
73
|
+
return False
|
|
74
|
+
config_path = directory / registry.CONFIG_FILENAME
|
|
75
|
+
if not config_path.is_file() or not (directory / registry.VOCAB_FILENAME).is_file():
|
|
76
|
+
return False
|
|
77
|
+
if not any((directory / weights).is_file() for weights in registry.WEIGHT_FILENAMES):
|
|
78
|
+
return False
|
|
79
|
+
try:
|
|
80
|
+
config = json.loads(config_path.read_text(encoding="utf-8"))
|
|
81
|
+
except (json.JSONDecodeError, OSError, UnicodeError):
|
|
82
|
+
return False
|
|
83
|
+
return isinstance(config, dict) and "encoder" in config
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _locate_bundle(root: Path) -> Path:
|
|
87
|
+
"""Return the directory holding the dygie bundle (``root`` or its ``model/`` subdir)."""
|
|
88
|
+
for sub in _BUNDLE_SUBDIRS:
|
|
89
|
+
candidate = root / sub if sub else root
|
|
90
|
+
if _looks_like_bundle(candidate):
|
|
91
|
+
return candidate
|
|
92
|
+
raise ModelNotFoundError(
|
|
93
|
+
f"No RadGraphIT model bundle found under {root} (looked in {root} and {root / 'model'}). "
|
|
94
|
+
f"A bundle must contain {registry.CONFIG_FILENAME} (with an 'encoder' block), "
|
|
95
|
+
f"{registry.VOCAB_FILENAME}, and a weights file ({' or '.join(registry.WEIGHT_FILENAMES)})."
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _verify(directory: Path) -> ResolvedModel:
|
|
100
|
+
"""Locate a manifest-backed bundle within ``directory`` and validate it completely."""
|
|
101
|
+
bundle = _locate_bundle(directory)
|
|
102
|
+
manifest_path = bundle / registry.MANIFEST_FILENAME
|
|
103
|
+
manifest = ModelManifest.from_path(manifest_path)
|
|
104
|
+
manifest.check_compatibility(_package_version())
|
|
105
|
+
manifest.verify_files(bundle)
|
|
106
|
+
logger.debug("Verified model bundle at %s (revision %s)", bundle, manifest.revision)
|
|
107
|
+
return ResolvedModel(directory=bundle, manifest=manifest)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def resolve_local(path: str | Path) -> ResolvedModel:
|
|
111
|
+
"""Resolve and verify a model bundle in an existing local directory.
|
|
112
|
+
|
|
113
|
+
Raises :class:`ModelNotFoundError` if the directory does not exist, and :class:`ManifestError`
|
|
114
|
+
(via :func:`_verify`) if its required manifest is missing, incompatible, or fails verification.
|
|
115
|
+
"""
|
|
116
|
+
directory = Path(path).expanduser()
|
|
117
|
+
if not directory.is_dir():
|
|
118
|
+
raise ModelNotFoundError(
|
|
119
|
+
f"Local model directory {directory} does not exist or is not a directory. Point "
|
|
120
|
+
"RadGraphIT.from_local(...) at a directory containing model_manifest.json, "
|
|
121
|
+
"config.json, vocab.json and the declared weights file, or at a parent holding them "
|
|
122
|
+
"under model/."
|
|
123
|
+
)
|
|
124
|
+
return _verify(directory)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def resolve_pretrained(
|
|
128
|
+
model_id: str,
|
|
129
|
+
*,
|
|
130
|
+
revision: str,
|
|
131
|
+
cache_dir: str | Path | None = None,
|
|
132
|
+
local_files_only: bool = False,
|
|
133
|
+
) -> ResolvedModel:
|
|
134
|
+
"""Download (or reuse cached) the model bundle from the Hub and verify it.
|
|
135
|
+
|
|
136
|
+
``model_id`` and ``revision`` must be non-empty. Revisions should be immutable commit SHAs or
|
|
137
|
+
tags; the package default is pinned to an exact commit.
|
|
138
|
+
|
|
139
|
+
The default RadGraphIT model is a **public, ungated** Hub repository, so no authentication is
|
|
140
|
+
required — anyone can download it with a plain ``pip install``.
|
|
141
|
+
|
|
142
|
+
Raises :class:`ModelNotFoundError` if the download fails for any reason (unknown repo, network,
|
|
143
|
+
offline-with-no-cache, or a repository that is private/gated instead of public).
|
|
144
|
+
"""
|
|
145
|
+
if not isinstance(model_id, str) or not model_id.strip():
|
|
146
|
+
raise ModelNotConfiguredError("model_id must be a non-empty Hugging Face repository id.")
|
|
147
|
+
if not isinstance(revision, str) or not revision.strip():
|
|
148
|
+
raise ModelNotConfiguredError(
|
|
149
|
+
"revision must be a non-empty immutable commit SHA or release tag."
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
manifest_locations = (
|
|
153
|
+
registry.MANIFEST_FILENAME,
|
|
154
|
+
f"model/{registry.MANIFEST_FILENAME}",
|
|
155
|
+
)
|
|
156
|
+
manifest_path: Path | None = None
|
|
157
|
+
manifest_location: str | None = None
|
|
158
|
+
manifest_errors: list[str] = []
|
|
159
|
+
for candidate in manifest_locations:
|
|
160
|
+
try:
|
|
161
|
+
downloaded = hf_hub_download(
|
|
162
|
+
repo_id=model_id,
|
|
163
|
+
filename=candidate,
|
|
164
|
+
revision=revision,
|
|
165
|
+
cache_dir=str(cache_dir) if cache_dir is not None else None,
|
|
166
|
+
local_files_only=local_files_only,
|
|
167
|
+
)
|
|
168
|
+
except Exception as exc:
|
|
169
|
+
manifest_errors.append(f"{candidate}: {exc}")
|
|
170
|
+
continue
|
|
171
|
+
manifest_path = Path(downloaded)
|
|
172
|
+
manifest_location = candidate
|
|
173
|
+
break
|
|
174
|
+
if manifest_path is None or manifest_location is None:
|
|
175
|
+
details = "; ".join(manifest_errors)
|
|
176
|
+
raise ModelNotFoundError(
|
|
177
|
+
f"Could not find {registry.MANIFEST_FILENAME} for model {model_id!r} at revision "
|
|
178
|
+
f"{revision!r}. A verified manifest is mandatory. Details: {details}"
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
manifest = ModelManifest.from_path(manifest_path)
|
|
182
|
+
manifest.check_compatibility(_package_version())
|
|
183
|
+
bundle_prefix = PurePosixPath(manifest_location).parent
|
|
184
|
+
prefix = "" if str(bundle_prefix) == "." else f"{bundle_prefix.as_posix()}/"
|
|
185
|
+
allow_patterns = [f"{prefix}{name}" for name in manifest.required_filenames()]
|
|
186
|
+
try:
|
|
187
|
+
local_dir = snapshot_download(
|
|
188
|
+
repo_id=model_id,
|
|
189
|
+
revision=revision,
|
|
190
|
+
cache_dir=str(cache_dir) if cache_dir is not None else None,
|
|
191
|
+
local_files_only=local_files_only,
|
|
192
|
+
allow_patterns=allow_patterns,
|
|
193
|
+
)
|
|
194
|
+
except Exception as exc:
|
|
195
|
+
raise ModelNotFoundError(
|
|
196
|
+
f"Could not download model {model_id!r} at revision {revision!r} from the Hugging "
|
|
197
|
+
f"Face Hub: {exc}. Check the repository id, the revision, your network connection, and "
|
|
198
|
+
"that the repository is public and ungated."
|
|
199
|
+
) from exc
|
|
200
|
+
return _verify(Path(local_dir))
|
radgraphit/cli.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Command-line interface: ``radgraphit predict``.
|
|
2
|
+
|
|
3
|
+
The CLI is the one place the package writes to stdout (the library itself never prints). It reads
|
|
4
|
+
one or more reports — from positional arguments, a ``--input-file`` (one report per line), or
|
|
5
|
+
stdin — runs inference, and writes the RadGraph-XL dictionary as JSON to stdout.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import logging
|
|
13
|
+
import sys
|
|
14
|
+
from collections.abc import Sequence
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from . import __version__
|
|
18
|
+
from .api import RadGraphIT
|
|
19
|
+
from .core.errors import RadGraphITError
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
23
|
+
parser = argparse.ArgumentParser(
|
|
24
|
+
prog="radgraphit",
|
|
25
|
+
description="Extract the RadGraph-XL graph from Italian radiology reports.",
|
|
26
|
+
)
|
|
27
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
28
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
29
|
+
|
|
30
|
+
predict = subparsers.add_parser("predict", help="Annotate one or more reports.")
|
|
31
|
+
source = predict.add_mutually_exclusive_group()
|
|
32
|
+
source.add_argument(
|
|
33
|
+
"--model-dir",
|
|
34
|
+
help="Path to a local model bundle (config.json, vocab.json, "
|
|
35
|
+
"model_manifest.json, weights). Use this for development/offline.",
|
|
36
|
+
)
|
|
37
|
+
source.add_argument(
|
|
38
|
+
"--model-id",
|
|
39
|
+
help="Hugging Face model repository (standard path). Default: the registry.",
|
|
40
|
+
)
|
|
41
|
+
predict.add_argument(
|
|
42
|
+
"--revision",
|
|
43
|
+
help="Revision (commit sha or tag) of the Hugging Face repository. Used with --model-id.",
|
|
44
|
+
)
|
|
45
|
+
predict.add_argument(
|
|
46
|
+
"--cache-dir", help="Cache directory for the model download (default: HF cache)."
|
|
47
|
+
)
|
|
48
|
+
predict.add_argument(
|
|
49
|
+
"--device",
|
|
50
|
+
default="auto",
|
|
51
|
+
help="Device: auto (default), cpu, cuda, cuda:<index>.",
|
|
52
|
+
)
|
|
53
|
+
predict.add_argument(
|
|
54
|
+
"--input-file",
|
|
55
|
+
help="Text file with one report per line (alternative to positional arguments).",
|
|
56
|
+
)
|
|
57
|
+
predict.add_argument(
|
|
58
|
+
"--indent", type=int, default=2, help="Indentation of the output JSON (default: 2)."
|
|
59
|
+
)
|
|
60
|
+
predict.add_argument(
|
|
61
|
+
"reports",
|
|
62
|
+
nargs="*",
|
|
63
|
+
help="One or more reports as arguments. If absent, reads from --input-file or stdin.",
|
|
64
|
+
)
|
|
65
|
+
return parser
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _collect_reports(args: argparse.Namespace) -> list[str]:
|
|
69
|
+
"""Gather reports from the single input source validated by :func:`main`."""
|
|
70
|
+
if args.reports:
|
|
71
|
+
return list(args.reports)
|
|
72
|
+
if args.input_file:
|
|
73
|
+
with Path(args.input_file).open(encoding="utf-8") as handle:
|
|
74
|
+
return [line.strip() for line in handle if line.strip()]
|
|
75
|
+
data = sys.stdin.read()
|
|
76
|
+
return [line.strip() for line in data.splitlines() if line.strip()]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _build_predictor(args: argparse.Namespace) -> RadGraphIT:
|
|
80
|
+
if args.model_dir:
|
|
81
|
+
return RadGraphIT.from_local(args.model_dir, device=args.device)
|
|
82
|
+
kwargs: dict[str, object] = {"device": args.device, "cache_dir": args.cache_dir}
|
|
83
|
+
if args.model_id:
|
|
84
|
+
kwargs["model_id"] = args.model_id
|
|
85
|
+
if args.revision:
|
|
86
|
+
kwargs["revision"] = args.revision
|
|
87
|
+
return RadGraphIT.from_pretrained(**kwargs) # type: ignore[arg-type]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
91
|
+
"""CLI entry point. Returns a process exit code."""
|
|
92
|
+
logging.basicConfig(level=logging.WARNING, format="%(name)s: %(message)s")
|
|
93
|
+
parser = _build_parser()
|
|
94
|
+
args = parser.parse_args(argv)
|
|
95
|
+
|
|
96
|
+
if args.command == "predict":
|
|
97
|
+
if args.reports and args.input_file:
|
|
98
|
+
print(
|
|
99
|
+
"radgraphit: positional reports and --input-file are mutually exclusive.",
|
|
100
|
+
file=sys.stderr,
|
|
101
|
+
)
|
|
102
|
+
return 2
|
|
103
|
+
if args.model_dir and (args.revision or args.cache_dir):
|
|
104
|
+
print(
|
|
105
|
+
"radgraphit: --revision/--cache-dir cannot be used with --model-dir.",
|
|
106
|
+
file=sys.stderr,
|
|
107
|
+
)
|
|
108
|
+
return 2
|
|
109
|
+
try:
|
|
110
|
+
reports = _collect_reports(args)
|
|
111
|
+
if not reports:
|
|
112
|
+
print("radgraphit: no report provided.", file=sys.stderr)
|
|
113
|
+
return 2
|
|
114
|
+
predictor = _build_predictor(args)
|
|
115
|
+
annotations = predictor.predict(reports)
|
|
116
|
+
except RadGraphITError as exc:
|
|
117
|
+
print(f"radgraphit: {exc}", file=sys.stderr)
|
|
118
|
+
return 1
|
|
119
|
+
except (OSError, UnicodeError) as exc:
|
|
120
|
+
print(f"radgraphit: {exc}", file=sys.stderr)
|
|
121
|
+
return 1
|
|
122
|
+
json.dump(annotations, sys.stdout, indent=args.indent, ensure_ascii=False)
|
|
123
|
+
sys.stdout.write("\n")
|
|
124
|
+
return 0
|
|
125
|
+
|
|
126
|
+
parser.error(f"unknown command {args.command!r}") # argparse exits; unreachable return below
|
|
127
|
+
return 2
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
if __name__ == "__main__":
|
|
131
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Core domain: immutable value objects, typed protocols, and the error hierarchy."""
|