doctosae 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- doctosae/__init__.py +98 -0
- doctosae/_ids.py +132 -0
- doctosae/cli.py +1035 -0
- doctosae/data/__init__.py +39 -0
- doctosae/data/dataset.py +278 -0
- doctosae/data/loaders.py +570 -0
- doctosae/data/registry.py +236 -0
- doctosae/data/splits.py +140 -0
- doctosae/embed.py +636 -0
- doctosae/hub.py +330 -0
- doctosae/interpret/__init__.py +49 -0
- doctosae/interpret/activations.py +320 -0
- doctosae/interpret/cards.py +157 -0
- doctosae/interpret/explain.py +184 -0
- doctosae/interpret/interpreter.py +483 -0
- doctosae/interpret/labeling.py +257 -0
- doctosae/interpret/llm.py +383 -0
- doctosae/interpret/prompts.py +285 -0
- doctosae/interpret/scoring.py +677 -0
- doctosae/interpret/selection.py +102 -0
- doctosae/interpret/tfidf.py +86 -0
- doctosae/metrics.py +303 -0
- doctosae/py.typed +0 -0
- doctosae/retrieval.py +165 -0
- doctosae/sae.py +387 -0
- doctosae/training/__init__.py +6 -0
- doctosae/training/config.py +116 -0
- doctosae/training/losses.py +111 -0
- doctosae/training/sampling.py +155 -0
- doctosae/training/trainer.py +820 -0
- doctosae-0.1.0.dist-info/METADATA +181 -0
- doctosae-0.1.0.dist-info/RECORD +35 -0
- doctosae-0.1.0.dist-info/WHEEL +4 -0
- doctosae-0.1.0.dist-info/entry_points.txt +2 -0
- doctosae-0.1.0.dist-info/licenses/LICENSE +202 -0
doctosae/__init__.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""DoctoSAE: sparse autoencoders for interpretable and transferable medical information retrieval.
|
|
2
|
+
|
|
3
|
+
Reference implementation of "Explanations that Transfer: Interpreting Medical Retrieval with Sparse
|
|
4
|
+
AutoEncoders" (ML4H 2026). A TopK sparse autoencoder is trained on the frozen embeddings of a text
|
|
5
|
+
encoder, with a reconstruction term and an InfoNCE term on the sparse codes; retrieval then runs on the
|
|
6
|
+
sparse codes and every feature can be described in natural language.
|
|
7
|
+
|
|
8
|
+
The public names below are imported on first access, so ``import doctosae`` (and the ``doctosae`` command
|
|
9
|
+
line) stays fast: PyTorch, pandas and sentence transformers load only when a model or a dataset is
|
|
10
|
+
used.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import importlib
|
|
16
|
+
from typing import TYPE_CHECKING, Any
|
|
17
|
+
|
|
18
|
+
__version__ = "0.1.0"
|
|
19
|
+
|
|
20
|
+
# Public name to the module that defines it.
|
|
21
|
+
_EXPORTS: dict[str, str] = {
|
|
22
|
+
"DoctoSAE": "doctosae.sae",
|
|
23
|
+
"load_pretrained": "doctosae.hub",
|
|
24
|
+
"list_pretrained": "doctosae.hub",
|
|
25
|
+
"pretrained_repo_id": "doctosae.hub",
|
|
26
|
+
"pretrained_checkpoint_id": "doctosae.hub",
|
|
27
|
+
"from_paper_checkpoint": "doctosae.hub",
|
|
28
|
+
"RetrievalDataset": "doctosae.data",
|
|
29
|
+
"load_dataset": "doctosae.data",
|
|
30
|
+
"list_datasets": "doctosae.data",
|
|
31
|
+
"load_beir_folder": "doctosae.data",
|
|
32
|
+
"Embedder": "doctosae.embed",
|
|
33
|
+
"EmbeddedDataset": "doctosae.embed",
|
|
34
|
+
"Trainer": "doctosae.training",
|
|
35
|
+
"TrainingConfig": "doctosae.training",
|
|
36
|
+
"evaluate": "doctosae.metrics",
|
|
37
|
+
"encode_corpus": "doctosae.retrieval",
|
|
38
|
+
"search": "doctosae.retrieval",
|
|
39
|
+
"Interpreter": "doctosae.interpret",
|
|
40
|
+
"FeatureCard": "doctosae.interpret",
|
|
41
|
+
"OpenAICompatibleLLM": "doctosae.interpret",
|
|
42
|
+
"TransformersLLM": "doctosae.interpret",
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
__all__ = [
|
|
46
|
+
"__version__",
|
|
47
|
+
"DoctoSAE",
|
|
48
|
+
"load_pretrained",
|
|
49
|
+
"list_pretrained",
|
|
50
|
+
"pretrained_repo_id",
|
|
51
|
+
"pretrained_checkpoint_id",
|
|
52
|
+
"from_paper_checkpoint",
|
|
53
|
+
"RetrievalDataset",
|
|
54
|
+
"load_dataset",
|
|
55
|
+
"list_datasets",
|
|
56
|
+
"load_beir_folder",
|
|
57
|
+
"Embedder",
|
|
58
|
+
"EmbeddedDataset",
|
|
59
|
+
"Trainer",
|
|
60
|
+
"TrainingConfig",
|
|
61
|
+
"evaluate",
|
|
62
|
+
"encode_corpus",
|
|
63
|
+
"search",
|
|
64
|
+
"Interpreter",
|
|
65
|
+
"FeatureCard",
|
|
66
|
+
"OpenAICompatibleLLM",
|
|
67
|
+
"TransformersLLM",
|
|
68
|
+
]
|
|
69
|
+
|
|
70
|
+
if TYPE_CHECKING: # static analysers and IDEs see the real objects
|
|
71
|
+
from doctosae.data import RetrievalDataset, list_datasets, load_beir_folder, load_dataset
|
|
72
|
+
from doctosae.embed import EmbeddedDataset, Embedder
|
|
73
|
+
from doctosae.hub import (
|
|
74
|
+
from_paper_checkpoint,
|
|
75
|
+
list_pretrained,
|
|
76
|
+
load_pretrained,
|
|
77
|
+
pretrained_checkpoint_id,
|
|
78
|
+
pretrained_repo_id,
|
|
79
|
+
)
|
|
80
|
+
from doctosae.interpret import FeatureCard, Interpreter, OpenAICompatibleLLM, TransformersLLM
|
|
81
|
+
from doctosae.metrics import evaluate
|
|
82
|
+
from doctosae.retrieval import encode_corpus, search
|
|
83
|
+
from doctosae.sae import DoctoSAE
|
|
84
|
+
from doctosae.training import Trainer, TrainingConfig
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def __getattr__(name: str) -> Any:
|
|
88
|
+
"""Import a public name from its module on first access (PEP 562), then cache it on the package."""
|
|
89
|
+
module = _EXPORTS.get(name)
|
|
90
|
+
if module is None:
|
|
91
|
+
raise AttributeError(f"module 'doctosae' has no attribute {name!r}")
|
|
92
|
+
value = getattr(importlib.import_module(module), name)
|
|
93
|
+
globals()[name] = value
|
|
94
|
+
return value
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def __dir__() -> list[str]:
|
|
98
|
+
return sorted(set(globals()) | set(__all__))
|
doctosae/_ids.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Model identifiers and embedding settings: what embeddings and checkpoints record, and checks on it.
|
|
2
|
+
|
|
3
|
+
Embeddings and checkpoints record the encoder that produced the vectors and the settings that change them
|
|
4
|
+
(revision, instructions, normalization, truncation), so that queries are always embedded the way the
|
|
5
|
+
training queries were. Whatever leaves the machine (a model card, a pushed ``config.json``) names a Hub id,
|
|
6
|
+
never a local path, which could reveal a user name or a folder layout.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
from collections.abc import Mapping
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from huggingface_hub.utils import HFValidationError, validate_repo_id
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"EMBEDDER_ALIASES",
|
|
20
|
+
"EMBEDDING_KEYS",
|
|
21
|
+
"embedding_differences",
|
|
22
|
+
"embedding_settings",
|
|
23
|
+
"is_hub_id",
|
|
24
|
+
"is_local_path",
|
|
25
|
+
"require_shareable_embedder",
|
|
26
|
+
"resolve_embedder",
|
|
27
|
+
"shareable_model_id",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
# Short names for the three embedders of the paper.
|
|
31
|
+
EMBEDDER_ALIASES: dict[str, str] = {
|
|
32
|
+
"bge": "BAAI/bge-large-en-v1.5",
|
|
33
|
+
"medembed": "abhinand/MedEmbed-large-v0.1",
|
|
34
|
+
"gemini": "gemini-embedding-001",
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
# Models whose id is written in its canonical case whatever the case it is given in: the Hub resolves ids
|
|
38
|
+
# without regard to case, so "baai/bge-large-en-v1.5" loads the same weights and must get the same defaults.
|
|
39
|
+
_KNOWN_MODELS = (*EMBEDDER_ALIASES.values(), "BAAI/bge-small-en-v1.5", "BAAI/bge-base-en-v1.5")
|
|
40
|
+
_CANONICAL = {model.lower(): model for model in _KNOWN_MODELS}
|
|
41
|
+
|
|
42
|
+
#: Settings of an embedder that change its vectors, in the order they are reported.
|
|
43
|
+
EMBEDDING_KEYS = (
|
|
44
|
+
"model",
|
|
45
|
+
"revision",
|
|
46
|
+
"query_instruction",
|
|
47
|
+
"document_instruction",
|
|
48
|
+
"normalize",
|
|
49
|
+
"max_seq_length",
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
_WINDOWS_DRIVE = re.compile(r"^[A-Za-z]:")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def resolve_embedder(embedder: str) -> str:
|
|
56
|
+
"""Full model id of an alias such as ``"bge"``, canonical case of a known model id, else ``embedder``."""
|
|
57
|
+
key = embedder.lower()
|
|
58
|
+
return EMBEDDER_ALIASES.get(key) or _CANONICAL.get(key, embedder)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def is_local_path(value: str) -> bool:
|
|
62
|
+
"""True when ``value`` names a file or folder of this machine rather than a Hub id."""
|
|
63
|
+
if value.startswith(("/", ".", "~")) or "\\" in value or _WINDOWS_DRIVE.match(value):
|
|
64
|
+
return True
|
|
65
|
+
try:
|
|
66
|
+
return Path(value).expanduser().exists()
|
|
67
|
+
except OSError:
|
|
68
|
+
return False
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def is_hub_id(value: str) -> bool:
|
|
72
|
+
"""True for a valid Hub repository id (``org/name`` or ``name``) that is not also a local path."""
|
|
73
|
+
if not value or is_local_path(value):
|
|
74
|
+
return False
|
|
75
|
+
try:
|
|
76
|
+
validate_repo_id(value)
|
|
77
|
+
except HFValidationError:
|
|
78
|
+
return False
|
|
79
|
+
return True
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def shareable_model_id(value: str | None) -> str | None:
|
|
83
|
+
"""``value`` when it can be shared (a Hub id or an API model name), else the last part of the local path.
|
|
84
|
+
|
|
85
|
+
A local path would publish a folder layout, and often a user name, in metadata and model cards.
|
|
86
|
+
"""
|
|
87
|
+
if value is None or not is_local_path(value):
|
|
88
|
+
return value
|
|
89
|
+
return re.split(r"[\\/]", value.rstrip("\\/"))[-1] or "local_model"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def require_shareable_embedder(embedder: str | None) -> None:
|
|
93
|
+
"""Refuse to upload a checkpoint whose embedder is a local path: ``sae.embedder`` must be a Hub id."""
|
|
94
|
+
if embedder is not None and is_local_path(embedder):
|
|
95
|
+
raise ValueError(
|
|
96
|
+
f"the embedder of this checkpoint is a local path ({shareable_model_id(embedder)!r}): set "
|
|
97
|
+
"sae.embedder to the Hub id of that model before uploading, so that the repository names no "
|
|
98
|
+
"local folder and other users can rebuild the encoder"
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def embedding_settings(
|
|
103
|
+
embedder: str | None, query_instruction: str | None, extra: Mapping[str, Any] | None = None
|
|
104
|
+
) -> dict[str, Any]:
|
|
105
|
+
"""Known settings of embeddings: ``extra`` (a recorded embedder configuration), the model and the query
|
|
106
|
+
instruction. ``None`` for the model or the instruction means unknown, so the key is left out."""
|
|
107
|
+
settings = dict(extra or {})
|
|
108
|
+
if embedder:
|
|
109
|
+
settings["model"] = embedder
|
|
110
|
+
if query_instruction is not None:
|
|
111
|
+
settings["query_instruction"] = query_instruction
|
|
112
|
+
return settings
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def embedding_differences(expected: Mapping[str, Any], found: Mapping[str, Any]) -> list[str]:
|
|
116
|
+
"""Settings recorded on both sides that differ, as readable strings (empty when they agree).
|
|
117
|
+
|
|
118
|
+
A setting missing on either side is unknown and never differs. Model ids are compared after alias and
|
|
119
|
+
case resolution, and a revision is compared only when both sides pin one.
|
|
120
|
+
"""
|
|
121
|
+
differences = []
|
|
122
|
+
for key in EMBEDDING_KEYS:
|
|
123
|
+
if key not in expected or key not in found:
|
|
124
|
+
continue
|
|
125
|
+
a, b = expected[key], found[key]
|
|
126
|
+
if key == "model":
|
|
127
|
+
a, b = resolve_embedder(str(a)), resolve_embedder(str(b))
|
|
128
|
+
elif key == "revision" and (a is None or b is None):
|
|
129
|
+
continue
|
|
130
|
+
if a != b:
|
|
131
|
+
differences.append(f"{key} {a!r} against {b!r}")
|
|
132
|
+
return differences
|