doctosae 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
doctosae/__init__.py ADDED
@@ -0,0 +1,98 @@
1
+ """DoctoSAE: sparse autoencoders for interpretable and transferable medical information retrieval.
2
+
3
+ Reference implementation of "Explanations that Transfer: Interpreting Medical Retrieval with Sparse
4
+ AutoEncoders" (ML4H 2026). A TopK sparse autoencoder is trained on the frozen embeddings of a text
5
+ encoder, with a reconstruction term and an InfoNCE term on the sparse codes; retrieval then runs on the
6
+ sparse codes and every feature can be described in natural language.
7
+
8
+ The public names below are imported on first access, so ``import doctosae`` (and the ``doctosae`` command
9
+ line) stays fast: PyTorch, pandas and sentence transformers load only when a model or a dataset is
10
+ used.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import importlib
16
+ from typing import TYPE_CHECKING, Any
17
+
18
+ __version__ = "0.1.0"
19
+
20
+ # Public name to the module that defines it.
21
+ _EXPORTS: dict[str, str] = {
22
+ "DoctoSAE": "doctosae.sae",
23
+ "load_pretrained": "doctosae.hub",
24
+ "list_pretrained": "doctosae.hub",
25
+ "pretrained_repo_id": "doctosae.hub",
26
+ "pretrained_checkpoint_id": "doctosae.hub",
27
+ "from_paper_checkpoint": "doctosae.hub",
28
+ "RetrievalDataset": "doctosae.data",
29
+ "load_dataset": "doctosae.data",
30
+ "list_datasets": "doctosae.data",
31
+ "load_beir_folder": "doctosae.data",
32
+ "Embedder": "doctosae.embed",
33
+ "EmbeddedDataset": "doctosae.embed",
34
+ "Trainer": "doctosae.training",
35
+ "TrainingConfig": "doctosae.training",
36
+ "evaluate": "doctosae.metrics",
37
+ "encode_corpus": "doctosae.retrieval",
38
+ "search": "doctosae.retrieval",
39
+ "Interpreter": "doctosae.interpret",
40
+ "FeatureCard": "doctosae.interpret",
41
+ "OpenAICompatibleLLM": "doctosae.interpret",
42
+ "TransformersLLM": "doctosae.interpret",
43
+ }
44
+
45
+ __all__ = [
46
+ "__version__",
47
+ "DoctoSAE",
48
+ "load_pretrained",
49
+ "list_pretrained",
50
+ "pretrained_repo_id",
51
+ "pretrained_checkpoint_id",
52
+ "from_paper_checkpoint",
53
+ "RetrievalDataset",
54
+ "load_dataset",
55
+ "list_datasets",
56
+ "load_beir_folder",
57
+ "Embedder",
58
+ "EmbeddedDataset",
59
+ "Trainer",
60
+ "TrainingConfig",
61
+ "evaluate",
62
+ "encode_corpus",
63
+ "search",
64
+ "Interpreter",
65
+ "FeatureCard",
66
+ "OpenAICompatibleLLM",
67
+ "TransformersLLM",
68
+ ]
69
+
70
+ if TYPE_CHECKING: # static analysers and IDEs see the real objects
71
+ from doctosae.data import RetrievalDataset, list_datasets, load_beir_folder, load_dataset
72
+ from doctosae.embed import EmbeddedDataset, Embedder
73
+ from doctosae.hub import (
74
+ from_paper_checkpoint,
75
+ list_pretrained,
76
+ load_pretrained,
77
+ pretrained_checkpoint_id,
78
+ pretrained_repo_id,
79
+ )
80
+ from doctosae.interpret import FeatureCard, Interpreter, OpenAICompatibleLLM, TransformersLLM
81
+ from doctosae.metrics import evaluate
82
+ from doctosae.retrieval import encode_corpus, search
83
+ from doctosae.sae import DoctoSAE
84
+ from doctosae.training import Trainer, TrainingConfig
85
+
86
+
87
+ def __getattr__(name: str) -> Any:
88
+ """Import a public name from its module on first access (PEP 562), then cache it on the package."""
89
+ module = _EXPORTS.get(name)
90
+ if module is None:
91
+ raise AttributeError(f"module 'doctosae' has no attribute {name!r}")
92
+ value = getattr(importlib.import_module(module), name)
93
+ globals()[name] = value
94
+ return value
95
+
96
+
97
+ def __dir__() -> list[str]:
98
+ return sorted(set(globals()) | set(__all__))
doctosae/_ids.py ADDED
@@ -0,0 +1,132 @@
1
+ """Model identifiers and embedding settings: what embeddings and checkpoints record, and checks on it.
2
+
3
+ Embeddings and checkpoints record the encoder that produced the vectors and the settings that change them
4
+ (revision, instructions, normalization, truncation), so that queries are always embedded the way the
5
+ training queries were. Whatever leaves the machine (a model card, a pushed ``config.json``) names a Hub id,
6
+ never a local path, which could reveal a user name or a folder layout.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import re
12
+ from collections.abc import Mapping
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ from huggingface_hub.utils import HFValidationError, validate_repo_id
17
+
18
+ __all__ = [
19
+ "EMBEDDER_ALIASES",
20
+ "EMBEDDING_KEYS",
21
+ "embedding_differences",
22
+ "embedding_settings",
23
+ "is_hub_id",
24
+ "is_local_path",
25
+ "require_shareable_embedder",
26
+ "resolve_embedder",
27
+ "shareable_model_id",
28
+ ]
29
+
30
+ # Short names for the three embedders of the paper.
31
+ EMBEDDER_ALIASES: dict[str, str] = {
32
+ "bge": "BAAI/bge-large-en-v1.5",
33
+ "medembed": "abhinand/MedEmbed-large-v0.1",
34
+ "gemini": "gemini-embedding-001",
35
+ }
36
+
37
+ # Models whose id is written in its canonical case whatever the case it is given in: the Hub resolves ids
38
+ # without regard to case, so "baai/bge-large-en-v1.5" loads the same weights and must get the same defaults.
39
+ _KNOWN_MODELS = (*EMBEDDER_ALIASES.values(), "BAAI/bge-small-en-v1.5", "BAAI/bge-base-en-v1.5")
40
+ _CANONICAL = {model.lower(): model for model in _KNOWN_MODELS}
41
+
42
+ #: Settings of an embedder that change its vectors, in the order they are reported.
43
+ EMBEDDING_KEYS = (
44
+ "model",
45
+ "revision",
46
+ "query_instruction",
47
+ "document_instruction",
48
+ "normalize",
49
+ "max_seq_length",
50
+ )
51
+
52
+ _WINDOWS_DRIVE = re.compile(r"^[A-Za-z]:")
53
+
54
+
55
+ def resolve_embedder(embedder: str) -> str:
56
+ """Full model id of an alias such as ``"bge"``, canonical case of a known model id, else ``embedder``."""
57
+ key = embedder.lower()
58
+ return EMBEDDER_ALIASES.get(key) or _CANONICAL.get(key, embedder)
59
+
60
+
61
+ def is_local_path(value: str) -> bool:
62
+ """True when ``value`` names a file or folder of this machine rather than a Hub id."""
63
+ if value.startswith(("/", ".", "~")) or "\\" in value or _WINDOWS_DRIVE.match(value):
64
+ return True
65
+ try:
66
+ return Path(value).expanduser().exists()
67
+ except OSError:
68
+ return False
69
+
70
+
71
+ def is_hub_id(value: str) -> bool:
72
+ """True for a valid Hub repository id (``org/name`` or ``name``) that is not also a local path."""
73
+ if not value or is_local_path(value):
74
+ return False
75
+ try:
76
+ validate_repo_id(value)
77
+ except HFValidationError:
78
+ return False
79
+ return True
80
+
81
+
82
+ def shareable_model_id(value: str | None) -> str | None:
83
+ """``value`` when it can be shared (a Hub id or an API model name), else the last part of the local path.
84
+
85
+ A local path would publish a folder layout, and often a user name, in metadata and model cards.
86
+ """
87
+ if value is None or not is_local_path(value):
88
+ return value
89
+ return re.split(r"[\\/]", value.rstrip("\\/"))[-1] or "local_model"
90
+
91
+
92
+ def require_shareable_embedder(embedder: str | None) -> None:
93
+ """Refuse to upload a checkpoint whose embedder is a local path: ``sae.embedder`` must be a Hub id."""
94
+ if embedder is not None and is_local_path(embedder):
95
+ raise ValueError(
96
+ f"the embedder of this checkpoint is a local path ({shareable_model_id(embedder)!r}): set "
97
+ "sae.embedder to the Hub id of that model before uploading, so that the repository names no "
98
+ "local folder and other users can rebuild the encoder"
99
+ )
100
+
101
+
102
+ def embedding_settings(
103
+ embedder: str | None, query_instruction: str | None, extra: Mapping[str, Any] | None = None
104
+ ) -> dict[str, Any]:
105
+ """Known settings of embeddings: ``extra`` (a recorded embedder configuration), the model and the query
106
+ instruction. ``None`` for the model or the instruction means unknown, so the key is left out."""
107
+ settings = dict(extra or {})
108
+ if embedder:
109
+ settings["model"] = embedder
110
+ if query_instruction is not None:
111
+ settings["query_instruction"] = query_instruction
112
+ return settings
113
+
114
+
115
+ def embedding_differences(expected: Mapping[str, Any], found: Mapping[str, Any]) -> list[str]:
116
+ """Settings recorded on both sides that differ, as readable strings (empty when they agree).
117
+
118
+ A setting missing on either side is unknown and never differs. Model ids are compared after alias and
119
+ case resolution, and a revision is compared only when both sides pin one.
120
+ """
121
+ differences = []
122
+ for key in EMBEDDING_KEYS:
123
+ if key not in expected or key not in found:
124
+ continue
125
+ a, b = expected[key], found[key]
126
+ if key == "model":
127
+ a, b = resolve_embedder(str(a)), resolve_embedder(str(b))
128
+ elif key == "revision" and (a is None or b is None):
129
+ continue
130
+ if a != b:
131
+ differences.append(f"{key} {a!r} against {b!r}")
132
+ return differences