trialmap 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trialmap/__init__.py +79 -0
- trialmap/_http.py +210 -0
- trialmap/_version.py +24 -0
- trialmap/assets/__init__.py +43 -0
- trialmap/assets/base.py +75 -0
- trialmap/assets/cache.py +129 -0
- trialmap/assets/names.py +165 -0
- trialmap/assets/resolver.py +247 -0
- trialmap/assets/sources.py +260 -0
- trialmap/cli.py +387 -0
- trialmap/client.py +487 -0
- trialmap/companies.py +261 -0
- trialmap/enums.py +406 -0
- trialmap/exceptions.py +40 -0
- trialmap/frame.py +130 -0
- trialmap/models/__init__.py +46 -0
- trialmap/models/_raw.py +1140 -0
- trialmap/models/study.py +238 -0
- trialmap/pipeline.py +180 -0
- trialmap/py.typed +0 -0
- trialmap/query.py +236 -0
- trialmap/rwe/__init__.py +51 -0
- trialmap/rwe/base.py +86 -0
- trialmap/rwe/cache.py +115 -0
- trialmap/rwe/collector.py +182 -0
- trialmap/rwe/sources.py +480 -0
- trialmap-0.1.0.dist-info/METADATA +398 -0
- trialmap-0.1.0.dist-info/RECORD +32 -0
- trialmap-0.1.0.dist-info/WHEEL +4 -0
- trialmap-0.1.0.dist-info/entry_points.txt +2 -0
- trialmap-0.1.0.dist-info/licenses/LICENSE +202 -0
- trialmap-0.1.0.dist-info/licenses/NOTICE +8 -0
trialmap/__init__.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""trialmap — ClinicalTrials.gov v2 API client, drug-code → asset resolution, company pipelines,
|
|
3
|
+
real-world evidence.
|
|
4
|
+
|
|
5
|
+
Quick start::
|
|
6
|
+
|
|
7
|
+
from trialmap import CTGovClient, Status
|
|
8
|
+
|
|
9
|
+
with CTGovClient() as ct:
|
|
10
|
+
for study in ct.iter_studies(query_spons='"National Cancer Institute"',
|
|
11
|
+
filter_overall_status=[Status.RECRUITING],
|
|
12
|
+
max_studies=100):
|
|
13
|
+
print(study.nct_id, study.phases, study.brief_title)
|
|
14
|
+
|
|
15
|
+
See :mod:`trialmap.assets`, :mod:`trialmap.companies` and :mod:`trialmap.pipeline` for the
|
|
16
|
+
asset-mapping workflows, and :mod:`trialmap.rwe` for real-world evidence.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
from ._version import __version__
|
|
23
|
+
except ImportError: # pragma: no cover - running from a source tree without a build
|
|
24
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
25
|
+
|
|
26
|
+
try:
|
|
27
|
+
__version__ = version("trialmap")
|
|
28
|
+
except PackageNotFoundError:
|
|
29
|
+
__version__ = "0.0.0"
|
|
30
|
+
|
|
31
|
+
from .client import BASE_URL, MAX_PAGE_SIZE, CTGovClient
|
|
32
|
+
from .companies import Company, TrialSponsor, companies_for_keywords, trials_for_companies
|
|
33
|
+
from .enums import (
|
|
34
|
+
AgencyClass,
|
|
35
|
+
InterventionType,
|
|
36
|
+
Phase,
|
|
37
|
+
Status,
|
|
38
|
+
StudyType,
|
|
39
|
+
)
|
|
40
|
+
from .exceptions import APIError, NotFoundError, RateLimitedError, ResolverError, TrialmapError
|
|
41
|
+
from .frame import study_rows, to_frame
|
|
42
|
+
from .models import DateInfo, Intervention, Organization, Sponsor, StudiesPage, Study
|
|
43
|
+
from .pipeline import Pipeline, PipelineAsset, build_pipeline
|
|
44
|
+
from .query import GeoDistance, StudyQuery, StudyQueryParams
|
|
45
|
+
|
|
46
|
+
__all__ = [
|
|
47
|
+
"BASE_URL",
|
|
48
|
+
"MAX_PAGE_SIZE",
|
|
49
|
+
"APIError",
|
|
50
|
+
"AgencyClass",
|
|
51
|
+
"CTGovClient",
|
|
52
|
+
"Company",
|
|
53
|
+
"DateInfo",
|
|
54
|
+
"GeoDistance",
|
|
55
|
+
"Intervention",
|
|
56
|
+
"InterventionType",
|
|
57
|
+
"NotFoundError",
|
|
58
|
+
"Organization",
|
|
59
|
+
"Phase",
|
|
60
|
+
"Pipeline",
|
|
61
|
+
"PipelineAsset",
|
|
62
|
+
"RateLimitedError",
|
|
63
|
+
"ResolverError",
|
|
64
|
+
"Sponsor",
|
|
65
|
+
"Status",
|
|
66
|
+
"StudiesPage",
|
|
67
|
+
"Study",
|
|
68
|
+
"StudyQuery",
|
|
69
|
+
"StudyQueryParams",
|
|
70
|
+
"StudyType",
|
|
71
|
+
"TrialSponsor",
|
|
72
|
+
"TrialmapError",
|
|
73
|
+
"__version__",
|
|
74
|
+
"build_pipeline",
|
|
75
|
+
"companies_for_keywords",
|
|
76
|
+
"study_rows",
|
|
77
|
+
"to_frame",
|
|
78
|
+
"trials_for_companies",
|
|
79
|
+
]
|
trialmap/_http.py
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Shared HTTP plumbing: an httpx client with timeouts, retries, backoff and rate limiting."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import email.utils
|
|
7
|
+
import logging
|
|
8
|
+
import random
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
from collections.abc import Callable, Mapping
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
from typing import TYPE_CHECKING, Any
|
|
14
|
+
|
|
15
|
+
import httpx
|
|
16
|
+
|
|
17
|
+
from .exceptions import APIError, NotFoundError, RateLimitedError
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from types import TracebackType
|
|
21
|
+
|
|
22
|
+
from typing_extensions import Self
|
|
23
|
+
|
|
24
|
+
log = logging.getLogger("trialmap.http")
|
|
25
|
+
|
|
26
|
+
RETRY_STATUSES = frozenset({429, 500, 502, 503, 504})
|
|
27
|
+
RETRY_EXCEPTIONS = (httpx.TransportError,) # connect/read/write timeouts, connection resets...
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def default_user_agent() -> str:
|
|
31
|
+
from . import __version__
|
|
32
|
+
|
|
33
|
+
return f"trialmap/{__version__} (+https://pypi.org/project/trialmap/) httpx/{httpx.__version__}"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class RateLimiter:
|
|
37
|
+
"""Minimum-interval limiter: at most ``rate`` acquisitions per second. Thread-safe."""
|
|
38
|
+
|
|
39
|
+
def __init__(
|
|
40
|
+
self,
|
|
41
|
+
rate: float,
|
|
42
|
+
*,
|
|
43
|
+
clock: Callable[[], float] = time.monotonic,
|
|
44
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
45
|
+
) -> None:
|
|
46
|
+
if rate <= 0:
|
|
47
|
+
raise ValueError("rate must be > 0")
|
|
48
|
+
self.interval = 1.0 / rate
|
|
49
|
+
self._clock = clock
|
|
50
|
+
self._sleep = sleep
|
|
51
|
+
self._next = 0.0
|
|
52
|
+
self._lock = threading.Lock()
|
|
53
|
+
|
|
54
|
+
def acquire(self) -> None:
|
|
55
|
+
with self._lock:
|
|
56
|
+
now = self._clock()
|
|
57
|
+
wait = self._next - now
|
|
58
|
+
self._next = max(now, self._next) + self.interval
|
|
59
|
+
if wait > 0:
|
|
60
|
+
self._sleep(wait)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def parse_retry_after(value: str | None, *, now: float | None = None) -> float | None:
|
|
64
|
+
"""Seconds to wait from a ``Retry-After`` header (delta-seconds or HTTP-date)."""
|
|
65
|
+
if not value:
|
|
66
|
+
return None
|
|
67
|
+
value = value.strip()
|
|
68
|
+
if value.isdigit():
|
|
69
|
+
return float(value)
|
|
70
|
+
try:
|
|
71
|
+
when = email.utils.parsedate_to_datetime(value)
|
|
72
|
+
except (TypeError, ValueError):
|
|
73
|
+
return None
|
|
74
|
+
if when.tzinfo is None:
|
|
75
|
+
when = when.replace(tzinfo=timezone.utc)
|
|
76
|
+
current = now if now is not None else datetime.now(timezone.utc).timestamp()
|
|
77
|
+
return max(0.0, when.timestamp() - current)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class HttpClient:
|
|
81
|
+
"""Thin wrapper around :class:`httpx.Client` used by the API client and every resolver.
|
|
82
|
+
|
|
83
|
+
Only idempotent requests are retried (GET, plus PubChem's read-only POST lookups).
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
base_url: prefix for relative paths.
|
|
87
|
+
timeout: seconds, applied to connect/read/write/pool.
|
|
88
|
+
max_retries: retries after the first attempt, on 429/5xx and transport errors.
|
|
89
|
+
backoff_factor: delay = backoff_factor * 2**attempt (+ jitter), capped at ``max_backoff``.
|
|
90
|
+
A server ``Retry-After`` header takes precedence.
|
|
91
|
+
max_backoff: cap for a single wait, in seconds.
|
|
92
|
+
rate_limit: max requests per second for this client, or ``None`` for no limit.
|
|
93
|
+
user_agent: ``User-Agent`` header (optional; a trialmap default is sent otherwise).
|
|
94
|
+
transport: custom httpx transport (tests use :class:`httpx.MockTransport`).
|
|
95
|
+
sleep: injectable sleep, for tests.
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
def __init__(
|
|
99
|
+
self,
|
|
100
|
+
*,
|
|
101
|
+
base_url: str = "",
|
|
102
|
+
timeout: float = 30.0,
|
|
103
|
+
max_retries: int = 4,
|
|
104
|
+
backoff_factor: float = 1.0,
|
|
105
|
+
max_backoff: float = 60.0,
|
|
106
|
+
rate_limit: float | None = None,
|
|
107
|
+
user_agent: str | None = None,
|
|
108
|
+
transport: httpx.BaseTransport | None = None,
|
|
109
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
110
|
+
) -> None:
|
|
111
|
+
if max_retries < 0:
|
|
112
|
+
raise ValueError("max_retries must be >= 0")
|
|
113
|
+
self.max_retries = max_retries
|
|
114
|
+
self.backoff_factor = backoff_factor
|
|
115
|
+
self.max_backoff = max_backoff
|
|
116
|
+
self._sleep = sleep
|
|
117
|
+
self.limiter = RateLimiter(rate_limit, sleep=sleep) if rate_limit else None
|
|
118
|
+
self._client = httpx.Client(
|
|
119
|
+
base_url=base_url,
|
|
120
|
+
timeout=httpx.Timeout(timeout),
|
|
121
|
+
headers={"User-Agent": user_agent or default_user_agent()},
|
|
122
|
+
follow_redirects=True, # /studies/{nctId} answers 301 for alias NCT IDs
|
|
123
|
+
transport=transport,
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
# ------------------------------------------------------------------ lifecycle
|
|
127
|
+
|
|
128
|
+
def close(self) -> None:
|
|
129
|
+
self._client.close()
|
|
130
|
+
|
|
131
|
+
def __enter__(self) -> Self:
|
|
132
|
+
return self
|
|
133
|
+
|
|
134
|
+
def __exit__(
|
|
135
|
+
self,
|
|
136
|
+
exc_type: type[BaseException] | None,
|
|
137
|
+
exc: BaseException | None,
|
|
138
|
+
tb: TracebackType | None,
|
|
139
|
+
) -> None:
|
|
140
|
+
self.close()
|
|
141
|
+
|
|
142
|
+
# ------------------------------------------------------------------ requests
|
|
143
|
+
|
|
144
|
+
def _delay(self, attempt: int, response: httpx.Response | None) -> float:
|
|
145
|
+
if response is not None:
|
|
146
|
+
retry_after = parse_retry_after(response.headers.get("Retry-After"))
|
|
147
|
+
if retry_after is not None:
|
|
148
|
+
return min(retry_after, self.max_backoff)
|
|
149
|
+
base = self.backoff_factor * (2**attempt)
|
|
150
|
+
return float(min(self.max_backoff, base + random.uniform(0, base / 2)))
|
|
151
|
+
|
|
152
|
+
def request(
|
|
153
|
+
self,
|
|
154
|
+
method: str,
|
|
155
|
+
url: str,
|
|
156
|
+
*,
|
|
157
|
+
params: Mapping[str, str] | None = None,
|
|
158
|
+
data: Mapping[str, str] | None = None,
|
|
159
|
+
headers: Mapping[str, str] | None = None,
|
|
160
|
+
ok_statuses: frozenset[int] = frozenset(),
|
|
161
|
+
) -> httpx.Response:
|
|
162
|
+
"""Send a request, retrying transient failures.
|
|
163
|
+
|
|
164
|
+
Returns the response for 2xx and for any status in ``ok_statuses`` (e.g. a resolver
|
|
165
|
+
treating 404 as "not found"); raises :class:`APIError` otherwise.
|
|
166
|
+
"""
|
|
167
|
+
attempt = 0
|
|
168
|
+
while True:
|
|
169
|
+
if self.limiter:
|
|
170
|
+
self.limiter.acquire()
|
|
171
|
+
response: httpx.Response | None = None
|
|
172
|
+
try:
|
|
173
|
+
response = self._client.request(
|
|
174
|
+
method, url, params=params, data=data, headers=headers
|
|
175
|
+
)
|
|
176
|
+
except RETRY_EXCEPTIONS as exc:
|
|
177
|
+
if attempt >= self.max_retries:
|
|
178
|
+
raise
|
|
179
|
+
log.warning("%s %s failed (%s); retrying", method, url, exc)
|
|
180
|
+
else:
|
|
181
|
+
if response.status_code not in RETRY_STATUSES or attempt >= self.max_retries:
|
|
182
|
+
break
|
|
183
|
+
log.warning(
|
|
184
|
+
"%s %s -> HTTP %s; retrying", method, response.url, response.status_code
|
|
185
|
+
)
|
|
186
|
+
self._sleep(self._delay(attempt, response))
|
|
187
|
+
attempt += 1
|
|
188
|
+
|
|
189
|
+
if response.is_success or response.status_code in ok_statuses:
|
|
190
|
+
return response
|
|
191
|
+
raise _error_for(response)
|
|
192
|
+
|
|
193
|
+
def get(
|
|
194
|
+
self, url: str, *, params: Mapping[str, str] | None = None, **kw: Any
|
|
195
|
+
) -> httpx.Response:
|
|
196
|
+
return self.request("GET", url, params=params, **kw)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _error_for(response: httpx.Response) -> APIError:
|
|
200
|
+
try:
|
|
201
|
+
message = response.text.strip()
|
|
202
|
+
except (httpx.ResponseNotRead, UnicodeDecodeError): # pragma: no cover - defensive
|
|
203
|
+
message = ""
|
|
204
|
+
url = str(response.request.url)
|
|
205
|
+
code = response.status_code
|
|
206
|
+
if code == 404:
|
|
207
|
+
return NotFoundError(code, message, url)
|
|
208
|
+
if code == 429:
|
|
209
|
+
return RateLimitedError(code, message, url)
|
|
210
|
+
return APIError(code, message, url)
|
trialmap/_version.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.1.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 1, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Drug-code → asset resolution.
|
|
3
|
+
|
|
4
|
+
>>> from trialmap.assets import AssetResolver # doctest: +SKIP
|
|
5
|
+
>>> AssetResolver().resolve("NSC 740").asset_name
|
|
6
|
+
'methotrexate'
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from .base import Resolution, Resolver
|
|
10
|
+
from .cache import MemoryCache, ResolutionCache, SQLiteCache, default_cache_path
|
|
11
|
+
from .names import (
|
|
12
|
+
DRUG_INTERVENTION_TYPES,
|
|
13
|
+
candidate_names,
|
|
14
|
+
clean_name,
|
|
15
|
+
is_drug_intervention,
|
|
16
|
+
is_non_asset,
|
|
17
|
+
normalize_key,
|
|
18
|
+
split_combination,
|
|
19
|
+
)
|
|
20
|
+
from .resolver import AssetResolver, DrugMatch
|
|
21
|
+
from .sources import ChemblResolver, MappingResolver, NciEvsResolver, PubChemResolver
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"DRUG_INTERVENTION_TYPES",
|
|
25
|
+
"AssetResolver",
|
|
26
|
+
"ChemblResolver",
|
|
27
|
+
"DrugMatch",
|
|
28
|
+
"MappingResolver",
|
|
29
|
+
"MemoryCache",
|
|
30
|
+
"NciEvsResolver",
|
|
31
|
+
"PubChemResolver",
|
|
32
|
+
"Resolution",
|
|
33
|
+
"ResolutionCache",
|
|
34
|
+
"Resolver",
|
|
35
|
+
"SQLiteCache",
|
|
36
|
+
"candidate_names",
|
|
37
|
+
"clean_name",
|
|
38
|
+
"default_cache_path",
|
|
39
|
+
"is_drug_intervention",
|
|
40
|
+
"is_non_asset",
|
|
41
|
+
"normalize_key",
|
|
42
|
+
"split_combination",
|
|
43
|
+
]
|
trialmap/assets/base.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""The resolver interface: implement :class:`Resolver` to plug in your own source."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import dataclasses
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Any, Protocol, runtime_checkable
|
|
9
|
+
|
|
10
|
+
__all__ = ["Resolution", "Resolver"]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True, slots=True)
|
|
14
|
+
class Resolution:
|
|
15
|
+
"""A drug name resolved to a named asset.
|
|
16
|
+
|
|
17
|
+
Attributes:
|
|
18
|
+
asset_name: preferred name, e.g. ``"methotrexate"`` (INN where the source has one).
|
|
19
|
+
source: resolver that produced it (``"chembl"``, ``"pubchem"``, ``"nci_evs"``...).
|
|
20
|
+
source_id: identifier in that source (``CHEMBL34259``, ``CID126941``, ``C2242``),
|
|
21
|
+
for attribution and follow-up.
|
|
22
|
+
url: human-readable page for ``source_id``.
|
|
23
|
+
molecule_type: e.g. ``"Antibody"``, ``"Small molecule"`` (ChEMBL only).
|
|
24
|
+
max_phase: highest development phase in the source (ChEMBL only; 4 = approved).
|
|
25
|
+
synonyms: other names/codes the source knows.
|
|
26
|
+
matched_on: which candidate name produced the hit (set by :class:`AssetResolver`).
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
asset_name: str
|
|
30
|
+
source: str
|
|
31
|
+
source_id: str | None = None
|
|
32
|
+
url: str | None = None
|
|
33
|
+
molecule_type: str | None = None
|
|
34
|
+
max_phase: float | None = None
|
|
35
|
+
synonyms: tuple[str, ...] = ()
|
|
36
|
+
matched_on: str | None = None
|
|
37
|
+
|
|
38
|
+
def to_dict(self) -> dict[str, Any]:
|
|
39
|
+
d = dataclasses.asdict(self)
|
|
40
|
+
d["synonyms"] = list(self.synonyms)
|
|
41
|
+
return d
|
|
42
|
+
|
|
43
|
+
@classmethod
|
|
44
|
+
def from_dict(cls, d: dict[str, Any]) -> Resolution:
|
|
45
|
+
fields = {f.name for f in dataclasses.fields(cls)}
|
|
46
|
+
kwargs = {k: v for k, v in d.items() if k in fields}
|
|
47
|
+
kwargs["synonyms"] = tuple(kwargs.get("synonyms") or ())
|
|
48
|
+
return cls(**kwargs)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@runtime_checkable
|
|
52
|
+
class Resolver(Protocol):
|
|
53
|
+
"""Anything with a ``name`` and a ``lookup`` method.
|
|
54
|
+
|
|
55
|
+
``lookup`` gets one already-cleaned candidate name and returns:
|
|
56
|
+
|
|
57
|
+
* a :class:`Resolution` on a match,
|
|
58
|
+
* ``None`` for a definite "not found" (cached as a miss),
|
|
59
|
+
* or raises :class:`~trialmap.exceptions.ResolverError` for transient failures (not cached;
|
|
60
|
+
the next resolver in the chain is tried).
|
|
61
|
+
|
|
62
|
+
Example — an internal pipeline list::
|
|
63
|
+
|
|
64
|
+
class InternalPipeline:
|
|
65
|
+
name = "internal"
|
|
66
|
+
|
|
67
|
+
def lookup(self, name: str) -> Resolution | None:
|
|
68
|
+
asset = MY_TABLE.get(name.upper())
|
|
69
|
+
return Resolution(asset, source=self.name) if asset else None
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def name(self) -> str: ...
|
|
74
|
+
|
|
75
|
+
def lookup(self, name: str) -> Resolution | None: ...
|
trialmap/assets/cache.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Lookup caches. The default is SQLite in the user cache dir, e.g.
|
|
3
|
+
``~/Library/Caches/trialmap/resolutions.sqlite3`` (macOS) or ``~/.cache/trialmap/...`` (Linux).
|
|
4
|
+
|
|
5
|
+
Hits and definite misses are cached; transient errors never are. Misses expire sooner than
|
|
6
|
+
hits (new INNs get assigned to research codes).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import sqlite3
|
|
13
|
+
import threading
|
|
14
|
+
import time
|
|
15
|
+
from datetime import timedelta
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Protocol
|
|
18
|
+
|
|
19
|
+
import platformdirs
|
|
20
|
+
|
|
21
|
+
from .base import Resolution
|
|
22
|
+
|
|
23
|
+
__all__ = ["MemoryCache", "ResolutionCache", "SQLiteCache", "default_cache_path"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def default_cache_path() -> Path:
|
|
27
|
+
return Path(platformdirs.user_cache_dir("trialmap")) / "resolutions.sqlite3"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ResolutionCache(Protocol):
|
|
31
|
+
def get(self, resolver: str, key: str) -> tuple[bool, Resolution | None]:
|
|
32
|
+
"""Return ``(found, value)``; ``value`` is ``None`` for a cached miss."""
|
|
33
|
+
...
|
|
34
|
+
|
|
35
|
+
def set(self, resolver: str, key: str, value: Resolution | None) -> None: ...
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class MemoryCache:
|
|
39
|
+
"""In-process cache (no expiry)."""
|
|
40
|
+
|
|
41
|
+
def __init__(self) -> None:
|
|
42
|
+
self._data: dict[tuple[str, str], Resolution | None] = {}
|
|
43
|
+
|
|
44
|
+
def get(self, resolver: str, key: str) -> tuple[bool, Resolution | None]:
|
|
45
|
+
k = (resolver, key)
|
|
46
|
+
return (k in self._data, self._data.get(k))
|
|
47
|
+
|
|
48
|
+
def set(self, resolver: str, key: str, value: Resolution | None) -> None:
|
|
49
|
+
self._data[(resolver, key)] = value
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class SQLiteCache:
|
|
53
|
+
"""Persistent cache.
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
path: database file; defaults to :func:`default_cache_path`.
|
|
57
|
+
hit_ttl: how long a match stays valid (``None`` = forever).
|
|
58
|
+
miss_ttl: how long a "not found" stays valid (``None`` = forever).
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
def __init__(
|
|
62
|
+
self,
|
|
63
|
+
path: str | Path | None = None,
|
|
64
|
+
*,
|
|
65
|
+
hit_ttl: timedelta | None = timedelta(days=180),
|
|
66
|
+
miss_ttl: timedelta | None = timedelta(days=30),
|
|
67
|
+
) -> None:
|
|
68
|
+
self.path = Path(path) if path else default_cache_path()
|
|
69
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
70
|
+
self.hit_ttl = hit_ttl
|
|
71
|
+
self.miss_ttl = miss_ttl
|
|
72
|
+
self._lock = threading.Lock()
|
|
73
|
+
self._db = sqlite3.connect(self.path, check_same_thread=False)
|
|
74
|
+
with self._db:
|
|
75
|
+
self._db.execute(
|
|
76
|
+
"CREATE TABLE IF NOT EXISTS resolutions ("
|
|
77
|
+
" resolver TEXT NOT NULL, key TEXT NOT NULL, value TEXT, created REAL NOT NULL,"
|
|
78
|
+
" PRIMARY KEY (resolver, key))"
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
def get(self, resolver: str, key: str) -> tuple[bool, Resolution | None]:
|
|
82
|
+
with self._lock:
|
|
83
|
+
row = self._db.execute(
|
|
84
|
+
"SELECT value, created FROM resolutions WHERE resolver = ? AND key = ?",
|
|
85
|
+
(resolver, key),
|
|
86
|
+
).fetchone()
|
|
87
|
+
if row is None:
|
|
88
|
+
return (False, None)
|
|
89
|
+
value, created = row
|
|
90
|
+
ttl = self.hit_ttl if value is not None else self.miss_ttl
|
|
91
|
+
if ttl is not None and time.time() - created > ttl.total_seconds():
|
|
92
|
+
return (False, None)
|
|
93
|
+
return (True, Resolution.from_dict(json.loads(value)) if value is not None else None)
|
|
94
|
+
|
|
95
|
+
def set(self, resolver: str, key: str, value: Resolution | None) -> None:
|
|
96
|
+
payload = json.dumps(value.to_dict()) if value is not None else None
|
|
97
|
+
with self._lock, self._db:
|
|
98
|
+
self._db.execute(
|
|
99
|
+
"INSERT OR REPLACE INTO resolutions (resolver, key, value, created)"
|
|
100
|
+
" VALUES (?, ?, ?, ?)",
|
|
101
|
+
(resolver, key, payload, time.time()),
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
def info(self) -> dict[str, int]:
|
|
105
|
+
"""Entry counts: ``{"hits": n, "misses": n}``."""
|
|
106
|
+
with self._lock:
|
|
107
|
+
hits, misses = self._db.execute(
|
|
108
|
+
"SELECT COALESCE(SUM(value IS NOT NULL), 0), COALESCE(SUM(value IS NULL), 0)"
|
|
109
|
+
" FROM resolutions"
|
|
110
|
+
).fetchone()
|
|
111
|
+
return {"hits": int(hits), "misses": int(misses)}
|
|
112
|
+
|
|
113
|
+
def clear(self, resolver: str | None = None) -> int:
|
|
114
|
+
"""Delete all entries (or one resolver's). Returns the number removed."""
|
|
115
|
+
with self._lock, self._db:
|
|
116
|
+
if resolver is None:
|
|
117
|
+
cur = self._db.execute("DELETE FROM resolutions")
|
|
118
|
+
else:
|
|
119
|
+
cur = self._db.execute("DELETE FROM resolutions WHERE resolver = ?", (resolver,))
|
|
120
|
+
return cur.rowcount
|
|
121
|
+
|
|
122
|
+
def close(self) -> None:
|
|
123
|
+
self._db.close()
|
|
124
|
+
|
|
125
|
+
def __enter__(self) -> SQLiteCache:
|
|
126
|
+
return self
|
|
127
|
+
|
|
128
|
+
def __exit__(self, *exc: object) -> None:
|
|
129
|
+
self.close()
|