trialmap 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trialmap/__init__.py +79 -0
- trialmap/_http.py +210 -0
- trialmap/_version.py +24 -0
- trialmap/assets/__init__.py +43 -0
- trialmap/assets/base.py +75 -0
- trialmap/assets/cache.py +129 -0
- trialmap/assets/names.py +165 -0
- trialmap/assets/resolver.py +247 -0
- trialmap/assets/sources.py +260 -0
- trialmap/cli.py +387 -0
- trialmap/client.py +487 -0
- trialmap/companies.py +261 -0
- trialmap/enums.py +406 -0
- trialmap/exceptions.py +40 -0
- trialmap/frame.py +130 -0
- trialmap/models/__init__.py +46 -0
- trialmap/models/_raw.py +1140 -0
- trialmap/models/study.py +238 -0
- trialmap/pipeline.py +180 -0
- trialmap/py.typed +0 -0
- trialmap/query.py +236 -0
- trialmap/rwe/__init__.py +51 -0
- trialmap/rwe/base.py +86 -0
- trialmap/rwe/cache.py +115 -0
- trialmap/rwe/collector.py +182 -0
- trialmap/rwe/sources.py +480 -0
- trialmap-0.1.0.dist-info/METADATA +398 -0
- trialmap-0.1.0.dist-info/RECORD +32 -0
- trialmap-0.1.0.dist-info/WHEEL +4 -0
- trialmap-0.1.0.dist-info/entry_points.txt +2 -0
- trialmap-0.1.0.dist-info/licenses/LICENSE +202 -0
- trialmap-0.1.0.dist-info/licenses/NOTICE +8 -0
trialmap/cli.py
ADDED
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Command-line interface: ``trialmap trials | resolve | rwe | cache``.
|
|
3
|
+
|
|
4
|
+
Writes CSV or JSON with the standard library only (pandas is not needed).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import csv
|
|
11
|
+
import json
|
|
12
|
+
import logging
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
from collections.abc import Sequence
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from . import __version__
|
|
20
|
+
from .assets import (
|
|
21
|
+
AssetResolver,
|
|
22
|
+
ChemblResolver,
|
|
23
|
+
MappingResolver,
|
|
24
|
+
NciEvsResolver,
|
|
25
|
+
PubChemResolver,
|
|
26
|
+
Resolver,
|
|
27
|
+
SQLiteCache,
|
|
28
|
+
default_cache_path,
|
|
29
|
+
)
|
|
30
|
+
from .client import CTGovClient
|
|
31
|
+
from .companies import Company, companies_for_keywords, trials_for_companies
|
|
32
|
+
from .exceptions import TrialmapError
|
|
33
|
+
from .pipeline import build_pipeline
|
|
34
|
+
from .rwe import (
|
|
35
|
+
CtgovObservationalSource,
|
|
36
|
+
EvidenceSource,
|
|
37
|
+
FdaAdverseEventsSource,
|
|
38
|
+
FdaApprovalsSource,
|
|
39
|
+
PubMedSource,
|
|
40
|
+
RweCollector,
|
|
41
|
+
SQLiteEvidenceCache,
|
|
42
|
+
default_rwe_cache_path,
|
|
43
|
+
pipeline_rwe,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
log = logging.getLogger("trialmap")
|
|
47
|
+
|
|
48
|
+
RESOLVERS = {
|
|
49
|
+
"chembl": ChemblResolver,
|
|
50
|
+
"pubchem": PubChemResolver,
|
|
51
|
+
"nci": NciEvsResolver,
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
EPILOG = """\
|
|
55
|
+
examples:
|
|
56
|
+
trialmap trials --companies "National Cancer Institute" "Mayo Clinic" --out nci
|
|
57
|
+
trialmap trials --companies-file companies.txt --format json
|
|
58
|
+
trialmap trials --keywords '"Atopic Dermatitis" OR Eczema' --field cond --max-studies 500
|
|
59
|
+
trialmap resolve "NSC 740" 10-1074 triapine
|
|
60
|
+
trialmap rwe methotrexate bevacizumab
|
|
61
|
+
trialmap trials --companies "National Cancer Institute" --max-studies 200 --rwe --out nci
|
|
62
|
+
trialmap cache info
|
|
63
|
+
|
|
64
|
+
API keys (optional, raise rate limits): OPENFDA_API_KEY, NCBI_API_KEY environment variables.
|
|
65
|
+
|
|
66
|
+
companies file: one company per line; aliases after "|", e.g.
|
|
67
|
+
NIAID | National Institute of Allergy and Infectious Diseases | Vaccine Research Center
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# ---------------------------------------------------------------- output
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def write_rows(rows: list[dict[str, Any]], path: Path, fmt: str) -> None:
|
|
75
|
+
if fmt == "json":
|
|
76
|
+
path.write_text(json.dumps(rows, indent=2, ensure_ascii=False, default=str) + "\n")
|
|
77
|
+
return
|
|
78
|
+
fieldnames: dict[str, None] = {}
|
|
79
|
+
for r in rows:
|
|
80
|
+
fieldnames.update(dict.fromkeys(r))
|
|
81
|
+
with path.open("w", newline="", encoding="utf-8") as fh:
|
|
82
|
+
writer = csv.DictWriter(fh, fieldnames=list(fieldnames))
|
|
83
|
+
writer.writeheader()
|
|
84
|
+
writer.writerows(rows)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
# ---------------------------------------------------------------- resolver setup
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def make_resolver(args: argparse.Namespace) -> AssetResolver:
|
|
91
|
+
chain: list[Resolver] = []
|
|
92
|
+
if args.mapping:
|
|
93
|
+
chain.append(MappingResolver.from_csv(args.mapping))
|
|
94
|
+
for name in (n.strip() for n in args.resolvers.split(",")):
|
|
95
|
+
if name not in RESOLVERS:
|
|
96
|
+
raise SystemExit(f"unknown resolver {name!r}; choose from {', '.join(RESOLVERS)}")
|
|
97
|
+
chain.append(RESOLVERS[name](user_agent=args.user_agent))
|
|
98
|
+
return AssetResolver(chain, cache=not args.no_cache, user_agent=args.user_agent)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _add_resolver_options(p: argparse.ArgumentParser) -> None:
|
|
102
|
+
g = p.add_argument_group("drug resolution")
|
|
103
|
+
g.add_argument(
|
|
104
|
+
"--resolvers",
|
|
105
|
+
default="chembl,pubchem",
|
|
106
|
+
help="comma-separated chain, tried in order (chembl, pubchem, nci). Default: %(default)s",
|
|
107
|
+
)
|
|
108
|
+
g.add_argument(
|
|
109
|
+
"--mapping",
|
|
110
|
+
metavar="CSV",
|
|
111
|
+
help="your own name→asset table (columns: name, asset), tried first",
|
|
112
|
+
)
|
|
113
|
+
g.add_argument("--no-cache", action="store_true", help="don't use the on-disk lookup cache")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
RWE_SOURCES = {
|
|
117
|
+
"ctgov": CtgovObservationalSource,
|
|
118
|
+
"fda": FdaApprovalsSource,
|
|
119
|
+
"faers": FdaAdverseEventsSource,
|
|
120
|
+
"pubmed": PubMedSource,
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def make_collector(args: argparse.Namespace) -> RweCollector:
|
|
125
|
+
sources: list[EvidenceSource] = []
|
|
126
|
+
for name in (n.strip() for n in args.rwe_sources.split(",")):
|
|
127
|
+
if name == "ctgov":
|
|
128
|
+
sources.append(CtgovObservationalSource(user_agent=args.user_agent))
|
|
129
|
+
elif name in ("fda", "faers"):
|
|
130
|
+
key = os.environ.get("OPENFDA_API_KEY") or None
|
|
131
|
+
sources.append(RWE_SOURCES[name](user_agent=args.user_agent, api_key=key))
|
|
132
|
+
elif name == "pubmed":
|
|
133
|
+
key = os.environ.get("NCBI_API_KEY") or None
|
|
134
|
+
sources.append(PubMedSource(user_agent=args.user_agent, api_key=key))
|
|
135
|
+
else:
|
|
136
|
+
raise SystemExit(f"unknown RWE source {name!r}; choose from {', '.join(RWE_SOURCES)}")
|
|
137
|
+
return RweCollector(sources, cache=not args.no_cache)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _add_rwe_options(p: argparse.ArgumentParser, *, with_no_cache: bool) -> None:
|
|
141
|
+
g = p.add_argument_group("real-world evidence")
|
|
142
|
+
g.add_argument(
|
|
143
|
+
"--rwe-sources",
|
|
144
|
+
default=",".join(RWE_SOURCES),
|
|
145
|
+
help="comma-separated (ctgov, fda, faers, pubmed). Default: %(default)s",
|
|
146
|
+
)
|
|
147
|
+
if with_no_cache:
|
|
148
|
+
g.add_argument("--no-cache", action="store_true", help="don't use the on-disk cache")
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
# ---------------------------------------------------------------- commands
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def cmd_trials(args: argparse.Namespace) -> int:
|
|
155
|
+
if args.rwe and args.no_resolve:
|
|
156
|
+
raise SystemExit("--rwe needs drug names: drop --no-resolve")
|
|
157
|
+
out = Path(args.out)
|
|
158
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
159
|
+
with CTGovClient(user_agent=args.user_agent) as ct:
|
|
160
|
+
if args.keywords:
|
|
161
|
+
rows = companies_for_keywords(
|
|
162
|
+
args.keywords,
|
|
163
|
+
field=args.field,
|
|
164
|
+
client=ct,
|
|
165
|
+
max_studies=args.max_studies,
|
|
166
|
+
sponsor_classes=None if args.all_sponsors else ("INDUSTRY",),
|
|
167
|
+
)
|
|
168
|
+
else:
|
|
169
|
+
lines = list(args.companies or [])
|
|
170
|
+
if args.companies_file:
|
|
171
|
+
text = Path(args.companies_file).read_text(encoding="utf-8")
|
|
172
|
+
lines += [ln for ln in text.splitlines() if ln.strip() and not ln.startswith("#")]
|
|
173
|
+
companies = [Company.parse(line) for line in lines]
|
|
174
|
+
rows = trials_for_companies(companies, client=ct, max_studies=args.max_studies)
|
|
175
|
+
|
|
176
|
+
if not rows:
|
|
177
|
+
print("No matching trials.", file=sys.stderr)
|
|
178
|
+
return 1
|
|
179
|
+
|
|
180
|
+
ext = args.format
|
|
181
|
+
trials_path = Path(f"{out}_trials.{ext}")
|
|
182
|
+
write_rows([r.to_row() for r in rows], trials_path, ext)
|
|
183
|
+
n_trials = len({r.nct_id for r in rows})
|
|
184
|
+
n_companies = len({r.company for r in rows})
|
|
185
|
+
print(f"{n_trials} trials, {n_companies} companies -> {trials_path}", file=sys.stderr)
|
|
186
|
+
|
|
187
|
+
if args.no_resolve:
|
|
188
|
+
pipeline = build_pipeline(rows)
|
|
189
|
+
else:
|
|
190
|
+
with make_resolver(args) as resolver:
|
|
191
|
+
pipeline = build_pipeline(rows, resolver)
|
|
192
|
+
|
|
193
|
+
drugs_path = Path(f"{out}_drugs.{ext}")
|
|
194
|
+
assets_path = Path(f"{out}_assets.{ext}")
|
|
195
|
+
write_rows(pipeline.drug_rows(), drugs_path, ext)
|
|
196
|
+
write_rows(pipeline.asset_rows(), assets_path, ext)
|
|
197
|
+
n_resolved = sum(d.resolved for d in pipeline.drugs)
|
|
198
|
+
print(
|
|
199
|
+
f"{len(pipeline.drugs)} drug names ({n_resolved} resolved) -> {drugs_path}\n"
|
|
200
|
+
f"{len(pipeline.assets)} company x asset rows -> {assets_path}",
|
|
201
|
+
file=sys.stderr,
|
|
202
|
+
)
|
|
203
|
+
if args.rwe:
|
|
204
|
+
with make_collector(args) as collector:
|
|
205
|
+
result = pipeline_rwe(pipeline, collector)
|
|
206
|
+
rwe_path = Path(f"{out}_rwe.{ext}")
|
|
207
|
+
write_rows(list(result.rows), rwe_path, ext)
|
|
208
|
+
failed = sum(bool(p.errors) for p in result.profiles)
|
|
209
|
+
note = f" ({failed} with source errors)" if failed else ""
|
|
210
|
+
print(f"{len(result.profiles)} assets with RWE{note} -> {rwe_path}", file=sys.stderr)
|
|
211
|
+
return 0
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def cmd_resolve(args: argparse.Namespace) -> int:
|
|
215
|
+
with make_resolver(args) as resolver:
|
|
216
|
+
matches = [resolver.resolve(name) for name in args.names]
|
|
217
|
+
rows = [m.to_row() for m in matches]
|
|
218
|
+
if args.json:
|
|
219
|
+
print(json.dumps(rows, indent=2, ensure_ascii=False))
|
|
220
|
+
else:
|
|
221
|
+
for m in matches:
|
|
222
|
+
r = m.resolution
|
|
223
|
+
if r:
|
|
224
|
+
extra = ", ".join(
|
|
225
|
+
x
|
|
226
|
+
for x in (
|
|
227
|
+
r.molecule_type,
|
|
228
|
+
f"max phase {r.max_phase:g}" if r.max_phase is not None else None,
|
|
229
|
+
)
|
|
230
|
+
if x
|
|
231
|
+
)
|
|
232
|
+
print(
|
|
233
|
+
f"{m.intervention}\t-> {r.asset_name}\t[{r.source} {r.source_id or ''}"
|
|
234
|
+
f"{'; ' + extra if extra else ''}]"
|
|
235
|
+
)
|
|
236
|
+
else:
|
|
237
|
+
note = f" (errors: {', '.join(m.errors)})" if m.errors else ""
|
|
238
|
+
print(f"{m.intervention}\t-> UNRESOLVED{note}")
|
|
239
|
+
return 0 if all(m.resolved for m in matches) else 2
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def cmd_rwe(args: argparse.Namespace) -> int:
|
|
243
|
+
with make_collector(args) as collector:
|
|
244
|
+
profiles = collector.collect_many(args.names)
|
|
245
|
+
if args.json:
|
|
246
|
+
payload = [
|
|
247
|
+
{
|
|
248
|
+
"drug": p.drug,
|
|
249
|
+
"evidence": [e.to_dict() for e in p.evidence],
|
|
250
|
+
"errors": list(p.errors),
|
|
251
|
+
}
|
|
252
|
+
for p in profiles
|
|
253
|
+
]
|
|
254
|
+
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
|
255
|
+
else:
|
|
256
|
+
for p in profiles:
|
|
257
|
+
print(p.drug)
|
|
258
|
+
for e in p.evidence:
|
|
259
|
+
metrics = " ".join(
|
|
260
|
+
f"{k}={v}" for k, v in e.metrics.items() if not isinstance(v, str) or v
|
|
261
|
+
)
|
|
262
|
+
print(f" {e.source}: {metrics}")
|
|
263
|
+
for name in p.errors:
|
|
264
|
+
print(f" {name}: ERROR (not cached; try again later)")
|
|
265
|
+
return 2 if any(p.errors for p in profiles) else 0
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def cmd_cache(args: argparse.Namespace) -> int:
|
|
269
|
+
clear = args.action == "clear"
|
|
270
|
+
if not (clear and args.source):
|
|
271
|
+
path = default_cache_path()
|
|
272
|
+
if not path.exists():
|
|
273
|
+
print(f"{path}: no cache yet")
|
|
274
|
+
else:
|
|
275
|
+
with SQLiteCache(path) as cache:
|
|
276
|
+
if clear:
|
|
277
|
+
print(f"removed {cache.clear(args.resolver)} entries from {path}")
|
|
278
|
+
else:
|
|
279
|
+
info = cache.info()
|
|
280
|
+
print(f"{path}\n hits: {info['hits']}\n misses: {info['misses']}")
|
|
281
|
+
if not (clear and args.resolver):
|
|
282
|
+
rwe_path = default_rwe_cache_path()
|
|
283
|
+
if not rwe_path.exists():
|
|
284
|
+
print(f"{rwe_path}: no cache yet")
|
|
285
|
+
else:
|
|
286
|
+
with SQLiteEvidenceCache(rwe_path) as rwe_cache:
|
|
287
|
+
if clear:
|
|
288
|
+
print(f"removed {rwe_cache.clear(args.source)} entries from {rwe_path}")
|
|
289
|
+
else:
|
|
290
|
+
counts = rwe_cache.info()
|
|
291
|
+
print(rwe_path)
|
|
292
|
+
for source, n in counts.items():
|
|
293
|
+
print(f" {source}: {n}")
|
|
294
|
+
if not counts:
|
|
295
|
+
print(" (empty)")
|
|
296
|
+
return 0
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
# ---------------------------------------------------------------- parser
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
303
|
+
parser = argparse.ArgumentParser(
|
|
304
|
+
prog="trialmap",
|
|
305
|
+
description="ClinicalTrials.gov trials → companies → drug assets.",
|
|
306
|
+
epilog=EPILOG,
|
|
307
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
308
|
+
)
|
|
309
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
310
|
+
parser.add_argument("-v", "--verbose", action="count", default=0, help="-v info, -vv debug")
|
|
311
|
+
parser.add_argument(
|
|
312
|
+
"--user-agent", help="User-Agent for API requests (optional; default: trialmap/<version>)"
|
|
313
|
+
)
|
|
314
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
315
|
+
|
|
316
|
+
p = sub.add_parser(
|
|
317
|
+
"trials",
|
|
318
|
+
help="trials for companies or keywords, plus their drug assets",
|
|
319
|
+
epilog=EPILOG,
|
|
320
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
321
|
+
)
|
|
322
|
+
mode = p.add_mutually_exclusive_group(required=True)
|
|
323
|
+
mode.add_argument("--companies", nargs="+", metavar="NAME", help="company names")
|
|
324
|
+
mode.add_argument(
|
|
325
|
+
"--companies-file", metavar="FILE", help="one company per line (aliases after |)"
|
|
326
|
+
)
|
|
327
|
+
mode.add_argument(
|
|
328
|
+
"--keywords", metavar="QUERY", help="Essie query, e.g. '\"Atopic Dermatitis\"'"
|
|
329
|
+
)
|
|
330
|
+
p.add_argument(
|
|
331
|
+
"--field",
|
|
332
|
+
choices=["term", "cond", "intr", "titles", "outc"],
|
|
333
|
+
default="term",
|
|
334
|
+
help="field searched by --keywords: term=all text, cond=conditions, intr=interventions",
|
|
335
|
+
)
|
|
336
|
+
p.add_argument("--all-sponsors", action="store_true", help="with --keywords: not only INDUSTRY")
|
|
337
|
+
p.add_argument("--max-studies", type=int, metavar="N", help="cap per company / per query")
|
|
338
|
+
p.add_argument("--no-resolve", action="store_true", help="skip drug → asset lookups")
|
|
339
|
+
p.add_argument("--format", choices=["csv", "json"], default="csv")
|
|
340
|
+
p.add_argument("--out", default="trialmap", help="output prefix (default: %(default)s)")
|
|
341
|
+
p.add_argument(
|
|
342
|
+
"--rwe", action="store_true", help="also write <out>_rwe: real-world evidence per asset"
|
|
343
|
+
)
|
|
344
|
+
_add_resolver_options(p)
|
|
345
|
+
_add_rwe_options(p, with_no_cache=False)
|
|
346
|
+
p.set_defaults(func=cmd_trials)
|
|
347
|
+
|
|
348
|
+
p = sub.add_parser("resolve", help="resolve drug codes/names to assets")
|
|
349
|
+
p.add_argument("names", nargs="+", metavar="NAME")
|
|
350
|
+
p.add_argument("--json", action="store_true", help="print JSON")
|
|
351
|
+
_add_resolver_options(p)
|
|
352
|
+
p.set_defaults(func=cmd_resolve)
|
|
353
|
+
|
|
354
|
+
p = sub.add_parser(
|
|
355
|
+
"rwe",
|
|
356
|
+
help="real-world evidence for drug names",
|
|
357
|
+
epilog="Counts are signals to review, not rates: FAERS counts reports, not patients.",
|
|
358
|
+
)
|
|
359
|
+
p.add_argument("names", nargs="+", metavar="DRUG", help="drug names, e.g. methotrexate")
|
|
360
|
+
p.add_argument("--json", action="store_true", help="print JSON, with the detail items")
|
|
361
|
+
_add_rwe_options(p, with_no_cache=True)
|
|
362
|
+
p.set_defaults(func=cmd_rwe)
|
|
363
|
+
|
|
364
|
+
p = sub.add_parser("cache", help="inspect or clear the lookup caches (drug names, RWE)")
|
|
365
|
+
p.add_argument("action", choices=["info", "clear"])
|
|
366
|
+
clear_one = p.add_mutually_exclusive_group()
|
|
367
|
+
clear_one.add_argument("--resolver", help="with clear: only this resolver's entries")
|
|
368
|
+
clear_one.add_argument("--source", help="with clear: only this RWE source's entries")
|
|
369
|
+
p.set_defaults(func=cmd_cache)
|
|
370
|
+
return parser
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
374
|
+
args = build_parser().parse_args(argv)
|
|
375
|
+
level = {0: logging.WARNING, 1: logging.INFO}.get(args.verbose, logging.DEBUG)
|
|
376
|
+
logging.basicConfig(level=level, format="%(levelname)s %(name)s: %(message)s")
|
|
377
|
+
try:
|
|
378
|
+
return int(args.func(args))
|
|
379
|
+
except TrialmapError as exc:
|
|
380
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
381
|
+
return 1
|
|
382
|
+
except KeyboardInterrupt: # pragma: no cover
|
|
383
|
+
return 130
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
if __name__ == "__main__": # pragma: no cover
|
|
387
|
+
raise SystemExit(main())
|