seroverse 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seroverse/__init__.py +0 -0
- seroverse/_version.py +24 -0
- seroverse/client/__init__.py +0 -0
- seroverse/client/base.py +287 -0
- seroverse/client/dsmz.py +234 -0
- seroverse/client/ena.py +93 -0
- seroverse/client/ncbi.py +329 -0
- seroverse/client/pathogenwatch.py +798 -0
- seroverse/core/__init__.py +0 -0
- seroverse/core/collections.py +280 -0
- seroverse/core/genome.py +135 -0
- seroverse/core/graph/__init__.py +0 -0
- seroverse/core/graph/align.py +328 -0
- seroverse/core/graph/arborescence.py +595 -0
- seroverse/core/graph/csr.py +1115 -0
- seroverse/core/graph/gnn.py +767 -0
- seroverse/core/graph/layout.py +407 -0
- seroverse/core/graph/models.py +1029 -0
- seroverse/core/interval.py +207 -0
- seroverse/core/io/__init__.py +0 -0
- seroverse/core/io/fasta.py +191 -0
- seroverse/core/seq/__init__.py +0 -0
- seroverse/core/seq/alphabet.py +922 -0
- seroverse/core/seq/clustering.py +1295 -0
- seroverse/core/seq/kmers.py +1387 -0
- seroverse/core/seq/pairwise.py +580 -0
- seroverse/core/seq/seq.py +663 -0
- seroverse/progress.py +203 -0
- seroverse/py.typed +0 -0
- seroverse-0.0.1.dist-info/METADATA +715 -0
- seroverse-0.0.1.dist-info/RECORD +33 -0
- seroverse-0.0.1.dist-info/WHEEL +4 -0
- seroverse-0.0.1.dist-info/licenses/LICENSE +674 -0
seroverse/__init__.py
ADDED
|
File without changes
|
seroverse/_version.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.0.1'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 0, 1)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
|
File without changes
|
seroverse/client/base.py
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
"""Unified asynchronous HTTP client and msgspec serialization helpers for SeroVerse."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import hashlib
|
|
5
|
+
import os
|
|
6
|
+
import time
|
|
7
|
+
from collections.abc import AsyncIterator, Coroutine
|
|
8
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
9
|
+
from io import BytesIO
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from types import TracebackType
|
|
12
|
+
from typing import Any, Self
|
|
13
|
+
|
|
14
|
+
import aiohttp
|
|
15
|
+
import msgspec
|
|
16
|
+
|
|
17
|
+
from seroverse.progress import current_progress
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Client:
|
|
21
|
+
"""High-performance asynchronous HTTP client powered by `aiohttp` and `msgspec`.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
base_url: Base endpoint URL.
|
|
25
|
+
api_key_header: Optional HTTP header key for API authorization.
|
|
26
|
+
api_key: Optional API key string.
|
|
27
|
+
user_agent: User-Agent header value.
|
|
28
|
+
timeout: Total request timeout in seconds.
|
|
29
|
+
max_concurrency: Maximum concurrent in-flight requests per event loop.
|
|
30
|
+
polite_delay: Optional minimum delay (in seconds) after live network requests.
|
|
31
|
+
recorder: Optional `cassetter` recorder instance.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
base_url: str,
|
|
37
|
+
api_key_header: str | None = None,
|
|
38
|
+
api_key: str | None = None,
|
|
39
|
+
user_agent: str = "seroverse-client/0.1",
|
|
40
|
+
timeout: float = 30.0,
|
|
41
|
+
*,
|
|
42
|
+
max_concurrency: int = 5,
|
|
43
|
+
polite_delay: float = 0.0,
|
|
44
|
+
recorder: Any | None = None,
|
|
45
|
+
) -> None:
|
|
46
|
+
self.base_url = base_url.rstrip("/")
|
|
47
|
+
self.api_key_header = api_key_header
|
|
48
|
+
self._api_key: str | None = api_key
|
|
49
|
+
self.user_agent = user_agent
|
|
50
|
+
self.timeout = aiohttp.ClientTimeout(total=timeout)
|
|
51
|
+
self.max_concurrency = max(1, int(max_concurrency))
|
|
52
|
+
self.polite_delay = float(polite_delay)
|
|
53
|
+
self.recorder = recorder
|
|
54
|
+
self._session: aiohttp.ClientSession | None = None
|
|
55
|
+
self._sem: asyncio.Semaphore | None = None
|
|
56
|
+
self._sem_loop: asyncio.AbstractEventLoop | None = None
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def api_key(self) -> str | None:
|
|
60
|
+
"""Return the configured API key string, or `None`."""
|
|
61
|
+
return self._api_key
|
|
62
|
+
|
|
63
|
+
@staticmethod
|
|
64
|
+
def encode_msgpack(obj: Any) -> bytes:
|
|
65
|
+
"""Serialize an object or dataclass tree into compact MessagePack bytes."""
|
|
66
|
+
return msgspec.msgpack.encode(obj)
|
|
67
|
+
|
|
68
|
+
@staticmethod
|
|
69
|
+
def decode_msgpack[T](data: bytes, type_: type[T]) -> T:
|
|
70
|
+
"""Deserialize MessagePack bytes directly into a typed `@dataclass(frozen=True, slots=True)` structure."""
|
|
71
|
+
return msgspec.msgpack.decode(data, type=type_)
|
|
72
|
+
|
|
73
|
+
@staticmethod
|
|
74
|
+
def save_msgpack(path: Path, obj: Any) -> None:
|
|
75
|
+
"""Atomically persist `obj` to `path` as binary MessagePack."""
|
|
76
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
77
|
+
tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")
|
|
78
|
+
tmp.write_bytes(msgspec.msgpack.encode(obj))
|
|
79
|
+
os.replace(tmp, path)
|
|
80
|
+
|
|
81
|
+
@staticmethod
|
|
82
|
+
def load_msgpack[T](path: Path, type_: type[T]) -> T:
|
|
83
|
+
"""Load and decode a binary MessagePack file into `type_`."""
|
|
84
|
+
return msgspec.msgpack.decode(path.read_bytes(), type=type_)
|
|
85
|
+
|
|
86
|
+
def __repr__(self) -> str:
|
|
87
|
+
key_repr = "'**********'" if self._api_key else "None"
|
|
88
|
+
return f"{self.__class__.__name__}(base_url='{self.base_url}', api_key={key_repr})"
|
|
89
|
+
|
|
90
|
+
def __cache_fingerprint__(self) -> tuple[str, str, str]:
|
|
91
|
+
"""Returns a deterministic, credential-safe fingerprint tuple for content-addressable caches."""
|
|
92
|
+
key_digest = hashlib.sha256((self._api_key or "").encode("utf-8")).hexdigest()[:16] if self._api_key else ""
|
|
93
|
+
return (type(self).__qualname__, self.base_url, key_digest)
|
|
94
|
+
|
|
95
|
+
def _headers(self) -> dict[str, str]:
|
|
96
|
+
headers = {"User-Agent": self.user_agent, "Accept": "application/json"}
|
|
97
|
+
if self.api_key_header and self._api_key:
|
|
98
|
+
headers[self.api_key_header] = self._api_key
|
|
99
|
+
return headers
|
|
100
|
+
|
|
101
|
+
def _get_semaphore(self) -> asyncio.Semaphore:
|
|
102
|
+
loop = asyncio.get_running_loop()
|
|
103
|
+
if self._sem is None or self._sem_loop is not loop:
|
|
104
|
+
self._sem = asyncio.Semaphore(self.max_concurrency)
|
|
105
|
+
self._sem_loop = loop
|
|
106
|
+
return self._sem
|
|
107
|
+
|
|
108
|
+
async def polite_sleep(self, t_req: float, delay: float | None = None) -> None:
|
|
109
|
+
"""Sleep politely after a live network request, skipping sleep on fast cassette replays."""
|
|
110
|
+
wait_s = self.polite_delay if delay is None else delay
|
|
111
|
+
if wait_s <= 0.0:
|
|
112
|
+
return
|
|
113
|
+
elapsed = time.perf_counter() - t_req
|
|
114
|
+
if not (self.recorder is not None and elapsed < 0.25) and elapsed > 0.02:
|
|
115
|
+
await asyncio.sleep(wait_s)
|
|
116
|
+
|
|
117
|
+
async def get_session(self) -> aiohttp.ClientSession:
|
|
118
|
+
"""Retrieve or instantiate the underlying `aiohttp.ClientSession` for the active event loop."""
|
|
119
|
+
current_loop = asyncio.get_running_loop()
|
|
120
|
+
if (
|
|
121
|
+
self._session is None
|
|
122
|
+
or self._session.closed
|
|
123
|
+
or getattr(self._session, "_loop", None) is not current_loop
|
|
124
|
+
or getattr(self._session, "_loop", current_loop).is_closed()
|
|
125
|
+
):
|
|
126
|
+
self._session = aiohttp.ClientSession(
|
|
127
|
+
headers=self._headers(),
|
|
128
|
+
timeout=self.timeout,
|
|
129
|
+
)
|
|
130
|
+
return self._session
|
|
131
|
+
|
|
132
|
+
async def __aenter__(self) -> Self:
|
|
133
|
+
await self.get_session()
|
|
134
|
+
return self
|
|
135
|
+
|
|
136
|
+
async def __aexit__(
|
|
137
|
+
self,
|
|
138
|
+
exc_type: type[BaseException] | None,
|
|
139
|
+
exc_val: BaseException | None,
|
|
140
|
+
exc_tb: TracebackType | None,
|
|
141
|
+
) -> None:
|
|
142
|
+
await self.close()
|
|
143
|
+
|
|
144
|
+
def __enter__(self) -> Self:
|
|
145
|
+
return self
|
|
146
|
+
|
|
147
|
+
def __exit__(
|
|
148
|
+
self,
|
|
149
|
+
exc_type: type[BaseException] | None,
|
|
150
|
+
exc_val: BaseException | None,
|
|
151
|
+
exc_tb: TracebackType | None,
|
|
152
|
+
) -> None:
|
|
153
|
+
self.close_sync()
|
|
154
|
+
|
|
155
|
+
async def close(self) -> None:
|
|
156
|
+
"""Close the underlying `aiohttp.ClientSession` if open."""
|
|
157
|
+
if self._session is not None:
|
|
158
|
+
if not self._session.closed:
|
|
159
|
+
loop = getattr(self._session, "_loop", None)
|
|
160
|
+
if loop is None or not loop.is_closed():
|
|
161
|
+
await self._session.close()
|
|
162
|
+
self._session = None
|
|
163
|
+
|
|
164
|
+
def close_sync(self) -> None:
|
|
165
|
+
"""Close the underlying `aiohttp.ClientSession` synchronously."""
|
|
166
|
+
if self._session is not None and not self._session.closed:
|
|
167
|
+
loop = getattr(self._session, "_loop", None)
|
|
168
|
+
if loop is not None and not loop.is_closed():
|
|
169
|
+
self.run_sync(self.close())
|
|
170
|
+
else:
|
|
171
|
+
self._session = None
|
|
172
|
+
|
|
173
|
+
def _url(self, endpoint: str) -> str:
|
|
174
|
+
if endpoint.startswith(("http://", "https://")):
|
|
175
|
+
return endpoint
|
|
176
|
+
return f"{self.base_url}/{endpoint.lstrip('/')}" if endpoint else self.base_url
|
|
177
|
+
|
|
178
|
+
async def request(self, method: str, endpoint: str, **kwargs: Any) -> Any:
|
|
179
|
+
"""Send an async HTTP request and return decoded JSON (via `msgspec`) or text."""
|
|
180
|
+
session = await self.get_session()
|
|
181
|
+
url = self._url(endpoint)
|
|
182
|
+
async with self._get_semaphore():
|
|
183
|
+
t_req = time.perf_counter()
|
|
184
|
+
try:
|
|
185
|
+
async with session.request(method, url, **kwargs) as resp:
|
|
186
|
+
raw = await resp.read()
|
|
187
|
+
if resp.status >= 400:
|
|
188
|
+
body = raw.decode("utf-8", errors="replace").strip()
|
|
189
|
+
reason = resp.reason or "HTTP Error"
|
|
190
|
+
detail = f"{reason}: {body[:300]}" if body else reason
|
|
191
|
+
raise aiohttp.ClientResponseError(
|
|
192
|
+
resp.request_info,
|
|
193
|
+
resp.history,
|
|
194
|
+
status=resp.status,
|
|
195
|
+
message=detail,
|
|
196
|
+
headers=resp.headers,
|
|
197
|
+
)
|
|
198
|
+
ctype = (resp.content_type or "").lower()
|
|
199
|
+
if "json" in ctype:
|
|
200
|
+
return msgspec.json.decode(raw)
|
|
201
|
+
return raw.decode("utf-8", errors="replace")
|
|
202
|
+
finally:
|
|
203
|
+
await self.polite_sleep(t_req)
|
|
204
|
+
|
|
205
|
+
async def get(self, endpoint: str, **kwargs: Any) -> Any:
|
|
206
|
+
"""Perform an async HTTP GET request."""
|
|
207
|
+
return await self.request("GET", endpoint, **kwargs)
|
|
208
|
+
|
|
209
|
+
async def post(self, endpoint: str, **kwargs: Any) -> Any:
|
|
210
|
+
"""Perform an async HTTP POST request."""
|
|
211
|
+
return await self.request("POST", endpoint, **kwargs)
|
|
212
|
+
|
|
213
|
+
async def delete(self, endpoint: str, **kwargs: Any) -> Any:
|
|
214
|
+
"""Perform an async HTTP DELETE request."""
|
|
215
|
+
return await self.request("DELETE", endpoint, **kwargs)
|
|
216
|
+
|
|
217
|
+
async def stream_bytes(self, method: str, endpoint: str, **kwargs: Any) -> bytes:
|
|
218
|
+
"""Send an async HTTP request and read raw response bytes."""
|
|
219
|
+
session = await self.get_session()
|
|
220
|
+
url = self._url(endpoint)
|
|
221
|
+
async with self._get_semaphore():
|
|
222
|
+
t_req = time.perf_counter()
|
|
223
|
+
try:
|
|
224
|
+
async with session.request(method, url, **kwargs) as resp:
|
|
225
|
+
resp.raise_for_status()
|
|
226
|
+
return await resp.read()
|
|
227
|
+
finally:
|
|
228
|
+
await self.polite_sleep(t_req)
|
|
229
|
+
|
|
230
|
+
async def get_json[T](self, endpoint: str, *, type_: type[T], **kwargs: Any) -> T:
|
|
231
|
+
"""Fetch `endpoint` via GET and decode JSON directly into `type_` using `msgspec`."""
|
|
232
|
+
raw = await self.stream_bytes("GET", endpoint, **kwargs)
|
|
233
|
+
return msgspec.json.decode(raw, type=type_)
|
|
234
|
+
|
|
235
|
+
async def post_json[T](self, endpoint: str, *, type_: type[T], **kwargs: Any) -> T:
|
|
236
|
+
"""Fetch `endpoint` via POST and decode JSON directly into `type_` using `msgspec`."""
|
|
237
|
+
raw = await self.stream_bytes("POST", endpoint, **kwargs)
|
|
238
|
+
return msgspec.json.decode(raw, type=type_)
|
|
239
|
+
|
|
240
|
+
async def stream_buffer(self, method: str, endpoint: str, **kwargs: Any) -> BytesIO:
|
|
241
|
+
"""Send an async HTTP request and return an in-memory `BytesIO` stream."""
|
|
242
|
+
b = await self.stream_bytes(method, endpoint, **kwargs)
|
|
243
|
+
return BytesIO(b)
|
|
244
|
+
|
|
245
|
+
async def stream_chunks(
|
|
246
|
+
self,
|
|
247
|
+
method: str,
|
|
248
|
+
endpoint: str,
|
|
249
|
+
chunk_size: int = 65536,
|
|
250
|
+
**kwargs: Any,
|
|
251
|
+
) -> AsyncIterator[bytes]:
|
|
252
|
+
"""Stream response bytes in chunks as an async generator, emitting progress when `Content-Length` is known."""
|
|
253
|
+
session = await self.get_session()
|
|
254
|
+
url = self._url(endpoint)
|
|
255
|
+
prog = current_progress()
|
|
256
|
+
async with session.request(method, url, **kwargs) as resp:
|
|
257
|
+
resp.raise_for_status()
|
|
258
|
+
total_bytes = resp.content_length if resp.content_length and resp.content_length > 0 else None
|
|
259
|
+
received = 0
|
|
260
|
+
async for chunk in resp.content.iter_chunked(chunk_size):
|
|
261
|
+
received += len(chunk)
|
|
262
|
+
if total_bytes is not None and prog.is_active:
|
|
263
|
+
prog.report(
|
|
264
|
+
min(1.0, received / total_bytes),
|
|
265
|
+
detail=f"Streaming {endpoint} ({received:,} / {total_bytes:,} B)",
|
|
266
|
+
)
|
|
267
|
+
yield chunk
|
|
268
|
+
|
|
269
|
+
def run_sync[T](self, coro: Coroutine[Any, Any, T]) -> T:
|
|
270
|
+
"""Execute a coroutine synchronously and close any session bound to the temporary event loop."""
|
|
271
|
+
|
|
272
|
+
async def _runner() -> T:
|
|
273
|
+
try:
|
|
274
|
+
return await coro
|
|
275
|
+
finally:
|
|
276
|
+
await self.close()
|
|
277
|
+
|
|
278
|
+
try:
|
|
279
|
+
loop = asyncio.get_running_loop()
|
|
280
|
+
except RuntimeError:
|
|
281
|
+
loop = None
|
|
282
|
+
|
|
283
|
+
if loop and loop.is_running():
|
|
284
|
+
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
285
|
+
future = pool.submit(lambda: asyncio.run(_runner()))
|
|
286
|
+
return future.result()
|
|
287
|
+
return asyncio.run(_runner())
|
seroverse/client/dsmz.py
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""DSMZ StrainInfo v2 and BacDive v2 REST API clients and culture-collection models."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import os
|
|
7
|
+
import urllib.parse
|
|
8
|
+
from collections.abc import Sequence
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import msgspec
|
|
13
|
+
|
|
14
|
+
from seroverse.client.base import Client
|
|
15
|
+
from seroverse.progress import current_progress
|
|
16
|
+
|
|
17
|
+
STRAININFO_V2_URL = "https://api.straininfo.dsmz.de/v2"
|
|
18
|
+
BACDIVE_V2_URL = "https://api.bacdive.dsmz.de/v2"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True, slots=True)
|
|
22
|
+
class StrainLiterature:
|
|
23
|
+
"""Immutable bibliographic entry linked to a StrainInfo culture deposit."""
|
|
24
|
+
|
|
25
|
+
id: int = 0
|
|
26
|
+
pubmed: int = 0
|
|
27
|
+
doi: str = ""
|
|
28
|
+
title: str = ""
|
|
29
|
+
year: int = 0
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True, slots=True)
|
|
33
|
+
class StrainPassport:
|
|
34
|
+
"""Consolidated DSMZ StrainInfo v2 passport with culture-collection synonyms, PMIDs, and INSDC accessions."""
|
|
35
|
+
|
|
36
|
+
si_id: int
|
|
37
|
+
taxon_name: str = ""
|
|
38
|
+
ncbi_taxid: int = 0
|
|
39
|
+
bacdive_id: int = 0
|
|
40
|
+
type_strain: bool = False
|
|
41
|
+
designations: tuple[str, ...] = ()
|
|
42
|
+
pmids: tuple[int, ...] = ()
|
|
43
|
+
dois: tuple[str, ...] = ()
|
|
44
|
+
sequence_accessions: tuple[str, ...] = ()
|
|
45
|
+
genome_accessions: tuple[str, ...] = ()
|
|
46
|
+
|
|
47
|
+
@classmethod
|
|
48
|
+
def from_raw_dict(cls, raw: dict[str, Any]) -> StrainPassport | None:
|
|
49
|
+
"""Parse a raw StrainInfo `/v2/data/strain/max/` JSON object into a `StrainPassport`."""
|
|
50
|
+
si_obj = raw.get("strain") if isinstance(raw.get("strain"), dict) else raw
|
|
51
|
+
if not isinstance(si_obj, dict):
|
|
52
|
+
return None
|
|
53
|
+
raw_id = si_obj.get("siID") if si_obj.get("siID") is not None else si_obj.get("id")
|
|
54
|
+
si_id = int(raw_id or 0) if str(raw_id or "").isdigit() else 0
|
|
55
|
+
if si_id <= 0:
|
|
56
|
+
return None
|
|
57
|
+
|
|
58
|
+
taxon = si_obj.get("taxon") or {}
|
|
59
|
+
taxon_name = str(taxon.get("name") or "").strip() if isinstance(taxon, dict) else ""
|
|
60
|
+
ncbi_taxid = (
|
|
61
|
+
int(taxon.get("ncbi") or 0) if isinstance(taxon, dict) and str(taxon.get("ncbi") or "").isdigit() else 0
|
|
62
|
+
)
|
|
63
|
+
raw_bd = si_obj.get("bdID") if si_obj.get("bdID") is not None else si_obj.get("bacdive")
|
|
64
|
+
bacdive_id = int(raw_bd or 0) if str(raw_bd or "").isdigit() else 0
|
|
65
|
+
type_strain = bool(si_obj.get("typeStrain") if "typeStrain" in si_obj else si_obj.get("type_strain"))
|
|
66
|
+
|
|
67
|
+
des_list: list[str] = []
|
|
68
|
+
rel = si_obj.get("relation") or {}
|
|
69
|
+
if isinstance(rel, dict):
|
|
70
|
+
for c_item in (rel.get("deposit") or ()) or (rel.get("culture") or ()):
|
|
71
|
+
if isinstance(c_item, dict):
|
|
72
|
+
sn = str(c_item.get("designation") or c_item.get("strain_number") or "").strip()
|
|
73
|
+
if sn and sn not in des_list:
|
|
74
|
+
des_list.append(sn)
|
|
75
|
+
for d_item in rel.get("designation") or ():
|
|
76
|
+
sn_d = str(d_item.get("designation") if isinstance(d_item, dict) else d_item or "").strip()
|
|
77
|
+
if sn_d and sn_d not in des_list:
|
|
78
|
+
des_list.append(sn_d)
|
|
79
|
+
for dep_item in raw.get("deposits") or ():
|
|
80
|
+
if isinstance(dep_item, dict) and (sn_dep := str(dep_item.get("designation") or "").strip()):
|
|
81
|
+
if sn_dep not in des_list:
|
|
82
|
+
des_list.append(sn_dep)
|
|
83
|
+
|
|
84
|
+
pmids: list[int] = []
|
|
85
|
+
dois: list[str] = []
|
|
86
|
+
for lit in si_obj.get("literature") or ():
|
|
87
|
+
if isinstance(lit, dict):
|
|
88
|
+
pm_raw = str(lit.get("pubmed") or "").strip()
|
|
89
|
+
if pm_raw.isdigit() and (pm_int := int(pm_raw)) > 0 and pm_int not in pmids:
|
|
90
|
+
pmids.append(pm_int)
|
|
91
|
+
doi_raw = str(lit.get("doi") or "").strip()
|
|
92
|
+
if doi_raw and doi_raw not in dois:
|
|
93
|
+
dois.append(doi_raw)
|
|
94
|
+
|
|
95
|
+
seq_accs: list[str] = []
|
|
96
|
+
gen_accs: list[str] = []
|
|
97
|
+
for seq in (si_obj.get("sequence") or ()) or (si_obj.get("sequences") or ()):
|
|
98
|
+
if isinstance(seq, dict) and (acc := str(seq.get("accessionNumber") or seq.get("accession") or "").strip()):
|
|
99
|
+
s_type = str(seq.get("type") or "").lower()
|
|
100
|
+
if "genome" in s_type or acc.startswith(("GCA_", "GCF_")):
|
|
101
|
+
if acc not in gen_accs:
|
|
102
|
+
gen_accs.append(acc)
|
|
103
|
+
elif s_type != "rrnaop" and acc not in seq_accs:
|
|
104
|
+
seq_accs.append(acc)
|
|
105
|
+
|
|
106
|
+
return cls(
|
|
107
|
+
si_id=si_id,
|
|
108
|
+
taxon_name=taxon_name,
|
|
109
|
+
ncbi_taxid=ncbi_taxid,
|
|
110
|
+
bacdive_id=bacdive_id,
|
|
111
|
+
type_strain=type_strain,
|
|
112
|
+
designations=tuple(des_list),
|
|
113
|
+
pmids=tuple(pmids),
|
|
114
|
+
dois=tuple(dois),
|
|
115
|
+
sequence_accessions=tuple(seq_accs),
|
|
116
|
+
genome_accessions=tuple(gen_accs),
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class StrainInfoClient(Client):
|
|
121
|
+
"""Asynchronous client for the public DSMZ StrainInfo v2 REST API."""
|
|
122
|
+
|
|
123
|
+
def __init__(
|
|
124
|
+
self,
|
|
125
|
+
*,
|
|
126
|
+
timeout: float = 30.0,
|
|
127
|
+
max_concurrency: int = 5,
|
|
128
|
+
polite_delay: float = 0.05,
|
|
129
|
+
recorder: Any | None = None,
|
|
130
|
+
) -> None:
|
|
131
|
+
super().__init__(
|
|
132
|
+
base_url=STRAININFO_V2_URL,
|
|
133
|
+
user_agent="seroverse-straininfo-client/0.1",
|
|
134
|
+
timeout=timeout,
|
|
135
|
+
max_concurrency=max_concurrency,
|
|
136
|
+
polite_delay=polite_delay,
|
|
137
|
+
recorder=recorder,
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
async def _search_ids(self, path: str) -> list[int]:
|
|
141
|
+
try:
|
|
142
|
+
raw_ids = await self.get_json(path, type_=list[int])
|
|
143
|
+
return [int(x) for x in raw_ids if int(x) > 0]
|
|
144
|
+
except Exception:
|
|
145
|
+
return []
|
|
146
|
+
|
|
147
|
+
async def search_by_taxon(self, taxon_name: str) -> list[int]:
|
|
148
|
+
"""Return all StrainInfo IDs (`SI-ID`s) for a species or genus name."""
|
|
149
|
+
return await self._search_ids(f"search/strain/tax_name/{urllib.parse.quote(taxon_name.strip())}")
|
|
150
|
+
|
|
151
|
+
async def search_by_cc_no(self, cc_no: str) -> list[int]:
|
|
152
|
+
"""Return matching `SI-ID`s for a culture collection number (e.g. `ATCC 25238`, `NCTC 9001`)."""
|
|
153
|
+
return await self._search_ids(f"search/strain/cc_no/{urllib.parse.quote(cc_no.strip())}")
|
|
154
|
+
|
|
155
|
+
async def search_by_designation(self, designation: str) -> list[int]:
|
|
156
|
+
"""Return matching `SI-ID`s for an arbitrary strain designation (e.g. `Prague 1550/57`)."""
|
|
157
|
+
return await self._search_ids(f"search/strain/des/{urllib.parse.quote(designation.strip())}")
|
|
158
|
+
|
|
159
|
+
async def search_by_collection(self, collection_code: str) -> list[int]:
|
|
160
|
+
"""Return all `SI-ID`s deposited in a culture collection (e.g. `CNCTC`, `NCTC`, `ATCC`)."""
|
|
161
|
+
return await self._search_ids(f"search/strain/cc/{urllib.parse.quote(collection_code.strip())}")
|
|
162
|
+
|
|
163
|
+
async def fetch_passports(self, si_ids: Sequence[int], batch_size: int = 50) -> list[StrainPassport]:
|
|
164
|
+
"""Batch-fetch `StrainPassport` records for `si_ids` via `/v2/data/strain/max/{ids}`."""
|
|
165
|
+
unique_ids = list(dict.fromkeys(int(x) for x in si_ids if int(x) > 0))
|
|
166
|
+
if not unique_ids:
|
|
167
|
+
return []
|
|
168
|
+
passports: list[StrainPassport] = []
|
|
169
|
+
chunks = [unique_ids[i : i + batch_size] for i in range(0, len(unique_ids), batch_size)]
|
|
170
|
+
prog = current_progress()
|
|
171
|
+
completed = 0
|
|
172
|
+
total = len(chunks)
|
|
173
|
+
|
|
174
|
+
async def _fetch_chunk(chunk: Sequence[int]) -> list[StrainPassport]:
|
|
175
|
+
nonlocal completed
|
|
176
|
+
joined = ",".join(str(x) for x in chunk)
|
|
177
|
+
try:
|
|
178
|
+
raw = await self.stream_bytes("GET", f"data/strain/max/{joined}")
|
|
179
|
+
items = msgspec.json.decode(raw)
|
|
180
|
+
if isinstance(items, list):
|
|
181
|
+
return [
|
|
182
|
+
p
|
|
183
|
+
for item in items
|
|
184
|
+
if isinstance(item, dict) and (p := StrainPassport.from_raw_dict(item)) is not None
|
|
185
|
+
]
|
|
186
|
+
except Exception:
|
|
187
|
+
pass
|
|
188
|
+
finally:
|
|
189
|
+
completed += 1
|
|
190
|
+
if prog.is_active:
|
|
191
|
+
prog.report(completed / total, detail=f"StrainInfo passports: batch {completed}/{total}")
|
|
192
|
+
return []
|
|
193
|
+
|
|
194
|
+
for res in await asyncio.gather(*[_fetch_chunk(c) for c in chunks]):
|
|
195
|
+
passports.extend(res)
|
|
196
|
+
return passports
|
|
197
|
+
|
|
198
|
+
async def fetch_taxon_passports(self, taxon_name: str, batch_size: int = 50) -> list[StrainPassport]:
|
|
199
|
+
"""Search all strains for `taxon_name` and batch-fetch their complete `StrainPassport` records."""
|
|
200
|
+
si_ids = await self.search_by_taxon(taxon_name)
|
|
201
|
+
return await self.fetch_passports(si_ids, batch_size=batch_size)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
class BacDiveClient(Client):
|
|
205
|
+
"""Asynchronous client for the DSMZ BacDive v2 REST API."""
|
|
206
|
+
|
|
207
|
+
def __init__(
|
|
208
|
+
self,
|
|
209
|
+
api_key: str | None = None,
|
|
210
|
+
*,
|
|
211
|
+
timeout: float = 30.0,
|
|
212
|
+
max_concurrency: int = 4,
|
|
213
|
+
polite_delay: float = 0.05,
|
|
214
|
+
recorder: Any | None = None,
|
|
215
|
+
) -> None:
|
|
216
|
+
token = api_key if api_key is not None else os.environ.get("BACDIVE_API_KEY", "")
|
|
217
|
+
super().__init__(
|
|
218
|
+
base_url=BACDIVE_V2_URL,
|
|
219
|
+
api_key_header="Authorization" if token else None,
|
|
220
|
+
api_key=f"Bearer {token}" if token and not token.startswith("Bearer ") else (token or None),
|
|
221
|
+
user_agent="seroverse-bacdive-client/0.1",
|
|
222
|
+
timeout=timeout,
|
|
223
|
+
max_concurrency=max_concurrency,
|
|
224
|
+
polite_delay=polite_delay,
|
|
225
|
+
recorder=recorder,
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
async def fetch_strain(self, bacdive_id: int) -> dict[str, Any]:
|
|
229
|
+
"""Fetch a BacDive strain record by its integer `bacdive_id`."""
|
|
230
|
+
try:
|
|
231
|
+
res = await self.get(f"fetch/{int(bacdive_id)}")
|
|
232
|
+
return res if isinstance(res, dict) else {}
|
|
233
|
+
except Exception:
|
|
234
|
+
return {}
|
seroverse/client/ena.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""Asynchronous client for the EMBL-EBI ENA Portal REST API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from seroverse.client.base import Client
|
|
10
|
+
|
|
11
|
+
ENA_PORTAL_URL = "https://www.ebi.ac.uk/ena/portal/api"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True, slots=True)
|
|
15
|
+
class ENASampleRecord:
|
|
16
|
+
"""Immutable ENA Portal `sample` record decoded via `msgspec`."""
|
|
17
|
+
|
|
18
|
+
sample_accession: str = ""
|
|
19
|
+
scientific_name: str = ""
|
|
20
|
+
strain: str = ""
|
|
21
|
+
serotype: str = ""
|
|
22
|
+
serovar: str = ""
|
|
23
|
+
tax_id: str = ""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True, slots=True)
|
|
27
|
+
class ENAAssemblyRecord:
|
|
28
|
+
"""Immutable ENA Portal `assembly` record decoded via `msgspec`."""
|
|
29
|
+
|
|
30
|
+
accession: str = ""
|
|
31
|
+
sample_accession: str = ""
|
|
32
|
+
scientific_name: str = ""
|
|
33
|
+
strain: str = ""
|
|
34
|
+
tax_id: str = ""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ENAClient(Client):
|
|
38
|
+
"""Asynchronous client for the EMBL-EBI ENA Portal API."""
|
|
39
|
+
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
*,
|
|
43
|
+
timeout: float = 40.0,
|
|
44
|
+
max_concurrency: int = 4,
|
|
45
|
+
polite_delay: float = 0.05,
|
|
46
|
+
recorder: Any | None = None,
|
|
47
|
+
) -> None:
|
|
48
|
+
super().__init__(
|
|
49
|
+
base_url=ENA_PORTAL_URL,
|
|
50
|
+
user_agent="seroverse-ena-client/0.1",
|
|
51
|
+
timeout=timeout,
|
|
52
|
+
max_concurrency=max_concurrency,
|
|
53
|
+
polite_delay=polite_delay,
|
|
54
|
+
recorder=recorder,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
async def search_samples(
|
|
58
|
+
self,
|
|
59
|
+
query: str,
|
|
60
|
+
fields: Sequence[str] = ("sample_accession", "scientific_name", "strain", "serotype", "serovar", "tax_id"),
|
|
61
|
+
limit: int = 0,
|
|
62
|
+
) -> list[ENASampleRecord]:
|
|
63
|
+
"""Query `search?result=sample` and decode directly into `list[ENASampleRecord]`."""
|
|
64
|
+
params = {
|
|
65
|
+
"result": "sample",
|
|
66
|
+
"query": query,
|
|
67
|
+
"fields": ",".join(fields),
|
|
68
|
+
"limit": str(limit),
|
|
69
|
+
"format": "json",
|
|
70
|
+
}
|
|
71
|
+
try:
|
|
72
|
+
return await self.get_json("search", type_=list[ENASampleRecord], params=params)
|
|
73
|
+
except Exception:
|
|
74
|
+
return []
|
|
75
|
+
|
|
76
|
+
async def search_assemblies(
|
|
77
|
+
self,
|
|
78
|
+
query: str,
|
|
79
|
+
fields: Sequence[str] = ("accession", "sample_accession", "scientific_name", "strain", "tax_id"),
|
|
80
|
+
limit: int = 0,
|
|
81
|
+
) -> list[ENAAssemblyRecord]:
|
|
82
|
+
"""Query `search?result=assembly` and decode directly into `list[ENAAssemblyRecord]`."""
|
|
83
|
+
params = {
|
|
84
|
+
"result": "assembly",
|
|
85
|
+
"query": query,
|
|
86
|
+
"fields": ",".join(fields),
|
|
87
|
+
"limit": str(limit),
|
|
88
|
+
"format": "json",
|
|
89
|
+
}
|
|
90
|
+
try:
|
|
91
|
+
return await self.get_json("search", type_=list[ENAAssemblyRecord], params=params)
|
|
92
|
+
except Exception:
|
|
93
|
+
return []
|