plasticdb 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- plasticdb/__init__.py +97 -0
- plasticdb/__main__.py +5 -0
- plasticdb/_cache.py +176 -0
- plasticdb/_client.py +642 -0
- plasticdb/_download.py +195 -0
- plasticdb/_endpoints.py +67 -0
- plasticdb/_fasta.py +53 -0
- plasticdb/_inputs.py +229 -0
- plasticdb/_pagination.py +437 -0
- plasticdb/_response.py +244 -0
- plasticdb/_version.py +1 -0
- plasticdb/cli/__init__.py +40 -0
- plasticdb/cli/_app.py +1330 -0
- plasticdb/cli/_config.py +113 -0
- plasticdb/cli/_core.py +686 -0
- plasticdb/cli/_output.py +250 -0
- plasticdb/errors.py +335 -0
- plasticdb/models/__init__.py +151 -0
- plasticdb/models/_base.py +51 -0
- plasticdb/models/catalogue.py +405 -0
- plasticdb/models/common.py +302 -0
- plasticdb/models/jobs.py +339 -0
- plasticdb/models/other.py +292 -0
- plasticdb/models/releases.py +84 -0
- plasticdb/py.typed +0 -0
- plasticdb/resources/__init__.py +25 -0
- plasticdb/resources/_base.py +13 -0
- plasticdb/resources/_filters.py +227 -0
- plasticdb/resources/catalogue.py +277 -0
- plasticdb/resources/downloads.py +251 -0
- plasticdb/resources/evidence.py +272 -0
- plasticdb/resources/jobs.py +932 -0
- plasticdb/resources/proteins.py +318 -0
- plasticdb/resources/releases.py +51 -0
- plasticdb-0.0.1.dist-info/METADATA +390 -0
- plasticdb-0.0.1.dist-info/RECORD +39 -0
- plasticdb-0.0.1.dist-info/WHEEL +4 -0
- plasticdb-0.0.1.dist-info/entry_points.txt +2 -0
- plasticdb-0.0.1.dist-info/licenses/LICENSE +21 -0
plasticdb/__init__.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Python client for the PlasticDB API (https://v2.plasticdb.org).
|
|
2
|
+
|
|
3
|
+
PlasticDB data is licensed CC BY 4.0: cite the dataset release you used
|
|
4
|
+
(``Client.cite()``).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import logging
|
|
8
|
+
|
|
9
|
+
from . import errors, models
|
|
10
|
+
from ._client import DEFAULT_BASE_URL, Citation, Client, Provenance
|
|
11
|
+
from ._pagination import Collection, ResultTable
|
|
12
|
+
from ._response import APIResponse
|
|
13
|
+
from ._version import __version__
|
|
14
|
+
from .errors import (
|
|
15
|
+
APIConnectionError,
|
|
16
|
+
APITimeoutError,
|
|
17
|
+
AuthenticationError,
|
|
18
|
+
BadRequestError,
|
|
19
|
+
ChecksumError,
|
|
20
|
+
ConflictError,
|
|
21
|
+
EngineeredVariantError,
|
|
22
|
+
GoneError,
|
|
23
|
+
IdempotencyKeyReusedError,
|
|
24
|
+
JobError,
|
|
25
|
+
JobFailedError,
|
|
26
|
+
JobTimeoutError,
|
|
27
|
+
LiveOnlyError,
|
|
28
|
+
MergedError,
|
|
29
|
+
MergedRecordWarning,
|
|
30
|
+
NotFoundError,
|
|
31
|
+
PayloadTooLargeError,
|
|
32
|
+
PermissionDenied,
|
|
33
|
+
PlasticDBError,
|
|
34
|
+
RateLimitError,
|
|
35
|
+
ReleaseError,
|
|
36
|
+
ReleaseNoArtefactsError,
|
|
37
|
+
ReleaseNotAvailableError,
|
|
38
|
+
ReleaseNotMountedError,
|
|
39
|
+
ReleaseUnsupportedError,
|
|
40
|
+
ResultsExpiredError,
|
|
41
|
+
ServerError,
|
|
42
|
+
ServiceUnavailableError,
|
|
43
|
+
UnexpectedResponseError,
|
|
44
|
+
UnknownReleaseError,
|
|
45
|
+
UnsupportedFilterError,
|
|
46
|
+
ValidationError,
|
|
47
|
+
)
|
|
48
|
+
from .resources.jobs import Job, JobProvenance
|
|
49
|
+
|
|
50
|
+
logging.getLogger("plasticdb").addHandler(logging.NullHandler())
|
|
51
|
+
|
|
52
|
+
__all__ = [
|
|
53
|
+
"DEFAULT_BASE_URL",
|
|
54
|
+
"APIConnectionError",
|
|
55
|
+
"APIResponse",
|
|
56
|
+
"APITimeoutError",
|
|
57
|
+
"AuthenticationError",
|
|
58
|
+
"BadRequestError",
|
|
59
|
+
"ChecksumError",
|
|
60
|
+
"Citation",
|
|
61
|
+
"Client",
|
|
62
|
+
"Collection",
|
|
63
|
+
"ConflictError",
|
|
64
|
+
"EngineeredVariantError",
|
|
65
|
+
"GoneError",
|
|
66
|
+
"IdempotencyKeyReusedError",
|
|
67
|
+
"Job",
|
|
68
|
+
"JobError",
|
|
69
|
+
"JobFailedError",
|
|
70
|
+
"JobProvenance",
|
|
71
|
+
"JobTimeoutError",
|
|
72
|
+
"LiveOnlyError",
|
|
73
|
+
"MergedError",
|
|
74
|
+
"MergedRecordWarning",
|
|
75
|
+
"NotFoundError",
|
|
76
|
+
"PayloadTooLargeError",
|
|
77
|
+
"PermissionDenied",
|
|
78
|
+
"PlasticDBError",
|
|
79
|
+
"Provenance",
|
|
80
|
+
"RateLimitError",
|
|
81
|
+
"ReleaseError",
|
|
82
|
+
"ReleaseNoArtefactsError",
|
|
83
|
+
"ReleaseNotAvailableError",
|
|
84
|
+
"ReleaseNotMountedError",
|
|
85
|
+
"ReleaseUnsupportedError",
|
|
86
|
+
"ResultTable",
|
|
87
|
+
"ResultsExpiredError",
|
|
88
|
+
"ServerError",
|
|
89
|
+
"ServiceUnavailableError",
|
|
90
|
+
"UnexpectedResponseError",
|
|
91
|
+
"UnknownReleaseError",
|
|
92
|
+
"UnsupportedFilterError",
|
|
93
|
+
"ValidationError",
|
|
94
|
+
"__version__",
|
|
95
|
+
"errors",
|
|
96
|
+
"models",
|
|
97
|
+
]
|
plasticdb/__main__.py
ADDED
plasticdb/_cache.py
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"""On-disk cache for answers from a past release's snapshot.
|
|
2
|
+
|
|
3
|
+
A past release is immutable, so its answers are kept indefinitely. Only
|
|
4
|
+
responses the server marks ``X-PlasticDB-Release-Source: snapshot`` are stored:
|
|
5
|
+
live answers (including the current release, which is served from the live
|
|
6
|
+
database) are never cached.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import contextlib
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import re
|
|
16
|
+
import tempfile
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
import httpx
|
|
20
|
+
|
|
21
|
+
from ._response import (
|
|
22
|
+
RELEASE_HEADER,
|
|
23
|
+
RELEASE_SOURCE_HEADER,
|
|
24
|
+
SHA256_HEADER,
|
|
25
|
+
SOURCE_SHA256_HEADER,
|
|
26
|
+
UNAVAILABLE_HEADER,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
_KEPT_HEADERS = (
|
|
30
|
+
RELEASE_HEADER,
|
|
31
|
+
RELEASE_SOURCE_HEADER,
|
|
32
|
+
UNAVAILABLE_HEADER,
|
|
33
|
+
SHA256_HEADER,
|
|
34
|
+
SOURCE_SHA256_HEADER,
|
|
35
|
+
"etag",
|
|
36
|
+
"content-type",
|
|
37
|
+
"content-disposition",
|
|
38
|
+
"link",
|
|
39
|
+
)
|
|
40
|
+
_SAFE = re.compile(r"[^A-Za-z0-9._-]")
|
|
41
|
+
_FORMAT = 1
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class ResponseCache:
|
|
45
|
+
def __init__(self, directory: str | os.PathLike[str]) -> None:
|
|
46
|
+
self.directory = Path(directory).expanduser()
|
|
47
|
+
|
|
48
|
+
def _paths(self, release: str, key: str) -> tuple[Path, Path]:
|
|
49
|
+
folder = self.directory / _SAFE.sub("_", release) / key[:2]
|
|
50
|
+
return folder / f"{key}.json", folder / f"{key}.body"
|
|
51
|
+
|
|
52
|
+
@staticmethod
|
|
53
|
+
def key(method: str, url: httpx.URL) -> str:
|
|
54
|
+
params = sorted(url.params.multi_items())
|
|
55
|
+
canonical = json.dumps(
|
|
56
|
+
[_FORMAT, method.upper(), str(url.copy_with(query=None)), params],
|
|
57
|
+
separators=(",", ":"),
|
|
58
|
+
)
|
|
59
|
+
return hashlib.sha256(canonical.encode()).hexdigest()
|
|
60
|
+
|
|
61
|
+
def get(self, release: str, key: str) -> tuple[int, httpx.Headers, bytes] | None:
|
|
62
|
+
meta_path, body_path = self._paths(release, key)
|
|
63
|
+
try:
|
|
64
|
+
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
|
65
|
+
body = body_path.read_bytes()
|
|
66
|
+
except (OSError, ValueError):
|
|
67
|
+
return None
|
|
68
|
+
if meta.get("format") != _FORMAT or meta.get("size") != len(body):
|
|
69
|
+
return None
|
|
70
|
+
if hashlib.sha256(body).hexdigest() != meta.get("sha256"):
|
|
71
|
+
return None
|
|
72
|
+
return int(meta["status"]), httpx.Headers(meta.get("headers") or {}), body
|
|
73
|
+
|
|
74
|
+
def put(self, release: str, key: str, response: httpx.Response) -> None:
|
|
75
|
+
meta_path, body_path = self._paths(release, key)
|
|
76
|
+
body = response.content
|
|
77
|
+
headers = {
|
|
78
|
+
name: response.headers[name] for name in _KEPT_HEADERS if name in response.headers
|
|
79
|
+
}
|
|
80
|
+
meta = {
|
|
81
|
+
"format": _FORMAT,
|
|
82
|
+
"status": response.status_code,
|
|
83
|
+
"url": str(response.request.url),
|
|
84
|
+
"headers": headers,
|
|
85
|
+
"size": len(body),
|
|
86
|
+
"sha256": hashlib.sha256(body).hexdigest(),
|
|
87
|
+
}
|
|
88
|
+
try:
|
|
89
|
+
meta_path.parent.mkdir(parents=True, exist_ok=True)
|
|
90
|
+
_atomic_write(body_path, body)
|
|
91
|
+
_atomic_write(meta_path, json.dumps(meta, indent=1).encode())
|
|
92
|
+
except OSError:
|
|
93
|
+
return
|
|
94
|
+
|
|
95
|
+
def get_file(self, release: str, key: str) -> tuple[int, httpx.Headers, Path] | None:
|
|
96
|
+
"""A cached download: its status, headers and the path of its verified body."""
|
|
97
|
+
meta_path, body_path = self._paths(release, key)
|
|
98
|
+
try:
|
|
99
|
+
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
|
100
|
+
size = body_path.stat().st_size
|
|
101
|
+
except (OSError, ValueError):
|
|
102
|
+
return None
|
|
103
|
+
if meta.get("format") != _FORMAT or meta.get("size") != size:
|
|
104
|
+
return None
|
|
105
|
+
try:
|
|
106
|
+
digest = sha256_file(body_path)
|
|
107
|
+
except OSError:
|
|
108
|
+
return None
|
|
109
|
+
if digest != meta.get("sha256"):
|
|
110
|
+
return None
|
|
111
|
+
return int(meta["status"]), httpx.Headers(meta.get("headers") or {}), body_path
|
|
112
|
+
|
|
113
|
+
def put_file(
|
|
114
|
+
self,
|
|
115
|
+
release: str,
|
|
116
|
+
key: str,
|
|
117
|
+
*,
|
|
118
|
+
source: Path,
|
|
119
|
+
status: int,
|
|
120
|
+
headers: httpx.Headers,
|
|
121
|
+
url: str,
|
|
122
|
+
sha256: str,
|
|
123
|
+
) -> None:
|
|
124
|
+
"""Keep a copy of a downloaded file (already hashed as ``sha256``)."""
|
|
125
|
+
import shutil
|
|
126
|
+
|
|
127
|
+
meta_path, body_path = self._paths(release, key)
|
|
128
|
+
kept = {name: headers[name] for name in _KEPT_HEADERS if name in headers}
|
|
129
|
+
meta = {
|
|
130
|
+
"format": _FORMAT,
|
|
131
|
+
"status": status,
|
|
132
|
+
"url": url,
|
|
133
|
+
"headers": kept,
|
|
134
|
+
"size": source.stat().st_size,
|
|
135
|
+
"sha256": sha256,
|
|
136
|
+
}
|
|
137
|
+
try:
|
|
138
|
+
meta_path.parent.mkdir(parents=True, exist_ok=True)
|
|
139
|
+
fd, tmp = tempfile.mkstemp(dir=body_path.parent, prefix=".tmp-")
|
|
140
|
+
os.close(fd)
|
|
141
|
+
try:
|
|
142
|
+
shutil.copyfile(source, tmp)
|
|
143
|
+
os.replace(tmp, body_path)
|
|
144
|
+
except BaseException:
|
|
145
|
+
with contextlib.suppress(OSError):
|
|
146
|
+
os.unlink(tmp)
|
|
147
|
+
raise
|
|
148
|
+
_atomic_write(meta_path, json.dumps(meta, indent=1).encode())
|
|
149
|
+
except OSError:
|
|
150
|
+
return
|
|
151
|
+
|
|
152
|
+
def clear(self, release: str | None = None) -> None:
|
|
153
|
+
import shutil
|
|
154
|
+
|
|
155
|
+
target = self.directory / _SAFE.sub("_", release) if release else self.directory
|
|
156
|
+
shutil.rmtree(target, ignore_errors=True)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def sha256_file(path: Path, chunk: int = 1 << 20) -> str:
|
|
160
|
+
digest = hashlib.sha256()
|
|
161
|
+
with path.open("rb") as fh:
|
|
162
|
+
for block in iter(lambda: fh.read(chunk), b""):
|
|
163
|
+
digest.update(block)
|
|
164
|
+
return digest.hexdigest()
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _atomic_write(path: Path, data: bytes) -> None:
|
|
168
|
+
fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".tmp-")
|
|
169
|
+
try:
|
|
170
|
+
with os.fdopen(fd, "wb") as fh:
|
|
171
|
+
fh.write(data)
|
|
172
|
+
os.replace(tmp, path)
|
|
173
|
+
except BaseException:
|
|
174
|
+
with contextlib.suppress(OSError):
|
|
175
|
+
os.unlink(tmp)
|
|
176
|
+
raise
|