seqevi 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seqevi/__init__.py +7 -0
- seqevi/__main__.py +6 -0
- seqevi/adapters/__init__.py +41 -0
- seqevi/adapters/base.py +131 -0
- seqevi/adapters/dbcan_cazyme.py +588 -0
- seqevi/adapters/eggnog.py +674 -0
- seqevi/adapters/interpro_pfam.py +757 -0
- seqevi/adapters/registry.py +68 -0
- seqevi/annotate.py +413 -0
- seqevi/api.py +390 -0
- seqevi/cli.py +610 -0
- seqevi/distribution/__init__.py +13 -0
- seqevi/distribution/manifest.py +199 -0
- seqevi/distribution/oci.py +490 -0
- seqevi/distribution/setup.py +752 -0
- seqevi/errors.py +73 -0
- seqevi/evidence.py +295 -0
- seqevi/execution_profile.py +526 -0
- seqevi/hashing.py +13 -0
- seqevi/kits/__init__.py +1 -0
- seqevi/kits/dbcan-cazyme.toml +35 -0
- seqevi/resource_lock.py +438 -0
- seqevi/result.py +682 -0
- seqevi/runner.py +163 -0
- seqevi/runtime_identity.py +104 -0
- seqevi/sequence.py +383 -0
- seqevi/service/__init__.py +11 -0
- seqevi/service/app.py +213 -0
- seqevi/service/config.py +38 -0
- seqevi/service/persistence.py +360 -0
- seqevi/store/__init__.py +14 -0
- seqevi/store/artifact.py +225 -0
- seqevi/store/client.py +311 -0
- seqevi/store/contract.py +33 -0
- seqevi/store/factory.py +38 -0
- seqevi/store/local.py +479 -0
- seqevi/store/migration.py +62 -0
- seqevi/store/migrations/__init__.py +1 -0
- seqevi/store/migrations/env.py +30 -0
- seqevi/store/migrations/versions/0001_initial_store.py +103 -0
- seqevi/store/migrations/versions/0002_artifact_byte_size_bigint.py +40 -0
- seqevi/store/migrations/versions/__init__.py +1 -0
- seqevi/store/schema.py +86 -0
- seqevi/store/transport.py +224 -0
- seqevi-0.2.0.dist-info/METADATA +333 -0
- seqevi-0.2.0.dist-info/RECORD +49 -0
- seqevi-0.2.0.dist-info/WHEEL +4 -0
- seqevi-0.2.0.dist-info/entry_points.txt +5 -0
- seqevi-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""Strict parsing for bundled managed runtime kit manifests."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import tomllib
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from importlib.resources import files
|
|
9
|
+
from pathlib import PurePosixPath
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from seqevi.adapters import AdapterName
|
|
13
|
+
from seqevi.errors import SetupError
|
|
14
|
+
|
|
15
|
+
_SCHEMA_VERSION = 1
|
|
16
|
+
_KIT_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*\Z")
|
|
17
|
+
_SHA256 = re.compile(r"[0-9a-f]{64}\Z")
|
|
18
|
+
_OCI_IMAGE = re.compile(
|
|
19
|
+
r"[a-z0-9][a-z0-9./_-]*(?::[A-Za-z0-9._-]+)?@sha256:[0-9a-f]{64}\Z"
|
|
20
|
+
)
|
|
21
|
+
_ROOT_KEYS = {
|
|
22
|
+
"schema_version",
|
|
23
|
+
"kit_id",
|
|
24
|
+
"adapter",
|
|
25
|
+
"platform",
|
|
26
|
+
"dbcan_version",
|
|
27
|
+
"diamond_version",
|
|
28
|
+
"image",
|
|
29
|
+
"resource",
|
|
30
|
+
}
|
|
31
|
+
_RESOURCE_KEYS = {"name", "version", "component"}
|
|
32
|
+
_COMPONENT_KEYS = {"name", "path", "size", "sha256"}
|
|
33
|
+
_DBCAN_COMPONENTS = {
|
|
34
|
+
("CAZy-diamond", "CAZy.dmnd"),
|
|
35
|
+
("dbCAN-HMM", "dbCAN.hmm"),
|
|
36
|
+
("dbCAN-sub-HMM", "dbCAN-sub.hmm"),
|
|
37
|
+
("fam-substrate-mapping", "fam-substrate-mapping.tsv"),
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True, slots=True)
|
|
42
|
+
class KitComponent:
|
|
43
|
+
"""One caller-owned resource file declared by a first-party kit."""
|
|
44
|
+
|
|
45
|
+
name: str
|
|
46
|
+
path: str
|
|
47
|
+
size: int
|
|
48
|
+
sha256: str
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(frozen=True, slots=True)
|
|
52
|
+
class KitManifest:
|
|
53
|
+
"""Validated immutable inputs for one managed runtime kit."""
|
|
54
|
+
|
|
55
|
+
schema_version: int
|
|
56
|
+
kit_id: str
|
|
57
|
+
adapter: AdapterName
|
|
58
|
+
platform: str
|
|
59
|
+
dbcan_version: str
|
|
60
|
+
diamond_version: str
|
|
61
|
+
image: str
|
|
62
|
+
resource_name: str
|
|
63
|
+
resource_version: str
|
|
64
|
+
components: tuple[KitComponent, ...]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def load_kit_manifest(name: str) -> KitManifest:
|
|
68
|
+
"""Load one bundled kit manifest by its user-facing setup name."""
|
|
69
|
+
|
|
70
|
+
if name != "dbcan-cazyme":
|
|
71
|
+
raise SetupError(f"unknown managed kit {name!r}; available kits: dbcan-cazyme")
|
|
72
|
+
try:
|
|
73
|
+
raw = files("seqevi.kits").joinpath(f"{name}.toml").read_bytes()
|
|
74
|
+
document = tomllib.loads(raw.decode("utf-8"))
|
|
75
|
+
except FileNotFoundError as error:
|
|
76
|
+
raise SetupError(f"managed kit manifest is missing for {name!r}") from error
|
|
77
|
+
except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as error:
|
|
78
|
+
raise SetupError(
|
|
79
|
+
f"cannot read managed kit manifest {name!r}: {error}"
|
|
80
|
+
) from error
|
|
81
|
+
return parse_kit_manifest(document, name=name)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def parse_kit_manifest(
|
|
85
|
+
document: dict[str, Any], *, name: str = "<document>"
|
|
86
|
+
) -> KitManifest:
|
|
87
|
+
"""Strictly validate a decoded first-party kit manifest."""
|
|
88
|
+
|
|
89
|
+
if set(document) != _ROOT_KEYS:
|
|
90
|
+
unknown = sorted(set(document) - _ROOT_KEYS)
|
|
91
|
+
missing = sorted(_ROOT_KEYS - set(document))
|
|
92
|
+
detail = []
|
|
93
|
+
if unknown:
|
|
94
|
+
detail.append(f"unknown keys: {', '.join(unknown)}")
|
|
95
|
+
if missing:
|
|
96
|
+
detail.append(f"missing keys: {', '.join(missing)}")
|
|
97
|
+
raise SetupError(
|
|
98
|
+
f"managed kit manifest {name} has invalid schema ({'; '.join(detail)})"
|
|
99
|
+
)
|
|
100
|
+
schema_version = document["schema_version"]
|
|
101
|
+
if isinstance(schema_version, bool) or schema_version != _SCHEMA_VERSION:
|
|
102
|
+
raise SetupError(
|
|
103
|
+
f"managed kit manifest schema_version must be {_SCHEMA_VERSION}"
|
|
104
|
+
)
|
|
105
|
+
kit_id = _string(document["kit_id"], "kit_id")
|
|
106
|
+
if _KIT_ID.fullmatch(kit_id) is None:
|
|
107
|
+
raise SetupError("managed kit manifest kit_id contains unsafe characters")
|
|
108
|
+
adapter_text = _string(document["adapter"], "adapter")
|
|
109
|
+
try:
|
|
110
|
+
adapter = AdapterName(adapter_text)
|
|
111
|
+
except ValueError as error:
|
|
112
|
+
raise SetupError(
|
|
113
|
+
f"managed kit manifest has unknown adapter {adapter_text!r}"
|
|
114
|
+
) from error
|
|
115
|
+
if adapter is not AdapterName.DBCAN_CAZYME:
|
|
116
|
+
raise SetupError("managed kit manifests currently support only dbcan-cazyme")
|
|
117
|
+
platform = _string(document["platform"], "platform")
|
|
118
|
+
if platform != "linux/amd64":
|
|
119
|
+
raise SetupError("dbCAN managed kit platform must be linux/amd64")
|
|
120
|
+
dbcan_version = _string(document["dbcan_version"], "dbcan_version")
|
|
121
|
+
diamond_version = _string(document["diamond_version"], "diamond_version")
|
|
122
|
+
image = _string(document["image"], "image")
|
|
123
|
+
if _OCI_IMAGE.fullmatch(image) is None:
|
|
124
|
+
raise SetupError("managed kit image must contain an immutable sha256 digest")
|
|
125
|
+
|
|
126
|
+
resource = document["resource"]
|
|
127
|
+
if not isinstance(resource, dict) or set(resource) != _RESOURCE_KEYS:
|
|
128
|
+
raise SetupError(
|
|
129
|
+
"managed kit resource must contain name, version and component"
|
|
130
|
+
)
|
|
131
|
+
resource_name = _string(resource["name"], "resource.name")
|
|
132
|
+
resource_version = _string(resource["version"], "resource.version")
|
|
133
|
+
raw_components = resource["component"]
|
|
134
|
+
if not isinstance(raw_components, list) or not raw_components:
|
|
135
|
+
raise SetupError("managed kit resource.component must be a non-empty array")
|
|
136
|
+
components: list[KitComponent] = []
|
|
137
|
+
names: set[str] = set()
|
|
138
|
+
paths: set[str] = set()
|
|
139
|
+
for index, raw in enumerate(raw_components, start=1):
|
|
140
|
+
if not isinstance(raw, dict) or set(raw) != _COMPONENT_KEYS:
|
|
141
|
+
raise SetupError(
|
|
142
|
+
f"managed kit resource component {index} has invalid schema"
|
|
143
|
+
)
|
|
144
|
+
component_name = _string(raw["name"], f"resource.component[{index}].name")
|
|
145
|
+
path = _relative_path(raw["path"], f"resource.component[{index}].path")
|
|
146
|
+
size = raw["size"]
|
|
147
|
+
if isinstance(size, bool) or not isinstance(size, int) or size < 0:
|
|
148
|
+
raise SetupError(
|
|
149
|
+
f"managed kit resource component {component_name!r} has invalid size"
|
|
150
|
+
)
|
|
151
|
+
sha256 = _string(raw["sha256"], f"resource.component[{index}].sha256")
|
|
152
|
+
if _SHA256.fullmatch(sha256) is None:
|
|
153
|
+
raise SetupError(
|
|
154
|
+
f"managed kit resource component {component_name!r} has invalid SHA-256"
|
|
155
|
+
)
|
|
156
|
+
if component_name in names or path in paths:
|
|
157
|
+
raise SetupError("managed kit resource components must be unique")
|
|
158
|
+
names.add(component_name)
|
|
159
|
+
paths.add(path)
|
|
160
|
+
components.append(KitComponent(component_name, path, size, sha256))
|
|
161
|
+
pairs = {(component.name, component.path) for component in components}
|
|
162
|
+
if pairs != _DBCAN_COMPONENTS:
|
|
163
|
+
raise SetupError(
|
|
164
|
+
"dbCAN managed kit must declare the exact four required resource files"
|
|
165
|
+
)
|
|
166
|
+
return KitManifest(
|
|
167
|
+
schema_version=schema_version,
|
|
168
|
+
kit_id=kit_id,
|
|
169
|
+
adapter=adapter,
|
|
170
|
+
platform=platform,
|
|
171
|
+
dbcan_version=dbcan_version,
|
|
172
|
+
diamond_version=diamond_version,
|
|
173
|
+
image=image,
|
|
174
|
+
resource_name=resource_name,
|
|
175
|
+
resource_version=resource_version,
|
|
176
|
+
components=tuple(components),
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _string(value: object, key: str) -> str:
|
|
181
|
+
if not isinstance(value, str) or not value.strip() or "\x00" in value:
|
|
182
|
+
raise SetupError(f"managed kit manifest key {key!r} must be a non-empty string")
|
|
183
|
+
return value
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _relative_path(value: object, key: str) -> str:
|
|
187
|
+
path = _string(value, key)
|
|
188
|
+
if "\\" in path:
|
|
189
|
+
raise SetupError(f"managed kit manifest key {key!r} must use POSIX paths")
|
|
190
|
+
parsed = PurePosixPath(path)
|
|
191
|
+
if (
|
|
192
|
+
path != parsed.as_posix()
|
|
193
|
+
or parsed.is_absolute()
|
|
194
|
+
or any(part in {"", ".", ".."} for part in parsed.parts)
|
|
195
|
+
):
|
|
196
|
+
raise SetupError(
|
|
197
|
+
f"managed kit manifest key {key!r} must be a safe relative path"
|
|
198
|
+
)
|
|
199
|
+
return path
|
|
@@ -0,0 +1,490 @@
|
|
|
1
|
+
"""Ephemeral Docker delegation for managed execution profiles."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import shutil
|
|
8
|
+
import subprocess
|
|
9
|
+
import tempfile
|
|
10
|
+
import uuid
|
|
11
|
+
from collections.abc import Mapping
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from urllib.parse import urlsplit
|
|
15
|
+
|
|
16
|
+
import duckdb
|
|
17
|
+
|
|
18
|
+
from seqevi import __version__
|
|
19
|
+
from seqevi.annotate import AnnotationMetrics, AnnotationSummary
|
|
20
|
+
from seqevi.errors import AnnotationError
|
|
21
|
+
from seqevi.evidence import sha256_digest
|
|
22
|
+
from seqevi.execution_profile import ExecutionProfile, ManagedRuntime
|
|
23
|
+
from seqevi.resource_lock import read_resource_lock
|
|
24
|
+
from seqevi.result import RESULT_FORMAT_VERSION
|
|
25
|
+
|
|
26
|
+
from .manifest import KitManifest, load_kit_manifest
|
|
27
|
+
|
|
28
|
+
_CONTAINER_INPUT = "/mnt/seqevi/input.fasta"
|
|
29
|
+
_CONTAINER_RESOURCE = "/mnt/seqevi/resource"
|
|
30
|
+
_CONTAINER_OUTPUT = "/mnt/seqevi/output"
|
|
31
|
+
_CONTAINER_STORE = "/mnt/seqevi/store"
|
|
32
|
+
_CONTAINER_ENTRYPOINT = "/opt/venv/bin/seqevi"
|
|
33
|
+
_CONTAINER_EXECUTABLE = "/opt/dbcan-venv/bin/run_dbcan"
|
|
34
|
+
_DEFAULT_DOCKER_TIMEOUT_SECONDS = 30.0
|
|
35
|
+
_PULL_TIMEOUT_SECONDS = 900.0
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True, slots=True)
|
|
39
|
+
class OciAnnotationResult:
|
|
40
|
+
"""Validated result produced by one managed container invocation.
|
|
41
|
+
|
|
42
|
+
This is an application-boundary value, not an adapter result. The adapter
|
|
43
|
+
remains unaware of Docker; callers receive the same summary and published
|
|
44
|
+
DuckDB path as the local execution path.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
output: Path
|
|
48
|
+
summary: AnnotationSummary
|
|
49
|
+
adapter: str
|
|
50
|
+
result_schema_id: str
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def run_oci_annotation(
|
|
54
|
+
*,
|
|
55
|
+
fasta: Path,
|
|
56
|
+
output: Path,
|
|
57
|
+
profile: ExecutionProfile,
|
|
58
|
+
store: str | Path | None,
|
|
59
|
+
threads: int,
|
|
60
|
+
timeout_seconds: float | None,
|
|
61
|
+
) -> OciAnnotationResult:
|
|
62
|
+
"""Delegate one v2 annotation through its digest-pinned Docker image.
|
|
63
|
+
|
|
64
|
+
The container is ephemeral and receives only the input FASTA, the
|
|
65
|
+
caller-owned resource, a narrow output staging directory and (for a local
|
|
66
|
+
Store) its Store directory. FASTA and resource binds are read-only; the
|
|
67
|
+
container uses the caller's numeric UID/GID and local Store execution uses
|
|
68
|
+
``--network none``.
|
|
69
|
+
|
|
70
|
+
Notes:
|
|
71
|
+
The managed profile and bundled kit are checked together before Docker
|
|
72
|
+
is launched. The function intentionally rejects credential-bearing
|
|
73
|
+
shared Store URLs until an external secret boundary is available.
|
|
74
|
+
Container cleanup is explicit so failures and cancellations do not
|
|
75
|
+
leave an annotation process behind.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
runtime = profile.runtime
|
|
79
|
+
if profile.version != 2 or runtime is None:
|
|
80
|
+
raise AnnotationError("OCI delegation requires a managed execution profile v2")
|
|
81
|
+
if profile.adapter.value != "dbcan-cazyme":
|
|
82
|
+
raise AnnotationError(
|
|
83
|
+
f"managed OCI adapter is not supported: {profile.adapter.value}"
|
|
84
|
+
)
|
|
85
|
+
if not fasta.is_file():
|
|
86
|
+
raise AnnotationError(f"annotation FASTA is not a readable file: {fasta}")
|
|
87
|
+
if output.exists():
|
|
88
|
+
raise AnnotationError(f"output path already exists: {output}")
|
|
89
|
+
if not output.parent.is_dir():
|
|
90
|
+
raise AnnotationError(
|
|
91
|
+
f"output parent directory does not exist: {output.parent}"
|
|
92
|
+
)
|
|
93
|
+
resource = profile.resource.resolve()
|
|
94
|
+
if not resource.is_dir() or not os.access(resource, os.R_OK):
|
|
95
|
+
raise AnnotationError(f"managed profile resource is not readable: {resource}")
|
|
96
|
+
|
|
97
|
+
manifest = load_kit_manifest("dbcan-cazyme")
|
|
98
|
+
_validate_runtime(runtime, manifest)
|
|
99
|
+
expected_resource_id = _resource_id_from_lock(resource, manifest)
|
|
100
|
+
docker = shutil.which("docker")
|
|
101
|
+
if docker is None:
|
|
102
|
+
raise AnnotationError("Docker executable is not available on PATH")
|
|
103
|
+
_ensure_image(docker, manifest, runtime.image)
|
|
104
|
+
|
|
105
|
+
store_mount, store_argument, network = _resolve_store(store)
|
|
106
|
+
stage = Path(tempfile.mkdtemp(prefix=f".{output.name}.", dir=output.parent))
|
|
107
|
+
staged_output = stage / output.name
|
|
108
|
+
container_name = f"seqevi-annotate-oci-{uuid.uuid4().hex[:16]}"
|
|
109
|
+
created = False
|
|
110
|
+
failure: AnnotationError | None = None
|
|
111
|
+
try:
|
|
112
|
+
command = _inner_command(
|
|
113
|
+
fasta=_CONTAINER_INPUT,
|
|
114
|
+
output=f"{_CONTAINER_OUTPUT}/{output.name}",
|
|
115
|
+
resource=_CONTAINER_RESOURCE,
|
|
116
|
+
store=store_argument,
|
|
117
|
+
threads=threads,
|
|
118
|
+
timeout_seconds=timeout_seconds,
|
|
119
|
+
)
|
|
120
|
+
mounts = (
|
|
121
|
+
_mount(fasta.resolve(), _CONTAINER_INPUT, readonly=True),
|
|
122
|
+
_mount(resource, _CONTAINER_RESOURCE, readonly=True),
|
|
123
|
+
_mount(stage, _CONTAINER_OUTPUT, readonly=False),
|
|
124
|
+
)
|
|
125
|
+
if store_mount is not None:
|
|
126
|
+
mounts += (_mount(store_mount, _CONTAINER_STORE, readonly=False),)
|
|
127
|
+
create_args = (
|
|
128
|
+
"create",
|
|
129
|
+
"--name",
|
|
130
|
+
container_name,
|
|
131
|
+
"--user",
|
|
132
|
+
f"{_uid()}:{_gid()}",
|
|
133
|
+
"--network",
|
|
134
|
+
network,
|
|
135
|
+
"--workdir",
|
|
136
|
+
"/work",
|
|
137
|
+
"--entrypoint",
|
|
138
|
+
_CONTAINER_ENTRYPOINT,
|
|
139
|
+
)
|
|
140
|
+
for mount in mounts:
|
|
141
|
+
create_args += ("--mount", mount)
|
|
142
|
+
created_result = _docker_call(
|
|
143
|
+
docker,
|
|
144
|
+
(*create_args, runtime.image, *command),
|
|
145
|
+
timeout_seconds=_DEFAULT_DOCKER_TIMEOUT_SECONDS,
|
|
146
|
+
action="create the managed annotation container",
|
|
147
|
+
)
|
|
148
|
+
if created_result.returncode != 0:
|
|
149
|
+
raise AnnotationError(
|
|
150
|
+
"Docker could not create the managed annotation container: "
|
|
151
|
+
+ _process_detail(created_result)
|
|
152
|
+
)
|
|
153
|
+
created = True
|
|
154
|
+
started = _docker_call(
|
|
155
|
+
docker,
|
|
156
|
+
("start", "--attach", container_name),
|
|
157
|
+
timeout_seconds=timeout_seconds,
|
|
158
|
+
action="run the managed annotation container",
|
|
159
|
+
)
|
|
160
|
+
if started.returncode != 0:
|
|
161
|
+
raise AnnotationError(
|
|
162
|
+
"managed annotation container failed: " + _process_detail(started)
|
|
163
|
+
)
|
|
164
|
+
payload = _json_result(started.stdout)
|
|
165
|
+
_validate_result(
|
|
166
|
+
staged_output,
|
|
167
|
+
payload,
|
|
168
|
+
profile=profile,
|
|
169
|
+
manifest=manifest,
|
|
170
|
+
resource_id=expected_resource_id,
|
|
171
|
+
)
|
|
172
|
+
_publish_without_overwrite(staged_output, output)
|
|
173
|
+
return OciAnnotationResult(
|
|
174
|
+
output=output,
|
|
175
|
+
summary=_summary_from_payload(payload, output=output),
|
|
176
|
+
adapter=str(payload["adapter"]),
|
|
177
|
+
result_schema_id=str(payload["result_schema"]),
|
|
178
|
+
)
|
|
179
|
+
except AnnotationError:
|
|
180
|
+
raise
|
|
181
|
+
except (OSError, ValueError, KeyError, TypeError, duckdb.Error) as error:
|
|
182
|
+
raise AnnotationError(f"managed OCI annotation failed: {error}") from error
|
|
183
|
+
finally:
|
|
184
|
+
if created:
|
|
185
|
+
removed = _docker_call(
|
|
186
|
+
docker,
|
|
187
|
+
("rm", "--force", container_name),
|
|
188
|
+
timeout_seconds=_DEFAULT_DOCKER_TIMEOUT_SECONDS,
|
|
189
|
+
action="remove the managed annotation container",
|
|
190
|
+
)
|
|
191
|
+
if removed.returncode != 0 and failure is None:
|
|
192
|
+
failure = AnnotationError(
|
|
193
|
+
"Docker could not remove the managed annotation container: "
|
|
194
|
+
+ _process_detail(removed)
|
|
195
|
+
)
|
|
196
|
+
shutil.rmtree(stage, ignore_errors=True)
|
|
197
|
+
if failure is not None:
|
|
198
|
+
raise failure
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _validate_runtime(runtime: ManagedRuntime, manifest: KitManifest) -> None:
|
|
202
|
+
if runtime.kind != "oci" or runtime.engine != "docker":
|
|
203
|
+
raise AnnotationError("managed profile runtime must be OCI Docker")
|
|
204
|
+
if runtime.kit_id != manifest.kit_id:
|
|
205
|
+
raise AnnotationError(
|
|
206
|
+
"managed profile kit_id conflicts with the bundled dbCAN kit"
|
|
207
|
+
)
|
|
208
|
+
if runtime.image != manifest.image:
|
|
209
|
+
raise AnnotationError(
|
|
210
|
+
"managed profile image conflicts with the bundled dbCAN kit"
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _ensure_image(docker: str, manifest: KitManifest, image: str) -> None:
|
|
215
|
+
inspected = _docker_call(
|
|
216
|
+
docker,
|
|
217
|
+
("image", "inspect", image),
|
|
218
|
+
timeout_seconds=_DEFAULT_DOCKER_TIMEOUT_SECONDS,
|
|
219
|
+
action="inspect the managed runtime image",
|
|
220
|
+
)
|
|
221
|
+
if inspected.returncode != 0:
|
|
222
|
+
pulled = _docker_call(
|
|
223
|
+
docker,
|
|
224
|
+
("pull", "--platform", manifest.platform, image),
|
|
225
|
+
timeout_seconds=_PULL_TIMEOUT_SECONDS,
|
|
226
|
+
action="pull the managed runtime image",
|
|
227
|
+
)
|
|
228
|
+
if pulled.returncode != 0:
|
|
229
|
+
raise AnnotationError(
|
|
230
|
+
"Docker could not pull the exact managed runtime image: "
|
|
231
|
+
+ _process_detail(pulled)
|
|
232
|
+
)
|
|
233
|
+
verified = _docker_call(
|
|
234
|
+
docker,
|
|
235
|
+
("image", "inspect", image),
|
|
236
|
+
timeout_seconds=_DEFAULT_DOCKER_TIMEOUT_SECONDS,
|
|
237
|
+
action="verify the managed runtime image",
|
|
238
|
+
)
|
|
239
|
+
if verified.returncode != 0:
|
|
240
|
+
raise AnnotationError(
|
|
241
|
+
"Docker did not retain the exact digest-pinned managed runtime image: "
|
|
242
|
+
+ _process_detail(verified)
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _resolve_store(store: str | Path | None) -> tuple[Path | None, str, str]:
|
|
247
|
+
if store is None:
|
|
248
|
+
raise AnnotationError(
|
|
249
|
+
"a Store path or URL is required via --store or SEQEVI_STORE"
|
|
250
|
+
)
|
|
251
|
+
raw = os.fspath(store)
|
|
252
|
+
if raw.startswith(("http://", "https://")):
|
|
253
|
+
parsed = urlsplit(raw)
|
|
254
|
+
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
|
255
|
+
raise AnnotationError(
|
|
256
|
+
"managed OCI shared Store URLs must not embed credentials or "
|
|
257
|
+
"query secrets; provide an external credential boundary"
|
|
258
|
+
)
|
|
259
|
+
return None, raw, "bridge"
|
|
260
|
+
if "://" in raw:
|
|
261
|
+
raise AnnotationError(f"unsupported Store URL scheme: {raw.split(':', 1)[0]}")
|
|
262
|
+
path = Path(raw).expanduser().resolve()
|
|
263
|
+
try:
|
|
264
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
265
|
+
except OSError as error:
|
|
266
|
+
raise AnnotationError(
|
|
267
|
+
f"cannot prepare local Store mount: {path}: {error}"
|
|
268
|
+
) from error
|
|
269
|
+
return path, _CONTAINER_STORE, "none"
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _inner_command(
|
|
273
|
+
*,
|
|
274
|
+
fasta: str,
|
|
275
|
+
output: str,
|
|
276
|
+
resource: str,
|
|
277
|
+
store: str,
|
|
278
|
+
threads: int,
|
|
279
|
+
timeout_seconds: float | None,
|
|
280
|
+
) -> tuple[str, ...]:
|
|
281
|
+
command = (
|
|
282
|
+
"annotate",
|
|
283
|
+
"--json",
|
|
284
|
+
"--fasta",
|
|
285
|
+
fasta,
|
|
286
|
+
"--output",
|
|
287
|
+
output,
|
|
288
|
+
"--adapter",
|
|
289
|
+
"dbcan-cazyme",
|
|
290
|
+
"--executable",
|
|
291
|
+
_CONTAINER_EXECUTABLE,
|
|
292
|
+
"--resource",
|
|
293
|
+
resource,
|
|
294
|
+
"--store",
|
|
295
|
+
store,
|
|
296
|
+
"--threads",
|
|
297
|
+
str(threads),
|
|
298
|
+
)
|
|
299
|
+
if timeout_seconds is not None:
|
|
300
|
+
command += ("--timeout-seconds", str(timeout_seconds))
|
|
301
|
+
return command
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _resource_id_from_lock(resource: Path, manifest: KitManifest) -> str:
|
|
305
|
+
lock = read_resource_lock(resource)
|
|
306
|
+
if lock is None:
|
|
307
|
+
raise AnnotationError(
|
|
308
|
+
"managed dbCAN resource has no seqevi.lock; rerun setup to verify it"
|
|
309
|
+
)
|
|
310
|
+
expected = {
|
|
311
|
+
component.path: (component.name, component.size, component.sha256)
|
|
312
|
+
for component in manifest.components
|
|
313
|
+
}
|
|
314
|
+
observed = {
|
|
315
|
+
component.relative_path: (component.name, component.size, component.sha256)
|
|
316
|
+
for component in lock.components
|
|
317
|
+
}
|
|
318
|
+
if (lock.resource_name, lock.resource_version) != (
|
|
319
|
+
manifest.resource_name,
|
|
320
|
+
manifest.resource_version,
|
|
321
|
+
) or observed != expected:
|
|
322
|
+
raise AnnotationError("managed dbCAN resource lock conflicts with the kit")
|
|
323
|
+
values = [(component.name, component.sha256) for component in manifest.components]
|
|
324
|
+
digest = sha256_digest(
|
|
325
|
+
json.dumps(values, separators=(",", ":"), ensure_ascii=True).encode()
|
|
326
|
+
)
|
|
327
|
+
return f"dbcan/{manifest.resource_version}/sha256:{digest}"
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _validate_result(
|
|
331
|
+
path: Path,
|
|
332
|
+
payload: Mapping[str, object],
|
|
333
|
+
*,
|
|
334
|
+
profile: ExecutionProfile,
|
|
335
|
+
manifest: KitManifest,
|
|
336
|
+
resource_id: str,
|
|
337
|
+
) -> None:
|
|
338
|
+
if not path.is_file():
|
|
339
|
+
raise AnnotationError("managed container did not publish a DuckDB result")
|
|
340
|
+
if payload.get("status") != "ok":
|
|
341
|
+
raise AnnotationError("managed container returned a non-success result")
|
|
342
|
+
expected = {
|
|
343
|
+
"adapter": profile.adapter.value,
|
|
344
|
+
"result_schema": "dbcan-cazyme/5",
|
|
345
|
+
}
|
|
346
|
+
for key, value in expected.items():
|
|
347
|
+
if payload.get(key) != value:
|
|
348
|
+
raise AnnotationError(f"managed container returned unexpected {key}")
|
|
349
|
+
with duckdb.connect(str(path), read_only=True) as connection:
|
|
350
|
+
row = connection.execute("SELECT * FROM _seqevi.metadata LIMIT 1").fetchone()
|
|
351
|
+
if row is None:
|
|
352
|
+
raise AnnotationError("managed result metadata is empty")
|
|
353
|
+
columns = [item[0] for item in connection.description]
|
|
354
|
+
metadata = dict(zip(columns, row, strict=True))
|
|
355
|
+
checks = {
|
|
356
|
+
"ResultFormatVersion": RESULT_FORMAT_VERSION,
|
|
357
|
+
"SeqEviVersion": __version__,
|
|
358
|
+
"Adapter": profile.adapter.value,
|
|
359
|
+
"AdapterContractVersion": "dbcan-cazyme/1",
|
|
360
|
+
"UpstreamTool": "dbCAN",
|
|
361
|
+
"UpstreamToolVersion": manifest.dbcan_version,
|
|
362
|
+
"ResourceID": resource_id,
|
|
363
|
+
}
|
|
364
|
+
for key, expected_value in checks.items():
|
|
365
|
+
if metadata.get(key) != expected_value:
|
|
366
|
+
raise AnnotationError(
|
|
367
|
+
f"managed result metadata {key} conflicts with the selected kit"
|
|
368
|
+
)
|
|
369
|
+
runtime_digest = metadata.get("ToolRuntimeDigest")
|
|
370
|
+
if not isinstance(runtime_digest, str) or not runtime_digest.startswith("sha256:"):
|
|
371
|
+
raise AnnotationError("managed result is missing ToolRuntimeDigest")
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _summary_from_payload(
|
|
375
|
+
payload: Mapping[str, object], *, output: Path
|
|
376
|
+
) -> AnnotationSummary:
|
|
377
|
+
counts = _mapping(payload.get("counts"), "counts")
|
|
378
|
+
metrics = _mapping(payload.get("metrics"), "metrics")
|
|
379
|
+
return AnnotationSummary(
|
|
380
|
+
input_records=_integer(counts, "input_records"),
|
|
381
|
+
unique_sequences=_integer(counts, "unique_sequences"),
|
|
382
|
+
cache_hits=_integer(counts, "cache_hits"),
|
|
383
|
+
computed=_integer(counts, "computed"),
|
|
384
|
+
hits=_integer(counts, "hits"),
|
|
385
|
+
no_hits=_integer(counts, "no_hits"),
|
|
386
|
+
output_dir=output,
|
|
387
|
+
metrics=AnnotationMetrics(
|
|
388
|
+
elapsed_seconds=_number(metrics, "elapsed_seconds"),
|
|
389
|
+
fasta_staging_seconds=0.0,
|
|
390
|
+
store_lookup_seconds=0.0,
|
|
391
|
+
adapter_seconds=0.0,
|
|
392
|
+
external_tool_seconds=0.0,
|
|
393
|
+
store_commit_seconds=0.0,
|
|
394
|
+
store_fetch_seconds=0.0,
|
|
395
|
+
package_seconds=_number(metrics, "package_seconds"),
|
|
396
|
+
peak_rss_kib=None,
|
|
397
|
+
store_lookup_batches=0,
|
|
398
|
+
store_commit_batches=0,
|
|
399
|
+
store_fetch_batches=0,
|
|
400
|
+
tool_batches=0,
|
|
401
|
+
unique_artifact_reads=0,
|
|
402
|
+
configured_threads=_integer(metrics, "configured_threads"),
|
|
403
|
+
),
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _publish_without_overwrite(staged: Path, output: Path) -> None:
|
|
408
|
+
try:
|
|
409
|
+
os.link(staged, output)
|
|
410
|
+
except FileExistsError as error:
|
|
411
|
+
raise AnnotationError(
|
|
412
|
+
f"output path appeared during managed annotation: {output}"
|
|
413
|
+
) from error
|
|
414
|
+
except OSError as error:
|
|
415
|
+
raise AnnotationError(
|
|
416
|
+
f"cannot publish managed annotation result: {error}"
|
|
417
|
+
) from error
|
|
418
|
+
staged.unlink()
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _mount(source: Path, target: str, *, readonly: bool) -> str:
|
|
422
|
+
suffix = ",readonly" if readonly else ""
|
|
423
|
+
return f"type=bind,source={source},target={target}{suffix}"
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _json_result(stdout: str) -> dict[str, object]:
|
|
427
|
+
lines = [line.strip() for line in stdout.splitlines() if line.strip()]
|
|
428
|
+
if not lines:
|
|
429
|
+
raise AnnotationError("managed container returned no JSON result")
|
|
430
|
+
try:
|
|
431
|
+
value = json.loads(lines[-1])
|
|
432
|
+
except json.JSONDecodeError as error:
|
|
433
|
+
raise AnnotationError("managed container returned invalid JSON") from error
|
|
434
|
+
if not isinstance(value, dict):
|
|
435
|
+
raise AnnotationError("managed container JSON result is not an object")
|
|
436
|
+
return value
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _mapping(value: object, name: str) -> Mapping[str, object]:
|
|
440
|
+
if not isinstance(value, Mapping):
|
|
441
|
+
raise AnnotationError(f"managed container JSON is missing {name}")
|
|
442
|
+
return value
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _integer(mapping: Mapping[str, object], name: str) -> int:
|
|
446
|
+
value = mapping.get(name)
|
|
447
|
+
if isinstance(value, bool) or not isinstance(value, int):
|
|
448
|
+
raise AnnotationError(f"managed container JSON has invalid {name}")
|
|
449
|
+
return value
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _number(mapping: Mapping[str, object], name: str) -> float:
|
|
453
|
+
value = mapping.get(name)
|
|
454
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
455
|
+
raise AnnotationError(f"managed container JSON has invalid {name}")
|
|
456
|
+
return float(value)
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _docker_call(
|
|
460
|
+
docker: str,
|
|
461
|
+
arguments: tuple[str, ...],
|
|
462
|
+
*,
|
|
463
|
+
timeout_seconds: float | None,
|
|
464
|
+
action: str,
|
|
465
|
+
) -> subprocess.CompletedProcess[str]:
|
|
466
|
+
try:
|
|
467
|
+
return subprocess.run(
|
|
468
|
+
[docker, *arguments],
|
|
469
|
+
check=False,
|
|
470
|
+
capture_output=True,
|
|
471
|
+
text=True,
|
|
472
|
+
timeout=timeout_seconds,
|
|
473
|
+
)
|
|
474
|
+
except subprocess.TimeoutExpired as error:
|
|
475
|
+
raise AnnotationError(f"timed out while trying to {action}") from error
|
|
476
|
+
except OSError as error:
|
|
477
|
+
raise AnnotationError(f"could not {action}: {error}") from error
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _process_detail(result: subprocess.CompletedProcess[str]) -> str:
|
|
481
|
+
detail = (result.stderr or result.stdout or "no diagnostic output").strip()
|
|
482
|
+
return detail[-2000:]
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _uid() -> int:
|
|
486
|
+
return getattr(os, "getuid", lambda: 0)()
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _gid() -> int:
|
|
490
|
+
return getattr(os, "getgid", lambda: 0)()
|