seqevi 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seqevi-0.2.0/LICENSE +21 -0
- seqevi-0.2.0/PKG-INFO +333 -0
- seqevi-0.2.0/README.md +299 -0
- seqevi-0.2.0/pyproject.toml +132 -0
- seqevi-0.2.0/src/seqevi/__init__.py +7 -0
- seqevi-0.2.0/src/seqevi/__main__.py +6 -0
- seqevi-0.2.0/src/seqevi/adapters/__init__.py +41 -0
- seqevi-0.2.0/src/seqevi/adapters/base.py +131 -0
- seqevi-0.2.0/src/seqevi/adapters/dbcan_cazyme.py +588 -0
- seqevi-0.2.0/src/seqevi/adapters/eggnog.py +674 -0
- seqevi-0.2.0/src/seqevi/adapters/interpro_pfam.py +757 -0
- seqevi-0.2.0/src/seqevi/adapters/registry.py +68 -0
- seqevi-0.2.0/src/seqevi/annotate.py +413 -0
- seqevi-0.2.0/src/seqevi/api.py +390 -0
- seqevi-0.2.0/src/seqevi/cli.py +610 -0
- seqevi-0.2.0/src/seqevi/distribution/__init__.py +13 -0
- seqevi-0.2.0/src/seqevi/distribution/manifest.py +199 -0
- seqevi-0.2.0/src/seqevi/distribution/oci.py +490 -0
- seqevi-0.2.0/src/seqevi/distribution/setup.py +752 -0
- seqevi-0.2.0/src/seqevi/errors.py +73 -0
- seqevi-0.2.0/src/seqevi/evidence.py +295 -0
- seqevi-0.2.0/src/seqevi/execution_profile.py +526 -0
- seqevi-0.2.0/src/seqevi/hashing.py +13 -0
- seqevi-0.2.0/src/seqevi/kits/__init__.py +1 -0
- seqevi-0.2.0/src/seqevi/kits/dbcan-cazyme.toml +35 -0
- seqevi-0.2.0/src/seqevi/resource_lock.py +438 -0
- seqevi-0.2.0/src/seqevi/result.py +682 -0
- seqevi-0.2.0/src/seqevi/runner.py +163 -0
- seqevi-0.2.0/src/seqevi/runtime_identity.py +104 -0
- seqevi-0.2.0/src/seqevi/sequence.py +383 -0
- seqevi-0.2.0/src/seqevi/service/__init__.py +11 -0
- seqevi-0.2.0/src/seqevi/service/app.py +213 -0
- seqevi-0.2.0/src/seqevi/service/config.py +38 -0
- seqevi-0.2.0/src/seqevi/service/persistence.py +360 -0
- seqevi-0.2.0/src/seqevi/store/__init__.py +14 -0
- seqevi-0.2.0/src/seqevi/store/artifact.py +225 -0
- seqevi-0.2.0/src/seqevi/store/client.py +311 -0
- seqevi-0.2.0/src/seqevi/store/contract.py +33 -0
- seqevi-0.2.0/src/seqevi/store/factory.py +38 -0
- seqevi-0.2.0/src/seqevi/store/local.py +479 -0
- seqevi-0.2.0/src/seqevi/store/migration.py +62 -0
- seqevi-0.2.0/src/seqevi/store/migrations/__init__.py +1 -0
- seqevi-0.2.0/src/seqevi/store/migrations/env.py +30 -0
- seqevi-0.2.0/src/seqevi/store/migrations/versions/0001_initial_store.py +103 -0
- seqevi-0.2.0/src/seqevi/store/migrations/versions/0002_artifact_byte_size_bigint.py +40 -0
- seqevi-0.2.0/src/seqevi/store/migrations/versions/__init__.py +1 -0
- seqevi-0.2.0/src/seqevi/store/schema.py +86 -0
- seqevi-0.2.0/src/seqevi/store/transport.py +224 -0
- seqevi-0.2.0/tests/__init__.py +0 -0
- seqevi-0.2.0/tests/support.py +241 -0
- seqevi-0.2.0/tests/test_annotate.py +393 -0
- seqevi-0.2.0/tests/test_api.py +91 -0
- seqevi-0.2.0/tests/test_artifact_store.py +68 -0
- seqevi-0.2.0/tests/test_cli.py +360 -0
- seqevi-0.2.0/tests/test_dbcan_cazyme.py +361 -0
- seqevi-0.2.0/tests/test_dbcan_image_assets.py +129 -0
- seqevi-0.2.0/tests/test_deployment_assets.py +288 -0
- seqevi-0.2.0/tests/test_eggnog.py +420 -0
- seqevi-0.2.0/tests/test_evidence.py +93 -0
- seqevi-0.2.0/tests/test_execution_profile.py +185 -0
- seqevi-0.2.0/tests/test_incremental_reuse.py +120 -0
- seqevi-0.2.0/tests/test_interpro_pfam.py +519 -0
- seqevi-0.2.0/tests/test_local_store.py +292 -0
- seqevi-0.2.0/tests/test_managed_setup.py +414 -0
- seqevi-0.2.0/tests/test_oci.py +260 -0
- seqevi-0.2.0/tests/test_resource_lock.py +172 -0
- seqevi-0.2.0/tests/test_result_prototype.py +305 -0
- seqevi-0.2.0/tests/test_runner.py +81 -0
- seqevi-0.2.0/tests/test_runtime_identity.py +64 -0
- seqevi-0.2.0/tests/test_sequence.py +159 -0
- seqevi-0.2.0/tests/test_shared_store.py +649 -0
seqevi-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Fuqing Zhang
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
seqevi-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: seqevi
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Content-addressed, reusable protein sequence annotation evidence.
|
|
5
|
+
Author-Email: Fuqing Zhang <fu.qing.zhang.work@gmail.com>, FuqingZhang <103730099+FuqingZh@users.noreply.github.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
8
|
+
Classifier: Environment :: Console
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
16
|
+
Project-URL: Homepage, https://github.com/FuqingZh/seqevi
|
|
17
|
+
Project-URL: Repository, https://github.com/FuqingZh/seqevi
|
|
18
|
+
Project-URL: Issues, https://github.com/FuqingZh/seqevi/issues
|
|
19
|
+
Requires-Python: >=3.12
|
|
20
|
+
Requires-Dist: alembic<2,>=1.18
|
|
21
|
+
Requires-Dist: biopython<2,>=1.87
|
|
22
|
+
Requires-Dist: duckdb<2,>=1.5
|
|
23
|
+
Requires-Dist: httpx<1,>=0.28
|
|
24
|
+
Requires-Dist: polars[pyarrow]<2,>=1.40
|
|
25
|
+
Requires-Dist: pydantic<3,>=2.12
|
|
26
|
+
Requires-Dist: sqlalchemy<3,>=2.0
|
|
27
|
+
Requires-Dist: typer<1,>=0.27
|
|
28
|
+
Provides-Extra: server
|
|
29
|
+
Requires-Dist: fastapi<1,>=0.116; extra == "server"
|
|
30
|
+
Requires-Dist: psycopg[binary]<4,>=3.2; extra == "server"
|
|
31
|
+
Requires-Dist: pydantic-settings<3,>=2.10; extra == "server"
|
|
32
|
+
Requires-Dist: uvicorn<1,>=0.35; extra == "server"
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
# SeqEvi
|
|
36
|
+
|
|
37
|
+
**SeqEvi: Sequence Evidence** is a content-addressed cache for reusable protein
|
|
38
|
+
sequence annotation evidence.
|
|
39
|
+
|
|
40
|
+
SeqEvi identifies proteins by canonical sequence content, determines which
|
|
41
|
+
sequences already have evidence under an exact annotation contract, runs an
|
|
42
|
+
external annotation tool only for cache misses, and exports an adapter-specific
|
|
43
|
+
single-file DuckDB result for the current FASTA.
|
|
44
|
+
|
|
45
|
+
## Status
|
|
46
|
+
|
|
47
|
+
The target architecture and v1.1 result contracts are approved. The SeqEvi
|
|
48
|
+
0.2.0 source tree uses strict protein sequence identity, the single-host
|
|
49
|
+
SQLite/POSIX Store, the external tool runner, exact cache-miss orchestration,
|
|
50
|
+
and atomic DuckDB result materialization.
|
|
51
|
+
|
|
52
|
+
SeqEvi 0.2.0 provides managed setup for dbCAN only. The `eggnog` and
|
|
53
|
+
`interpro-pfam` adapters remain supported through explicit runtimes and named
|
|
54
|
+
host profiles; managed setup for them is later feature work. The
|
|
55
|
+
[Slice D gate record](docs/benchmarks/20260806-v1.4-dbcan-public-release-gate.md)
|
|
56
|
+
preserves the incomplete original public-user run and its subsequent acceptance
|
|
57
|
+
decision. In particular, a repeat pull from that run's site is a deferred
|
|
58
|
+
transport check rather than a 0.2.0 release blocker.
|
|
59
|
+
|
|
60
|
+
The `interpro-pfam` and eggNOG-mapper 2.x `eggnog` adapters are implemented with
|
|
61
|
+
native-output validation and fixture parity coverage. The eggNOG adapter passes
|
|
62
|
+
direct parity against eggNOG-mapper 2.1.13 and eggNOG DB 5.0.2. The InterPro
|
|
63
|
+
adapter passes direct parity against InterProScan 5.77-108.0 with InterPro data
|
|
64
|
+
108.0 and Pfam 38.1. The Phase 5 shared Store service, HTTP client, streamed
|
|
65
|
+
POSIX artifacts, and PostgreSQL persistence are implemented and covered by
|
|
66
|
+
local/shared plus provisioned PostgreSQL integration tests. Phase 6 resource
|
|
67
|
+
locks avoid repeated hashing of large immutable database files and provide an
|
|
68
|
+
explicit full-content verification command. Annotation now uses atomic FASTA
|
|
69
|
+
staging, file-backed artifacts, bounded Store batches, adapter-native Parquet
|
|
70
|
+
artifacts, and an operational thread setting.
|
|
71
|
+
|
|
72
|
+
## Why SeqEvi
|
|
73
|
+
|
|
74
|
+
Two FASTA files do not need to be identical to reuse annotation. If a new FASTA
|
|
75
|
+
contains sequences seen in earlier projects, SeqEvi reuses the immutable
|
|
76
|
+
evidence for those sequences and annotates only novel content.
|
|
77
|
+
|
|
78
|
+
```text
|
|
79
|
+
FASTA A: 2000 new sequences -> annotate 2000
|
|
80
|
+
FASTA B: 1000 sequences from A -> annotate 0
|
|
81
|
+
FASTA C: 1000 from A + 500 novel -> annotate 500
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Reuse is exact. Tool runtime, annotation resource, semantic parameters, or
|
|
85
|
+
adapter contract changes produce a different evidence key and never silently
|
|
86
|
+
fall back to an older result.
|
|
87
|
+
|
|
88
|
+
## Intended CLI
|
|
89
|
+
|
|
90
|
+
For repeated use, keep one machine-local TOML per adapter runtime under
|
|
91
|
+
`${XDG_CONFIG_HOME:-~/.config}/seqevi/profiles/`:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
seqevi profile init eggnog-5.0.2 --adapter eggnog
|
|
95
|
+
seqevi profile init interpro-pfam-38.1 --adapter interpro-pfam
|
|
96
|
+
seqevi profile init dbcan-5.2.9 --adapter dbcan-cazyme
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Each command creates a complete adapter-specific TOML file and refuses to
|
|
100
|
+
replace an existing profile. After editing the machine-local paths, inspect and
|
|
101
|
+
validate profiles without launching either annotation runtime:
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
seqevi profile list
|
|
105
|
+
seqevi profile show eggnog-5.0.2
|
|
106
|
+
seqevi profile validate \
|
|
107
|
+
--config "${XDG_CONFIG_HOME:-$HOME/.config}/seqevi/profiles/eggnog-5.0.2.toml"
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
`profile show` resolves paths and operational defaults but reports only
|
|
111
|
+
environment variable names, never their values. The original complete
|
|
112
|
+
templates remain available through `profile example --adapter ADAPTER`.
|
|
113
|
+
|
|
114
|
+
These profile commands configure SeqEvi; they do not install annotation
|
|
115
|
+
software or databases. Managed setup is available only for dbCAN and uses a
|
|
116
|
+
runtime image published by SeqEvi. It supports a read-only preview and an
|
|
117
|
+
explicit apply:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
seqevi setup dbcan-cazyme \
|
|
121
|
+
--resource /data/dbcan/db_v5-2-9_5-5-2026/raw \
|
|
122
|
+
--dry-run
|
|
123
|
+
|
|
124
|
+
seqevi setup dbcan-cazyme \
|
|
125
|
+
--resource /data/dbcan/db_v5-2-9_5-5-2026/raw \
|
|
126
|
+
--dry-run --json
|
|
127
|
+
|
|
128
|
+
seqevi setup dbcan-cazyme \
|
|
129
|
+
--resource /data/dbcan/db_v5-2-9_5-5-2026/raw \
|
|
130
|
+
--yes
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
`--dry-run` never mutates state. `--yes` pulls the immutable image only when
|
|
134
|
+
needed, verifies the caller-owned four-file resource, creates `seqevi.lock`
|
|
135
|
+
when the resource permits it, runs an ephemeral read-only smoke, and publishes
|
|
136
|
+
the v2 profile atomically. It never downloads or copies the database. Slice C
|
|
137
|
+
now dispatches a managed dbCAN annotation through an ephemeral Docker
|
|
138
|
+
container with the same caller UID/GID, read-only FASTA/resource mounts and a
|
|
139
|
+
local-Store `--network none` boundary:
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
seqevi annotate \
|
|
143
|
+
--profile dbcan-cazyme \
|
|
144
|
+
--store /data/seqevi-store \
|
|
145
|
+
--fasta proteins.fasta \
|
|
146
|
+
--output results/dbcan.duckdb
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
The dispatcher and cleanup boundary are covered by fixture tests. Real
|
|
150
|
+
direct-candidate versus managed-v2 scientific equality and later-process replay
|
|
151
|
+
passed the release gate. A validation harness used an immutable local image ID
|
|
152
|
+
built from the exact published inputs when site GHCR transport is unavailable;
|
|
153
|
+
the public setup/profile surface remains pinned to the bundled GHCR digest and
|
|
154
|
+
exposes no image override.
|
|
155
|
+
|
|
156
|
+
The real local/shared Store acceptance for eggNOG and InterPro/Pfam is recorded
|
|
157
|
+
in the [result-consumption runtime report](docs/benchmarks/20260805-v1.0-result-consumption-runtime-acceptance.md).
|
|
158
|
+
The managed dbCAN distribution gate is tracked in the
|
|
159
|
+
[runtime image release review](docs/architecture/20260805-v1.1-dbcan-runtime-image-release-review.md).
|
|
160
|
+
|
|
161
|
+
Run repeated annotations by name:
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
seqevi annotate \
|
|
165
|
+
--profile eggnog-5.0.2 \
|
|
166
|
+
--fasta proteins.fasta \
|
|
167
|
+
--output results/eggnog.duckdb
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
seqevi annotate \
|
|
172
|
+
--profile interpro-pfam-38.1 \
|
|
173
|
+
--fasta proteins.fasta \
|
|
174
|
+
--store https://seqevi.example.org \
|
|
175
|
+
--output results/pfam.duckdb
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
```bash
|
|
179
|
+
seqevi annotate \
|
|
180
|
+
--profile dbcan-5.2.9 \
|
|
181
|
+
--fasta proteins.fasta \
|
|
182
|
+
--output results/dbcan.duckdb
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
An exact profile file can be selected with `--config PATH`. Complete explicit
|
|
186
|
+
mode remains available:
|
|
187
|
+
|
|
188
|
+
```bash
|
|
189
|
+
seqevi annotate \
|
|
190
|
+
--adapter eggnog \
|
|
191
|
+
--fasta proteins.fasta \
|
|
192
|
+
--store /data/seqevi-store \
|
|
193
|
+
--output results/eggnog.duckdb \
|
|
194
|
+
--executable /opt/eggnog-mapper/emapper.py \
|
|
195
|
+
--resource /data/eggnog-5.0.2 \
|
|
196
|
+
--threads 8
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
seqevi annotate \
|
|
201
|
+
--adapter interpro-pfam \
|
|
202
|
+
--fasta proteins.fasta \
|
|
203
|
+
--store https://seqevi.example.org \
|
|
204
|
+
--output results/pfam.duckdb \
|
|
205
|
+
--executable /opt/interproscan/interproscan.sh \
|
|
206
|
+
--resource /data/interproscan-5.77-108.0/data
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
Shared deployments expose the same Store contract:
|
|
210
|
+
|
|
211
|
+
```bash
|
|
212
|
+
seqevi serve \
|
|
213
|
+
--database-url postgresql+psycopg://seqevi@postgres/seqevi \
|
|
214
|
+
--artifacts-dir /data/seqevi-artifacts
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
The supported user-systemd deployment through the host rootful Docker daemon is
|
|
218
|
+
documented in the
|
|
219
|
+
[service runbook](docs/operations/20260727-v0.1.0-loopback-service-runbook.md).
|
|
220
|
+
The service image contains SeqEvi and its server dependencies only; annotation
|
|
221
|
+
executables and databases remain external.
|
|
222
|
+
|
|
223
|
+
Initialize or audit a database resource lock independently of annotation:
|
|
224
|
+
|
|
225
|
+
```bash
|
|
226
|
+
seqevi resource verify \
|
|
227
|
+
--adapter eggnog \
|
|
228
|
+
--executable /opt/eggnog-mapper/emapper.py \
|
|
229
|
+
--resource /data/eggnog-5.0.2
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
## V1 Scope
|
|
233
|
+
|
|
234
|
+
- Protein FASTA input with strict, deterministic canonicalization.
|
|
235
|
+
- GA4GH `SQ.` sequence identifiers plus MD5 compatibility aliases.
|
|
236
|
+
- Exact, immutable evidence keys.
|
|
237
|
+
- Explicit `eggnog`, `interpro-pfam`, and official-runtime-validated
|
|
238
|
+
`dbcan-cazyme` adapters. dbCAN direct/local/shared scientific acceptance is
|
|
239
|
+
complete; publishing the managed runtime image remains separate work, and
|
|
240
|
+
annotation databases remain caller supplied.
|
|
241
|
+
- Local SQLite/POSIX Store and shared PostgreSQL/POSIX Store service.
|
|
242
|
+
- One self-describing DuckDB result per invocation; adapter-native normalized
|
|
243
|
+
evidence remains Parquet inside the incremental Store.
|
|
244
|
+
|
|
245
|
+
SeqEvi does not infer species, manage projects, schedule workflows, install
|
|
246
|
+
third-party tools, distribute annotation databases, or merge unrelated adapter
|
|
247
|
+
schemas.
|
|
248
|
+
|
|
249
|
+
## Documentation
|
|
250
|
+
|
|
251
|
+
Start with [the documentation index](docs/README.md).
|
|
252
|
+
|
|
253
|
+
- [Architecture overview](docs/architecture/20260720-v1.0-seqevi-architecture.md)
|
|
254
|
+
- [Sequence and evidence contract](docs/architecture/20260804-v1.1-sequence-evidence-contract.md)
|
|
255
|
+
- [Adapter contract](docs/architecture/20260804-v1.1-adapter-contract.md)
|
|
256
|
+
- [Result consumption contract](docs/architecture/20260804-v1.1-result-consumption-contract.md)
|
|
257
|
+
- [Execution profile contract](docs/architecture/20260724-v1.0-execution-profile-contract.md)
|
|
258
|
+
- [Storage and deployment architecture](docs/architecture/20260729-v1.1-storage-deployment-architecture.md)
|
|
259
|
+
- [MVP implementation plan](docs/implementation-plan/20260720-v1.0-mvp-implementation-plan.md)
|
|
260
|
+
- [Execution profile implementation plan](docs/implementation-plan/20260724-v1.0-execution-profile-implementation-plan.md)
|
|
261
|
+
- [Validation strategy](docs/testing/20260720-v1.0-validation-strategy.md)
|
|
262
|
+
- [Annotate runtime and bounded-memory plan](docs/implementation-plan/20260722-v1.0-annotate-bounded-memory-plan.md)
|
|
263
|
+
- [Bounded-memory and operational performance](docs/benchmarks/20260722-v1.0-bounded-memory-performance.md)
|
|
264
|
+
- [InterProScan Pfam runtime validation](docs/benchmarks/20260723-v1.0-interproscan-runtime-validation.md)
|
|
265
|
+
- [dbCAN CAZyme adapter implementation plan](docs/implementation-plan/20260804-v1.0-dbcan-cazyme-adapter-implementation-plan.md)
|
|
266
|
+
- [DuckDB result-consumption runtime acceptance](docs/benchmarks/20260805-v1.0-result-consumption-runtime-acceptance.md)
|
|
267
|
+
- [dbCAN runtime image release review](docs/architecture/20260805-v1.1-dbcan-runtime-image-release-review.md)
|
|
268
|
+
|
|
269
|
+
Accepted managed-boundary documents; Slice B setup and smoke plus Slice C OCI
|
|
270
|
+
execution and real candidate acceptance are implemented, while v1 profiles
|
|
271
|
+
remain compatible:
|
|
272
|
+
|
|
273
|
+
- [Managed adapter onboarding roadmap v1.1](docs/implementation-plan/20260805-v1.1-managed-adapter-onboarding-implementation-plan.md)
|
|
274
|
+
- [Managed-distribution architecture v1.2](docs/architecture/20260806-v1.2-managed-adapter-distribution-architecture.md)
|
|
275
|
+
- [Execution profile v2.2 contract](docs/architecture/20260806-v2.2-execution-profile-contract.md)
|
|
276
|
+
|
|
277
|
+
## Python And Result Discovery
|
|
278
|
+
|
|
279
|
+
The public Python API returns DuckDB's native relation, so the same object can
|
|
280
|
+
be queried from a notebook or passed to Arrow/Polars without a SeqEvi wrapper:
|
|
281
|
+
|
|
282
|
+
```python
|
|
283
|
+
import seqevi
|
|
284
|
+
|
|
285
|
+
annotations = seqevi.annotate(
|
|
286
|
+
"proteins.faa",
|
|
287
|
+
profile="interpro-pfam-38.1",
|
|
288
|
+
output="results/pfam.duckdb",
|
|
289
|
+
)
|
|
290
|
+
print(annotations.columns)
|
|
291
|
+
pfam = annotations.select("InputID", "SignatureAccession")
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
An existing result can be opened read-only with `seqevi.scan_annotations()`. If
|
|
295
|
+
the adapter columns are not known in advance, inspect the native relation or
|
|
296
|
+
the stable catalog first:
|
|
297
|
+
|
|
298
|
+
```python
|
|
299
|
+
annotations = seqevi.scan_annotations("results/pfam.duckdb")
|
|
300
|
+
print(annotations.columns)
|
|
301
|
+
print(annotations.pl(lazy=True).collect_schema())
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
The normal protein-level join key is `InputID`. `SequenceID` is the content
|
|
305
|
+
identity used for exact Store reuse. InterPro/Pfam keeps one-to-many domain
|
|
306
|
+
rows, so aggregate it before joining to a one-row-per-protein table when that
|
|
307
|
+
is the desired grain. SQL, R, and workflow tasks can open the same file and
|
|
308
|
+
query `main.annotations`; `_seqevi.column_info`, `_seqevi.table_info`, and
|
|
309
|
+
`_seqevi.metadata` provide column descriptions, row grain, and provenance.
|
|
310
|
+
|
|
311
|
+
SeqEvi 0.2.0 is a deliberate output cutover from the 0.1.0 directory Data
|
|
312
|
+
Package. Existing 0.1.0 packages remain readable by their own Data Package
|
|
313
|
+
tools, but new SeqEvi invocations publish DuckDB only; rerun an annotation to
|
|
314
|
+
produce the new result file.
|
|
315
|
+
|
|
316
|
+
## External Tools
|
|
317
|
+
|
|
318
|
+
Annotation runtimes and databases are supplied by the user. The current CLI has
|
|
319
|
+
no `seqevi setup` command. SeqEvi v1 targets
|
|
320
|
+
[eggNOG-mapper](https://github.com/eggnogdb/eggnog-mapper) and
|
|
321
|
+
[InterProScan](https://www.ebi.ac.uk/interpro/interproscan.html) with the Pfam
|
|
322
|
+
application, and [dbCAN](https://github.com/bcb-unl/run_dbcan) for protein-level
|
|
323
|
+
CAZyme annotation. A future managed path may publish a SeqEvi-maintained runtime
|
|
324
|
+
image built from locked upstream inputs after runtime compliance review; it would
|
|
325
|
+
not be an upstream-official image. Annotation databases remain separate and
|
|
326
|
+
are never bundled in the wheel or runtime image. The proposed managed path
|
|
327
|
+
uses a public, digest-pinned
|
|
328
|
+
`ghcr.io/fuqingzh/seqevi-dbcan` runtime package; callers continue to provide the
|
|
329
|
+
database path, and internal registry mirrors remain deployment policy.
|
|
330
|
+
|
|
331
|
+
## License
|
|
332
|
+
|
|
333
|
+
SeqEvi is distributed under the [MIT License](LICENSE).
|
seqevi-0.2.0/README.md
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
# SeqEvi
|
|
2
|
+
|
|
3
|
+
**SeqEvi: Sequence Evidence** is a content-addressed cache for reusable protein
|
|
4
|
+
sequence annotation evidence.
|
|
5
|
+
|
|
6
|
+
SeqEvi identifies proteins by canonical sequence content, determines which
|
|
7
|
+
sequences already have evidence under an exact annotation contract, runs an
|
|
8
|
+
external annotation tool only for cache misses, and exports an adapter-specific
|
|
9
|
+
single-file DuckDB result for the current FASTA.
|
|
10
|
+
|
|
11
|
+
## Status
|
|
12
|
+
|
|
13
|
+
The target architecture and v1.1 result contracts are approved. The SeqEvi
|
|
14
|
+
0.2.0 source tree uses strict protein sequence identity, the single-host
|
|
15
|
+
SQLite/POSIX Store, the external tool runner, exact cache-miss orchestration,
|
|
16
|
+
and atomic DuckDB result materialization.
|
|
17
|
+
|
|
18
|
+
SeqEvi 0.2.0 provides managed setup for dbCAN only. The `eggnog` and
|
|
19
|
+
`interpro-pfam` adapters remain supported through explicit runtimes and named
|
|
20
|
+
host profiles; managed setup for them is later feature work. The
|
|
21
|
+
[Slice D gate record](docs/benchmarks/20260806-v1.4-dbcan-public-release-gate.md)
|
|
22
|
+
preserves the incomplete original public-user run and its subsequent acceptance
|
|
23
|
+
decision. In particular, a repeat pull from that run's site is a deferred
|
|
24
|
+
transport check rather than a 0.2.0 release blocker.
|
|
25
|
+
|
|
26
|
+
The `interpro-pfam` and eggNOG-mapper 2.x `eggnog` adapters are implemented with
|
|
27
|
+
native-output validation and fixture parity coverage. The eggNOG adapter passes
|
|
28
|
+
direct parity against eggNOG-mapper 2.1.13 and eggNOG DB 5.0.2. The InterPro
|
|
29
|
+
adapter passes direct parity against InterProScan 5.77-108.0 with InterPro data
|
|
30
|
+
108.0 and Pfam 38.1. The Phase 5 shared Store service, HTTP client, streamed
|
|
31
|
+
POSIX artifacts, and PostgreSQL persistence are implemented and covered by
|
|
32
|
+
local/shared plus provisioned PostgreSQL integration tests. Phase 6 resource
|
|
33
|
+
locks avoid repeated hashing of large immutable database files and provide an
|
|
34
|
+
explicit full-content verification command. Annotation now uses atomic FASTA
|
|
35
|
+
staging, file-backed artifacts, bounded Store batches, adapter-native Parquet
|
|
36
|
+
artifacts, and an operational thread setting.
|
|
37
|
+
|
|
38
|
+
## Why SeqEvi
|
|
39
|
+
|
|
40
|
+
Two FASTA files do not need to be identical to reuse annotation. If a new FASTA
|
|
41
|
+
contains sequences seen in earlier projects, SeqEvi reuses the immutable
|
|
42
|
+
evidence for those sequences and annotates only novel content.
|
|
43
|
+
|
|
44
|
+
```text
|
|
45
|
+
FASTA A: 2000 new sequences -> annotate 2000
|
|
46
|
+
FASTA B: 1000 sequences from A -> annotate 0
|
|
47
|
+
FASTA C: 1000 from A + 500 novel -> annotate 500
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Reuse is exact. Tool runtime, annotation resource, semantic parameters, or
|
|
51
|
+
adapter contract changes produce a different evidence key and never silently
|
|
52
|
+
fall back to an older result.
|
|
53
|
+
|
|
54
|
+
## Intended CLI
|
|
55
|
+
|
|
56
|
+
For repeated use, keep one machine-local TOML per adapter runtime under
|
|
57
|
+
`${XDG_CONFIG_HOME:-~/.config}/seqevi/profiles/`:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
seqevi profile init eggnog-5.0.2 --adapter eggnog
|
|
61
|
+
seqevi profile init interpro-pfam-38.1 --adapter interpro-pfam
|
|
62
|
+
seqevi profile init dbcan-5.2.9 --adapter dbcan-cazyme
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Each command creates a complete adapter-specific TOML file and refuses to
|
|
66
|
+
replace an existing profile. After editing the machine-local paths, inspect and
|
|
67
|
+
validate profiles without launching either annotation runtime:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
seqevi profile list
|
|
71
|
+
seqevi profile show eggnog-5.0.2
|
|
72
|
+
seqevi profile validate \
|
|
73
|
+
--config "${XDG_CONFIG_HOME:-$HOME/.config}/seqevi/profiles/eggnog-5.0.2.toml"
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
`profile show` resolves paths and operational defaults but reports only
|
|
77
|
+
environment variable names, never their values. The original complete
|
|
78
|
+
templates remain available through `profile example --adapter ADAPTER`.
|
|
79
|
+
|
|
80
|
+
These profile commands configure SeqEvi; they do not install annotation
|
|
81
|
+
software or databases. Managed setup is available only for dbCAN and uses a
|
|
82
|
+
runtime image published by SeqEvi. It supports a read-only preview and an
|
|
83
|
+
explicit apply:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
seqevi setup dbcan-cazyme \
|
|
87
|
+
--resource /data/dbcan/db_v5-2-9_5-5-2026/raw \
|
|
88
|
+
--dry-run
|
|
89
|
+
|
|
90
|
+
seqevi setup dbcan-cazyme \
|
|
91
|
+
--resource /data/dbcan/db_v5-2-9_5-5-2026/raw \
|
|
92
|
+
--dry-run --json
|
|
93
|
+
|
|
94
|
+
seqevi setup dbcan-cazyme \
|
|
95
|
+
--resource /data/dbcan/db_v5-2-9_5-5-2026/raw \
|
|
96
|
+
--yes
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
`--dry-run` never mutates state. `--yes` pulls the immutable image only when
|
|
100
|
+
needed, verifies the caller-owned four-file resource, creates `seqevi.lock`
|
|
101
|
+
when the resource permits it, runs an ephemeral read-only smoke, and publishes
|
|
102
|
+
the v2 profile atomically. It never downloads or copies the database. Slice C
|
|
103
|
+
now dispatches a managed dbCAN annotation through an ephemeral Docker
|
|
104
|
+
container with the same caller UID/GID, read-only FASTA/resource mounts and a
|
|
105
|
+
local-Store `--network none` boundary:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
seqevi annotate \
|
|
109
|
+
--profile dbcan-cazyme \
|
|
110
|
+
--store /data/seqevi-store \
|
|
111
|
+
--fasta proteins.fasta \
|
|
112
|
+
--output results/dbcan.duckdb
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
The dispatcher and cleanup boundary are covered by fixture tests. Real
|
|
116
|
+
direct-candidate versus managed-v2 scientific equality and later-process replay
|
|
117
|
+
passed the release gate. A validation harness used an immutable local image ID
|
|
118
|
+
built from the exact published inputs when site GHCR transport is unavailable;
|
|
119
|
+
the public setup/profile surface remains pinned to the bundled GHCR digest and
|
|
120
|
+
exposes no image override.
|
|
121
|
+
|
|
122
|
+
The real local/shared Store acceptance for eggNOG and InterPro/Pfam is recorded
|
|
123
|
+
in the [result-consumption runtime report](docs/benchmarks/20260805-v1.0-result-consumption-runtime-acceptance.md).
|
|
124
|
+
The managed dbCAN distribution gate is tracked in the
|
|
125
|
+
[runtime image release review](docs/architecture/20260805-v1.1-dbcan-runtime-image-release-review.md).
|
|
126
|
+
|
|
127
|
+
Run repeated annotations by name:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
seqevi annotate \
|
|
131
|
+
--profile eggnog-5.0.2 \
|
|
132
|
+
--fasta proteins.fasta \
|
|
133
|
+
--output results/eggnog.duckdb
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
seqevi annotate \
|
|
138
|
+
--profile interpro-pfam-38.1 \
|
|
139
|
+
--fasta proteins.fasta \
|
|
140
|
+
--store https://seqevi.example.org \
|
|
141
|
+
--output results/pfam.duckdb
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
seqevi annotate \
|
|
146
|
+
--profile dbcan-5.2.9 \
|
|
147
|
+
--fasta proteins.fasta \
|
|
148
|
+
--output results/dbcan.duckdb
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
An exact profile file can be selected with `--config PATH`. Complete explicit
|
|
152
|
+
mode remains available:
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
seqevi annotate \
|
|
156
|
+
--adapter eggnog \
|
|
157
|
+
--fasta proteins.fasta \
|
|
158
|
+
--store /data/seqevi-store \
|
|
159
|
+
--output results/eggnog.duckdb \
|
|
160
|
+
--executable /opt/eggnog-mapper/emapper.py \
|
|
161
|
+
--resource /data/eggnog-5.0.2 \
|
|
162
|
+
--threads 8
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
seqevi annotate \
|
|
167
|
+
--adapter interpro-pfam \
|
|
168
|
+
--fasta proteins.fasta \
|
|
169
|
+
--store https://seqevi.example.org \
|
|
170
|
+
--output results/pfam.duckdb \
|
|
171
|
+
--executable /opt/interproscan/interproscan.sh \
|
|
172
|
+
--resource /data/interproscan-5.77-108.0/data
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Shared deployments expose the same Store contract:
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
seqevi serve \
|
|
179
|
+
--database-url postgresql+psycopg://seqevi@postgres/seqevi \
|
|
180
|
+
--artifacts-dir /data/seqevi-artifacts
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
The supported user-systemd deployment through the host rootful Docker daemon is
|
|
184
|
+
documented in the
|
|
185
|
+
[service runbook](docs/operations/20260727-v0.1.0-loopback-service-runbook.md).
|
|
186
|
+
The service image contains SeqEvi and its server dependencies only; annotation
|
|
187
|
+
executables and databases remain external.
|
|
188
|
+
|
|
189
|
+
Initialize or audit a database resource lock independently of annotation:
|
|
190
|
+
|
|
191
|
+
```bash
|
|
192
|
+
seqevi resource verify \
|
|
193
|
+
--adapter eggnog \
|
|
194
|
+
--executable /opt/eggnog-mapper/emapper.py \
|
|
195
|
+
--resource /data/eggnog-5.0.2
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
## V1 Scope
|
|
199
|
+
|
|
200
|
+
- Protein FASTA input with strict, deterministic canonicalization.
|
|
201
|
+
- GA4GH `SQ.` sequence identifiers plus MD5 compatibility aliases.
|
|
202
|
+
- Exact, immutable evidence keys.
|
|
203
|
+
- Explicit `eggnog`, `interpro-pfam`, and official-runtime-validated
|
|
204
|
+
`dbcan-cazyme` adapters. dbCAN direct/local/shared scientific acceptance is
|
|
205
|
+
complete; publishing the managed runtime image remains separate work, and
|
|
206
|
+
annotation databases remain caller supplied.
|
|
207
|
+
- Local SQLite/POSIX Store and shared PostgreSQL/POSIX Store service.
|
|
208
|
+
- One self-describing DuckDB result per invocation; adapter-native normalized
|
|
209
|
+
evidence remains Parquet inside the incremental Store.
|
|
210
|
+
|
|
211
|
+
SeqEvi does not infer species, manage projects, schedule workflows, install
|
|
212
|
+
third-party tools, distribute annotation databases, or merge unrelated adapter
|
|
213
|
+
schemas.
|
|
214
|
+
|
|
215
|
+
## Documentation
|
|
216
|
+
|
|
217
|
+
Start with [the documentation index](docs/README.md).
|
|
218
|
+
|
|
219
|
+
- [Architecture overview](docs/architecture/20260720-v1.0-seqevi-architecture.md)
|
|
220
|
+
- [Sequence and evidence contract](docs/architecture/20260804-v1.1-sequence-evidence-contract.md)
|
|
221
|
+
- [Adapter contract](docs/architecture/20260804-v1.1-adapter-contract.md)
|
|
222
|
+
- [Result consumption contract](docs/architecture/20260804-v1.1-result-consumption-contract.md)
|
|
223
|
+
- [Execution profile contract](docs/architecture/20260724-v1.0-execution-profile-contract.md)
|
|
224
|
+
- [Storage and deployment architecture](docs/architecture/20260729-v1.1-storage-deployment-architecture.md)
|
|
225
|
+
- [MVP implementation plan](docs/implementation-plan/20260720-v1.0-mvp-implementation-plan.md)
|
|
226
|
+
- [Execution profile implementation plan](docs/implementation-plan/20260724-v1.0-execution-profile-implementation-plan.md)
|
|
227
|
+
- [Validation strategy](docs/testing/20260720-v1.0-validation-strategy.md)
|
|
228
|
+
- [Annotate runtime and bounded-memory plan](docs/implementation-plan/20260722-v1.0-annotate-bounded-memory-plan.md)
|
|
229
|
+
- [Bounded-memory and operational performance](docs/benchmarks/20260722-v1.0-bounded-memory-performance.md)
|
|
230
|
+
- [InterProScan Pfam runtime validation](docs/benchmarks/20260723-v1.0-interproscan-runtime-validation.md)
|
|
231
|
+
- [dbCAN CAZyme adapter implementation plan](docs/implementation-plan/20260804-v1.0-dbcan-cazyme-adapter-implementation-plan.md)
|
|
232
|
+
- [DuckDB result-consumption runtime acceptance](docs/benchmarks/20260805-v1.0-result-consumption-runtime-acceptance.md)
|
|
233
|
+
- [dbCAN runtime image release review](docs/architecture/20260805-v1.1-dbcan-runtime-image-release-review.md)
|
|
234
|
+
|
|
235
|
+
Accepted managed-boundary documents; Slice B setup and smoke plus Slice C OCI
|
|
236
|
+
execution and real candidate acceptance are implemented, while v1 profiles
|
|
237
|
+
remain compatible:
|
|
238
|
+
|
|
239
|
+
- [Managed adapter onboarding roadmap v1.1](docs/implementation-plan/20260805-v1.1-managed-adapter-onboarding-implementation-plan.md)
|
|
240
|
+
- [Managed-distribution architecture v1.2](docs/architecture/20260806-v1.2-managed-adapter-distribution-architecture.md)
|
|
241
|
+
- [Execution profile v2.2 contract](docs/architecture/20260806-v2.2-execution-profile-contract.md)
|
|
242
|
+
|
|
243
|
+
## Python And Result Discovery
|
|
244
|
+
|
|
245
|
+
The public Python API returns DuckDB's native relation, so the same object can
|
|
246
|
+
be queried from a notebook or passed to Arrow/Polars without a SeqEvi wrapper:
|
|
247
|
+
|
|
248
|
+
```python
|
|
249
|
+
import seqevi
|
|
250
|
+
|
|
251
|
+
annotations = seqevi.annotate(
|
|
252
|
+
"proteins.faa",
|
|
253
|
+
profile="interpro-pfam-38.1",
|
|
254
|
+
output="results/pfam.duckdb",
|
|
255
|
+
)
|
|
256
|
+
print(annotations.columns)
|
|
257
|
+
pfam = annotations.select("InputID", "SignatureAccession")
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
An existing result can be opened read-only with `seqevi.scan_annotations()`. If
|
|
261
|
+
the adapter columns are not known in advance, inspect the native relation or
|
|
262
|
+
the stable catalog first:
|
|
263
|
+
|
|
264
|
+
```python
|
|
265
|
+
annotations = seqevi.scan_annotations("results/pfam.duckdb")
|
|
266
|
+
print(annotations.columns)
|
|
267
|
+
print(annotations.pl(lazy=True).collect_schema())
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
The normal protein-level join key is `InputID`. `SequenceID` is the content
|
|
271
|
+
identity used for exact Store reuse. InterPro/Pfam keeps one-to-many domain
|
|
272
|
+
rows, so aggregate it before joining to a one-row-per-protein table when that
|
|
273
|
+
is the desired grain. SQL, R, and workflow tasks can open the same file and
|
|
274
|
+
query `main.annotations`; `_seqevi.column_info`, `_seqevi.table_info`, and
|
|
275
|
+
`_seqevi.metadata` provide column descriptions, row grain, and provenance.
|
|
276
|
+
|
|
277
|
+
SeqEvi 0.2.0 is a deliberate output cutover from the 0.1.0 directory Data
|
|
278
|
+
Package. Existing 0.1.0 packages remain readable by their own Data Package
|
|
279
|
+
tools, but new SeqEvi invocations publish DuckDB only; rerun an annotation to
|
|
280
|
+
produce the new result file.
|
|
281
|
+
|
|
282
|
+
## External Tools
|
|
283
|
+
|
|
284
|
+
Annotation runtimes and databases are supplied by the user. The current CLI has
|
|
285
|
+
no `seqevi setup` command. SeqEvi v1 targets
|
|
286
|
+
[eggNOG-mapper](https://github.com/eggnogdb/eggnog-mapper) and
|
|
287
|
+
[InterProScan](https://www.ebi.ac.uk/interpro/interproscan.html) with the Pfam
|
|
288
|
+
application, and [dbCAN](https://github.com/bcb-unl/run_dbcan) for protein-level
|
|
289
|
+
CAZyme annotation. A future managed path may publish a SeqEvi-maintained runtime
|
|
290
|
+
image built from locked upstream inputs after runtime compliance review; it would
|
|
291
|
+
not be an upstream-official image. Annotation databases remain separate and
|
|
292
|
+
are never bundled in the wheel or runtime image. The proposed managed path
|
|
293
|
+
uses a public, digest-pinned
|
|
294
|
+
`ghcr.io/fuqingzh/seqevi-dbcan` runtime package; callers continue to provide the
|
|
295
|
+
database path, and internal registry mirrors remain deployment policy.
|
|
296
|
+
|
|
297
|
+
## License
|
|
298
|
+
|
|
299
|
+
SeqEvi is distributed under the [MIT License](LICENSE).
|