orfdata 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- orfdata-0.2.0/MANIFEST.in +3 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/PKG-INFO +4 -3
- {orfdata-0.1.0 → orfdata-0.2.0}/README.md +3 -2
- orfdata-0.2.0/deploy.sh +18 -0
- orfdata-0.2.0/docs/design.md +84 -0
- orfdata-0.2.0/docs/reference.md +145 -0
- orfdata-0.2.0/docs/species.md +58 -0
- orfdata-0.2.0/lint.sh +3 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/__init__.py +3 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/catalog.py +17 -5
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/cli.py +8 -1
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/model.py +37 -3
- orfdata-0.2.0/orfdata/reference.py +208 -0
- orfdata-0.2.0/orfdata/version.py +1 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata.egg-info/PKG-INFO +4 -3
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata.egg-info/SOURCES.txt +15 -1
- orfdata-0.2.0/test.sh +3 -0
- orfdata-0.2.0/tests/data/references/README.md +24 -0
- orfdata-0.2.0/tests/data/references/canis_lupus_familiaris.json +148 -0
- orfdata-0.2.0/tests/data/references/homo_sapiens.json +148 -0
- orfdata-0.2.0/tests/data/references/mus_musculus.json +148 -0
- orfdata-0.2.0/tests/data/references/saccharomyces_cerevisiae.json +72 -0
- orfdata-0.2.0/tests/test_reference_context.py +230 -0
- orfdata-0.1.0/orfdata/version.py +0 -1
- {orfdata-0.1.0 → orfdata-0.2.0}/LICENSE +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/build.py +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/data/orf-evidence/AUTHOR-LICENSE.txt +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/data/orf-evidence/NOTICE.txt +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/data/orf-evidence/catalog.json.gz +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/data/orf-evidence/manifest.json +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/data/uorf-evidence/AUTHOR-LICENSE.txt +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/data/uorf-evidence/NOTICE.txt +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/data/uorf-evidence/catalog.json.gz +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/data/uorf-evidence/manifest.json +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/initiation_context.py +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata/sequence.py +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata.egg-info/dependency_links.txt +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata.egg-info/entry_points.txt +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata.egg-info/requires.txt +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/orfdata.egg-info/top_level.txt +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/pyproject.toml +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/setup.cfg +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/tests/test_initiation_context.py +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/tests/test_reference_orf.py +0 -0
- {orfdata-0.1.0 → orfdata-0.2.0}/tests/test_uorf_catalog.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: orfdata
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Reference open reading frames and experimental translation evidence on annotated transcripts
|
|
5
5
|
Author: OpenVax
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -90,8 +90,9 @@ RNA products. Reference evidence stays separate from evidence for a variant or
|
|
|
90
90
|
fusion. See the [API and integration guide](docs/reference.md).
|
|
91
91
|
|
|
92
92
|
The catalogue API supports other species, with explicit species and assembly
|
|
93
|
-
checks. **Bundled evidence is currently human-only
|
|
94
|
-
|
|
93
|
+
checks. **Bundled experimental evidence is currently human-only**. Offline tests
|
|
94
|
+
bind real human, mouse, dog and yeast reference slices; those fixtures are not
|
|
95
|
+
new experimental ORF catalogues. [Species and reference identity](docs/species.md).
|
|
95
96
|
|
|
96
97
|
Install with `pip install orfdata[pyensembl]`. See the
|
|
97
98
|
[quantitative, multispecies data roadmap](https://github.com/openvax/orfdata/issues/3)
|
|
@@ -68,8 +68,9 @@ RNA products. Reference evidence stays separate from evidence for a variant or
|
|
|
68
68
|
fusion. See the [API and integration guide](docs/reference.md).
|
|
69
69
|
|
|
70
70
|
The catalogue API supports other species, with explicit species and assembly
|
|
71
|
-
checks. **Bundled evidence is currently human-only
|
|
72
|
-
|
|
71
|
+
checks. **Bundled experimental evidence is currently human-only**. Offline tests
|
|
72
|
+
bind real human, mouse, dog and yeast reference slices; those fixtures are not
|
|
73
|
+
new experimental ORF catalogues. [Species and reference identity](docs/species.md).
|
|
73
74
|
|
|
74
75
|
Install with `pip install orfdata[pyensembl]`. See the
|
|
75
76
|
[quantitative, multispecies data roadmap](https://github.com/openvax/orfdata/issues/3)
|
orfdata-0.2.0/deploy.sh
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Publish only a reviewed version from clean, synchronized main.
|
|
3
|
+
set -euo pipefail
|
|
4
|
+
PYTHON="${PYTHON:-python}"
|
|
5
|
+
[[ "$(git branch --show-current)" == main ]] || { echo 'Release from main only' >&2; exit 1; }
|
|
6
|
+
[[ -z "$(git status --porcelain)" ]] || { echo 'Release requires a clean tree' >&2; exit 1; }
|
|
7
|
+
git fetch origin main
|
|
8
|
+
[[ "$(git rev-parse HEAD)" == "$(git rev-parse origin/main)" ]] || { echo 'Synchronize main first' >&2; exit 1; }
|
|
9
|
+
version="$($PYTHON -c 'from orfdata.version import __version__; print(__version__)')"
|
|
10
|
+
if git rev-parse "v$version" >/dev/null 2>&1; then echo 'Version tag already exists' >&2; exit 1; fi
|
|
11
|
+
./lint.sh
|
|
12
|
+
./test.sh
|
|
13
|
+
"$PYTHON" -m build
|
|
14
|
+
"$PYTHON" -m twine check "dist/orfdata-$version.tar.gz" "dist/orfdata-$version-py3-none-any.whl"
|
|
15
|
+
if [[ "${1:-}" == --dry-run ]]; then exit 0; fi
|
|
16
|
+
"$PYTHON" -m twine upload "dist/orfdata-$version.tar.gz" "dist/orfdata-$version-py3-none-any.whl"
|
|
17
|
+
git tag "v$version"
|
|
18
|
+
git push origin "v$version"
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# General ORF reference and evidence layer
|
|
2
|
+
|
|
3
|
+
Status: architectural correction to the unmerged oncoref/Varcode/Isovar PRs.
|
|
4
|
+
Working distribution name: orfdata. No release is authorized by this document.
|
|
5
|
+
|
|
6
|
+
## Ownership
|
|
7
|
+
|
|
8
|
+
PyEnsembl remains responsible for reference genomes, annotations, transcripts,
|
|
9
|
+
exons and canonical proteins. A small separate library adds source-defined ORFs
|
|
10
|
+
on those transcripts and their experimental evidence. oncoref remains a consumer
|
|
11
|
+
for cancer-specific analysis. Neither Varcode nor Isovar owns reference datasets.
|
|
12
|
+
PepData's immune-epitope/property scope is not a fit.
|
|
13
|
+
|
|
14
|
+
The new library has a dependency-light data layer and a PyEnsembl-compatible
|
|
15
|
+
binding API. It must not import Varcode, Isovar or oncoref. No new PyEnsembl core
|
|
16
|
+
API is needed to bind a Transcript; avoid reverse dependencies or monkey-patching.
|
|
17
|
+
|
|
18
|
+
## Public objects
|
|
19
|
+
|
|
20
|
+
- ORFModel: immutable source ORF identity, assembly, species (NCBI taxon ID and
|
|
21
|
+
scientific name), translation table, parent transcript, spliced coding blocks,
|
|
22
|
+
protein sequence, ORF class. ORF class is distinct from transcript biotype.
|
|
23
|
+
- ORFEvidence: ribosome observations and peptide-level MS observations with their
|
|
24
|
+
source, accession, mapping ambiguity, assay and quality. Native evidence does
|
|
25
|
+
not become evidence for a modified product; orthology is not direct evidence.
|
|
26
|
+
- ORFRecord: model plus evidence, retaining legacy flat accessors for Isovar.
|
|
27
|
+
- ReferenceORF: validated binding of a record to a PyEnsembl Transcript, retaining
|
|
28
|
+
the original transcript (including noncoding biotype), cDNA coding interval,
|
|
29
|
+
coding sequence and actual resolved annotation/version. Multiple ORFs per RNA
|
|
30
|
+
are normal. Annotation lookup never relabels the transcript protein-coding.
|
|
31
|
+
- ORFCatalog: explicitly one species/assembly per catalogue. Query by transcript,
|
|
32
|
+
gene, locus, class or evidence. for_transcript(transcript) validates and returns
|
|
33
|
+
ReferenceORFs. Load custom versioned catalogues offline. The initial bundled
|
|
34
|
+
human data is identified as human-only; no implied mouse evidence.
|
|
35
|
+
- Varcode ORFAnnotation: reference ORF plus effect and realized products. Exports
|
|
36
|
+
separately identify reference model/evidence and modifications, preserving
|
|
37
|
+
phase/splice alternatives. A private transcript-engine adapter permits reuse
|
|
38
|
+
of existing variant logic without exposing a fake transcript as the base ORF.
|
|
39
|
+
- Isovar: attach those same annotations to RNA products with candidate-specific
|
|
40
|
+
RNA support, maintaining conditional translation and unresolved RNA polarity.
|
|
41
|
+
|
|
42
|
+
## Generality and compatibility
|
|
43
|
+
|
|
44
|
+
Support uORF, overlapping uORF, internal/out-of-frame, downstream and noncoding-
|
|
45
|
+
transcript ORFs with the same model and evidence schema. Preserve source classes
|
|
46
|
+
verbatim rather than equating them with Ensembl transcript biotypes. Complete
|
|
47
|
+
stopped ORFs are the initial binding contract; partial models are rejected
|
|
48
|
+
explicitly rather than given invented boundaries.
|
|
49
|
+
|
|
50
|
+
A generic v1 schema requires species, assembly, translation table and source
|
|
51
|
+
provenance. Legacy Isovar human data may be read only through its explicit known
|
|
52
|
+
human/GRCh38 provenance. Keep legacy Isovar uorf_* imports/call shapes working.
|
|
53
|
+
New generic APIs use orf terminology; defaults remain explicit about dataset scope.
|
|
54
|
+
|
|
55
|
+
Multi-species path: one versioned catalogue per organism/assembly, importer input
|
|
56
|
+
using the same model/evidence schema, optional organism-specific source adapters
|
|
57
|
+
(e.g. OpenProt). Test a mouse noncoding transcript with real PyEnsembl objects,
|
|
58
|
+
plus human coding transcripts, different frames, minus strand, splice junctions,
|
|
59
|
+
assembly/species mismatches and unknown translation tables. Synthetic examples
|
|
60
|
+
are identified as test fixtures, never scientific observations. OpenProt already
|
|
61
|
+
publishes multiple species with Ensembl/RefSeq mappings and evidence filters.
|
|
62
|
+
|
|
63
|
+
## Review corrections
|
|
64
|
+
|
|
65
|
+
Remove the oncoref runtime dependency from both consumers and withdraw its ORF
|
|
66
|
+
PR when the replacement is reviewable. Keep unrelated lint fixes independently.
|
|
67
|
+
Review evidence retention for duplicate reference codons (retain all copies),
|
|
68
|
+
explicit translation tables and noncoding parents; preserve all Sid alternatives.
|
|
69
|
+
Document unsupported de novo initiation discovery and partial ORFs honestly.
|
|
70
|
+
|
|
71
|
+
## Verification
|
|
72
|
+
|
|
73
|
+
New library: existing evidence/context/builder tests, generic data round-trip,
|
|
74
|
+
PyEnsembl binding and multi-species isolation, independently installed wheel.
|
|
75
|
+
Varcode/Isovar: focused tests then required full lint/test scripts. Regenerate
|
|
76
|
+
Sid outputs from the original packaged reads. Existing human evidence counts and
|
|
77
|
+
native peptide attribution must remain unchanged. Do not claim release until CI,
|
|
78
|
+
merge and PyPI publication actually complete; publication remains unapproved.
|
|
79
|
+
|
|
80
|
+
Primary sources:
|
|
81
|
+
- https://www.nature.com/articles/s41587-022-01369-0
|
|
82
|
+
- https://www.nature.com/articles/s41586-026-10459-x
|
|
83
|
+
- https://archive.openprot.org/p/help
|
|
84
|
+
- https://github.com/openvax/pyensembl
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# Using orfdata
|
|
2
|
+
|
|
3
|
+
See the [README](../README.md) for the purpose, ORF types and basic example.
|
|
4
|
+
|
|
5
|
+
## Objects and queries
|
|
6
|
+
|
|
7
|
+
`load_catalog()` loads the bundled reference data offline. Catalogue access uses
|
|
8
|
+
only the Python standard library; the optional `pyensembl` extra supplies
|
|
9
|
+
PyEnsembl. Binding uses a transcript you have already loaded.
|
|
10
|
+
|
|
11
|
+
`catalog.for_transcript(transcript)` returns a `ReferenceORF` for each matching
|
|
12
|
+
ORF. Each has:
|
|
13
|
+
|
|
14
|
+
- `model`: source identity, ORF type, coordinates, species and protein sequence.
|
|
15
|
+
- `evidence`: ribosome observations and peptide-level MS observations.
|
|
16
|
+
- `transcript`: the original PyEnsembl transcript, including its original biotype.
|
|
17
|
+
- `coding_sequence` and `protein_sequence`: checked against that transcript.
|
|
18
|
+
|
|
19
|
+
Binding checks the assembly, available species information, transcript identity,
|
|
20
|
+
full spliced coding path, protein sequence and terminal stop. A matching gene
|
|
21
|
+
name alone is insufficient. Incompatible annotations raise an error.
|
|
22
|
+
|
|
23
|
+
To inspect source records without loading a transcript:
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
from orfdata import load_catalog
|
|
27
|
+
|
|
28
|
+
catalog = load_catalog()
|
|
29
|
+
records = catalog.query(gene="TPST1", evidence="ms")
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Evidence
|
|
33
|
+
|
|
34
|
+
Use `orf.evidence.has_evidence("ribosome")`, `"shotgun"` (protease-digest MS),
|
|
35
|
+
`"mhc"` (including human HLA), or `"ms"` (any supporting MS observation).
|
|
36
|
+
`"hla"` selects human HLA observations specifically. Source observations whose
|
|
37
|
+
MS assay type is unspecified remain labelled `"ms_unclassified"`.
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
orf.evidence.has_evidence("ms", require_unique=True, require_reviewed=True)
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
This requires a supporting spectrum review and a single protein mapping
|
|
44
|
+
**within that source's search scope**. It does not imply uniqueness across every
|
|
45
|
+
possible protein. Inspect `orf.evidence.peptides` for peptide sequences, source
|
|
46
|
+
accessions, matching intervals, mapping counts and review quality. Missing counts
|
|
47
|
+
remain unknown; low or mixed spectrum reviews do not count as supporting MS.
|
|
48
|
+
|
|
49
|
+
The distinction between translation, peptide detection and functional protein
|
|
50
|
+
evidence follows the [TransCODE study](https://www.nature.com/articles/s41586-026-10459-x).
|
|
51
|
+
Source evidence belongs to the exact reference ORF. It is not automatically
|
|
52
|
+
evidence for a different isoform, a mutant protein, a fusion or another species.
|
|
53
|
+
|
|
54
|
+
## Variants and RNA products
|
|
55
|
+
|
|
56
|
+
Varcode keeps the reference object alongside its modification:
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
from pyensembl import EnsemblRelease
|
|
60
|
+
from orfdata import load_catalog
|
|
61
|
+
from varcode import Variant, annotate_orf_variants
|
|
62
|
+
|
|
63
|
+
genome = EnsemblRelease(95, species="human") # already downloaded and indexed
|
|
64
|
+
transcript = genome.transcript_by_id("ENST00000304842")
|
|
65
|
+
orf, = load_catalog().for_transcript(transcript)
|
|
66
|
+
variant = Variant("7", 66205486, "C", "G", genome=genome)
|
|
67
|
+
annotation = annotate_orf_variants([variant], orf)
|
|
68
|
+
assert annotation.reference is orf
|
|
69
|
+
print(annotation.effect) # P2A in the TPST1 upstream ORF
|
|
70
|
+
result = annotation.to_dict() # reference, evidence and altered products
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Isovar's `annotate_sv_orf_evidence` attaches the same reference annotations to
|
|
74
|
+
reconstructed RNA products and preserves each candidate's RNA support. For
|
|
75
|
+
example, the native TPST1 peptide KIILSKISV is absent from the Sid TPST1–CRCP
|
|
76
|
+
alternatives, so its MS evidence does not support those altered products.
|
|
77
|
+
|
|
78
|
+
The reference integration is available in Varcode 10.12.0; the Isovar 1.50.0
|
|
79
|
+
integration is in review. The stricter reference-context contract is coordinated
|
|
80
|
+
with Varcode 10.13.0 and Isovar 1.51.0.
|
|
81
|
+
|
|
82
|
+
## Data and limits
|
|
83
|
+
|
|
84
|
+
The bundled human GRCh38 catalogue, dataset `2026-10-11.1`, draws on the
|
|
85
|
+
[GENCODE Phase I reference](https://www.nature.com/articles/s41587-022-01369-0),
|
|
86
|
+
[TransCODE author exports](https://www.nature.com/articles/s41586-026-10459-x),
|
|
87
|
+
and the [community microprotein spectrum benchmark](https://www.nature.com/articles/s41467-025-68002-x).
|
|
88
|
+
All 7,264 ORF paths and proteins were checked against Ensembl 101 GTF, coding RNA
|
|
89
|
+
and noncoding RNA FASTAs. The 2,186 models with supporting MS observations use
|
|
90
|
+
the packaged source-specific filters; this is not a count of proven functional
|
|
91
|
+
proteins. Packaged manifests retain source accessions, checksums, licenses,
|
|
92
|
+
processing rules and dataset version.
|
|
93
|
+
|
|
94
|
+
The current binder accepts complete ORFs with a terminal stop, NCBI translation
|
|
95
|
+
table 1, and ATG/CTG/TTG initiation codons. All bundled models start at ATG.
|
|
96
|
+
Other genetic codes, initiation conventions and incomplete ORFs need additional
|
|
97
|
+
support. Transcript-boundary contexts can be partial; initiation efficiency and
|
|
98
|
+
effects on the main coding region are not established by these data.
|
|
99
|
+
|
|
100
|
+
Circular-RNA translation, stop-codon readthrough and translation from new
|
|
101
|
+
transcripts or fusions are not covered by this reference catalogue. Those need
|
|
102
|
+
additional source data and, where necessary, model support. ORF discovery is
|
|
103
|
+
outside the current API.
|
|
104
|
+
|
|
105
|
+
`load_uorf_catalog()` preserves Isovar's upstream-only snapshot: 3,771 models,
|
|
106
|
+
dataset `2026-10-08.2`. New code should use the broader `load_catalog()` API.
|
|
107
|
+
|
|
108
|
+
## Other species
|
|
109
|
+
|
|
110
|
+
[ReferenceContext and ScopedTarget](species.md) keep taxa, strains, assemblies
|
|
111
|
+
and annotation versions separate. Custom genomes without species/taxonomy need
|
|
112
|
+
an explicit context when binding; source-pinned contexts require an exact match.
|
|
113
|
+
|
|
114
|
+
`load_catalog(path)` reads a custom versioned JSON catalogue. Each catalogue is
|
|
115
|
+
scoped to one species and assembly; each model records its NCBI taxon, scientific
|
|
116
|
+
name, assembly, translation table and source. Human evidence is never transferred
|
|
117
|
+
to a mouse ortholog. Mouse bindings are tested with synthetic ORFs on PyEnsembl
|
|
118
|
+
transcripts; there is no bundled mouse evidence.
|
|
119
|
+
|
|
120
|
+
To build a catalogue, create `ORFModel` objects from source coordinates and
|
|
121
|
+
sequences, attach source observations with `ORFRecord.from_model(model, evidence)`,
|
|
122
|
+
and construct `ORFCatalog(records, metadata)`. Its `to_dict()` produces the
|
|
123
|
+
`orfdata.catalog.v1` JSON representation. Genomic coding blocks are 1-based,
|
|
124
|
+
inclusive, sorted by genomic position, and include the terminal stop codon.
|
|
125
|
+
|
|
126
|
+
[OpenProt](https://archive.openprot.org/p/help) is a possible source for further species,
|
|
127
|
+
with organism/assembly-specific models, transcript mappings and experimental
|
|
128
|
+
evidence. A future importer must pin releases, preserve ambiguity and licensing,
|
|
129
|
+
and validate coordinate conventions. No OpenProt importer is included yet.
|
|
130
|
+
|
|
131
|
+
## Rebuild and verify
|
|
132
|
+
|
|
133
|
+
From a development checkout with the PyEnsembl dependency installed:
|
|
134
|
+
|
|
135
|
+
```sh
|
|
136
|
+
python -m pip install -e '.[dev]'
|
|
137
|
+
python -m orfdata.build --source-dir /path/to/sources --prepare-sources \
|
|
138
|
+
--all-orfs --output orfdata/data/orf-evidence
|
|
139
|
+
./lint.sh
|
|
140
|
+
./test.sh
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Source preparation downloads pinned public inputs and needs Rscript to extract
|
|
144
|
+
the author export. Ordinary queries need neither R nor spreadsheet libraries.
|
|
145
|
+
See the [design notes](design.md) for the model and library boundaries.
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Species and references
|
|
2
|
+
|
|
3
|
+
ORFdata uses the same objects for human, mouse, dog, yeast and custom species.
|
|
4
|
+
Every ORF states its NCBI species taxon, assembly and genetic code. A
|
|
5
|
+
`ReferenceContext` also pins the annotation provider/version and can retain a
|
|
6
|
+
versioned assembly accession, strain taxon/label and sequence-set SHA-256.
|
|
7
|
+
|
|
8
|
+
```python
|
|
9
|
+
from orfdata import ReferenceContext, ScopedTarget
|
|
10
|
+
|
|
11
|
+
yeast = ReferenceContext(
|
|
12
|
+
taxon_id=4932, species="saccharomyces_cerevisiae",
|
|
13
|
+
genome_build="R64-1-1", assembly_accession="GCA_000146045.2",
|
|
14
|
+
annotation_name="ensembl", annotation_version="115",
|
|
15
|
+
strain="S288C", strain_taxon_id=559292,
|
|
16
|
+
)
|
|
17
|
+
target = ScopedTarget(yeast, "protein", "YFL039C", "ensembl:115")
|
|
18
|
+
print(target.key)
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Target keys include the reference, source namespace, kind and source identifier.
|
|
22
|
+
Identical local IDs in different species, reference versions or datasets remain
|
|
23
|
+
separate. Known aliases such as `canis_familiaris` and
|
|
24
|
+
`canis_lupus_familiaris` resolve to dog taxon 9615 through a versioned mapping;
|
|
25
|
+
the original name stays in exports. Assembly names are never automatically
|
|
26
|
+
equated, and strain identity is never discarded. Unknown species require their
|
|
27
|
+
own explicit taxonomy context; they do not require a schema change.
|
|
28
|
+
|
|
29
|
+
`ReferenceContext.from_genome(genome)` resolves available PyEnsembl metadata
|
|
30
|
+
offline. A custom Genome without species/taxonomy needs an explicit context:
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
bound = record.bind(transcript, context=yeast)
|
|
34
|
+
# Or: catalog.for_transcript(transcript, context=yeast)
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
The caller declares metadata absent from a custom genome; any metadata already
|
|
38
|
+
present must agree. The binder still checks transcript version, strand, complete
|
|
39
|
+
spliced path, coding sequence and protein. An ORF model with a
|
|
40
|
+
`reference_context` requires that exact context. Older catalogues without a
|
|
41
|
+
pinned annotation context can bind an identical transcript in another release;
|
|
42
|
+
the actual resolved context and transcript/coding/protein hashes are exported.
|
|
43
|
+
No binding downloads a reference or performs liftover.
|
|
44
|
+
|
|
45
|
+
`context.validate_genome(genome)` checks available metadata against an explicit
|
|
46
|
+
context. Genomic alleles can use `include_annotation=False`: their coordinates
|
|
47
|
+
are independent of a gene annotation release, while taxonomy, strain and
|
|
48
|
+
assembly checks still apply. This does not relax ORF or RNA-product binding.
|
|
49
|
+
|
|
50
|
+
`ScopedTarget` supports genes, transcripts, ORFs, proteins, ambiguous protein
|
|
51
|
+
groups and peptides. A parent's gene or RNA measurement remains at its original
|
|
52
|
+
resolution; this identity layer does not copy it onto every ORF. Orthology and
|
|
53
|
+
equal peptide sequences do not transfer observations between species.
|
|
54
|
+
|
|
55
|
+
The binder currently supports complete stopped ORFs under NCBI table 1, with
|
|
56
|
+
ATG/CTG/TTG initiation. Unsupported codes and initiation codons fail explicitly.
|
|
57
|
+
In particular, yeast nuclear and mitochondrial codes differ; a contig named
|
|
58
|
+
`MT` never selects a code implicitly. See [NCBI's code definitions](https://www.ncbi.nlm.nih.gov/Taxonomy/Utils/wprintgc.cgi).
|
orfdata-0.2.0/lint.sh
ADDED
|
@@ -14,6 +14,7 @@ from .catalog import (
|
|
|
14
14
|
)
|
|
15
15
|
from .initiation_context import KOZAK_PREFERENCES, InitiationContext, annotate_initiation_context
|
|
16
16
|
from .model import ORFModel, ReferenceORF
|
|
17
|
+
from .reference import ReferenceContext, ScopedTarget
|
|
17
18
|
from .version import __version__
|
|
18
19
|
|
|
19
20
|
__all__ = [
|
|
@@ -25,6 +26,8 @@ __all__ = [
|
|
|
25
26
|
"PeptideEvidence",
|
|
26
27
|
"ORFModel",
|
|
27
28
|
"ReferenceORF",
|
|
29
|
+
"ReferenceContext",
|
|
30
|
+
"ScopedTarget",
|
|
28
31
|
"load_catalog",
|
|
29
32
|
"UORFCatalog",
|
|
30
33
|
"UORFRecord",
|
|
@@ -15,6 +15,7 @@ from typing import Optional
|
|
|
15
15
|
|
|
16
16
|
from .initiation_context import KOZAK_PREFERENCES, InitiationContext
|
|
17
17
|
from .model import ORFModel, ReferenceORF, _species
|
|
18
|
+
from .reference import ReferenceContext
|
|
18
19
|
from .sequence import CODONS, reverse_complement_dna
|
|
19
20
|
|
|
20
21
|
EVIDENCE_METHODS = ("ribosome", "shotgun", "mhc", "hla", "ms_unclassified", "ms")
|
|
@@ -114,6 +115,7 @@ class ORFRecord:
|
|
|
114
115
|
species: Optional[str] = None
|
|
115
116
|
translation_table: int = 1
|
|
116
117
|
source: str = ""
|
|
118
|
+
reference_context: Optional[ReferenceContext] = None
|
|
117
119
|
|
|
118
120
|
@property
|
|
119
121
|
def model(self):
|
|
@@ -133,6 +135,7 @@ class ORFRecord:
|
|
|
133
135
|
species=self.species,
|
|
134
136
|
translation_table=self.translation_table,
|
|
135
137
|
source=self.source,
|
|
138
|
+
reference_context=self.reference_context,
|
|
136
139
|
)
|
|
137
140
|
|
|
138
141
|
@property
|
|
@@ -144,6 +147,8 @@ class ORFRecord:
|
|
|
144
147
|
"""Construct a record from the generic model and independent evidence."""
|
|
145
148
|
evidence = ORFEvidence() if evidence is None else evidence
|
|
146
149
|
fields = model.to_dict()
|
|
150
|
+
if model.reference_context is not None:
|
|
151
|
+
fields["reference_context"] = model.reference_context
|
|
147
152
|
fields["biotype"] = fields.pop("orf_type")
|
|
148
153
|
return cls(
|
|
149
154
|
**fields,
|
|
@@ -152,13 +157,15 @@ class ORFRecord:
|
|
|
152
157
|
initiation_context=initiation_context,
|
|
153
158
|
)
|
|
154
159
|
|
|
155
|
-
def bind(self, transcript):
|
|
160
|
+
def bind(self, transcript, *, context=None):
|
|
156
161
|
"""Validate and bind this ORF to a PyEnsembl transcript without relabelling it."""
|
|
157
|
-
return ReferenceORF(self, transcript)
|
|
162
|
+
return ReferenceORF(self, transcript, context=context)
|
|
158
163
|
|
|
159
164
|
def to_dict(self):
|
|
160
165
|
"""Return a serializable copy of this record."""
|
|
161
166
|
result = asdict(self)
|
|
167
|
+
if self.reference_context is None:
|
|
168
|
+
result.pop("reference_context")
|
|
162
169
|
if self.initiation_context is not None:
|
|
163
170
|
result["initiation_context"] = self.initiation_context.to_dict()
|
|
164
171
|
return result
|
|
@@ -348,7 +355,7 @@ class ORFCatalog:
|
|
|
348
355
|
if r.genome_build != self.genome_build or r.strand not in ("+", "-"):
|
|
349
356
|
raise ValueError(f"Invalid assembly or strand for {r.orf_id}")
|
|
350
357
|
r.model # Validate identity and geometry, including non-upstream ORFs.
|
|
351
|
-
if (r.taxon_id, r.species) != (self.taxon_id, self.species):
|
|
358
|
+
if (r.taxon_id, _species(r.species)) != (self.taxon_id, _species(self.species)):
|
|
352
359
|
raise ValueError("Catalogue and ORF species disagree")
|
|
353
360
|
if (
|
|
354
361
|
not r.coding_blocks
|
|
@@ -425,7 +432,7 @@ class ORFCatalog:
|
|
|
425
432
|
"""Return an exact source ORF ID; unknown IDs raise KeyError."""
|
|
426
433
|
return self._by_id[orf_id]
|
|
427
434
|
|
|
428
|
-
def for_transcript(self, transcript, **filters):
|
|
435
|
+
def for_transcript(self, transcript, *, context=None, **filters):
|
|
429
436
|
"""Return validated ORFs on this transcript, with optional evidence filters.
|
|
430
437
|
|
|
431
438
|
A source model that disagrees with the selected annotation raises rather
|
|
@@ -436,8 +443,11 @@ class ORFCatalog:
|
|
|
436
443
|
species = getattr(transcript.genome, "species", None)
|
|
437
444
|
if species is not None and _species(species) != _species(self.species):
|
|
438
445
|
raise ValueError("Catalogue transcript species mismatch")
|
|
446
|
+
resolved = ReferenceContext.from_genome(transcript.genome, context=context)
|
|
447
|
+
if resolved.taxon_id != self.taxon_id:
|
|
448
|
+
raise ValueError("Catalogue transcript taxon mismatch")
|
|
439
449
|
return tuple(
|
|
440
|
-
record.bind(transcript)
|
|
450
|
+
record.bind(transcript, context=resolved)
|
|
441
451
|
for record in self.query(**filters)
|
|
442
452
|
if record.transcript_id.split(".")[0] == str(transcript.id).split(".")[0]
|
|
443
453
|
)
|
|
@@ -576,6 +586,8 @@ def _load_catalog(path=None, *, expected_sha256=None, resource="orf-evidence"):
|
|
|
576
586
|
)
|
|
577
587
|
row["coding_blocks"] = tuple(tuple(x) for x in row["coding_blocks"])
|
|
578
588
|
row["ribosome_studies"] = tuple(row["ribosome_studies"])
|
|
589
|
+
if row.get("reference_context") is not None:
|
|
590
|
+
row["reference_context"] = ReferenceContext(**row["reference_context"])
|
|
579
591
|
if row.get("initiation_context") is not None:
|
|
580
592
|
context = dict(row["initiation_context"])
|
|
581
593
|
assessment = context.pop("assessment", None)
|
|
@@ -120,8 +120,10 @@ def run(argv=None, prog=None, *, catalog_loader=load_catalog):
|
|
|
120
120
|
sys.stdout.write("\n")
|
|
121
121
|
elif args.format == "fasta":
|
|
122
122
|
for r in records:
|
|
123
|
+
reference = "" if r.reference_context is None else " reference=" + r.reference_context.key
|
|
124
|
+
scope = "" if isinstance(catalog, UORFCatalog) else f" taxon={r.taxon_id}{reference}"
|
|
123
125
|
sys.stdout.write(
|
|
124
|
-
f">{r.orf_id} gene={r.gene_name} transcript={r.transcript_id} assembly={r.genome_build} biotype={r.biotype}\n{r.protein_sequence}\n"
|
|
126
|
+
f">{r.orf_id} gene={r.gene_name} transcript={r.transcript_id} assembly={r.genome_build} biotype={r.biotype}{scope}\n{r.protein_sequence}\n"
|
|
125
127
|
)
|
|
126
128
|
else:
|
|
127
129
|
fields = (
|
|
@@ -143,12 +145,17 @@ def run(argv=None, prog=None, *, catalog_loader=load_catalog):
|
|
|
143
145
|
"peptides_json",
|
|
144
146
|
"initiation_context_json",
|
|
145
147
|
)
|
|
148
|
+
scoped = not isinstance(catalog, UORFCatalog)
|
|
149
|
+
if scoped:
|
|
150
|
+
fields += ("reference_context_json",)
|
|
146
151
|
writer = csv.DictWriter(sys.stdout, fieldnames=fields, delimiter="\t", lineterminator="\n")
|
|
147
152
|
writer.writeheader()
|
|
148
153
|
for r in records:
|
|
149
154
|
row = r.to_dict()
|
|
150
155
|
for field in ("coding_blocks", "ribosome_studies", "peptides", "initiation_context"):
|
|
151
156
|
row[field + "_json"] = json.dumps(row.pop(field), separators=(",", ":"))
|
|
157
|
+
if scoped:
|
|
158
|
+
row["reference_context_json"] = json.dumps(row.pop("reference_context", None), separators=(",", ":"))
|
|
152
159
|
writer.writerow(row)
|
|
153
160
|
|
|
154
161
|
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
"""Reference ORF identity and its binding to an annotated transcript."""
|
|
2
2
|
|
|
3
|
+
import hashlib
|
|
3
4
|
from dataclasses import asdict, dataclass
|
|
5
|
+
from typing import Optional
|
|
4
6
|
|
|
7
|
+
from .reference import ReferenceContext, canonical_species, species_taxon
|
|
5
8
|
from .sequence import CODONS
|
|
6
9
|
|
|
7
10
|
|
|
@@ -29,10 +32,21 @@ class ORFModel:
|
|
|
29
32
|
species: str
|
|
30
33
|
translation_table: int = 1
|
|
31
34
|
source: str = ""
|
|
35
|
+
reference_context: Optional[ReferenceContext] = None
|
|
32
36
|
|
|
33
37
|
def __post_init__(self):
|
|
34
38
|
if type(self.taxon_id) is not int or self.taxon_id < 1 or not self.species:
|
|
35
39
|
raise ValueError("An explicit species and positive NCBI taxon_id are required")
|
|
40
|
+
known = species_taxon(self.species)
|
|
41
|
+
if known is not None and known != self.taxon_id:
|
|
42
|
+
raise ValueError("ORF species name and taxon_id disagree")
|
|
43
|
+
if self.reference_context is not None:
|
|
44
|
+
if not isinstance(self.reference_context, ReferenceContext):
|
|
45
|
+
raise TypeError("reference_context must be a ReferenceContext")
|
|
46
|
+
if (self.reference_context.taxon_id != self.taxon_id
|
|
47
|
+
or self.reference_context.genome_build != self.genome_build
|
|
48
|
+
or canonical_species(self.reference_context.species) != canonical_species(self.species)):
|
|
49
|
+
raise ValueError("ORF and reference context disagree")
|
|
36
50
|
if type(self.translation_table) is not int or self.translation_table < 1:
|
|
37
51
|
raise ValueError("An explicit NCBI translation_table is required")
|
|
38
52
|
if not all((self.orf_id, self.transcript_id, self.genome_build, self.orf_type)):
|
|
@@ -58,7 +72,10 @@ class ORFModel:
|
|
|
58
72
|
return positions[::-1] if self.strand == "-" else positions
|
|
59
73
|
|
|
60
74
|
def to_dict(self):
|
|
61
|
-
|
|
75
|
+
result = asdict(self)
|
|
76
|
+
if self.reference_context is None:
|
|
77
|
+
result.pop("reference_context")
|
|
78
|
+
return result
|
|
62
79
|
|
|
63
80
|
|
|
64
81
|
def transcript_positions(transcript):
|
|
@@ -78,7 +95,7 @@ def _contig(value):
|
|
|
78
95
|
|
|
79
96
|
|
|
80
97
|
def _species(value):
|
|
81
|
-
return
|
|
98
|
+
return canonical_species(value)
|
|
82
99
|
|
|
83
100
|
|
|
84
101
|
@dataclass(frozen=True, init=False)
|
|
@@ -96,8 +113,9 @@ class ReferenceORF:
|
|
|
96
113
|
coding_sequence: str
|
|
97
114
|
transcript_sequence: str
|
|
98
115
|
genomic_positions: tuple
|
|
116
|
+
context: ReferenceContext
|
|
99
117
|
|
|
100
|
-
def __init__(self, record, transcript):
|
|
118
|
+
def __init__(self, record, transcript, *, context=None):
|
|
101
119
|
model = record.model
|
|
102
120
|
genome = transcript.genome
|
|
103
121
|
if genome.reference_name != model.genome_build:
|
|
@@ -108,6 +126,11 @@ class ReferenceORF:
|
|
|
108
126
|
taxon = getattr(genome, "taxon_id", None)
|
|
109
127
|
if taxon is not None and taxon != model.taxon_id:
|
|
110
128
|
raise ValueError("ORF transcript taxon mismatch")
|
|
129
|
+
resolved_context = ReferenceContext.from_genome(genome, context=context)
|
|
130
|
+
if resolved_context.taxon_id != model.taxon_id:
|
|
131
|
+
raise ValueError("ORF transcript taxon mismatch")
|
|
132
|
+
if model.reference_context is not None and model.reference_context.key != resolved_context.key:
|
|
133
|
+
raise ValueError("ORF transcript reference context mismatch (annotation, strain or assembly)")
|
|
111
134
|
if str(transcript.id).split(".")[0] != model.transcript_id.split(".")[0]:
|
|
112
135
|
raise ValueError("ORF transcript identity mismatch")
|
|
113
136
|
# If the source explicitly pins a transcript version, respect it.
|
|
@@ -154,6 +177,7 @@ class ReferenceORF:
|
|
|
154
177
|
coding_sequence=coding,
|
|
155
178
|
transcript_sequence=sequence,
|
|
156
179
|
genomic_positions=positions,
|
|
180
|
+
context=resolved_context,
|
|
157
181
|
).items():
|
|
158
182
|
object.__setattr__(self, name, value)
|
|
159
183
|
|
|
@@ -175,6 +199,16 @@ class ReferenceORF:
|
|
|
175
199
|
return dict(
|
|
176
200
|
model=self.model.to_dict(),
|
|
177
201
|
evidence=self.evidence.to_dict(),
|
|
202
|
+
reference_context=self.context.to_dict(),
|
|
203
|
+
reference_key=self.context.key,
|
|
204
|
+
sequence_fingerprints={
|
|
205
|
+
name: hashlib.sha256(sequence.encode("ascii")).hexdigest()
|
|
206
|
+
for name, sequence in (
|
|
207
|
+
("transcript_sha256", self.transcript_sequence),
|
|
208
|
+
("coding_sha256", self.coding_sequence),
|
|
209
|
+
("protein_sha256", self.protein_sequence),
|
|
210
|
+
)
|
|
211
|
+
},
|
|
178
212
|
resolved_transcript=dict(
|
|
179
213
|
id=transcript.id,
|
|
180
214
|
version=getattr(transcript, "version", None),
|