orfdata 0.30.0__tar.gz → 0.31.0__tar.gz

{orfdata-0.30.0/orfdata.egg-info → orfdata-0.31.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orfdata
-Version: 0.30.0
+Version: 0.31.0
 Summary: Reference proteomes with evidence-backed noncanonical ORFs on PyEnsembl genomes
 Author: OpenVax
 License-Expression: Apache-2.0

{orfdata-0.30.0 → orfdata-0.31.0}/docs/proteome.md

@@ -43,6 +43,33 @@ Mouse, dog, yeast and other PyEnsembl species work the same way; mouse Ensembl
 annotation data. An uninstalled genome raises an error naming the
 `pyensembl install` command.
 
+## RefSeq proteins (experimental)
+
+Build from NCBI's protein FASTA and genome-download feature table, with pinned
+byte checksums and an explicit `ReferenceContext`:
+
+```python
+from orfdata import ReferenceContext
+
+context = ReferenceContext(9615, "dog", "UU_Cfam_GSD_1.0", "NCBI RefSeq", "106",
+                           assembly_accession="GCF_011100685.1")
+reference = Proteome.from_refseq(
+    "protein.faa.gz", "feature_table.txt.gz", reference=context,
+    expected_sha256={"protein": protein_sha256, "feature_table": feature_table_sha256},
+)
+reference.save("dog.proteome.json.gz")
+```
+
+This preserves NCBI Gene IDs, versioned RNA/protein accessions, every mapped
+occurrence and all FASTA proteins. Wrong checksums, assemblies, missing proteins
+and conflicting assignments fail. CDS rows without protein products are counted
+in `layers[0]["cds_without_protein"]`.
+
+The annotated layer retains both asset hashes and identifier namespaces. Its
+`genomic_spans` maps each protein accession to spans in occurrence order. These
+are genomic envelopes, so occurrence CDS offsets/blocks and transcript biotypes
+remain unknown. No annotation indexing or downloads occur.
+
 ## Choose the evidence
 
 `with_extra_orfs(evidence=...)` selects catalogue ORFs by evidence and adds each

{orfdata-0.30.0 → orfdata-0.31.0}/orfdata/proteome.py

@@ -466,6 +466,19 @@ class Proteome:
                      added=len(entries))
         return cls(reference, entries, (layer,), genome=genome)
 
+    @classmethod
+    def from_refseq(cls, protein_fasta, feature_table, *, reference, expected_sha256):
+        """Read pinned RefSeq proteins and NCBI Gene/transcript occurrences offline.
+
+        ``expected_sha256`` requires ``protein`` and ``feature_table`` byte hashes.
+        The NCBI genome-download feature table supplies accession joins, not
+        spliced CDS coordinates. All FASTA proteins must have source mappings.
+        """
+        from .refseq import refseq_proteome
+
+        return refseq_proteome(cls, protein_fasta, feature_table,
+                              reference=reference, expected_sha256=expected_sha256)
+
     @classmethod
     def build(cls, genome, evidence="ms", *, catalog=None, viral=(), extra_orfs=True, context=None,
               cache=False, cache_root=None, catalogs=None):

orfdata-0.31.0/orfdata/refseq.py

@@ -0,0 +1,123 @@
+"""Pinned NCBI genome-download feature tables and reference proteins."""
+
+import csv
+import gzip
+import hashlib
+import io
+import re
+from collections import defaultdict
+from dataclasses import replace
+from pathlib import Path
+
+from .proteome import Occurrence, ProteomeEntry, _sha256
+from .reference import ReferenceContext
+
+_ACCESSION = re.compile(r"[A-Z]{2}_[0-9]+\.[1-9][0-9]*")
+
+
+def _source(path, expected):
+    if not isinstance(expected, str) or not re.fullmatch(r"[0-9a-f]{64}", expected):
+        raise ValueError("RefSeq sources require lowercase SHA-256 byte checksums")
+    raw = Path(path).read_bytes()
+    if hashlib.sha256(raw).hexdigest() != expected:
+        raise ValueError(f"RefSeq source checksum mismatch: {path}")
+    return (gzip.decompress(raw) if raw.startswith(b"\x1f\x8b") else raw).decode("ascii")
+
+
+def _proteins(text):
+    result, accession, chunks = {}, None, []
+    for line in text.splitlines() + [">"]:
+        if line.startswith(">"):
+            if accession is not None:
+                sequence = "".join(chunks).removesuffix("*")
+                if not sequence or not re.fullmatch(r"[A-Z]+", sequence):
+                    raise ValueError(f"Invalid RefSeq protein sequence: {accession}")
+                if accession in result:
+                    raise ValueError(f"Duplicate RefSeq protein accession: {accession}")
+                result[accession] = sequence
+            accession, chunks = (line[1:].split() or [None])[0], []
+            if accession is not None and not _ACCESSION.fullmatch(accession):
+                raise ValueError("RefSeq proteins require versioned accessions")
+        elif line.strip():
+            if accession is None:
+                raise ValueError("RefSeq FASTA sequence precedes its header")
+            chunks.append(line.strip())
+    if not result:
+        raise ValueError("RefSeq protein inventory is empty")
+    return result
+
+
+def refseq_proteome(cls, protein_fasta, feature_table, *, reference, expected_sha256):
+    if (not isinstance(reference, ReferenceContext) or reference.annotation_name != "NCBI RefSeq"
+            or not reference.annotation_version or not reference.assembly_accession):
+        raise ValueError("RefSeq requires explicit taxon, assembly accession and NCBI RefSeq annotation")
+    if not isinstance(expected_sha256, dict) or set(expected_sha256) != {"protein", "feature_table"}:
+        raise ValueError("RefSeq expected_sha256 requires protein and feature_table")
+    # Read exactly the verified bytes. Subsequent file changes cannot affect this build.
+    proteins = _proteins(_source(protein_fasta, expected_sha256["protein"]))
+    text = _source(feature_table, expected_sha256["feature_table"])
+    reader = csv.DictReader(io.StringIO(text), delimiter="\t")
+    required = {"# feature", "class", "assembly", "product_accession", "related_accession",
+                "GeneID", "symbol", "genomic_accession", "strand", "product_length", "start", "end"}
+    if not reader.fieldnames or not required.issubset(reader.fieldnames):
+        raise ValueError("Missing NCBI genome-download feature-table columns")
+    if len(set(reader.fieldnames)) != len(reader.fieldnames):
+        raise ValueError("Duplicate RefSeq feature-table columns")
+    occurrences, transcripts = defaultdict(dict), {}
+    untranslated = 0
+    for row in reader:
+        if None in row or any(row.get(name) is None for name in required):
+            raise ValueError("Malformed RefSeq feature-table row")
+        if row["assembly"] != reference.assembly_accession:
+            raise ValueError("RefSeq feature-table assembly disagrees with reference")
+        if row["# feature"] != "CDS":
+            continue
+        protein, transcript, gene = row["product_accession"], row["related_accession"], row["GeneID"]
+        if row["class"] == "without_protein" and not protein and not transcript:
+            untranslated += 1
+            continue
+        if (row["class"] != "with_protein" or not _ACCESSION.fullmatch(protein)
+                or not _ACCESSION.fullmatch(transcript) or not re.fullmatch(r"[1-9][0-9]*", gene)
+                or not row["genomic_accession"] or row["strand"] not in {"+", "-"}):
+            raise ValueError("Incomplete RefSeq CDS accession/gene mapping")
+        if protein not in proteins:
+            raise ValueError(f"Mapped RefSeq protein absent from FASTA: {protein}")
+        if row["product_length"] != str(len(proteins[protein])):
+            raise ValueError(f"RefSeq feature-table protein length disagrees with FASTA: {protein}")
+        assignment = (gene, protein)
+        if transcript in transcripts and transcripts[transcript] != assignment:
+            raise ValueError(f"Conflicting RefSeq transcript assignment: {transcript}")
+        transcripts[transcript] = assignment
+        if (not re.fullmatch(r"[1-9][0-9]*", row["start"])
+                or not re.fullmatch(r"[1-9][0-9]*", row["end"])
+                or int(row["start"]) > int(row["end"])):
+            raise ValueError("Invalid RefSeq source genomic span")
+        occurrence = Occurrence(transcript, gene, row["symbol"] or None, None,
+                                row["genomic_accession"], row["strand"], None, None, (), True)
+        key = (transcript, gene, row["genomic_accession"], row["strand"], int(row["start"]), int(row["end"]))
+        if key in occurrences[protein] and occurrences[protein][key] != occurrence:
+            raise ValueError(f"Conflicting RefSeq source occurrence: {transcript}")
+        occurrences[protein][key] = occurrence
+    missing = set(proteins) - set(occurrences)
+    if missing:
+        raise ValueError("RefSeq FASTA proteins lack feature-table mappings: " + ", ".join(sorted(missing)[:5]))
+    source_hash, sequence_hash = _sha256(expected_sha256), _sha256(sorted(proteins.items()))
+    if (reference.source_sha256 not in (None, source_hash)
+            or reference.sequence_sha256 not in (None, sequence_hash)):
+        raise ValueError("RefSeq source/sequence checksums disagree with reference")
+    reference = replace(reference, source_sha256=source_hash, sequence_sha256=sequence_hash)
+    source = f"NCBI RefSeq:{reference.annotation_version}"
+    entries = [ProteomeEntry(accession, proteins[accession], "annotated", reference.taxon_id,
+                source, occurrences=tuple(occurrences[accession][key] for key in sorted(occurrences[accession])),
+                source_binding="feature-table-accession-join; spliced CDS coordinates unavailable")
+               for accession in sorted(proteins)]
+    layer = dict(kind="annotated", role="host", source=source, reference=reference.key,
+                 added=len(entries), content_provenance=dict(expected_sha256),
+                 identifier_namespaces={"gene": "NCBI Gene", "transcript": "NCBI RefSeq RNA",
+                                        "protein": "NCBI RefSeq protein"},
+                 cds_without_protein=untranslated, mapping="feature-table-accession-join")
+    # Same accession can have multiple genomic placements. Keep their spans
+    # indexed exactly like entry.occurrences without inventing spliced CDS blocks.
+    layer['genomic_spans'] = {accession: [list(key[-2:]) for key in sorted(occurrences[accession])]
+                             for accession in sorted(occurrences)}
+    return cls(reference, entries, (layer,))

orfdata-0.31.0/orfdata/version.py

@@ -0,0 +1 @@
+__version__ = "0.31.0"

{orfdata-0.30.0 → orfdata-0.31.0/orfdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orfdata
-Version: 0.30.0
+Version: 0.31.0
 Summary: Reference proteomes with evidence-backed noncanonical ORFs on PyEnsembl genomes
 Author: OpenVax
 License-Expression: Apache-2.0

{orfdata-0.30.0 → orfdata-0.31.0}/orfdata.egg-info/SOURCES.txt

@@ -53,6 +53,7 @@ orfdata/proteome.py
 orfdata/proteome_cli.py
 orfdata/reference.py
 orfdata/reference_mapping.py
+orfdata/refseq.py
 orfdata/rna.py
 orfdata/search_reference.py
 orfdata/sequence.py
@@ -115,6 +116,7 @@ tests/test_proteome.py
 tests/test_reference_context.py
 tests/test_reference_mapping.py
 tests/test_reference_orf.py
+tests/test_refseq.py
 tests/test_region_targets.py
 tests/test_riborf.py
 tests/test_search_reference.py

orfdata-0.31.0/tests/test_refseq.py

@@ -0,0 +1,122 @@
+"""RefSeq source accession joins preserve identities without a symbol heuristic."""
+
+import csv
+import gzip
+import hashlib
+from dataclasses import replace
+
+import pytest
+
+from orfdata import Proteome, ReferenceContext
+
+
+def source(tmp_path, *, rows=None, fasta=None, compressed=False):
+    fields = ['# feature', 'class', 'assembly', 'product_accession', 'related_accession',
+              'GeneID', 'symbol', 'genomic_accession', 'strand', 'product_length', 'start', 'end']
+    if rows is None:
+        rows = [dict(zip(fields, ['CDS', 'with_protein', 'GCF_011100685.1', 'XP_123.1', 'XM_123.1',
+                    '123', 'GENE', 'NC_123.1', '+', '6', '1', '200'])),
+                dict(zip(fields, ['CDS', 'with_protein', 'GCF_011100685.1', 'XP_123.1', 'XM_124.2',
+                    '124', 'OTHER', 'NC_124.1', '-', '6', '2', '201']))]
+    protein, features = tmp_path / 'protein.faa', tmp_path / 'feature_table.txt'
+    protein.write_text(fasta if fasta is not None else '>XP_123.1 source description\nACDEFG\n')
+    with features.open('w') as stream:
+        writer = csv.DictWriter(stream, fieldnames=fields, delimiter='\t')
+        writer.writeheader()
+        writer.writerows(rows)
+    if compressed:
+        for path in (protein, features):
+            path.write_bytes(gzip.compress(path.read_bytes(), mtime=0))
+    hashes = {role: hashlib.sha256(path.read_bytes()).hexdigest()
+              for role, path in (('protein', protein), ('feature_table', features))}
+    context = ReferenceContext(9615, 'dog', 'UU_Cfam_GSD_1.0', 'NCBI RefSeq', '106',
+                               assembly_accession='GCF_011100685.1')
+    return protein, features, context, hashes, rows
+
+
+def build(args):
+    protein, features, reference, hashes, _ = args
+    return Proteome.from_refseq(protein, features, reference=reference, expected_sha256=hashes)
+
+
+@pytest.mark.parametrize('compressed', [False, True])
+def test_refseq_keeps_all_gene_occurrences_and_offline_provenance(tmp_path, compressed):
+    args = source(tmp_path, compressed=compressed)
+    result = build(args)
+    entry, = result.entries
+    assert entry.sequence == 'ACDEFG'
+    assert [o.gene_id for o in entry.occurrences] == ['123', '124']
+    assert [o.transcript_id for o in entry.occurrences] == ['XM_123.1', 'XM_124.2']
+    assert all((o.start, o.end, o.coding_blocks, o.transcript_biotype) == (None, None, (), None)
+               for o in entry.occurrences)
+    assert dict(result.layers[0]['content_provenance']) == args[3]
+    assert result.layers[0]['genomic_spans']['XP_123.1'] == [[1, 200], [2, 201]]
+    saved = tmp_path / 'proteome.json.gz'
+    result.save(saved)
+    args[0].unlink()
+    args[1].unlink()
+    replay = Proteome.load(saved)
+    assert replay.identity == result.identity
+    assert replay.entries == result.entries
+    assert replay.layers == result.layers
+
+
+def test_refseq_preserves_repeated_placements_and_untranslated_cds_count(tmp_path):
+    args = source(tmp_path)
+    rows = args[-1]
+    rows.append(dict(rows[0], start='1001', end='1200'))
+    rows.append(dict(rows[0], **{'class': 'without_protein', 'product_accession': '', 'related_accession': ''}))
+    result = build(source(tmp_path, rows=rows))
+    assert len(result.entries[0].occurrences) == 3
+    assert result.layers[0]['cds_without_protein'] == 1
+    assert result.layers[0]['genomic_spans']['XP_123.1'] == [[1, 200], [1001, 1200], [2, 201]]
+
+
+@pytest.mark.parametrize('change', [dict(assembly='GCF_011100685.2'), dict(GeneID='GENE'),
+    dict(related_accession='XM_123'), dict(product_accession='XP_123'), dict(product_length='7'),
+    dict(strand='?'), dict(start='200', end='1'), dict(genomic_accession='')])
+def test_refseq_rejects_invalid_and_conflicting_source_fields(tmp_path, change):
+    args = source(tmp_path)
+    rows = args[-1]
+    rows[0].update(change)
+    with pytest.raises(ValueError):
+        build(source(tmp_path, rows=rows))
+
+
+@pytest.mark.parametrize('fasta', ['>XP_123.1\nACDEFG\n>XP_123.1\nACDEFG\n',
+    '>XP_123\nACDEFG\n', 'ACDEFG\n', '>XP_123.1\nAC*EFG\n',
+    '>XP_123.1\nACDEFG\n>XP_999.1\nACDEFG\n', ''])
+def test_refseq_rejects_invalid_or_unmapped_fasta_proteins(tmp_path, fasta):
+    with pytest.raises(ValueError):
+        build(source(tmp_path, fasta=fasta))
+
+
+def test_refseq_rejects_checksum_changes_and_missing_sources(tmp_path):
+    args = source(tmp_path)
+    args[0].write_text('changed')
+    with pytest.raises(ValueError, match='checksum mismatch'):
+        build(args)
+    args = source(tmp_path)
+    with pytest.raises(ValueError, match='requires protein and feature_table'):
+        Proteome.from_refseq(args[0], args[1], reference=args[2], expected_sha256={'protein': args[3]['protein']})
+    with pytest.raises(ValueError, match='checksums disagree'):
+        Proteome.from_refseq(args[0], args[1], reference=replace(args[2], source_sha256='0' * 64),
+                            expected_sha256=args[3])
+
+
+def test_refseq_rejects_one_transcript_assigned_to_different_gene(tmp_path):
+    args = source(tmp_path)
+    rows = args[-1]
+    rows[1]['related_accession'] = rows[0]['related_accession']
+    with pytest.raises(ValueError, match='Conflicting RefSeq transcript'):
+        build(source(tmp_path, rows=rows))
+
+
+def test_refseq_rejects_short_or_extra_cells_at_ingest(tmp_path):
+    for row in ('CDS\twith_protein\n', '\t'.join(['CDS'] * 13) + '\n'):
+        args = source(tmp_path)
+        header = args[1].read_text().splitlines()[0]
+        args[1].write_text(header + '\n' + row)
+        args[3]['feature_table'] = hashlib.sha256(args[1].read_bytes()).hexdigest()
+        with pytest.raises(ValueError, match='Malformed RefSeq feature-table row'):
+            build(args)

orfdata-0.30.0/orfdata/version.py

@@ -1 +0,0 @@
-__version__ = "0.30.0"