orfdata 0.37.0__tar.gz → 0.38.2__tar.gz

{orfdata-0.37.0/orfdata.egg-info → orfdata-0.38.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orfdata
-Version: 0.37.0
+Version: 0.38.2
 Summary: Reference proteomes with evidence-backed noncanonical ORFs on PyEnsembl genomes
 Author: OpenVax
 License-Expression: Apache-2.0

{orfdata-0.37.0 → orfdata-0.38.2}/docs/proteome.md

@@ -281,6 +281,15 @@ other.
   `source:id`. `catalog.get` and `ExtraORFs.for_orf` accept any of them.
 - **ID collisions.** When two sources use one ID for different ORFs, both
   become `source:id`, and the bare ID raises rather than picking one.
+  Direct `ORFCatalog(records, metadata)` construction follows the same rule,
+  retaining each native record in `members` without merging its evidence.
+  IDs must still be unique within a source. If a source or native ID contains
+  punctuation that would collide with a generated ID, generated components
+  are percent-escaped; native IDs remain available through the source records.
+  Use `catalog.get("native-id", source="study")` or
+  `orfs.for_orf("native-id", source="study")` for explicit source selection.
+  A combined string such as `study:native-id` raises if it also identifies
+  another native record; explicit source selection resolves that ambiguity.
 - **Evidence.** Each source's evidence is kept, and evidence rules see the union:
   a peptide counts once per sequence. A Ribo-seq study two catalogues label
   differently counts once when their `ribosome_study_pmids` metadata give it

{orfdata-0.37.0 → orfdata-0.38.2}/docs/species.md

@@ -71,3 +71,12 @@ must declare table 2, and a yeast one table 3. See
 products, such as Varcode and Isovar, must translate them with that code.
 For example, OpenProt's IP_306399 on human MT-ND4 begins ATA and ends AGG. It
 binds under table 2; table 1 would read those codons as isoleucine and arginine.
+
+OpenProt import manifests can declare `processing.translation_tables` by exact
+native transcript ID, for example `{"ENST00000361381.2": 2}` for the qualified
+human IP_306399 row. The declaration belongs to the pinned source/reference
+manifest and is retained in catalogue provenance. Mitochondrial rows require
+such a declaration; unsupported codes and IDs absent from the input fail.
+The importer retains the native coordinate conversion and aggregate MS/TE
+counts. Binding still checks the exact transcript sequence and protein.
+This does not publish a complete human or yeast OpenProt catalogue (#18).

{orfdata-0.37.0 → orfdata-0.38.2}/orfdata/catalog.py

@@ -15,6 +15,7 @@ from dataclasses import asdict, dataclass, replace
 from importlib.resources import files
 from pathlib import Path
 from typing import NamedTuple, Optional
+from urllib.parse import quote
 
 from .evidence import EVIDENCE_METHODS as EVIDENCE_METHODS  # Re-exported for existing imports.
 from .evidence import MS_METHODS, evidence_rule, stored_method
@@ -402,25 +403,52 @@ class ORFRecord:
         }
 
 
+def _qualify_colliding_ids(records):
+    """Give colliding source records unique catalogue IDs, retaining their native records."""
+    records = tuple(records)
+    keys = [(r.source, r.orf_id) for r in records]
+    if len(set(keys)) != len(keys):
+        raise ValueError("Duplicate ORF identifiers within one source")
+    names = [r.orf_id for r in records]
+    qualified = set()
+    while True:
+        counts = Counter(names)
+        collisions = {i for i, name in enumerate(names) if counts[name] > 1}
+        if not collisions:
+            break
+        # Qualify all participants, including a native ID that equals another
+        # record's generated ID. Escaping makes source/ID boundaries unambiguous.
+        qualified.update(collisions)
+        names = [f"{quote(r.source, safe='')}:{quote(r.orf_id, safe='')}" if i in qualified
+                 else r.orf_id for i, r in enumerate(records)]
+    return tuple(r if name == r.orf_id else replace(
+        r, orf_id=name, members=r.members or (r,), lifted_from=None, liftover=None)
+        for r, name in zip(records, names))
+
+
 class ORFCatalog:
     """Offline ORF records for one explicitly identified species and assembly."""
 
     def __init__(self, records, metadata):
-        self.records = tuple(records)
+        self.records = _qualify_colliding_ids(records)
         self._metadata = deepcopy(metadata)
         self.genome_build = metadata["genome_build"]
         self.taxon_id = metadata["taxon_id"]
         self.species = metadata["species"]
         self._by_id = {r.orf_id: r for r in self.records}
-        if len(self._by_id) != len(self.records):
-            raise ValueError("Duplicate ORF identifiers")
         self._aliases = {}
+        self._by_source_id = {}
         for r in self.records:
             names = {f"{m.source}:{m.orf_id}" for m in r.members or (r,)} | {m.orf_id for m in r.members}
             # A source accession split by coding path, "IP_1@ENST...", still answers to "IP_1".
             names |= {m.orf_id.split("@")[0] for m in r.members or (r,) if "@" in m.orf_id}
             for alias in names - {r.orf_id}:
                 self._aliases.setdefault(alias, []).append(r)
+            for member in r.members or (r,):
+                for name in {member.orf_id, member.orf_id.split("@")[0]}:
+                    matches = self._by_source_id.setdefault((member.source, name), [])
+                    if not any(match is r for match in matches):
+                        matches.append(r)
         for r in self.records:
             if r.genome_build != self.genome_build or r.strand not in ("+", "-"):
                 raise ValueError(f"Invalid assembly or strand for {r.orf_id}")
@@ -503,14 +531,17 @@ class ORFCatalog:
         """Return a copy of dataset version, sources, scope and provenance."""
         return deepcopy(self._metadata)
 
-    def get(self, orf_id):
+    def get(self, orf_id, *, source=None):
         """Return the ORF with this ID, bare or as ``source:id``; unknown IDs raise KeyError.
 
         In a merged catalogue every source's ID finds its ORF. A bare ID that
         two sources use for different ORFs raises rather than choosing one.
+        ``source`` selects a native ID explicitly, including when a source or
+        native ID contains a colon and the combined string is ambiguous.
         """
-        found = list(dict.fromkeys(
-            ([self._by_id[orf_id]] if orf_id in self._by_id else []) + list(self._aliases.get(orf_id, ()))))
+        matches = (self._by_source_id.get((source, orf_id), ()) if source is not None else
+                   ([self._by_id[orf_id]] if orf_id in self._by_id else []) + list(self._aliases.get(orf_id, ())))
+        found = list({id(record): record for record in matches}.values())
         if len(found) > 1:
             raise ValueError(f"{orf_id} names several ORFs: {', '.join(r.orf_id for r in found)}")
         if not found:
@@ -780,21 +811,20 @@ def merge_catalogs(catalogs):
                     groups.append(group)
                 group[0].add(origin[id(record)])
                 group[1].append(record)
-    uses = Counter(orf_id for _, members in groups for orf_id in {m.orf_id for m in members})
     records = []
     for _, members in groups:
         primary = members[0]
-        orf_id = primary.orf_id if uses[primary.orf_id] == 1 else f"{primary.source}:{primary.orf_id}"
-        if len(members) == 1 and orf_id == primary.orf_id:
+        if len(members) == 1:
             records.append(primary)
             continue
         pinned = {None if m.reference_context is None else m.reference_context.key for m in members}
         records.append(replace(
             # The merged ORF is no one source's lift; each member keeps its own lift provenance.
-            primary, orf_id=orf_id, members=tuple(members), lifted_from=None, liftover=None,
+            primary, members=tuple(members), lifted_from=None, liftover=None,
             initiation_context=_shared_context(primary, members),
             # Sources pinned to different annotations of one assembly leave the ORF unpinned.
             reference_context=primary.reference_context if len(pinned) == 1 else None, **_union(members, study)))
+    records = _qualify_colliding_ids(records)
     paths = {}
     for record in records:
         paths.setdefault(_path_key(record), set()).add((_merge_key(record), record.orf_id))

{orfdata-0.37.0 → orfdata-0.38.2}/orfdata/proteome.py

@@ -779,17 +779,20 @@ class ExtraORFs:
                 found.append(orf)
         return tuple(found)
 
-    def for_orf(self, orf_id):
+    def for_orf(self, orf_id, *, source=None):
         """Every transcript placement of one ORF, the source's own transcript first.
 
         After merging catalogues, any source's ID finds the ORF, bare or as
         ``source:id``; a bare ID two sources use for different ORFs raises.
+        Pass ``source`` to select the native ID in one source explicitly.
         """
         if self.catalog is not None:
             try:
-                orf_id = self.catalog.get(orf_id).orf_id
+                orf_id = self.catalog.get(orf_id, source=source).orf_id
             except KeyError:
                 return ()
+        elif source is not None:
+            raise ValueError("Source-qualified lookup requires the source catalogue")
         return tuple(sorted((o for o in self.orfs if o.orf_id == orf_id),
                             key=lambda o: (not o.on_source_transcript, _versioned(o.transcript))))
 

{orfdata-0.37.0 → orfdata-0.38.2}/orfdata/sources/openprot.py

@@ -31,6 +31,7 @@ from ..catalog import ORFCatalog, ORFEvidence, ORFRecord
 from ..evidence_summary import EvidenceSummary
 from ..model import ORFModel
 from ..reference import ScopedTarget
+from ..sequence import genetic_code
 
 MS_DEFINITION = "Sum of study-specific unique-peptide counts; not global peptide uniqueness."
 TE_DEFINITION = "Count of studies reporting one or more significant translation detections."
@@ -83,6 +84,9 @@ def _summary(reference, accession, protein, method, metric, value, definition, u
 def import_orfs(inputs, output, manifest):
     """Write the catalogue ``manifest`` describes from its OpenProt TSV, BED and FASTA archives."""
     processing = manifest.processing
+    translation_tables = processing.get("translation_tables", {})
+    for code in translation_tables.values():
+        genetic_code(code)  # Unsupported source declarations must never fall back.
     members = processing["members"]
     rows = list(csv.DictReader(io.StringIO(_member(inputs["tsv"], members["tsv"])), delimiter="\t"))
     bed = [line.split("\t") for line in _member(inputs["bed"], members["bed"]).splitlines() if line.strip()]
@@ -92,13 +96,16 @@ def import_orfs(inputs, output, manifest):
     tsv = next(s for s in manifest.sources if s.asset_id == "tsv")
     if len(bed) != len(rows):
         raise ValueError("OpenProt TSV and BED row counts disagree")
+    unknown = set(translation_tables) - {row["transcript accession"] for row in rows}
+    if unknown:
+        raise ValueError("OpenProt translation tables name absent source transcripts: %s" % sorted(unknown))
 
     paths, unresolved, scores = defaultdict(dict), [], {}
     for line, (row, region) in enumerate(zip(rows, bed), start=2):
         accession, transcript = row["protein accession"], row["transcript accession"]
-        if row["chr"] in ("MT", "M"):  # Needs table 2 and its starts, e.g. human IP_306399 begins ATA (#55).
+        if row["chr"] in ("MT", "M") and transcript not in translation_tables:
             raise ValueError(f"OpenProt TSV line {line} ({accession}) is mitochondrial; the importer handles "
-                             "nuclear ORFs only")
+                             "it only with an explicit source-transcript translation table")
         length = int(row["protein length (a.a.)"])
         blocks = _blocks(region)
         protein = proteins.get(accession)
@@ -116,7 +123,7 @@ def import_orfs(inputs, output, manifest):
         if not transcript.startswith(processing["transcript_prefix"]):
             unresolved.append(dict(protein=accession, transcript=transcript, reason=processing["unresolved_reason"]))
             continue
-        key = (row["chr"], region[5], blocks)
+        key = (row["chr"], region[5], blocks, translation_tables.get(transcript, 1))
         found = paths[accession].setdefault(key, dict(protein=protein, genes={}, types=set()))
         found["genes"][transcript] = row["gene symbol"].split(".")[0]  # Overlapping genes can share a path.
         found["types"].add(row["protein type"])
@@ -124,7 +131,7 @@ def import_orfs(inputs, output, manifest):
     records = []
     for accession, found in sorted(paths.items()):
         ms, te = scores[accession]
-        for (contig, strand, blocks), path in sorted(found.items(), key=lambda item: min(item[1]["genes"])):
+        for (contig, strand, blocks, table), path in sorted(found.items(), key=lambda item: min(item[1]["genes"])):
             transcript = min(path["genes"])
             if len(path["types"]) != 1:
                 raise ValueError(f"OpenProt gives {accession} several protein types")
@@ -132,7 +139,7 @@ def import_orfs(inputs, output, manifest):
             model = ORFModel(accession if len(found) == 1 else f"{accession}@{transcript}", transcript,
                              path["genes"][transcript], "", contig, strand, blocks, protein, path["types"].pop(),
                              record_reference.genome_build, record_reference.taxon_id, record_reference.species,
-                             source="openprot", reference_context=record_reference)
+                             source="openprot", reference_context=record_reference, translation_table=table)
             summaries = tuple(_summary(summary_reference, accession, protein, *values, tsv.url, tsv.sha256) for values in (
                 ("ms", "MS score", ms, MS_DEFINITION), ("ribosome", "TE Translation Event score", te, TE_DEFINITION)))
             records.append(ORFRecord.from_model(model, ORFEvidence(summaries=summaries)))

orfdata-0.38.2/orfdata/version.py

@@ -0,0 +1 @@
+__version__ = "0.38.2"

{orfdata-0.37.0 → orfdata-0.38.2/orfdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orfdata
-Version: 0.37.0
+Version: 0.38.2
 Summary: Reference proteomes with evidence-backed noncanonical ORFs on PyEnsembl genomes
 Author: OpenVax
 License-Expression: Apache-2.0

{orfdata-0.37.0 → orfdata-0.38.2}/orfdata.egg-info/SOURCES.txt

@@ -118,6 +118,7 @@ tests/conftest.py
 tests/test_catalog_registry.py
 tests/test_copy.py
 tests/test_cross_reference_pairing.py
+tests/test_dog_ncorf_search.py
 tests/test_dog_peptides.py
 tests/test_downloads.py
 tests/test_ensembl_search_reference.py

orfdata-0.38.2/tests/test_dog_ncorf_search.py

@@ -0,0 +1,62 @@
+import importlib.util
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+SCRIPTS = Path(__file__).parents[1] / "scripts" / "experimental" / "dog-ncorf-search"
+pytestmark = pytest.mark.skipif(not SCRIPTS.is_dir(), reason="experimental scripts are not in the sdist")
+
+
+def _load(name):
+    spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_dog_search_qvalues_include_complete_score_ties():
+    qvalues = _load("fdr").qvalues
+    scores = np.array([10.0, 9.0, 9.0, 8.0])
+    decoys = np.array([False, False, True, True])
+    expected = np.array([0.0, 0.5, 0.5, 1.0])
+    np.testing.assert_allclose(qvalues(scores, decoys), expected)
+    np.testing.assert_allclose(qvalues(scores[[0, 2, 1, 3]], decoys[[0, 2, 1, 3]]), expected)
+
+
+def test_dog_search_normalizes_non_blosum_residues():
+    normalize = _load("homology").blosum62_sequence
+    assert normalize("ACUOJX*") == "ACCKXX*"
+    with pytest.raises(ValueError, match="unsupported"):
+        normalize("AC-")
+
+
+def test_dog_search_driver_fails_when_a_run_has_a_failure_marker(tmp_path):
+    work = tmp_path / "search"
+    (work / "db").mkdir(parents=True)
+    (work / "work").mkdir()
+    (work / "db" / "dog_db.fasta").write_text(">CANON000000\nPEPTIDE\n")
+    (work / "runs.tsv").write_text(
+        "failed\tfixture\thttps://example.invalid/archive.zip\tfailed.mzML\t1\t1\n")
+    (work / "work" / "failed.fetch_failed").write_text("intentional test marker\n")
+
+    sage = tmp_path / "sage"
+    sage.write_text("#!/bin/sh\n[ \"${1:-}\" = --version ] && echo 'sage test'\n")
+    sage.chmod(0o755)
+    python = tmp_path / "python"
+    python.write_text(
+        f"#!/bin/sh\n"
+        "case \"$*\" in\n"
+        "  *'import pandas, pyarrow, inflate64'*) exit 0;;\n"
+        "esac\n"
+        f"exec {sys.executable} \"$@\"\n")
+    python.chmod(0o755)
+    env = dict(os.environ, SAGE=str(sage), PYTHON=str(python), MIN_FREE_GB="0")
+    result = subprocess.run(
+        ["bash", str(SCRIPTS / "driver.sh"), str(work)], env=env,
+        capture_output=True, text=True, check=False)
+    assert result.returncode == 1
+    assert "INCOMPLETE searched=0 failed=1" in result.stdout

{orfdata-0.37.0 → orfdata-0.38.2}/tests/test_genetic_codes.py

@@ -1,6 +1,9 @@
 """Declared NCBI genetic codes bind native mitochondrial ORFs and fail closed otherwise."""
 
+import csv
+import io
 import json
+import zipfile
 from dataclasses import replace
 from pathlib import Path
 
@@ -122,3 +125,54 @@ def test_codon_annotation_uses_the_declared_code(table, expected):
     assert codon["alternate_codon"] == "ATG" and codon["alternate_codon_translation"] == "M"
     with pytest.raises(ValueError, match="translation table"):
         replace(orf, translation_table=4).annotate_initiation_snv(103, "C", "A", genome_build="R64-1-1")
+
+
+@pytest.mark.parametrize("native", FIXTURES[:1], ids=[FIXTURES[0]["name"]], indirect=True)
+def test_native_openprot_import_accepts_only_a_declared_supported_code(native, tmp_path):
+    """Replay the pinned native row; archive wrappers/BED serialization are fixture-only."""
+    from orfdata import CatalogRegistry, load_catalog
+    from orfdata.sources.openprot import import_orfs
+
+    example, original, transcript = native
+    row = example["native"]
+    tsv = io.StringIO()
+    writer = csv.DictWriter(tsv, fieldnames=list(row), delimiter="\t", lineterminator="\n")
+    writer.writeheader()
+    writer.writerow(row)
+    a, b = original.coding_blocks[0]
+    bed = f"chrMT\t{a - 1}\t{b}\t{original.orf_id}\t0\t+\t{a - 1}\t{b}\t0\t1\t{b - a + 1}\t0\n"
+    contents = dict(tsv=tsv.getvalue(), bed=bed,
+                    fasta=example["native_fasta_header"] + "\n" + original.protein_sequence + "\n")
+    inputs = {}
+    for kind, text in contents.items():
+        path = tmp_path / f"fixture.{kind}.zip"
+        with zipfile.ZipFile(path, "w") as archive:
+            archive.writestr(f"fixture.{kind}", text)
+        inputs[kind] = path
+    base = CatalogRegistry().info("openprot-mouse")
+    summary = example["summaries"][0]
+    manifest = replace(base, catalog_id="openprot-human-fixture",
+                       references=(original.reference_context,
+                                   ReferenceContext(**summary["target"]["reference"])),
+                       sources=tuple(replace(s, url=summary["source_url"], sha256=summary["source_sha256"])
+                                     if s.asset_id == "tsv" else s for s in base.sources),
+                       processing=dict(base.processing, transcript_prefix="ENST",
+                                       members={kind: f"fixture.{kind}" for kind in contents},
+                                       translation_tables={original.transcript_id: 2}),
+                       coverage=dict(records=1, source_rows=1, resolved_rows=1, unresolved_rows=0,
+                                     accessions=1, resolved_accessions=1))
+    output = tmp_path / "catalog.json.gz"
+    import_orfs(inputs, output, manifest)
+    imported = load_catalog(output).get(original.orf_id)
+    assert imported.translation_table == 2
+    assert imported.coding_blocks == original.coding_blocks
+    assert imported.bind(transcript).protein_sequence == original.protein_sequence
+    assert {s.metric: s.value for s in imported.summaries} == {
+        "MS score": 2, "TE Translation Event score": 0}
+    assert imported.summaries[0].source_sha256 == summary["source_sha256"]
+    for declarations, error in [({}, "is mitochondrial"),
+                                 ({original.transcript_id: 4}, "translation table"),
+                                 ({"missing-transcript": 2}, "absent source transcripts")]:
+        with pytest.raises(ValueError, match=error):
+            import_orfs(inputs, output, replace(manifest, processing=dict(
+                manifest.processing, translation_tables=declarations)))

{orfdata-0.37.0 → orfdata-0.38.2}/tests/test_merge_catalogs.py

@@ -62,6 +62,64 @@ def test_one_id_for_different_orfs_is_qualified_and_never_guessed():
         merged.get("missing")
 
 
+def test_direct_catalogue_preserves_colliding_source_records_and_lookup(tmp_path):
+    first = replace(orf("shared", ribosome_studies=("study-a",)), source="a")
+    second = replace(orf("shared", ribosome_studies=("study-b",)), source="b")
+    catalog = ORFCatalog([first, second], metadata("combined-v1", "combined"))
+    assert [r.orf_id for r in catalog.records] == ["a:shared", "b:shared"]
+    for original in (first, second):
+        record = catalog.get("shared", source=original.source)
+        assert record.members == (original,)
+        assert catalog.get(f"{original.source}:shared") is record
+        assert record.ribosome_studies == original.ribosome_studies
+    with pytest.raises(ValueError, match="shared names several ORFs"):
+        catalog.get("shared")
+    with pytest.raises(KeyError):
+        catalog.get("shared", source="absent")
+    path = tmp_path / "collisions.json.gz"
+    catalog.save(path)
+    loaded = load_catalog(path)
+    assert loaded.to_dict() == catalog.to_dict()
+    assert loaded.get("shared", source="a").members == (first,)
+    with pytest.raises(ValueError, match="names several ORFs"):
+        loaded.get("shared")
+
+
+def test_source_qualified_ids_cannot_overwrite_native_ids_or_delimiter_collisions():
+    records = [replace(orf(identifier), source=source) for source, identifier in (
+        ("a", "x"), ("b", "x"), ("a", "a:x"), ("a:b", "c"), ("a", "b:c"))]
+    catalog = ORFCatalog(records, metadata("combined-v1", "combined"))
+    assert len({r.orf_id for r in catalog.records}) == len(records)
+    for original in records:
+        resolved = catalog.get(original.orf_id, source=original.source)
+        assert original in (resolved.members or (resolved,))
+    for ambiguous in ("x", "a:x", "a:b:c"):
+        with pytest.raises(ValueError, match="names several ORFs"):
+            catalog.get(ambiguous)
+
+
+@pytest.mark.parametrize("changed", [False, True])
+def test_duplicate_ids_within_one_source_still_raise(changed):
+    original = orf("same")
+    other = replace(original, protein_sequence="MKPA") if changed else original
+    with pytest.raises(ValueError, match="Duplicate ORF identifiers"):
+        ORFCatalog([original, other], metadata("fixture-v1", "synthetic"))
+
+
+def test_direct_collisions_keep_distinct_placements_and_proteome_records(genome):  # noqa: F811
+    records = [replace(orf("shared"), source="a"),
+               replace(orf("shared", blocks=((218, 226),), protein="MK"), source="b")]
+    catalog = ORFCatalog(records, metadata("combined-v1", "combined"))
+    found = extra_orfs(genome, catalog=catalog)
+    assert {o.protein_sequence for o in found.for_orf("shared", source="a")} == {"MKPG"}
+    assert {o.protein_sequence for o in found.for_orf("shared", source="b")} == {"MK"}
+    with pytest.raises(ValueError, match="names several ORFs"):
+        found.for_orf("shared")
+    proteins = Proteome.from_genome(genome).with_extra_orfs(None, catalog=catalog)
+    assert {(p.accession, p.sequence) for p in proteins if p.kind == "extra_orf"} == {
+        ("a:shared", "MKPG"), ("b:shared", "MK")}
+
+
 def test_incompatible_catalogues_are_not_merged():
     first = make_catalog()
     assert merge_catalogs([first]) is first

orfdata-0.37.0/orfdata/version.py

@@ -1 +0,0 @@
-__version__ = "0.37.0"