bam2tensor 2.6__tar.gz → 2.7__tar.gz

{bam2tensor-2.6 → bam2tensor-2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bam2tensor
-Version: 2.6
+Version: 2.7
 Summary: Convert bisulfite-seq and EM-seq BAM files to sparse tensor representations of DNA methylation
 Project-URL: Homepage, https://github.com/mcwdsi/bam2tensor
 Project-URL: Repository, https://github.com/mcwdsi/bam2tensor

{bam2tensor-2.6 → bam2tensor-2.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "bam2tensor"
-version = "2.6"
+version = "2.7"
 description = "Convert bisulfite-seq and EM-seq BAM files to sparse tensor representations of DNA methylation"
 authors = [{ name = "Nick Semenkovich", email = "semenko@alum.mit.edu" }]
 license = "MIT"

{bam2tensor-2.6 → bam2tensor-2.7}/src/bam2tensor/__init__.py

@@ -50,4 +50,4 @@ See Also:
     - https://mcwdsi.github.io/bam2tensor for full documentation
 """
 
-__version__ = "2.6"
+__version__ = "2.7"

{bam2tensor-2.6 → bam2tensor-2.7}/src/bam2tensor/embedding.py

@@ -45,6 +45,9 @@ import numpy as np
 from tqdm import tqdm
 from Bio import SeqIO
 
+from bam2tensor import __version__
+from bam2tensor.metadata import compute_fasta_sha256
+
 
 class GenomeMethylationEmbedding:
     """Manages CpG site positions and coordinate conversions for a reference genome.
@@ -173,7 +176,12 @@ class GenomeMethylationEmbedding:
                     window_size == self.window_size
                 ), "Window size does not match cached window size!"
             except FileNotFoundError as e:
-                if self.verbose:
+                # Stale-cache rejections (version or FASTA SHA-256 mismatch)
+                # raise FileNotFoundError too — always surface those so users
+                # are not silently regenerating a cache they thought was valid.
+                if os.path.exists(self.cache_file):
+                    print(f"Discarding stale embedding cache: {e}")
+                elif self.verbose:
                     print("Could not load methylation embedding from cache: " + str(e))
 
         if not cache_available:
@@ -224,6 +232,9 @@ class GenomeMethylationEmbedding:
         The cache file is named "{genome_name}.cache.json.gz" and contains:
         - genome_name: The genome identifier
         - fasta_source: Path to the original FASTA file
+        - fasta_sha256: SHA-256 of the FASTA file bytes (for cache validation)
+        - bam2tensor_version: Version that produced this cache
+        - total_cpg_sites: Total CpG count across all included chromosomes
         - expected_chromosomes: List of included chromosomes
         - window_size: The window_size parameter (for compatibility checking)
         - cpg_sites_dict: Dictionary of chromosome -> list of CpG positions
@@ -241,9 +252,13 @@ class GenomeMethylationEmbedding:
 
         assert len(self.cpg_sites_dict) > 0, "CpG sites dict is empty!"
 
+        total_cpg_sites = sum(len(v) for v in self.cpg_sites_dict.values())
         cache_data = {
             "genome_name": self.genome_name,
             "fasta_source": self.fasta_source,
+            "fasta_sha256": compute_fasta_sha256(self.fasta_source),
+            "bam2tensor_version": __version__,
+            "total_cpg_sites": total_cpg_sites,
             "expected_chromosomes": self.expected_chromosomes,
             "window_size": self.window_size,
             "cpg_sites_dict": self.cpg_sites_dict,
@@ -263,38 +278,66 @@ class GenomeMethylationEmbedding:
         restore all CpG site data. If successful, this avoids the slow
         FASTA parsing step.
 
+        Provenance is validated before the cached data is trusted: the
+        cache must have been written by the same major.minor of
+        bam2tensor and must reference a FASTA file with the same SHA-256
+        as the current ``fasta_source``. A stale cache is rejected with
+        a ``FileNotFoundError`` so the caller falls through to a fresh
+        FASTA parse and overwrites the stale cache on save.
+
         Returns:
             True if the cache was successfully loaded.
 
         Raises:
-            FileNotFoundError: If the cache file does not exist.
+            FileNotFoundError: If the cache file does not exist, or if
+                the cache is stale (version mismatch or FASTA SHA-256
+                mismatch).
 
         Note:
-            After loading, the caller should verify that expected_chromosomes
-            and window_size match the current configuration, as this method
-            overwrites those attributes with cached values.
+            After loading, the caller should verify that
+            ``expected_chromosomes`` and ``window_size`` match the current
+            configuration, as this method overwrites those attributes
+            with cached values.
         """
 
-        if os.path.exists(self.cache_file):
-            if self.verbose:
-                print(f"\tReading embedding from cache: {self.cache_file}")
+        if not os.path.exists(self.cache_file):
+            raise FileNotFoundError("No cache of embedding found.")
 
-            # TODO: Add type hinting via TypedDicts?
-            # e.g. https://stackoverflow.com/questions/51291722/define-a-jsonable-type-using-mypy-pep-526
-            with gzip.open(self.cache_file, "rt") as f:
-                self.cache_data = json.load(f)
+        if self.verbose:
+            print(f"\tReading embedding from cache: {self.cache_file}")
 
-            # Load the cached data
-            self.genome_name = self.cache_data["genome_name"]
-            self.fasta_source = self.cache_data["fasta_source"]
-            self.expected_chromosomes = self.cache_data["expected_chromosomes"]
-            self.window_size = self.cache_data["window_size"]
-            self.cpg_sites_dict = self.cache_data["cpg_sites_dict"]
+        with gzip.open(self.cache_file, "rt") as f:
+            self.cache_data = json.load(f)
 
-            if self.verbose:
-                print(f"\tCached genome fasta source: {self.fasta_source}")
-        else:
-            raise FileNotFoundError("No cache of embedding found.")
+        # Validate cache provenance: stale caches predating v2.7 used a
+        # case-sensitive CpG search that silently dropped roughly half
+        # the CpG sites in soft-masked FASTAs (e.g. UCSC's hg38.fa.gz).
+        cached_version = self.cache_data.get("bam2tensor_version")
+        if cached_version != __version__:
+            raise FileNotFoundError(
+                f"Stale cache {self.cache_file!r}: written by bam2tensor "
+                f"{cached_version!r}, current is {__version__!r}. "
+                "Regenerating."
+            )
+
+        cached_fasta_sha256 = self.cache_data.get("fasta_sha256")
+        current_fasta_sha256 = compute_fasta_sha256(self.fasta_source)
+        if cached_fasta_sha256 != current_fasta_sha256:
+            raise FileNotFoundError(
+                f"Stale cache {self.cache_file!r}: FASTA SHA-256 mismatch "
+                f"(cache={cached_fasta_sha256}, current={current_fasta_sha256}). "
+                "Regenerating."
+            )
+
+        # Load the cached data
+        self.genome_name = self.cache_data["genome_name"]
+        self.fasta_source = self.cache_data["fasta_source"]
+        self.expected_chromosomes = self.cache_data["expected_chromosomes"]
+        self.window_size = self.cache_data["window_size"]
+        self.cpg_sites_dict = self.cache_data["cpg_sites_dict"]
+
+        if self.verbose:
+            print(f"\tCached genome fasta source: {self.fasta_source}")
 
         return True
 
@@ -350,7 +393,11 @@ class GenomeMethylationEmbedding:
                 if self.verbose:
                     tqdm.write(f"\tSkipping chromosome {seqrecord.id}")
                 continue
-            sequence = seqrecord.seq
+            # Upper-case the sequence so soft-masked FASTAs (UCSC's default
+            # hg38.fa.gz uses lowercase for RepeatMasker/TRF regions) do not
+            # silently drop CpGs in repeats — that is roughly half of all
+            # CpGs in the human genome.
+            sequence = seqrecord.seq.upper()
 
             # Find all CpG sites
             # The pos+1 is because we want to store the 1-based position, because .bed is wild and arguably 1-based maybe:

{bam2tensor-2.6 → bam2tensor-2.7}/src/bam2tensor/metadata.py

@@ -27,17 +27,44 @@ Example:
         hg38
 """
 
+import hashlib
 import io
 import json
 import zipfile
 import zlib
+from typing import TYPE_CHECKING
 
 import numpy as np
 
-from bam2tensor.embedding import GenomeMethylationEmbedding
+if TYPE_CHECKING:
+    # Avoid a runtime circular import: embedding.py imports compute_fasta_sha256
+    # from this module, and this module only needs the embedding type for
+    # annotations.
+    from bam2tensor.embedding import GenomeMethylationEmbedding
 
 
-def compute_cpg_index_crc32(embedding: GenomeMethylationEmbedding) -> str:
+def compute_fasta_sha256(fasta_source: str) -> str:
+    """Compute the SHA-256 of a FASTA file's bytes on disk.
+
+    Used to stamp the CpG-site cache (see
+    :py:class:`bam2tensor.embedding.GenomeMethylationEmbedding`) so a
+    cache can be rejected when the underlying FASTA changes (e.g. a
+    user swaps a soft-masked build for a hard-masked one).
+
+    Args:
+        fasta_source: Path to the reference FASTA file.
+
+    Returns:
+        The hex-encoded SHA-256 digest of the file's bytes.
+    """
+    h = hashlib.sha256()
+    with open(fasta_source, "rb") as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def compute_cpg_index_crc32(embedding: "GenomeMethylationEmbedding") -> str:
     """Compute a CRC32 checksum over the CpG site positions in an embedding.
 
     The checksum captures the exact column mapping of the sparse matrix:

{bam2tensor-2.6 → bam2tensor-2.7}/tests/test_embedding.py

@@ -1,6 +1,10 @@
+import gzip
+import json
 import os
+
 import pytest
-from bam2tensor import embedding
+
+from bam2tensor import __version__, embedding
 
 # Generate a fresh, uncached embedding
 test_embedding = embedding.GenomeMethylationEmbedding(
@@ -241,3 +245,219 @@ def test_save_cache_verbose(tmp_path, capsys) -> None:
         assert "Saved embedding cache" in captured.out
     finally:
         os.chdir(original_cwd)
+
+
+# -- Regression: soft-masked FASTAs must enumerate every CpG -----------------
+
+
+def _write_softmasked_fasta(path: str) -> None:
+    """Write a tiny mixed-case FASTA with CG/cg/Cg/cG to ``path``.
+
+    UCSC's default ``hg38.fa.gz`` is soft-masked (lowercase = RepeatMasker
+    / Tandem Repeats Finder). Prior to v2.7 the CpG enumeration used a
+    case-sensitive search and silently dropped roughly half the CpGs in
+    soft-masked references — this fixture pins the regression.
+    """
+    # 1-based positions of the C in each C[gG] (0-based + 1):
+    #   CG @ pos  6, cg @ pos 11, Cg @ pos 17, cG @ pos 23
+    seq = "AAAAACGAAAcgAAAACgAAAAcGAAAAA"
+    with open(path, "w") as fh:
+        fh.write(">chr1\n")
+        fh.write(seq + "\n")
+
+
+def test_soft_masked_fasta_enumerates_all_cpgs(tmp_path) -> None:
+    """All CpGs are found regardless of case (regression for v2.7 case fix).
+
+    Bug: prior to v2.7, ``sequence.find('CG')`` on a soft-masked FASTA
+    missed every lowercase or mixed-case ``cg``/``Cg``/``cG``. On real
+    UCSC hg38 that silently dropped ~53% of all CpG sites.
+    """
+    fasta = tmp_path / "softmasked.fa"
+    _write_softmasked_fasta(str(fasta))
+
+    original_cwd = os.getcwd()
+    os.chdir(tmp_path)
+    try:
+        emb = embedding.GenomeMethylationEmbedding(
+            "softmasked_test",
+            expected_chromosomes=["chr1"],
+            fasta_source=str(fasta),
+            window_size=150,
+            skip_cache=True,
+            verbose=False,
+        )
+    finally:
+        os.chdir(original_cwd)
+
+    assert emb.total_cpg_sites == 4
+    assert emb.cpg_sites_dict["chr1"] == [6, 11, 17, 23]
+
+
+# -- Cache provenance: stamp + reject stale ---------------------------------
+
+
+def _read_cache(path: str) -> dict:
+    """Read and decode a gzipped JSON cache file."""
+    with gzip.open(path, "rt") as f:
+        return json.load(f)
+
+
+def test_cache_stamps_provenance(tmp_path) -> None:
+    """Saved cache embeds bam2tensor_version, fasta_sha256, total_cpg_sites."""
+    original_cwd = os.getcwd()
+    os.chdir(tmp_path)
+    try:
+        emb = embedding.GenomeMethylationEmbedding(
+            "provenance_test",
+            expected_chromosomes=["chr1", "chr2", "chr3"],
+            fasta_source=os.path.join(original_cwd, "tests/test_fasta.fa"),
+            window_size=150,
+            skip_cache=False,
+            verbose=False,
+        )
+        data = _read_cache("provenance_test.cache.json.gz")
+    finally:
+        os.chdir(original_cwd)
+
+    assert data["bam2tensor_version"] == __version__
+    assert data["total_cpg_sites"] == emb.total_cpg_sites
+    assert isinstance(data["fasta_sha256"], str)
+    assert len(data["fasta_sha256"]) == 64  # hex-encoded SHA-256
+
+
+def test_cache_rejected_on_fasta_sha256_mismatch(tmp_path, capsys) -> None:
+    """A cache whose ``fasta_sha256`` no longer matches the current FASTA is discarded.
+
+    This is the primary safety net for the upgrade to v2.7: any user
+    with a pre-v2.7 cache (or any user who switches between soft- and
+    hard-masked FASTAs) will get a clean re-parse rather than silently
+    misaligned column indices.
+    """
+    original_cwd = os.getcwd()
+    os.chdir(tmp_path)
+    try:
+        # First run: generate cache against the real test FASTA.
+        emb1 = embedding.GenomeMethylationEmbedding(
+            "sha_mismatch_test",
+            expected_chromosomes=["chr1", "chr2", "chr3"],
+            fasta_source=os.path.join(original_cwd, "tests/test_fasta.fa"),
+            window_size=150,
+            skip_cache=False,
+            verbose=False,
+        )
+        cache_path = "sha_mismatch_test.cache.json.gz"
+        assert os.path.exists(cache_path)
+
+        # Tamper with the cached fasta_sha256 so it can no longer match
+        # the on-disk FASTA. This simulates a user upgrading bam2tensor
+        # with an inherited v2.5 cache whose enumeration is stale.
+        data = _read_cache(cache_path)
+        data["fasta_sha256"] = "0" * 64
+        with gzip.open(cache_path, "wt") as f:
+            json.dump(data, f)
+
+        # Second run: the stale cache must be rejected and regenerated.
+        emb2 = embedding.GenomeMethylationEmbedding(
+            "sha_mismatch_test",
+            expected_chromosomes=["chr1", "chr2", "chr3"],
+            fasta_source=os.path.join(original_cwd, "tests/test_fasta.fa"),
+            window_size=150,
+            skip_cache=False,
+            verbose=False,
+        )
+
+        captured = capsys.readouterr()
+        assert "Discarding stale embedding cache" in captured.out
+        assert "FASTA SHA-256 mismatch" in captured.out
+
+        # Regenerated cache: same content as the first run.
+        assert emb2.total_cpg_sites == emb1.total_cpg_sites
+        # And the on-disk cache now stamps the real SHA-256, not the tampered one.
+        refreshed = _read_cache(cache_path)
+        assert refreshed["fasta_sha256"] != "0" * 64
+    finally:
+        os.chdir(original_cwd)
+
+
+def test_cache_rejected_on_version_mismatch(tmp_path, capsys) -> None:
+    """A cache stamped with an older bam2tensor version is discarded."""
+    original_cwd = os.getcwd()
+    os.chdir(tmp_path)
+    try:
+        embedding.GenomeMethylationEmbedding(
+            "version_mismatch_test",
+            expected_chromosomes=["chr1", "chr2", "chr3"],
+            fasta_source=os.path.join(original_cwd, "tests/test_fasta.fa"),
+            window_size=150,
+            skip_cache=False,
+            verbose=False,
+        )
+        cache_path = "version_mismatch_test.cache.json.gz"
+
+        # Roll the version back to a release that predates the
+        # case-sensitivity fix; any non-current version should trigger
+        # rejection.
+        data = _read_cache(cache_path)
+        data["bam2tensor_version"] = "2.5"
+        with gzip.open(cache_path, "wt") as f:
+            json.dump(data, f)
+
+        embedding.GenomeMethylationEmbedding(
+            "version_mismatch_test",
+            expected_chromosomes=["chr1", "chr2", "chr3"],
+            fasta_source=os.path.join(original_cwd, "tests/test_fasta.fa"),
+            window_size=150,
+            skip_cache=False,
+            verbose=False,
+        )
+
+        captured = capsys.readouterr()
+        assert "Discarding stale embedding cache" in captured.out
+        assert "bam2tensor" in captured.out
+        assert "2.5" in captured.out
+    finally:
+        os.chdir(original_cwd)
+
+
+def test_cache_rejected_when_provenance_absent(tmp_path, capsys) -> None:
+    """A pre-v2.7 cache without provenance fields is treated as stale.
+
+    Users on v2.5 / v2.6 have caches with no ``bam2tensor_version`` or
+    ``fasta_sha256``. After upgrading, those caches must be discarded so
+    the case-sensitivity fix actually takes effect on their data.
+    """
+    original_cwd = os.getcwd()
+    os.chdir(tmp_path)
+    try:
+        embedding.GenomeMethylationEmbedding(
+            "legacy_cache_test",
+            expected_chromosomes=["chr1", "chr2", "chr3"],
+            fasta_source=os.path.join(original_cwd, "tests/test_fasta.fa"),
+            window_size=150,
+            skip_cache=False,
+            verbose=False,
+        )
+        cache_path = "legacy_cache_test.cache.json.gz"
+
+        # Strip provenance fields to mimic a v2.5-era cache file.
+        data = _read_cache(cache_path)
+        data.pop("bam2tensor_version", None)
+        data.pop("fasta_sha256", None)
+        data.pop("total_cpg_sites", None)
+        with gzip.open(cache_path, "wt") as f:
+            json.dump(data, f)
+
+        embedding.GenomeMethylationEmbedding(
+            "legacy_cache_test",
+            expected_chromosomes=["chr1", "chr2", "chr3"],
+            fasta_source=os.path.join(original_cwd, "tests/test_fasta.fa"),
+            window_size=150,
+            skip_cache=False,
+            verbose=False,
+        )
+
+        captured = capsys.readouterr()
+        assert "Discarding stale embedding cache" in captured.out
+    finally:
+        os.chdir(original_cwd)

{bam2tensor-2.6 → bam2tensor-2.7}/tests/test_inspect.py

@@ -6,7 +6,7 @@ import numpy as np
 import scipy.sparse
 from click.testing import CliRunner
 
-from bam2tensor import __main__
+from bam2tensor import __main__, __version__
 from bam2tensor.inspect import _format_size
 from bam2tensor.inspect import main as inspect_main
 from bam2tensor.metadata import write_npz_metadata, write_npz_tlen
@@ -122,7 +122,7 @@ def test_inspect_end_to_end(tmp_path) -> None:
     assert result.exit_code == 0
     assert "test" in result.output  # genome_name
     assert "CpG index CRC32:" in result.output
-    assert "v2.6" in result.output
+    assert f"v{__version__}" in result.output
 
 
 def test_format_size_bytes() -> None:

{bam2tensor-2.6 → bam2tensor-2.7}/uv.lock

@@ -63,7 +63,7 @@ wheels = [
 
 [[package]]
 name = "bam2tensor"
-version = "2.6"
+version = "2.7"
 source = { editable = "." }
 dependencies = [
     { name = "biopython" },