sniffcell 0.5.0__tar.gz

sniffcell-0.5.0/.github/workflows/python-publish.yml

@@ -0,0 +1,70 @@
+# This workflow will upload a Python Package to PyPI when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  release-build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Build release distributions
+        run: |
+          # NOTE: put your own distribution build steps here.
+          python -m pip install build
+          # python -m build
+
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-dists
+          path: dist/
+
+  pypi-publish:
+    runs-on: ubuntu-latest
+    needs:
+      - release-build
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
+
+    # Dedicated environments with protections for publishing are strongly recommended.
+    # For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules
+    environment:
+      name: pypi
+      # OPTIONAL: uncomment and update to include your PyPI project URL in the deployment status:
+      url: https://pypi.org/p/sniffcell
+      #
+      # ALTERNATIVE: if your GitHub Release name is the PyPI project version string
+      # ALTERNATIVE: exactly, uncomment the following line instead:
+      # url: https://pypi.org/project/YOURPROJECT/${{ github.event.release.name }}
+
+    steps:
+      - name: Retrieve release distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: release-dists
+          path: dist/
+
+      - name: Publish release distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/

sniffcell-0.5.0/.gitignore

@@ -0,0 +1,23 @@
+.ipynb_checkpoints/*
+
+# Ignore Python cache files
+__pycache__/
+src/__pycache__/*
+*.py[cod]
+
+# Ignore Jupyter Notebook checkpoints
+.ipynb_checkpoints/
+
+# Ignore virtual environment directories
+venv/
+env/
+
+dataset/
+sniffmeth_deconv_output/
+notebooks/
+.vscode/
+
+src/archived_src/*
+test/
+
+atlas/

sniffcell-0.5.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Yilei Fu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sniffcell-0.5.0/PKG-INFO

@@ -0,0 +1,48 @@
+Metadata-Version: 2.4
+Name: sniffcell
+Version: 0.5.0
+Summary: SniffCell: Annotate SVs cell type based on CpG methylation
+Home-page: https://github.com/Fu-Yilei/SniffCell
+Author: Yilei Fu
+Author-email: yilei.fu@bcm.edu
+License: MIT
+Project-URL: Bug Tracker, https://github.com/Fu-Yilei/SniffCell/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pysam>=0.21.0
+Requires-Dist: edlib>=1.3.9
+Requires-Dist: psutil>=5.9.4
+Requires-Dist: numpy>=2.2.0
+Requires-Dist: pandas>=2.3.0
+Requires-Dist: scipy
+Requires-Dist: tqdm
+Requires-Dist: scikit-learn
+Dynamic: author
+Dynamic: author-email
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: summary
+
+# SniffCell - Identifying cell type specific SV from long-read bulk sequenced tissue only
+
+SniffCell is a tool designed to analyze DNA methylation changes associated with structural variations (SVs), including mosaic SVs. It processes primary alignments from BAM files and provides detailed outputs for visualization and analysis.
+
+
+    positional arguments:
+    {find,deconv,anno,svanno,dmsv}
+        find                Find cell type-specific DMRs.
+        deconv              Deconvolve cell-type composition from methylation data.
+        anno                Annotate variants with cell-type-specific methylation.
+        svanno              Use pre-annotated reads csv to annotate variants' cell types
+        dmsv                Find out which SV's supporting reads have differential methylation compared to non-supporting reads.
+
+    options:
+    -h, --help            show this help message and exit
+    -v, --version         show program's version number and exit

sniffcell-0.5.0/README.md

@@ -0,0 +1,16 @@
+# SniffCell - Identifying cell type specific SV from long-read bulk sequenced tissue only
+
+SniffCell is a tool designed to analyze DNA methylation changes associated with structural variations (SVs), including mosaic SVs. It processes primary alignments from BAM files and provides detailed outputs for visualization and analysis.
+
+
+    positional arguments:
+    {find,deconv,anno,svanno,dmsv}
+        find                Find cell type-specific DMRs.
+        deconv              Deconvolve cell-type composition from methylation data.
+        anno                Annotate variants with cell-type-specific methylation.
+        svanno              Use pre-annotated reads csv to annotate variants' cell types
+        dmsv                Find out which SV's supporting reads have differential methylation compared to non-supporting reads.
+
+    options:
+    -h, --help            show this help message and exit
+    -v, --version         show program's version number and exit

sniffcell-0.5.0/pyproject.toml

@@ -0,0 +1,8 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]

sniffcell-0.5.0/setup.cfg

@@ -0,0 +1,42 @@
+[metadata]
+name = sniffcell
+version = 0.5.0
+author = Yilei Fu
+author_email = yilei.fu@bcm.edu
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://github.com/Fu-Yilei/SniffCell
+project_urls = 
+	Bug Tracker = https://github.com/Fu-Yilei/SniffCell/issues
+classifiers = 
+	Programming Language :: Python :: 3
+	License :: OSI Approved :: MIT License
+	Operating System :: OS Independent
+
+[options]
+package_dir = 
+	= src
+packages = find:
+python_requires = >=3.10
+install_requires = 
+	pysam>=0.21.0
+	edlib>=1.3.9
+	psutil>=5.9.4
+	numpy>=2.2.0
+	pandas>=2.3.0
+	scipy
+	tqdm
+	scikit-learn
+include_package_data = True
+
+[options.packages.find]
+where = src
+
+[options.entry_points]
+console_scripts = 
+	sniffcell = sniffcell.main:main
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

sniffcell-0.5.0/setup.py

@@ -0,0 +1,14 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='sniffcell',
+    version='0.5.0',
+    packages=find_packages(),
+    url='https://github.com/Fu-Yilei/SniffCell',
+    license='MIT',
+    author='Yilei Fu',
+    author_email='yilei.fu@bcm.edu',
+    description='SniffCell: Annotate SVs cell type based on CpG methylation',
+    long_description=open('README.md').read(),
+    long_description_content_type='text/markdown',
+)

sniffcell-0.5.0/src/sniffcell/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.0"

sniffcell-0.5.0/src/sniffcell/anno/anno.py

@@ -0,0 +1,230 @@
+import os
+import pandas as pd
+from sniffcell.anno.kmeans import kmeans_cluster_cells
+from sniffcell.anno.methyl_matrix import methyl_matrix_from_bam
+from sniffcell.anno.filter_bed_based_on_variants import filter_bed_based_on_variants
+from sniffcell.anno.vcf_to_df import read_vcf_to_df
+from sniffcell.anno.variant_assignment import assign_sv_celltypes
+from tqdm import tqdm
+import multiprocessing as mp
+import numpy as np
+import logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(processName)s] %(levelname)s: %(message)s")
+
+def _one_dmr(args):
+    # args: (row_dict, input_bam, reference_fasta)
+    logger = logging.getLogger("anno._one_dmr")
+
+    row, input_file, reference = args
+    chrom = str(row["chr"])
+    start = int(row["start"])
+    end   = int(row["end"])
+
+    best_group = str(row["best_group"])
+    best_dir   = row.get("best_dir", None)
+
+    # collect arbitrary cell types from mean_* columns in THIS row
+    cell_types = []
+    for k, v in row.items():
+        if isinstance(k, str) and k.startswith("mean_") and k not in ("mean_best_value", "mean_rest_value", "mean_margin"):
+            cell_types.append(k[len("mean_"):])
+    if best_group not in cell_types:
+        cell_types.append(best_group)
+    cell_types = sorted(dict.fromkeys(cell_types))  # stable order
+
+    # logger.info(f"[{chrom}:{start}-{end}] best_group={best_group} best_dir={best_dir} "
+    #             f"cell_types={cell_types} (n={len(cell_types)})")
+
+    try:
+        # load methylation matrix + CpG positions
+        mm, cpgs = methyl_matrix_from_bam(
+            input_file, reference, chrom=chrom, start=start, end=end, return_positions=True
+        )
+        n_reads_raw = 0 if mm is None else mm.shape[0]
+        n_cpgs = len(cpgs)
+        if n_cpgs == 0:
+            logger.warning(f"[{chrom}:{start}-{end}] no CpGs found; skipping")
+            return None
+
+        # drop rows that are entirely NaN across CpGs
+        mm = mm.dropna(how="all")
+        if mm.empty or mm.shape[0] < 2:
+            logger.warning(f"[{chrom}:{start}-{end}] usable_reads={mm.shape[0] if not mm.empty else 0} "
+                           f"(raw={n_reads_raw}) < 2; skipping")
+            return None
+
+        # call your untouched kmeans wrapper to assign target vs Other
+        dmr_row = {
+            "best_group": best_group,
+            "best_dir": best_dir,
+            "mean_best_value": row.get("mean_best_value", np.nan),
+            "mean_rest_value": row.get("mean_rest_value", np.nan),
+        }
+        out = kmeans_cluster_cells(mm, dmr_row=dmr_row)
+        
+        # CpG bounds from cpgs
+        cpgstart = int(cpgs[0])
+        cpgend   = int(cpgs[-1])
+
+        # read names from MultiIndex level 0 if present
+        if isinstance(mm.index, pd.MultiIndex) and "read_name" in mm.index.names:
+            readnames = mm.index.get_level_values("read_name").astype(str).values
+        else:
+            readnames = (mm.index.astype(str).values if mm.index.dtype == object
+                         else np.array([f"read_{i}" for i in range(len(mm))], dtype=str))
+
+        # target mask from your output column
+        mask_target = (out["celltype_or_other"].astype(str).str.lower()
+                       == best_group.strip().lower()).values
+        # build variable-length code strings
+        pos = {ct: i for i, ct in enumerate(cell_types)}
+        t_idx = pos[best_group]
+        target_bits = ["0"] * len(cell_types); target_bits[t_idx] = "1"
+        other_bits  = ["1"] * len(cell_types); other_bits[t_idx]  = "0"
+        target_code = "".join(target_bits)
+        other_code  = "".join(other_bits)
+        code_col = np.where(mask_target, target_code, other_code)
+
+        # --- per-read assignments (each read = one row / index) ---
+        assign_df = pd.DataFrame({
+            "chr": chrom,
+            "start": start,
+            "end": end,
+            "cpgstart": cpgstart,
+            "cpgend": cpgend,
+            "code_order": "|".join(cell_types),
+            "code": code_col,
+        }, index=pd.Index(readnames, name="readname"))
+
+        # per-block means (per-read mean methylation, then avg by target vs other)
+        X_imp = mm.astype(float).copy().fillna(mm.astype(float).mean())
+        read_mean = X_imp.mean(axis=1).values
+        tgt_mean = float(np.nanmean(read_mean[mask_target])) if mask_target.any() else np.nan
+        oth_mean = float(np.nanmean(read_mean[~mask_target])) if (~mask_target).any() else np.nan
+
+        # logger.info(f"[{chrom}:{start}-{end}] target_mean={tgt_mean:.4f} other_mean={oth_mean:.4f} "
+        #             f"cpg_bounds={cpgstart}-{cpgend}")
+
+        state_payload = {
+            "chr": chrom, "start": start, "end": end,
+            "cpgstart": cpgstart, "cpgend": cpgend,
+        }
+        for ct in cell_types:
+            state_payload[f"{ct}_methylation"] = tgt_mean if ct == best_group else oth_mean
+        state_df = pd.DataFrame([state_payload])
+
+        return assign_df, state_df
+
+    except Exception as e:
+        logger.exception(f"[{chrom}:{start}-{end}] failed with error")
+        return None
+
+def sv_anno(args):
+    logger = logging.getLogger("anno.sv_anno")
+    logger.info("Starting SV annotation from pre-annotated reads")
+    if args.command == "svanno":    
+        input_file = args.input
+    else:
+        input_file = os.path.join(args.output, "reads_classification.tsv")
+    if args.kanpig_read_names is not None:
+        logger.info(f"Using kanpig read names from: {args.kanpig_read_names}")
+    else:
+        logger.info("No kanpig read names provided; using Sniffles read names from VCF")
+    sv_assignment_df = assign_sv_celltypes(read_vcf_to_df(args.vcf, kanpig_read_names=args.kanpig_read_names), pd.read_csv(input_file, sep="\t", index_col=0))
+    sv_assignment_df.to_csv(os.path.join(args.output, "sv_assignment.tsv"), sep="\t", index=False)
+
+
+
+
+def anno_main(args):
+    # print(args)
+    # return
+    logger = logging.getLogger("anno.main")
+
+    bed_file   = args.bed
+    base_out   = args.output    # writes <output>.reads.tsv and <output>.blocks.tsv
+    input_file = args.input
+    reference  = args.reference
+    threads    = int(args.threads)
+    window     = int(args.window)
+    logger.info(f"Starting annotation: bed={bed_file} bam={input_file} ref={reference} "
+                f"threads={threads} out_base={base_out}")
+
+    # Output paths
+    reads_out  = os.path.join(base_out, "reads_classification.tsv")
+    blocks_out = os.path.join(base_out, "blocks_classification.tsv")
+
+    # Load and (optionally) filter BED
+    bed = pd.read_csv(bed_file, sep="\t")
+    logger.info(f"Loaded BED with {len(bed)} DMR rows")
+
+    sv_df = read_vcf_to_df(args.vcf)
+    filtered_bed = filter_bed_based_on_variants(bed, sv_df=sv_df, window=window)
+
+    for col in ["chr", "start", "end", "best_group", "best_dir"]:
+        if col not in filtered_bed.columns:
+            logger.error(f"BED missing required column: {col}")
+            raise ValueError(f"BED missing required column: {col}")
+
+    n_tasks = len(filtered_bed)
+    logger.info(f"Filtered BED to {n_tasks} DMRs after variant overlap filtering, window size = {window}")
+
+    tasks = [(dict(row), input_file, reference) for _, row in filtered_bed.iterrows()]
+
+    # --- Prepare outputs: truncate files and reset header flags ---
+    # We'll only write headers on the first real chunk for each file.
+    open(reads_out,  "w").close()
+    open(blocks_out, "w").close()
+    reads_header_written  = False
+    blocks_header_written = False
+    blocks_cols_locked: list[str] | None = None  # we lock schema to the first block we see
+
+    # Stream results and append immediately
+    with mp.Pool(threads) as pool:
+        for res in tqdm(pool.imap(_one_dmr, tasks, chunksize=1),
+                        total=n_tasks, desc="Processing DMRs"):
+            if res is None:
+                continue
+
+            a_df, s_df = res
+
+            # --- APPEND READS ---
+            if a_df is not None and not a_df.empty:
+                if not reads_header_written:
+                    # first write: include header and index (readname)
+                    a_df.to_csv(reads_out, sep="\t", index=True, mode="a", header=True)
+                    reads_header_written = True
+                else:
+                    a_df.to_csv(reads_out, sep="\t", index=True, mode="a", header=False)
+
+            # --- APPEND BLOCKS (variable columns across DMRs) ---
+            if s_df is not None and not s_df.empty:
+                if not blocks_header_written:
+                    # lock the schema to the first encountered block columns
+                    blocks_cols_locked = list(s_df.columns)
+                    s_df.to_csv(blocks_out, sep="\t", index=False, mode="a", header=True)
+                    blocks_header_written = True
+                else:
+                    # align columns to locked header; drop extras, add missing as NaN
+                    assert blocks_cols_locked is not None
+                    s_df_aligned = s_df.reindex(columns=blocks_cols_locked)
+                    s_df_aligned.to_csv(blocks_out, sep="\t", index=False, mode="a", header=False)
+
+    # If nothing was written, emit empty files with headers to be friendly downstream
+    if not reads_header_written:
+        empty_reads = pd.DataFrame(
+            columns=["chr","start","end","cpgstart","cpgend","code_order","code"]
+        )
+        empty_reads.index.name = "readname"
+        empty_reads.to_csv(reads_out, sep="\t", index=True, header=True)
+        logger.warning("No per-read assignments generated; wrote empty reads header only")
+
+    if not blocks_header_written:
+        pd.DataFrame(columns=["chr","start","end","cpgstart","cpgend"]).to_csv(
+            blocks_out, sep="\t", index=False, header=True
+        )
+        logger.warning("No block states generated; wrote empty blocks header only")
+    sv_anno(args)
+    logger.info("Annotation complete")
+
+

sniffcell-0.5.0/src/sniffcell/anno/filter_bed_based_on_variants.py

@@ -0,0 +1,85 @@
+import numpy as np
+import pandas as pd
+
+def filter_bed_based_on_variants(bed_df: pd.DataFrame, sv_df: pd.DataFrame, window: int = 5000) -> pd.DataFrame:
+    # 1) Normalize column names
+    bed = bed_df.copy() 
+    sv  = sv_df.rename(columns={'chr': 'chr'}).copy()
+    if 'supporting_reads' not in sv.columns:
+        exit("VCF DataFrame missing 'supporting_reads' column required for SV assignment.")
+    # 2) Normalize chromosome naming (strip or add 'chr' to match)
+    def _norm_chr(x):
+        x = str(x)
+        return x[3:] if x.startswith('chr') else x
+    bed['chr'] = bed['chr'].map(_norm_chr)
+    sv['chr']  = sv['chr'].map(_norm_chr)
+
+    # 3) Enforce integer dtype
+    bed[['start','end']] = bed[['start','end']].astype(np.int64)
+    sv[['ref_start','ref_end']] = sv[['ref_start','ref_end']].astype(np.int64)
+
+    # 4) Convert VCF (1-based inclusive) -> BED-style half-open
+    #    [ref_start-1, ref_end) in 0-based half-open. Equivalently:
+    #    start0 = ref_start - 1; end0_exclusive = ref_end
+    sv_start0 = (sv['ref_start'].to_numpy(np.int64) - 1)
+    sv_end0   = sv['ref_end'].to_numpy(np.int64)      # already exclusive after conversion
+
+    # Handle insertions robustly: if END < POS (some callers), clamp end to POS.
+    bad = sv_end0 < (sv_start0 + 1)
+    if np.any(bad):
+        sv_end0[bad] = sv_start0[bad] + 1
+
+    sv = sv.assign(_start=sv_start0, _end=sv_end0)
+
+    # 5) Optional: make chr categorical for faster groupbys
+    bed['chr'] = bed['chr'].astype('category')
+    sv['chr']  = sv['chr'].astype('category')
+
+    out_mask = np.zeros(len(bed), dtype=bool)
+
+    # Build per-chrom sorted arrays of SV starts/ends (already sorted? we still ensure monotonic)
+    sv_idx = {}
+    for chrom, sdf in sv.groupby('chr', sort=False, observed=False):
+        starts = sdf['_start'].to_numpy(np.int64)
+        ends   = sdf['_end'].to_numpy(np.int64)
+        # Ensure sorted (cheap if already sorted)
+        order = np.argsort(starts, kind='mergesort')
+        starts = starts[order]
+        ends   = ends[order]  # maintain pairing order
+        # We also want an array of ends sorted to use searchsorted independently.
+        ends_sorted = np.sort(ends, kind='mergesort')
+        sv_idx[chrom] = (starts, ends, ends_sorted)
+
+
+    for chrom, bdf in bed.groupby('chr', sort=False, observed=False):
+        if chrom not in sv_idx:
+            continue
+        starts, _, ends_sorted = sv_idx[chrom]
+        if starts.size == 0:
+            continue
+
+        # Core (original) BED interval
+        core_start = bdf['start'].to_numpy(np.int64)
+        core_end   = bdf['end'].to_numpy(np.int64)
+
+        # Padded interval
+        bed_starts = core_start - window
+        bed_ends   = core_end   + window
+
+        # ---------- 1) Overlap with *padded* interval (half-open, as before) ----------
+        n_start_lt_end_pad = np.searchsorted(starts,      bed_ends,   side='left')
+        n_end_le_start_pad = np.searchsorted(ends_sorted, bed_starts, side='right')
+        overlap_padded = (n_start_lt_end_pad - n_end_le_start_pad) > 0
+
+        # ---------- 2) Overlap with *core* interval (treat breakpoint as overlap) ------
+        # Closed-interval style: sv_start <= core_end AND sv_end >= core_start
+        n_start_le_end_core = np.searchsorted(starts,      core_end,   side='right')
+        n_end_lt_start_core = np.searchsorted(ends_sorted, core_start, side='left')
+        overlap_core = (n_start_le_end_core - n_end_lt_start_core) > 0
+
+        # ---------- 3) We want SV in padding, but NOT in core --------------------------
+        keep = overlap_padded & (~overlap_core)
+        out_mask[bdf.index] = keep
+
+
+    return bed_df.loc[out_mask]