deep-cnv 0.0.1__tar.gz

deep_cnv-0.0.1/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.3
+Name: deep-cnv
+Version: 0.0.1
+Summary: Deep learning for Copy Number Variation detection from SNP array data
+Requires-Dist: eir-dl>=0.25.0
+Requires-Dist: fastapi>=0.128.0
+Requires-Dist: numpy>=1.26.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: polars>=1.37.1
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: uvicorn>=0.40.0
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+

deep_cnv-0.0.1/pyproject.toml

@@ -0,0 +1,45 @@
+[project]
+name = "deep-cnv"
+version = "0.0.1"
+description = "Deep learning for Copy Number Variation detection from SNP array data"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "eir-dl>=0.25.0",
+    "fastapi>=0.128.0",
+    "numpy>=1.26.0",
+    "pandas>=2.0.0",
+    "polars>=1.37.1",
+    "pydantic>=2.12.5",
+    "pyyaml>=6.0.3",
+    "uvicorn>=0.40.0",
+]
+
+[project.scripts]
+deep-cnv-server = "deep_cnv.training.streaming_server:main"
+deep-cnv-preprocess = "deep_cnv.data_processing.run_preprocess:main"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0.0",
+    "ruff>=0.4.0",
+]
+
+[build-system]
+requires = ["uv_build>=0.12.19,<0.13"]
+build-backend = "uv_build"
+
+[tool.ruff]
+line-length = 100
+target-version = "py313"
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "F",
+    "I",
+    "UP",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

deep_cnv-0.0.1/pyproject.toml.orig

@@ -0,0 +1,40 @@
+[project]
+name = "deep-cnv"
+version = "0.0.1"
+description = "Deep learning for Copy Number Variation detection from SNP array data"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "eir-dl>=0.25.0",
+    "fastapi>=0.128.0",
+    "numpy>=1.26.0",
+    "pandas>=2.0.0",
+    "polars>=1.37.1",
+    "pydantic>=2.12.5",
+    "pyyaml>=6.0.3",
+    "uvicorn>=0.40.0",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0.0",
+    "ruff>=0.4.0",
+]
+
+[project.scripts]
+deep-cnv-server = "deep_cnv.training.streaming_server:main"
+deep-cnv-preprocess = "deep_cnv.data_processing.run_preprocess:main"
+
+[build-system]
+requires = ["uv_build>=0.12.19,<0.13"]
+build-backend = "uv_build"
+
+[tool.ruff]
+line-length = 100
+target-version = "py313"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

deep_cnv-0.0.1/src/deep_cnv/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

deep_cnv-0.0.1/src/deep_cnv/data_processing/data_utils.py

@@ -0,0 +1,38 @@
+import gzip
+from pathlib import Path
+
+import pandas as pd
+
+
+def load_cnvs(file_path: Path) -> pd.DataFrame:
+    df = pd.read_csv(file_path, sep="\t")
+    return df
+
+
+def read_tabix(
+    tabix_path: Path,
+    snppos: set[int] | None,
+) -> pd.DataFrame:
+    with gzip.open(tabix_path, "rb") as f:
+        df = pd.read_csv(
+            f,
+            sep="\t",
+            header=None,
+            names=["chr", "start", "end", "lrr_raw", "baf", "lrr_adj"],
+        )
+    if snppos is not None:
+        df = df[df["start"].isin(snppos)]
+
+    return df.dropna(subset=["lrr_adj", "baf"]).reset_index(drop=True)
+
+
+def load_samples_to_exclude_from_training(file_path: Path | None) -> set[str]:
+    if file_path is None:
+        return set()
+
+    if not file_path.exists():
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    df_test_set_to_skip = pd.read_csv(file_path, sep="\t")
+
+    return set(df_test_set_to_skip["sample_ID"])

deep_cnv-0.0.1/src/deep_cnv/data_processing/preprocess.py

@@ -0,0 +1,281 @@
+from concurrent.futures import ProcessPoolExecutor
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from eir.utils.logging import get_logger
+
+from deep_cnv.data_processing.data_utils import (
+    load_cnvs,
+    load_samples_to_exclude_from_training,
+    read_tabix,
+)
+from deep_cnv.data_processing.region_processing.job_logic import (
+    CnvJob,
+    NormalJob,
+    build_cnv_jobs,
+    build_custom_cn2_jobs,
+    build_random_normal_jobs,
+)
+from deep_cnv.data_processing.region_processing.regions import (
+    CnvRegion,
+    RegionResult,
+    extract_cnv_region,
+    extract_random_normal_region,
+    save_region,
+)
+from deep_cnv.data_processing.setup.config import DataProcessingConfig, DataSourceConfig
+
+logger = get_logger(name=__name__)
+
+
+def _extract_cnvs_from_sample(job: CnvJob) -> list[CnvRegion]:
+    try:
+        df_tabix = read_tabix(
+            tabix_path=job.tabix_path,
+            snppos=None,
+        )
+    except FileNotFoundError:
+        logger.error(f"Tabix file not found for sample {job.sample_id}: {job.tabix_path}")
+        return []
+
+    regions = []
+
+    for row in job.cnv_rows:
+        region = extract_cnv_region(
+            df_tabix=df_tabix,
+            chrom=int(row["chr"]),
+            cnv_start=int(row["start"]),
+            cnv_end=int(row["end"]),
+            cn_state=int(row["CN"]),
+            flank_snps=job.flank_snps,
+        )
+        if region is not None:
+            regions.append(region)
+    return regions
+
+
+def _extract_normal_regions_from_sample(job: NormalJob) -> list[CnvRegion]:
+    rng = np.random.default_rng(seed=job.seed)
+    try:
+        tabix_df = read_tabix(
+            tabix_path=job.tabix_path,
+            snppos=None,
+        )
+    except FileNotFoundError:
+        logger.error(f"Tabix file not found for sample {job.sample_id}: {job.tabix_path}.")
+        return []
+
+    regions = []
+
+    # informed CNV 2 case, these are visually confirmed to be 2
+    if job.cnv_rows is not None:
+        for row in job.cnv_rows:
+            region = extract_cnv_region(
+                df_tabix=tabix_df,
+                chrom=int(row["chr"]),
+                cnv_start=int(row["start"]),
+                cnv_end=int(row["end"]),
+                cn_state=2,
+                flank_snps=job.region_size // 2,
+            )
+
+            # this is a bit of a hack, but the streaming server expects cn_start and
+            # cn_end to be None for normal regions, i.e. we have this check
+            #  "cnv" if pd.notna(row["cnv_start"]) else "normal" there
+            if region is not None:
+                regions.append(replace(region, cnv_start=None, cnv_end=None))
+
+    # purely random case
+    for _ in range(job.normal_regions_per_sample):
+        assert job.cnv_rows is None
+        region = extract_random_normal_region(
+            tabix_df=tabix_df,
+            region_size=job.region_size,
+            rng=rng,
+        )
+        if region is not None:
+            regions.append(region)
+    return regions
+
+
+@dataclass
+class SourceData:
+    cnvs: pd.DataFrame
+    samples: pd.DataFrame
+    samples_to_exclude: set[str]
+    all_sample_ids: set[str]
+    cnv_ids: set[str]
+
+
+def set_up_source_data(data_config: DataSourceConfig) -> SourceData:
+    dc = data_config
+
+    cnvs = load_cnvs(file_path=dc.cnvs_file)
+    cnvs["PN"] = cnvs["sample_ID"].str.split("_").str[0]
+
+    samples = pd.read_csv(filepath_or_buffer=dc.samples_file, sep="\t")
+
+    samples_to_exclude = load_samples_to_exclude_from_training(file_path=dc.test_set_file)
+
+    logger.info(f"Will exclude {len(samples_to_exclude)} PNs from training")
+
+    samples_filtered = samples[~samples["PN"].isin(samples_to_exclude)]
+    cnvs_filtered = cnvs[~cnvs["PN"].isin(samples_to_exclude)]
+
+    logger.info(f"Samples after filtering: {len(samples_filtered)}")
+    logger.info(f"CNVs after filtering: {len(cnvs_filtered)}")
+
+    all_sample_ids = set(samples_filtered["PN"])
+    cnv_ids = set(cnvs_filtered["PN"].astype(str).unique())
+
+    logger.info(
+        f"Samples with CNV (PN in CNV file): {len(cnv_ids)}, total samples: {len(all_sample_ids)}"
+    )
+
+    return SourceData(
+        cnvs=cnvs_filtered,
+        samples=samples_filtered,
+        samples_to_exclude=samples_to_exclude,
+        all_sample_ids=all_sample_ids,
+        cnv_ids=cnv_ids,
+    )
+
+
+def run_cnv_multiprocessing_extraction(
+    n_workers: int,
+    cnv_jobs: list[CnvJob],
+    output_dir: Path,
+) -> list[RegionResult]:
+    region_idx = 0
+
+    records: list[RegionResult] = []
+
+    with ProcessPoolExecutor(max_workers=n_workers) as executor:
+        job_fn_mp_iter = zip(cnv_jobs, executor.map(_extract_cnvs_from_sample, cnv_jobs))
+
+        for job, regions in job_fn_mp_iter:
+            for region in regions:
+                region_id = f"cnv_{region_idx:06d}"
+                record = save_region(
+                    region=region,
+                    output_dir=output_dir / "train" / "cnv",
+                    region_id=region_id,
+                    sample_id=job.sample_id,
+                )
+                records.append(record)
+                region_idx += 1
+
+    logger.info(f"{region_idx} CNV regions saved")
+
+    return records
+
+
+def run_normal_multiprocessing_extraction(
+    n_workers: int,
+    normal_jobs: list[NormalJob],
+    output_dir: Path,
+) -> list[RegionResult]:
+    region_idx = 0
+
+    records: list[RegionResult] = []
+
+    with ProcessPoolExecutor(max_workers=n_workers) as executor:
+        job_fn_mp_iter = zip(
+            normal_jobs, executor.map(_extract_normal_regions_from_sample, normal_jobs)
+        )
+
+        for job, regions in job_fn_mp_iter:
+            for region in regions:
+                region_id = f"normal_{region_idx:06d}"
+                record = save_region(
+                    region=region,
+                    output_dir=output_dir / "train" / "normal",
+                    region_id=region_id,
+                    sample_id=job.sample_id,
+                )
+                records.append(record)
+                region_idx += 1
+
+    logger.info(f"{region_idx} normal regions saved")
+
+    return records
+
+
+def save_metadata(records: list[RegionResult], output_dir: Path) -> None:
+    df = pd.DataFrame(records)
+    df.to_csv(output_dir / "train" / "metadata.csv", index=False)
+    logger.info(f"Train: {len(records)} regions")
+
+    if len(df) > 0:
+        logger.info(f"cn_state counts:\n{df['cn_state'].value_counts().sort_index().to_string()}")
+
+
+def process_dataset(
+    output_dir: Path,
+    data_config: DataSourceConfig,
+    processing_config: DataProcessingConfig,
+) -> None:
+    dc = data_config
+    pc = processing_config
+
+    rng = np.random.default_rng(seed=pc.random_seed)
+
+    source_data = set_up_source_data(data_config=dc)
+    sd = source_data
+
+    (output_dir / "train" / "cnv").mkdir(parents=True, exist_ok=True)
+    (output_dir / "train" / "normal").mkdir(parents=True, exist_ok=True)
+
+    logger.info("Extracting CNV regions...")
+
+    id_to_path_map = dict(zip(sd.samples["PN"], sd.samples["file_path_tabix"]))
+    cnv_jobs = build_cnv_jobs(
+        cnvs=sd.cnvs,
+        id_to_path_map=id_to_path_map,
+        flank_snps=pc.flank_snps,
+    )
+
+    cnv_records = run_cnv_multiprocessing_extraction(
+        n_workers=pc.n_workers,
+        cnv_jobs=cnv_jobs,
+        output_dir=output_dir,
+    )
+
+    logger.info(
+        f"Extracting normal regions from {len(sd.all_sample_ids) - len(sd.cnv_ids)} samples..."
+    )
+    normal_sample_ids = sorted([sid for sid in sd.all_sample_ids if sid not in sd.cnv_ids])
+
+    random_jobs = build_random_normal_jobs(
+        sample_ids=normal_sample_ids,
+        id_to_path_map=id_to_path_map,
+        normal_regions_per_sample=pc.normal_regions_per_sample,
+        region_size=2 * pc.flank_snps,
+        rng=rng,
+    )
+    custom_jobs = build_custom_cn2_jobs(
+        id_to_path_map=id_to_path_map,
+        samples_to_exclude=sd.samples_to_exclude,
+        flank_snps=pc.flank_snps,
+        samples_file=dc.samples_file,
+        custom_normal_samples_file=dc.custom_normal_samples_file,
+    )
+    normal_jobs = random_jobs + custom_jobs
+
+    logger.info(
+        f"Built {len(random_jobs)} random normal jobs "
+        f"({pc.normal_regions_per_sample} regions each) "
+        f"and {len(custom_jobs)} custom CN2 jobs for multiprocessing..."
+    )
+
+    normal_records = run_normal_multiprocessing_extraction(
+        n_workers=pc.n_workers,
+        normal_jobs=normal_jobs,
+        output_dir=output_dir,
+    )
+
+    all_records = cnv_records + normal_records
+
+    save_metadata(records=all_records, output_dir=output_dir)

deep_cnv-0.0.1/src/deep_cnv/data_processing/region_processing/job_logic.py

@@ -0,0 +1,116 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from eir.utils.logging import get_logger
+
+logger = get_logger(name=__name__)
+
+
+@dataclass(frozen=True)
+class CnvJob:
+    sample_id: str
+    tabix_path: Path
+    cnv_rows: list[dict]
+    flank_snps: int
+
+
+def build_cnv_jobs(
+    cnvs: pd.DataFrame,
+    id_to_path_map: dict[str, str],
+    flank_snps: int,
+) -> list[CnvJob]:
+    jobs = [
+        CnvJob(
+            sample_id=str(sid),
+            tabix_path=Path(id_to_path_map[sid].replace(".tbi", "")),
+            cnv_rows=sample_cnvs.to_dict("records"),
+            flank_snps=flank_snps,
+        )
+        for sid, sample_cnvs in cnvs.groupby("PN")
+    ]
+
+    logger.info(f"Built {len(jobs)} CNV jobs for multiprocessing...")
+
+    return jobs
+
+
+@dataclass(frozen=True)
+class NormalJob:
+    sample_id: str
+    tabix_path: Path
+    cnv_rows: list[dict] | None
+    normal_regions_per_sample: int
+    region_size: int
+    seed: int | None
+
+
+def build_random_normal_jobs(
+    sample_ids: list[str],
+    id_to_path_map: dict[str, str],
+    normal_regions_per_sample: int,
+    region_size: int,
+    rng: np.random.Generator,
+) -> list[NormalJob]:
+
+    seeds = rng.integers(0, 2**31, size=len(sample_ids)).tolist()
+
+    random_normal_jobs = [
+        NormalJob(
+            sample_id=sid,
+            tabix_path=Path(id_to_path_map[sid].replace(".tbi", "")),
+            cnv_rows=None,
+            normal_regions_per_sample=normal_regions_per_sample,
+            region_size=region_size,
+            seed=int(seed),
+        )
+        for sid, seed in zip(sample_ids, seeds)
+    ]
+
+    return random_normal_jobs
+
+
+
+def build_custom_cn2_jobs(
+    id_to_path_map: dict[str, str],
+    samples_to_exclude: set[str],
+    flank_snps: int,
+    samples_file: Path,
+    custom_normal_samples_file: Path | None,
+) -> list[NormalJob]:
+    custom_file = custom_normal_samples_file
+
+    if custom_file is None:
+        logger.info("No custom CNV2 file passed in - will only use randomly drawn samples.")
+        return []
+
+    custom_cnv2_regions = pd.read_csv(custom_file, sep="\t")
+    custom_cnv2_regions["PN"] = custom_cnv2_regions["sample_ID"].str.split("_").str[0]
+
+    is_excluded = custom_cnv2_regions["PN"].isin(samples_to_exclude)
+    logger.info(f"Excluding {is_excluded.sum()} custom CNV2 regions belonging to test set PNs")
+
+    custom_cnv2_regions = custom_cnv2_regions[~is_excluded]
+
+    missing_pns = set(custom_cnv2_regions["PN"]) - set(id_to_path_map)
+    if missing_pns:
+        raise ValueError(
+            f"{len(missing_pns)} PNs in {custom_file} not found in {samples_file}: "
+            f"{sorted(missing_pns)[:10]}"
+        )
+
+    jobs = []
+    for sid, sample_cnv2_rows in custom_cnv2_regions.groupby("PN"):
+        jobs.append(
+            NormalJob(
+                sample_id=sid,
+                tabix_path=Path(id_to_path_map[sid].replace(".tbi", "")),
+                cnv_rows=sample_cnv2_rows.to_dict("records"),
+                normal_regions_per_sample=0,
+                region_size=2 * flank_snps,
+                seed=None,
+            )
+        )
+
+    return jobs

deep_cnv-0.0.1/src/deep_cnv/data_processing/region_processing/regions.py

@@ -0,0 +1,181 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+
+@dataclass
+class CnvRegion:
+    chrom: int
+    cn_state: int
+    signals: np.ndarray  # shape (N, 2): LRR, BAF
+    positions: np.ndarray  # shape (N,): genomic position per SNP
+    cnv_start: int | None = None  # None for normal (CN2) regions
+    cnv_end: int | None = None  # None for normal (CN2) regions
+
+
+def extract_cnv_region(
+    df_tabix: pd.DataFrame,
+    chrom: int,
+    cnv_start: int,
+    cnv_end: int,
+    cn_state: int,
+    flank_snps: int = 1000,
+    use_adjusted_lrr: bool = True,
+) -> CnvRegion | None:
+    chrom_df = df_tabix[df_tabix["chr"] == chrom].sort_values("start")
+    if chrom_df.empty:
+        return None
+
+    positions = chrom_df["start"].values
+
+    cnv_start_idx = np.searchsorted(positions, cnv_start)
+    cnv_end_idx = np.searchsorted(positions, cnv_end, side="right")
+
+    region_start_idx = max(0, cnv_start_idx - flank_snps)
+    region_end_idx = min(len(positions), cnv_end_idx + flank_snps)
+
+    region_df = chrom_df.iloc[region_start_idx:region_end_idx]
+    lrr_col = "lrr_adj" if use_adjusted_lrr else "lrr_raw"
+    signals = region_df[[lrr_col, "baf"]].values.astype(np.float32)
+
+    return CnvRegion(
+        chrom=chrom,
+        cn_state=cn_state,
+        signals=signals,
+        positions=region_df["start"].values,
+        cnv_start=cnv_start,
+        cnv_end=cnv_end,
+    )
+
+
+def extract_random_normal_region(
+    tabix_df: pd.DataFrame,
+    region_size: int,
+    rng: np.random.Generator,
+    use_adjusted_lrr: bool = True,
+) -> CnvRegion | None:
+    if tabix_df.empty:
+        return None
+
+    chromosomes = tabix_df["chr"].unique()
+    chrom = int(rng.choice(chromosomes))
+
+    chrom_df = tabix_df[tabix_df["chr"] == chrom].sort_values("start")
+    if len(chrom_df) < region_size:
+        return None
+
+    max_start_idx = len(chrom_df) - region_size
+    start_idx = int(rng.integers(0, max_start_idx + 1))
+
+    region_df = chrom_df.iloc[start_idx : start_idx + region_size]
+    lrr_col = "lrr_adj" if use_adjusted_lrr else "lrr_raw"
+    signals = region_df[[lrr_col, "baf"]].values.astype(np.float32)
+
+    return CnvRegion(
+        chrom=chrom,
+        cn_state=2,
+        signals=signals,
+        positions=region_df["start"].values,
+    )
+
+
+def load_region(region_path: Path, metadata_row: dict) -> CnvRegion:
+    data = np.load(file=region_path)
+    cnv_start = metadata_row["cnv_start"]
+    cnv_end = metadata_row["cnv_end"]
+
+    cnv_start = int(cnv_start) if cnv_start is not None and not np.isnan(cnv_start) else None
+    cnv_end = int(cnv_end) if cnv_end is not None and not np.isnan(cnv_end) else None
+
+    return CnvRegion(
+        chrom=int(metadata_row["chrom"]),
+        cn_state=int(metadata_row["cn_state"]),
+        signals=data["signals"],
+        positions=data["positions"],
+        cnv_start=cnv_start,
+        cnv_end=cnv_end,
+    )
+
+
+def sample_window(
+    region: CnvRegion,
+    window_size: int,
+    rng: np.random.Generator,
+) -> tuple[np.ndarray, np.ndarray] | None:
+    if len(region.positions) < window_size:
+        return None
+
+    max_start = len(region.positions) - window_size
+    start_idx = int(rng.integers(0, max_start + 1))
+
+    return (
+        region.signals[start_idx : start_idx + window_size],
+        region.positions[start_idx : start_idx + window_size],
+    )
+
+
+def make_labels(
+    positions: np.ndarray,
+    cnv_start: int | None,
+    cnv_end: int | None,
+    cn_state: int,
+) -> np.ndarray:
+    labels = np.full(len(positions), 2, dtype=np.int8)
+
+    if cnv_start is not None and cnv_end is not None:
+        mask = (positions >= cnv_start) & (positions <= cnv_end)
+        labels[mask] = cn_state
+
+    return labels
+
+
+@dataclass
+class RegionResult:
+    region_id: str
+    sample_id: str
+    chrom: int
+    cnv_start: int | None
+    cnv_end: int | None
+    cn_state: int
+    n_snps: int
+    n_flank_left: int | None
+    n_flank_right: int | None
+
+
+def save_region(
+    region: CnvRegion,
+    output_dir: Path,
+    region_id: str,
+    sample_id: str,
+) -> RegionResult:
+    np.savez_compressed(
+        output_dir / f"{region_id}.npz",
+        signals=region.signals,
+        positions=region.positions,
+    )
+
+    if region.cnv_start is not None and region.cnv_end is not None:
+        n_flank_left = int(np.searchsorted(region.positions, region.cnv_start))
+        n_flank_right = int(
+            len(region.positions) - np.searchsorted(region.positions, region.cnv_end, side="right")
+        )
+
+    else:
+        n_flank_left = None
+        n_flank_right = None
+
+    result = RegionResult(
+        region_id=region_id,
+        sample_id=sample_id,
+        chrom=region.chrom,
+        cnv_start=region.cnv_start,
+        cnv_end=region.cnv_end,
+        cn_state=region.cn_state,
+        n_snps=len(region.positions),
+        n_flank_left=n_flank_left,
+        n_flank_right=n_flank_right,
+    )
+
+    return result

deep_cnv-0.0.1/src/deep_cnv/data_processing/run_preprocess.py

@@ -0,0 +1,24 @@
+import argparse
+from pathlib import Path
+
+from deep_cnv.data_processing.preprocess import process_dataset
+from deep_cnv.data_processing.setup.config import load_data_config, load_processing_config
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data-config", type=Path, required=True)
+    parser.add_argument("--processing-config", type=Path, required=True)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    args = parser.parse_args()
+
+    data_config = load_data_config(config_path=args.data_config)
+    processing_config = load_processing_config(config_path=args.processing_config)
+
+    process_dataset(
+        output_dir=args.output_dir, data_config=data_config, processing_config=processing_config
+    )
+
+
+if __name__ == "__main__":
+    main()

deep_cnv-0.0.1/src/deep_cnv/data_processing/setup/config.py

@@ -0,0 +1,19 @@
+from yaml import safe_load
+
+from deep_cnv.data_processing.setup.schemas import DataProcessingConfig, DataSourceConfig
+
+
+def load_data_config(config_path: str) -> DataSourceConfig:
+    with open(config_path) as f:
+        config_dict = safe_load(
+            stream=f,
+        )
+
+    return DataSourceConfig(**config_dict)
+
+
+def load_processing_config(config_path: str) -> DataProcessingConfig:
+    with open(config_path) as f:
+        config_dict = safe_load(stream=f)
+
+    return DataProcessingConfig(**config_dict)

deep_cnv-0.0.1/src/deep_cnv/data_processing/setup/schemas.py

@@ -0,0 +1,32 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class DataSourceConfig:
+    cnvs_file: Path
+    samples_file: Path
+    test_set_file: Path | None = None
+    custom_normal_samples_file: Path | None = None
+
+    def __post_init__(self) -> None:
+        str_to_path_keys = [
+            "cnvs_file",
+            "samples_file",
+            "test_set_file",
+            "custom_normal_samples_file",
+        ]
+        for key in str_to_path_keys:
+            value = getattr(self, key)
+            if isinstance(value, str):
+                setattr(self, key, Path(value))
+
+
+@dataclass
+class DataProcessingConfig:
+    """ """
+
+    flank_snps: int = 1000
+    normal_regions_per_sample: int = 3
+    random_seed: int = 42
+    n_workers: int = 8

deep_cnv-0.0.1/src/deep_cnv/inference/predict.py

@@ -0,0 +1,66 @@
+import base64
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import requests
+
+from deep_cnv.data_processing.region_processing.regions import CnvRegion, load_region
+
+
+def _encode_array(array: np.ndarray) -> str:
+    return base64.b64encode(array.tobytes()).decode("utf-8")
+
+
+def _decode_array(encoded: str, dtype: np.dtype, shape: tuple) -> np.ndarray:
+    return np.frombuffer(base64.b64decode(encoded), dtype=dtype).reshape(shape)
+
+
+def predict(
+    signals_batch: list[np.ndarray],
+    url: str = "http://localhost:8000/predict",
+    output_name: str = "snp_signals",
+) -> list[np.ndarray]:
+    payload = []
+    for signals in signals_batch:
+        payload.append(
+            {
+                "snp_signals": _encode_array(array=signals),
+            }
+        )
+
+    response = requests.post(url=url, json=payload)
+    response.raise_for_status()
+    results = response.json()["result"]
+
+    predictions = []
+    for item in results:
+        raw = item[output_name]
+        n_bytes = len(base64.b64decode(raw))
+        n_floats = n_bytes // 4
+        window_size = signals_batch[0].shape[-1]
+        if n_floats == 5 * window_size:
+            shape = (5, window_size)
+        elif n_floats == 5 * 1 * window_size:
+            shape = (5, 1, window_size)
+        else:
+            raise ValueError(
+                f"Unexpected prediction size: {n_floats} floats (window_size={window_size})"
+            )
+        arr = _decode_array(encoded=raw, dtype=np.float32, shape=shape)
+        predictions.append(arr)
+
+    return predictions
+
+
+def load_regions_with_metadata(regions_dir: Path) -> list[tuple[CnvRegion, dict]]:
+    metadata = pd.read_csv(regions_dir / "metadata.csv")
+    regions = []
+    for _, row in metadata.iterrows():
+        subdir = "cnv" if pd.notna(row["cnv_start"]) else "normal"
+        path = regions_dir / subdir / f"{row['region_id']}.npz"
+        if not path.exists():
+            continue
+        region = load_region(region_path=path, metadata_row=row.to_dict())
+        regions.append((region, row.to_dict()))
+    return regions

deep_cnv-0.0.1/src/deep_cnv/inference/predict_batched.py

@@ -0,0 +1,9 @@
+_config: dict = {
+    "serve_url": "http://localhost:8001/predict",
+}
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--serve-url", type=str, default=_config["serve_url"])

deep_cnv-0.0.1/src/deep_cnv/training/data_augmentation.py

@@ -0,0 +1,70 @@
+import numpy as np
+
+
+def augment_signals(
+    signals: np.ndarray,
+    rng: np.random.Generator,
+    lrr_shift_std: float = 0.05,
+    lrr_noise_std: float = 0.03,
+    lrr_scale_range: tuple[float, float] = (0.85, 1.15),
+    baf_jitter_std: float = 0.015,
+    probe_dropout_rate: float = 0.02,
+    lrr_mask_rate: float = 0.0,
+    baf_mask_rate: float = 0.0,
+) -> np.ndarray:
+    signals = signals.copy()
+    n = len(signals)
+    lrr = signals[:, 0]
+    baf = signals[:, 1]
+
+    if lrr_mask_rate > 0 and rng.random() < lrr_mask_rate:
+        lrr[:] = rng.normal(scale=0.15, size=n).astype(np.float32)
+    else:
+        lrr += rng.normal(scale=lrr_shift_std)
+        lrr += rng.normal(scale=lrr_noise_std, size=n).astype(np.float32)
+        lrr *= rng.uniform(low=lrr_scale_range[0], high=lrr_scale_range[1])
+
+    if baf_mask_rate > 0 and rng.random() < baf_mask_rate:
+        baf[:] = rng.uniform(low=0.0, high=1.0, size=n).astype(np.float32)
+    else:
+        baf += rng.normal(scale=baf_jitter_std, size=n).astype(np.float32)
+        np.clip(baf, 0.0, 1.0, out=baf)
+
+    if probe_dropout_rate > 0:
+        drop_mask = rng.random(size=n) < probe_dropout_rate
+        lrr[drop_mask] = 0.0
+        baf[drop_mask] = 0.5
+
+    return signals
+
+
+def inject_fake_lrr_bump(
+    signals: np.ndarray,
+    rng: np.random.Generator,
+    min_snps: int = 50,
+    max_snps: int = 300,
+    shift_range: tuple[float, float] = (0.1, 0.4),
+    taper_snps: int = 10,
+) -> np.ndarray:
+    signals = signals.copy()
+    n = len(signals)
+    lrr = signals[:, 0]
+
+    width = rng.integers(low=min_snps, high=min(max_snps, n) + 1)
+    start = rng.integers(low=0, high=n - width + 1)
+    end = start + width
+
+    shift = rng.uniform(low=shift_range[0], high=shift_range[1])
+
+    bump = np.zeros(n, dtype=np.float32)
+    bump[start:end] = shift
+
+    left_taper = min(taper_snps, width // 2)
+    right_taper = min(taper_snps, width // 2)
+    if left_taper > 0:
+        bump[start : start + left_taper] *= np.linspace(0, 1, left_taper, dtype=np.float32)
+    if right_taper > 0:
+        bump[end - right_taper : end] *= np.linspace(1, 0, right_taper, dtype=np.float32)
+
+    lrr += bump
+    return signals

deep_cnv-0.0.1/src/deep_cnv/training/streaming_server.py

@@ -0,0 +1,375 @@
+import argparse
+import base64
+from collections.abc import Iterator
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from threading import Lock
+
+import numpy as np
+import polars as pl
+import uvicorn
+from eir.setup.streaming_data_setup.protocol import PROTOCOL_VERSION
+from eir.utils.logging import get_logger
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+from pydantic import BaseModel
+
+from deep_cnv.data_processing.region_processing.regions import (
+    CnvRegion,
+    load_region,
+    make_labels,
+    sample_window,
+)
+from deep_cnv.training.data_augmentation import augment_signals, inject_fake_lrr_bump
+
+logger = get_logger(name=__name__)
+
+app = FastAPI()
+
+
+@dataclass
+class LoadArgs:
+    region_id: str
+    region_path: Path
+    metadata_row: dict
+    window_size: int
+
+
+def _load_region_entry(load_args: LoadArgs) -> tuple[str, CnvRegion] | None:
+    la = load_args
+
+    if not la.region_path.exists():
+        return None
+
+    loaded_region = load_region(region_path=la.region_path, metadata_row=la.metadata_row)
+
+    if len(loaded_region.positions) < la.window_size:
+        return None
+
+    return (la.region_id, loaded_region)
+
+
+def _build_load_args(
+    df_metadata: pl.DataFrame, regions_dir: Path, window_size: int
+) -> Iterator[LoadArgs]:
+    for row in df_metadata.iter_rows(named=True):
+        region_id = row["region_id"]
+        cnv_start = row["cnv_start"]
+
+        region_folder = "cnv" if cnv_start is not None else "normal"
+        region_path = regions_dir / region_folder / f"{region_id}.npz"
+
+        yield LoadArgs(
+            region_id=region_id,
+            region_path=region_path,
+            metadata_row=row,
+            window_size=window_size,
+        )
+
+
+class RegionDataset:
+    def __init__(
+        self,
+        regions_dir: Path,
+        window_size: int = 1024,
+        cnv_fraction: float = 0.7,
+        seed: int = 42,
+        n_workers: int = 4,
+    ) -> None:
+        self.window_size = window_size
+        self.cnv_fraction = cnv_fraction
+        self.rng = np.random.default_rng(seed=seed)
+        self._lock = Lock()
+        self.position = 0
+        self.validation_ids: set[str] = set()
+
+        logger.info(f"Loading regions from {regions_dir}...")
+        metadata = pl.read_csv(source=regions_dir / "metadata.csv")
+
+        load_args_iterable = _build_load_args(
+            df_metadata=metadata,
+            regions_dir=regions_dir,
+            window_size=window_size,
+        )
+        load_args = list(load_args_iterable)
+
+        self.cnv_regions = []
+        self.normal_regions = []
+        with ThreadPoolExecutor(max_workers=n_workers) as executor:
+            for entry in executor.map(_load_region_entry, load_args):
+                if entry is None:
+                    continue
+                _, region = entry
+                if region.cnv_start is not None:
+                    self.cnv_regions.append(entry)
+                else:
+                    self.normal_regions.append(entry)
+
+        skipped = sum(1 for a in load_args if not a.region_path.exists())
+        logger.info(
+            f"Loaded {len(self.cnv_regions)} CNV + {len(self.normal_regions)} normal regions "
+            f"({skipped} skipped), {len(metadata)} total in metadata"
+        )
+
+    def reset(self) -> None:
+        with self._lock:
+            self.position = 0
+
+    def get_batch(self, batch_size: int) -> list[dict]:
+        batch = []
+        attempts = 0
+        max_attempts = batch_size * 10
+
+        with self._lock:
+            while len(batch) < batch_size and attempts < max_attempts:
+                attempts += 1
+                sample_id = f"sample_{self.position}"
+                self.position += 1
+
+                if sample_id in self.validation_ids:
+                    continue
+
+                if self.rng.random() < self.cnv_fraction:
+                    pool = self.cnv_regions
+                else:
+                    pool = self.normal_regions
+                region_id, region = pool[int(self.rng.integers(0, len(pool)))]
+
+                result = sample_window(
+                    region=region,
+                    window_size=self.window_size,
+                    rng=self.rng,
+                )
+                if result is None:
+                    continue
+
+                signals, positions = result
+                labels = make_labels(
+                    positions=positions,
+                    cnv_start=region.cnv_start,
+                    cnv_end=region.cnv_end,
+                    cn_state=region.cn_state,
+                )
+
+                signals = signals.astype(np.float32)
+
+                if self.rng.random() < 0.5:
+                    signals = signals[::-1].copy()
+                    labels = labels[::-1].copy()
+
+                signals = augment_signals(
+                    signals=signals,
+                    rng=self.rng,
+                    lrr_mask_rate=_config["lrr_mask_rate"],
+                    baf_mask_rate=_config["baf_mask_rate"],
+                )
+
+                fake_dup_rate = _config["fake_dup_rate"]
+                is_cn2 = np.all(labels == 2)
+                if fake_dup_rate > 0 and is_cn2 and self.rng.random() < fake_dup_rate:
+                    signals = inject_fake_lrr_bump(
+                        signals=signals,
+                        rng=self.rng,
+                    )
+
+                labels = labels.astype(np.float32)
+                labels_encoded = base64.b64encode(labels.tobytes()).decode("utf-8")
+
+                combined = np.transpose(signals, (1, 0))[:, np.newaxis, :]
+                inputs = {
+                    "snp_signals": base64.b64encode(combined.tobytes()).decode("utf-8"),
+                }
+
+                output_name = _config["output_name"]
+                batch.append(
+                    {
+                        "inputs": inputs,
+                        "target_labels": {
+                            output_name: {
+                                output_name: labels_encoded,
+                            },
+                        },
+                        "sample_id": sample_id,
+                    }
+                )
+
+        return batch
+
+
+# ---------------------------------------------------------------------------
+# Protocol models
+# ---------------------------------------------------------------------------
+
+
+class InputInfo(BaseModel):
+    type: str
+    shape: list[int] | None = None
+
+
+class OutputInfo(BaseModel):
+    type: str
+    shape: list[int] | None = None
+
+
+class DatasetInfo(BaseModel):
+    inputs: dict[str, InputInfo]
+    outputs: dict[str, OutputInfo]
+
+
+# ---------------------------------------------------------------------------
+# Server
+# ---------------------------------------------------------------------------
+
+
+_config: dict = {
+    "regions_dir": Path("data", "real", "regions", "train"),
+    "window_size": 1024,
+    "cnv_fraction": 0.7,
+    "output_name": "snp_signals",
+    "lrr_mask_rate": 0.0,
+    "baf_mask_rate": 0.0,
+    "fake_dup_rate": 0.0,
+    "n_workers": 4,
+}
+dataset: RegionDataset | None = None
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global dataset
+    dataset = RegionDataset(
+        regions_dir=_config["regions_dir"],
+        window_size=_config["window_size"],
+        cnv_fraction=_config["cnv_fraction"],
+        n_workers=_config["n_workers"],
+    )
+    yield
+
+
+app = FastAPI(lifespan=lifespan)
+
+
+async def _handshake(websocket: WebSocket) -> bool:
+    await websocket.accept()
+    msg = await websocket.receive_json()
+    if msg.get("type") != "handshake" or msg.get("version") != PROTOCOL_VERSION:
+        await websocket.send_json(
+            {"type": "error", "payload": {"message": "Incompatible protocol version"}}
+        )
+        await websocket.close()
+        return False
+    await websocket.send_json({"type": "handshake", "version": PROTOCOL_VERSION})
+    return True
+
+
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket) -> None:
+    if not await _handshake(websocket=websocket):
+        return
+
+    try:
+        while True:
+            data = await websocket.receive_json()
+            msg_type = data.get("type")
+
+            if msg_type == "getInfo":
+                window_size = _config["window_size"]
+                output_name = _config["output_name"]
+
+                inputs = {
+                    "snp_signals": InputInfo(type="array", shape=[2, 1, window_size]),
+                }
+
+                info = DatasetInfo(
+                    inputs=inputs,
+                    outputs={output_name: OutputInfo(type="array", shape=[1, 1, window_size])},
+                )
+
+                await websocket.send_json({"type": "info", "payload": info.model_dump()})
+
+            elif msg_type == "getData":
+                batch_size = data.get("payload", {}).get("batch_size", 32)
+                batch = dataset.get_batch(batch_size=batch_size)
+
+                if not batch:
+                    await websocket.send_json({"type": "data", "payload": ["terminate"]})
+                    break
+
+                await websocket.send_json({"type": "data", "payload": batch})
+
+            elif msg_type == "reset":
+                dataset.reset()
+
+                await websocket.send_json(
+                    {"type": "resetConfirmation", "payload": {"message": "Reset successful"}}
+                )
+
+                await websocket.send_json(
+                    {"type": "reset", "payload": {"message": "Reset command received"}}
+                )
+
+            elif msg_type == "setValidationIds":
+                validation_ids = data.get("payload", {}).get("validation_ids", [])
+                dataset.validation_ids = set(validation_ids)
+
+                await websocket.send_json(
+                    {
+                        "type": "validationIdsConfirmation",
+                        "payload": {"message": f"Received {len(validation_ids)} validation IDs"},
+                    }
+                )
+
+            elif msg_type == "status":
+                await websocket.send_json(
+                    {
+                        "type": "status",
+                        "payload": {"current_position": dataset.position},
+                    }
+                )
+
+            elif msg_type == "heartbeat":
+                await websocket.send_json({"type": "heartbeat"})
+
+            else:
+                logger.warning(f"Unknown message type: {msg_type}")
+
+    except WebSocketDisconnect:
+        logger.info("Client disconnected")
+    finally:
+        dataset.reset()
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", default="0.0.0.0")
+    parser.add_argument("--port", type=int, default=8000)
+    parser.add_argument("--regions-dir", type=Path, default=_config["regions_dir"])
+    parser.add_argument("--window-size", type=int, default=_config["window_size"])
+    parser.add_argument("--cnv-fraction", type=float, default=_config["cnv_fraction"])
+    parser.add_argument("--output-name", type=str, default=_config["output_name"])
+    parser.add_argument("--lrr-mask-rate", type=float, default=_config["lrr_mask_rate"])
+    parser.add_argument("--baf-mask-rate", type=float, default=_config["baf_mask_rate"])
+    parser.add_argument("--fake-dup-rate", type=float, default=_config["fake_dup_rate"])
+    parser.add_argument("--n-workers", type=int, default=_config["n_workers"])
+    args = parser.parse_args()
+
+    _config["regions_dir"] = args.regions_dir
+    _config["window_size"] = args.window_size
+    _config["cnv_fraction"] = args.cnv_fraction
+    _config["output_name"] = args.output_name
+    _config["lrr_mask_rate"] = args.lrr_mask_rate
+    _config["baf_mask_rate"] = args.baf_mask_rate
+    _config["fake_dup_rate"] = args.fake_dup_rate
+    _config["n_workers"] = args.n_workers
+
+    uvicorn.run(app=app, host=args.host, port=args.port, ws_ping_timeout=3600)
+
+
+if __name__ == "__main__":
+    main()