embkit 0.1.0__tar.gz → 0.2__tar.gz

embkit-0.2/PKG-INFO

@@ -0,0 +1,69 @@
+Metadata-Version: 2.4
+Name: embkit
+Version: 0.2
+Summary: Embedding Kit for embedding models
+Author-email: Kyle Ellrott <ellrott@ohsu.edu>, Raphael Kirchgaessner <kirchgae@ohsu.edu>
+License: MIT
+Keywords: embeddings,machine learning,deep learning
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: torch>=2.0
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: click
+Requires-Dist: scikit-learn
+Requires-Dist: scipy
+Requires-Dist: gripql
+Requires-Dist: tqdm
+Requires-Dist: biopython
+Requires-Dist: fair-esm
+
+
+# Embedding Kit
+
+Methods for data normalization, embedding, synthesis and transformation
+
+
+## Installation
+```
+pip install embkit
+```
+
+# Training a model
+```
+embkit model train-vae ./experiments/tcga/tumor.normalized.tsv --epochs 120
+```
+
+## Development
+
+To install the library locally use:  
+```pip install -e .```  
+```python setup.py build```  
+```python setup.py install```  
+
+### To run tests use:
+```bash
+coverage run --source=embkit -m unittest discover -s tests
+```
+
+To generate a coverage report use:
+```bash
+coverage html
+```
+
+To open the coverage report in a browser, run:
+#### MacOS:
+```bash
+open htmlcov/index.html
+```
+
+#### Linux:
+```bash
+xdg-open htmlcov/index.html
+```
+
+#### Windows:
+```bash
+start htmlcov\index.html
+```
+

embkit-0.2/README.md

@@ -0,0 +1,49 @@
+
+# Embedding Kit
+
+Methods for data normalization, embedding, synthesis and transformation
+
+
+## Installation
+```
+pip install embkit
+```
+
+# Training a model
+```
+embkit model train-vae ./experiments/tcga/tumor.normalized.tsv --epochs 120
+```
+
+## Development
+
+To install the library locally use:  
+```pip install -e .```  
+```python setup.py build```  
+```python setup.py install```  
+
+### To run tests use:
+```bash
+coverage run --source=embkit -m unittest discover -s tests
+```
+
+To generate a coverage report use:
+```bash
+coverage html
+```
+
+To open the coverage report in a browser, run:
+#### MacOS:
+```bash
+open htmlcov/index.html
+```
+
+#### Linux:
+```bash
+xdg-open htmlcov/index.html
+```
+
+#### Windows:
+```bash
+start htmlcov\index.html
+```
+

{embkit-0.1.0 → embkit-0.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "embkit"
-version = "0.1.0"
+version = "0.2"
 description = "Embedding Kit for embedding models"
 readme = "README.md"
 requires-python = ">=3.8"
@@ -19,11 +19,19 @@ dependencies = [
     "pandas",
     "numpy",
     "click",
-    "scikit-learn"
+    "scikit-learn",
+    "scipy",
+    "gripql",
+    "tqdm",
+    "biopython",
+    "fair-esm"
 ]
 
 [tool.setuptools]
 package-dir = { "" = "src" }
 
 [tool.setuptools.packages.find]
-where = ["src"]
+where = ["src"]
+
+[project.scripts]
+embkit = "embkit.__main__:cli"

embkit-0.2/src/embkit/__init__.py

@@ -0,0 +1,62 @@
+"""
+Embedding Kit
+
+The base module contains frequently used methods and functions
+"""
+import logging
+import torch
+import pandas as pd
+import numpy as np
+from torch.utils.data import TensorDataset, DataLoader
+
+logger = logging.getLogger(__name__)
+
+def get_device() -> torch.device:
+    """
+    get_device - scan system and return default device
+    """
+    if torch.cuda.is_available():
+        logger.info("Using CUDA GPU")
+        return torch.device("cuda")
+    if torch.backends.mps.is_available():
+        logger.info("Using Metal GPU")
+        return torch.device("mps")
+    logger.info("Using CPU")
+    return torch.device("cpu")
+
+def dataframe_tensor(df: pd.DataFrame,device=None) -> torch.Tensor:
+    """
+    dataframe_tensor
+
+    take a pandas dataframe, create a tensor    
+    """
+    if device is None:
+        device = get_device()
+    return torch.from_numpy(df.values.astype(np.float32)).to(device)
+
+def tensor_dataframe(tn: torch.Tensor, index=None, columns=None) -> pd.DataFrame:
+    """
+    dataframe_tensor
+
+    take a torch tensor, create a pandas dataframe
+    """
+    return pd.DataFrame(tn.cpu().detach().numpy(), index=index, columns=columns)
+
+def dataframe_loader(df: pd.DataFrame,
+                     batch_size = 256, shuffle=True,
+                     device=None) -> torch.utils.data.DataLoader:
+    """
+    dataframe_loader
+
+    :param df: Pandas Dataframe to be converted into passed to DataLoader
+    :param batch_size: value to be passed to DataLoader batch_size
+    :param shuffle: value to be passed to DataLoader shuffle
+    :param device: Device the where tensor should be moved. If None, use get_device 
+    """
+
+    if device is None:
+        device = get_device()
+    x = torch.from_numpy(df.values.astype(np.float32)).to(device)
+    dataset = TensorDataset(x)
+    data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle)
+    return data_loader

embkit-0.2/src/embkit/__main__.py

@@ -0,0 +1,81 @@
+import click
+import logging
+
+from .commands import model, matrix, cbio_cmd, protein, datasets, align
+
+@click.group(invoke_without_command=True)
+@click.option('--verbose', is_flag=True, help="Enable verbose logging")
+@click.pass_context
+def cli(ctx, verbose):
+    """Embedding Kit CLI.
+
+    Use 'embkit help [COMMAND]' for details on a specific command.
+    """
+    # Set up logging based on the verbose flag
+    logging.basicConfig(
+        level=logging.DEBUG if verbose else logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(message)s"
+    )
+
+    ctx.ensure_object(dict)
+    ctx.obj["VERBOSE"] = verbose
+
+    if ctx.invoked_subcommand is None:
+        click.echo(ctx.get_help())
+
+
+# Register our commands here
+cli.add_command(model)
+cli.add_command(matrix)
+cli.add_command(cbio_cmd)
+cli.add_command(protein)
+cli.add_command(datasets)
+cli.add_command(align)
+
+
+# ---- 'help' command ----
+@cli.command("help", context_settings=dict(ignore_unknown_options=True))
+@click.argument("path", nargs=-1)
+@click.pass_context
+def help_cmd(ctx, path):
+    """
+    Show help for the CLI or a specific command.
+
+    Examples:
+      embkit help
+      embkit help model
+      embkit help model train
+    """
+    # No args -> show top-level help plus a neat command list summary
+    if not path:
+        click.echo(ctx.parent.get_help())
+        click.echo("\nCommands:")
+        # Sorted, with 1-line summaries
+        for name in sorted(cli.commands):
+            cmd = cli.commands[name]
+            summary = (cmd.help or cmd.short_help or "").strip().splitlines()[0]
+            click.echo(f"  {name:15s} {summary}")
+        return
+
+    # Resolve dotted/space-separated path to a nested command
+    cmd = cli
+    info_name = []
+    parent = ctx.parent  # start from top-level context
+    for part in path:
+        if not hasattr(cmd, "get_command"):
+            click.echo(cli.get_help(ctx)) # pragma: no cover
+        nxt_cmd = cmd.get_command(parent, part)
+        if nxt_cmd is None:
+            raise click.UsageError(f"Unknown command: {' '.join(path)}")
+        info_name.append(part)
+        cmd = nxt_cmd
+        parent = click.Context(cmd, info_name=part, parent=parent)
+
+    # Print help for the resolved command
+    click.echo(cmd.get_help(parent))
+
+
+cli_main = cli
+
+if __name__ == "__main__": # pragma: no cover
+    cli()

embkit-0.2/src/embkit/align.py

@@ -0,0 +1,151 @@
+import numpy as np
+import pandas as pd
+from scipy.stats import spearmanr
+from scipy.optimize import linear_sum_assignment
+from hopcroftkarp import HopcroftKarp
+
+
+def calc_rmsd(array1, array2):
+    """
+    Calculates the Root Mean Square Deviation (RMSD) between two arrays.
+    Parameters:
+        array1 (numpy.ndarray): The first array.
+        array2 (numpy.ndarray): The second array, must be of the same length as array1.
+
+    Returns:
+        float: The RMSD between the two arrays.
+    """
+    if len(array1) != len(array2):
+        raise ValueError("Arrays must be of the same length")
+    diff = array1 - array2
+    squared_diff = diff ** 2
+    mean_squared_diff = np.mean(squared_diff)
+    rmsd = np.sqrt(mean_squared_diff)
+
+    return rmsd
+
+
+def matrix_spearman_alignment_linear(a, b, cutoff=0.0):
+    """
+    matrix_spearman_alignment_linear
+
+    Uses the linear_sum_assignment algorithm to create an optimal
+    mapping between two matrices by maximizing the total Spearman
+    correlation score.
+    """
+    # Identify overlapping columns
+    isect = a.columns.intersection(b.columns)
+
+    combined_df = pd.concat([a[isect], b[isect]], axis=0)
+    o = spearmanr(combined_df, axis=1)
+
+    a_count = a.shape[0]
+    sdf = pd.DataFrame(o.correlation[a_count:, :a_count], index=b.index, columns=a.index)
+
+    rows, cols = linear_sum_assignment(-sdf)
+
+    out_a = []
+    out_b = []
+    out_score = []
+
+    for r, c in zip(rows, cols):
+        score = sdf.iloc[r, c]
+        if score >= cutoff:
+            out_a.append(a.index[c])
+            out_b.append(b.index[r])
+            out_score.append(score)
+
+    return out_a, out_b, out_score
+
+
+def matrix_spearman_alignment_hopkraft(a, b, cutoff=0.0):
+    """
+    matrix_spearman_alignment_hopkraft
+
+    Uses the HopcroftKarp maximum *cardinality* matching algorithm.
+    It finds the LARGEST NUMBER of matches, not the BEST (highest score)
+    match.
+
+    Note: This is fundamentally different from linear_sum_assignment.
+    """
+    isect = a.columns.intersection(b.columns)
+
+    combined_df = pd.concat([a[isect], b[isect]], axis=0)
+    o = spearmanr(combined_df, axis=1)
+
+    a_count = a.shape[0]
+    sdf = pd.DataFrame(o.correlation[a_count:, :a_count], index=b.index, columns=a.index)
+
+    m = {}
+    s_result = sdf.apply(lambda x: x.nlargest(10).index, axis=0)
+    for k, row in s_result.items():
+        m[k] = row.tolist()
+
+    id_map = HopcroftKarp(m).maximum_matching(keys_only=True)
+
+    out_a = []
+    out_b = []
+    out_score = []
+
+    for k, v in id_map.items():
+        c = sdf.loc[v, k]
+        if c >= cutoff:
+            out_a.append(k)
+            out_b.append(v)
+            out_score.append(c)
+
+    return out_a, out_b, out_score
+
+
+def procrustes(X, Y):
+    """
+    Computes the Procrustes transformation (optimal PURE rotation)
+    to align X to Y, correcting for reflections.
+
+    Args:
+        X: The first matrix (N_points, N_dims).
+        Y: The second matrix (N_points, N_dims).
+
+    Returns:
+        R: The optimal rotation matrix (guaranteed det(R) = +1).
+    """
+    assert X.shape == Y.shape
+
+    M = np.dot(X.T, Y)
+
+    U, S, Vt = np.linalg.svd(M)
+
+    R = np.dot(U, Vt)
+
+    if np.linalg.det(R) < 0:
+        Vt_corrected = Vt.copy()
+        Vt_corrected[-1, :] *= -1
+        R = np.dot(U, Vt_corrected)
+
+    return R
+
+
+def procrustes_scale(X, Y):
+    """
+    Compute the procrustes transformation (R), then compute a factor (k) to rescale the src matrix. 
+    
+    To apply transformtion:
+    src.dot(R) * k
+
+    
+    Args:
+        X: The first matrix (N_points, N_dims).
+        Y: The second matrix (N_points, N_dims).
+
+    Returns:
+        R: The optimal rotation matrix (guaranteed det(R) = +1).
+        k: Scaling factor
+
+    """
+    R = procrustes(X,Y)
+    A = np.array(X.dot(R))
+    B = np.array(Y)
+    numerators = np.sum(A * B, axis=0)
+    denominators = np.sum(A * A, axis=0)
+    k = np.divide(numerators, denominators, out=np.zeros_like(numerators), where=denominators!=0)
+    return R, k

{embkit-0.1.0 → embkit-0.2}/src/embkit/bmeg.py

@@ -1,3 +1,10 @@
+
+"""
+BMEG
+
+Data retrieval from BMEG instance
+"""
+
 import gripql
 
 def connect(url="https://bmeg.io/grip", graph="rc6", cred_file="./bmeg_credentials.json"):

embkit-0.2/src/embkit/c_bio/__init__.py

@@ -0,0 +1 @@
+from .api import CBIOAPI

embkit-0.2/src/embkit/c_bio/api.py

@@ -0,0 +1,13 @@
+import requests
+
+
+class CBIOAPI:
+
+    @staticmethod
+    def list_studies():
+        try:
+            studies = requests.get("https://www.cbioportal.org/api/studies").json()
+            return studies
+        except requests.RequestException as e:
+            print(f"Error fetching studies: {e}")
+            return None

embkit-0.2/src/embkit/commands/__init__.py

@@ -0,0 +1,20 @@
+
+"""
+Command Line Methods
+
+embkit matrix <> : Matrix methods for normalization and transformation
+
+embkit model <> : Methods of model training and application
+
+embkit cbio <> : cBIO methods for data querying and download
+
+"""
+
+from .model import model
+from .matrix import matrix
+from .cbio import cbio_cmd
+from .datasets import datasets
+from .protein import protein
+from .align import align
+
+__all__ = ["model", "matrix", "datasets", "protein"]

embkit-0.2/src/embkit/commands/align.py

@@ -0,0 +1,35 @@
+"""
+
+Command line tools for aligning embedding spaces
+
+"""
+
+import click
+import pandas as pd
+from ..align import matrix_spearman_alignment_linear, matrix_spearman_alignment_hopkraft
+
+align = click.Group(name="align", help="Embedding Alignment methods")
+
+@align.command(name="pair")
+@click.argument("matrix1")
+@click.argument("matrix2")
+@click.option("--method", "-m", required=False, type=str, default="linear")
+@click.option("--cutoff", "-c", required=False, type=float, default=0.5)
+def pair(matrix1, matrix2, method, cutoff):
+    """
+    Given two matrices, use feature rank correlation to
+    create list of pairs
+    """
+
+    m1 = pd.read_csv(matrix1, sep="\t", index_col=0)
+    m2 = pd.read_csv(matrix2, sep="\t", index_col=0)
+    
+    if method == "hopkraft":
+        out = matrix_spearman_alignment_hopkraft(m1, m2)
+        for k, v in out.items():
+            print(f"{k}\t{v[0]}\t{v[1]}")
+
+    elif method == "linear":
+        out = matrix_spearman_alignment_linear(m1, m2, cutoff)
+        for k, v in out.items():
+            print(f"{k}\t{v[0]}\t{v[1]}")

embkit-0.2/src/embkit/commands/cbio.py

@@ -0,0 +1,63 @@
+import click
+from ..datasets import CBIOPortal
+from ..c_bio import CBIOAPI
+import logging
+
+logger = logging.getLogger(__name__)
+
+cbio_cmd = click.Group(name="cbio", help="cBIO commands.")
+
+
+@cbio_cmd.command(name="studies", help="List cbio studies.")
+@click.pass_context
+def list_studies(ctx):
+    """
+    Command to list available cBIO studies.
+
+    Parameters:
+        ctx: Click context object.
+
+    Returns:
+        None
+    """
+    api = CBIOAPI()
+    studies = api.list_studies()
+
+    if studies:
+        for study in studies:
+            click.echo(f"Study ID: {study['studyId']}, Name: {study['name']}")
+    else:
+        click.echo("No studies found or an error occurred while fetching studies.")
+
+
+@cbio_cmd.command(name="download", help="Download cbio dataset.")
+@click.option("--save_path", "-s", type=click.Path(exists=False, writable=True, path_type=str),
+              help="Path to save the dataset.")
+@click.option("--study_id", "-si", type=str, show_default=True, help="Id of the study to download.")
+@click.pass_context
+def download_tmp(ctx, save_path: str | None, study_id: str):
+    """
+    Command to download a cBIO dataset by specifying the study ID.
+
+
+    Parameters:
+        ctx: Click context object.
+        save_path: Path to save the dataset.
+        study_id: ID of the study to download.
+
+    Returns:
+        None
+    """
+
+    if study_id is None:
+        click.echo("Study Id not specified. Please use -si option to specify the study id.")
+        return
+
+    cbio_portal: CBIOPortal = CBIOPortal(save_path=save_path, study_id=study_id, download=True)
+    cbio_portal.download()
+    cbio_portal.unpack()
+
+
+# needed for testing
+if __name__ == "__main__":
+    cbio_cmd()  # pragma: no cover

embkit-0.2/src/embkit/commands/datasets.py

@@ -0,0 +1,61 @@
+import click
+import pandas as pd
+from ..preprocessing import load_raw_hugo, load_gct
+from ..datasets import GTEx, Hugo, SIF
+
+datasets = click.Group(name="datasets", help="Datasets commands.")
+
+
+@datasets.command()
+@click.option("--data_type", "-t", required=False, type=str, help="Name of the dataset to download.")
+@click.option("--output_folder", "-f", required=True, type=str, help="The folder to download the dataset into.")
+def gtex(data_type: str, output_folder: str):
+    """Download GTEx dataset."""
+
+    if data_type not in GTEx.NAMES:
+        click.echo(f"Dataset name '{data_type}' is not recognized. Available options are: {list(GTEx.NAMES.keys())}")
+        return
+
+    GTEx(data_type=data_type, save_path=output_folder)
+
+
+
+@datasets.command()
+@click.option("--output_folder", "-f", required=True, type=str, help="The folder to download the dataset into.")
+def sif(output_folder: str):
+    """Download SIF dataset."""
+    SIF(save_path=output_folder)
+
+
+
+@datasets.command()
+@click.option("--output_folder", "-f", required=True, type=str, help="The folder to download the dataset into.")
+@click.option("--gtex_conversion", "-gtex", required=False, is_flag=True, type=bool, default=False,
+              help="Whether to convert the hugo data to a gtex version.")
+def hugo(gtex_conversion: bool, output_folder: str):
+    """Download Hugo dataset."""
+
+    click.echo(f"Downloading hugo dataset into '{output_folder}'")
+    hugo_data: Hugo = Hugo(save_path=output_folder)
+
+    if gtex_conversion:
+        click.echo("Converting hugo dataset using GTEx dataset...")
+        click.echo("Downloading required dataset (GTEx)...")
+        gtex_data: GTEx = GTEx(save_path=output_folder)
+        click.echo("Loading GTEx dataset...")
+        gtex_df: pd.DataFrame = load_gct(gtex_data)
+        click.echo("Loading Hugo dataset...")
+        hugo_df: pd.DataFrame = load_raw_hugo(hugo_data)
+
+        # select protein coding genes and build a dict that translates from ensembl gene id to Hugo name
+        gene_names = hugo_df[hugo_df["locus_group"] == "protein-coding gene"][["symbol", "ensembl_gene_id"]].set_index(
+            "ensembl_gene_id")["symbol"].to_dict()
+
+        # rename columns using HUGO names
+        df_select = gtex_df.rename(columns=lambda x: gene_names.get(x.split(".")[0], x))
+        # remove columns that aren't renamed (including all non-protein coding columns)
+        df_select = df_select.loc[:, df_select.columns.map(lambda x: not x.startswith("ENSG0"))]
+        output_path = f"{hugo_data.save_path}/gtex.hugo.tsv"
+        click.echo(f"Saving converted dataset to '{output_path}...'")
+        df_select.to_csv(output_path, sep="\t")
+        click.echo("Done.")

embkit-0.2/src/embkit/commands/matrix.py

@@ -0,0 +1,36 @@
+import click
+import pandas as pd
+
+matrix = click.Group(name="matrix", help="Model commands.")
+
+@matrix.command()
+@click.argument("srcs", nargs=-1)
+@click.option("--out", "-o", required=True)
+@click.option("--features", default=None, type=click.Path(exists=True, dir_okay=False, readable=True, path_type=str))
+@click.option("--col-quantile", "col_quantile", default=False, is_flag=True)
+@click.option("--quantile-max", "quantile_max", default=0.9, type=float)
+@click.option("--precision", "-p", type=int, default=5)
+def normalize(srcs, out, features, col_quantile, quantile_max, precision):
+
+    features_list = None
+    if features is not None:
+        features = []
+        with open(features, encoding="ascii") as handle:
+            features_list = list(line.rstrip() for line in handle)
+
+    dfs = []
+    for i in srcs:
+        df = pd.read_csv(i, sep="\t", index_col=0)
+        if features is not None:
+            df = df[features]
+        dfs.append(df)
+    if len(dfs) == 0:
+        click.echo("No matrices defined")
+        return
+    df = pd.concat( dfs )
+
+    if not col_quantile:
+        normDF = (df.transpose() / df.quantile(quantile_max, axis=1)).transpose().clip(upper=1.0, lower=0.0).fillna(0.0)
+    else:
+        normDF = (df / df.quantile(quantile_max)).clip(upper=1.0, lower=0.0).fillna(0.0)
+    normDF.round(decimals=precision).to_csv(out, sep="\t")