mlmolprop 1.9.1__py3-none-any.whl

mlmolprop/__init__.py

@@ -0,0 +1,92 @@
+"""mlmolprop: cheminformatics and QSAR modeling toolkit built on RDKit and scikit-learn."""
+
+from importlib.metadata import version
+
+from .basic import (
+    RMSEP_CV_C,
+    F,
+    analyse,
+    makecolumn,
+    press,
+    press_m,
+    press_root,
+    q2r2,
+    r2test,
+    twodlist,
+)
+from .correlation import find_correlation
+from .descriptors import CI, dataframe, desc, tanimoto
+from .fingerprint import Fingerprint, explicit_bitvect_to_numpy_array, make_fingerprints
+from .image import highlight, images_to_dataframe, svg_files_to_png
+from .importance import lime_explain, partial
+from .model import (
+    Model,
+    ModelC,
+    clus_uns,
+    metr,
+    plot_confusion_matrix,
+    plot_w,
+    rocc,
+    safe_divide,
+)
+from .molprep import RDimage, mol_enumerate, mol_prep, moltosvg
+from .plots import plot_features
+from .processing import (
+    SelectKBest_selector,
+    VarianceThreshold_selector,
+    average_bygroup,
+    data_prep,
+    file2list,
+    file2object,
+    list2file,
+    object2file,
+)
+
+__version__ = version("mlmolprop")
+
+__all__ = [
+    "CI",
+    "RMSEP_CV_C",
+    "F",
+    "Fingerprint",
+    "Model",
+    "ModelC",
+    "RDimage",
+    "SelectKBest_selector",
+    "VarianceThreshold_selector",
+    "analyse",
+    "average_bygroup",
+    "clus_uns",
+    "data_prep",
+    "dataframe",
+    "desc",
+    "explicit_bitvect_to_numpy_array",
+    "file2list",
+    "file2object",
+    "find_correlation",
+    "highlight",
+    "images_to_dataframe",
+    "lime_explain",
+    "list2file",
+    "make_fingerprints",
+    "makecolumn",
+    "metr",
+    "mol_enumerate",
+    "mol_prep",
+    "moltosvg",
+    "object2file",
+    "partial",
+    "plot_confusion_matrix",
+    "plot_features",
+    "plot_w",
+    "press",
+    "press_m",
+    "press_root",
+    "q2r2",
+    "r2test",
+    "rocc",
+    "safe_divide",
+    "svg_files_to_png",
+    "tanimoto",
+    "twodlist",
+]

mlmolprop/basic.py

@@ -0,0 +1,154 @@
+"""Core statistics helpers used for QSAR model evaluation."""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def _check_same_length(a: np.ndarray, b: np.ndarray) -> None:
+    if a.shape != b.shape:
+        raise ValueError(f"inputs must be the same length, got {len(a)} and {len(b)}")
+
+
+def press(observed, predicted) -> float:
+    """Sum of squared residuals (a.k.a. PRESS/RSS) between two sequences."""
+    observed = np.asarray(observed, dtype=float)
+    predicted = np.asarray(predicted, dtype=float)
+    _check_same_length(observed, predicted)
+    return float(np.sum((observed - predicted) ** 2))
+
+
+def press_root(observed, predicted) -> float:
+    """Sum of absolute residuals between two sequences."""
+    observed = np.asarray(observed, dtype=float)
+    predicted = np.asarray(predicted, dtype=float)
+    _check_same_length(observed, predicted)
+    return float(np.sum(np.abs(observed - predicted)))
+
+
+def press_m(values, reference) -> float:
+    """Sum of squared deviations of ``values`` from ``mean(reference)``.
+
+    ``reference`` may be a precomputed scalar mean (``mean`` of a scalar is
+    itself, so this is a no-op) or an array-like whose mean is used as the
+    reference point. Both call styles are used elsewhere in this module.
+    """
+    values = np.asarray(values, dtype=float)
+    reference_mean = np.mean(reference)
+    return float(np.sum((values - reference_mean) ** 2))
+
+
+def q2r2(obs, pred) -> float:
+    """Q2/R2 = 1 - PRESS / total sum of squares of ``obs``."""
+    obs = np.asarray(obs, dtype=float)
+    ss_tot = press_m(obs, np.mean(obs))
+    if ss_tot == 0:
+        return 0.0
+    return 1 - press(obs, pred) / ss_tot
+
+
+def r2test(obs, pred, train) -> float:
+    """External R2 of a test set, using the training set's mean as reference."""
+    ss_tot = press_m(obs, train)
+    if ss_tot == 0:
+        return 0.0
+    return 1 - press(obs, pred) / ss_tot
+
+
+def RMSEP_CV_C(obs, pred, k: int = 0) -> float:
+    """Root mean square error, optionally adjusted for degrees of freedom.
+
+    With ``k=0`` this is the plain RMSE (external test set or LOO-CV set).
+    With ``k>0`` the denominator is adjusted by the number of predictors/
+    components ``k`` (training-set calibration RMSE, a.k.a. SEE).
+    """
+    ssr = press(obs, pred)
+    n = len(obs)
+    denom = n if k == 0 else n - k - 1
+    if denom <= 0:
+        raise ValueError(f"not enough observations ({n}) for k={k} predictors")
+    return float(np.sqrt(ssr / denom))
+
+
+def F(obs, pred, k: int) -> float:
+    """F-statistic (MSR / MSE) for a training-set regression with k predictors."""
+    obs = np.asarray(obs, dtype=float)
+    n = len(obs)
+    denom = n - k - 1
+    if denom <= 0:
+        raise ValueError(f"not enough observations ({n}) for k={k} predictors")
+    msr = press_m(pred, np.mean(obs)) / k
+    mse = press(obs, pred) / denom
+    return msr / mse
+
+
+def analyse(ytrain, y_pred_train, ytest, y_pred_test, ycv1, ycv2, k: int) -> dict:
+    """Compute a standard set of QSAR model-quality metrics.
+
+    Parameters
+    ----------
+    ytrain, y_pred_train : array-like
+        Observed and predicted values on the training set.
+    ytest, y_pred_test : array-like
+        Observed and predicted values on the external test set.
+    ycv1, ycv2 : array-like
+        Observed and predicted values under cross-validation.
+    k : int
+        Number of predictors (or latent components). The cross-validation
+        formula used here assumes leave-one-out CV; it is not a valid
+        adjustment for k-fold CV.
+
+    Returns
+    -------
+    dict
+        R2, R2_Adj, R2_test, F, q2, RMSE/MAE for train, test, and CV.
+    """
+    n_train = len(ytrain)
+    r2 = q2r2(ytrain, y_pred_train)
+    # Both R2_Adj and F encode classical OLS degrees-of-freedom (n > k+1, k
+    # literally-fit linear parameters) -- meaningful for M in {mlr, pls,
+    # lasso, ...}, but not a real constraint for RF/SVM/tree-style
+    # regressors, whose capacity isn't controlled by feature count the same
+    # way. Reporting NaN here instead of raising/dividing-by-zero mirrors
+    # ModelC(), which has no such constraint, and lets those model types run
+    # on any feature count.
+    if n_train - k - 1 > 0:
+        r2_adj = 1 - (((n_train - 1) / (n_train - k - 1)) * (1 - r2))
+    else:
+        r2_adj = float("nan")
+    q2 = q2r2(ycv1, ycv2)
+    r2_test = r2test(ytest, y_pred_test, ytrain)
+    try:
+        f = F(ytrain, y_pred_train, k)
+    except ValueError:
+        f = float("nan")
+    rmse_train = RMSEP_CV_C(ytrain, y_pred_train)
+    mae_train = press_root(ytrain, y_pred_train) / n_train
+    rmse_cv = RMSEP_CV_C(ycv1, ycv2)
+    mae_cv = press_root(ycv1, ycv2) / len(ycv2)
+    rmse_test = RMSEP_CV_C(ytest, y_pred_test)
+    mae_test = press_root(ytest, y_pred_test) / len(ytest)
+    return {
+        "R2": r2,
+        "R2_Adj": r2_adj,
+        "R2_test": r2_test,
+        "F": f,
+        "q2": q2,
+        "RMSE_train": rmse_train,
+        "MAE_train": mae_train,
+        "RMSE_test": rmse_test,
+        "MAE_test": mae_test,
+        "RMSE_CV": rmse_cv,
+        "MAE_CV": mae_cv,
+    }
+
+
+def twodlist(m: int, n: int) -> list[list[str]]:
+    """Build an m x n list of lists pre-filled with the placeholder "none"."""
+    return [["none"] * n for _ in range(m)]
+
+
+def makecolumn(data, c: int) -> list[list]:
+    """Return the first ``c`` columns of ``data`` as a list of Python lists."""
+    arr = np.asarray(data)
+    return [list(arr[:, j]) for j in range(c)]

mlmolprop/correlation.py

@@ -0,0 +1,80 @@
+"""Detect and select for removal highly correlated numeric features."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+
+def find_correlation(data: pd.DataFrame, threshold: float = 0.9) -> list[str]:
+    """Find features that are redundant due to high pairwise correlation.
+
+    For every pair of features whose absolute correlation exceeds
+    ``threshold``, one feature from the pair is kept and the rest are
+    marked for removal, so that at most one representative of each
+    correlated cluster survives.
+
+    Parameters
+    ----------
+    data : pandas.DataFrame
+        Numeric feature matrix (columns are features).
+    threshold : float, default 0.9
+        Absolute correlation threshold in ``(0, 1]``; pairs whose absolute
+        correlation exceeds this value are considered redundant.
+
+    Returns
+    -------
+    list[str]
+        Column names recommended for removal.
+    """
+    if not 0 < threshold <= 1:
+        raise ValueError(f"threshold must be in (0, 1], got {threshold}")
+
+    data = pd.DataFrame(data)
+    columns = list(data.columns)
+    corr_arr = data.corr().abs().to_numpy()
+
+    # Union-find over pairs whose correlation exceeds threshold, so columns
+    # linked only transitively (A~B and B~C both exceed threshold, even if
+    # A~C doesn't) end up in the same cluster rather than being resolved as
+    # two independent pairs that can each keep a representative that turns
+    # out to be correlated with the other pair's representative.
+    parent = {c: c for c in columns}
+
+    def find(c: str) -> str:
+        while parent[c] != c:
+            parent[c] = parent[parent[c]]
+            c = parent[c]
+        return c
+
+    def union(a: str, b: str) -> None:
+        ra, rb = find(a), find(b)
+        if ra != rb:
+            parent[ra] = rb
+
+    # Scanning every pair via DataFrame.loc in a Python loop is O(n^2) with
+    # substantial per-lookup overhead -- minutes, not seconds, past a few
+    # hundred columns (e.g. ~2 min for 2048 ECFP4 bits). np.triu_indices
+    # walks the upper triangle in the same row-major (i, then j > i) order
+    # the original nested loop used, so the union() call sequence -- and
+    # therefore which survivor each cluster keeps -- is unchanged; only the
+    # "is this pair above threshold" scan itself is vectorized.
+    iu, ju = np.triu_indices(len(columns), k=1)
+    above = corr_arr[iu, ju] > threshold
+    for i, j in zip(iu[above], ju[above], strict=True):
+        union(columns[i], columns[j])
+
+    clusters: dict[str, list[str]] = {}
+    for c in columns:
+        clusters.setdefault(find(c), []).append(c)
+
+    to_remove: list[str] = []
+    for members in clusters.values():
+        if len(members) > 1:
+            # Keep the last column in original order, matching this
+            # function's existing convention for a simple two-column match
+            # (the earlier column is removed, the later one kept).
+            survivor = members[-1]
+            to_remove.extend(m for m in members if m != survivor)
+
+    return to_remove

mlmolprop/descriptors.py

@@ -0,0 +1,279 @@
+"""Compute RDKit molecular descriptors and build QSAR-ready datasets."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+from rdkit import DataStructs
+from rdkit.Chem import (
+    AllChem,
+    Descriptors,
+    MACCSkeys,
+    PandasTools,
+    SDMolSupplier,
+    SmilesMolSupplier,
+)
+from rdkit.Chem.Fingerprints import FingerprintMols
+from rdkit.ML.Descriptors import MoleculeDescriptors
+
+from .basic import makecolumn, twodlist
+
+_SOURCE_TYPES = {"sdf", "smi", "molecule"}
+_DESCRIPTOR_SETS = {"all", "positive", "interpretable"}
+
+positive_prefixes = [
+    "fr_",  # fragment counts (~110 descriptors)
+    "Num",  # counts: NumAtoms, NumRings, etc.
+    "MolWt",
+    "ExactMolWt",
+    "HeavyAtomMolWt",
+    "TPSA",
+    "LabuteASA",
+    "SPS",
+    "qed",
+    "PEOE_VSA",
+    "SMR_VSA",
+    "SlogP_VSA",  # surface area families
+]
+# Filter RDKit's full descriptor list to only those starting with the above prefixes
+positive_desc_names = [
+    name
+    for name, _ in Descriptors.descList
+    if any(name.startswith(p) for p in positive_prefixes)
+]
+
+easy_prefixes = ["fr_", "Num", "MolWt", "ExactMolWt", "HeavyAtomMolWt", "TPSA"]
+easy_desc_names = [
+    name
+    for name, _ in Descriptors.descList
+    if any(name.startswith(p) for p in easy_prefixes)
+]
+
+
+def desc(
+    molecules, source: str = "molecule", delimiter: str = ",", selected_des: str = "all"
+) -> list:
+    """Compute RDKit descriptors for a set of molecules.
+
+    Parameters
+    ----------
+    molecules
+        Path to an SDF/SMILES file, or (if ``source="molecule"``) an
+        already-loaded list of RDKit Mol objects. Each molecule must have
+        a ``_Name`` property set.
+    source : {"sdf", "smi", "molecule"}, default "molecule"
+    delimiter : str, default ","
+        Field delimiter, used only when ``source="smi"``.
+    selected_des : {"all", "positive", "interpretable"}, default "all"
+        "positive" restricts to descriptors that are inherently
+        non-negative (counts, fragment counts, surface areas, ...);
+        "interpretable" restricts to a smaller, easy-to-explain subset.
+
+    Returns
+    -------
+    list
+        ``[names, descriptor_names, descriptor_values]``: molecule names,
+        the computed descriptor names, and a list of per-molecule
+        descriptor value tuples.
+    """
+    if source == "sdf":
+        molecules_list = SDMolSupplier(molecules)
+    elif source == "smi":
+        molecules_list = SmilesMolSupplier(
+            molecules, delimiter=delimiter, titleLine=True, smilesColumn=0, nameColumn=1
+        )
+    elif source == "molecule":
+        molecules_list = molecules
+    else:
+        raise ValueError(
+            f"source must be one of {sorted(_SOURCE_TYPES)}, got {source!r}"
+        )
+
+    if selected_des == "all":
+        descriptor_names = [x[0] for x in Descriptors.descList]
+    elif selected_des == "positive":
+        descriptor_names = positive_desc_names
+    elif selected_des == "interpretable":
+        descriptor_names = easy_desc_names
+    else:
+        raise ValueError(
+            f"selected_des must be one of {sorted(_DESCRIPTOR_SETS)}, got {selected_des!r}"
+        )
+
+    names = len(molecules_list) * ["null"]
+    values = len(molecules_list) * ["null"]
+    calc = MoleculeDescriptors.MolecularDescriptorCalculator(descriptor_names)
+    for i in range(len(molecules_list)):
+        values[i] = calc.CalcDescriptors(molecules_list[i])
+        names[i] = molecules_list[i].GetProp("_Name")
+
+    return [names, descriptor_names, values]
+
+
+def dataframe(
+    desc_result,
+    input_activities: str | None,
+    output: str,
+    target: str = "IC50",
+    sdf_file: str | None = None,
+    source: str = "des",
+) -> pd.DataFrame:
+    """Merge computed descriptors/fingerprints with an activity table into a QSAR dataset.
+
+    Parameters
+    ----------
+    desc_result
+        Output of :func:`desc` (``source="des"``), or raw fingerprint
+        data indexable by ``pandas.DataFrame`` (``source in
+        {"finger", "image"}``).
+    input_activities : str or None
+        Path to a CSV of activities with a "name" column matching the
+        molecule names in ``desc_result``. If ``None``, the descriptor/
+        fingerprint DataFrame is written out as-is -- useful for scoring
+        an external set with no known activity.
+    output : str
+        Path the merged (or standalone) DataFrame is written to as CSV.
+    target : str, default "IC50"
+        Activity column to pull from ``sdf_file`` when generating
+        ``input_activities`` from an SDF.
+    sdf_file : str or None
+        If given, activities are extracted from this SDF's "ID" and
+        ``target`` fields and written to ``input_activities`` before
+        merging.
+    source : {"des", "finger", "image"}, default "des"
+        Shape of ``desc_result``.
+
+    Returns
+    -------
+    pandas.DataFrame
+    """
+    if sdf_file is not None:
+        sdftable = PandasTools.LoadSDF(sdf_file)
+        activity_df = sdftable.loc[:, ["ID", target]].rename(columns={"ID": "name"})
+        activity_df.to_csv(input_activities, index=False)
+
+    if source in ("finger", "image"):
+        df1 = pd.DataFrame(desc_result)
+    elif source == "des":
+        df1 = pd.DataFrame(desc_result[2], index=desc_result[0], columns=desc_result[1])
+    else:
+        raise ValueError(
+            f"source must be one of 'des', 'finger', 'image', got {source!r}"
+        )
+
+    if input_activities is not None:
+        reader = pd.read_csv(input_activities)
+        reader["name"] = reader["name"].astype(str)
+        merged = pd.merge(reader, df1, right_index=True, left_on="name")
+    else:
+        merged = df1
+
+    merged.to_csv(output, index=False)
+    return merged
+
+
+def tanimoto(molecules_list, names, output: str, kind: str = "fps") -> pd.DataFrame:
+    """Compute a pairwise Tanimoto similarity matrix for a list of molecules.
+
+    Parameters
+    ----------
+    molecules_list : list of rdkit.Chem.Mol
+    names : list
+        Molecule names/labels, used as the DataFrame's index and columns.
+    output : str
+        Path the similarity matrix is written to as CSV.
+    kind : {"fps", "maccs", "ecfp", "fcfp"}, default "fps"
+        Fingerprint used for similarity: Daylight ("fps"), MACCS keys,
+        Morgan/ECFP, or feature-based Morgan/FCFP.
+    """
+    if kind == "fps":
+        fps = [FingerprintMols.FingerprintMol(m) for m in molecules_list]
+    elif kind == "maccs":
+        fps = [MACCSkeys.GenMACCSKeys(m) for m in molecules_list]
+    elif kind == "ecfp":
+        fps = [AllChem.GetMorganFingerprint(m, 6) for m in molecules_list]
+    elif kind == "fcfp":
+        fps = [
+            AllChem.GetMorganFingerprint(m, 6, useFeatures=True) for m in molecules_list
+        ]
+    else:
+        raise ValueError(
+            f"kind must be one of 'fps', 'maccs', 'ecfp', 'fcfp', got {kind!r}"
+        )
+
+    n = len(fps)
+    table = twodlist(n, n)
+    for i, x in enumerate(fps):
+        for j, y in enumerate(fps):
+            table[i][j] = DataStructs.TanimotoSimilarity(x, y)
+
+    df = pd.DataFrame(table, index=names, columns=names)
+    df.to_csv(output)
+    return df
+
+
+def CI(
+    X_train, X_test, v_names, train_names, test_names, cutoff: float
+) -> list[pd.DataFrame]:
+    """Standardize train/test features against training-set statistics and flag outliers.
+
+    For each feature, z-scores (``|value - train_mean| / train_std``) are
+    computed for both train and test sets using the *training* set's mean
+    and standard deviation (features with zero training variance get a
+    z-score of 0). ``out_train``/``out_test`` keep only the z-scores that
+    exceed ``cutoff``, leaving the placeholder string ``"none"``
+    elsewhere -- matching the original R ``CI_Test`` this was ported from.
+
+    Returns
+    -------
+    list[pandas.DataFrame]
+        ``[table_train, table_test, out_train, out_test]``, each a
+        z-score table indexed by ``train_names``/``test_names``.
+    """
+    train = np.array(X_train)
+    test = np.array(X_test)
+    r, c = len(train), len(train[0])
+    r2 = len(test)
+
+    table_train = twodlist(r, c)
+    table_test = twodlist(r2, c)
+    out_train = twodlist(r, c)
+    out_test = twodlist(r2, c)
+
+    for i in range(r):
+        table_train[i] = train[i]
+        if i < r2:
+            table_test[i] = test[i]
+
+    column = makecolumn(table_train, c)
+    col_means = [np.mean(column[j]) for j in range(c)]
+    col_stds = [np.std(column[j]) for j in range(c)]
+
+    for i in range(r):
+        for j in range(c):
+            if col_stds[j] != 0:
+                table_train[i][j] = np.abs(
+                    (table_train[i][j] - col_means[j]) / col_stds[j]
+                )
+                if i < r2:
+                    table_test[i][j] = np.abs(
+                        (table_test[i][j] - col_means[j]) / col_stds[j]
+                    )
+            else:
+                table_train[i][j] = 0
+                if i < r2:
+                    table_test[i][j] = 0
+
+    for i in range(r):
+        for j in range(c):
+            if table_train[i][j] > cutoff:
+                out_train[i][j] = table_train[i][j]
+                if i < r2 and table_test[i][j] > cutoff:
+                    out_test[i][j] = table_test[i][j]
+
+    return [
+        pd.DataFrame(table_train, columns=v_names, index=train_names),
+        pd.DataFrame(table_test, columns=v_names, index=test_names),
+        pd.DataFrame(out_train, columns=v_names, index=train_names),
+        pd.DataFrame(out_test, columns=v_names, index=test_names),
+    ]