pythonflex 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

pythonflex/preprocessing.py

@@ -0,0 +1,221 @@
+import os
+import pandas as pd
+import numpy as np
+from .utils import dsave, dload
+from tqdm import tqdm
+from .logging_config import log
+from importlib import resources
+from pathlib import Path
+tqdm.pandas()
+
+
+def get_example_data_path(filename: str):
+    return resources.files("pyflex.data").joinpath("dataset").joinpath(filename)
+
+
+def _load_file(filepath, ext):
+    loaders = {
+        ".csv": lambda f: pd.read_csv(f, index_col=0),
+        ".xlsx": lambda f: pd.read_excel(f, index_col=0),
+        ".parquet": pd.read_parquet,
+        ".p": pd.read_parquet
+    }
+    if ext not in loaders:
+        raise ValueError(f"Unsupported file extension: {ext}")
+
+    return loaders[ext](filepath)
+
+
+def load_datasets(files, continue_with_common_genes=False):
+    preprocessing = dload("config")["preprocessing"]
+    data_dict= {}     
+
+    for filename, meta in files.items():
+        if isinstance(meta, pd.DataFrame):
+            df = meta
+        elif isinstance(meta, dict):
+            filepath = meta["path"]
+            if isinstance(filepath, pd.DataFrame):
+                df = filepath
+            else:
+                ext = os.path.splitext(filepath)[1]
+                df = _load_file(filepath, ext)
+        else:
+            raise ValueError(f"Unsupported data structure for '{filename}': {type(meta)}")
+
+        df.index = df.index.str.split().str[0]
+        if preprocessing.get('normalize'):
+            log.info(f"{filename}: Normalization.")
+            df = (df - df.mean()) / df.std(ddof=0)
+
+        if preprocessing.get('drop_na'):
+            log.info(f"{filename}: Dropping missing values.")
+            df = df.dropna(how="any")
+
+        if preprocessing.get('fill_na'):
+            log.info(f"{filename}: Filling missing values with column mean.")
+            #df = df.T.fillna(df.mean(axis=1)).T
+            df = data_imputation(df)
+
+            
+        data_dict[filename] = df
+
+    common_genes = get_common_genes(data_dict)
+    if continue_with_common_genes:
+        log.info(f"Continuing with common genes: {len(common_genes)}")
+        for filename, df in data_dict.items():
+            if df.index.isin(common_genes).any():
+                data_dict[filename] = df.loc[common_genes]
+    
+    dsave({
+        "datasets": data_dict,
+        "sorting": {
+            k: v.get("sort", "high") if isinstance(v, dict) else "high"
+            for k, v in files.items()
+        }
+    }, "input")
+    log.done(f"Datasets loaded.")
+    return data_dict, common_genes  
+
+
+
+
+def drop_bad_samples(df, max_na=0.1):
+    total_elements = df.shape[0] * df.shape[1]
+    percent_nan = np.isnan(df.values).sum() / total_elements if total_elements > 0 else 0
+    has_nan_per_sample = np.isnan(df.values).any(axis=0) # how many samples has NA.
+
+    log.info(f"Total: {total_elements}, Percent NaN: {percent_nan:.2%}, Samples with NaN: {np.sum(has_nan_per_sample)} / {df.shape[1]}")
+
+    num_genes = df.shape[0]  # E.g., 1178 (total rows/genes)
+    na_per_sample = np.isnan(df.values).sum(axis=0) / num_genes  # Fraction NA per sample (column)
+    good_samples = na_per_sample <= max_na  # Keep if <=10% NA
+    data_filtered = df.loc[:, good_samples]  # Drop bad samples (those >10% NA)
+
+    log.info(f"Filtered samples: {data_filtered.shape[1]} (removed {df.shape[1] - data_filtered.shape[1]} samples with >{max_na*100:.0f}% NAs)")
+    return data_filtered
+
+
+
+def data_imputation(df):
+    log.info("Imputing missing values with gene means ...")
+    gene_means = np.nanmean(df.values, axis=1)  # 1D array: means per gene
+    data_values = df.values.copy()
+    rows, cols = np.where(np.isnan(data_values))
+    if len(rows) > 0:
+        data_values[rows, cols] = np.take(gene_means, rows)
+
+    df = pd.DataFrame(data_values, index=df.index, columns=df.columns)
+    log.info(f"Data after imputation: {df.shape[0]} genes, {df.shape[1]} samples")
+    return df
+
+
+
+
+def get_common_genes(datasets):
+    log.started("Finding common genes across datasets.")
+    gene_sets = [set(df.index) for df in datasets.values()]
+    common_genes = set.intersection(*gene_sets)
+    log.done(f"Common genes found: {len(common_genes)}")
+    dsave(common_genes, "common", "common_genes")
+    return list(common_genes)
+
+
+def filter_matrix_by_genes(matrix, genes_present_in_terms):
+    log.started("Filtering matrix using genes present in terms.")
+    genes = matrix.index.intersection(genes_present_in_terms)
+    matrix = matrix.loc[genes, genes]
+    log.done(f"Filtering matrix: {matrix.shape}")
+    return matrix.loc[genes, genes]
+    
+
+
+
+def load_gold_standard():
+    
+    config = dload("config") 
+    common_genes = dload("common", "common_genes")
+
+    gold_standard_source = config['gold_standard']
+    log.started(f"Loading gold standard: {gold_standard_source}, Min complex size: {config['min_genes_in_complex']}, Jaccard filtering: {config['jaccard']}")
+    if not common_genes:
+        raise ValueError("Common genes not found.")
+
+    # Define gold standard file paths for predefined sources
+    gold_standard_files = {
+        "CORUM": "gold_standard/CORUM.parquet",
+        "GOBP": "gold_standard/GOBP.parquet",
+        "PATHWAY": "gold_standard/PATHWAY.parquet"
+    }
+
+    if gold_standard_source in gold_standard_files:
+        # Load predefined gold standard from package resources
+        filename = gold_standard_files[gold_standard_source]
+        filename_path = resources.files("pyflex.data").joinpath(filename)
+        if not filename_path.exists():  # Check if the file exists
+            raise ValueError(f"Invalid Gold Standard type: {gold_standard_source}. File not found.")
+        terms = pd.read_parquet(filename_path)  # type: ignore
+    elif Path(gold_standard_source).suffix.lower() == '.csv':
+        # Load user-provided custom gold standard from CSV file
+        filename_path = Path(gold_standard_source)
+        if not filename_path.exists():
+            raise ValueError(f"Custom gold standard CSV file not found: {gold_standard_source}")
+        log.done(f"Loading custom gold standard from CSV: {gold_standard_source}")
+        terms = pd.read_csv(filename_path)  
+    else:
+        raise ValueError(f"Invalid gold standard source: {gold_standard_source}. Must be one of {list(gold_standard_files.keys())} or a path to a .csv file.")
+
+    common_genes_set = set(common_genes)
+    terms["used_genes"] = terms["Genes"].apply(lambda x: list(set(x.split(";")) & common_genes_set))
+    terms["n_used_genes"] = terms["used_genes"].apply(len)
+    log.info(f"Applying min_genes_in_complex filtering: {config['min_genes_in_complex']}")
+    terms = terms[terms["n_used_genes"] >= config['min_genes_in_complex']]
+    terms["hash"] = terms["used_genes"].apply(lambda x: [hash(i) for i in x])
+
+    if config['jaccard']:
+        log.info("Applying Jaccard filtering. Remove terms with identical gene sets.")
+        terms = filter_duplicate_terms(terms)
+
+
+    genes_present_in_terms = list(set(terms["used_genes"].explode().unique()) & common_genes_set)
+    # if there is column called "ID", set it as index
+    if "ID" in terms.columns:
+        terms = terms.set_index("ID")
+
+    dsave(terms, "common", "terms")
+    dsave(genes_present_in_terms, "common", "genes_present_in_terms")
+    log.done("Gold standard loading completed.")
+    return terms, genes_present_in_terms
+
+
+
+
+
+def filter_duplicate_terms(terms):
+    log.started("Filtering duplicate terms using optimized method.")
+    
+    # Precompute frozen gene sets and hash them
+    terms = terms.copy()
+    terms["gene_set"] = terms["used_genes"].map(lambda x: frozenset(x))
+    
+    # Group by identical gene sets
+    grouped = terms.groupby("gene_set", sort=False)
+    
+    # Identify duplicate clusters (groups with >1 term)
+    duplicate_clusters = []
+    for _, group in grouped:
+        if len(group) > 1:
+            duplicate_clusters.append(group["ID"].values)
+    
+    # Determine which IDs to keep (smallest ID in each duplicate cluster)
+    keep_ids = set(terms["ID"])
+    for cluster in duplicate_clusters:
+        sorted_ids = sorted(cluster)
+        keep_ids.difference_update(sorted_ids[1:])  # Remove all but smallest ID
+    
+    # Filter and clean up
+    filtered = terms[terms["ID"].isin(keep_ids)].copy()
+    filtered.drop(columns=["gene_set"], inplace=True)
+    
+    log.done(f"{len(terms) - len(filtered)} terms removed due to identical gene sets.")
+    return filtered

pythonflex/utils.py

@@ -0,0 +1,100 @@
+import os
+import re  # For sanitization (built-in, minimal regex)
+import tempfile
+import joblib
+import numpy as np
+import pandas as pd
+
+# Constants
+TMP_ROOT = ".tmp"
+VALID_EXTS = {".feather", ".npy", ".pkl"}
+
+# Helper to sanitize names (make filesystem-safe)
+def _sanitize(name):
+    if not name:
+        return "data"
+    # Replace forbidden/problematic chars with '_', collapse multiples, strip edges
+    safe = re.sub(r'[<>:"/\\|?*$,\s]+', '_', str(name).strip())
+    safe = re.sub(r'_+', '_', safe).strip('_')
+    return safe if safe else "data"
+
+# Helper to get safe path
+def _safe_path(category, name=None, ext=".pkl"):
+    safe_category = _sanitize(category)
+    dir_path = os.path.join(TMP_ROOT, safe_category)
+    os.makedirs(dir_path, exist_ok=True)
+    safe_name = _sanitize(name) if name else "data"
+    return os.path.join(dir_path, f"{safe_name}{ext}")
+
+# Save function
+def dsave(data, category, name=None, path=None):  # 'path' ignored for compatibility with old code
+    # If data is dict and no name, recurse on each item
+    if name is None and isinstance(data, dict):
+        for k, v in data.items():
+            dsave(v, category, k)
+        return
+
+    # Choose best extension based on type
+    if isinstance(data, pd.DataFrame):
+        ext = ".feather"
+        save_func = lambda p: data.to_feather(p)
+    elif isinstance(data, np.ndarray):
+        ext = ".npy"
+        save_func = lambda p: np.save(p, data, allow_pickle=False)
+    else:
+        ext = ".pkl"
+        save_func = lambda p: joblib.dump(data, p, compress=0)  # Add compress=3 if needed
+
+    target = _safe_path(category, name, ext)
+
+    # Atomic save: Write to temp file, then rename
+    with tempfile.NamedTemporaryFile(dir=os.path.dirname(target), delete=False) as tf:
+        tmp_path = tf.name
+        tf.close()  # Close so save_func can write
+        save_func(tmp_path)
+    os.replace(tmp_path, target)  # Atomic move
+
+# Load function
+def dload(category, name=None, path=None):  # 'path' ignored for compatibility
+    dir_path = os.path.join(TMP_ROOT, _sanitize(category))
+
+    if not os.path.exists(dir_path):
+        return {}
+
+    if name is None:
+        # Load all in category as dict
+        out = {}
+        for filename in os.listdir(dir_path):
+            if not any(filename.endswith(ext) for ext in VALID_EXTS):
+                continue
+            k = os.path.splitext(filename)[0]  # Key from filename (without ext)
+            full_path = os.path.join(dir_path, filename)
+            try:
+                if filename.endswith(".feather"):
+                    out[k] = pd.read_feather(full_path)
+                elif filename.endswith(".npy"):
+                    out[k] = np.load(full_path, mmap_mode="r")  # MMap for perf
+                elif filename.endswith(".pkl"):
+                    out[k] = joblib.load(full_path, mmap_mode="r")  # MMap for perf
+            except (EOFError, ValueError, OSError):
+                print(f"Warning: '{full_path}' is corrupted. Skipping...")
+                os.remove(full_path)  # Delete corrupted file
+        return out
+
+    # Load specific name (try extensions in order)
+    for ext in VALID_EXTS:
+        target = _safe_path(category, name, ext)
+        if os.path.exists(target):
+            try:
+                if ext == ".feather":
+                    return pd.read_feather(target)
+                elif ext == ".npy":
+                    return np.load(target, mmap_mode="r")  # MMap for perf
+                elif ext == ".pkl":
+                    return joblib.load(target, mmap_mode="r")  # MMap for perf
+            except (EOFError, ValueError, OSError):
+                print(f"Warning: '{target}' is corrupted. Deleting and returning {{}}...")
+                os.remove(target)
+                return {}
+    return {}
+

{pythonflex-0.1.2.dist-info → pythonflex-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pythonflex
-Version: 0.1.2
+Version: 0.1.3
 Summary: pythonFLEX is a benchmarking toolkit for evaluating CRISPR screen results against biological gold standards. The toolkit computes gene-level and complex-level performance metrics, helping researchers systematically assess the biological relevance and resolution of their CRISPR screening data.
 Author-email: Yasir Demirtaş <tyasird@hotmail.com>
 Requires-Python: >=3.9

pythonflex-0.1.3.dist-info/RECORD

@@ -0,0 +1,21 @@
+pythonflex/__init__.py,sha256=rz-8y-zEJQR3ThbhlIUV0N1q7Z-4UNHpHbOglXFp52c,1292
+pythonflex/analysis.py,sha256=Bu4wK7bnSlaXFRoqUjAGlQP5CoGYsrr1x_ZJRE1AuSc,52966
+pythonflex/logging_config.py,sha256=QrwI9NSpi026rGKgitky_zP67aH6lo7rKA2TTAyrm1E,2011
+pythonflex/plotting.py,sha256=ydC5CHIFl9GAGkYFyoXZ9v4ESKxTL_4vvoOkSR8hvCU,19508
+pythonflex/preprocessing.py,sha256=wBuN3qfe7-qczzLtugOtnV3Un4QK4Kc_G8AZ1MMlhNk,8575
+pythonflex/utils.py,sha256=nyVlGu5OXpz5YPj48hXueL5ja88sQ2PUiJ76c4USg4A,3886
+pythonflex/data/dataset/liver_cell_lines_500_genes.csv,sha256=qfKsqPjL41Y1GuxxAhc-MfaNO0mX6Qju_SeynKSpEiM,238639
+pythonflex/data/dataset/melanoma_cell_lines_500_genes.csv,sha256=ByxcaDDqLlRtAyuCKhHeFQCitIBk2-Q4Hn6k8BNUF6c,620887
+pythonflex/data/dataset/neuroblastoma_cell_lines_500_genes.csv,sha256=IxJI8E-smagbxHRvTjvQLZxuu89MuCw8XrReMSsViUI,365992
+pythonflex/data/gold_standard/CORUM.parquet,sha256=AkLiflQAeQ6K3HG-PIdLbZ8vEF9GtNObtlY7TkxHyaw,131858
+pythonflex/data/gold_standard/GOBP.parquet,sha256=YQGhRcHSiN_cMKytCUYNCfcDwYj9L3TLFNwobiS2f3M,1025099
+pythonflex/data/gold_standard/PATHWAY.parquet,sha256=bFRDe3PQ_TFc7B1uZuynwOGcgxESLaOy1Zt5gpQ1Oso,277386
+pythonflex/data/gold_standard/corum.csv,sha256=2rZeyr2Ghm7f-gFxCZnhPtxI2jxRoiZMUEH2EJwAgsI,208889
+pythonflex/data/gold_standard/gobp.csv,sha256=TO9yfx9mO8WkXvWfSB-pFId9T8xYfqdZpshAXC0Fyj8,1739167
+pythonflex/data/gold_standard/pathway.csv,sha256=J3HKVLUZ_Oxucmn_14ieYp3Wr2lcKtp0nIl4_8_K2Yc,489424
+pythonflex/examples/basic_usage.py,sha256=RBtJFphAE6NpCmSm9tMYN6FSdcpbWQzwkTlpX2ZejvI,2387
+pythonflex/examples/dataset_filtering.py,sha256=YBHgkbHj9SpIj-qLxQhHS4UWjERv4MHyb2t_fR6AONE,971
+pythonflex-0.1.3.dist-info/METADATA,sha256=U9S0qymqzZ0NUxAAy7Vjn9YXqRaSxdcSm3zI2wd-aoI,3928
+pythonflex-0.1.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pythonflex-0.1.3.dist-info/entry_points.txt,sha256=37liK1baI_CRVDivpjsn8JDClL9_YeTTuSMAZ3Ty7oE,47
+pythonflex-0.1.3.dist-info/RECORD,,

pythonflex-0.1.2.dist-info/RECORD

@@ -1,4 +0,0 @@
-pythonflex-0.1.2.dist-info/METADATA,sha256=LGYUDuQ3o_ANpHA6qovfk1w1VqNuz0pb-YwBs8tMNO0,3928
-pythonflex-0.1.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-pythonflex-0.1.2.dist-info/entry_points.txt,sha256=37liK1baI_CRVDivpjsn8JDClL9_YeTTuSMAZ3Ty7oE,47
-pythonflex-0.1.2.dist-info/RECORD,,