pythonflex 0.3.3__py3-none-any.whl → 0.4__py3-none-any.whl

pythonflex/preprocessing.py

@@ -1,7 +1,7 @@
 import os
 import pandas as pd
 import numpy as np
-from .utils import dsave, dload
+from .utils import dsave, dload, normalize_analysis_genes
 from tqdm import tqdm
 from .logging_config import log
 tqdm.pandas()
@@ -68,7 +68,16 @@ def _load_file(filepath, ext):
 def load_datasets(files, continue_with_common_genes=False):
     config = dload("config")
     preprocessing = config["preprocessing"]
-    use_common_genes = config.get("use_common_genes", True)
+    analysis_genes_raw = config.get("analysis_genes", "")
+    analysis_genes_missing = (
+        analysis_genes_raw is None or str(analysis_genes_raw).strip() == ""
+    )
+    analysis_genes = normalize_analysis_genes(
+        analysis_genes_raw,
+        legacy_use_common_genes=(
+            config.get("use_common_genes") if analysis_genes_missing else None
+        ),
+    )
     data_dict= {}     
 
     for filename, meta in files.items():
@@ -103,14 +112,16 @@ def load_datasets(files, continue_with_common_genes=False):
 
     common_genes = get_common_genes(data_dict)
 
-    # Apply common gene filtering only if use_common_genes is True
-    if use_common_genes and (continue_with_common_genes or use_common_genes):
-        log.info(f"Applying common gene filtering: {len(common_genes)} genes")
-        for filename, df in data_dict.items():
-            if df.index.isin(common_genes).any():
-                data_dict[filename] = df.loc[common_genes]
-    elif not use_common_genes:
-        log.info(f"Skipping common gene filtering (use_common_genes=False). Common genes found: {len(common_genes)}")
+    # Apply common gene filtering only when analysis_genes='shared' (or forced by arg)
+    if analysis_genes == "shared" or continue_with_common_genes:
+        log.info(f"Applying common gene filtering: {len(common_genes)} genes")
+        for filename, df in data_dict.items():
+            if df.index.isin(common_genes).any():
+                data_dict[filename] = df.loc[common_genes]
+    else:
+        log.info(
+            f"Skipping common gene filtering (analysis_genes='dataset_specific'). Common genes found: {len(common_genes)}"
+        )
     
     dsave({
         "datasets": data_dict,
@@ -175,7 +186,7 @@ def filter_matrix_by_genes(matrix, genes_present_in_terms):
     genes = matrix.index.intersection(genes_present_in_terms)
     matrix = matrix.loc[genes, genes]
     log.done(f"Filtering matrix: {matrix.shape}")
-    return matrix.loc[genes, genes]
+    return matrix
     
 
 
@@ -185,11 +196,25 @@ def load_gold_standard():
     package_dir = return_package_dir()
     data_dir_path = os.path.join(package_dir, 'data')
     
-    config = dload("config") 
-    use_common_genes = config.get("use_common_genes", True)
+    config = dload("config") 
+    analysis_genes_raw = config.get("analysis_genes", "")
+    analysis_genes_missing = (
+        analysis_genes_raw is None or str(analysis_genes_raw).strip() == ""
+    )
+    analysis_genes = normalize_analysis_genes(
+        analysis_genes_raw,
+        legacy_use_common_genes=(
+            config.get("use_common_genes") if analysis_genes_missing else None
+        ),
+    )
 
     gold_standard_source = config['gold_standard']
-    log.started(f"Loading gold standard: {gold_standard_source}, Min complex size: {config['min_genes_in_complex']}, Jaccard filtering: {config['jaccard']}, use_common_genes: {use_common_genes}")
+    jaccard_enabled = bool(config.get("jaccard", False))
+    log.done(
+        f"Loading gold standard: {gold_standard_source}, Min complex size: {config['min_genes_in_complex']}, "
+        f"Jaccard filtering: {jaccard_enabled} (exact duplicate used_genes after dataset filtering), "
+        f"analysis_genes: {analysis_genes}"
+    )
 
     # Define gold standard file paths for predefined sources
     gold_standard_files = {
@@ -217,34 +242,12 @@ def load_gold_standard():
 
     # Store raw gold standard for later per-dataset filtering
     terms["all_genes"] = terms["Genes"].apply(lambda x: list(set(x.split(";"))))
-    log.info(f"Gold standard loaded with {len(terms)} terms")
+    log.done(f"Gold standard loaded with {len(terms)} terms")
 
     # Basic filtering by minimum complex size (before gene filtering)
     terms["n_all_genes"] = terms["all_genes"].apply(len) 
     terms = terms[terms["n_all_genes"] >= config['min_genes_in_complex']]
-    log.info(f"After min_genes_in_complex filtering: {len(terms)} terms")
-
-    if config['jaccard']:
-        log.info("Applying Jaccard filtering. Remove terms with identical gene sets.")
-        # Use all genes for jaccard filtering
-        terms["gene_set"] = terms["all_genes"].map(lambda x: frozenset(x))
-        grouped = terms.groupby("gene_set", sort=False)
-        duplicate_clusters = []
-        for _, group in grouped:
-            if len(group) > 1:
-                duplicate_clusters.append(group["ID"].values if "ID" in group.columns else group.index.values)
-        
-        keep_ids = set(terms["ID"] if "ID" in terms.columns else terms.index)
-        for cluster in duplicate_clusters:
-            sorted_ids = sorted(cluster)
-            keep_ids.difference_update(sorted_ids[1:])
-        
-        if "ID" in terms.columns:
-            terms = terms[terms["ID"].isin(keep_ids)].copy()
-        else:
-            terms = terms[terms.index.isin(keep_ids)].copy()
-        terms.drop(columns=["gene_set"], inplace=True, errors='ignore')
-        log.info(f"After Jaccard filtering: {len(terms)} terms")
+    log.done(f"After min_genes_in_complex filtering: {len(terms)} terms")
 
     # if there is column called "ID", set it as index
     if "ID" in terms.columns:
@@ -258,31 +261,30 @@ def load_gold_standard():
 
 
 
-def filter_duplicate_terms(terms):
-    log.started("Filtering duplicate terms using optimized method.")
-    
-    # Precompute frozen gene sets and hash them
+def filter_duplicate_terms(terms: pd.DataFrame) -> pd.DataFrame:
+    """Backward-compatible wrapper for exact-duplicate filtering.
+
+    This removes exact duplicate `used_genes` sets and keeps the smallest ID.
+    """
+    log.started("Filtering duplicate terms using exact used_genes sets.")
+    before = len(terms)
     terms = terms.copy()
     terms["gene_set"] = terms["used_genes"].map(lambda x: frozenset(x))
-    
-    # Group by identical gene sets
     grouped = terms.groupby("gene_set", sort=False)
-    
-    # Identify duplicate clusters (groups with >1 term)
-    duplicate_clusters = []
+
+    id_values = terms["ID"] if "ID" in terms.columns else terms.index
+    keep_ids = set(id_values)
     for _, group in grouped:
-        if len(group) > 1:
-            duplicate_clusters.append(group["ID"].values)
-    
-    # Determine which IDs to keep (smallest ID in each duplicate cluster)
-    keep_ids = set(terms["ID"])
-    for cluster in duplicate_clusters:
+        if len(group) <= 1:
+            continue
+        cluster = group["ID"].values if "ID" in group.columns else group.index.values
         sorted_ids = sorted(cluster)
-        keep_ids.difference_update(sorted_ids[1:])  # Remove all but smallest ID
-    
-    # Filter and clean up
-    filtered = terms[terms["ID"].isin(keep_ids)].copy()
-    filtered.drop(columns=["gene_set"], inplace=True)
-    
-    log.done(f"{len(terms) - len(filtered)} terms removed due to identical gene sets.")
+        keep_ids.difference_update(sorted_ids[1:])
+
+    if "ID" in terms.columns:
+        filtered = terms[terms["ID"].isin(keep_ids)].copy()
+    else:
+        filtered = terms[terms.index.isin(keep_ids)].copy()
+    filtered.drop(columns=["gene_set"], inplace=True, errors="ignore")
+    log.done(f"{before - len(filtered)} terms removed due to identical gene sets.")
     return filtered

pythonflex/utils.py

@@ -5,12 +5,39 @@ import joblib
 import numpy as np
 import pandas as pd
 
+from .logging_config import log
+
 # Constants
 TMP_ROOT = ".tmp"
-VALID_EXTS = {".parquet", ".npy", ".pkl"}  # Removed .feather
-
-# Helper to sanitize names (make filesystem-safe)
-def _sanitize(name):
+VALID_EXTS = {".parquet", ".npy", ".pkl"}  # Removed .feather
+
+ANALYSIS_GENES_ALIASES = {
+    "shared": "shared",
+    "common": "shared",
+    "dataset_specific": "dataset_specific",
+    "dataset": "dataset_specific",
+}
+
+
+def normalize_analysis_genes(value=None, legacy_use_common_genes=None):
+    """Normalize analysis_genes config values while keeping legacy names working."""
+    if value is None or str(value).strip() == "":
+        if legacy_use_common_genes is not None:
+            return "shared" if bool(legacy_use_common_genes) else "dataset_specific"
+        value = "shared"
+
+    key = str(value).strip().lower().replace("-", "_").replace(" ", "_")
+    if key not in ANALYSIS_GENES_ALIASES:
+        raise ValueError(
+            "config['analysis_genes'] must be one of "
+            "['shared', 'dataset_specific'] "
+            "(legacy aliases ['common', 'dataset'] are also supported), "
+            f"got {value!r}"
+        )
+    return ANALYSIS_GENES_ALIASES[key]
+
+# Helper to sanitize names (make filesystem-safe)
+def _sanitize(name):
     if not name:
         return "data"
     # Replace forbidden/problematic chars with '_', collapse multiples, strip edges
@@ -59,7 +86,7 @@ def dload(category, name=None, path=None):
     dir_path = os.path.join(TMP_ROOT, _sanitize(category))
 
     if not os.path.exists(dir_path):
-        return {}
+        return {} if name is None else None
 
     if name is None:
         # Load all in category as dict
@@ -77,7 +104,7 @@ def dload(category, name=None, path=None):
                 elif filename.endswith(".pkl"):
                     out[k] = joblib.load(full_path, mmap_mode="r")
             except (EOFError, ValueError, OSError):
-                print(f"Warning: '{full_path}' is corrupted. Skipping...")
+                log.warning(f"'{full_path}' is corrupted. Skipping...")
                 os.remove(full_path)
         return out
 
@@ -93,9 +120,9 @@ def dload(category, name=None, path=None):
                 elif ext == ".pkl":
                     return joblib.load(target, mmap_mode="r")
             except (EOFError, ValueError, OSError) as e:
-                print(f"Warning: '{target}' is corrupted ({e}). Trying next format...")
+                log.warning(f"'{target}' is corrupted ({e}). Trying next format...")
                 os.remove(target)
                 continue
     
-    print(f"Warning: No valid file found for {category}/{name}")
-    return {}
+    log.warning(f"No valid file found for {category}/{name}")
+    return None

{pythonflex-0.3.3.dist-info → pythonflex-0.4.dist-info}/METADATA

@@ -1,8 +1,9 @@
 Metadata-Version: 2.4
 Name: pythonflex
-Version: 0.3.3
+Version: 0.4
 Summary: pythonFLEX is a benchmarking toolkit for evaluating CRISPR screen results against biological gold standards. The toolkit computes gene-level and complex-level performance metrics, helping researchers systematically assess the biological relevance and resolution of their CRISPR screening data.
 Author-email: Yasir Demirtaş <tyasird@hotmail.com>
+License-File: LICENSE
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
@@ -114,6 +115,10 @@ default_config = {
     "gold_standard": "GOBP",
     "color_map": "RdYlBu",
     "jaccard": True,
+    # Which genes define the evaluated space:
+    # - 'common'  : intersect terms with genes common across datasets
+    # - 'dataset' : intersect terms with genes present in each dataset
+    "analysis_genes": "common",
     "plotting": {
         "save_plot": True,
         "output_type": "png",
@@ -124,7 +129,7 @@ default_config = {
     },
     "corr_function": "numpy",
     "logging": {  
-        "visible_levels": ["DONE","STARTED"]  # "PROGRESS", "STARTED", ,"INFO","WARNING"
+        "visible_levels": ["DONE","INFO", "WARNING"]  # "PROGRESS", "STARTED", ,"INFO","WARNING"
     }
 }
 
@@ -138,7 +143,7 @@ terms, genes_in_terms = flex.load_gold_standard()
 
 # Run analysis
 for name, dataset in data.items():
-    df, pr_auc = flex.pra(name, dataset)
+    df = flex.pra(name, dataset)
     fpc = flex.pra_percomplex(name, dataset, is_corr=False) 
     cc = flex.complex_contributions(name)
 
@@ -149,8 +154,8 @@ flex.plot_percomplex_scatter()
 flex.plot_percomplex_scatter_bysize()
 flex.plot_significant_complexes()
 flex.plot_complex_contributions()
+flex.plot_mpr_summary(variants="unfiltered")
 
-# Save Result CSVspyflex.save_results_to_csv()
 flex.save_results_to_csv()
 
 

pythonflex-0.4.dist-info/RECORD

@@ -0,0 +1,32 @@
+pythonflex/__init__.py,sha256=D3go2UnojWzRAFr8ahjrPwDTR3np__hDHqaAhBqUGHc,2640
+pythonflex/analysis.py,sha256=H164EqsG5ewgshaYQOXUOh5pjF7TcWXGsmQvWzHFT6M,64615
+pythonflex/logging_config.py,sha256=iqRKK18zvtfV_-bYHWrXtSZywiUtYxoHkw0ZnVORQBQ,2015
+pythonflex/old_functions.py,sha256=regtkNGCS3ph0OBssAg8Sg1ivn1-kiRB54Q7xIVeQ4E,19031
+pythonflex/plotting.py,sha256=Ie0F79q-DSkn8YzytjJf2FQthfvY_OWlR3B_sJMpXac,89247
+pythonflex/preprocessing.py,sha256=XwLul6MQN2Gzjg-_JH0I14qklRslO8f6GCoYk3JpWpA,11251
+pythonflex/utils.py,sha256=WqsCEnccNGIEwWuq_r34mIqDBBuu8mkFXW4zPIm0IwA,4665
+pythonflex/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pythonflex/data/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pythonflex/data/dataset/liver_cell_lines_500_genes.csv,sha256=qfKsqPjL41Y1GuxxAhc-MfaNO0mX6Qju_SeynKSpEiM,238639
+pythonflex/data/dataset/melanoma_cell_lines_500_genes.csv,sha256=ByxcaDDqLlRtAyuCKhHeFQCitIBk2-Q4Hn6k8BNUF6c,620887
+pythonflex/data/dataset/neuroblastoma_cell_lines_500_genes.csv,sha256=IxJI8E-smagbxHRvTjvQLZxuu89MuCw8XrReMSsViUI,365992
+pythonflex/data/gold_standard/CORUM.parquet,sha256=AkLiflQAeQ6K3HG-PIdLbZ8vEF9GtNObtlY7TkxHyaw,131858
+pythonflex/data/gold_standard/GOBP.parquet,sha256=YQGhRcHSiN_cMKytCUYNCfcDwYj9L3TLFNwobiS2f3M,1025099
+pythonflex/data/gold_standard/PATHWAY.parquet,sha256=bFRDe3PQ_TFc7B1uZuynwOGcgxESLaOy1Zt5gpQ1Oso,277386
+pythonflex/data/gold_standard/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pythonflex/data/gold_standard/corum.csv,sha256=2rZeyr2Ghm7f-gFxCZnhPtxI2jxRoiZMUEH2EJwAgsI,208889
+pythonflex/data/gold_standard/gobp.csv,sha256=TO9yfx9mO8WkXvWfSB-pFId9T8xYfqdZpshAXC0Fyj8,1739167
+pythonflex/data/gold_standard/pathway.csv,sha256=J3HKVLUZ_Oxucmn_14ieYp3Wr2lcKtp0nIl4_8_K2Yc,489424
+pythonflex/examples/basic_usage.py,sha256=06IPn_yon9DwXefpfb0Fo6uG_5CyRCT_ZCMZSPM-3Ww,2695
+pythonflex/examples/manuscript.py,sha256=AyzycXys9BII4TZFyBi-bXksAWlrTcFZHCdc2gu8D0U,2984
+pythonflex/examples/runtime/runtime_benchmark.py,sha256=kjOELNHzbk8Y-DDe1IPl75R2UNnxfdgxZBR_SDdcQgc,5688
+pythonflex/examples/runtime/runtime_benchmark_10_runs_memmap.py,sha256=kXQ8ycnqmWxbjoEWoyT1DC3O1JWl90tlNvA7KVskfz4,14832
+pythonflex/examples/runtime/runtime_benchmark_corum_njobs.py,sha256=n6ON4cRHuN5y1YU32OH2lsXF-TPNQ4B2635MtwCOXKY,6361
+pythonflex/examples/runtime/runtime_benchmark_gobp_njobs_chunks.py,sha256=sC3cMV8MxFqT8GWFgskkIMx6jz7if6BWWkhC7jip0ks,9130
+pythonflex/examples/runtime/runtime_benchmark_gobp_optimization.py,sha256=1Us2sISWbjaM2hahQ1hG-XSrlTaWzcDnvVQrItHo_ls,11934
+pythonflex/examples/runtime/runtime_benchmark_repeated.py,sha256=6-NV1aLbScdmH13heu6_rROAJk0L1VZ1byO95kMtzX8,9666
+pythonflex-0.4.dist-info/METADATA,sha256=TxzZaLktNrLQvfHu2xcbWPr_nOt7VaHz2sO24BFm7-Q,4463
+pythonflex-0.4.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+pythonflex-0.4.dist-info/entry_points.txt,sha256=37liK1baI_CRVDivpjsn8JDClL9_YeTTuSMAZ3Ty7oE,47
+pythonflex-0.4.dist-info/licenses/LICENSE,sha256=buBzPy38DV0g95acwIGUqsWZwhbpc0tQXnz6nBhjyS8,1091
+pythonflex-0.4.dist-info/RECORD,,

{pythonflex-0.3.3.dist-info → pythonflex-0.4.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.29.0
+Generator: hatchling 1.30.1
 Root-Is-Purelib: true
 Tag: py3-none-any

pythonflex-0.4.dist-info/licenses/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2026, Yasir Demirtas, Maximilian Billmann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

pythonflex-0.3.3.dist-info/RECORD

@@ -1,24 +0,0 @@
-pythonflex/__init__.py,sha256=MoDbdVhclK_PF_u9vzN4ntWX6hTRAKfvkTiDisIci5o,1748
-pythonflex/analysis.py,sha256=gKJ4cYA_TWYe521nAXizMqChd36A90TWfDf595fw_0M,77760
-pythonflex/logging_config.py,sha256=iqRKK18zvtfV_-bYHWrXtSZywiUtYxoHkw0ZnVORQBQ,2015
-pythonflex/plotting.py,sha256=AOzgyhJX5bPMoGs2ih2zbA30Dm-OoWpk8MNBC-9OQ94,75981
-pythonflex/preprocessing.py,sha256=jIeyB2SPPac-svtjB-zGe3vIyOSVB-SxYIFyNFFiCsY,11440
-pythonflex/utils.py,sha256=7toGnKbA_TKBtHz1HLk7ckWM0bjuFw_Byhp6ZUJaNs4,3694
-pythonflex/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pythonflex/data/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pythonflex/data/dataset/liver_cell_lines_500_genes.csv,sha256=qfKsqPjL41Y1GuxxAhc-MfaNO0mX6Qju_SeynKSpEiM,238639
-pythonflex/data/dataset/melanoma_cell_lines_500_genes.csv,sha256=ByxcaDDqLlRtAyuCKhHeFQCitIBk2-Q4Hn6k8BNUF6c,620887
-pythonflex/data/dataset/neuroblastoma_cell_lines_500_genes.csv,sha256=IxJI8E-smagbxHRvTjvQLZxuu89MuCw8XrReMSsViUI,365992
-pythonflex/data/gold_standard/CORUM.parquet,sha256=AkLiflQAeQ6K3HG-PIdLbZ8vEF9GtNObtlY7TkxHyaw,131858
-pythonflex/data/gold_standard/GOBP.parquet,sha256=YQGhRcHSiN_cMKytCUYNCfcDwYj9L3TLFNwobiS2f3M,1025099
-pythonflex/data/gold_standard/PATHWAY.parquet,sha256=bFRDe3PQ_TFc7B1uZuynwOGcgxESLaOy1Zt5gpQ1Oso,277386
-pythonflex/data/gold_standard/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pythonflex/data/gold_standard/corum.csv,sha256=2rZeyr2Ghm7f-gFxCZnhPtxI2jxRoiZMUEH2EJwAgsI,208889
-pythonflex/data/gold_standard/gobp.csv,sha256=TO9yfx9mO8WkXvWfSB-pFId9T8xYfqdZpshAXC0Fyj8,1739167
-pythonflex/data/gold_standard/pathway.csv,sha256=J3HKVLUZ_Oxucmn_14ieYp3Wr2lcKtp0nIl4_8_K2Yc,489424
-pythonflex/examples/basic_usage.py,sha256=dizQXYPJWjW7-2d2G29a8qYCBRIsKhrvxOxyXtudK30,2265
-pythonflex/examples/manuscript.py,sha256=V28vIBFmrxGsE_YhvouRFiLKWC9CorbOx9Ed3B2L8bQ,2810
-pythonflex-0.3.3.dist-info/METADATA,sha256=l5CnF5hX_qgnhMEHnTQbK9ZrBJrIRKzbYeCVCC7Wv1M,4226
-pythonflex-0.3.3.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
-pythonflex-0.3.3.dist-info/entry_points.txt,sha256=37liK1baI_CRVDivpjsn8JDClL9_YeTTuSMAZ3Ty7oE,47
-pythonflex-0.3.3.dist-info/RECORD,,