pythonflex 0.3.3__tar.gz → 0.3.4__tar.gz

{pythonflex-0.3.3 → pythonflex-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pythonflex
-Version: 0.3.3
+Version: 0.3.4
 Summary: pythonFLEX is a benchmarking toolkit for evaluating CRISPR screen results against biological gold standards. The toolkit computes gene-level and complex-level performance metrics, helping researchers systematically assess the biological relevance and resolution of their CRISPR screening data.
 Author-email: Yasir Demirtaş <tyasird@hotmail.com>
 Classifier: License :: OSI Approved :: MIT License
@@ -114,6 +114,7 @@ default_config = {
     "gold_standard": "GOBP",
     "color_map": "RdYlBu",
     "jaccard": True,
+    "jaccard_threshold": 1.0,  # set e.g. 0.90 to remove highly similar terms
     "plotting": {
         "save_plot": True,
         "output_type": "png",
@@ -124,7 +125,7 @@ default_config = {
     },
     "corr_function": "numpy",
     "logging": {  
-        "visible_levels": ["DONE","STARTED"]  # "PROGRESS", "STARTED", ,"INFO","WARNING"
+        "visible_levels": ["DONE","INFO", "WARNING"]  # "PROGRESS", "STARTED", ,"INFO","WARNING"
     }
 }
 
@@ -149,8 +150,10 @@ flex.plot_percomplex_scatter()
 flex.plot_percomplex_scatter_bysize()
 flex.plot_significant_complexes()
 flex.plot_complex_contributions()
+flex.plot_mpr_tp_multi(show_filters="all")
+flex.plot_mpr_complexes_multi(show_filters="all")
+flex.plot_mpr_complexes_auc_scores("all")
 
-# Save Result CSVspyflex.save_results_to_csv()
 flex.save_results_to_csv()
 
 

{pythonflex-0.3.3 → pythonflex-0.3.4}/README.md

@@ -83,6 +83,7 @@ default_config = {
     "gold_standard": "GOBP",
     "color_map": "RdYlBu",
     "jaccard": True,
+    "jaccard_threshold": 1.0,  # set e.g. 0.90 to remove highly similar terms
     "plotting": {
         "save_plot": True,
         "output_type": "png",
@@ -93,7 +94,7 @@ default_config = {
     },
     "corr_function": "numpy",
     "logging": {  
-        "visible_levels": ["DONE","STARTED"]  # "PROGRESS", "STARTED", ,"INFO","WARNING"
+        "visible_levels": ["DONE","INFO", "WARNING"]  # "PROGRESS", "STARTED", ,"INFO","WARNING"
     }
 }
 
@@ -118,8 +119,10 @@ flex.plot_percomplex_scatter()
 flex.plot_percomplex_scatter_bysize()
 flex.plot_significant_complexes()
 flex.plot_complex_contributions()
+flex.plot_mpr_tp_multi(show_filters="all")
+flex.plot_mpr_complexes_multi(show_filters="all")
+flex.plot_mpr_complexes_auc_scores("all")
 
-# Save Result CSVspyflex.save_results_to_csv()
 flex.save_results_to_csv()
 
 

{pythonflex-0.3.3 → pythonflex-0.3.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pythonflex"
-version = "0.3.3"
+version = "0.3.4"
 description = "pythonFLEX is a benchmarking toolkit for evaluating CRISPR screen results against biological gold standards. The toolkit computes gene-level and complex-level performance metrics, helping researchers systematically assess the biological relevance and resolution of their CRISPR screening data."
 readme = "README.md"
 authors = [

{pythonflex-0.3.3 → pythonflex-0.3.4}/src/pythonflex/analysis.py

@@ -43,6 +43,7 @@ def initialize(config={}):
         "gold_standard": "CORUM",
         "color_map": "RdYlBu",
         "jaccard": True,
+        "jaccard_threshold": 1.0,
         "use_common_genes": True,
         "plotting": {
             "save_plot": True,

{pythonflex-0.3.3 → pythonflex-0.3.4}/src/pythonflex/examples/basic_usage.py

@@ -31,7 +31,8 @@ default_config = {
     "output_folder": "CORUM",
     "gold_standard": "CORUM",
     "color_map": "BuGn",
-    "jaccard": False,
+    "jaccard": True,
+    "jaccard_threshold": 1,
     "use_common_genes": False,  # Set to False for individual dataset-gold standard intersections
     "plotting": {
         "save_plot": True,
@@ -61,6 +62,7 @@ for name, dataset in data.items():
     fpc = flex.pra_percomplex(name, dataset, is_corr=False) 
     cc = flex.complex_contributions(name)
     flex.mpr_prepare(name)  
+    
 
 
 
@@ -73,15 +75,13 @@ for name, dataset in data.items():
 # flex.plot_percomplex_scatter(n_top=20)
 # flex.plot_percomplex_scatter_bysize()
 # flex.plot_complex_contributions()
-#%%
-#flex.plot_mpr_tp_multi(show_filters="all")
-flex.plot_mpr_complexes_multi(show_filters="all")
+# flex.plot_mpr_tp_multi(show_filters="all")
+# flex.plot_mpr_complexes_multi(show_filters="all")
+# flex.plot_mpr_complexes_auc_scores("all")
+
+
 
 #%%
 # Save results to CSV
-flex.save_results_to_csv()
-
+# flex.save_results_to_csv()
 
-# %%
-flex.plot_mpr_complexes_auc_scores("all")
-# %%

{pythonflex-0.3.3 → pythonflex-0.3.4}/src/pythonflex/examples/manuscript.py

@@ -58,6 +58,7 @@ default_config = {
     "gold_standard": "CORUM",
     "color_map": "BuGn",
     "jaccard": False,
+        "jaccard_threshold": 1.0,
     "use_common_genes": False,  # Set to False for individual dataset-gold standard intersections
     "plotting": {
         "save_plot": True,

{pythonflex-0.3.3 → pythonflex-0.3.4}/src/pythonflex/preprocessing.py

@@ -189,7 +189,18 @@ def load_gold_standard():
     use_common_genes = config.get("use_common_genes", True)
 
     gold_standard_source = config['gold_standard']
-    log.started(f"Loading gold standard: {gold_standard_source}, Min complex size: {config['min_genes_in_complex']}, Jaccard filtering: {config['jaccard']}, use_common_genes: {use_common_genes}")
+    jaccard_enabled = bool(config.get("jaccard", False))
+    jaccard_threshold_raw = config.get("jaccard_threshold", 1.0)
+    try:
+        jaccard_threshold = float(jaccard_threshold_raw)  # type: ignore[arg-type]
+    except (TypeError, ValueError):
+        raise ValueError(
+            f"config['jaccard_threshold'] must be a number in (0, 1], got {jaccard_threshold_raw!r}"
+        )
+    log.done(
+        f"Loading gold standard: {gold_standard_source}, Min complex size: {config['min_genes_in_complex']}, "
+        f"Jaccard filtering: {jaccard_enabled} (threshold={jaccard_threshold}), use_common_genes: {use_common_genes}"
+    )
 
     # Define gold standard file paths for predefined sources
     gold_standard_files = {
@@ -217,34 +228,44 @@ def load_gold_standard():
 
     # Store raw gold standard for later per-dataset filtering
     terms["all_genes"] = terms["Genes"].apply(lambda x: list(set(x.split(";"))))
-    log.info(f"Gold standard loaded with {len(terms)} terms")
+    log.done(f"Gold standard loaded with {len(terms)} terms")
 
     # Basic filtering by minimum complex size (before gene filtering)
     terms["n_all_genes"] = terms["all_genes"].apply(len) 
     terms = terms[terms["n_all_genes"] >= config['min_genes_in_complex']]
-    log.info(f"After min_genes_in_complex filtering: {len(terms)} terms")
-
-    if config['jaccard']:
-        log.info("Applying Jaccard filtering. Remove terms with identical gene sets.")
-        # Use all genes for jaccard filtering
-        terms["gene_set"] = terms["all_genes"].map(lambda x: frozenset(x))
-        grouped = terms.groupby("gene_set", sort=False)
-        duplicate_clusters = []
-        for _, group in grouped:
-            if len(group) > 1:
-                duplicate_clusters.append(group["ID"].values if "ID" in group.columns else group.index.values)
-        
-        keep_ids = set(terms["ID"] if "ID" in terms.columns else terms.index)
-        for cluster in duplicate_clusters:
-            sorted_ids = sorted(cluster)
-            keep_ids.difference_update(sorted_ids[1:])
-        
-        if "ID" in terms.columns:
-            terms = terms[terms["ID"].isin(keep_ids)].copy()
+    log.done(f"After min_genes_in_complex filtering: {len(terms)} terms")
+
+    if jaccard_enabled:
+        if not (0.0 < jaccard_threshold <= 1.0):
+            raise ValueError(f"config['jaccard_threshold'] must be in (0, 1], got {jaccard_threshold}")
+
+        if jaccard_threshold >= 1.0:
+            log.done("Applying Jaccard filtering (threshold=1.0). Removing terms with identical gene sets.")
+            # Use all genes for jaccard filtering
+            terms["gene_set"] = terms["all_genes"].map(lambda x: frozenset(x))
+            grouped = terms.groupby("gene_set", sort=False)
+            duplicate_clusters = []
+            for _, group in grouped:
+                if len(group) > 1:
+                    duplicate_clusters.append(group["ID"].values if "ID" in group.columns else group.index.values)
+
+            keep_ids = set(terms["ID"] if "ID" in terms.columns else terms.index)
+            for cluster in duplicate_clusters:
+                sorted_ids = sorted(cluster)
+                keep_ids.difference_update(sorted_ids[1:])
+
+            if "ID" in terms.columns:
+                terms = terms[terms["ID"].isin(keep_ids)].copy()
+            else:
+                terms = terms[terms.index.isin(keep_ids)].copy()
+            terms.drop(columns=["gene_set"], inplace=True, errors="ignore")
+            log.done(f"After Jaccard filtering: {len(terms)} terms")
         else:
-            terms = terms[terms.index.isin(keep_ids)].copy()
-        terms.drop(columns=["gene_set"], inplace=True, errors='ignore')
-        log.info(f"After Jaccard filtering: {len(terms)} terms")
+            log.done(
+                f"Applying Jaccard filtering (threshold={jaccard_threshold}). Removing highly similar terms."
+            )
+            terms = _filter_terms_by_jaccard_threshold(terms, threshold=jaccard_threshold, genes_col="all_genes")
+            log.done(f"After Jaccard filtering: {len(terms)} terms")
 
     # if there is column called "ID", set it as index
     if "ID" in terms.columns:
@@ -255,6 +276,149 @@ def load_gold_standard():
     return terms, None  # Return None for genes_present_in_terms - will be computed per dataset
 
 
+class _UnionFind:
+    def __init__(self, n: int):
+        self.parent = list(range(n))
+        self.rank = [0] * n
+
+    def find(self, x: int) -> int:
+        while self.parent[x] != x:
+            self.parent[x] = self.parent[self.parent[x]]
+            x = self.parent[x]
+        return x
+
+    def union(self, a: int, b: int) -> None:
+        ra, rb = self.find(a), self.find(b)
+        if ra == rb:
+            return
+        if self.rank[ra] < self.rank[rb]:
+            self.parent[ra] = rb
+        elif self.rank[ra] > self.rank[rb]:
+            self.parent[rb] = ra
+        else:
+            self.parent[rb] = ra
+            self.rank[ra] += 1
+
+
+def _safe_id_sort_key(val):
+    """Sort key that prefers numeric ordering when IDs look like ints."""
+    try:
+        return (0, int(val))
+    except Exception:
+        return (1, str(val))
+
+
+def _jaccard_similarity(a: set, b: set) -> float:
+    if not a and not b:
+        return 1.0
+    if not a or not b:
+        return 0.0
+    inter = len(a.intersection(b))
+    if inter == 0:
+        return 0.0
+    union = len(a) + len(b) - inter
+    return inter / union
+
+
+def _filter_terms_by_jaccard_threshold(terms: pd.DataFrame, threshold: float, genes_col: str = "all_genes") -> pd.DataFrame:
+    """Remove near-duplicate terms whose gene sets have Jaccard similarity >= threshold.
+
+    Keeps one representative per similarity-connected component (smallest ID).
+    This uses an exact Jaccard similarity join with prefix-filter candidate generation.
+    """
+    if not (0.0 < threshold < 1.0):
+        # threshold == 1.0 handled elsewhere; invalid values rejected earlier
+        return terms
+
+    # Build IDs and gene sets
+    id_col = "ID" if "ID" in terms.columns else None
+    term_ids = (terms["ID"].tolist() if id_col else terms.index.tolist())
+    gene_sets = []
+    for genes in terms[genes_col].tolist():
+        gene_sets.append(set(genes))
+
+    sizes = [len(s) for s in gene_sets]
+    if len(gene_sets) <= 1:
+        return terms
+
+    # Global token frequency for ordering (rare tokens first)
+    from collections import Counter, defaultdict
+    freq = Counter()
+    for s in gene_sets:
+        freq.update(s)
+
+    def sort_tokens(s: set):
+        return sorted(s, key=lambda tok: (freq.get(tok, 0), str(tok)))
+
+    # Process smaller sets first (helps size filtering and keeps index smaller)
+    order = sorted(range(len(gene_sets)), key=lambda i: (sizes[i], _safe_id_sort_key(term_ids[i])))
+    ordered_tokens = [sort_tokens(gene_sets[i]) for i in range(len(gene_sets))]
+
+    # Inverted index over prefix tokens
+    inv_index = defaultdict(list)  # token -> list of processed term indices (original idx)
+
+    uf = _UnionFind(len(gene_sets))
+
+    # Precompute prefix lengths
+    import math
+    prefix_len = []
+    for i in range(len(gene_sets)):
+        m = sizes[i]
+        # PPJoin prefix length for Jaccard threshold
+        p = m - math.ceil(threshold * m) + 1
+        if p < 0:
+            p = 0
+        if p > m:
+            p = m
+        prefix_len.append(p)
+
+    # Candidate generation + exact verification
+    for idx_pos, i in enumerate(order):
+        tokens_i = ordered_tokens[i]
+        p_i = prefix_len[i]
+
+        # Count shared prefix tokens with previously indexed sets
+        candidate_overlap_lb = defaultdict(int)
+        for tok in tokens_i[:p_i]:
+            for j in inv_index.get(tok, []):
+                # size filter: if too different in size, cannot meet Jaccard threshold
+                if sizes[j] < threshold * sizes[i]:
+                    continue
+                if sizes[j] > sizes[i] / threshold:
+                    continue
+                candidate_overlap_lb[j] += 1
+            inv_index[tok].append(i)
+
+        if not candidate_overlap_lb:
+            continue
+
+        set_i = gene_sets[i]
+        for j in candidate_overlap_lb.keys():
+            # Exact verification
+            sim = _jaccard_similarity(set_i, gene_sets[j])
+            if sim >= threshold:
+                uf.union(i, j)
+
+    # Choose representative (smallest ID) for each connected component
+    components = {}
+    for i in range(len(gene_sets)):
+        root = uf.find(i)
+        components.setdefault(root, []).append(i)
+
+    keep_original_indices = set()
+    for members in components.values():
+        # Keep smallest ID among members
+        keep = min(members, key=lambda k: _safe_id_sort_key(term_ids[k]))
+        keep_original_indices.add(keep)
+
+    if id_col:
+        keep_ids = {term_ids[i] for i in keep_original_indices}
+        return terms[terms["ID"].isin(keep_ids)].copy()
+    else:
+        keep_index = {term_ids[i] for i in keep_original_indices}
+        return terms[terms.index.isin(keep_index)].copy()
+
+