pythonflex 0.3.2__tar.gz → 0.3.4__tar.gz

{pythonflex-0.3.2 → pythonflex-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pythonflex
-Version: 0.3.2
+Version: 0.3.4
 Summary: pythonFLEX is a benchmarking toolkit for evaluating CRISPR screen results against biological gold standards. The toolkit computes gene-level and complex-level performance metrics, helping researchers systematically assess the biological relevance and resolution of their CRISPR screening data.
 Author-email: Yasir Demirtaş <tyasird@hotmail.com>
 Classifier: License :: OSI Approved :: MIT License
@@ -114,6 +114,7 @@ default_config = {
     "gold_standard": "GOBP",
     "color_map": "RdYlBu",
     "jaccard": True,
+    "jaccard_threshold": 1.0,  # set e.g. 0.90 to remove highly similar terms
     "plotting": {
         "save_plot": True,
         "output_type": "png",
@@ -124,7 +125,7 @@ default_config = {
     },
     "corr_function": "numpy",
     "logging": {  
-        "visible_levels": ["DONE","STARTED"]  # "PROGRESS", "STARTED", ,"INFO","WARNING"
+        "visible_levels": ["DONE","INFO", "WARNING"]  # "PROGRESS", "STARTED", ,"INFO","WARNING"
     }
 }
 
@@ -149,8 +150,10 @@ flex.plot_percomplex_scatter()
 flex.plot_percomplex_scatter_bysize()
 flex.plot_significant_complexes()
 flex.plot_complex_contributions()
+flex.plot_mpr_tp_multi(show_filters="all")
+flex.plot_mpr_complexes_multi(show_filters="all")
+flex.plot_mpr_complexes_auc_scores("all")
 
-# Save Result CSVspyflex.save_results_to_csv()
 flex.save_results_to_csv()
 
 

{pythonflex-0.3.2 → pythonflex-0.3.4}/README.md

@@ -83,6 +83,7 @@ default_config = {
     "gold_standard": "GOBP",
     "color_map": "RdYlBu",
     "jaccard": True,
+    "jaccard_threshold": 1.0,  # set e.g. 0.90 to remove highly similar terms
     "plotting": {
         "save_plot": True,
         "output_type": "png",
@@ -93,7 +94,7 @@ default_config = {
     },
     "corr_function": "numpy",
     "logging": {  
-        "visible_levels": ["DONE","STARTED"]  # "PROGRESS", "STARTED", ,"INFO","WARNING"
+        "visible_levels": ["DONE","INFO", "WARNING"]  # "PROGRESS", "STARTED", ,"INFO","WARNING"
     }
 }
 
@@ -118,8 +119,10 @@ flex.plot_percomplex_scatter()
 flex.plot_percomplex_scatter_bysize()
 flex.plot_significant_complexes()
 flex.plot_complex_contributions()
+flex.plot_mpr_tp_multi(show_filters="all")
+flex.plot_mpr_complexes_multi(show_filters="all")
+flex.plot_mpr_complexes_auc_scores("all")
 
-# Save Result CSVspyflex.save_results_to_csv()
 flex.save_results_to_csv()
 
 

{pythonflex-0.3.2 → pythonflex-0.3.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pythonflex"
-version = "0.3.2"
+version = "0.3.4"
 description = "pythonFLEX is a benchmarking toolkit for evaluating CRISPR screen results against biological gold standards. The toolkit computes gene-level and complex-level performance metrics, helping researchers systematically assess the biological relevance and resolution of their CRISPR screening data."
 readme = "README.md"
 authors = [

{pythonflex-0.3.2 → pythonflex-0.3.4}/src/pythonflex/__init__.py

@@ -5,7 +5,7 @@ from .analysis import initialize, pra, pra_percomplex, fast_corr, perform_corr,
 from .plotting import (
     adjust_text_positions, plot_precision_recall_curve, plot_aggregated_pra, plot_iqr_pra, plot_all_runs_pra, plot_percomplex_scatter,
     plot_percomplex_scatter_bysize, plot_complex_contributions, plot_significant_complexes, plot_auc_scores,
-    plot_mpr_tp, plot_mpr_complexes, plot_mpr_tp_multi, plot_mpr_complexes_multi
+    plot_mpr_tp, plot_mpr_complexes, plot_mpr_tp_multi, plot_mpr_complexes_multi, plot_mpr_complexes_auc_scores
 )
 
 __all__ = [ "log", "get_example_data_path", "fast_corr",
@@ -14,7 +14,7 @@ __all__ = [ "log", "get_example_data_path", "fast_corr",
     "perform_corr", "is_symmetric", "binary", "has_mirror_of_first_pair", "convert_full_to_half_matrix",
     "drop_mirror_pairs", "quick_sort", "complex_contributions", "adjust_text_positions", "plot_precision_recall_curve",
     "plot_aggregated_pra", "plot_iqr_pra", "plot_all_runs_pra", "plot_percomplex_scatter", "plot_percomplex_scatter_bysize", "plot_complex_contributions",
-    "plot_significant_complexes", "plot_auc_scores", "save_results_to_csv", "update_matploblib_config",
+    "plot_significant_complexes", "plot_auc_scores", "plot_mpr_complexes_auc_scores", "save_results_to_csv", "update_matploblib_config",
     "mpr_prepare", "plot_mpr_tp", "plot_mpr_complexes",
     "plot_mpr_tp_multi", "plot_mpr_complexes_multi"
 ]

{pythonflex-0.3.2 → pythonflex-0.3.4}/src/pythonflex/analysis.py

@@ -43,6 +43,7 @@ def initialize(config={}):
         "gold_standard": "CORUM",
         "color_map": "RdYlBu",
         "jaccard": True,
+        "jaccard_threshold": 1.0,
         "use_common_genes": True,
         "plotting": {
             "save_plot": True,
@@ -844,7 +845,7 @@ def quick_sort(df, ascending=False):
     log.done("Pair-wise matrix sorting.")
     return sorted_df
 
-def save_results_to_csv(categories = ["complex_contributions", "pr_auc", "pra_percomplex"]):
+def save_results_to_csv(categories = ["complex_contributions", "pr_auc", "pra_percomplex", "mpr_complexes_auc"]):
 
     config = dload("config")  # Load config to get output folder
     output_folder = Path(config.get("output_folder", "output"))
@@ -856,6 +857,18 @@ def save_results_to_csv(categories = ["complex_contributions", "pr_auc", "pra_pe
         if data is None:
             log.warning(f"No data found for category '{category}'. Skipping save.")
             continue
+
+        if category == "mpr_complexes_auc" and isinstance(data, dict):
+            # Dict[dataset_name -> Dict[filter_key -> auc]]
+            try:
+                df = pd.DataFrame.from_dict(data, orient="index")
+                df.index.name = "Dataset"
+                csv_path = output_folder / f"{category}.csv"
+                df.to_csv(csv_path, index=True)
+                log.info(f"Saved '{category}' to {csv_path}")
+            except Exception as e:
+                log.warning(f"Failed to convert and save '{category}': {e}")
+            continue
         
         if category == "pr_auc" and isinstance(data, dict):
             # Special handling: Convert dict to DataFrame (assuming keys are indices, values are data)
@@ -1312,6 +1325,64 @@ def _mpr_module_coverage(contrib_df, terms, tp_th=1, percent_th=0.1):
     return coverage
 
 
+def _mpr_complexes_auc(
+    coverage: np.ndarray,
+    precision_cutoffs: np.ndarray,
+    max_complexes: float = 200.0,
+) -> float:
+    """Compute AUC for the Fig. 1F-style mPR curve (#complexes vs precision).
+
+    The plot uses:
+      x = #covered complexes (capped at `max_complexes`, shown on a log axis)
+      y = precision cutoff
+
+    We compute a normalized AUC by integrating precision over the *normalized*
+    coverage axis:
+        AUC = \int y \, d(x/max_complexes)
+
+    This yields a score in [0, 1] (or NaN if insufficient data).
+    """
+    cov = np.asarray(coverage, dtype=float)
+    prec = np.asarray(precision_cutoffs, dtype=float)
+
+    if cov.size == 0 or prec.size == 0:
+        return 0.0
+
+    # Match plot_mpr_complexes_multi(): only count cov>0 (log-x cannot show 0)
+    mask = (
+        np.isfinite(cov)
+        & np.isfinite(prec)
+        & (cov > 0)
+        & (cov <= max_complexes)
+        & (prec >= 0)
+        & (prec <= 1.0)
+    )
+    if not np.any(mask):
+        return 0.0
+
+    x_cov = cov[mask]
+    y = prec[mask]
+
+    # x-axis is log-scaled in the plot; normalize so cov=1 -> 0, cov=max_complexes -> 1
+    # (This matches the plot's tick hack where 1 is labeled as "0".)
+    x = np.log10(x_cov) / np.log10(float(max_complexes))
+
+    # Sort by x and collapse duplicate x values by taking max y (upper envelope)
+    order = np.argsort(x)
+    x = x[order]
+    y = y[order]
+
+    x_unique = np.unique(x)
+    if x_unique.size != x.size:
+        y = np.array([float(np.nanmax(y[x == xv])) for xv in x_unique], dtype=float)
+        x = x_unique
+
+    if x.size < 2:
+        return 0.0
+
+    return float(np.trapz(y, x))
+
+
 
 
 
@@ -1379,6 +1450,7 @@ def mpr_prepare(
 
     tp_curves = {}
     coverage_curves = {}
+    complexes_auc = {}
     precision_cutoffs = None
 
     for label, removed in filter_sets.items():
@@ -1393,6 +1465,7 @@ def mpr_prepare(
                 "precision": np.array([], dtype=float),
             }
             coverage_curves[label] = np.zeros(0, dtype=float)
+            complexes_auc[label] = float("nan")
             continue
 
         tp_cum = true.cumsum()
@@ -1417,11 +1490,17 @@ def mpr_prepare(
             percent_th=percent_th,
         )
         coverage_curves[label] = cov
+        complexes_auc[label] = _mpr_complexes_auc(
+            cov,
+            precision_cutoffs,
+            max_complexes=200.0,
+        )
 
     mpr_data = {
         "precision_cutoffs": precision_cutoffs,
         "tp_curves": tp_curves,
         "coverage_curves": coverage_curves,
+        "complexes_auc": complexes_auc,
         "filters": {
             "no_mtRibo_ETCI": sorted(mtRibo_ids),
             "no_small_highAUPRC": sorted(small_hi_ids),
@@ -1435,6 +1514,9 @@ def mpr_prepare(
 
     dsave(mpr_data, "mpr", name)
 
+    # Convenience: store AUCs as their own category for easy export / plotting.
+    dsave(complexes_auc, "mpr_complexes_auc", name)
+
 
 
 ### OLD FUNCTIONS

{pythonflex-0.3.2 → pythonflex-0.3.4}/src/pythonflex/examples/basic_usage.py

@@ -31,7 +31,8 @@ default_config = {
     "output_folder": "CORUM",
     "gold_standard": "CORUM",
     "color_map": "BuGn",
-    "jaccard": False,
+    "jaccard": True,
+    "jaccard_threshold": 1,
     "use_common_genes": False,  # Set to False for individual dataset-gold standard intersections
     "plotting": {
         "save_plot": True,
@@ -61,26 +62,26 @@ for name, dataset in data.items():
     fpc = flex.pra_percomplex(name, dataset, is_corr=False) 
     cc = flex.complex_contributions(name)
     flex.mpr_prepare(name)  
+    
 
 
 
 
 #%%
 # Generate plots
-flex.plot_precision_recall_curve()
-flex.plot_auc_scores()
-flex.plot_significant_complexes()
-flex.plot_percomplex_scatter(n_top=20)
-flex.plot_percomplex_scatter_bysize()
-flex.plot_complex_contributions()
-##
-flex.plot_mpr_tp_multi()
-flex.plot_mpr_complexes_multi()
+# flex.plot_precision_recall_curve()
+# flex.plot_auc_scores()
+# flex.plot_significant_complexes()
+# flex.plot_percomplex_scatter(n_top=20)
+# flex.plot_percomplex_scatter_bysize()
+# flex.plot_complex_contributions()
+# flex.plot_mpr_tp_multi(show_filters="all")
+# flex.plot_mpr_complexes_multi(show_filters="all")
+# flex.plot_mpr_complexes_auc_scores("all")
+
+
 
 #%%
 # Save results to CSV
-flex.save_results_to_csv()
+# flex.save_results_to_csv()
 
-# %%
-flex.plot_mpr_complexes_multi(show_filters="no_mtRibo_ETCI")
-# %%

pythonflex-0.3.4/src/pythonflex/examples/manuscript.py

@@ -0,0 +1,112 @@
+"""
+Basic usage example of the pythonFLEX package.
+Demonstrates initialization, data loading, analysis, and plotting.
+"""
+#%%
+import pythonflex as flex
+import pandas as pd
+
+gene_effect = pd.read_csv('C:/Users/yd/Desktop/projects/_datasets/depmap/25Q2/gene_effect.csv', index_col=0)
+
+skin = pd.read_csv('C:/Users/yd/Desktop/projects/_datasets/depmap/25Q2/subset/skin_cell_lines.csv', index_col=0)
+
+soft = pd.read_csv('C:/Users/yd/Desktop/projects/_datasets/depmap/25Q2/subset/soft_tissue_cell_lines.csv', index_col=0)
+
+
+cholesky = pd.read_csv('C:/Users/yd/Desktop/projects/_datasets/depmap/25Q2/25Q2_chronos_whitened_Cholesky.csv', index_col=0).T
+
+# inputs = {
+#     "All Screens": {
+#         "path": gene_effect, 
+#         "sort": "high",
+#         "color": "#000000"
+#     },
+#     "Skin": {
+#         "path": skin, 
+#         "sort": "high",
+#         "color": "#FF0000"
+#     },
+#     "Soft Tissue": {
+#         "path": soft, 
+#         "sort": "high",
+#         "color": "#FFFF00"
+#     },
+# }
+
+
+inputs = {
+    "DM All Screens": {
+        "path": gene_effect, 
+        "sort": "high",
+        "color": "#000000"
+    },
+    "DM Cholesky Whitening": {
+        "path": cholesky, 
+        "sort": "high",
+        "color": "#FF0000"
+    },
+   
+}
+
+
+
+
+default_config = {
+    "min_genes_in_complex": 2,
+    "min_genes_per_complex_analysis": 3,
+    "output_folder": "CORUM_DMvsCholesky",
+    "gold_standard": "CORUM",
+    "color_map": "BuGn",
+    "jaccard": False,
+        "jaccard_threshold": 1.0,
+    "use_common_genes": False,  # Set to False for individual dataset-gold standard intersections
+    "plotting": {
+        "save_plot": True,
+        "output_type": "pdf",
+    },
+    "preprocessing": {
+        "fill_na": True,
+        "normalize": False,
+    },
+    "corr_function": "numpy",
+    "logging": {  
+        "visible_levels": ["DONE"]  
+        # "PROGRESS", "STARTED", ,"INFO","WARNING"
+    }
+}
+
+# Initialize logger, config, and output folder
+flex.initialize(default_config)
+
+# Load datasets and gold standard terms
+data, _ = flex.load_datasets(inputs)
+terms, genes_in_terms = flex.load_gold_standard()
+
+# Run analysis
+for name, dataset in data.items():
+    pra = flex.pra(name, dataset, is_corr=False)
+    fpc = flex.pra_percomplex(name, dataset, is_corr=False) 
+    cc = flex.complex_contributions(name)
+    flex.mpr_prepare(name)  
+
+
+
+
+#%%
+# Generate plots
+flex.plot_precision_recall_curve()
+flex.plot_auc_scores()
+flex.plot_significant_complexes()
+flex.plot_percomplex_scatter(n_top=20)
+flex.plot_percomplex_scatter_bysize()
+flex.plot_complex_contributions()
+##
+#%%
+flex.plot_mpr_tp_multi(show_filters="all")
+flex.plot_mpr_complexes_multi(show_filters="all")
+
+# Save results to CSV
+flex.save_results_to_csv()
+
+# %%
+# %%

{pythonflex-0.3.2 → pythonflex-0.3.4}/src/pythonflex/plotting.py

@@ -1220,6 +1220,107 @@ def plot_auc_scores():
     plt.close(fig)
     return pra_dict
 
+
+def plot_mpr_complexes_auc_scores(filter_key: str = "all"):
+    """Plot AUC scores for the mPR complexes curve (Fig 1F-style).
+
+    Requires `mpr_prepare()` to have been run for each dataset.
+
+    Parameters
+    ----------
+    filter_key : str
+        One of: "all", "no_mtRibo_ETCI", "no_small_highAUPRC".
+
+    Returns
+    -------
+    pd.Series
+        AUC values indexed by dataset name (sorted descending).
+    """
+    config = dload("config")
+    plot_config = config["plotting"]
+    mpr_auc_dict = dload("mpr_complexes_auc")
+    input_colors = dload("input", "colors")
+
+    if input_colors:
+        input_colors = {_sanitize(k): v for k, v in input_colors.items()}
+
+    if not isinstance(mpr_auc_dict, dict) or not mpr_auc_dict:
+        log.warning(
+            "No mPR complexes AUC data found. Run mpr_prepare() first (it stores 'mpr_complexes_auc')."
+        )
+        return pd.Series(dtype=float)
+
+    # Build Series: dataset -> auc
+    auc_by_dataset = {}
+    for dataset, per_filter in mpr_auc_dict.items():
+        if not isinstance(per_filter, dict):
+            continue
+        val = per_filter.get(filter_key)
+        if val is None:
+            continue
+        try:
+            auc_by_dataset[dataset] = float(val)
+        except (TypeError, ValueError):
+            continue
+
+    if not auc_by_dataset:
+        log.warning(
+            f"No mPR complexes AUC scores found for filter '{filter_key}'. Available filters: {list(FILTER_STYLES.keys())}"
+        )
+        return pd.Series(dtype=float)
+
+    s = pd.Series(auc_by_dataset).sort_values(ascending=False)
+    datasets = list(s.index)
+    auc_scores = list(s.values)
+
+    fig, ax = plt.subplots()
+
+    # Color logic (match other bar plots)
+    cmap_name = config.get("color_map", "tab10")
+    try:
+        cmap = get_cmap(cmap_name)
+    except ValueError:
+        cmap = get_cmap("tab10")
+
+    num_datasets = len(datasets)
+    if num_datasets <= 10 and cmap_name == "tab10":
+        default_colors = [cmap(i) for i in range(num_datasets)]
+    else:
+        default_colors = [cmap(float(i) / max(num_datasets - 1, 1)) for i in range(num_datasets)]
+
+    final_colors = []
+    for i, dataset in enumerate(datasets):
+        color = input_colors.get(dataset) if input_colors else None
+        if color is None:
+            color = default_colors[i]
+        final_colors.append(color)
+
+    ax.bar(datasets, auc_scores, color=final_colors, edgecolor="black")
+
+    ymax = max([v for v in auc_scores if np.isfinite(v)], default=0.0)
+    ax.set_ylim(0, ymax + 0.01)
+    ax.set_ylabel("mPR complexes AUC")
+    plt.xticks(rotation=45, ha="right")
+
+    # Styling consistent with other plots
+    ax.grid(visible=False, which="both", axis="both")
+    ax.set_axisbelow(False)
+    ax.spines["top"].set_visible(False)
+    ax.spines["right"].set_visible(False)
+
+    if plot_config.get("save_plot", False):
+        output_type = plot_config.get("output_type", "pdf")
+        output_folder = Path(config["output_folder"])
+        output_folder.mkdir(parents=True, exist_ok=True)
+        output_path = output_folder / f"mpr_complexes_auc_{filter_key}.{output_type}"
+        plt.savefig(output_path, bbox_inches="tight", format=output_type)
+
+    if plot_config.get("show_plot", True):
+        plt.show()
+
+    plt.close(fig)
+    return s
+
 # -----------------------------------------------------------------------------
 # mPR plots (Fig. 1E and Fig. 1F)
 # -----------------------------------------------------------------------------
@@ -1620,6 +1721,8 @@ def plot_mpr_complexes_multi(
     outname=None,
     linewidth=1.8,
     show_filters=("all", "no_mtRibo_ETCI", "no_small_highAUPRC"),
+    show_markers="auto",
+    marker_size=20,
 ):
     """
     Plot module-level PR (#complexes vs precision) for multiple datasets.
@@ -1644,6 +1747,11 @@ def plot_mpr_complexes_multi(
         Line width for all curves
     show_filters : tuple of str
         Which filters to show. Default is all three.
+    show_markers : bool or "auto"
+        If True, draw markers on curves to make short curves visible.
+        If "auto" (default), markers are drawn only for curves with <= 10 points.
+    marker_size : int
+        Scatter marker size (points^2) when markers are shown.
     
     Returns
     -------
@@ -1730,13 +1838,26 @@ def plot_mpr_complexes_multi(
             prec_plot = precision_cutoffs[mask]
             
             style = FILTER_STYLES.get(filter_key, {})
-            ax.plot(
-                cov_plot,
-                prec_plot,
-                color=color,
-                linestyle=style.get("linestyle", "-"),
-                linewidth=linewidth,
-            )
+
+            # Decide marker visibility
+            if show_markers == "auto":
+                use_markers = (cov_plot.size <= 10)
+            else:
+                use_markers = bool(show_markers)
+
+            if cov_plot.size == 1:
+                # A single point is effectively invisible as a line; draw a marker.
+                ax.scatter(cov_plot, prec_plot, color=color, s=marker_size, zorder=3)
+            else:
+                ax.plot(
+                    cov_plot,
+                    prec_plot,
+                    color=color,
+                    linestyle=style.get("linestyle", "-"),
+                    linewidth=linewidth,
+                    marker=("o" if use_markers else None),
+                    markersize=(3 if use_markers else None),
+                )
     
     # Configure axes
     ax.set_xscale("log")

{pythonflex-0.3.2 → pythonflex-0.3.4}/src/pythonflex/preprocessing.py

@@ -189,7 +189,18 @@ def load_gold_standard():
     use_common_genes = config.get("use_common_genes", True)
 
     gold_standard_source = config['gold_standard']
-    log.started(f"Loading gold standard: {gold_standard_source}, Min complex size: {config['min_genes_in_complex']}, Jaccard filtering: {config['jaccard']}, use_common_genes: {use_common_genes}")
+    jaccard_enabled = bool(config.get("jaccard", False))
+    jaccard_threshold_raw = config.get("jaccard_threshold", 1.0)
+    try:
+        jaccard_threshold = float(jaccard_threshold_raw)  # type: ignore[arg-type]
+    except (TypeError, ValueError):
+        raise ValueError(
+            f"config['jaccard_threshold'] must be a number in (0, 1], got {jaccard_threshold_raw!r}"
+        )
+    log.done(
+        f"Loading gold standard: {gold_standard_source}, Min complex size: {config['min_genes_in_complex']}, "
+        f"Jaccard filtering: {jaccard_enabled} (threshold={jaccard_threshold}), use_common_genes: {use_common_genes}"
+    )
 
     # Define gold standard file paths for predefined sources
     gold_standard_files = {
@@ -217,34 +228,44 @@ def load_gold_standard():
 
     # Store raw gold standard for later per-dataset filtering
     terms["all_genes"] = terms["Genes"].apply(lambda x: list(set(x.split(";"))))
-    log.info(f"Gold standard loaded with {len(terms)} terms")
+    log.done(f"Gold standard loaded with {len(terms)} terms")
 
     # Basic filtering by minimum complex size (before gene filtering)
     terms["n_all_genes"] = terms["all_genes"].apply(len) 
     terms = terms[terms["n_all_genes"] >= config['min_genes_in_complex']]
-    log.info(f"After min_genes_in_complex filtering: {len(terms)} terms")
-
-    if config['jaccard']:
-        log.info("Applying Jaccard filtering. Remove terms with identical gene sets.")
-        # Use all genes for jaccard filtering
-        terms["gene_set"] = terms["all_genes"].map(lambda x: frozenset(x))
-        grouped = terms.groupby("gene_set", sort=False)
-        duplicate_clusters = []
-        for _, group in grouped:
-            if len(group) > 1:
-                duplicate_clusters.append(group["ID"].values if "ID" in group.columns else group.index.values)
-        
-        keep_ids = set(terms["ID"] if "ID" in terms.columns else terms.index)
-        for cluster in duplicate_clusters:
-            sorted_ids = sorted(cluster)
-            keep_ids.difference_update(sorted_ids[1:])
-        
-        if "ID" in terms.columns:
-            terms = terms[terms["ID"].isin(keep_ids)].copy()
+    log.done(f"After min_genes_in_complex filtering: {len(terms)} terms")
+
+    if jaccard_enabled:
+        if not (0.0 < jaccard_threshold <= 1.0):
+            raise ValueError(f"config['jaccard_threshold'] must be in (0, 1], got {jaccard_threshold}")
+
+        if jaccard_threshold >= 1.0:
+            log.done("Applying Jaccard filtering (threshold=1.0). Removing terms with identical gene sets.")
+            # Use all genes for jaccard filtering
+            terms["gene_set"] = terms["all_genes"].map(lambda x: frozenset(x))
+            grouped = terms.groupby("gene_set", sort=False)
+            duplicate_clusters = []
+            for _, group in grouped:
+                if len(group) > 1:
+                    duplicate_clusters.append(group["ID"].values if "ID" in group.columns else group.index.values)
+
+            keep_ids = set(terms["ID"] if "ID" in terms.columns else terms.index)
+            for cluster in duplicate_clusters:
+                sorted_ids = sorted(cluster)
+                keep_ids.difference_update(sorted_ids[1:])
+
+            if "ID" in terms.columns:
+                terms = terms[terms["ID"].isin(keep_ids)].copy()
+            else:
+                terms = terms[terms.index.isin(keep_ids)].copy()
+            terms.drop(columns=["gene_set"], inplace=True, errors="ignore")
+            log.done(f"After Jaccard filtering: {len(terms)} terms")
         else:
-            terms = terms[terms.index.isin(keep_ids)].copy()
-        terms.drop(columns=["gene_set"], inplace=True, errors='ignore')
-        log.info(f"After Jaccard filtering: {len(terms)} terms")
+            log.done(
+                f"Applying Jaccard filtering (threshold={jaccard_threshold}). Removing highly similar terms."
+            )
+            terms = _filter_terms_by_jaccard_threshold(terms, threshold=jaccard_threshold, genes_col="all_genes")
+            log.done(f"After Jaccard filtering: {len(terms)} terms")
 
     # if there is column called "ID", set it as index
     if "ID" in terms.columns:
@@ -255,6 +276,149 @@ def load_gold_standard():
     return terms, None  # Return None for genes_present_in_terms - will be computed per dataset
 
 
+class _UnionFind:
+    def __init__(self, n: int):
+        self.parent = list(range(n))
+        self.rank = [0] * n
+
+    def find(self, x: int) -> int:
+        while self.parent[x] != x:
+            self.parent[x] = self.parent[self.parent[x]]
+            x = self.parent[x]
+        return x
+
+    def union(self, a: int, b: int) -> None:
+        ra, rb = self.find(a), self.find(b)
+        if ra == rb:
+            return
+        if self.rank[ra] < self.rank[rb]:
+            self.parent[ra] = rb
+        elif self.rank[ra] > self.rank[rb]:
+            self.parent[rb] = ra
+        else:
+            self.parent[rb] = ra
+            self.rank[ra] += 1
+
+
+def _safe_id_sort_key(val):
+    """Sort key that prefers numeric ordering when IDs look like ints."""
+    try:
+        return (0, int(val))
+    except Exception:
+        return (1, str(val))
+
+
+def _jaccard_similarity(a: set, b: set) -> float:
+    if not a and not b:
+        return 1.0
+    if not a or not b:
+        return 0.0
+    inter = len(a.intersection(b))
+    if inter == 0:
+        return 0.0
+    union = len(a) + len(b) - inter
+    return inter / union
+
+
+def _filter_terms_by_jaccard_threshold(terms: pd.DataFrame, threshold: float, genes_col: str = "all_genes") -> pd.DataFrame:
+    """Remove near-duplicate terms whose gene sets have Jaccard similarity >= threshold.
+
+    Keeps one representative per similarity-connected component (smallest ID).
+    This uses an exact Jaccard similarity join with prefix-filter candidate generation.
+    """
+    if not (0.0 < threshold < 1.0):
+        # threshold == 1.0 handled elsewhere; invalid values rejected earlier
+        return terms
+
+    # Build IDs and gene sets
+    id_col = "ID" if "ID" in terms.columns else None
+    term_ids = (terms["ID"].tolist() if id_col else terms.index.tolist())
+    gene_sets = []
+    for genes in terms[genes_col].tolist():
+        gene_sets.append(set(genes))
+
+    sizes = [len(s) for s in gene_sets]
+    if len(gene_sets) <= 1:
+        return terms
+
+    # Global token frequency for ordering (rare tokens first)
+    from collections import Counter, defaultdict
+    freq = Counter()
+    for s in gene_sets:
+        freq.update(s)
+
+    def sort_tokens(s: set):
+        return sorted(s, key=lambda tok: (freq.get(tok, 0), str(tok)))
+
+    # Process smaller sets first (helps size filtering and keeps index smaller)
+    order = sorted(range(len(gene_sets)), key=lambda i: (sizes[i], _safe_id_sort_key(term_ids[i])))
+    ordered_tokens = [sort_tokens(gene_sets[i]) for i in range(len(gene_sets))]
+
+    # Inverted index over prefix tokens
+    inv_index = defaultdict(list)  # token -> list of processed term indices (original idx)
+
+    uf = _UnionFind(len(gene_sets))
+
+    # Precompute prefix lengths
+    import math
+    prefix_len = []
+    for i in range(len(gene_sets)):
+        m = sizes[i]
+        # PPJoin prefix length for Jaccard threshold
+        p = m - math.ceil(threshold * m) + 1
+        if p < 0:
+            p = 0
+        if p > m:
+            p = m
+        prefix_len.append(p)
+
+    # Candidate generation + exact verification
+    for idx_pos, i in enumerate(order):
+        tokens_i = ordered_tokens[i]
+        p_i = prefix_len[i]
+
+        # Count shared prefix tokens with previously indexed sets
+        candidate_overlap_lb = defaultdict(int)
+        for tok in tokens_i[:p_i]:
+            for j in inv_index.get(tok, []):
+                # size filter: if too different in size, cannot meet Jaccard threshold
+                if sizes[j] < threshold * sizes[i]:
+                    continue
+                if sizes[j] > sizes[i] / threshold:
+                    continue
+                candidate_overlap_lb[j] += 1
+            inv_index[tok].append(i)
+
+        if not candidate_overlap_lb:
+            continue
+
+        set_i = gene_sets[i]
+        for j in candidate_overlap_lb.keys():
+            # Exact verification
+            sim = _jaccard_similarity(set_i, gene_sets[j])
+            if sim >= threshold:
+                uf.union(i, j)
+
+    # Choose representative (smallest ID) for each connected component
+    components = {}
+    for i in range(len(gene_sets)):
+        root = uf.find(i)
+        components.setdefault(root, []).append(i)
+
+    keep_original_indices = set()
+    for members in components.values():
+        # Keep smallest ID among members
+        keep = min(members, key=lambda k: _safe_id_sort_key(term_ids[k]))
+        keep_original_indices.add(keep)
+
+    if id_col:
+        keep_ids = {term_ids[i] for i in keep_original_indices}
+        return terms[terms["ID"].isin(keep_ids)].copy()
+    else:
+        keep_index = {term_ids[i] for i in keep_original_indices}
+        return terms[terms.index.isin(keep_index)].copy()
+
+