sdufseval 1.0.4__py3-none-any.whl → 1.0.6__py3-none-any.whl

sdufseval/fseval.py

@@ -4,9 +4,8 @@ import time
 import warnings
 import numpy as np
 import pandas as pd
-from .eval import unsupervised_eval, supervised_eval
-from .loader import load_dataset
-
+from eval import unsupervised_eval, supervised_eval
+from loader import load_dataset
 
 class FSEVAL:
     def __init__(self, 
@@ -17,22 +16,19 @@ class FSEVAL:
                  unsupervised_iter=10, 
                  eval_type="both", 
                  metrics=None, 
-                 experiments=None):
-        """
-        Feature Selection Evaluation Suite.
-        """
+                 experiments=None,
+                 save_all=False):
         self.output_dir = output_dir
         self.cv = cv
         self.avg_steps = avg_steps
         self.supervised_iter = supervised_iter
         self.unsupervised_iter = unsupervised_iter
         self.eval_type = eval_type
+        self.save_all = save_all
         
-        # Metric configuration
         all_metrics = ["CLSACC", "NMI", "ACC", "AUC"]
         self.selected_metrics = metrics if metrics else all_metrics
         
-        # Experiment/Scale configuration
         self.scales = {}
         target_exps = experiments if experiments else ["10Percent", "100Percent"]
         if "10Percent" in target_exps:
@@ -44,24 +40,30 @@ class FSEVAL:
             os.makedirs(self.output_dir)
 
     def random_baseline(self, X, **kwargs):
-        """
-        Randomly assigns importance scores to features.
-        Internal method for lower-bound baseline.
-        """
         return np.random.rand(X.shape[1])
 
+    def _should_skip(self, ds_name, methods):
+        for m_info in methods:
+            for scale_name in self.scales.keys():
+                last_met = self.selected_metrics[-1]
+                fname = os.path.join(self.output_dir, f"{m_info['name']}_{last_met}_{scale_name}.csv")
+                
+                if not os.path.exists(fname):
+                    return False
+                
+                df = pd.read_csv(fname)
+                if 'Dataset' not in df.columns or ds_name not in df['Dataset'].values:
+                    return False
+        return True
+
     def run(self, datasets, methods, classifier=None):
-        """
-        Executes the benchmark for given datasets and FS methods.
-        
-        Args:
-            datasets: List of dataset names.
-            methods: List of dicts {'name': str, 'func': callable, 'stochastic': bool}.
-            classifier: Optional sklearn classifier instance to pass to supervised_eval.
-        """
         warnings.filterwarnings("ignore")
+        
         for ds_name in datasets:
-            print(f"\n>>> Benchmarking Dataset: {ds_name}")
+            if self._should_skip(ds_name, methods):
+                print(f">>> Skipping {ds_name}")
+                continue
+
             X, y_raw = load_dataset(ds_name)
             if X is None: continue
             
@@ -71,89 +73,74 @@ class FSEVAL:
             for m_info in methods:
                 name = m_info['name']
                 fs_func = m_info['func']
-                # Stochastic methods run 10 times and average
                 repeats = self.avg_steps if m_info.get('stochastic', False) else 1
                 
-                # Internal storage for current dataset results
                 ds_results = {s: {met: [] for met in self.selected_metrics} for s in self.scales}
 
                 for r in range(repeats):
-                    print(f"  [{name}] Progress: {r+1}/{repeats}")
-                    
-                    # Get feature ranking
+                    print(f"  [{name}] {ds_name} - Run {r+1}/{repeats}")
                     scores = fs_func(X)
                     indices = np.argsort(scores)[::-1]
 
                     for scale_name, percentages in self.scales.items():
                         row = {met: {'Dataset': ds_name} for met in self.selected_metrics}
-                        
                         for p in percentages:
                             k = max(1, min(math.ceil(p * n_features), n_features))
                             X_subset = X[:, indices[:k]]
 
-                            # Run evaluators
-                            c_acc, nmi, acc, auc = np.nan, np.nan, np.nan, np.nan
-                            
+                            res = {"CLSACC": np.nan, "NMI": np.nan, "ACC": np.nan, "AUC": np.nan}
                             if self.eval_type in ["unsupervised", "both"]:
-                                c_acc, nmi = unsupervised_eval(X_subset, y, avg_steps=self.unsupervised_iter)
-                            
+                                res["CLSACC"], res["NMI"] = unsupervised_eval(X_subset, y, avg_steps=self.unsupervised_iter)
                             if self.eval_type in ["supervised", "both"]:
-                                # Passes classifier (None or instance) to eval.py
-                                acc, auc = supervised_eval(X_subset, y, classifier=classifier, cv=self.cv, avg_steps=self.supervised_iter)
+                                res["ACC"], res["AUC"] = supervised_eval(X_subset, y, classifier=classifier, cv=self.cv, avg_steps=self.supervised_iter)
 
-                            # Map metrics to columns
-                            mapping = {"CLSACC": c_acc, "NMI": nmi, "ACC": acc, "AUC": auc}
                             for met in self.selected_metrics:
-                                row[met][p] = mapping[met]
+                                row[met][p] = res[met]
                         
                         for met in self.selected_metrics:
                             ds_results[scale_name][met].append(row[met])
 
-                # Save/Update results for this method/dataset
                 self._save_results(name, ds_results)
 
-    
+    def _save_results(self, method_name, ds_results):
+        for scale, metrics in ds_results.items():
+            for met_name, rows in metrics.items():
+                df_new = pd.DataFrame(rows)
+                
+                if not self.save_all:
+                    df_new = df_new.groupby('Dataset').mean().reset_index()
+                
+                df_new.columns = df_new.columns.astype(str)
+                fname = os.path.join(self.output_dir, f"{method_name}_{met_name}_{scale}.csv")
+                
+                if os.path.exists(fname):
+                    df_old = pd.read_csv(fname)
+                    df_old.columns = df_old.columns.astype(str)
+                    
+                    if self.save_all:
+                        df_final = pd.concat([df_old, df_new], ignore_index=True)
+                    else:
+                        df_final = pd.concat([df_old, df_new]).drop_duplicates(subset=['Dataset'], keep='last')
+                else:
+                    df_final = df_new
+                
+                df_final.to_csv(fname, index=False)
+
     def timer(self, methods, vary_param='both', time_limit=3600):
-        """
-        Runs a standalone runtime analysis experiment with a time cap.
-        
-        Args:
-            methods: List of dicts {'name': str, 'func': callable}.
-            vary_param: 'features', 'instances', or 'both'.
-            time_limit: Max seconds per method before it is skipped.
-        """
-        
-        # Determine which experiments to run
         experiments = []
         if vary_param in ['features', 'both']:
-            experiments.append({
-                'name': 'features',
-                'fixed_val': 100,
-                'range': range(1000, 20001, 500),
-                'file': 'time_analysis_features.csv'
-            })
+            experiments.append({'name': 'features', 'fixed_val': 100, 'range': range(1000, 20001, 500), 'file': 'time_analysis_features.csv'})
         if vary_param in ['instances', 'both']:
-            experiments.append({
-                'name': 'instances',
-                'fixed_val': 100,
-                'range': range(1000, 20001, 500),
-                'file': 'time_analysis_instances.csv'
-            })
+            experiments.append({'name': 'instances', 'fixed_val': 100, 'range': range(1000, 20001, 500), 'file': 'time_analysis_instances.csv'})
 
         for exp in experiments:
             vary_type = exp['name']
             val_range = exp['range']
             filename = os.path.join(self.output_dir, exp['file'])
-            
-            # Tracking for this specific experiment
             timed_out_methods = set()
             results = {m['name']: [] for m in methods}
             
-            print(f"\n--- Starting Experiment: Varying {vary_type} ---")
-            print(f"Time limit: {time_limit}s | Output: {filename}")
-
             for val in val_range:
-                # 1. Generate synthetic data based on vary_param
                 if vary_type == 'features':
                     n_samples, n_features = exp['fixed_val'], val
                 else:
@@ -162,62 +149,27 @@ class FSEVAL:
                 try:
                     X = np.random.rand(n_samples, n_features)
                 except MemoryError:
-                    print(f"  FATAL: MemoryError: Failed to allocate {n_samples}x{n_features} data.")
                     for m in methods:
                         results[m['name']].append(-1 if m['name'] in timed_out_methods else np.nan)
                     continue
 
-                # 2. Run each method
                 for m_info in methods:
                     name = m_info['name']
                     func = m_info['func']
-                    
-                    # Check if method has already timed out in this experiment
                     if name in timed_out_methods:
                         results[name].append(-1)
                         continue
                     
                     try:
                         start_time = time.time()
-                        
-                        # Execute the method (assuming benchmark format)
                         func(X)
-                        
                         duration = time.time() - start_time
-                        
                         if duration > time_limit:
-                            print(f"  - {name:<18}: {duration:.4f}s (TIMEOUT - skipping future runs)")
                             timed_out_methods.add(name)
-                        else:
-                            print(f"  - {name:<18}: {duration:.4f}s")
-                        
                         results[name].append(duration)
-                        
-                    except Exception as e:
-                        print(f"  - {name:<18}: FAILED ({type(e).__name__})")
+                    except Exception:
                         results[name].append(np.nan)
 
-            # 3. Save results to CSV
-            try:
-                df_results = pd.DataFrame.from_dict(results, orient='index', columns=list(val_range))
-                df_results.index.name = 'Method'
-                df_results.to_csv(filename)
-                print(f"\n--- Results saved to {filename} ---")
-            except Exception as e:
-                print(f"\n--- FAILED to save results: {e} ---")
-
-
-    def _save_results(self, method_name, ds_results):
-        """Aggregates repeats and saves to disk after each dataset."""
-        for scale, metrics in ds_results.items():
-            for met_name, rows in metrics.items():
-                df_new = pd.DataFrame(rows).groupby('Dataset').mean().reset_index()
-                fname = os.path.join(self.output_dir, f"{method_name}_{met_name}_{scale}.csv")
-                
-                if os.path.exists(fname):
-                    df_old = pd.read_csv(fname)
-                    df_final = pd.concat([df_old, df_new]).drop_duplicates(subset=['Dataset'], keep='last')
-                else:
-                    df_final = df_new
-                
-                df_final.to_csv(fname, index=False)
+            df_results = pd.DataFrame.from_dict(results, orient='index', columns=list(val_range))
+            df_results.index.name = 'Method'
+            df_results.to_csv(filename)

{sdufseval-1.0.4.dist-info → sdufseval-1.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdufseval
-Version: 1.0.4
+Version: 1.0.6
 Summary: Evaluation and Benchmark Tool for Feature Selection
 Project-URL: Homepage, https://github.com/mrajabinasab/FSEVAL
 Project-URL: Bug Tracker, https://github.com/mrajabinasab/FSEVAL/issues
@@ -124,6 +124,7 @@ Initializes the evalutation and benchmark object.
 | **`eval_type`** | both | "supervised", "unsupervised", or "both". |
 | **`metrics`** | ["CLSACC", "NMI", "ACC", "AUC"] | Evaluation metrics to calculate. |
 | **`experiments`** | ["10Percent", "100Percent"] | Which feature ratio grids to evaluate. |
+| **`save_all`** | False | Save the results of all runs of the stochastic methods separately. |
 
 ### ⚙️ `run(datasets, methods, classifier=None)`
 

sdufseval-1.0.6.dist-info/RECORD

@@ -0,0 +1,7 @@
+sdufseval/__init__.py,sha256=KIZIPxldHV2TLZVTW1jP6L8-tDw7-iMgmW-e-DeTdvo,271
+sdufseval/eval.py,sha256=445pNvgOo_fcdOKNElDsflDZ6iptxPRKGz5wuxLYPUE,2573
+sdufseval/fseval.py,sha256=VNFFYB72GCesInKeFJftGDg9pd2eI75gkArIPQrf5sY,7594
+sdufseval/loader.py,sha256=YUMSAdi2zcg2MizcGlnCxhsV5Y5cikL1hfk5ofwaI6s,2286
+sdufseval-1.0.6.dist-info/METADATA,sha256=Ja5jQbLYP9Fkb5BnZxQ6S1YiM4BkiD4SEYVIIGtgDwI,5892
+sdufseval-1.0.6.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+sdufseval-1.0.6.dist-info/RECORD,,

sdufseval-1.0.4.dist-info/RECORD

@@ -1,7 +0,0 @@
-sdufseval/__init__.py,sha256=KIZIPxldHV2TLZVTW1jP6L8-tDw7-iMgmW-e-DeTdvo,271
-sdufseval/eval.py,sha256=445pNvgOo_fcdOKNElDsflDZ6iptxPRKGz5wuxLYPUE,2573
-sdufseval/fseval.py,sha256=LPBKfXEj6lHSKPSEVv1b9U7V_7jXNBWpd5snD355_Rc,9513
-sdufseval/loader.py,sha256=YUMSAdi2zcg2MizcGlnCxhsV5Y5cikL1hfk5ofwaI6s,2286
-sdufseval-1.0.4.dist-info/METADATA,sha256=1qm0ZQSe4UhcAazHwOdLdoIISx3QpxhKI4qPbtPswOU,5796
-sdufseval-1.0.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-sdufseval-1.0.4.dist-info/RECORD,,