sdufseval 1.0.1__tar.gz → 1.0.2__tar.gz

{sdufseval-1.0.1 → sdufseval-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdufseval
-Version: 1.0.1
+Version: 1.0.2
 Summary: Evaluation and Benchmark Tool for Feature Selection
 Project-URL: Homepage, https://github.com/mrajabinasab/FSEVAL
 Project-URL: Bug Tracker, https://github.com/mrajabinasab/FSEVAL/issues
@@ -62,7 +62,7 @@ import numpy as np
 
 if __name__ == "__main__":
 
-    # The 23 real datasets
+    # The 23 benchmark datasets
     DATASETS_TO_RUN = [
         'ALLAML', 'CLL_SUB_111', 'COIL20', 'Carcinom', 'GLIOMA', 'GLI_85', 
         'Isolet', 'ORL', 'Prostate_GE', 'SMK_CAN_187', 'TOX_171', 'Yale', 
@@ -73,7 +73,7 @@ if __name__ == "__main__":
     # Initialize FSEVAL
     evaluator = FSEVAL(output_dir="benchmark_results", avg_steps=10)
 
-    # Configuration for methods using the class internal random_baseline
+    # Configuration for methods
     methods_list = [
         {
             'name': 'Random', 
@@ -87,8 +87,19 @@ if __name__ == "__main__":
         }
     ]
     
-    # Run Benchmark (Defaults to RF)
+    # --- 1. Run Standard Benchmark ---
+    # Evaluates methods on real-world datasets across different feature scales
     evaluator.run(DATASETS_TO_RUN, methods_list)
+
+    # --- 2. Run Runtime Analysis ---
+    # Performs scalability testing on synthetic data with a time cap.
+    # vary_param='both' triggers both 'features' and 'instances' experiments.
+    print("\n>>> Starting Scalability Analysis...")
+    evaluator.timer(
+        methods=methods_list, 
+        vary_param='both', 
+        time_limit=3600  # 1 hour limit 
+    )
 ```
 
 ## Data Loading
@@ -107,9 +118,9 @@ Initializes the evalutation and benchmark object.
 | :--- | :--- | :--- |
 | **`output_dir`** | results | Folder where CSV result files are saved. |
 | **`cv`** | 5 | Cross-validation folds (supervised only). |
-| **`avg_steps`** | 10 | Number of random restarts / seeds to average over. |
-| **`eval_type`** | both | Number of random restarts / seeds to average over. |
-| **`metrics`** | ["CLSACC", "NMI", "ACC", "AUC"] | "supervised", "unsupervised", or "both". |
+| **`avg_steps`** | 10 | Number of random restarts / seeds to average over.|
+| **`eval_type`** | both | "supervised", "unsupervised", or "both". |
+| **`metrics`** | ["CLSACC", "NMI", "ACC", "AUC"] | Evaluation metrics to calculate. |
 | **`experiments`** | ["10Percent", "100Percent"] | Which feature ratio grids to evaluate. |
 
 ### ⚙️ `run(datasets, methods, classifier=None)`
@@ -122,6 +133,16 @@ Initializes the evalutation and benchmark object.
 | **`methods`** | List[dict] | "[{""name"": str, ""func"": callable, ""stochastic"": bool}, ...]" |
 | **`classifier`** | sklearn classifier | Classifier for supervised eval (default: RandomForestClassifier) |
 
+### ⚙️ `timer(methods, vary_param='features', time_limit=3600)`
+
+Runs a runtime analysis on the methods.
+
+| Argument | Type | Description |
+| :--- | :--- | :--- |
+| **`methods`** | List[dict] | "[{""name"": str, ""func"": callable, ""stochastic"": bool}, ...]" |
+| **`vary_param`** | ["CLSACC", "NMI", "ACC", "AUC"] | "features", "instances", or "both". |
+| **`time_limit`** | 3600 | Terminate the method after reecording first time it exceeds this limit. |
+
 #  Dashboard
 
 There is a Feature Selection Evaluation Dashboard based on the benchmarks provided by FSEVAL, available on:

{sdufseval-1.0.1 → sdufseval-1.0.2}/README.md

@@ -39,7 +39,7 @@ import numpy as np
 
 if __name__ == "__main__":
 
-    # The 23 real datasets
+    # The 23 benchmark datasets
     DATASETS_TO_RUN = [
         'ALLAML', 'CLL_SUB_111', 'COIL20', 'Carcinom', 'GLIOMA', 'GLI_85', 
         'Isolet', 'ORL', 'Prostate_GE', 'SMK_CAN_187', 'TOX_171', 'Yale', 
@@ -50,7 +50,7 @@ if __name__ == "__main__":
     # Initialize FSEVAL
     evaluator = FSEVAL(output_dir="benchmark_results", avg_steps=10)
 
-    # Configuration for methods using the class internal random_baseline
+    # Configuration for methods
     methods_list = [
         {
             'name': 'Random', 
@@ -64,8 +64,19 @@ if __name__ == "__main__":
         }
     ]
     
-    # Run Benchmark (Defaults to RF)
+    # --- 1. Run Standard Benchmark ---
+    # Evaluates methods on real-world datasets across different feature scales
     evaluator.run(DATASETS_TO_RUN, methods_list)
+
+    # --- 2. Run Runtime Analysis ---
+    # Performs scalability testing on synthetic data with a time cap.
+    # vary_param='both' triggers both 'features' and 'instances' experiments.
+    print("\n>>> Starting Scalability Analysis...")
+    evaluator.timer(
+        methods=methods_list, 
+        vary_param='both', 
+        time_limit=3600  # 1 hour limit 
+    )
 ```
 
 ## Data Loading
@@ -84,9 +95,9 @@ Initializes the evalutation and benchmark object.
 | :--- | :--- | :--- |
 | **`output_dir`** | results | Folder where CSV result files are saved. |
 | **`cv`** | 5 | Cross-validation folds (supervised only). |
-| **`avg_steps`** | 10 | Number of random restarts / seeds to average over. |
-| **`eval_type`** | both | Number of random restarts / seeds to average over. |
-| **`metrics`** | ["CLSACC", "NMI", "ACC", "AUC"] | "supervised", "unsupervised", or "both". |
+| **`avg_steps`** | 10 | Number of random restarts / seeds to average over.|
+| **`eval_type`** | both | "supervised", "unsupervised", or "both". |
+| **`metrics`** | ["CLSACC", "NMI", "ACC", "AUC"] | Evaluation metrics to calculate. |
 | **`experiments`** | ["10Percent", "100Percent"] | Which feature ratio grids to evaluate. |
 
 ### ⚙️ `run(datasets, methods, classifier=None)`
@@ -99,6 +110,16 @@ Initializes the evalutation and benchmark object.
 | **`methods`** | List[dict] | "[{""name"": str, ""func"": callable, ""stochastic"": bool}, ...]" |
 | **`classifier`** | sklearn classifier | Classifier for supervised eval (default: RandomForestClassifier) |
 
+### ⚙️ `timer(methods, vary_param='features', time_limit=3600)`
+
+Runs a runtime analysis on the methods.
+
+| Argument | Type | Description |
+| :--- | :--- | :--- |
+| **`methods`** | List[dict] | "[{""name"": str, ""func"": callable, ""stochastic"": bool}, ...]" |
+| **`vary_param`** | ["CLSACC", "NMI", "ACC", "AUC"] | "features", "instances", or "both". |
+| **`time_limit`** | 3600 | Terminate the method after reecording first time it exceeds this limit. |
+
 #  Dashboard
 
 There is a Feature Selection Evaluation Dashboard based on the benchmarks provided by FSEVAL, available on:

{sdufseval-1.0.1 → sdufseval-1.0.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sdufseval"
-version = "1.0.1"
+version = "1.0.2"
 authors = [
   { name="Muhammad Rajabinasab", email="muhammad.rajabinasab@outlook.com" },
 ]

{sdufseval-1.0.1 → sdufseval-1.0.2}/sdufseval/fseval.py

@@ -1,5 +1,6 @@
 import os
 import math
+import time
 import warnings
 import numpy as np
 import pandas as pd
@@ -107,6 +108,101 @@ class FSEVAL:
                 # Save/Update results for this method/dataset
                 self._save_results(name, ds_results)
 
+    
+    def timer(self, methods, vary_param='both', time_limit=3600):
+        """
+        Runs a standalone runtime analysis experiment with a time cap.
+        
+        Args:
+            methods: List of dicts {'name': str, 'func': callable}.
+            vary_param: 'features', 'instances', or 'both'.
+            time_limit: Max seconds per method before it is skipped.
+        """
+        
+        # Determine which experiments to run
+        experiments = []
+        if vary_param in ['features', 'both']:
+            experiments.append({
+                'name': 'features',
+                'fixed_val': 100,
+                'range': range(1000, 20001, 500),
+                'file': 'time_analysis_features.csv'
+            })
+        if vary_param in ['instances', 'both']:
+            experiments.append({
+                'name': 'instances',
+                'fixed_val': 100,
+                'range': range(1000, 20001, 500),
+                'file': 'time_analysis_instances.csv'
+            })
+
+        for exp in experiments:
+            vary_type = exp['name']
+            val_range = exp['range']
+            filename = os.path.join(self.output_dir, exp['file'])
+            
+            # Tracking for this specific experiment
+            timed_out_methods = set()
+            results = {m['name']: [] for m in methods}
+            
+            print(f"\n--- Starting Experiment: Varying {vary_type} ---")
+            print(f"Time limit: {time_limit}s | Output: {filename}")
+
+            for val in val_range:
+                # 1. Generate synthetic data based on vary_param
+                if vary_type == 'features':
+                    n_samples, n_features = exp['fixed_val'], val
+                else:
+                    n_samples, n_features = val, exp['fixed_val']
+
+                try:
+                    X = np.random.rand(n_samples, n_features)
+                except MemoryError:
+                    print(f"  FATAL: MemoryError: Failed to allocate {n_samples}x{n_features} data.")
+                    for m in methods:
+                        results[m['name']].append(-1 if m['name'] in timed_out_methods else np.nan)
+                    continue
+
+                # 2. Run each method
+                for m_info in methods:
+                    name = m_info['name']
+                    func = m_info['func']
+                    
+                    # Check if method has already timed out in this experiment
+                    if name in timed_out_methods:
+                        results[name].append(-1)
+                        continue
+                    
+                    try:
+                        start_time = time.time()
+                        
+                        # Execute the method (assuming benchmark format)
+                        func(X)
+                        
+                        duration = time.time() - start_time
+                        
+                        if duration > time_limit:
+                            print(f"  - {name:<18}: {duration:.4f}s (TIMEOUT - skipping future runs)")
+                            timed_out_methods.add(name)
+                        else:
+                            print(f"  - {name:<18}: {duration:.4f}s")
+                        
+                        results[name].append(duration)
+                        
+                    except Exception as e:
+                        print(f"  - {name:<18}: FAILED ({type(e).__name__})")
+                        results[name].append(np.nan)
+
+            # 3. Save results to CSV
+            try:
+                df_results = pd.DataFrame.from_dict(results, orient='index', columns=list(val_range))
+                df_results.index.name = 'Method'
+                df_results.to_csv(filename)
+                print(f"\n--- Results saved to {filename} ---")
+            except Exception as e:
+                print(f"\n--- FAILED to save results: {e} ---")
+
+
     def _save_results(self, method_name, ds_results):
         """Aggregates repeats and saves to disk after each dataset."""
         for scale, metrics in ds_results.items():