ImportanceScore 1.1.2__tar.gz

importancescore-1.1.2/ImportanceScore/ImportanceScore/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.1"

importancescore-1.1.2/ImportanceScore/ImportanceScore/csv_table.py

@@ -0,0 +1,114 @@
+from pathlib import Path
+import sys
+from typing import List, Optional
+
+import pandas as pd
+from PySide6.QtCore import Qt
+from PySide6.QtWidgets import (QApplication, QHBoxLayout, QLabel, QLineEdit, QMainWindow,
+                               QPushButton, QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget, )
+
+
+class CsvTable(QWidget):
+    """
+    QWidget for viewing a CSV data in a table - supports find and sort
+    """
+    def __init__(self, parent: Optional[QWidget] = None) -> None:
+        super().__init__(parent)
+        main_layout = QVBoxLayout(self)
+        self.table_widget = QTableWidget()
+        self.table_widget.setSortingEnabled(True)
+        self.table_widget.setEditTriggers(QTableWidget.EditTrigger.NoEditTriggers)
+        main_layout.addWidget(self.table_widget)
+
+        find_layout = QHBoxLayout()
+        self.find_input = QLineEdit()
+        self.find_input.textChanged.connect(self._reset_search)
+
+        self.find_button = QPushButton("Find")
+        self.find_button.clicked.connect(self._on_find_clicked)
+
+        find_layout.addWidget(QLabel("Find:"))
+        find_layout.addWidget(self.find_input)
+        find_layout.addWidget(self.find_button)
+        find_layout.addStretch()
+        main_layout.addLayout(find_layout)
+
+        self._search_results = []
+        self._current_search_index = -1
+
+    def clear(self):
+        """ Clear table contents """
+        self.table_widget.clear()
+
+    def load(self, path: Path, remove_columns: Optional[List[str]] = None):
+        """Load and displays a CSV file.  Remove columns if specified """
+        try:
+            print(f"Loading {path}")
+            self.table_widget.clear()
+
+            df = pd.read_csv(path)
+            if remove_columns:
+                df = df.drop(columns=remove_columns, errors='ignore')
+
+            self.table_widget.setRowCount(df.shape[0])
+            self.table_widget.setColumnCount(df.shape[1])
+            self.table_widget.setHorizontalHeaderLabels(df.columns)
+
+            for row_idx, row_data in enumerate(df.values):
+                for col_idx, value in enumerate(row_data):
+                    if isinstance(value, (int, float)):
+                        if value > 2147483647:   # Max C++ int
+                            item = QTableWidgetItem(str(value))
+                        else:
+                            item = QTableWidgetItem(value)
+                        item.setData(Qt.ItemDataRole.EditRole, value)
+                    else:
+                        txt = "" if pd.isna(value) else str(value)
+                        item = QTableWidgetItem(txt)
+
+                    self.table_widget.setItem(row_idx, col_idx, item)
+
+            self.table_widget.resizeColumnsToContents()
+            print("   Load complete")
+
+        except Exception as e:
+            print(f"err {e}")
+            self.table_widget.setItem(0, 0, QTableWidgetItem(f"Error loading file"))
+
+
+    def _reset_search(self):
+        """
+        Resets the search state. Called whenever the find text is changed.
+        """
+        self.table_widget.clearSelection()
+        self._search_results = []
+        self._current_search_index = -1
+
+    def _on_find_clicked(self):
+        """
+        Handles a click on the "Find" button.
+        - If it's a new search (or text has changed), it finds all occurrences.
+        - If it's a subsequent search, it finds the next occurrence.
+        """
+        search_text = self.find_input.text()
+
+        # If the index is -1, it means this is a new search.
+        if self._current_search_index == -1:
+            self._search_results = self.table_widget.findItems(
+                search_text, Qt.MatchFlag.MatchContains
+            )
+            if not self._search_results:
+                return
+
+        # If we have results, move to the next one.
+        if self._search_results:
+            self._current_search_index = (self._current_search_index + 1) % len(self._search_results)
+            self._highlight_current_search_result()
+
+    def _highlight_current_search_result(self):
+        """Scrolls to and selects the current search item."""
+        if not self._search_results:
+            return
+        item = self._search_results[self._current_search_index]
+        self.table_widget.setCurrentItem(item)
+

importancescore-1.1.2/ImportanceScore/ImportanceScore/custom_editor.py

@@ -0,0 +1,50 @@
+# custom_editor.py
+from PySide6.QtGui import QColor, QSyntaxHighlighter, QTextCharFormat, QFont
+from pygments import highlight
+from pygments.lexers import YamlLexer
+from pygments.token import Token
+from pygments.styles import get_style_by_name
+
+class PygmentsHighlighter(QSyntaxHighlighter):
+    """A QSyntaxHighlighter that uses the Pygments library for styling."""
+
+    def __init__(self, parent):
+        super().__init__(parent)
+
+        # The lexer is responsible for breaking text into tokens
+        self.lexer = YamlLexer()
+
+        # 1. Get a Pygments style.
+        # 2. Create a dictionary mapping token types to QTextCharFormat objects.
+        self.styles = {}
+        style = get_style_by_name('dracula')
+        for token, s in style:
+            q_format = QTextCharFormat()
+            if s['color']:
+                q_format.setForeground(QColor(f"#{s['color']}"))
+
+            self.styles[token] = q_format
+
+    def highlightBlock(self, text: str):
+        """
+        This method is called by Qt for each block of text to be highlighted.
+        """
+        # Get the tokens from Pygments
+        tokens = self.lexer.get_tokens_unprocessed(text)
+
+        start_index = 0
+        for index, token_type, value in tokens:
+            # Find the correct style for this token
+            # We traverse up the token hierarchy (e.g., from Number.Integer to Number to Token)
+            # until we find a style defined.
+            current_format = None
+            temp_token_type = token_type
+            while current_format is None:
+                current_format = self.styles.get(temp_token_type)
+                if temp_token_type is Token:
+                    break
+                temp_token_type = temp_token_type.parent
+
+            # Apply the format if one was found
+            if current_format:
+                self.setFormat(index, len(value), current_format)

importancescore-1.1.2/ImportanceScore/ImportanceScore/data_loader.py

@@ -0,0 +1,211 @@
+# data_loader.py
+
+import sys
+
+import pandas as pd
+
+from ImportanceScore.model import get_target_column
+
+
+class DataLoader:
+    """
+    Handles loading, validating, and merging of feature and target datasets.
+
+    This class encapsulates the data loading logic, using a configuration
+    dictionary to dynamically handle different data schemas and requirements.
+    """
+
+    def __init__(self, model_config: dict):
+        """
+        Initializes the DataLoader with a model configuration.
+
+        Args:
+            model_config (dict): A dictionary containing model settings, including
+                                 the `id_column` and `regressor_target`.
+        """
+        if not isinstance(model_config, dict):
+            raise TypeError("model_config must be a dictionary.")
+
+        self.config = model_config
+        self.id_column = self.config['id_column']
+        self.name_key = self.config['name_column']
+        self.model_name = self.config['model']
+        self.target_column = get_target_column(self.config, self.model_name)
+
+    @staticmethod
+    def _safe_read_csv(path, dtype=None, required_cols=None, label=None):
+        """Internal method to safely read a CSV with validation."""
+        try:
+            df = pd.read_csv(path, dtype=dtype)
+        except Exception as e:
+            print(f"  ❌ Error loading {label}: {path}. {e}")
+            sys.exit(1)
+
+        if required_cols:
+            missing = [col for col in required_cols if col not in df.columns]
+            if missing:
+                print(f"  ❌ {label or path} is missing required columns: {missing}")
+                sys.exit(1)
+        return df
+
+    def _validate_input_files(self, df_features, df_targets):
+        """Internal method to ensure required columns exist in dataframes."""
+        required_target_cols = [self.id_column, self.target_column]
+        for col in required_target_cols:
+            if col not in df_targets.columns:
+                raise ValueError(f"Missing '{col}' in targets file.")
+        if self.id_column not in df_features.columns:
+            raise ValueError(f"Missing '{self.id_column}' in features file.")
+
+    def load_and_merge_data(self, features_path: str, targets_path: str) -> pd.DataFrame:
+        """
+        Loads, validates, and merges feature and target data.
+
+        Args:
+            features_path (str): Path to the features CSV.
+            targets_path (str): Path to the targets CSV.
+
+        Returns:
+            pd.DataFrame: A merged DataFrame ready for preprocessing.
+        """
+        required_final_cols = [self.id_column, self.name_key, self.target_column]
+
+        df_features = self._safe_read_csv(
+            features_path, dtype={self.id_column: str}, required_cols=[self.id_column],
+            label="Features CSV"
+        )
+        df_targets = self._safe_read_csv(
+            targets_path, dtype={self.id_column: str},
+            required_cols=[self.id_column, self.target_column], label="Targets CSV"
+        )
+
+        try:
+            if self.name_key in df_targets.columns:
+                df_targets = df_targets.drop(columns=[self.name_key])
+
+            self._validate_input_files(df_features, df_targets)
+            df = pd.merge(df_features, df_targets, on=self.id_column, how="left")
+
+            missing = [col for col in required_final_cols if col not in df.columns]
+            if missing:
+                raise ValueError(f"Missing required columns after merge: {', '.join(missing)}")
+
+            return df
+        except Exception as e:
+            print(f"Error loading data: {e}")
+            sys.exit(1)
+
+
+# Schema for Model config
+MODEL_CONFIG_SCHEMA = {
+    'config_type': {'type': 'string','required': True,'allowed': ["ModelConfig"] },
+    'anomaly_group_key': {'type': 'string', 'required': False},
+    'model': {
+        'type': 'string', 'required': True,
+        'allowed': ['WLM', 'RFR', 'GBT', 'LOGR', 'SVM', 'SVR', 'LR']
+    },
+    'id_column': {'type': 'string', 'required': True},
+    'name_column': {'type': 'string', 'required': True},
+    'regressor_target': {'type': 'string', 'required': True},
+    'base_score_column': {'type': 'string', 'required': False},
+
+    'scaling': {
+        'type': 'dict', 'required': False, 'schema': {
+            'min': {'type': 'number', 'required': True},
+            'max': {'type': 'number', 'required': True},
+        }
+    },
+
+    'text_weight_columns': {
+        'type': 'list', 'schema': {'type': 'string'}, 'required': False, 'default': []
+    }, 'one_hot_columns': {
+        'type': 'list', 'schema': {'type': 'string'}, 'required': False, 'default': []
+    }, 'fillna': {
+        'type': 'dict', 'required': False, 'valuesrules': {'type': ['number', 'string', 'boolean']}
+    }, 'conditional_adjustment': {
+        'type': 'dict', 'required': False, 'default': {}
+    }, 'ignore_columns': {
+        'type': 'list', 'schema': {'type': 'string'}, 'required': False, 'default': []
+    }, 'output_columns': {
+        'type': 'list', 'schema': {'type': 'string'}, 'required': True
+    },
+
+    'feature_interactions': {
+        'type': 'list', 'required': False, 'schema': {
+            'type': 'dict', 'schema': {
+                'output_column': {'type': 'string', 'required': True},
+                'input_columns': {'type': 'list', 'schema': {'type': 'string'}, 'required': True},
+                'method': {'type': 'string', 'allowed': ['logical_or'], 'required': True}
+            }
+        }
+    },
+
+    #  clip_outliers
+    'clip_outliers': {
+        'type': 'dict', 'required': False, 'valuesrules': {
+            'type': 'dict', 'schema': {
+                'method': {'type': 'string', 'allowed': ['IQR', 'threshold'], 'required': True},
+
+                # Parameters for the 'IQR' method
+                'factor': {'type': 'number', 'required': False, 'default': 1.5},
+
+                # Parameters for the new 'threshold' method
+                'min': {'type': 'number', 'required': False},
+                'max': {'type': 'number', 'required': False},
+            }
+        }
+    },
+
+    #  scaler section
+    'scaler': {
+        'type': 'dict', 'required': False, 'valuesrules': {
+            'type': 'string',
+            'allowed': ['standard', 'minmax', 'robust', 'power', 'sublinear', 'quantile', 'none']
+        }
+    },
+
+    'smoothing': {
+        'type': 'dict', 'required': False, 'schema': {
+            'method': {'type': 'string', 'allowed': ['sigmoid', 'none'], 'required': True},
+            'midpoint': {'type': 'number', 'required': False},
+            'steepness': {'type': 'number', 'required': False}
+        }
+    }
+}
+
+
+
+CLASSIFICATION_SCHEMA = {
+    'config_type': {'type': 'string','required': True,'allowed': ["Classification"] },
+    'keys': {'type': 'dict', 'required': True},
+    'output_tag': {'type': 'string', 'required': True},
+    'score_key': {'type': 'string', 'required': True},
+    'style': {'type': 'string', 'required': False},
+    'features': {'type': 'list', 'required': True},
+    'require_name': {
+        'type': 'boolean',
+        'required': False, # It's an optional setting
+        'default': True    # A sensible default: usually we want named features
+    },
+
+    'debug_ids': {
+        'type': 'list', 'schema': {'type': 'integer'},  # assuming node_row IDs are integers
+        'required': False,
+    },
+
+    'enrichment': {
+        'type': 'list',
+        'required': False,
+        'schema': {
+            'type': 'dict',
+            'schema': {
+                'file_suffix': {'type': 'string', 'required': True},
+                'columns': {
+                    'type': 'list',
+                    'required': True,
+                    'schema': {'type': 'string'}
+                }
+            }
+        }
+    },
+}

importancescore-1.1.2/ImportanceScore/ImportanceScore/features.py

@@ -0,0 +1,155 @@
+# features.py
+"""
+This module provides a robust FeaturePreprocessor class that acts as a
+data-driven orchestrator for the feature preprocessing pipeline.
+"""
+import logging
+import sys
+from typing import Tuple, Dict, Any, List
+
+import pandas as pd
+
+from ImportanceScore import transformations
+from ImportanceScore.model import get_target_column
+
+
+# --- Module-Level Helper for Logging ---
+def _setup_logger(level) -> logging.Logger:
+    """Initializes and configures the logger for the feature pipeline."""
+    logger = logging.getLogger("__file__")
+    logger.setLevel(level)
+    if not logger.handlers:
+        handler = logging.StreamHandler(sys.stdout)
+        formatter = logging.Formatter("%(message)s")
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    return logger
+
+
+class FeaturePreprocessor:
+    """
+    Orchestrates the feature preprocessing pipeline by executing a configurable
+    sequence of independent transformation strategies.
+    """
+    def __init__(self, model_config: Dict[str, Any], category: str, model_name: str):
+        """Initializes the FeaturePreprocessor."""
+        self.config = model_config
+        self.category = category
+        self.model_name = model_name
+        self.target_col_name = get_target_column(self.config, self.model_name)
+        self.logger = _setup_logger(4)
+
+        # This property will store the names of columns created during one-hot encoding.
+        self.one_hot_column_names: List[str] = []
+
+        self.pipeline_steps = [
+            ("feature_interactions", transformations.apply_feature_interactions),
+            ("fillna", lambda df, cfg, log: transformations.apply_fillna(
+                df, cfg, [self.target_col_name], log, self.category
+            )),
+            ("text_weight_columns", lambda df, cfg, log: transformations.apply_text_weight_scoring(
+                df, {"columns": cfg}, self.category, log
+            )),
+            ("conditional_adjustment", transformations.apply_conditional_adjustment),
+        ]
+
+    def transform(
+            self, raw_dataframe: pd.DataFrame, training: bool = True
+    ) -> Tuple[pd.DataFrame, pd.Series, pd.Series]:
+        """Executes the full preprocessing pipeline in the correct order."""
+        self.logger.info("\n🔷Preprocessing Features:\n")
+        self.logger.info(f"   {len(raw_dataframe)}  records.")
+
+        try:
+            processed_df = raw_dataframe.copy()
+
+            # --- Step 1: Handle the special case of one-hot encoding first ---
+            one_hot_config = {"columns": self.config.get("one_hot_columns", [])}
+            processed_df, self.one_hot_column_names = transformations.apply_one_hot_encoding(
+                processed_df, one_hot_config, self.logger
+            )
+
+            # --- Step 2: Run the rest of the data-driven pipeline ---
+            for config_key, transform_func in self.pipeline_steps:
+                step_config = self.config.get(config_key, {})
+                processed_df = transform_func(processed_df, step_config, self.logger)
+
+            # --- Step 3: Post-Transformation Orchestration ---
+            y, labeled_mask = self._extract_labels(processed_df, training)
+            feature_matrix_X = self._prune_columns(processed_df)
+            feature_matrix_X = transformations.clip_outliers(
+                feature_matrix_X, self.config.get("clip_outliers", {}), self.logger
+            )
+            feature_matrix_X = transformations.scale_features(
+                feature_matrix_X, self.config.get("scaler", {}), self.logger
+            )
+            self._validate_feature_coverage(feature_matrix_X)
+
+            return feature_matrix_X, y, labeled_mask
+
+        except (ValueError, KeyError) as e:
+            self.logger.error(f"\n   ❌ Preprocessing Error: {e}")
+            self.logger.error("      Processing stopped. Please correct the error above and rerun.")
+            sys.exit(1)
+
+    # --- Orchestration-Level Helper Methods ---
+
+    def _extract_labels(self, df: pd.DataFrame, training: bool) -> Tuple[pd.Series, pd.Series]:
+        """Extracts the target variable (y) and a mask of labeled rows."""
+        if not training:
+            return pd.Series([pd.NA] * len(df), dtype="Int64"), pd.Series([False] * len(df))
+
+        if self.target_col_name not in df.columns:
+            raise KeyError(
+                f"Label column '{self.target_col_name}' is required for training but is missing."
+            )
+
+        dtype = "Int64" if self.config.get("task_type") == "classification" else "float64"
+        y = df[self.target_col_name].astype(dtype)
+        return y, y.notna()
+
+    def _prune_columns(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Removes ignored and non-numeric columns to create the feature matrix."""
+        self.logger.info("➡️ Pruning columns to create feature matrix...")
+
+        ignore_cols = self.config.get("ignore_columns", [])
+        df.drop(columns=ignore_cols, errors="ignore", inplace=True)
+
+        numeric_dtypes = ["int64", "float64", "bool"]
+        non_numeric_cols = df.select_dtypes(exclude=numeric_dtypes).columns.tolist()
+        df.drop(columns=non_numeric_cols, errors="ignore", inplace=True)
+
+        self.logger.info(f"  • Ignored columns dropped: {ignore_cols}")
+        self.logger.info(f"  • Non-numeric columns dropped: {non_numeric_cols or 'none'}")
+
+        return df
+
+    def _validate_feature_coverage(self, X: pd.DataFrame) -> None:
+        """Validates that each feature column has some non-zero coverage."""
+        self.logger.info("\n✳️ Feature column coverage:\n")
+        for col in X.columns:
+            _validate_tag(X, col, threshold=1, logger=self.logger)
+
+        self.logger.info("\n")
+
+# ---  Utility Functions ---
+
+def _validate_tag(df: pd.DataFrame, column: str, threshold: float = None, logger=None) -> bool:
+    """Ensures a given column has sufficient non-zero coverage."""
+    if column not in df.columns:
+        logger.info(f"❌ Column '{column}' not found for coverage validation.")
+        return False
+
+    pct_filled = df[column].fillna(0).ne(0).mean()
+
+    if threshold is None:
+        logger.info(f"↪ Column '{column}': {pct_filled:.0%} filled")
+        return True
+
+    is_valid = pct_filled >= (threshold / 100)
+    status_emoji = "🆗" if is_valid else "⚠️"
+    column_name = f"{column}:"
+    logger.info(
+        f"      {status_emoji} {column_name:<20} {pct_filled:>7.2%} filled"
+    )
+    return is_valid