featcopilot 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

featcopilot/selection/statistical.py

@@ -98,67 +98,137 @@ class StatisticalSelector(BaseSelector):
     def _compute_mutual_info(self, X: pd.DataFrame, y: np.ndarray) -> np.ndarray:
         """Compute mutual information scores."""
         from sklearn.feature_selection import mutual_info_classif, mutual_info_regression
+        from sklearn.preprocessing import LabelEncoder
 
-        # Determine if classification or regression
-        unique_y = len(np.unique(y))
-        is_classification = unique_y < 20 and y.dtype in [np.int32, np.int64, "object"]
-
-        X_array = X.fillna(0).values
+        # Encode string labels if needed
+        y_encoded = y
+        if y.dtype == object or y.dtype.kind in ("U", "S"):
+            le = LabelEncoder()
+            y_encoded = le.fit_transform(y)
 
-        if is_classification:
-            scores = mutual_info_classif(X_array, y, random_state=42)
-        else:
-            scores = mutual_info_regression(X_array, y, random_state=42)
+        # Determine if classification or regression
+        # Check if target is categorical (object type) or has discrete integer values
+        unique_y = len(np.unique(y_encoded))
+        is_classification = (
+            y.dtype == object
+            or y.dtype.kind in ("U", "S")
+            or (np.issubdtype(y_encoded.dtype, np.integer) and unique_y <= len(y_encoded) * 0.1)
+        )
+
+        # Filter to numeric columns only
+        numeric_cols = X.select_dtypes(include=[np.number]).columns.tolist()
+        scores = np.zeros(len(X.columns))
+
+        if numeric_cols:
+            X_numeric = X[numeric_cols].fillna(0).values
+            numeric_indices = [X.columns.get_loc(c) for c in numeric_cols]
+
+            if is_classification:
+                numeric_scores = mutual_info_classif(X_numeric, y_encoded, random_state=42)
+            else:
+                numeric_scores = mutual_info_regression(X_numeric, y_encoded, random_state=42)
+
+            for i, idx in enumerate(numeric_indices):
+                scores[idx] = numeric_scores[i]
 
         return scores
 
     def _compute_f_test(self, X: pd.DataFrame, y: np.ndarray) -> np.ndarray:
         """Compute F-test scores."""
         from sklearn.feature_selection import f_classif, f_regression
+        from sklearn.preprocessing import LabelEncoder
+
+        # Encode string labels if needed
+        y_encoded = y
+        if y.dtype == object or y.dtype.kind in ("U", "S"):
+            le = LabelEncoder()
+            y_encoded = le.fit_transform(y)
 
-        unique_y = len(np.unique(y))
-        is_classification = unique_y < 20
+        # Determine if classification or regression
+        unique_y = len(np.unique(y_encoded))
+        is_classification = (
+            y.dtype == object
+            or y.dtype.kind in ("U", "S")
+            or (np.issubdtype(y_encoded.dtype, np.integer) and unique_y <= len(y_encoded) * 0.1)
+        )
 
-        X_array = X.fillna(0).values
+        # Filter to numeric columns only
+        numeric_cols = X.select_dtypes(include=[np.number]).columns.tolist()
+        scores = np.zeros(len(X.columns))
 
-        if is_classification:
-            scores, _ = f_classif(X_array, y)
-        else:
-            scores, _ = f_regression(X_array, y)
+        if numeric_cols:
+            X_numeric = X[numeric_cols].fillna(0).values
+            numeric_indices = [X.columns.get_loc(c) for c in numeric_cols]
+
+            if is_classification:
+                numeric_scores, _ = f_classif(X_numeric, y_encoded)
+            else:
+                numeric_scores, _ = f_regression(X_numeric, y_encoded)
+
+            # Handle NaN scores
+            numeric_scores = np.nan_to_num(numeric_scores, 0)
+
+            for i, idx in enumerate(numeric_indices):
+                scores[idx] = numeric_scores[i]
 
-        # Handle NaN scores
-        scores = np.nan_to_num(scores, 0)
         return scores
 
     def _compute_chi2(self, X: pd.DataFrame, y: np.ndarray) -> np.ndarray:
         """Compute chi-square scores (for non-negative features)."""
         from sklearn.feature_selection import chi2
+        from sklearn.preprocessing import LabelEncoder
 
-        X_array = X.fillna(0).values
+        # Encode string labels if needed
+        y_encoded = y
+        if y.dtype == object or y.dtype.kind in ("U", "S"):
+            le = LabelEncoder()
+            y_encoded = le.fit_transform(y)
 
-        # Chi2 requires non-negative values
-        X_positive = X_array - X_array.min(axis=0) + 1e-8
+        # Filter to numeric columns only
+        numeric_cols = X.select_dtypes(include=[np.number]).columns.tolist()
+        scores = np.zeros(len(X.columns))
 
-        try:
-            scores, _ = chi2(X_positive, y)
-            scores = np.nan_to_num(scores, 0)
-        except Exception:
-            # Fallback to mutual information
-            scores = self._compute_mutual_info(X, y)
+        if numeric_cols:
+            X_numeric = X[numeric_cols].fillna(0).values
+            numeric_indices = [X.columns.get_loc(c) for c in numeric_cols]
+
+            # Chi2 requires non-negative values
+            X_positive = X_numeric - X_numeric.min(axis=0) + 1e-8
+
+            try:
+                numeric_scores, _ = chi2(X_positive, y_encoded)
+                numeric_scores = np.nan_to_num(numeric_scores, 0)
+            except Exception:
+                # Fallback to mutual information
+                return self._compute_mutual_info(X, y)
+
+            for i, idx in enumerate(numeric_indices):
+                scores[idx] = numeric_scores[i]
 
         return scores
 
     def _compute_correlation(self, X: pd.DataFrame, y: np.ndarray) -> np.ndarray:
         """Compute absolute correlation with target."""
-        scores = []
-        for col in X.columns:
+        from sklearn.preprocessing import LabelEncoder
+
+        # Encode string labels if needed
+        y_encoded = y
+        if y.dtype == object or y.dtype.kind in ("U", "S"):
+            le = LabelEncoder()
+            y_encoded = le.fit_transform(y).astype(float)
+
+        numeric_cols = X.select_dtypes(include=[np.number]).columns.tolist()
+        scores = np.zeros(len(X.columns))
+
+        for col in numeric_cols:
             try:
-                corr = np.abs(np.corrcoef(X[col].fillna(0).values, y)[0, 1])
-                scores.append(corr if not np.isnan(corr) else 0)
+                idx = X.columns.get_loc(col)
+                corr = np.abs(np.corrcoef(X[col].fillna(0).values, y_encoded)[0, 1])
+                scores[idx] = corr if not np.isnan(corr) else 0
             except Exception:
-                scores.append(0)
+                pass
 
-        return np.array(scores)
+        return scores
 
     def _select_features(self) -> None:
         """Select features based on scores."""

featcopilot/selection/unified.py

@@ -46,6 +46,7 @@ class FeatureSelector(BaseSelector):
         max_features: Optional[int] = None,
         correlation_threshold: float = 0.95,
         combination: str = "union",
+        original_features: Optional[set[str]] = None,
         verbose: bool = False,
         **kwargs,
     ):
@@ -54,6 +55,7 @@ class FeatureSelector(BaseSelector):
         self.max_features = max_features
         self.correlation_threshold = correlation_threshold
         self.combination = combination  # 'union' or 'intersection'
+        self.original_features = original_features or set()
         self.verbose = verbose
         self._selectors: dict[str, BaseSelector] = {}
         self._method_scores: dict[str, dict[str, float]] = {}
@@ -76,6 +78,10 @@ class FeatureSelector(BaseSelector):
         X = self._validate_input(X)
         y = np.array(y)
 
+        # Identify categorical/text columns (can't be scored by numeric methods)
+        categorical_cols = X.select_dtypes(include=["object", "category"]).columns.tolist()
+        numeric_cols = X.select_dtypes(include=[np.number]).columns.tolist()
+
         # Initialize and fit each selector
         for method in self.methods:
             selector = self._create_selector(method)
@@ -86,11 +92,27 @@ class FeatureSelector(BaseSelector):
         # Combine scores from all methods
         self._combine_scores(X.columns.tolist())
 
+        # Give categorical columns a minimum score so they're not filtered out
+        # Original categorical columns are important for models that can handle them
+        if categorical_cols:
+            # Get the median score of numeric features to use as baseline for categorical
+            numeric_scores = [v for k, v in self._feature_scores.items() if k in numeric_cols and v > 0]
+            if numeric_scores:
+                baseline_score = np.median(numeric_scores)
+            else:
+                baseline_score = 0.5  # Default if no numeric scores
+
+            for col in categorical_cols:
+                if col in self.original_features:
+                    # Original categorical columns get a baseline score
+                    self._feature_scores[col] = max(self._feature_scores.get(col, 0), baseline_score)
+
         # Apply redundancy elimination
         if self.correlation_threshold < 1.0:
             eliminator = RedundancyEliminator(
                 correlation_threshold=self.correlation_threshold,
                 importance_scores=self._feature_scores,
+                original_features=self.original_features,
                 verbose=self.verbose,
             )
             eliminator.fit(X)
@@ -149,13 +171,42 @@ class FeatureSelector(BaseSelector):
         """Make final feature selection."""
         sorted_features = sorted(self._feature_scores.items(), key=lambda x: x[1], reverse=True)
 
+        # Always include original features first
+        original_selected = []
+        derived_selected = []
+
+        for name, score in sorted_features:
+            if name in self.original_features:
+                original_selected.append(name)
+            else:
+                # Only include derived features with meaningful importance (> 1% of max)
+                max_score = max(self._feature_scores.values()) if self._feature_scores else 1.0
+                importance_threshold = max_score * 0.01  # 1% threshold
+                if score >= importance_threshold:
+                    derived_selected.append(name)
+                elif self.verbose:
+                    logger.debug(f"Excluding low-importance feature {name} (score={score:.4f})")
+
+        # Apply max_features limit only to derived features
         if self.max_features is not None:
-            sorted_features = sorted_features[: self.max_features]
+            # Reserve slots for original features, then fill with top derived
+            n_derived = max(0, self.max_features - len(original_selected))
+            derived_selected = derived_selected[:n_derived]
 
-        self._selected_features = [name for name, _ in sorted_features]
+        self._selected_features = original_selected + derived_selected
+
+        # Ensure we never have fewer features than original
+        if len(self._selected_features) < len(self.original_features):
+            # This should not happen, but add all original features as safety
+            for f in self.original_features:
+                if f not in self._selected_features:
+                    self._selected_features.append(f)
 
         if self.verbose:
-            logger.info(f"FeatureSelector: Selected {len(self._selected_features)} features")
+            logger.info(
+                f"FeatureSelector: Selected {len(self._selected_features)} features "
+                f"({len(original_selected)} original + {len(derived_selected)} derived)"
+            )
 
     def transform(self, X: Union[pd.DataFrame, np.ndarray], **kwargs) -> pd.DataFrame:
         """Select features from data."""

featcopilot/stores/__init__.py

@@ -7,9 +7,11 @@ and serving in production ML systems.
 
 from featcopilot.stores.base import BaseFeatureStore, FeatureStoreConfig
 from featcopilot.stores.feast_store import FeastFeatureStore
+from featcopilot.stores.rule_store import TransformRuleStore
 
 __all__ = [
     "BaseFeatureStore",
     "FeatureStoreConfig",
     "FeastFeatureStore",
+    "TransformRuleStore",
 ]

featcopilot/stores/rule_store.py

@@ -0,0 +1,343 @@
+"""Persistent storage for transform rules.
+
+Provides JSON-file based storage for saving, loading, and searching
+reusable transform rules.
+"""
+
+import json
+import os
+from pathlib import Path
+from typing import Optional
+
+from featcopilot.core.transform_rule import TransformRule
+from featcopilot.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class TransformRuleStore:
+    """
+    Persistent storage for transform rules.
+
+    Stores rules in a JSON file for reuse across sessions and datasets.
+    Supports searching by tags, description similarity, and column patterns.
+
+    Parameters
+    ----------
+    path : str, optional
+        Path to the JSON file for storage. Defaults to ~/.featcopilot/rules.json
+
+    Examples
+    --------
+    >>> store = TransformRuleStore()
+    >>> store.save_rule(rule)
+    >>> matching = store.find_matching_rules(columns=["price", "quantity"])
+    >>> all_rules = store.list_rules()
+    """
+
+    DEFAULT_PATH = "~/.featcopilot/rules.json"
+
+    def __init__(self, path: Optional[str] = None):
+        self.path = Path(os.path.expanduser(path or self.DEFAULT_PATH))
+        self._rules: dict[str, TransformRule] = {}
+        self._ensure_directory()
+        self._load()
+
+    def _ensure_directory(self) -> None:
+        """Ensure the storage directory exists."""
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+
+    def _load(self) -> None:
+        """Load rules from storage file."""
+        if self.path.exists():
+            try:
+                with open(self.path, encoding="utf-8") as f:
+                    data = json.load(f)
+                    self._rules = {rule_id: TransformRule.from_dict(rule_data) for rule_id, rule_data in data.items()}
+                logger.debug(f"Loaded {len(self._rules)} rules from {self.path}")
+            except (json.JSONDecodeError, KeyError) as e:
+                logger.warning(f"Failed to load rules from {self.path}: {e}")
+                self._rules = {}
+        else:
+            self._rules = {}
+
+    def _save(self) -> None:
+        """Save rules to storage file."""
+        try:
+            with open(self.path, "w", encoding="utf-8") as f:
+                data = {rule_id: rule.to_dict() for rule_id, rule in self._rules.items()}
+                json.dump(data, f, indent=2)
+            logger.debug(f"Saved {len(self._rules)} rules to {self.path}")
+        except OSError as e:
+            logger.error(f"Failed to save rules to {self.path}: {e}")
+            raise
+
+    def save_rule(self, rule: TransformRule) -> str:
+        """
+        Save a rule to the store.
+
+        Parameters
+        ----------
+        rule : TransformRule
+            The rule to save
+
+        Returns
+        -------
+        str
+            The rule's ID
+        """
+        self._rules[rule.id] = rule
+        self._save()
+        logger.info(f"Saved rule '{rule.name}' with ID {rule.id}")
+        return rule.id
+
+    def get_rule(self, rule_id: str) -> Optional[TransformRule]:
+        """
+        Get a rule by ID.
+
+        Parameters
+        ----------
+        rule_id : str
+            The rule's ID
+
+        Returns
+        -------
+        TransformRule or None
+            The rule if found, None otherwise
+        """
+        return self._rules.get(rule_id)
+
+    def get_rule_by_name(self, name: str) -> Optional[TransformRule]:
+        """
+        Get a rule by name.
+
+        Parameters
+        ----------
+        name : str
+            The rule's name
+
+        Returns
+        -------
+        TransformRule or None
+            The first rule matching the name, None if not found
+        """
+        for rule in self._rules.values():
+            if rule.name == name:
+                return rule
+        return None
+
+    def delete_rule(self, rule_id: str) -> bool:
+        """
+        Delete a rule by ID.
+
+        Parameters
+        ----------
+        rule_id : str
+            The rule's ID
+
+        Returns
+        -------
+        bool
+            True if deleted, False if not found
+        """
+        if rule_id in self._rules:
+            del self._rules[rule_id]
+            self._save()
+            logger.info(f"Deleted rule {rule_id}")
+            return True
+        return False
+
+    def list_rules(self, tags: Optional[list[str]] = None) -> list[TransformRule]:
+        """
+        List all rules, optionally filtered by tags.
+
+        Parameters
+        ----------
+        tags : list[str], optional
+            Filter rules that have all specified tags
+
+        Returns
+        -------
+        list[TransformRule]
+            List of matching rules
+        """
+        rules = list(self._rules.values())
+
+        if tags:
+            rules = [r for r in rules if all(t in r.tags for t in tags)]
+
+        return rules
+
+    def find_matching_rules(
+        self,
+        columns: Optional[list[str]] = None,
+        description: Optional[str] = None,
+        tags: Optional[list[str]] = None,
+        min_usage: int = 0,
+    ) -> list[tuple[TransformRule, dict[str, str]]]:
+        """
+        Find rules that can be applied to the given context.
+
+        Parameters
+        ----------
+        columns : list[str], optional
+            Available column names to match against
+        description : str, optional
+            Description to search for (keyword matching)
+        tags : list[str], optional
+            Required tags
+        min_usage : int, default=0
+            Minimum usage count
+
+        Returns
+        -------
+        list[tuple[TransformRule, dict]]
+            List of (rule, column_mapping) tuples for applicable rules,
+            sorted by usage count (most used first)
+        """
+        results: list[tuple[TransformRule, dict[str, str]]] = []
+
+        for rule in self._rules.values():
+            # Filter by usage count
+            if rule.usage_count < min_usage:
+                continue
+
+            # Filter by tags
+            if tags and not all(t in rule.tags for t in tags):
+                continue
+
+            # Filter by description keywords
+            if description:
+                keywords = description.lower().split()
+                rule_text = f"{rule.name} {rule.description}".lower()
+                if not any(kw in rule_text for kw in keywords):
+                    continue
+
+            # Check column compatibility
+            mapping: dict[str, str] = {}
+            if columns:
+                matches, mapping = rule.matches_columns(columns)
+                if not matches:
+                    continue
+
+            results.append((rule, mapping))
+
+        # Sort by usage count (descending)
+        results.sort(key=lambda x: x[0].usage_count, reverse=True)
+
+        return results
+
+    def search_by_description(self, query: str, limit: int = 10) -> list[TransformRule]:
+        """
+        Search rules by description similarity.
+
+        Parameters
+        ----------
+        query : str
+            Search query
+        limit : int, default=10
+            Maximum number of results
+
+        Returns
+        -------
+        list[TransformRule]
+            Matching rules sorted by relevance
+        """
+        query_words = set(query.lower().split())
+        scored_rules: list[tuple[float, TransformRule]] = []
+
+        for rule in self._rules.values():
+            rule_words = set(f"{rule.name} {rule.description}".lower().split())
+
+            # Simple word overlap scoring
+            overlap = len(query_words & rule_words)
+            if overlap > 0:
+                score = overlap / len(query_words)
+                scored_rules.append((score, rule))
+
+        # Sort by score descending
+        scored_rules.sort(key=lambda x: x[0], reverse=True)
+
+        return [rule for _, rule in scored_rules[:limit]]
+
+    def import_rules(self, path: str, merge: bool = True) -> int:
+        """
+        Import rules from another JSON file.
+
+        Parameters
+        ----------
+        path : str
+            Path to import from
+        merge : bool, default=True
+            If True, merge with existing rules. If False, replace all.
+
+        Returns
+        -------
+        int
+            Number of rules imported
+        """
+        import_path = Path(os.path.expanduser(path))
+
+        if not import_path.exists():
+            raise FileNotFoundError(f"Import file not found: {path}")
+
+        with open(import_path, encoding="utf-8") as f:
+            data = json.load(f)
+
+        if not merge:
+            self._rules = {}
+
+        count = 0
+        for _rule_id, rule_data in data.items():
+            rule = TransformRule.from_dict(rule_data)
+            self._rules[rule.id] = rule
+            count += 1
+
+        self._save()
+        logger.info(f"Imported {count} rules from {path}")
+
+        return count
+
+    def export_rules(self, path: str, tags: Optional[list[str]] = None) -> int:
+        """
+        Export rules to a JSON file.
+
+        Parameters
+        ----------
+        path : str
+            Path to export to
+        tags : list[str], optional
+            Only export rules with these tags
+
+        Returns
+        -------
+        int
+            Number of rules exported
+        """
+        export_path = Path(os.path.expanduser(path))
+        export_path.parent.mkdir(parents=True, exist_ok=True)
+
+        rules_to_export = self.list_rules(tags=tags)
+
+        with open(export_path, "w", encoding="utf-8") as f:
+            data = {r.id: r.to_dict() for r in rules_to_export}
+            json.dump(data, f, indent=2)
+
+        logger.info(f"Exported {len(rules_to_export)} rules to {path}")
+
+        return len(rules_to_export)
+
+    def clear(self) -> None:
+        """Remove all rules from the store."""
+        self._rules = {}
+        self._save()
+        logger.info("Cleared all rules")
+
+    def __len__(self) -> int:
+        return len(self._rules)
+
+    def __contains__(self, rule_id: str) -> bool:
+        return rule_id in self._rules
+
+    def __iter__(self):
+        return iter(self._rules.values())

featcopilot/transformers/sklearn_compat.py

@@ -90,7 +90,7 @@ class AutoFeatureEngineer(BaseEstimator, TransformerMixin):
     Parameters
     ----------
     engines : list, default=['tabular']
-        Engines to use ('tabular', 'timeseries', 'text', 'llm')
+        Engines to use ('tabular', 'timeseries', 'relational', 'text', 'llm')
     max_features : int, optional
         Maximum features to generate/select
     selection_methods : list, default=['mutual_info', 'importance']
@@ -199,6 +199,8 @@ class AutoFeatureEngineer(BaseEstimator, TransformerMixin):
             return TimeSeriesEngine(max_features=self.max_features, verbose=self.verbose)
         elif engine_name == "text":
             return TextEngine(max_features=self.max_features, verbose=self.verbose)
+        elif engine_name == "relational":
+            return RelationalEngine(max_features=self.max_features, verbose=self.verbose)
         elif engine_name == "llm":
             from featcopilot.llm.semantic_engine import SemanticEngine
 
@@ -294,12 +296,19 @@ class AutoFeatureEngineer(BaseEstimator, TransformerMixin):
         self.fit(X, y, column_descriptions, task_description, **fit_params)
         result = self.transform(X)
 
+        # Track original features (input columns) vs derived features
+        if isinstance(X, np.ndarray):
+            original_features = {f"feature_{i}" for i in range(X.shape[1])}
+        else:
+            original_features = set(X.columns)
+
         # Apply feature selection if enabled and y is provided
         if apply_selection and y is not None and self.max_features:
             self._selector = FeatureSelector(
                 methods=self.selection_methods,
                 max_features=self.max_features,
                 correlation_threshold=self.correlation_threshold,
+                original_features=original_features,
                 verbose=self.verbose,
             )
             result = self._selector.fit_transform(result, y)