rapidsegment 1.0.0__py3-none-any.whl

rapidsegment/scorer.py

@@ -0,0 +1,174 @@
+"""
+Strategic Segmentation Scorecard Engine
+=========================================
+Creating scorecards for multi-segmented populations using vectorized DuckDB aggregations.
+
+Author: Bishwarup Biswas + Gemini
+Python Version: 3.9+
+"""
+
+import json
+import logging
+import os
+from typing import Any, Dict, List, Union
+import duckdb
+from datetime import datetime
+
+# 1. Get current date and time
+now = datetime.now()
+
+# Example output: "2026_07_11_22_24_30"
+timestamp = now.strftime("%Y_%m_%d_%H_%M_%S")
+
+
+# Configure Production Module Logger
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s | %(levelname)-8s | [%(filename)s:%(lineno)d] | %(message)s",
+)
+logger = logging.getLogger("StrategicEngine")
+
+class StrategicSegmentScore:
+    """High-Throughput Vectorized Scorecard Engine.
+
+    Computes segment weights via Harmonic Mean and applies deciling
+    over large datasets natively inside DuckDB's out-of-core engine.
+    """
+
+    def __init__(
+        self, target_col: str, primary_key: str, segment_cols: List[str]
+    ) -> None:
+        self.target_col = target_col
+        self.primary_key = primary_key
+        self.segment_cols = segment_cols
+        self.model_artifact: Dict[str, Any] = {}
+
+    def calculate_and_export_weights(
+        self, data: Any, export_path: str = f"scored_experiment_{timestamp}.json"
+    ) -> Dict[str, Any]:
+        """Calculates harmonic weights and derives decile boundaries via vectorized execution."""
+        logger.info(f"Initializing out-of-core DuckDB scorecard engine...")
+
+        # Memory Optimization: File-backed engine for 100M+ rows
+        if os.path.exists(f"score_experiment_{timestamp}.db"):
+            os.remove(f"score_experiment_{timestamp}.db")
+        ctx = duckdb.connect(f"score_experiment_{timestamp}.db")
+        ctx.execute("CREATE OR REPLACE TABLE df AS SELECT * FROM data")
+
+        # Step 1: Baseline metrics + Vectorized multi-segment aggregation (O(1) Scan)
+        agg_expressions = [
+            f'COUNT(CASE WHEN "{col}" = 1 THEN 1 END) AS "{col}_cnt", '
+            f'SUM(CASE WHEN "{col}" = 1 THEN "{self.target_col}" ELSE 0 END) AS "{col}_ev"'
+            for col in self.segment_cols
+        ]
+
+        master_sql = f"""
+            SELECT 
+                COUNT(*) AS total_pop, 
+                SUM(CAST("{self.target_col}" AS DOUBLE)) AS total_ev,
+                {', '.join(agg_expressions)}
+            FROM df
+        """
+
+        master_res = ctx.execute(master_sql).fetchone()
+        if not master_res:
+            raise RuntimeError("Database engine failed to return aggregations.")
+
+        total_population, total_events = master_res[0], master_res[1]
+
+        if total_population == 0 or total_events == 0:
+            raise ValueError(
+                "Invalid Dataset: Population and total events must be greater than zero."
+            )
+
+        baseline_rate = total_events / total_population
+        zero_inflation_rate = 1.0 - baseline_rate
+
+        # Step 2: Unpack vectorized SQL aggregations into weight lookup
+        logger.info("Computing harmonic scorecard weights...")
+        weights_lookup: Dict[str, Dict[str, Union[int, float]]] = {}
+
+        for idx, seg_col in enumerate(self.segment_cols):
+            seg_count = master_res[2 + (idx * 2)] or 0
+            seg_events = master_res[2 + (idx * 2) + 1] or 0
+
+            if seg_count == 0 or seg_events == 0:
+                logger.warning(f"Segment '{seg_col}' has zero volume or events. Setting weight=0.")
+                weights_lookup[seg_col] = {
+                    "weight": 0, "lift": 0.0, "response_rate": 0.0, "capture_rate": 0.0,
+                }
+                continue
+
+            response_rate = seg_events / seg_count
+            capture_rate = seg_events / total_events
+            lift = response_rate / baseline_rate
+
+            harmonic_mean = 2 * ((response_rate * capture_rate) / (response_rate + capture_rate))
+            raw_weight = lift * harmonic_mean * 100.0
+
+            weights_lookup[seg_col] = {
+                "weight": int(round(raw_weight)),
+                "lift": round(lift, 4),
+                "response_rate": round(response_rate, 4),
+                "capture_rate": round(capture_rate, 4),
+            }
+
+        # Step 3: Direct C++ SQL Matrix Math (Replaces NumPy Memory Bottleneck)
+        logger.info("Scoring 100M rows natively via C++ database engine...")
+        scored_cols = list(weights_lookup.keys())
+        
+        if not scored_cols:
+            raise ValueError("Scorecard Failure: No valid segments found to score.")
+
+        score_terms = [f'(CAST("{col}" AS DOUBLE) * {weights_lookup[col]["weight"]})' for col in scored_cols]
+        score_math_expr = " + ".join(score_terms)
+
+        ctx.execute(f"""
+            CREATE OR REPLACE TABLE scored_population AS 
+            SELECT "{self.primary_key}", ({score_math_expr}) AS total_score 
+            FROM df
+        """)
+
+        logger.info(f"Dataset Zero-Inflation Rate: {zero_inflation_rate:.2%}")
+
+        # Step 4: Out-of-Core Decile Boundary Profiling via SQL Quantiles
+        logger.info("Calibrating deciles across active populations...")
+
+        # Handle zero-inflation masking completely on the database side
+        filter_clause = "WHERE total_score > 0" if zero_inflation_rate >= 0.80 else ""
+
+        # Calculate all 10 deciles in a single table scan using highly optimized C++ quantiles
+        quantile_query = f"""
+            SELECT QUANTILE_DISC(total_score, [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) 
+            FROM scored_population
+            {filter_clause}
+        """
+        quantiles_res = ctx.execute(quantile_query).fetchone()
+        
+        if not quantiles_res or not quantiles_res[0]:
+            raise ValueError("Scorecard Failure: 0 customers triggered any segment rules.")
+            
+        quantiles = quantiles_res[0]
+        quantiles = quantiles[::-1] # Reverse list to match descending order paradigm
+        decile_thresholds = {str(i+1): int(quantiles[i]) for i in range(10)}
+
+        # Capture Active Population dynamically from the filtered subset
+        active_pop_size = ctx.execute(f"SELECT COUNT(*) FROM scored_population {filter_clause}").fetchone()[0]
+
+        # Step 5: JSON Generation
+        self.model_artifact = {
+            "model_metadata": {
+                "total_training_population": int(total_population),
+                "active_scored_population": int(active_pop_size),
+                "active_population_pct": round((active_pop_size / total_population) * 100.0, 2),
+                "baseline_event_rate": round(baseline_rate, 4),
+            },
+            "segment_weights": weights_lookup,
+            "decile_min_thresholds": decile_thresholds,
+        }
+
+        with open(export_path, "w", encoding="utf-8") as f:
+            json.dump(self.model_artifact, f, indent=4)
+
+        ctx.close()
+        return self.model_artifact

rapidsegment/utils/__init__.py

@@ -0,0 +1,3 @@
+from .data_loader import UniversalDataLoader
+
+__all__ = ["UniversalDataLoader"]

rapidsegment/utils/data_loader.py

@@ -0,0 +1,170 @@
+"""
+Unified Data Ingestion Layer for Strategic Analytics
+===================================================
+A multi-format data loader abstraction supporting Local Files (CSV, Parquet, Arrow, Excel),
+In-Memory PyArrow Tables, and Google Cloud BigQuery Storage API streams.
+"""
+
+import os
+import logging
+from typing import Any, Optional, Union, List
+import pyarrow as pa
+import pyarrow.csv as pa_csv
+import pyarrow.parquet as pa_pq
+import pyarrow.compute as pc  # Added for vectorized schema casting
+
+logger = logging.getLogger("StrategicEngine.DataLoader")
+
+
+class UniversalDataLoader:
+    """Handles multi-source data ingestion, normalizing inputs into highly optimized
+    in-memory structures compatible with vectorized down-stream compute engines.
+    """
+
+    def __init__(
+        self,
+        project_id: Optional[str] = None,
+        dataset_id: Optional[str] = None,
+        table_id: Optional[str] = None,
+        file_path: Optional[str] = None,
+    ) -> None:
+        self.project_id = project_id
+        self.dataset_id = dataset_id
+        self.table_id = table_id
+        self.file_path = file_path
+
+    def load(self, fallback_data: Optional[Any] = None) -> Union[pa.Table, str]:
+        """Auto-detects the source configuration parameters and loads the dataset.
+
+        Returns:
+            An ingestion asset (pa.Table or DuckDB macro string) ready for downstream consumption.
+        """
+        # Scenario 1: Direct In-Memory Object Check
+        if fallback_data is not None:
+            if isinstance(fallback_data, pa.Table):
+                logger.info("Ingesting directly provided in-memory PyArrow Table.")
+                return self._cast_table_numerics_to_float(fallback_data)
+            return fallback_data
+
+        # Scenario 2: BigQuery Parameters Detected
+        if self.dataset_id and self.table_id:
+            return self._load_from_bigquery()
+
+        # Scenario 3: Local File Path Detection
+        if self.file_path:
+            return self._load_from_file()
+
+        raise ValueError(
+            "Invalid Configuration: You must provide either a valid `file_path`, "
+            "BigQuery identifiers, or pass an explicit `fallback_data` object."
+        )
+
+    def _cast_table_numerics_to_float(self, table: pa.Table) -> pa.Table:
+        """Inspects a PyArrow Table schema and casts all numeric columns to float64."""
+        if not isinstance(table, pa.Table):
+            return table
+            
+        new_columns = []
+        new_fields = []
+        
+        for i, field in enumerate(table.schema):
+            # Evaluate if the type is integer, floating, or decimal
+            if (pa.types.is_integer(field.type) or 
+                pa.types.is_floating(field.type) or 
+                pa.types.is_decimal(field.type)):
+                
+                try:
+                    # Cast the underlying data array to float64
+                    casted_col = pc.cast(table.column(i), pa.float64(), safe=False)
+                    new_columns.append(casted_col)
+                    new_fields.append(pa.field(field.name, pa.float64(), nullable=field.nullable))
+                except Exception as e:
+                    logger.warning(f"Failed to cast column {field.name} to float64. Reason: {e}")
+                    new_columns.append(table.column(i))
+                    new_fields.append(field)
+            else:
+                new_columns.append(table.column(i))
+                new_fields.append(field)
+                
+        return pa.Table.from_arrays(new_columns, schema=pa.schema(new_fields))
+
+    def _load_from_file(self) -> pa.Table:
+        """Parses local files using high-performance C++ Arrow readers."""
+        if not os.path.exists(self.file_path):
+            raise FileNotFoundError(f"Data file not found at: {self.file_path}")
+
+        ext = os.path.splitext(self.file_path)[-1].lower()
+        table = None
+        
+        if ext == ".parquet":
+            table = pa_pq.read_table(self.file_path)
+        elif ext == ".csv":
+            table = pa_csv.read_csv(self.file_path)
+        elif ext in [".arrow", ".feather"]:
+            with pa.memory_map(self.file_path, "r") as source:
+                table = pa.ipc.open_file(source).read_all()
+        elif ext in [".xlsx", ".xls"]:
+            table = self._load_excel_to_arrow()
+        else:
+            raise ValueError(f"Unsupported format: '{ext}'.")
+            
+        # Apply standard numerical conversion to all successfully loaded local tables
+        return self._cast_table_numerics_to_float(table)
+
+    def _load_excel_to_arrow(self) -> pa.Table:
+        """Parses Excel using positional tracking to prevent column misalignment."""
+        logger.info("Parsing Excel spreadsheet via positional column tracking...")
+        try:
+            import openpyxl
+            wb = openpyxl.load_workbook(self.file_path, data_only=True, read_only=True)
+            sheet = wb.active
+            rows = sheet.iter_rows(values_only=True)
+            
+            headers = next(rows)
+            if not headers:
+                raise ValueError("The Excel file appears to be empty.")
+            
+            # Fix: Track column names by index to prevent header-shift corruption
+            column_names = [f"{h}" if h else f"_col_{i}" for i, h in enumerate(headers)]
+            data_columns = {name: [] for name in column_names}
+
+            for row in rows:
+                for i, name in enumerate(column_names):
+                    val = row[i] if i < len(row) else None
+                    data_columns[name].append(val)
+            
+            wb.close()
+            return pa.Table.from_pydict(data_columns)
+        except ImportError:
+            raise ImportError("Dependency missing: `pip install openpyxl` required.")
+
+    def _load_from_bigquery(self) -> Union[pa.Table, str]:
+        """Resolves BigQuery ingestion using cost-optimized metadata inspection."""
+        try:
+            from google.cloud import bigquery
+            bq_client = bigquery.Client(project=self.project_id)
+            full_table_ref = f"{self.project_id or bq_client.project}.{self.dataset_id}.{self.table_id}"
+            
+            # Optimization: Use get_table() instead of INFORMATION_SCHEMA to avoid slot costs
+            table = bq_client.get_table(full_table_ref)
+            
+            select_clauses = []
+            for field in table.schema:
+                # Type normalization: Extended to intercept all kinds of integers, decimals, and floats
+                if field.field_type in ("NUMERIC", "BIGNUMERIC", "DECIMAL", "INTEGER", "INT64", "FLOAT", "FLOAT64"):
+                    select_clauses.append(f"SAFE_CAST(`{field.name}` AS FLOAT64) AS `{field.name}`")
+                else:
+                    select_clauses.append(f"`{field.name}`")
+            
+            query = f"SELECT {', '.join(select_clauses)} FROM `{full_table_ref}`"
+            
+            # No need to run _cast_table_numerics_to_float here as BQ natively hands back FLOAT64 based on the query.
+            return bq_client.query(query).to_arrow()
+
+        except ImportError:
+            # Fallback for environments without the GCP library installed
+            if not self.dataset_id or not self.table_id:
+                raise ValueError("BigQuery coordinates (dataset_id, table_id) required for scan.")
+            
+            logger.warning("google-cloud-bigquery not found. Returning native DuckDB scan macro.")
+            return f"bigquery_scan('{self.project_id or 'default'}', '{self.dataset_id}', '{self.table_id}')"

rapidsegment/utils/on_gcp_feature_selection.py

@@ -0,0 +1,271 @@
+import logging
+from typing import List, Tuple, Optional
+import duckdb
+from google.cloud import bigquery
+
+# Configure enterprise-grade logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+class BigQueryFeatureSelector:
+    """
+    WARNING: This utility is designed for large-scale datasets and may incur costs on Google BigQuery.
+    A highly scalable feature selection utility designed for enterprise analytics pipelines.
+    Computes Information Value (IV) and standard deviation natively within Google BigQuery.
+    
+    Optimizations:
+    - Single-hit schema extraction to reduce API latency.
+    - Dynamic Binary Profiling: Automatically detects numeric 0/1 flags and treats them 
+      categorically.
+    - Batched SQL execution to safeguard BigQuery slot availability.
+    - DuckDB integration: Streams data via PyArrow to bypass Pandas completely.
+
+    Attributes:
+        project_id (str): The Google Cloud project ID.
+        dataset_id (str): The BigQuery dataset ID.
+        table_id (str): The BigQuery table ID.
+        target_column (str): The name of the binary target variable (must be 0 or 1).
+        iv_threshold (float): Minimum Information Value required to retain a feature.
+        stddev_threshold (float): Minimum standard deviation required to retain a feature.
+        bins (int): The number of quantiles (ntiles) to use for the naive IV calculation.
+        batch_size (int): Number of columns to process per BigQuery job to prevent timeout.
+        binary_columns (Optional[List[str]]): Explicit list of binary columns. If None, auto-detected.
+        client (google.cloud.bigquery.Client): The BigQuery client instance.
+    """
+
+    def __init__(
+        self,
+        project_id: str,
+        dataset_id: str,
+        table_id: str,
+        target_column: str,
+        iv_threshold: float = 0.02,
+        stddev_threshold: float = 1e-5,
+        min_bin_n_event: int = 1,        
+        min_bin_n_nonevent: int = 1,     
+        bins: int = 10,
+        batch_size: int = 15,
+        binary_columns: Optional[List[str]] = None,
+        bq_client: Optional[bigquery.Client] = None
+    ):
+        self.project_id = project_id
+        self.dataset_id = dataset_id
+        self.table_id = table_id
+        self.target_column = target_column
+        self.iv_threshold = iv_threshold
+        self.stddev_threshold = stddev_threshold
+        self.min_bin_n_event = min_bin_n_event
+        self.min_bin_n_nonevent = min_bin_n_nonevent
+        self.bins = bins
+        self.batch_size = batch_size
+        self.binary_columns = binary_columns
+        
+        self.client = bq_client if bq_client else bigquery.Client(project=self.project_id)
+        self.full_table_path = f"`{self.project_id}.{self.dataset_id}.{self.table_id}`"
+
+    def _detect_binary_numerical_columns(self, numerical_columns: List[str]) -> List[str]:
+        """Executes a single row query to count distinct values in numerical columns."""
+        if not numerical_columns:
+            return []
+
+        logger.info("Performing dynamic profiling on numerical columns to detect binary flags...")
+        
+        select_expressions = [
+            f"COUNT(DISTINCT {col}) AS `{col}`" for col in numerical_columns
+        ]
+        query = f"SELECT {', '.join(select_expressions)} FROM {self.full_table_path}"
+        
+        try:
+            job = self.client.query(query)
+            # Natively fetch the single resulting row iterator
+            row = next(job.result())
+            
+            # Extract column names where the distinct count is <= 2
+            detected_binary = [
+                col for col in numerical_columns if row[col] <= 2
+            ]
+            
+            if detected_binary:
+                logger.info(f"Dynamically detected {len(detected_binary)} binary numeric columns: {detected_binary}")
+            else:
+                logger.info("No implicit binary numeric columns discovered.")
+                
+            return detected_binary
+
+        except Exception as e:
+            logger.error(f"Failed to dynamically profile numeric columns: {e}. Proceeding without auto-detection.")
+            return []
+
+    def _get_table_schema(self) -> Tuple[List[str], List[str]]:
+        """Retrieves the table schema and categorizes columns."""
+        logger.info(f"Fetching metadata schema for table: {self.full_table_path}")
+        table_ref = self.client.get_table(f"{self.project_id}.{self.dataset_id}.{self.table_id}")
+        
+        num_types = {"INTEGER", "FLOAT", "NUMERIC", "BIGNUMERIC"}
+        cat_types = {"STRING", "BOOLEAN"}
+        
+        raw_numerical = []
+        categorical_columns = []
+        
+        for field in table_ref.schema:
+            if field.name == self.target_column:
+                continue
+                
+            if field.field_type in num_types:
+                raw_numerical.append(field.name)
+            elif field.field_type in cat_types:
+                categorical_columns.append(field.name)
+        
+        # Handle Binary Conversions
+        if self.binary_columns is not None:
+            logger.info("Applying user-defined binary column overrides.")
+            actual_binary = [col for col in self.binary_columns if col in raw_numerical]
+        else:
+            actual_binary = self._detect_binary_numerical_columns(raw_numerical)
+            
+        numerical_columns = [col for col in raw_numerical if col not in actual_binary]
+        categorical_columns.extend(actual_binary)
+                
+        logger.info(f"Final categorization: {len(numerical_columns)} continuous features, "
+                    f"{len(categorical_columns)} categorical/binary features.")
+        return numerical_columns, categorical_columns
+
+    def _build_batch_query(self, numerical_chunk: List[str], categorical_chunk: List[str]) -> str:
+        """Constructs the optimized SQL query for a specific batch of columns."""
+        sql_parts = [f"""
+        WITH global_stats AS (
+            SELECT 
+                COUNTIF({self.target_column} = 0) AS total_goods,
+                COUNTIF({self.target_column} = 1) AS total_bads
+            FROM {self.full_table_path}
+        )
+        """]
+        
+        union_queries = []
+
+        # Defensive constraint clause shared across numerical and categorical blocks to ensure minimum bin counts for IV calculation and prevent any target leakage into feature space. This is crucial for enterprise-grade feature selection where data integrity is paramount.
+        iv_calculation_template = f"""
+        SUM(
+        CASE
+            WHEN
+            goods_in_bin < {self.min_bin_n_nonevent}
+            OR bads_in_bin < {self.min_bin_n_event}
+            THEN 0
+            ELSE
+            (
+                (goods_in_bin / NULLIF(total_goods, 0))
+                - (bads_in_bin / NULLIF(total_bads, 0)))
+            * LN(
+                ((goods_in_bin / NULLIF(total_goods, 0)) + 0.0001)
+                / ((bads_in_bin / NULLIF(total_bads, 0)) + 0.0001))
+            END) AS naive_iv
+        """
+        
+        for col in numerical_chunk:
+            part = f"""
+            SELECT
+            '{col}' AS feature_name,
+            (SELECT STDDEV({col}) FROM {self.full_table_path}) AS feature_stddev,
+            {iv_calculation_template}
+            FROM
+            (
+                SELECT
+                bin,
+                COUNTIF({self.target_column} = 0) AS goods_in_bin,
+                COUNTIF({self.target_column} = 1) AS bads_in_bin
+                FROM
+                (
+                    SELECT
+                    NTILE({self.bins}) OVER (ORDER BY {col}) AS bin,
+                    {self.target_column}
+                    FROM {self.full_table_path}
+                    WHERE {col} IS NOT NULL
+                )
+                GROUP BY bin
+            )
+            CROSS JOIN global_stats
+            """
+            union_queries.append(part)
+
+        for col in categorical_chunk:
+            part = f"""
+            SELECT
+            '{col}' AS feature_name,
+            9999.0 AS feature_stddev,
+            {iv_calculation_template}
+            FROM
+            (
+                SELECT
+                CAST({col} AS STRING) AS bin,
+                COUNTIF({self.target_column} = 0) AS goods_in_bin,
+                COUNTIF({self.target_column} = 1) AS bads_in_bin
+                FROM {self.full_table_path}
+                WHERE {col} IS NOT NULL
+                GROUP BY 1
+            )
+            CROSS JOIN global_stats
+
+            """
+            union_queries.append(part)
+
+        sql_parts.append("\nUNION ALL\n".join(union_queries))
+        return "\n".join(sql_parts)
+
+    def screen_features(self) -> duckdb.DuckDBPyRelation:
+        """Executes the pipeline and filters/sorts results via local DuckDB."""
+        numerical_columns, categorical_columns = self._get_table_schema()
+        
+        con = duckdb.connect()
+        
+        if not numerical_columns and not categorical_columns:
+            logger.warning("No valid numerical or categorical columns found to screen.")
+            return con.sql("SELECT NULL AS feature_name, NULL AS feature_stddev, NULL AS naive_iv WHERE 1=0")
+
+        con.execute("""
+            CREATE TABLE all_results (
+                feature_name VARCHAR, 
+                feature_stddev DOUBLE, 
+                naive_iv DOUBLE
+            )
+        """)
+
+        # Process numerical columns
+        for i in range(0, len(numerical_columns), self.batch_size):
+            chunk = numerical_columns[i:i + self.batch_size]
+            logger.info(f"Processing numerical batch {i // self.batch_size + 1}...")
+            query = self._build_batch_query(numerical_chunk=chunk, categorical_chunk=[])
+            
+            arrow_table = self.client.query(query).to_arrow()
+            con.execute("INSERT INTO all_results SELECT * FROM arrow_table")
+
+        # Process categorical/binary columns
+        for i in range(0, len(categorical_columns), self.batch_size):
+            chunk = categorical_columns[i:i + self.batch_size]
+            logger.info(f"Processing categorical/binary batch {i // self.batch_size + 1}...")
+            query = self._build_batch_query(numerical_chunk=[], categorical_chunk=chunk)
+            
+            arrow_table = self.client.query(query).to_arrow()
+            con.execute("INSERT INTO all_results SELECT * FROM arrow_table")
+
+        # Apply thresholds and sort inside DuckDB
+        retained_features_rel = con.sql(f"""
+            SELECT 
+                feature_name, 
+                feature_stddev, 
+                naive_iv
+            FROM all_results
+            WHERE feature_stddev > {self.stddev_threshold}
+              AND naive_iv >= {self.iv_threshold}
+            ORDER BY naive_iv DESC
+        """)
+
+        retained_count = con.sql("SELECT COUNT(*) FROM retained_features_rel").fetchone()[0]
+        total_screened = len(numerical_columns) + len(categorical_columns)
+        dropped_count = total_screened - retained_count
+        
+        logger.info(f"Screening complete. Retained: {retained_count} | Dropped: {dropped_count}")
+        
+        return retained_features_rel

rapidsegment/utils/undersampler.sql

@@ -0,0 +1,14 @@
+-- sample SQL script to downsample a dataset in BigQuery
+
+CREATE OR REPLACE TABLE `your_dataset.downsampled_training_data` AS
+SELECT * 
+FROM `your_project.your_dataset.your_table`
+WHERE target_y = 1  -- Keep 100% of minority class
+
+UNION ALL
+
+SELECT * 
+FROM `your_project.your_dataset.your_table`
+WHERE target_y = 0
+  -- Generates a deterministic pseudo-random float between 0 and 1 per row
+  AND ABS(MOD(FARM_FINGERPRINT(CAST(row_id AS STRING)), 100)) < 10; -- Keeps exactly 10% of Class 0