rapidsegment 1.2.1__tar.gz → 1.2.2__tar.gz

{rapidsegment-1.2.1 → rapidsegment-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rapidsegment
-Version: 1.2.1
+Version: 1.2.2
 Summary: A fast, multi-segment population scoring and combinatorial heuristic rule extraction engine.
 Author: Bishwarup Biswas
 Author-email: Bishwarup Biswas <bishwarup1429@gmail.com>

{rapidsegment-1.2.1 → rapidsegment-1.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "rapidsegment"
-version = "1.2.1"
+version = "1.2.2"
 description = "A fast, multi-segment population scoring and combinatorial heuristic rule extraction engine."
 readme = "README.md"
 license = "MIT"

{rapidsegment-1.2.1 → rapidsegment-1.2.2}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "rapidsegment"
-version = "1.2.1"
+version = "1.2.2"
 description = "A fast, multi-segment population scoring and combinatorial heuristic rule extraction engine."
 readme = "README.md"
 authors = [

{rapidsegment-1.2.1 → rapidsegment-1.2.2}/src/rapidsegment/builder.py

@@ -95,8 +95,7 @@ class StrategicSegmentBuilder:
         feature_groups: Optional[Dict[str, List[str]]] = None,
         ignore_features: Optional[List[str]] = None,
         sort_priority: str = "rate_lift_count",
-        binning_method: str = "optimal",  
-        naive_bins: int = 5 ,
+        binning_method: str = "optimal_cart",  # "naive" | "optimal_cart" | "optimal_quantile" | "optimal" (alias of cart)        naive_bins: int = 5 ,
         max_expansion_hops: int = 0,
         selection_metric: str = "iv",
         expand_log_mode: str = "none",
@@ -121,8 +120,12 @@ class StrategicSegmentBuilder:
             feature_groups: Mapping of business categories to columns (e.g. {'risk': ['scr', 'bal']}).
             ignore_features: Explicit list of columns to drop prior to IV calculation.
             sort_priority: Ranking criteria for selecting champion segments.
-            binning_method: Which binning engine to use ('optimal' or 'naive').
-            naive_bins: Number of quantile bins used when binning_method is 'naive'.
+            binning_method: Binning engine:
+                - "naive": DuckDB equal-frequency quantiles (supports adjacent-bin expansion).
+                - "optimal_cart": OptBinning with CART prebinning (target-aware cuts).
+                - "optimal_quantile": OptBinning with quantile prebinning (more stable).
+                - "optimal": alias for "optimal_cart" (backward compatible).
+            naive_bins: Number of quantile bins when binning_method is "naive".
             selection_metric: Metric used to rank features for top_n_vars selection ("iv" or "response_rate").
             max_expansion_hops: Adjacent-bin merging hop distance limit (0 disables expansion).
             expand_log_mode: Controls verbosity of adjacent-bin expansion logging ("none", "summary", "champion", "full").
@@ -159,6 +162,16 @@ class StrategicSegmentBuilder:
         self.diagnostics_: List[Dict[str, Any]] = []
         self.stop_reason: Optional[str] = None
         self.binning_method = binning_method  
+        # Normalize aliases
+        _bm = (binning_method or "optimal_cart").lower().strip()
+        if _bm == "optimal":
+            _bm = "optimal_cart"
+        if _bm not in ("naive", "optimal_cart", "optimal_quantile"):
+            raise ValueError(
+                f"binning_method must be one of 'naive', 'optimal_cart', "
+                f"'optimal_quantile', or 'optimal' (alias of cart); got {_bm!r}"
+            )
+        self.binning_method = _bm
         self.naive_bins = naive_bins     
         self.max_expansion_hops = max(0, int(max_expansion_hops))
         self.selection_metric = selection_metric
@@ -230,35 +243,42 @@ class StrategicSegmentBuilder:
         """
         priority = self.sort_priority
         if priority == "lift_count_rate":
-            return (rule["lift"], rule["count"], rule["rate"])
+            key = (rule["lift"], rule["count"], rule["rate"])
         elif priority == "count_lift_rate":
-            return (rule["count"], rule["lift"], rule["rate"])
+            key = (rule["count"], rule["lift"], rule["rate"])
         elif priority == "rate_lift_count":
-            return (rule["rate"], rule["lift"], rule["count"])
+            key = (rule["rate"], rule["lift"], rule["count"])
         elif priority == "lift_rate_count":
-            return (rule["lift"], rule["rate"], rule["count"])
+            key = (rule["lift"], rule["rate"], rule["count"])
         elif priority == "count_rate_lift":
-            return (rule["count"], rule["rate"], rule["lift"])
+            key = (rule["count"], rule["rate"], rule["lift"])
         elif priority == "rate_count_lift":
-            return (rule["rate"], rule["count"], rule["lift"])
+            key = (rule["rate"], rule["count"], rule["lift"])
         elif priority == "events_lift_rate":
-            return (rule["events"], rule["lift"], rule["rate"])
+            key = (rule["events"], rule["lift"], rule["rate"])
         elif priority == "events_rate_lift":
-            return (rule["events"], rule["rate"], rule["lift"])
+            key = (rule["events"], rule["rate"], rule["lift"])
         elif priority == "lift_events_rate":
-            return (rule["lift"], rule["events"], rule["rate"])
+            key = (rule["lift"], rule["events"], rule["rate"])
         elif priority == "rate_events_lift":
-            return (rule["rate"], rule["events"], rule["lift"])
+            key = (rule["rate"], rule["events"], rule["lift"])
         elif priority == "events_count_rate":
-            return (rule["events"], rule["count"], rule["rate"])
+            key = (rule["events"], rule["count"], rule["rate"])
         elif priority == "events_rate_count":
-            return (rule["events"], rule["rate"], rule["count"])
+            key = (rule["events"], rule["rate"], rule["count"])
         elif priority == "count_events_rate":
-            return (rule["count"], rule["events"], rule["rate"])
+            key = (rule["count"], rule["events"], rule["rate"])
         elif priority == "rate_events_count":
-            return (rule["rate"], rule["events"], rule["count"])
+            key = (rule["rate"], rule["events"], rule["count"])
         else:
-            return (rule["lift"], rule["rate"], rule["count"])
+            key = (rule["lift"], rule["rate"], rule["count"])
+        # Deterministic tie-breaker: when every configured priority dimension
+        # is exactly equal between two candidates, the winner must not depend
+        # on incoming list order (which itself can vary run-to-run due to
+        # Python's per-process string hash randomization affecting set/dict
+        # iteration order, and unordered SQL GROUP BY results). Sorting on
+        # the rule string last makes ties resolve identically every run.
+        return key + (rule.get("rule", ""),)
 
     def compute_iv_ranking_and_bin(
         self,
@@ -380,14 +400,25 @@ class StrategicSegmentBuilder:
                     # Transform bins array in DuckDB
                     transformed_bins = thread_con.execute(
                         f"""
-                        SELECT {case_expr} AS bin_label FROM current_df
+                        SELECT
+                            "__rs_row_id",
+                            {case_expr} AS bin_label
+                        FROM current_df
+                        ORDER BY "__rs_row_id"
                         """
                     ).fetchnumpy()["bin_label"].astype(str)
 
                 else:
                     # Optimal binning fallback
                     data_dict = thread_con.execute(
-                        f'SELECT "{col}", "{self.target}" FROM current_df'
+                        f'''
+                        SELECT
+                            "__rs_row_id",
+                            "{col}",
+                            "{self.target}"
+                        FROM current_df
+                        ORDER BY "__rs_row_id"
+                        '''
                     ).fetchnumpy()
 
                     col_arr_raw = data_dict[col]
@@ -396,10 +427,48 @@ class StrategicSegmentBuilder:
                     col_arr = col_arr_raw.filled(np.nan if np.issubdtype(col_arr_raw.dtype, np.number) else None) if isinstance(col_arr_raw, np.ma.MaskedArray) else col_arr_raw
                     target_arr = target_arr_raw.filled(0) if isinstance(target_arr_raw, np.ma.MaskedArray) else target_arr_raw
 
+                    # Deterministic row order before CART prebinning.
+                    # DuckDB SELECT has no ORDER BY; OptBinning's default prebinning
+                    # (CART) can split differently when tied samples arrive in a
+                    # different order, which changes bins/IV and cascades into
+                    # different segments across runs.
+                    #
+                    # IMPORTANT: only the *fitting* input is sorted. The transform
+                    # below must run on the ORIGINAL (unsorted) arrays so the
+                    # returned bin labels stay aligned with "__rs_row_id" / the
+                    # row order of current_df. Otherwise binned_df pairs the wrong
+                    # target values with the wrong bin labels, corrupting both the
+                    # event counts of 1-way rules and the row alignment of 2/3-way
+                    # rules (which is what produced spurious candidates whose SQL
+                    # re-validation then failed against the real table).
+                    try:
+                        col_arr = np.asarray(col_arr)
+                        target_arr = np.asarray(target_arr)
+                        if np.issubdtype(col_arr.dtype, np.number):
+                            order = np.lexsort((
+                                target_arr,
+                                np.nan_to_num(col_arr.astype(float, copy=False), nan=np.inf),
+                            ))
+                        else:
+                            col_as_str = col_arr.astype(str)
+                            order = np.lexsort((target_arr, col_as_str))
+                        col_fit = col_arr[order]
+                        target_fit = target_arr[order]
+                    except Exception:
+                        col_fit = col_arr
+                        target_fit = target_arr
+
                     # Fit an Optimal Binning model to the current feature/target pair so we can
                     # create monotonic, high-signal bins without relying on a fixed quantile grid.
-                    optb = OptimalBinning(name=col, dtype=dtype)
-                    optb.fit(col_arr, target_arr)
+                    prebinning_method = (
+                        "quantile" if self.binning_method == "optimal_quantile" else "cart"
+                    )
+                    optb = OptimalBinning(
+                        name=col,
+                        dtype=dtype,
+                        prebinning_method=prebinning_method,
+                    )
+                    optb.fit(col_fit, target_fit)
 
                     bin_table = optb.binning_table.build()
                     iv_val = float(bin_table["IV"].values[-1])
@@ -469,11 +538,12 @@ class StrategicSegmentBuilder:
             ranking.append({"variable": col, "iv": iv, "max_rr": rr})
             if bins is not None:
                 precomputed_bins[col] = bins
-
+        # Stable sort: primary metric descending, variable name ascending on ties
+        # so top_n_vars is identical across runs when scores collide.
         if self.selection_metric == "response_rate":
-            ranking.sort(key=lambda x: x["max_rr"], reverse=True)
+            ranking.sort(key=lambda x: (-x["max_rr"], x["variable"]))
         else:
-            ranking.sort(key=lambda x: x["iv"], reverse=True)
+            ranking.sort(key=lambda x: (-x["iv"], x["variable"]))
 
         return ranking, precomputed_bins
 
@@ -834,6 +904,11 @@ class StrategicSegmentBuilder:
         parts = [p.strip() for p in rule_str.split("&")]
         sql_conditions: List[str] = []
 
+        def _quote_sql_ident(ident: str) -> str:
+            # Quote every emitted column identifier so DuckDB reserved words
+            # (e.g. "default") and exotic column names do not break the predicate.
+            return '"' + str(ident).replace('"', '""') + '"'
+
         def _quote_sql_string(val: Any) -> str:
             txt = str(val)
             # escape single quotes for SQL and wrap in single quotes
@@ -908,10 +983,10 @@ class StrategicSegmentBuilder:
                     range_conds = []
                     if overall_lower.lower() != "-inf":
                         op = ">=" if overall_lower_char == "[" else ">"
-                        range_conds.append(f"{col} {op} {overall_lower}")
+                        range_conds.append(f"{_quote_sql_ident(col)} {op} {overall_lower}")
                     if overall_upper.lower() != "inf":
                         op = "<=" if overall_upper_char == "]" else "<"
-                        range_conds.append(f"{col} {op} {overall_upper}")
+                        range_conds.append(f"{_quote_sql_ident(col)} {op} {overall_upper}")
         
                     if range_conds:
                         sql_conditions.append(" AND ".join(range_conds))
@@ -932,13 +1007,13 @@ class StrategicSegmentBuilder:
                 )
                 if is_single_nested:
                     cleaned = _strip_wrapping_quotes(interval[1:-1])
-                    sql_conditions.append(f"{col} = {_quote_sql_string(cleaned)}")
+                    sql_conditions.append(f"{_quote_sql_ident(col)} = {_quote_sql_string(cleaned)}")
                     continue
                 if cat_tokens:
                     cleaned_items = [_strip_wrapping_quotes(t) for t in cat_tokens]
                     formatted_items = ", ".join(_quote_sql_string(item) for item in cleaned_items if item)
                     if formatted_items:
-                        sql_conditions.append(f"{col} IN ({formatted_items})")
+                        sql_conditions.append(f"{_quote_sql_ident(col)} IN ({formatted_items})")
                     continue
         
             # 2. Explicit Categoricals
@@ -988,12 +1063,12 @@ class StrategicSegmentBuilder:
                     ]
                 )
                 if formatted_items:
-                    sql_conditions.append(f"{col} IN ({formatted_items})")
+                    sql_conditions.append(f"{_quote_sql_ident(col)} IN ({formatted_items})")
                 continue
-        
+
             # 3. Special or Missing values
             if interval in ["Special", "Missing"]:
-                sql_conditions.append(f"{col} IS NULL")
+                sql_conditions.append(f"{_quote_sql_ident(col)} IS NULL")
                 continue
         
             # 4. Standard Intervals [lo, hi) or Single Bracket Values [val]
@@ -1005,7 +1080,7 @@ class StrategicSegmentBuilder:
                         raw_items = [_strip_wrapping_quotes(x) for x in inner.split(",") if x.strip()]
                         formatted_items = ", ".join(_quote_sql_string(item) for item in raw_items)
                         if formatted_items:
-                            sql_conditions.append(f"{col} IN ({formatted_items})")
+                            sql_conditions.append(f"{_quote_sql_ident(col)} IN ({formatted_items})")
                         continue
                     left_char, right_char = interval[0], interval[-1]
                     lower_str, upper_str = [x.strip() for x in inner.split(",", 1)]
@@ -1013,10 +1088,10 @@ class StrategicSegmentBuilder:
                     range_conds = []
                     if lower_str.lower() != "-inf":
                         op = ">=" if left_char == "[" else ">"
-                        range_conds.append(f"{col} {op} {lower_str}")
+                        range_conds.append(f"{_quote_sql_ident(col)} {op} {lower_str}")
                     if upper_str.lower() != "inf":
                         op = "<=" if right_char == "]" else "<"
-                        range_conds.append(f"{col} {op} {upper_str}")
+                        range_conds.append(f"{_quote_sql_ident(col)} {op} {upper_str}")
         
                     if range_conds:
                         sql_conditions.append(" AND ".join(range_conds))
@@ -1025,11 +1100,11 @@ class StrategicSegmentBuilder:
                     clean_val = _strip_wrapping_quotes(inner)
         
                     if col_is_categorical:
-                        sql_conditions.append(f"{col} = {_quote_sql_string(clean_val)}")
+                        sql_conditions.append(f"{_quote_sql_ident(col)} = {_quote_sql_string(clean_val)}")
                     elif clean_val.replace(".", "", 1).replace("-", "", 1).isdigit():
-                        sql_conditions.append(f"{col} = {clean_val}")
+                        sql_conditions.append(f"{_quote_sql_ident(col)} = {clean_val}")
                     else:
-                        sql_conditions.append(f"{col} = {_quote_sql_string(clean_val)}")
+                        sql_conditions.append(f"{_quote_sql_ident(col)} = {_quote_sql_string(clean_val)}")
         
         return " AND ".join(f"({cond})" for cond in sql_conditions)
 
@@ -1069,14 +1144,17 @@ class StrategicSegmentBuilder:
         con.execute(f"SET memory_limit = '{target_memory_gb}GB';")
         if db_temp_dir:
             con.execute(f"PRAGMA temp_directory='{db_temp_dir}';")
-        con.execute("SET preserve_insertion_order = false;")
+        # con.execute("SET preserve_insertion_order = false;")
         
         logger.info(
             f"⚙️ DuckDB Configured for Disk Spilling: Threads={target_threads}/{total_cores}, "
             f"MemoryLimit={target_memory_gb}GB, TempDir={self.db_temp_dir}"
         )
         logger.info(f"📊 Sort priority: {self.sort_priority}")
-        logger.info(f"📦 Binning method: {self.binning_method} (naive_bins={self.naive_bins})")
+        logger.info(
+            f"📦 Binning method: {self.binning_method}"
+            + (f" (naive_bins={self.naive_bins})" if self.binning_method == "naive" else "")
+        )
         try:
             con.register("input_data_view", data)
             # Cast the target to DOUBLE up front so boolean targets work with AVG,
@@ -1089,9 +1167,15 @@ class StrategicSegmentBuilder:
                     f"Available columns: {sorted(probe_cols)}"
                 )
             con.execute(
-                f'CREATE OR REPLACE TABLE current_df AS '
-                f'SELECT * REPLACE (CAST("{self.target}" AS DOUBLE) AS "{self.target}") '
-                f'FROM input_data_view'
+                f'''
+                CREATE OR REPLACE TABLE current_df AS
+                SELECT
+                    ROW_NUMBER() OVER () AS "__rs_row_id",
+                    * REPLACE (
+                        CAST("{self.target}" AS DOUBLE) AS "{self.target}"
+                    )
+                FROM input_data_view
+                '''
             )
 
             cols_info = con.execute("DESCRIBE current_df").fetchall()
@@ -1107,9 +1191,15 @@ class StrategicSegmentBuilder:
             if self.enable_diversity:
                 self._validate_feature_groups(all_cols)
 
-            eligible_cols = [
-                c for c in all_cols if c != self.target and c not in self.ignore_features
-            ]
+            eligible_cols = sorted(
+                c
+                for c in all_cols
+                if (
+                    c != self.target
+                    and c != "__rs_row_id"
+                    and c not in self.ignore_features
+                )
+)
             self.feature_usage_counts = {col: 0 for col in eligible_cols}
 
             if self.param_grid:
@@ -1211,7 +1301,11 @@ class StrategicSegmentBuilder:
                     break
 
                 raw_target_arr = con.execute(
-                    f'SELECT "{self.target}" FROM current_df'
+                    f'''
+                    SELECT "{self.target}"
+                    FROM current_df
+                    ORDER BY "__rs_row_id"
+                    '''
                 ).fetchnumpy()[self.target]
 
                 if isinstance(raw_target_arr, np.ma.MaskedArray):
@@ -1231,14 +1325,16 @@ class StrategicSegmentBuilder:
                     break
 
                 con.execute("DROP TABLE IF EXISTS binned_df")
-                # Materialize binned_data via a pandas DataFrame and register it with DuckDB
-                try:
-                    binned_df_pd = pd.DataFrame(binned_data)
-                    con.register("binned_data_df", binned_df_pd)
-                    con.execute("CREATE TABLE binned_df AS SELECT * FROM binned_data_df")
-                except Exception:
-                    # Fallback to previous direct table creation if DataFrame registration fails
-                    con.execute("CREATE TABLE binned_df AS SELECT * FROM binned_data")
+
+                con.register("binned_data_view", binned_data)
+
+                con.execute("""
+                    CREATE TABLE binned_df AS
+                    SELECT *
+                    FROM binned_data_view
+                """)
+
+                con.unregister("binned_data_view")
 
                 global_min_sample = (
                     min(exp["min_sample_size"] for exp in experiments)
@@ -1269,7 +1365,7 @@ class StrategicSegmentBuilder:
                 # Level 2 (Pairs)
                 valid_2way_sets = set()
                 if len(valid_1way_vars) >= 2 and (self.enable_2way or self.enable_3way):
-                    combos_2 = [c for c in combinations(valid_1way_vars, 2) if self.is_diverse(c)]
+                    combos_2 = [c for c in combinations(sorted(valid_1way_vars), 2) if self.is_diverse(c)]
                     if combos_2:
                         res_2 = self._agg_combinations(con, combos_2, original_base_rate)
                         if res_2:
@@ -1280,7 +1376,7 @@ class StrategicSegmentBuilder:
                 # Level 3 (Triplets)
                 if self.enable_3way and len(valid_1way_vars) >= 3 and valid_2way_sets:
                     combos_3 = [
-                        c for c in combinations(valid_1way_vars, 3)
+                        c for c in combinations(sorted(valid_1way_vars), 3)
                         if self.is_diverse(c) and all(frozenset(p) in valid_2way_sets for p in combinations(c, 2))
                     ]
                     if combos_3:
@@ -1305,6 +1401,10 @@ class StrategicSegmentBuilder:
                         grid_candidates.append(top_match)
 
                 if not grid_candidates:
+                    self.diagnostics_[-1]["candidate_funnel"] = {
+                        "total_candidates_before_grid": len(all_candidate_rules),
+                        "candidates_after_grid": 0,
+                    }
                     self.stop_reason = "No candidate rules met the minimum grid thresholds (sample size / lift)."
                     logger.info("⏹️ No candidates cleared the grid. Stopping.")
                     break
@@ -1864,4 +1964,4 @@ class StrategicSegmentBuilder:
                 })
     
         health_report_df = pd.DataFrame(rows)
-        return health_report_df
+        return health_report_df