SuperModelingFactory 0.2.0__py3-none-any.whl

Modeling_Tool/Core/__init__.py

@@ -0,0 +1,99 @@
+from .Binning_Tool import (
+    Binning,
+    chi2_binning,
+    run_binning,
+    super_binning,
+    get_max_nbins,
+    get_decision_tree_binning_edges,
+    NumVarBinning,
+    cre_pvt,
+    merge_bins,
+    observed_laplace,
+    cat_2_list,
+    get_bin_range,
+    get_bin_range_list,
+    chi2_auto_binning,
+    quick_binning,
+)
+
+def __getattr__(name):
+    if name == "ODPSRunner":
+        from .ODPS_Tool import ODPSRunner as _ODPSRunner
+        return _ODPSRunner
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+from .Slope_Tool import (
+    calculate_slope_sklearn,
+    calculate_slope_scipy,
+    calculate_slope_numpy,
+    calculate_slope_manual,
+    SlopeCalculator,
+)
+
+from .sample_weight_utils import (
+    resolve_sample_weight,
+    validate_sample_weight,
+    weighted_sum,
+    weighted_mean,
+    weighted_rate,
+)
+
+from .utils import (
+    cut2pieces,
+    proc_freq,
+    read_attr_list,
+    write_attr_list,
+    odds_score,
+    parse_odps_schema,
+    npnan2none,
+    drop_tmp_cols,
+    mkdir_if_not_exist,
+    parse_sql_file,
+    calc_woe,
+    calc_iv,
+    scoring,
+    get_missing_indicator,
+    upload_score,
+    DataFrameProcessor,
+    FilePathManager,
+    DateTimeUtils,
+    WOEIVCalculator,
+    get_feature_names,
+    get_feature_names_lgb,
+    get_feature_names_xgb,
+    get_feature_names_batch,
+    pull_attributes_in_batch,
+)
+from .Model_Registry_Tool import (
+    make_model_artifact,
+    save_model,
+    load_model,
+    load_model_metadata,
+)
+
+from .XOR_Encryptor import TextEncryptor
+
+from .kDataFrame import kSeries, kDataFrame
+
+__all__ = [
+    'Binning', 'chi2_binning', 'run_binning', 'super_binning',
+    'get_max_nbins', 'get_decision_tree_binning_edges', 'NumVarBinning',
+    'cre_pvt', 'merge_bins', 'observed_laplace', 'cat_2_list',
+    'get_bin_range', 'get_bin_range_list', 'chi2_auto_binning', 'quick_binning',
+    'ODPSRunner',
+    'calculate_slope_sklearn', 'calculate_slope_scipy',
+    'calculate_slope_numpy', 'calculate_slope_manual',
+    'SlopeCalculator',
+    'resolve_sample_weight', 'validate_sample_weight',
+    'weighted_sum', 'weighted_mean', 'weighted_rate',
+    'cut2pieces', 'proc_freq', 'read_attr_list', 'write_attr_list',
+    'odds_score', 'parse_odps_schema', 'npnan2none', 'drop_tmp_cols',
+    'mkdir_if_not_exist', 'parse_sql_file', 'calc_woe', 'calc_iv',
+    'save_model', 'load_model', 'load_model_metadata', 'make_model_artifact',
+    'scoring', 'get_missing_indicator', 'upload_score',
+    'get_feature_names', 'get_feature_names_lgb', 'get_feature_names_xgb',
+    'get_feature_names_batch', 'pull_attributes_in_batch',
+    'DataFrameProcessor', 'FilePathManager', 'DateTimeUtils', 'WOEIVCalculator',
+    'TextEncryptor',
+    'kSeries', 'kDataFrame',
+]

Modeling_Tool/Core/kDataFrame.py

@@ -0,0 +1,228 @@
+import logging
+
+import pandas as pd
+from pandas import DataFrame
+from pandas import Series
+import numpy as np
+
+# Available only in newer pandas versions. Older Airflow images should skip it.
+try:
+    pd.set_option('future.no_silent_downcasting', True)
+except (KeyError, ValueError):
+    pass
+pd.options.mode.chained_assignment = None  # default='warn'
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+
+class kSeries(Series):
+    @property
+    def _constructor(self):
+        return kSeries
+
+    @property
+    def _constructor_expanddim(self):
+        return kDataFrame
+    
+    def __init__(self, data, *args, **kwargs):
+        super().__init__(data, *args, **kwargs)
+        
+    def to_pdSeries(self, inplace: bool = False):
+        """
+        Convert kSeries back to pandas Series.
+        """
+        if inplace:
+            data = self
+        else:
+            data = self.copy()
+
+        df = Series(data)
+        return df
+    
+    def odds_score(self):
+        """
+        Calculate the odd score based on the given probability score.
+
+        Odds Score = (count of sth happening) / (count of sth not happening)
+        Odds Score = pb_score / (1-pb_score)  [This ranges from 0 to infinity.]
+        Log Odds Score = np.log(pb_score / (1-pb_score))  [This ranges from -infinity to +infinity]
+        This is helpful for solving binary classification problem.
+        Odds Ratio: The ratio of odds.
+        """
+        pb_score = self
+        a = (20 / np.log(2))
+        b = (np.log(15) + np.log(pb_score/(1 - pb_score)))
+        return (500 - a * b)
+    
+    def scale_score(self):
+        """
+        Scale the model scores (for internt segment of MCI model)
+        """
+        data = self.copy()
+        def app_func(x):
+            scores = x * 1.112
+            if (scores > 0.9999999):
+                return 0.9999999
+            return scores
+        return data.apply(app_func)
+    
+    def proc_freq(self) -> pd.DataFrame:
+        """
+        Implement the Python version "proc freq" query in SAS.
+        """
+        data = self.copy()
+        f = data.value_counts(dropna = False)
+        p = data.value_counts(dropna = False, normalize = True)
+        df = pd.concat([f,p], axis = 1, keys = ['frequency', 'percent'])
+        df['cumFrequency'] = df['frequency'].cumsum()
+        df['cumPercent'] = df['percent'].cumsum()
+        return df
+    
+    
+        
+class kDataFrame(DataFrame):
+    """
+    The extension class of Pandas DataFrame including more useful methods to manipulate dataset.
+    
+    Parameters:
+    -----------
+    data [DataFrame]: Any pandas dataframe dataset.
+    
+    Return:
+    -------
+    data [kDataFrame]: the extension object of dataframe.
+    
+    """
+    _metadata = ['added_property']
+    added_property = 1  # This will be passed to copies
+
+    @property
+    def _constructor(self):
+        return kDataFrame
+
+    @property
+    def _constructor_sliced(self):
+        return kSeries
+    
+    def __init__(self, data, *args, **kwargs):
+        super().__init__(data, *args, **kwargs)
+        
+    def move_column(self, colname: str, idx: int, inplace: bool = False):
+        """
+        To move a column into specific place by index.
+        """
+        if inplace:
+            data = self
+        else:
+            data = self.copy()
+        colarray = data.columns.tolist()
+        colarray.remove(colname)
+        colarray.insert(idx, colname)
+        data = data[colarray]
+        return data
+
+    def convert_to_vintage(self, vintage_colname: str = 'VINTAGE', by: str = 'TRAN_TMS', inplace: bool = False):
+        """
+        To obtain a vintage column by a time/data column.
+        """
+        if inplace:
+            data = self
+        else:
+            data = self.copy()
+        import re
+        data[vintage_colname] = data[by].apply(lambda x: re.search("\d{4}-\d{2}", x).group().replace('-', ''))
+        if inplace:
+            self = data
+        return data
+
+    def col_filter_regex(self, regex: str = ".*?of_co_at_12m", case_sensitive = True, inplace: bool = False):
+        """
+        To filter the DataFrame columns by regular expression.
+        """
+        if inplace:
+            data = self
+        else:
+            data = self.copy()
+
+        fltr = data.columns[data.columns.str.contains(regex, regex = True, case = case_sensitive)]
+        return data[fltr]
+    
+    def row_filter_regex(self, col = None, regex: str = None, case_sensitive = True,
+                         as_index = False, inplace: bool = False):
+        """
+        To filter the string format row using regex.
+        """
+        if inplace:
+            data = self
+        else:
+            data = self.copy()
+            
+        fltr = data[col].astype('str').str.contains(pat = regex, regex = True, case = case_sensitive)
+        if as_index:
+            return data[fltr].set_index(col)
+        return data[fltr]
+    
+    def scale_score(self, pb_score: str):
+        """
+        Scale the model scores (for internt segment of MCI model)
+        """
+        data = self.copy()
+        def app_func(x):
+            scores = x * 1.112
+            if (scores > 0.9999999):
+                return 0.9999999
+            return scores
+        return data[pb_score].apply(app_func)
+    
+    def proc_freq(self, var: str):
+        """
+        Implement the Python version "proc freq" query in SAS.
+        """
+        data = self.copy()
+        f = data[var].value_counts(dropna = False)
+        p = data[var].value_counts(dropna = False, normalize = True)
+        df = pd.concat([f,p], axis = 1, keys = ['frequency', 'percent'])
+        df['cumFrequency'] = df['frequency'].cumsum()
+        df['cumPercent'] = df['percent'].cumsum()
+        return df
+    
+    def unify_table_col_names(self, how: str = "lowercase", inplace: bool = False):
+        """
+        Unify the format of column names.
+        """
+        
+        if inplace:
+            data = self
+        else:
+            data = self.copy()
+        
+        cols = data.columns
+        if how.lower() == "lower" or how.lower() == "lowercase":
+            res = [name.lower() for name in cols]
+        if how.lower() == "upper" or how.lower() == "uppercase":
+            res = [name.upper() for name in cols]
+        if how.lower() == "cap" or how.lower() == "capitalize":
+            res = [name.capitalize() for name in cols]
+        data.columns = res
+        return data
+    
+    def convert_strlist_to_list(self, col: str):
+        """
+        cast string-type lists in a specified Series into real lists.
+        """
+        import re
+        
+        data = self.copy()
+        str_col = kSeries([re.findall("\w+", str(x)) for x in data[col]])
+        return str_col
+    
+    def to_pdDataFrame(self, inplace: bool = False):
+        """
+        Convert df_extension back to DataFrame.
+        """
+        if inplace:
+            data = self
+        else:
+            data = self.copy()
+            
+        df = DataFrame(data)
+        return df

Modeling_Tool/Core/kDataFrame.pyi

@@ -0,0 +1,43 @@
+# =============================================================================
+# Modeling_Tool.Core.kDataFrame
+# -----------------------------------------------------------------------------
+# Copyright (c) 2026 Kyle Sun <github.com/Kyle-J-Sun>. All rights reserved.
+# SuperModelingFactory — Licensed under the Business Source License 1.1.
+#
+# This stub describes the public API of a closed-source module compiled to a
+# native extension (.so / .pyd). The original source is not distributed.
+# Production / commercial use requires a separate commercial license.
+#
+# FINGERPRINT: SMF-KDATAFRAME-76c89511
+#   (Unique trace marker. Do not remove or alter — used for plagiarism
+#    detection across the public internet.)
+# =============================================================================
+
+import logging
+import pandas as pd
+from pandas import DataFrame
+from pandas import Series
+import numpy as np
+
+class kSeries(Series):
+    def _constructor(self): ...
+    def _constructor_expanddim(self): ...
+    def __init__(self, data, *args, **kwargs): ...
+    def to_pdSeries(self, inplace: bool = False): ...
+    def odds_score(self): ...
+    def scale_score(self): ...
+    def proc_freq(self) -> pd.DataFrame: ...
+
+class kDataFrame(DataFrame):
+    def _constructor(self): ...
+    def _constructor_sliced(self): ...
+    def __init__(self, data, *args, **kwargs): ...
+    def move_column(self, colname: str, idx: int, inplace: bool = False): ...
+    def convert_to_vintage(self, vintage_colname: str = 'VINTAGE', by: str = 'TRAN_TMS', inplace: bool = False): ...
+    def col_filter_regex(self, regex: str = '.*?of_co_at_12m', case_sensitive = True, inplace: bool = False): ...
+    def row_filter_regex(self, col = None, regex: str = None, case_sensitive = True, as_index = False, inplace: bool = False): ...
+    def scale_score(self, pb_score: str): ...
+    def proc_freq(self, var: str): ...
+    def unify_table_col_names(self, how: str = 'lowercase', inplace: bool = False): ...
+    def convert_strlist_to_list(self, col: str): ...
+    def to_pdDataFrame(self, inplace: bool = False): ...

Modeling_Tool/Core/sample_weight_utils.py

@@ -0,0 +1,77 @@
+# encoding: utf-8
+"""Sample-weight resolution and weighted aggregation helpers."""
+from __future__ import annotations
+
+import numpy as np
+
+
+def resolve_sample_weight(
+    data=None,
+    weight_col=None,
+    sample_weight=None,
+    expected_len=None,
+    wgt=None,
+    wgt_col=None,
+):
+    """Resolve sample weights from a DataFrame column or array.
+
+    Accepts ``weight_col`` / ``wgt_col`` and ``sample_weight`` / ``wgt`` aliases.
+    Returns ``None`` when no weight source is provided.
+    """
+    if weight_col is None:
+        weight_col = wgt_col
+    if sample_weight is None:
+        sample_weight = wgt
+
+    if sample_weight is not None and weight_col is not None:
+        raise ValueError("Provide either weight_col or sample_weight, not both.")
+
+    if weight_col is not None:
+        if data is None:
+            raise ValueError("data is required when weight_col is provided.")
+        if weight_col not in data.columns:
+            raise KeyError("weight column '{0}' not found in data".format(weight_col))
+        sample_weight = data[weight_col].values
+
+    if sample_weight is None:
+        return None
+
+    return validate_sample_weight(sample_weight, expected_len=expected_len)
+
+
+def validate_sample_weight(weights, expected_len=None):
+    """Validate and return a 1-D float numpy weight vector."""
+    w = np.asarray(weights, dtype=float)
+    if w.ndim != 1:
+        raise ValueError("sample_weight must be 1-dimensional.")
+    if expected_len is not None and len(w) != expected_len:
+        raise ValueError(
+            "sample_weight length {0} != expected {1}".format(len(w), expected_len)
+        )
+    if not np.all(np.isfinite(w)):
+        raise ValueError("sample_weight must be finite (no NaN/inf).")
+    if np.any(w < 0):
+        raise ValueError("sample_weight must be non-negative.")
+    return w
+
+
+def weighted_sum(values, weights):
+    """Weighted sum of values."""
+    v = np.asarray(values, dtype=float)
+    w = np.asarray(weights, dtype=float)
+    return float(np.sum(v * w))
+
+
+def weighted_mean(values, weights):
+    """Weighted mean; returns NaN when total weight is zero."""
+    w = np.asarray(weights, dtype=float)
+    total = float(np.sum(w))
+    if total == 0:
+        return np.nan
+    return weighted_sum(values, weights) / total
+
+
+def weighted_rate(mask, weights):
+    """Weighted rate of True / 1 values in mask."""
+    m = np.asarray(mask, dtype=float)
+    return weighted_mean(m, weights)