wizata-dsapi 2.0.0.dev23__tar.gz → 2.0.0.dev25__tar.gz

{wizata_dsapi-2.0.0.dev23/wizata_dsapi.egg-info → wizata_dsapi-2.0.0.dev25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wizata_dsapi
-Version: 2.0.0.dev23
+Version: 2.0.0.dev25
 Summary: Wizata Data Science Toolkit
 Author: Wizata S.A.
 Author-email: info@wizata.com

wizata_dsapi-2.0.0.dev25/wizata_dsapi/models/__init__.py

@@ -0,0 +1 @@
+from .common import linear_regression, logistic_regression, isolation_forest, gradiant_boost_classifier, setpoint_optimizer, SetpointOptimizer

wizata_dsapi-2.0.0.dev25/wizata_dsapi/models/common.py

@@ -0,0 +1,272 @@
+import wizata_dsapi
+
+import pandas
+import numpy
+
+import sklearn
+import sklearn.linear_model
+import sklearn.ensemble
+import sklearn.neighbors
+import sklearn.pipeline
+import sklearn.preprocessing
+
+
+def extract_target_feat(context: wizata_dsapi.Context, single: bool = True):
+    """
+    return a list of target_feat columns names if not single value or the single value target feat name
+    raise an error if configuration mismatch
+    """
+
+    if "target_feat" not in context.properties:
+        raise ValueError(f"training script requires a proper target_feat")
+
+    target_feat = context.properties["target_feat"]
+    if isinstance(target_feat, str):
+        if single:
+            return target_feat
+        else:
+            return [target_feat]
+    elif isinstance(target_feat, list):
+        if single:
+            if len(target_feat) == 1:
+                return target_feat[0]
+            else:
+                raise ValueError(f"expecting only one target_feat but found {len(target_feat)}")
+        else:
+            return [target_feat]
+    else:
+        raise TypeError(f'target_feat must be a str or a list of str but found {target_feat.__class__.__name__}')
+
+
+def linear_regression(context: wizata_dsapi.Context):
+    """Train a linear regression model on all features to predict a single target column."""
+    df = context.dataframe
+
+    model_config = context.get_model_config()
+    if not model_config.has_target_feat():
+        raise ValueError(f'linear_regression requires a target feat')
+    target_feat_name = context.properties["target_feat"]
+
+    x = df.drop(columns=[target_feat_name])
+    y = df[target_feat_name]
+
+    model = sklearn.linear_model.LinearRegression()
+    model.fit(x, y)
+
+    context.set_model(model, features=x.columns)
+
+
+def logistic_regression(context: wizata_dsapi.Context):
+    """Train a logistic regression classifier on all features to predict a binary target column."""
+    df = context.dataframe
+
+    model_config = context.get_model_config()
+    if not model_config.has_target_feat():
+        raise ValueError(f'logistic_regression requires a target feat')
+    target_feat_name = context.properties["target_feat"]
+
+    x = df.drop(columns=[target_feat_name])
+    y = df[target_feat_name]
+
+    model = sklearn.linear_model.LogisticRegression()
+    model.fit(x, y.astype(int))
+
+    context.set_model(model, features=x.columns)
+
+
+def isolation_forest(context: wizata_dsapi.Context):
+    """Train an Isolation Forest for unsupervised anomaly detection using a sensitivity level (1-5)."""
+
+    model_config = context.get_model_config()
+    if model_config.has_target_feat():
+        raise ValueError(f'isolation_forest does not requires a target feat')
+
+    try:
+        if context.properties['sensitivity'] is None:
+            raise KeyError("sensitivity is none")
+        sensitivity = int(context.properties['sensitivity'])
+        sensitivities = [0.05, 0.15, 0.25, 0.35, 0.4]
+        contamination = sensitivities[sensitivity - 1]
+    except Exception as e:
+        raise ValueError(f'cannot extract sensitivity integer from 0 to 4 due to {e}')
+
+    df = context.dataframe.copy()
+    model = sklearn.ensemble.IsolationForest(contamination=contamination)
+    df['isolation_forest_predict'] = model.fit_predict(df)
+    context.set_model(model, features=df.columns)
+    return df
+
+
+def gradiant_boost_classifier(context: wizata_dsapi.Context):
+    """Train a Gradient Boosting classifier on all features to predict a target column."""
+    df = context.dataframe
+
+    model_config = context.get_model_config()
+    if not model_config.has_target_feat():
+        raise ValueError(f'gradiant_boost_classifier requires a target feat')
+    target_feat_name = context.properties["target_feat"]
+
+    x = df.drop(columns=[target_feat_name])
+    y = df[target_feat_name]
+
+    model = sklearn.ensemble.GradientBoostingClassifier(random_state=0).fit(x, y)
+    context.set_model(model, features=df.columns)
+
+
+class SetpointOptimizer:
+    """
+    Wraps a KNN quality forecaster (StandardScaler + KNeighborsRegressor) with a grid-search
+    setpoint recommendation method.
+
+    At training time, the underlying pipeline learns quality = f(telemetry + setpoints) from
+    historical data. At inference, recommend(X) keeps each row's telemetry fixed and grid-searches
+    over stored setpoint bounds (5th-95th percentile of train data) to find the combination that
+    minimizes (or maximizes) predicted quality.
+
+    :ivar pipeline: fitted sklearn Pipeline (StandardScaler -> KNeighborsRegressor).
+    :ivar setpoint_cols: ordered list of setpoint column names auto-detected at train time.
+    :ivar feature_cols: ordered list of all feature columns used at training (telemetry + setpoints).
+    :ivar bounds: dict mapping each setpoint column name to a (low, high) tuple.
+    :ivar direction: 'minimize' or 'maximize' the target quality.
+    :ivar grid_size: number of points per setpoint axis in the grid search.
+    """
+
+    def __init__(self, pipeline, setpoint_cols, feature_cols, bounds, direction, grid_size):
+        self.pipeline = pipeline
+        self.setpoint_cols = list(setpoint_cols)
+        self.feature_cols = list(feature_cols)
+        self.bounds = dict(bounds)
+        self.direction = direction
+        self.grid_size = int(grid_size)
+
+    def predict(self, X):
+        """Return predicted quality for each row — dual-use for validation or regular predict-mode pipelines."""
+        return self.pipeline.predict(X)
+
+    def recommend(self, X):
+        """
+        For each row of X, return the grid-search best setpoint combination.
+        :param X: features matrix (DataFrame or ndarray) with columns matching feature_cols.
+        :return: ndarray of shape [n_rows, n_setpoints] in the order of self.setpoint_cols.
+        """
+        if not isinstance(X, pandas.DataFrame):
+            X = pandas.DataFrame(X, columns=self.feature_cols)
+
+        grids = [
+            numpy.linspace(self.bounds[sp][0], self.bounds[sp][1], self.grid_size)
+            for sp in self.setpoint_cols
+        ]
+        mesh = numpy.array(numpy.meshgrid(*grids)).reshape(len(self.setpoint_cols), -1).T
+
+        recs = numpy.zeros((len(X), len(self.setpoint_cols)))
+        for i, (_, row) in enumerate(X.iterrows()):
+            candidates = pandas.DataFrame(
+                numpy.tile(row.values, (len(mesh), 1)),
+                columns=self.feature_cols
+            )
+            for j, sp in enumerate(self.setpoint_cols):
+                candidates[sp] = mesh[:, j]
+            preds = self.pipeline.predict(candidates)
+            if self.direction == "maximize":
+                best_idx = int(numpy.argmax(preds))
+            else:
+                best_idx = int(numpy.argmin(preds))
+            recs[i] = mesh[best_idx]
+
+        return recs
+
+    def output_names(self, suffix: str = "_recommended"):
+        """Suggest MLModelConfig.output_columns_names matching the setpoint order (e.g. for UI prefill)."""
+        return [f"{sp}{suffix}" for sp in self.setpoint_cols]
+
+    def get_inference_contract(self):
+        """Self-describing inference contract — the platform reads this to override MLModelConfig at runtime.
+        Returns the function to invoke (.recommend) and the output column names that will be produced, in order."""
+        return {
+            "function": "recommend",
+            "output_columns_names": self.output_names(),
+        }
+
+
+def setpoint_optimizer(context: wizata_dsapi.Context):
+    """Train a KNN-based setpoint optimizer that learns quality = f(telemetry + setpoints) and at
+    inference recommends optimal setpoint values. Setpoint columns are auto-detected via
+    BusinessType.SET_POINTS on context.datapoints; bounds are the 5th-95th percentile of train data.
+
+    Required MLModelConfig:
+      - train_script = 'wizata.models.setpoint_optimizer'
+      - target_feat  = '<quality column name>'
+      - function     = 'recommend'  (or 'predict' for quality forecasting only)
+      - output_columns_names = ['<sp1>_recommended', '<sp2>_recommended', ...] in the order setpoints appear.
+        The trained model exposes `.output_names()` to suggest a matching default.
+
+    Properties:
+      - k: KNN neighbors (default 5)
+      - grid_size: points per setpoint axis (default 10 — total cost is grid_size^n_setpoints per row)
+      - direction: 'minimize' (default) or 'maximize' the target
+    """
+    df = context.dataframe
+
+    model_config = context.get_model_config()
+    if not model_config.has_target_feat():
+        raise ValueError(f'setpoint_optimizer requires a target_feat (the quality column to optimize)')
+
+    target = context.properties["target_feat"]
+    if isinstance(target, list):
+        if len(target) != 1:
+            raise ValueError(f'setpoint_optimizer requires exactly one target_feat column')
+        target = target[0]
+
+    if target not in df.columns:
+        raise ValueError(f"target_feat '{target}' not found in dataframe columns")
+
+    datapoints = context.datapoints or {}
+    setpoint_cols = [
+        col for col, dp in datapoints.items()
+        if col in df.columns
+        and col != target
+        and dp.business_type == wizata_dsapi.BusinessType.SET_POINTS
+    ]
+
+    if not setpoint_cols:
+        raise ValueError(
+            "no setpoint datapoints found in context (BusinessType.SET_POINTS) — "
+            "the optimizer needs at least one setpoint column to optimize"
+        )
+
+    k = int(context.properties.get("k", 5))
+    grid_size = int(context.properties.get("grid_size", 10))
+    direction = context.properties.get("direction", "minimize")
+    if direction not in ("minimize", "maximize"):
+        raise ValueError(f"direction must be 'minimize' or 'maximize', got '{direction}'")
+
+    x = df.drop(columns=[target])
+    y = df[target]
+
+    pipeline = sklearn.pipeline.Pipeline([
+        ("scaler", sklearn.preprocessing.StandardScaler()),
+        ("knn", sklearn.neighbors.KNeighborsRegressor(n_neighbors=k)),
+    ])
+    pipeline.fit(x, y)
+
+    bounds = {
+        sp: (float(x[sp].quantile(0.05)), float(x[sp].quantile(0.95)))
+        for sp in setpoint_cols
+    }
+
+    optimizer = SetpointOptimizer(
+        pipeline=pipeline,
+        setpoint_cols=setpoint_cols,
+        feature_cols=list(x.columns),
+        bounds=bounds,
+        direction=direction,
+        grid_size=grid_size,
+    )
+
+    context.set_model(optimizer, features=x.columns)
+
+
+
+
+
+

wizata_dsapi-2.0.0.dev25/wizata_dsapi/scripts/__init__.py

@@ -0,0 +1 @@
+from .common import filter_df, fillna, clustering, merge, normalize, interpolate, remove_outliers, resample, rolling, diff, lag, clip, steady_state_filter, pca, setpoint_deviation, formula, target_feat_to_binary

wizata_dsapi-2.0.0.dev25/wizata_dsapi/scripts/common.py

@@ -0,0 +1,372 @@
+import re
+
+import wizata_dsapi
+
+import pandas
+import numpy
+
+import sklearn
+import sklearn.cluster
+import sklearn.metrics
+import sklearn.ensemble
+import sklearn.preprocessing
+import sklearn.decomposition
+
+
+_FORMULA_SAFE_FUNCTIONS = {
+    "abs": numpy.abs,
+    "sqrt": numpy.sqrt,
+    "log": numpy.log,
+    "log10": numpy.log10,
+    "exp": numpy.exp,
+    "clip": numpy.clip,
+    "round": numpy.round,
+    "min": numpy.minimum,
+    "max": numpy.maximum,
+}
+
+
+def filter_df(context: wizata_dsapi.Context):
+    """Filter dataframe rows using pandas query strings from the 'filters' property list."""
+
+    if "filters" not in context.properties or not isinstance(context.properties['filters'], list):
+        raise ValueError(f'there is no list *filters* in properties - please set them on context or config')
+
+    df = context.dataframe.copy()
+
+    filters = context.properties['filters']
+    for filter_row in filters:
+        try:
+            df = df.query(filter_row)
+        except pandas.errors.ParserError as e:
+            raise ValueError(f"error parsing filter string '{filter_row}': {e}")
+
+    return df
+
+
+def clustering(context: wizata_dsapi.Context):
+    """K-means clustering with automatic cluster count selection via silhouette score."""
+    df = context.dataframe.copy()
+    scaler = sklearn.preprocessing.StandardScaler()
+    df_clustering_scaler = scaler.fit_transform(df)
+
+    range_n_clusters = list(range(2, min(10, df_clustering_scaler.shape[0])))
+    silhouette_avg = []
+    for num_clusters in range_n_clusters:
+        kmeans = sklearn.cluster.KMeans(n_clusters=num_clusters)
+        kmeans.fit(df_clustering_scaler)
+        cluster_labels = kmeans.labels_
+        unique, counts = numpy.unique(cluster_labels, return_counts=True)
+
+        if len(unique) >= 2:
+            silhouette_avg.append(sklearn.metrics.silhouette_score(df_clustering_scaler, cluster_labels))
+        else:
+            silhouette_avg.append(numpy.nan)
+
+    if numpy.isnan(silhouette_avg).all():
+        df['cluster_labels'] = 0
+    else:
+        best_nb_clusters = silhouette_avg.index(max(silhouette_avg)) + 2
+        kmeans = sklearn.cluster.KMeans(n_clusters=best_nb_clusters)
+        kmeans.fit(df_clustering_scaler)
+        cluster_labels = kmeans.labels_
+        df['cluster_labels'] = cluster_labels
+        df['cluster_labels'] = df['cluster_labels'].apply(lambda x: int(x + 1))
+
+    return df
+
+
+def merge(context: wizata_dsapi.Context):
+    """Merge multiple dataframes by index using outer join (configurable via 'how' property)."""
+    dataframes = context.current_dataframes()
+    if len(dataframes) <= 1:
+        raise ValueError(f'there is not enough dataframes to concat')
+
+    how = "outer"
+    if "how" in context.properties:
+        how = context.properties["how"]
+
+    df = None
+    for key in dataframes:
+        if df is None:
+            df = dataframes[key]
+        else:
+            df = df.merge(dataframes[key], how=how, left_index=True, right_index=True)
+    return df
+
+
+def fillna(context: wizata_dsapi.Context):
+    """Fill missing values per column using the 'fillna' property dict mapping column names to fill values."""
+    df = context.dataframe
+
+    if "fillna" not in context.properties:
+        raise KeyError(f'please set a property dict fillna')
+
+    for key in context.properties["fillna"]:
+        df[key] = df[key].fillna(value=context.properties["fillna"][key])
+
+    return df
+
+
+def normalize(context: wizata_dsapi.Context):
+    """Normalize all numeric columns using 'minmax' (default) or 'zscore' scaling (configurable via 'method' property)."""
+    df = context.dataframe.copy()
+
+    method = context.properties.get("method", "minmax")
+    columns = df.select_dtypes(include="number").columns.tolist()
+
+    if not columns:
+        raise ValueError(f'no numeric columns to normalize')
+
+    if method == "minmax":
+        scaler = sklearn.preprocessing.MinMaxScaler()
+    elif method == "zscore":
+        scaler = sklearn.preprocessing.StandardScaler()
+    else:
+        raise ValueError(f"unknown normalize method '{method}', use 'minmax' or 'zscore'")
+
+    df[columns] = scaler.fit_transform(df[columns])
+    return df
+
+
+def interpolate(context: wizata_dsapi.Context):
+    """Interpolate missing values in numeric columns. The 'method' property selects the pandas interpolation method: 'time' (default, respects timestamp spacing), 'linear', 'nearest', 'pad', 'polynomial', or 'spline'."""
+    df = context.dataframe.copy()
+
+    method = context.properties.get("method", "time")
+
+    return df.interpolate(method=method)
+
+
+def remove_outliers(context: wizata_dsapi.Context):
+    """Drop rows containing outliers in any numeric column. The 'method' property selects 'iqr' (default, Tukey 1.5*IQR rule) or 'zscore' (drops rows beyond ±3 sigma)."""
+    df = context.dataframe.copy()
+
+    method = context.properties.get("method", "iqr")
+    columns = df.select_dtypes(include="number").columns.tolist()
+
+    if not columns:
+        return df
+
+    if method == "iqr":
+        q1 = df[columns].quantile(0.25)
+        q3 = df[columns].quantile(0.75)
+        iqr = q3 - q1
+        lower = q1 - 1.5 * iqr
+        upper = q3 + 1.5 * iqr
+        mask = ((df[columns] >= lower) & (df[columns] <= upper)).all(axis=1)
+    elif method == "zscore":
+        mean = df[columns].mean()
+        std = df[columns].std()
+        z = (df[columns] - mean).abs().divide(std)
+        mask = (z.fillna(0) <= 3).all(axis=1)
+    else:
+        raise ValueError(f"unknown remove_outliers method '{method}', use 'iqr' or 'zscore'")
+
+    return df[mask]
+
+
+def resample(context: wizata_dsapi.Context):
+    """Resample the dataframe to a new time frequency. Property 'freq' is required (pandas offset alias, e.g. '1min', '5min', '1H', '1D'); 'agg' selects the aggregation ('mean' default, 'sum', 'min', 'max', 'first', 'last', 'median')."""
+    df = context.dataframe
+
+    if "freq" not in context.properties:
+        raise KeyError(f"please set a 'freq' property (pandas offset alias, e.g. '1min', '1H')")
+
+    freq = context.properties["freq"]
+    agg = context.properties.get("agg", "mean")
+
+    return df.resample(freq).agg(agg)
+
+
+def rolling(context: wizata_dsapi.Context):
+    """Apply a rolling-window aggregation over all numeric columns. Property 'window' is required (integer number of rows); 'agg' selects the aggregation ('mean' default, 'sum', 'std', 'min', 'max', 'median')."""
+    df = context.dataframe
+
+    if "window" not in context.properties:
+        raise KeyError(f"please set a 'window' property (integer number of rows)")
+
+    window = context.properties["window"]
+    agg = context.properties.get("agg", "mean")
+
+    return df.rolling(window=window).agg(agg)
+
+
+def diff(context: wizata_dsapi.Context):
+    """Compute discrete differences (rate of change) across all numeric columns. Property 'periods' (default 1) is the number of rows to shift before subtracting — use 1 for first derivative, higher for longer horizons."""
+    df = context.dataframe
+
+    periods = context.properties.get("periods", 1)
+
+    return df.diff(periods=periods)
+
+
+def lag(context: wizata_dsapi.Context):
+    """Add lagged versions of all numeric columns as new '<col>_lag<N>' columns, preserving originals. Property 'periods' is required (integer number of rows to shift back)."""
+    df = context.dataframe
+
+    if "periods" not in context.properties:
+        raise KeyError(f"please set a 'periods' property (integer number of rows to lag)")
+
+    periods = context.properties["periods"]
+    columns = df.select_dtypes(include="number").columns.tolist()
+
+    for col in columns:
+        df[f"{col}_lag{periods}"] = df[col].shift(periods)
+
+    return df
+
+
+def clip(context: wizata_dsapi.Context):
+    """Clip all numeric columns to a physical range. Properties 'min' and/or 'max' (floats) bound the values; at least one of the two must be provided."""
+    df = context.dataframe
+
+    lower = context.properties.get("min")
+    upper = context.properties.get("max")
+
+    if lower is None and upper is None:
+        raise KeyError(f"please set at least one of 'min' or 'max' properties")
+
+    columns = df.select_dtypes(include="number").columns.tolist()
+    df[columns] = df[columns].clip(lower=lower, upper=upper)
+
+    return df
+
+
+def steady_state_filter(context: wizata_dsapi.Context):
+    """Keep only rows where all numeric columns are in steady state (rolling std over 'window' rows stays below 'tolerance'). Drops transients and start-up periods — a standard preprocessing step before process modeling."""
+    df = context.dataframe
+
+    if "window" not in context.properties:
+        raise KeyError(f"please set a 'window' property (integer number of rows)")
+    if "tolerance" not in context.properties:
+        raise KeyError(f"please set a 'tolerance' property (max allowed rolling std)")
+
+    window = context.properties["window"]
+    tolerance = context.properties["tolerance"]
+
+    columns = df.select_dtypes(include="number").columns.tolist()
+    if not columns:
+        return df
+
+    rolling_std = df[columns].rolling(window=window).std()
+    mask = (rolling_std <= tolerance).all(axis=1)
+
+    return df[mask]
+
+
+def pca(context: wizata_dsapi.Context):
+    """Reduce numeric columns to principal components, replacing them with 'PC1', 'PC2', ... Property 'n_components' is required — an integer (number of components) or a float in (0, 1] (minimum explained variance ratio). NaN values must be handled upstream (e.g. with interpolate or fillna)."""
+    df = context.dataframe
+
+    if "n_components" not in context.properties:
+        raise KeyError(f"please set an 'n_components' property (int or float in (0,1])")
+
+    n_components = context.properties["n_components"]
+    columns = df.select_dtypes(include="number").columns.tolist()
+
+    if not columns:
+        raise ValueError(f"no numeric columns to reduce")
+
+    if df[columns].isna().any().any():
+        raise ValueError(f"PCA cannot handle NaN values — run interpolate or fillna upstream")
+
+    model = sklearn.decomposition.PCA(n_components=n_components)
+    transformed = model.fit_transform(df[columns])
+
+    pc_cols = [f"PC{i + 1}" for i in range(transformed.shape[1])]
+    return pandas.DataFrame(transformed, index=df.index, columns=pc_cols)
+
+
+def setpoint_deviation(context: wizata_dsapi.Context):
+    """Add '<measurement>_deviation' columns computed as (measurement - setpoint) for each setpoint datapoint in the dataframe. Setpoints are auto-detected via BusinessType.SET_POINTS and paired with a non-setpoint datapoint sharing the same category_id and twin_id. Ambiguous or unpaired setpoints are skipped."""
+    df = context.dataframe
+    datapoints = context.datapoints or {}
+
+    setpoints = {
+        col: dp for col, dp in datapoints.items()
+        if col in df.columns and dp.business_type == wizata_dsapi.BusinessType.SET_POINTS
+    }
+
+    if not setpoints:
+        raise ValueError(f"no setpoint datapoints found in context (BusinessType.SET_POINTS)")
+
+    paired = 0
+    for sp_col, sp_dp in setpoints.items():
+        candidates = [
+            col for col, dp in datapoints.items()
+            if col in df.columns
+            and col != sp_col
+            and dp.business_type != wizata_dsapi.BusinessType.SET_POINTS
+            and dp.category_id is not None
+            and dp.category_id == sp_dp.category_id
+            and dp.twin_id == sp_dp.twin_id
+        ]
+        if len(candidates) == 1:
+            meas_col = candidates[0]
+            df[f"{meas_col}_deviation"] = df[meas_col] - df[sp_col]
+            paired += 1
+
+    if paired == 0:
+        raise ValueError(f"no setpoint/measurement pairs could be resolved — check that paired datapoints share category_id and twin_id")
+
+    return df
+
+
+def formula(context: wizata_dsapi.Context):
+    """Add a new column computed from a user-defined math expression over existing columns. Property 'expression' is required — intuitive math syntax referencing column names (e.g. 'temp_1 + temp_2', '(p_in - p_out) / p_in * 100', 'sqrt(vibration_x**2 + vibration_y**2)', 'clip(temperature, 0, 500)', 'log(power + 1)'). Supports arithmetic operators (+, -, *, /, %, **) and these functions: abs, sqrt, log, log10, exp, clip, round, min, max. Column names with spaces must be wrapped in backticks (e.g. '`motor temp` * 2'). Property 'result' (default 'result') names the output column."""
+    df = context.dataframe
+
+    if "expression" not in context.properties:
+        raise KeyError(f"please set an 'expression' property (e.g. 'col_a + col_b')")
+
+    expression = context.properties["expression"]
+    result = context.properties.get("result", "result")
+
+    # Whitelist validation: strip everything we allow, flag any leftover token.
+    sanitized = expression
+    for col in sorted(df.columns, key=len, reverse=True):
+        sanitized = sanitized.replace(f"`{col}`", "").replace(col, "")
+    for fn in _FORMULA_SAFE_FUNCTIONS:
+        sanitized = sanitized.replace(fn, "")
+    sanitized = re.sub(r"[\d\.\+\-\*/\%\(\)\s,\^]", "", sanitized)
+    if sanitized.strip():
+        raise ValueError(
+            f"expression contains disallowed tokens: '{sanitized.strip()}' — "
+            f"only column names, numbers, arithmetic operators, and these functions are allowed: "
+            f"{list(_FORMULA_SAFE_FUNCTIONS.keys())}"
+        )
+
+    df[result] = df.eval(expression, local_dict=_FORMULA_SAFE_FUNCTIONS, engine="python")
+    return df
+
+
+def target_feat_to_binary(context: wizata_dsapi.Context):
+    """Convert a target feature column to binary (0/1) using a threshold operator (lt, lte, gt, gte)."""
+    df = context.dataframe
+
+    if "target_feat" not in context.properties:
+        raise KeyError(f'please set a target feature to transform to binary class')
+
+    target_feat = context.properties["target_feat"]["sensor"]
+    operator = context.properties["target_feat"]["operator"]
+    threshold = context.properties["target_feat"]["threshold"]
+
+    if operator == 'lt':
+        df[target_feat] = numpy.where(df[target_feat] < threshold, 1, 0)
+    elif operator == 'lte':
+        df[target_feat] = numpy.where(df[target_feat] <= threshold, 1, 0)
+    elif operator == 'gt':
+        df[target_feat] = numpy.where(df[target_feat] > threshold, 1, 0)
+    elif operator == 'gte':
+        df[target_feat] = numpy.where(df[target_feat] >= threshold, 1, 0)
+    else:
+        raise KeyError(f'operator type for binarisation not know')
+
+    # Check if at least 1 value of each class
+    if df[target_feat].nunique() == 1:
+        raise KeyError(f'classification model requires 2 classes, only one was detected')
+    elif df[target_feat].nunique() > 2:
+        raise KeyError(f'classification model requires 2 classes, more than 2 were detected')
+
+    return df

wizata_dsapi-2.0.0.dev25/wizata_dsapi/version.py

@@ -0,0 +1 @@
+__version__ = "2.0.0.dev25"

{wizata_dsapi-2.0.0.dev23 → wizata_dsapi-2.0.0.dev25/wizata_dsapi.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wizata_dsapi
-Version: 2.0.0.dev23
+Version: 2.0.0.dev25
 Summary: Wizata Data Science Toolkit
 Author: Wizata S.A.
 Author-email: info@wizata.com

wizata_dsapi-2.0.0.dev23/wizata_dsapi/models/__init__.py

@@ -1 +0,0 @@
-from .common import linear_regression, logistic_regression, isolation_forest, gradiant_boost_classifier

wizata_dsapi-2.0.0.dev23/wizata_dsapi/models/common.py

@@ -1,116 +0,0 @@
-import wizata_dsapi
-
-import pandas
-import numpy
-
-import sklearn
-import sklearn.linear_model
-import sklearn.ensemble
-
-
-def extract_target_feat(context: wizata_dsapi.Context, single: bool = True):
-    """
-    return a list of target_feat columns names if not single value or the single value target feat name
-    raise an error if configuration mismatch
-    """
-
-    if "target_feat" not in context.properties:
-        raise ValueError(f"training script requires a proper target_feat")
-
-    target_feat = context.properties["target_feat"]
-    if isinstance(target_feat, str):
-        if single:
-            return target_feat
-        else:
-            return [target_feat]
-    elif isinstance(target_feat, list):
-        if single:
-            if len(target_feat) == 1:
-                return target_feat[0]
-            else:
-                raise ValueError(f"expecting only one target_feat but found {len(target_feat)}")
-        else:
-            return [target_feat]
-    else:
-        raise TypeError(f'target_feat must be a str or a list of str but found {target_feat.__class__.__name__}')
-
-
-def linear_regression(context: wizata_dsapi.Context):
-    """Train a linear regression model on all features to predict a single target column."""
-    df = context.dataframe
-
-    model_config = context.get_model_config()
-    if not model_config.has_target_feat():
-        raise ValueError(f'linear_regression requires a target feat')
-    target_feat_name = context.properties["target_feat"]
-
-    x = df.drop(columns=[target_feat_name])
-    y = df[target_feat_name]
-
-    model = sklearn.linear_model.LinearRegression()
-    model.fit(x, y)
-
-    context.set_model(model, features=x.columns)
-
-
-def logistic_regression(context: wizata_dsapi.Context):
-    """Train a logistic regression classifier on all features to predict a binary target column."""
-    df = context.dataframe
-
-    model_config = context.get_model_config()
-    if not model_config.has_target_feat():
-        raise ValueError(f'logistic_regression requires a target feat')
-    target_feat_name = context.properties["target_feat"]
-
-    x = df.drop(columns=[target_feat_name])
-    y = df[target_feat_name]
-
-    model = sklearn.linear_model.LogisticRegression()
-    model.fit(x, y.astype(int))
-
-    context.set_model(model, features=x.columns)
-
-
-def isolation_forest(context: wizata_dsapi.Context):
-    """Train an Isolation Forest for unsupervised anomaly detection using a sensitivity level (1-5)."""
-
-    model_config = context.get_model_config()
-    if model_config.has_target_feat():
-        raise ValueError(f'isolation_forest does not requires a target feat')
-
-    try:
-        if context.properties['sensitivity'] is None:
-            raise KeyError("sensitivity is none")
-        sensitivity = int(context.properties['sensitivity'])
-        sensitivities = [0.05, 0.15, 0.25, 0.35, 0.4]
-        contamination = sensitivities[sensitivity - 1]
-    except Exception as e:
-        raise ValueError(f'cannot extract sensitivity integer from 0 to 4 due to {e}')
-
-    df = context.dataframe.copy()
-    model = sklearn.ensemble.IsolationForest(contamination=contamination)
-    df['isolation_forest_predict'] = model.fit_predict(df)
-    context.set_model(model, features=df.columns)
-    return df
-
-
-def gradiant_boost_classifier(context: wizata_dsapi.Context):
-    """Train a Gradient Boosting classifier on all features to predict a target column."""
-    df = context.dataframe
-
-    model_config = context.get_model_config()
-    if not model_config.has_target_feat():
-        raise ValueError(f'gradiant_boost_classifier requires a target feat')
-    target_feat_name = context.properties["target_feat"]
-
-    x = df.drop(columns=[target_feat_name])
-    y = df[target_feat_name]
-
-    model = sklearn.ensemble.GradientBoostingClassifier(random_state=0).fit(x, y)
-    context.set_model(model, features=df.columns)
-
-
-
-
-
-

wizata_dsapi-2.0.0.dev23/wizata_dsapi/scripts/__init__.py

@@ -1 +0,0 @@
-from .common import filter_df, fillna, clustering, merge, target_feat_to_binary

wizata_dsapi-2.0.0.dev23/wizata_dsapi/scripts/common.py

@@ -1,122 +0,0 @@
-import wizata_dsapi
-
-import pandas
-import numpy
-
-import sklearn
-import sklearn.cluster
-import sklearn.metrics
-import sklearn.ensemble
-
-
-def filter_df(context: wizata_dsapi.Context):
-    """Filter dataframe rows using pandas query strings from the 'filters' property list."""
-
-    if "filters" not in context.properties or not isinstance(context.properties['filters'], list):
-        raise ValueError(f'there is no list *filters* in properties - please set them on context or config')
-
-    df = context.dataframe.copy()
-
-    filters = context.properties['filters']
-    for filter_row in filters:
-        try:
-            df = df.query(filter_row)
-        except pandas.errors.ParserError as e:
-            raise ValueError(f"error parsing filter string '{filter_row}': {e}")
-
-    return df
-
-
-def clustering(context: wizata_dsapi.Context):
-    """K-means clustering with automatic cluster count selection via silhouette score."""
-    df = context.dataframe.copy()
-    scaler = sklearn.preprocessing.StandardScaler()
-    df_clustering_scaler = scaler.fit_transform(df)
-
-    range_n_clusters = list(range(2, min(10, df_clustering_scaler.shape[0])))
-    silhouette_avg = []
-    for num_clusters in range_n_clusters:
-        kmeans = sklearn.cluster.KMeans(n_clusters=num_clusters)
-        kmeans.fit(df_clustering_scaler)
-        cluster_labels = kmeans.labels_
-        unique, counts = numpy.unique(cluster_labels, return_counts=True)
-
-        if len(unique) >= 2:
-            silhouette_avg.append(sklearn.metrics.silhouette_score(df_clustering_scaler, cluster_labels))
-        else:
-            silhouette_avg.append(numpy.nan)
-
-    if numpy.isnan(silhouette_avg).all():
-        df['cluster_labels'] = 0
-    else:
-        best_nb_clusters = silhouette_avg.index(max(silhouette_avg)) + 2
-        kmeans = sklearn.cluster.KMeans(n_clusters=best_nb_clusters)
-        kmeans.fit(df_clustering_scaler)
-        cluster_labels = kmeans.labels_
-        df['cluster_labels'] = cluster_labels
-        df['cluster_labels'] = df['cluster_labels'].apply(lambda x: int(x + 1))
-
-    return df
-
-
-def merge(context: wizata_dsapi.Context):
-    """Merge multiple dataframes by index using outer join (configurable via 'how' property)."""
-    dataframes = context.current_dataframes()
-    if len(dataframes) <= 1:
-        raise ValueError(f'there is not enough dataframes to concat')
-
-    how = "outer"
-    if "how" in context.properties:
-        how = context.properties["how"]
-
-    df = None
-    for key in dataframes:
-        if df is None:
-            df = dataframes[key]
-        else:
-            df = df.merge(dataframes[key], how=how, left_index=True, right_index=True)
-    return df
-
-
-def fillna(context: wizata_dsapi.Context):
-    """Fill missing values per column using the 'fillna' property dict mapping column names to fill values."""
-    df = context.dataframe
-
-    if "fillna" not in context.properties:
-        raise KeyError(f'please set a property dict fillna')
-
-    for key in context.properties["fillna"]:
-        df[key] = df[key].fillna(value=context.properties["fillna"][key])
-
-    return df
-
-
-def target_feat_to_binary(context: wizata_dsapi.Context):
-    """Convert a target feature column to binary (0/1) using a threshold operator (lt, lte, gt, gte)."""
-    df = context.dataframe
-
-    if "target_feat" not in context.properties:
-        raise KeyError(f'please set a target feature to transform to binary class')
-
-    target_feat = context.properties["target_feat"]["sensor"]
-    operator = context.properties["target_feat"]["operator"]
-    threshold = context.properties["target_feat"]["threshold"]
-
-    if operator == 'lt':
-        df[target_feat] = numpy.where(df[target_feat] < threshold, 1, 0)
-    elif operator == 'lte':
-        df[target_feat] = numpy.where(df[target_feat] <= threshold, 1, 0)
-    elif operator == 'gt':
-        df[target_feat] = numpy.where(df[target_feat] > threshold, 1, 0)
-    elif operator == 'gte':
-        df[target_feat] = numpy.where(df[target_feat] >= threshold, 1, 0)
-    else:
-        raise KeyError(f'operator type for binarisation not know')
-
-    # Check if at least 1 value of each class
-    if df[target_feat].nunique() == 1:
-        raise KeyError(f'classification model requires 2 classes, only one was detected')
-    elif df[target_feat].nunique() > 2:
-        raise KeyError(f'classification model requires 2 classes, more than 2 were detected')
-
-    return df

wizata_dsapi-2.0.0.dev23/wizata_dsapi/version.py

@@ -1 +0,0 @@
-__version__ = "2.0.0.dev23"