wizata-dsapi 2.0.0.dev22__tar.gz → 2.0.0.dev24__tar.gz

{wizata_dsapi-2.0.0.dev22/wizata_dsapi.egg-info → wizata_dsapi-2.0.0.dev24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wizata_dsapi
-Version: 2.0.0.dev22
+Version: 2.0.0.dev24
 Summary: Wizata Data Science Toolkit
 Author: Wizata S.A.
 Author-email: info@wizata.com

{wizata_dsapi-2.0.0.dev22 → wizata_dsapi-2.0.0.dev24}/wizata_dsapi/mlmodel.py

@@ -472,6 +472,7 @@ class MLModelConfig(ApiDto):
         self.train_test_split_pct = train_test_split_pct
         self.train_test_split_type = train_test_split_type
         self.function = function
+        self.properties = None
         self.properties_mapping = properties_mapping
 
         # features management
@@ -540,6 +541,8 @@ class MLModelConfig(ApiDto):
             self.train_test_split_type = obj["train_test_split_type"]
         if "function" in obj.keys() and obj["function"] is not None:
             self.function = obj["function"]
+        if "properties" in obj and obj["properties"] is not None:
+            self.properties = obj["properties"]
         if "properties_mapping" in obj:
             self.properties_mapping = obj["properties_mapping"]
 
@@ -617,6 +620,8 @@ class MLModelConfig(ApiDto):
             obj["features"] = self.features
         if self.features_from_file is not None:
             obj["features_from_file"] = self.features_from_file
+        if self.properties is not None and isinstance(self.properties, dict):
+            obj["properties"] = self.properties
         if self.properties_mapping is not None and isinstance(self.properties_mapping, dict):
             obj["properties_mapping"] = self.properties_mapping
 

{wizata_dsapi-2.0.0.dev22 → wizata_dsapi-2.0.0.dev24}/wizata_dsapi/script.py

@@ -187,7 +187,8 @@ class Script(ApiDto):
         return "json"
 
     def __init__(self, script_id=None, description=None, function=None,
-                 script_type=None, version=None, library=None, category=None):
+                 script_type=None, version=None, library=None, category=None,
+                 properties=None):
 
         # Id
         if script_id is None:
@@ -202,6 +203,7 @@ class Script(ApiDto):
         self.library = library
         self.module = None
         self.category = category
+        self.properties = properties if properties is not None else {}
 
         # Source code property (not serialized)
         self.source = None
@@ -271,6 +273,8 @@ class Script(ApiDto):
             obj["source"] = self.source
         if self.category is not None:
             obj["category"] = self.category.value
+        if self.properties:
+            obj["properties"] = self.properties
         return obj
 
     def to_json(self, target: str = None):
@@ -304,6 +308,8 @@ class Script(ApiDto):
             self.source = obj["source"]
         if "category" in obj and obj["category"] is not None:
             self.category = ScriptCategory(obj["category"])
+        if "properties" in obj and obj["properties"] is not None:
+            self.properties = obj["properties"]
 
     def copy(self, myfunction):
         """

wizata_dsapi-2.0.0.dev24/wizata_dsapi/scripts/__init__.py

@@ -0,0 +1 @@
+from .common import filter_df, fillna, clustering, merge, normalize, interpolate, remove_outliers, resample, rolling, diff, lag, clip, steady_state_filter, pca, setpoint_deviation, formula, target_feat_to_binary

wizata_dsapi-2.0.0.dev24/wizata_dsapi/scripts/common.py

@@ -0,0 +1,372 @@
+import re
+
+import wizata_dsapi
+
+import pandas
+import numpy
+
+import sklearn
+import sklearn.cluster
+import sklearn.metrics
+import sklearn.ensemble
+import sklearn.preprocessing
+import sklearn.decomposition
+
+
+_FORMULA_SAFE_FUNCTIONS = {
+    "abs": numpy.abs,
+    "sqrt": numpy.sqrt,
+    "log": numpy.log,
+    "log10": numpy.log10,
+    "exp": numpy.exp,
+    "clip": numpy.clip,
+    "round": numpy.round,
+    "min": numpy.minimum,
+    "max": numpy.maximum,
+}
+
+
+def filter_df(context: wizata_dsapi.Context):
+    """Filter dataframe rows using pandas query strings from the 'filters' property list."""
+
+    if "filters" not in context.properties or not isinstance(context.properties['filters'], list):
+        raise ValueError(f'there is no list *filters* in properties - please set them on context or config')
+
+    df = context.dataframe.copy()
+
+    filters = context.properties['filters']
+    for filter_row in filters:
+        try:
+            df = df.query(filter_row)
+        except pandas.errors.ParserError as e:
+            raise ValueError(f"error parsing filter string '{filter_row}': {e}")
+
+    return df
+
+
+def clustering(context: wizata_dsapi.Context):
+    """K-means clustering with automatic cluster count selection via silhouette score."""
+    df = context.dataframe.copy()
+    scaler = sklearn.preprocessing.StandardScaler()
+    df_clustering_scaler = scaler.fit_transform(df)
+
+    range_n_clusters = list(range(2, min(10, df_clustering_scaler.shape[0])))
+    silhouette_avg = []
+    for num_clusters in range_n_clusters:
+        kmeans = sklearn.cluster.KMeans(n_clusters=num_clusters)
+        kmeans.fit(df_clustering_scaler)
+        cluster_labels = kmeans.labels_
+        unique, counts = numpy.unique(cluster_labels, return_counts=True)
+
+        if len(unique) >= 2:
+            silhouette_avg.append(sklearn.metrics.silhouette_score(df_clustering_scaler, cluster_labels))
+        else:
+            silhouette_avg.append(numpy.nan)
+
+    if numpy.isnan(silhouette_avg).all():
+        df['cluster_labels'] = 0
+    else:
+        best_nb_clusters = silhouette_avg.index(max(silhouette_avg)) + 2
+        kmeans = sklearn.cluster.KMeans(n_clusters=best_nb_clusters)
+        kmeans.fit(df_clustering_scaler)
+        cluster_labels = kmeans.labels_
+        df['cluster_labels'] = cluster_labels
+        df['cluster_labels'] = df['cluster_labels'].apply(lambda x: int(x + 1))
+
+    return df
+
+
+def merge(context: wizata_dsapi.Context):
+    """Merge multiple dataframes by index using outer join (configurable via 'how' property)."""
+    dataframes = context.current_dataframes()
+    if len(dataframes) <= 1:
+        raise ValueError(f'there is not enough dataframes to concat')
+
+    how = "outer"
+    if "how" in context.properties:
+        how = context.properties["how"]
+
+    df = None
+    for key in dataframes:
+        if df is None:
+            df = dataframes[key]
+        else:
+            df = df.merge(dataframes[key], how=how, left_index=True, right_index=True)
+    return df
+
+
+def fillna(context: wizata_dsapi.Context):
+    """Fill missing values per column using the 'fillna' property dict mapping column names to fill values."""
+    df = context.dataframe
+
+    if "fillna" not in context.properties:
+        raise KeyError(f'please set a property dict fillna')
+
+    for key in context.properties["fillna"]:
+        df[key] = df[key].fillna(value=context.properties["fillna"][key])
+
+    return df
+
+
+def normalize(context: wizata_dsapi.Context):
+    """Normalize all numeric columns using 'minmax' (default) or 'zscore' scaling (configurable via 'method' property)."""
+    df = context.dataframe.copy()
+
+    method = context.properties.get("method", "minmax")
+    columns = df.select_dtypes(include="number").columns.tolist()
+
+    if not columns:
+        raise ValueError(f'no numeric columns to normalize')
+
+    if method == "minmax":
+        scaler = sklearn.preprocessing.MinMaxScaler()
+    elif method == "zscore":
+        scaler = sklearn.preprocessing.StandardScaler()
+    else:
+        raise ValueError(f"unknown normalize method '{method}', use 'minmax' or 'zscore'")
+
+    df[columns] = scaler.fit_transform(df[columns])
+    return df
+
+
+def interpolate(context: wizata_dsapi.Context):
+    """Interpolate missing values in numeric columns. The 'method' property selects the pandas interpolation method: 'time' (default, respects timestamp spacing), 'linear', 'nearest', 'pad', 'polynomial', or 'spline'."""
+    df = context.dataframe.copy()
+
+    method = context.properties.get("method", "time")
+
+    return df.interpolate(method=method)
+
+
+def remove_outliers(context: wizata_dsapi.Context):
+    """Drop rows containing outliers in any numeric column. The 'method' property selects 'iqr' (default, Tukey 1.5*IQR rule) or 'zscore' (drops rows beyond ±3 sigma)."""
+    df = context.dataframe.copy()
+
+    method = context.properties.get("method", "iqr")
+    columns = df.select_dtypes(include="number").columns.tolist()
+
+    if not columns:
+        return df
+
+    if method == "iqr":
+        q1 = df[columns].quantile(0.25)
+        q3 = df[columns].quantile(0.75)
+        iqr = q3 - q1
+        lower = q1 - 1.5 * iqr
+        upper = q3 + 1.5 * iqr
+        mask = ((df[columns] >= lower) & (df[columns] <= upper)).all(axis=1)
+    elif method == "zscore":
+        mean = df[columns].mean()
+        std = df[columns].std()
+        z = (df[columns] - mean).abs().divide(std)
+        mask = (z.fillna(0) <= 3).all(axis=1)
+    else:
+        raise ValueError(f"unknown remove_outliers method '{method}', use 'iqr' or 'zscore'")
+
+    return df[mask]
+
+
+def resample(context: wizata_dsapi.Context):
+    """Resample the dataframe to a new time frequency. Property 'freq' is required (pandas offset alias, e.g. '1min', '5min', '1H', '1D'); 'agg' selects the aggregation ('mean' default, 'sum', 'min', 'max', 'first', 'last', 'median')."""
+    df = context.dataframe
+
+    if "freq" not in context.properties:
+        raise KeyError(f"please set a 'freq' property (pandas offset alias, e.g. '1min', '1H')")
+
+    freq = context.properties["freq"]
+    agg = context.properties.get("agg", "mean")
+
+    return df.resample(freq).agg(agg)
+
+
+def rolling(context: wizata_dsapi.Context):
+    """Apply a rolling-window aggregation over all numeric columns. Property 'window' is required (integer number of rows); 'agg' selects the aggregation ('mean' default, 'sum', 'std', 'min', 'max', 'median')."""
+    df = context.dataframe
+
+    if "window" not in context.properties:
+        raise KeyError(f"please set a 'window' property (integer number of rows)")
+
+    window = context.properties["window"]
+    agg = context.properties.get("agg", "mean")
+
+    return df.rolling(window=window).agg(agg)
+
+
+def diff(context: wizata_dsapi.Context):
+    """Compute discrete differences (rate of change) across all numeric columns. Property 'periods' (default 1) is the number of rows to shift before subtracting — use 1 for first derivative, higher for longer horizons."""
+    df = context.dataframe
+
+    periods = context.properties.get("periods", 1)
+
+    return df.diff(periods=periods)
+
+
+def lag(context: wizata_dsapi.Context):
+    """Add lagged versions of all numeric columns as new '<col>_lag<N>' columns, preserving originals. Property 'periods' is required (integer number of rows to shift back)."""
+    df = context.dataframe
+
+    if "periods" not in context.properties:
+        raise KeyError(f"please set a 'periods' property (integer number of rows to lag)")
+
+    periods = context.properties["periods"]
+    columns = df.select_dtypes(include="number").columns.tolist()
+
+    for col in columns:
+        df[f"{col}_lag{periods}"] = df[col].shift(periods)
+
+    return df
+
+
+def clip(context: wizata_dsapi.Context):
+    """Clip all numeric columns to a physical range. Properties 'min' and/or 'max' (floats) bound the values; at least one of the two must be provided."""
+    df = context.dataframe
+
+    lower = context.properties.get("min")
+    upper = context.properties.get("max")
+
+    if lower is None and upper is None:
+        raise KeyError(f"please set at least one of 'min' or 'max' properties")
+
+    columns = df.select_dtypes(include="number").columns.tolist()
+    df[columns] = df[columns].clip(lower=lower, upper=upper)
+
+    return df
+
+
+def steady_state_filter(context: wizata_dsapi.Context):
+    """Keep only rows where all numeric columns are in steady state (rolling std over 'window' rows stays below 'tolerance'). Drops transients and start-up periods — a standard preprocessing step before process modeling."""
+    df = context.dataframe
+
+    if "window" not in context.properties:
+        raise KeyError(f"please set a 'window' property (integer number of rows)")
+    if "tolerance" not in context.properties:
+        raise KeyError(f"please set a 'tolerance' property (max allowed rolling std)")
+
+    window = context.properties["window"]
+    tolerance = context.properties["tolerance"]
+
+    columns = df.select_dtypes(include="number").columns.tolist()
+    if not columns:
+        return df
+
+    rolling_std = df[columns].rolling(window=window).std()
+    mask = (rolling_std <= tolerance).all(axis=1)
+
+    return df[mask]
+
+
+def pca(context: wizata_dsapi.Context):
+    """Reduce numeric columns to principal components, replacing them with 'PC1', 'PC2', ... Property 'n_components' is required — an integer (number of components) or a float in (0, 1] (minimum explained variance ratio). NaN values must be handled upstream (e.g. with interpolate or fillna)."""
+    df = context.dataframe
+
+    if "n_components" not in context.properties:
+        raise KeyError(f"please set an 'n_components' property (int or float in (0,1])")
+
+    n_components = context.properties["n_components"]
+    columns = df.select_dtypes(include="number").columns.tolist()
+
+    if not columns:
+        raise ValueError(f"no numeric columns to reduce")
+
+    if df[columns].isna().any().any():
+        raise ValueError(f"PCA cannot handle NaN values — run interpolate or fillna upstream")
+
+    model = sklearn.decomposition.PCA(n_components=n_components)
+    transformed = model.fit_transform(df[columns])
+
+    pc_cols = [f"PC{i + 1}" for i in range(transformed.shape[1])]
+    return pandas.DataFrame(transformed, index=df.index, columns=pc_cols)
+
+
+def setpoint_deviation(context: wizata_dsapi.Context):
+    """Add '<measurement>_deviation' columns computed as (measurement - setpoint) for each setpoint datapoint in the dataframe. Setpoints are auto-detected via BusinessType.SET_POINTS and paired with a non-setpoint datapoint sharing the same category_id and twin_id. Ambiguous or unpaired setpoints are skipped."""
+    df = context.dataframe
+    datapoints = context.datapoints or {}
+
+    setpoints = {
+        col: dp for col, dp in datapoints.items()
+        if col in df.columns and dp.business_type == wizata_dsapi.BusinessType.SET_POINTS
+    }
+
+    if not setpoints:
+        raise ValueError(f"no setpoint datapoints found in context (BusinessType.SET_POINTS)")
+
+    paired = 0
+    for sp_col, sp_dp in setpoints.items():
+        candidates = [
+            col for col, dp in datapoints.items()
+            if col in df.columns
+            and col != sp_col
+            and dp.business_type != wizata_dsapi.BusinessType.SET_POINTS
+            and dp.category_id is not None
+            and dp.category_id == sp_dp.category_id
+            and dp.twin_id == sp_dp.twin_id
+        ]
+        if len(candidates) == 1:
+            meas_col = candidates[0]
+            df[f"{meas_col}_deviation"] = df[meas_col] - df[sp_col]
+            paired += 1
+
+    if paired == 0:
+        raise ValueError(f"no setpoint/measurement pairs could be resolved — check that paired datapoints share category_id and twin_id")
+
+    return df
+
+
+def formula(context: wizata_dsapi.Context):
+    """Add a new column computed from a user-defined math expression over existing columns. Property 'expression' is required — intuitive math syntax referencing column names (e.g. 'temp_1 + temp_2', '(p_in - p_out) / p_in * 100', 'sqrt(vibration_x**2 + vibration_y**2)', 'clip(temperature, 0, 500)', 'log(power + 1)'). Supports arithmetic operators (+, -, *, /, %, **) and these functions: abs, sqrt, log, log10, exp, clip, round, min, max. Column names with spaces must be wrapped in backticks (e.g. '`motor temp` * 2'). Property 'result' (default 'result') names the output column."""
+    df = context.dataframe
+
+    if "expression" not in context.properties:
+        raise KeyError(f"please set an 'expression' property (e.g. 'col_a + col_b')")
+
+    expression = context.properties["expression"]
+    result = context.properties.get("result", "result")
+
+    # Whitelist validation: strip everything we allow, flag any leftover token.
+    sanitized = expression
+    for col in sorted(df.columns, key=len, reverse=True):
+        sanitized = sanitized.replace(f"`{col}`", "").replace(col, "")
+    for fn in _FORMULA_SAFE_FUNCTIONS:
+        sanitized = sanitized.replace(fn, "")
+    sanitized = re.sub(r"[\d\.\+\-\*/\%\(\)\s,\^]", "", sanitized)
+    if sanitized.strip():
+        raise ValueError(
+            f"expression contains disallowed tokens: '{sanitized.strip()}' — "
+            f"only column names, numbers, arithmetic operators, and these functions are allowed: "
+            f"{list(_FORMULA_SAFE_FUNCTIONS.keys())}"
+        )
+
+    df[result] = df.eval(expression, local_dict=_FORMULA_SAFE_FUNCTIONS, engine="python")
+    return df
+
+
+def target_feat_to_binary(context: wizata_dsapi.Context):
+    """Convert a target feature column to binary (0/1) using a threshold operator (lt, lte, gt, gte)."""
+    df = context.dataframe
+
+    if "target_feat" not in context.properties:
+        raise KeyError(f'please set a target feature to transform to binary class')
+
+    target_feat = context.properties["target_feat"]["sensor"]
+    operator = context.properties["target_feat"]["operator"]
+    threshold = context.properties["target_feat"]["threshold"]
+
+    if operator == 'lt':
+        df[target_feat] = numpy.where(df[target_feat] < threshold, 1, 0)
+    elif operator == 'lte':
+        df[target_feat] = numpy.where(df[target_feat] <= threshold, 1, 0)
+    elif operator == 'gt':
+        df[target_feat] = numpy.where(df[target_feat] > threshold, 1, 0)
+    elif operator == 'gte':
+        df[target_feat] = numpy.where(df[target_feat] >= threshold, 1, 0)
+    else:
+        raise KeyError(f'operator type for binarisation not know')
+
+    # Check if at least 1 value of each class
+    if df[target_feat].nunique() == 1:
+        raise KeyError(f'classification model requires 2 classes, only one was detected')
+    elif df[target_feat].nunique() > 2:
+        raise KeyError(f'classification model requires 2 classes, more than 2 were detected')
+
+    return df

wizata_dsapi-2.0.0.dev24/wizata_dsapi/version.py

@@ -0,0 +1 @@
+__version__ = "2.0.0.dev24"

{wizata_dsapi-2.0.0.dev22 → wizata_dsapi-2.0.0.dev24/wizata_dsapi.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wizata_dsapi
-Version: 2.0.0.dev22
+Version: 2.0.0.dev24
 Summary: Wizata Data Science Toolkit
 Author: Wizata S.A.
 Author-email: info@wizata.com

wizata_dsapi-2.0.0.dev22/wizata_dsapi/scripts/__init__.py

@@ -1 +0,0 @@
-from .common import filter_df, fillna, clustering, merge, target_feat_to_binary

wizata_dsapi-2.0.0.dev22/wizata_dsapi/scripts/common.py

@@ -1,122 +0,0 @@
-import wizata_dsapi
-
-import pandas
-import numpy
-
-import sklearn
-import sklearn.cluster
-import sklearn.metrics
-import sklearn.ensemble
-
-
-def filter_df(context: wizata_dsapi.Context):
-    """Filter dataframe rows using pandas query strings from the 'filters' property list."""
-
-    if "filters" not in context.properties or not isinstance(context.properties['filters'], list):
-        raise ValueError(f'there is no list *filters* in properties - please set them on context or config')
-
-    df = context.dataframe.copy()
-
-    filters = context.properties['filters']
-    for filter_row in filters:
-        try:
-            df = df.query(filter_row)
-        except pandas.errors.ParserError as e:
-            raise ValueError(f"error parsing filter string '{filter_row}': {e}")
-
-    return df
-
-
-def clustering(context: wizata_dsapi.Context):
-    """K-means clustering with automatic cluster count selection via silhouette score."""
-    df = context.dataframe.copy()
-    scaler = sklearn.preprocessing.StandardScaler()
-    df_clustering_scaler = scaler.fit_transform(df)
-
-    range_n_clusters = list(range(2, min(10, df_clustering_scaler.shape[0])))
-    silhouette_avg = []
-    for num_clusters in range_n_clusters:
-        kmeans = sklearn.cluster.KMeans(n_clusters=num_clusters)
-        kmeans.fit(df_clustering_scaler)
-        cluster_labels = kmeans.labels_
-        unique, counts = numpy.unique(cluster_labels, return_counts=True)
-
-        if len(unique) >= 2:
-            silhouette_avg.append(sklearn.metrics.silhouette_score(df_clustering_scaler, cluster_labels))
-        else:
-            silhouette_avg.append(numpy.nan)
-
-    if numpy.isnan(silhouette_avg).all():
-        df['cluster_labels'] = 0
-    else:
-        best_nb_clusters = silhouette_avg.index(max(silhouette_avg)) + 2
-        kmeans = sklearn.cluster.KMeans(n_clusters=best_nb_clusters)
-        kmeans.fit(df_clustering_scaler)
-        cluster_labels = kmeans.labels_
-        df['cluster_labels'] = cluster_labels
-        df['cluster_labels'] = df['cluster_labels'].apply(lambda x: int(x + 1))
-
-    return df
-
-
-def merge(context: wizata_dsapi.Context):
-    """Merge multiple dataframes by index using outer join (configurable via 'how' property)."""
-    dataframes = context.current_dataframes()
-    if len(dataframes) <= 1:
-        raise ValueError(f'there is not enough dataframes to concat')
-
-    how = "outer"
-    if "how" in context.properties:
-        how = context.properties["how"]
-
-    df = None
-    for key in dataframes:
-        if df is None:
-            df = dataframes[key]
-        else:
-            df = df.merge(dataframes[key], how=how, left_index=True, right_index=True)
-    return df
-
-
-def fillna(context: wizata_dsapi.Context):
-    """Fill missing values per column using the 'fillna' property dict mapping column names to fill values."""
-    df = context.dataframe
-
-    if "fillna" not in context.properties:
-        raise KeyError(f'please set a property dict fillna')
-
-    for key in context.properties["fillna"]:
-        df[key] = df[key].fillna(value=context.properties["fillna"][key])
-
-    return df
-
-
-def target_feat_to_binary(context: wizata_dsapi.Context):
-    """Convert a target feature column to binary (0/1) using a threshold operator (lt, lte, gt, gte)."""
-    df = context.dataframe
-
-    if "target_feat" not in context.properties:
-        raise KeyError(f'please set a target feature to transform to binary class')
-
-    target_feat = context.properties["target_feat"]["sensor"]
-    operator = context.properties["target_feat"]["operator"]
-    threshold = context.properties["target_feat"]["threshold"]
-
-    if operator == 'lt':
-        df[target_feat] = numpy.where(df[target_feat] < threshold, 1, 0)
-    elif operator == 'lte':
-        df[target_feat] = numpy.where(df[target_feat] <= threshold, 1, 0)
-    elif operator == 'gt':
-        df[target_feat] = numpy.where(df[target_feat] > threshold, 1, 0)
-    elif operator == 'gte':
-        df[target_feat] = numpy.where(df[target_feat] >= threshold, 1, 0)
-    else:
-        raise KeyError(f'operator type for binarisation not know')
-
-    # Check if at least 1 value of each class
-    if df[target_feat].nunique() == 1:
-        raise KeyError(f'classification model requires 2 classes, only one was detected')
-    elif df[target_feat].nunique() > 2:
-        raise KeyError(f'classification model requires 2 classes, more than 2 were detected')
-
-    return df

wizata_dsapi-2.0.0.dev22/wizata_dsapi/version.py

@@ -1 +0,0 @@
-__version__ = "2.0.0.dev22"