wizata-dsapi 2.2.0.dev19__tar.gz → 2.2.0.dev20__tar.gz

{wizata_dsapi-2.2.0.dev19 → wizata_dsapi-2.2.0.dev20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wizata_dsapi
-Version: 2.2.0.dev19
+Version: 2.2.0.dev20
 Summary: Wizata Data Science Toolkit
 Author: Wizata S.A.
 Author-email: info@wizata.com

{wizata_dsapi-2.2.0.dev19 → wizata_dsapi-2.2.0.dev20}/wizata_dsapi/__init__.py

@@ -5,6 +5,8 @@ from .paged_query_result import PagedQueryResult
 from .plot import Plot
 from .mlmodel import ModelInfo, ModelList, MLModelConfig, ModelFile, ModelIdentifierInfo
 from .request import Request, filter_map, RequestGroup, RequestGroupMap, DynamicSelector
+from .formula import (FormulaError, apply_formulas, evaluate_formula, normalize_formulas, check_result_name,
+                      formula_catalog, FORMULA_OPERATORS, FORMULA_FUNCTIONS, FORMULA_CONSTANTS)
 from .execution import Execution, ExecutionStatus, ExecutionStepLog, AbortedException
 from .execution_log import ExecutionLog
 from .experiment import Experiment
@@ -45,7 +47,8 @@ from .wizard_function import WizardStep, WizardFunction
 from .wizard_request import WizardRequest
 
 # Pipeline Entities (Dto)
-from .pipeline import Pipeline, PipelineStep, StepType, WriteConfig, AlertConfig, PipelineIO, RunnerStatus, AlertType
+from .pipeline import Pipeline, PipelineStep, StepType, WriteConfig, AlertConfig, FormulaConfig, PipelineIO, RunnerStatus, \
+    AlertType
 from .context import Context
 from .ilogger import ILogger
 from .pipeline_image import PipelineImage

wizata_dsapi-2.2.0.dev20/wizata_dsapi/formula.py

@@ -0,0 +1,493 @@
+"""
+Calculated columns (DEV-6112).
+
+One resolver for the two places a formula runs, so they cannot disagree:
+
+- the query option ``Request.formulas``, applied by ``QueryClient.execute`` after
+  everything else in the query, so the Explorer gets the results in the same
+  ``/dsapi/data`` call;
+- the ``wizata.scripts.formula`` pipeline step, where "Open as pipeline" moves that
+  list unchanged.
+
+An expression is parsed, never handed to ``eval``. Column names are swapped for
+placeholders, ``^ & | ~`` for ``** and or not``, the text is parsed with :mod:`ast`,
+and only numbers, columns, the operators, the functions and the constants of the
+catalog are evaluated. Anything else (an attribute, an index, a string, a lambda) is
+refused: since the query option, expressions run inside the DS API process.
+
+:func:`formula_catalog` is what the front-end reads (DS API ``GET /dsapi/formulas/``).
+"""
+import ast
+import functools
+import math
+import re
+
+import numpy
+import pandas
+
+
+MAX_FORMULAS = 50
+MAX_EXPRESSION_LENGTH = 2000
+MAX_FORMULA_NODES = 500
+
+# A result is named by the user, like a datapoint: 2 to 128 characters, no whitespace.
+# '/' and '|' are refused too: a result column named "<dp>/<field>" or "<dp>|<scope>"
+# would read as a datapoint field or a split datapoint in the query result format.
+RESULT_NAME_MIN_LENGTH = 2
+RESULT_NAME_MAX_LENGTH = 128
+RESULT_NAME_FORBIDDEN = " /|`"
+FORMULA_RESULT_PATTERN = re.compile(r"^[^\s/|`]{%d,%d}$" % (RESULT_NAME_MIN_LENGTH, RESULT_NAME_MAX_LENGTH))
+
+
+class FormulaError(ValueError):
+    """An expression or a formula list the resolver refuses (a 400 on the DS API)."""
+
+
+# ── catalog ──────────────────────────────────────────────────────────────────
+# `sign` is what the front-end inserts, `word` how it reads, `accepts` every spelling
+# the resolver takes for it. Each `example` is evaluated by the tests against columns
+# a, b and c, so the catalog cannot list what the resolver refuses.
+
+FORMULA_OPERATORS = [
+    {"key": "add", "category": "arithmetic", "sign": "+", "word": "plus", "accepts": ["+"],
+     "description": "Adds the right value to the left one.",
+     "example": "a + b"},
+    {"key": "subtract", "category": "arithmetic", "sign": "-", "word": "minus", "accepts": ["-"],
+     "description": "Subtracts the right value from the left one. In front of a single value, negates it.",
+     "example": "a - b"},
+    {"key": "multiply", "category": "arithmetic", "sign": "*", "word": "times", "accepts": ["*"],
+     "description": "Multiplies the left value by the right one.",
+     "example": "a * b"},
+    {"key": "divide", "category": "arithmetic", "sign": "/", "word": "divided by", "accepts": ["/"],
+     "description": "Divides the left value by the right one. Dividing by zero gives an infinite value, "
+                    "zero by zero an empty one.",
+     "example": "a / b"},
+    {"key": "modulo", "category": "arithmetic", "sign": "%", "word": "modulo", "accepts": ["%"],
+     "description": "Remainder of the left value divided by the right one.",
+     "example": "a % 2"},
+    {"key": "power", "category": "arithmetic", "sign": "**", "word": "to the power of", "accepts": ["**", "^"],
+     "description": "Raises the left value to the power of the right one; ^ means the same. It binds tighter "
+                    "than the other operators: a + b ^ 2 squares b only.",
+     "example": "a ^ 2"},
+    {"key": "less", "category": "comparison", "sign": "<", "word": "less than", "accepts": ["<"],
+     "description": "True where the left value is less than the right one.",
+     "example": "a < b"},
+    {"key": "less_equal", "category": "comparison", "sign": "<=", "word": "less than or equal to", "accepts": ["<="],
+     "description": "True where the left value is less than or equal to the right one.",
+     "example": "a <= b"},
+    {"key": "greater", "category": "comparison", "sign": ">", "word": "greater than", "accepts": [">"],
+     "description": "True where the left value is greater than the right one.",
+     "example": "a > b"},
+    {"key": "greater_equal", "category": "comparison", "sign": ">=", "word": "greater than or equal to",
+     "accepts": [">="],
+     "description": "True where the left value is greater than or equal to the right one.",
+     "example": "a >= b"},
+    {"key": "equal", "category": "comparison", "sign": "==", "word": "equal to", "accepts": ["=="],
+     "description": "True where both values are equal.",
+     "example": "a == b"},
+    {"key": "not_equal", "category": "comparison", "sign": "!=", "word": "not equal to", "accepts": ["!="],
+     "description": "True where the values differ.",
+     "example": "a != b"},
+    {"key": "and", "category": "logical", "sign": "&", "word": "and", "accepts": ["&", "and"],
+     "description": "True where both conditions are true. Comparisons are evaluated first, so "
+                    "a > 1 & b < 2 needs no brackets.",
+     "example": "a > 1 and b < 2"},
+    {"key": "or", "category": "logical", "sign": "|", "word": "or", "accepts": ["|", "or"],
+     "description": "True where at least one of the conditions is true. Comparisons are evaluated first.",
+     "example": "a > 1 or b < 2"},
+    {"key": "not", "category": "logical", "sign": "~", "word": "not", "accepts": ["~", "not"],
+     "description": "True where the condition is false. It applies to the whole comparison after it: "
+                    "not a > 1.",
+     "example": "not a > 1"},
+]
+
+
+def _round(x, decimals=0):
+    if not isinstance(decimals, (int, float, numpy.number)) or isinstance(decimals, (bool, numpy.bool_)):
+        raise FormulaError("round: the number of decimals must be a number, e.g. round(x, 2)")
+    return numpy.round(x, int(decimals))
+
+
+_FUNCTION_DEFINITIONS = [
+    ("abs", "abs(x)", 1, 1, numpy.abs, "Absolute value.", "abs(a - b)"),
+    ("sqrt", "sqrt(x)", 1, 1, numpy.sqrt, "Square root; empty for a negative value.", "sqrt(a)"),
+    ("log", "log(x)", 1, 1, numpy.log, "Natural logarithm; empty for a negative value.", "log(a)"),
+    ("log10", "log10(x)", 1, 1, numpy.log10, "Base-10 logarithm; empty for a negative value.", "log10(a)"),
+    ("exp", "exp(x)", 1, 1, numpy.exp, "e raised to the value.", "exp(a)"),
+    ("round", "round(x, decimals)", 1, 2, _round,
+     "Rounds to the given number of decimals, 0 when left out. The decimals must be a number.", "round(a, 1)"),
+    ("clip", "clip(x, low, high)", 3, 3, numpy.clip, "Limits each value to the range low to high.",
+     "clip(a, 0, 3)"),
+    ("min", "min(x, y, ...)", 2, None, lambda *values: functools.reduce(numpy.minimum, values),
+     "The smallest of the values, row by row.", "min(a, b, c)"),
+    ("max", "max(x, y, ...)", 2, None, lambda *values: functools.reduce(numpy.maximum, values),
+     "The largest of the values, row by row.", "max(a, b, c)"),
+    ("where", "where(condition, if_true, if_false)", 3, 3, numpy.where,
+     "if_true where the condition is true, else if_false.", "where(a > b, a, b)"),
+]
+
+FORMULA_FUNCTIONS = [
+    {"name": name, "signature": signature, "min_args": min_args, "max_args": max_args,
+     "description": description, "example": example}
+    for name, signature, min_args, max_args, _, description, example in _FUNCTION_DEFINITIONS
+]
+
+FORMULA_CONSTANTS = [
+    {"name": "pi", "value": math.pi, "description": "The ratio of a circle's circumference to its diameter."},
+    {"name": "e", "value": math.e, "description": "Euler's number, the base of the natural logarithm."},
+]
+
+_FUNCTIONS = {name: (function, min_args, max_args, signature)
+              for name, signature, min_args, max_args, function, _, _ in _FUNCTION_DEFINITIONS}
+_CONSTANTS = {constant["name"]: numpy.float64(constant["value"]) for constant in FORMULA_CONSTANTS}
+_OPERATOR_WORDS = {"and", "or", "not"}
+# Names a result may not take: a bare reference to it would read as something else.
+_KEYWORDS = set(_FUNCTIONS) | set(_CONSTANTS) | _OPERATOR_WORDS | {"True", "False"}
+
+
+def formula_catalog() -> dict:
+    """What the front-end needs to build and check a formula: the operators (sign, word,
+    description), the functions, the constants and the result naming rule."""
+    return {
+        "operators": [dict(operator, accepts=list(operator["accepts"])) for operator in FORMULA_OPERATORS],
+        "functions": [dict(function) for function in FORMULA_FUNCTIONS],
+        "constants": [dict(constant) for constant in FORMULA_CONSTANTS],
+        "result_name": {
+            "min_length": RESULT_NAME_MIN_LENGTH,
+            "max_length": RESULT_NAME_MAX_LENGTH,
+            "forbidden": ["whitespace", "/", "|", "`"],
+            "pattern": FORMULA_RESULT_PATTERN.pattern,
+            "reserved": sorted(_KEYWORDS),
+        },
+        "column_reference": "A column is written by its name: mt1_bearing2 - mt1_bearing1. Backticks are only "
+                            "needed for a name holding a space, which a datapoint never does.",
+        "limits": {"formulas": MAX_FORMULAS, "expression_length": MAX_EXPRESSION_LENGTH},
+    }
+
+
+# ── formula lists ────────────────────────────────────────────────────────────
+
+def check_result_name(name, position: int = None):
+    """Refuse a result name that is missing or breaks the datapoint naming convention."""
+    where = f"formula {position}: " if position is not None else ""
+    if name is None or (isinstance(name, str) and not name.strip()):
+        raise FormulaError(f"{where}a formula needs a 'result' name for its column")
+    if not isinstance(name, str) or not FORMULA_RESULT_PATTERN.match(name):
+        raise FormulaError(f"{where}result name {name!r} must be {RESULT_NAME_MIN_LENGTH} to "
+                           f"{RESULT_NAME_MAX_LENGTH} characters, without spaces, '/', '|' or '`' "
+                           f"(the datapoint naming convention)")
+    if name in _KEYWORDS:
+        raise FormulaError(f"{where}{name!r} is a formula keyword and cannot name a result")
+
+
+def normalize_formulas(formulas) -> list:
+    """Check a ``[{expression, result}]`` list and return it with only those two keys.
+
+    None is no formula. Every formula names its result (JP, 2026-10-09: the front-end may
+    generate the name, the API never does), names are unique, and the list holds at most
+    MAX_FORMULAS formulas.
+    """
+    if formulas is None:
+        return []
+    if not isinstance(formulas, list):
+        raise FormulaError(f"'formulas' must be a list of {{'expression', 'result'}}, got {type(formulas).__name__}")
+    if len(formulas) > MAX_FORMULAS:
+        raise FormulaError(f"'formulas' holds {len(formulas)} formulas, at most {MAX_FORMULAS} are allowed")
+    normalized = []
+    seen = set()
+    for position, item in enumerate(formulas, start=1):
+        if not isinstance(item, dict):
+            raise FormulaError(f"formula {position} must be an object with 'expression' and 'result'")
+        expression = item.get("expression")
+        if not isinstance(expression, str) or not expression.strip():
+            raise FormulaError(f"formula {position}: 'expression' is required")
+        _check_length(expression)
+        result = item.get("result")
+        check_result_name(result, position)
+        if result in seen:
+            raise FormulaError(f"formula {position}: result {result!r} is used twice")
+        seen.add(result)
+        normalized.append({"expression": expression, "result": result})
+    return normalized
+
+
+def apply_formulas(df: pandas.DataFrame, formulas: list, strict: bool = True) -> pandas.DataFrame:
+    """A copy of *df* with one column per formula, computed in order, so a formula can use
+    an earlier result.
+
+    :param strict: the rules of a ``formulas`` list (named, conventional and new results).
+        False is for the legacy ``expression`` / ``result`` pair of existing pipelines,
+        whose result may be any name, an existing column included.
+    """
+    if df is None:
+        raise FormulaError("there is no dataframe to compute the formulas on")
+    if strict:
+        formulas = normalize_formulas(formulas)
+    out = df.copy(deep=False)
+    for formula in formulas:
+        result = formula["result"]
+        if strict and result in _column_names(out):
+            raise FormulaError(f"formula {result!r}: {result!r} is already a column, choose another name")
+        if len(out.index) == 0:
+            values = pandas.Series(dtype="float64", index=out.index)
+        else:
+            try:
+                values = evaluate_formula(out, formula["expression"])
+            except FormulaError as e:
+                raise FormulaError(f"formula {result!r}: {e}") from e
+        out[_column_key(out, result)] = values
+    return out
+
+
+# ── one expression ───────────────────────────────────────────────────────────
+
+def evaluate_formula(df: pandas.DataFrame, expression: str) -> pandas.Series:
+    """The values of *expression* for each row of *df*, as a Series on its index."""
+    if not isinstance(expression, str) or not expression.strip():
+        raise FormulaError("the expression is empty")
+    _check_length(expression)
+    names = _column_names(df)
+    placeholders = {name: f"__wz_col_{i}__" for i, name in enumerate(names)}
+    text = _substitute(expression, placeholders)
+    for sign, word in (("^", "**"), ("&", " and "), ("|", " or "), ("~", " not ")):
+        text = text.replace(sign, word)
+    try:
+        tree = ast.parse(text.strip(), mode="eval")
+    except SyntaxError as e:
+        hint = " (compare with ==)" if re.search(r"(?<![<>=!])=(?!=)", expression) else ""
+        raise FormulaError(f"invalid expression {expression!r}: {e.msg}{hint}") from None
+    if sum(1 for _ in ast.walk(tree)) > MAX_FORMULA_NODES:
+        raise FormulaError(f"the expression is too complex (more than {MAX_FORMULA_NODES} elements)")
+    columns = {placeholder: names[name] for name, placeholder in placeholders.items()}
+    evaluator = _Evaluator(df, columns, {placeholder: name for name, placeholder in placeholders.items()})
+    with numpy.errstate(all="ignore"):
+        try:
+            value = evaluator.visit(tree)
+        except FormulaError:
+            raise
+        except Exception as e:
+            raise FormulaError(f"cannot compute {expression!r}: {e}") from None
+    return _as_series(value, df.index)
+
+
+def _check_length(expression: str):
+    if len(expression) > MAX_EXPRESSION_LENGTH:
+        raise FormulaError(f"the expression has {len(expression)} characters, "
+                           f"at most {MAX_EXPRESSION_LENGTH} are allowed")
+
+
+def _column_names(df: pandas.DataFrame) -> dict:
+    """``{name: column key}``. A MultiIndex column is named ``<dp>/<field>``, as the query
+    result format names it."""
+    names = {}
+    for column in df.columns:
+        if isinstance(column, tuple):
+            name = "/".join(str(part) for part in column if part is not None and part != "")
+        else:
+            name = str(column)
+        names.setdefault(name, column)
+    return names
+
+
+def _column_key(df: pandas.DataFrame, name: str):
+    if isinstance(df.columns, pandas.MultiIndex):
+        return (name,) + ("",) * (df.columns.nlevels - 1)
+    return name
+
+
+_IDENTIFIER = re.compile(r"[A-Za-z0-9_]")
+_NUMBER = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$")
+
+
+def _substitute(expression: str, placeholders: dict) -> str:
+    """Swap column names for their placeholders: exactly between backticks, and as whole
+    names elsewhere, longest first."""
+    parts = expression.split("`")
+    if len(parts) % 2 == 0:
+        raise FormulaError("a backtick is not closed: write `column name`")
+    bare = sorted((name for name in placeholders
+                   if name and name not in _OPERATOR_WORDS and not _NUMBER.match(name)),
+                  key=len, reverse=True)
+    out = []
+    for position, part in enumerate(parts):
+        if position % 2 == 1:
+            if part not in placeholders:
+                raise FormulaError(f"unknown column `{part}`")
+            out.append(f" {placeholders[part]} ")
+        else:
+            out.append(_substitute_bare(part, bare, placeholders))
+    return "".join(out)
+
+
+def _substitute_bare(text: str, names: list, placeholders: dict) -> str:
+    out = []
+    i = 0
+    while i < len(text):
+        found = None
+        for name in names:
+            if not text.startswith(name, i):
+                continue
+            end = i + len(name)
+            if _IDENTIFIER.match(name[0]) and i > 0 and _IDENTIFIER.match(text[i - 1]):
+                continue
+            if _IDENTIFIER.match(name[-1]) and end < len(text) and _IDENTIFIER.match(text[end]):
+                continue
+            if name in _FUNCTIONS and text[end:].lstrip().startswith("("):
+                continue
+            found = name
+            break
+        if found is None:
+            out.append(text[i])
+            i += 1
+        else:
+            out.append(f" {placeholders[found]} ")
+            i += len(found)
+    return "".join(out)
+
+
+def _as_series(value, index) -> pandas.Series:
+    if isinstance(value, pandas.Series):
+        return value
+    array = numpy.asarray(value)
+    if array.ndim == 0:
+        return pandas.Series(array.item(), index=index)
+    if array.ndim == 1 and len(array) == len(index):
+        return pandas.Series(array, index=index)
+    raise FormulaError("the expression does not give one value per row")
+
+
+_BINARY = {
+    ast.Add: lambda a, b: a + b,
+    ast.Sub: lambda a, b: a - b,
+    ast.Mult: lambda a, b: a * b,
+    ast.Div: lambda a, b: a / b,
+    ast.Mod: lambda a, b: a % b,
+    ast.Pow: lambda a, b: a ** b,
+}
+_COMPARE = {
+    ast.Lt: lambda a, b: a < b,
+    ast.LtE: lambda a, b: a <= b,
+    ast.Gt: lambda a, b: a > b,
+    ast.GtE: lambda a, b: a >= b,
+    ast.Eq: lambda a, b: a == b,
+    ast.NotEq: lambda a, b: a != b,
+}
+_REFUSED = {
+    ast.Attribute: "attribute access (a.b)",
+    ast.Subscript: "indexing (a[0])",
+    ast.Lambda: "a lambda",
+    ast.IfExp: "an inline if (use where(condition, if_true, if_false))",
+    ast.List: "a list",
+    ast.Tuple: "a tuple",
+    ast.Dict: "a dictionary",
+    ast.Set: "a set",
+    ast.JoinedStr: "a string",
+    ast.NamedExpr: "an assignment",
+    ast.Starred: "unpacking (*)",
+}
+
+
+class _Evaluator:
+    """Evaluates the parsed expression, node by node. Every node type it does not know
+    is refused."""
+
+    def __init__(self, df: pandas.DataFrame, columns: dict, names: dict):
+        self.df = df
+        self.columns = columns          # placeholder -> column key
+        self.names = names              # placeholder -> column name, for messages
+
+    def visit(self, node):
+        if isinstance(node, ast.Expression):
+            return self.visit(node.body)
+        if isinstance(node, ast.Constant):
+            return self._constant(node.value)
+        if isinstance(node, ast.Name):
+            return self._name(node.id)
+        if isinstance(node, ast.BinOp):
+            operation = _BINARY.get(type(node.op))
+            if operation is None:
+                raise FormulaError(f"operator {_operator(node.op)} is not supported")
+            return operation(self.visit(node.left), self.visit(node.right))
+        if isinstance(node, ast.UnaryOp):
+            operand = self.visit(node.operand)
+            if isinstance(node.op, ast.USub):
+                return -operand
+            if isinstance(node.op, ast.UAdd):
+                return +operand
+            if isinstance(node.op, ast.Not):
+                return numpy.logical_not(operand)
+            raise FormulaError(f"operator {_operator(node.op)} is not supported")
+        if isinstance(node, ast.BoolOp):
+            combine = numpy.logical_and if isinstance(node.op, ast.And) else numpy.logical_or
+            return functools.reduce(combine, [self.visit(value) for value in node.values])
+        if isinstance(node, ast.Compare):
+            return self._compare(node)
+        if isinstance(node, ast.Call):
+            return self._call(node)
+        what = _REFUSED.get(type(node), type(node).__name__)
+        raise FormulaError(f"{what} is not allowed in a formula")
+
+    @staticmethod
+    def _constant(value):
+        if isinstance(value, bool):
+            return numpy.bool_(value)
+        if isinstance(value, (int, float)):
+            # float64, never a Python int: 9 ** 9 ** 9 overflows to inf instead of computing
+            return numpy.float64(value)
+        raise FormulaError(f"{value!r} is not a number: write column names without quotes, "
+                           f"or between backticks")
+
+    def _name(self, name: str):
+        if name in self.columns:
+            return self.df[self.columns[name]]
+        if name.startswith("__"):
+            raise FormulaError(f"expression contains disallowed tokens: {name!r}")
+        if name in _CONSTANTS:
+            return _CONSTANTS[name]
+        if name in _FUNCTIONS:
+            raise FormulaError(f"{name!r} is a function: call it as {_FUNCTIONS[name][3]}")
+        raise FormulaError(f"unknown column {name!r}: write a column by its name, or between backticks")
+
+    def _compare(self, node: ast.Compare):
+        left = self.visit(node.left)
+        result = None
+        for op, comparator in zip(node.ops, node.comparators):
+            compare = _COMPARE.get(type(op))
+            if compare is None:
+                raise FormulaError(f"comparison {_operator(op)} is not supported")
+            right = self.visit(comparator)
+            current = compare(left, right)
+            result = current if result is None else numpy.logical_and(result, current)
+            left = right
+        return result
+
+    def _call(self, node: ast.Call):
+        if not isinstance(node.func, ast.Name):
+            raise FormulaError(f"{_REFUSED.get(type(node.func), 'calling an expression')} is not allowed in a formula")
+        name = node.func.id
+        if name in self.columns:
+            raise FormulaError(f"column {self.names[name]!r} is not a function")
+        if name not in _FUNCTIONS:
+            raise FormulaError(f"unknown function {name!r}")
+        function, min_args, max_args, signature = _FUNCTIONS[name]
+        if node.keywords:
+            raise FormulaError(f"{name}: write the arguments without names, as {signature}")
+        count = len(node.args)
+        if count < min_args or (max_args is not None and count > max_args):
+            raise FormulaError(f"{name} takes {_arity(min_args, max_args)}, got {count}: {signature}")
+        return function(*[self.visit(argument) for argument in node.args])
+
+
+def _arity(min_args: int, max_args) -> str:
+    if max_args is None:
+        return f"at least {min_args} values"
+    if min_args == max_args:
+        return f"{min_args} value{'s' if min_args > 1 else ''}"
+    return f"{min_args} to {max_args} values"
+
+
+def _operator(op) -> str:
+    return {ast.FloorDiv: "'//'", ast.MatMult: "'@'", ast.LShift: "'<<'", ast.RShift: "'>>'",
+            ast.BitAnd: "'&'", ast.BitOr: "'|'", ast.BitXor: "'^'", ast.Invert: "'~'",
+            ast.In: "'in'", ast.NotIn: "'not in'", ast.Is: "'is'", ast.IsNot: "'is not'"}.get(type(op),
+                                                                                             type(op).__name__)

{wizata_dsapi-2.2.0.dev19 → wizata_dsapi-2.2.0.dev20}/wizata_dsapi/pipeline.py

@@ -13,6 +13,7 @@ from .template import Template, TemplateProperty
 from .twin import Twin
 from .twinregistration import TwinRegistration, TwinRegistrationProperty
 from .datapoint import DataPoint
+from .formula import normalize_formulas
 
 from enum import Enum
 
@@ -44,6 +45,8 @@ class StepType(Enum):
         - "write"
         - "plot"
         - "df_save"
+        - "alert"
+        - "formula": calculated columns on one dataframe (DEV-6112)
     """
     QUERY = 'query'
     SCRIPT = 'script'
@@ -52,6 +55,7 @@ class StepType(Enum):
     PLOT = 'plot'
     DF_SAVE = 'df_save'
     ALERT = 'alert'
+    FORMULA = 'formula'
 
 
 class AlertType(Enum):
@@ -120,6 +124,39 @@ class WriteConfig:
         return obj
 
 
+class FormulaConfig:
+    """
+    Configuration for a FORMULA pipeline step (DEV-6112).
+
+    Calculated columns ``[{"expression": ..., "result": ...}]``, computed in order on the
+    step's one input dataframe by the resolver behind the query option ``Request.formulas``
+    (:mod:`wizata_dsapi.formula`). "Open as pipeline" moves a query's list here unchanged,
+    so both give the same frame. The step outputs the input plus the new columns.
+    """
+
+    def __init__(self, formulas: list = None):
+        self.formulas = normalize_formulas(formulas)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        config = cls()
+        config.from_json(data)
+        return config
+
+    def from_json(self, obj: dict):
+        if not isinstance(obj, dict):
+            raise ValueError(f"a formula step config must be an object with 'formulas', got {type(obj).__name__}")
+        self.formulas = normalize_formulas(obj.get("formulas"))
+        if not self.formulas:
+            raise ValueError("a formula step needs at least one formula in 'formulas'")
+
+    def to_dict(self) -> dict:
+        return {"formulas": [dict(formula) for formula in self.formulas]}
+
+    def to_json(self, target: str = None) -> dict:
+        return self.to_dict()
+
+
 class AlertConfig:
     """
     Configuration for an ALERT pipeline step.
@@ -400,6 +437,9 @@ class PipelineStep(ApiDto):
             elif self.step_type == StepType.ALERT:
                 self.config = AlertConfig()
                 self.config.from_json(obj['config'])
+            elif self.step_type == StepType.FORMULA:
+                self.config = FormulaConfig()
+                self.config.from_json(obj['config'])
             else:
                 self.config = obj['config']
         self.inputs = []
@@ -433,6 +473,8 @@ class PipelineStep(ApiDto):
                 obj["config"] = self.config.to_json()
             elif self.step_type == StepType.ALERT:
                 obj["config"] = self.config.to_json()
+            elif self.step_type == StepType.FORMULA:
+                obj["config"] = self.config.to_json()
             else:
                 obj["config"] = self.config
 
@@ -662,6 +704,8 @@ class Pipeline(ApiDto):
             raise RuntimeError(f'write step must have exactly one input and no outputs')
         if step.step_type == StepType.ALERT and (len(step.inputs) != 1 or len(step.outputs) > 0):
             raise RuntimeError(f'alert step must have exactly one input and no outputs')
+        if step.step_type == StepType.FORMULA and (len(step.inputs) != 1 or len(step.outputs) != 1):
+            raise RuntimeError(f'formula step must have exactly one input and one output')
         for output in step.outputs_names():
             if output in produced:
                 raise RuntimeError(f'output {output} is already produced.')
@@ -852,6 +896,32 @@ class Pipeline(ApiDto):
 
         self.steps.append(step)
 
+    def add_formula(self,
+                    formulas,
+                    input_df,
+                    output_df):
+        """
+        Add a formula step: calculated columns on one dataframe (DEV-6112).
+
+        :param formulas: ``[{"expression": ..., "result": ...}]`` or a FormulaConfig. Every
+            result is named; see ``wizata_dsapi.formula_catalog()``.
+        :param input_df: str, dict or PipelineIO of the dataframe the formulas read.
+        :param output_df: str, dict or PipelineIO of the dataframe they produce: the input
+            plus one column per formula.
+        """
+        config = formulas if isinstance(formulas, FormulaConfig) else FormulaConfig(formulas=formulas)
+        if not config.formulas:
+            raise ValueError("a formula step needs at least one formula")
+        if input_df is None or output_df is None:
+            raise ValueError('please provide input_df and output_df (str, dict or PipelineIO)')
+
+        step = PipelineStep()
+        step.step_type = StepType.FORMULA
+        step.config = config
+        step.inputs.append(PipelineIO.from_obj(input_df))
+        step.outputs.append(PipelineIO.from_obj(output_df))
+        self.steps.append(step)
+
     def add_alert(self,
                   config: AlertConfig = None,
                   df_name: str = None,

{wizata_dsapi-2.2.0.dev19 → wizata_dsapi-2.2.0.dev20}/wizata_dsapi/request.py

@@ -3,6 +3,7 @@ from datetime import datetime, timedelta, timezone
 from .dataframe_toolkit import generate_epoch
 from .api_dto import ApiDto
 from .datapoint import BusinessType
+from .formula import normalize_formulas
 
 filter_map = {
     '>': 'gt',
@@ -168,6 +169,9 @@ class Request(ApiDto):
     :ivar list products: product keys to keep on inventoryMovement datapoints (all when None). With
         no datapoints, it selects the inventoryMovement datapoints holding them (under the twin
         when the request names one).
+    :ivar list formulas: calculated columns ``[{"expression": ..., "result": ...}]``, computed in
+        order after everything else in the query (DEV-6112). Each result is named, see
+        ``wizata_dsapi.formula_catalog()`` for the operators, functions and naming rule.
     """
 
     @classmethod
@@ -203,7 +207,8 @@ class Request(ApiDto):
                  bucket=None,
                  tags=None,
                  scopes=None,
-                 products=None):
+                 products=None,
+                 formulas=None):
         if request_id is None:
             request_id = uuid.uuid4()
         self.request_id = request_id
@@ -248,6 +253,7 @@ class Request(ApiDto):
         self.tags = tags
         self.scopes = scopes
         self.products = products
+        self.formulas = normalize_formulas(formulas) or None
 
         self.null = null
 
@@ -631,6 +637,8 @@ class Request(ApiDto):
             query['scopes'] = self.scopes
         if self.products is not None:
             query['products'] = self.products
+        if getattr(self, "formulas", None):                  # absent on a Request pickled before 12.2
+            query['formulas'] = [dict(formula) for formula in self.formulas]
 
         if self.target_feat is not None:
             query["target_feat"] = {
@@ -775,6 +783,9 @@ class Request(ApiDto):
                                  f"for every product), got {products!r}"[:300])
             self.products = products
 
+        if "formulas" in json_data.keys():
+            self.formulas = normalize_formulas(json_data["formulas"]) or None
+
         if "connections" in json_data.keys():
             self.connections = json_data["connections"]
 

{wizata_dsapi-2.2.0.dev19 → wizata_dsapi-2.2.0.dev20}/wizata_dsapi/scripts/common.py

@@ -1,6 +1,5 @@
-import re
-
 import wizata_dsapi
+from ..formula import apply_formulas
 
 import pandas
 import numpy
@@ -13,19 +12,6 @@ import sklearn.preprocessing
 import sklearn.decomposition
 
 
-_FORMULA_SAFE_FUNCTIONS = {
-    "abs": numpy.abs,
-    "sqrt": numpy.sqrt,
-    "log": numpy.log,
-    "log10": numpy.log10,
-    "exp": numpy.exp,
-    "clip": numpy.clip,
-    "round": numpy.round,
-    "min": numpy.minimum,
-    "max": numpy.maximum,
-}
-
-
 def filter_df(context: wizata_dsapi.Context):
     """Filter dataframe rows using pandas query strings from the 'filters' property list."""
 
@@ -313,48 +299,32 @@ def setpoint_deviation(context: wizata_dsapi.Context):
     return df
 
 
-def formula(context: wizata_dsapi.Context):
-    """Add a new column computed from a user-defined math expression over existing columns. Property 'expression' is required — intuitive math syntax referencing column names (e.g. 'temp_1 + temp_2', '(p_in - p_out) / p_in * 100', 'sqrt(vibration_x**2 + vibration_y**2)', 'clip(temperature, 0, 500)', 'log(power + 1)'). Supports arithmetic operators (+, -, *, /, %, **) and these functions: abs, sqrt, log, log10, exp, clip, round, min, max. Column names with spaces must be wrapped in backticks (e.g. '`motor temp` * 2'). Property 'result' (default 'result') names the output column."""
-    df = context.dataframe
+def _step_properties(context: wizata_dsapi.Context) -> dict:
+    """The properties of the step being run. ``context.properties`` also holds those of every
+    step before it (``map_context`` never takes them back), so a second formula step without
+    ``result`` used to reuse the first step's (DEV-6112). Outside a pipeline step (a unit
+    test, a direct call), the context's own properties."""
+    step = getattr(context, "step", None)
+    config = getattr(step, "config", None)
+    if isinstance(config, wizata_dsapi.ScriptConfig) and config.properties is not None:
+        return config.properties
+    return context.properties or {}
 
-    if "expression" not in context.properties:
-        raise KeyError(f"please set an 'expression' property (e.g. 'col_a + col_b')")
-
-    expression = context.properties["expression"]
-    result = context.properties.get("result", "result")
-
-    # Replace every column name (backticked or bare) with a safe placeholder
-    # identifier before evaluation. Column names can contain characters that
-    # are illegal in a Python/pandas eval expression — e.g. tag IDs like
-    # 'DE_AL_Ark1_1:04800F1?Value', whose numeric segment '04800' is otherwise
-    # parsed as a leading-zero integer literal and raises a SyntaxError
-    # (DEV-5912). Substituting the whole name atomically sidesteps this.
-    placeholders = {}  # placeholder -> real column name
-    substituted = expression
-    for i, col in enumerate(sorted(df.columns, key=len, reverse=True)):
-        placeholder = f"__wz_col_{i}__"
-        substituted = substituted.replace(f"`{col}`", placeholder).replace(col, placeholder)
-        placeholders[placeholder] = col
-
-    # Whitelist validation: strip placeholders + allowed functions, flag any
-    # leftover token as disallowed.
-    sanitized = substituted
-    for placeholder in placeholders:
-        sanitized = sanitized.replace(placeholder, "")
-    for fn in _FORMULA_SAFE_FUNCTIONS:
-        sanitized = sanitized.replace(fn, "")
-    sanitized = re.sub(r"[\d\.\+\-\*/\%\(\)\s,\^]", "", sanitized)
-    if sanitized.strip():
-        raise ValueError(
-            f"expression contains disallowed tokens: '{sanitized.strip()}' — "
-            f"only column names, numbers, arithmetic operators, and these functions are allowed: "
-            f"{list(_FORMULA_SAFE_FUNCTIONS.keys())}"
-        )
-
-    # Evaluate against a frame whose columns are the safe placeholder names.
-    safe_df = df.rename(columns={real: ph for ph, real in placeholders.items()})
-    df[result] = safe_df.eval(substituted, local_dict=_FORMULA_SAFE_FUNCTIONS, engine="python")
-    return df
+
+def formula(context: wizata_dsapi.Context):
+    """Legacy: add one column computed from a math expression over existing columns. New pipelines use a formula step (step type 'formula', DEV-6112), which computes a named list of formulas with the same resolver as the query option Request.formulas. Property 'expression' is required; 'result' (default 'result') names the output column and may replace a column. Expressions use column names (between backticks when they hold spaces), numbers, + - * / % ** (or ^), comparisons < <= > >= == !=, and / or / not (or & | ~), the functions abs, sqrt, log, log10, exp, round, clip, min, max, where, and the constants pi and e."""
+    properties = _step_properties(context)
+
+    if "formulas" in properties:
+        raise ValueError("'formulas' belongs to a formula step (step type 'formula'); "
+                         "this script takes 'expression' and 'result'")
+    if "expression" not in properties:
+        raise KeyError("please set an 'expression' property (e.g. 'col_a + col_b'), "
+                       "or use a formula step (step type 'formula')")
+
+    return apply_formulas(context.dataframe,
+                          [{"expression": properties["expression"], "result": properties.get("result", "result")}],
+                          strict=False)
 
 
 def target_feat_to_binary(context: wizata_dsapi.Context):

wizata_dsapi-2.2.0.dev20/wizata_dsapi/version.py

@@ -0,0 +1 @@
+__version__ = "2.2.0.dev20"

{wizata_dsapi-2.2.0.dev19 → wizata_dsapi-2.2.0.dev20}/wizata_dsapi/wizata_dsapi_client.py

@@ -28,6 +28,7 @@ from .api_dto import ApiDto, VarType, ApiDtoInterface
 from .business_label import BusinessLabel
 from .plot import Plot
 from .request import Request
+from .formula import normalize_formulas
 from .mlmodel import ModelInfo, ModelList, ModelFile
 from .experiment import Experiment
 from .script import Script
@@ -779,7 +780,8 @@ class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
                        bucket: str = None,
                        tags: dict = None,
                        scopes: list = None,
-                       products: list = None) -> Request:
+                       products: list = None,
+                       formulas: list = None) -> Request:
         """
         Build the Request behind a time-series query.
 
@@ -825,6 +827,9 @@ class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
         if products is not None:
             request.products = products
 
+        if formulas is not None:
+            request.formulas = normalize_formulas(formulas) or None
+
         if options is not None:
             request.options = options
 
@@ -857,7 +862,8 @@ class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
               bucket: str = None,
               tags: dict = None,
               scopes: list = None,
-              products: list = None) -> pandas.DataFrame:
+              products: list = None,
+              formulas: list = None) -> pandas.DataFrame:
         """
         Query a dataframe from API.
         :param agg_method:
@@ -878,13 +884,15 @@ class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
             ``options={"inventory": "movements" | "balances" | "stock"}`` chooses what they return.
             With no ``datapoints``, the inventoryMovement datapoints holding those products are
             selected for you: every storage, or those under ``twin`` and its children (DEV-6089).
+        :param formulas: calculated columns ``[{"expression": "a - b", "result": "gap"}]``, computed
+            after the query, in order (DEV-6112). See ``wizata_dsapi.formula_catalog()``.
         :return: dataframe
         """
         return self._query(request=self._build_request(
             datapoints=datapoints, start=start, end=end, interval=interval,
             agg_method=agg_method, template=template, twin=twin, null=null,
             filters=filters, options=options, group=group, field=field,
-            bucket=bucket, tags=tags, scopes=scopes, products=products))
+            bucket=bucket, tags=tags, scopes=scopes, products=products, formulas=formulas))
 
     def batch_query(self,
                     step: timedelta,

{wizata_dsapi-2.2.0.dev19 → wizata_dsapi-2.2.0.dev20}/wizata_dsapi.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wizata_dsapi
-Version: 2.2.0.dev19
+Version: 2.2.0.dev20
 Summary: Wizata Data Science Toolkit
 Author: Wizata S.A.
 Author-email: info@wizata.com

{wizata_dsapi-2.2.0.dev19 → wizata_dsapi-2.2.0.dev20}/wizata_dsapi.egg-info/SOURCES.txt

@@ -25,6 +25,7 @@ wizata_dsapi/evaluation.py
 wizata_dsapi/execution.py
 wizata_dsapi/execution_log.py
 wizata_dsapi/experiment.py
+wizata_dsapi/formula.py
 wizata_dsapi/graylog_log.py
 wizata_dsapi/group_system.py
 wizata_dsapi/ilogger.py

wizata_dsapi-2.2.0.dev19/wizata_dsapi/version.py

@@ -1 +0,0 @@
-__version__ = "2.2.0.dev19"