wizata-dsapi 2.2.0.dev18__py3-none-any.whl → 2.2.0.dev20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
wizata_dsapi/__init__.py CHANGED
@@ -5,6 +5,8 @@ from .paged_query_result import PagedQueryResult
5
5
  from .plot import Plot
6
6
  from .mlmodel import ModelInfo, ModelList, MLModelConfig, ModelFile, ModelIdentifierInfo
7
7
  from .request import Request, filter_map, RequestGroup, RequestGroupMap, DynamicSelector
8
+ from .formula import (FormulaError, apply_formulas, evaluate_formula, normalize_formulas, check_result_name,
9
+ formula_catalog, FORMULA_OPERATORS, FORMULA_FUNCTIONS, FORMULA_CONSTANTS)
8
10
  from .execution import Execution, ExecutionStatus, ExecutionStepLog, AbortedException
9
11
  from .execution_log import ExecutionLog
10
12
  from .experiment import Experiment
@@ -45,7 +47,8 @@ from .wizard_function import WizardStep, WizardFunction
45
47
  from .wizard_request import WizardRequest
46
48
 
47
49
  # Pipeline Entities (Dto)
48
- from .pipeline import Pipeline, PipelineStep, StepType, WriteConfig, AlertConfig, PipelineIO, RunnerStatus, AlertType
50
+ from .pipeline import Pipeline, PipelineStep, StepType, WriteConfig, AlertConfig, FormulaConfig, PipelineIO, RunnerStatus, \
51
+ AlertType
49
52
  from .context import Context
50
53
  from .ilogger import ILogger
51
54
  from .pipeline_image import PipelineImage
@@ -61,6 +64,7 @@ from .notification import NotificationScope, DeviceToken, AlertRule, AlertActive
61
64
  from .edge_device import EdgeDevice, EdgeDeviceStatus, EdgeDeviceType, EdgeDeviceProvisioning
62
65
  from .edge_module import EdgeModule, EdgeModuleType, WIZATA_MODULES
63
66
  from .edge_config import EdgeConsumer, EdgeWriter, EdgeTrigger, EdgeFile, EdgeConfig, EdgeCommand
67
+ from .edge_package import EdgePackage, EdgePackageOperation, EdgeRelease
64
68
  from .connection import Connection, ConnectionMapping, ConnectionDirection, TransformKind, \
65
69
  EdgeDeviceKey, EdgeDeviceKeyStatus
66
70
  from .connection_types import ConnectionType, CONNECTION_TYPES, TRANSFORM_SCHEMAS, get_connection_type, \
@@ -59,6 +59,7 @@ class EdgeDevice(ApiDto):
59
59
  :ivar bool managed: the platform keeps this device's manifest ("deploy modules").
60
60
  :ivar dict applied_modules: what was last applied: module -> {image, env names}.
61
61
  :ivar str reconcile_error: why the last apply failed, until one succeeds.
62
+ :ivar int last_seen_at: last heartbeat of a manual edge, epoch ms (DEV-6109).
62
63
  """
63
64
 
64
65
  @classmethod
@@ -96,7 +97,8 @@ class EdgeDevice(ApiDto):
96
97
  runner_python_version: str = None,
97
98
  managed: bool = None,
98
99
  applied_modules: dict = None,
99
- reconcile_error: str = None):
100
+ reconcile_error: str = None,
101
+ last_seen_at: int = None):
100
102
  self.device_id = device_id
101
103
  self.name = name
102
104
  self.twin_name = twin_name
@@ -117,6 +119,7 @@ class EdgeDevice(ApiDto):
117
119
  self.managed = managed
118
120
  self.applied_modules = applied_modules
119
121
  self.reconcile_error = reconcile_error
122
+ self.last_seen_at = last_seen_at
120
123
 
121
124
  def api_id(self) -> str:
122
125
  return self.device_id
@@ -173,6 +176,8 @@ class EdgeDevice(ApiDto):
173
176
  self.applied_modules = obj["appliedModules"]
174
177
  if "reconcileError" in obj and obj["reconcileError"] is not None:
175
178
  self.reconcile_error = obj["reconcileError"]
179
+ if "lastSeenAt" in obj and obj["lastSeenAt"] is not None:
180
+ self.last_seen_at = int(obj["lastSeenAt"])
176
181
 
177
182
  def to_json(self, target: str = None):
178
183
  obj = {}
@@ -215,4 +220,6 @@ class EdgeDevice(ApiDto):
215
220
  obj["appliedModules"] = self.applied_modules
216
221
  if self.reconcile_error is not None:
217
222
  obj["reconcileError"] = self.reconcile_error
223
+ if self.last_seen_at is not None:
224
+ obj["lastSeenAt"] = self.last_seen_at
218
225
  return obj
@@ -0,0 +1,128 @@
1
+ from enum import Enum
2
+
3
+ from .api_dto import Dto
4
+
5
+
6
+ class EdgePackageOperation(Enum):
7
+ """
8
+ What a package for a manual edge does (DEV-6109). A manual edge serves a validated
9
+ (21 CFR Part 11) site: the platform never changes it, the site's operator runs each
10
+ package as a procedure.
11
+
12
+ - ``pipeline``: the pipelines it runs — triggers.json and the pipeline images;
13
+ - ``platform``: the platform version — docker-compose.yml, the module settings and
14
+ the profile files of a release;
15
+ - ``full``: everything, for the first install or any other change (connections,
16
+ writers, functions).
17
+ """
18
+ PIPELINE = "pipeline"
19
+ PLATFORM = "platform"
20
+ FULL = "full"
21
+
22
+
23
+ class EdgeRelease(Dto):
24
+ """
25
+ The edge images of a release, as Wizata.Jobs publishes them (DEV-6109).
26
+
27
+ ``edge_images.json`` in Wizata.Jobs stays where they are pinned. The Jobs consumer
28
+ writes them here at its start, so the DS API can compose a manual edge's platform
29
+ package with the images the environment runs.
30
+
31
+ :ivar str id: digest of the images: the same images are the same release.
32
+ :ivar str version: what published them, e.g. ``wizata-packages 2.2.0``.
33
+ :ivar dict images: module -> image; ``pipeline_runner`` maps a Python version to one.
34
+ :ivar int published_at: when last published, epoch ms.
35
+ """
36
+
37
+ def __init__(self, id=None, version=None, images=None, published_at=None):
38
+ self.id = id
39
+ self.version = version
40
+ self.images = images
41
+ self.published_at = published_at
42
+
43
+ @classmethod
44
+ def from_dict(cls, data: dict) -> "EdgeRelease":
45
+ return cls(id=data.get("id"), version=data.get("version"), images=data.get("images"),
46
+ published_at=data.get("publishedAt"))
47
+
48
+ def to_dict(self) -> dict:
49
+ return {"id": self.id, "version": self.version, "images": self.images,
50
+ "publishedAt": self.published_at}
51
+
52
+
53
+ class EdgePackage(Dto):
54
+ """
55
+ A procedure composed for a manual edge, and its record (DEV-6109): what it contains,
56
+ who generated and downloaded it, and when the edge reported it applied.
57
+
58
+ The package itself holds secrets (the module settings) and is never stored: it is
59
+ rendered again at each download, and refused if its contents changed since it was
60
+ generated. The record holds names and digests only.
61
+
62
+ :ivar str id: package id (UUID). The edge reports it once installed.
63
+ :ivar str edge_device_id: the device it is for.
64
+ :ivar EdgePackageOperation operation: pipeline, platform or full.
65
+ :ivar int sequence: 1, 2, 3... per device.
66
+ :ivar str release_id: the ``EdgeRelease`` its module images come from (platform, full).
67
+ :ivar str contents_hash: digest of the contents, secrets excluded.
68
+ :ivar dict contents: ``files`` (path -> sha256), ``settings`` (module -> variable
69
+ names), ``modules`` (module -> image), ``images`` (pipeline image ids),
70
+ ``warnings``.
71
+ :ivar dict diff: against what the edge runs (its applied packages): added, changed
72
+ and removed files, pipeline images, module images.
73
+ :ivar list procedure: the steps, each ``{"title", "command"}``.
74
+ :ivar str generated_by: who generated it.
75
+ :ivar int generated_at: epoch ms.
76
+ :ivar list downloads: each ``{"by", "at"}``.
77
+ :ivar int applied_at: when it was first reported installed, epoch ms.
78
+ :ivar str applied_by: ``edge`` (its heartbeat), or the admin who confirmed it for an
79
+ edge that cannot reach the platform.
80
+ """
81
+
82
+ def __init__(self, id=None, edge_device_id=None, operation=None, sequence=None,
83
+ release_id=None, contents_hash=None, contents=None, diff=None, procedure=None,
84
+ generated_by=None, generated_at=None, downloads=None, applied_at=None,
85
+ applied_by=None):
86
+ self.id = id
87
+ self.edge_device_id = edge_device_id
88
+ self.operation = operation
89
+ self.sequence = sequence
90
+ self.release_id = release_id
91
+ self.contents_hash = contents_hash
92
+ self.contents = contents
93
+ self.diff = diff
94
+ self.procedure = procedure
95
+ self.generated_by = generated_by
96
+ self.generated_at = generated_at
97
+ self.downloads = downloads
98
+ self.applied_at = applied_at
99
+ self.applied_by = applied_by
100
+
101
+ @classmethod
102
+ def from_dict(cls, data: dict) -> "EdgePackage":
103
+ operation = data.get("operation")
104
+ return cls(id=data.get("id"), edge_device_id=data.get("edgeDeviceId"),
105
+ operation=EdgePackageOperation(operation) if operation else None,
106
+ sequence=data.get("sequence"), release_id=data.get("releaseId"),
107
+ contents_hash=data.get("contentsHash"), contents=data.get("contents"),
108
+ diff=data.get("diff"), procedure=data.get("procedure"),
109
+ generated_by=data.get("generatedBy"), generated_at=data.get("generatedAt"),
110
+ downloads=data.get("downloads"), applied_at=data.get("appliedAt"),
111
+ applied_by=data.get("appliedBy"))
112
+
113
+ def to_dict(self) -> dict:
114
+ return {"id": self.id, "edgeDeviceId": self.edge_device_id,
115
+ "operation": self.operation.value if self.operation else None,
116
+ "sequence": self.sequence, "releaseId": self.release_id,
117
+ "contentsHash": self.contents_hash, "contents": self.contents,
118
+ "diff": self.diff, "procedure": self.procedure,
119
+ "generatedBy": self.generated_by, "generatedAt": self.generated_at,
120
+ "downloads": self.downloads or [], "appliedAt": self.applied_at,
121
+ "appliedBy": self.applied_by}
122
+
123
+ def marker(self) -> dict:
124
+ """What the package installs as ``conf/_packages/<id>.json`` and the edge reports."""
125
+ return {"id": self.id, "edgeDeviceId": self.edge_device_id,
126
+ "operation": self.operation.value if self.operation else None,
127
+ "sequence": self.sequence, "releaseId": self.release_id,
128
+ "contentsHash": self.contents_hash}
@@ -0,0 +1,493 @@
1
+ """
2
+ Calculated columns (DEV-6112).
3
+
4
+ One resolver for the two places a formula runs, so they cannot disagree:
5
+
6
+ - the query option ``Request.formulas``, applied by ``QueryClient.execute`` after
7
+ everything else in the query, so the Explorer gets the results in the same
8
+ ``/dsapi/data`` call;
9
+ - the ``wizata.scripts.formula`` pipeline step, where "Open as pipeline" moves that
10
+ list unchanged.
11
+
12
+ An expression is parsed, never handed to ``eval``. Column names are swapped for
13
+ placeholders, ``^ & | ~`` for ``** and or not``, the text is parsed with :mod:`ast`,
14
+ and only numbers, columns, the operators, the functions and the constants of the
15
+ catalog are evaluated. Anything else (an attribute, an index, a string, a lambda) is
16
+ refused: since the query option, expressions run inside the DS API process.
17
+
18
+ :func:`formula_catalog` is what the front-end reads (DS API ``GET /dsapi/formulas/``).
19
+ """
20
+ import ast
21
+ import functools
22
+ import math
23
+ import re
24
+
25
+ import numpy
26
+ import pandas
27
+
28
+
29
+ MAX_FORMULAS = 50
30
+ MAX_EXPRESSION_LENGTH = 2000
31
+ MAX_FORMULA_NODES = 500
32
+
33
+ # A result is named by the user, like a datapoint: 2 to 128 characters, no whitespace.
34
+ # '/' and '|' are refused too: a result column named "<dp>/<field>" or "<dp>|<scope>"
35
+ # would read as a datapoint field or a split datapoint in the query result format.
36
+ RESULT_NAME_MIN_LENGTH = 2
37
+ RESULT_NAME_MAX_LENGTH = 128
38
+ RESULT_NAME_FORBIDDEN = " /|`"
39
+ FORMULA_RESULT_PATTERN = re.compile(r"^[^\s/|`]{%d,%d}$" % (RESULT_NAME_MIN_LENGTH, RESULT_NAME_MAX_LENGTH))
40
+
41
+
42
+ class FormulaError(ValueError):
43
+ """An expression or a formula list the resolver refuses (a 400 on the DS API)."""
44
+
45
+
46
+ # ── catalog ──────────────────────────────────────────────────────────────────
47
+ # `sign` is what the front-end inserts, `word` how it reads, `accepts` every spelling
48
+ # the resolver takes for it. Each `example` is evaluated by the tests against columns
49
+ # a, b and c, so the catalog cannot list what the resolver refuses.
50
+
51
+ FORMULA_OPERATORS = [
52
+ {"key": "add", "category": "arithmetic", "sign": "+", "word": "plus", "accepts": ["+"],
53
+ "description": "Adds the right value to the left one.",
54
+ "example": "a + b"},
55
+ {"key": "subtract", "category": "arithmetic", "sign": "-", "word": "minus", "accepts": ["-"],
56
+ "description": "Subtracts the right value from the left one. In front of a single value, negates it.",
57
+ "example": "a - b"},
58
+ {"key": "multiply", "category": "arithmetic", "sign": "*", "word": "times", "accepts": ["*"],
59
+ "description": "Multiplies the left value by the right one.",
60
+ "example": "a * b"},
61
+ {"key": "divide", "category": "arithmetic", "sign": "/", "word": "divided by", "accepts": ["/"],
62
+ "description": "Divides the left value by the right one. Dividing by zero gives an infinite value, "
63
+ "zero by zero an empty one.",
64
+ "example": "a / b"},
65
+ {"key": "modulo", "category": "arithmetic", "sign": "%", "word": "modulo", "accepts": ["%"],
66
+ "description": "Remainder of the left value divided by the right one.",
67
+ "example": "a % 2"},
68
+ {"key": "power", "category": "arithmetic", "sign": "**", "word": "to the power of", "accepts": ["**", "^"],
69
+ "description": "Raises the left value to the power of the right one; ^ means the same. It binds tighter "
70
+ "than the other operators: a + b ^ 2 squares b only.",
71
+ "example": "a ^ 2"},
72
+ {"key": "less", "category": "comparison", "sign": "<", "word": "less than", "accepts": ["<"],
73
+ "description": "True where the left value is less than the right one.",
74
+ "example": "a < b"},
75
+ {"key": "less_equal", "category": "comparison", "sign": "<=", "word": "less than or equal to", "accepts": ["<="],
76
+ "description": "True where the left value is less than or equal to the right one.",
77
+ "example": "a <= b"},
78
+ {"key": "greater", "category": "comparison", "sign": ">", "word": "greater than", "accepts": [">"],
79
+ "description": "True where the left value is greater than the right one.",
80
+ "example": "a > b"},
81
+ {"key": "greater_equal", "category": "comparison", "sign": ">=", "word": "greater than or equal to",
82
+ "accepts": [">="],
83
+ "description": "True where the left value is greater than or equal to the right one.",
84
+ "example": "a >= b"},
85
+ {"key": "equal", "category": "comparison", "sign": "==", "word": "equal to", "accepts": ["=="],
86
+ "description": "True where both values are equal.",
87
+ "example": "a == b"},
88
+ {"key": "not_equal", "category": "comparison", "sign": "!=", "word": "not equal to", "accepts": ["!="],
89
+ "description": "True where the values differ.",
90
+ "example": "a != b"},
91
+ {"key": "and", "category": "logical", "sign": "&", "word": "and", "accepts": ["&", "and"],
92
+ "description": "True where both conditions are true. Comparisons are evaluated first, so "
93
+ "a > 1 & b < 2 needs no brackets.",
94
+ "example": "a > 1 and b < 2"},
95
+ {"key": "or", "category": "logical", "sign": "|", "word": "or", "accepts": ["|", "or"],
96
+ "description": "True where at least one of the conditions is true. Comparisons are evaluated first.",
97
+ "example": "a > 1 or b < 2"},
98
+ {"key": "not", "category": "logical", "sign": "~", "word": "not", "accepts": ["~", "not"],
99
+ "description": "True where the condition is false. It applies to the whole comparison after it: "
100
+ "not a > 1.",
101
+ "example": "not a > 1"},
102
+ ]
103
+
104
+
105
+ def _round(x, decimals=0):
106
+ if not isinstance(decimals, (int, float, numpy.number)) or isinstance(decimals, (bool, numpy.bool_)):
107
+ raise FormulaError("round: the number of decimals must be a number, e.g. round(x, 2)")
108
+ return numpy.round(x, int(decimals))
109
+
110
+
111
+ _FUNCTION_DEFINITIONS = [
112
+ ("abs", "abs(x)", 1, 1, numpy.abs, "Absolute value.", "abs(a - b)"),
113
+ ("sqrt", "sqrt(x)", 1, 1, numpy.sqrt, "Square root; empty for a negative value.", "sqrt(a)"),
114
+ ("log", "log(x)", 1, 1, numpy.log, "Natural logarithm; empty for a negative value.", "log(a)"),
115
+ ("log10", "log10(x)", 1, 1, numpy.log10, "Base-10 logarithm; empty for a negative value.", "log10(a)"),
116
+ ("exp", "exp(x)", 1, 1, numpy.exp, "e raised to the value.", "exp(a)"),
117
+ ("round", "round(x, decimals)", 1, 2, _round,
118
+ "Rounds to the given number of decimals, 0 when left out. The decimals must be a number.", "round(a, 1)"),
119
+ ("clip", "clip(x, low, high)", 3, 3, numpy.clip, "Limits each value to the range low to high.",
120
+ "clip(a, 0, 3)"),
121
+ ("min", "min(x, y, ...)", 2, None, lambda *values: functools.reduce(numpy.minimum, values),
122
+ "The smallest of the values, row by row.", "min(a, b, c)"),
123
+ ("max", "max(x, y, ...)", 2, None, lambda *values: functools.reduce(numpy.maximum, values),
124
+ "The largest of the values, row by row.", "max(a, b, c)"),
125
+ ("where", "where(condition, if_true, if_false)", 3, 3, numpy.where,
126
+ "if_true where the condition is true, else if_false.", "where(a > b, a, b)"),
127
+ ]
128
+
129
+ FORMULA_FUNCTIONS = [
130
+ {"name": name, "signature": signature, "min_args": min_args, "max_args": max_args,
131
+ "description": description, "example": example}
132
+ for name, signature, min_args, max_args, _, description, example in _FUNCTION_DEFINITIONS
133
+ ]
134
+
135
+ FORMULA_CONSTANTS = [
136
+ {"name": "pi", "value": math.pi, "description": "The ratio of a circle's circumference to its diameter."},
137
+ {"name": "e", "value": math.e, "description": "Euler's number, the base of the natural logarithm."},
138
+ ]
139
+
140
+ _FUNCTIONS = {name: (function, min_args, max_args, signature)
141
+ for name, signature, min_args, max_args, function, _, _ in _FUNCTION_DEFINITIONS}
142
+ _CONSTANTS = {constant["name"]: numpy.float64(constant["value"]) for constant in FORMULA_CONSTANTS}
143
+ _OPERATOR_WORDS = {"and", "or", "not"}
144
+ # Names a result may not take: a bare reference to it would read as something else.
145
+ _KEYWORDS = set(_FUNCTIONS) | set(_CONSTANTS) | _OPERATOR_WORDS | {"True", "False"}
146
+
147
+
148
+ def formula_catalog() -> dict:
149
+ """What the front-end needs to build and check a formula: the operators (sign, word,
150
+ description), the functions, the constants and the result naming rule."""
151
+ return {
152
+ "operators": [dict(operator, accepts=list(operator["accepts"])) for operator in FORMULA_OPERATORS],
153
+ "functions": [dict(function) for function in FORMULA_FUNCTIONS],
154
+ "constants": [dict(constant) for constant in FORMULA_CONSTANTS],
155
+ "result_name": {
156
+ "min_length": RESULT_NAME_MIN_LENGTH,
157
+ "max_length": RESULT_NAME_MAX_LENGTH,
158
+ "forbidden": ["whitespace", "/", "|", "`"],
159
+ "pattern": FORMULA_RESULT_PATTERN.pattern,
160
+ "reserved": sorted(_KEYWORDS),
161
+ },
162
+ "column_reference": "A column is written by its name: mt1_bearing2 - mt1_bearing1. Backticks are only "
163
+ "needed for a name holding a space, which a datapoint never does.",
164
+ "limits": {"formulas": MAX_FORMULAS, "expression_length": MAX_EXPRESSION_LENGTH},
165
+ }
166
+
167
+
168
+ # ── formula lists ────────────────────────────────────────────────────────────
169
+
170
+ def check_result_name(name, position: int = None):
171
+ """Refuse a result name that is missing or breaks the datapoint naming convention."""
172
+ where = f"formula {position}: " if position is not None else ""
173
+ if name is None or (isinstance(name, str) and not name.strip()):
174
+ raise FormulaError(f"{where}a formula needs a 'result' name for its column")
175
+ if not isinstance(name, str) or not FORMULA_RESULT_PATTERN.match(name):
176
+ raise FormulaError(f"{where}result name {name!r} must be {RESULT_NAME_MIN_LENGTH} to "
177
+ f"{RESULT_NAME_MAX_LENGTH} characters, without spaces, '/', '|' or '`' "
178
+ f"(the datapoint naming convention)")
179
+ if name in _KEYWORDS:
180
+ raise FormulaError(f"{where}{name!r} is a formula keyword and cannot name a result")
181
+
182
+
183
+ def normalize_formulas(formulas) -> list:
184
+ """Check a ``[{expression, result}]`` list and return it with only those two keys.
185
+
186
+ None is no formula. Every formula names its result (JP, 2026-10-09: the front-end may
187
+ generate the name, the API never does), names are unique, and the list holds at most
188
+ MAX_FORMULAS formulas.
189
+ """
190
+ if formulas is None:
191
+ return []
192
+ if not isinstance(formulas, list):
193
+ raise FormulaError(f"'formulas' must be a list of {{'expression', 'result'}}, got {type(formulas).__name__}")
194
+ if len(formulas) > MAX_FORMULAS:
195
+ raise FormulaError(f"'formulas' holds {len(formulas)} formulas, at most {MAX_FORMULAS} are allowed")
196
+ normalized = []
197
+ seen = set()
198
+ for position, item in enumerate(formulas, start=1):
199
+ if not isinstance(item, dict):
200
+ raise FormulaError(f"formula {position} must be an object with 'expression' and 'result'")
201
+ expression = item.get("expression")
202
+ if not isinstance(expression, str) or not expression.strip():
203
+ raise FormulaError(f"formula {position}: 'expression' is required")
204
+ _check_length(expression)
205
+ result = item.get("result")
206
+ check_result_name(result, position)
207
+ if result in seen:
208
+ raise FormulaError(f"formula {position}: result {result!r} is used twice")
209
+ seen.add(result)
210
+ normalized.append({"expression": expression, "result": result})
211
+ return normalized
212
+
213
+
214
+ def apply_formulas(df: pandas.DataFrame, formulas: list, strict: bool = True) -> pandas.DataFrame:
215
+ """A copy of *df* with one column per formula, computed in order, so a formula can use
216
+ an earlier result.
217
+
218
+ :param strict: the rules of a ``formulas`` list (named, conventional and new results).
219
+ False is for the legacy ``expression`` / ``result`` pair of existing pipelines,
220
+ whose result may be any name, an existing column included.
221
+ """
222
+ if df is None:
223
+ raise FormulaError("there is no dataframe to compute the formulas on")
224
+ if strict:
225
+ formulas = normalize_formulas(formulas)
226
+ out = df.copy(deep=False)
227
+ for formula in formulas:
228
+ result = formula["result"]
229
+ if strict and result in _column_names(out):
230
+ raise FormulaError(f"formula {result!r}: {result!r} is already a column, choose another name")
231
+ if len(out.index) == 0:
232
+ values = pandas.Series(dtype="float64", index=out.index)
233
+ else:
234
+ try:
235
+ values = evaluate_formula(out, formula["expression"])
236
+ except FormulaError as e:
237
+ raise FormulaError(f"formula {result!r}: {e}") from e
238
+ out[_column_key(out, result)] = values
239
+ return out
240
+
241
+
242
+ # ── one expression ───────────────────────────────────────────────────────────
243
+
244
+ def evaluate_formula(df: pandas.DataFrame, expression: str) -> pandas.Series:
245
+ """The values of *expression* for each row of *df*, as a Series on its index."""
246
+ if not isinstance(expression, str) or not expression.strip():
247
+ raise FormulaError("the expression is empty")
248
+ _check_length(expression)
249
+ names = _column_names(df)
250
+ placeholders = {name: f"__wz_col_{i}__" for i, name in enumerate(names)}
251
+ text = _substitute(expression, placeholders)
252
+ for sign, word in (("^", "**"), ("&", " and "), ("|", " or "), ("~", " not ")):
253
+ text = text.replace(sign, word)
254
+ try:
255
+ tree = ast.parse(text.strip(), mode="eval")
256
+ except SyntaxError as e:
257
+ hint = " (compare with ==)" if re.search(r"(?<![<>=!])=(?!=)", expression) else ""
258
+ raise FormulaError(f"invalid expression {expression!r}: {e.msg}{hint}") from None
259
+ if sum(1 for _ in ast.walk(tree)) > MAX_FORMULA_NODES:
260
+ raise FormulaError(f"the expression is too complex (more than {MAX_FORMULA_NODES} elements)")
261
+ columns = {placeholder: names[name] for name, placeholder in placeholders.items()}
262
+ evaluator = _Evaluator(df, columns, {placeholder: name for name, placeholder in placeholders.items()})
263
+ with numpy.errstate(all="ignore"):
264
+ try:
265
+ value = evaluator.visit(tree)
266
+ except FormulaError:
267
+ raise
268
+ except Exception as e:
269
+ raise FormulaError(f"cannot compute {expression!r}: {e}") from None
270
+ return _as_series(value, df.index)
271
+
272
+
273
+ def _check_length(expression: str):
274
+ if len(expression) > MAX_EXPRESSION_LENGTH:
275
+ raise FormulaError(f"the expression has {len(expression)} characters, "
276
+ f"at most {MAX_EXPRESSION_LENGTH} are allowed")
277
+
278
+
279
+ def _column_names(df: pandas.DataFrame) -> dict:
280
+ """``{name: column key}``. A MultiIndex column is named ``<dp>/<field>``, as the query
281
+ result format names it."""
282
+ names = {}
283
+ for column in df.columns:
284
+ if isinstance(column, tuple):
285
+ name = "/".join(str(part) for part in column if part is not None and part != "")
286
+ else:
287
+ name = str(column)
288
+ names.setdefault(name, column)
289
+ return names
290
+
291
+
292
+ def _column_key(df: pandas.DataFrame, name: str):
293
+ if isinstance(df.columns, pandas.MultiIndex):
294
+ return (name,) + ("",) * (df.columns.nlevels - 1)
295
+ return name
296
+
297
+
298
+ _IDENTIFIER = re.compile(r"[A-Za-z0-9_]")
299
+ _NUMBER = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$")
300
+
301
+
302
+ def _substitute(expression: str, placeholders: dict) -> str:
303
+ """Swap column names for their placeholders: exactly between backticks, and as whole
304
+ names elsewhere, longest first."""
305
+ parts = expression.split("`")
306
+ if len(parts) % 2 == 0:
307
+ raise FormulaError("a backtick is not closed: write `column name`")
308
+ bare = sorted((name for name in placeholders
309
+ if name and name not in _OPERATOR_WORDS and not _NUMBER.match(name)),
310
+ key=len, reverse=True)
311
+ out = []
312
+ for position, part in enumerate(parts):
313
+ if position % 2 == 1:
314
+ if part not in placeholders:
315
+ raise FormulaError(f"unknown column `{part}`")
316
+ out.append(f" {placeholders[part]} ")
317
+ else:
318
+ out.append(_substitute_bare(part, bare, placeholders))
319
+ return "".join(out)
320
+
321
+
322
+ def _substitute_bare(text: str, names: list, placeholders: dict) -> str:
323
+ out = []
324
+ i = 0
325
+ while i < len(text):
326
+ found = None
327
+ for name in names:
328
+ if not text.startswith(name, i):
329
+ continue
330
+ end = i + len(name)
331
+ if _IDENTIFIER.match(name[0]) and i > 0 and _IDENTIFIER.match(text[i - 1]):
332
+ continue
333
+ if _IDENTIFIER.match(name[-1]) and end < len(text) and _IDENTIFIER.match(text[end]):
334
+ continue
335
+ if name in _FUNCTIONS and text[end:].lstrip().startswith("("):
336
+ continue
337
+ found = name
338
+ break
339
+ if found is None:
340
+ out.append(text[i])
341
+ i += 1
342
+ else:
343
+ out.append(f" {placeholders[found]} ")
344
+ i += len(found)
345
+ return "".join(out)
346
+
347
+
348
+ def _as_series(value, index) -> pandas.Series:
349
+ if isinstance(value, pandas.Series):
350
+ return value
351
+ array = numpy.asarray(value)
352
+ if array.ndim == 0:
353
+ return pandas.Series(array.item(), index=index)
354
+ if array.ndim == 1 and len(array) == len(index):
355
+ return pandas.Series(array, index=index)
356
+ raise FormulaError("the expression does not give one value per row")
357
+
358
+
359
+ _BINARY = {
360
+ ast.Add: lambda a, b: a + b,
361
+ ast.Sub: lambda a, b: a - b,
362
+ ast.Mult: lambda a, b: a * b,
363
+ ast.Div: lambda a, b: a / b,
364
+ ast.Mod: lambda a, b: a % b,
365
+ ast.Pow: lambda a, b: a ** b,
366
+ }
367
+ _COMPARE = {
368
+ ast.Lt: lambda a, b: a < b,
369
+ ast.LtE: lambda a, b: a <= b,
370
+ ast.Gt: lambda a, b: a > b,
371
+ ast.GtE: lambda a, b: a >= b,
372
+ ast.Eq: lambda a, b: a == b,
373
+ ast.NotEq: lambda a, b: a != b,
374
+ }
375
+ _REFUSED = {
376
+ ast.Attribute: "attribute access (a.b)",
377
+ ast.Subscript: "indexing (a[0])",
378
+ ast.Lambda: "a lambda",
379
+ ast.IfExp: "an inline if (use where(condition, if_true, if_false))",
380
+ ast.List: "a list",
381
+ ast.Tuple: "a tuple",
382
+ ast.Dict: "a dictionary",
383
+ ast.Set: "a set",
384
+ ast.JoinedStr: "a string",
385
+ ast.NamedExpr: "an assignment",
386
+ ast.Starred: "unpacking (*)",
387
+ }
388
+
389
+
390
+ class _Evaluator:
391
+ """Evaluates the parsed expression, node by node. Every node type it does not know
392
+ is refused."""
393
+
394
+ def __init__(self, df: pandas.DataFrame, columns: dict, names: dict):
395
+ self.df = df
396
+ self.columns = columns # placeholder -> column key
397
+ self.names = names # placeholder -> column name, for messages
398
+
399
+ def visit(self, node):
400
+ if isinstance(node, ast.Expression):
401
+ return self.visit(node.body)
402
+ if isinstance(node, ast.Constant):
403
+ return self._constant(node.value)
404
+ if isinstance(node, ast.Name):
405
+ return self._name(node.id)
406
+ if isinstance(node, ast.BinOp):
407
+ operation = _BINARY.get(type(node.op))
408
+ if operation is None:
409
+ raise FormulaError(f"operator {_operator(node.op)} is not supported")
410
+ return operation(self.visit(node.left), self.visit(node.right))
411
+ if isinstance(node, ast.UnaryOp):
412
+ operand = self.visit(node.operand)
413
+ if isinstance(node.op, ast.USub):
414
+ return -operand
415
+ if isinstance(node.op, ast.UAdd):
416
+ return +operand
417
+ if isinstance(node.op, ast.Not):
418
+ return numpy.logical_not(operand)
419
+ raise FormulaError(f"operator {_operator(node.op)} is not supported")
420
+ if isinstance(node, ast.BoolOp):
421
+ combine = numpy.logical_and if isinstance(node.op, ast.And) else numpy.logical_or
422
+ return functools.reduce(combine, [self.visit(value) for value in node.values])
423
+ if isinstance(node, ast.Compare):
424
+ return self._compare(node)
425
+ if isinstance(node, ast.Call):
426
+ return self._call(node)
427
+ what = _REFUSED.get(type(node), type(node).__name__)
428
+ raise FormulaError(f"{what} is not allowed in a formula")
429
+
430
+ @staticmethod
431
+ def _constant(value):
432
+ if isinstance(value, bool):
433
+ return numpy.bool_(value)
434
+ if isinstance(value, (int, float)):
435
+ # float64, never a Python int: 9 ** 9 ** 9 overflows to inf instead of computing
436
+ return numpy.float64(value)
437
+ raise FormulaError(f"{value!r} is not a number: write column names without quotes, "
438
+ f"or between backticks")
439
+
440
+ def _name(self, name: str):
441
+ if name in self.columns:
442
+ return self.df[self.columns[name]]
443
+ if name.startswith("__"):
444
+ raise FormulaError(f"expression contains disallowed tokens: {name!r}")
445
+ if name in _CONSTANTS:
446
+ return _CONSTANTS[name]
447
+ if name in _FUNCTIONS:
448
+ raise FormulaError(f"{name!r} is a function: call it as {_FUNCTIONS[name][3]}")
449
+ raise FormulaError(f"unknown column {name!r}: write a column by its name, or between backticks")
450
+
451
+ def _compare(self, node: ast.Compare):
452
+ left = self.visit(node.left)
453
+ result = None
454
+ for op, comparator in zip(node.ops, node.comparators):
455
+ compare = _COMPARE.get(type(op))
456
+ if compare is None:
457
+ raise FormulaError(f"comparison {_operator(op)} is not supported")
458
+ right = self.visit(comparator)
459
+ current = compare(left, right)
460
+ result = current if result is None else numpy.logical_and(result, current)
461
+ left = right
462
+ return result
463
+
464
+ def _call(self, node: ast.Call):
465
+ if not isinstance(node.func, ast.Name):
466
+ raise FormulaError(f"{_REFUSED.get(type(node.func), 'calling an expression')} is not allowed in a formula")
467
+ name = node.func.id
468
+ if name in self.columns:
469
+ raise FormulaError(f"column {self.names[name]!r} is not a function")
470
+ if name not in _FUNCTIONS:
471
+ raise FormulaError(f"unknown function {name!r}")
472
+ function, min_args, max_args, signature = _FUNCTIONS[name]
473
+ if node.keywords:
474
+ raise FormulaError(f"{name}: write the arguments without names, as {signature}")
475
+ count = len(node.args)
476
+ if count < min_args or (max_args is not None and count > max_args):
477
+ raise FormulaError(f"{name} takes {_arity(min_args, max_args)}, got {count}: {signature}")
478
+ return function(*[self.visit(argument) for argument in node.args])
479
+
480
+
481
+ def _arity(min_args: int, max_args) -> str:
482
+ if max_args is None:
483
+ return f"at least {min_args} values"
484
+ if min_args == max_args:
485
+ return f"{min_args} value{'s' if min_args > 1 else ''}"
486
+ return f"{min_args} to {max_args} values"
487
+
488
+
489
+ def _operator(op) -> str:
490
+ return {ast.FloorDiv: "'//'", ast.MatMult: "'@'", ast.LShift: "'<<'", ast.RShift: "'>>'",
491
+ ast.BitAnd: "'&'", ast.BitOr: "'|'", ast.BitXor: "'^'", ast.Invert: "'~'",
492
+ ast.In: "'in'", ast.NotIn: "'not in'", ast.Is: "'is'", ast.IsNot: "'is not'"}.get(type(op),
493
+ type(op).__name__)
wizata_dsapi/pipeline.py CHANGED
@@ -13,6 +13,7 @@ from .template import Template, TemplateProperty
13
13
  from .twin import Twin
14
14
  from .twinregistration import TwinRegistration, TwinRegistrationProperty
15
15
  from .datapoint import DataPoint
16
+ from .formula import normalize_formulas
16
17
 
17
18
  from enum import Enum
18
19
 
@@ -44,6 +45,8 @@ class StepType(Enum):
44
45
  - "write"
45
46
  - "plot"
46
47
  - "df_save"
48
+ - "alert"
49
+ - "formula": calculated columns on one dataframe (DEV-6112)
47
50
  """
48
51
  QUERY = 'query'
49
52
  SCRIPT = 'script'
@@ -52,6 +55,7 @@ class StepType(Enum):
52
55
  PLOT = 'plot'
53
56
  DF_SAVE = 'df_save'
54
57
  ALERT = 'alert'
58
+ FORMULA = 'formula'
55
59
 
56
60
 
57
61
  class AlertType(Enum):
@@ -120,6 +124,39 @@ class WriteConfig:
120
124
  return obj
121
125
 
122
126
 
127
+ class FormulaConfig:
128
+ """
129
+ Configuration for a FORMULA pipeline step (DEV-6112).
130
+
131
+ Calculated columns ``[{"expression": ..., "result": ...}]``, computed in order on the
132
+ step's one input dataframe by the resolver behind the query option ``Request.formulas``
133
+ (:mod:`wizata_dsapi.formula`). "Open as pipeline" moves a query's list here unchanged,
134
+ so both give the same frame. The step outputs the input plus the new columns.
135
+ """
136
+
137
+ def __init__(self, formulas: list = None):
138
+ self.formulas = normalize_formulas(formulas)
139
+
140
+ @classmethod
141
+ def from_dict(cls, data: dict):
142
+ config = cls()
143
+ config.from_json(data)
144
+ return config
145
+
146
+ def from_json(self, obj: dict):
147
+ if not isinstance(obj, dict):
148
+ raise ValueError(f"a formula step config must be an object with 'formulas', got {type(obj).__name__}")
149
+ self.formulas = normalize_formulas(obj.get("formulas"))
150
+ if not self.formulas:
151
+ raise ValueError("a formula step needs at least one formula in 'formulas'")
152
+
153
+ def to_dict(self) -> dict:
154
+ return {"formulas": [dict(formula) for formula in self.formulas]}
155
+
156
+ def to_json(self, target: str = None) -> dict:
157
+ return self.to_dict()
158
+
159
+
123
160
  class AlertConfig:
124
161
  """
125
162
  Configuration for an ALERT pipeline step.
@@ -400,6 +437,9 @@ class PipelineStep(ApiDto):
400
437
  elif self.step_type == StepType.ALERT:
401
438
  self.config = AlertConfig()
402
439
  self.config.from_json(obj['config'])
440
+ elif self.step_type == StepType.FORMULA:
441
+ self.config = FormulaConfig()
442
+ self.config.from_json(obj['config'])
403
443
  else:
404
444
  self.config = obj['config']
405
445
  self.inputs = []
@@ -433,6 +473,8 @@ class PipelineStep(ApiDto):
433
473
  obj["config"] = self.config.to_json()
434
474
  elif self.step_type == StepType.ALERT:
435
475
  obj["config"] = self.config.to_json()
476
+ elif self.step_type == StepType.FORMULA:
477
+ obj["config"] = self.config.to_json()
436
478
  else:
437
479
  obj["config"] = self.config
438
480
 
@@ -662,6 +704,8 @@ class Pipeline(ApiDto):
662
704
  raise RuntimeError(f'write step must have exactly one input and no outputs')
663
705
  if step.step_type == StepType.ALERT and (len(step.inputs) != 1 or len(step.outputs) > 0):
664
706
  raise RuntimeError(f'alert step must have exactly one input and no outputs')
707
+ if step.step_type == StepType.FORMULA and (len(step.inputs) != 1 or len(step.outputs) != 1):
708
+ raise RuntimeError(f'formula step must have exactly one input and one output')
665
709
  for output in step.outputs_names():
666
710
  if output in produced:
667
711
  raise RuntimeError(f'output {output} is already produced.')
@@ -852,6 +896,32 @@ class Pipeline(ApiDto):
852
896
 
853
897
  self.steps.append(step)
854
898
 
899
+ def add_formula(self,
900
+ formulas,
901
+ input_df,
902
+ output_df):
903
+ """
904
+ Add a formula step: calculated columns on one dataframe (DEV-6112).
905
+
906
+ :param formulas: ``[{"expression": ..., "result": ...}]`` or a FormulaConfig. Every
907
+ result is named; see ``wizata_dsapi.formula_catalog()``.
908
+ :param input_df: str, dict or PipelineIO of the dataframe the formulas read.
909
+ :param output_df: str, dict or PipelineIO of the dataframe they produce: the input
910
+ plus one column per formula.
911
+ """
912
+ config = formulas if isinstance(formulas, FormulaConfig) else FormulaConfig(formulas=formulas)
913
+ if not config.formulas:
914
+ raise ValueError("a formula step needs at least one formula")
915
+ if input_df is None or output_df is None:
916
+ raise ValueError('please provide input_df and output_df (str, dict or PipelineIO)')
917
+
918
+ step = PipelineStep()
919
+ step.step_type = StepType.FORMULA
920
+ step.config = config
921
+ step.inputs.append(PipelineIO.from_obj(input_df))
922
+ step.outputs.append(PipelineIO.from_obj(output_df))
923
+ self.steps.append(step)
924
+
855
925
  def add_alert(self,
856
926
  config: AlertConfig = None,
857
927
  df_name: str = None,
wizata_dsapi/request.py CHANGED
@@ -3,6 +3,7 @@ from datetime import datetime, timedelta, timezone
3
3
  from .dataframe_toolkit import generate_epoch
4
4
  from .api_dto import ApiDto
5
5
  from .datapoint import BusinessType
6
+ from .formula import normalize_formulas
6
7
 
7
8
  filter_map = {
8
9
  '>': 'gt',
@@ -168,6 +169,9 @@ class Request(ApiDto):
168
169
  :ivar list products: product keys to keep on inventoryMovement datapoints (all when None). With
169
170
  no datapoints, it selects the inventoryMovement datapoints holding them (under the twin
170
171
  when the request names one).
172
+ :ivar list formulas: calculated columns ``[{"expression": ..., "result": ...}]``, computed in
173
+ order after everything else in the query (DEV-6112). Each result is named, see
174
+ ``wizata_dsapi.formula_catalog()`` for the operators, functions and naming rule.
171
175
  """
172
176
 
173
177
  @classmethod
@@ -203,7 +207,8 @@ class Request(ApiDto):
203
207
  bucket=None,
204
208
  tags=None,
205
209
  scopes=None,
206
- products=None):
210
+ products=None,
211
+ formulas=None):
207
212
  if request_id is None:
208
213
  request_id = uuid.uuid4()
209
214
  self.request_id = request_id
@@ -248,6 +253,7 @@ class Request(ApiDto):
248
253
  self.tags = tags
249
254
  self.scopes = scopes
250
255
  self.products = products
256
+ self.formulas = normalize_formulas(formulas) or None
251
257
 
252
258
  self.null = null
253
259
 
@@ -631,6 +637,8 @@ class Request(ApiDto):
631
637
  query['scopes'] = self.scopes
632
638
  if self.products is not None:
633
639
  query['products'] = self.products
640
+ if getattr(self, "formulas", None): # absent on a Request pickled before 12.2
641
+ query['formulas'] = [dict(formula) for formula in self.formulas]
634
642
 
635
643
  if self.target_feat is not None:
636
644
  query["target_feat"] = {
@@ -775,6 +783,9 @@ class Request(ApiDto):
775
783
  f"for every product), got {products!r}"[:300])
776
784
  self.products = products
777
785
 
786
+ if "formulas" in json_data.keys():
787
+ self.formulas = normalize_formulas(json_data["formulas"]) or None
788
+
778
789
  if "connections" in json_data.keys():
779
790
  self.connections = json_data["connections"]
780
791
 
@@ -1,6 +1,5 @@
1
- import re
2
-
3
1
  import wizata_dsapi
2
+ from ..formula import apply_formulas
4
3
 
5
4
  import pandas
6
5
  import numpy
@@ -13,19 +12,6 @@ import sklearn.preprocessing
13
12
  import sklearn.decomposition
14
13
 
15
14
 
16
- _FORMULA_SAFE_FUNCTIONS = {
17
- "abs": numpy.abs,
18
- "sqrt": numpy.sqrt,
19
- "log": numpy.log,
20
- "log10": numpy.log10,
21
- "exp": numpy.exp,
22
- "clip": numpy.clip,
23
- "round": numpy.round,
24
- "min": numpy.minimum,
25
- "max": numpy.maximum,
26
- }
27
-
28
-
29
15
  def filter_df(context: wizata_dsapi.Context):
30
16
  """Filter dataframe rows using pandas query strings from the 'filters' property list."""
31
17
 
@@ -313,48 +299,32 @@ def setpoint_deviation(context: wizata_dsapi.Context):
313
299
  return df
314
300
 
315
301
 
316
- def formula(context: wizata_dsapi.Context):
317
- """Add a new column computed from a user-defined math expression over existing columns. Property 'expression' is required — intuitive math syntax referencing column names (e.g. 'temp_1 + temp_2', '(p_in - p_out) / p_in * 100', 'sqrt(vibration_x**2 + vibration_y**2)', 'clip(temperature, 0, 500)', 'log(power + 1)'). Supports arithmetic operators (+, -, *, /, %, **) and these functions: abs, sqrt, log, log10, exp, clip, round, min, max. Column names with spaces must be wrapped in backticks (e.g. '`motor temp` * 2'). Property 'result' (default 'result') names the output column."""
318
- df = context.dataframe
302
+ def _step_properties(context: wizata_dsapi.Context) -> dict:
303
+ """The properties of the step being run. ``context.properties`` also holds those of every
304
+ step before it (``map_context`` never takes them back), so a second formula step without
305
+ ``result`` used to reuse the first step's (DEV-6112). Outside a pipeline step (a unit
306
+ test, a direct call), the context's own properties."""
307
+ step = getattr(context, "step", None)
308
+ config = getattr(step, "config", None)
309
+ if isinstance(config, wizata_dsapi.ScriptConfig) and config.properties is not None:
310
+ return config.properties
311
+ return context.properties or {}
319
312
 
320
- if "expression" not in context.properties:
321
- raise KeyError(f"please set an 'expression' property (e.g. 'col_a + col_b')")
322
-
323
- expression = context.properties["expression"]
324
- result = context.properties.get("result", "result")
325
-
326
- # Replace every column name (backticked or bare) with a safe placeholder
327
- # identifier before evaluation. Column names can contain characters that
328
- # are illegal in a Python/pandas eval expression — e.g. tag IDs like
329
- # 'DE_AL_Ark1_1:04800F1?Value', whose numeric segment '04800' is otherwise
330
- # parsed as a leading-zero integer literal and raises a SyntaxError
331
- # (DEV-5912). Substituting the whole name atomically sidesteps this.
332
- placeholders = {} # placeholder -> real column name
333
- substituted = expression
334
- for i, col in enumerate(sorted(df.columns, key=len, reverse=True)):
335
- placeholder = f"__wz_col_{i}__"
336
- substituted = substituted.replace(f"`{col}`", placeholder).replace(col, placeholder)
337
- placeholders[placeholder] = col
338
-
339
- # Whitelist validation: strip placeholders + allowed functions, flag any
340
- # leftover token as disallowed.
341
- sanitized = substituted
342
- for placeholder in placeholders:
343
- sanitized = sanitized.replace(placeholder, "")
344
- for fn in _FORMULA_SAFE_FUNCTIONS:
345
- sanitized = sanitized.replace(fn, "")
346
- sanitized = re.sub(r"[\d\.\+\-\*/\%\(\)\s,\^]", "", sanitized)
347
- if sanitized.strip():
348
- raise ValueError(
349
- f"expression contains disallowed tokens: '{sanitized.strip()}' — "
350
- f"only column names, numbers, arithmetic operators, and these functions are allowed: "
351
- f"{list(_FORMULA_SAFE_FUNCTIONS.keys())}"
352
- )
353
-
354
- # Evaluate against a frame whose columns are the safe placeholder names.
355
- safe_df = df.rename(columns={real: ph for ph, real in placeholders.items()})
356
- df[result] = safe_df.eval(substituted, local_dict=_FORMULA_SAFE_FUNCTIONS, engine="python")
357
- return df
313
+
314
+ def formula(context: wizata_dsapi.Context):
315
+ """Legacy: add one column computed from a math expression over existing columns. New pipelines use a formula step (step type 'formula', DEV-6112), which computes a named list of formulas with the same resolver as the query option Request.formulas. Property 'expression' is required; 'result' (default 'result') names the output column and may replace a column. Expressions use column names (between backticks when they hold spaces), numbers, + - * / % ** (or ^), comparisons < <= > >= == !=, and / or / not (or & | ~), the functions abs, sqrt, log, log10, exp, round, clip, min, max, where, and the constants pi and e."""
316
+ properties = _step_properties(context)
317
+
318
+ if "formulas" in properties:
319
+ raise ValueError("'formulas' belongs to a formula step (step type 'formula'); "
320
+ "this script takes 'expression' and 'result'")
321
+ if "expression" not in properties:
322
+ raise KeyError("please set an 'expression' property (e.g. 'col_a + col_b'), "
323
+ "or use a formula step (step type 'formula')")
324
+
325
+ return apply_formulas(context.dataframe,
326
+ [{"expression": properties["expression"], "result": properties.get("result", "result")}],
327
+ strict=False)
358
328
 
359
329
 
360
330
  def target_feat_to_binary(context: wizata_dsapi.Context):
wizata_dsapi/version.py CHANGED
@@ -1 +1 @@
1
- __version__ = "2.2.0.dev18"
1
+ __version__ = "2.2.0.dev20"
@@ -28,6 +28,7 @@ from .api_dto import ApiDto, VarType, ApiDtoInterface
28
28
  from .business_label import BusinessLabel
29
29
  from .plot import Plot
30
30
  from .request import Request
31
+ from .formula import normalize_formulas
31
32
  from .mlmodel import ModelInfo, ModelList, ModelFile
32
33
  from .experiment import Experiment
33
34
  from .script import Script
@@ -779,7 +780,8 @@ class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
779
780
  bucket: str = None,
780
781
  tags: dict = None,
781
782
  scopes: list = None,
782
- products: list = None) -> Request:
783
+ products: list = None,
784
+ formulas: list = None) -> Request:
783
785
  """
784
786
  Build the Request behind a time-series query.
785
787
 
@@ -825,6 +827,9 @@ class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
825
827
  if products is not None:
826
828
  request.products = products
827
829
 
830
+ if formulas is not None:
831
+ request.formulas = normalize_formulas(formulas) or None
832
+
828
833
  if options is not None:
829
834
  request.options = options
830
835
 
@@ -857,7 +862,8 @@ class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
857
862
  bucket: str = None,
858
863
  tags: dict = None,
859
864
  scopes: list = None,
860
- products: list = None) -> pandas.DataFrame:
865
+ products: list = None,
866
+ formulas: list = None) -> pandas.DataFrame:
861
867
  """
862
868
  Query a dataframe from API.
863
869
  :param agg_method:
@@ -878,13 +884,15 @@ class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
878
884
  ``options={"inventory": "movements" | "balances" | "stock"}`` chooses what they return.
879
885
  With no ``datapoints``, the inventoryMovement datapoints holding those products are
880
886
  selected for you: every storage, or those under ``twin`` and its children (DEV-6089).
887
+ :param formulas: calculated columns ``[{"expression": "a - b", "result": "gap"}]``, computed
888
+ after the query, in order (DEV-6112). See ``wizata_dsapi.formula_catalog()``.
881
889
  :return: dataframe
882
890
  """
883
891
  return self._query(request=self._build_request(
884
892
  datapoints=datapoints, start=start, end=end, interval=interval,
885
893
  agg_method=agg_method, template=template, twin=twin, null=null,
886
894
  filters=filters, options=options, group=group, field=field,
887
- bucket=bucket, tags=tags, scopes=scopes, products=products))
895
+ bucket=bucket, tags=tags, scopes=scopes, products=products, formulas=formulas))
888
896
 
889
897
  def batch_query(self,
890
898
  step: timedelta,
@@ -3004,13 +3012,14 @@ class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
3004
3012
  return "edge/" + "/".join(quoted)
3005
3013
 
3006
3014
  def _edge_request(self, method: str, route: str, body=None, headers=None,
3007
- expected=(200,), stream=False):
3015
+ expected=(200,), stream=False, timeout=None):
3008
3016
  response = requests.request(
3009
3017
  method,
3010
3018
  self.__url() + route,
3011
3019
  headers={**self.__header(), **(headers or {})},
3012
3020
  data=json.dumps(body) if body is not None else None,
3013
3021
  stream=stream,
3022
+ timeout=timeout,
3014
3023
  )
3015
3024
  if response.status_code not in expected:
3016
3025
  raise self.__raise_error(response)
@@ -3080,6 +3089,21 @@ class WizataDSAPIClient(ApiInterface, ApiDtoInterface):
3080
3089
  os.remove(temporary)
3081
3090
  return path
3082
3091
 
3092
+ def send_edge_heartbeat(self, edge_device_id: str, packages: list = None,
3093
+ timeout=(5, 15)) -> dict:
3094
+ """
3095
+ A manual edge's heartbeat (DEV-6109): it is alive, and these packages (the ids in
3096
+ its ``conf/_packages``) are installed. The platform records when it was seen, and
3097
+ each package as installed the first time it is listed.
3098
+
3099
+ :param timeout: (connect, read) seconds — a site that blocks outbound traffic must
3100
+ not hold the caller.
3101
+ :return: ``{"lastSeenAt", "applied": [ids newly recorded], "unknown": [ids]}``.
3102
+ """
3103
+ response = self._edge_request("POST", self._edge_route(edge_device_id, "heartbeat"),
3104
+ body={"packages": list(packages or [])}, timeout=timeout)
3105
+ return response.json()
3106
+
3083
3107
 
3084
3108
  def api() -> WizataDSAPIClient:
3085
3109
  """
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: wizata_dsapi
3
- Version: 2.2.0.dev18
3
+ Version: 2.2.0.dev20
4
4
  Summary: Wizata Data Science Toolkit
5
5
  Author: Wizata S.A.
6
6
  Author-email: info@wizata.com
@@ -1,4 +1,4 @@
1
- wizata_dsapi/__init__.py,sha256=bo7yxZNkKqHuMhjec9b6YpnFoqMNrASN2IBxOzE5SP4,3920
1
+ wizata_dsapi/__init__.py,sha256=ASXZxRPiRNt0-QspmCMZKBpqLdVaSKgggmVlbPH2NoU,4219
2
2
  wizata_dsapi/api_config.py,sha256=zOgvpkB3FE2ny_2vAui7B1hcAMFCLvfb2ep3m-b7dIw,8270
3
3
  wizata_dsapi/api_dto.py,sha256=F9HjNa_vqYhdEMZwwk8E3SmCxuMBcxFnMp1eE9dxLvg,2910
4
4
  wizata_dsapi/api_interface.py,sha256=mCq0-rlwWL4GKB2QnxSPuyTecYOXXknmPOlQe_SEf2I,14670
@@ -15,12 +15,14 @@ wizata_dsapi/deployment.py,sha256=DCycwDPqsE3YoZtoqaye4V8-G2pqHP84vkNwpY2op5s,10
15
15
  wizata_dsapi/ds_dataframe.py,sha256=Sk2JRUuTRJzko3HosJnbK34STpgSJUlqxLq8w_E5VM4,2089
16
16
  wizata_dsapi/dsapi_json_encoder.py,sha256=_NJE2IncJCS-juOxV_IQWHq8WpaanKe2aWpGNZNUzyk,2062
17
17
  wizata_dsapi/edge_config.py,sha256=YvLL2pz42NhmgZK4-fxoI1p_jWMKT31tlys_RLOVQ4s,9131
18
- wizata_dsapi/edge_device.py,sha256=ogzfy-hFNqVcRwn6NI9vV1nE87W4yFF3oE5_uY6nvGQ,9323
18
+ wizata_dsapi/edge_device.py,sha256=zucfEzgHtUwkDv1gkhC82RxBTeoh1IE4IwbAFECYMGU,9702
19
19
  wizata_dsapi/edge_module.py,sha256=HqYKAjdKGd_0bci5-uckBm3HgBhqj4YRPXlQRYZh7eY,4636
20
+ wizata_dsapi/edge_package.py,sha256=BJZQMOhujtd-zDfYTgEgbkAHLE7hQjlnP1MvVpyl_sc,6078
20
21
  wizata_dsapi/evaluation.py,sha256=kB61SD66uRBBbKqiES7XZERn77bwhacbyufneSD4s8c,2222
21
22
  wizata_dsapi/execution.py,sha256=JuU8qZDcUZxtCvQwwvM_luwHi18GQ2jbBSxAjc2cSlM,14148
22
23
  wizata_dsapi/execution_log.py,sha256=TRchBeDvQp7QfdCN4oiu5NxrB05JYCKAFYRJs9Dch3c,17106
23
24
  wizata_dsapi/experiment.py,sha256=sYhdeliJdYdbTd0HPyTE7LVHB7QDnw6ynw36jGjJa3c,4773
25
+ wizata_dsapi/formula.py,sha256=i_YqGG_4LmGxM4MBGzWCpq1-yp6lc-6zwRMMYLxwuoQ,23450
24
26
  wizata_dsapi/graylog_log.py,sha256=KaHzv9HSqzmg8Cjr1vrsqeggd3tvrDkSN1TXaxCwsPs,4808
25
27
  wizata_dsapi/group_system.py,sha256=6rUKe0_J3YWACysyBlzuw_TEpKNXgLOMxhpWsNxOzwY,1708
26
28
  wizata_dsapi/ilogger.py,sha256=YKBTRp7pHYtBlnzroGRPT91_bMVJT93Pft3fUGcFSIM,488
@@ -31,11 +33,11 @@ wizata_dsapi/mobile_asset.py,sha256=FBEvrOaXkT8YaLv1coemqNFJaoY-NVUcgNKPFgUzfP4,
31
33
  wizata_dsapi/model_toolkit.py,sha256=UNyw5CFSgZeXydQFsiDIRTjoMeqIsdyIIuiwumLW5bA,1574
32
34
  wizata_dsapi/notification.py,sha256=rzAo0e-LcGqsmatSlZL3YhUu6JvDj6p643CxDfI9nMc,24147
33
35
  wizata_dsapi/paged_query_result.py,sha256=QOpJMAS6rxJfiz3a5oSfc0gOvwQeZTEhOtBoSso5vFw,2877
34
- wizata_dsapi/pipeline.py,sha256=mMkNzFQsC9s1ZuC0HrfLG1BsZCnYYlrrhPlo6Out1Zs,40872
36
+ wizata_dsapi/pipeline.py,sha256=9ufOxoGSVaRYRfv9WM2zLURrSbxeAIO6l-AZ4PJSLps,43922
35
37
  wizata_dsapi/pipeline_image.py,sha256=WtMY0_ncba9WzZ96KQINiYshJegQdArzhspKTGWBgVA,7758
36
38
  wizata_dsapi/plot.py,sha256=KLeYj6xpG_XFSDuR4A8Iqp2fmAVLeDsHaUOVT2sRUYM,4033
37
39
  wizata_dsapi/product.py,sha256=EiiT9icS2xr8aN3STre4X7CsaJ33tt7jmOnQPD9pA6U,8139
38
- wizata_dsapi/request.py,sha256=3gpnUMQJomndu5mTCatczmJ1sCs9sDlRKeeycj9bAYE,29560
40
+ wizata_dsapi/request.py,sha256=o4MZmy53kSnIoMje3-TyhUTdXj_EZVfcFlb8YKMx2Sk,30269
39
41
  wizata_dsapi/script.py,sha256=1Mjw3BNxN_R5dx0f55T0WuebqqRr_69rsK5yuQZOjmQ,19467
40
42
  wizata_dsapi/search.py,sha256=pHX-xGIB0E6gJkTxXmLjFe_b8-0gei7hfkW0JnxzCl0,4691
41
43
  wizata_dsapi/solution_component.py,sha256=8gbZWx2h_xUqI_pAXa3goqAnR5Y-GDMii8MeGlaK1IE,9531
@@ -46,10 +48,10 @@ wizata_dsapi/trigger.py,sha256=waOX2AhrLvFzNFRQ535j6iemgg_QJP2QsWGL2ipljFg,8848
46
48
  wizata_dsapi/twin.py,sha256=8AnrxSaL7dMrILXJ3-GGmmUz078j3qqwzDTe6i86xqA,12672
47
49
  wizata_dsapi/twinregistration.py,sha256=iXt55AVGAQNl0jmxa3ZVynPqlSsIunFf4sLfp-MWTw0,13244
48
50
  wizata_dsapi/user.py,sha256=-hchKAMZULu9P00RUHDqu_5AT4NI5BYOJd8XCBrt3eQ,1950
49
- wizata_dsapi/version.py,sha256=d3EON8QcgPlCOSGrfATCw2ZdOkkFmhxLoiga35vwr1M,28
51
+ wizata_dsapi/version.py,sha256=Ej92cnhM5g3Pwri9IPc8X2C0NhC-UawJM0AUXdnAxLM,28
50
52
  wizata_dsapi/wizard_function.py,sha256=RbM7W7Gf-6Rhp_1dU9DBYkHaciknGAGvuAndhAS_vyo,942
51
53
  wizata_dsapi/wizard_request.py,sha256=v6BaqKLKvTWmUSo0_gda9FabAQz5x_-GOH1Av50GzFo,3762
52
- wizata_dsapi/wizata_dsapi_client.py,sha256=owSiTro0Y_1_RRLYXqI4cD861-toUfVSjbIchFlCsfc,129493
54
+ wizata_dsapi/wizata_dsapi_client.py,sha256=uoeUsaob9CsR1VojBuyM5Yq0wqUVDkCKorrb8kE4uTg,130806
53
55
  wizata_dsapi/words.py,sha256=tV8CqzCqODZCV7PgBxBF5exBxeF_ya9t5DiUy-cg6Sg,1535
54
56
  wizata_dsapi/models/__init__.py,sha256=O3Tusyhl8hFb-5QaEBY7DJISRB8Gf_Xbxxr-aFDR1_U,380
55
57
  wizata_dsapi/models/common.py,sha256=5BC3Dc6kMvqPGxHgoCuGVOynu_F1MWh7EJSzXIICSSg,27196
@@ -57,9 +59,9 @@ wizata_dsapi/plots/__init__.py,sha256=ueFVcCVPZmYOBRCkZOrSGG5L2FdS6m3H_3SXq2gBsu
57
59
  wizata_dsapi/plots/common.py,sha256=7F4WMBaddubnCpdSafgaI9U9Sz0Sngg0VREP_vnIMno,21274
58
60
  wizata_dsapi/plots/theme.py,sha256=D4sSgEwabXVxF3z1GV8DivXJbbY61KgxXAWfjUbNDAc,3594
59
61
  wizata_dsapi/scripts/__init__.py,sha256=9fRdTMuuXcqMkIsUPf5dwFsTWwBzKteQdAMVkyy2acQ,212
60
- wizata_dsapi/scripts/common.py,sha256=T2cP8aGPg8OgIvBWuQBNonVtph_dHlNFubByuYrLb0o,15675
61
- wizata_dsapi-2.2.0.dev18.dist-info/licenses/LICENSE.txt,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
62
- wizata_dsapi-2.2.0.dev18.dist-info/METADATA,sha256=yEBEIykVqXavw3uBRjbi-A7lj53Ps9Qgj2k2jaAtuco,14038
63
- wizata_dsapi-2.2.0.dev18.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
64
- wizata_dsapi-2.2.0.dev18.dist-info/top_level.txt,sha256=-OeTJbEnh5DuWyTOHtvw0Dw3LRg3G27TNS6W4ZtfwPs,13
65
- wizata_dsapi-2.2.0.dev18.dist-info/RECORD,,
62
+ wizata_dsapi/scripts/common.py,sha256=8LJK23Za0Yy4Dda1feKpKELav8806MH-hxFJYrklVOY,14758
63
+ wizata_dsapi-2.2.0.dev20.dist-info/licenses/LICENSE.txt,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
64
+ wizata_dsapi-2.2.0.dev20.dist-info/METADATA,sha256=xKiGgGaanscGQRqW-QYSjLGfGcqI-jq52HHKBJcFTpw,14038
65
+ wizata_dsapi-2.2.0.dev20.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
66
+ wizata_dsapi-2.2.0.dev20.dist-info/top_level.txt,sha256=-OeTJbEnh5DuWyTOHtvw0Dw3LRg3G27TNS6W4ZtfwPs,13
67
+ wizata_dsapi-2.2.0.dev20.dist-info/RECORD,,