wizata-dsapi 2.2.0.dev20__tar.gz → 2.2.0.dev21__tar.gz

{wizata_dsapi-2.2.0.dev20 → wizata_dsapi-2.2.0.dev21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wizata_dsapi
-Version: 2.2.0.dev20
+Version: 2.2.0.dev21
 Summary: Wizata Data Science Toolkit
 Author: Wizata S.A.
 Author-email: info@wizata.com

{wizata_dsapi-2.2.0.dev20 → wizata_dsapi-2.2.0.dev21}/wizata_dsapi/__init__.py

@@ -65,10 +65,13 @@ from .edge_device import EdgeDevice, EdgeDeviceStatus, EdgeDeviceType, EdgeDevic
 from .edge_module import EdgeModule, EdgeModuleType, WIZATA_MODULES
 from .edge_config import EdgeConsumer, EdgeWriter, EdgeTrigger, EdgeFile, EdgeConfig, EdgeCommand
 from .edge_package import EdgePackage, EdgePackageOperation, EdgeRelease
-from .connection import Connection, ConnectionMapping, ConnectionDirection, TransformKind, \
-    EdgeDeviceKey, EdgeDeviceKeyStatus
-from .connection_types import ConnectionType, CONNECTION_TYPES, TRANSFORM_SCHEMAS, get_connection_type, \
-    list_connection_types
+from .connection import Connection, ConnectionChannel, ConnectionSignal, ConnectionDirection, \
+    TransformKind, EdgeDeviceKey, EdgeDeviceKeyStatus
+from .connection_types import ConnectionType, ChannelType, CONNECTION_TYPES, SYSTEM_TYPES, \
+    TRANSFORM_SCHEMAS, get_connection_type, list_connection_types
+from .connection_policy import DatapointPolicy, PolicyError, canonical_node_id
+from .connection_discovery import ConnectionSeenName, ConnectionSeenReport, ConnectionBrowse, \
+    SEEN_OUTCOMES, BROWSE_REQUESTED, BROWSE_STATUSES, BROWSE_EXPIRES_SECONDS
 from .product import ProductKind, Product, ProductComponent, BUILTIN_PRODUCT_KINDS, is_valid_product_key
 from .inventory import (InventoryBalance, InventoryStorage, InventoryStorageDatapoint, inventory_mode,
                         InventoryStock, InventoryStockProduct, InventoryStockLine, InventoryStockCount, INVENTORY_OPTION, INVENTORY_MODES,

{wizata_dsapi-2.2.0.dev20 → wizata_dsapi-2.2.0.dev21}/wizata_dsapi/connection.py

@@ -2,16 +2,18 @@
 Edge connection DTOs (DEV-6013).
 
 An edge connector used to be one untyped dict in the device's ``consumers.json``,
-credentials inline. It is now split in two entities, the same for every type:
+credentials inline. It is now (12.2 redesign) three entities, the same for every type:
 
 Connection — how to reach a source or a target (endpoint, credentials, certificates),
-  bound to one edge device. ``config`` is clear; ``secrets`` holds one sealed envelope
-  per secret field, readable by the owning edge only.
+  bound to one edge device and entered once. ``config`` is clear; ``secrets`` holds one
+  sealed envelope per secret field, readable by the owning edge only. ``cloud`` (hub) and
+  ``edge`` (tsdb) are the device's built-in connections.
 
-ConnectionMapping — one thing to read or write through a connection (an OPC node, an
-  MQTT topic, a Modbus register, an S7 block entry, a queue). ``address`` and
-  ``options`` are shaped by the connection type (see ``connection_types``); the other
-  fields are common to every type.
+ConnectionChannel — a reader or a writer of a connection: what it reads or writes
+  (``spec``, shaped by the type) and its datapoint policy (all / list / mapping).
+
+ConnectionSignal — one row of a policy: a reader's name and the datapoint it goes on as,
+  or a datapoint and where a writer sends it.
 
 EdgeDeviceKey — the public half of the key pair an edge generated for itself. Secrets
   are sealed for the edge's active key; rotating adds a row and retires the previous.
@@ -74,6 +76,17 @@ class Connection:
         self.secrets: dict = {}
         self.created_at: Optional[datetime] = None
         self.updated_at: Optional[datetime] = None
+        # read-only summaries a search adds: [ConnectionChannel], without their rows
+        self.readers: Optional[list] = None
+        self.writers: Optional[list] = None
+
+    @property
+    def system(self) -> bool:
+        """A device's built-in ``cloud`` / ``edge`` connection (DEV-6013): created by the
+        platform, never deleted, only its writer's policy edited."""
+        from .connection_types import get_connection_type
+        connection_type = get_connection_type(self.connection_type)
+        return bool(connection_type and connection_type.system)
 
     def secrets_set(self) -> list:
         """Names of the secret fields holding a value."""
@@ -99,6 +112,7 @@ class Connection:
             obj["connectionType"] = self.connection_type
         if self.edge_device_id is not None:
             obj["edgeDeviceId"] = self.edge_device_id
+        obj["system"] = self.system
         obj["config"] = dict(self.config or {})
         if include_secrets:
             obj["secrets"] = dict(self.secrets or {})
@@ -113,6 +127,10 @@ class Connection:
             obj["createdAt"] = _ts_out(self.created_at)
         if self.updated_at is not None:
             obj["updatedAt"] = _ts_out(self.updated_at)
+        if self.readers is not None:
+            obj["readers"] = [c.to_dict() for c in self.readers]
+        if self.writers is not None:
+            obj["writers"] = [c.to_dict() for c in self.writers]
         return obj
 
     @classmethod
@@ -144,50 +162,70 @@ class Connection:
                 f"type={self.connection_type!r}, edge={self.edge_device_id!r})")
 
 
-class ConnectionMapping:
+class ConnectionChannel:
     """
-    ConnectionMapping entity stored in PostgreSQL ``"ConnectionMapping"`` table.
-    PK ``id`` (UUID). FK ``connection_id`` → ``Connection.id``, cascade on delete.
-
-    - ``direction``: READ (source → platform) or WRITE (platform → target).
-    - ``address``: where, shaped by the type (``{"node_id": ...}``, ``{"topic": ...}``).
-    - ``hardware_id``: the datapoint. Optional where the source names it (OPC browse
-      with ``children``, a topic function fanning out to many datapoints).
-    - ``poll_interval_ms``: polling cadence for polled types; ignored when pushed.
-    - ``transform``: ``{"kind": "expression", "expression": "x / 10"}`` or
-      ``{"kind": "function", "filepath": ..., "function_name": ...}``.
-    - ``options``: type-specific tuning (``qos``, ``batch_size``, ``pack``...).
+    A reader or a writer of a connection (DEV-6013, 12.2 redesign), stored in PostgreSQL
+    ``"ConnectionChannel"``. PK ``id`` (UUID); FK ``connection_id`` → ``Connection.id``,
+    cascade on delete.
+
+    - ``direction``: READ (a reader, a consumers.json entry) or WRITE (a writer, a
+      writers.json entry).
+    - ``key``: unique per device and direction; it is the consumer or writer ``id`` on the
+      edge, so it is what the edge's logs and seen names call it.
+    - ``spec``: what it reads or writes, shaped by the connection type
+      (``connection_types``): OPC nodes and cadence, MQTT topics and function, a queue...
+    - ``policy_mode`` / ``prefix``: its datapoint policy (``connection_policy``); the rows
+      of the policy are its ``ConnectionSignal`` s.
+    - ``enabled``: a disabled reader or writer is not rendered.
+    - ``signals``: on create only, the policy rows to write with it (read back from search).
+    - ``signal_count`` / ``mapped_count``: read-only, how many rows and how many name a
+      datapoint.
     """
 
     def __init__(self):
         self.id: Optional[str] = None
         self.connection_id: Optional[str] = None
+        self.edge_device_id: Optional[str] = None
         self.direction: ConnectionDirection = ConnectionDirection.READ
-        self.address: dict = {}
-        self.hardware_id: Optional[str] = None
-        self.poll_interval_ms: Optional[int] = None
-        self.transform: Optional[dict] = None
-        self.options: dict = {}
+        self.key: Optional[str] = None
+        self.name: Optional[str] = None
+        self.enabled: bool = True
+        self.spec: dict = {}
+        self.policy_mode: Optional[str] = None
+        self.prefix: Optional[str] = None
+        self.signals: Optional[list] = None
+        self.signal_count: Optional[int] = None
+        self.mapped_count: Optional[int] = None
         self.created_at: Optional[datetime] = None
         self.updated_at: Optional[datetime] = None
 
+    @property
+    def is_reader(self) -> bool:
+        return ConnectionDirection(self.direction) == ConnectionDirection.READ
+
     def to_dict(self) -> dict:
         obj = {}
         if self.id is not None:
             obj["id"] = str(self.id)
         if self.connection_id is not None:
             obj["connectionId"] = str(self.connection_id)
-        obj["direction"] = (self.direction.value
-                            if isinstance(self.direction, ConnectionDirection)
+        if self.edge_device_id is not None:
+            obj["edgeDeviceId"] = self.edge_device_id
+        obj["direction"] = (self.direction.value if isinstance(self.direction, ConnectionDirection)
                             else self.direction)
-        obj["address"] = dict(self.address or {})
-        if self.hardware_id is not None:
-            obj["hardwareId"] = self.hardware_id
-        if self.poll_interval_ms is not None:
-            obj["pollIntervalMs"] = self.poll_interval_ms
-        if self.transform is not None:
-            obj["transform"] = dict(self.transform)
-        obj["options"] = dict(self.options or {})
+        if self.key is not None:
+            obj["key"] = self.key
+        if self.name is not None:
+            obj["name"] = self.name
+        obj["enabled"] = bool(self.enabled)
+        obj["spec"] = dict(self.spec or {})
+        obj["policy"] = {"mode": self.policy_mode, "prefix": self.prefix or ""}
+        if self.signals is not None:
+            obj["signals"] = [s.to_dict() if hasattr(s, "to_dict") else dict(s) for s in self.signals]
+        if self.signal_count is not None:
+            obj["signalCount"] = self.signal_count
+        if self.mapped_count is not None:
+            obj["mappedCount"] = self.mapped_count
         if self.created_at is not None:
             obj["createdAt"] = _ts_out(self.created_at)
         if self.updated_at is not None:
@@ -195,19 +233,32 @@ class ConnectionMapping:
         return obj
 
     @classmethod
-    def from_dict(cls, data: dict) -> "ConnectionMapping":
+    def from_dict(cls, data: dict) -> "ConnectionChannel":
         obj = cls()
         obj.id = data.get("id")
         obj.connection_id = data.get("connectionId") or data.get("connection_id")
+        obj.edge_device_id = data.get("edgeDeviceId") or data.get("edge_device_id")
         raw_direction = data.get("direction")
         if raw_direction is not None:
             obj.direction = ConnectionDirection(str(raw_direction).lower())
-        obj.address = data.get("address") or {}
-        obj.hardware_id = data.get("hardwareId") or data.get("hardware_id")
-        poll = data.get("pollIntervalMs", data.get("poll_interval_ms"))
-        obj.poll_interval_ms = int(poll) if poll is not None else None
-        obj.transform = data.get("transform")
-        obj.options = data.get("options") or {}
+        obj.key = data.get("key")
+        obj.name = data.get("name")
+        enabled = data.get("enabled")
+        obj.enabled = True if enabled is None else bool(enabled)
+        obj.spec = data.get("spec") or {}
+        policy = data.get("policy")
+        if isinstance(policy, dict):
+            obj.policy_mode = policy.get("mode")
+            obj.prefix = policy.get("prefix")
+        else:
+            obj.policy_mode = data.get("policyMode") or data.get("policy_mode")
+            obj.prefix = data.get("prefix")
+        signals = data.get("signals")
+        if signals is not None:
+            obj.signals = [s if isinstance(s, ConnectionSignal) else ConnectionSignal.from_dict(s)
+                           for s in signals]
+        obj.signal_count = data.get("signalCount")
+        obj.mapped_count = data.get("mappedCount")
         obj.created_at = _ts_in(data.get("createdAt") or data.get("created_at"))
         obj.updated_at = _ts_in(data.get("updatedAt") or data.get("updated_at"))
         return obj
@@ -216,14 +267,73 @@ class ConnectionMapping:
         return str(self.id) if self.id else None
 
     def api_key(self) -> str:
-        return self.api_id()
+        return self.key
+
+    def set_id(self, value):
+        self.id = str(value)
+
+    def __repr__(self) -> str:
+        return (f"ConnectionChannel(id={self.id!r}, key={self.key!r}, direction={self.direction!r}, "
+                f"connection={self.connection_id!r}, policy={self.policy_mode!r})")
+
+
+class ConnectionSignal:
+    """
+    One row of a reader's or writer's datapoint policy (DEV-6013, 12.2 redesign), stored in
+    PostgreSQL ``"ConnectionSignal"``. PK ``id`` (UUID); FK ``channel_id`` →
+    ``ConnectionChannel.id``, cascade on delete. Unique per channel: ``tag``, and
+    ``hardware_id`` when set.
+
+    - For a **reader**, ``tag`` is the reader's own name for what it read (an OPC node id,
+      a point name, a function's output) and ``hardware_id`` the datapoint it goes on as.
+      In a ``list`` policy a row with no ``hardware_id`` lets the tag through (prefixed).
+    - For a **writer**, ``hardware_id`` is the datapoint and ``tag`` where it goes (the OPC
+      node, the name written). In a ``list`` policy a row with no ``tag`` lets the
+      datapoint through.
+    - A row with both always renames, whatever the mode.
+    """
+
+    def __init__(self, tag: str = None, hardware_id: str = None):
+        self.id: Optional[str] = None
+        self.channel_id: Optional[str] = None
+        self.tag: Optional[str] = tag
+        self.hardware_id: Optional[str] = hardware_id
+        self.created_at: Optional[datetime] = None
+        self.updated_at: Optional[datetime] = None
+
+    def to_dict(self) -> dict:
+        obj = {}
+        if self.id is not None:
+            obj["id"] = str(self.id)
+        if self.channel_id is not None:
+            obj["channelId"] = str(self.channel_id)
+        obj["tag"] = self.tag
+        obj["hardwareId"] = self.hardware_id
+        if self.created_at is not None:
+            obj["createdAt"] = _ts_out(self.created_at)
+        if self.updated_at is not None:
+            obj["updatedAt"] = _ts_out(self.updated_at)
+        return obj
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "ConnectionSignal":
+        obj = cls()
+        obj.id = data.get("id")
+        obj.channel_id = data.get("channelId") or data.get("channel_id")
+        obj.tag = data.get("tag")
+        obj.hardware_id = data.get("hardwareId") or data.get("hardware_id")
+        obj.created_at = _ts_in(data.get("createdAt") or data.get("created_at"))
+        obj.updated_at = _ts_in(data.get("updatedAt") or data.get("updated_at"))
+        return obj
+
+    def api_id(self) -> str:
+        return str(self.id) if self.id else None
 
     def set_id(self, value):
         self.id = str(value)
 
     def __repr__(self) -> str:
-        return (f"ConnectionMapping(id={self.id!r}, connection={self.connection_id!r}, "
-                f"direction={self.direction!r}, hardware_id={self.hardware_id!r})")
+        return f"ConnectionSignal(tag={self.tag!r}, hardware_id={self.hardware_id!r})"
 
 
 class EdgeDeviceKey:

wizata_dsapi-2.2.0.dev21/wizata_dsapi/connection_discovery.py

@@ -0,0 +1,196 @@
+"""
+What an edge tells the platform about its connections (DEV-6013, release 12.2):
+
+- **seen names**: every name a reader produced, and what its policy did with it. A new
+  reader lets nothing through (an empty list), so this is how a user sees what to allow.
+  **Names, counts and times, never values**: what a policy dropped must not leave the edge
+  through the report.
+- **browse results**: the nodes of an OPC-UA server, browsed by the edge on a platform
+  request (``browse_opc`` command) or by hand on a manual edge (``browse.py``).
+
+Stored in PostgreSQL (alembic 0029): ``"ConnectionSeenName"``, ``"ConnectionSeenReport"``,
+``"ConnectionBrowse"``.
+"""
+import uuid
+from datetime import datetime
+from typing import Optional
+
+from .connection import _ts_in, _ts_out
+
+SEEN_OUTCOMES = ("passed", "renamed", "dropped")
+
+BROWSE_REQUESTED = "requested"
+BROWSE_STATUSES = (BROWSE_REQUESTED, "ok", "partial", "failed", "refused")
+#: A request the edge never answered reads as expired after this (the command is served
+#: 15 minutes, a browse runs up to 5, and its upload is retried up to 15).
+BROWSE_EXPIRES_SECONDS = 3600
+
+
+class ConnectionSeenName:
+    """
+    One name a reader produced (PK ``channel_id`` + ``name``), as the edge last reported it.
+
+    - ``name``: the reader's native name (OPC node id or display name, a point name, what a
+      function emitted), before its policy.
+    - ``hardware_id`` / ``outcome``: what the policy the edge ran made of it (``passed``,
+      ``renamed``, ``dropped``); ``hardware_id`` is None when dropped.
+    - ``count``: messages since the edge's reader started; ``first_seen`` / ``last_seen``.
+    - ``value_type``: ``number``, ``text``, ``bool``... never the value.
+    - ``current_hardware_id`` / ``current_outcome``: read-only, what the policy stored now
+      makes of the name (the edge applies it on its next sync).
+    """
+
+    def __init__(self, channel_id: str = None, name: str = None):
+        self.channel_id: Optional[str] = channel_id
+        self.name: Optional[str] = name
+        self.hardware_id: Optional[str] = None
+        self.outcome: Optional[str] = None
+        self.count: int = 0
+        self.first_seen: Optional[datetime] = None
+        self.last_seen: Optional[datetime] = None
+        self.value_type: Optional[str] = None
+        self.reported_at: Optional[datetime] = None
+        self.current_hardware_id: Optional[str] = None
+        self.current_outcome: Optional[str] = None
+
+    def to_dict(self) -> dict:
+        obj = {"channelId": str(self.channel_id) if self.channel_id else None, "name": self.name,
+               "hardwareId": self.hardware_id, "outcome": self.outcome, "count": self.count,
+               "firstSeen": _ts_out(self.first_seen), "lastSeen": _ts_out(self.last_seen),
+               "valueType": self.value_type, "reportedAt": _ts_out(self.reported_at)}
+        if self.current_outcome is not None:
+            obj["current"] = {"hardwareId": self.current_hardware_id, "outcome": self.current_outcome}
+        return obj
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "ConnectionSeenName":
+        obj = cls(channel_id=data.get("channelId"), name=data.get("name"))
+        obj.hardware_id = data.get("hardwareId")
+        obj.outcome = data.get("outcome")
+        obj.count = int(data.get("count") or 0)
+        obj.first_seen = _ts_in(data.get("firstSeen"))
+        obj.last_seen = _ts_in(data.get("lastSeen"))
+        obj.value_type = data.get("valueType")
+        obj.reported_at = _ts_in(data.get("reportedAt"))
+        return obj
+
+    def __repr__(self) -> str:
+        return f"ConnectionSeenName(name={self.name!r}, outcome={self.outcome!r})"
+
+
+class ConnectionSeenReport:
+    """
+    The last report of one reader (PK ``channel_id``): the digest of the policy the edge
+    ran, so a user sees whether the edge applies the stored one yet; names beyond the
+    edge's cap (``overflow``) and messages it could not name (``invalid``).
+    """
+
+    def __init__(self, channel_id: str = None):
+        self.channel_id: Optional[str] = channel_id
+        self.policy_digest: Optional[str] = None
+        self.overflow: int = 0
+        self.invalid: int = 0
+        self.reported_at: Optional[datetime] = None
+
+    def to_dict(self) -> dict:
+        return {"channelId": str(self.channel_id) if self.channel_id else None,
+                "policyDigest": self.policy_digest, "overflow": self.overflow,
+                "invalid": self.invalid, "reportedAt": _ts_out(self.reported_at)}
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "ConnectionSeenReport":
+        obj = cls(channel_id=data.get("channelId"))
+        obj.policy_digest = data.get("policyDigest")
+        obj.overflow = int(data.get("overflow") or 0)
+        obj.invalid = int(data.get("invalid") or 0)
+        obj.reported_at = _ts_in(data.get("reportedAt"))
+        return obj
+
+
+class ConnectionBrowse:
+    """
+    One OPC-UA browse of a connection (PK ``id``, the command id the edge answers with).
+
+    - ``source``: ``platform`` (requested here, sent as a ``browse_opc`` command) or
+      ``edge`` (run by hand on a manual edge and uploaded).
+    - ``status``: ``requested`` until the edge answers, then the edge's ``ok``,
+      ``partial`` (deadline or node cap reached), ``failed`` or ``refused``. Read as
+      ``expired`` when a request got no answer within ``BROWSE_EXPIRES_SECONDS``.
+    - ``roots``, ``depth``, ``max_nodes``, ``timeout_seconds``: what was asked (the edge
+      clamps them).
+    - ``server``: ``{url, namespaces}``; ``nodes``: what the browser returned (node id,
+      names, class, data type, access, ``readable``, ``hasChildren``, parent, depth).
+    - ``pages`` / ``complete``: the edge uploads at most 1000 nodes per page; ``complete``
+      once its last page arrived.
+    """
+
+    def __init__(self, id: str = None, connection_id: str = None, edge_device_id: str = None):
+        self.id: Optional[str] = id
+        self.connection_id: Optional[str] = connection_id
+        self.edge_device_id: Optional[str] = edge_device_id
+        self.source: str = "platform"
+        self.status: str = BROWSE_REQUESTED
+        self.roots: Optional[list] = None
+        self.depth: Optional[int] = None
+        self.max_nodes: Optional[int] = None
+        self.timeout_seconds: Optional[int] = None
+        self.requested_by: Optional[str] = None
+        self.requested_at: Optional[datetime] = None
+        self.started_at: Optional[str] = None
+        self.finished_at: Optional[str] = None
+        self.error: Optional[str] = None
+        self.truncated: bool = False
+        self.server: Optional[dict] = None
+        self.nodes: list = []
+        self.node_count: int = 0
+        self.pages: int = 0
+        self.complete: bool = False
+
+    def api_id(self) -> str:
+        return str(self.id) if self.id else None
+
+    def set_id(self, value):
+        self.id = str(value)
+
+    def get_id_type(self) -> type:
+        return uuid.UUID
+
+    def to_dict(self, include_nodes: bool = True) -> dict:
+        obj = {"id": str(self.id) if self.id else None,
+               "connectionId": str(self.connection_id) if self.connection_id else None,
+               "edgeDeviceId": self.edge_device_id, "source": self.source, "status": self.status,
+               "roots": self.roots, "depth": self.depth, "maxNodes": self.max_nodes,
+               "timeoutSeconds": self.timeout_seconds, "requestedBy": self.requested_by,
+               "requestedAt": _ts_out(self.requested_at), "startedAt": self.started_at,
+               "finishedAt": self.finished_at, "error": self.error, "truncated": bool(self.truncated),
+               "server": self.server, "nodeCount": self.node_count, "pages": self.pages,
+               "complete": bool(self.complete)}
+        if include_nodes:
+            obj["nodes"] = list(self.nodes or [])
+        return obj
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "ConnectionBrowse":
+        obj = cls(id=data.get("id"), connection_id=data.get("connectionId"),
+                  edge_device_id=data.get("edgeDeviceId"))
+        obj.source = data.get("source") or "platform"
+        obj.status = data.get("status") or BROWSE_REQUESTED
+        obj.roots = data.get("roots")
+        obj.depth = data.get("depth")
+        obj.max_nodes = data.get("maxNodes")
+        obj.timeout_seconds = data.get("timeoutSeconds")
+        obj.requested_by = data.get("requestedBy")
+        obj.requested_at = _ts_in(data.get("requestedAt"))
+        obj.started_at = data.get("startedAt")
+        obj.finished_at = data.get("finishedAt")
+        obj.error = data.get("error")
+        obj.truncated = bool(data.get("truncated"))
+        obj.server = data.get("server")
+        obj.nodes = list(data.get("nodes") or [])
+        obj.node_count = int(data.get("nodeCount") or len(obj.nodes))
+        obj.pages = int(data.get("pages") or 0)
+        obj.complete = bool(data.get("complete"))
+        return obj
+
+    def __repr__(self) -> str:
+        return f"ConnectionBrowse(id={self.id!r}, status={self.status!r}, nodes={self.node_count})"