syndesi 0.5.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

syndesi/adapters/tracehub.py

@@ -0,0 +1,229 @@
+# File : tracehub.py
+# Author : Sébastien Deriaz
+# License : GPL
+"""
+Trace module
+
+A single global instance of TraceHub allows all the syndesi modules and workers
+to emit trace events
+"""
+
+from __future__ import annotations
+
+import json
+import socket
+import struct
+import time
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+from syndesi.adapters.stop_conditions import StopConditionType
+
+STOP_CONDITION_INDICATOR = {
+    StopConditionType.CONTINUATION : "Cont",
+    StopConditionType.FRAGMENT : "Frag",
+    StopConditionType.LENGTH : "Len",
+    StopConditionType.TERMINATION : "Term",
+    StopConditionType.TOTAL : "Tot",
+    StopConditionType.TIMEOUT : "Time"
+}
+
+
+@dataclass(frozen=True)
+class TraceEvent:
+    """
+    Base trace event
+    """
+    descriptor : str
+    timestamp : float
+    t : str = field(default="", init=False)
+
+@dataclass(frozen=True)
+class OpenEvent(TraceEvent):
+    """
+    Adapter open trace event
+    """
+    t : str = field(default="open", init=False)
+
+@dataclass(frozen=True)
+class FragmentEvent(TraceEvent):
+    """
+    Fragment received trace event
+    """
+    data : str
+    length : int
+    t : str = field(default="fragment", init=False)
+
+@dataclass(frozen=True)
+class CloseEvent(TraceEvent):
+    """
+    Adapter close trace event
+    """
+    t : str = field(default="close", init=False)
+
+@dataclass(frozen=True)
+class ReadEvent(TraceEvent):
+    """
+    Adapter read trace event
+    """
+    data : str
+    t : str = field(default="read", init=False)
+    length : int
+    stop_condition_indicator : str
+
+@dataclass(frozen=True)
+class WriteEvent(TraceEvent):
+    """
+    Adapter write trace event
+    """
+    data : str
+    length : int
+    t : str = field(default="write", init=False)
+
+EVENTS : list[type[TraceEvent]] = [FragmentEvent, OpenEvent, CloseEvent, ReadEvent, WriteEvent]
+
+EVENTS_MAP : dict[str, type[TraceEvent]]= {
+    e.t : e for e in EVENTS
+}
+
+DEFAULT_MULTICAST_GROUP = "239.255.42.99"
+DEFAULT_MULTICAST_PORT = 12000
+
+def json_to_trace_event(payload : dict[str, Any]) -> TraceEvent:
+    """
+    Convert json data to TraceEvent
+    """
+    payload_type = payload.get("t", None)
+    if payload_type in EVENTS_MAP:
+        arguments = payload.copy()
+        arguments.pop("t")
+        return EVENTS_MAP[payload_type](**arguments)
+
+    raise ValueError(f"Could not parse payload : {payload}")
+
+class _TraceHub:
+    TTL = 1
+    LOOPBACK = True
+    TRUNCATE_LENGTH = 50
+
+    _udp_addr = (DEFAULT_MULTICAST_GROUP, DEFAULT_MULTICAST_PORT)
+
+    TRUNCATION_TERMINATION = "..."
+    # CHARACTERS_REPLACEMENT = {
+    #     '\n' : '\\n',
+    #     '\r' : '\\r',
+    #     '\t' : '\t',
+
+    # }
+
+    def __init__(self) -> None:
+        #self._udp_sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, 1)
+        #self._udp_sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 1)
+
+        self._udp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
+        self._udp_sock.setblocking(False)
+        self._udp_sock.setsockopt(
+            socket.IPPROTO_IP,
+            socket.IP_MULTICAST_IF,
+            socket.inet_aton("127.0.0.1")
+        )
+        self._udp_sock.setsockopt(
+            socket.IPPROTO_IP,
+            socket.IP_MULTICAST_TTL,
+            struct.pack("b", self.TTL)
+        )
+        self._udp_sock.setsockopt(
+            socket.IPPROTO_IP,
+            socket.IP_MULTICAST_LOOP,
+            struct.pack("b", 1 if self.LOOPBACK else 0)
+        )
+        self._udp_dropped = 0
+
+    def emit_open(self, descriptor : str) -> None:
+        """
+        Emit an open trace event
+        """
+        self._emit_event(OpenEvent(descriptor, time.time()))
+
+    def emit_close(self, descriptor : str) -> None:
+        """
+        Emit a close trace event
+        """
+        self._emit_event(CloseEvent(descriptor, time.time()))
+
+    def _format_data(self, data : bytes) -> str:
+        if len(data) > 4*self.TRUNCATE_LENGTH:
+            # Pre-truncate to avoid working with super long data
+            data = data[:4*self.TRUNCATE_LENGTH]
+
+        str_data = repr(data)[2:-1]
+
+        truncated_str = str_data[:self.TRUNCATE_LENGTH]
+
+        if len(str_data) != len(truncated_str):
+            return truncated_str[:-len(self.TRUNCATION_TERMINATION)] + self.TRUNCATION_TERMINATION
+        return truncated_str
+
+    def emit_fragment(self, descriptor : str, data : bytes) -> None:
+        """
+        Emit a fragment trace event
+        """
+        self._emit_event(FragmentEvent(
+            descriptor,
+            time.time(),
+            self._format_data(data),
+            len(data)
+        ))
+
+    def emit_write(self, descriptor : str, data : bytes) -> None:
+        """
+        Emit a write trace event
+        """
+        self._emit_event(WriteEvent(
+            descriptor,
+            time.time(),
+            self._format_data(data),
+            len(data)
+        ))
+
+    def emit_read(
+            self,
+            descriptor : str,
+            data : bytes,
+            stop_condition_type : StopConditionType
+        ) -> None:
+        """
+        Emit a read trace event
+        """
+
+        indicator = STOP_CONDITION_INDICATOR[stop_condition_type]
+
+        self._emit_event(ReadEvent(
+            descriptor,
+            time.time(),
+            self._format_data(data),
+            len(data),
+            indicator
+        ))
+
+    def _emit_event(self, ev: TraceEvent) -> None:
+        d = asdict(ev)
+        # meta = d.setdefault("meta", {})
+        # if isinstance(meta, dict):
+        #     meta.setdefault("pid", os.getpid())
+
+        payload = json.dumps(d, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
+
+        # if len(payload) > self._udp_max:
+        #     if isinstance(meta, dict):
+        #         meta["truncated"] = True
+        #     payload = payload[: self._udp_max]
+
+        try:
+            self._udp_sock.sendto(payload, self._udp_addr)
+        except (BlockingIOError, InterruptedError, OSError):
+            # drop rather than blocking I/O paths
+            self._udp_dropped += 1
+
+# Public singleton
+tracehub = _TraceHub()

syndesi/adapters/visa.py

@@ -15,9 +15,12 @@ from collections.abc import Callable
 from dataclasses import dataclass
 from enum import Enum
 from types import EllipsisType
+from typing import cast
+
+import pyvisa
+from pyvisa.resources import MessageBasedResource
 
 from syndesi.adapters.adapter_worker import (
-    AdapterDisconnectedEvent,
     AdapterEvent,
     HasFileno,
 )
@@ -28,13 +31,20 @@ from syndesi.tools.errors import AdapterReadError
 from .adapter import Adapter
 from .timeout import Timeout
 
-# --- Runtime optional import
-try:
-    import pyvisa  # type: ignore
-    from pyvisa.resources import Resource  # type: ignore
-except ImportError:
-    pyvisa = None
-    Resource = None
+
+class QueueEvent:
+    """VISA adapter queue event"""
+
+
+class DisconnectedEvent(QueueEvent):
+    """VISA queue disconnected event"""
+
+
+@dataclass
+class FragmentEvent(QueueEvent):
+    """VISA queue new fragment event"""
+
+    fragment: Fragment
 
 
 @dataclass
@@ -100,6 +110,8 @@ class Visa(Adapter):
     It uses pyvisa under the hood
     """
 
+    THREAD_STOP_DELAY = 0.2
+
     def __init__(
         self,
         descriptor: str,
@@ -109,21 +121,12 @@ class Visa(Adapter):
         timeout: None | float | Timeout | EllipsisType = ...,
         encoding: str = "utf-8",
         event_callback: Callable[[AdapterEvent], None] | None = None,
+        auto_open: bool = False,
     ) -> None:
-        super().__init__(
-            descriptor=VisaDescriptor.from_string(descriptor),
-            alias=alias,
-            stop_conditions=stop_conditions,
-            timeout=timeout,
-            encoding=encoding,
-            event_callback=event_callback,
-        )
 
         self._worker_descriptor: VisaDescriptor
         self._descriptor: VisaDescriptor
 
-        self._logger.info("Setting up VISA IP adapter")
-
         if pyvisa is None:
             raise ImportError(
                 "Missing optional dependency 'pyvisa'. Install with:\n"
@@ -131,7 +134,9 @@ class Visa(Adapter):
             )
 
         self._rm = pyvisa.ResourceManager()
-        self._inst: Resource | None = None  # annotation only; no runtime import needed
+        self._inst: MessageBasedResource | None = (
+            None  # annotation only; no runtime import needed
+        )
 
         # We need a socket pair because VISA doesn't expose a selectable fileno/socket
         # So we create a thread to read data and push that to the socket
@@ -142,18 +147,19 @@ class Visa(Adapter):
         self._stop_lock = threading.Lock()
         self.stop = False
 
-        self._fragment_lock = threading.Lock()
-        self._fragment: Fragment | None = None
-        self._event_queue: queue.Queue[AdapterEvent] = queue.Queue()
+        self._event_queue: queue.Queue[QueueEvent] = queue.Queue()
 
-        self._thread = threading.Thread(
-            target=self._internal_thread,
-            args=(self._inst, self._event_queue),
-            daemon=True,
-        )
-        self._thread.start()
+        self._thread: threading.Thread | None = None
 
-        self._inst_lock = threading.Lock()
+        super().__init__(
+            descriptor=VisaDescriptor.from_string(descriptor),
+            alias=alias,
+            stop_conditions=stop_conditions,
+            timeout=timeout,
+            encoding=encoding,
+            event_callback=event_callback,
+            auto_open=auto_open,
+        )
 
     def _default_timeout(self) -> Timeout:
         return Timeout(response=5, action="error")
@@ -186,83 +192,98 @@ class Visa(Adapter):
         return available_resources
 
     def _worker_close(self) -> None:
-        with self._inst_lock:
-            if self._inst is not None:
-                self._inst.close()
-            self._opened = False
+        super()._worker_close()
+        # with self._inst_lock:
+        # Stop the thread
+        if self._thread is not None:
             with self._stop_lock:
                 self.stop = True
+                self._thread.join(timeout=self.THREAD_STOP_DELAY)
+
+        # if self._inst is not None:
+        #     self._inst.close()
+        self._opened = False
 
     def _worker_write(self, data: bytes) -> None:
+        super()._worker_write(data)
         # TODO : Add try around write
-        with self._inst_lock:
-            if self._inst is not None:
-                self._inst.write_raw(data)
+        # TODO : We assume that the instance is thread safe because
+        # it would slow things down to have a lock because the internal thread
+        # would release it every cycle (50ms)
+
+        # with self._inst_lock:
+        if self._inst is not None:
+            self._inst.write_raw(data)
 
     def _worker_read(self, fragment_timestamp: float) -> Fragment:
         self._notify_recv.recv(1)
-        if not self._event_queue.empty():
-            event = self._event_queue.get()
-            if isinstance(event, AdapterDisconnectedEvent):
-                # return Fragment(b"", fragment_timestamp)
-                raise AdapterReadError("Could not read from adapter")
-
-        with self._fragment_lock:
-            if self._fragment is None:
-                raise AdapterReadError("Invalid fragment")
-            output = self._fragment
-            self._fragment = None
-            return output
+        event = self._event_queue.get(block=False, timeout=None)
+
+        if isinstance(event, DisconnectedEvent):
+            # Signal that the adapter disconnected
+            return Fragment(b"", fragment_timestamp)
+        if isinstance(event, FragmentEvent):
+            return event.fragment
+
+        raise AdapterReadError("Invalid queue event")
 
     def _worker_open(self) -> None:
+        super()._worker_open()
         self._worker_check_descriptor()
 
-        if self._inst is None:
-            # NOTE: self._rm is always defined in __init__ when pyvisa is present
-            self._inst = self._rm.open_resource(self._worker_descriptor.descriptor)
+        if self._thread is not None:
+            self.close()
 
-        if not self._opened:
-            # These attributes exist on pyvisa resources
-            self._inst.write_termination = ""
-            self._inst.read_termination = None
-            self._opened = True
+        # if self._inst is None:
+        # NOTE: self._rm is always defined in __init__ when pyvisa is present
 
-        # TODO : Tell the thread to open
+        self._inst = cast(
+            MessageBasedResource,
+            self._rm.open_resource(self._worker_descriptor.descriptor),
+        )
+        self._inst.write_termination = ""
+        self._inst.read_termination = None
 
-    def _internal_thread(
-        self,
-        inst: Resource,
-        event_queue: queue.Queue[AdapterEvent],
-    ) -> None:
-        timeout = 2000
+        self._opened = True
+
+        self._thread = threading.Thread(
+            target=self._internal_thread,
+            args=(self._inst,),
+            daemon=True,
+        )
+        self._thread.start()
+
+    def _internal_thread(self, instance: MessageBasedResource) -> None:
+        timeout = 50e-3
         while True:
             payload = b""
-            with self._fragment_lock:
-                self._fragment = None  # Fragment(b"", None)
+            fragment: Fragment | None = None
             try:
-                inst.timeout = timeout
+                instance.timeout = timeout
             except pyvisa.InvalidSession:
-                pass
+                return
             try:
                 while True:
                     # Read up to an error
-                    payload += inst.read_bytes(1)
-                    inst.timeout = 0
+                    payload += instance.read_bytes(1)  # TODO : Maybe test with read_raw
+                    instance.timeout = 0
             except pyvisa.VisaIOError:
                 # Timeout
                 if payload:
-                    with self._fragment_lock:
-                        if self._fragment is None:
-                            self._fragment = Fragment(payload, time.time())
-                        else:
-                            self._fragment.data += payload
+                    if fragment is None:
+                        fragment = Fragment(payload, time.time())
+                    else:
+                        fragment.data += payload
                     # Tell the session that there's data (write to a virtual socket)
+                    self._event_queue.put(FragmentEvent(fragment))
                     self._notify_send.send(b"1")
             except (TypeError, pyvisa.InvalidSession, BrokenPipeError):
-                event_queue.put(AdapterDisconnectedEvent())
+                self._event_queue.put(DisconnectedEvent())
                 self._notify_send.send(b"1")
+
             with self._stop_lock:
                 if self.stop:
+                    instance.close()
                     break
 
     def _selectable(self) -> HasFileno | None:

syndesi/cli/shell.py

@@ -128,10 +128,9 @@ class AdapterShell:
         self._parser.add_argument(
             "-t",
             "--timeout",
-            nargs="+",
             type=float,
             required=False,
-            default=[2],
+            default=2.0,
             help="Adapter timeout (response)",
         )
         self._parser.add_argument(
@@ -202,7 +201,9 @@ class AdapterShell:
                 auto_open=False,
             )
         elif kind == AdapterType.VISA:
-            self.adapter = Visa(descriptor=args.descriptor, timeout=timeout)
+            self.adapter = Visa(
+                descriptor=args.descriptor, timeout=timeout, auto_open=False
+            )
 
         self.adapter.set_default_timeout(Timeout(action="return_empty"))
 
@@ -245,7 +246,8 @@ class AdapterShell:
         else:
             self.shell.run()
             self.shell.print(
-                f"Opened adapter {self.adapter.descriptor}", style=Shell.Style.NOTE
+                f"Opened adapter {self.adapter.get_descriptor()}",
+                style=Shell.Style.NOTE,
             )
 
     def on_command(self, command: str) -> None:
@@ -276,5 +278,4 @@ class AdapterShell:
             )
         elif isinstance(event, ProtocolFrameEvent):
             data = event.frame.get_payload()
-            # TODO : Catch data from delimited with formatting
             self.shell.print(data)

syndesi/component.py

@@ -57,7 +57,7 @@ class Frame(Generic[T]):
     """
 
     stop_timestamp: float | None
-    stop_condition_type: StopConditionType | None  # None if it's a response timeout
+    stop_condition_type: StopConditionType
     previous_read_buffer_used: bool
     response_delay: float | None
 
@@ -133,7 +133,7 @@ class Component(ABC, Generic[T]):
     """
 
     def __init__(self, logger_alias: LoggerAlias) -> None:
-        super().__init__()
+        #super().__init__()
         self._logger = logging.getLogger(logger_alias.value)
 
     # ==== open ====

syndesi/protocols/delimited.py

@@ -9,23 +9,11 @@ command-like formats with specified delimiters (like \\n, \\r, \\r\\n, etc...)
 from collections.abc import Callable
 from types import EllipsisType
 
-from syndesi.adapters.adapter_worker import (
-    AdapterDisconnectedEvent,
-    AdapterEvent,
-    AdapterFrameEvent,
-)
-
 from ..adapters.adapter import Adapter
 from ..adapters.stop_conditions import StopCondition, Termination
 from ..adapters.timeout import Timeout
 from ..component import AdapterFrame, ReadScope
-from .protocol import (
-    Protocol,
-    ProtocolDisconnectedEvent,
-    ProtocolEvent,
-    ProtocolFrame,
-    ProtocolFrameEvent,
-)
+from .protocol import Protocol, ProtocolEvent, ProtocolFrame
 
 
 class DelimitedFrame(ProtocolFrame[str]):
@@ -70,7 +58,10 @@ class Delimited(Protocol[str]):
         event_callback: Callable[[ProtocolEvent], None] | None = None,
         receive_termination: str | None = None,
     ) -> None:
-        if not isinstance(termination, str) or isinstance(termination, bytes):
+        self._encoding = encoding
+        if isinstance(termination, bytes):
+            termination = termination.decode(self._encoding)
+        elif not isinstance(termination, str):
             raise ValueError(
                 f"end argument must be of type str or bytes, not {type(termination)}"
             )
@@ -79,7 +70,6 @@ class Delimited(Protocol[str]):
         else:
             self._receive_termination = receive_termination
         self._termination = termination
-        self._encoding = encoding
         self._response_formatting = format_response
 
         adapter.set_stop_conditions(
@@ -89,8 +79,6 @@ class Delimited(Protocol[str]):
 
         self._adapter.set_event_callback(self._on_event)
 
-        # TODO : Disable encoding/decoding when encoding==None
-
     def __str__(self) -> str:
         if self._receive_termination == self._termination:
             return f"Delimited({self._adapter},{repr(self._termination)})"
@@ -153,16 +141,16 @@ class Delimited(Protocol[str]):
         )
         return frame.get_payload()
 
-    def _on_event(self, event: AdapterEvent) -> None:
+    # def _on_event(self, event: AdapterEvent) -> None:
 
-        if self._event_callback is not None:
-            output_event: ProtocolEvent | None = None
-            if isinstance(event, AdapterDisconnectedEvent):
-                output_event = ProtocolDisconnectedEvent()
-            if isinstance(event, AdapterFrameEvent):
-                output_event = ProtocolFrameEvent(
-                    frame=self._adapter_to_protocol(event.frame)
-                )
+    #     if self._event_callback is not None:
+    #         output_event: ProtocolEvent | None = None
+    #         if isinstance(event, AdapterDisconnectedEvent):
+    #             output_event = ProtocolDisconnectedEvent()
+    #         if isinstance(event, AdapterFrameEvent):
+    #             output_event = ProtocolFrameEvent(
+    #                 frame=self._adapter_to_protocol(event.frame)
+    #             )
 
-            if output_event is not None:
-                self._event_callback(output_event)
+    #         if output_event is not None:
+    #             self._event_callback(output_event)

syndesi/protocols/modbus.py

@@ -43,7 +43,6 @@ from math import ceil
 from types import EllipsisType
 from typing import cast
 
-from syndesi.adapters.adapter_worker import AdapterEvent
 from syndesi.component import AdapterFrame
 
 from ..adapters.adapter import Adapter
@@ -1422,8 +1421,6 @@ class Modbus(Protocol[ModbusSDU]):
     def _default_timeout(self) -> Timeout | None:
         return Timeout(response=1, action="error")
 
-    def _on_event(self, event: AdapterEvent) -> None: ...
-
     def _protocol_to_adapter(self, protocol_payload: ModbusSDU) -> bytes:
         if isinstance(protocol_payload, SerialLineOnlySDU):
             if self._modbus_type == ModbusType.TCP: