SnowSignal 0.1.1__py3-none-any.whl → 0.1.4.2__py3-none-any.whl

snowsignal/configure.py

@@ -20,12 +20,13 @@ class ConfigArgs(NamedTuple):
     mesh_port: int
     other_relays: list[str]
     log_level: str
+    decode_pvaccess: bool
 
 
 def configure(argv: Sequence[str] | None = None) -> ConfigArgs:
     """Setup configuration for the SnowSignal service"""
 
-    p = configargparse.ArgParser()
+    p = configargparse.ArgParser(prog="snowsignal")
     # Remember to add new arguments to the Args class above!
     p.add_argument(
         "-t",
@@ -66,6 +67,7 @@ def configure(argv: Sequence[str] | None = None) -> ConfigArgs:
         default="info",
         help="Logging level",
     )
+    p.add_argument("--decode-pvaccess", action="store_true", help="Attempt to decode and log to INFO pvaccess messages")
     # Remember to add new arguments to the Args class above!
 
     # config = p.parse_args(argv)
@@ -82,10 +84,12 @@ def configure(argv: Sequence[str] | None = None) -> ConfigArgs:
             loglevel = logging.INFO
         case "debug":
             loglevel = logging.DEBUG
+        case _:
+            loglevel = logging.WARNING
 
     if loglevel < logging.INFO:
         logging.basicConfig(
-            format="%(asctime)s - %(levelname)s - " "%(name)s.%(funcName)s: %(message)s",
+            format="%(asctime)s - %(levelname)s - %(name)s.%(funcName)s: %(message)s",
             encoding="utf-8",
             level=loglevel,
         )
@@ -100,7 +104,7 @@ def configure(argv: Sequence[str] | None = None) -> ConfigArgs:
             "Broadcast port (%i) and mesh port (%i) may not be the same", config.broadcast_port, config.mesh_port
         )
         raise ValueError(
-            f"Broadcast port ({config.broadcast_port}) and " f"mesh port ({config.mesh_port}) may not be the same"
+            f"Broadcast port ({config.broadcast_port}) and mesh port ({config.mesh_port}) may not be the same"
         )
 
     return config

snowsignal/dockerfile

@@ -2,8 +2,10 @@
 # psutils requirement in requirements.txt
 FROM python:3.12-slim
 
+ARG PIPARGS=
+
 # Token has read_api and read_repository permissions and so does not need to be kept secret
 # However it will expire
-RUN pip install SnowSignal --index-url https://gitlab.stfc.ac.uk/api/v4/projects/5671/packages/pypi/simple 
+RUN pip install SnowSignal ${PIPARGS} --no-cache-dir --index-url https://gitlab.stfc.ac.uk/api/v4/projects/5671/packages/pypi/simple 
 
 CMD ["python", "-m", "snowsignal"]

snowsignal/netutils.py

@@ -65,9 +65,7 @@ def get_from_iface(
         if snicaddr.family == family:
             return getattr(snicaddr, attribute)
 
-    raise ResourceNotFoundException(
-        f"Could not identify the {family}, " "{attribute} associated with interface {iface}"
-    )
+    raise ResourceNotFoundException(f"Could not identify the {family}, {attribute} associated with interface {iface}")
 
 
 def get_localipv4_from_iface(iface: str) -> str:

snowsignal/packet.py

@@ -48,6 +48,10 @@ class Packet:
     ip_chksum: int | None = None
     ip_src_addr: str | None = None
     ip_dst_addr: str | None = None
+    ip_length: int | None = None
+    ipv4_identification: int | None = None
+    ipv4_more_fragments: bool = False
+    ipv4_fragmented_offset: int | None = None
 
     udp_src_port: int | None = None
     udp_dst_port: int | None = None
@@ -93,6 +97,14 @@ class Packet:
         if self.ip_version != 4:
             return
 
+        self.ip_length = iph[2]
+        self.ipv4_identification = iph[3]
+
+        # Decode fragment data
+        fragment_flag_and_offset = iph[4]
+        self.ipv4_more_fragments = bool(fragment_flag_and_offset & 0x2000)
+        self.ipv4_fragmented_offset = fragment_flag_and_offset & 0x1FFF
+
         # Calculate the length (of the header?)
         ihl = version_ihl & 0xF
         self._iph_length = ihl * 4
@@ -172,3 +184,21 @@ class Packet:
     def change_ethernet_source(self, newmac) -> None:
         """Change packet Ethernet source to a new MAC address"""
         self.raw = self.raw[0:6] + newmac + self.raw[12:]
+
+    def is_ipv4_fragmented(self) -> bool:
+        """Check if the IPv4 packet is fragmented"""
+        # Check for IP packet fragmentation, only IPv4 packets can be fragmented
+        # https://en.wikipedia.org/wiki/IPv4#Fragmentation_and_reassembly
+        # If a fragment then the More Fragments flag is set True for the first fragment and subsequent flags until the
+        # last segment in which the More Fragments flag is False. However, the Fragment Offset is only zero
+        # in the first fragment.
+        # We can identify a last fragment by the More Fragments flag being False and the Fragment Offset being non-zero.
+        # A non-fragmented IP packet will have the More Fragments flag set False and the Fragment Offset equal to zero.
+        logger.debug(
+            "ip_version = %i, ipv4_more_fragments = %s, ipv4_fragmented_offset = %i",
+            self.ip_version,
+            self.ipv4_more_fragments,
+            self.ipv4_fragmented_offset,
+        )
+
+        return self.ip_version == 4 and (self.ipv4_more_fragments or self.ipv4_fragmented_offset != 0)

snowsignal/pva_packet.py

@@ -0,0 +1,394 @@
+"""
+Decode and understand PVAccess Protocol packets
+Protocol specification is available here: https://docs.epics-controls.org/en/latest/pv-access/protocol.html
+"""
+
+import dataclasses
+import logging
+import socket
+import struct
+import traceback
+from enum import Enum, IntEnum, unique
+from struct import unpack
+
+from .packet import BadPacketException, Packet
+
+logger = logging.getLogger(__name__)
+
+
+@unique
+class PVAccessMessageType(Enum):
+    """PVAccess Application Message Type"""
+
+    BEACON = 0x00
+    VALIDATION = 0x01
+    ECHO = 0x02
+    SEARCH_REQUEST = 0x03
+    SEARCH_RESPONSE = 0x04
+    CREATE_CHANNEL = 0x07
+    DESTROY_CHANNEL = 0x08
+    GET = 0x0A
+    PUT = 0x0B
+    PUTGET = 0x0C
+    MONITOR = 0x0D
+    ARRAY = 0x0E
+    DESTROY_REQUEST = 0x0F
+    CHANNEL_PROCESS = 0x10
+    GET_INTROSPECT = 0x11
+    MESSAGE = 0x12
+    CHANNEL_RPC = 0x14
+    CANCEL_REQUEST = 0x15
+
+
+@unique
+class Endianness(IntEnum):
+    """Endianness of message"""
+
+    LITTLEEND = 0
+    BIGEND = 1
+
+    def unpack_char(self) -> str:
+        """Convert endianness to a Python struct format string, see
+        https://docs.python.org/3/library/struct.html#format-strings"""
+        match self.value:
+            case self.LITTLEEND:
+                return "<"
+            case self.BIGEND:
+                return ">"
+            case _:
+                raise IndexError("Unknown / Impossible Endianness")
+
+
+@dataclasses.dataclass
+class PVAccessMessageHeader:
+    """PVAccess Message Header decoder
+
+    https://docs.epics-controls.org/en/latest/pv-access/Protocol-Messages.html#message-header
+    """
+
+    @unique
+    class MessageType(IntEnum):
+        """Message header is type application or control"""
+
+        APPLICATION = 0
+        CONTROL = 1
+
+    @unique
+    class Segmentation(Enum):
+        """Message is segmented, and what type of segment"""
+
+        NOT_SEGMENTED = 0
+        SEGMENT_START = 1
+        SEGMENT_MIDDLE = 2
+        SEGMENT_END = 3
+
+    @unique
+    class Role(IntEnum):
+        """Message is from a client or server"""
+
+        CLIENT = 0
+        SERVER = 1
+
+    raw: bytes  # The raw original bytes of the header
+
+    magic: int  # Must be 0xCA for PVAccess
+    version: int
+    msgtype: MessageType
+    segmented: Segmentation
+    role: Role
+    endian: Endianness
+    message_command: PVAccessMessageType
+    payload_size: int
+
+    def __init__(self, raw: bytes) -> None:
+        """Decode message header bytes"""
+
+        self.raw = raw
+
+        try:
+            msg_header = self.raw[0:8]
+
+            # Decode the first four bytes of the header
+            # This also serves as a loose confirmation that this is a PVAccess protocol message
+            # due to the magic bytes. Due to chance we'll only try to process mistakenly one
+            # in 256 times. We decode only the first four bytes initially because we need to
+            # know the endianness of the message
+            pvh = unpack("BBBB", msg_header[0:4])
+            if pvh[0] != 0xCA:
+                logger.debug("Magic bytes were %s instead of 0xCA", hex(pvh[0]))
+                raise BadPacketException(f"Magic bytes were {hex(pvh[0])} instead of 0xCA")
+
+            self.magic = pvh[0]
+            self.version = pvh[1]
+            self.message_command = PVAccessMessageType(pvh[3])
+
+            # Flags are packed in individual bits, or in one case a pair of bits, within a single byte
+            # We need to know the endianness to decode the integer that follows
+            flags = int(pvh[2])
+            self.msgtype = self.MessageType(flags & 1)
+            self.segmented = self.Segmentation((flags >> 4) & 0b11)
+            self.role = self.Role((flags >> 6) & 1)
+            self.endian = Endianness((flags >> 7) & 1)
+
+            # Payload size, here we need to know the endianness
+            pvhsize = unpack(f"{self.endian.unpack_char()}I", msg_header[4:8])
+            self.payload_size = pvhsize[0]
+
+        except Exception as e:
+            raise BadPacketException from e
+
+
+def decode_pvaccess_size(payload: bytes, endianness: Endianness, start_byte: int = 0) -> tuple[int, int]:
+    """Decode a string or array size in the PVAccess Protocol format.
+
+    https://docs.epics-controls.org/en/latest/pv-access/Protocol-Encoding.html#sizes
+
+    We require a set of bytes to decode and optionally where in the bytes to start. This means that
+    part of a message payload starting at the size or the entire payload with a pointer to the start
+    of the size may be passed in. We then return a tuple of the size and a pointer to the byte after
+    the decode."""
+
+    endchar = endianness.unpack_char()
+
+    # Calculate the length of the string or array. Usually this is encoded in a single byte, but
+    # if the value of that byte is 254 then there is an integer which holds the size instead
+    sizedecode_byte = unpack("B", payload[start_byte : start_byte + 1])
+    bytes_for_decode = 1
+
+    if sizedecode_byte[0] == 254:
+        sizedecode_int = unpack(f"{endchar}I", payload[start_byte + 1 : start_byte + 5])
+
+        pvasize = sizedecode_int[0]
+        bytes_for_decode = 5
+    elif sizedecode_byte[0] == 255:
+        # I'm not sure why we're not using 0 to represent 0?
+        pvasize = 0
+    else:
+        pvasize = sizedecode_byte[0]
+
+    return (pvasize, start_byte + bytes_for_decode)
+
+
+def decode_pvaccess_string(payload: bytes, endianness: Endianness, start_byte: int = 0) -> tuple[str, int]:
+    """
+    Decode a PVAccess Protocol string
+
+    https://docs.epics-controls.org/en/latest/pv-access/Protocol-Encoding.html#strings
+
+    This is basically an integer size specified in the usual PVAccess Protocol way
+    (see decode_pvaccess_size()) and then an array of UTF-8 bytes
+    """
+
+    # First get the length of the string
+    (pvastrlen, new_start_byte) = decode_pvaccess_size(payload, endianness, start_byte)
+
+    # Then get the string
+    pvastr_unpacked = unpack(f"!{pvastrlen}s", payload[new_start_byte : new_start_byte + pvastrlen])
+    pvastr = pvastr_unpacked[0].decode("utf8")
+
+    return (pvastr, new_start_byte + pvastrlen)
+
+
+@dataclasses.dataclass
+class PVAccessBeaconMessage:
+    """
+    PVAccess Beacon Message decode
+
+    https://docs.epics-controls.org/en/latest/pv-access/Protocol-Messages.html#cmd-beacon-0x00
+    """
+
+    raw: bytes
+
+    guid: str
+    flags: bytes
+    beacon_sequence_id: int
+    change_count: int
+    server_address: str  # IPv4Address | IPv6Address
+    server_port: int
+    protocol: str
+
+    def __init__(self, raw: bytes, endianness: Endianness) -> None:
+        """Decode beacon message payload"""
+
+        self.raw = raw
+
+        try:
+            msg_payload = self.raw
+
+            endchar = endianness.unpack_char()
+            pbm = unpack(f"{endchar}12sBBH16sH", msg_payload[0:34])
+
+            self.guid = pbm[0].hex()
+            self.flags = pbm[1]
+            self.beacon_sequence_id = pbm[2]
+            self.change_count = pbm[3]
+            self.server_address = pbm[4].hex()  # ip_address(pbm[4])
+            self.server_port = pbm[5]
+
+            (pvastr, _) = decode_pvaccess_string(msg_payload[34:], endianness)
+            self.protocol = pvastr
+        except Exception as e:
+            raise BadPacketException from e
+
+
+@dataclasses.dataclass
+class PVAccessSearchMessage:
+    """PVAccess Search Request Message decode
+
+    https://docs.epics-controls.org/en/latest/pv-access/Protocol-Messages.html#cmd-search-0x03
+    """
+
+    raw: bytes
+
+    search_sequence_id: int
+    flags: bytes
+    reponse_address: str  # IPv4Address | IPv6Address
+    response_port: int
+    protocols: list[str]
+
+    @dataclasses.dataclass
+    class Channel:
+        search_instance_id: int
+        channelname: str
+
+        def __repr__(self) -> str:
+            return f"{self.search_instance_id} / {self.channelname}"
+
+    channels: list[Channel]
+
+    def __repr__(self) -> str:
+        return f"sid: {self.search_sequence_id} flags: {self.flags} raddr: {self.reponse_address} rport: {self.response_port} protos: {self.protocols}"
+
+    def __init__(self, raw: bytes, endianness: Endianness) -> None:
+        """Decode search message payload"""
+
+        self.raw = raw
+
+        try:
+            msg_payload = self.raw
+
+            endchar = endianness.unpack_char()
+            psm = unpack(f"{endchar}IB3x16sH", msg_payload[0:26])
+
+            self.search_sequence_id = psm[0]
+            self.flags = psm[1]
+            self.reponse_address = psm[2].hex()  # ip_address(pbm[4])
+            self.response_port = psm[3]
+
+            # Decode the array of protocol strings
+            (protocol_strings_count, payload_pointer) = decode_pvaccess_size(msg_payload, endianness, 26)
+            self.protocols: list[str] = []
+            for x in range(protocol_strings_count):
+                (protocol_string, payload_pointer) = decode_pvaccess_string(msg_payload, endianness, payload_pointer)
+                self.protocols.append(protocol_string)
+
+            # Decode the array of channel searches
+            # Note that the spec uses a different size / count for this
+            cc_unpacked = unpack(f"{endchar}H", msg_payload[payload_pointer : payload_pointer + 2])
+            channels_count = cc_unpacked[0]
+            payload_pointer = payload_pointer + 2
+
+            # Get list of channels. This is an array of structs, where the structs are an integer identifier
+            # and a channel name string. We catch struct unpack exceptions because an earlier version of SnowSignal
+            # was rebroadcasting truncated strings
+            self.channels: list[PVAccessSearchMessage.Channel] = []
+            try:
+                for x in range(channels_count):
+                    # Get search instance ID
+                    chans_unpacked = unpack(f"{endchar}I", msg_payload[payload_pointer : payload_pointer + 4])
+                    search_instance_id = chans_unpacked[0]
+                    payload_pointer = payload_pointer + 4
+
+                    # Get channelname string
+                    (channame_string, payload_pointer) = decode_pvaccess_string(
+                        msg_payload, endianness, payload_pointer
+                    )
+
+                    self.channels.append(self.Channel(search_instance_id, channame_string))
+            except struct.error as e:
+                logger.debug(
+                    "Unexpected termination of search channel array, possible truncated packet or malformed channel count"
+                )
+                logger.debug(
+                    "%s, channels_count: %i no_channels_found: %i channels_found: %s",
+                    self,
+                    channels_count,
+                    len(self.channels),
+                    self.channels,
+                )
+                raise BadPacketException from e
+
+        except Exception as e:
+            raise BadPacketException from e
+
+
+def log_pvaccess(payload: bytes, packet_src_ip: str | None, source: str = "Rebroadcasting") -> None:
+    """Log details of a PVAccess message payload"""
+    # We trap all BadPacketExceptions because
+    try:
+        # Decode the PVAcccess Protocol message header
+        pvamgshdr = PVAccessMessageHeader(payload)
+
+        # Construct a string describing the source for the later log messages
+        if packet_src_ip:
+            try:
+                # Try to determine the name of the source machine, but such a thing may not exist
+                srcname = socket.gethostbyaddr(packet_src_ip)[0] if packet_src_ip else "Unknown"
+                srcstring = f"{packet_src_ip} --> {srcname}"
+            except socket.herror:
+                srcstring = f"{packet_src_ip}"
+        else:
+            srcstring = "Unspecified"
+
+        messagehdr_str = (
+            f"{source} {pvamgshdr.message_command.name} (v{pvamgshdr.version}) "
+            f"[Flags: {pvamgshdr.msgtype.name},{pvamgshdr.segmented.name},{pvamgshdr.role.name},{pvamgshdr.endian.name}] "
+            f"from {srcstring}"
+        )
+
+        pva_message = payload[8:]
+        match pvamgshdr.message_command:
+            case PVAccessMessageType.BEACON:
+                pvabeaconmsg = PVAccessBeaconMessage(pva_message, pvamgshdr.endian)
+                logger.info(
+                    "%s: self-identifies as %s %s:%i;%s with update counters beacon:%i, PVs:%i",
+                    messagehdr_str,
+                    pvabeaconmsg.protocol,
+                    pvabeaconmsg.server_address,
+                    pvabeaconmsg.server_port,
+                    pvabeaconmsg.guid,
+                    pvabeaconmsg.beacon_sequence_id,
+                    pvabeaconmsg.change_count,
+                )
+            case PVAccessMessageType.SEARCH_REQUEST:
+                # Seems to work for pvxs and Phoebus sources
+                pvasearchmsg = PVAccessSearchMessage(pva_message, pvamgshdr.endian)
+                logger.info(
+                    "%s: self-identifies as %s:%i (seq id %i) with protocols %s searching for %s",
+                    messagehdr_str,
+                    pvasearchmsg.reponse_address,
+                    pvasearchmsg.response_port,
+                    pvasearchmsg.search_sequence_id,
+                    pvasearchmsg.protocols,
+                    pvasearchmsg.channels,
+                )
+            case _:
+                # Currently unsupported / unexpected
+                logger.info("Decode of %s message is currently unsupported", pvamgshdr.message_command.name)
+
+    except BadPacketException:
+        # Ignore packets we can't decode
+        logger.info("Packet not decoded; invalid or malformed PVAccess Protocol?")
+        logger.info("Bad PVAccess packet from %s : %s", packet_src_ip, payload)
+        logging.error(traceback.format_exc())
+
+
+def log_pvaccess_packet(packet: Packet) -> None:
+    """Details of a PVAccess message packet"""
+    # Check packet is minimum length to support a PVAccess protocol header
+    if packet.udp_length and packet.udp_length >= 8:
+        payload = packet.get_udp_payload()
+        packet_src_ip = packet.ip_src_addr
+        log_pvaccess(payload, packet_src_ip, "Received")
+    else:
+        logger.debug("Received from %s payload that was not PVAccess Protocol message", packet.ip_src_addr)

snowsignal/snowsignal.py

@@ -1,6 +1,7 @@
 """SnowSignal - UDP Broadcast Relay"""
 
 import asyncio
+import importlib.metadata
 import ipaddress
 import logging
 import os
@@ -89,7 +90,10 @@ async def main(argv: Sequence[str] | None = None, loop_forever: bool = True):
 
     # Configure this relay
     config = configure(argv)
-    logger.info("Starting with configuration %s", config)
+    try:
+        logger.info("Starting version %s with configuration %s", importlib.metadata.version("snowsignal"), config)
+    except importlib.metadata.PackageNotFoundError:
+        logger.info("Starting local version with configuration %s", config)
 
     # Get the local IP address
     # TODO: Properly support IPv6
@@ -116,7 +120,7 @@ async def main(argv: Sequence[str] | None = None, loop_forever: bool = True):
     # Loop forever, but if in swarm mode periodically recheck the relays
     while loop_forever:
         await asyncio.sleep(10)
-        if swarmmode:
+        if swarmmode and not config.other_relays:
             # Check to see if remote relays have changed
             # e.g. containers have restarted
             udp_relay_transmit.set_remote_relays(discover_relays())

snowsignal/udp_relay_receive.py

@@ -16,6 +16,8 @@ from typing import Any
 
 from .configure import ConfigArgs
 from .netutils import get_broadcast_from_iface, get_macaddress_from_iface
+from .packet import BadPacketException, Packet
+from .pva_packet import log_pvaccess
 
 logger = logging.getLogger(__name__)
 
@@ -38,8 +40,10 @@ class UDPRelayReceive(asyncio.DatagramProtocol):
 
         if config:
             self._iface = config.target_interface
+            self._decode_pvaccess = config.decode_pvaccess
         else:
             self._iface = "eth0"
+            self._decode_pvaccess = False
 
         # Assume the MAC address is immutable
         self._mac = get_macaddress_from_iface(self._iface)
@@ -64,6 +68,7 @@ class UDPRelayReceive(asyncio.DatagramProtocol):
         """Calculate UDP checksum, using the IP and UDP parts of the packet,
         and change the existing packet UDP checksum with the newly calculcated
         checksum"""
+        logger.debug("Recalculating UDP checksum")
 
         # The UDP checksum algorithm is defined in RFC768
         # https://www.rfc-editor.org/rfc/rfc768.txt
@@ -117,7 +122,7 @@ class UDPRelayReceive(asyncio.DatagramProtocol):
         return ip_packet
 
     def datagram_received(self, data: bytes, addr: tuple[str | Any, int]) -> None:
-        """Receive a UDP message and forward it to the remote relays"""
+        """Receive a UDP message and forward it any listeners on our local broadcast network segment"""
         logger.debug(
             "Received from %s for rebroadcast on port %i message: %r",
             addr,
@@ -126,19 +131,24 @@ class UDPRelayReceive(asyncio.DatagramProtocol):
         )
 
         # Simple verification of the received payload, and remove the bytes
-        # confirming that this is for us
+        # confirming that this is for us. We also remove the ethernet frame as the
+        # sendto() below will take care of that part
         if data[0:2] == b"SS":
             data = data[2:]
         else:
             logger.debug("Malformed packet received")
             return
 
-        # TODO: Apply any filters
+        # Check if this is a UDP packet and if it is recalculate the UDP checksum
+        try:
+            packet = Packet(data)
+            packet.decode_ip()
+            packet.decode_udp()
+            data = self.recalculate_udp_checksum(data[14:])
+        except BadPacketException:
+            pass
 
-        # We can't use the data as is for some reason but need to recalculate the
-        # UDP checksum. We also remove the ethernet frame as the sendto() below
-        # will take care of that part
-        data = self.recalculate_udp_checksum(data[14:])
+        # TODO: Apply any filters
 
         # TODO: The code above does not change the IP source address
         # If we're on a different network segment then we should switch the
@@ -146,7 +156,7 @@ class UDPRelayReceive(asyncio.DatagramProtocol):
         # then require recomputing checksums. Note: is this required? We
         # are sending to the broadcast address in the sendto() below
 
-        # TODO: Logic to validate what we're receiving as a PVAccess message
+        # TODO: Logic to validate what we're receiving a PVAccess message
         # Note that although doing the validation on receipt means we're doing
         # it for every relay (instead of once if we did it on send), it's much
         # safer to do it on receipt since it means we don't have to trust the
@@ -156,11 +166,35 @@ class UDPRelayReceive(asyncio.DatagramProtocol):
         # this asynchronous but unfortunately that wouldn't allow us to
         # control the IP headers. Is there another way to resolve that?
 
+        # Use this unusual conditional in order to avoid expensive
+        # decoding operations when we're not debugging
+        if self._decode_pvaccess and logger.isEnabledFor(logging.INFO):
+            # Construct a fake ethernet header so we can decode the IP address using existing
+            # functionality. This will only work if the original source used IPv4.
+            # TODO: This indicates a flaw in the current logic. The received broadcast and the
+            # rebroadcast must use the same IP version (IPv4 or IPv6) or the message will
+            # be mangled
+            packet = Packet(b"\xff\xff\xff\xff\xff\xff\x02B\xac\x16\x00\x02\x08\x00" + data)
+            packet.decode_ip()
+            packet.decode_udp()
+            logger.debug(Packet)
+
+            packet_src_ip = packet.ip_src_addr
+
+            log_pvaccess(data[28:], packet_src_ip)
+
         # Finally broadcast the new packet
         # It doesn't feel much simpler but we're not using fully raw sockets here
         # but instead letting Python do the work of handling the Ethernet frames
         sendbytes = self._rebroad_sock.sendto(data, (self._broadcast_addr, self.broadcast_port))
-        logger.debug("Broadcast UDP packet of length %d on iface %s: %s", sendbytes, self._iface, data)
+        logger.debug(
+            "Broadcast UDP packet of length %d to (%s,%s) on iface %s: %s",
+            sendbytes,
+            self._broadcast_addr,
+            self.broadcast_port,
+            self._iface,
+            data,
+        )
 
     async def start(self) -> None:
         """Start the UDP server that listens for messages from other relays and broadcasts them"""

snowsignal/udp_relay_transmit.py

@@ -18,17 +18,38 @@ import ipaddress
 import logging
 import socket
 from collections.abc import Sequence
+from dataclasses import dataclass
+
+import cachetools
 
 from .configure import ConfigArgs
 from .netutils import get_localhost_macs, human_readable_mac, identify_pkttype, machine_readable_mac
 from .packet import BadPacketException, EthernetProtocol, Packet
+from .pva_packet import log_pvaccess_packet
 
 logger = logging.getLogger(__name__)
 
 
+@dataclass(init=True, repr=True, eq=True, frozen=True)
+class FragID:
+    """Enough details about an IPv4 fragment to identify its later parts"""
+
+    fragid: int
+    src: str
+    dst: str
+
+
 class UDPRelayTransmit:
     """Listen for UDP broadcasts and transmit to the other relays"""
 
+    # We need a cache to store potential UDP fragments
+    _fragcache = cachetools.TTLCache(maxsize=1024, ttl=1)
+
+    # This next bit is supplied only so that the testing may override it
+    # define ETH_P_ALL    0x0003          /* Every packet (be careful!!!) */
+    # define ETH_P_IP     0x0800          IP packets only; I believe this is IPv4
+    _packet_filter = 0x0800
+
     def __init__(
         self,
         remote_relays: Sequence[ipaddress.IPv4Address | ipaddress.IPv6Address | str],
@@ -54,8 +75,10 @@ class UDPRelayTransmit:
         # otherwise use some sensible defaults
         if config:
             self._iface = config.target_interface
+            self._decode_pvaccess = config.decode_pvaccess
         else:
             self._iface = "eth0"
+            self._decode_pvaccess = False
 
         self._macs = get_localhost_macs()
         self._macs = [machine_readable_mac(x) for x in self._macs]
@@ -89,7 +112,13 @@ class UDPRelayTransmit:
             with socket.socket(sock_family, socket.SOCK_DGRAM) as s:
                 s.setblocking(False)
                 loop = asyncio.get_running_loop()
-                await loop.sock_sendto(s, msgbytes, (str(remote_relay), self.remote_port))
+                bytessent = await loop.sock_sendto(s, msgbytes, (str(remote_relay), self.remote_port))
+                if bytessent < len(msgbytes):
+                    logger.warning(
+                        "Sent truncated message to other SnowSignal nodes; was %i, should be %i",
+                        bytessent,
+                        len(msgbytes),
+                    )
 
     def l1filter(self, ifname: str) -> bool:
         """Check the network interface is as expected"""
@@ -134,27 +163,82 @@ class UDPRelayTransmit:
     def l4filter(self, packet: Packet) -> bool:
         """Tests to perform on Level4 of packet, i.e. UDP Protocol"""
         if packet.udp_dst_port != self.local_port:
-            logger.debug("Wrong UDP destination port: %i", packet.udp_dst_port)
+            logger.debug("Wrong UDP destination port: %i on packet %s", packet.udp_dst_port, packet)
             return False
 
         return True
 
+    def filter_fragment(self, packet: Packet) -> bool:
+        """
+        We have identified this as a UDP packet fragment. If it the first fragment we apply the
+        l4filter and then cache an identifier so that we may apply the result of that filter to
+        the subsequent fragments.
+        """
+
+        # If this is the first packet then if will have a fragment offset of 0. Importantly,
+        # we should still be able to evaluate it as a UDP packet and thus see if it satisfies
+        # our filters. If it doesn't then neither will subsequent fragments. However, if it
+        # does satisfy the filter then so will subsequent fragments which won't have UDP
+        # headers for us to evaluate
+        fragid = FragID(packet.ipv4_identification, packet.ip_src_addr, packet.ip_dst_addr)  # type: ignore
+
+        # If the Fragment Offset is zero then this is the first fragment
+        if packet.ipv4_fragmented_offset == 0:
+            # The first fragment should be a valid UDP packet
+            packet.decode_udp()
+            # If it passes the l4filter then we cache its identifier so that subsequent fragments may be passed
+            if self.l4filter(packet):
+                logger.debug(
+                    "Fragment (%i/%i), a first fragment, passed l4filter",
+                    packet.ipv4_identification,
+                    packet.ipv4_fragmented_offset,
+                )
+                self._fragcache[fragid] = packet
+            else:
+                logger.debug(
+                    "Fragment (%i/%i) failed l4filter",
+                    packet.ipv4_identification,
+                    packet.ipv4_fragmented_offset,
+                )
+                return False
+        else:
+            # This is a fragment but not the first one. It will therefore not have a valid UDP header to decode.
+            # Instead we check if its identifier is in the cache from an inspection of the first fragement. If it is
+            # then the first fragment passed the l4filter and therefore this one does too.
+            if not self._fragcache[fragid]:
+                logger.debug(
+                    "Fragment (%i/%i) not recognised as continuation of l4filter approved packet",
+                    packet.ipv4_identification,
+                    packet.ipv4_fragmented_offset,
+                )
+                return False
+
+            # If this is the final fragment, indicated by the More Fragments flag being False but the Fragment Offset
+            # being >0, then we need to remove its identifier from the fragment cache
+            if not packet.ipv4_more_fragments:
+                self._fragcache.pop(fragid)
+
+        return True
+
     async def start(self) -> None:
         """Monitor for UDP broadcasts on the specified port"""
+
+        logger.debug("UDPRelayTransmit starting to listen for raw packets")
+
         # create a AF_PACKET type raw socket (thats basically packet level)
         # define ETH_P_ALL    0x0003          /* Every packet (be careful!!!) */
         # define ETH_P_IP     0x0800          IP packets only; I believe this is IPv4
         with socket.socket(
-            socket.AF_PACKET,
+            socket.AF_PACKET,  # type: ignore - not available on Windows
             socket.SOCK_RAW,
-            socket.ntohs(0x0800),  # pylint: disable=no-member
+            socket.ntohs(self._packet_filter),
         ) as sock:
             sock.setblocking(False)
 
             while self._loop_forever:
                 loop = asyncio.get_running_loop()
 
-                raw_packet = await loop.sock_recvfrom(sock, 1024)
+                raw_packet = await loop.sock_recvfrom(sock, 65536)
                 (ifname, proto, pkttype, hatype, addr) = raw_packet[1]
                 raw_packet = raw_packet[0]
                 logger.debug(
@@ -188,21 +272,38 @@ class UDPRelayTransmit:
                         self._loop_forever = self._continue_while_loop()
                         continue
 
-                    # Check Level 4 transport protocol, i.e. UDP
-                    packet.decode_udp()
-                    if not self.l4filter(packet):
-                        logger.debug("Failed l4filter")
-                        self._loop_forever = self._continue_while_loop()
-                        continue
+                    # Check for IP packet fragmentation, only IPv4 packets can be fragmented
+                    # https://en.wikipedia.org/wiki/IPv4#Fragmentation_and_reassembly
+                    if not packet.is_ipv4_fragmented():
+                        # This is an ordinary unfragmented IP packet
+                        # Check Level 4 transport protocol, i.e. UDP
+                        packet.decode_udp()
+                        if not self.l4filter(packet):
+                            logger.debug("Failed l4filter")
+                            self._loop_forever = self._continue_while_loop()
+                            continue
+                    else:
+                        if not self.filter_fragment(packet):  # Note we may l4filter in this call
+                            logger.debug("Failed l4filter of fragment")
+                            self._loop_forever = self._continue_while_loop()
+                            continue
+
                 except BadPacketException as bpe:
                     logger.debug("Malformed packet %r", bpe)
                     self._loop_forever = self._continue_while_loop()
                     continue
 
+                # Use this unusual conditional in order to avoid expensive
+                # decoding operations when we're not debugging
+                if self._decode_pvaccess and logger.isEnabledFor(logging.INFO):
+                    log_pvaccess_packet(packet)
+
                 # Send to other relays
                 await self._send_to_relays_packet(packet)
                 self._loop_forever = self._continue_while_loop()
 
+        logger.warning("UDPRelayTransmit no longer listening for raw packets")
+
     def _continue_while_loop(self) -> bool:
         """This function exists purely to allow unit testing of the start() function above"""
         return self._loop_forever

{snowsignal-0.1.1.dist-info → snowsignal-0.1.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: SnowSignal
-Version: 0.1.1
+Version: 0.1.4.2
 Summary: UDP Broadcast Relay
 Project-URL: Repository, https://github.com/ISISNeutronMuon/SnowSignal
 Author-email: Ivan Finch <ivan.finch@stfc.ac.uk>
@@ -16,14 +16,15 @@ Classifier: Programming Language :: Python
 Classifier: Topic :: System :: Networking
 Classifier: Typing :: Typed
 Requires-Python: >=3.11
+Requires-Dist: cachetools>=5.5
 Requires-Dist: configargparse>=1.7
 Requires-Dist: psutil>=5.9
 Provides-Extra: dist
 Requires-Dist: build>=1.2; extra == 'dist'
-Requires-Dist: twine>=5.1; extra == 'dist'
+Requires-Dist: hatch>=1.14; extra == 'dist'
 Provides-Extra: test
 Requires-Dist: coverage>=7.6; extra == 'test'
-Requires-Dist: ruff>0.6; extra == 'test'
+Requires-Dist: ruff>0.9; extra == 'test'
 Requires-Dist: scapy~=2.0; extra == 'test'
 Description-Content-Type: text/markdown
 

snowsignal-0.1.4.2.dist-info/RECORD

@@ -0,0 +1,14 @@
+snowsignal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+snowsignal/__main__.py,sha256=AehSEZINUEFv4Y2zFjUHpBifeLZNE-KFA2X9vZ890LU,255
+snowsignal/configure.py,sha256=piAZXf36YhW4cgCYl3gVcv0WOo7jTwRJ5qMnm_Zz3Kc,3392
+snowsignal/dockerfile,sha256=_WDaJrTD_iYESymw5e2a33BE0T8YwYDunkQwDKm7ajs,426
+snowsignal/netutils.py,sha256=OCQwhFHtw3uMfD2IelxNztdN5COPLm8Dy8YyD9Z6foI,4777
+snowsignal/packet.py,sha256=upIR1S6_jDRWuzzZ13355oFT78f2CyaujYm7b_LLa1c,7234
+snowsignal/pva_packet.py,sha256=OutH928pUB4V5tDClaiVkcgzY6zxl6inFPb-uDCRs5Y,14153
+snowsignal/snowsignal.py,sha256=Q-mQ9KKMF6lOWvM1lgyoSOwILTB7am2AmhSFosaecd8,5070
+snowsignal/udp_relay_receive.py,sha256=9Hdsm8PrEjcdXsldVk9nh25NuzgARmXw-tZNVhGnVkE,9747
+snowsignal/udp_relay_transmit.py,sha256=CIvNhMQs5XcEVYK-gcGRYdbU6uKrbQ92IB2WYjp924U,13467
+snowsignal-0.1.4.2.dist-info/METADATA,sha256=cEZ_34eVrp-BGLSBjKxkZ7zOR11wHwLTAe_il1BBCqs,11541
+snowsignal-0.1.4.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+snowsignal-0.1.4.2.dist-info/licenses/LICENSE,sha256=Xyl-Ykl-HUYLhQ4bu65pmUbqf8c-deJjxoq7ScjEcrI,1526
+snowsignal-0.1.4.2.dist-info/RECORD,,

{snowsignal-0.1.1.dist-info → snowsignal-0.1.4.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.25.0
+Generator: hatchling 1.27.0
 Root-Is-Purelib: true
 Tag: py3-none-any

snowsignal-0.1.1.dist-info/RECORD

@@ -1,13 +0,0 @@
-snowsignal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-snowsignal/__main__.py,sha256=AehSEZINUEFv4Y2zFjUHpBifeLZNE-KFA2X9vZ890LU,255
-snowsignal/configure.py,sha256=e_dLvanmrPVs6RG1KOZPzscRJF9SbYlwstDMqBJSU90,3180
-snowsignal/dockerfile,sha256=HeJUva1v8-QBjU_yd8NNPSo3YEDYLdHLiSkF3Yf-28Q,386
-snowsignal/netutils.py,sha256=ZQ0cs3JF0Ec1DRWQsLOaLEcBRx6iGK4A6EQITIPac84,4794
-snowsignal/packet.py,sha256=CNnFDpoFdtIwAMBenXrRrOK7UpHAGsOXlI1QOcSWyeM,5696
-snowsignal/snowsignal.py,sha256=bG8I6tiFTixTiRYtDGcETbfqbTwZTb_nVxYvZeCGmvo,4822
-snowsignal/udp_relay_receive.py,sha256=uySOhRKLd2L1epHXYUxHP6M_Ey6NQ4LcN7MKtbH5pdQ,8331
-snowsignal/udp_relay_transmit.py,sha256=gxdS1dVPB4srkRtNA_UPOz3jDZsbfZNjL68VF8DtFuU,8626
-snowsignal-0.1.1.dist-info/METADATA,sha256=PHJ9-4_asNKK_g_cjajA0bDFhSUi9aRHc3rfMXSy7fE,11507
-snowsignal-0.1.1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-snowsignal-0.1.1.dist-info/licenses/LICENSE,sha256=Xyl-Ykl-HUYLhQ4bu65pmUbqf8c-deJjxoq7ScjEcrI,1526
-snowsignal-0.1.1.dist-info/RECORD,,