AcraNetwork 1.2.1__tar.gz → 1.2.4__tar.gz

{acranetwork-1.2.1 → acranetwork-1.2.4}/AcraNetwork/IRIG106/Chapter7/__init__.py

@@ -23,6 +23,7 @@ import logging
 import typing
 from enum import IntEnum
 
+
 ch7_logger = logging.getLogger(__name__)
 
 
@@ -34,6 +35,11 @@ class PTDPContent(IntEnum):
     ETHERNET_MAC = 0x4
     IP = 0x5
     CHAPTER_24 = 0x6
+    ILLEGAL = 0xF
+
+    @classmethod
+    def _missing_(cls, value):
+        return PTDPContent.ILLEGAL
 
 
 class PTDPFragment(IntEnum):
@@ -41,12 +47,17 @@ class PTDPFragment(IntEnum):
     FIRST = 0x1
     MIDDLE = 0x2
     LAST = 0x3
+    ILLEGAL = 0x4
+
+    @classmethod
+    def _missing_(cls, value):
+        return PTDPFragment.ILLEGAL
 
 
 PTDP_HDR_LEN = 0x6  # 24bits x2
 PTFR_HDR_LEN = 0x4  # 1 byte unprotected and 3 bytes protected
 
-
+PTDT_LLP_TRAILER_LEN = 1
 PTDP_MAX_LEN = 0x800
 
 
@@ -77,10 +88,25 @@ def datapkts_to_ptfr(
     # Create a new frame
     ptfr = _new_ptfr(ptfr_len, streamid, golay)
     for ptdp in datapkts_to_ptdp(eth_ch10_packets):
-        # Encapsulate the data in PTDP packets
-        # Add the payload to the PTFR frame. IF we get a remainder then this from is full
+
+        remaining_space = ptfr.remaining_space()
+        # logging.debug(f"Adding ptdp {ptdp} with remaining space {remaining_space}")
+        # If we have a low latency packet but not enough space to insert it full then
+        # insert a fill packet
+        if ptdp.low_latency and remaining_space <= (len(ptdp) + PTDT_LLP_TRAILER_LEN):
+            # logging.warning(f"Need to add fill packet to fill remaining space={remaining_space}")
+            fill = ptdp_fill(remaining_space)
+            remainder = ptfr.add_payload(fill.pack(), False)
+            yield ptfr
+            ptfr = _new_ptfr(ptfr_len, streamid, golay)
+            if len(remainder) > 0:
+                ptfr.add_payload(remainder)
+                # logging.debug(f"Added remainder of len {len(remainder)}. Remainder={ptfr.remaining_space()}")
+
+        # Add the packet and if there is remainder push out the ptfr packet and start a new one
         remainder = ptfr.add_payload(ptdp.pack(), ptdp.low_latency)
-        while remainder != bytes():
+        ch7_logger.debug(f"PTDP ({ptdp}) to be added to PTRF leaving remainder of len={len(remainder)}")
+        while not ptfr.has_space():
             # Spit out the full frame
             yield ptfr
             # Create anew frame
@@ -93,10 +119,11 @@ def datapkts_to_ptfr(
                 remainder = remainder[ptfr_len:]  # If we have more data then save it for the next frame
                 ptfr = _new_ptfr(ptfr_len, streamid, golay)
 
-            # Point to the first offset of the frame.
-            ptfr.ptdp_offset = len(remainder)
-            # Add the PTDS to the frame
-            remainder = ptfr.add_payload(remainder)
+            if len(remainder) > 0:
+                # Point to the first offset of the frame.
+                ptfr.ptdp_offset = len(remainder)
+                # Add the PTDS to the frame
+                remainder = ptfr.add_payload(remainder)
 
 
 def datapkts_to_ptdp(eth_ch10_packets: typing.Iterable[bytes, bool]) -> typing.Generator[PTDP, None, None]:
@@ -164,6 +191,7 @@ class PTDP(object):
             )
 
         self._payload = val
+        self.length = len(val)
 
     def pack(self) -> bytes:
         """
@@ -198,7 +226,7 @@ class PTDP(object):
             raise PTDPLengthError("GolayHdr=len={}. Must be corrupted".format(self.length))
         elif len(buffer[6:]) < self.length:
             raise PTDPRemainingData(
-                "Buffer length={} GolayHdr=len={} fragment={} content={}".format(
+                "Not a full PTDP packet. Rest likely in next packet . Buffer length={} GolayHdr=len={} fragment={} content={}".format(
                     len(buffer[6:]), self.length, self.fragment, self.content
                 )
             )
@@ -222,9 +250,18 @@ class PTDP(object):
         return not self.__eq__(other)
 
     def __repr__(self):
-        return "PTDP: Len={} Content={} Fragment={} LowLatency={}".format(
-            self.length, self.content, self.fragment, self.low_latency
-        )
+        return f"PTDP: Len={self.length} Content={repr(self.content)} Fragment={repr(self.fragment)} LowLatency={self.low_latency}"
+
+
+def ptdp_fill(total_len_min: int) -> PTDP:
+    _p = PTDP()
+    _p.content = PTDPContent.FILL
+    if total_len_min < (PTDP_HDR_LEN + 1):
+        payload_len = 1
+    else:
+        payload_len = total_len_min - PTDP_HDR_LEN
+    _p.payload = struct.pack(f">{payload_len}B", *([0xFF] * payload_len))
+    return _p
 
 
 class PTFR(object):
@@ -255,11 +292,17 @@ class PTFR(object):
     def add_payload(self, buffer: bytes, is_llp: bool = False) -> bytes:
         """
         Add the buffer to the payload ensuring not to go over the length field
-        Return the remained
+        Return the remainder
         """
+        if len(buffer) == 0:
+            raise Exception("Can't add payload of len = 0")
+        if is_llp and len(buffer) + len(self._payload) > self.length:
+            raise Exception(
+                f"LLP packet ({len(buffer)}) cannot be added (ptfrlen={self.length}) as it will push the payload ({len(self.payload)}) into the next frame. Add a fill word"
+            )
         if is_llp and len(self._payload) > 0 and self.llp:
             ch7_logger.debug(
-                "Adding an LLP buffer to some existing llp payload. Shifting original data and adding llp at "
+                f"Adding an LLP buffer({len(buffer)}) to some existing llp payload ({len(self._payload)}). Shifting original data and adding llp at "
                 "start. len={}".format(len(buffer) + 1)
             )
             self.ptdp_offset += len(buffer) + 1
@@ -276,10 +319,11 @@ class PTFR(object):
         elif is_llp and len(self.payload) == 0:
             ch7_logger.debug("Adding first LLP buffer len={}".format(len(buffer) + 1))
             self.llp = True
-            self._payload = buffer + struct.pack(">B", 0xFF)
+            self._payload = buffer + struct.pack(">B", 0x00)  # Further LLPs will be added in front of this one
+            self.ptdp_offset += len(buffer) + 1
         else:
-            ch7_logger.debug("Adding a normal PTDP buffer len={}".format(len(buffer)))
             self._payload += buffer
+            ch7_logger.debug(f"Adding a normal PTDP buffer of len={len(buffer)} to a total len={len(self._payload)}")
 
         if len(self._payload) > self.length:
             len_to_take = self.length - len(self._payload)
@@ -289,6 +333,12 @@ class PTFR(object):
         else:
             return bytes()
 
+    def has_space(self, expected_addition: int = 0) -> bool:
+        return (len(self.payload) + expected_addition) < self.length
+
+    def remaining_space(self) -> int:
+        return self.length - len(self.payload)
+
     def pack(self) -> bytes:
         """
         Convert a PTFR object into a string for transmission. This will return the packed string and the remainder string
@@ -325,10 +375,8 @@ class PTFR(object):
 
     def check_offsets(self, act_offset: int) -> bool:
         if (act_offset != self.ptdp_offset) and (self.ptdp_offset != 2047):
-            ch7_logger.warning(
-                "Offset of unpacked PTDP packet ({}) does not match the declared offset ({})".format(
-                    act_offset, self.ptdp_offset
-                )
+            ch7_logger.debug(
+                f"Offset of unpacked PTDP packet ({act_offset}) does not match the declared offset ({self.ptdp_offset})"
             )
             return False
         elif self.ptdp_offset != 2047:
@@ -383,15 +431,20 @@ class PTFR(object):
                 )
             )
 
+        do_offset_check = True
         if is_llp:
             byte_offset = 0
         elif remainder is None:  # Fake the offset if we have jumped into the middle of a data stream
             byte_offset = self.ptdp_offset
         else:
             byte_offset = -1 * len(remainder)
-        do_offset_check = True
+            if len(remainder) == 0:
+                do_offset_check = False
+
         offset_check_count = 0
+
         while aligned:
+            ch7_logger.debug(f"Starting to check buf of lenght={len(buf)}")
             p = PTDP(self._golay)
             try:
                 buf = p.unpack(buf)
@@ -402,6 +455,7 @@ class PTFR(object):
                         len(buf), e, self.ptdp_offset
                     )
                 )
+
                 yield (None, None, e)
             except PTDPRemainingData as e:
                 aligned = False
@@ -416,12 +470,16 @@ class PTFR(object):
                 yield (None, buf, e)
             else:
                 if not is_llp and do_offset_check and byte_offset >= 0:
+                    # ch7_logger.debug(f"do_offset_check={do_offset_check} byte_offset={byte_offset}")
                     self.check_offsets(byte_offset)
                     do_offset_check = False
                     offset_check_count += 1
                 elif not is_llp and not do_offset_check and offset_check_count < 1:
                     do_offset_check = True
                     byte_offset += len(p)
+                    # ch7_logger.debug(
+                    #    f"Enable do_offset_check because {offset_check_count} < 1 byte_offset={byte_offset} len_p={len(p)}"
+                    # )
                 elif not is_llp:
                     byte_offset += len(p)
 
@@ -433,6 +491,7 @@ class PTFR(object):
                     (next_llp,) = struct.unpack_from(">B", buf)
                     # Check if the next PTDP is low latency before yielding
                     if next_llp == 0xFF:
+                        ch7_logger.debug("Next packet is LLP")
                         is_llp = True
                         buf = buf[1:]
                         byte_offset += len(p) + 1
@@ -456,8 +515,11 @@ class PTFR(object):
                             if len(remainder) > 0:
                                 do_offset_check = False
 
+                # ch7_logger.debug(f"Returning p={repr(p)} and no remainder")
                 yield (p, bytes(), "")
 
+        # ch7_logger.debug("------PTFR expired-----")
+
     def __eq__(self, other: PTFR):
         if not isinstance(other, PTFR):
             return False
@@ -472,6 +534,6 @@ class PTFR(object):
         return not self.__eq__(other)
 
     def __repr__(self):
-        return "PTFR: Length={} StreamID={:#0X} Offset={} LowLatency={}\n".format(
-            self.length, self.streamid, self.ptdp_offset, self.llp
+        return (
+            f"PTFR: Length={self.length} StreamID={self.streamid:#0X} Offset={self.ptdp_offset} LowLatency={self.llp}\n"
         )

{acranetwork-1.2.1 → acranetwork-1.2.4}/AcraNetwork/__version__.py

@@ -106,4 +106,7 @@
 # 1.1.9   - Fixed ARINC ch10 intra packet header
 # 1.2.0   - Moved Chapter 7 into the IRIG106 folder. Removed old Chapter10 directory
 # 1.2.1   - Fixed logging in IP error message
-__version__ = "1.2.1"
+# 1.2.2   - Fixed chapter 7 packet generation
+# 1.2.3   - Last release was nto successful
+# 1.2.4   - Removed exit in the ch7
+__version__ = "1.2.4"

{acranetwork-1.2.1 → acranetwork-1.2.4}/AcraNetwork.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AcraNetwork
-Version: 1.2.1
+Version: 1.2.4
 Summary: Classes and utilities to support Flight Test Instrumentation Ethernet networks
 Home-page: https://github.com/diarmuidcwc/AcraNetwork
 Author: Diarmuid Collins

{acranetwork-1.2.1 → acranetwork-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AcraNetwork
-Version: 1.2.1
+Version: 1.2.4
 Summary: Classes and utilities to support Flight Test Instrumentation Ethernet networks
 Home-page: https://github.com/diarmuidcwc/AcraNetwork
 Author: Diarmuid Collins

{acranetwork-1.2.1 → acranetwork-1.2.4}/examples/validate_pcap.py

@@ -1,15 +1,12 @@
 #!/usr/bin/env python
 
-
+"""
 # Very rudimentary (but fast) validation of recorded data
 # Finds all inetx packets and validate no missing sequence numbers
 # It will also take the url to an axn mem and verify data after downloading
+"""
 
 
-import sys
-
-sys.path.append("..")
-sys.path.append(".")
 import AcraNetwork.Pcap as pcap
 import glob
 import os.path
@@ -24,13 +21,12 @@ from os import mkdir, remove
 import datetime
 from dataclasses import dataclass, field
 import typing
+import zstandard as zstd
+import sys
 
+VERSION = "0.5.0"
 
-VERSION = "0.4.0"
-
-logging.basicConfig(
-    level=logging.INFO, format="%(levelname)-6s %(asctime)-15s %(message)s"
-)
+logging.basicConfig(level=logging.INFO, format="%(levelname)-6s %(asctime)-15s %(message)s")
 
 
 @dataclass
@@ -56,7 +52,7 @@ class Streams:
             return 0
         return int(self.length * 8 * self.pkt_count / (self.end_ts - self.start_ts))
 
-    def timelen(self) ->float:
+    def timelen(self) -> float:
         return self.end_ts - self.start_ts
 
     def drops_to_hist(self):
@@ -66,7 +62,7 @@ class Streams:
         bin_cnt = [0] * bins
         for i in range(bins):
             for s in self.sequence_list:
-                #print(f"{i}:{start_seq}:{s}:{bin_wdth}")
+                # print(f"{i}:{start_seq}:{s}:{bin_wdth}")
                 if (start_seq + bin_wdth * i) <= s < (start_seq + bin_wdth * (i + 1)):
                     bin_cnt[i] += 1
         rstring = "|"
@@ -83,9 +79,7 @@ class Streams:
 
 def create_parser():
     # Argument parser
-    parser = argparse.ArgumentParser(
-        description="Validate inetx sequence numbers quickly in a pcap file"
-    )
+    parser = argparse.ArgumentParser(description="Validate inetx sequence numbers quickly in a pcap file")
     # Common
     parser.add_argument(
         "--folder",
@@ -122,9 +116,7 @@ def create_parser():
         default=False,
         help="print a rough histogram of where the drops happened",
     )
-    parser.add_argument(
-        "--version", action="version", version="%(prog)s {}".format(VERSION)
-    )
+    parser.add_argument("--version", action="version", version="%(prog)s {}".format(VERSION))
 
     return parser
 
@@ -141,6 +133,11 @@ def uri_validator(x):
         return False
 
 
+def is_compressed(x: str) -> bool:
+    (_root, _ext) = os.path.splitext(x)
+    return _ext == ".zst"
+
+
 def main(args):
     roll_over = pow(2, 32)
     fnames = {}
@@ -174,7 +171,7 @@ def main(args):
         first_pcap_time = None
         last_pcap_time = None
         packet_data_vol = 0
-        tmp_folder = "httpdl"
+        tmp_folder = ".tmp"
         loss = 0
         floss = 0
         outf = ""
@@ -195,12 +192,27 @@ def main(args):
                     out_file.write(chunk)
 
                 dlspeed = data_len * 8 / (time.time() - sd) / 1e6
-                logging.info(
-                    "Downloaded {} at {:.1f}Mbps and wrote to {}".format(
-                        pfile, dlspeed, outf
-                    )
-                )
+                logging.info("Downloaded {} at {:.1f}Mbps and wrote to {}".format(pfile, dlspeed, outf))
             p = pcap.Pcap(outf)
+        elif is_compressed(pfile):
+
+            if not os.path.exists(tmp_folder):
+                mkdir(tmp_folder, 0o755)
+            fname_pcap, _ext = os.path.splitext(os.path.basename(pfile))
+            decompressed_f = os.path.join(tmp_folder, fname_pcap)
+            with open(pfile, "rb") as compressed:
+                with open(decompressed_f, "wb") as decompressed:
+                    dctx = zstd.ZstdDecompressor()
+                    dctx.copy_stream(compressed, decompressed)
+
+            # Get sizes
+            compressed_size = os.path.getsize(pfile)
+            decompressed_size = os.path.getsize(decompressed_f)
+
+            # Compute compression ratio
+            compression_ratio = decompressed_size / compressed_size
+            logging.info(f"Compression ratio: {compression_ratio:.2f}x")
+            p = pcap.Pcap(decompressed_f)
         else:
             p = pcap.Pcap(pfile)
         prev_rec_ts = None
@@ -215,34 +227,23 @@ def main(args):
             if prev_rec_ts is not None:
                 if prev_rec_ts > last_pcap_time:
                     delta = prev_rec_ts - last_pcap_time
-                    logging.warning(
-                        f"Record={i + 1} Record timestamp negative jump {delta}s")
+                    logging.warning(f"Record={i + 1} Record timestamp negative jump {delta}s")
             prev_rec_ts = last_pcap_time
 
             packet_data_vol += len(r.payload)
             total_pkt_count += 1
-            if len(r.payload) >= (
-                26 + 0x28
-            ):  # For short packets don't try to decode them as inetx
+            if len(r.payload) >= (26 + 0x28):  # For short packets don't try to decode them as inetx
                 # pull out the key fields
-                (
-                    dst_port,
-                    udp_len,
-                    checksum,
-                    control,
-                    stream_id,
-                    seq,
-                    _len,
-                    ptpsec,
-                    ptpnsec
-                ) = struct.unpack_from(">HHHIIIIII", r.payload, 0x24)
+                (dst_port, udp_len, checksum, control, stream_id, seq, _len, ptpsec, ptpnsec) = struct.unpack_from(
+                    ">HHHIIIIII", r.payload, 0x24
+                )
                 if control == args.control:
                     if stream_id in streams:
                         stream = streams[stream_id]
                         if seq != (stream.sequence + 1) % roll_over:
-                            pkt_ts = datetime.datetime.fromtimestamp(
-                                r.sec + r.usec * 1e-6
-                            ).strftime("%H:%M:%S.%f %d %b")
+                            pkt_ts = datetime.datetime.fromtimestamp(r.sec + r.usec * 1e-6).strftime(
+                                "%H:%M:%S.%f %d %b"
+                            )
                             if seq < stream.sequence:
                                 logging.warning(
                                     "Source Restarted. File={} PktNum={} StreamID={:#0X} PrevSeq={} "
@@ -267,22 +268,31 @@ def main(args):
                                             stream.sequence,
                                             seq,
                                             loss,
-                                            loss * stream.length
+                                            loss * stream.length,
                                         )
                                     )
                                 loss_count += loss
-                                loss_data += (loss * stream.length)
+                                loss_data += loss * stream.length
                                 stream.dropcnt += loss
                                 stream.sequence_list.append(stream.sequence + 1)
                                 floss += loss
                         stream.sequence = seq
                         stream.pkt_count += 1
-                        stream.end_ts = ptpsec + ptpnsec/1e9
+                        stream.end_ts = ptpsec + ptpnsec / 1e9
                         stream.datavol += len(r.payload)
                     else:
-                        stream = Streams(stream_id, seq, 1, ptpsec + ptpnsec/1e9,
-                                         ptpsec + ptpnsec/1e6, 0, 0, len(r.payload),
-                                         len(r.payload), [])
+                        stream = Streams(
+                            stream_id,
+                            seq,
+                            1,
+                            ptpsec + ptpnsec / 1e9,
+                            ptpsec + ptpnsec / 1e6,
+                            0,
+                            0,
+                            len(r.payload),
+                            len(r.payload),
+                            [],
+                        )
                         streams[stream_id] = stream
                     inetx_pkts_validate += 1
                     data_count_bytes += len(r.payload)
@@ -295,16 +305,12 @@ def main(args):
             dr = 100
 
         try:
-            ave_rec_rate_mbps = (
-                (packet_data_vol * 8) / (last_pcap_time - first_pcap_time) / 1e6
-            )
+            ave_rec_rate_mbps = (packet_data_vol * 8) / (last_pcap_time - first_pcap_time) / 1e6
         except:
             ave_rec_rate_mbps = 0
         sids_found = len(streams)
         if first_pcap_time is not None:
-            file_stamp = datetime.datetime.fromtimestamp(first_pcap_time).strftime(
-                "%H:%M:%S %d %b"
-            )
+            file_stamp = datetime.datetime.fromtimestamp(first_pcap_time).strftime("%H:%M:%S %d %b")
         else:
             file_stamp = "unknown"
         info_str = (
@@ -319,9 +325,7 @@ def main(args):
         if args.verbose:
             if len(streams) > 0:
                 logging.info(
-                    "{:>7s} {:>9s} {:>9s} {:>9s} {:>9s}".format(
-                        "SID", "Seq", "LostCount", "ResetCnt", "Length"
-                    )
+                    "{:>7s} {:>9s} {:>9s} {:>9s} {:>9s}".format("SID", "Seq", "LostCount", "ResetCnt", "Length")
                 )
             for sid, stream in streams.items():
                 logging.info(
@@ -337,12 +341,24 @@ def main(args):
             remove(outf)
         elif is_url and not args.verbose:
             remove(outf)
+        if is_compressed(pfile):
+            remove(decompressed_f)
+
     print("\n")
     if len(streams) > 0:
         logging.info(
             "{:>7s} {:>15s} {:>9s} {:>9s} {:>9s} {:>9s} {:>9s} {:>18s} {:>12s} {:>12s} {:>12s}".format(
-                "SID", "Cnt", "LostCount", "ResetCnt", "Length", "PPS", "Mbps", "Elapsed Time(s)",
-                "DataVol(MB)", "DropVol(Bytes)", "BitRate(Mbps)"
+                "SID",
+                "Cnt",
+                "LostCount",
+                "ResetCnt",
+                "Length",
+                "PPS",
+                "Mbps",
+                "Elapsed Time(s)",
+                "DataVol(MB)",
+                "DropVol(Bytes)",
+                "BitRate(Mbps)",
             )
         )
     for sid, stream in sorted(streams.items()):
@@ -352,9 +368,18 @@ def main(args):
             _hist = ""
         logging.info(
             "{:#07X} {:15,d} {:9d} {:9d} {:9d} {:9d} {:9.1f} {:18.1f} {:12,.1f} {:12,d} {:12,.1f} {}".format(
-                sid, stream.pkt_count, stream.dropcnt, stream.rstcnt, stream.length,
-                stream.pps(), stream.bitrate()/1e6, stream.timelen(), stream.datavol/1e6,
-                stream.dropcnt * stream.length, stream.datavol * 8 / (stream.timelen() * 1e6), _hist
+                sid,
+                stream.pkt_count,
+                stream.dropcnt,
+                stream.rstcnt,
+                stream.length,
+                stream.pps(),
+                stream.bitrate() / 1e6,
+                stream.timelen(),
+                stream.datavol / 1e6,
+                stream.dropcnt * stream.length,
+                stream.datavol * 8 / (stream.timelen() * 1e6),
+                _hist,
             )
         )
 

{acranetwork-1.2.1 → acranetwork-1.2.4}/test/test_ch7.py

@@ -8,10 +8,14 @@ import struct
 import logging
 from pstats import Stats
 import cProfile
+import typing
+import random
 
 
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 
+logging.basicConfig(level=logging.INFO, format="%(levelname)s:%(funcName)s:%(lineno)s:%(message)s")
+
 
 def buf_generator(count, llp_count=0):
     running_count = 0
@@ -25,6 +29,23 @@ def buf_generator(count, llp_count=0):
         yield buf_len, low_latency
 
 
+def ptfr_to_pcm_frame(
+    count: int,
+):
+    remainder = bytes()
+    pcm_frame_len = 1024
+    offset_ptfr = 30
+    zero_buf = struct.pack(">B", 0) * offset_ptfr
+    pcm_frame = zero_buf
+    for ptfr in ch7.datapkts_to_ptfr(buf_generator(count)):
+        pcm_frame += ptfr.pack()
+        if len(pcm_frame) >= pcm_frame_len:
+            remainder = pcm_frame[pcm_frame_len:]
+            pcm_frame = pcm_frame[:pcm_frame_len]
+            yield pcm_frame
+            pcm_frame = zero_buf + remainder
+
+
 class TestCaseCh7(unittest.TestCase):
     def test_basic(self):
         ch7_pkt = ch7.PTFR()
@@ -85,7 +106,7 @@ class TestGenerators(unittest.TestCase):
             # print(repr(ptdp_pkt))
             self.assertIsInstance(ptdp_pkt, ch7.PTDP)
 
-    def test_ptfr_geenerator(self):
+    def test_ptfr_generator(self):
         remainder = bytes()
         for ptfr in ch7.datapkts_to_ptfr(buf_generator(5), ptfr_len=200):
             # print(repr(ptfr))
@@ -223,9 +244,10 @@ class TestRealEthernet(unittest.TestCase):
                         r.payload = eth_p
                         # Verify the size of the packets and it's probably good. I could unpack them here too
                         # self.assertEqual(2*64+eth.UDP.UDP_HEADER_SIZE+eth.IP.IP_HEADER_SIZE+eth.Ethernet.HEADERLEN, len(eth_p))
-                        if eth_p not in TestRealEthernet.pkts_sent:
-                            self.assertTrue(False)
-                        pkt_count += 1
+                        if p.content != ch7.PTDPContent.FILL:
+                            if eth_p not in TestRealEthernet.pkts_sent:
+                                self.assertTrue(False)
+                            pkt_count += 1
                         pf.write(r)
                         eth_p = bytes()
                     ptdp_idx += 1
@@ -239,7 +261,6 @@ class TestRealEthernet(unittest.TestCase):
         llc_count = 0
         fill_count = 0
         mac_count = 0
-        logging.basicConfig(level=logging.WARN)
         remainder = b""
         for i in range(3, 6):
             f = open(THIS_DIR + "/ptfr_{}.bin".format(i), "rb")
@@ -267,5 +288,111 @@ class TestRealEthernet(unittest.TestCase):
         self.assertEqual(4, mac_count)
 
 
+def get_pkts(some_low_latency: bool = False, max_len: int = 178) -> typing.Generator[tuple[bytes, bool], None, None]:
+
+    count = 0
+    while True:
+        # pkt_len = random.randint(2, 180)
+        pkt_len = (count % max_len) + 2
+        paylaod_int = [pkt_len] + [count] * (pkt_len - 1)
+        payload = struct.pack(f">{pkt_len}Q", *paylaod_int)
+        count += 1
+        llc_pkts = [True, False, False, False, False, False, False]
+        # llc_pkts = [True, True, True, True, True, True, True]
+        if some_low_latency:
+            low_latency = llc_pkts[count % 7]
+        else:
+            low_latency = False
+        logging.debug(f"TX: Generated payload of length {pkt_len*8} count={count}")
+        yield payload, low_latency
+
+
+def get_pcm_frame(offset_ptfr: int = 0, some_low_latency: bool = False, max_len: int = 178):
+    pcm_frame_len = 1024
+    ptfr_len = pcm_frame_len - offset_ptfr - 4
+    zero_buf = struct.pack(">B", 0) * offset_ptfr
+    for ptfr in ch7.datapkts_to_ptfr(get_pkts(some_low_latency, max_len), ptfr_len=ptfr_len):
+        pcm_frame = zero_buf + ptfr.pack()
+        logging.debug(f"TX pcm_frame_len={len(pcm_frame)} ptfr_len={ptfr_len}")
+        yield pcm_frame
+
+
+def missing_elements(L):
+    start, end = L[0], L[-1]
+    return sorted(set(range(start, end + 1)).difference(L))
+
+
+class TestRandomSizedDecom(unittest.TestCase):
+    def test_no_llc(self):
+        offset = 0
+        first_PTFR = True
+        eth_p = bytes()
+        prev_eth_count = None
+        remainder = None
+        count = 0
+        for frame in get_pcm_frame(offset, some_low_latency=False):
+            ch7_pkt = ch7.PTFR()
+            ch7_buffer = frame[offset:]
+            ch7_pkt.length = len(ch7_buffer)
+            ch7_pkt.unpack(ch7_buffer)
+            count += 1
+            if count > 10000:
+                return
+
+            for p, remainder, e in ch7_pkt.get_aligned_payload(first_PTFR, remainder):
+                first_PTFR = False
+                if p is not None:
+                    if p.length != 0:
+                        if p.fragment == ch7.PTDPFragment.COMPLETE or p.fragment == ch7.PTDPFragment.LAST:
+                            eth_p += p.payload
+                            logging.debug(repr(p))
+                            self.assertGreaterEqual(len(eth_p), 16)
+                            (expected_len, count) = struct.unpack_from(">QQ", eth_p, 0x0)
+                            logging.debug(f"RX payload count={count} len={len(eth_p)}")
+                            if prev_eth_count is not None:
+                                if prev_eth_count + 1 != count:
+                                    self.assertEqual(prev_eth_count + 1, count)
+                                self.assertEqual(expected_len * 8, len(eth_p))
+
+                            prev_eth_count = count
+                            eth_p = bytes()
+
+    def test_some_llc(self):
+        offset = 0
+        first_PTFR = True
+        eth_p = bytes()
+        remainder = None
+        golay = ch7.Golay.Golay()
+        count = 0
+        numbers_found = []
+        for frame in get_pcm_frame(offset, some_low_latency=True, max_len=50):
+            ch7_pkt = ch7.PTFR(golay)
+            ch7_buffer = frame[offset:]
+            ch7_pkt.length = len(ch7_buffer)
+            ch7_pkt.unpack(ch7_buffer)
+            count += 1
+            if count > 10000:
+                return
+
+            for p, remainder, e in ch7_pkt.get_aligned_payload(first_PTFR, remainder):
+                first_PTFR = False
+                if p is not None:
+                    if p.length != 0:
+                        if p.fragment == ch7.PTDPFragment.COMPLETE or p.fragment == ch7.PTDPFragment.LAST:
+                            eth_p += p.payload
+                            logging.debug(repr(p))
+                            if p.content != ch7.PTDPContent.FILL:
+                                self.assertGreaterEqual(len(eth_p), 16)
+                                (expected_len, count) = struct.unpack_from(">QQ", eth_p, 0x0)
+                                logging.debug(f"RX payload count={count} len={len(eth_p)}")
+                                numbers_found.append(count)
+                                self.assertEqual(expected_len * 8, len(eth_p))
+
+                            eth_p = bytes()
+
+        numbers_found.sort()
+        self.assertEqual(missing_elements(numbers_found), [])
+
+
 if __name__ == "__main__":
     unittest.main()