network-core 0.1.0__py3-none-any.whl

network_core/conn.py

@@ -0,0 +1,188 @@
+# Define PacketInfo class
+from typing import List
+from .dataModels import PacketType, TransPortType, PacketInfo, FiveTuple
+from scapy.all import UDP, TCP, IP, IPv6  # type: ignore
+
+
+def getPacketType(packet):
+    """
+    Here packet is a scapy packet
+    """
+    if TCP in packet:
+        return PacketType.TCP
+    elif UDP in packet:
+        return PacketType.UDP
+    return PacketType.UNKNOWN
+
+
+def getPacketInfoFromPacket(packet, direction: int):
+    packet_type = getPacketType(packet)
+
+    payload_bytes = b""
+    if UDP in packet:
+        payload_bytes = bytes(packet[UDP].payload)
+    elif TCP in packet:
+        payload_bytes = bytes(packet[TCP].payload)
+
+    return PacketInfo(
+        length=len(packet),
+        timestamp=float(packet.time),
+        direction=direction,
+        packet_type=packet_type,
+        transport_payload=payload_bytes,
+        other_info=(
+            {"seq": packet[TCP].seq} if packet_type == PacketType.TCP else dict()
+        ),
+    )
+
+
+class Connection:
+    def __init__(
+        self,
+        five_tuple: FiveTuple,
+        other_info=None,
+    ):
+        self.five_tuple = five_tuple
+        self.connection_type: set[PacketType] = set()
+        self.packet_stream: List[PacketInfo] = []
+        self.other_info = other_info
+        self.__sni = None
+        self.is_quic = False
+
+    @staticmethod
+    def getTransPortType(packet):
+        if TCP in packet:
+            return TransPortType.TCP
+        elif UDP in packet:
+            return TransPortType.UDP
+        return TransPortType.UNKNOWN
+
+    @staticmethod
+    def getConnFromPacket(packet):
+        transport_type = Connection.getTransPortType(packet)
+
+        packet_type = getPacketType(packet)
+        if packet_type == PacketType.UNKNOWN:
+            return None
+        ip_layer = IP if IP in packet else IPv6
+
+        if transport_type == TransPortType.TCP:
+            ft = FiveTuple(
+                src_ip=packet[ip_layer].src,
+                dst_ip=packet[ip_layer].dst,
+                src_port=packet[TCP].sport,
+                dst_port=packet[TCP].dport,
+                transport_type=transport_type,
+            )
+            return Connection(five_tuple=ft)
+        elif transport_type == TransPortType.UDP:
+            ft = FiveTuple(
+                src_ip=packet[ip_layer].src,
+                dst_ip=packet[ip_layer].dst,
+                src_port=packet[UDP].sport,
+                dst_port=packet[UDP].dport,
+                transport_type=transport_type,
+            )
+            return Connection(five_tuple=ft)
+        return None
+
+    def addPacket(self, packet: PacketInfo):
+        if packet.packet_type == PacketType.UNKNOWN:
+            assert False, "Packet Type is Unknown"
+        self.connection_type.add(packet.packet_type)
+        self.packet_stream.append(packet)
+
+    def __len__(self):
+        return len(self.packet_stream)
+
+    def setSNI(self, sni: str):
+        self.__sni = sni
+
+    def getSNI(self):
+        return self.__sni
+
+    def getPacketsOfTypes(self, packet_types: set[PacketType]) -> List[PacketInfo]:
+        return list(filter(lambda x: x.packet_type in packet_types, self.packet_stream))
+
+    def getRemovedPacketsOfTypes(self, packet_types: set[PacketType]):
+        return list(
+            filter(lambda x: x.packet_type not in packet_types, self.packet_stream)
+        )
+
+    def getPacketsOfDirection(self, direction: int) -> list[PacketInfo]:
+        return [p for p in self.packet_stream if p.direction == direction]
+
+    def getSubConnection(self, packet_types: List[PacketType]):
+        packets = self.getPacketsOfTypes(packet_types=set(packet_types))
+
+        new_conn = Connection(five_tuple=self.five_tuple)
+        for packet in packets:
+            new_conn.addPacket(packet=packet)
+        return new_conn
+
+    def getDuration(self):
+        """
+        Returns the duration of connection on seconds
+        """
+        if len(self.packet_stream) == 0:
+            return 0
+
+        return self.packet_stream[-1].timestamp - self.packet_stream[0].timestamp
+
+    def sort(self):
+        self.packet_stream.sort(key=lambda x: x.timestamp)
+
+    def pps(self):
+        num_packets = len(self.packet_stream)
+        duration = self.getDuration()
+        if num_packets == 0 or duration == 0:
+            return 0
+
+        return num_packets / duration
+
+    def removePayload(self):
+        """
+        This is an irreversable operation, removes packets inplace
+        """
+        for packet in self.packet_stream:
+            packet.transport_payload = bytes()
+
+    @property
+    def down_bytes(self):
+        total = 0
+        for p in self.packet_stream:
+            if p.direction == 1:
+                total += p.length
+        return total
+
+    @property
+    def up_bytes(self):
+        total = 0
+        for p in self.packet_stream:
+            if p.direction == 0:
+                total += p.length
+        return total
+
+    @property
+    def down_packets(self):
+        total = 0
+        for p in self.packet_stream:
+            if p.direction == 1:
+                total += 1
+        return total
+
+    @property
+    def up_packets(self):
+        total = 0
+        for p in self.packet_stream:
+            if p.direction == 0:
+                total += 1
+        return total
+
+    @property
+    def start_timestamp(self):
+        return self.packet_stream[0].timestamp
+
+    @property
+    def end_timestamp(self):
+        return self.packet_stream[-1].timestamp

network_core/connOps.py

@@ -0,0 +1,206 @@
+from .conn import Connection
+from .dataModels import PacketInfo, PacketType, FiveTuple
+from .utils.dt import convertUNIXToHumanReadable
+
+import numpy as np
+
+
+class ConnStats:
+    def __init__(self, conn: Connection):
+        self.conn = conn
+
+    def getUpStreamBytes(self):
+        packet_stream = self.conn.packet_stream
+
+        up_stream_bytes = 0
+        for packet in packet_stream:
+            if packet.direction == 0:
+                up_stream_bytes += packet.length
+        return up_stream_bytes
+
+    def getDownStreamBytes(self):
+        packet_stream = self.conn.packet_stream
+
+        down_stream_bytes = 0
+        for packet in packet_stream:
+            if packet.direction == 1:
+                down_stream_bytes += packet.length
+        return down_stream_bytes
+
+    def getConnSpan(self):
+        packet_stream = self.conn.packet_stream
+        return packet_stream[-1].timestamp - packet_stream[0].timestamp
+
+    def getPacketTypeCounts(self):
+        packet_stream = self.conn.packet_stream
+        packet_type_counts = dict()
+        for packet in packet_stream:
+            packet_type_counts[packet.packet_type] = 1 + packet_type_counts.get(
+                packet.packet_type, 0
+            )
+        return packet_type_counts
+
+    def getStartTime(self):
+        return convertUNIXToHumanReadable(self.conn.packet_stream[0].timestamp)
+
+    def getEndTime(self):
+        return convertUNIXToHumanReadable(self.conn.packet_stream[-1].timestamp)
+
+
+def filterConnections(
+    connections: list[Connection],
+    connection_types: list[PacketType],
+    min_packets: int | None = None,
+    min_duration_in_seconds: float | None = None,
+):
+    """
+    filter connections by connection types, the connection must have atleast one packet of the types in the connection_types
+    :param connections: List of connections to filter
+    :param connection_types: List of connection types to filter by
+    :return: Filtered connections
+    """
+
+    def doesContainConnectionTypes(
+        connection: Connection, connection_types: list[PacketType]
+    ):
+        for connection_type in connection_types:
+            if connection_type not in connection.connection_type:
+                return False
+        return True
+
+    f_connections = [
+        connection
+        for connection in connections
+        if doesContainConnectionTypes(connection, connection_types)
+    ]
+
+    if min_packets is not None:
+        f_connections = [
+            connection for connection in f_connections if len(connection) >= min_packets
+        ]
+    if min_duration_in_seconds is not None:
+        f_connections = [
+            connection
+            for connection in f_connections
+            if ConnStats(connection).getConnSpan() >= min_duration_in_seconds
+        ]
+    return f_connections
+
+
+def combine_connections(connections: list[Connection], enforce_transport=False):
+    """
+    Combine connections with the same src_ip, src_port, dst_ip, dst_port and transport_type in the end. The first connection's ip and port will be used
+    :param connections: List of connections to combine
+    :param enforce_transport: If True, enforce that all connections have the same transport_type
+    :return: Combined connection sorted by timestamp
+    """
+    if len(connections) == 0:
+        return None
+    if len(connections) == 1:
+        return connections[0]
+
+    combined_connection = Connection(
+        five_tuple=connections[0].five_tuple,
+    )
+    for connection in connections:
+        if len(connection) == 0:
+            continue
+        if enforce_transport:
+            assert (
+                connection.five_tuple.transport_type
+                == combined_connection.five_tuple.transport_type
+            ), f"{connection.five_tuple.transport_type}, {combined_connection.five_tuple.transport_type}, {len(connection)}"
+        for packet in connection.packet_stream:
+            combined_connection.addPacket(packet)
+    # important to sort it in order of timestamp
+    combined_connection.packet_stream = sorted(
+        combined_connection.packet_stream, key=lambda x: x.timestamp
+    )
+    return combined_connection
+
+
+def printConns(connections: list[Connection]):
+    """
+    Utility to print connections
+    """
+
+    def splitTimeString(time_str):
+        return time_str.split(" ")[1]
+
+    for conn in connections:
+        stat = ConnStats(conn)
+        print(
+            conn.five_tuple,
+            f"SNI {conn.getSNI()}",
+            f"Start time {splitTimeString(stat.getStartTime())}",
+            f"End Time {splitTimeString(stat.getEndTime())}",
+            f"Duration {stat.getConnSpan()}",
+            f"Upstream Bytes {stat.getUpStreamBytes()}",
+            f"Downstream Bytes {stat.getDownStreamBytes()}",
+            conn.connection_type,
+            f"Num Packets {len(conn)}",
+        )
+
+
+def getPacketsInInterval(
+    packet_stream: list[PacketInfo],
+    start_time_timestamp: float,
+    end_timestamp: float,
+) -> list[PacketInfo]:
+    """
+    packet_stream is assumed to be sorted, by timestamp
+    Also, ideally packet_stream should contain packets from the same direction.
+
+    Returns packets from  with timestamp larger or equal to start_timestamp and just smaller than end_timestamp
+    It then becomes the function user's responsibility to validate if there are enough packets in the interval.
+    """
+
+    def bSearch(time: float):
+        start_index = 0
+        end_index = len(packet_stream) - 1
+        just_left_of_time, just_right_or_equal_of_time = None, None
+        while start_index <= end_index:
+            mid_index = (start_index + end_index) // 2
+            mid_timestamp = packet_stream[mid_index].timestamp
+            if mid_timestamp >= time:
+                just_right_or_equal_of_time = mid_index
+                end_index = mid_index - 1
+            elif mid_timestamp < time:
+                just_left_of_time = mid_index
+                start_index = mid_index + 1
+
+        return just_left_of_time, just_right_or_equal_of_time
+
+    _, left = bSearch(time=start_time_timestamp)
+    right, _ = bSearch(time=end_timestamp)
+
+    if (left is None or right is None) or left > right:
+        return []
+
+    return packet_stream[left : right + 1]
+
+
+def normalizePacketStream(packet_stream: list[PacketInfo]) -> dict:
+    def normIAT(iat: np.ndarray, limit=200) -> np.ndarray:
+        """
+        iats are in miliseconds
+        """
+        iat = np.clip(iat, 0, limit)  # Same as capping
+        return np.log1p(iat) / np.log1p(limit)
+
+    sizes = np.array([p.length for p in packet_stream])
+    times = np.array([p.timestamp * 100 for p in packet_stream])
+    iats = np.diff(times) if len(times) > 1 else np.array([])
+
+    return {
+        "mean_size": sizes.mean(),
+        "std_size": sizes.std(),
+        "burstiness": (sizes.max() - sizes.min()) / (sizes.mean() + 1e-5),
+        "mean_iat": iats.mean(),
+        "std_iat": iats.std(),
+        "packet_count": len(sizes),
+        "unique_sizes": len(set(sizes)),
+        "lengths": (sizes / 1500).tolist(),
+        "iats": [0] + normIAT(iats).tolist(),
+        "timestamps": [p.timestamp for p in packet_stream],
+    }

network_core/dataModels.py

@@ -0,0 +1,62 @@
+from enum import Enum
+from dataclasses import dataclass, field
+
+
+class PacketType(Enum):
+    TLS = 0
+    DTLS = 1
+    RTP = 2
+    RTCP = 3
+    STUN = 4
+    TURN = 5
+    QUIC = 6
+
+    TCP = 7
+    UDP = 8
+    UNKNOWN = 9
+
+
+class TransPortType(Enum):
+    TCP = 0
+    UDP = 1
+    UNKNOWN = 2
+
+
+@dataclass
+class FiveTuple:
+    src_ip: str
+    dst_ip: str
+    src_port: int
+    dst_port: int
+    transport_type: TransPortType
+
+    def _get_key(self):
+        return (
+            self.src_ip,
+            self.dst_ip,
+            self.src_port,
+            self.dst_port,
+            self.transport_type.name,
+        )
+
+    def __hash__(self) -> int:
+        return hash(self._get_key())
+
+    def rev_ft(self) -> "FiveTuple":
+        return FiveTuple(
+            src_ip=self.dst_ip,
+            src_port=self.dst_port,
+            dst_ip=self.src_ip,
+            dst_port=self.src_port,
+            transport_type=self.transport_type,
+        )
+
+
+@dataclass
+class PacketInfo:
+    length: int
+    timestamp: float
+    direction: int
+    packet_type: PacketType
+    transport_payload: bytes = field(default_factory=bytes)
+    other_info: dict = field(default_factory=dict)

network_core/utils/csvIO.py

@@ -0,0 +1,9 @@
+import csv
+from typing import Iterator
+
+
+def read_csv_to_dicts(file_path) -> Iterator[dict]:
+    with open(file_path, newline="") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            yield row

network_core/utils/dt.py

@@ -0,0 +1,47 @@
+import datetime, pytz
+
+
+def convertUNIXToDT(timestamp: float, timezone="Australia/Sydney") -> datetime.datetime:
+    australian_timezone = pytz.timezone(timezone)
+    utc_dt = datetime.datetime.fromtimestamp(timestamp, tz=pytz.utc)
+    return utc_dt.astimezone(australian_timezone)
+
+
+def convertUNIXToHumanReadable(timestamp: float, timezone="Australia/Sydney") -> str:
+    dt = convertUNIXToDT(timestamp=timestamp, timezone=timezone)
+    ms = dt.microsecond // 1000  # get milliseconds, doing this so it isn't 0
+    return dt.strftime("%Y-%m-%d %H:%M:%S") + f".{ms}"
+
+
+def getDatetimeFromTime(time: datetime.time) -> datetime.datetime:
+    return datetime.datetime.combine(datetime.datetime.today(), time=time)
+
+
+def addBufferToTime(time: datetime.time, buffer_in_seconds: float) -> datetime.time:
+    delta = datetime.timedelta(seconds=buffer_in_seconds)
+    return (getDatetimeFromTime(time=time) + delta).time()
+
+
+def fast_strptime(s: str) -> float:
+    """
+    2026-01-11 12:25:31.689768 +0530 IST to UNIX
+    """
+    parts = s.split()
+
+    # Drop trailing timezone name (like 'UTC', 'AEDT', etc.)
+    if len(parts) > 3:
+        s = " ".join(parts[:3])
+    else:
+        s = " ".join(parts)
+
+    # Handle too many fractional digits (datetime only supports microseconds)
+    date, time_str, offset = s.split()
+    if "." in time_str:
+        main, frac = time_str.split(".")
+        frac = frac[:6]  # truncate nanoseconds to microseconds
+        time_str = f"{main}.{frac}"
+    s = f"{date} {time_str} {offset}"
+
+    fmt = "%Y-%m-%d %H:%M:%S.%f %z" if "." in time_str else "%Y-%m-%d %H:%M:%S %z"
+
+    return datetime.datetime.strptime(s, fmt).timestamp()

network_core/utils/jsonIO.py

@@ -0,0 +1,124 @@
+import datetime
+import json
+from typing import Union, List, Dict, Iterator
+
+import orjson
+import numpy as np
+
+
+def numpy_to_lists(obj):
+    """
+    Recursively convert NumPy arrays (and NumPy scalars) back to Python lists/int/float.
+    """
+    if isinstance(obj, np.ndarray):
+        # Convert array to nested list
+        return [numpy_to_lists(x) for x in obj]
+    elif isinstance(obj, (np.integer, np.floating)):
+        # Convert NumPy scalar to Python scalar
+        return obj.item()
+    elif isinstance(obj, dict):
+        return {k: numpy_to_lists(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        # Already a list, convert its elements recursively
+        return [numpy_to_lists(x) for x in obj]
+    else:
+        return obj
+
+
+class JSONCustomEncoder(json.JSONEncoder):
+    """
+    Custom JSON encoder that recursively converts NumPy arrays and scalars
+    to lists and native Python types.
+    """
+
+    def default(self, obj):
+        # Use your recursive conversion
+        obj_converted = numpy_to_lists(obj)
+        # After conversion, let the base class handle normal types
+        return super().default(obj_converted)
+
+
+def lists_to_numpy(obj: dict | list):
+    if isinstance(obj, list):
+        return np.array([lists_to_numpy(x) for x in obj])
+    elif isinstance(obj, dict):
+        return {k: lists_to_numpy(v) for k, v in obj.items()}
+    else:
+        return obj
+
+
+class JSONCustomDecoder(json.JSONDecoder):
+    """
+    Custom JSON decoder to convert lists back into NumPy arrays.
+    """
+
+    def __init__(self, *args, **kwargs):
+        # Use object_hook to intercept all dicts/lists during decoding
+        super().__init__(object_hook=self.object_hook, *args, **kwargs)
+
+    def object_hook(self, obj):
+        return lists_to_numpy(obj)
+
+
+def saveDctAsJSON(path: str, data: Union[Dict, List[Dict]], append: bool = False):
+    """
+    Save a dict or list of dicts as JSON.
+
+    ALWAYS SAVES AT A LINE SEPERATED JSON
+
+    - Dict: saved as a single pretty-printed JSON object.
+    - List[Dict]: line-delimited JSON.
+    """
+    mode = "a" if append else "w"
+    with open(path, mode) as f:
+        if isinstance(data, dict):
+            json.dump(data, f, separators=(",", ":"), cls=JSONCustomEncoder)
+            f.write("\n")
+        elif isinstance(data, list):
+            # Write each item as a line-delimited JSON object
+            for item in data:
+                json.dump(item, f)
+                f.write("\n")
+
+
+def loadJSONAsDct(
+    path: str, use_custom_decoder: bool = False
+) -> Union[Dict, List[Dict]]:
+    """
+    Load JSON from a file. Handles:
+    - Single dict (pretty-printed or one-line)
+    - List of dicts (array or line-delimited)
+    """
+
+    with open(path, "r") as f:
+        text = f.read().strip()
+        if not text:
+            return {}  # empty file
+        try:
+            return json.loads(text)  # dict or list saved as JSON array
+        except json.JSONDecodeError:
+            # fallback: line-delimited JSON
+            result = []
+            with open(path, "r") as f2:
+                for line in f2:
+                    line = line.strip()
+                    if line:
+                        result.append(
+                            json.loads(
+                                line,
+                                cls=JSONCustomDecoder if use_custom_decoder else None,
+                            )
+                        )
+            return result
+
+
+def loadJsonLines(path: str, np_convert: bool = False) -> Iterator[dict]:
+    with open(path, "rb") as f:  # orjson expects bytes
+        for line in f:
+            line = line.strip()
+            if line:
+                yield (
+                    orjson.loads(line)
+                    if not np_convert
+                    else lists_to_numpy(orjson.loads(line))
+                )  # type: ignore

network_core-0.1.0.dist-info/METADATA

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: network_core
+Version: 0.1.0
+Summary: Core networking utilities and data models
+Author: Your Name
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: pytz
+Requires-Dist: orjson
+Requires-Dist: scapy

network_core-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+network_core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+network_core/conn.py,sha256=-wngAzOBUtyOIUKLmFnf0YXwSrtbqaJIQuqA6sAe2dY,5444
+network_core/connOps.py,sha256=9frmJ6_JgiTyceuIcx5x3NWJbjAizLA7s_TuZ-9P9wo,7135
+network_core/dataModels.py,sha256=XKV6yYI9b13FzZX_1uaxzS7Och4vQSevC88Q-LpQDnA,1167
+network_core/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+network_core/utils/csvIO.py,sha256=tUqL_2VZR9bMUGX_3mOuvgL_3ihokEDVMXuXZqjR3Lk,220
+network_core/utils/dt.py,sha256=V1v7Ug3DVZoVyVFQZfRPpeVdxXpmPT7Re-Amy4RL6zY,1660
+network_core/utils/jsonIO.py,sha256=oq0vBeiQEV_yHd07kQq1X5HLSvJnz7VAAonAOb8wFXI,3850
+network_core-0.1.0.dist-info/METADATA,sha256=9d6qdHcNMF_r0GnS_sL2wrl_VynTPxxsfJaCDg0SIg4,294
+network_core-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+network_core-0.1.0.dist-info/top_level.txt,sha256=wnMFOdmtkPtdr2Dm9LYLVttaV_JUMuxDQQux5YmCMo0,13
+network_core-0.1.0.dist-info/RECORD,,

network_core-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

network_core-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+network_core