network-core 0.5.4__tar.gz → 0.5.6__tar.gz

network_core-0.5.6/MANIFEST.in

@@ -0,0 +1 @@
+recursive-include go_src *

{network_core-0.5.4 → network_core-0.5.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: network_core
-Version: 0.5.4
+Version: 0.5.6
 Summary: Core networking utilities and data models
 Author: Your Name
 Requires-Python: >=3.9

network_core-0.5.6/go_src/commons/constants.go

@@ -0,0 +1,31 @@
+package commons
+
+import (
+	"encoding/hex"
+	"time"
+)
+
+var PACKET_BATCH_WRITE_SIZE int = 8000
+var TIME_BUFFER time.Duration = 5 * time.Minute
+
+var MIN_FLOW_DURATION time.Duration = 2 * time.Second
+var MIN_FLOW_PACKETS int = 10
+
+var SNI_MAX_PACKETS int = 20
+
+var TCP_PROTO int = 6
+var UDP_PROTO int = 17
+
+// SNI stuff
+
+var QUIC_TYPE_MAP = map[int]string{
+	0: "initial",
+	1: "0-RTT",
+	2: "handshake",
+	3: "retry",
+}
+
+var QuicSalt, _ = hex.DecodeString("38762cf7f55934b34d179ae6a4c80cadccbb7f0a")
+var TLS_SNI_EXT_TYPE int = 0
+var TLS_ALPN_EXT_TYPE int = 16
+var QUIC_INITIAL_TYPE int = 0

network_core-0.5.6/go_src/commons/data.go

@@ -0,0 +1,84 @@
+package commons
+
+import (
+	"fmt"
+	"time"
+)
+
+type Flow struct {
+	FlowID    int
+	FiveTuple *FiveTuple
+	StartTime time.Time
+	EndTime   time.Time
+	Packets   []PacketInfo
+	SNI       string
+	ALPN      string // first ALPN protocol from TLS ClientHello, or "nan"
+	Closed    bool
+}
+
+type PacketInfo struct {
+	PacketLength      int
+	Timestamp         time.Time
+	Direction         uint8
+	Transport_payload []byte
+	TCP_SEQ           uint32
+}
+
+type FiveTuple struct {
+	ClientIP   string
+	ServerIP   string
+	ClientPort uint16
+	ServerPort uint16
+	Protocol   uint8
+}
+
+func (ft *FiveTuple) String() string {
+	return fmt.Sprintf("%s:%d->%s:%d->(Protocol: %d)",
+		ft.ClientIP, ft.ClientPort, ft.ServerIP, ft.ServerPort, ft.Protocol)
+}
+
+type SavePacket struct {
+	FlowId       int
+	PacketLength int
+	Timestamp    time.Time
+	Direction    uint8
+}
+
+type SaveFlow struct {
+	FlowId    int
+	FiveTuple *FiveTuple
+	StartTime time.Time
+	EndTime   time.Time
+	SNI       string
+	ALPN      string
+}
+
+type SaveJob struct {
+	Flows        []*Flow
+	SaveFlowData bool
+}
+
+// SNIInfo holds the resolved SNI and ALPN for a 5-tuple; used for carry-forward
+// across session-resumed flows that share the same TCP socket.
+type SNIInfo struct {
+	SNI  string
+	ALPN string
+}
+
+// SNI related
+
+type TLS_CH_Parsed struct {
+	Valid bool
+	SNIs  []string
+	ALPN  []string // e.g. ["h2"] or ["http/1.1"]
+}
+
+type QUIC_CRYPTO_Frame struct {
+	offset  int
+	payload []byte
+}
+
+type QUIC_LH_PARSED struct {
+	valid  bool
+	frames []QUIC_CRYPTO_Frame
+}

network_core-0.5.6/go_src/commons/flowOps.go

@@ -0,0 +1,45 @@
+package commons
+
+func alpnFirst(alpns []string) string {
+	if len(alpns) > 0 {
+		return alpns[0]
+	}
+	return "nan"
+}
+
+func (flow *Flow) Get_Set_SNI() {
+
+	if flow.FiveTuple.Protocol == uint8(TCP_PROTO) {
+		// I will try both direction
+
+		tls_sni := TLS_sni_extractor(get_transport_packets_of_direction(flow, 0, SNI_MAX_PACKETS))
+		if !tls_sni.Valid {
+			tls_sni = TLS_sni_extractor(get_transport_packets_of_direction(flow, 1, SNI_MAX_PACKETS))
+		}
+		if tls_sni.Valid && len(tls_sni.SNIs) > 0 {
+			flow.SNI = tls_sni.SNIs[0]
+		} else {
+			flow.SNI = "nan"
+		}
+		if tls_sni.Valid {
+			flow.ALPN = alpnFirst(tls_sni.ALPN)
+		} else {
+			flow.ALPN = "nan"
+		}
+
+	} else if flow.FiveTuple.Protocol == uint8(UDP_PROTO) {
+
+		quic_sni := QUIC_SNI_extractor(get_transport_packets_of_direction(flow, 0, SNI_MAX_PACKETS))
+		if !quic_sni.Valid {
+			quic_sni = QUIC_SNI_extractor(get_transport_packets_of_direction(flow, 1, SNI_MAX_PACKETS))
+		}
+		if quic_sni.Valid {
+			flow.SNI = quic_sni.SNIs[0]
+		} else {
+			flow.SNI = "nan"
+		}
+		flow.ALPN = "nan" // QUIC path does not extract ALPN yet
+
+	}
+
+}

network_core-0.5.6/go_src/commons/packetOps.go

@@ -0,0 +1,67 @@
+package commons
+
+import (
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+)
+
+func GetFiveTupleFromPacket(packet *gopacket.Packet) (FiveTuple, bool) {
+	var clientIP, serverIP string
+	// Check for IPv4 layer
+
+	ipv4Layer := (*packet).Layer(layers.LayerTypeIPv4)
+	if ipv4Layer != nil {
+		ipv4, _ := ipv4Layer.(*layers.IPv4)
+		clientIP = ipv4.SrcIP.String()
+		serverIP = ipv4.DstIP.String()
+	} else {
+		// Check for IPv6 layer
+		ipv6Layer := (*packet).Layer(layers.LayerTypeIPv6)
+		if ipv6Layer != nil {
+			ipv6, _ := ipv6Layer.(*layers.IPv6)
+			clientIP = ipv6.SrcIP.String()
+			serverIP = ipv6.DstIP.String()
+		} else {
+			// Neither IPv4 nor IPv6 present, return false
+			return FiveTuple{}, false
+		}
+	}
+
+	// Extract TCP/UDP information
+	var srcPort, dstPort uint16
+	var protocol uint8
+	if tcpLayer := (*packet).Layer(layers.LayerTypeTCP); tcpLayer != nil {
+		tcp, _ := tcpLayer.(*layers.TCP)
+		srcPort = uint16(tcp.SrcPort)
+		dstPort = uint16(tcp.DstPort)
+		protocol = 6 // TCP
+	} else if udpLayer := (*packet).Layer(layers.LayerTypeUDP); udpLayer != nil {
+		udp, _ := udpLayer.(*layers.UDP)
+		srcPort = uint16(udp.SrcPort)
+		dstPort = uint16(udp.DstPort)
+		protocol = 17 // UDP
+	} else {
+		return FiveTuple{}, false
+	}
+
+	// Construct tuple
+	tp := FiveTuple{
+		ClientIP:   clientIP,
+		ServerIP:   serverIP,
+		ClientPort: srcPort,
+		ServerPort: dstPort,
+		Protocol:   protocol,
+	}
+	return tp, true
+
+}
+
+func GetRevFromFiveTuple(five_tuple FiveTuple) FiveTuple {
+	return FiveTuple{
+		ClientIP:   five_tuple.ServerIP,
+		ServerIP:   five_tuple.ClientIP,
+		ClientPort: five_tuple.ServerPort,
+		ServerPort: five_tuple.ClientPort,
+		Protocol:   five_tuple.Protocol,
+	}
+}

network_core-0.5.6/go_src/commons/quicCrypto.go

@@ -0,0 +1,124 @@
+package commons
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/binary"
+)
+
+func hkdfExtract(salt, ikm []byte) []byte {
+	mac := hmac.New(sha256.New, salt)
+	mac.Write(ikm)
+	return mac.Sum(nil)
+}
+
+func hkdfExpand(secret, info []byte, length int) []byte {
+	var result []byte
+	var t []byte
+	counter := byte(1)
+
+	for len(result) < length {
+		h := hmac.New(sha256.New, secret)
+		h.Write(t)
+		h.Write(info)
+		h.Write([]byte{counter})
+		t = h.Sum(nil)
+		result = append(result, t...)
+		counter++
+	}
+	return result[:length]
+}
+
+func hkdfExpandLabel(secret []byte, label string, length int) []byte {
+	fullLabel := append([]byte("tls13 "), []byte(label)...)
+
+	info := make([]byte, 0, 2+1+len(fullLabel)+1)
+	tmp := make([]byte, 2)
+	binary.BigEndian.PutUint16(tmp, uint16(length))
+	info = append(info, tmp...)
+	info = append(info, byte(len(fullLabel)))
+	info = append(info, fullLabel...)
+	info = append(info, 0x00)
+
+	return hkdfExpand(secret, info, length)
+}
+
+func getSecretsAndKeys(dcid []byte, salt []byte) (hpKey, clientKey, clientIV []byte) {
+	initialSecret := hkdfExtract(salt, dcid)
+	clientSecret := hkdfExpandLabel(initialSecret, "client in", 32)
+
+	hpKey = hkdfExpandLabel(clientSecret, "quic hp", 16)
+	clientKey = hkdfExpandLabel(clientSecret, "quic key", 16)
+	clientIV = hkdfExpandLabel(clientSecret, "quic iv", 12)
+
+	return
+}
+
+func removeHeaderProtection(payload []byte, pnOffset int, hpKey []byte) (packetNumber uint64, pnLen int, unprotected []byte, ok bool) {
+	if pnOffset+4+16 > len(payload) {
+		return 0, 0, nil, false
+	}
+
+	sample := payload[pnOffset+4 : pnOffset+4+16]
+
+	block, err := aes.NewCipher(hpKey)
+	if err != nil {
+		return 0, 0, nil, false
+	}
+
+	mask := make([]byte, 16)
+	block.Encrypt(mask, sample)
+
+	unprotected = make([]byte, len(payload))
+	copy(unprotected, payload)
+
+	unprotected[0] ^= mask[0] & 0x0f
+	pnLen = int(unprotected[0]&0x03) + 1
+
+	for i := 0; i < pnLen; i++ {
+		unprotected[pnOffset+i] ^= mask[i+1]
+	}
+
+	var pn uint64
+	for i := 0; i < pnLen; i++ {
+		pn = (pn << 8) | uint64(unprotected[pnOffset+i])
+	}
+
+	return pn, pnLen, unprotected, true
+}
+
+func decryptQUICPayload(
+	encryptedPayload []byte,
+	packetNumber uint64,
+	clientKey []byte,
+	clientIV []byte,
+	associatedData []byte,
+) ([]byte, bool) {
+
+	block, err := aes.NewCipher(clientKey)
+	if err != nil {
+		return nil, false
+	}
+
+	aead, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, false
+	}
+
+	pnBytes := make([]byte, 12)
+	binary.BigEndian.PutUint64(pnBytes[4:], packetNumber)
+
+	nonce := make([]byte, 12)
+	for i := 0; i < 12; i++ {
+		nonce[i] = clientIV[i] ^ pnBytes[i]
+	}
+
+	plaintext, err := aead.Open(nil, nonce, encryptedPayload, associatedData)
+	if err != nil {
+		return nil, false
+	}
+
+	return plaintext, true
+}

network_core-0.5.6/go_src/commons/quicParser.go

@@ -0,0 +1,277 @@
+package commons
+
+import (
+	"slices"
+)
+
+/*
+
+def is_quic_long_header(payload):
+    """
+    Checks if the given payload is a QUIC long header packet.
+    1. It first checks if the payload looks like QUIC using looks_like_quic
+    2. Then it checks if the first byte indicates a long header (most significant bit is 1)
+    3. Finally, it checks if the version field (bytes 1-4) is non-zero.
+    Returns True if all conditions are met, False otherwise.
+    """
+    if looks_like_quic(payload) == False:
+        return False
+    if len(payload) < 6:
+        return False
+
+    if (payload[0] & 0x80) == 0:
+        return False
+
+    version_id = int.from_bytes(payload[1:5], byteorder="big")
+    if version_id == 0:
+        return False
+    return True
+
+
+*/
+
+func is_quic_long_header(payload []byte) bool {
+	/*
+		1 1 T T P P N N
+		| | | | | | | |
+		| | | | | | +-+- Packet Number Length
+		| | | | +-+-+--- Packet Type (Initial/0-RTT/Handshake/Retry)
+		| | +-+--------- Reserved bits
+		| +------------- Fixed bit (must be 1)
+		+--------------- Header Form (1 = Long header)
+
+	*/
+	if len(payload) < 6 {
+		return false
+	}
+
+	if (payload[0] & 0x40) == 0 {
+		return false
+	}
+
+	if (payload[0] & 0x80) == 0 {
+		return false
+	}
+
+	version_id, ok := readUint32(payload, 1)
+	if !ok {
+		return false
+	}
+	if version_id == 0 {
+		return false
+	}
+
+	return true
+}
+
+func parse_quic_plain_text(plainText []byte) ([]QUIC_CRYPTO_Frame, bool) {
+	parseIndex := 0
+	frames := make([]QUIC_CRYPTO_Frame, 0)
+
+	for parseIndex < len(plainText) {
+		frameType, newIndex, ok := read_var_int(plainText, parseIndex)
+		if !ok {
+			return frames, false
+		}
+		parseIndex = newIndex
+
+		switch frameType {
+
+		case 0x00: // PADDING
+			continue
+
+		case 0x01: // PING
+			continue
+
+		case 0x06: // CRYPTO
+			offset, parseIndex2, ok := read_var_int(plainText, parseIndex)
+			if !ok {
+				return frames, false
+			}
+			parseIndex = parseIndex2
+
+			length, parseIndex2, ok := read_var_int(plainText, parseIndex)
+			if !ok {
+				return frames, false
+			}
+			parseIndex = parseIndex2
+
+			if parseIndex+int(length) > len(plainText) {
+				return frames, false
+			}
+
+			frames = append(frames, QUIC_CRYPTO_Frame{
+				offset:  int(offset),
+				payload: plainText[parseIndex : parseIndex+int(length)],
+			})
+
+			parseIndex += int(length)
+
+		default:
+			// Stop parsing but keep what we found
+			return frames, len(frames) > 0
+		}
+	}
+
+	return frames, len(frames) > 0
+}
+
+func quic_long_header_parser(payload []byte) QUIC_LH_PARSED {
+	if !is_quic_long_header(payload) {
+		return QUIC_LH_PARSED{valid: false}
+	}
+
+	parse_index := 0
+
+	// checking if the packet is initial
+	if int((payload[parse_index]>>4)&0x03) != QUIC_INITIAL_TYPE {
+		return QUIC_LH_PARSED{valid: false}
+	}
+	parse_index += 1
+
+	// version
+	_, ok := readUint32(payload, parse_index)
+	if !ok {
+		return QUIC_LH_PARSED{valid: false}
+	}
+	parse_index += 4
+
+	// dcid
+	dcid_len, ok := readUint8(payload, parse_index)
+	if !ok {
+		return QUIC_LH_PARSED{valid: false}
+	}
+	parse_index += 1
+	if parse_index+dcid_len > len(payload) {
+		return QUIC_LH_PARSED{valid: false}
+	}
+	dcid := payload[parse_index : parse_index+dcid_len]
+	parse_index += dcid_len
+
+	// scid
+	scid_len, ok := readUint8(payload, parse_index)
+	if !ok {
+		return QUIC_LH_PARSED{valid: false}
+	}
+	parse_index += 1
+	parse_index += scid_len
+
+	if parse_index >= len(payload) {
+		return QUIC_LH_PARSED{valid: false}
+	}
+
+	//  token
+	token_len, parse_index, ok := read_var_int(payload, parse_index)
+	if !ok {
+		return QUIC_LH_PARSED{valid: false}
+	}
+
+	parse_index += int(token_len)
+	if parse_index >= len(payload) {
+		return QUIC_LH_PARSED{valid: false}
+	}
+
+	// length
+	payload_length, parse_index, ok := read_var_int(payload, parse_index)
+	if !ok {
+		return QUIC_LH_PARSED{valid: false}
+	}
+
+	if parse_index+int(payload_length) > len(payload) { // not >= but >
+		return QUIC_LH_PARSED{valid: false}
+	}
+
+	/*
+			# right now parse_index is pointing to the start of packet number
+		    # I first need to unprotect the header to get the packet number length
+		    # Start header protection removal
+
+	*/
+
+	hp_key, client_key, client_iv := getSecretsAndKeys(dcid, QuicSalt)
+
+	packet_number, pn_len, hp_less_payload, ok := removeHeaderProtection(payload, parse_index, hp_key)
+	if !ok {
+		return QUIC_LH_PARSED{valid: false}
+	}
+
+	// moving past packet length
+	parse_index += pn_len
+
+	if int(payload_length) < pn_len {
+		return QUIC_LH_PARSED{valid: false}
+	}
+
+	plain_text, ok := decryptQUICPayload(payload[parse_index:parse_index+int(payload_length)-pn_len], packet_number, client_key, client_iv, hp_less_payload[:parse_index])
+	if !ok {
+		return QUIC_LH_PARSED{valid: false}
+	}
+
+	frames, ok := parse_quic_plain_text(plain_text)
+	if !ok {
+		return QUIC_LH_PARSED{valid: false}
+	}
+
+	return QUIC_LH_PARSED{valid: true, frames: frames}
+
+}
+
+func quicFrameStitcher(packets []PacketInfo) []byte {
+	data := make(map[int]byte)
+
+	for _, pkt := range packets {
+		parsed := quic_long_header_parser(pkt.Transport_payload)
+		if !parsed.valid {
+			continue
+		}
+
+		for _, f := range parsed.frames {
+			for i := 0; i < len(f.payload); i++ {
+				pos := f.offset + i
+				if _, exists := data[pos]; !exists { // ignore retransmits
+					data[pos] = f.payload[i]
+				}
+			}
+		}
+	}
+
+	if len(data) == 0 {
+		return nil
+	}
+
+	keys := make([]int, 0, len(data))
+	for k := range data {
+		keys = append(keys, k)
+	}
+	slices.Sort(keys)
+
+	mn := keys[0]
+	mx := keys[len(keys)-1]
+
+	// Detect missing gaps
+	for i := mn; i <= mx; i++ {
+		if _, ok := data[i]; !ok {
+			return nil // incomplete CRYPTO stream
+		}
+	}
+
+	full := make([]byte, mx-mn+1)
+	for _, k := range keys {
+		full[k-mn] = data[k]
+	}
+
+	return full
+}
+
+func QUIC_SNI_extractor(packets []PacketInfo) TLS_CH_Parsed {
+	// All packets in the same direction
+
+	full_data := quicFrameStitcher(packets)
+
+	if full_data == nil {
+		return TLS_CH_Parsed{Valid: false}
+	}
+
+	return tls_ch_parser(full_data)
+
+}

network_core-0.5.6/go_src/commons/savingMethods.go

@@ -0,0 +1,41 @@
+package commons
+
+import (
+	"strconv"
+)
+
+type SaveStructCSV interface {
+	GetStringArr() []string
+	GetHeader() []string
+}
+
+func (flow SaveFlow) GetStringArr() []string {
+	return []string{
+		strconv.Itoa(flow.FlowId),
+		flow.FiveTuple.String(),
+		flow.StartTime.String(),
+		flow.EndTime.String(),
+		flow.SNI,
+		flow.ALPN,
+	}
+}
+
+func (flow SaveFlow) GetHeader() []string {
+	return []string{"FlowId", "FiveTuple", "StartTime", "EndTime", "SNI", "ALPN"}
+}
+
+func (packet SavePacket) GetHeader() []string {
+	return []string{
+		"FlowId", "PacketLength", "Timestamp", "Direction"}
+}
+
+func (packet SavePacket) GetStringArr() []string {
+	cols := []string{
+		strconv.Itoa(packet.FlowId),
+		strconv.Itoa(packet.PacketLength),
+		packet.Timestamp.String(),
+		strconv.Itoa(int(packet.Direction)),
+	}
+
+	return cols
+}