FlowAnalyzer 0.4.3__py3-none-any.whl → 0.4.5__py3-none-any.whl

FlowAnalyzer/FlowAnalyzer.py

@@ -1,47 +1,20 @@
-import contextlib
-import csv
-import gzip
 import os
 import sqlite3
 import subprocess
-from dataclasses import dataclass
-from typing import Iterable, NamedTuple, Optional, Tuple
-from urllib import parse
+from concurrent.futures import ThreadPoolExecutor
+from typing import Iterable, Optional
 
 from .logging_config import logger
+from .Models import HttpPair, Request, Response
+from .PacketParser import PacketParser
 from .Path import get_default_tshark_path
 
 
-@dataclass
-class Request:
-    __slots__ = ("frame_num", "header", "file_data", "full_uri", "time_epoch")
-    frame_num: int
-    header: bytes
-    file_data: bytes
-    full_uri: str
-    time_epoch: float
-
-
-@dataclass
-class Response:
-    __slots__ = ("frame_num", "header", "file_data", "time_epoch", "_request_in")
-    frame_num: int
-    header: bytes
-    file_data: bytes
-    time_epoch: float
-    _request_in: Optional[int]
-
-
-class HttpPair(NamedTuple):
-    request: Optional[Request]
-    response: Optional[Response]
-
-
 class FlowAnalyzer:
     """
     FlowAnalyzer 流量分析器 (智能缓存版)
     特点：
-    1. Tshark -> Pipe -> ijson -> SQLite (无中间JSON文件)
+    1. Tshark -> Pipe -> ThreadPool -> SQLite
     2. 智能校验：自动比对 Filter 和文件修改时间，防止缓存错乱
     3. 存储优化：数据库文件生成在流量包同级目录下
     """
@@ -80,7 +53,6 @@ class FlowAnalyzer:
 
             # === 第一步：配对查询 ===
             # 利用 SQLite 的 LEFT JOIN 直接匹配请求和响应
-            # 避免将所有数据加载到 Python 内存中
             sql_pair = """
             SELECT 
                 req.frame_num, req.header, req.file_data, req.full_uri, req.time_epoch,  -- 0-4 (Request)
@@ -94,19 +66,15 @@ class FlowAnalyzer:
 
             # 流式遍历结果，内存占用极低
             for row in cursor:
-                # 构建 Request 对象
-                # 注意处理 NULL 情况，虽然 requests 表理论上不为空，但防万一用 or b''
                 req = Request(frame_num=row[0], header=row[1] or b"", file_data=row[2] or b"", full_uri=row[3] or "", time_epoch=row[4])
 
                 resp = None
-                # 如果 row[5] (Response frame_num) 不为空，说明匹配到了响应
                 if row[5] is not None:
                     resp = Response(frame_num=row[5], header=row[6] or b"", file_data=row[7] or b"", time_epoch=row[8], _request_in=row[9])
 
                 yield HttpPair(request=req, response=resp)
 
             # === 第二步：孤儿响应查询 ===
-            # 找出那些有 request_in 但找不到对应 Request 的响应包
             sql_orphan = """
             SELECT frame_num, header, file_data, time_epoch, request_in
             FROM responses
@@ -126,33 +94,21 @@ class FlowAnalyzer:
     def get_json_data(file_path: str, display_filter: str, tshark_path: Optional[str] = None) -> str:
         """
         获取数据路径 (智能校验版)。
-
-        逻辑：
-        1. 根据 PCAP 路径推算 DB 路径 (位于 PCAP 同级目录)。
-        2. 检查 DB 是否存在。
-        3. 检查 Filter 和文件元数据是否一致。
-        4. 若一致返回路径，不一致则重新解析。
         """
         if not os.path.exists(file_path):
             raise FileNotFoundError("流量包路径不存在：%s" % file_path)
 
-        # --- 修改处：获取流量包的绝对路径和所在目录 ---
         abs_file_path = os.path.abspath(file_path)
-        pcap_dir = os.path.dirname(abs_file_path)  # 获取文件所在的文件夹
+        pcap_dir = os.path.dirname(abs_file_path)
         base_name = os.path.splitext(os.path.basename(abs_file_path))[0]
-
-        # 将 db_path 拼接在流量包所在的目录下
         db_path = os.path.join(pcap_dir, f"{base_name}.db")
-        # ----------------------------------------
 
-        # --- 校验环节 ---
         if FlowAnalyzer._is_cache_valid(db_path, abs_file_path, display_filter):
             logger.debug(f"缓存校验通过 (Filter匹配且文件未变)，使用缓存: [{db_path}]")
             return db_path
         else:
             logger.debug(f"缓存失效或不存在 (Filter变更或文件更新)，开始重新解析...")
 
-        # --- 解析环节 ---
         tshark_path = FlowAnalyzer.get_tshark_path(tshark_path)
         FlowAnalyzer._stream_tshark_to_db(abs_file_path, display_filter, tshark_path, db_path)
 
@@ -160,17 +116,10 @@ class FlowAnalyzer:
 
     @staticmethod
     def get_db_data(file_path: str, display_filter: str, tshark_path: Optional[str] = None) -> str:
-        """
-        获取数据库路径 (get_json_data 的语义化别名)。
-        新项目建议使用此方法名，get_json_data 保留用于兼容旧习惯。
-        """
         return FlowAnalyzer.get_json_data(file_path, display_filter, tshark_path)
 
     @staticmethod
     def _is_cache_valid(db_path: str, pcap_path: str, current_filter: str) -> bool:
-        """
-        检查缓存有效性：对比 Filter 字符串和文件元数据
-        """
         if not os.path.exists(db_path) or os.path.getsize(db_path) == 0:
             return False
 
@@ -188,7 +137,6 @@ class FlowAnalyzer:
 
                 cached_filter, cached_mtime, cached_size = row
 
-                # 容差 0.1秒
                 if cached_filter == current_filter and cached_size == current_size and abs(cached_mtime - current_mtime) < 0.1:
                     return True
                 else:
@@ -203,16 +151,7 @@ class FlowAnalyzer:
 
     @staticmethod
     def _stream_tshark_to_db(pcap_path: str, display_filter: str, tshark_path: str, db_path: str):
-        """流式解析并存入DB，同时记录元数据"""
-        # 增加 CSV 字段大小限制，防止超大包报错
-        # 将限制设置为系统最大值，注意 32位系统不要超过 2GB (但 Python int通常是动态的，保险起见设大一点)
-        # Windows下 sys.maxsize 通常足够大
-        try:
-            csv.field_size_limit(500 * 1024 * 1024)  # 500 MB
-        except Exception:
-            # 如果失败，尝试取最大值
-            csv.field_size_limit(int(2**31 - 1))
-
+        """流式解析并存入DB (多线程版)"""
         if os.path.exists(db_path):
             os.remove(db_path)
 
@@ -235,7 +174,6 @@ class FlowAnalyzer:
             """)
             conn.commit()
 
-        # 修改命令为 -T fields 模式
         command = [
             tshark_path,
             "-r",
@@ -244,7 +182,6 @@ class FlowAnalyzer:
             f"({display_filter})",
             "-T",
             "fields",
-            # 指定输出字段
             "-e",
             "http.response.code",  # 0
             "-e",
@@ -261,81 +198,91 @@ class FlowAnalyzer:
             "exported_pdu.exported_pdu",  # 6
             "-e",
             "http.request.full_uri",  # 7
-            # 格式控制
+            "-e",
+            "tcp.segment.count",  # 8
             "-E",
-            "header=n",  # 不输出表头
+            "header=n",
             "-E",
-            "separator=|",  # 使用 | 分割 (比逗号更安全)
+            "separator=/t",
             "-E",
-            "quote=d",  # 双引号包裹
+            "quote=n",
             "-E",
-            "occurrence=f",  # 每个字段只取第一个值 (First)
+            "occurrence=f",
         ]
 
         logger.debug(f"执行 Tshark: {command}")
+        BATCH_SIZE = 2000
+        MAX_PENDING_BATCHES = 20  # 控制内存中待处理的批次数量 (Backpressure)
 
-        # 使用 utf-8 编码读取 stdout text mode
-        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.dirname(os.path.abspath(pcap_path)), encoding="utf-8", errors="replace")
-
-        db_req_rows = []
-        db_resp_rows = []
-        BATCH_SIZE = 5000
+        # 使用 ThreadPoolExecutor 并行处理数据
+        max_workers = min(32, (os.cpu_count() or 1) + 4)
 
+        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.dirname(os.path.abspath(pcap_path)))
         try:
-            # 使用 csv.reader 解析 stdout 流
-            reader = csv.reader(process.stdout, delimiter="|", quotechar='"')  # type: ignore
             with sqlite3.connect(db_path) as conn:
                 cursor = conn.cursor()
 
-                for row in reader:
-                    # row 是一个列表，对应上面的 -e 顺序
-                    # [code, req_in, reassembled, frame, payload, epoch, pdu, uri]
-                    if not row:
-                        continue
-
-                    try:
-                        # 解析数据
-                        frame_num, request_in, time_epoch, full_uri, full_request = FlowAnalyzer.parse_packet_data(row)
+                with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                    current_batch = []
+                    pending_futures = []  # List[Future]
 
-                        if not full_request:
-                            continue
+                    def write_results_to_db(results):
+                        """将一批处理好的结果写入数据库"""
+                        if not results:
+                            return
 
-                        header, file_data = FlowAnalyzer.extract_http_file_data(full_request)
+                        db_req_rows = []
+                        db_resp_rows = []
 
-                        # 判断是请求还是响应
-                        # http.response.code (index 0) 是否为空
-                        if row[0]:
-                            # Response
-                            db_resp_rows.append((frame_num, header, file_data, time_epoch, request_in))
-                        else:
-                            # Request
-                            db_req_rows.append((frame_num, header, file_data, full_uri, time_epoch))
+                        for item in results:
+                            if item["type"] == "response":
+                                db_resp_rows.append((item["frame_num"], item["header"], item["file_data"], item["time_epoch"], item["request_in"]))
+                            else:
+                                db_req_rows.append((item["frame_num"], item["header"], item["file_data"], item["full_uri"], item["time_epoch"]))
 
-                        # 批量插入
-                        if len(db_req_rows) >= BATCH_SIZE:
+                        if db_req_rows:
                             cursor.executemany("INSERT OR REPLACE INTO requests VALUES (?,?,?,?,?)", db_req_rows)
-                            db_req_rows.clear()
-                        if len(db_resp_rows) >= BATCH_SIZE:
+                        if db_resp_rows:
                             cursor.executemany("INSERT OR REPLACE INTO responses VALUES (?,?,?,?,?)", db_resp_rows)
-                            db_resp_rows.clear()
 
-                    except Exception as e:
-                        # 偶尔可能会有解析失败的行，跳过即可
-                        pass
+                    def submit_batch():
+                        """提交当前批次到线程池"""
+                        if not current_batch:
+                            return
 
-                # 插入剩余数据
-                if db_req_rows:
-                    cursor.executemany("INSERT OR REPLACE INTO requests VALUES (?,?,?,?,?)", db_req_rows)
-                if db_resp_rows:
-                    cursor.executemany("INSERT OR REPLACE INTO responses VALUES (?,?,?,?,?)", db_resp_rows)
+                        # Copy batch data for the thread (list slicing is fast)
+                        batch_data = current_batch[:]
+                        future = executor.submit(PacketParser.process_batch, batch_data)
+                        pending_futures.append(future)
+                        current_batch.clear()
 
-                # --- 优化点：插入完数据后再创建索引，速度更快 ---
-                cursor.execute("CREATE INDEX idx_resp_req_in ON responses(request_in)")
+                    # --- Main Pipeline Loop ---
+                    if process.stdout:
+                        for line in process.stdout:
+                            current_batch.append(line)
+
+                            if len(current_batch) >= BATCH_SIZE:
+                                submit_batch()
+
+                                # Backpressure: 如果积压的任务太多，主线程暂停读取，先处理掉最早的一个
+                                # 这样既保证了 Pipeline 流动，又防止内存爆掉
+                                if len(pending_futures) >= MAX_PENDING_BATCHES:
+                                    oldest_future = pending_futures.pop(0)
+                                    write_results_to_db(oldest_future.result())
+
+                    # --- Drain Pipeline ---
+                    # 提交剩余数据
+                    submit_batch()
+
+                    # 等待所有剩余任务完成
+                    for future in pending_futures:
+                        write_results_to_db(future.result())
 
+                # 创建索引和元数据
+                cursor.execute("CREATE INDEX idx_resp_req_in ON responses(request_in)")
                 pcap_mtime = os.path.getmtime(pcap_path)
                 pcap_size = os.path.getsize(pcap_path)
                 cursor.execute("INSERT INTO meta_info (filter, pcap_path, pcap_mtime, pcap_size) VALUES (?, ?, ?, ?)", (display_filter, pcap_path, pcap_mtime, pcap_size))
-
                 conn.commit()
 
         except Exception as e:
@@ -346,131 +293,6 @@ class FlowAnalyzer:
             if process.poll() is None:
                 process.terminate()
 
-    # --- 辅助静态方法 ---
-
-    @staticmethod
-    def parse_packet_data(row: list) -> Tuple[int, int, float, str, str]:
-        # row definition:
-        # 0: http.response.code
-        # 1: http.request_in
-        # 2: tcp.reassembled.data
-        # 3: frame.number
-        # 4: tcp.payload
-        # 5: frame.time_epoch
-        # 6: exported_pdu.exported_pdu
-        # 7: http.request.full_uri
-
-        frame_num = int(row[3])
-        request_in = int(row[1]) if row[1] else frame_num
-        full_uri = parse.unquote(row[7]) if row[7] else ""
-        time_epoch = float(row[5])
-
-        if row[2]:
-            full_request = row[2]
-        elif row[4]:
-            full_request = row[4]
-        else:
-            full_request = row[6] if row[6] else ""
-
-        return frame_num, request_in, time_epoch, full_uri, full_request
-
-    @staticmethod
-    def split_http_headers(file_data: bytes) -> Tuple[bytes, bytes]:
-        headerEnd = file_data.find(b"\r\n\r\n")
-        if headerEnd != -1:
-            return file_data[: headerEnd + 4], file_data[headerEnd + 4 :]
-        elif file_data.find(b"\n\n") != -1:
-            headerEnd = file_data.index(b"\n\n") + 2
-            return file_data[:headerEnd], file_data[headerEnd:]
-        return b"", file_data
-
-    @staticmethod
-    def dechunck_http_response(file_data: bytes) -> bytes:
-        """解码分块TCP数据"""
-        if not file_data:
-            return b""
-
-        chunks = []
-        cursor = 0
-        total_len = len(file_data)
-
-        while cursor < total_len:
-            # 1. 寻找当前 Chunk Size 行的结束符 (\n)
-            newline_idx = file_data.find(b"\n", cursor)
-            if newline_idx == -1:
-                # 找不到换行符，说明格式不对，抛出异常让外层处理
-                raise ValueError("Not chunked data")
-
-            # 2. 提取并解析十六进制大小
-            size_line = file_data[cursor:newline_idx].strip()
-
-            # 处理可能的空行 (例如上一个 Chunk 后的 CRLF)
-            if not size_line:
-                cursor = newline_idx + 1
-                continue
-
-            # 这里不要捕获 ValueError，如果解析失败，直接抛出
-            # 说明这根本不是 chunk size，而是普通数据
-            chunk_size = int(size_line, 16)
-
-            # Chunk Size 为 0 表示传输结束
-            if chunk_size == 0:
-                break
-
-            # 3. 定位数据区域
-            data_start = newline_idx + 1
-            data_end = data_start + chunk_size
-
-            if data_end > total_len:
-                # 数据被截断，尽力读取
-                chunks.append(file_data[data_start:])
-                break
-
-            # 4. 提取数据
-            chunks.append(file_data[data_start:data_end])
-
-            # 5. 移动游标
-            cursor = data_end
-            # 跳过尾随的 \r 和 \n
-            while cursor < total_len and file_data[cursor] in (13, 10):
-                cursor += 1
-
-        return b"".join(chunks)
-
-    @staticmethod
-    def extract_http_file_data(full_request: str) -> Tuple[bytes, bytes]:
-        """提取HTTP请求或响应中的文件数据 (修复版)"""
-        # 1. 基础校验
-        if not full_request:
-            return b"", b""
-
-        try:
-            # 转为二进制
-            raw_bytes = bytes.fromhex(full_request)
-
-            # 分割 Header 和 Body
-            header, file_data = FlowAnalyzer.split_http_headers(raw_bytes)
-
-            # 处理 Chunked 编码
-            with contextlib.suppress(Exception):
-                file_data = FlowAnalyzer.dechunck_http_response(file_data)
-
-            # 处理 Gzip 压缩
-            with contextlib.suppress(Exception):
-                if file_data.startswith(b"\x1f\x8b"):
-                    file_data = gzip.decompress(file_data)
-
-            return header, file_data
-
-        except ValueError as e:
-            # 专门捕获 Hex 转换错误，并打印出来，方便你调试
-            # 如果你在控制台看到这个错误，说明 Tshark 输出的数据格式非常奇怪
-            logger.error(f"Hex转换失败: {str(e)[:100]}... 原数据片段: {full_request[:50]}")
-            return b"", b""
-        except Exception as e:
-            logger.error(f"解析HTTP数据未知错误: {e}")
-            return b"", b""
-
     @staticmethod
     def get_tshark_path(tshark_path: Optional[str]) -> str:
         default_tshark_path = get_default_tshark_path()

FlowAnalyzer/Models.py

@@ -0,0 +1,27 @@
+from dataclasses import dataclass
+from typing import NamedTuple, Optional
+
+
+@dataclass
+class Request:
+    __slots__ = ("frame_num", "header", "file_data", "full_uri", "time_epoch")
+    frame_num: int
+    header: bytes
+    file_data: bytes
+    full_uri: str
+    time_epoch: float
+
+
+@dataclass
+class Response:
+    __slots__ = ("frame_num", "header", "file_data", "time_epoch", "_request_in")
+    frame_num: int
+    header: bytes
+    file_data: bytes
+    time_epoch: float
+    _request_in: Optional[int]
+
+
+class HttpPair(NamedTuple):
+    request: Optional[Request]
+    response: Optional[Response]

FlowAnalyzer/PacketParser.py

@@ -0,0 +1,183 @@
+import binascii
+import contextlib
+import gzip
+from typing import List, Optional, Tuple
+from urllib import parse
+
+from .logging_config import logger
+
+
+class PacketParser:
+    @staticmethod
+    def parse_packet_data(row: list) -> Tuple[int, int, float, str, bytes]:
+        """
+        解析 Tshark 输出的一行数据
+        row definition (all bytes):
+        0: http.response.code
+        1: http.request_in
+        2: tcp.reassembled.data
+        3: frame.number
+        4: tcp.payload
+        5: frame.time_epoch
+        6: exported_pdu.exported_pdu
+        7: http.request.full_uri 
+        8: tcp.segment.count
+        """
+        frame_num = int(row[3])
+        request_in = int(row[1]) if row[1] else frame_num
+        # Decode only URI to string
+        full_uri = parse.unquote(row[7].decode("utf-8", errors="replace")) if row[7] else ""
+        time_epoch = float(row[5])
+
+        # Logic for Raw Packet (Header Source)
+        # Previous index 9 is now 8 since we removed http.file_data
+        is_reassembled = len(row) > 8 and row[8]
+
+        if is_reassembled and row[2]:
+            full_request = row[2]
+        elif row[4]:
+            full_request = row[4]
+        else:
+            # Fallback (e.g. Exported PDU)
+            full_request = row[2] if row[2] else (row[6] if row[6] else b"")
+
+        return frame_num, request_in, time_epoch, full_uri, full_request
+
+    @staticmethod
+    def split_http_headers(file_data: bytes) -> Tuple[bytes, bytes]:
+        headerEnd = file_data.find(b"\r\n\r\n")
+        if headerEnd != -1:
+            return file_data[: headerEnd + 4], file_data[headerEnd + 4 :]
+        elif file_data.find(b"\n\n") != -1:
+            headerEnd = file_data.index(b"\n\n") + 2
+            return file_data[:headerEnd], file_data[headerEnd:]
+        return b"", file_data
+
+    @staticmethod
+    def dechunk_http_response(file_data: bytes) -> bytes:
+        """解码分块TCP数据"""
+        if not file_data:
+            return b""
+
+        chunks = []
+        cursor = 0
+        total_len = len(file_data)
+
+        while cursor < total_len:
+            newline_idx = file_data.find(b"\n", cursor)
+            if newline_idx == -1:
+                # If no newline found, maybe it's just remaining data (though strictly should end with 0 chunk)
+                # But for robustness we might perform a "best effort" or just stop.
+                # raising ValueError("Not chunked data") might be too aggressive if we are just "trying" to dechunk
+                # Let's assume non-chunked if strict format not found
+                raise ValueError("Not chunked data")
+
+            size_line = file_data[cursor:newline_idx].strip()
+            if not size_line:
+                cursor = newline_idx + 1
+                continue
+
+            try:
+                chunk_size = int(size_line, 16)
+            except ValueError:
+                raise ValueError("Invalid chunk size")
+
+            if chunk_size == 0:
+                break
+
+            data_start = newline_idx + 1
+            data_end = data_start + chunk_size
+
+            # Robustness check
+            if data_start > total_len:
+                break
+
+            if data_end > total_len:
+                chunks.append(file_data[data_start:])
+                break
+
+            chunks.append(file_data[data_start:data_end])
+
+            cursor = data_end
+            # Skip CRLF after chunk data
+            while cursor < total_len and file_data[cursor] in (13, 10):
+                cursor += 1
+
+        return b"".join(chunks)
+
+    @staticmethod
+    def extract_http_file_data(full_request: bytes) -> Tuple[bytes, bytes]:
+        """
+        提取HTTP请求或响应中的文件数据 (混合模式 - 二进制优化版)
+        """
+        header = b""
+        file_data = b""
+
+        if not full_request:
+            return b"", b""
+        try:
+            raw_bytes = binascii.unhexlify(full_request)
+            header, body_part = PacketParser.split_http_headers(raw_bytes)
+
+            with contextlib.suppress(Exception):
+                body_part = PacketParser.dechunk_http_response(body_part)
+
+            with contextlib.suppress(Exception):
+                if body_part.startswith(b"\x1f\x8b"):
+                    body_part = gzip.decompress(body_part)
+
+            file_data = body_part
+            return header, file_data
+
+        except binascii.Error:
+            logger.error("Hex转换失败")
+            return b"", b""
+        except Exception as e:
+            logger.error(f"解析HTTP数据未知错误: {e}")
+            return b"", b""
+
+    @staticmethod
+    def process_row(line: bytes) -> Optional[dict]:
+        """
+        处理单行数据，返回结构化结果供主线程写入
+        """
+        line = line.rstrip(b"\r\n")
+        if not line:
+            return None
+
+        row = line.split(b"\t")
+        try:
+            frame_num, request_in, time_epoch, full_uri, full_request = PacketParser.parse_packet_data(row)
+
+            if not full_request:
+                return None
+
+            header, file_data = PacketParser.extract_http_file_data(full_request)
+
+            # row[0] is http.response.code (bytes)
+            is_response = bool(row[0])
+
+            return {
+                "type": "response" if is_response else "request",
+                "frame_num": frame_num,
+                "header": header,
+                "file_data": file_data,
+                "time_epoch": time_epoch,
+                "request_in": request_in,  # Only useful for Response
+                "full_uri": full_uri,  # Only useful for Request
+            }
+
+        except Exception:
+            return None
+
+    @staticmethod
+    def process_batch(lines: List[bytes]) -> List[dict]:
+        """
+        批量处理行数据，减少函数调用开销
+        """
+        results = []
+        for line in lines:
+            res = PacketParser.process_row(line)
+            if res:
+                results.append(res)
+        return results

FlowAnalyzer/PcapSplitter.py

@@ -0,0 +1,128 @@
+import os
+import time
+from collections import defaultdict
+from typing import List, Tuple
+
+import dpkt
+
+
+class PcapSplitter:
+    """
+    Encapsulates logic to split a PCAP/PCAPNG file into multiple smaller PCAP files
+    based on TCP flows, dynamically balanced for parallel processing.
+    """
+
+    def __init__(self, pcap_file: str, output_dir: str):
+        self.pcap_file = pcap_file
+        self.output_dir = output_dir
+
+    def get_stream_key(self, tcp, ip) -> Tuple:
+        """Generate a 5-tuple key for the flow."""
+        src = ip.src
+        dst = ip.dst
+        sport = tcp.sport
+        dport = tcp.dport
+        # Canonicalize bidirectional flows to the same key
+        key1 = (src, dst, sport, dport)
+        key2 = (dst, src, dport, sport)
+        return key1 if key1 < key2 else key2
+
+    def split(self, threshold_mb: int = 10, default_chunks: int = 3) -> List[str]:
+        """
+        Split the pcap file into balanced chunks based on stream volume (bytes).
+        Uses a Greedy Partition Algorithm (Longest Processing Time first).
+
+        Args:
+            threshold_mb: File size threshold in MB. If smaller, do not split.
+            default_chunks: Number of chunks to split into if threshold is exceeded.
+
+        Returns:
+            List of generated file paths (or original file if not split).
+        """
+        if not os.path.exists(self.pcap_file):
+            raise FileNotFoundError(f"PCAP file not found: {self.pcap_file}")
+
+        file_size_mb = os.path.getsize(self.pcap_file) / (1024 * 1024)
+        if file_size_mb < threshold_mb:
+            print(f"File size {file_size_mb:.2f}MB < {threshold_mb}MB. Skipping split.")
+            return [self.pcap_file]
+
+        os.makedirs(self.output_dir, exist_ok=True)
+
+        start_time = time.time()
+        # Dictionary to store packets: stream_key -> list of (ts, buf)
+        streams = defaultdict(list)
+        # Dictionary to store total size: stream_key -> total_bytes
+        stream_sizes = defaultdict(int)
+
+        # 1. Read and Group Packets
+        print(f"Reading {self.pcap_file}...")
+        with open(self.pcap_file, "rb") as f:
+            if self.pcap_file.lower().endswith(".pcapng"):
+                reader = dpkt.pcapng.Reader(f)
+            else:
+                reader = dpkt.pcap.Reader(f)
+
+            for ts, buf in reader:
+                try:
+                    eth = dpkt.ethernet.Ethernet(buf)
+                    if not isinstance(eth.data, dpkt.ip.IP):
+                        continue
+                    ip = eth.data
+                    if not isinstance(ip.data, dpkt.tcp.TCP):
+                        continue
+                    tcp = ip.data
+
+                    key = self.get_stream_key(tcp, ip)
+                    streams[key].append((ts, buf))
+                    stream_sizes[key] += len(buf)
+                except Exception:
+                    continue
+
+        total_streams = len(streams)
+        print(f"Found {total_streams} TCP streams.")
+
+        if total_streams == 0:
+            print("No TCP streams found to split.")
+            return []
+
+        # 2. Assign Streams to Buckets (Greedy LPT Algorithm)
+        num_chunks = min(default_chunks, total_streams)
+
+        # Sort streams by size (descending)
+        sorted_streams = sorted(stream_sizes.items(), key=lambda item: item[1], reverse=True)
+
+        # Buckets: list of (current_size, batch_index, list_of_keys)
+        # We perform standard list sort to find min bucket, sufficient for small N
+        buckets = [[0, i, []] for i in range(num_chunks)]
+
+        for key, size in sorted_streams:
+            # Find bucket with smallest current size
+            buckets.sort(key=lambda x: x[0])
+            smallest_bucket = buckets[0]
+
+            # Add stream to this bucket
+            smallest_bucket[0] += size
+            smallest_bucket[2].append(key)
+
+        print(f"Splitting into {num_chunks} files with volume balancing...")
+        generated_files = []
+
+        # 3. Write Batches
+        # Sort buckets by index ensures file naming order 0, 1, 2...
+        buckets.sort(key=lambda x: x[1])
+
+        for size, i, batch_keys in buckets:
+            out_file_path = os.path.join(self.output_dir, f"batch_{i}.pcap")
+            generated_files.append(out_file_path)
+
+            with open(out_file_path, "wb") as f:
+                writer = dpkt.pcap.Writer(f)
+                for key in batch_keys:
+                    for ts, buf in streams[key]:
+                        writer.writepkt(buf, ts)
+
+            print(f"  - Created {os.path.basename(out_file_path)}: {len(batch_keys)} streams ({size/1024/1024:.2f} MB)")
+
+        print(f"Split completed in {time.time() - start_time:.2f}s")
+        return generated_files

FlowAnalyzer/logging_config.py

@@ -15,8 +15,9 @@ def configure_logger(logger_name, level=logging.DEBUG) -> logging.Logger:
     console_handler.setFormatter(formatter)
     return logger
 
+
 logger = configure_logger("FlowAnalyzer", logging.INFO)
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     logger = configure_logger("FlowAnalyzer")
     logger.info("This is a test!")

{flowanalyzer-0.4.3.dist-info → flowanalyzer-0.4.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: FlowAnalyzer
-Version: 0.4.3
+Version: 0.4.5
 Summary: FlowAnalyzer是一个流量分析器，用于解析和处理tshark导出的JSON数据文件
 Home-page: https://github.com/Byxs20/FlowAnalyzer
 Author: Byxs20
@@ -15,6 +15,14 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: summary
 
 # FlowAnalyzer
 
@@ -28,9 +36,11 @@ License-File: LICENSE
 
 为了解决传统解析方式慢、内存占用高的问题，FlowAnalyzer 进行了核心架构升级：**流式解析 + SQLite 智能缓存**。
 
-### 1. ⚡️ 高性能流式解析
-- **极低内存占用**：不再将整个 JSON 读入内存。通过 `subprocess` 管道对接 Tshark 输出，结合 `ijson` 进行增量解析。
-- **无中间文件**：解析过程中不生成体积巨大的临时 JSON 文件，直接入库。
+### 1. ⚡️ 高性能流式解析 (多线程流水线)
+- **多线程并行**：采用 `ThreadPoolExecutor` 构建流水线，主线程负责读取 Tshark 输出，子线程并行解析数据包，充分利用多核 CPU。
+- **批量处理**：引入 Batch 机制（默认 2000 包/批），大幅减少数据库事务开销和 Python 函数调用损耗。
+- **内存背压控制 (Backpressure)**：智能监控待处理队列长度，防止在处理高速流量时内存溢出。
+- **极低内存占用**：不再将整个 JSON 读入内存。通过 `subprocess` 管道流式处理，解析过程中不生成体积巨大的临时文件。
 
 ### 2. 💾 智能缓存机制
 - **自动缓存**：首次分析 `test.pcap` 时，会自动生成同级目录下的 `test.db`。
@@ -51,7 +61,7 @@ License-File: LICENSE
 
 | 特性         | 旧版架构                      | **新版架构 (FlowAnalyzer)**         |
 | :----------- | :---------------------------- | :---------------------------------- |
-| **解析流程** | 生成巨大 JSON -> 全量读入内存 | Tshark流 -> 管道 -> ijson -> SQLite |
+| **解析流程** | 生成巨大 JSON -> 全量读入内存 | Tshark流 -> 多线程Batch解析 -> SQLite |
 | **内存占用** | 极高 (易 OOM)                 | **极低 (内存稳定)**                 |
 | **二次加载** | 需重新解析                    | **直接读取 DB (0秒)**               |
 | **磁盘占用** | 巨大的临时 JSON 文件          | 轻量级 SQLite 文件                  |
@@ -63,11 +73,11 @@ License-File: LICENSE
 请确保您的环境中已安装 Python 3 和 Tshark (Wireshark)。
 
 ```bash
-# 安装 FlowAnalyzer 及其依赖 ijson
-pip3 install FlowAnalyzer ijson
+# 安装 FlowAnalyzer
+pip3 install FlowAnalyzer
 
 # 或者使用国内源加速
-pip3 install FlowAnalyzer ijson -i https://pypi.org/simple
+pip3 install FlowAnalyzer -i https://pypi.org/simple
 ```
 
 ---

flowanalyzer-0.4.5.dist-info/RECORD

@@ -0,0 +1,12 @@
+FlowAnalyzer/FlowAnalyzer.py,sha256=9SshWk5wf0XATI7W4eBiIpzEqeGFyQJs3on5ox-zrNQ,12666
+FlowAnalyzer/Models.py,sha256=2x7nPJIAyLTC1oiGFlW4mELDPgthk2IsmuyearT-MSQ,622
+FlowAnalyzer/PacketParser.py,sha256=vdXUMFteSlIbOJ4y4_ikUIL3HwBCFBBgjevNL0jLozE,6174
+FlowAnalyzer/Path.py,sha256=E5VvucTftp8VTQUffFzFWHotQEYtZL-j7IQPOaleiug,130
+FlowAnalyzer/PcapSplitter.py,sha256=0E_vmLYYsE_gD34XTwG1XPx5kBg8ZchJspQEnkBoIdY,4855
+FlowAnalyzer/__init__.py,sha256=vfiHONPTrvjUU3MwhjFOEo3sWfzlhkA6gOLn_4UJ7sg,70
+FlowAnalyzer/logging_config.py,sha256=fnBlvoimteQ38IBlQBV9fdLQvfAlRgGhcvLpUC3YunA,732
+flowanalyzer-0.4.5.dist-info/licenses/LICENSE,sha256=ybAV0ECduYBZCpjkHyNALVWRRmT_eM0BDgqUszhwEFU,1080
+flowanalyzer-0.4.5.dist-info/METADATA,sha256=oyoNqX8eZkkiNTrkz8qrZ6T7ofrW0lnFSxXoV2Q1wIU,6099
+flowanalyzer-0.4.5.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+flowanalyzer-0.4.5.dist-info/top_level.txt,sha256=2MtvAF6dEe_eHipw_6G5pFLb2uOCbGnlH0bC4iBtm5A,13
+flowanalyzer-0.4.5.dist-info/RECORD,,

{flowanalyzer-0.4.3.dist-info → flowanalyzer-0.4.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.2)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

flowanalyzer-0.4.3.dist-info/RECORD

@@ -1,9 +0,0 @@
-FlowAnalyzer/FlowAnalyzer.py,sha256=9seSOamepCnejHYRKLWym9Eu0lbxCgn7p3hE2WUZstk,18964
-FlowAnalyzer/Path.py,sha256=E5VvucTftp8VTQUffFzFWHotQEYtZL-j7IQPOaleiug,130
-FlowAnalyzer/__init__.py,sha256=vfiHONPTrvjUU3MwhjFOEo3sWfzlhkA6gOLn_4UJ7sg,70
-FlowAnalyzer/logging_config.py,sha256=-RntNJhrBiW7ToXIP1WJjZ4Yf9jmZQ1PTX_er3tDxhw,730
-flowanalyzer-0.4.3.dist-info/LICENSE,sha256=ybAV0ECduYBZCpjkHyNALVWRRmT_eM0BDgqUszhwEFU,1080
-flowanalyzer-0.4.3.dist-info/METADATA,sha256=W6BhXCna1TYeTVd_gY5Q63xjbckhRpomHYErrtS5fBM,5588
-flowanalyzer-0.4.3.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-flowanalyzer-0.4.3.dist-info/top_level.txt,sha256=2MtvAF6dEe_eHipw_6G5pFLb2uOCbGnlH0bC4iBtm5A,13
-flowanalyzer-0.4.3.dist-info/RECORD,,