FlowAnalyzer 0.4.1__py3-none-any.whl → 0.4.3__py3-none-any.whl

FlowAnalyzer/FlowAnalyzer.py

@@ -1,14 +1,13 @@
 import contextlib
+import csv
 import gzip
 import os
 import sqlite3
 import subprocess
 from dataclasses import dataclass
-from typing import Dict, Iterable, NamedTuple, Optional, Tuple
+from typing import Iterable, NamedTuple, Optional, Tuple
 from urllib import parse
 
-import ijson
-
 from .logging_config import logger
 from .Path import get_default_tshark_path
 
@@ -69,57 +68,54 @@ class FlowAnalyzer:
         if not os.path.exists(self.db_path):
             raise FileNotFoundError(f"未找到数据文件或缓存数据库: {self.db_path}，请先调用 get_json_data 生成。")
 
-    def _load_from_db(self) -> Tuple[Dict[int, Request], Dict[int, Response]]:
-        """从 SQLite 数据库加载数据"""
-        requests, responses = {}, {}
-        try:
-            with sqlite3.connect(self.db_path) as conn:
-                cursor = conn.cursor()
+    def generate_http_dict_pairs(self) -> Iterable[HttpPair]:
+        """生成HTTP请求和响应信息的字典对 (SQL JOIN 高性能版)"""
+        if not os.path.exists(self.db_path):
+            return
 
-                # 简单防错检查
-                try:
-                    cursor.execute("SELECT count(*) FROM requests")
-                    if cursor.fetchone()[0] == 0:
-                        cursor.execute("SELECT count(*) FROM responses")
-                        if cursor.fetchone()[0] == 0:
-                            return {}, {}
-                except sqlite3.OperationalError:
-                    logger.error("数据库损坏或表不存在")
-                    return {}, {}
-
-                logger.debug(f"正在加载缓存数据: {self.db_path}")
-
-                # 加载 Requests
-                cursor.execute("SELECT frame_num, header, file_data, full_uri, time_epoch FROM requests")
-                for row in cursor.fetchall():
-                    requests[row[0]] = Request(row[0], row[1], row[2], row[3], row[4])
-
-                # 加载 Responses
-                cursor.execute("SELECT frame_num, header, file_data, time_epoch, request_in FROM responses")
-                for row in cursor.fetchall():
-                    responses[row[0]] = Response(row[0], row[1], row[2], row[3], row[4])
-
-                return requests, responses
-        except sqlite3.Error as e:
-            logger.error(f"读取数据库出错: {e}")
-            return {}, {}
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            # 开启查询优化
+            cursor.execute("PRAGMA query_only = 1;")
+
+            # === 第一步：配对查询 ===
+            # 利用 SQLite 的 LEFT JOIN 直接匹配请求和响应
+            # 避免将所有数据加载到 Python 内存中
+            sql_pair = """
+            SELECT 
+                req.frame_num, req.header, req.file_data, req.full_uri, req.time_epoch,  -- 0-4 (Request)
+                resp.frame_num, resp.header, resp.file_data, resp.time_epoch, resp.request_in -- 5-9 (Response)
+            FROM requests req
+            LEFT JOIN responses resp ON req.frame_num = resp.request_in
+            ORDER BY req.frame_num ASC
+            """
+
+            cursor.execute(sql_pair)
+
+            # 流式遍历结果，内存占用极低
+            for row in cursor:
+                # 构建 Request 对象
+                # 注意处理 NULL 情况，虽然 requests 表理论上不为空，但防万一用 or b''
+                req = Request(frame_num=row[0], header=row[1] or b"", file_data=row[2] or b"", full_uri=row[3] or "", time_epoch=row[4])
+
+                resp = None
+                # 如果 row[5] (Response frame_num) 不为空，说明匹配到了响应
+                if row[5] is not None:
+                    resp = Response(frame_num=row[5], header=row[6] or b"", file_data=row[7] or b"", time_epoch=row[8], _request_in=row[9])
 
-    def generate_http_dict_pairs(self) -> Iterable[HttpPair]:
-        """生成HTTP请求和响应信息的字典对"""
-        requests, responses = self._load_from_db()
-        response_map = {r._request_in: r for r in responses.values()}
-        yielded_resps = set()
-
-        for req_id, req in requests.items():
-            resp = response_map.get(req_id)
-            if resp:
-                yielded_resps.add(resp.frame_num)
                 yield HttpPair(request=req, response=resp)
-            else:
-                yield HttpPair(request=req, response=None)
 
-        for resp in responses.values():
-            if resp.frame_num not in yielded_resps:
+            # === 第二步：孤儿响应查询 ===
+            # 找出那些有 request_in 但找不到对应 Request 的响应包
+            sql_orphan = """
+            SELECT frame_num, header, file_data, time_epoch, request_in
+            FROM responses
+            WHERE request_in NOT IN (SELECT frame_num FROM requests)
+            """
+            cursor.execute(sql_orphan)
+
+            for row in cursor:
+                resp = Response(frame_num=row[0], header=row[1] or b"", file_data=row[2] or b"", time_epoch=row[3], _request_in=row[4])
                 yield HttpPair(request=None, response=resp)
 
     # =========================================================================
@@ -208,6 +204,14 @@ class FlowAnalyzer:
     @staticmethod
     def _stream_tshark_to_db(pcap_path: str, display_filter: str, tshark_path: str, db_path: str):
         """流式解析并存入DB，同时记录元数据"""
+        # 增加 CSV 字段大小限制，防止超大包报错
+        # 将限制设置为系统最大值，注意 32位系统不要超过 2GB (但 Python int通常是动态的，保险起见设大一点)
+        # Windows下 sys.maxsize 通常足够大
+        try:
+            csv.field_size_limit(500 * 1024 * 1024)  # 500 MB
+        except Exception:
+            # 如果失败，尝试取最大值
+            csv.field_size_limit(int(2**31 - 1))
 
         if os.path.exists(db_path):
             os.remove(db_path)
@@ -231,6 +235,7 @@ class FlowAnalyzer:
             """)
             conn.commit()
 
+        # 修改命令为 -T fields 模式
         command = [
             tshark_path,
             "-r",
@@ -238,55 +243,75 @@ class FlowAnalyzer:
             "-Y",
             f"({display_filter})",
             "-T",
-            "json",
+            "fields",
+            # 指定输出字段
             "-e",
-            "http.response.code",
+            "http.response.code",  # 0
             "-e",
-            "http.request_in",
+            "http.request_in",  # 1
             "-e",
-            "tcp.reassembled.data",
+            "tcp.reassembled.data",  # 2
             "-e",
-            "frame.number",
+            "frame.number",  # 3
             "-e",
-            "tcp.payload",
+            "tcp.payload",  # 4
             "-e",
-            "frame.time_epoch",
+            "frame.time_epoch",  # 5
             "-e",
-            "exported_pdu.exported_pdu",
+            "exported_pdu.exported_pdu",  # 6
             "-e",
-            "http.request.full_uri",
+            "http.request.full_uri",  # 7
+            # 格式控制
+            "-E",
+            "header=n",  # 不输出表头
+            "-E",
+            "separator=|",  # 使用 | 分割 (比逗号更安全)
+            "-E",
+            "quote=d",  # 双引号包裹
+            "-E",
+            "occurrence=f",  # 每个字段只取第一个值 (First)
         ]
 
         logger.debug(f"执行 Tshark: {command}")
 
-        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.dirname(os.path.abspath(pcap_path)))
+        # 使用 utf-8 编码读取 stdout text mode
+        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.dirname(os.path.abspath(pcap_path)), encoding="utf-8", errors="replace")
 
         db_req_rows = []
         db_resp_rows = []
         BATCH_SIZE = 5000
 
         try:
-            parser = ijson.items(process.stdout, "item")
-
+            # 使用 csv.reader 解析 stdout 流
+            reader = csv.reader(process.stdout, delimiter="|", quotechar='"')  # type: ignore
             with sqlite3.connect(db_path) as conn:
                 cursor = conn.cursor()
 
-                for packet in parser:
-                    layers = packet.get("_source", {}).get("layers", {})
-                    if not layers:
+                for row in reader:
+                    # row 是一个列表，对应上面的 -e 顺序
+                    # [code, req_in, reassembled, frame, payload, epoch, pdu, uri]
+                    if not row:
                         continue
 
                     try:
-                        frame_num, request_in, time_epoch, full_uri, full_request = FlowAnalyzer.parse_packet_data(layers)
+                        # 解析数据
+                        frame_num, request_in, time_epoch, full_uri, full_request = FlowAnalyzer.parse_packet_data(row)
+
                         if not full_request:
                             continue
+
                         header, file_data = FlowAnalyzer.extract_http_file_data(full_request)
 
-                        if layers.get("http.response.code"):
+                        # 判断是请求还是响应
+                        # http.response.code (index 0) 是否为空
+                        if row[0]:
+                            # Response
                             db_resp_rows.append((frame_num, header, file_data, time_epoch, request_in))
                         else:
+                            # Request
                             db_req_rows.append((frame_num, header, file_data, full_uri, time_epoch))
 
+                        # 批量插入
                         if len(db_req_rows) >= BATCH_SIZE:
                             cursor.executemany("INSERT OR REPLACE INTO requests VALUES (?,?,?,?,?)", db_req_rows)
                             db_req_rows.clear()
@@ -294,14 +319,19 @@ class FlowAnalyzer:
                             cursor.executemany("INSERT OR REPLACE INTO responses VALUES (?,?,?,?,?)", db_resp_rows)
                             db_resp_rows.clear()
 
-                    except Exception:
+                    except Exception as e:
+                        # 偶尔可能会有解析失败的行，跳过即可
                         pass
 
+                # 插入剩余数据
                 if db_req_rows:
                     cursor.executemany("INSERT OR REPLACE INTO requests VALUES (?,?,?,?,?)", db_req_rows)
                 if db_resp_rows:
                     cursor.executemany("INSERT OR REPLACE INTO responses VALUES (?,?,?,?,?)", db_resp_rows)
 
+                # --- 优化点：插入完数据后再创建索引，速度更快 ---
+                cursor.execute("CREATE INDEX idx_resp_req_in ON responses(request_in)")
+
                 pcap_mtime = os.path.getmtime(pcap_path)
                 pcap_size = os.path.getsize(pcap_path)
                 cursor.execute("INSERT INTO meta_info (filter, pcap_path, pcap_mtime, pcap_size) VALUES (?, ?, ?, ?)", (display_filter, pcap_path, pcap_mtime, pcap_size))
@@ -319,18 +349,29 @@ class FlowAnalyzer:
     # --- 辅助静态方法 ---
 
     @staticmethod
-    def parse_packet_data(packet: dict) -> Tuple[int, int, float, str, str]:
-        frame_num = int(packet["frame.number"][0])
-        request_in = int(packet["http.request_in"][0]) if packet.get("http.request_in") else frame_num
-        full_uri = parse.unquote(packet["http.request.full_uri"][0]) if packet.get("http.request.full_uri") else ""
-        time_epoch = float(packet["frame.time_epoch"][0])
-
-        if packet.get("tcp.reassembled.data"):
-            full_request = packet["tcp.reassembled.data"][0]
-        elif packet.get("tcp.payload"):
-            full_request = packet["tcp.payload"][0]
+    def parse_packet_data(row: list) -> Tuple[int, int, float, str, str]:
+        # row definition:
+        # 0: http.response.code
+        # 1: http.request_in
+        # 2: tcp.reassembled.data
+        # 3: frame.number
+        # 4: tcp.payload
+        # 5: frame.time_epoch
+        # 6: exported_pdu.exported_pdu
+        # 7: http.request.full_uri
+
+        frame_num = int(row[3])
+        request_in = int(row[1]) if row[1] else frame_num
+        full_uri = parse.unquote(row[7]) if row[7] else ""
+        time_epoch = float(row[5])
+
+        if row[2]:
+            full_request = row[2]
+        elif row[4]:
+            full_request = row[4]
         else:
-            full_request = packet["exported_pdu.exported_pdu"][0] if packet.get("exported_pdu.exported_pdu") else ""
+            full_request = row[6] if row[6] else ""
+
         return frame_num, request_in, time_epoch, full_uri, full_request
 
     @staticmethod
@@ -345,10 +386,7 @@ class FlowAnalyzer:
 
     @staticmethod
     def dechunck_http_response(file_data: bytes) -> bytes:
-        """解码分块TCP数据 (修复版)
-        注意：如果数据不是 Chunked 格式，此函数必须抛出异常，
-        以便外层逻辑回退到使用原始数据。
-        """
+        """解码分块TCP数据"""
         if not file_data:
             return b""
 

{flowanalyzer-0.4.1.dist-info → flowanalyzer-0.4.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: FlowAnalyzer
-Version: 0.4.1
+Version: 0.4.3
 Summary: FlowAnalyzer是一个流量分析器，用于解析和处理tshark导出的JSON数据文件
 Home-page: https://github.com/Byxs20/FlowAnalyzer
 Author: Byxs20
@@ -15,14 +15,6 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license-file
-Dynamic: summary
 
 # FlowAnalyzer
 

flowanalyzer-0.4.3.dist-info/RECORD

@@ -0,0 +1,9 @@
+FlowAnalyzer/FlowAnalyzer.py,sha256=9seSOamepCnejHYRKLWym9Eu0lbxCgn7p3hE2WUZstk,18964
+FlowAnalyzer/Path.py,sha256=E5VvucTftp8VTQUffFzFWHotQEYtZL-j7IQPOaleiug,130
+FlowAnalyzer/__init__.py,sha256=vfiHONPTrvjUU3MwhjFOEo3sWfzlhkA6gOLn_4UJ7sg,70
+FlowAnalyzer/logging_config.py,sha256=-RntNJhrBiW7ToXIP1WJjZ4Yf9jmZQ1PTX_er3tDxhw,730
+flowanalyzer-0.4.3.dist-info/LICENSE,sha256=ybAV0ECduYBZCpjkHyNALVWRRmT_eM0BDgqUszhwEFU,1080
+flowanalyzer-0.4.3.dist-info/METADATA,sha256=W6BhXCna1TYeTVd_gY5Q63xjbckhRpomHYErrtS5fBM,5588
+flowanalyzer-0.4.3.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+flowanalyzer-0.4.3.dist-info/top_level.txt,sha256=2MtvAF6dEe_eHipw_6G5pFLb2uOCbGnlH0bC4iBtm5A,13
+flowanalyzer-0.4.3.dist-info/RECORD,,

{flowanalyzer-0.4.1.dist-info → flowanalyzer-0.4.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.10.2)
+Generator: setuptools (75.3.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

flowanalyzer-0.4.1.dist-info/RECORD

@@ -1,9 +0,0 @@
-FlowAnalyzer/FlowAnalyzer.py,sha256=ciuWFPQWQgYqjdL_u7ck4BNIsQNx00HLOjr6lSkfzMg,17348
-FlowAnalyzer/Path.py,sha256=E5VvucTftp8VTQUffFzFWHotQEYtZL-j7IQPOaleiug,130
-FlowAnalyzer/__init__.py,sha256=vfiHONPTrvjUU3MwhjFOEo3sWfzlhkA6gOLn_4UJ7sg,70
-FlowAnalyzer/logging_config.py,sha256=-RntNJhrBiW7ToXIP1WJjZ4Yf9jmZQ1PTX_er3tDxhw,730
-flowanalyzer-0.4.1.dist-info/licenses/LICENSE,sha256=ybAV0ECduYBZCpjkHyNALVWRRmT_eM0BDgqUszhwEFU,1080
-flowanalyzer-0.4.1.dist-info/METADATA,sha256=WD01CpYRDVbT8RA5GwTKYZPv8Fa06_-4ZuiTAa5SfeE,5767
-flowanalyzer-0.4.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-flowanalyzer-0.4.1.dist-info/top_level.txt,sha256=2MtvAF6dEe_eHipw_6G5pFLb2uOCbGnlH0bC4iBtm5A,13
-flowanalyzer-0.4.1.dist-info/RECORD,,