FlowAnalyzer 0.3.9__py3-none-any.whl → 0.4.1__py3-none-any.whl

FlowAnalyzer/FlowAnalyzer.py

@@ -1,17 +1,21 @@
 import contextlib
 import gzip
-import hashlib
-import json
 import os
+import sqlite3
 import subprocess
+from dataclasses import dataclass
 from typing import Dict, Iterable, NamedTuple, Optional, Tuple
 from urllib import parse
 
+import ijson
+
 from .logging_config import logger
 from .Path import get_default_tshark_path
 
 
-class Request(NamedTuple):
+@dataclass
+class Request:
+    __slots__ = ("frame_num", "header", "file_data", "full_uri", "time_epoch")
     frame_num: int
     header: bytes
     file_data: bytes
@@ -19,12 +23,14 @@ class Request(NamedTuple):
     time_epoch: float
 
 
-class Response(NamedTuple):
+@dataclass
+class Response:
+    __slots__ = ("frame_num", "header", "file_data", "time_epoch", "_request_in")
     frame_num: int
     header: bytes
     file_data: bytes
-    request_in: int
     time_epoch: float
+    _request_in: Optional[int]
 
 
 class HttpPair(NamedTuple):
@@ -33,293 +39,405 @@ class HttpPair(NamedTuple):
 
 
 class FlowAnalyzer:
-    """FlowAnalyzer是一个流量分析器，用于解析和处理tshark导出的JSON数据文件"""
-
-    def __init__(self, json_path: str):
-        """初始化FlowAnalyzer对象
-
-        Parameters
-        ----------
-        json_path : str
-            tshark导出的JSON文件路径
+    """
+    FlowAnalyzer 流量分析器 (智能缓存版)
+    特点：
+    1. Tshark -> Pipe -> ijson -> SQLite (无中间JSON文件)
+    2. 智能校验：自动比对 Filter 和文件修改时间，防止缓存错乱
+    3. 存储优化：数据库文件生成在流量包同级目录下
+    """
+
+    def __init__(self, db_path: str):
         """
-        self.json_path = json_path
-        self.check_json_file()
-
-    def check_json_file(self):
-        # sourcery skip: replace-interpolation-with-fstring
-        """检查JSON文件是否存在并非空
-
-        Raises
-        ------
-        FileNotFoundError
-            当JSON文件不存在时抛出异常
-        ValueError
-            当JSON文件内容为空时抛出异常
+        初始化 FlowAnalyzer
+        :param db_path: 数据库文件路径 (由 get_json_data 返回)
         """
-        if not os.path.exists(self.json_path):
-            raise FileNotFoundError("您的tshark导出的JSON文件没有找到！JSON路径：%s" % self.json_path)
-
-        if os.path.getsize(self.json_path) == 0:
-            raise ValueError("您的tshark导出的JSON文件内容为空！JSON路径：%s" % self.json_path)
-
-    def parse_packet(self, packet: dict) -> Tuple[int, int, float, str, str]:
-        """解析Json中的关键信息字段
-
-        Parameters
-        ----------
-        packet : dict
-            传入Json字典
-
-        Returns
-        -------
-        Tuple[int, int, float, str, str]
-            frame_num, request_in, time_epoch, full_uri, full_request
-        """
-        frame_num = int(packet["frame.number"][0])
-        request_in = int(packet["http.request_in"][0]) if packet.get("http.request_in") else frame_num
-        full_uri = parse.unquote(packet["http.request.full_uri"][0]) if packet.get("http.request.full_uri") else ""
-        time_epoch = float(packet["frame.time_epoch"][0])
-
-        if packet.get("tcp.reassembled.data"):
-            full_request = packet["tcp.reassembled.data"][0]
-        elif packet.get("tcp.payload"):
-            full_request = packet["tcp.payload"][0]
+        # 路径兼容处理
+        if db_path.endswith(".json"):
+            possible_db = db_path + ".db"
+            if os.path.exists(possible_db):
+                self.db_path = possible_db
+            else:
+                self.db_path = db_path
         else:
-            # exported_pdu.exported_pdu
-            full_request = packet["exported_pdu.exported_pdu"][0]
-        return frame_num, request_in, time_epoch, full_uri, full_request
+            self.db_path = db_path
 
-    def parse_http_json(self) -> Tuple[Dict[int, Request], Dict[int, Response]]:
-        """解析JSON数据文件中的HTTP请求和响应信息
+        self.check_db_file()
 
-        Returns
-        -------
-        tuple
-            包含请求字典和响应列表的元组
-        """
-        with open(self.json_path, "r", encoding="utf-8") as f:
-            data = json.load(f)
+    def check_db_file(self):
+        """检查数据库文件是否存在"""
+        if not os.path.exists(self.db_path):
+            raise FileNotFoundError(f"未找到数据文件或缓存数据库: {self.db_path}，请先调用 get_json_data 生成。")
 
+    def _load_from_db(self) -> Tuple[Dict[int, Request], Dict[int, Response]]:
+        """从 SQLite 数据库加载数据"""
         requests, responses = {}, {}
-        for packet in data:
-            packet = packet["_source"]["layers"]
-            frame_num, request_in, time_epoch, full_uri, full_request = self.parse_packet(packet)
-            header, file_data = self.extract_http_file_data(full_request)
-
-            # 请求包使用 full_uri 来记录请求 url  返回包使用 request_in 来记录请求包的序号
-            if packet.get("http.response.code"):
-                responses[frame_num] = Response(
-                    frame_num=frame_num,
-                    request_in=request_in,
-                    header=header,
-                    file_data=file_data,
-                    time_epoch=time_epoch,
-                )
-            else:
-                requests[frame_num] = Request(
-                    frame_num=frame_num, header=header, file_data=file_data, time_epoch=time_epoch, full_uri=full_uri
-                )
-        return requests, responses
-
-    def generate_http_dict_pairs(self) -> Iterable[HttpPair]:  # sourcery skip: use-named-expression
-        """生成HTTP请求和响应信息的字典对
-        Yields
-        ------
-        Iterable[HttpPair]
-            包含请求和响应信息的字典迭代器
-        """
-        requests, responses = self.parse_http_json()
-        response_map = {r.request_in: r for r in responses.values()}
-        yielded_resps = []
+        try:
+            with sqlite3.connect(self.db_path) as conn:
+                cursor = conn.cursor()
+
+                # 简单防错检查
+                try:
+                    cursor.execute("SELECT count(*) FROM requests")
+                    if cursor.fetchone()[0] == 0:
+                        cursor.execute("SELECT count(*) FROM responses")
+                        if cursor.fetchone()[0] == 0:
+                            return {}, {}
+                except sqlite3.OperationalError:
+                    logger.error("数据库损坏或表不存在")
+                    return {}, {}
+
+                logger.debug(f"正在加载缓存数据: {self.db_path}")
+
+                # 加载 Requests
+                cursor.execute("SELECT frame_num, header, file_data, full_uri, time_epoch FROM requests")
+                for row in cursor.fetchall():
+                    requests[row[0]] = Request(row[0], row[1], row[2], row[3], row[4])
+
+                # 加载 Responses
+                cursor.execute("SELECT frame_num, header, file_data, time_epoch, request_in FROM responses")
+                for row in cursor.fetchall():
+                    responses[row[0]] = Response(row[0], row[1], row[2], row[3], row[4])
+
+                return requests, responses
+        except sqlite3.Error as e:
+            logger.error(f"读取数据库出错: {e}")
+            return {}, {}
+
+    def generate_http_dict_pairs(self) -> Iterable[HttpPair]:
+        """生成HTTP请求和响应信息的字典对"""
+        requests, responses = self._load_from_db()
+        response_map = {r._request_in: r for r in responses.values()}
+        yielded_resps = set()
+
         for req_id, req in requests.items():
             resp = response_map.get(req_id)
             if resp:
-                yielded_resps.append(resp)
-                resp = resp._replace(request_in=None)
+                yielded_resps.add(resp.frame_num)
                 yield HttpPair(request=req, response=resp)
             else:
                 yield HttpPair(request=req, response=None)
 
-        for resp in response_map.values():
-            if resp not in yielded_resps:
-                resp = resp._replace(request_in=None)
+        for resp in responses.values():
+            if resp.frame_num not in yielded_resps:
                 yield HttpPair(request=None, response=resp)
 
+    # =========================================================================
+    #  静态方法区域：包含校验逻辑和流式处理
+    # =========================================================================
+
     @staticmethod
-    def get_hash(file_path: str, display_filter: str) -> str:
-        with open(file_path, "rb") as f:
-            return hashlib.md5(f.read() + display_filter.encode()).hexdigest()
+    def get_json_data(file_path: str, display_filter: str, tshark_path: Optional[str] = None) -> str:
+        """
+        获取数据路径 (智能校验版)。
+
+        逻辑：
+        1. 根据 PCAP 路径推算 DB 路径 (位于 PCAP 同级目录)。
+        2. 检查 DB 是否存在。
+        3. 检查 Filter 和文件元数据是否一致。
+        4. 若一致返回路径，不一致则重新解析。
+        """
+        if not os.path.exists(file_path):
+            raise FileNotFoundError("流量包路径不存在：%s" % file_path)
+
+        # --- 修改处：获取流量包的绝对路径和所在目录 ---
+        abs_file_path = os.path.abspath(file_path)
+        pcap_dir = os.path.dirname(abs_file_path)  # 获取文件所在的文件夹
+        base_name = os.path.splitext(os.path.basename(abs_file_path))[0]
+
+        # 将 db_path 拼接在流量包所在的目录下
+        db_path = os.path.join(pcap_dir, f"{base_name}.db")
+        # ----------------------------------------
+
+        # --- 校验环节 ---
+        if FlowAnalyzer._is_cache_valid(db_path, abs_file_path, display_filter):
+            logger.debug(f"缓存校验通过 (Filter匹配且文件未变)，使用缓存: [{db_path}]")
+            return db_path
+        else:
+            logger.debug(f"缓存失效或不存在 (Filter变更或文件更新)，开始重新解析...")
+
+        # --- 解析环节 ---
+        tshark_path = FlowAnalyzer.get_tshark_path(tshark_path)
+        FlowAnalyzer._stream_tshark_to_db(abs_file_path, display_filter, tshark_path, db_path)
+
+        return db_path
+
+    @staticmethod
+    def get_db_data(file_path: str, display_filter: str, tshark_path: Optional[str] = None) -> str:
+        """
+        获取数据库路径 (get_json_data 的语义化别名)。
+        新项目建议使用此方法名，get_json_data 保留用于兼容旧习惯。
+        """
+        return FlowAnalyzer.get_json_data(file_path, display_filter, tshark_path)
+
+    @staticmethod
+    def _is_cache_valid(db_path: str, pcap_path: str, current_filter: str) -> bool:
+        """
+        检查缓存有效性：对比 Filter 字符串和文件元数据
+        """
+        if not os.path.exists(db_path) or os.path.getsize(db_path) == 0:
+            return False
+
+        try:
+            current_mtime = os.path.getmtime(pcap_path)
+            current_size = os.path.getsize(pcap_path)
+
+            with sqlite3.connect(db_path) as conn:
+                cursor = conn.cursor()
+                cursor.execute("SELECT filter, pcap_mtime, pcap_size FROM meta_info LIMIT 1")
+                row = cursor.fetchone()
+
+                if not row:
+                    return False
+
+                cached_filter, cached_mtime, cached_size = row
+
+                # 容差 0.1秒
+                if cached_filter == current_filter and cached_size == current_size and abs(cached_mtime - current_mtime) < 0.1:
+                    return True
+                else:
+                    logger.debug(f"校验失败: 缓存Filter={cached_filter} vs 当前={current_filter}")
+                    return False
+
+        except sqlite3.OperationalError:
+            return False
+        except Exception as e:
+            logger.warning(f"缓存校验出错: {e}，将重新解析")
+            return False
 
     @staticmethod
-    def extract_json_file(file_name: str, display_filter: str, tshark_path: str, tshark_work_dir: str, json_work_path: str) -> None:
+    def _stream_tshark_to_db(pcap_path: str, display_filter: str, tshark_path: str, db_path: str):
+        """流式解析并存入DB，同时记录元数据"""
+
+        if os.path.exists(db_path):
+            os.remove(db_path)
+
+        with sqlite3.connect(db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute("PRAGMA synchronous = OFF")
+            cursor.execute("PRAGMA journal_mode = MEMORY")
+
+            cursor.execute("CREATE TABLE requests (frame_num INTEGER PRIMARY KEY, header BLOB, file_data BLOB, full_uri TEXT, time_epoch REAL)")
+            cursor.execute("CREATE TABLE responses (frame_num INTEGER PRIMARY KEY, header BLOB, file_data BLOB, time_epoch REAL, request_in INTEGER)")
+
+            cursor.execute("""
+                CREATE TABLE meta_info (
+                    id INTEGER PRIMARY KEY, 
+                    filter TEXT, 
+                    pcap_path TEXT, 
+                    pcap_mtime REAL, 
+                    pcap_size INTEGER
+                )
+            """)
+            conn.commit()
+
         command = [
             tshark_path,
-            "-r", file_name,
-            "-Y", f"({display_filter})",
-            "-T", "json",
-            "-e", "http.response.code",
-            "-e", "http.request_in",
-            "-e", "tcp.reassembled.data",
-            "-e", "frame.number",
-            "-e", "tcp.payload",
-            "-e", "frame.time_epoch",
-            "-e", "exported_pdu.exported_pdu",
-            "-e", "http.request.full_uri",
+            "-r",
+            pcap_path,
+            "-Y",
+            f"({display_filter})",
+            "-T",
+            "json",
+            "-e",
+            "http.response.code",
+            "-e",
+            "http.request_in",
+            "-e",
+            "tcp.reassembled.data",
+            "-e",
+            "frame.number",
+            "-e",
+            "tcp.payload",
+            "-e",
+            "frame.time_epoch",
+            "-e",
+            "exported_pdu.exported_pdu",
+            "-e",
+            "http.request.full_uri",
         ]
-        logger.debug(f"导出Json命令: {command}")
-        
-        with open(json_work_path, "wb") as output_file:
-            process = subprocess.Popen(
-                command,
-                stdout=output_file,
-                stderr=subprocess.PIPE,
-                cwd=tshark_work_dir
-            )
-            _, stderr = process.communicate()
-        logger.debug(f"导出Json文件路径: {json_work_path}")
-
-        if stderr and b"WARNING" not in stderr:
-            try:
-                print(f"[Warning/Error]: {stderr.decode('utf-8')}")
-            except Exception:
-                print(f"[Warning/Error]: {stderr.decode('gbk')}")
 
-    @staticmethod
-    def add_md5sum(json_work_path: str, md5_sum: str) -> None:
-        with open(json_work_path, "r", encoding="utf-8") as f:
-            data = json.load(f)
-        data[0]["MD5Sum"] = md5_sum
+        logger.debug(f"执行 Tshark: {command}")
 
-        with open(json_work_path, "w", encoding="utf-8") as f:
-            json.dump(data, f, indent=2)
+        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.dirname(os.path.abspath(pcap_path)))
 
-    @staticmethod
-    def get_json_data(file_path: str, display_filter: str, tshark_path: Optional[str] = None) -> str:
-        # sourcery skip: replace-interpolation-with-fstring
-        """获取JSON数据并保存至文件，保存目录是当前工作目录，也就是您运行脚本所在目录
-
-        Parameters
-        ----------
-        file_path : str
-            待处理的数据文件路径
-        display_filter : str
-            WireShark的显示过滤器
-
-        Returns
-        -------
-        str
-            保存JSON数据的文件路径
-        """
-        if not os.path.exists(file_path):
-            raise FileNotFoundError("您的填写的流量包没有找到！流量包路径：%s" % file_path)
-
-        md5_sum = FlowAnalyzer.get_hash(file_path, display_filter)
-        logger.debug(f"md5校验值: {md5_sum}")
-        
-        work_dir = os.getcwd()
-        tshark_command_work_dir = os.path.dirname(os.path.abspath(file_path))
-        json_work_path = os.path.join(work_dir, "output.json")
-        file_name = os.path.basename(file_path)
-
-        if os.path.exists(json_work_path):
-            try:
-                with open(json_work_path, "r", encoding="utf-8") as f:
-                    data = json.load(f)
-                    if data[0].get("MD5Sum") == md5_sum:
-                        logger.debug("匹配md5校验无误，自动返回Json文件路径!")
-                        return json_work_path
-            except Exception:
-                logger.debug("默认的Json文件无法被正常解析, 正在重新生成Json文件中")
-        
-        tshark_path = FlowAnalyzer.get_tshark_path(tshark_path)
-        FlowAnalyzer.extract_json_file(file_name, display_filter, tshark_path, tshark_command_work_dir, json_work_path)
-        FlowAnalyzer.add_md5sum(json_work_path, md5_sum)
-        return json_work_path
+        db_req_rows = []
+        db_resp_rows = []
+        BATCH_SIZE = 5000
 
-    @staticmethod
-    def get_tshark_path(tshark_path: Optional[str]) -> str:
-        default_tshark_path = get_default_tshark_path()
-        if not os.path.exists(default_tshark_path):
-            logger.debug("没有检测到tshark存在, 请查看并检查tshark_path")
-        else:
-            logger.debug("检测到默认tshark存在!")
+        try:
+            parser = ijson.items(process.stdout, "item")
 
-        if tshark_path is None:
-            logger.debug("您没有传入tshark_path, 请传入tshark_path")
-        elif not os.path.exists(tshark_path):
-            logger.debug("传入的tshark_path不存在, 请查看并检查tshark_path")
+            with sqlite3.connect(db_path) as conn:
+                cursor = conn.cursor()
 
-        use_tshark_path = None
-        if os.path.exists(default_tshark_path):
-            use_tshark_path = default_tshark_path
+                for packet in parser:
+                    layers = packet.get("_source", {}).get("layers", {})
+                    if not layers:
+                        continue
 
-        if tshark_path is not None and os.path.exists(tshark_path):
-            use_tshark_path = tshark_path
+                    try:
+                        frame_num, request_in, time_epoch, full_uri, full_request = FlowAnalyzer.parse_packet_data(layers)
+                        if not full_request:
+                            continue
+                        header, file_data = FlowAnalyzer.extract_http_file_data(full_request)
 
-        if use_tshark_path is None:
-            logger.critical("您没有配置 tshark_path 并且没有在参数中传入 tshark_path")
-            exit(-1)
-        return use_tshark_path
+                        if layers.get("http.response.code"):
+                            db_resp_rows.append((frame_num, header, file_data, time_epoch, request_in))
+                        else:
+                            db_req_rows.append((frame_num, header, file_data, full_uri, time_epoch))
+
+                        if len(db_req_rows) >= BATCH_SIZE:
+                            cursor.executemany("INSERT OR REPLACE INTO requests VALUES (?,?,?,?,?)", db_req_rows)
+                            db_req_rows.clear()
+                        if len(db_resp_rows) >= BATCH_SIZE:
+                            cursor.executemany("INSERT OR REPLACE INTO responses VALUES (?,?,?,?,?)", db_resp_rows)
+                            db_resp_rows.clear()
+
+                    except Exception:
+                        pass
+
+                if db_req_rows:
+                    cursor.executemany("INSERT OR REPLACE INTO requests VALUES (?,?,?,?,?)", db_req_rows)
+                if db_resp_rows:
+                    cursor.executemany("INSERT OR REPLACE INTO responses VALUES (?,?,?,?,?)", db_resp_rows)
+
+                pcap_mtime = os.path.getmtime(pcap_path)
+                pcap_size = os.path.getsize(pcap_path)
+                cursor.execute("INSERT INTO meta_info (filter, pcap_path, pcap_mtime, pcap_size) VALUES (?, ?, ?, ?)", (display_filter, pcap_path, pcap_mtime, pcap_size))
+
+                conn.commit()
+
+        except Exception as e:
+            logger.error(f"解析错误: {e}")
+            if process.poll() is None:
+                process.terminate()
+        finally:
+            if process.poll() is None:
+                process.terminate()
 
-    def split_http_headers(self, file_data: bytes) -> Tuple[bytes, bytes]:
+    # --- 辅助静态方法 ---
+
+    @staticmethod
+    def parse_packet_data(packet: dict) -> Tuple[int, int, float, str, str]:
+        frame_num = int(packet["frame.number"][0])
+        request_in = int(packet["http.request_in"][0]) if packet.get("http.request_in") else frame_num
+        full_uri = parse.unquote(packet["http.request.full_uri"][0]) if packet.get("http.request.full_uri") else ""
+        time_epoch = float(packet["frame.time_epoch"][0])
+
+        if packet.get("tcp.reassembled.data"):
+            full_request = packet["tcp.reassembled.data"][0]
+        elif packet.get("tcp.payload"):
+            full_request = packet["tcp.payload"][0]
+        else:
+            full_request = packet["exported_pdu.exported_pdu"][0] if packet.get("exported_pdu.exported_pdu") else ""
+        return frame_num, request_in, time_epoch, full_uri, full_request
+
+    @staticmethod
+    def split_http_headers(file_data: bytes) -> Tuple[bytes, bytes]:
         headerEnd = file_data.find(b"\r\n\r\n")
         if headerEnd != -1:
-            headerEnd += 4
-            return file_data[:headerEnd], file_data[headerEnd:]
+            return file_data[: headerEnd + 4], file_data[headerEnd + 4 :]
         elif file_data.find(b"\n\n") != -1:
             headerEnd = file_data.index(b"\n\n") + 2
             return file_data[:headerEnd], file_data[headerEnd:]
-        else:
-            print("[Warning] 没有找到headers和response的划分位置!")
-            return b"", file_data
-
-    def dechunck_http_response(self, file_data: bytes) -> bytes:
-        """解码分块TCP数据
-
-        Parameters
-        ----------
-        file_data : bytes
-            已经切割掉headers的TCP数据
+        return b"", file_data
 
-        Returns
-        -------
-        bytes
-            解码分块后的TCP数据
+    @staticmethod
+    def dechunck_http_response(file_data: bytes) -> bytes:
+        """解码分块TCP数据 (修复版)
+        注意：如果数据不是 Chunked 格式，此函数必须抛出异常，
+        以便外层逻辑回退到使用原始数据。
         """
+        if not file_data:
+            return b""
+
         chunks = []
-        chunkSizeEnd = file_data.find(b"\n") + 1
-        lineEndings = b"\r\n" if bytes([file_data[chunkSizeEnd - 2]]) == b"\r" else b"\n"
-        lineEndingsLength = len(lineEndings)
-        while True:
-            chunkSize = int(file_data[:chunkSizeEnd], 16)
-            if not chunkSize:
+        cursor = 0
+        total_len = len(file_data)
+
+        while cursor < total_len:
+            # 1. 寻找当前 Chunk Size 行的结束符 (\n)
+            newline_idx = file_data.find(b"\n", cursor)
+            if newline_idx == -1:
+                # 找不到换行符，说明格式不对，抛出异常让外层处理
+                raise ValueError("Not chunked data")
+
+            # 2. 提取并解析十六进制大小
+            size_line = file_data[cursor:newline_idx].strip()
+
+            # 处理可能的空行 (例如上一个 Chunk 后的 CRLF)
+            if not size_line:
+                cursor = newline_idx + 1
+                continue
+
+            # 这里不要捕获 ValueError，如果解析失败，直接抛出
+            # 说明这根本不是 chunk size，而是普通数据
+            chunk_size = int(size_line, 16)
+
+            # Chunk Size 为 0 表示传输结束
+            if chunk_size == 0:
                 break
 
-            chunks.append(file_data[chunkSizeEnd : chunkSize + chunkSizeEnd])
-            file_data = file_data[chunkSizeEnd + chunkSize + lineEndingsLength :]
-            chunkSizeEnd = file_data.find(lineEndings) + lineEndingsLength
-        return b"".join(chunks)
+            # 3. 定位数据区域
+            data_start = newline_idx + 1
+            data_end = data_start + chunk_size
 
-    def extract_http_file_data(self, full_request: str) -> Tuple[bytes, bytes]:
-        """提取HTTP请求或响应中的文件数据
+            if data_end > total_len:
+                # 数据被截断，尽力读取
+                chunks.append(file_data[data_start:])
+                break
 
-        Parameters
-        ----------
-        full_request : bytes
-            HTTP请求或响应的原始字节流
+            # 4. 提取数据
+            chunks.append(file_data[data_start:data_end])
 
-        Returns
-        -------
-        tuple
-            包含header和file_data的元组
-        """
-        header, file_data = self.split_http_headers(bytes.fromhex(full_request))
+            # 5. 移动游标
+            cursor = data_end
+            # 跳过尾随的 \r 和 \n
+            while cursor < total_len and file_data[cursor] in (13, 10):
+                cursor += 1
+
+        return b"".join(chunks)
 
-        with contextlib.suppress(Exception):
-            file_data = self.dechunck_http_response(file_data)
+    @staticmethod
+    def extract_http_file_data(full_request: str) -> Tuple[bytes, bytes]:
+        """提取HTTP请求或响应中的文件数据 (修复版)"""
+        # 1. 基础校验
+        if not full_request:
+            return b"", b""
+
+        try:
+            # 转为二进制
+            raw_bytes = bytes.fromhex(full_request)
+
+            # 分割 Header 和 Body
+            header, file_data = FlowAnalyzer.split_http_headers(raw_bytes)
+
+            # 处理 Chunked 编码
+            with contextlib.suppress(Exception):
+                file_data = FlowAnalyzer.dechunck_http_response(file_data)
+
+            # 处理 Gzip 压缩
+            with contextlib.suppress(Exception):
+                if file_data.startswith(b"\x1f\x8b"):
+                    file_data = gzip.decompress(file_data)
+
+            return header, file_data
+
+        except ValueError as e:
+            # 专门捕获 Hex 转换错误，并打印出来，方便你调试
+            # 如果你在控制台看到这个错误，说明 Tshark 输出的数据格式非常奇怪
+            logger.error(f"Hex转换失败: {str(e)[:100]}... 原数据片段: {full_request[:50]}")
+            return b"", b""
+        except Exception as e:
+            logger.error(f"解析HTTP数据未知错误: {e}")
+            return b"", b""
 
-        with contextlib.suppress(Exception):
-            if file_data.startswith(b"\x1F\x8B"):
-                file_data = gzip.decompress(file_data)
-        return header, file_data
+    @staticmethod
+    def get_tshark_path(tshark_path: Optional[str]) -> str:
+        default_tshark_path = get_default_tshark_path()
+        use_path = tshark_path if tshark_path and os.path.exists(tshark_path) else default_tshark_path
+        if not use_path or not os.path.exists(use_path):
+            logger.critical("未找到 Tshark，请检查路径配置")
+            exit(-1)
+        return use_path

flowanalyzer-0.4.1.dist-info/METADATA

@@ -0,0 +1,153 @@
+Metadata-Version: 2.4
+Name: FlowAnalyzer
+Version: 0.4.1
+Summary: FlowAnalyzer是一个流量分析器，用于解析和处理tshark导出的JSON数据文件
+Home-page: https://github.com/Byxs20/FlowAnalyzer
+Author: Byxs20
+Author-email: 97766819@qq.com
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: summary
+
+# FlowAnalyzer
+
+[![PyPI version](https://img.shields.io/pypi/v/FlowAnalyzer.svg)](https://pypi.org/project/FlowAnalyzer/) [![License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) ![Python 3.6+](https://img.shields.io/badge/python-3.6+-blue.svg)
+
+**FlowAnalyzer** 是一个高效的 Python 流量分析库，基于 `Tshark` 进行底层解析。它专为处理大流量包（Large PCAP）设计，采用流式解析与 SQLite 缓存架构，彻底解决内存溢出问题，实现秒级二次加载。
+
+---
+
+## 🚀 核心特性：智能缓存与流式架构
+
+为了解决传统解析方式慢、内存占用高的问题，FlowAnalyzer 进行了核心架构升级：**流式解析 + SQLite 智能缓存**。
+
+### 1. ⚡️ 高性能流式解析
+- **极低内存占用**：不再将整个 JSON 读入内存。通过 `subprocess` 管道对接 Tshark 输出，结合 `ijson` 进行增量解析。
+- **无中间文件**：解析过程中不生成体积巨大的临时 JSON 文件，直接入库。
+
+### 2. 💾 智能缓存机制
+- **自动缓存**：首次分析 `test.pcap` 时，会自动生成同级目录下的 `test.db`。
+- **秒级加载**：二次分析时，直接读取 SQLite 数据库，跳过漫长的 Tshark 解析过程（速度提升 100 倍+）。
+
+### 3. 🛡️ 智能校验 (Smart Validation)
+为了防止“修改了过滤规则但误读旧缓存”的问题，内置了严格的元数据校验机制。每次运行时自动比对指纹：
+
+| 校验项                  | 说明                                                         |
+| :---------------------- | :----------------------------------------------------------- |
+| **过滤规则 (Filter)**   | 检查本次传入的 Tshark 过滤器（如 `http contains flag`）是否与缓存一致。 |
+| **文件指纹 (Metadata)** | 检查原始 PCAP 文件的 **修改时间 (MTime)** 和 **文件大小 (Size)**。 |
+
+- ✅ **命中缓存**：规则一致且文件未变 → **0秒等待，直接加载**。
+- 🔄 **缓存失效**：规则变更或文件更新 → **自动重新解析并更新数据库**。
+
+### 4. 性能对比
+
+| 特性         | 旧版架构                      | **新版架构 (FlowAnalyzer)**         |
+| :----------- | :---------------------------- | :---------------------------------- |
+| **解析流程** | 生成巨大 JSON -> 全量读入内存 | Tshark流 -> 管道 -> ijson -> SQLite |
+| **内存占用** | 极高 (易 OOM)                 | **极低 (内存稳定)**                 |
+| **二次加载** | 需重新解析                    | **直接读取 DB (0秒)**               |
+| **磁盘占用** | 巨大的临时 JSON 文件          | 轻量级 SQLite 文件                  |
+
+---
+
+## 📦 安装
+
+请确保您的环境中已安装 Python 3 和 Tshark (Wireshark)。
+
+```bash
+# 安装 FlowAnalyzer 及其依赖 ijson
+pip3 install FlowAnalyzer ijson
+
+# 或者使用国内源加速
+pip3 install FlowAnalyzer ijson -i https://pypi.org/simple
+```
+
+---
+
+## 🛠️ 快速上手
+
+### 1. 基础使用
+
+```python
+from FlowAnalyzer import FlowAnalyzer
+
+# 流量包路径
+pcap_path = r"tests/demo.pcap"
+# 过滤规则
+display_filter = "http"
+
+# 1. 获取数据库数据 (自动处理解析、缓存和校验)
+# 返回的是生成的 .db 文件路径
+db_path = FlowAnalyzer.get_db_data(pcap_path, display_filter)
+# 兼容老的函数名 get_json_data
+# db_path = FlowAnalyzer.get_json_data(pcap_path, display_filter)
+
+# 2. 初始化分析器
+analyzer = FlowAnalyzer(db_path)
+
+# 3. 遍历 HTTP 流
+print("[+] 开始分析 HTTP 流...")
+for pair in analyzer.generate_http_dict_pairs():
+    if pair.request:
+        print(f"Frame: {pair.request.frame_num} | URI: {pair.request.full_uri}")
+        # 获取请求体数据
+        # print(pair.request.file_data)
+```
+
+### 2. 配置 Tshark 路径
+
+如果您的 `tshark` 不在系统环境变量中，程序可能会报错。您有两种方式进行配置：
+
+**方法一：代码中指定 (推荐)**
+
+在调用 `get_db_data` 时直接传入路径：
+
+```python
+tshark_ex = r"D:\Program Files\Wireshark\tshark.exe"
+
+FlowAnalyzer.get_db_data(pcap_path, display_filter, tshark_path=tshark_ex)
+```
+
+**方法二：修改默认配置**
+
+如果安装目录固定，可以修改库文件中的默认路径：
+找到 `python安装目录\Lib\site-packages\FlowAnalyzer\Path.py`，修改 `tshark_path` 变量。
+
+---
+
+## 📝 测试
+
+```bash
+$ git clone https://github.com/Byxs20/FlowAnalyzer.git
+$ cd ./FlowAnalyzer/
+$ python tests/demo.py
+```
+
+**运行预期结果：**
+
+```text
+[+] 正在处理第1个HTTP流!
+序号: 2请求包, 请求头: b'POST /upload/php_eval_xor_base64.php HTTP/1.1 ...
+```
+
+---
+
+## 📄 License
+
+This project is licensed under the [MIT License](LICENSE).

flowanalyzer-0.4.1.dist-info/RECORD

@@ -0,0 +1,9 @@
+FlowAnalyzer/FlowAnalyzer.py,sha256=ciuWFPQWQgYqjdL_u7ck4BNIsQNx00HLOjr6lSkfzMg,17348
+FlowAnalyzer/Path.py,sha256=E5VvucTftp8VTQUffFzFWHotQEYtZL-j7IQPOaleiug,130
+FlowAnalyzer/__init__.py,sha256=vfiHONPTrvjUU3MwhjFOEo3sWfzlhkA6gOLn_4UJ7sg,70
+FlowAnalyzer/logging_config.py,sha256=-RntNJhrBiW7ToXIP1WJjZ4Yf9jmZQ1PTX_er3tDxhw,730
+flowanalyzer-0.4.1.dist-info/licenses/LICENSE,sha256=ybAV0ECduYBZCpjkHyNALVWRRmT_eM0BDgqUszhwEFU,1080
+flowanalyzer-0.4.1.dist-info/METADATA,sha256=WD01CpYRDVbT8RA5GwTKYZPv8Fa06_-4ZuiTAa5SfeE,5767
+flowanalyzer-0.4.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+flowanalyzer-0.4.1.dist-info/top_level.txt,sha256=2MtvAF6dEe_eHipw_6G5pFLb2uOCbGnlH0bC4iBtm5A,13
+flowanalyzer-0.4.1.dist-info/RECORD,,

{FlowAnalyzer-0.3.9.dist-info → flowanalyzer-0.4.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (72.1.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

FlowAnalyzer-0.3.9.dist-info/METADATA

@@ -1,71 +0,0 @@
-Metadata-Version: 2.1
-Name: FlowAnalyzer
-Version: 0.3.9
-Summary: FlowAnalyzer是一个流量分析器，用于解析和处理tshark导出的JSON数据文件
-Home-page: https://github.com/Byxs20/FlowAnalyzer
-Author: Byxs20
-Author-email: 97766819@qq.com
-Classifier: Development Status :: 3 - Alpha
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
-
-# FlowAnalyzer
-
-# 安装
-
-使用 `pip` 安装：
-
-```
-pip3 install FlowAnalyzer
-```
-
-```
-pip3 install FlowAnalyzer -i https://pypi.org/simple
-```
-
-# 快速上手
-
-## 配置
-
-如果您安装 `WireShark` 没有修改安装目录，默认 `tshark` 路径会如下：
-
-```python
-# windows
-tshark_path = r"C:\Program Files\Wireshark\tshark.exe"
-```
-
-`Linux`, `MacOS` 默认路径不清楚，需要看下面的**纠正路径**，**确定路径没有问题，那也无需任何配置即可使用！**
-
-## 纠正路径
-
-修改 `python安装目录\Lib\site-packages\FlowAnalyzer\Path.py` 中的变量 `tshark_path` 改为**tshark正确路径**
-
-## 测试
-
-```
-$ git clone https://github.com/Byxs20/FlowAnalyzer.git
-$ cd ./FlowAnalyzer/
-$ python tests\demo.py
-```
-
-运行结果：
-
-```
-[+] 正在处理第1个HTTP流!
-序号: 2请求包, 请求头: b'POST /upload/php_eval_xor_base64.php HTTP/1.1\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0\r\n
-...
-```
-
-# Contributing
-Feel free to submit issues or pull requests if you have any suggestions, improvements, or bug reports.
-
-# License
-
-This project is licensed under the [MIT License.](LICENSE)

FlowAnalyzer-0.3.9.dist-info/RECORD

@@ -1,9 +0,0 @@
-FlowAnalyzer/FlowAnalyzer.py,sha256=ErHea4wQEeGmCgAmWr4xmEuKSSYfXE0kFe7It0xD6Is,12203
-FlowAnalyzer/Path.py,sha256=E5VvucTftp8VTQUffFzFWHotQEYtZL-j7IQPOaleiug,130
-FlowAnalyzer/__init__.py,sha256=vfiHONPTrvjUU3MwhjFOEo3sWfzlhkA6gOLn_4UJ7sg,70
-FlowAnalyzer/logging_config.py,sha256=-RntNJhrBiW7ToXIP1WJjZ4Yf9jmZQ1PTX_er3tDxhw,730
-FlowAnalyzer-0.3.9.dist-info/LICENSE,sha256=ybAV0ECduYBZCpjkHyNALVWRRmT_eM0BDgqUszhwEFU,1080
-FlowAnalyzer-0.3.9.dist-info/METADATA,sha256=OcwMs0sqeUmUv1Y-9NWDaGFswMupCLf-FuJYr68DQX8,1956
-FlowAnalyzer-0.3.9.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-FlowAnalyzer-0.3.9.dist-info/top_level.txt,sha256=2MtvAF6dEe_eHipw_6G5pFLb2uOCbGnlH0bC4iBtm5A,13
-FlowAnalyzer-0.3.9.dist-info/RECORD,,