FlowAnalyzer 0.4.6__py3-none-any.whl → 0.4.7__py3-none-any.whl

FlowAnalyzer/FlowAnalyzer.py

@@ -56,7 +56,7 @@ class FlowAnalyzer:
             sql_pair = """
             SELECT 
                 req.frame_num, req.header, req.file_data, req.full_uri, req.time_epoch,  -- 0-4 (Request)
-                resp.frame_num, resp.header, resp.file_data, resp.time_epoch, resp.request_in -- 5-9 (Response)
+                resp.frame_num, resp.header, resp.file_data, resp.time_epoch, resp.request_in, resp.status_code -- 5-10 (Response)
             FROM requests req
             LEFT JOIN responses resp ON req.frame_num = resp.request_in
             ORDER BY req.frame_num ASC
@@ -70,20 +70,20 @@ class FlowAnalyzer:
 
                 resp = None
                 if row[5] is not None:
-                    resp = Response(frame_num=row[5], header=row[6] or b"", file_data=row[7] or b"", time_epoch=row[8], _request_in=row[9])
+                    resp = Response(frame_num=row[5], header=row[6] or b"", file_data=row[7] or b"", time_epoch=row[8], _request_in=row[9], status_code=row[10] or 0)
 
                 yield HttpPair(request=req, response=resp)
 
             # === 第二步：孤儿响应查询 ===
             sql_orphan = """
-            SELECT frame_num, header, file_data, time_epoch, request_in
+            SELECT frame_num, header, file_data, time_epoch, request_in, status_code
             FROM responses
             WHERE request_in NOT IN (SELECT frame_num FROM requests)
             """
             cursor.execute(sql_orphan)
 
             for row in cursor:
-                resp = Response(frame_num=row[0], header=row[1] or b"", file_data=row[2] or b"", time_epoch=row[3], _request_in=row[4])
+                resp = Response(frame_num=row[0], header=row[1] or b"", file_data=row[2] or b"", time_epoch=row[3], _request_in=row[4], status_code=row[5] or 0)
                 yield HttpPair(request=None, response=resp)
 
     # =========================================================================
@@ -161,7 +161,7 @@ class FlowAnalyzer:
             cursor.execute("PRAGMA journal_mode = MEMORY")
 
             cursor.execute("CREATE TABLE requests (frame_num INTEGER PRIMARY KEY, header BLOB, file_data BLOB, full_uri TEXT, time_epoch REAL)")
-            cursor.execute("CREATE TABLE responses (frame_num INTEGER PRIMARY KEY, header BLOB, file_data BLOB, time_epoch REAL, request_in INTEGER)")
+            cursor.execute("CREATE TABLE responses (frame_num INTEGER PRIMARY KEY, header BLOB, file_data BLOB, time_epoch REAL, request_in INTEGER, status_code INTEGER)")
 
             cursor.execute("""
                 CREATE TABLE meta_info (
@@ -174,50 +174,29 @@ class FlowAnalyzer:
             """)
             conn.commit()
 
+        lua_script_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "tshark.lua")
+
+        # Pass filter via environment variable
+        env = os.environ.copy()
+        env["flowanalyzer_filter"] = display_filter
+
         command = [
             tshark_path,
             "-r",
             pcap_path,
-            "-Y",
-            f"({display_filter})",
-            "-T",
-            "fields",
-            "-e",
-            "http.response.code",  # 0
-            "-e",
-            "http.request_in",  # 1
-            "-e",
-            "tcp.reassembled.data",  # 2
-            "-e",
-            "frame.number",  # 3
-            "-e",
-            "tcp.payload",  # 4
-            "-e",
-            "frame.time_epoch",  # 5
-            "-e",
-            "exported_pdu.exported_pdu",  # 6
-            "-e",
-            "http.request.full_uri",  # 7
-            "-e",
-            "tcp.segment.count",  # 8
-            "-E",
-            "header=n",
-            "-E",
-            "separator=/t",
-            "-E",
-            "quote=n",
-            "-E",
-            "occurrence=f",
+            "-q",
+            "-X",
+            f"lua_script:{lua_script_path}",
         ]
 
-        logger.debug(f"执行 Tshark: {command}")
+        logger.debug(f"执行 Tshark: {' '.join(command)}")
         BATCH_SIZE = 2000
         MAX_PENDING_BATCHES = 20  # 控制内存中待处理的批次数量 (Backpressure)
 
         # 使用 ThreadPoolExecutor 并行处理数据
         max_workers = min(32, (os.cpu_count() or 1) + 4)
 
-        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.dirname(os.path.abspath(pcap_path)))
+        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.dirname(os.path.abspath(pcap_path)), env=env, encoding="utf-8", errors="replace")
         try:
             with sqlite3.connect(db_path) as conn:
                 cursor = conn.cursor()
@@ -236,14 +215,14 @@ class FlowAnalyzer:
 
                         for item in results:
                             if item["type"] == "response":
-                                db_resp_rows.append((item["frame_num"], item["header"], item["file_data"], item["time_epoch"], item["request_in"]))
+                                db_resp_rows.append((item["frame_num"], item["header"], item["file_data"], item["time_epoch"], item["request_in"], item.get("status_code", 0)))
                             else:
                                 db_req_rows.append((item["frame_num"], item["header"], item["file_data"], item["full_uri"], item["time_epoch"]))
 
                         if db_req_rows:
                             cursor.executemany("INSERT OR REPLACE INTO requests VALUES (?,?,?,?,?)", db_req_rows)
                         if db_resp_rows:
-                            cursor.executemany("INSERT OR REPLACE INTO responses VALUES (?,?,?,?,?)", db_resp_rows)
+                            cursor.executemany("INSERT OR REPLACE INTO responses VALUES (?,?,?,?,?,?)", db_resp_rows)
 
                     def submit_batch():
                         """提交当前批次到线程池"""
@@ -259,6 +238,10 @@ class FlowAnalyzer:
                     # --- Main Pipeline Loop ---
                     if process.stdout:
                         for line in process.stdout:
+                            # Strip newline
+                            line = line.strip()
+                            if not line:
+                                continue
                             current_batch.append(line)
 
                             if len(current_batch) >= BATCH_SIZE:

FlowAnalyzer/Models.py

@@ -8,17 +8,18 @@ class Request:
     frame_num: int
     header: bytes
     file_data: bytes
-    full_uri: str
     time_epoch: float
+    full_uri: str
 
 
 @dataclass
 class Response:
-    __slots__ = ("frame_num", "header", "file_data", "time_epoch", "_request_in")
+    __slots__ = ("frame_num", "header", "file_data", "time_epoch", "status_code", "_request_in")
     frame_num: int
     header: bytes
     file_data: bytes
     time_epoch: float
+    status_code: int
     _request_in: Optional[int]
 
 

FlowAnalyzer/PacketParser.py

@@ -2,46 +2,79 @@ import binascii
 import contextlib
 import gzip
 from typing import List, Optional, Tuple
-from urllib import parse
 
 from .logging_config import logger
 
 
 class PacketParser:
     @staticmethod
-    def parse_packet_data(row: list) -> Tuple[int, int, float, str, bytes]:
+    def process_batch(lines: List[str]) -> List[dict]:
         """
-        解析 Tshark 输出的一行数据
-        row definition (all bytes):
-        0: http.response.code
-        1: http.request_in
-        2: tcp.reassembled.data
-        3: frame.number
-        4: tcp.payload
-        5: frame.time_epoch
-        6: exported_pdu.exported_pdu
-        7: http.request.full_uri
-        8: tcp.segment.count
+        批量处理行数据
         """
-        frame_num = int(row[3])
-        request_in = int(row[1]) if row[1] else frame_num
-        # Decode only URI to string
-        full_uri = parse.unquote(row[7].decode("utf-8", errors="replace")) if row[7] else ""
-        time_epoch = float(row[5])
-
-        # Logic for Raw Packet (Header Source)
-        # Previous index 9 is now 8 since we removed http.file_data
-        is_reassembled = len(row) > 8 and row[8]
-
-        if is_reassembled and row[2]:
-            full_request = row[2]
-        elif row[4]:
-            full_request = row[4]
-        else:
-            # Fallback (e.g. Exported PDU)
-            full_request = row[2] if row[2] else (row[6] if row[6] else b"")
-
-        return frame_num, request_in, time_epoch, full_uri, full_request
+        results = []
+        for line in lines:
+            res = PacketParser.process_row(line)
+            if res:
+                results.append(res)
+        return results
+
+    @staticmethod
+    def process_row(line: str) -> Optional[dict]:
+        """
+        解析 Tshark Lua 脚本输出的一行数据
+        Columns:
+        0: type ("req" / "rep" / "data")
+        1: frame.number
+        2: time_epoch
+        3: header_hex
+        4: file_data_hex (Body)
+        5: uri_or_code
+        6: request_in
+        """
+        try:
+            parts = line.split("\t")
+            if len(parts) < 6:
+                return None
+
+            p_type = parts[0]
+            frame_num = int(parts[1])
+            time_epoch = float(parts[2])
+
+            # Hex string -> Bytes
+            # parts[3] might be empty string
+            header = binascii.unhexlify(parts[3]) if parts[3] else b""
+            file_data = binascii.unhexlify(parts[4]) if parts[4] else b""
+
+            uri_or_code = parts[5]
+            request_in_str = parts[6] if len(parts) > 6 else ""
+
+            if p_type == "req":
+                return {"type": "request", "frame_num": frame_num, "header": header, "file_data": file_data, "time_epoch": time_epoch, "full_uri": uri_or_code, "request_in": None}
+            elif p_type == "rep":
+                request_in = int(request_in_str) if request_in_str else 0
+                try:
+                    status_code = int(uri_or_code)
+                except (ValueError, TypeError):
+                    status_code = 0
+
+                return {
+                    "type": "response",
+                    "frame_num": frame_num,
+                    "header": header,
+                    "file_data": file_data,
+                    "time_epoch": time_epoch,
+                    "request_in": request_in,
+                    "status_code": status_code,
+                    "full_uri": "",
+                }
+            else:
+                # 'data' or unknown, ignore for now based on current logic
+                return None
+
+        except Exception as e:
+            logger.debug(f"Packet parse error: {e} | Line: {line[:100]}...")
+            return None
 
     @staticmethod
     def split_http_headers(file_data: bytes) -> Tuple[bytes, bytes]:
@@ -141,49 +174,3 @@ class PacketParser:
         except Exception as e:
             logger.error(f"解析HTTP数据未知错误: {e}")
             return b"", b""
-
-    @staticmethod
-    def process_row(line: bytes) -> Optional[dict]:
-        """
-        处理单行数据，返回结构化结果供主线程写入
-        """
-        line = line.rstrip(b"\r\n")
-        if not line:
-            return None
-
-        row = line.split(b"\t")
-        try:
-            frame_num, request_in, time_epoch, full_uri, full_request = PacketParser.parse_packet_data(row)
-
-            if not full_request:
-                return None
-
-            header, file_data = PacketParser.extract_http_file_data(full_request)
-
-            # row[0] is http.response.code (bytes)
-            is_response = bool(row[0])
-
-            return {
-                "type": "response" if is_response else "request",
-                "frame_num": frame_num,
-                "header": header,
-                "file_data": file_data,
-                "time_epoch": time_epoch,
-                "request_in": request_in,  # Only useful for Response
-                "full_uri": full_uri,  # Only useful for Request
-            }
-
-        except Exception:
-            return None
-
-    @staticmethod
-    def process_batch(lines: List[bytes]) -> List[dict]:
-        """
-        批量处理行数据，减少函数调用开销
-        """
-        results = []
-        for line in lines:
-            res = PacketParser.process_row(line)
-            if res:
-                results.append(res)
-        return results

FlowAnalyzer/tshark.lua

@@ -0,0 +1,196 @@
+-- =========================================================================
+-- 1. 字段定义
+-- =========================================================================
+local f_resp_code = Field.new("http.response.code")
+local f_full_uri = Field.new("http.request.full_uri")
+local f_frame_num = Field.new("frame.number")
+local f_time_epoch = Field.new("frame.time_epoch")
+local f_reassembled = Field.new("tcp.reassembled.data")
+local f_payload = Field.new("tcp.payload")
+local f_file_data = Field.new("http.file_data")
+local f_seg_count = Field.new("tcp.segment.count")
+local f_retrans = Field.new("tcp.analysis.retransmission")
+local f_request_in = Field.new("http.request_in")
+-- [新增] 替换 Header 源的字段
+local f_exported_pdu = Field.new("exported_pdu.exported_pdu")
+
+-- =========================================================================
+-- 2. 获取过滤器
+-- =========================================================================
+local user_filter = os.getenv("flowanalyzer_filter")
+if not user_filter or user_filter == "" then
+    user_filter = "http"
+end
+
+-- =========================================================================
+-- 3. 初始化监听器
+-- =========================================================================
+local tap = Listener.new("frame", user_filter)
+
+-- =========================================================================
+-- 4. 辅助函数
+-- =========================================================================
+
+local function val_to_str(val)
+    if val == nil then
+        return ""
+    end
+    return tostring(val)
+end
+
+-- 查找 Header 结束位置
+local function find_header_split_pos(hex_str)
+    if not hex_str then
+        return nil
+    end
+
+    -- 1. 找 0D0A0D0A (CRLF CRLF)
+    local start_idx = 1
+    while true do
+        local s, e = string.find(hex_str, "0D0A0D0A", start_idx, true)
+        if not s then
+            break
+        end
+        if s % 2 == 1 then
+            return s
+        end -- 确保字节对齐
+        start_idx = s + 1
+    end
+
+    -- 2. 找 0A0A (LF LF)
+    start_idx = 1
+    while true do
+        local s, e = string.find(hex_str, "0A0A", start_idx, true)
+        if not s then
+            break
+        end
+        if s % 2 == 1 then
+            return s
+        end
+        start_idx = s + 1
+    end
+    return nil
+end
+
+-- [核心性能优化] 智能提取 Header Hex
+-- 即使 Body 不限制大小，Header 依然建议只扫描前 2KB，因为 Header 不会那么长
+local function extract_header_smart(field_info)
+    if not field_info then
+        return ""
+    end
+
+    local range = field_info.range
+    local total_len = range:len()
+
+    -- 预览前 2KB
+    local cap_len = 2048
+    if total_len < cap_len then
+        cap_len = total_len
+    end
+
+    -- [关键] 转为 Hex 并强制转为大写
+    local preview_hex = string.upper(range(0, cap_len):bytes():tohex())
+
+    -- 查找分隔符
+    local pos = find_header_split_pos(preview_hex)
+
+    if pos then
+        return string.sub(preview_hex, 1, pos - 1)
+    else
+        return preview_hex
+    end
+end
+
+-- 直接获取完整 Hex
+local function get_full_hex(field_info)
+    if not field_info then
+        return ""
+    end
+    -- 强制转大写，保持格式一致
+    return string.upper(field_info.range:bytes():tohex())
+end
+
+-- =========================================================================
+-- 5. 主处理逻辑
+-- =========================================================================
+function tap.packet(pinfo, tvb)
+    -- 过滤 TCP 重传
+    if f_retrans() then
+        return
+    end
+
+    local frame_num = f_frame_num()
+    if not frame_num then
+        return
+    end
+
+    -- === 1. 确定类型 (req/rep) 和 信息 (URI/Code) ===
+    local col_type = "data"
+    local col_uri_or_code = ""
+
+    local code = f_resp_code()
+    local uri = f_full_uri()
+
+    if code then
+        col_type = "rep"
+        col_uri_or_code = tostring(code)
+    elseif uri then
+        col_type = "req"
+        col_uri_or_code = tostring(uri)
+    end
+
+    -- === 2. 基础信息 ===
+    local col_frame = tostring(frame_num)
+    local col_time = val_to_str(f_time_epoch())
+
+    -- === 3. Header Hex ===
+    -- 逻辑：Exported PDU > TCP Reassembled > TCP Payload
+    local col_header_hex = ""
+
+    local exp_pdu = f_exported_pdu()
+
+    if exp_pdu then
+        col_header_hex = extract_header_smart(exp_pdu)
+    else
+        local seq_count = f_seg_count()
+        local reass = nil
+        if seq_count then
+            reass = f_reassembled()
+        end
+
+        if reass then
+            col_header_hex = extract_header_smart(reass)
+        else
+            local pay = f_payload()
+            if pay then
+                col_header_hex = extract_header_smart(pay)
+            end
+        end
+    end
+
+    -- === 4. File Data (Body Hex) ===
+    -- [修改] 移除大小判断，无条件转换所有 Body
+    local col_file_data = ""
+    local fd = f_file_data()
+
+    if fd then
+        col_file_data = get_full_hex(fd)
+    end
+
+    -- === 5. Request In (仅响应包有) ===
+    local col_req_in = ""
+    local req_in = f_request_in()
+    if req_in then
+        col_req_in = tostring(req_in)
+    end
+
+    -- === 输出 (Tab 分隔) ===
+    print(table.concat({col_type, -- 1. req / rep
+    col_frame, -- 2. Frame Number
+    col_time, -- 3. Time Epoch
+    col_header_hex, -- 4. Header Bytes (Hex)
+    col_file_data, -- 5. File Data (Hex) [完整数据，不跳过]
+    col_uri_or_code, -- 6. URI / Code
+    col_req_in -- 7. Request In
+    }, "\t"))
+end

{flowanalyzer-0.4.6.dist-info → flowanalyzer-0.4.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: FlowAnalyzer
-Version: 0.4.6
+Version: 0.4.7
 Summary: FlowAnalyzer是一个流量分析器，用于解析和处理tshark导出的JSON数据文件
 Home-page: https://github.com/Byxs20/FlowAnalyzer
 Author: Byxs20

flowanalyzer-0.4.7.dist-info/RECORD

@@ -0,0 +1,13 @@
+FlowAnalyzer/FlowAnalyzer.py,sha256=5e4fCTvKcAb5wyloNy8pUOEZQflJ3ePfzIfkYnbbiuA,12549
+FlowAnalyzer/Models.py,sha256=onh3HaSpdKZPJOmXvKSIwGC19y9dIILKXm-FbKpBkCM,659
+FlowAnalyzer/PacketParser.py,sha256=N45Szx6E8RPuxvdtWZjveAYQm1jRvauRh-7epOqMv7s,5979
+FlowAnalyzer/Path.py,sha256=E5VvucTftp8VTQUffFzFWHotQEYtZL-j7IQPOaleiug,130
+FlowAnalyzer/PcapSplitter.py,sha256=0E_vmLYYsE_gD34XTwG1XPx5kBg8ZchJspQEnkBoIdY,4855
+FlowAnalyzer/__init__.py,sha256=vfiHONPTrvjUU3MwhjFOEo3sWfzlhkA6gOLn_4UJ7sg,70
+FlowAnalyzer/logging_config.py,sha256=fnBlvoimteQ38IBlQBV9fdLQvfAlRgGhcvLpUC3YunA,732
+FlowAnalyzer/tshark.lua,sha256=OrfGdiodlJo3AkI27sUr-0fvfSnbsIOO3kKqAhO7BWY,5745
+flowanalyzer-0.4.7.dist-info/licenses/LICENSE,sha256=ybAV0ECduYBZCpjkHyNALVWRRmT_eM0BDgqUszhwEFU,1080
+flowanalyzer-0.4.7.dist-info/METADATA,sha256=y5DqyrvLjb9akcbdf52tp3TDb2sudxliB4nUACbHBlM,6099
+flowanalyzer-0.4.7.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+flowanalyzer-0.4.7.dist-info/top_level.txt,sha256=2MtvAF6dEe_eHipw_6G5pFLb2uOCbGnlH0bC4iBtm5A,13
+flowanalyzer-0.4.7.dist-info/RECORD,,

flowanalyzer-0.4.6.dist-info/RECORD

@@ -1,12 +0,0 @@
-FlowAnalyzer/FlowAnalyzer.py,sha256=9SshWk5wf0XATI7W4eBiIpzEqeGFyQJs3on5ox-zrNQ,12666
-FlowAnalyzer/Models.py,sha256=2x7nPJIAyLTC1oiGFlW4mELDPgthk2IsmuyearT-MSQ,622
-FlowAnalyzer/PacketParser.py,sha256=So3iD2ykkWpT0e3aLjBdx_ohoNscD-oAt4bfr_oRqgo,6331
-FlowAnalyzer/Path.py,sha256=E5VvucTftp8VTQUffFzFWHotQEYtZL-j7IQPOaleiug,130
-FlowAnalyzer/PcapSplitter.py,sha256=0E_vmLYYsE_gD34XTwG1XPx5kBg8ZchJspQEnkBoIdY,4855
-FlowAnalyzer/__init__.py,sha256=vfiHONPTrvjUU3MwhjFOEo3sWfzlhkA6gOLn_4UJ7sg,70
-FlowAnalyzer/logging_config.py,sha256=fnBlvoimteQ38IBlQBV9fdLQvfAlRgGhcvLpUC3YunA,732
-flowanalyzer-0.4.6.dist-info/licenses/LICENSE,sha256=ybAV0ECduYBZCpjkHyNALVWRRmT_eM0BDgqUszhwEFU,1080
-flowanalyzer-0.4.6.dist-info/METADATA,sha256=j9Bw-2Sr1dx_DatRtxo56WE0BB1-WMOoIhfoSoSYk-Y,6099
-flowanalyzer-0.4.6.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-flowanalyzer-0.4.6.dist-info/top_level.txt,sha256=2MtvAF6dEe_eHipw_6G5pFLb2uOCbGnlH0bC4iBtm5A,13
-flowanalyzer-0.4.6.dist-info/RECORD,,