stackforge 0.6.5__tar.gz → 0.6.6__tar.gz

{stackforge-0.6.5 → stackforge-0.6.6}/Cargo.lock

@@ -1074,7 +1074,7 @@ dependencies = [
 
 [[package]]
 name = "stackforge"
-version = "0.6.5"
+version = "0.6.6"
 dependencies = [
  "pyo3",
  "stackforge-automata",
@@ -1083,7 +1083,7 @@ dependencies = [
 
 [[package]]
 name = "stackforge-automata"
-version = "0.6.5"
+version = "0.6.6"
 dependencies = [
  "bytes",
  "stackforge-core",
@@ -1091,7 +1091,7 @@ dependencies = [
 
 [[package]]
 name = "stackforge-core"
-version = "0.6.5"
+version = "0.6.6"
 dependencies = [
  "aes",
  "aes-gcm",

{stackforge-0.6.5 → stackforge-0.6.6}/Cargo.toml

@@ -21,7 +21,7 @@ shared-version = true
 tag-name = "v{{version}}"
 
 [workspace.package]
-version = "0.6.5"
+version = "0.6.6"
 edition = "2024"
 license = "GPL-3.0-only"
 authors = ["Stackforge Contributors"]

{stackforge-0.6.5 → stackforge-0.6.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stackforge
-Version: 0.6.5
+Version: 0.6.6
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: System Administrators
@@ -359,17 +359,29 @@ Enable verbose mode to see progress feedback on stderr during extraction:
 ```python
 # Quick verbose flag on the function call
 conversations = extract_flows("capture.pcap", verbose=True)
-# [stackforge] Opening capture file: capture.pcap
-# [stackforge] Starting streaming flow extraction...
-# [stackforge] Processed 10000 packets (342 flows so far)
-# [stackforge] Processed 20000 packets (587 flows so far)
-# [stackforge] Flow extraction complete: 612 conversations
 
 # Or via FlowConfig
 config = FlowConfig(verbose=True)
 conversations = extract_flows("capture.pcap", config=config)
 ```
 
+Verbose output shows real-time progress with processing rate, memory usage, ETA, and spill stats:
+
+```
+[+] stackforge flow extraction engine
+[+] File: capture.pcap (2.3 GB)
+[+] Mode: streaming (packets read from disk on-the-fly)
+[+] Memory budget: 1.00 GB
+[+] Processing...
+
+    [1m 23s] 100,000 pkts | 1,234 flows | 85,432/s (avg 72,150/s) | mem ~45.2 MB
+    [2m 48s] 200,000 pkts | 2,567 flows | 78,901/s (avg 71,428/s) | mem ~89.1 MB | 3 spills
+
+[+] Finalizing (sorting 88,254 flows)...
+[+] Complete: 88,254 flows extracted
+[+] Wall time: 1h 12m
+```
+
 Customize timeouts, buffer limits, and memory budget with `FlowConfig`:
 
 ```python
@@ -389,6 +401,8 @@ For large captures, set a memory budget so reassembly buffers automatically spil
 config = FlowConfig(
     memory_budget=256 * 1024 * 1024,  # 256 MB RAM budget
     spill_dir="/tmp/stackforge-spill", # optional custom spill directory
+    store_packet_indices=False,        # save ~8 bytes/pkt on large captures
+    progress_interval=500_000,         # report every 500K packets (default: 100K)
 )
 conversations = extract_flows("large_capture.pcapng", config=config)
 ```

{stackforge-0.6.5 → stackforge-0.6.6}/README.md

@@ -335,17 +335,29 @@ Enable verbose mode to see progress feedback on stderr during extraction:
 ```python
 # Quick verbose flag on the function call
 conversations = extract_flows("capture.pcap", verbose=True)
-# [stackforge] Opening capture file: capture.pcap
-# [stackforge] Starting streaming flow extraction...
-# [stackforge] Processed 10000 packets (342 flows so far)
-# [stackforge] Processed 20000 packets (587 flows so far)
-# [stackforge] Flow extraction complete: 612 conversations
 
 # Or via FlowConfig
 config = FlowConfig(verbose=True)
 conversations = extract_flows("capture.pcap", config=config)
 ```
 
+Verbose output shows real-time progress with processing rate, memory usage, ETA, and spill stats:
+
+```
+[+] stackforge flow extraction engine
+[+] File: capture.pcap (2.3 GB)
+[+] Mode: streaming (packets read from disk on-the-fly)
+[+] Memory budget: 1.00 GB
+[+] Processing...
+
+    [1m 23s] 100,000 pkts | 1,234 flows | 85,432/s (avg 72,150/s) | mem ~45.2 MB
+    [2m 48s] 200,000 pkts | 2,567 flows | 78,901/s (avg 71,428/s) | mem ~89.1 MB | 3 spills
+
+[+] Finalizing (sorting 88,254 flows)...
+[+] Complete: 88,254 flows extracted
+[+] Wall time: 1h 12m
+```
+
 Customize timeouts, buffer limits, and memory budget with `FlowConfig`:
 
 ```python
@@ -365,6 +377,8 @@ For large captures, set a memory budget so reassembly buffers automatically spil
 config = FlowConfig(
     memory_budget=256 * 1024 * 1024,  # 256 MB RAM budget
     spill_dir="/tmp/stackforge-spill", # optional custom spill directory
+    store_packet_indices=False,        # save ~8 bytes/pkt on large captures
+    progress_interval=500_000,         # report every 500K packets (default: 100K)
 )
 conversations = extract_flows("large_capture.pcapng", config=config)
 ```

{stackforge-0.6.5 → stackforge-0.6.6}/crates/stackforge-core/src/flow/mod.rs

@@ -174,6 +174,18 @@ pub fn extract_flows_with_config(
             format_duration(elapsed),
             format_count(rate as usize)
         );
+
+        let (total_drops, flows_with_drops) = count_dropped_segments(&conversations);
+        if total_drops > 0 {
+            eprintln!(
+                "[!] Warning: {} TCP segments dropped across {} flows (buffer/fragment limits exceeded)",
+                format_count(total_drops as usize),
+                format_count(flows_with_drops),
+            );
+            eprintln!(
+                "[!] Tip: increase max_reassembly_buffer or max_ooo_fragments to capture more data"
+            );
+        }
         eprintln!();
     }
     Ok(conversations)
@@ -265,11 +277,40 @@ where
             format_count(conversations.len())
         );
         eprintln!("[+] Wall time: {}", format_duration(elapsed));
+
+        // Report reassembly drops
+        let (total_drops, flows_with_drops) = count_dropped_segments(&conversations);
+        if total_drops > 0 {
+            eprintln!(
+                "[!] Warning: {} TCP segments dropped across {} flows (buffer/fragment limits exceeded)",
+                format_count(total_drops as usize),
+                format_count(flows_with_drops),
+            );
+            eprintln!(
+                "[!] Tip: increase max_reassembly_buffer or max_ooo_fragments to capture more data"
+            );
+        }
         eprintln!();
     }
     Ok(conversations)
 }
 
+/// Count total dropped TCP segments across all conversations.
+fn count_dropped_segments(conversations: &[ConversationState]) -> (u64, usize) {
+    let mut total_drops: u64 = 0;
+    let mut flows_with_drops: usize = 0;
+    for conv in conversations {
+        if let ProtocolState::Tcp(ref tcp) = conv.protocol_state {
+            let drops = tcp.total_dropped_segments();
+            if drops > 0 {
+                total_drops += drops;
+                flows_with_drops += 1;
+            }
+        }
+    }
+    (total_drops, flows_with_drops)
+}
+
 /// Extract flows directly from a capture file (PCAP or PcapNG).
 ///
 /// Streams packets from disk — never loads the entire file into memory.

{stackforge-0.6.5 → stackforge-0.6.6}/crates/stackforge-core/src/flow/table.rs

@@ -160,20 +160,31 @@ impl ConversationTable {
 
     /// Spill reassembly buffers to disk until under budget.
     ///
-    /// Limits the number of entries scanned per call to avoid O(n) full-table
-    /// walks that become expensive as the flow count grows.
+    /// Two limits prevent runaway iteration:
+    /// - `max_spills`: stop after spilling this many buffers (actual work done)
+    /// - `max_skip`: stop after skipping this many already-spilled/non-TCP entries
+    ///   without finding anything to free (avoids scanning the entire table when
+    ///   most flows are already on disk)
     fn maybe_spill(&self) {
-        let mut scanned = 0;
-        let max_scan = 256; // Cap per-call work to avoid stalls
+        let mut spills = 0;
+        let max_spills = 64;
+        let mut consecutive_skips = 0;
+        let max_skip = 512;
+
         for mut entry in self.conversations.iter_mut() {
-            if !self.memory_tracker.is_over_budget() || scanned >= max_scan {
+            if !self.memory_tracker.is_over_budget() || spills >= max_spills {
+                break;
+            }
+            if consecutive_skips >= max_skip {
+                // Most nearby entries are already spilled — stop scanning
                 break;
             }
-            scanned += 1;
+
             if let ProtocolState::Tcp(ref mut tcp_state) = entry.value_mut().protocol_state {
-                // Only try to spill buffers that are still in memory
+                // Skip buffers already on disk
                 if tcp_state.reassembler_fwd.is_spilled() && tcp_state.reassembler_rev.is_spilled()
                 {
+                    consecutive_skips += 1;
                     continue;
                 }
                 let freed_fwd = tcp_state
@@ -189,7 +200,13 @@ impl ConversationTable {
                     self.memory_tracker.subtract(total_freed);
                     self.spill_count
                         .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+                    spills += 1;
+                    consecutive_skips = 0; // Reset — we found something useful
+                } else {
+                    consecutive_skips += 1;
                 }
+            } else {
+                consecutive_skips += 1;
             }
         }
     }

{stackforge-0.6.5 → stackforge-0.6.6}/crates/stackforge-core/src/flow/tcp_state.rs

@@ -104,6 +104,10 @@ pub struct TcpConversationState {
     pub reassembler_fwd: TcpReassembler,
     /// Stream reassembly for reverse direction.
     pub reassembler_rev: TcpReassembler,
+    /// Number of forward segments dropped due to buffer/fragment limits.
+    pub dropped_segments_fwd: u64,
+    /// Number of reverse segments dropped due to buffer/fragment limits.
+    pub dropped_segments_rev: u64,
 }
 
 impl TcpConversationState {
@@ -115,9 +119,17 @@ impl TcpConversationState {
             reverse_endpoint: TcpEndpointState::new(),
             reassembler_fwd: TcpReassembler::new(),
             reassembler_rev: TcpReassembler::new(),
+            dropped_segments_fwd: 0,
+            dropped_segments_rev: 0,
         }
     }
 
+    /// Total dropped segments across both directions.
+    #[must_use]
+    pub fn total_dropped_segments(&self) -> u64 {
+        self.dropped_segments_fwd + self.dropped_segments_rev
+    }
+
     /// Process a TCP packet, updating connection state and reassembly buffers.
     ///
     /// `direction` indicates whether this packet is Forward (`addr_a` → `addr_b`)
@@ -219,9 +231,20 @@ impl TcpConversationState {
 
                 // Process payload through reassembler
                 if !payload.is_empty() {
-                    // Ignore reassembly errors (buffer full, etc.) — they don't
-                    // affect connection state tracking
-                    let _ = reassembler.process_segment(seq, payload, config);
+                    if let Err(e) = reassembler.process_segment(seq, payload, config) {
+                        // Buffer full or too many fragments — track the drop
+                        match direction {
+                            FlowDirection::Forward => self.dropped_segments_fwd += 1,
+                            FlowDirection::Reverse => self.dropped_segments_rev += 1,
+                        }
+                        // Log once at thresholds to avoid flooding stderr
+                        let total = self.dropped_segments_fwd + self.dropped_segments_rev;
+                        if total == 1 || total.is_power_of_two() {
+                            eprintln!(
+                                "[!] stackforge: TCP reassembly dropped segment ({e}), {total} total drops for this flow"
+                            );
+                        }
+                    }
                 }
 
                 if flags.fin {

{stackforge-0.6.5 → stackforge-0.6.6}/src/lib.rs

@@ -4254,6 +4254,34 @@ impl PyConversation {
         }
     }
 
+    /// Number of TCP segments dropped due to reassembly buffer/fragment limits.
+    /// Returns 0 for non-TCP flows.
+    #[getter]
+    fn dropped_segments(&self) -> u64 {
+        match &self.inner.protocol_state {
+            stackforge_core::ProtocolState::Tcp(tcp) => tcp.total_dropped_segments(),
+            _ => 0,
+        }
+    }
+
+    /// Number of forward-direction TCP segments dropped.
+    #[getter]
+    fn dropped_segments_fwd(&self) -> u64 {
+        match &self.inner.protocol_state {
+            stackforge_core::ProtocolState::Tcp(tcp) => tcp.dropped_segments_fwd,
+            _ => 0,
+        }
+    }
+
+    /// Number of reverse-direction TCP segments dropped.
+    #[getter]
+    fn dropped_segments_rev(&self) -> u64 {
+        match &self.inner.protocol_state {
+            stackforge_core::ProtocolState::Tcp(tcp) => tcp.dropped_segments_rev,
+            _ => 0,
+        }
+    }
+
     /// Z-Wave home ID, or None for non-Z-Wave flows.
     #[getter]
     fn zwave_home_id(&self) -> Option<u32> {