stackforge 0.6.1__tar.gz → 0.6.3__tar.gz

{stackforge-0.6.1 → stackforge-0.6.3}/Cargo.lock

@@ -1074,7 +1074,7 @@ dependencies = [
 
 [[package]]
 name = "stackforge"
-version = "0.6.1"
+version = "0.6.3"
 dependencies = [
  "pyo3",
  "stackforge-automata",
@@ -1083,7 +1083,7 @@ dependencies = [
 
 [[package]]
 name = "stackforge-automata"
-version = "0.6.1"
+version = "0.6.3"
 dependencies = [
  "bytes",
  "stackforge-core",
@@ -1091,7 +1091,7 @@ dependencies = [
 
 [[package]]
 name = "stackforge-core"
-version = "0.6.1"
+version = "0.6.3"
 dependencies = [
  "aes",
  "aes-gcm",

{stackforge-0.6.1 → stackforge-0.6.3}/Cargo.toml

@@ -21,7 +21,7 @@ shared-version = true
 tag-name = "v{{version}}"
 
 [workspace.package]
-version = "0.6.1"
+version = "0.6.3"
 edition = "2024"
 license = "GPL-3.0-only"
 authors = ["Stackforge Contributors"]

{stackforge-0.6.1 → stackforge-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stackforge
-Version: 0.6.1
+Version: 0.6.3
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: System Administrators
@@ -354,6 +354,22 @@ packets = rdpcap("capture.pcap")
 conversations = extract_flows_from_packets(packets)
 ```
 
+Enable verbose mode to see progress feedback on stderr during extraction:
+
+```python
+# Quick verbose flag on the function call
+conversations = extract_flows("capture.pcap", verbose=True)
+# [stackforge] Opening capture file: capture.pcap
+# [stackforge] Starting streaming flow extraction...
+# [stackforge] Processed 10000 packets (342 flows so far)
+# [stackforge] Processed 20000 packets (587 flows so far)
+# [stackforge] Flow extraction complete: 612 conversations
+
+# Or via FlowConfig
+config = FlowConfig(verbose=True)
+conversations = extract_flows("capture.pcap", config=config)
+```
+
 Customize timeouts, buffer limits, and memory budget with `FlowConfig`:
 
 ```python

{stackforge-0.6.1 → stackforge-0.6.3}/README.md

@@ -330,6 +330,22 @@ packets = rdpcap("capture.pcap")
 conversations = extract_flows_from_packets(packets)
 ```
 
+Enable verbose mode to see progress feedback on stderr during extraction:
+
+```python
+# Quick verbose flag on the function call
+conversations = extract_flows("capture.pcap", verbose=True)
+# [stackforge] Opening capture file: capture.pcap
+# [stackforge] Starting streaming flow extraction...
+# [stackforge] Processed 10000 packets (342 flows so far)
+# [stackforge] Processed 20000 packets (587 flows so far)
+# [stackforge] Flow extraction complete: 612 conversations
+
+# Or via FlowConfig
+config = FlowConfig(verbose=True)
+conversations = extract_flows("capture.pcap", config=config)
+```
+
 Customize timeouts, buffer limits, and memory budget with `FlowConfig`:
 
 ```python

{stackforge-0.6.1 → stackforge-0.6.3}/crates/stackforge-core/src/flow/config.rs

@@ -30,6 +30,8 @@ pub struct FlowConfig {
     pub memory_budget: Option<usize>,
     /// Directory for spill files (None = system temp dir).
     pub spill_dir: Option<PathBuf>,
+    /// Print progress feedback to stderr during flow extraction (default: false).
+    pub verbose: bool,
 }
 
 impl Default for FlowConfig {
@@ -46,6 +48,7 @@ impl Default for FlowConfig {
             track_max_flow_len: false,
             memory_budget: None,
             spill_dir: None,
+            verbose: false,
         }
     }
 }

{stackforge-0.6.1 → stackforge-0.6.3}/crates/stackforge-core/src/flow/mod.rs

@@ -75,14 +75,35 @@ pub fn extract_flows_with_config(
     packets: &[CapturedPacket],
     config: FlowConfig,
 ) -> Result<Vec<ConversationState>, FlowError> {
+    let verbose = config.verbose;
+    let total = packets.len();
     let table = ConversationTable::new(config);
 
+    if verbose {
+        eprintln!("[stackforge] Starting flow extraction ({total} packets)...");
+    }
+
     for (index, captured) in packets.iter().enumerate() {
         let timestamp = captured.metadata.timestamp;
         table.ingest_packet(&captured.packet, timestamp, index)?;
+
+        if verbose && (index + 1) % 10_000 == 0 {
+            eprintln!(
+                "[stackforge] Processed {}/{total} packets ({} flows so far)",
+                index + 1,
+                table.conversation_count(),
+            );
+        }
     }
 
-    Ok(table.into_conversations())
+    let conversations = table.into_conversations();
+    if verbose {
+        eprintln!(
+            "[stackforge] Flow extraction complete: {total} packets -> {} conversations",
+            conversations.len(),
+        );
+    }
+    Ok(conversations)
 }
 
 /// Extract flows from a streaming packet source (iterator).
@@ -100,16 +121,36 @@ pub fn extract_flows_streaming<I>(
 where
     I: Iterator<Item = Result<CapturedPacket, PacketError>>,
 {
+    let verbose = config.verbose;
     let table = ConversationTable::new(config);
 
+    if verbose {
+        eprintln!("[stackforge] Starting streaming flow extraction...");
+    }
+
     for (index, result) in packets.enumerate() {
         let captured = result.map_err(FlowError::PacketError)?;
         let timestamp = captured.metadata.timestamp;
         table.ingest_packet(&captured.packet, timestamp, index)?;
         // `captured` is dropped here — packet memory freed immediately
+
+        if verbose && (index + 1) % 10_000 == 0 {
+            eprintln!(
+                "[stackforge] Processed {} packets ({} flows so far)",
+                index + 1,
+                table.conversation_count(),
+            );
+        }
     }
 
-    Ok(table.into_conversations())
+    let conversations = table.into_conversations();
+    if verbose {
+        eprintln!(
+            "[stackforge] Flow extraction complete: {} conversations",
+            conversations.len(),
+        );
+    }
+    Ok(conversations)
 }
 
 /// Extract flows directly from a capture file (PCAP or PcapNG).
@@ -120,6 +161,13 @@ pub fn extract_flows_from_file(
     path: impl AsRef<Path>,
     config: FlowConfig,
 ) -> Result<Vec<ConversationState>, FlowError> {
+    let verbose = config.verbose;
+    if verbose {
+        eprintln!(
+            "[stackforge] Opening capture file: {}",
+            path.as_ref().display(),
+        );
+    }
     let iter = CaptureIterator::open(path).map_err(FlowError::PacketError)?;
     extract_flows_streaming(iter, config)
 }

{stackforge-0.6.1 → stackforge-0.6.3}/crates/stackforge-core/src/flow/spill.rs

@@ -43,17 +43,18 @@ impl ReassemblyStorage {
         self.len() == 0
     }
 
-    /// Append data. Only valid when `InMemory`.
+    /// Append data. Works for both in-memory and on-disk storage.
     ///
-    /// # Panics
-    /// Panics if storage has been spilled to disk. Callers must ensure data
-    /// is not appended after a spill (in practice, spilling only happens
-    /// for completed/idle flows).
+    /// When in-memory, appends to the `Vec`. When on-disk, appends to the
+    /// temporary file and updates the tracked length.
     pub fn extend_from_slice(&mut self, data: &[u8]) {
         match self {
             Self::InMemory(v) => v.extend_from_slice(data),
-            Self::OnDisk { .. } => {
-                panic!("cannot extend spilled storage; data already on disk");
+            Self::OnDisk { file, len } => {
+                file.write_all(data)
+                    .expect("failed to append to spill file");
+                file.flush().expect("failed to flush spill file");
+                *len += data.len();
             },
         }
     }
@@ -258,6 +259,22 @@ mod tests {
         assert!(s.is_empty());
     }
 
+    #[test]
+    fn test_reassembly_storage_extend_after_spill() {
+        let mut s = ReassemblyStorage::new();
+        s.extend_from_slice(b"hello ");
+        s.spill_to_disk(None).unwrap();
+        assert!(s.is_spilled());
+
+        // Appending after spill should work (writes to file)
+        s.extend_from_slice(b"world");
+        assert_eq!(s.len(), 11);
+        assert!(s.is_spilled());
+
+        let data = s.read_all().unwrap();
+        assert_eq!(data, b"hello world");
+    }
+
     #[test]
     fn test_reassembly_storage_drain_spilled() {
         let mut s = ReassemblyStorage::new();

{stackforge-0.6.1 → stackforge-0.6.3}/crates/stackforge-core/src/flow/table.rs

@@ -125,17 +125,21 @@ impl ConversationTable {
         // Update conversation status from protocol state
         conv.update_status();
 
-        // Track memory for TCP reassembly buffers
+        // Track memory for TCP reassembly buffers (only for in-memory data)
         if self.memory_tracker.has_budget() {
-            if matches!(conv.protocol_state, ProtocolState::Tcp(_)) {
-                // Track bytes that TCP payload may have added
-                let tcp_payload_len = packet.tcp().map_or(0, |tcp| {
-                    let data_offset = tcp.data_offset(buf).unwrap_or(5) as usize * 4;
-                    let payload_start = tcp.index.start + data_offset;
-                    buf.len().saturating_sub(payload_start)
-                });
-                if tcp_payload_len > 0 {
-                    self.memory_tracker.add(tcp_payload_len);
+            if let ProtocolState::Tcp(ref tcp_state) = conv.protocol_state {
+                // Only track if at least one reassembler is still in memory
+                let fwd_spilled = tcp_state.reassembler_fwd.is_spilled();
+                let rev_spilled = tcp_state.reassembler_rev.is_spilled();
+                if !fwd_spilled || !rev_spilled {
+                    let tcp_payload_len = packet.tcp().map_or(0, |tcp| {
+                        let data_offset = tcp.data_offset(buf).unwrap_or(5) as usize * 4;
+                        let payload_start = tcp.index.start + data_offset;
+                        buf.len().saturating_sub(payload_start)
+                    });
+                    if tcp_payload_len > 0 {
+                        self.memory_tracker.add(tcp_payload_len);
+                    }
                 }
             }
         }

{stackforge-0.6.1 → stackforge-0.6.3}/src/lib.rs

@@ -4076,6 +4076,7 @@ impl PyFlowConfig {
         track_max_flow_len=false,
         memory_budget=None,
         spill_dir=None,
+        verbose=false,
     ))]
     #[allow(clippy::too_many_arguments)]
     fn new(
@@ -4089,6 +4090,7 @@ impl PyFlowConfig {
         track_max_flow_len: bool,
         memory_budget: Option<usize>,
         spill_dir: Option<String>,
+        verbose: bool,
     ) -> Self {
         Self {
             inner: stackforge_core::FlowConfig {
@@ -4104,6 +4106,7 @@ impl PyFlowConfig {
                 track_max_flow_len,
                 memory_budget,
                 spill_dir: spill_dir.map(std::path::PathBuf::from),
+                verbose,
                 ..stackforge_core::FlowConfig::default()
             },
         }
@@ -4451,9 +4454,16 @@ impl PyConversation {
 /// Returns:
 ///     List of Conversation objects sorted by start time.
 #[pyfunction]
-#[pyo3(signature = (pcap_path, config=None))]
-fn extract_flows(pcap_path: &str, config: Option<PyFlowConfig>) -> PyResult<Vec<PyConversation>> {
-    let flow_config = config.map(|c| c.inner).unwrap_or_default();
+#[pyo3(signature = (pcap_path, config=None, verbose=false))]
+fn extract_flows(
+    pcap_path: &str,
+    config: Option<PyFlowConfig>,
+    verbose: bool,
+) -> PyResult<Vec<PyConversation>> {
+    let mut flow_config = config.map(|c| c.inner).unwrap_or_default();
+    if verbose {
+        flow_config.verbose = true;
+    }
 
     // Use streaming extraction — never loads all packets into memory at once
     let conversations =
@@ -4484,10 +4494,11 @@ fn extract_flows(pcap_path: &str, config: Option<PyFlowConfig>) -> PyResult<Vec<
 /// Returns:
 ///     List of Conversation objects sorted by start time.
 #[pyfunction]
-#[pyo3(signature = (packets, config=None))]
+#[pyo3(signature = (packets, config=None, verbose=false))]
 fn extract_flows_from_packets(
     packets: Vec<PyRef<'_, PyPacket>>,
     config: Option<PyFlowConfig>,
+    verbose: bool,
 ) -> PyResult<Vec<PyConversation>> {
     let captured: Vec<stackforge_core::CapturedPacket> = packets
         .iter()
@@ -4502,7 +4513,10 @@ fn extract_flows_from_packets(
         })
         .collect();
 
-    let flow_config = config.map(|c| c.inner).unwrap_or_default();
+    let mut flow_config = config.map(|c| c.inner).unwrap_or_default();
+    if verbose {
+        flow_config.verbose = true;
+    }
 
     let conversations = stackforge_core::extract_flows_with_config(&captured, flow_config)
         .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(format!("{e}")))?;