stackforge 0.6.0__tar.gz → 0.6.2__tar.gz

{stackforge-0.6.0 → stackforge-0.6.2}/Cargo.lock

@@ -377,6 +377,22 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "errno"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "fastrand"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
+
 [[package]]
 name = "ff"
 version = "0.13.1"
@@ -512,6 +528,12 @@ version = "0.2.178"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091"
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039"
+
 [[package]]
 name = "lock_api"
 version = "0.4.14"
@@ -543,6 +565,15 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df39d232f5c40b0891c10216992c2f250c054105cb1e56f0fc9032db6203ecc1"
 
+[[package]]
+name = "memmap2"
+version = "0.9.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "memoffset"
 version = "0.9.1"
@@ -918,6 +949,19 @@ dependencies = [
  "semver",
 ]
 
+[[package]]
+name = "rustix"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34"
+dependencies = [
+ "bitflags 2.11.0",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.22"
@@ -1030,7 +1074,7 @@ dependencies = [
 
 [[package]]
 name = "stackforge"
-version = "0.6.0"
+version = "0.6.2"
 dependencies = [
  "pyo3",
  "stackforge-automata",
@@ -1039,7 +1083,7 @@ dependencies = [
 
 [[package]]
 name = "stackforge-automata"
-version = "0.6.0"
+version = "0.6.2"
 dependencies = [
  "bytes",
  "stackforge-core",
@@ -1047,7 +1091,7 @@ dependencies = [
 
 [[package]]
 name = "stackforge-core"
-version = "0.6.0"
+version = "0.6.2"
 dependencies = [
  "aes",
  "aes-gcm",
@@ -1061,6 +1105,7 @@ dependencies = [
  "hkdf",
  "hmac",
  "md-5",
+ "memmap2",
  "p256",
  "pcap-file",
  "pnet_datalink",
@@ -1069,6 +1114,7 @@ dependencies = [
  "sha1",
  "sha2",
  "smallvec",
+ "tempfile",
  "thiserror 2.0.17",
  "x25519-dalek",
 ]
@@ -1128,6 +1174,19 @@ version = "0.13.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba"
 
+[[package]]
+name = "tempfile"
+version = "3.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1"
+dependencies = [
+ "fastrand",
+ "getrandom 0.3.4",
+ "once_cell",
+ "rustix",
+ "windows-sys",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.69"
@@ -1254,6 +1313,15 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
 
+[[package]]
+name = "windows-sys"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
+dependencies = [
+ "windows-link",
+]
+
 [[package]]
 name = "windows-targets"
 version = "0.48.5"

{stackforge-0.6.0 → stackforge-0.6.2}/Cargo.toml

@@ -21,7 +21,7 @@ shared-version = true
 tag-name = "v{{version}}"
 
 [workspace.package]
-version = "0.6.0"
+version = "0.6.2"
 edition = "2024"
 license = "GPL-3.0-only"
 authors = ["Stackforge Contributors"]
@@ -32,6 +32,8 @@ bytes = "1.11.0"
 smallvec = "1.15.1"
 thiserror = "2.0.17"
 dashmap = "6.1"
+memmap2 = "0.9"
+tempfile = "3"
 stackforge-core = { path = "crates/stackforge-core" }
 stackforge-automata = { path = "crates/stackforge-automata" }
 

{stackforge-0.6.0 → stackforge-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stackforge
-Version: 0.6.0
+Version: 0.6.2
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: System Administrators
@@ -35,9 +35,10 @@ Project-URL: Repository, https://github.com/LaBackDoor/stackforge
 
 - **Scapy-style API** — Stack layers with `Ether() / IP() / TCP()`, set fields with keyword arguments
 - **High Performance** — Core logic in Rust, zero-copy parsing, copy-on-write mutation
-- **Broad Protocol Support** — Ethernet, ARP, IPv4/IPv6, TCP, UDP, ICMP/ICMPv6 (with echo correlation), DNS, HTTP/1.x, HTTP/2, QUIC, L2TP, 802.11 (Wi-Fi), 802.15.4 (Zigbee), and custom protocols
-- **Stateful Flow Extraction** — Extract bidirectional conversations from PCAP files with TCP state tracking, stream reassembly, UDP timeout handling, and optional max packet/flow length tracking
-- **PCAP I/O** — Read and write pcap files with `rdpcap()` / `wrpcap()`
+- **Broad Protocol Support** — Ethernet, ARP, IPv4/IPv6, TCP, UDP, ICMP/ICMPv6 (with echo correlation), DNS, HTTP/1.x, HTTP/2, QUIC, L2TP, MQTT, MQTT-SN, Modbus, Z-Wave, FTP, TFTP, SMTP, POP3, IMAP, 802.11 (Wi-Fi), 802.15.4 (Zigbee), and custom protocols
+- **Stateful Flow Extraction** — Extract bidirectional conversations from PCAP/PcapNG files with TCP state tracking, stream reassembly, UDP timeout handling, and optional max packet/flow length tracking
+- **Memory-Budgeted Streaming** — Process gigabyte-sized captures without loading everything into RAM; set a memory budget and reassembly buffers automatically spill to memory-mapped temp files
+- **PCAP & PcapNG I/O** — Read and write both classic PCAP and PcapNG files with auto-detection via `rdpcap()` / `wrpcap()` / `wrpcapng()`
 - **Python Bindings** — Seamless integration via PyO3/maturin
 - **Custom Protocols** — Define runtime protocols with `CustomLayer` and typed fields
 
@@ -101,10 +102,10 @@ print(pkt.summary())                    # "Ethernet / IPv4 / TCP"
 print(pkt.show())                       # detailed layer view
 ```
 
-### Read and write PCAP files
+### Read and write PCAP / PcapNG files
 
 ```python
-from stackforge import rdpcap, wrpcap, PcapReader, Ether, IP, TCP
+from stackforge import rdpcap, wrpcap, wrpcapng, PcapReader, Ether, IP, TCP
 
 # Write packets to a pcap file
 packets = [
@@ -113,13 +114,20 @@ packets = [
 ]
 wrpcap("capture.pcap", packets)
 
-# Read all packets at once
-packets = rdpcap("capture.pcap")
+# Write PcapNG format explicitly
+wrpcapng("capture.pcapng", packets)
+
+# wrpcap auto-detects format from extension
+wrpcap("capture.pcapng", packets)  # writes PcapNG
+
+# Read any format (auto-detected)
+packets = rdpcap("capture.pcap")    # classic PCAP
+packets = rdpcap("capture.pcapng")  # PcapNG — same API
 for pkt in packets:
     print(pkt.summary())
 
-# Stream large pcap files
-for pkt in PcapReader("large_capture.pcap"):
+# Stream large captures (works with both formats)
+for pkt in PcapReader("large_capture.pcapng"):
     print(pkt.summary())
 ```
 
@@ -281,6 +289,36 @@ pkt.parse()
 print(pkt.has_layer(LayerKind.L2tp))
 ```
 
+### IoT Protocols
+
+```python
+from stackforge import MQTT, MQTTSN, Modbus, ZWave
+
+# MQTT (auto-detected on TCP port 1883)
+pkt = Ether() / IP() / TCP(dport=1883) / MQTT(msg_type=1)  # CONNECT
+
+# MQTT-SN (auto-detected on UDP port 1883)
+pkt = Ether() / IP() / UDP(dport=1883) / MQTTSN(msg_type=0x04)  # PUBLISH
+
+# Modbus TCP (auto-detected on TCP port 502)
+pkt = Ether() / IP() / TCP(dport=502) / Modbus(func_code=3, data=b"\x00\x01\x00\x0a")
+
+# Z-Wave (wireless, not auto-detected over TCP/UDP)
+pkt = ZWave(home_id=0x12345678, src=1, dst=2, cmd_class=0x25, cmd=0x01)
+```
+
+### Email & File Transfer Protocols
+
+```python
+from stackforge import FTP, TFTP, SMTP, POP3, IMAP
+
+# FTP (TCP port 21), SMTP (TCP ports 25/587/465), POP3 (TCP port 110), IMAP (TCP port 143)
+# All auto-detected during packet parsing
+
+# TFTP (UDP port 69)
+pkt = Ether() / IP() / UDP(dport=69) / TFTP(opcode=1, filename="test.txt", mode="octet")
+```
+
 ### Stateful Flow Extraction
 
 Extract bidirectional conversations from PCAP captures with full TCP state machine tracking, stream reassembly, and UDP timeout-based flow grouping.
@@ -316,7 +354,7 @@ packets = rdpcap("capture.pcap")
 conversations = extract_flows_from_packets(packets)
 ```
 
-Customize timeouts and buffer limits with `FlowConfig`:
+Customize timeouts, buffer limits, and memory budget with `FlowConfig`:
 
 ```python
 config = FlowConfig(
@@ -327,6 +365,20 @@ config = FlowConfig(
 conversations = extract_flows("capture.pcap", config=config)
 ```
 
+#### Memory-Budgeted Flow Extraction
+
+For large captures, set a memory budget so reassembly buffers automatically spill to disk when RAM is tight:
+
+```python
+config = FlowConfig(
+    memory_budget=256 * 1024 * 1024,  # 256 MB RAM budget
+    spill_dir="/tmp/stackforge-spill", # optional custom spill directory
+)
+conversations = extract_flows("large_capture.pcapng", config=config)
+```
+
+Packets stream from disk one at a time (never loaded all at once). When TCP reassembly buffers exceed the budget, the largest buffers are transparently spilled to memory-mapped temp files and read back on demand. Temp files are automatically cleaned up via RAII.
+
 Optional: Track maximum packet sizes during flow extraction:
 
 ```python

{stackforge-0.6.0 → stackforge-0.6.2}/README.md

@@ -11,9 +11,10 @@
 
 - **Scapy-style API** — Stack layers with `Ether() / IP() / TCP()`, set fields with keyword arguments
 - **High Performance** — Core logic in Rust, zero-copy parsing, copy-on-write mutation
-- **Broad Protocol Support** — Ethernet, ARP, IPv4/IPv6, TCP, UDP, ICMP/ICMPv6 (with echo correlation), DNS, HTTP/1.x, HTTP/2, QUIC, L2TP, 802.11 (Wi-Fi), 802.15.4 (Zigbee), and custom protocols
-- **Stateful Flow Extraction** — Extract bidirectional conversations from PCAP files with TCP state tracking, stream reassembly, UDP timeout handling, and optional max packet/flow length tracking
-- **PCAP I/O** — Read and write pcap files with `rdpcap()` / `wrpcap()`
+- **Broad Protocol Support** — Ethernet, ARP, IPv4/IPv6, TCP, UDP, ICMP/ICMPv6 (with echo correlation), DNS, HTTP/1.x, HTTP/2, QUIC, L2TP, MQTT, MQTT-SN, Modbus, Z-Wave, FTP, TFTP, SMTP, POP3, IMAP, 802.11 (Wi-Fi), 802.15.4 (Zigbee), and custom protocols
+- **Stateful Flow Extraction** — Extract bidirectional conversations from PCAP/PcapNG files with TCP state tracking, stream reassembly, UDP timeout handling, and optional max packet/flow length tracking
+- **Memory-Budgeted Streaming** — Process gigabyte-sized captures without loading everything into RAM; set a memory budget and reassembly buffers automatically spill to memory-mapped temp files
+- **PCAP & PcapNG I/O** — Read and write both classic PCAP and PcapNG files with auto-detection via `rdpcap()` / `wrpcap()` / `wrpcapng()`
 - **Python Bindings** — Seamless integration via PyO3/maturin
 - **Custom Protocols** — Define runtime protocols with `CustomLayer` and typed fields
 
@@ -77,10 +78,10 @@ print(pkt.summary())                    # "Ethernet / IPv4 / TCP"
 print(pkt.show())                       # detailed layer view
 ```
 
-### Read and write PCAP files
+### Read and write PCAP / PcapNG files
 
 ```python
-from stackforge import rdpcap, wrpcap, PcapReader, Ether, IP, TCP
+from stackforge import rdpcap, wrpcap, wrpcapng, PcapReader, Ether, IP, TCP
 
 # Write packets to a pcap file
 packets = [
@@ -89,13 +90,20 @@ packets = [
 ]
 wrpcap("capture.pcap", packets)
 
-# Read all packets at once
-packets = rdpcap("capture.pcap")
+# Write PcapNG format explicitly
+wrpcapng("capture.pcapng", packets)
+
+# wrpcap auto-detects format from extension
+wrpcap("capture.pcapng", packets)  # writes PcapNG
+
+# Read any format (auto-detected)
+packets = rdpcap("capture.pcap")    # classic PCAP
+packets = rdpcap("capture.pcapng")  # PcapNG — same API
 for pkt in packets:
     print(pkt.summary())
 
-# Stream large pcap files
-for pkt in PcapReader("large_capture.pcap"):
+# Stream large captures (works with both formats)
+for pkt in PcapReader("large_capture.pcapng"):
     print(pkt.summary())
 ```
 
@@ -257,6 +265,36 @@ pkt.parse()
 print(pkt.has_layer(LayerKind.L2tp))
 ```
 
+### IoT Protocols
+
+```python
+from stackforge import MQTT, MQTTSN, Modbus, ZWave
+
+# MQTT (auto-detected on TCP port 1883)
+pkt = Ether() / IP() / TCP(dport=1883) / MQTT(msg_type=1)  # CONNECT
+
+# MQTT-SN (auto-detected on UDP port 1883)
+pkt = Ether() / IP() / UDP(dport=1883) / MQTTSN(msg_type=0x04)  # PUBLISH
+
+# Modbus TCP (auto-detected on TCP port 502)
+pkt = Ether() / IP() / TCP(dport=502) / Modbus(func_code=3, data=b"\x00\x01\x00\x0a")
+
+# Z-Wave (wireless, not auto-detected over TCP/UDP)
+pkt = ZWave(home_id=0x12345678, src=1, dst=2, cmd_class=0x25, cmd=0x01)
+```
+
+### Email & File Transfer Protocols
+
+```python
+from stackforge import FTP, TFTP, SMTP, POP3, IMAP
+
+# FTP (TCP port 21), SMTP (TCP ports 25/587/465), POP3 (TCP port 110), IMAP (TCP port 143)
+# All auto-detected during packet parsing
+
+# TFTP (UDP port 69)
+pkt = Ether() / IP() / UDP(dport=69) / TFTP(opcode=1, filename="test.txt", mode="octet")
+```
+
 ### Stateful Flow Extraction
 
 Extract bidirectional conversations from PCAP captures with full TCP state machine tracking, stream reassembly, and UDP timeout-based flow grouping.
@@ -292,7 +330,7 @@ packets = rdpcap("capture.pcap")
 conversations = extract_flows_from_packets(packets)
 ```
 
-Customize timeouts and buffer limits with `FlowConfig`:
+Customize timeouts, buffer limits, and memory budget with `FlowConfig`:
 
 ```python
 config = FlowConfig(
@@ -303,6 +341,20 @@ config = FlowConfig(
 conversations = extract_flows("capture.pcap", config=config)
 ```
 
+#### Memory-Budgeted Flow Extraction
+
+For large captures, set a memory budget so reassembly buffers automatically spill to disk when RAM is tight:
+
+```python
+config = FlowConfig(
+    memory_budget=256 * 1024 * 1024,  # 256 MB RAM budget
+    spill_dir="/tmp/stackforge-spill", # optional custom spill directory
+)
+conversations = extract_flows("large_capture.pcapng", config=config)
+```
+
+Packets stream from disk one at a time (never loaded all at once). When TCP reassembly buffers exceed the budget, the largest buffers are transparently spilled to memory-mapped temp files and read back on demand. Temp files are automatically cleaned up via RAII.
+
 Optional: Track maximum packet sizes during flow extraction:
 
 ```python

{stackforge-0.6.0 → stackforge-0.6.2}/crates/stackforge-core/Cargo.toml

@@ -15,6 +15,8 @@ default-net = "0.22.0"
 rand = { version = "0.9.2", optional = true }
 pcap-file = "2"
 dashmap = { workspace = true }
+memmap2 = { workspace = true }
+tempfile = { workspace = true }
 
 # TLS crypto dependencies
 hmac = { version = "0.12", optional = true }

{stackforge-0.6.0 → stackforge-0.6.2}/crates/stackforge-core/src/flow/config.rs

@@ -1,3 +1,4 @@
+use std::path::PathBuf;
 use std::time::Duration;
 
 /// Configuration for the flow extraction engine.
@@ -24,6 +25,11 @@ pub struct FlowConfig {
     pub track_max_packet_len: bool,
     /// Track maximum flow length per direction (default: false).
     pub track_max_flow_len: bool,
+    /// Total RAM budget for flow extraction (None = unlimited).
+    /// When set, reassembly buffers will be spilled to disk when exceeded.
+    pub memory_budget: Option<usize>,
+    /// Directory for spill files (None = system temp dir).
+    pub spill_dir: Option<PathBuf>,
 }
 
 impl Default for FlowConfig {
@@ -38,6 +44,8 @@ impl Default for FlowConfig {
             eviction_interval: Duration::from_secs(30),
             track_max_packet_len: false,
             track_max_flow_len: false,
+            memory_budget: None,
+            spill_dir: None,
         }
     }
 }

{stackforge-0.6.0 → stackforge-0.6.2}/crates/stackforge-core/src/flow/error.rs

@@ -20,6 +20,9 @@ pub enum FlowError {
     #[error("too many discontinuous fragments ({count}, limit {limit})")]
     TooManyFragments { count: usize, limit: usize },
 
+    #[error("disk spill I/O error: {0}")]
+    SpillError(String),
+
     #[error(transparent)]
     PacketError(#[from] PacketError),
 }

{stackforge-0.6.0 → stackforge-0.6.2}/crates/stackforge-core/src/flow/mod.rs

@@ -29,6 +29,7 @@ pub mod config;
 pub mod error;
 pub mod icmp_state;
 pub mod key;
+pub mod spill;
 pub mod state;
 pub mod table;
 pub mod tcp_reassembly;
@@ -51,9 +52,11 @@ pub use tcp_state::{TcpConnectionState, TcpConversationState, TcpEndpointState};
 pub use udp_state::UdpFlowState;
 
 use std::collections::HashMap;
+use std::path::Path;
 
+use crate::error::PacketError;
 use crate::layer::LayerKind;
-use crate::pcap::CapturedPacket;
+use crate::pcap::{CaptureIterator, CapturedPacket};
 
 /// Extract bidirectional conversations from a list of captured packets.
 ///
@@ -82,6 +85,45 @@ pub fn extract_flows_with_config(
     Ok(table.into_conversations())
 }
 
+/// Extract flows from a streaming packet source (iterator).
+///
+/// Does not require all packets in memory simultaneously — each packet is
+/// processed and then dropped. Only conversation state (metadata + reassembly
+/// buffers) is retained.
+///
+/// If `config.memory_budget` is set, reassembly buffers will be spilled to
+/// disk when the budget is exceeded.
+pub fn extract_flows_streaming<I>(
+    packets: I,
+    config: FlowConfig,
+) -> Result<Vec<ConversationState>, FlowError>
+where
+    I: Iterator<Item = Result<CapturedPacket, PacketError>>,
+{
+    let table = ConversationTable::new(config);
+
+    for (index, result) in packets.enumerate() {
+        let captured = result.map_err(FlowError::PacketError)?;
+        let timestamp = captured.metadata.timestamp;
+        table.ingest_packet(&captured.packet, timestamp, index)?;
+        // `captured` is dropped here — packet memory freed immediately
+    }
+
+    Ok(table.into_conversations())
+}
+
+/// Extract flows directly from a capture file (PCAP or PcapNG).
+///
+/// Streams packets from disk — never loads the entire file into memory.
+/// The file format is auto-detected from magic bytes.
+pub fn extract_flows_from_file(
+    path: impl AsRef<Path>,
+    config: FlowConfig,
+) -> Result<Vec<ConversationState>, FlowError> {
+    let iter = CaptureIterator::open(path).map_err(FlowError::PacketError)?;
+    extract_flows_streaming(iter, config)
+}
+
 /// Extract Z-Wave conversations from a list of captured packets.
 ///
 /// Z-Wave is a wireless protocol not carried over IP, so it needs its own
@@ -155,6 +197,7 @@ mod tests {
             metadata: PcapMetadata {
                 timestamp: Duration::from_secs(timestamp_secs),
                 orig_len: 0,
+                ..Default::default()
             },
         }
     }