netflower 1.0.0__tar.gz

netflower-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Luiz Henrique
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

netflower-1.0.0/PKG-INFO

@@ -0,0 +1,178 @@
+Metadata-Version: 2.4
+Name: netflower
+Version: 1.0.0
+Summary: High-performance network flow extractor with live capture support for edge devices
+Author-email: Luiz Henrique <luizlinkezio@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2026 Luiz Henrique
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Repository, https://github.com/luiz-linkezio/pcapflower
+Keywords: pcap,pcapng,network,flow,CICFlowMeter,ids,intrusion detection,live capture
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: System :: Networking :: Monitoring
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: dpkt>=1.9.8
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Dynamic: license-file
+
+<p align="center">
+  <img src="assets/icon.png" alt="netflower" width="128" />
+</p>
+
+<h1 align="center">netflower</h1>
+
+<p align="center">High-performance network flow extractor for edge devices.</p>
+
+**netflower** extracts bidirectional network flows — either from a live interface or from `.pcap` / `.pcapng` files — and produces 82 features compatible with the [CICFlowMeter](https://www.unb.ca/cic/research/applications.html) feature set.
+
+## Features
+
+- **Live capture** — capture flows in real time from a network interface; flows are emitted only when complete (TCP FIN/RST or idle timeout), never cut by an arbitrary boundary
+- **PCAP flows to CSV** — convert `.pcap` / `.pcapng` files to flow-based CSV, with optional parallel processing
+- **Edge-optimized** — `dpkt` for packet parsing (~10–25× faster than Scapy on ARM), Welford's online algorithm for O(1) memory per flow, batch-buffered CSV output
+- **No extra pip dependencies** for live capture — libpcap is accessed via `ctypes`
+
+## Installation
+
+```bash
+pip install netflower
+```
+
+Live capture requires libpcap on the system:
+
+```bash
+# Linux
+sudo apt install libpcap-dev
+
+# macOS
+brew install libpcap
+```
+
+## Usage
+
+### Live capture
+
+Captures packets from a network interface and emits each flow the moment it completes.
+
+```python
+from netflower import capture_live
+
+def on_flow(flow):
+    print(flow["src_ip"], flow["dst_ip"], flow["flow_duration"])
+
+handle = capture_live("eth0", on_flow=on_flow)
+handle.start()
+
+# ... rest of your program ...
+
+handle.stop()
+```
+
+As a context manager:
+
+```python
+with capture_live("eth0", on_flow=on_flow) as handle:
+    handle.start()
+    import time; time.sleep(60)
+```
+
+> **Note:** live capture requires root or `CAP_NET_RAW` privilege.
+
+### PCAP flows to CSV
+
+Converts a `.pcap` or `.pcapng` file into a flow-based CSV.
+
+```python
+from netflower import convert_pcap_to_csv
+
+n = convert_pcap_to_csv("capture.pcap", "flows.csv")
+print(f"Extracted {n} flows")
+
+# Parallel — use all available CPUs
+n = convert_pcap_to_csv("capture.pcap", "flows.csv", n_jobs=-1)
+```
+
+## API
+
+### `capture_live(interface, on_flow, **kwargs) → CaptureHandle`
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `interface` | — | Network interface name (e.g. `"eth0"`) |
+| `on_flow` | — | Callable receiving a flow `dict` when a flow completes |
+| `idle_timeout` | `30.0` | Seconds of inactivity before a flow is emitted |
+| `flow_timeout` | `120.0` | Absolute max flow duration before forced emit |
+| `save_pcap` | `False` | Save raw packets of each completed flow to a `.pcap` file |
+| `pcap_dir` | `None` | Directory for `.pcap` files (required when `save_pcap=True`) |
+
+`CaptureHandle` exposes `.start()`, `.stop()`, and context manager support.
+
+### `convert_pcap_to_csv(input_path, output_path, **kwargs) → int`
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `input_path` | — | Path to `.pcap` or `.pcapng` file |
+| `output_path` | — | Path for the output `.csv` (created or overwritten) |
+| `flow_timeout` | `120.0` | Seconds of inactivity before a flow is evicted |
+| `gc_interval` | `1000` | Run idle-flow GC every N packets |
+| `buffer_rows` | `500` | Rows buffered in memory before flushing to disk |
+| `n_jobs` | `1` | Worker processes. `-1` uses all available CPUs |
+
+Returns the number of flow rows written.
+
+## Flow features
+
+Both APIs produce the same **82 features** per flow:
+
+- Flow identity: source/destination IP, port, protocol, timestamp
+- Duration, bytes/s, and packets/s (forward, backward, combined)
+- Packet length statistics (mean, std, min, max, variance)
+- Inter-arrival time statistics (flow, forward, backward)
+- TCP flag counts (FIN, SYN, RST, PSH, ACK, URG, ECE, CWR)
+- Active/idle period statistics
+- Bulk transfer metrics (forward and backward)
+- Subflow metrics
+- Initial TCP window sizes
+
+## Supported formats
+
+- **pcap** — standard libpcap format
+- **pcapng** — next-generation capture format
+
+Only **IPv4 TCP and UDP** flows are extracted; other protocols are silently skipped.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

netflower-1.0.0/README.md

@@ -0,0 +1,129 @@
+<p align="center">
+  <img src="assets/icon.png" alt="netflower" width="128" />
+</p>
+
+<h1 align="center">netflower</h1>
+
+<p align="center">High-performance network flow extractor for edge devices.</p>
+
+**netflower** extracts bidirectional network flows — either from a live interface or from `.pcap` / `.pcapng` files — and produces 82 features compatible with the [CICFlowMeter](https://www.unb.ca/cic/research/applications.html) feature set.
+
+## Features
+
+- **Live capture** — capture flows in real time from a network interface; flows are emitted only when complete (TCP FIN/RST or idle timeout), never cut by an arbitrary boundary
+- **PCAP flows to CSV** — convert `.pcap` / `.pcapng` files to flow-based CSV, with optional parallel processing
+- **Edge-optimized** — `dpkt` for packet parsing (~10–25× faster than Scapy on ARM), Welford's online algorithm for O(1) memory per flow, batch-buffered CSV output
+- **No extra pip dependencies** for live capture — libpcap is accessed via `ctypes`
+
+## Installation
+
+```bash
+pip install netflower
+```
+
+Live capture requires libpcap on the system:
+
+```bash
+# Linux
+sudo apt install libpcap-dev
+
+# macOS
+brew install libpcap
+```
+
+## Usage
+
+### Live capture
+
+Captures packets from a network interface and emits each flow the moment it completes.
+
+```python
+from netflower import capture_live
+
+def on_flow(flow):
+    print(flow["src_ip"], flow["dst_ip"], flow["flow_duration"])
+
+handle = capture_live("eth0", on_flow=on_flow)
+handle.start()
+
+# ... rest of your program ...
+
+handle.stop()
+```
+
+As a context manager:
+
+```python
+with capture_live("eth0", on_flow=on_flow) as handle:
+    handle.start()
+    import time; time.sleep(60)
+```
+
+> **Note:** live capture requires root or `CAP_NET_RAW` privilege.
+
+### PCAP flows to CSV
+
+Converts a `.pcap` or `.pcapng` file into a flow-based CSV.
+
+```python
+from netflower import convert_pcap_to_csv
+
+n = convert_pcap_to_csv("capture.pcap", "flows.csv")
+print(f"Extracted {n} flows")
+
+# Parallel — use all available CPUs
+n = convert_pcap_to_csv("capture.pcap", "flows.csv", n_jobs=-1)
+```
+
+## API
+
+### `capture_live(interface, on_flow, **kwargs) → CaptureHandle`
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `interface` | — | Network interface name (e.g. `"eth0"`) |
+| `on_flow` | — | Callable receiving a flow `dict` when a flow completes |
+| `idle_timeout` | `30.0` | Seconds of inactivity before a flow is emitted |
+| `flow_timeout` | `120.0` | Absolute max flow duration before forced emit |
+| `save_pcap` | `False` | Save raw packets of each completed flow to a `.pcap` file |
+| `pcap_dir` | `None` | Directory for `.pcap` files (required when `save_pcap=True`) |
+
+`CaptureHandle` exposes `.start()`, `.stop()`, and context manager support.
+
+### `convert_pcap_to_csv(input_path, output_path, **kwargs) → int`
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `input_path` | — | Path to `.pcap` or `.pcapng` file |
+| `output_path` | — | Path for the output `.csv` (created or overwritten) |
+| `flow_timeout` | `120.0` | Seconds of inactivity before a flow is evicted |
+| `gc_interval` | `1000` | Run idle-flow GC every N packets |
+| `buffer_rows` | `500` | Rows buffered in memory before flushing to disk |
+| `n_jobs` | `1` | Worker processes. `-1` uses all available CPUs |
+
+Returns the number of flow rows written.
+
+## Flow features
+
+Both APIs produce the same **82 features** per flow:
+
+- Flow identity: source/destination IP, port, protocol, timestamp
+- Duration, bytes/s, and packets/s (forward, backward, combined)
+- Packet length statistics (mean, std, min, max, variance)
+- Inter-arrival time statistics (flow, forward, backward)
+- TCP flag counts (FIN, SYN, RST, PSH, ACK, URG, ECE, CWR)
+- Active/idle period statistics
+- Bulk transfer metrics (forward and backward)
+- Subflow metrics
+- Initial TCP window sizes
+
+## Supported formats
+
+- **pcap** — standard libpcap format
+- **pcapng** — next-generation capture format
+
+Only **IPv4 TCP and UDP** flows are extracted; other protocols are silently skipped.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

netflower-1.0.0/netflower/__init__.py

@@ -0,0 +1,5 @@
+from ._convert import convert_pcap_to_csv
+from ._capture import capture_live
+
+__all__ = ["convert_pcap_to_csv", "capture_live"]
+__version__ = "1.0.0"

netflower-1.0.0/netflower/_capture.py

@@ -0,0 +1,222 @@
+"""
+Live packet capture using system libpcap via ctypes.
+
+Usage:
+    from netflower import capture_live
+
+    handle = capture_live("eth0", on_flow=lambda flow: print(flow), idle_timeout=30)
+    handle.start()
+    # ... do work ...
+    handle.stop()
+"""
+
+import ctypes
+import threading
+
+from ._libpcap import (
+    PcapHandler_cb, PktHdr,
+    pcap_open_live, pcap_loop, pcap_breakloop, pcap_close,
+    pcap_dump_open, pcap_dump, pcap_dump_close,
+)
+from ._parser import _parse_packet
+from ._session import FlowSession
+from ._constants import FLOW_TIMEOUT
+
+
+class CallbackWriter:
+    """Writer that calls on_flow(dict) instead of writing to a CSV file."""
+
+    def __init__(self, on_flow: callable) -> None:
+        self._on_flow = on_flow
+
+    def write(self, flow_dict: dict) -> None:
+        self._on_flow(flow_dict)
+
+
+class _PcapSavingWriter:
+    """
+    Wraps on_flow callable. When a flow is emitted, also dumps its raw packets
+    to a .pcap file in pcap_dir before clearing the buffer.
+    """
+
+    def __init__(self, on_flow: callable, pcap_dir: str, pcap_handle) -> None:
+        self._on_flow = on_flow
+        self._pcap_dir = pcap_dir
+        self._pcap_handle = pcap_handle
+        # flow_key -> list of (PktHdr, bytes)
+        self._buffers: dict[tuple, list] = {}
+
+    def write(self, flow_dict: dict) -> None:
+        key = (
+            flow_dict.get("protocol"),
+            flow_dict.get("src_ip"),
+            flow_dict.get("dst_ip"),
+            flow_dict.get("src_port"),
+            flow_dict.get("dst_port"),
+        )
+        packets = self._buffers.pop(key, [])
+        if packets:
+            ts_us = int(flow_dict.get("timestamp", 0) * 1_000_000)
+            fname = (
+                f"{self._pcap_dir}/"
+                f"{flow_dict.get('src_ip')}-{flow_dict.get('src_port')}-"
+                f"{flow_dict.get('dst_ip')}-{flow_dict.get('dst_port')}-"
+                f"{flow_dict.get('protocol')}-{ts_us}.pcap"
+            ).encode()
+            dumper = pcap_dump_open(self._pcap_handle, fname)
+            if dumper:
+                for hdr, data in packets:
+                    pcap_dump(dumper, ctypes.byref(hdr), data)
+                pcap_dump_close(dumper)
+        self._on_flow(flow_dict)
+
+    def buffer_packet(self, key: tuple, hdr: PktHdr, data: bytes) -> None:
+        if key not in self._buffers:
+            self._buffers[key] = []
+        self._buffers[key].append((hdr, data))
+
+
+class CaptureHandle:
+    """
+    Handle returned by capture_live(). Controls start/stop of capture thread.
+
+    Parameters
+    ----------
+    interface:    Network interface name as bytes (e.g. b"eth0").
+    on_flow:      Callable receiving a flow dict when a flow completes.
+    idle_timeout: Seconds of inactivity before a flow is emitted.
+    flow_timeout: Absolute max flow duration before forced emit.
+    save_pcap:    If True, save raw packets of each completed flow to pcap_dir.
+    pcap_dir:     Directory path for .pcap files. Required when save_pcap=True.
+    """
+
+    _GC_INTERVAL = 1000
+
+    def __init__(
+        self,
+        interface: bytes,
+        on_flow: callable,
+        idle_timeout: float,
+        flow_timeout: float,
+        save_pcap: bool,
+        pcap_dir: str | None,
+    ) -> None:
+        if save_pcap and pcap_dir is None:
+            raise ValueError("pcap_dir is required when save_pcap=True")
+        self._interface = interface
+        self._on_flow = on_flow
+        self._idle_timeout = idle_timeout
+        self._flow_timeout = flow_timeout
+        self._save_pcap = save_pcap
+        self._pcap_dir = pcap_dir
+        self._handle = None
+        self._thread: threading.Thread | None = None
+        self._session = None
+        self._writer = None
+        self._callback = None
+
+    def start(self) -> None:
+        errbuf = ctypes.create_string_buffer(256)
+        self._handle = pcap_open_live(self._interface, 65535, 1, 1000, errbuf)
+        if not self._handle:
+            raise RuntimeError(f"pcap_open_live failed: {errbuf.value.decode()}")
+
+        if self._save_pcap:
+            self._writer = _PcapSavingWriter(self._on_flow, self._pcap_dir, self._handle)
+        else:
+            self._writer = CallbackWriter(self._on_flow)
+
+        self._session = FlowSession(self._writer, self._flow_timeout)
+        self._pkt_counter = 0
+
+        @PcapHandler_cb
+        def _callback(user, pkthdr_p, pkt_data):
+            hdr = pkthdr_p.contents
+            ts = hdr.ts_sec + hdr.ts_usec / 1_000_000
+            buf = ctypes.string_at(pkt_data, hdr.caplen)
+            parsed = _parse_packet(buf)
+            if parsed is None:
+                return
+            src_ip, dst_ip, src_port, dst_port, proto, pkt_len, hdr_len, pay_len, flags, win = parsed
+            if self._save_pcap:
+                fwd_key = (proto, src_ip, dst_ip, src_port, dst_port)
+                bwd_key = (proto, dst_ip, src_ip, dst_port, src_port)
+                key = fwd_key if (fwd_key in self._session._flows
+                                  or bwd_key not in self._session._flows) else bwd_key
+                self._writer.buffer_packet(key, hdr, buf)
+            self._session.process(ts, src_ip, dst_ip, src_port, dst_port,
+                                  proto, pkt_len, hdr_len, pay_len, flags, win)
+            self._pkt_counter += 1
+            if self._pkt_counter % self._GC_INTERVAL == 0:
+                self._session.gc(ts)
+
+        # Keep reference to prevent ctypes GC
+        self._callback = _callback
+
+        self._thread = threading.Thread(
+            target=pcap_loop,
+            args=(self._handle, -1, self._callback, None),
+            daemon=True,
+        )
+        self._thread.start()
+
+    def stop(self) -> None:
+        if self._handle:
+            pcap_breakloop(self._handle)
+        if self._thread:
+            self._thread.join(timeout=5)
+        if self._session:
+            self._session.flush_all()
+        if self._handle:
+            pcap_close(self._handle)
+            self._handle = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_):
+        self.stop()
+
+
+def capture_live(
+    interface: str,
+    on_flow: callable,
+    idle_timeout: float = 30.0,
+    flow_timeout: float = FLOW_TIMEOUT,
+    save_pcap: bool = False,
+    pcap_dir: str | None = None,
+) -> CaptureHandle:
+    """
+    Capture live packets from a network interface and emit completed flows.
+
+    Requires root or CAP_NET_RAW privilege.
+
+    Parameters
+    ----------
+    interface:    Network interface name (e.g. "eth0").
+    on_flow:      Called with a flow dict (82 fields) when a flow completes.
+                  Same column names as convert_pcap_to_csv output.
+    idle_timeout: Seconds of inactivity before a flow is emitted (default 30).
+    flow_timeout: Absolute max flow duration in seconds (default 120).
+    save_pcap:    Save raw packets of each completed flow to a .pcap file.
+    pcap_dir:     Directory for .pcap files. Required when save_pcap=True.
+
+    Returns
+    -------
+    CaptureHandle — call .start() to begin capture, .stop() to end.
+
+    Example
+    -------
+    >>> handle = capture_live("eth0", on_flow=lambda f: print(f["flow_duration"]))
+    >>> handle.start()
+    >>> import time; time.sleep(60)
+    >>> handle.stop()
+    """
+    return CaptureHandle(
+        interface=interface.encode() if isinstance(interface, str) else interface,
+        on_flow=on_flow,
+        idle_timeout=idle_timeout,
+        flow_timeout=flow_timeout,
+        save_pcap=save_pcap,
+        pcap_dir=pcap_dir,
+    )

netflower-1.0.0/netflower/_constants.py

@@ -0,0 +1,18 @@
+FLOW_TIMEOUT = 120.0    # seconds of inactivity before a flow is flushed
+ACTIVE_TIMEOUT = 5.0    # gap > this separates active from idle periods
+CLUMP_TIMEOUT = 1.0     # gap > this starts a new subflow
+BULK_BOUND = 4          # minimum packets required to register a bulk transfer
+CSV_BUFFER_ROWS = 500   # rows buffered in memory before writing to disk
+
+# TCP flag bitmasks (RFC 793 / RFC 3168)
+TCP_FIN = 0x01
+TCP_SYN = 0x02
+TCP_RST = 0x04
+TCP_PSH = 0x08
+TCP_ACK = 0x10
+TCP_URG = 0x20
+TCP_ECE = 0x40
+TCP_CWR = 0x80
+
+FORWARD = 0
+BACKWARD = 1