netflume 0.1.0__py3-none-any.whl

netflume/__init__.py

@@ -0,0 +1,109 @@
+"""NetFlow v5, NetFlow v9 and IPFIX collection and parsing, as a library.
+
+Built for programs rather than for terminals: a daemon writing flows to a
+database, a process forwarding a subset of fields over MQTT, an analyser
+replaying a capture. Standard library only, so it installs anywhere Python
+does and adds nothing to a deployment's dependency surface.
+
+Two layers, and either can be used without the other:
+
+**Parsing.** :func:`~netflume.parse.parse_message` and friends turn
+bytes into records. No socket, no clock the caller cannot control, no state
+beyond the :class:`~netflume.parse.TemplateStore` you hand in. Feed it
+from a pcap, a queue or a test fixture.
+
+**Collection.** :class:`Collector` binds a UDP socket and yields what arrives::
+
+    from netflume import Collector
+
+    with Collector(port=2055) as collector:
+        for flow in collector.flows():
+            print(flow.src_addr, flow.dst_addr, flow.octets)
+
+A flow is a plain dict of normalised keys, the same keys whichever version the
+exporter speaks, and :class:`~netflume.flow.Flow` is an opt-in typed view over
+that dict which resolves the field aliases and keeps the dict attached. The
+README documents both shapes.
+
+Nothing here prints. Sampling rates, export gaps and undecodable datagrams are
+:mod:`~netflume.events` objects from
+:meth:`~netflume.decoder.Decoder.take_events`, and are also written to
+the ``netflume`` logger for callers who want no more than that. As a
+library should, this package adds nothing but a NullHandler; configure a real
+handler, or records go nowhere.
+
+Hostname resolution is not here. It was an optional module that nothing else
+in this package imported, and it now lives on its own as ``lanname``: reverse
+DNS, mDNS and NetBIOS behind a cache, non-blocking, and off until a mode is
+chosen. Install it alongside if a consumer wants names. Nothing here needs it.
+"""
+
+import logging
+
+from .collector import DEFAULT_PORT, DEFAULT_RCVBUF, Collector
+from .decoder import Decoder, Message
+from .events import DecodeError, ExportGap, SamplingChange
+from .flow import METADATA_KEYS, MODELLED_FIELDS, Flow
+from .ie import FLOW_END_REASON, IE, PROTO_NAMES, TCP_FLAG_BITS
+from .parse import (
+                    IPFIX_HDR,
+                    SUPPORTED_VERSIONS,
+                    V5_HDR,
+                    V5_REC,
+                    V9_HDR,
+                    TemplateStore,
+                    decode_value,
+                    flow_duration,
+                    flow_endpoints,
+                    flow_timestamp,
+                    parse_data_record,
+                    parse_message,
+                    parse_v5,
+                    parse_v9_or_ipfix,
+                    read_template_fields,
+                    record_min_length,
+)
+from .sampling import SamplingWatch, sampling_rate
+from .sequence import (
+                    MAX_PLAUSIBLE_GAP,
+                    MAX_REORDER,
+                    RESYNC_AFTER,
+                    SEQ_MODULUS,
+                    SequenceWatch,
+)
+from .values import (
+                    ADDR_KINDS,
+                    addr_kind,
+                    flow_end_reason_name,
+                    proto_name,
+                    service_name,
+                    tcp_flags_str,
+)
+
+__version__ = "0.1.0"
+
+# A library that logs to an unconfigured root logger prints to stderr, which is
+# exactly what this package exists not to do.
+logging.getLogger(__name__).addHandler(logging.NullHandler())
+
+__all__ = [
+    # collection
+    "Collector", "DEFAULT_PORT", "DEFAULT_RCVBUF", "Decoder", "Message",
+    # the flow shape
+    "Flow", "MODELLED_FIELDS", "METADATA_KEYS",
+    # parsing
+    "TemplateStore", "parse_message", "parse_v5", "parse_v9_or_ipfix",
+    "parse_data_record", "read_template_fields", "record_min_length",
+    "decode_value", "flow_endpoints", "flow_timestamp", "flow_duration",
+    "SUPPORTED_VERSIONS", "V5_HDR", "V5_REC", "V9_HDR", "IPFIX_HDR",
+    # what the exporter says about itself
+    "SamplingWatch", "sampling_rate", "SequenceWatch", "SEQ_MODULUS",
+    "MAX_PLAUSIBLE_GAP", "MAX_REORDER", "RESYNC_AFTER",
+    # events
+    "ExportGap", "SamplingChange", "DecodeError",
+    # field level helpers
+    "addr_kind", "ADDR_KINDS", "service_name", "proto_name", "tcp_flags_str",
+    "flow_end_reason_name", "IE", "PROTO_NAMES", "FLOW_END_REASON",
+    "TCP_FLAG_BITS",
+    "__version__",
+]

netflume/collector.py

@@ -0,0 +1,285 @@
+"""Binding the socket and handing back flows.
+
+:class:`Collector` is a UDP socket, a :class:`~netflume.decoder.Decoder`
+and three ways to get at what arrives:
+
+* ``for rec, hdr in collector:`` gives plain dicts, the cheapest form
+* ``for flow in collector.flows():`` gives typed objects, built on demand
+* ``message = collector.poll(timeout)`` gives one datagram, or None, for a
+  caller who owns the event loop
+
+It prints nothing, exits nothing, and reads no arguments. Everything it would
+otherwise have to say arrives as an event from
+:meth:`~netflume.decoder.Decoder.take_events` or on the
+``netflume`` logger.
+"""
+
+import errno
+import logging
+import selectors
+import socket
+import time
+
+from .decoder import Decoder
+
+__all__ = ["Collector", "DEFAULT_PORT", "DEFAULT_RCVBUF"]
+
+log = logging.getLogger(__name__)
+
+#: The port every NetFlow exporter defaults to. Not registered to anything and
+#: not privileged, so a daemon need not run as root.
+DEFAULT_PORT = 2055
+
+#: 4 MB of kernel receive buffer. UDP has no backpressure: a buffer that fills
+#: while this process is busy drops datagrams silently and the only trace is a
+#: sequence gap. Ask for more than the system default, and do not fail if the
+#: system says no.
+DEFAULT_RCVBUF = 4 * 1024 * 1024
+
+#: The largest datagram to accept. A NetFlow export never approaches this;
+#: it is simply the largest a UDP payload can be.
+MAX_DATAGRAM = 65535
+
+
+class Collector:
+    """A bound UDP socket that yields decoded NetFlow.
+
+    ::
+
+        with Collector(port=2055) as collector:
+            for rec, hdr in collector:
+                store(rec, hdr)
+
+    The socket is bound in the constructor, so a port already in use raises
+    OSError there rather than at first read. Use ``with``, or call
+    :meth:`close`, so the socket is released promptly; the alternative is a
+    port left held until the garbage collector gets round to it.
+
+    Iteration is endless by design: a collector with nothing to report is a
+    quiet network, not a finished job. Break out of the loop, or call
+    :meth:`stop` from a signal handler or another thread.
+    """
+
+    def __init__(self, port=DEFAULT_PORT, bind="0.0.0.0", decoder=None,
+                 timeout=1.0, rcvbuf=DEFAULT_RCVBUF, reuse_address=True,
+                 sock=None):
+        """Bind and get ready to receive.
+
+        port, bind    where to listen. "0.0.0.0" is every IPv4 interface; name
+                      one interface if the machine is multi-homed and only one
+                      of them faces the exporters.
+        decoder       an existing Decoder, if you want to share template state
+                      with something else or to have turned tracking off.
+        timeout       how long the blocking reads inside iteration wait before
+                      coming up for air. It bounds how long :meth:`stop` takes
+                      to be noticed; it is not a deadline on receiving.
+        rcvbuf        kernel receive buffer to request, or None to leave the
+                      system default alone.
+        reuse_address set SO_REUSEADDR before binding. Default True, which lets
+                      a restart of this process take the port back immediately
+                      instead of waiting for the old socket to clear. The cost
+                      is that on UDP it also lets a *second* process bind the
+                      same port, and then only one of them receives: a
+                      collector that looks bound and healthy while another
+                      process quietly takes its traffic. Pass False to make
+                      that clash raise OSError here instead of going silent.
+        sock          an already-bound socket to use instead of making one,
+                      which is how a test drives this without a real exporter.
+        """
+        self.decoder = decoder if decoder is not None else Decoder()
+        self.timeout = timeout
+        self._closed = False
+        self._stopping = False
+
+        if sock is None:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            if reuse_address:
+                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            if rcvbuf:
+                try:
+                    sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, rcvbuf)
+                except OSError as exc:
+                    log.debug("could not set SO_RCVBUF to %d: %s", rcvbuf, exc)
+            try:
+                sock.bind((bind, port))
+            except OSError:
+                sock.close()
+                raise
+        self.socket = sock
+        self.socket.setblocking(False)
+
+        # A socketpair rather than a pipe: select() on Windows takes sockets
+        # and nothing else, and stop() has to be able to interrupt a wait from
+        # another thread or a signal handler.
+        self._waker_r, self._waker_w = socket.socketpair()
+        self._waker_r.setblocking(False)
+        self._selector = selectors.DefaultSelector()
+        self._selector.register(self.socket, selectors.EVENT_READ)
+        self._selector.register(self._waker_r, selectors.EVENT_READ)
+
+        log.info("listening for NetFlow/IPFIX on %s", self.address)
+
+    # == properties ==========================================================
+
+    @property
+    def address(self):
+        """The (host, port) actually bound, which matters when port was 0."""
+        return self.socket.getsockname()
+
+    @property
+    def stats(self):
+        """The decoder's running counters. See the README for the keys."""
+        return self.decoder.stats
+
+    def fileno(self):
+        """The socket's descriptor, so this can go straight into a caller's
+        own ``selectors`` or ``asyncio`` loop. Pair it with
+        ``poll(timeout=0)``, which never blocks."""
+        return self.socket.fileno()
+
+    # == receiving ===========================================================
+
+    def poll(self, timeout=0):
+        """Wait up to `timeout` seconds for one datagram and decode it.
+
+        Returns a :class:`~netflume.decoder.Message`, or None if
+        nothing arrived in time or what arrived would not decode. The two are
+        deliberately not distinguished here: neither is an error, and a caller
+        who cares can read :meth:`~netflume.decoder.Decoder.take_events`.
+
+        ``timeout=0`` polls without blocking, ``timeout=None`` waits
+        indefinitely.
+
+        Raises ValueError once :meth:`close` has been called. Iteration and
+        :meth:`flows` stop quietly instead, because a loop that has been shut
+        down is finishing normally, whereas asking a closed collector for one
+        more datagram is a mistake worth hearing about.
+        """
+        if self._closed:
+            raise ValueError("collector is closed")
+        data, addr = self._receive(timeout, honour_stop=False)
+        if data is None:
+            return None
+        return self.decoder.decode(data, addr[0])
+
+    def messages(self, timeout=None):
+        """Yield one :class:`~netflume.decoder.Message` per datagram.
+
+        Endless, and skips datagrams that do not decode. Use this rather than
+        :meth:`__iter__` when the sequence number, the export time or the
+        option records matter, since those belong to a message, not a flow.
+        """
+        if timeout is None:
+            timeout = self.timeout
+        while not self._stopping and not self._closed:
+            data, addr = self._receive(timeout)
+            if data is None:
+                continue
+            message = self.decoder.decode(data, addr[0])
+            if message is not None:
+                yield message
+
+    def __iter__(self):
+        """Yield (record, header) for every flow, as plain dicts.
+
+        The header comes along because a v5 or v9 record cannot be timestamped
+        without it. This is the cheapest shape: no objects are built beyond
+        what the parser already made.
+        """
+        for message in self.messages():
+            header = message.header
+            for rec in message.flows:
+                yield rec, header
+
+    def flows(self, now=None):
+        """Yield a :class:`~netflume.flow.Flow` for every flow.
+
+        The typed shape: aliases resolved, timestamp derived, sampling rate
+        attached, and the original dict still on ``flow.raw``.
+        """
+        for message in self.messages():
+            for flow in message.typed_flows(now=now):
+                yield flow
+
+    # == shutting down =======================================================
+
+    def stop(self):
+        """Ask the iterators to finish. Safe from a signal handler or thread.
+
+        Returns immediately; the loop stops after at most one `timeout`. The
+        socket is left open, so a stopped collector can be read again with
+        :meth:`poll`. Call :meth:`close` to release it.
+        """
+        self._stopping = True
+        try:
+            self._waker_w.send(b"\x00")
+        except OSError:
+            pass
+
+    def close(self):
+        """Release the socket. Idempotent, and implied by leaving a ``with``."""
+        if self._closed:
+            return
+        self._closed = True
+        self._stopping = True
+        self._selector.close()
+        for handle in (self.socket, self._waker_r, self._waker_w):
+            try:
+                handle.close()
+            except OSError:
+                pass
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        self.close()
+        return False
+
+    # == the one place that touches the socket ===============================
+
+    def _receive(self, timeout, honour_stop=True):
+        """(data, addr), or (None, None) if nothing was ready in time.
+
+        `honour_stop` is what separates the two callers. An iterator wants a
+        stop() to end its wait immediately, however long the timeout was.
+        poll() is a single explicit read and answers the question it was
+        asked, stopped or not.
+        """
+        deadline = None if timeout is None else time.monotonic() + timeout
+        while True:
+            remaining = (None if deadline is None
+                         else max(0.0, deadline - time.monotonic()))
+            ready = self._selector.select(remaining)
+            if not ready:
+                return None, None
+
+            woken = False
+            for key, _events in ready:
+                if key.fileobj is self._waker_r:
+                    woken = True
+                    try:
+                        self._waker_r.recv(4096)   # drain, it carries no data
+                    except OSError:
+                        pass
+                    continue
+                try:
+                    return self.socket.recvfrom(MAX_DATAGRAM)
+                except OSError as exc:
+                    if exc.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
+                        return None, None
+                    # WSAECONNRESET: Windows reports an ICMP port-unreachable
+                    # caused by an earlier send as an error on the *next* read
+                    # of a UDP socket. Nothing is wrong with this socket.
+                    if getattr(exc, "winerror", None) == 10054:
+                        log.debug("ignoring a stale ICMP error on the socket")
+                        return None, None
+                    raise
+
+            # Only the wakeup fired. Go round again with whatever time is left,
+            # rather than reporting nothing: stop() must not cost the caller a
+            # datagram that was already queued behind it.
+            if not woken or (honour_stop and self._stopping):
+                return None, None
+            if deadline is not None and time.monotonic() >= deadline:
+                return None, None

netflume/decoder.py

@@ -0,0 +1,238 @@
+"""Turning datagrams into messages, and remembering what that implies.
+
+:class:`Decoder` is the parsing layer with its state attached: the template
+store that has to outlive a datagram, the sequence watch that can only find a
+gap by comparing one message with the last, and the sampling watch that has to
+remember what an exporter said an hour ago. It has no socket in it, so a caller
+holding bytes from a pcap, a queue or a test fixture gets the same answers a
+live collector would.
+"""
+
+import logging
+import struct
+from collections import Counter, deque
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Mapping
+
+from .events import DecodeError
+from .flow import Flow
+from .parse import SUPPORTED_VERSIONS, TemplateStore, parse_v5, parse_v9_or_ipfix
+from .sampling import SamplingWatch
+from .sequence import SequenceWatch
+
+__all__ = ["Decoder", "MAX_PENDING_EVENTS", "Message"]
+
+log = logging.getLogger(__name__)
+
+#: Events held for :meth:`Decoder.take_events` before the oldest are dropped.
+#: A collector on an open UDP port receives things that are not NetFlow, such
+#: as scanners, sFlow on the same port, or a misconfigured exporter, and each
+#: one raises a DecodeError. A caller that never drains would otherwise
+#: accumulate one per bad datagram for as long as the process runs. Drops are
+#: counted in ``stats["events_dropped"]`` so that the loss is visible rather
+#: than silent.
+MAX_PENDING_EVENTS = 10000
+
+
+@dataclass
+class Message:
+    """One export datagram, decoded.
+
+    The message is the unit that arrives on the wire, and some things are only
+    true of a message rather than of a flow: the sequence number, the export
+    time, and whether anything was lost getting here.
+    """
+
+    #: The header dict: version, exporter, domain, sequence, unix_secs,
+    #: sys_uptime. Needed to timestamp a v5 or v9 flow, so keep it with the
+    #: records rather than discarding it.
+    header: Mapping[str, Any]
+    #: Flow records, as plain dicts of normalised keys.
+    flows: List[Dict[str, Any]] = field(default_factory=list)
+    #: Option records: what the exporter says about itself, not about traffic.
+    #: Sampling rates are read out of these automatically; interface names and
+    #: the like are left for the caller to use or ignore.
+    options: List[Dict[str, Any]] = field(default_factory=list)
+    #: Exports the sequence counter says never arrived before this message. 0
+    #: is the normal answer, and is also the answer while the stream is still
+    #: being learned. See :class:`~netflume.sequence.SequenceWatch`.
+    gap: int = 0
+    #: The 1-in-N rate in force for this exporter, 1 if unsampled or unstated.
+    sampling_rate: int = 1
+
+    @property
+    def exporter(self):
+        return self.header.get("exporter")
+
+    @property
+    def version(self):
+        return self.header.get("version")
+
+    @property
+    def sequence(self):
+        return self.header.get("sequence")
+
+    def typed_flows(self, now=None):
+        """The flow records as :class:`~netflume.flow.Flow` objects.
+
+        Built on demand, not up front: a consumer that wants two fields out of
+        a dict should not pay to construct thirty.
+        """
+        return [Flow.from_record(rec, self.header, self.sampling_rate, now=now)
+                for rec in self.flows]
+
+    def __len__(self):
+        return len(self.flows)
+
+    def __bool__(self):
+        """Always True. A Message exists or it is None; there is no empty one.
+
+        Without this, ``__len__`` makes a message carrying only template sets,
+        only option records, or only a sequence gap evaluate false, and the
+        natural ``if message:`` after :meth:`~netflume.collector.Collector.poll`
+        silently discards all three. Ask ``if message is not None`` to test for
+        a decode, and ``len(message)`` or ``message.flows`` for traffic.
+        """
+        return True
+
+
+class Decoder:
+    """Stateful decoding of one exporter's datagrams, or of many.
+
+    Templates, sequence tracking and sampling rates are all keyed by
+    (exporter, observation domain) inside, so one Decoder can serve every device
+    sending to a collector, and a device running several domains keeps them
+    apart. It is not thread safe; give each thread its own, or put one behind a
+    queue.
+
+    The per-exporter tables have ceilings, namely MAX_TEMPLATES, MAX_STREAMS
+    and MAX_SAMPLING_STREAMS, because a UDP source address is whatever the
+    sender typed, and anything keyed by one and never evicted is a memory leak
+    that anyone able to reach the socket can pull on.
+    """
+
+    def __init__(self, track_sequence=True, track_sampling=True):
+        self.templates = TemplateStore()
+        self.stats = Counter()
+        self.sequence = SequenceWatch() if track_sequence else None
+        self.sampling = SamplingWatch() if track_sampling else None
+        self._events = deque()
+
+    def decode(self, data, exporter):
+        """Decode one datagram. Returns a :class:`Message`, or None.
+
+        None means nothing usable was in it: too short, a version this package
+        does not decode, or a body that would not parse. Every one of those is
+        counted in :attr:`stats` and queued as a
+        :class:`~netflume.events.DecodeError`; none of them raises,
+        because a decoder that dies on one bad packet is useless on a real
+        network.
+
+        A Message with no flows in it is not a failure. Template sets, option
+        records and data sets whose template has not arrived yet all produce
+        one, and an exporter's first few datagrams are routinely all template.
+        """
+        self.stats["packets"] += 1
+        self.stats["bytes_rx"] += len(data)
+
+        if len(data) < 2:
+            return self._fail(exporter, "short", "fewer than two bytes")
+
+        version = struct.unpack_from("!H", data, 0)[0]
+        if version not in SUPPORTED_VERSIONS:
+            self.stats["unsupported_version"] += 1
+            return self._fail(exporter, "unsupported",
+                              f"version {version}", counted=True)
+
+        try:
+            if version == 5:
+                hdr, records, opts = parse_v5(data, exporter)
+            else:
+                hdr, records, opts = parse_v9_or_ipfix(
+                    data, exporter, self.templates, self.stats)
+        except Exception as exc:      # keep listening even on a bad datagram
+            self.stats["parse_errors"] += 1
+            return self._fail(exporter, "malformed",
+                              f"{type(exc).__name__}: {exc}", counted=True)
+
+        if hdr is None:
+            return self._fail(exporter, "short", "truncated header")
+
+        self.stats[f"v{version}_msgs"] += 1
+        self.stats["flows"] += len(records)
+        self.stats["option_records"] += len(opts)
+
+        gap = 0
+        if self.sequence is not None:
+            gap = self.sequence.observe(exporter, hdr.get("domain"), version,
+                                        hdr.get("sequence"), len(records),
+                                        len(opts))
+            if gap:
+                self.stats["missed_exports"] += gap
+            self._queue(self.sequence.take_events())
+
+        rate = 1
+        if self.sampling is not None:
+            domain = hdr.get("domain")
+            for opt in opts:
+                self.sampling.note(exporter, domain, opt)
+            self._queue(self.sampling.take_events())
+            rate = self.sampling.rate_for(exporter, domain)
+
+        return Message(header=hdr, flows=records, options=opts, gap=gap,
+                       sampling_rate=rate)
+
+    def flows(self, data, exporter, typed=False, now=None):
+        """Decode a datagram straight to its flows, discarding the rest.
+
+        The convenience form for a caller who has bytes and wants records. The
+        header is still needed to timestamp them, so `typed=True` is the shape
+        to use if you want start times; the dict form leaves that to you.
+        """
+        message = self.decode(data, exporter)
+        if message is None:
+            return []
+        return message.typed_flows(now=now) if typed else message.flows
+
+    # == what the caller may want to know ====================================
+
+    def _queue(self, events):
+        """Add events to the pending queue, oldest dropped past the ceiling."""
+        for event in events:
+            if len(self._events) >= MAX_PENDING_EVENTS:
+                self._events.popleft()
+                self.stats["events_dropped"] += 1
+            self._events.append(event)
+
+    def take_events(self):
+        """Hand over everything raised since the last call, and forget it.
+
+        A list of :class:`~netflume.events.ExportGap`,
+        :class:`~netflume.events.SamplingChange` and
+        :class:`~netflume.events.DecodeError`. Empty is the normal
+        answer on a healthy network, so this is cheap to call in a loop.
+        """
+        events = list(self._events)
+        self._events.clear()
+        return events
+
+    def sampling_rate(self, exporter, domain=None):
+        """The 1-in-N rate last advertised, or 1.
+
+        Rates are scoped to an observation domain, so name one. Omitting it
+        answers for the exporter as a whole, and only when its domains agree.
+        See :meth:`~netflume.sampling.SamplingWatch.rate_for`.
+        """
+        return self.sampling.rate_for(exporter, domain) if self.sampling else 1
+
+    def export_gaps(self):
+        """Cumulative loss per stream, as ExportGap records, worst first."""
+        return self.sequence.gaps() if self.sequence else []
+
+    def _fail(self, exporter, reason, detail, counted=False):
+        if not counted:
+            self.stats["malformed"] += 1
+        error = DecodeError(exporter, reason, detail)
+        self._queue((error,))
+        log.debug("%s", error)
+        return None