fluidattacks-agent 0.1.2__py3-none-any.whl

fluidattacks_agent/observer.py

@@ -0,0 +1,137 @@
+"""Watch what a workload imports, without taking part in importing it."""
+
+import sys
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from types import ModuleType
+from typing import Protocol
+
+from fluidattacks_agent.distributions import Dist, distribution_of
+from fluidattacks_agent.report import Ecosystem, Evidence, Granularity, Record, keyed
+
+# the probe rides inside a workload we do not own, so its bookkeeping is
+# bounded the same way the reader's is
+MAX_MODULES = 4096
+
+# our own package would otherwise land in the workload's inventory
+OURS = "fluidattacks_agent"
+
+
+@dataclass
+class ImportObserver:
+    """
+    A meta path finder that claims nothing and only takes note.
+
+    The import system asks for nothing but ``find_spec``, so there is no base
+    class to derive from, and returning ``None`` from every lookup leaves the
+    finders behind it to decide the outcome.
+    """
+
+    # how many times each module was looked up, by its whole dotted name, so a
+    # record can name the module that was seen and not only the package holding
+    # it. The name is what bounds the ceiling below
+    seen: dict[str, int] = field(default_factory=dict)
+    moved: set[str] = field(default_factory=set)
+    # modules seen past the ceiling, so the loss is never silent
+    suppressed: int = 0
+
+    def note(self, fullname: str) -> str | None:
+        """Take note of a module looked up, naming it whenever there is room."""
+        if fullname.split(".", maxsplit=1)[0] == OURS:
+            return None
+        if fullname in self.seen:
+            self.seen[fullname] += 1
+            self.moved.add(fullname)
+            return fullname
+        # the ceiling bounds how many distinct names are held, not how often a
+        # name already held is counted
+        if len(self.seen) >= MAX_MODULES:
+            self.suppressed += 1
+            return None
+        self.seen[fullname] = 1
+        self.moved.add(fullname)
+        return fullname
+
+    def find_spec(
+        self,
+        fullname: str,
+        _path: object = None,
+        _target: ModuleType | None = None,
+    ) -> None:
+        """Note the module being looked up, then defer to the finders behind."""
+        self.note(fullname)
+
+    def records(
+        self,
+        modules: dict[str, ModuleType],
+        installed: dict[str, Dist],
+        carried: Mapping[tuple[str, str], int],
+    ) -> list[Record]:
+        """
+        Name the distribution behind every module that finished importing.
+
+        A lookup is only an attempt: an import that raised never reaches
+        ``modules``, and reporting it would claim a package loaded when it did
+        not. Intersecting the two is what makes this evidence rather than
+        intent.
+
+        Both maps are taken by the caller so this stays a pure fold, and so the
+        distribution scan happens once per report instead of once per module.
+        """
+        found: list[Record] = []
+        scanned = bool(installed)
+        for name in sorted(self.moved):
+            times = self.seen[name]
+            if times <= carried.get(keyed(Evidence.IMPORTED, name), 0):
+                self.moved.discard(name)
+                continue
+            module = modules.get(name)
+            if module is None:
+                # kept, because a lookup mid-import finishes after this flush
+                continue
+            dist = distribution_of(name, installed)
+            if dist is None:
+                if scanned:
+                    self.moved.discard(name)
+                continue
+            found.append(
+                Record(
+                    ecosystem=Ecosystem.PYPI,
+                    name=dist.name,
+                    version=dist.version,
+                    confidence=dist.confidence,
+                    symbol=name,
+                    granularity=granularity_of(module),
+                    evidence=Evidence.IMPORTED,
+                    frequency=times,
+                ),
+            )
+        return found
+
+
+def granularity_of(module: ModuleType) -> Granularity:
+    """Say whether a module that imported is a package or a module inside one."""
+    # a package is the one that carries the path it searches for its own
+    # submodules, so this is read off the module rather than guessed at the name
+    return Granularity.PACKAGE if hasattr(module, "__path__") else Granularity.MODULE
+
+
+class Finder(Protocol):
+    def find_spec(
+        self,
+        fullname: str,
+        path: object = None,
+        target: ModuleType | None = None,
+        /,
+    ) -> None: ...
+
+
+def install(finder: Finder) -> None:
+    """Put the finder ahead of the ones that do the real work."""
+    sys.meta_path.insert(0, finder)
+
+
+def uninstall(finder: Finder) -> None:
+    """Take the finder back out, leaving the chain as it was found."""
+    while finder in sys.meta_path:
+        sys.meta_path.remove(finder)

fluidattacks_agent/outbox.py

@@ -0,0 +1,64 @@
+"""Hold what has not travelled yet, bounded by what holding it costs."""
+
+import threading
+from collections import deque
+from dataclasses import dataclass, field
+from typing import Final
+
+from fluidattacks_agent.batch import Batch
+
+# what evidence waiting to travel may cost the process it waits inside. Bodies
+# arrive packed, so the same ceiling holds about three and a half times the
+# report text it would have held unpacked, and a report is capped at a mebibyte
+# before it is packed at all
+MAX_QUEUED: Final = 32
+MAX_QUEUED_BYTES: Final = 2 << 20
+
+
+@dataclass
+class Outbox:
+    """Batches waiting to travel, bounded by how many and by how much."""
+
+    limit: int = MAX_QUEUED
+    ceiling: int = MAX_QUEUED_BYTES
+    # a repr of this would spell every byte of every report still held, and a
+    # host that walks its objects for a log line is the reason there is a probe
+    held: deque[Batch] = field(default_factory=deque, repr=False)
+    weight: int = 0
+    # the workload's threads put and delivery takes, so the queue and what it
+    # weighs have to move as one step or the ceiling is not a ceiling
+    guard: threading.Lock = field(default_factory=threading.Lock, repr=False)
+
+    def put(self, batch: Batch) -> bool:
+        """Take a batch if it fits, and say plainly whether it did."""
+        with self.guard:
+            if len(self.held) >= self.limit:
+                return False
+            if self.weight + len(batch.body) > self.ceiling:
+                return False
+            self.held.append(batch)
+            self.weight += len(batch.body)
+            return True
+
+    def take(self) -> Batch | None:
+        """Give up the batch that has waited longest, or nothing if none has."""
+        with self.guard:
+            if not self.held:
+                return None
+            batch = self.held.popleft()
+            self.weight -= len(batch.body)
+            return batch
+
+    def forked(self) -> None:
+        """Leave a child able to put, holding none of its parent's batches."""
+        # a lock held by the thread draining this at the moment of the fork is
+        # held in the child forever: measured, a child inheriting one blocked on
+        # its first put, and a put here happens inside a host callback. A fresh
+        # one is unheld by construction, and the child has no second thread for
+        # it to be held against
+        self.guard = threading.Lock()
+        # not a loss to count: these are keyed to the parent's instance, the
+        # parent still holds the same bytes under the same key, and saying
+        # evidence was lost here would be saying so of evidence in flight
+        self.held.clear()
+        self.weight = 0

fluidattacks_agent/patience.py

@@ -0,0 +1,38 @@
+"""Say how long to wait before asking a far end again, and how long at first."""
+
+import os
+from typing import Final
+
+# the shortest and the longest a refused attempt is waited out
+FLOOR: Final = 1.0
+CEILING: Final = 300.0
+
+# 1.0 << 9 is already past the ceiling, so nothing above this is computed
+DOUBLINGS: Final = 9
+
+# where a refusal that is about nothing of ours starts, rather than climbing
+TOPMOST: Final = DOUBLINGS + 1
+
+HALF: Final = 0.5
+
+DRAWN: Final = 2
+
+
+def fraction() -> float:
+    """Draw a share of a window out of the kernel's own entropy."""
+    # os is already imported; random would cost 1.3 ms of every start
+    return int.from_bytes(os.urandom(DRAWN), "big") / (1 << (8 * DRAWN))
+
+
+def standoff(tries: int, share: float, ceiling: float = CEILING) -> float:
+    """Give the wait a refused attempt earns, out of a window that doubles."""
+    if tries <= 0:
+        return 0.0
+    widest = min(ceiling, FLOOR * (1 << min(tries - 1, DOUBLINGS)))
+    # half the window is drawn, which is what keeps replicas apart
+    return widest * (HALF + share * HALF)
+
+
+def splay(interval: float, share: float) -> float:
+    """Move a first contact off the second every other replica picked."""
+    return interval * share

fluidattacks_agent/post.py

@@ -0,0 +1,175 @@
+"""Say what travels beside a batch, hand it over, and read what came back."""
+
+import contextlib
+import enum
+from typing import TYPE_CHECKING, Final, Protocol
+
+if TYPE_CHECKING:
+    import ssl
+    from collections.abc import Callable
+
+from fluidattacks_agent.batch import BATCH_TAG, Batch
+from fluidattacks_agent.settings import Delivery
+
+# the body is signed as the bytes it is, so nothing may re-encode it on the
+# way. Content-Encoding invites a proxy to normalise exactly that, and one that
+# did would break every signature without touching a byte of the evidence
+CONTENT_TYPE: Final = "application/gzip"
+
+# so that a customer's proxy has something to allow which is not a bare
+# address, and a constant, so nothing of the workload's is spelt to reach it
+AGENT: Final = "fluidattacks-agent/1"
+
+# a socket that never answers must not become a probe that never delivers
+TIMEOUT: Final = 10.0
+
+# read only far enough to let the socket close
+MAX_ANSWER: Final = 4096
+
+PORT: Final = 443
+
+FORMAT: Final = "X-Watches-Format"
+GROUP: Final = "X-Watches-Group"
+WORKLOAD: Final = "X-Watches-Workload"
+INSTANCE: Final = "X-Watches-Instance"
+SEQ: Final = "X-Watches-Seq"
+STREAM: Final = "X-Watches-Stream"
+SIGNATURE: Final = "X-Watches-Signature"
+
+# asked to wait, or asked too early: the answers that are about the moment
+AGAIN: Final = frozenset({408, 425, 429})
+
+LOWEST_GOOD: Final = 200
+FIRST_BAD: Final = 300
+FIRST_ERROR: Final = 500
+
+
+class Answer(Protocol):
+    """What a far end sent back, of which nothing but the status is read."""
+
+    status: int
+
+    def read(self, amount: int, /) -> bytes: ...
+
+
+class Connection(Protocol):
+    """A connection to the endpoint, open and ours to close."""
+
+    def request(
+        self,
+        method: str,
+        path: str,
+        /,
+        *,
+        body: bytes,
+        headers: dict[str, str],
+    ) -> None: ...
+
+    def getresponse(self) -> Answer: ...
+
+    def close(self) -> None: ...
+
+
+class Verdict(enum.Enum):
+    """What became of a batch handed over, and nothing of what it held."""
+
+    DELIVERED = "delivered"
+    # this same batch again, later
+    RETRY = "retry"
+    # nothing of ours is being taken at all, so stop offering
+    STOP = "stop"
+
+
+def judged(status: int) -> Verdict:
+    """Say what an answer means for the batch that earned it."""
+    if LOWEST_GOOD <= status < FIRST_BAD:
+        return Verdict.DELIVERED
+    if status in AGAIN or status >= FIRST_ERROR:
+        return Verdict.RETRY
+    # everything else is the endpoint saying no to us rather than to this
+    # batch: a redirect it wants followed, a credential it will not take, a
+    # path that is not there. No answer makes this destroy a report, because a
+    # blanket refusal cannot be told from one meant for a single batch, and
+    # stopping costs nothing: a full outbox refuses a flush before that flush
+    # charges the records it rendered, so they are offered again
+    return Verdict.STOP
+
+
+def headers(batch: Batch) -> dict[str, str]:
+    """Name everything the far end needs to rebuild what was signed."""
+    # the credential is in none of these. What proves the sender is the proof
+    # over the batch, and a bearer would be readable by everything in between
+    return {
+        "Content-Type": CONTENT_TYPE,
+        "User-Agent": AGENT,
+        FORMAT: BATCH_TAG,
+        GROUP: batch.group,
+        WORKLOAD: batch.workload,
+        INSTANCE: batch.instance,
+        SEQ: str(batch.seq),
+        STREAM: batch.stream,
+        SIGNATURE: batch.signature,
+    }
+
+
+def targeted(path: str, query: str) -> str:
+    """Name the resource to ask for, keeping what the endpoint routes by."""
+    # a query is the endpoint's and a fragment is the client's own
+    return f"{path or '/'}?{query}" if query else path or "/"
+
+
+def proven() -> "ssl.SSLContext":
+    """Give a context the far end has to prove itself on, hostname included."""
+    import ssl  # noqa: PLC0415
+
+    return ssl.create_default_context()
+
+
+def opened(host: str, port: int, bound: float) -> Connection:
+    """Open a bounded connection to a far end that proved who it is."""
+    # fourteen milliseconds of imports, paid on the thread that delivers
+    import http.client  # noqa: PLC0415
+
+    return http.client.HTTPSConnection(host, port, timeout=bound, context=proven())
+
+
+def posted(
+    delivery: Delivery,
+    batch: Batch,
+    bound: float = TIMEOUT,
+    connect: "Callable[[str, int, float], Connection]" = opened,
+) -> Verdict:
+    """Offer one batch to the endpoint, and never raise at the thread."""
+    # bounded by whoever offers, since an exit has less of a workload's time
+    # to spend than a carrier at rest does
+    from http.client import HTTPException  # noqa: PLC0415
+    from urllib.parse import urlsplit  # noqa: PLC0415
+
+    try:
+        split = urlsplit(delivery.endpoint)
+        host = split.hostname
+        if split.scheme != "https" or not host:
+            return Verdict.STOP
+        held = connect(host, split.port or PORT, bound)
+    except (ValueError, OSError):
+        # a setting a later attempt resolves no differently
+        return Verdict.STOP
+    try:
+        # nothing follows a redirect: where a batch goes is not the far end's
+        held.request(
+            "POST",
+            targeted(split.path, split.query),
+            body=batch.body,
+            headers=headers(batch),
+        )
+        answer = held.getresponse()
+        answer.read(MAX_ANSWER)
+        return judged(answer.status)
+    except (OSError, HTTPException):
+        return Verdict.RETRY
+    except ValueError:
+        # a header the client will not encode: not transient either way
+        return Verdict.STOP
+    finally:
+        with contextlib.suppress(Exception):
+            held.close()

fluidattacks_agent/report.py

@@ -0,0 +1,286 @@
+"""
+Serialise frequency into the watches-exec report the agent reads.
+
+The bounds mirror ``watches/crates/domain/src/exec.rs``. A record the reader
+would refuse is dropped here and counted, so the reader's refusal counter stays
+a signal about the wire rather than about us.
+"""
+
+import re
+from dataclasses import dataclass
+from enum import Enum
+from typing import Final
+
+REPORT_TAG: Final = "watches-exec/1"
+ABSENT: Final = "-"
+
+MAX_LINE: Final = 4096
+MAX_REPORT: Final = 1 << 20
+ROOM: Final = 256
+MAX_NAME: Final = 214
+MAX_VERSION: Final = 64
+MAX_SYMBOL: Final = 1024
+MAX_ATTRIBUTE: Final = 64
+
+# PEP 503: a run of - _ . is one -, and case never distinguishes two projects
+SEPARATORS: Final = re.compile(r"[-_.]+")
+
+# the tail of a line is tagged rather than positional, so stating one more thing
+# about a record never changes what a line that stated nothing already meant
+CONFIDENCE: Final = "c"
+GRANULARITY: Final = "g"
+WHEN: Final = "w"
+USED: Final = "u"
+
+MAX_WHEN: Final = 3_153_600_000_000
+MAX_USED: Final = 4_102_444_800_000
+
+# a kind outside the three rungs: the six fixed columns describe a symbol, and
+# this describes the window the symbols were seen in
+WINDOW: Final = "s"
+DROPPED: Final = "d"
+UNSTAMPED: Final = "n"
+BEYOND_READ: Final = "sl"
+BEYOND_IMPORTED: Final = "si"
+BEYOND_EXECUTED: Final = "sx"
+MONITORED: Final = "m"
+ALONE: Final = "a"
+BACKLOG: Final = "b"
+
+
+class Evidence(Enum):
+    """What the interpreter did with the symbol."""
+
+    # weaker than IMPORTED: the files were read and no module resulted. Not
+    # named "loaded", which reads as a synonym for imported. The tag stays "l"
+    # because changing it would mean a new REPORT_TAG for no gain in meaning
+    READ = "l"
+    IMPORTED = "i"
+    EXECUTED = "x"
+
+
+class Confidence(Enum):
+    """How well the version on a record is known, in the reader's words."""
+
+    EXACT = "exact"
+    # the module is shipped by more than one installed distribution, so the
+    # number is real and the distribution it is attributed to is a guess
+    INFERRED = "inferred"
+    UNKNOWN = "unknown"
+
+
+class Granularity(Enum):
+    """What the symbol on a record stands for, in the reader's words."""
+
+    PACKAGE = "package"
+    MODULE = "module"
+    FUNCTION = "function"
+
+
+class Ecosystem(Enum):
+    """
+    The tags the reader accepts, mirroring its ``Ecosystem::as_str``.
+
+    A closed set rather than a string: the reader refuses the whole line for a
+    tag it does not know, so a free-text column could emit records it would
+    reject. Falling behind a new tag on its side only costs us the ability to
+    name that ecosystem, which is the harmless direction.
+    """
+
+    PYPI = "pypi"
+    NPM = "npm"
+    MAVEN = "maven"
+    RUBYGEMS = "rubygems"
+    HEX = "hex"
+    NATIVE = "native"
+    GO = "go"
+    PACKAGIST = "packagist"
+    NUGET = "nuget"
+
+
+@dataclass(frozen=True)
+class Name:
+    """
+    A package name in the one spelling the reader keys its inventory by.
+
+    Constructing one is the only way to hold one and construction normalizes,
+    so no path reaches the wire with a name the reader could not join. The rule
+    is PEP 503, which is the rule for every record this probe emits.
+    """
+
+    value: str
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "value", SEPARATORS.sub("-", self.value).lower())
+
+
+@dataclass(frozen=True)
+class Record:
+    """One observation: a package, and the symbol it imported or ran."""
+
+    ecosystem: Ecosystem
+    name: Name
+    version: str | None
+    confidence: Confidence
+    symbol: str
+    granularity: Granularity
+    evidence: Evidence
+    frequency: int
+    when: int | None = None
+    used: int | None = None
+
+
+@dataclass(frozen=True)
+class Window:
+    """What the probe held back, in the words the reader folds them by."""
+
+    # sightings that found no room once an observer stopped taking new symbols,
+    # per rung. Sightings and not symbols: a name never admitted is counted
+    # again every time it comes back, so this bounds the blind spot from above
+    beyond_read: int = 0
+    beyond_imported: int = 0
+    beyond_executed: int = 0
+    # whether the executed rung exists in this process at all, and whether the
+    # probe is the only tool watching. Without the second, a count on an
+    # executed record cannot be read: it means windows the code ran in only
+    # while the probe is alone, and means seen at least once beside a coverage
+    # or profiling tool, which re-arms the events for everyone
+    monitored: bool = False
+    alone: bool = False
+    # flushes it would not write because its own undrained reports had piled up
+    backlog: int = 0
+
+
+@dataclass(frozen=True)
+class Report:
+    """A serialised report, and what building it could not put on the wire."""
+
+    text: str
+    dropped: int
+    unstamped: int
+    # records that did not fit, for the caller to put on the report after this
+    held: tuple[Record, ...] = ()
+
+
+def keyed(evidence: Evidence, symbol: str) -> tuple[str, str]:
+    """Name a record the way a probe holds what it has already carried."""
+    return (evidence.value, symbol)
+
+
+def _writable(value: str, limit: int) -> bool:
+    # ``isprintable`` is stricter than the reader's control-character check and
+    # never looser, so anything we emit is something it accepts. It is asked
+    # first because it is what rules out the characters encoding would raise on
+    return bool(value) and value.isprintable() and len(value.encode()) <= limit
+
+
+def readable(version: str) -> bool:
+    """Say whether a version is one the reader would take as written."""
+    return _writable(version, MAX_VERSION)
+
+
+def _known(record: Record) -> bool:
+    # a version nobody could read, and a version claimed to be read that is not
+    # there, both say more than was witnessed, and the reader refuses either
+    return (record.version is None) == (record.confidence is Confidence.UNKNOWN)
+
+
+def _stamp(tag: str, value: int | None, ceiling: int) -> str | None:
+    """Render a stamp, taking only a whole number a clock could have produced."""
+    if not isinstance(value, int) or isinstance(value, bool):
+        return None
+    return f"{tag}={value}" if 0 <= value <= ceiling else None
+
+
+def _line(record: Record) -> str | None:
+    version = ABSENT if record.version is None else record.version
+    bounded = (
+        (record.name.value, MAX_NAME),
+        (record.symbol, MAX_SYMBOL),
+        (version, MAX_VERSION),
+        (record.confidence.value, MAX_ATTRIBUTE),
+        (record.granularity.value, MAX_ATTRIBUTE),
+    )
+    if record.frequency < 1 or not _known(record):
+        return None
+    if any(not _writable(value, limit) for value, limit in bounded):
+        return None
+    stamps = (
+        _stamp(WHEN, record.when, MAX_WHEN),
+        _stamp(USED, record.used, MAX_USED),
+    )
+    line = "\t".join(
+        (
+            record.evidence.value,
+            record.ecosystem.value,
+            record.name.value,
+            version,
+            record.symbol,
+            str(record.frequency),
+            f"{CONFIDENCE}={record.confidence.value}",
+            f"{GRANULARITY}={record.granularity.value}",
+            *(stamp for stamp in stamps if stamp is not None),
+        ),
+    )
+    return line if len(line.encode()) <= MAX_LINE else None
+
+
+def _stated(tag: str, value: int) -> str | None:
+    # never clamped, unlike a record's own fields: the reader pins a number
+    # past its ceiling rather than refusing it, and the pin is what says the
+    # true count is larger. Clamping here would hand it an honest looking value
+    # and charge the difference away. A zero it reads the same as unsaid is
+    # left unsaid, which keeps a quiet window to one short line
+    if not isinstance(value, int) or isinstance(value, bool) or value < 1:
+        return None
+    return f"{tag}={value}"
+
+
+def _window(window: Window, dropped: int, unstamped: int) -> str:
+    stated = (
+        _stated(DROPPED, dropped),
+        _stated(UNSTAMPED, unstamped),
+        _stated(BEYOND_READ, window.beyond_read),
+        _stated(BEYOND_IMPORTED, window.beyond_imported),
+        _stated(BEYOND_EXECUTED, window.beyond_executed),
+        f"{MONITORED}=1" if window.monitored else None,
+        f"{ALONE}=1" if window.alone else None,
+        _stated(BACKLOG, window.backlog),
+    )
+    return "\t".join((WINDOW, *(pair for pair in stated if pair is not None)))
+
+
+def render(
+    records: list[Record],
+    window: Window | None = None,
+    dropped: int = 0,
+    unstamped: int = 0,
+) -> Report:
+    """Render records, dropping and counting any the reader would refuse."""
+    lines = [REPORT_TAG]
+    size = len(REPORT_TAG) + 1
+    held: tuple[Record, ...] = ()
+    for index, record in enumerate(records):
+        line = _line(record)
+        if line is None:
+            dropped += 1
+            continue
+        room = len(line.encode()) + 1
+        if size + room > MAX_REPORT - ROOM:
+            held = tuple(records[index:])
+            break
+        size += room
+        # asked of the record and not of the rendered line: a symbol may spell
+        # the tag itself, and searching the line would read that as a date
+        if _stamp(WHEN, record.when, MAX_WHEN) is None:
+            unstamped += 1
+        lines.append(line)
+    # on the last report of a flush, once every refusal has been counted
+    if window is not None and not held:
+        lines.append(_window(window, dropped, unstamped))
+    return Report(
+        text="\n".join([*lines, ""]),
+        dropped=dropped,
+        unstamped=unstamped,
+        held=held,
+    )