fluidattacks-agent 0.1.2__py3-none-any.whl

fluidattacks_agent/batch.py

@@ -0,0 +1,161 @@
+"""Pack a report into a batch the centre can order, verify and decompress."""
+
+import os
+import threading
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Final
+
+from fluidattacks_agent.settings import Delivery, Secret
+
+BATCH_TAG: Final = "watches-batch/1"
+
+# what the centre keys on. The whitepaper's unit is a machine that outlives a
+# reboot; ours is an interpreter that may live for seconds, so what names one is
+# minted per start and never reused
+INSTANCE_BYTES: Final = 16
+
+# a probe's own health is not a workload's evidence and must not be filed as it
+EXEC: Final = "exec"
+
+# what orders one instance's batches. Wrapping would land on a key already
+# ingested and be discarded as a duplicate, so the counter refuses instead
+MAX_SEQ: Final = 1 << 53
+
+# a flush runs inside a host callback, so this is the workload's own latency:
+# measured on a mebibyte of real report text, level one costs 4.5 ms for 3.1x
+# and level six costs 18.6 ms for 3.7x. Fourteen milliseconds of somebody
+# else's request is not worth sixty kibibytes
+COMPRESSION: Final = 1
+
+
+def packed(text: str) -> bytes:
+    """Compress a report, the same way every time it holds the same thing."""
+    # here and not above: a flush is what needs this, and two milliseconds of
+    # interpreter start is not owed by a workload that never reaches one
+    import gzip  # noqa: PLC0415
+
+    # mtime zero because a gzip header otherwise carries a clock, and a probe
+    # that asserts no phase of its own should not assert one here either
+    return gzip.compress(text.encode(), compresslevel=COMPRESSION, mtime=0)
+
+
+@dataclass(frozen=True, slots=True)
+class Batch:
+    """One delivery: what names it, what orders it, and what proves it."""
+
+    instance: str
+    seq: int
+    stream: str
+    workload: str
+    group: str
+    body: bytes
+    signature: str
+
+
+def minted() -> str:
+    """Name this interpreter, once, out of the kernel's own entropy."""
+    # not secrets.token_hex: the same source at the same 0.8 us a call, but
+    # importing it costs 3.5 ms of a start this package spent months shortening
+    return os.urandom(INSTANCE_BYTES).hex()
+
+
+def _stated(*parts: str) -> bytes:
+    # length-prefixed over the encoded bytes, so that no two different batches
+    # can spell alike whatever the values hold: being unambiguous is a property
+    # of this form rather than a promise made somewhere else
+    return b"\n".join(b"%d:%s" % (len(spelt), spelt) for spelt in map(str.encode, parts))
+
+
+def derived(token: Secret) -> bytes:
+    """Make a key that signs, and is not the credential that authorises."""
+    import hmac  # noqa: PLC0415
+
+    # so a signature cannot be replayed as a credential, and a credential
+    # cannot be read back out of a signature
+    return hmac.digest(token.held.encode(), BATCH_TAG.encode(), "sha256")
+
+
+def batched(
+    delivery: Delivery,
+    instance: str,
+    seq: int,
+    stream: str,
+    text: str,
+) -> Batch | None:
+    """Name a report, order it, pack it, and prove where it came from."""
+    if not 0 < seq < MAX_SEQ:
+        return None
+    import hmac  # noqa: PLC0415
+
+    body = packed(text)
+    workload = delivery.workload or ""
+    group = delivery.group or ""
+    proof = hmac.new(derived(delivery.token), digestmod="sha256")
+    # what travels beside the body is signed with it, because a proxy that can
+    # rewrite which instance or which tenant sent this misfiles it without
+    # touching a byte of the evidence
+    proof.update(_stated(BATCH_TAG, group, workload, instance, str(seq), stream))
+    proof.update(body)
+    return Batch(
+        instance=instance,
+        seq=seq,
+        stream=stream,
+        workload=workload,
+        group=group,
+        body=body,
+        signature=proof.hexdigest(),
+    )
+
+
+@dataclass
+class Batcher:
+    """One instance's batches, in the order that instance made them."""
+
+    instance: str = ""
+    seq: int = 0
+    # two host callbacks can be inside a flush at the same time, because what
+    # keeps a second one out is a plain attribute and not a mutex: measured,
+    # eight threads asking for a place got the same one in all three hundred
+    # rounds, and two batches sharing a place have one of them discarded
+    guard: threading.Lock = field(default_factory=threading.Lock, repr=False)
+
+    def named(self) -> str:
+        """Say what names this interpreter, minting it the first time asked."""
+        if not self.instance:
+            self.instance = minted()
+        return self.instance
+
+    def next(
+        self,
+        delivery: Delivery,
+        text: str,
+        taken: Callable[[Batch], bool],
+        stream: str = EXEC,
+    ) -> Batch | None:
+        """Build the next batch, and spend its place only once it is held."""
+        # taking the place, building against it and offering it are one step, so
+        # that no two batches can be built against the same place and no place
+        # is spent on a batch that was turned away. Whoever takes it is asked
+        # from in here for that reason, and never asks anything back of this
+        with self.guard:
+            held = self.seq + 1
+            batch = batched(delivery, self.named(), held, stream, text)
+            if batch is None or not taken(batch):
+                return None
+            # a place spent on a batch nothing is holding would leave a hole,
+            # and a hole is how evidence lost in flight is meant to look
+            self.seq = held
+            return batch
+
+    def forked(self) -> None:
+        """Forget what named this instance, because a child of it is another."""
+        # a lock held by another thread at the moment of the fork is held in the
+        # child forever, and a fresh one cannot be: the child has no second
+        # thread, so there is nothing for this one to be held against
+        self.guard = threading.Lock()
+        # a child inherits the name and the count and goes on emitting the keys
+        # its parent is emitting: measured across a fork, every key after it
+        # collided, and ingest discards a collision without saying so
+        self.instance = ""
+        self.seq = 0

fluidattacks_agent/deliver.py

@@ -0,0 +1,178 @@
+"""Carry what a workload reported out of the process it was reported in."""
+
+import contextlib
+import errno
+import os
+import threading
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Final
+
+from fluidattacks_agent.batch import Batch, Batcher
+from fluidattacks_agent.outbox import Outbox
+from fluidattacks_agent.patience import TOPMOST, fraction, splay, standoff
+from fluidattacks_agent.post import TIMEOUT, Verdict, posted
+from fluidattacks_agent.settings import Delivery
+from fluidattacks_agent.sink import Stalled
+
+# how old the centre's picture may be while nothing new is being found
+INTERVAL: Final = 15.0
+
+# what a workload's own exit may be delayed by while what is held goes out
+PARTING: Final = 2.0
+
+NAME: Final = "fluidattacks-agent"
+
+
+@dataclass
+class Deliverer:
+    """A sink that puts a report on its way out rather than in a directory."""
+
+    told: Delivery
+    batcher: Batcher = field(default_factory=Batcher)
+    outbox: Outbox = field(default_factory=Outbox)
+    carry: Callable[[Delivery, Batch, float], Verdict] = posted
+    # kept in hand rather than put back, so its place stays its place
+    holding: Batch | None = field(default=None, repr=False)
+    interval: float = INTERVAL
+    share: Callable[[], float] = fraction
+    # a wake says there is something to carry, never that it is welcome
+    due: float = 0.0
+    tries: int = 0
+    woken: threading.Event = field(default_factory=threading.Event, repr=False)
+    thread: threading.Thread | None = field(default=None, repr=False)
+    # the thread field alone, and never held across anything that can block
+    guard: threading.Lock = field(default_factory=threading.Lock, repr=False)
+    # one pass at a time: what is taken but not delivered lives in one place
+    passing: threading.Lock = field(default_factory=threading.Lock, repr=False)
+    # told once a period went by with the far end in reach and nothing left to
+    # carry, so what the probe still holds goes out before it is any older
+    remind: Callable[[], None] = field(default=lambda: None, repr=False)
+    # when the probe was last reminded: a far end that refuses and then takes
+    # ends a wait every standoff, and a wait ending is not a period going by
+    reminded: float = 0.0
+    leaving: bool = False
+    delivered: int = 0
+
+    def write(self, text: str) -> None:
+        """Take a report for delivery, or refuse and be counted for it."""
+        if self.batcher.next(self.told, text, self.outbox.put) is None:
+            raise Stalled(errno.ENOSPC, "nothing taken", len(text))
+        self._rouse()
+
+    def _rouse(self) -> None:
+        """Say there is something to carry, and find a carrier if there is none."""
+        with self.guard:
+            # a carrier that stopped takes delivery with it, and one is not
+            # worth starting for a process already on its way out
+            if not self.leaving and (self.thread is None or not self.thread.is_alive()):
+                # replicas start in the same second, so this is drawn; and
+                # once, so that it can never overwrite a standoff
+                if not self.due:
+                    self.due = time.monotonic() + splay(self.interval, self.share())
+                # nothing here may wait on the carrier: a thread importing what
+                # a post needs blocks while its starter is mid-import
+                self.thread = threading.Thread(target=self._serving, name=NAME, daemon=True)
+                self.thread.start()
+        self.woken.set()
+
+    def _serving(self) -> None:
+        while not self.leaving:
+            # a thread that dies takes delivery with it and tells nobody
+            with contextlib.suppress(Exception):
+                roused = self.woken.wait(self._waiting())
+                self.woken.clear()
+                if time.monotonic() < self.due:
+                    continue
+                self.offering()
+                # a period, never a wake: a wake says there is something to
+                # carry, and asking for more on one made each flush bring the
+                # next. Nothing in hand, or the far end is still refusing
+                now = time.monotonic()
+                if (
+                    not roused
+                    and self.holding is None
+                    and not self.leaving
+                    and now - self.reminded >= self.interval
+                ):
+                    self.reminded = now
+                    self.remind()
+
+    def parting(self) -> None:
+        """Give whatever is held one bounded chance to travel, then give up."""
+        self.leaving = True
+        self.woken.set()
+        limit = time.monotonic() + PARTING
+        # a carrier mid-pass is doing this work, so the wait is that budget
+        if not self.passing.acquire(timeout=PARTING):
+            return
+        try:
+            with contextlib.suppress(Exception):
+                # bounded by what is left, or the bound is only the moment a
+                # last attempt may begin at
+                while (left := limit - time.monotonic()) > 0 and self._offered(left):
+                    pass
+        finally:
+            self.passing.release()
+
+    def forked(self) -> None:
+        """Leave a child holding none of its parent's, and owing itself a carrier."""
+        # a lock or event held at the moment of the fork is held in the child
+        # forever, so each is made again rather than reset
+        self.guard = threading.Lock()
+        self.passing = threading.Lock()
+        self.woken = threading.Event()
+        self.thread = None
+        self.holding = None
+        # every worker of a pre-forking server was born in the same second
+        self.due = 0.0
+        self.tries = 0
+        self.reminded = 0.0
+        # a child is not on its way out merely because its parent was
+        self.leaving = False
+        self.batcher.forked()
+        self.outbox.forked()
+
+    def registered(self) -> None:
+        """Ask the interpreter to say when this process has become two."""
+        # cannot be taken back once given, so exactly one per process
+        if hasattr(os, "register_at_fork"):
+            os.register_at_fork(after_in_child=self.forked)
+
+    def offering(self) -> None:
+        """Offer what is waiting, one pass at a time, stopping at a refusal."""
+        if not self.passing.acquire(blocking=False):
+            return
+        try:
+            for _ in range(self.outbox.limit):
+                if not self._offered(TIMEOUT):
+                    return
+        finally:
+            self.passing.release()
+
+    def _offered(self, bound: float) -> bool:
+        """Offer one batch, and say whether offering another is worth trying."""
+        batch = self.holding or self.outbox.take()
+        if batch is None:
+            return False
+        # held before the attempt, so a cut-off leaves it findable
+        self.holding = batch
+        if (verdict := self.carry(self.told, batch, bound)) is not Verdict.DELIVERED:
+            self._stood_off(verdict)
+            return False
+        self.holding = None
+        self.due = 0.0
+        self.tries = 0
+        self.delivered += 1
+        return True
+
+    def _stood_off(self, verdict: Verdict) -> None:
+        """Wait longer for each refusal, and longest for one about nothing of ours."""
+        self.tries = TOPMOST if verdict is Verdict.STOP else self.tries + 1
+        self.due = time.monotonic() + standoff(self.tries, self.share())
+
+    def _waiting(self) -> float:
+        """Wait out a standoff if there is one, and otherwise a pass."""
+        left = self.due - time.monotonic()
+        return left if left > 0 else self.interval

fluidattacks_agent/distributions.py

@@ -0,0 +1,80 @@
+"""Map an imported module to the distribution that installed it."""
+
+from dataclasses import dataclass
+
+from fluidattacks_agent.report import Confidence, Name, readable
+
+
+@dataclass(frozen=True)
+class Dist:
+    """The distribution behind a module, and how well its version is known."""
+
+    name: Name
+    version: str | None
+    confidence: Confidence
+
+
+@dataclass(frozen=True)
+class Declared:
+    """The versions read off installed metadata, and the ones that would not."""
+
+    versions: dict[Name, str]
+    unreadable: int
+
+
+def declared() -> Declared:
+    """Read the version every installed distribution declares for itself."""
+    # imported here and not above: reading metadata costs twenty milliseconds of
+    # interpreter start, and it is a flush that needs it, not an import
+    from importlib.metadata import distributions  # noqa: PLC0415
+
+    found: dict[Name, str] = {}
+    unreadable = 0
+    for dist in distributions():
+        # metadata is a file the workload owns: one unreadable distribution
+        # must not cost the map, since without it the probe reports nothing
+        try:
+            metadata = dist.metadata
+            name, version = metadata["Name"], metadata["Version"]
+        except Exception:  # noqa: BLE001
+            unreadable += 1
+            continue
+        # the type keys it the way it will be looked up, which also folds two
+        # spellings of one project together, since PEP 503 says they are one
+        if name is not None and version is not None:
+            found.setdefault(Name(name), version)
+    return Declared(versions=found, unreadable=unreadable)
+
+
+def resolve(names: list[str], versions: dict[Name, str]) -> Dist:
+    """Choose the distribution behind a module, and say how sure that is."""
+    name = Name(names[0])
+    version = versions.get(name)
+    # a version the reader would refuse is one we could not read, and dropping
+    # the record over it would lose the package instead of the version
+    if version is None or not readable(version):
+        return Dist(name=name, version=None, confidence=Confidence.UNKNOWN)
+    known = Confidence.EXACT if len(names) == 1 else Confidence.INFERRED
+    return Dist(name=name, version=version, confidence=known)
+
+
+def distribution_map() -> dict[str, Dist]:
+    """
+    Read which distribution installed each top-level module, and its version.
+
+    This walks every installed distribution's metadata, so it is taken once per
+    report rather than once per module. The probe spends the workload's own cpu.
+    """
+    from importlib.metadata import packages_distributions  # noqa: PLC0415
+
+    versions = declared().versions
+    return {
+        module: resolve(names, versions)
+        for module, names in packages_distributions().items()
+        if names
+    }
+
+
+def distribution_of(module: str, installed: dict[str, Dist]) -> Dist | None:
+    """Name the distribution a module came from, submodules included."""
+    return installed.get(module.split(".", maxsplit=1)[0])

fluidattacks_agent/executions.py

@@ -0,0 +1,124 @@
+"""Watch code inside a package start running, where the interpreter allows it."""
+
+import sys
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import PurePath
+from typing import Final
+
+from fluidattacks_agent.distributions import Dist, distribution_of
+from fluidattacks_agent.report import Ecosystem, Evidence, Granularity, Record, keyed
+
+# 3.15 makes the import below lazy from this; every version we support
+# ignores it, so nothing changes until the floor moves
+__lazy_modules__ = ["pathlib"]
+
+# a real application runs some 26,000 distinct functions while it starts, and
+# what filled the ceiling is what never gets named again for the process's life
+MAX_FUNCTIONS: Final = 32768
+
+OURS: Final = "fluidattacks_agent"
+
+SOURCE: Final = ".py"
+
+INITIALIZER: Final = "__init__"
+PARENT: Final = ".."
+
+TOOL: Final = 3
+TOOL_NAME: Final = "fluidattacks-agent"
+
+
+TOOLS: Final = 6
+
+
+def available() -> bool:
+    """Say whether this interpreter can report code starting at all."""
+    return hasattr(sys, "monitoring")
+
+
+def alone(tool: int = TOOL) -> bool:
+    """Say whether this is the only tool watching: restart_events takes none."""
+    held = (sys.monitoring.get_tool(other) for other in range(TOOLS) if other != tool)
+    return not any(name is not None for name in held)
+
+
+def module_of(filename: str, sites: tuple[str, ...]) -> str | None:
+    """Name the module whose source a running code object came from."""
+    for site in sites:
+        prefix = site + "/"
+        if not filename.startswith(prefix) or not filename.endswith(SOURCE):
+            continue
+        held = PurePath(filename[len(prefix) :])
+        parts = list(held.parts)
+        if not parts or held.is_absolute() or PARENT in parts:
+            return None
+        stem = parts.pop()[: -len(SOURCE)]
+        if not stem:
+            return None
+        if stem != INITIALIZER:
+            parts.append(stem)
+        return ".".join(parts) or None
+    return None
+
+
+@dataclass
+class RunObserver:
+    """Functions that started, bounded the way the reader's own map is."""
+
+    sites: tuple[str, ...] = ()
+    seen: dict[str, int] = field(default_factory=dict)
+    moved: set[str] = field(default_factory=set)
+    suppressed: int = 0
+
+    def note(self, filename: str, qualname: str) -> str | None:
+        """Take note of code starting, naming the function whenever there is room."""
+        module = module_of(filename, self.sites)
+        if module is None:
+            return None
+        if module.split(".", maxsplit=1)[0] == OURS:
+            return None
+        qualified = f"{module}.{qualname}"
+        if qualified in self.seen:
+            self.seen[qualified] += 1
+            self.moved.add(qualified)
+            return qualified
+        if len(self.seen) >= MAX_FUNCTIONS:
+            self.suppressed += 1
+            return None
+        self.seen[qualified] = 1
+        self.moved.add(qualified)
+        return qualified
+
+    def records(
+        self,
+        installed: dict[str, Dist],
+        carried: Mapping[tuple[str, str], int],
+    ) -> list[Record]:
+        """Report every function whose package a distribution installed."""
+        found: list[Record] = []
+        scanned = bool(installed)
+        for qualified in sorted(self.moved):
+            times = self.seen[qualified]
+            # what a report carried comes back only if note sees it again,
+            # and no distribution will own what a settled scan does not
+            if times <= carried.get(keyed(Evidence.EXECUTED, qualified), 0):
+                self.moved.discard(qualified)
+                continue
+            dist = distribution_of(qualified, installed)
+            if dist is None:
+                if scanned:
+                    self.moved.discard(qualified)
+                continue
+            found.append(
+                Record(
+                    ecosystem=Ecosystem.PYPI,
+                    name=dist.name,
+                    version=dist.version,
+                    confidence=dist.confidence,
+                    symbol=qualified,
+                    granularity=Granularity.FUNCTION,
+                    evidence=Evidence.EXECUTED,
+                    frequency=times,
+                ),
+            )
+        return found

fluidattacks_agent/gate.py

@@ -0,0 +1,10 @@
+"""Start the probe on being imported, and only if a workload wants it."""
+
+from fluidattacks_agent.switch import switched_on
+
+# importing this runs it, because a site directory is what imports it. A
+# workload that said no imports nothing above the switch: 26 ms of a start
+if switched_on():
+    from fluidattacks_agent.startup import start
+
+    start()

fluidattacks_agent/loads.py

@@ -0,0 +1,109 @@
+"""Account for package code the interpreter read without importing it."""
+
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from types import ModuleType
+
+from fluidattacks_agent.distributions import Dist, distribution_of
+from fluidattacks_agent.report import Ecosystem, Evidence, Granularity, Record, keyed
+
+MAX_PACKAGES = 4096
+
+OURS = "fluidattacks_agent"
+
+METADATA = (".dist-info", ".egg-info")
+
+CACHE = "__pycache__"
+
+INITIALIZER = "__init__"
+PARENT = ".."
+
+CODE = (".py", ".pyc", ".so", ".pyd")
+
+
+def package_of(path: str, sites: tuple[str, ...]) -> str | None:
+    """Name the top level package a read file belongs to, when it is code."""
+    if not path.endswith(CODE):
+        return None
+    for site in sites:
+        prefix = site + os.sep
+        if not path.startswith(prefix):
+            continue
+        held = path[len(prefix) :]
+        # a step back out names a file the package it sits under does not own
+        if f"{os.sep}{PARENT}{os.sep}" in held:
+            return None
+        head, _, tail = held.partition(os.sep)
+        if head == CACHE:
+            head = tail.split(os.sep, maxsplit=1)[0]
+        if head.endswith(METADATA):
+            return None
+        named = head.split(".", maxsplit=1)[0]
+        return None if named == INITIALIZER else named or None
+    return None
+
+
+@dataclass
+class LoadObserver:
+    """Packages whose code was read, bounded the way the reader's map is."""
+
+    sites: tuple[str, ...] = ()
+    # how many files of each package were read, not merely whether any was
+    seen: dict[str, int] = field(default_factory=dict)
+    moved: set[str] = field(default_factory=set)
+    suppressed: int = 0
+
+    def note(self, path: str) -> str | None:
+        """Take note of a file opened, naming the package whenever there is room."""
+        package = package_of(path, self.sites)
+        if package is None or package == OURS:
+            return None
+        if package in self.seen:
+            self.seen[package] += 1
+            self.moved.add(package)
+            return package
+        if len(self.seen) >= MAX_PACKAGES:
+            self.suppressed += 1
+            return None
+        self.seen[package] = 1
+        self.moved.add(package)
+        return package
+
+    def records(
+        self,
+        modules: dict[str, ModuleType],
+        installed: dict[str, Dist],
+        carried: Mapping[tuple[str, str], int],
+    ) -> list[Record]:
+        """Report the packages that were read and never became modules."""
+        found: list[Record] = []
+        scanned = bool(installed)
+        for package in sorted(self.moved):
+            times = self.seen[package]
+            if times <= carried.get(keyed(Evidence.READ, package), 0):
+                self.moved.discard(package)
+                continue
+            if package in modules:
+                # kept, because a name can leave sys.modules and be read again
+                continue
+            dist = distribution_of(package, installed)
+            if dist is None:
+                if scanned:
+                    self.moved.discard(package)
+                continue
+            found.append(
+                Record(
+                    ecosystem=Ecosystem.PYPI,
+                    name=dist.name,
+                    version=dist.version,
+                    confidence=dist.confidence,
+                    symbol=package,
+                    # a read file names the package it sits in, and naming the
+                    # module it is would claim the interpreter reached it
+                    granularity=Granularity.PACKAGE,
+                    evidence=Evidence.READ,
+                    frequency=times,
+                ),
+            )
+        return found