fluidattacks-agent 0.1.2__py3-none-any.whl

fluidattacks_agent/startup.py

@@ -0,0 +1,439 @@
+"""Start the probe inside a workload and leave reports where the agent drains."""
+
+import _thread
+import atexit
+import contextlib
+import os
+import sys
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field, replace
+from types import CodeType, ModuleType
+from typing import Final
+
+from fluidattacks_agent.distributions import Dist, distribution_map
+from fluidattacks_agent.executions import TOOL, TOOL_NAME, RunObserver, alone, available
+from fluidattacks_agent.loads import LoadObserver
+from fluidattacks_agent.observer import ImportObserver, install
+from fluidattacks_agent.report import Evidence, Record, Window, keyed, render
+from fluidattacks_agent.settings import Delivery, asked, delivery
+from fluidattacks_agent.sink import FileSink, Holding, Sink, Stalled
+from fluidattacks_agent.switch import switched_on
+
+# first sightings between flushes. The carrier's clock brings the rest
+FLUSH_EVERY: Final = 128
+
+# what a site directory is given so that installing the package starts the
+# probe. The agent writes the same line when it injects one it did not install
+PTH_NAME: Final = "fluidattacks_agent.pth"
+# the gate and not this module: a workload that switched the probe off pays for
+# reading the switch and for nothing above it
+PTH_LINE: Final = "import fluidattacks_agent.gate\n"
+
+MS: Final = 1_000_000
+SECOND: Final = 1000
+
+START: Final = time.monotonic_ns()
+
+# the reader reads u= as milliseconds since the epoch. Taken once and advanced
+# by the monotonic clock, so a stepped clock cannot walk a stamp backwards
+ORIGIN: Final = time.time_ns() // MS
+
+
+def reached() -> int:
+    """Say how far into the probe's own life this moment is, in milliseconds."""
+    return (time.monotonic_ns() - START) // MS
+
+
+@dataclass
+class Probe:
+    """An observer whose reports are left behind as the workload runs."""
+
+    sink: Sink = field(default_factory=FileSink)
+    observer: ImportObserver = field(default_factory=ImportObserver)
+    loads: LoadObserver = field(default_factory=LoadObserver)
+    runs: RunObserver = field(default_factory=RunObserver)
+    distributions: Callable[[], dict[str, Dist]] = distribution_map
+    clock: Callable[[], int] = reached
+    origin: int = ORIGIN
+    # what each record was last reported as, so a flush carries the increment
+    reported: dict[tuple[str, str], int] = field(default_factory=dict)
+    # the same, for the sightings each observer found no room for
+    stated: dict[str, int] = field(default_factory=dict)
+    first: dict[tuple[str, str], int] = field(default_factory=dict)
+    last: dict[tuple[str, str], int] = field(default_factory=dict)
+    scanned: dict[str, Dist] | None = None
+    pending: int = 0
+    flushing: bool = False
+    stalled: int = 0
+    charged: int = 0
+    # owed until a report states them: a flush can end after charging a record
+    # and before saying it refused one, and a charged record is never offered again
+    refused: int = 0
+    undated: int = 0
+    monitoring: bool = False
+    looking: bool = False
+    # sampled when the events were armed, not when the report is written: the
+    # later reading governs the next window rather than the one reported
+    solitary: bool = False
+    # set off the workload's threads, acted on by the next sighting on one of
+    # them, since what is held is walked nowhere else
+    overdue: bool = False
+    # every thread sees overdue at once, so the flush is gated by a lock nothing
+    # waits on: a second caller finds it taken and returns. From _thread, which
+    # every interpreter start has already paid for, and threading is not
+    gate: _thread.LockType = field(default_factory=_thread.allocate_lock, repr=False)
+
+    def find_spec(
+        self,
+        fullname: str,
+        _path: object = None,
+        _target: ModuleType | None = None,
+    ) -> None:
+        """Note the lookup, and leave a report once enough have piled up."""
+        held = self.looking
+        self.looking = True
+        try:
+            with contextlib.suppress(Exception):
+                self._looked(fullname)
+        finally:
+            self.looking = held
+
+    def _looked(self, fullname: str) -> None:
+        if self.flushing:
+            return
+        module = self.observer.note(fullname)
+        self._pile(0 if module is None else self._mark(Evidence.IMPORTED, module))
+
+    def read(self, path: str) -> None:
+        """Note a file the workload opened, and report as those pile up too."""
+        # a flush of ours reads and imports on its own account, and none of that
+        # is the workload using anything
+        if self.flushing:
+            return
+        package = self.loads.note(path)
+        # tested here and not inside: the hook runs on every open the host
+        # makes, and all but a few are not ours to stamp
+        self._pile(0 if package is None else self._mark(Evidence.READ, package))
+
+    def ran(self, filename: str, qualname: str) -> None:
+        """Note code starting, and report as first sightings pile up too."""
+        if self.flushing:
+            return
+        symbol = self.runs.note(filename, qualname)
+        self._pile(0 if symbol is None else self._mark(Evidence.EXECUTED, symbol))
+
+    def _mark(self, evidence: Evidence, symbol: str) -> int:
+        """Stamp a sighting, saying whether it was the first of its symbol."""
+        key = keyed(evidence, symbol)
+        seen = self.clock()
+        self.last[key] = seen
+        if key in self.first:
+            return 0
+        self.first[key] = seen
+        return 1
+
+    def remind(self) -> None:
+        """Have the next sighting bring a flush, however few have piled up."""
+        # a flag and never a flush: the carrier's clock calls this from a
+        # thread of its own, and a flush runs on the workload's alone
+        self.overdue = True
+
+    def _pile(self, added: int) -> None:
+        # first sightings only: a flush re-arms every code object, so counting
+        # one seen again, or one with no room, made each flush bring the next
+        self.pending += added
+        if self.overdue or self.pending >= FLUSH_EVERY:
+            # cleared before the call, because monitoring reports the entry of
+            # flush itself and would otherwise pile up and call it again
+            self.pending = 0
+            self.flush()
+
+    def flush(self) -> None:
+        """Write what has not been reported, without ever raising at a caller."""
+        if not self.gate.acquire(blocking=False):
+            return
+        self.flushing = True
+        # spent here, so a reminder given while this runs brings one more flush
+        self.overdue = False
+        try:
+            with contextlib.suppress(Exception):
+                if self._sighted():
+                    self._write_fresh(self._installed())
+        finally:
+            self.pending = 0
+            self.flushing = False
+            self.gate.release()
+            self._rearm()
+
+    def forked(self) -> None:
+        """Leave a child able to flush, whatever its parent was doing at the fork."""
+        # a lock another thread held at the fork is held in the child forever,
+        # and a flag it set stays set: the child has no thread left to clear it
+        self.gate = _thread.allocate_lock()
+        self.flushing = False
+        self.overdue = False
+        self.pending = 0
+
+    def _owed(self) -> bool:
+        """Say whether a count is waiting for a report that can state it."""
+        return bool(self.refused or self.undated)
+
+    def _sighted(self) -> bool:
+        """Say whether this window holds anything a report could carry."""
+        return bool(
+            self._owed()
+            or self.loads.seen
+            or self.runs.seen
+            # a lookup that never became a module is dropped when the records are
+            # folded, so on its own it is not cause to walk any metadata
+            or any(name in sys.modules for name in self.observer.seen)
+            or any(self._beyond().values())
+            or self.stalled != self.charged,
+        )
+
+    def _rearm(self) -> None:
+        if self.monitoring:
+            with contextlib.suppress(Exception):
+                self.solitary = alone()
+                if self.solitary:
+                    sys.monitoring.restart_events()
+
+    def _installed(self) -> dict[str, Dist]:
+        if self.scanned is not None:
+            return self.scanned
+        if self.looking:
+            # the walk imports, and a lookup of ours runs while the interpreter
+            # holds the lock every other import is waiting on. Not remembered,
+            # so the next flush outside one still reads it
+            return {}
+        try:
+            scanned = self.distributions()
+        except ImportError:
+            # what the walk imports goes through whatever the workload put on
+            # the import path, and a refusal there can be this once only
+            return {}
+        except Exception:  # noqa: BLE001
+            # a scan the workload can make raise is remembered as having found
+            # nothing, since suppressing it would leave nothing scanned and buy
+            # the workload a full metadata walk on every flush that follows
+            self.scanned = {}
+            return self.scanned
+        self.scanned = scanned
+        return scanned
+
+    def _write_fresh(self, installed: dict[str, Dist]) -> None:
+        seen = [
+            *self.loads.records(sys.modules, installed, self.reported),
+            *self.observer.records(sys.modules, installed, self.reported),
+            *self.runs.records(installed, self.reported),
+        ]
+        fresh = [record for record in map(self._since, seen) if record is not None]
+        counted = self._beyond()
+        beyond = {tag: n - self.stated.get(tag, 0) for tag, n in counted.items()}
+        if not fresh and not any(beyond.values()) and not self._owed():
+            return
+        # the reader sums what every report states, so this is the increment
+        stalled = self.stalled
+        window: Window | None = self._window(beyond, stalled - self.charged)
+        pending = fresh
+        while True:
+            report = render(pending, window, self.refused, self.undated)
+            self._write(report.text)
+            self._charge(pending[: len(pending) - len(report.held)])
+            held = list(report.held)
+            self.refused = report.dropped if held else 0
+            self.undated = report.unstamped if held else 0
+            if not held:
+                self.stated.update(counted)
+                self.charged = stalled
+                return
+            # a render holding everything back would write files forever, and
+            # the suppression above would never let it be seen
+            if len(held) >= len(pending):
+                return
+            pending = held
+
+    def _charge(self, written: list[Record]) -> None:
+        for record in written:
+            key = _key(record)
+            self.reported[key] = self.reported.get(key, 0) + record.frequency
+
+    def _beyond(self) -> dict[str, int]:
+        return {
+            Evidence.READ.value: self.loads.suppressed,
+            Evidence.IMPORTED.value: self.observer.suppressed,
+            Evidence.EXECUTED.value: self.runs.suppressed,
+        }
+
+    def _window(self, beyond: dict[str, int], backlog: int) -> Window:
+        return Window(
+            backlog=backlog,
+            beyond_read=beyond[Evidence.READ.value],
+            beyond_imported=beyond[Evidence.IMPORTED.value],
+            beyond_executed=beyond[Evidence.EXECUTED.value],
+            monitored=self.monitoring,
+            alone=self.monitoring and self.solitary,
+        )
+
+    def _since(self, record: Record) -> Record | None:
+        # the reader adds up what every report of a record says, so a report
+        # carries what has happened since the last one and never the running
+        # total, which would count each sighting once per flush that follows it
+        key = _key(record)
+        since = record.frequency - self.reported.get(key, 0)
+        if since < 1:
+            return None
+        return replace(
+            record,
+            frequency=since,
+            when=self.first.get(key),
+            used=self._used(key),
+        )
+
+    def _used(self, key: tuple[str, str]) -> int | None:
+        """Date the last sighting of a symbol as of this report, to the second."""
+        seen = self.last.get(key)
+        if seen is None:
+            return None
+        # what drains a report holds its own clock to whole seconds and drops
+        # any date past it, so a finer one here is one it can only throw away
+        return (self.origin + seen) // SECOND * SECOND
+
+    def _write(self, text: str) -> None:
+        try:
+            self.sink.write(text)
+        except Stalled:
+            # counted here and not in the sink, because what a window missed is
+            # the probe's to state and a sink knows nothing of windows
+            self.stalled += 1
+            raise
+
+
+def _key(record: Record) -> tuple[str, str]:
+    return keyed(record.evidence, record.symbol)
+
+
+def site_dirs() -> tuple[str, ...]:
+    """Name the directories the workload installs its distributions into."""
+    found: list[str] = []
+    for entry in sys.path:
+        if not entry.endswith("site-packages"):
+            continue
+        for form in (entry, os.path.realpath(entry)):
+            if form not in found:
+                found.append(form)
+    return tuple(found)
+
+
+def monitor(probe: Probe) -> Callable[[CodeType, int], object]:
+    """Build the callback the interpreter calls the first time code starts."""
+
+    def started(code: CodeType, _offset: int) -> object:
+        with contextlib.suppress(Exception):
+            probe.ran(code.co_filename, code.co_qualname)
+        return sys.monitoring.DISABLE
+
+    return started
+
+
+def observe_runs(probe: Probe, tool: int = TOOL) -> bool:
+    """Ask the interpreter to report code starting, if it knows how."""
+    if not available():
+        return False
+    try:
+        events = sys.monitoring.events
+        sys.monitoring.use_tool_id(tool, TOOL_NAME)
+    except Exception:  # noqa: BLE001
+        return False
+    try:
+        sys.monitoring.register_callback(tool, events.PY_START, monitor(probe))
+        sys.monitoring.set_events(tool, events.PY_START)
+    except Exception:  # noqa: BLE001
+        with contextlib.suppress(Exception):
+            sys.monitoring.free_tool_id(tool)
+        return False
+    with contextlib.suppress(Exception):
+        probe.solitary = alone(tool)
+    return True
+
+
+def watch(probe: Probe) -> Callable[[str, tuple[object, ...]], None]:
+    """Build the audit hook that notes every file the workload opens."""
+
+    def hook(event: str, args: tuple[object, ...]) -> None:
+        if event != "open":
+            return
+        try:
+            target = args[0]
+            if isinstance(target, str):
+                probe.read(target)
+        except Exception:  # noqa: BLE001
+            return
+
+    return hook
+
+
+def running() -> Probe | None:
+    """Find the probe already watching this process, if one of ours is."""
+    for finder in sys.meta_path:
+        if isinstance(finder, Probe):
+            return finder
+    return None
+
+
+def start(sink: Sink | None = None) -> Probe | None:
+    """Put the probe on the import path, unless one is there or it is unwanted."""
+    try:
+        return _start(sink)
+    except Exception:  # noqa: BLE001
+        # every interpreter the workload runs reaches this line, and what runs
+        # it prints whatever escapes to the host's stderr and carries on
+        return None
+
+
+def chosen(remind: Callable[[], None]) -> Sink:
+    """
+    Give a workload what it asked for, and something loud if it asked badly.
+
+    ``remind`` is what a carrier calls, off the workload's threads, once a
+    period has gone by with the far end in reach and nothing left to carry.
+    """
+    if not asked():
+        return FileSink()
+    told = delivery()
+    if told is None:
+        # half configured must be loud, not indistinguishable from quiet
+        return Holding()
+    return carrier(told, remind)
+
+
+def carrier(told: Delivery, remind: Callable[[], None]) -> Sink:
+    """Build the one thing that carries reports out, and wire it to this process."""
+    # imported here, so a workload that never asked pays for none of it
+    from fluidattacks_agent.deliver import Deliverer  # noqa: PLC0415
+
+    held = Deliverer(told=told, remind=remind)
+    # before the flush handler, since exit handlers run in reverse order
+    atexit.register(held.parting)
+    held.registered()
+    return held
+
+
+def _start(sink: Sink | None) -> Probe | None:
+    held = running()
+    if held is not None:
+        return held
+    if not switched_on():
+        return None
+    sites = site_dirs()
+    probe = Probe(loads=LoadObserver(sites=sites), runs=RunObserver(sites=sites))
+    # after the probe, because what carries its reports is told what to remind
+    probe.sink = chosen(probe.remind) if sink is None else sink
+    if hasattr(os, "register_at_fork"):
+        os.register_at_fork(after_in_child=probe.forked)
+    install(probe)
+    sys.addaudithook(watch(probe))
+    probe.monitoring = observe_runs(probe)
+    atexit.register(probe.flush)
+    return probe

fluidattacks_agent/switch.py

@@ -0,0 +1,22 @@
+"""Say whether a workload left the probe switched on, before anything is paid."""
+
+import os
+from collections.abc import Mapping
+
+# no Final: importing typing would cost 1.3 ms of a start nobody asked for
+SWITCH = "FLUIDATTACKS_AGENT"
+
+# every spelling of no, and none of the words the agent's own --probe takes for
+# yes: two of its three mean observe there and cannot mean the opposite here
+OFF = frozenset({"0", "off", "false", "no", "disabled", "none"})
+
+
+def switched_on(environ: Mapping[str, str] | None = None) -> bool:
+    """Say whether the probe was left switched on, taking silence for yes."""
+    try:
+        held = os.environ if environ is None else environ
+        value = held.get(SWITCH)
+        return not value or value.strip().lower() not in OFF
+    except Exception:  # noqa: BLE001
+        # a switch that cannot be read is not something to read as consent
+        return False

fluidattacks_agent-0.1.2.dist-info/METADATA

@@ -0,0 +1,95 @@
+Metadata-Version: 2.5
+Name: fluidattacks-agent
+Version: 0.1.2
+Summary: In-process probe reporting what a Python workload imports and runs
+Project-URL: Homepage, https://fluidattacks.com
+Project-URL: Source, https://gitlab.com/fluidattacks/universe/-/tree/trunk/watches/agents/python
+Author-email: Development <development@fluidattacks.com>
+License: MPL-2.0
+Keywords: dependencies,observability,reachability,runtime,sbom
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Security
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+# fluidattacks-agent
+
+Reports what a running Python workload actually imports and runs, so that a
+dependency inventory can say which of its findings are reachable at runtime and
+which are not.
+
+It is a library, not a service. It observes the interpreter it is installed in,
+writes what it saw, and does nothing else. It takes no dependencies: the
+standard library only, because it is installed into workloads we do not own.
+
+## Installing
+
+```
+pip install fluidattacks-agent
+```
+
+That is the whole setup. A `.pth` file at the root of the wheel starts the probe
+at interpreter start, before the workload's own program runs, so nothing has to
+be imported or called by hand.
+
+## Turning it off
+
+```
+FLUIDATTACKS_AGENT=off
+```
+
+Also `0`, `false`, `no`, `disabled` or `none`. A workload that says no pays for
+reading the setting and for nothing above it. Saying nothing is taken for yes,
+because installing the package is the consent.
+
+## What it observes
+
+- distributions whose modules were imported, and which modules
+- functions that were executed, where the interpreter offers that
+- how often, in windows, and when a symbol was first reached
+
+Only what an installed distribution owns is attributed. The standard library and
+a workload's own first-party code produce no records.
+
+## Where reports go
+
+By default, one file per report under `/tmp/.watches-exec`, for a collector to
+drain. Naming an endpoint sends them instead:
+
+| | |
+|---|---|
+| `FLUIDATTACKS_AGENT_ENDPOINT` | where reports are posted, `https://` only |
+| `FLUIDATTACKS_AGENT_TOKEN_FILE` | a file holding the credential, preferred |
+| `FLUIDATTACKS_AGENT_TOKEN` | the credential itself, read only if no file is named |
+| `FLUIDATTACKS_AGENT_GROUP` | what the reports are filed under |
+| `FLUIDATTACKS_AGENT_WORKLOAD` | what this workload is called |
+
+A file is preferred over a variable because a file can be mode 400, while an
+environment variable is readable by any process of the same user.
+
+An endpoint named without enough beside it to reach is a misconfiguration, not a
+reason to fall back: the probe then holds nothing and counts every report it
+refused, so a half-configured deployment is loud rather than a directory filling
+up where nobody drains it.
+
+## What travels, and what does not
+
+Reports are gzipped and signed with a key derived from the credential; the
+credential itself never travels, appears in no record, and is in no exception.
+The far end must prove who it is — certificate chain and hostname both — and no
+redirect is followed.
+
+What a report contains is distribution names, versions, module and function
+names, and counts. No arguments, no return values, no file contents, no
+environment.
+
+## Licence
+
+MPL-2.0

fluidattacks_agent-0.1.2.dist-info/RECORD

@@ -0,0 +1,20 @@
+fluidattacks_agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fluidattacks_agent/batch.py,sha256=AE8RyRKfa5qANjQVZKcH3ath1Y2H0Gv9OBmq09P95NE,6143
+fluidattacks_agent/deliver.py,sha256=5Mig_seLUGTmwLtnPbUxNbEMBH6c5qhBuQv2qgQzN-0,7673
+fluidattacks_agent/distributions.py,sha256=ecBT8gk25fuTVghCEsNPjgqpvwBk3cj1_dZI01QP74w,2990
+fluidattacks_agent/executions.py,sha256=L269j15UqiBAU_I6VDR4LAsxlCHzeNE7ypws3JWTHd8,4255
+fluidattacks_agent/gate.py,sha256=lGn4k2GLN6vmAzHFirXRryv8CCaaVfHu4JTKpKvISZw,354
+fluidattacks_agent/loads.py,sha256=0v8QpMyI4VUI84TMoP1bOW4z4pRE4q887R6hTJ-qzhc,3725
+fluidattacks_agent/observer.py,sha256=89LpH37WEfjsdniksl1STZsF6UYLb_9brSoEb0mm6OQ,4880
+fluidattacks_agent/outbox.py,sha256=Wjr1fd5xIi6of33dYS_LRMxSzDh83WhsgHLVjCQ5FDA,2679
+fluidattacks_agent/patience.py,sha256=qMgUJhvyHNDJAn0FgHHRck0h25kLFHoJdTIYQmYXUuE,1229
+fluidattacks_agent/post.py,sha256=hMrzpt1AmN2VLjRyWcz2n1nIZcoMaZ7901GL2BXRgzs,5728
+fluidattacks_agent/report.py,sha256=wdBLaZaEvIcduA3HhxRNtHluXpGTpyGCEr3KFUE3j2k,9532
+fluidattacks_agent/settings.py,sha256=e7yezXwVtcfzpK5zsozhamayIHCF1wfOKipaM6zEJjk,4301
+fluidattacks_agent/sink.py,sha256=19d81osEynUh7iz0XXsIc5kAWkDHsJfaLoAj_0ydfMM,6027
+fluidattacks_agent/startup.py,sha256=yF5BlFsNuH5S14mvaFwzHC2b70WHFbFzCVYSG8MwCUM,17226
+fluidattacks_agent/switch.py,sha256=HQ9ONU7iHEkUlLhSFa5t4NkBQM1Y0rLkbMJdGGJwVMw,900
+fluidattacks_agent.pth,sha256=layRid5SG6BO2eLwjJ33U7d50i4m8BO9HrL4g99bftE,31
+fluidattacks_agent-0.1.2.dist-info/METADATA,sha256=3DvPkwAnchJ0OwuVn-KW0G4jwCRssBZXhTmofXr6-D4,3657
+fluidattacks_agent-0.1.2.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+fluidattacks_agent-0.1.2.dist-info/RECORD,,

fluidattacks_agent-0.1.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

fluidattacks_agent.pth

@@ -0,0 +1 @@
+import fluidattacks_agent.gate