fluidattacks-agent 0.1.1__tar.gz → 0.1.2__tar.gz

{fluidattacks_agent-0.1.1 → fluidattacks_agent-0.1.2}/.gitignore

@@ -1,4 +1,4 @@
-# Rust build artifacts (top-level crate and the ebpf/ sub-crates)
+# Rust build artifacts
 target/
 
 # Python tooling caches from the probe's lint and test targets

{fluidattacks_agent-0.1.1 → fluidattacks_agent-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: fluidattacks-agent
-Version: 0.1.1
+Version: 0.1.2
 Summary: In-process probe reporting what a Python workload imports and runs
 Project-URL: Homepage, https://fluidattacks.com
 Project-URL: Source, https://gitlab.com/fluidattacks/universe/-/tree/trunk/watches/agents/python

{fluidattacks_agent-0.1.1 → fluidattacks_agent-0.1.2}/fluidattacks_agent/deliver.py

@@ -46,6 +46,12 @@ class Deliverer:
     guard: threading.Lock = field(default_factory=threading.Lock, repr=False)
     # one pass at a time: what is taken but not delivered lives in one place
     passing: threading.Lock = field(default_factory=threading.Lock, repr=False)
+    # told once a period went by with the far end in reach and nothing left to
+    # carry, so what the probe still holds goes out before it is any older
+    remind: Callable[[], None] = field(default=lambda: None, repr=False)
+    # when the probe was last reminded: a far end that refuses and then takes
+    # ends a wait every standoff, and a wait ending is not a period going by
+    reminded: float = 0.0
     leaving: bool = False
     delivered: int = 0
 
@@ -75,10 +81,23 @@ class Deliverer:
         while not self.leaving:
             # a thread that dies takes delivery with it and tells nobody
             with contextlib.suppress(Exception):
-                self.woken.wait(self._waiting())
+                roused = self.woken.wait(self._waiting())
                 self.woken.clear()
-                if time.monotonic() >= self.due:
-                    self.offering()
+                if time.monotonic() < self.due:
+                    continue
+                self.offering()
+                # a period, never a wake: a wake says there is something to
+                # carry, and asking for more on one made each flush bring the
+                # next. Nothing in hand, or the far end is still refusing
+                now = time.monotonic()
+                if (
+                    not roused
+                    and self.holding is None
+                    and not self.leaving
+                    and now - self.reminded >= self.interval
+                ):
+                    self.reminded = now
+                    self.remind()
 
     def parting(self) -> None:
         """Give whatever is held one bounded chance to travel, then give up."""
@@ -109,6 +128,7 @@ class Deliverer:
         # every worker of a pre-forking server was born in the same second
         self.due = 0.0
         self.tries = 0
+        self.reminded = 0.0
         # a child is not on its way out merely because its parent was
         self.leaving = False
         self.batcher.forked()

{fluidattacks_agent-0.1.1 → fluidattacks_agent-0.1.2}/fluidattacks_agent/executions.py

@@ -13,7 +13,9 @@ from fluidattacks_agent.report import Ecosystem, Evidence, Granularity, Record,
 # ignores it, so nothing changes until the floor moves
 __lazy_modules__ = ["pathlib"]
 
-MAX_FUNCTIONS: Final = 4096
+# a real application runs some 26,000 distinct functions while it starts, and
+# what filled the ceiling is what never gets named again for the process's life
+MAX_FUNCTIONS: Final = 32768
 
 OURS: Final = "fluidattacks_agent"
 

{fluidattacks_agent-0.1.1 → fluidattacks_agent-0.1.2}/fluidattacks_agent/startup.py

@@ -1,5 +1,6 @@
 """Start the probe inside a workload and leave reports where the agent drains."""
 
+import _thread
 import atexit
 import contextlib
 import os
@@ -19,7 +20,8 @@ from fluidattacks_agent.settings import Delivery, asked, delivery
 from fluidattacks_agent.sink import FileSink, Holding, Sink, Stalled
 from fluidattacks_agent.switch import switched_on
 
-FLUSH_EVERY: Final = 32
+# first sightings between flushes. The carrier's clock brings the rest
+FLUSH_EVERY: Final = 128
 
 # what a site directory is given so that installing the package starts the
 # probe. The agent writes the same line when it injects one it did not install
@@ -74,6 +76,13 @@ class Probe:
     # sampled when the events were armed, not when the report is written: the
     # later reading governs the next window rather than the one reported
     solitary: bool = False
+    # set off the workload's threads, acted on by the next sighting on one of
+    # them, since what is held is walked nowhere else
+    overdue: bool = False
+    # every thread sees overdue at once, so the flush is gated by a lock nothing
+    # waits on: a second caller finds it taken and returns. From _thread, which
+    # every interpreter start has already paid for, and threading is not
+    gate: _thread.LockType = field(default_factory=_thread.allocate_lock, repr=False)
 
     def find_spec(
         self,
@@ -93,10 +102,8 @@ class Probe:
     def _looked(self, fullname: str) -> None:
         if self.flushing:
             return
-        held = self.observer.suppressed
         module = self.observer.note(fullname)
-        fresh = 0 if module is None else self._mark(Evidence.IMPORTED, module)
-        self._pile(fresh or int(self.observer.suppressed != held))
+        self._pile(0 if module is None else self._mark(Evidence.IMPORTED, module))
 
     def read(self, path: str) -> None:
         """Note a file the workload opened, and report as those pile up too."""
@@ -104,21 +111,17 @@ class Probe:
         # is the workload using anything
         if self.flushing:
             return
-        held = self.loads.suppressed
         package = self.loads.note(path)
         # tested here and not inside: the hook runs on every open the host
         # makes, and all but a few are not ours to stamp
-        fresh = 0 if package is None else self._mark(Evidence.READ, package)
-        self._pile(fresh or int(self.loads.suppressed != held))
+        self._pile(0 if package is None else self._mark(Evidence.READ, package))
 
     def ran(self, filename: str, qualname: str) -> None:
         """Note code starting, and report as first sightings pile up too."""
         if self.flushing:
             return
-        held = self.runs.suppressed
         symbol = self.runs.note(filename, qualname)
-        fresh = 0 if symbol is None else self._mark(Evidence.EXECUTED, symbol)
-        self._pile(fresh or int(self.runs.suppressed != held))
+        self._pile(0 if symbol is None else self._mark(Evidence.EXECUTED, symbol))
 
     def _mark(self, evidence: Evidence, symbol: str) -> int:
         """Stamp a sighting, saying whether it was the first of its symbol."""
@@ -130,11 +133,17 @@ class Probe:
         self.first[key] = seen
         return 1
 
+    def remind(self) -> None:
+        """Have the next sighting bring a flush, however few have piled up."""
+        # a flag and never a flush: the carrier's clock calls this from a
+        # thread of its own, and a flush runs on the workload's alone
+        self.overdue = True
+
     def _pile(self, added: int) -> None:
-        # a sighting with no room counts too, or a probe at its ceiling stops
-        # flushing exactly when it has the most to say
+        # first sightings only: a flush re-arms every code object, so counting
+        # one seen again, or one with no room, made each flush bring the next
         self.pending += added
-        if self.pending >= FLUSH_EVERY:
+        if self.overdue or self.pending >= FLUSH_EVERY:
             # cleared before the call, because monitoring reports the entry of
             # flush itself and would otherwise pile up and call it again
             self.pending = 0
@@ -142,9 +151,11 @@ class Probe:
 
     def flush(self) -> None:
         """Write what has not been reported, without ever raising at a caller."""
-        if self.flushing:
+        if not self.gate.acquire(blocking=False):
             return
         self.flushing = True
+        # spent here, so a reminder given while this runs brings one more flush
+        self.overdue = False
         try:
             with contextlib.suppress(Exception):
                 if self._sighted():
@@ -152,8 +163,18 @@ class Probe:
         finally:
             self.pending = 0
             self.flushing = False
+            self.gate.release()
             self._rearm()
 
+    def forked(self) -> None:
+        """Leave a child able to flush, whatever its parent was doing at the fork."""
+        # a lock another thread held at the fork is held in the child forever,
+        # and a flag it set stays set: the child has no thread left to clear it
+        self.gate = _thread.allocate_lock()
+        self.flushing = False
+        self.overdue = False
+        self.pending = 0
+
     def _owed(self) -> bool:
         """Say whether a count is waiting for a report that can state it."""
         return bool(self.refused or self.undated)
@@ -371,23 +392,28 @@ def start(sink: Sink | None = None) -> Probe | None:
         return None
 
 
-def chosen() -> Sink:
-    """Give a workload what it asked for, and something loud if it asked badly."""
+def chosen(remind: Callable[[], None]) -> Sink:
+    """
+    Give a workload what it asked for, and something loud if it asked badly.
+
+    ``remind`` is what a carrier calls, off the workload's threads, once a
+    period has gone by with the far end in reach and nothing left to carry.
+    """
     if not asked():
         return FileSink()
     told = delivery()
     if told is None:
         # half configured must be loud, not indistinguishable from quiet
         return Holding()
-    return carrier(told)
+    return carrier(told, remind)
 
 
-def carrier(told: Delivery) -> Sink:
+def carrier(told: Delivery, remind: Callable[[], None]) -> Sink:
     """Build the one thing that carries reports out, and wire it to this process."""
     # imported here, so a workload that never asked pays for none of it
     from fluidattacks_agent.deliver import Deliverer  # noqa: PLC0415
 
-    held = Deliverer(told=told)
+    held = Deliverer(told=told, remind=remind)
     # before the flush handler, since exit handlers run in reverse order
     atexit.register(held.parting)
     held.registered()
@@ -401,11 +427,11 @@ def _start(sink: Sink | None) -> Probe | None:
     if not switched_on():
         return None
     sites = site_dirs()
-    probe = Probe(
-        sink=chosen() if sink is None else sink,
-        loads=LoadObserver(sites=sites),
-        runs=RunObserver(sites=sites),
-    )
+    probe = Probe(loads=LoadObserver(sites=sites), runs=RunObserver(sites=sites))
+    # after the probe, because what carries its reports is told what to remind
+    probe.sink = chosen(probe.remind) if sink is None else sink
+    if hasattr(os, "register_at_fork"):
+        os.register_at_fork(after_in_child=probe.forked)
     install(probe)
     sys.addaudithook(watch(probe))
     probe.monitoring = observe_runs(probe)

{fluidattacks_agent-0.1.1 → fluidattacks_agent-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fluidattacks-agent"
-version = "0.1.1"
+version = "0.1.2"
 description = "In-process probe reporting what a Python workload imports and runs"
 readme = "README.md"
 authors = [{ name = "Development", email = "development@fluidattacks.com" }]