fluidattacks-agent 0.2.0__tar.gz → 0.3.0__tar.gz

{fluidattacks_agent-0.2.0 → fluidattacks_agent-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: fluidattacks-agent
-Version: 0.2.0
+Version: 0.3.0
 Summary: In-process probe reporting what a Python workload imports and runs
 Project-URL: Homepage, https://fluidattacks.com
 Project-URL: Source, https://gitlab.com/fluidattacks/universe/-/tree/trunk/watches/agents/python

fluidattacks_agent-0.3.0/fluidattacks_agent/executions.py

@@ -0,0 +1,162 @@
+"""Watch code inside a package start running, where the interpreter allows it."""
+
+import sys
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from pathlib import PurePath
+from typing import Final
+
+from fluidattacks_agent.distributions import Dist, distribution_of
+from fluidattacks_agent.report import (
+    Ecosystem,
+    Evidence,
+    Granularity,
+    Record,
+    Sighting,
+    dated,
+    merged,
+    reached,
+    sighted,
+    trimmed,
+)
+
+# 3.15 makes the import below lazy from this; every version we support
+# ignores it, so nothing changes until the floor moves
+__lazy_modules__ = ["pathlib"]
+
+# what one window holds at most: a real application runs some 26,000 distinct
+# functions while it starts, and the start is the busiest window there is. A
+# name refused past it is counted, and named again the next window it runs in
+MAX_FUNCTIONS: Final = 32768
+
+OURS: Final = "fluidattacks_agent"
+
+SOURCE: Final = ".py"
+
+INITIALIZER: Final = "__init__"
+PARENT: Final = ".."
+
+TOOL: Final = 3
+TOOL_NAME: Final = "fluidattacks-agent"
+
+
+TOOLS: Final = 6
+
+
+def available() -> bool:
+    """Say whether this interpreter can report code starting at all."""
+    return hasattr(sys, "monitoring")
+
+
+def alone(tool: int = TOOL) -> bool:
+    """Say whether this is the only tool watching: restart_events takes none."""
+    held = (sys.monitoring.get_tool(other) for other in range(TOOLS) if other != tool)
+    return not any(name is not None for name in held)
+
+
+def module_of(filename: str, sites: tuple[str, ...]) -> str | None:
+    """Name the module whose source a running code object came from."""
+    for site in sites:
+        prefix = site + "/"
+        if not filename.startswith(prefix) or not filename.endswith(SOURCE):
+            continue
+        held = PurePath(filename[len(prefix) :])
+        parts = list(held.parts)
+        if not parts or held.is_absolute() or PARENT in parts:
+            return None
+        stem = parts.pop()[: -len(SOURCE)]
+        if not stem:
+            return None
+        if stem != INITIALIZER:
+            parts.append(stem)
+        return ".".join(parts) or None
+    return None
+
+
+@dataclass
+class RunObserver:
+    """Functions that started in the window under way, and in no window before it."""
+
+    sites: tuple[str, ...] = ()
+    # the window's own: a fold takes the whole map, and the reader adds the
+    # windows up. So the bound is on a window and not on the process's life
+    seen: dict[str, Sighting] = field(default_factory=dict)
+    # what a fold took and did not report, for the next fold to offer again.
+    # The fold's own: no thread of the workload writes beside it. Bounded by
+    # the same ceiling, and what finds no room is counted with the refusals
+    owed: dict[str, Sighting] = field(default_factory=dict)
+    suppressed: int = 0
+    unkept: int = 0
+
+    def note(
+        self,
+        filename: str,
+        qualname: str,
+        clock: Callable[[], int] = reached,
+    ) -> str | None:
+        """Take note of code starting, naming the function whenever the window has room."""
+        module = module_of(filename, self.sites)
+        if module is None:
+            return None
+        if module.split(".", maxsplit=1)[0] == OURS:
+            return None
+        qualified = f"{module}.{qualname}"
+        # bound once: a fold may swap the window out meanwhile, and a sighting
+        # must land in the window it was counted against, never in the next
+        seen = self.seen
+        held = seen.get(qualified)
+        if held is None and len(seen) >= MAX_FUNCTIONS:
+            self.suppressed += 1
+            return None
+        seen[qualified] = sighted(held, clock())
+        return qualified
+
+    def drain(self) -> dict[str, Sighting]:
+        """Take the window whole, and what the last fold owed, leaving the next one empty."""
+        taken, self.seen = self.seen, {}
+        # into a map of the fold's own, so what a late sighting lands in the
+        # window taken is at most lost, and never written over by the fold
+        offered = self.owed
+        self.owed = {}
+        merged(offered, taken)
+        return offered
+
+    def restore(self, taken: Mapping[str, Sighting]) -> None:
+        """Keep for the next fold what this one took and did not report, as far as there is room."""
+        merged(self.owed, taken)
+        self.unkept += trimmed(self.owed, MAX_FUNCTIONS)
+
+    def records(
+        self,
+        taken: Mapping[str, Sighting],
+        installed: dict[str, Dist],
+        origin: int,
+    ) -> tuple[list[Record], dict[str, Sighting]]:
+        """Report every function a distribution owns, and say what the next window keeps."""
+        found: list[Record] = []
+        kept: dict[str, Sighting] = {}
+        scanned = bool(installed)
+        for qualified in sorted(taken):
+            held = taken[qualified]
+            dist = distribution_of(qualified, installed)
+            if dist is None:
+                # no distribution will own what a settled scan does not, and a
+                # scan that has found nothing yet may still own all of these
+                if not scanned:
+                    kept[qualified] = held
+                continue
+            found.append(
+                Record(
+                    ecosystem=Ecosystem.PYPI,
+                    name=dist.name,
+                    version=dist.version,
+                    confidence=dist.confidence,
+                    symbol=qualified,
+                    granularity=Granularity.FUNCTION,
+                    evidence=Evidence.EXECUTED,
+                    frequency=held.times,
+                    when=held.first,
+                    used=dated(origin, held.last),
+                ),
+            )
+        return found, kept

{fluidattacks_agent-0.2.0 → fluidattacks_agent-0.3.0}/fluidattacks_agent/loads.py

@@ -1,13 +1,25 @@
 """Account for package code the interpreter read without importing it."""
 
 import os
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
 from dataclasses import dataclass, field
 from types import ModuleType
 
 from fluidattacks_agent.distributions import Dist, distribution_of
-from fluidattacks_agent.report import Ecosystem, Evidence, Granularity, Record, keyed
+from fluidattacks_agent.report import (
+    Ecosystem,
+    Evidence,
+    Granularity,
+    Record,
+    Sighting,
+    dated,
+    merged,
+    reached,
+    sighted,
+    trimmed,
+)
 
+# what one window holds at most, and not the process's life
 MAX_PACKAGES = 4096
 
 OURS = "fluidattacks_agent"
@@ -46,51 +58,67 @@ def package_of(path: str, sites: tuple[str, ...]) -> str | None:
 
 @dataclass
 class LoadObserver:
-    """Packages whose code was read, bounded the way the reader's map is."""
+    """Packages whose code was read in the window under way."""
 
     sites: tuple[str, ...] = ()
     # how many files of each package were read, not merely whether any was
-    seen: dict[str, int] = field(default_factory=dict)
-    moved: set[str] = field(default_factory=set)
+    seen: dict[str, Sighting] = field(default_factory=dict)
+    # what a fold took and did not report, for the next fold to offer again.
+    # The fold's own: no thread of the workload writes beside it. Bounded by
+    # the same ceiling, and what finds no room is counted with the refusals
+    owed: dict[str, Sighting] = field(default_factory=dict)
     suppressed: int = 0
+    unkept: int = 0
 
-    def note(self, path: str) -> str | None:
-        """Take note of a file opened, naming the package whenever there is room."""
+    def note(self, path: str, clock: Callable[[], int] = reached) -> str | None:
+        """Take note of a file opened, naming the package whenever the window has room."""
         package = package_of(path, self.sites)
         if package is None or package == OURS:
             return None
-        if package in self.seen:
-            self.seen[package] += 1
-            self.moved.add(package)
-            return package
-        if len(self.seen) >= MAX_PACKAGES:
+        seen = self.seen
+        held = seen.get(package)
+        if held is None and len(seen) >= MAX_PACKAGES:
             self.suppressed += 1
             return None
-        self.seen[package] = 1
-        self.moved.add(package)
+        seen[package] = sighted(held, clock())
         return package
 
+    def drain(self) -> dict[str, Sighting]:
+        """Take the window whole, and what the last fold owed, leaving the next one empty."""
+        taken, self.seen = self.seen, {}
+        # into a map of the fold's own, so what a late sighting lands in the
+        # window taken is at most lost, and never written over by the fold
+        offered = self.owed
+        self.owed = {}
+        merged(offered, taken)
+        return offered
+
+    def restore(self, taken: Mapping[str, Sighting]) -> None:
+        """Keep for the next fold what this one took and did not report, as far as there is room."""
+        merged(self.owed, taken)
+        self.unkept += trimmed(self.owed, MAX_PACKAGES)
+
     def records(
         self,
+        taken: Mapping[str, Sighting],
         modules: dict[str, ModuleType],
         installed: dict[str, Dist],
-        carried: Mapping[tuple[str, str], int],
-    ) -> list[Record]:
-        """Report the packages that were read and never became modules."""
+        origin: int,
+    ) -> tuple[list[Record], dict[str, Sighting]]:
+        """Report the packages read that became no module, and say what the next window keeps."""
         found: list[Record] = []
+        kept: dict[str, Sighting] = {}
         scanned = bool(installed)
-        for package in sorted(self.moved):
-            times = self.seen[package]
-            if times <= carried.get(keyed(Evidence.READ, package), 0):
-                self.moved.discard(package)
-                continue
+        for package in sorted(taken):
+            held = taken[package]
             if package in modules:
-                # kept, because a name can leave sys.modules and be read again
+                # the stronger evidence has it. A name that leaves sys.modules
+                # and is read again is a sighting of the window that reads it
                 continue
             dist = distribution_of(package, installed)
             if dist is None:
-                if scanned:
-                    self.moved.discard(package)
+                if not scanned:
+                    kept[package] = held
                 continue
             found.append(
                 Record(
@@ -103,7 +131,9 @@ class LoadObserver:
                     # module it is would claim the interpreter reached it
                     granularity=Granularity.PACKAGE,
                     evidence=Evidence.READ,
-                    frequency=times,
+                    frequency=held.times,
+                    when=held.first,
+                    used=dated(origin, held.last),
                 ),
             )
-        return found
+        return found, kept

{fluidattacks_agent-0.2.0 → fluidattacks_agent-0.3.0}/fluidattacks_agent/observer.py

@@ -1,16 +1,27 @@
 """Watch what a workload imports, without taking part in importing it."""
 
 import sys
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
 from dataclasses import dataclass, field
 from types import ModuleType
 from typing import Protocol
 
 from fluidattacks_agent.distributions import Dist, distribution_of
-from fluidattacks_agent.report import Ecosystem, Evidence, Granularity, Record, keyed
-
-# the probe rides inside a workload we do not own, so its bookkeeping is
-# bounded the same way the reader's is
+from fluidattacks_agent.report import (
+    Ecosystem,
+    Evidence,
+    Granularity,
+    Record,
+    Sighting,
+    dated,
+    merged,
+    reached,
+    sighted,
+    trimmed,
+)
+
+# the probe rides inside a workload we do not own, so what one window holds is
+# bounded, and the bound is on the window rather than on the process's life
 MAX_MODULES = 4096
 
 # our own package would otherwise land in the workload's inventory
@@ -27,29 +38,29 @@ class ImportObserver:
     finders behind it to decide the outcome.
     """
 
-    # how many times each module was looked up, by its whole dotted name, so a
-    # record can name the module that was seen and not only the package holding
-    # it. The name is what bounds the ceiling below
-    seen: dict[str, int] = field(default_factory=dict)
-    moved: set[str] = field(default_factory=set)
-    # modules seen past the ceiling, so the loss is never silent
+    # how many times each module was looked up in the window under way, by its
+    # whole dotted name, so a record can name the module that was seen and not
+    # only the package holding it. The name is what bounds the ceiling below
+    seen: dict[str, Sighting] = field(default_factory=dict)
+    # what a fold took and did not report, for the next fold to offer again.
+    # The fold's own: no thread of the workload writes beside it. Bounded by
+    # the same ceiling, and what finds no room is counted with the refusals
+    owed: dict[str, Sighting] = field(default_factory=dict)
     suppressed: int = 0
+    unkept: int = 0
 
-    def note(self, fullname: str) -> str | None:
-        """Take note of a module looked up, naming it whenever there is room."""
+    def note(self, fullname: str, clock: Callable[[], int] = reached) -> str | None:
+        """Take note of a module looked up, naming it whenever the window has room."""
         if fullname.split(".", maxsplit=1)[0] == OURS:
             return None
-        if fullname in self.seen:
-            self.seen[fullname] += 1
-            self.moved.add(fullname)
-            return fullname
-        # the ceiling bounds how many distinct names are held, not how often a
-        # name already held is counted
-        if len(self.seen) >= MAX_MODULES:
+        seen = self.seen
+        held = seen.get(fullname)
+        # the ceiling bounds how many distinct names a window holds, not how
+        # often a name already held is counted
+        if held is None and len(seen) >= MAX_MODULES:
             self.suppressed += 1
             return None
-        self.seen[fullname] = 1
-        self.moved.add(fullname)
+        seen[fullname] = sighted(held, clock())
         return fullname
 
     def find_spec(
@@ -61,38 +72,53 @@ class ImportObserver:
         """Note the module being looked up, then defer to the finders behind."""
         self.note(fullname)
 
+    def drain(self) -> dict[str, Sighting]:
+        """Take the window whole, and what the last fold owed, leaving the next one empty."""
+        taken, self.seen = self.seen, {}
+        # into a map of the fold's own, so what a late sighting lands in the
+        # window taken is at most lost, and never written over by the fold
+        offered = self.owed
+        self.owed = {}
+        merged(offered, taken)
+        return offered
+
+    def restore(self, taken: Mapping[str, Sighting]) -> None:
+        """Keep for the next fold what this one took and did not report, as far as there is room."""
+        merged(self.owed, taken)
+        self.unkept += trimmed(self.owed, MAX_MODULES)
+
     def records(
         self,
+        taken: Mapping[str, Sighting],
         modules: dict[str, ModuleType],
         installed: dict[str, Dist],
-        carried: Mapping[tuple[str, str], int],
-    ) -> list[Record]:
+        origin: int,
+    ) -> tuple[list[Record], dict[str, Sighting]]:
         """
         Name the distribution behind every module that finished importing.
 
         A lookup is only an attempt: an import that raised never reaches
         ``modules``, and reporting it would claim a package loaded when it did
         not. Intersecting the two is what makes this evidence rather than
-        intent.
+        intent. What has not landed yet is what the next window keeps.
 
         Both maps are taken by the caller so this stays a pure fold, and so the
         distribution scan happens once per report instead of once per module.
         """
         found: list[Record] = []
+        kept: dict[str, Sighting] = {}
         scanned = bool(installed)
-        for name in sorted(self.moved):
-            times = self.seen[name]
-            if times <= carried.get(keyed(Evidence.IMPORTED, name), 0):
-                self.moved.discard(name)
-                continue
+        for name in sorted(taken):
+            held = taken[name]
             module = modules.get(name)
             if module is None:
-                # kept, because a lookup mid-import finishes after this flush
+                # kept, because a lookup mid-import finishes after this fold
+                kept[name] = held
                 continue
             dist = distribution_of(name, installed)
             if dist is None:
-                if scanned:
-                    self.moved.discard(name)
+                if not scanned:
+                    kept[name] = held
                 continue
             found.append(
                 Record(
@@ -103,10 +129,12 @@ class ImportObserver:
                     symbol=name,
                     granularity=granularity_of(module),
                     evidence=Evidence.IMPORTED,
-                    frequency=times,
+                    frequency=held.times,
+                    when=held.first,
+                    used=dated(origin, held.last),
                 ),
             )
-        return found
+        return found, kept
 
 
 def granularity_of(module: ModuleType) -> Granularity:

{fluidattacks_agent-0.2.0 → fluidattacks_agent-0.3.0}/fluidattacks_agent/report.py

@@ -7,9 +7,11 @@ a signal about the wire rather than about us.
 """
 
 import re
+import time
+from collections.abc import Mapping
 from dataclasses import dataclass
 from enum import Enum
-from typing import Final
+from typing import Final, NamedTuple
 
 REPORT_TAG: Final = "watches-exec/1"
 ABSENT: Final = "-"
@@ -35,6 +37,15 @@ USED: Final = "u"
 MAX_WHEN: Final = 3_153_600_000_000
 MAX_USED: Final = 4_102_444_800_000
 
+MS: Final = 1_000_000
+SECOND: Final = 1000
+
+START: Final = time.monotonic_ns()
+
+# the reader reads u= as milliseconds since the epoch. Taken once and advanced
+# by the monotonic clock, so a stepped clock cannot walk a stamp backwards
+ORIGIN: Final = time.time_ns() // MS
+
 # a kind outside the three rungs: the six fixed columns describe a symbol, and
 # this describes the window the symbols were seen in
 WINDOW: Final = "s"
@@ -162,9 +173,52 @@ class Report:
     held: tuple[Record, ...] = ()
 
 
-def keyed(evidence: Evidence, symbol: str) -> tuple[str, str]:
-    """Name a record the way a probe holds what it has already carried."""
-    return (evidence.value, symbol)
+def reached() -> int:
+    """Say how far into the probe's own life this moment is, in milliseconds."""
+    return (time.monotonic_ns() - START) // MS
+
+
+def dated(origin: int, at: int) -> int:
+    """Date a moment of the probe's life by the epoch, to the second."""
+    # what drains a report holds its own clock to whole seconds and drops any
+    # date past it, so a finer one here is one it can only throw away
+    return (origin + at) // SECOND * SECOND
+
+
+class Sighting(NamedTuple):
+    """What a window holds of one symbol: how often, and when it was first and last seen."""
+
+    times: int
+    first: int
+    last: int
+
+
+def sighted(held: Sighting | None, now: int) -> Sighting:
+    """Fold one more sighting into what the window holds of its symbol."""
+    return Sighting(1, now, now) if held is None else Sighting(held.times + 1, held.first, now)
+
+
+def merged(into: dict[str, Sighting], taken: Mapping[str, Sighting]) -> None:
+    """Fold what one window holds into another."""
+    # a copy of the items, taken in one step: the window taken may still be
+    # written by a thread that bound it before the fold swapped it out
+    for symbol, held in list(taken.items()):
+        kept = into.get(symbol)
+        into[symbol] = (
+            held
+            if kept is None
+            else Sighting(
+                kept.times + held.times,
+                min(kept.first, held.first),
+                max(kept.last, held.last),
+            )
+        )
+
+
+def trimmed(held: dict[str, Sighting], ceiling: int) -> int:
+    """Drop what a window holds past its ceiling, and say how many sightings that was."""
+    beyond = list(held)[ceiling:]
+    return sum(held.pop(symbol).times for symbol in beyond)
 
 
 def _writable(value: str, limit: int) -> bool:

{fluidattacks_agent-0.2.0 → fluidattacks_agent-0.3.0}/fluidattacks_agent/startup.py

@@ -5,9 +5,8 @@ import atexit
 import contextlib
 import os
 import sys
-import time
 from collections.abc import Callable
-from dataclasses import dataclass, field, replace
+from dataclasses import dataclass, field
 from types import CodeType, ModuleType
 from typing import TYPE_CHECKING, Final
 
@@ -15,7 +14,15 @@ from fluidattacks_agent.distributions import Dist, distribution_map
 from fluidattacks_agent.executions import TOOL, TOOL_NAME, RunObserver, alone, available
 from fluidattacks_agent.loads import LoadObserver
 from fluidattacks_agent.observer import ImportObserver, install
-from fluidattacks_agent.report import Evidence, Record, Window, keyed, render
+from fluidattacks_agent.report import (
+    ORIGIN,
+    Evidence,
+    Record,
+    Sighting,
+    Window,
+    reached,
+    render,
+)
 from fluidattacks_agent.settings import Delivery, delivery
 from fluidattacks_agent.sink import PARTING, Sink, Stalled
 from fluidattacks_agent.switch import switched_on
@@ -30,24 +37,10 @@ PTH_NAME: Final = "fluidattacks_agent.pth"
 # reading the switch and for nothing above it
 PTH_LINE: Final = "import fluidattacks_agent.gate\n"
 
-MS: Final = 1_000_000
-SECOND: Final = 1000
-
 # what a workload's fork may be delayed by while a fold under way ends: a fold
 # imports, and a lock held across a fork is held in the child forever
 FORKING: Final = 0.5
 
-START: Final = time.monotonic_ns()
-
-# the reader reads u= as milliseconds since the epoch. Taken once and advanced
-# by the monotonic clock, so a stepped clock cannot walk a stamp backwards
-ORIGIN: Final = time.time_ns() // MS
-
-
-def reached() -> int:
-    """Say how far into the probe's own life this moment is, in milliseconds."""
-    return (time.monotonic_ns() - START) // MS
-
 
 @dataclass
 class Probe:
@@ -60,12 +53,10 @@ class Probe:
     distributions: Callable[[], dict[str, Dist]] = distribution_map
     clock: Callable[[], int] = reached
     origin: int = ORIGIN
-    # what each record was last reported as, so a flush carries the increment
-    reported: dict[tuple[str, str], int] = field(default_factory=dict)
-    # the same, for the sightings each observer found no room for
+    # what the sightings each observer found no room for were last stated as, so
+    # a report carries the increment. The sightings themselves are never held
+    # past the window that saw them: the reader adds the windows up
     stated: dict[str, int] = field(default_factory=dict)
-    first: dict[tuple[str, str], int] = field(default_factory=dict)
-    last: dict[tuple[str, str], int] = field(default_factory=dict)
     scanned: dict[str, Dist] | None = None
     # set while a fold runs, with the thread running it: that thread's own
     # reads and imports are the probe's, and every other thread's are the
@@ -108,34 +99,21 @@ class Probe:
     def _looked(self, fullname: str) -> None:
         if self._folding():
             return
-        module = self.observer.note(fullname)
-        if module is not None:
-            self._mark(Evidence.IMPORTED, module)
+        self.observer.note(fullname, self.clock)
 
     def read(self, path: str) -> None:
         """Note a file the workload opened."""
         if self._folding():
             return
-        package = self.loads.note(path)
-        # tested here and not inside: the hook runs on every open the host
-        # makes, and all but a few are not ours to stamp
-        if package is not None:
-            self._mark(Evidence.READ, package)
+        # the hook runs on every open the host makes, and all but a few are ours
+        # to stamp, so the clock is read inside and only for those
+        self.loads.note(path, self.clock)
 
     def ran(self, filename: str, qualname: str) -> None:
         """Note code starting."""
         if self._folding():
             return
-        symbol = self.runs.note(filename, qualname)
-        if symbol is not None:
-            self._mark(Evidence.EXECUTED, symbol)
-
-    def _mark(self, evidence: Evidence, symbol: str) -> None:
-        """Stamp a sighting: when its symbol was first reached, and this once."""
-        key = keyed(evidence, symbol)
-        seen = self.clock()
-        self.last[key] = seen
-        self.first.setdefault(key, seen)
+        self.runs.note(filename, qualname, self.clock)
 
     def flush(self) -> None:
         """Write what has not been reported, waiting out a fold under way."""
@@ -188,15 +166,16 @@ class Probe:
 
     def _sighted(self) -> bool:
         """Say whether this window holds anything a report could carry."""
-        # what moved and not what is held, since only that can be a record; and
-        # a copy of it, since the workload's threads add to it meanwhile
+        # a copy of the lookups, since the workload's threads add to them meanwhile
         return bool(
             self._owed()
-            or self.loads.moved
-            or self.runs.moved
-            # a lookup that never became a module is dropped when the records are
+            or self.loads.seen
+            or self.loads.owed
+            or self.runs.seen
+            or self.runs.owed
+            # a lookup that never became a module is kept when the records are
             # folded, so on its own it is not cause to walk any metadata
-            or any(name in sys.modules for name in list(self.observer.moved))
+            or any(name in sys.modules for name in [*list(self.observer.seen), *self.observer.owed])
             or any(self._beyond().values())
             or self.stalled != self.charged,
         )
@@ -227,16 +206,66 @@ class Probe:
         return scanned
 
     def _write_fresh(self, installed: dict[str, Dist]) -> None:
-        seen = [
-            *self.loads.records(sys.modules, installed, self.reported),
-            *self.observer.records(sys.modules, installed, self.reported),
-            *self.runs.records(installed, self.reported),
-        ]
-        fresh = [record for record in map(self._since, seen) if record is not None]
+        # every window is taken whole before any is read, so a sighting the
+        # workload makes meanwhile lands in the next one and in nothing older
+        taken = {
+            Evidence.READ: self.loads.drain(),
+            Evidence.IMPORTED: self.observer.drain(),
+            Evidence.EXECUTED: self.runs.drain(),
+        }
+        try:
+            read, keep_read = self.loads.records(
+                taken[Evidence.READ],
+                sys.modules,
+                installed,
+                self.origin,
+            )
+            imported, keep_imported = self.observer.records(
+                taken[Evidence.IMPORTED],
+                sys.modules,
+                installed,
+                self.origin,
+            )
+            ran, keep_ran = self.runs.records(taken[Evidence.EXECUTED], installed, self.origin)
+        except Exception:
+            # nothing taken is lost to a fold that could not even read it
+            self._restore(taken)
+            raise
+        # given back at once: the next window keeps these whatever this fold's fate
+        self._restore(
+            {
+                Evidence.READ: keep_read,
+                Evidence.IMPORTED: keep_imported,
+                Evidence.EXECUTED: keep_ran,
+            },
+        )
+        fresh = [*read, *imported, *ran]
         counted = self._beyond()
         beyond = {tag: n - self.stated.get(tag, 0) for tag, n in counted.items()}
         if not fresh and not any(beyond.values()) and not self._owed():
             return
+        # what this fold offers and has not landed yet, by rung: a record a
+        # report never carried goes back to its window rather than being lost
+        offered = {
+            evidence: {record.symbol: taken[evidence][record.symbol] for record in records}
+            for evidence, records in (
+                (Evidence.READ, read),
+                (Evidence.IMPORTED, imported),
+                (Evidence.EXECUTED, ran),
+            )
+        }
+        try:
+            self._land(fresh, beyond, counted, offered)
+        finally:
+            self._restore(offered)
+
+    def _land(
+        self,
+        fresh: list[Record],
+        beyond: dict[str, int],
+        counted: dict[str, int],
+        offered: dict[Evidence, dict[str, Sighting]],
+    ) -> None:
         # the reader sums what every report states, so this is the increment
         stalled = self.stalled
         window: Window | None = self._window(beyond, stalled - self.charged)
@@ -244,7 +273,9 @@ class Probe:
         while True:
             report = render(pending, window, self.refused, self.undated)
             self._write(report.text)
-            self._charge(pending[: len(pending) - len(report.held)])
+            # landed, or dropped and counted: neither is the window's any more
+            for record in pending[: len(pending) - len(report.held)]:
+                offered[record.evidence].pop(record.symbol, None)
             held = list(report.held)
             self.refused = report.dropped if held else 0
             self.undated = report.unstamped if held else 0
@@ -258,16 +289,17 @@ class Probe:
                 return
             pending = held
 
-    def _charge(self, written: list[Record]) -> None:
-        for record in written:
-            key = _key(record)
-            self.reported[key] = self.reported.get(key, 0) + record.frequency
+    def _restore(self, kept: dict[Evidence, dict[str, Sighting]]) -> None:
+        self.loads.restore(kept[Evidence.READ])
+        self.observer.restore(kept[Evidence.IMPORTED])
+        self.runs.restore(kept[Evidence.EXECUTED])
 
     def _beyond(self) -> dict[str, int]:
+        # what a window had no room for, and what a fold had no room to keep
         return {
-            Evidence.READ.value: self.loads.suppressed,
-            Evidence.IMPORTED.value: self.observer.suppressed,
-            Evidence.EXECUTED.value: self.runs.suppressed,
+            Evidence.READ.value: self.loads.suppressed + self.loads.unkept,
+            Evidence.IMPORTED.value: self.observer.suppressed + self.observer.unkept,
+            Evidence.EXECUTED.value: self.runs.suppressed + self.runs.unkept,
         }
 
     def _window(self, beyond: dict[str, int], backlog: int) -> Window:
@@ -280,30 +312,6 @@ class Probe:
             alone=self.monitoring and self.solitary,
         )
 
-    def _since(self, record: Record) -> Record | None:
-        # the reader adds up what every report of a record says, so a report
-        # carries what has happened since the last one and never the running
-        # total, which would count each sighting once per flush that follows it
-        key = _key(record)
-        since = record.frequency - self.reported.get(key, 0)
-        if since < 1:
-            return None
-        return replace(
-            record,
-            frequency=since,
-            when=self.first.get(key),
-            used=self._used(key),
-        )
-
-    def _used(self, key: tuple[str, str]) -> int | None:
-        """Date the last sighting of a symbol as of this report, to the second."""
-        seen = self.last.get(key)
-        if seen is None:
-            return None
-        # what drains a report holds its own clock to whole seconds and drops
-        # any date past it, so a finer one here is one it can only throw away
-        return (self.origin + seen) // SECOND * SECOND
-
     def _write(self, text: str) -> None:
         try:
             self.sink.write(text)
@@ -314,10 +322,6 @@ class Probe:
             raise
 
 
-def _key(record: Record) -> tuple[str, str]:
-    return keyed(record.evidence, record.symbol)
-
-
 def site_dirs() -> tuple[str, ...]:
     """Name the directories the workload installs its distributions into."""
     found: list[str] = []

{fluidattacks_agent-0.2.0 → fluidattacks_agent-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fluidattacks-agent"
-version = "0.2.0"
+version = "0.3.0"
 description = "In-process probe reporting what a Python workload imports and runs"
 readme = "README.md"
 authors = [{ name = "Development", email = "development@fluidattacks.com" }]

fluidattacks_agent-0.2.0/fluidattacks_agent/executions.py

@@ -1,124 +0,0 @@
-"""Watch code inside a package start running, where the interpreter allows it."""
-
-import sys
-from collections.abc import Mapping
-from dataclasses import dataclass, field
-from pathlib import PurePath
-from typing import Final
-
-from fluidattacks_agent.distributions import Dist, distribution_of
-from fluidattacks_agent.report import Ecosystem, Evidence, Granularity, Record, keyed
-
-# 3.15 makes the import below lazy from this; every version we support
-# ignores it, so nothing changes until the floor moves
-__lazy_modules__ = ["pathlib"]
-
-# a real application runs some 26,000 distinct functions while it starts, and
-# what filled the ceiling is what never gets named again for the process's life
-MAX_FUNCTIONS: Final = 32768
-
-OURS: Final = "fluidattacks_agent"
-
-SOURCE: Final = ".py"
-
-INITIALIZER: Final = "__init__"
-PARENT: Final = ".."
-
-TOOL: Final = 3
-TOOL_NAME: Final = "fluidattacks-agent"
-
-
-TOOLS: Final = 6
-
-
-def available() -> bool:
-    """Say whether this interpreter can report code starting at all."""
-    return hasattr(sys, "monitoring")
-
-
-def alone(tool: int = TOOL) -> bool:
-    """Say whether this is the only tool watching: restart_events takes none."""
-    held = (sys.monitoring.get_tool(other) for other in range(TOOLS) if other != tool)
-    return not any(name is not None for name in held)
-
-
-def module_of(filename: str, sites: tuple[str, ...]) -> str | None:
-    """Name the module whose source a running code object came from."""
-    for site in sites:
-        prefix = site + "/"
-        if not filename.startswith(prefix) or not filename.endswith(SOURCE):
-            continue
-        held = PurePath(filename[len(prefix) :])
-        parts = list(held.parts)
-        if not parts or held.is_absolute() or PARENT in parts:
-            return None
-        stem = parts.pop()[: -len(SOURCE)]
-        if not stem:
-            return None
-        if stem != INITIALIZER:
-            parts.append(stem)
-        return ".".join(parts) or None
-    return None
-
-
-@dataclass
-class RunObserver:
-    """Functions that started, bounded the way the reader's own map is."""
-
-    sites: tuple[str, ...] = ()
-    seen: dict[str, int] = field(default_factory=dict)
-    moved: set[str] = field(default_factory=set)
-    suppressed: int = 0
-
-    def note(self, filename: str, qualname: str) -> str | None:
-        """Take note of code starting, naming the function whenever there is room."""
-        module = module_of(filename, self.sites)
-        if module is None:
-            return None
-        if module.split(".", maxsplit=1)[0] == OURS:
-            return None
-        qualified = f"{module}.{qualname}"
-        if qualified in self.seen:
-            self.seen[qualified] += 1
-            self.moved.add(qualified)
-            return qualified
-        if len(self.seen) >= MAX_FUNCTIONS:
-            self.suppressed += 1
-            return None
-        self.seen[qualified] = 1
-        self.moved.add(qualified)
-        return qualified
-
-    def records(
-        self,
-        installed: dict[str, Dist],
-        carried: Mapping[tuple[str, str], int],
-    ) -> list[Record]:
-        """Report every function whose package a distribution installed."""
-        found: list[Record] = []
-        scanned = bool(installed)
-        for qualified in sorted(self.moved):
-            times = self.seen[qualified]
-            # what a report carried comes back only if note sees it again,
-            # and no distribution will own what a settled scan does not
-            if times <= carried.get(keyed(Evidence.EXECUTED, qualified), 0):
-                self.moved.discard(qualified)
-                continue
-            dist = distribution_of(qualified, installed)
-            if dist is None:
-                if scanned:
-                    self.moved.discard(qualified)
-                continue
-            found.append(
-                Record(
-                    ecosystem=Ecosystem.PYPI,
-                    name=dist.name,
-                    version=dist.version,
-                    confidence=dist.confidence,
-                    symbol=qualified,
-                    granularity=Granularity.FUNCTION,
-                    evidence=Evidence.EXECUTED,
-                    frequency=times,
-                ),
-            )
-        return found