fluidattacks-agent 0.1.2__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/PKG-INFO +7 -8
  2. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/README.md +6 -7
  3. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/deliver.py +37 -33
  4. fluidattacks_agent-0.3.0/fluidattacks_agent/executions.py +162 -0
  5. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/loads.py +57 -27
  6. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/observer.py +63 -35
  7. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/post.py +10 -0
  8. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/report.py +58 -4
  9. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/settings.py +0 -9
  10. fluidattacks_agent-0.3.0/fluidattacks_agent/sink.py +16 -0
  11. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/startup.py +178 -174
  12. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/pyproject.toml +1 -1
  13. fluidattacks_agent-0.1.2/fluidattacks_agent/executions.py +0 -124
  14. fluidattacks_agent-0.1.2/fluidattacks_agent/sink.py +0 -163
  15. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/.gitignore +0 -0
  16. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/__init__.py +0 -0
  17. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/batch.py +0 -0
  18. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/distributions.py +0 -0
  19. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/gate.py +0 -0
  20. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/outbox.py +0 -0
  21. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/patience.py +0 -0
  22. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/switch.py +0 -0
  23. {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent.pth +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: fluidattacks-agent
3
- Version: 0.1.2
3
+ Version: 0.3.0
4
4
  Summary: In-process probe reporting what a Python workload imports and runs
5
5
  Project-URL: Homepage, https://fluidattacks.com
6
6
  Project-URL: Source, https://gitlab.com/fluidattacks/universe/-/tree/trunk/watches/agents/python
@@ -26,7 +26,7 @@ dependency inventory can say which of its findings are reachable at runtime and
26
26
  which are not.
27
27
 
28
28
  It is a library, not a service. It observes the interpreter it is installed in,
29
- writes what it saw, and does nothing else. It takes no dependencies: the
29
+ sends what it saw, and does nothing else. It takes no dependencies: the
30
30
  standard library only, because it is installed into workloads we do not own.
31
31
 
32
32
  ## Installing
@@ -60,8 +60,8 @@ a workload's own first-party code produce no records.
60
60
 
61
61
  ## Where reports go
62
62
 
63
- By default, one file per report under `/tmp/.watches-exec`, for a collector to
64
- drain. Naming an endpoint sends them instead:
63
+ Naming an endpoint is what makes the probe observe at all. A workload that
64
+ names none starts nothing:
65
65
 
66
66
  | | |
67
67
  |---|---|
@@ -74,10 +74,9 @@ drain. Naming an endpoint sends them instead:
74
74
  A file is preferred over a variable because a file can be mode 400, while an
75
75
  environment variable is readable by any process of the same user.
76
76
 
77
- An endpoint named without enough beside it to reach is a misconfiguration, not a
78
- reason to fall back: the probe then holds nothing and counts every report it
79
- refused, so a half-configured deployment is loud rather than a directory filling
80
- up where nobody drains it.
77
+ An endpoint named without enough beside it to reach — no credential, or not
78
+ `https://` — is a misconfiguration, and the probe starts nothing rather than
79
+ observe what it could never deliver.
81
80
 
82
81
  ## What travels, and what does not
83
82
 
@@ -5,7 +5,7 @@ dependency inventory can say which of its findings are reachable at runtime and
5
5
  which are not.
6
6
 
7
7
  It is a library, not a service. It observes the interpreter it is installed in,
8
- writes what it saw, and does nothing else. It takes no dependencies: the
8
+ sends what it saw, and does nothing else. It takes no dependencies: the
9
9
  standard library only, because it is installed into workloads we do not own.
10
10
 
11
11
  ## Installing
@@ -39,8 +39,8 @@ a workload's own first-party code produce no records.
39
39
 
40
40
  ## Where reports go
41
41
 
42
- By default, one file per report under `/tmp/.watches-exec`, for a collector to
43
- drain. Naming an endpoint sends them instead:
42
+ Naming an endpoint is what makes the probe observe at all. A workload that
43
+ names none starts nothing:
44
44
 
45
45
  | | |
46
46
  |---|---|
@@ -53,10 +53,9 @@ drain. Naming an endpoint sends them instead:
53
53
  A file is preferred over a variable because a file can be mode 400, while an
54
54
  environment variable is readable by any process of the same user.
55
55
 
56
- An endpoint named without enough beside it to reach is a misconfiguration, not a
57
- reason to fall back: the probe then holds nothing and counts every report it
58
- refused, so a half-configured deployment is loud rather than a directory filling
59
- up where nobody drains it.
56
+ An endpoint named without enough beside it to reach — no credential, or not
57
+ `https://` — is a misconfiguration, and the probe starts nothing rather than
58
+ observe what it could never deliver.
60
59
 
61
60
  ## What travels, and what does not
62
61
 
@@ -12,18 +12,18 @@ from typing import Final
12
12
  from fluidattacks_agent.batch import Batch, Batcher
13
13
  from fluidattacks_agent.outbox import Outbox
14
14
  from fluidattacks_agent.patience import TOPMOST, fraction, splay, standoff
15
- from fluidattacks_agent.post import TIMEOUT, Verdict, posted
15
+ from fluidattacks_agent.post import TIMEOUT, Verdict, posted, prepared
16
16
  from fluidattacks_agent.settings import Delivery
17
- from fluidattacks_agent.sink import Stalled
17
+ from fluidattacks_agent.sink import PARTING, Stalled
18
18
 
19
- # how old the centre's picture may be while nothing new is being found
19
+ # how old the centre's picture may be: the period the window is folded on
20
20
  INTERVAL: Final = 15.0
21
21
 
22
- # what a workload's own exit may be delayed by while what is held goes out
23
- PARTING: Final = 2.0
24
-
25
22
  NAME: Final = "fluidattacks-agent"
26
23
 
24
+ # before any fold, so the first wait to end folds whatever the probe holds
25
+ NEVER: Final = float("-inf")
26
+
27
27
 
28
28
  @dataclass
29
29
  class Deliverer:
@@ -46,23 +46,27 @@ class Deliverer:
46
46
  guard: threading.Lock = field(default_factory=threading.Lock, repr=False)
47
47
  # one pass at a time: what is taken but not delivered lives in one place
48
48
  passing: threading.Lock = field(default_factory=threading.Lock, repr=False)
49
- # told once a period went by with the far end in reach and nothing left to
50
- # carry, so what the probe still holds goes out before it is any older
51
- remind: Callable[[], None] = field(default=lambda: None, repr=False)
52
- # when the probe was last reminded: a far end that refuses and then takes
53
- # ends a wait every standoff, and a wait ending is not a period going by
54
- reminded: float = 0.0
49
+ # what renders the probe's window into reports. Called here, on the
50
+ # carrier's own thread and clock, so a workload's threads render nothing
51
+ fold: Callable[[], None] = field(default=lambda: None, repr=False)
52
+ folded: float = NEVER
55
53
  leaving: bool = False
56
54
  delivered: int = 0
57
55
 
58
56
  def write(self, text: str) -> None:
59
57
  """Take a report for delivery, or refuse and be counted for it."""
58
+ # here, inside the fold that made the report, so what a post imports is
59
+ # imported under the lock a fork waits out and not on the pass after
60
+ prepared()
60
61
  if self.batcher.next(self.told, text, self.outbox.put) is None:
61
62
  raise Stalled(errno.ENOSPC, "nothing taken", len(text))
62
- self._rouse()
63
+ # a fold of the carrier's own writes here, and its pass follows in the
64
+ # same turn: only another thread has a carrier to wake
65
+ if threading.current_thread() is not self.thread:
66
+ self.rouse()
63
67
 
64
- def _rouse(self) -> None:
65
- """Say there is something to carry, and find a carrier if there is none."""
68
+ def rouse(self) -> None:
69
+ """Start the carrier if there is none, and say there is something for it."""
66
70
  with self.guard:
67
71
  # a carrier that stopped takes delivery with it, and one is not
68
72
  # worth starting for a process already on its way out
@@ -81,23 +85,17 @@ class Deliverer:
81
85
  while not self.leaving:
82
86
  # a thread that dies takes delivery with it and tells nobody
83
87
  with contextlib.suppress(Exception):
84
- roused = self.woken.wait(self._waiting())
88
+ self.woken.wait(self._waiting())
85
89
  self.woken.clear()
86
- if time.monotonic() < self.due:
90
+ now = time.monotonic()
91
+ if now < self.due:
87
92
  continue
93
+ # on this clock and no other: a far end refusing and then taking
94
+ # ends a wait every standoff, and a wait ending is not a period
95
+ if now - self.folded >= self.interval:
96
+ self.folded = now
97
+ self.fold()
88
98
  self.offering()
89
- # a period, never a wake: a wake says there is something to
90
- # carry, and asking for more on one made each flush bring the
91
- # next. Nothing in hand, or the far end is still refusing
92
- now = time.monotonic()
93
- if (
94
- not roused
95
- and self.holding is None
96
- and not self.leaving
97
- and now - self.reminded >= self.interval
98
- ):
99
- self.reminded = now
100
- self.remind()
101
99
 
102
100
  def parting(self) -> None:
103
101
  """Give whatever is held one bounded chance to travel, then give up."""
@@ -128,11 +126,15 @@ class Deliverer:
128
126
  # every worker of a pre-forking server was born in the same second
129
127
  self.due = 0.0
130
128
  self.tries = 0
131
- self.reminded = 0.0
129
+ self.folded = NEVER
132
130
  # a child is not on its way out merely because its parent was
133
131
  self.leaving = False
134
132
  self.batcher.forked()
135
133
  self.outbox.forked()
134
+ # the clock is this thread's, and the child has none until it is
135
+ # started again. Safe here because threading's own fork handler, which
136
+ # registered first, has already run
137
+ self.rouse()
136
138
 
137
139
  def registered(self) -> None:
138
140
  """Ask the interpreter to say when this process has become two."""
@@ -173,6 +175,8 @@ class Deliverer:
173
175
  self.due = time.monotonic() + standoff(self.tries, self.share())
174
176
 
175
177
  def _waiting(self) -> float:
176
- """Wait out a standoff if there is one, and otherwise a pass."""
177
- left = self.due - time.monotonic()
178
- return left if left > 0 else self.interval
178
+ """Wait out a standoff if there is one, and otherwise until the next fold."""
179
+ now = time.monotonic()
180
+ if self.due > now:
181
+ return self.due - now
182
+ return max(0.0, self.folded + self.interval - now)
@@ -0,0 +1,162 @@
1
+ """Watch code inside a package start running, where the interpreter allows it."""
2
+
3
+ import sys
4
+ from collections.abc import Callable, Mapping
5
+ from dataclasses import dataclass, field
6
+ from pathlib import PurePath
7
+ from typing import Final
8
+
9
+ from fluidattacks_agent.distributions import Dist, distribution_of
10
+ from fluidattacks_agent.report import (
11
+ Ecosystem,
12
+ Evidence,
13
+ Granularity,
14
+ Record,
15
+ Sighting,
16
+ dated,
17
+ merged,
18
+ reached,
19
+ sighted,
20
+ trimmed,
21
+ )
22
+
23
+ # 3.15 makes the import below lazy from this; every version we support
24
+ # ignores it, so nothing changes until the floor moves
25
+ __lazy_modules__ = ["pathlib"]
26
+
27
+ # what one window holds at most: a real application runs some 26,000 distinct
28
+ # functions while it starts, and the start is the busiest window there is. A
29
+ # name refused past it is counted, and named again the next window it runs in
30
+ MAX_FUNCTIONS: Final = 32768
31
+
32
+ OURS: Final = "fluidattacks_agent"
33
+
34
+ SOURCE: Final = ".py"
35
+
36
+ INITIALIZER: Final = "__init__"
37
+ PARENT: Final = ".."
38
+
39
+ TOOL: Final = 3
40
+ TOOL_NAME: Final = "fluidattacks-agent"
41
+
42
+
43
+ TOOLS: Final = 6
44
+
45
+
46
+ def available() -> bool:
47
+ """Say whether this interpreter can report code starting at all."""
48
+ return hasattr(sys, "monitoring")
49
+
50
+
51
+ def alone(tool: int = TOOL) -> bool:
52
+ """Say whether this is the only tool watching: restart_events takes none."""
53
+ held = (sys.monitoring.get_tool(other) for other in range(TOOLS) if other != tool)
54
+ return not any(name is not None for name in held)
55
+
56
+
57
+ def module_of(filename: str, sites: tuple[str, ...]) -> str | None:
58
+ """Name the module whose source a running code object came from."""
59
+ for site in sites:
60
+ prefix = site + "/"
61
+ if not filename.startswith(prefix) or not filename.endswith(SOURCE):
62
+ continue
63
+ held = PurePath(filename[len(prefix) :])
64
+ parts = list(held.parts)
65
+ if not parts or held.is_absolute() or PARENT in parts:
66
+ return None
67
+ stem = parts.pop()[: -len(SOURCE)]
68
+ if not stem:
69
+ return None
70
+ if stem != INITIALIZER:
71
+ parts.append(stem)
72
+ return ".".join(parts) or None
73
+ return None
74
+
75
+
76
+ @dataclass
77
+ class RunObserver:
78
+ """Functions that started in the window under way, and in no window before it."""
79
+
80
+ sites: tuple[str, ...] = ()
81
+ # the window's own: a fold takes the whole map, and the reader adds the
82
+ # windows up. So the bound is on a window and not on the process's life
83
+ seen: dict[str, Sighting] = field(default_factory=dict)
84
+ # what a fold took and did not report, for the next fold to offer again.
85
+ # The fold's own: no thread of the workload writes beside it. Bounded by
86
+ # the same ceiling, and what finds no room is counted with the refusals
87
+ owed: dict[str, Sighting] = field(default_factory=dict)
88
+ suppressed: int = 0
89
+ unkept: int = 0
90
+
91
+ def note(
92
+ self,
93
+ filename: str,
94
+ qualname: str,
95
+ clock: Callable[[], int] = reached,
96
+ ) -> str | None:
97
+ """Take note of code starting, naming the function whenever the window has room."""
98
+ module = module_of(filename, self.sites)
99
+ if module is None:
100
+ return None
101
+ if module.split(".", maxsplit=1)[0] == OURS:
102
+ return None
103
+ qualified = f"{module}.{qualname}"
104
+ # bound once: a fold may swap the window out meanwhile, and a sighting
105
+ # must land in the window it was counted against, never in the next
106
+ seen = self.seen
107
+ held = seen.get(qualified)
108
+ if held is None and len(seen) >= MAX_FUNCTIONS:
109
+ self.suppressed += 1
110
+ return None
111
+ seen[qualified] = sighted(held, clock())
112
+ return qualified
113
+
114
+ def drain(self) -> dict[str, Sighting]:
115
+ """Take the window whole, and what the last fold owed, leaving the next one empty."""
116
+ taken, self.seen = self.seen, {}
117
+ # into a map of the fold's own, so what a late sighting lands in the
118
+ # window taken is at most lost, and never written over by the fold
119
+ offered = self.owed
120
+ self.owed = {}
121
+ merged(offered, taken)
122
+ return offered
123
+
124
+ def restore(self, taken: Mapping[str, Sighting]) -> None:
125
+ """Keep for the next fold what this one took and did not report, as far as there is room."""
126
+ merged(self.owed, taken)
127
+ self.unkept += trimmed(self.owed, MAX_FUNCTIONS)
128
+
129
+ def records(
130
+ self,
131
+ taken: Mapping[str, Sighting],
132
+ installed: dict[str, Dist],
133
+ origin: int,
134
+ ) -> tuple[list[Record], dict[str, Sighting]]:
135
+ """Report every function a distribution owns, and say what the next window keeps."""
136
+ found: list[Record] = []
137
+ kept: dict[str, Sighting] = {}
138
+ scanned = bool(installed)
139
+ for qualified in sorted(taken):
140
+ held = taken[qualified]
141
+ dist = distribution_of(qualified, installed)
142
+ if dist is None:
143
+ # no distribution will own what a settled scan does not, and a
144
+ # scan that has found nothing yet may still own all of these
145
+ if not scanned:
146
+ kept[qualified] = held
147
+ continue
148
+ found.append(
149
+ Record(
150
+ ecosystem=Ecosystem.PYPI,
151
+ name=dist.name,
152
+ version=dist.version,
153
+ confidence=dist.confidence,
154
+ symbol=qualified,
155
+ granularity=Granularity.FUNCTION,
156
+ evidence=Evidence.EXECUTED,
157
+ frequency=held.times,
158
+ when=held.first,
159
+ used=dated(origin, held.last),
160
+ ),
161
+ )
162
+ return found, kept
@@ -1,13 +1,25 @@
1
1
  """Account for package code the interpreter read without importing it."""
2
2
 
3
3
  import os
4
- from collections.abc import Mapping
4
+ from collections.abc import Callable, Mapping
5
5
  from dataclasses import dataclass, field
6
6
  from types import ModuleType
7
7
 
8
8
  from fluidattacks_agent.distributions import Dist, distribution_of
9
- from fluidattacks_agent.report import Ecosystem, Evidence, Granularity, Record, keyed
9
+ from fluidattacks_agent.report import (
10
+ Ecosystem,
11
+ Evidence,
12
+ Granularity,
13
+ Record,
14
+ Sighting,
15
+ dated,
16
+ merged,
17
+ reached,
18
+ sighted,
19
+ trimmed,
20
+ )
10
21
 
22
+ # what one window holds at most, and not the process's life
11
23
  MAX_PACKAGES = 4096
12
24
 
13
25
  OURS = "fluidattacks_agent"
@@ -46,51 +58,67 @@ def package_of(path: str, sites: tuple[str, ...]) -> str | None:
46
58
 
47
59
  @dataclass
48
60
  class LoadObserver:
49
- """Packages whose code was read, bounded the way the reader's map is."""
61
+ """Packages whose code was read in the window under way."""
50
62
 
51
63
  sites: tuple[str, ...] = ()
52
64
  # how many files of each package were read, not merely whether any was
53
- seen: dict[str, int] = field(default_factory=dict)
54
- moved: set[str] = field(default_factory=set)
65
+ seen: dict[str, Sighting] = field(default_factory=dict)
66
+ # what a fold took and did not report, for the next fold to offer again.
67
+ # The fold's own: no thread of the workload writes beside it. Bounded by
68
+ # the same ceiling, and what finds no room is counted with the refusals
69
+ owed: dict[str, Sighting] = field(default_factory=dict)
55
70
  suppressed: int = 0
71
+ unkept: int = 0
56
72
 
57
- def note(self, path: str) -> str | None:
58
- """Take note of a file opened, naming the package whenever there is room."""
73
+ def note(self, path: str, clock: Callable[[], int] = reached) -> str | None:
74
+ """Take note of a file opened, naming the package whenever the window has room."""
59
75
  package = package_of(path, self.sites)
60
76
  if package is None or package == OURS:
61
77
  return None
62
- if package in self.seen:
63
- self.seen[package] += 1
64
- self.moved.add(package)
65
- return package
66
- if len(self.seen) >= MAX_PACKAGES:
78
+ seen = self.seen
79
+ held = seen.get(package)
80
+ if held is None and len(seen) >= MAX_PACKAGES:
67
81
  self.suppressed += 1
68
82
  return None
69
- self.seen[package] = 1
70
- self.moved.add(package)
83
+ seen[package] = sighted(held, clock())
71
84
  return package
72
85
 
86
+ def drain(self) -> dict[str, Sighting]:
87
+ """Take the window whole, and what the last fold owed, leaving the next one empty."""
88
+ taken, self.seen = self.seen, {}
89
+ # into a map of the fold's own, so what a late sighting lands in the
90
+ # window taken is at most lost, and never written over by the fold
91
+ offered = self.owed
92
+ self.owed = {}
93
+ merged(offered, taken)
94
+ return offered
95
+
96
+ def restore(self, taken: Mapping[str, Sighting]) -> None:
97
+ """Keep for the next fold what this one took and did not report, as far as there is room."""
98
+ merged(self.owed, taken)
99
+ self.unkept += trimmed(self.owed, MAX_PACKAGES)
100
+
73
101
  def records(
74
102
  self,
103
+ taken: Mapping[str, Sighting],
75
104
  modules: dict[str, ModuleType],
76
105
  installed: dict[str, Dist],
77
- carried: Mapping[tuple[str, str], int],
78
- ) -> list[Record]:
79
- """Report the packages that were read and never became modules."""
106
+ origin: int,
107
+ ) -> tuple[list[Record], dict[str, Sighting]]:
108
+ """Report the packages read that became no module, and say what the next window keeps."""
80
109
  found: list[Record] = []
110
+ kept: dict[str, Sighting] = {}
81
111
  scanned = bool(installed)
82
- for package in sorted(self.moved):
83
- times = self.seen[package]
84
- if times <= carried.get(keyed(Evidence.READ, package), 0):
85
- self.moved.discard(package)
86
- continue
112
+ for package in sorted(taken):
113
+ held = taken[package]
87
114
  if package in modules:
88
- # kept, because a name can leave sys.modules and be read again
115
+ # the stronger evidence has it. A name that leaves sys.modules
116
+ # and is read again is a sighting of the window that reads it
89
117
  continue
90
118
  dist = distribution_of(package, installed)
91
119
  if dist is None:
92
- if scanned:
93
- self.moved.discard(package)
120
+ if not scanned:
121
+ kept[package] = held
94
122
  continue
95
123
  found.append(
96
124
  Record(
@@ -103,7 +131,9 @@ class LoadObserver:
103
131
  # module it is would claim the interpreter reached it
104
132
  granularity=Granularity.PACKAGE,
105
133
  evidence=Evidence.READ,
106
- frequency=times,
134
+ frequency=held.times,
135
+ when=held.first,
136
+ used=dated(origin, held.last),
107
137
  ),
108
138
  )
109
- return found
139
+ return found, kept
@@ -1,16 +1,27 @@
1
1
  """Watch what a workload imports, without taking part in importing it."""
2
2
 
3
3
  import sys
4
- from collections.abc import Mapping
4
+ from collections.abc import Callable, Mapping
5
5
  from dataclasses import dataclass, field
6
6
  from types import ModuleType
7
7
  from typing import Protocol
8
8
 
9
9
  from fluidattacks_agent.distributions import Dist, distribution_of
10
- from fluidattacks_agent.report import Ecosystem, Evidence, Granularity, Record, keyed
11
-
12
- # the probe rides inside a workload we do not own, so its bookkeeping is
13
- # bounded the same way the reader's is
10
+ from fluidattacks_agent.report import (
11
+ Ecosystem,
12
+ Evidence,
13
+ Granularity,
14
+ Record,
15
+ Sighting,
16
+ dated,
17
+ merged,
18
+ reached,
19
+ sighted,
20
+ trimmed,
21
+ )
22
+
23
+ # the probe rides inside a workload we do not own, so what one window holds is
24
+ # bounded, and the bound is on the window rather than on the process's life
14
25
  MAX_MODULES = 4096
15
26
 
16
27
  # our own package would otherwise land in the workload's inventory
@@ -27,29 +38,29 @@ class ImportObserver:
27
38
  finders behind it to decide the outcome.
28
39
  """
29
40
 
30
- # how many times each module was looked up, by its whole dotted name, so a
31
- # record can name the module that was seen and not only the package holding
32
- # it. The name is what bounds the ceiling below
33
- seen: dict[str, int] = field(default_factory=dict)
34
- moved: set[str] = field(default_factory=set)
35
- # modules seen past the ceiling, so the loss is never silent
41
+ # how many times each module was looked up in the window under way, by its
42
+ # whole dotted name, so a record can name the module that was seen and not
43
+ # only the package holding it. The name is what bounds the ceiling below
44
+ seen: dict[str, Sighting] = field(default_factory=dict)
45
+ # what a fold took and did not report, for the next fold to offer again.
46
+ # The fold's own: no thread of the workload writes beside it. Bounded by
47
+ # the same ceiling, and what finds no room is counted with the refusals
48
+ owed: dict[str, Sighting] = field(default_factory=dict)
36
49
  suppressed: int = 0
50
+ unkept: int = 0
37
51
 
38
- def note(self, fullname: str) -> str | None:
39
- """Take note of a module looked up, naming it whenever there is room."""
52
+ def note(self, fullname: str, clock: Callable[[], int] = reached) -> str | None:
53
+ """Take note of a module looked up, naming it whenever the window has room."""
40
54
  if fullname.split(".", maxsplit=1)[0] == OURS:
41
55
  return None
42
- if fullname in self.seen:
43
- self.seen[fullname] += 1
44
- self.moved.add(fullname)
45
- return fullname
46
- # the ceiling bounds how many distinct names are held, not how often a
47
- # name already held is counted
48
- if len(self.seen) >= MAX_MODULES:
56
+ seen = self.seen
57
+ held = seen.get(fullname)
58
+ # the ceiling bounds how many distinct names a window holds, not how
59
+ # often a name already held is counted
60
+ if held is None and len(seen) >= MAX_MODULES:
49
61
  self.suppressed += 1
50
62
  return None
51
- self.seen[fullname] = 1
52
- self.moved.add(fullname)
63
+ seen[fullname] = sighted(held, clock())
53
64
  return fullname
54
65
 
55
66
  def find_spec(
@@ -61,38 +72,53 @@ class ImportObserver:
61
72
  """Note the module being looked up, then defer to the finders behind."""
62
73
  self.note(fullname)
63
74
 
75
+ def drain(self) -> dict[str, Sighting]:
76
+ """Take the window whole, and what the last fold owed, leaving the next one empty."""
77
+ taken, self.seen = self.seen, {}
78
+ # into a map of the fold's own, so what a late sighting lands in the
79
+ # window taken is at most lost, and never written over by the fold
80
+ offered = self.owed
81
+ self.owed = {}
82
+ merged(offered, taken)
83
+ return offered
84
+
85
+ def restore(self, taken: Mapping[str, Sighting]) -> None:
86
+ """Keep for the next fold what this one took and did not report, as far as there is room."""
87
+ merged(self.owed, taken)
88
+ self.unkept += trimmed(self.owed, MAX_MODULES)
89
+
64
90
  def records(
65
91
  self,
92
+ taken: Mapping[str, Sighting],
66
93
  modules: dict[str, ModuleType],
67
94
  installed: dict[str, Dist],
68
- carried: Mapping[tuple[str, str], int],
69
- ) -> list[Record]:
95
+ origin: int,
96
+ ) -> tuple[list[Record], dict[str, Sighting]]:
70
97
  """
71
98
  Name the distribution behind every module that finished importing.
72
99
 
73
100
  A lookup is only an attempt: an import that raised never reaches
74
101
  ``modules``, and reporting it would claim a package loaded when it did
75
102
  not. Intersecting the two is what makes this evidence rather than
76
- intent.
103
+ intent. What has not landed yet is what the next window keeps.
77
104
 
78
105
  Both maps are taken by the caller so this stays a pure fold, and so the
79
106
  distribution scan happens once per report instead of once per module.
80
107
  """
81
108
  found: list[Record] = []
109
+ kept: dict[str, Sighting] = {}
82
110
  scanned = bool(installed)
83
- for name in sorted(self.moved):
84
- times = self.seen[name]
85
- if times <= carried.get(keyed(Evidence.IMPORTED, name), 0):
86
- self.moved.discard(name)
87
- continue
111
+ for name in sorted(taken):
112
+ held = taken[name]
88
113
  module = modules.get(name)
89
114
  if module is None:
90
- # kept, because a lookup mid-import finishes after this flush
115
+ # kept, because a lookup mid-import finishes after this fold
116
+ kept[name] = held
91
117
  continue
92
118
  dist = distribution_of(name, installed)
93
119
  if dist is None:
94
- if scanned:
95
- self.moved.discard(name)
120
+ if not scanned:
121
+ kept[name] = held
96
122
  continue
97
123
  found.append(
98
124
  Record(
@@ -103,10 +129,12 @@ class ImportObserver:
103
129
  symbol=name,
104
130
  granularity=granularity_of(module),
105
131
  evidence=Evidence.IMPORTED,
106
- frequency=times,
132
+ frequency=held.times,
133
+ when=held.first,
134
+ used=dated(origin, held.last),
107
135
  ),
108
136
  )
109
- return found
137
+ return found, kept
110
138
 
111
139
 
112
140
  def granularity_of(module: ModuleType) -> Granularity:
@@ -118,6 +118,16 @@ def targeted(path: str, query: str) -> str:
118
118
  return f"{path or '/'}?{query}" if query else path or "/"
119
119
 
120
120
 
121
+ def prepared() -> None:
122
+ """Import what a post needs, where a fork cannot catch an import half done."""
123
+ # a fork taken while another thread holds a module's import lock leaves
124
+ # that lock held in the child forever, so these are taken under the fold's
125
+ # own lock, which a fork waits out, and never first on the thread that posts
126
+ import http.client # noqa: F401, PLC0415
127
+ import ssl # noqa: F401, PLC0415
128
+ import urllib.parse # noqa: F401, PLC0415
129
+
130
+
121
131
  def proven() -> "ssl.SSLContext":
122
132
  """Give a context the far end has to prove itself on, hostname included."""
123
133
  import ssl # noqa: PLC0415