fluidattacks-agent 0.1.2__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/PKG-INFO +7 -8
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/README.md +6 -7
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/deliver.py +37 -33
- fluidattacks_agent-0.3.0/fluidattacks_agent/executions.py +162 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/loads.py +57 -27
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/observer.py +63 -35
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/post.py +10 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/report.py +58 -4
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/settings.py +0 -9
- fluidattacks_agent-0.3.0/fluidattacks_agent/sink.py +16 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/startup.py +178 -174
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/pyproject.toml +1 -1
- fluidattacks_agent-0.1.2/fluidattacks_agent/executions.py +0 -124
- fluidattacks_agent-0.1.2/fluidattacks_agent/sink.py +0 -163
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/.gitignore +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/__init__.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/batch.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/distributions.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/gate.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/outbox.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/patience.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent/switch.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.3.0}/fluidattacks_agent.pth +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: fluidattacks-agent
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: In-process probe reporting what a Python workload imports and runs
|
|
5
5
|
Project-URL: Homepage, https://fluidattacks.com
|
|
6
6
|
Project-URL: Source, https://gitlab.com/fluidattacks/universe/-/tree/trunk/watches/agents/python
|
|
@@ -26,7 +26,7 @@ dependency inventory can say which of its findings are reachable at runtime and
|
|
|
26
26
|
which are not.
|
|
27
27
|
|
|
28
28
|
It is a library, not a service. It observes the interpreter it is installed in,
|
|
29
|
-
|
|
29
|
+
sends what it saw, and does nothing else. It takes no dependencies: the
|
|
30
30
|
standard library only, because it is installed into workloads we do not own.
|
|
31
31
|
|
|
32
32
|
## Installing
|
|
@@ -60,8 +60,8 @@ a workload's own first-party code produce no records.
|
|
|
60
60
|
|
|
61
61
|
## Where reports go
|
|
62
62
|
|
|
63
|
-
|
|
64
|
-
|
|
63
|
+
Naming an endpoint is what makes the probe observe at all. A workload that
|
|
64
|
+
names none starts nothing:
|
|
65
65
|
|
|
66
66
|
| | |
|
|
67
67
|
|---|---|
|
|
@@ -74,10 +74,9 @@ drain. Naming an endpoint sends them instead:
|
|
|
74
74
|
A file is preferred over a variable because a file can be mode 400, while an
|
|
75
75
|
environment variable is readable by any process of the same user.
|
|
76
76
|
|
|
77
|
-
An endpoint named without enough beside it to reach
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
up where nobody drains it.
|
|
77
|
+
An endpoint named without enough beside it to reach — no credential, or not
|
|
78
|
+
`https://` — is a misconfiguration, and the probe starts nothing rather than
|
|
79
|
+
observe what it could never deliver.
|
|
81
80
|
|
|
82
81
|
## What travels, and what does not
|
|
83
82
|
|
|
@@ -5,7 +5,7 @@ dependency inventory can say which of its findings are reachable at runtime and
|
|
|
5
5
|
which are not.
|
|
6
6
|
|
|
7
7
|
It is a library, not a service. It observes the interpreter it is installed in,
|
|
8
|
-
|
|
8
|
+
sends what it saw, and does nothing else. It takes no dependencies: the
|
|
9
9
|
standard library only, because it is installed into workloads we do not own.
|
|
10
10
|
|
|
11
11
|
## Installing
|
|
@@ -39,8 +39,8 @@ a workload's own first-party code produce no records.
|
|
|
39
39
|
|
|
40
40
|
## Where reports go
|
|
41
41
|
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
Naming an endpoint is what makes the probe observe at all. A workload that
|
|
43
|
+
names none starts nothing:
|
|
44
44
|
|
|
45
45
|
| | |
|
|
46
46
|
|---|---|
|
|
@@ -53,10 +53,9 @@ drain. Naming an endpoint sends them instead:
|
|
|
53
53
|
A file is preferred over a variable because a file can be mode 400, while an
|
|
54
54
|
environment variable is readable by any process of the same user.
|
|
55
55
|
|
|
56
|
-
An endpoint named without enough beside it to reach
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
up where nobody drains it.
|
|
56
|
+
An endpoint named without enough beside it to reach — no credential, or not
|
|
57
|
+
`https://` — is a misconfiguration, and the probe starts nothing rather than
|
|
58
|
+
observe what it could never deliver.
|
|
60
59
|
|
|
61
60
|
## What travels, and what does not
|
|
62
61
|
|
|
@@ -12,18 +12,18 @@ from typing import Final
|
|
|
12
12
|
from fluidattacks_agent.batch import Batch, Batcher
|
|
13
13
|
from fluidattacks_agent.outbox import Outbox
|
|
14
14
|
from fluidattacks_agent.patience import TOPMOST, fraction, splay, standoff
|
|
15
|
-
from fluidattacks_agent.post import TIMEOUT, Verdict, posted
|
|
15
|
+
from fluidattacks_agent.post import TIMEOUT, Verdict, posted, prepared
|
|
16
16
|
from fluidattacks_agent.settings import Delivery
|
|
17
|
-
from fluidattacks_agent.sink import Stalled
|
|
17
|
+
from fluidattacks_agent.sink import PARTING, Stalled
|
|
18
18
|
|
|
19
|
-
# how old the centre's picture may be
|
|
19
|
+
# how old the centre's picture may be: the period the window is folded on
|
|
20
20
|
INTERVAL: Final = 15.0
|
|
21
21
|
|
|
22
|
-
# what a workload's own exit may be delayed by while what is held goes out
|
|
23
|
-
PARTING: Final = 2.0
|
|
24
|
-
|
|
25
22
|
NAME: Final = "fluidattacks-agent"
|
|
26
23
|
|
|
24
|
+
# before any fold, so the first wait to end folds whatever the probe holds
|
|
25
|
+
NEVER: Final = float("-inf")
|
|
26
|
+
|
|
27
27
|
|
|
28
28
|
@dataclass
|
|
29
29
|
class Deliverer:
|
|
@@ -46,23 +46,27 @@ class Deliverer:
|
|
|
46
46
|
guard: threading.Lock = field(default_factory=threading.Lock, repr=False)
|
|
47
47
|
# one pass at a time: what is taken but not delivered lives in one place
|
|
48
48
|
passing: threading.Lock = field(default_factory=threading.Lock, repr=False)
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
# ends a wait every standoff, and a wait ending is not a period going by
|
|
54
|
-
reminded: float = 0.0
|
|
49
|
+
# what renders the probe's window into reports. Called here, on the
|
|
50
|
+
# carrier's own thread and clock, so a workload's threads render nothing
|
|
51
|
+
fold: Callable[[], None] = field(default=lambda: None, repr=False)
|
|
52
|
+
folded: float = NEVER
|
|
55
53
|
leaving: bool = False
|
|
56
54
|
delivered: int = 0
|
|
57
55
|
|
|
58
56
|
def write(self, text: str) -> None:
|
|
59
57
|
"""Take a report for delivery, or refuse and be counted for it."""
|
|
58
|
+
# here, inside the fold that made the report, so what a post imports is
|
|
59
|
+
# imported under the lock a fork waits out and not on the pass after
|
|
60
|
+
prepared()
|
|
60
61
|
if self.batcher.next(self.told, text, self.outbox.put) is None:
|
|
61
62
|
raise Stalled(errno.ENOSPC, "nothing taken", len(text))
|
|
62
|
-
|
|
63
|
+
# a fold of the carrier's own writes here, and its pass follows in the
|
|
64
|
+
# same turn: only another thread has a carrier to wake
|
|
65
|
+
if threading.current_thread() is not self.thread:
|
|
66
|
+
self.rouse()
|
|
63
67
|
|
|
64
|
-
def
|
|
65
|
-
"""
|
|
68
|
+
def rouse(self) -> None:
|
|
69
|
+
"""Start the carrier if there is none, and say there is something for it."""
|
|
66
70
|
with self.guard:
|
|
67
71
|
# a carrier that stopped takes delivery with it, and one is not
|
|
68
72
|
# worth starting for a process already on its way out
|
|
@@ -81,23 +85,17 @@ class Deliverer:
|
|
|
81
85
|
while not self.leaving:
|
|
82
86
|
# a thread that dies takes delivery with it and tells nobody
|
|
83
87
|
with contextlib.suppress(Exception):
|
|
84
|
-
|
|
88
|
+
self.woken.wait(self._waiting())
|
|
85
89
|
self.woken.clear()
|
|
86
|
-
|
|
90
|
+
now = time.monotonic()
|
|
91
|
+
if now < self.due:
|
|
87
92
|
continue
|
|
93
|
+
# on this clock and no other: a far end refusing and then taking
|
|
94
|
+
# ends a wait every standoff, and a wait ending is not a period
|
|
95
|
+
if now - self.folded >= self.interval:
|
|
96
|
+
self.folded = now
|
|
97
|
+
self.fold()
|
|
88
98
|
self.offering()
|
|
89
|
-
# a period, never a wake: a wake says there is something to
|
|
90
|
-
# carry, and asking for more on one made each flush bring the
|
|
91
|
-
# next. Nothing in hand, or the far end is still refusing
|
|
92
|
-
now = time.monotonic()
|
|
93
|
-
if (
|
|
94
|
-
not roused
|
|
95
|
-
and self.holding is None
|
|
96
|
-
and not self.leaving
|
|
97
|
-
and now - self.reminded >= self.interval
|
|
98
|
-
):
|
|
99
|
-
self.reminded = now
|
|
100
|
-
self.remind()
|
|
101
99
|
|
|
102
100
|
def parting(self) -> None:
|
|
103
101
|
"""Give whatever is held one bounded chance to travel, then give up."""
|
|
@@ -128,11 +126,15 @@ class Deliverer:
|
|
|
128
126
|
# every worker of a pre-forking server was born in the same second
|
|
129
127
|
self.due = 0.0
|
|
130
128
|
self.tries = 0
|
|
131
|
-
self.
|
|
129
|
+
self.folded = NEVER
|
|
132
130
|
# a child is not on its way out merely because its parent was
|
|
133
131
|
self.leaving = False
|
|
134
132
|
self.batcher.forked()
|
|
135
133
|
self.outbox.forked()
|
|
134
|
+
# the clock is this thread's, and the child has none until it is
|
|
135
|
+
# started again. Safe here because threading's own fork handler, which
|
|
136
|
+
# registered first, has already run
|
|
137
|
+
self.rouse()
|
|
136
138
|
|
|
137
139
|
def registered(self) -> None:
|
|
138
140
|
"""Ask the interpreter to say when this process has become two."""
|
|
@@ -173,6 +175,8 @@ class Deliverer:
|
|
|
173
175
|
self.due = time.monotonic() + standoff(self.tries, self.share())
|
|
174
176
|
|
|
175
177
|
def _waiting(self) -> float:
|
|
176
|
-
"""Wait out a standoff if there is one, and otherwise
|
|
177
|
-
|
|
178
|
-
|
|
178
|
+
"""Wait out a standoff if there is one, and otherwise until the next fold."""
|
|
179
|
+
now = time.monotonic()
|
|
180
|
+
if self.due > now:
|
|
181
|
+
return self.due - now
|
|
182
|
+
return max(0.0, self.folded + self.interval - now)
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""Watch code inside a package start running, where the interpreter allows it."""
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
from collections.abc import Callable, Mapping
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from pathlib import PurePath
|
|
7
|
+
from typing import Final
|
|
8
|
+
|
|
9
|
+
from fluidattacks_agent.distributions import Dist, distribution_of
|
|
10
|
+
from fluidattacks_agent.report import (
|
|
11
|
+
Ecosystem,
|
|
12
|
+
Evidence,
|
|
13
|
+
Granularity,
|
|
14
|
+
Record,
|
|
15
|
+
Sighting,
|
|
16
|
+
dated,
|
|
17
|
+
merged,
|
|
18
|
+
reached,
|
|
19
|
+
sighted,
|
|
20
|
+
trimmed,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
# 3.15 makes the import below lazy from this; every version we support
|
|
24
|
+
# ignores it, so nothing changes until the floor moves
|
|
25
|
+
__lazy_modules__ = ["pathlib"]
|
|
26
|
+
|
|
27
|
+
# what one window holds at most: a real application runs some 26,000 distinct
|
|
28
|
+
# functions while it starts, and the start is the busiest window there is. A
|
|
29
|
+
# name refused past it is counted, and named again the next window it runs in
|
|
30
|
+
MAX_FUNCTIONS: Final = 32768
|
|
31
|
+
|
|
32
|
+
OURS: Final = "fluidattacks_agent"
|
|
33
|
+
|
|
34
|
+
SOURCE: Final = ".py"
|
|
35
|
+
|
|
36
|
+
INITIALIZER: Final = "__init__"
|
|
37
|
+
PARENT: Final = ".."
|
|
38
|
+
|
|
39
|
+
TOOL: Final = 3
|
|
40
|
+
TOOL_NAME: Final = "fluidattacks-agent"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
TOOLS: Final = 6
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def available() -> bool:
|
|
47
|
+
"""Say whether this interpreter can report code starting at all."""
|
|
48
|
+
return hasattr(sys, "monitoring")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def alone(tool: int = TOOL) -> bool:
|
|
52
|
+
"""Say whether this is the only tool watching: restart_events takes none."""
|
|
53
|
+
held = (sys.monitoring.get_tool(other) for other in range(TOOLS) if other != tool)
|
|
54
|
+
return not any(name is not None for name in held)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def module_of(filename: str, sites: tuple[str, ...]) -> str | None:
|
|
58
|
+
"""Name the module whose source a running code object came from."""
|
|
59
|
+
for site in sites:
|
|
60
|
+
prefix = site + "/"
|
|
61
|
+
if not filename.startswith(prefix) or not filename.endswith(SOURCE):
|
|
62
|
+
continue
|
|
63
|
+
held = PurePath(filename[len(prefix) :])
|
|
64
|
+
parts = list(held.parts)
|
|
65
|
+
if not parts or held.is_absolute() or PARENT in parts:
|
|
66
|
+
return None
|
|
67
|
+
stem = parts.pop()[: -len(SOURCE)]
|
|
68
|
+
if not stem:
|
|
69
|
+
return None
|
|
70
|
+
if stem != INITIALIZER:
|
|
71
|
+
parts.append(stem)
|
|
72
|
+
return ".".join(parts) or None
|
|
73
|
+
return None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclass
|
|
77
|
+
class RunObserver:
|
|
78
|
+
"""Functions that started in the window under way, and in no window before it."""
|
|
79
|
+
|
|
80
|
+
sites: tuple[str, ...] = ()
|
|
81
|
+
# the window's own: a fold takes the whole map, and the reader adds the
|
|
82
|
+
# windows up. So the bound is on a window and not on the process's life
|
|
83
|
+
seen: dict[str, Sighting] = field(default_factory=dict)
|
|
84
|
+
# what a fold took and did not report, for the next fold to offer again.
|
|
85
|
+
# The fold's own: no thread of the workload writes beside it. Bounded by
|
|
86
|
+
# the same ceiling, and what finds no room is counted with the refusals
|
|
87
|
+
owed: dict[str, Sighting] = field(default_factory=dict)
|
|
88
|
+
suppressed: int = 0
|
|
89
|
+
unkept: int = 0
|
|
90
|
+
|
|
91
|
+
def note(
|
|
92
|
+
self,
|
|
93
|
+
filename: str,
|
|
94
|
+
qualname: str,
|
|
95
|
+
clock: Callable[[], int] = reached,
|
|
96
|
+
) -> str | None:
|
|
97
|
+
"""Take note of code starting, naming the function whenever the window has room."""
|
|
98
|
+
module = module_of(filename, self.sites)
|
|
99
|
+
if module is None:
|
|
100
|
+
return None
|
|
101
|
+
if module.split(".", maxsplit=1)[0] == OURS:
|
|
102
|
+
return None
|
|
103
|
+
qualified = f"{module}.{qualname}"
|
|
104
|
+
# bound once: a fold may swap the window out meanwhile, and a sighting
|
|
105
|
+
# must land in the window it was counted against, never in the next
|
|
106
|
+
seen = self.seen
|
|
107
|
+
held = seen.get(qualified)
|
|
108
|
+
if held is None and len(seen) >= MAX_FUNCTIONS:
|
|
109
|
+
self.suppressed += 1
|
|
110
|
+
return None
|
|
111
|
+
seen[qualified] = sighted(held, clock())
|
|
112
|
+
return qualified
|
|
113
|
+
|
|
114
|
+
def drain(self) -> dict[str, Sighting]:
|
|
115
|
+
"""Take the window whole, and what the last fold owed, leaving the next one empty."""
|
|
116
|
+
taken, self.seen = self.seen, {}
|
|
117
|
+
# into a map of the fold's own, so what a late sighting lands in the
|
|
118
|
+
# window taken is at most lost, and never written over by the fold
|
|
119
|
+
offered = self.owed
|
|
120
|
+
self.owed = {}
|
|
121
|
+
merged(offered, taken)
|
|
122
|
+
return offered
|
|
123
|
+
|
|
124
|
+
def restore(self, taken: Mapping[str, Sighting]) -> None:
|
|
125
|
+
"""Keep for the next fold what this one took and did not report, as far as there is room."""
|
|
126
|
+
merged(self.owed, taken)
|
|
127
|
+
self.unkept += trimmed(self.owed, MAX_FUNCTIONS)
|
|
128
|
+
|
|
129
|
+
def records(
|
|
130
|
+
self,
|
|
131
|
+
taken: Mapping[str, Sighting],
|
|
132
|
+
installed: dict[str, Dist],
|
|
133
|
+
origin: int,
|
|
134
|
+
) -> tuple[list[Record], dict[str, Sighting]]:
|
|
135
|
+
"""Report every function a distribution owns, and say what the next window keeps."""
|
|
136
|
+
found: list[Record] = []
|
|
137
|
+
kept: dict[str, Sighting] = {}
|
|
138
|
+
scanned = bool(installed)
|
|
139
|
+
for qualified in sorted(taken):
|
|
140
|
+
held = taken[qualified]
|
|
141
|
+
dist = distribution_of(qualified, installed)
|
|
142
|
+
if dist is None:
|
|
143
|
+
# no distribution will own what a settled scan does not, and a
|
|
144
|
+
# scan that has found nothing yet may still own all of these
|
|
145
|
+
if not scanned:
|
|
146
|
+
kept[qualified] = held
|
|
147
|
+
continue
|
|
148
|
+
found.append(
|
|
149
|
+
Record(
|
|
150
|
+
ecosystem=Ecosystem.PYPI,
|
|
151
|
+
name=dist.name,
|
|
152
|
+
version=dist.version,
|
|
153
|
+
confidence=dist.confidence,
|
|
154
|
+
symbol=qualified,
|
|
155
|
+
granularity=Granularity.FUNCTION,
|
|
156
|
+
evidence=Evidence.EXECUTED,
|
|
157
|
+
frequency=held.times,
|
|
158
|
+
when=held.first,
|
|
159
|
+
used=dated(origin, held.last),
|
|
160
|
+
),
|
|
161
|
+
)
|
|
162
|
+
return found, kept
|
|
@@ -1,13 +1,25 @@
|
|
|
1
1
|
"""Account for package code the interpreter read without importing it."""
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
|
-
from collections.abc import Mapping
|
|
4
|
+
from collections.abc import Callable, Mapping
|
|
5
5
|
from dataclasses import dataclass, field
|
|
6
6
|
from types import ModuleType
|
|
7
7
|
|
|
8
8
|
from fluidattacks_agent.distributions import Dist, distribution_of
|
|
9
|
-
from fluidattacks_agent.report import
|
|
9
|
+
from fluidattacks_agent.report import (
|
|
10
|
+
Ecosystem,
|
|
11
|
+
Evidence,
|
|
12
|
+
Granularity,
|
|
13
|
+
Record,
|
|
14
|
+
Sighting,
|
|
15
|
+
dated,
|
|
16
|
+
merged,
|
|
17
|
+
reached,
|
|
18
|
+
sighted,
|
|
19
|
+
trimmed,
|
|
20
|
+
)
|
|
10
21
|
|
|
22
|
+
# what one window holds at most, and not the process's life
|
|
11
23
|
MAX_PACKAGES = 4096
|
|
12
24
|
|
|
13
25
|
OURS = "fluidattacks_agent"
|
|
@@ -46,51 +58,67 @@ def package_of(path: str, sites: tuple[str, ...]) -> str | None:
|
|
|
46
58
|
|
|
47
59
|
@dataclass
|
|
48
60
|
class LoadObserver:
|
|
49
|
-
"""Packages whose code was read
|
|
61
|
+
"""Packages whose code was read in the window under way."""
|
|
50
62
|
|
|
51
63
|
sites: tuple[str, ...] = ()
|
|
52
64
|
# how many files of each package were read, not merely whether any was
|
|
53
|
-
seen: dict[str,
|
|
54
|
-
|
|
65
|
+
seen: dict[str, Sighting] = field(default_factory=dict)
|
|
66
|
+
# what a fold took and did not report, for the next fold to offer again.
|
|
67
|
+
# The fold's own: no thread of the workload writes beside it. Bounded by
|
|
68
|
+
# the same ceiling, and what finds no room is counted with the refusals
|
|
69
|
+
owed: dict[str, Sighting] = field(default_factory=dict)
|
|
55
70
|
suppressed: int = 0
|
|
71
|
+
unkept: int = 0
|
|
56
72
|
|
|
57
|
-
def note(self, path: str) -> str | None:
|
|
58
|
-
"""Take note of a file opened, naming the package whenever
|
|
73
|
+
def note(self, path: str, clock: Callable[[], int] = reached) -> str | None:
|
|
74
|
+
"""Take note of a file opened, naming the package whenever the window has room."""
|
|
59
75
|
package = package_of(path, self.sites)
|
|
60
76
|
if package is None or package == OURS:
|
|
61
77
|
return None
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
return package
|
|
66
|
-
if len(self.seen) >= MAX_PACKAGES:
|
|
78
|
+
seen = self.seen
|
|
79
|
+
held = seen.get(package)
|
|
80
|
+
if held is None and len(seen) >= MAX_PACKAGES:
|
|
67
81
|
self.suppressed += 1
|
|
68
82
|
return None
|
|
69
|
-
|
|
70
|
-
self.moved.add(package)
|
|
83
|
+
seen[package] = sighted(held, clock())
|
|
71
84
|
return package
|
|
72
85
|
|
|
86
|
+
def drain(self) -> dict[str, Sighting]:
|
|
87
|
+
"""Take the window whole, and what the last fold owed, leaving the next one empty."""
|
|
88
|
+
taken, self.seen = self.seen, {}
|
|
89
|
+
# into a map of the fold's own, so what a late sighting lands in the
|
|
90
|
+
# window taken is at most lost, and never written over by the fold
|
|
91
|
+
offered = self.owed
|
|
92
|
+
self.owed = {}
|
|
93
|
+
merged(offered, taken)
|
|
94
|
+
return offered
|
|
95
|
+
|
|
96
|
+
def restore(self, taken: Mapping[str, Sighting]) -> None:
|
|
97
|
+
"""Keep for the next fold what this one took and did not report, as far as there is room."""
|
|
98
|
+
merged(self.owed, taken)
|
|
99
|
+
self.unkept += trimmed(self.owed, MAX_PACKAGES)
|
|
100
|
+
|
|
73
101
|
def records(
|
|
74
102
|
self,
|
|
103
|
+
taken: Mapping[str, Sighting],
|
|
75
104
|
modules: dict[str, ModuleType],
|
|
76
105
|
installed: dict[str, Dist],
|
|
77
|
-
|
|
78
|
-
) -> list[Record]:
|
|
79
|
-
"""Report the packages that
|
|
106
|
+
origin: int,
|
|
107
|
+
) -> tuple[list[Record], dict[str, Sighting]]:
|
|
108
|
+
"""Report the packages read that became no module, and say what the next window keeps."""
|
|
80
109
|
found: list[Record] = []
|
|
110
|
+
kept: dict[str, Sighting] = {}
|
|
81
111
|
scanned = bool(installed)
|
|
82
|
-
for package in sorted(
|
|
83
|
-
|
|
84
|
-
if times <= carried.get(keyed(Evidence.READ, package), 0):
|
|
85
|
-
self.moved.discard(package)
|
|
86
|
-
continue
|
|
112
|
+
for package in sorted(taken):
|
|
113
|
+
held = taken[package]
|
|
87
114
|
if package in modules:
|
|
88
|
-
#
|
|
115
|
+
# the stronger evidence has it. A name that leaves sys.modules
|
|
116
|
+
# and is read again is a sighting of the window that reads it
|
|
89
117
|
continue
|
|
90
118
|
dist = distribution_of(package, installed)
|
|
91
119
|
if dist is None:
|
|
92
|
-
if scanned:
|
|
93
|
-
|
|
120
|
+
if not scanned:
|
|
121
|
+
kept[package] = held
|
|
94
122
|
continue
|
|
95
123
|
found.append(
|
|
96
124
|
Record(
|
|
@@ -103,7 +131,9 @@ class LoadObserver:
|
|
|
103
131
|
# module it is would claim the interpreter reached it
|
|
104
132
|
granularity=Granularity.PACKAGE,
|
|
105
133
|
evidence=Evidence.READ,
|
|
106
|
-
frequency=times,
|
|
134
|
+
frequency=held.times,
|
|
135
|
+
when=held.first,
|
|
136
|
+
used=dated(origin, held.last),
|
|
107
137
|
),
|
|
108
138
|
)
|
|
109
|
-
return found
|
|
139
|
+
return found, kept
|
|
@@ -1,16 +1,27 @@
|
|
|
1
1
|
"""Watch what a workload imports, without taking part in importing it."""
|
|
2
2
|
|
|
3
3
|
import sys
|
|
4
|
-
from collections.abc import Mapping
|
|
4
|
+
from collections.abc import Callable, Mapping
|
|
5
5
|
from dataclasses import dataclass, field
|
|
6
6
|
from types import ModuleType
|
|
7
7
|
from typing import Protocol
|
|
8
8
|
|
|
9
9
|
from fluidattacks_agent.distributions import Dist, distribution_of
|
|
10
|
-
from fluidattacks_agent.report import
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
10
|
+
from fluidattacks_agent.report import (
|
|
11
|
+
Ecosystem,
|
|
12
|
+
Evidence,
|
|
13
|
+
Granularity,
|
|
14
|
+
Record,
|
|
15
|
+
Sighting,
|
|
16
|
+
dated,
|
|
17
|
+
merged,
|
|
18
|
+
reached,
|
|
19
|
+
sighted,
|
|
20
|
+
trimmed,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
# the probe rides inside a workload we do not own, so what one window holds is
|
|
24
|
+
# bounded, and the bound is on the window rather than on the process's life
|
|
14
25
|
MAX_MODULES = 4096
|
|
15
26
|
|
|
16
27
|
# our own package would otherwise land in the workload's inventory
|
|
@@ -27,29 +38,29 @@ class ImportObserver:
|
|
|
27
38
|
finders behind it to decide the outcome.
|
|
28
39
|
"""
|
|
29
40
|
|
|
30
|
-
# how many times each module was looked up
|
|
31
|
-
# record can name the module that was seen and not
|
|
32
|
-
# it. The name is what bounds the ceiling below
|
|
33
|
-
seen: dict[str,
|
|
34
|
-
|
|
35
|
-
#
|
|
41
|
+
# how many times each module was looked up in the window under way, by its
|
|
42
|
+
# whole dotted name, so a record can name the module that was seen and not
|
|
43
|
+
# only the package holding it. The name is what bounds the ceiling below
|
|
44
|
+
seen: dict[str, Sighting] = field(default_factory=dict)
|
|
45
|
+
# what a fold took and did not report, for the next fold to offer again.
|
|
46
|
+
# The fold's own: no thread of the workload writes beside it. Bounded by
|
|
47
|
+
# the same ceiling, and what finds no room is counted with the refusals
|
|
48
|
+
owed: dict[str, Sighting] = field(default_factory=dict)
|
|
36
49
|
suppressed: int = 0
|
|
50
|
+
unkept: int = 0
|
|
37
51
|
|
|
38
|
-
def note(self, fullname: str) -> str | None:
|
|
39
|
-
"""Take note of a module looked up, naming it whenever
|
|
52
|
+
def note(self, fullname: str, clock: Callable[[], int] = reached) -> str | None:
|
|
53
|
+
"""Take note of a module looked up, naming it whenever the window has room."""
|
|
40
54
|
if fullname.split(".", maxsplit=1)[0] == OURS:
|
|
41
55
|
return None
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
# name already held is counted
|
|
48
|
-
if len(self.seen) >= MAX_MODULES:
|
|
56
|
+
seen = self.seen
|
|
57
|
+
held = seen.get(fullname)
|
|
58
|
+
# the ceiling bounds how many distinct names a window holds, not how
|
|
59
|
+
# often a name already held is counted
|
|
60
|
+
if held is None and len(seen) >= MAX_MODULES:
|
|
49
61
|
self.suppressed += 1
|
|
50
62
|
return None
|
|
51
|
-
|
|
52
|
-
self.moved.add(fullname)
|
|
63
|
+
seen[fullname] = sighted(held, clock())
|
|
53
64
|
return fullname
|
|
54
65
|
|
|
55
66
|
def find_spec(
|
|
@@ -61,38 +72,53 @@ class ImportObserver:
|
|
|
61
72
|
"""Note the module being looked up, then defer to the finders behind."""
|
|
62
73
|
self.note(fullname)
|
|
63
74
|
|
|
75
|
+
def drain(self) -> dict[str, Sighting]:
|
|
76
|
+
"""Take the window whole, and what the last fold owed, leaving the next one empty."""
|
|
77
|
+
taken, self.seen = self.seen, {}
|
|
78
|
+
# into a map of the fold's own, so what a late sighting lands in the
|
|
79
|
+
# window taken is at most lost, and never written over by the fold
|
|
80
|
+
offered = self.owed
|
|
81
|
+
self.owed = {}
|
|
82
|
+
merged(offered, taken)
|
|
83
|
+
return offered
|
|
84
|
+
|
|
85
|
+
def restore(self, taken: Mapping[str, Sighting]) -> None:
|
|
86
|
+
"""Keep for the next fold what this one took and did not report, as far as there is room."""
|
|
87
|
+
merged(self.owed, taken)
|
|
88
|
+
self.unkept += trimmed(self.owed, MAX_MODULES)
|
|
89
|
+
|
|
64
90
|
def records(
|
|
65
91
|
self,
|
|
92
|
+
taken: Mapping[str, Sighting],
|
|
66
93
|
modules: dict[str, ModuleType],
|
|
67
94
|
installed: dict[str, Dist],
|
|
68
|
-
|
|
69
|
-
) -> list[Record]:
|
|
95
|
+
origin: int,
|
|
96
|
+
) -> tuple[list[Record], dict[str, Sighting]]:
|
|
70
97
|
"""
|
|
71
98
|
Name the distribution behind every module that finished importing.
|
|
72
99
|
|
|
73
100
|
A lookup is only an attempt: an import that raised never reaches
|
|
74
101
|
``modules``, and reporting it would claim a package loaded when it did
|
|
75
102
|
not. Intersecting the two is what makes this evidence rather than
|
|
76
|
-
intent.
|
|
103
|
+
intent. What has not landed yet is what the next window keeps.
|
|
77
104
|
|
|
78
105
|
Both maps are taken by the caller so this stays a pure fold, and so the
|
|
79
106
|
distribution scan happens once per report instead of once per module.
|
|
80
107
|
"""
|
|
81
108
|
found: list[Record] = []
|
|
109
|
+
kept: dict[str, Sighting] = {}
|
|
82
110
|
scanned = bool(installed)
|
|
83
|
-
for name in sorted(
|
|
84
|
-
|
|
85
|
-
if times <= carried.get(keyed(Evidence.IMPORTED, name), 0):
|
|
86
|
-
self.moved.discard(name)
|
|
87
|
-
continue
|
|
111
|
+
for name in sorted(taken):
|
|
112
|
+
held = taken[name]
|
|
88
113
|
module = modules.get(name)
|
|
89
114
|
if module is None:
|
|
90
|
-
# kept, because a lookup mid-import finishes after this
|
|
115
|
+
# kept, because a lookup mid-import finishes after this fold
|
|
116
|
+
kept[name] = held
|
|
91
117
|
continue
|
|
92
118
|
dist = distribution_of(name, installed)
|
|
93
119
|
if dist is None:
|
|
94
|
-
if scanned:
|
|
95
|
-
|
|
120
|
+
if not scanned:
|
|
121
|
+
kept[name] = held
|
|
96
122
|
continue
|
|
97
123
|
found.append(
|
|
98
124
|
Record(
|
|
@@ -103,10 +129,12 @@ class ImportObserver:
|
|
|
103
129
|
symbol=name,
|
|
104
130
|
granularity=granularity_of(module),
|
|
105
131
|
evidence=Evidence.IMPORTED,
|
|
106
|
-
frequency=times,
|
|
132
|
+
frequency=held.times,
|
|
133
|
+
when=held.first,
|
|
134
|
+
used=dated(origin, held.last),
|
|
107
135
|
),
|
|
108
136
|
)
|
|
109
|
-
return found
|
|
137
|
+
return found, kept
|
|
110
138
|
|
|
111
139
|
|
|
112
140
|
def granularity_of(module: ModuleType) -> Granularity:
|
|
@@ -118,6 +118,16 @@ def targeted(path: str, query: str) -> str:
|
|
|
118
118
|
return f"{path or '/'}?{query}" if query else path or "/"
|
|
119
119
|
|
|
120
120
|
|
|
121
|
+
def prepared() -> None:
|
|
122
|
+
"""Import what a post needs, where a fork cannot catch an import half done."""
|
|
123
|
+
# a fork taken while another thread holds a module's import lock leaves
|
|
124
|
+
# that lock held in the child forever, so these are taken under the fold's
|
|
125
|
+
# own lock, which a fork waits out, and never first on the thread that posts
|
|
126
|
+
import http.client # noqa: F401, PLC0415
|
|
127
|
+
import ssl # noqa: F401, PLC0415
|
|
128
|
+
import urllib.parse # noqa: F401, PLC0415
|
|
129
|
+
|
|
130
|
+
|
|
121
131
|
def proven() -> "ssl.SSLContext":
|
|
122
132
|
"""Give a context the far end has to prove itself on, hostname included."""
|
|
123
133
|
import ssl # noqa: PLC0415
|