fluidattacks-agent 0.1.1__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/.gitignore +1 -1
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/PKG-INFO +7 -8
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/README.md +6 -7
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/deliver.py +38 -14
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/executions.py +3 -1
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/post.py +10 -0
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/settings.py +0 -9
- fluidattacks_agent-0.2.0/fluidattacks_agent/sink.py +16 -0
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/startup.py +112 -86
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/pyproject.toml +1 -1
- fluidattacks_agent-0.1.1/fluidattacks_agent/sink.py +0 -163
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/__init__.py +0 -0
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/batch.py +0 -0
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/distributions.py +0 -0
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/gate.py +0 -0
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/loads.py +0 -0
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/observer.py +0 -0
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/outbox.py +0 -0
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/patience.py +0 -0
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/report.py +0 -0
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent/switch.py +0 -0
- {fluidattacks_agent-0.1.1 → fluidattacks_agent-0.2.0}/fluidattacks_agent.pth +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: fluidattacks-agent
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: In-process probe reporting what a Python workload imports and runs
|
|
5
5
|
Project-URL: Homepage, https://fluidattacks.com
|
|
6
6
|
Project-URL: Source, https://gitlab.com/fluidattacks/universe/-/tree/trunk/watches/agents/python
|
|
@@ -26,7 +26,7 @@ dependency inventory can say which of its findings are reachable at runtime and
|
|
|
26
26
|
which are not.
|
|
27
27
|
|
|
28
28
|
It is a library, not a service. It observes the interpreter it is installed in,
|
|
29
|
-
|
|
29
|
+
sends what it saw, and does nothing else. It takes no dependencies: the
|
|
30
30
|
standard library only, because it is installed into workloads we do not own.
|
|
31
31
|
|
|
32
32
|
## Installing
|
|
@@ -60,8 +60,8 @@ a workload's own first-party code produce no records.
|
|
|
60
60
|
|
|
61
61
|
## Where reports go
|
|
62
62
|
|
|
63
|
-
|
|
64
|
-
|
|
63
|
+
Naming an endpoint is what makes the probe observe at all. A workload that
|
|
64
|
+
names none starts nothing:
|
|
65
65
|
|
|
66
66
|
| | |
|
|
67
67
|
|---|---|
|
|
@@ -74,10 +74,9 @@ drain. Naming an endpoint sends them instead:
|
|
|
74
74
|
A file is preferred over a variable because a file can be mode 400, while an
|
|
75
75
|
environment variable is readable by any process of the same user.
|
|
76
76
|
|
|
77
|
-
An endpoint named without enough beside it to reach
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
up where nobody drains it.
|
|
77
|
+
An endpoint named without enough beside it to reach — no credential, or not
|
|
78
|
+
`https://` — is a misconfiguration, and the probe starts nothing rather than
|
|
79
|
+
observe what it could never deliver.
|
|
81
80
|
|
|
82
81
|
## What travels, and what does not
|
|
83
82
|
|
|
@@ -5,7 +5,7 @@ dependency inventory can say which of its findings are reachable at runtime and
|
|
|
5
5
|
which are not.
|
|
6
6
|
|
|
7
7
|
It is a library, not a service. It observes the interpreter it is installed in,
|
|
8
|
-
|
|
8
|
+
sends what it saw, and does nothing else. It takes no dependencies: the
|
|
9
9
|
standard library only, because it is installed into workloads we do not own.
|
|
10
10
|
|
|
11
11
|
## Installing
|
|
@@ -39,8 +39,8 @@ a workload's own first-party code produce no records.
|
|
|
39
39
|
|
|
40
40
|
## Where reports go
|
|
41
41
|
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
Naming an endpoint is what makes the probe observe at all. A workload that
|
|
43
|
+
names none starts nothing:
|
|
44
44
|
|
|
45
45
|
| | |
|
|
46
46
|
|---|---|
|
|
@@ -53,10 +53,9 @@ drain. Naming an endpoint sends them instead:
|
|
|
53
53
|
A file is preferred over a variable because a file can be mode 400, while an
|
|
54
54
|
environment variable is readable by any process of the same user.
|
|
55
55
|
|
|
56
|
-
An endpoint named without enough beside it to reach
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
up where nobody drains it.
|
|
56
|
+
An endpoint named without enough beside it to reach — no credential, or not
|
|
57
|
+
`https://` — is a misconfiguration, and the probe starts nothing rather than
|
|
58
|
+
observe what it could never deliver.
|
|
60
59
|
|
|
61
60
|
## What travels, and what does not
|
|
62
61
|
|
|
@@ -12,18 +12,18 @@ from typing import Final
|
|
|
12
12
|
from fluidattacks_agent.batch import Batch, Batcher
|
|
13
13
|
from fluidattacks_agent.outbox import Outbox
|
|
14
14
|
from fluidattacks_agent.patience import TOPMOST, fraction, splay, standoff
|
|
15
|
-
from fluidattacks_agent.post import TIMEOUT, Verdict, posted
|
|
15
|
+
from fluidattacks_agent.post import TIMEOUT, Verdict, posted, prepared
|
|
16
16
|
from fluidattacks_agent.settings import Delivery
|
|
17
|
-
from fluidattacks_agent.sink import Stalled
|
|
17
|
+
from fluidattacks_agent.sink import PARTING, Stalled
|
|
18
18
|
|
|
19
|
-
# how old the centre's picture may be
|
|
19
|
+
# how old the centre's picture may be: the period the window is folded on
|
|
20
20
|
INTERVAL: Final = 15.0
|
|
21
21
|
|
|
22
|
-
# what a workload's own exit may be delayed by while what is held goes out
|
|
23
|
-
PARTING: Final = 2.0
|
|
24
|
-
|
|
25
22
|
NAME: Final = "fluidattacks-agent"
|
|
26
23
|
|
|
24
|
+
# before any fold, so the first wait to end folds whatever the probe holds
|
|
25
|
+
NEVER: Final = float("-inf")
|
|
26
|
+
|
|
27
27
|
|
|
28
28
|
@dataclass
|
|
29
29
|
class Deliverer:
|
|
@@ -46,17 +46,27 @@ class Deliverer:
|
|
|
46
46
|
guard: threading.Lock = field(default_factory=threading.Lock, repr=False)
|
|
47
47
|
# one pass at a time: what is taken but not delivered lives in one place
|
|
48
48
|
passing: threading.Lock = field(default_factory=threading.Lock, repr=False)
|
|
49
|
+
# what renders the probe's window into reports. Called here, on the
|
|
50
|
+
# carrier's own thread and clock, so a workload's threads render nothing
|
|
51
|
+
fold: Callable[[], None] = field(default=lambda: None, repr=False)
|
|
52
|
+
folded: float = NEVER
|
|
49
53
|
leaving: bool = False
|
|
50
54
|
delivered: int = 0
|
|
51
55
|
|
|
52
56
|
def write(self, text: str) -> None:
|
|
53
57
|
"""Take a report for delivery, or refuse and be counted for it."""
|
|
58
|
+
# here, inside the fold that made the report, so what a post imports is
|
|
59
|
+
# imported under the lock a fork waits out and not on the pass after
|
|
60
|
+
prepared()
|
|
54
61
|
if self.batcher.next(self.told, text, self.outbox.put) is None:
|
|
55
62
|
raise Stalled(errno.ENOSPC, "nothing taken", len(text))
|
|
56
|
-
|
|
63
|
+
# a fold of the carrier's own writes here, and its pass follows in the
|
|
64
|
+
# same turn: only another thread has a carrier to wake
|
|
65
|
+
if threading.current_thread() is not self.thread:
|
|
66
|
+
self.rouse()
|
|
57
67
|
|
|
58
|
-
def
|
|
59
|
-
"""
|
|
68
|
+
def rouse(self) -> None:
|
|
69
|
+
"""Start the carrier if there is none, and say there is something for it."""
|
|
60
70
|
with self.guard:
|
|
61
71
|
# a carrier that stopped takes delivery with it, and one is not
|
|
62
72
|
# worth starting for a process already on its way out
|
|
@@ -77,8 +87,15 @@ class Deliverer:
|
|
|
77
87
|
with contextlib.suppress(Exception):
|
|
78
88
|
self.woken.wait(self._waiting())
|
|
79
89
|
self.woken.clear()
|
|
80
|
-
|
|
81
|
-
|
|
90
|
+
now = time.monotonic()
|
|
91
|
+
if now < self.due:
|
|
92
|
+
continue
|
|
93
|
+
# on this clock and no other: a far end refusing and then taking
|
|
94
|
+
# ends a wait every standoff, and a wait ending is not a period
|
|
95
|
+
if now - self.folded >= self.interval:
|
|
96
|
+
self.folded = now
|
|
97
|
+
self.fold()
|
|
98
|
+
self.offering()
|
|
82
99
|
|
|
83
100
|
def parting(self) -> None:
|
|
84
101
|
"""Give whatever is held one bounded chance to travel, then give up."""
|
|
@@ -109,10 +126,15 @@ class Deliverer:
|
|
|
109
126
|
# every worker of a pre-forking server was born in the same second
|
|
110
127
|
self.due = 0.0
|
|
111
128
|
self.tries = 0
|
|
129
|
+
self.folded = NEVER
|
|
112
130
|
# a child is not on its way out merely because its parent was
|
|
113
131
|
self.leaving = False
|
|
114
132
|
self.batcher.forked()
|
|
115
133
|
self.outbox.forked()
|
|
134
|
+
# the clock is this thread's, and the child has none until it is
|
|
135
|
+
# started again. Safe here because threading's own fork handler, which
|
|
136
|
+
# registered first, has already run
|
|
137
|
+
self.rouse()
|
|
116
138
|
|
|
117
139
|
def registered(self) -> None:
|
|
118
140
|
"""Ask the interpreter to say when this process has become two."""
|
|
@@ -153,6 +175,8 @@ class Deliverer:
|
|
|
153
175
|
self.due = time.monotonic() + standoff(self.tries, self.share())
|
|
154
176
|
|
|
155
177
|
def _waiting(self) -> float:
|
|
156
|
-
"""Wait out a standoff if there is one, and otherwise
|
|
157
|
-
|
|
158
|
-
|
|
178
|
+
"""Wait out a standoff if there is one, and otherwise until the next fold."""
|
|
179
|
+
now = time.monotonic()
|
|
180
|
+
if self.due > now:
|
|
181
|
+
return self.due - now
|
|
182
|
+
return max(0.0, self.folded + self.interval - now)
|
|
@@ -13,7 +13,9 @@ from fluidattacks_agent.report import Ecosystem, Evidence, Granularity, Record,
|
|
|
13
13
|
# ignores it, so nothing changes until the floor moves
|
|
14
14
|
__lazy_modules__ = ["pathlib"]
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
# a real application runs some 26,000 distinct functions while it starts, and
|
|
17
|
+
# what filled the ceiling is what never gets named again for the process's life
|
|
18
|
+
MAX_FUNCTIONS: Final = 32768
|
|
17
19
|
|
|
18
20
|
OURS: Final = "fluidattacks_agent"
|
|
19
21
|
|
|
@@ -118,6 +118,16 @@ def targeted(path: str, query: str) -> str:
|
|
|
118
118
|
return f"{path or '/'}?{query}" if query else path or "/"
|
|
119
119
|
|
|
120
120
|
|
|
121
|
+
def prepared() -> None:
|
|
122
|
+
"""Import what a post needs, where a fork cannot catch an import half done."""
|
|
123
|
+
# a fork taken while another thread holds a module's import lock leaves
|
|
124
|
+
# that lock held in the child forever, so these are taken under the fold's
|
|
125
|
+
# own lock, which a fork waits out, and never first on the thread that posts
|
|
126
|
+
import http.client # noqa: F401, PLC0415
|
|
127
|
+
import ssl # noqa: F401, PLC0415
|
|
128
|
+
import urllib.parse # noqa: F401, PLC0415
|
|
129
|
+
|
|
130
|
+
|
|
121
131
|
def proven() -> "ssl.SSLContext":
|
|
122
132
|
"""Give a context the far end has to prove itself on, hostname included."""
|
|
123
133
|
import ssl # noqa: PLC0415
|
|
@@ -91,15 +91,6 @@ def _credential(value: str | None) -> Secret | None:
|
|
|
91
91
|
return Secret(named) if _readable(named) else None
|
|
92
92
|
|
|
93
93
|
|
|
94
|
-
def asked(environ: Mapping[str, str] | None = None) -> bool:
|
|
95
|
-
"""Say whether the workload asked for its reports to be delivered at all."""
|
|
96
|
-
try:
|
|
97
|
-
held = os.environ if environ is None else environ
|
|
98
|
-
return _setting(held, ENDPOINT) is not None or ENDPOINT in held
|
|
99
|
-
except Exception: # noqa: BLE001
|
|
100
|
-
return False
|
|
101
|
-
|
|
102
|
-
|
|
103
94
|
def delivery(environ: Mapping[str, str] | None = None) -> Delivery | None:
|
|
104
95
|
"""Read where reports go, or nothing at all if it cannot be read whole."""
|
|
105
96
|
try:
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Somewhere a report can go, and the refusal that says it did not."""
|
|
2
|
+
|
|
3
|
+
from typing import Final, Protocol
|
|
4
|
+
|
|
5
|
+
# what a workload's own exit may be delayed by while what is held goes out
|
|
6
|
+
PARTING: Final = 2.0
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Stalled(OSError):
|
|
10
|
+
"""Nothing was taken, and what was not taken is still owed."""
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Sink(Protocol):
|
|
14
|
+
"""Somewhere a report can go, which may refuse it and must never lie."""
|
|
15
|
+
|
|
16
|
+
def write(self, text: str) -> None: ...
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
"""Start the probe inside a workload and
|
|
1
|
+
"""Start the probe inside a workload and hand its reports to what carries them."""
|
|
2
2
|
|
|
3
|
+
import _thread
|
|
3
4
|
import atexit
|
|
4
5
|
import contextlib
|
|
5
6
|
import os
|
|
@@ -8,18 +9,19 @@ import time
|
|
|
8
9
|
from collections.abc import Callable
|
|
9
10
|
from dataclasses import dataclass, field, replace
|
|
10
11
|
from types import CodeType, ModuleType
|
|
11
|
-
from typing import Final
|
|
12
|
+
from typing import TYPE_CHECKING, Final
|
|
12
13
|
|
|
13
14
|
from fluidattacks_agent.distributions import Dist, distribution_map
|
|
14
15
|
from fluidattacks_agent.executions import TOOL, TOOL_NAME, RunObserver, alone, available
|
|
15
16
|
from fluidattacks_agent.loads import LoadObserver
|
|
16
17
|
from fluidattacks_agent.observer import ImportObserver, install
|
|
17
18
|
from fluidattacks_agent.report import Evidence, Record, Window, keyed, render
|
|
18
|
-
from fluidattacks_agent.settings import Delivery,
|
|
19
|
-
from fluidattacks_agent.sink import
|
|
19
|
+
from fluidattacks_agent.settings import Delivery, delivery
|
|
20
|
+
from fluidattacks_agent.sink import PARTING, Sink, Stalled
|
|
20
21
|
from fluidattacks_agent.switch import switched_on
|
|
21
22
|
|
|
22
|
-
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from fluidattacks_agent.deliver import Deliverer
|
|
23
25
|
|
|
24
26
|
# what a site directory is given so that installing the package starts the
|
|
25
27
|
# probe. The agent writes the same line when it injects one it did not install
|
|
@@ -31,6 +33,10 @@ PTH_LINE: Final = "import fluidattacks_agent.gate\n"
|
|
|
31
33
|
MS: Final = 1_000_000
|
|
32
34
|
SECOND: Final = 1000
|
|
33
35
|
|
|
36
|
+
# what a workload's fork may be delayed by while a fold under way ends: a fold
|
|
37
|
+
# imports, and a lock held across a fork is held in the child forever
|
|
38
|
+
FORKING: Final = 0.5
|
|
39
|
+
|
|
34
40
|
START: Final = time.monotonic_ns()
|
|
35
41
|
|
|
36
42
|
# the reader reads u= as milliseconds since the epoch. Taken once and advanced
|
|
@@ -45,9 +51,9 @@ def reached() -> int:
|
|
|
45
51
|
|
|
46
52
|
@dataclass
|
|
47
53
|
class Probe:
|
|
48
|
-
"""An observer whose reports are
|
|
54
|
+
"""An observer whose reports are handed on as the workload runs."""
|
|
49
55
|
|
|
50
|
-
sink: Sink
|
|
56
|
+
sink: Sink
|
|
51
57
|
observer: ImportObserver = field(default_factory=ImportObserver)
|
|
52
58
|
loads: LoadObserver = field(default_factory=LoadObserver)
|
|
53
59
|
runs: RunObserver = field(default_factory=RunObserver)
|
|
@@ -61,8 +67,13 @@ class Probe:
|
|
|
61
67
|
first: dict[tuple[str, str], int] = field(default_factory=dict)
|
|
62
68
|
last: dict[tuple[str, str], int] = field(default_factory=dict)
|
|
63
69
|
scanned: dict[str, Dist] | None = None
|
|
64
|
-
|
|
70
|
+
# set while a fold runs, with the thread running it: that thread's own
|
|
71
|
+
# reads and imports are the probe's, and every other thread's are the
|
|
72
|
+
# workload's, so a fold no longer costs anyone else their evidence
|
|
65
73
|
flushing: bool = False
|
|
74
|
+
folder: int = 0
|
|
75
|
+
# whether a fork in progress holds the gate, so the parent gives it back
|
|
76
|
+
pausing: bool = False
|
|
66
77
|
stalled: int = 0
|
|
67
78
|
charged: int = 0
|
|
68
79
|
# owed until a report states them: a flush can end after charging a record
|
|
@@ -70,10 +81,13 @@ class Probe:
|
|
|
70
81
|
refused: int = 0
|
|
71
82
|
undated: int = 0
|
|
72
83
|
monitoring: bool = False
|
|
73
|
-
looking: bool = False
|
|
74
84
|
# sampled when the events were armed, not when the report is written: the
|
|
75
85
|
# later reading governs the next window rather than the one reported
|
|
76
86
|
solitary: bool = False
|
|
87
|
+
# one fold at a time: the carrier takes this without waiting, the exit with
|
|
88
|
+
# a bound. From _thread, which every interpreter start has already paid
|
|
89
|
+
# for, and threading is not
|
|
90
|
+
gate: _thread.LockType = field(default_factory=_thread.allocate_lock, repr=False)
|
|
77
91
|
|
|
78
92
|
def find_spec(
|
|
79
93
|
self,
|
|
@@ -81,92 +95,108 @@ class Probe:
|
|
|
81
95
|
_path: object = None,
|
|
82
96
|
_target: ModuleType | None = None,
|
|
83
97
|
) -> None:
|
|
84
|
-
"""Note the lookup, and
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
98
|
+
"""Note the lookup, and claim nothing."""
|
|
99
|
+
with contextlib.suppress(Exception):
|
|
100
|
+
self._looked(fullname)
|
|
101
|
+
|
|
102
|
+
def _folding(self) -> bool:
|
|
103
|
+
# a fold of ours reads and imports on its own account, and none of that
|
|
104
|
+
# is the workload using anything. Its own thread only: the cost of
|
|
105
|
+
# asking is paid while a fold runs and at no other time
|
|
106
|
+
return self.flushing and _thread.get_ident() == self.folder
|
|
92
107
|
|
|
93
108
|
def _looked(self, fullname: str) -> None:
|
|
94
|
-
if self.
|
|
109
|
+
if self._folding():
|
|
95
110
|
return
|
|
96
|
-
held = self.observer.suppressed
|
|
97
111
|
module = self.observer.note(fullname)
|
|
98
|
-
|
|
99
|
-
|
|
112
|
+
if module is not None:
|
|
113
|
+
self._mark(Evidence.IMPORTED, module)
|
|
100
114
|
|
|
101
115
|
def read(self, path: str) -> None:
|
|
102
|
-
"""Note a file the workload opened
|
|
103
|
-
|
|
104
|
-
# is the workload using anything
|
|
105
|
-
if self.flushing:
|
|
116
|
+
"""Note a file the workload opened."""
|
|
117
|
+
if self._folding():
|
|
106
118
|
return
|
|
107
|
-
held = self.loads.suppressed
|
|
108
119
|
package = self.loads.note(path)
|
|
109
120
|
# tested here and not inside: the hook runs on every open the host
|
|
110
121
|
# makes, and all but a few are not ours to stamp
|
|
111
|
-
|
|
112
|
-
|
|
122
|
+
if package is not None:
|
|
123
|
+
self._mark(Evidence.READ, package)
|
|
113
124
|
|
|
114
125
|
def ran(self, filename: str, qualname: str) -> None:
|
|
115
|
-
"""Note code starting
|
|
116
|
-
if self.
|
|
126
|
+
"""Note code starting."""
|
|
127
|
+
if self._folding():
|
|
117
128
|
return
|
|
118
|
-
held = self.runs.suppressed
|
|
119
129
|
symbol = self.runs.note(filename, qualname)
|
|
120
|
-
|
|
121
|
-
|
|
130
|
+
if symbol is not None:
|
|
131
|
+
self._mark(Evidence.EXECUTED, symbol)
|
|
122
132
|
|
|
123
|
-
def _mark(self, evidence: Evidence, symbol: str) ->
|
|
124
|
-
"""Stamp a sighting
|
|
133
|
+
def _mark(self, evidence: Evidence, symbol: str) -> None:
|
|
134
|
+
"""Stamp a sighting: when its symbol was first reached, and this once."""
|
|
125
135
|
key = keyed(evidence, symbol)
|
|
126
136
|
seen = self.clock()
|
|
127
137
|
self.last[key] = seen
|
|
128
|
-
|
|
129
|
-
return 0
|
|
130
|
-
self.first[key] = seen
|
|
131
|
-
return 1
|
|
132
|
-
|
|
133
|
-
def _pile(self, added: int) -> None:
|
|
134
|
-
# a sighting with no room counts too, or a probe at its ceiling stops
|
|
135
|
-
# flushing exactly when it has the most to say
|
|
136
|
-
self.pending += added
|
|
137
|
-
if self.pending >= FLUSH_EVERY:
|
|
138
|
-
# cleared before the call, because monitoring reports the entry of
|
|
139
|
-
# flush itself and would otherwise pile up and call it again
|
|
140
|
-
self.pending = 0
|
|
141
|
-
self.flush()
|
|
138
|
+
self.first.setdefault(key, seen)
|
|
142
139
|
|
|
143
140
|
def flush(self) -> None:
|
|
144
|
-
"""Write what has not been reported,
|
|
145
|
-
|
|
141
|
+
"""Write what has not been reported, waiting out a fold under way."""
|
|
142
|
+
# the exit's: a fold under way is the last window being rendered, and
|
|
143
|
+
# what this renders after it is whatever that one had not
|
|
144
|
+
self._fold(self.gate.acquire(timeout=PARTING))
|
|
145
|
+
|
|
146
|
+
def fold(self) -> None:
|
|
147
|
+
"""Write what has not been reported, unless a fold is already under way."""
|
|
148
|
+
# the carrier's: nothing here waits, since the next period comes
|
|
149
|
+
self._fold(self.gate.acquire(blocking=False))
|
|
150
|
+
|
|
151
|
+
def _fold(self, acquired: bool) -> None:
|
|
152
|
+
if not acquired:
|
|
146
153
|
return
|
|
154
|
+
self.folder = _thread.get_ident()
|
|
147
155
|
self.flushing = True
|
|
148
156
|
try:
|
|
149
157
|
with contextlib.suppress(Exception):
|
|
150
158
|
if self._sighted():
|
|
151
159
|
self._write_fresh(self._installed())
|
|
152
160
|
finally:
|
|
153
|
-
self.pending = 0
|
|
154
161
|
self.flushing = False
|
|
162
|
+
self.gate.release()
|
|
155
163
|
self._rearm()
|
|
156
164
|
|
|
165
|
+
def forking(self) -> None:
|
|
166
|
+
"""Hold a fold under way until the fork is done, for as long as a fold takes."""
|
|
167
|
+
# bounded, because the fold may itself be waiting on an import the
|
|
168
|
+
# forking thread holds: past the bound the fork goes on as it would have
|
|
169
|
+
self.pausing = self.gate.acquire(timeout=FORKING)
|
|
170
|
+
|
|
171
|
+
def resumed(self) -> None:
|
|
172
|
+
"""Let the parent fold again once its fork is done."""
|
|
173
|
+
if self.pausing:
|
|
174
|
+
self.pausing = False
|
|
175
|
+
self.gate.release()
|
|
176
|
+
|
|
177
|
+
def forked(self) -> None:
|
|
178
|
+
"""Leave a child able to fold, whatever its parent was doing at the fork."""
|
|
179
|
+
# a lock another thread held at the fork is held in the child forever,
|
|
180
|
+
# and a flag it set stays set: the child has no thread left to clear it
|
|
181
|
+
self.gate = _thread.allocate_lock()
|
|
182
|
+
self.flushing = False
|
|
183
|
+
self.pausing = False
|
|
184
|
+
|
|
157
185
|
def _owed(self) -> bool:
|
|
158
186
|
"""Say whether a count is waiting for a report that can state it."""
|
|
159
187
|
return bool(self.refused or self.undated)
|
|
160
188
|
|
|
161
189
|
def _sighted(self) -> bool:
|
|
162
190
|
"""Say whether this window holds anything a report could carry."""
|
|
191
|
+
# what moved and not what is held, since only that can be a record; and
|
|
192
|
+
# a copy of it, since the workload's threads add to it meanwhile
|
|
163
193
|
return bool(
|
|
164
194
|
self._owed()
|
|
165
|
-
or self.loads.
|
|
166
|
-
or self.runs.
|
|
195
|
+
or self.loads.moved
|
|
196
|
+
or self.runs.moved
|
|
167
197
|
# a lookup that never became a module is dropped when the records are
|
|
168
198
|
# folded, so on its own it is not cause to walk any metadata
|
|
169
|
-
or any(name in sys.modules for name in self.observer.
|
|
199
|
+
or any(name in sys.modules for name in list(self.observer.moved))
|
|
170
200
|
or any(self._beyond().values())
|
|
171
201
|
or self.stalled != self.charged,
|
|
172
202
|
)
|
|
@@ -181,11 +211,6 @@ class Probe:
|
|
|
181
211
|
def _installed(self) -> dict[str, Dist]:
|
|
182
212
|
if self.scanned is not None:
|
|
183
213
|
return self.scanned
|
|
184
|
-
if self.looking:
|
|
185
|
-
# the walk imports, and a lookup of ours runs while the interpreter
|
|
186
|
-
# holds the lock every other import is waiting on. Not remembered,
|
|
187
|
-
# so the next flush outside one still reads it
|
|
188
|
-
return {}
|
|
189
214
|
try:
|
|
190
215
|
scanned = self.distributions()
|
|
191
216
|
except ImportError:
|
|
@@ -361,53 +386,54 @@ def running() -> Probe | None:
|
|
|
361
386
|
return None
|
|
362
387
|
|
|
363
388
|
|
|
364
|
-
def start(
|
|
389
|
+
def start() -> Probe | None:
|
|
365
390
|
"""Put the probe on the import path, unless one is there or it is unwanted."""
|
|
366
391
|
try:
|
|
367
|
-
return _start(
|
|
392
|
+
return _start()
|
|
368
393
|
except Exception: # noqa: BLE001
|
|
369
394
|
# every interpreter the workload runs reaches this line, and what runs
|
|
370
395
|
# it prints whatever escapes to the host's stderr and carries on
|
|
371
396
|
return None
|
|
372
397
|
|
|
373
398
|
|
|
374
|
-
def
|
|
375
|
-
"""
|
|
376
|
-
if not asked():
|
|
377
|
-
return FileSink()
|
|
378
|
-
told = delivery()
|
|
379
|
-
if told is None:
|
|
380
|
-
# half configured must be loud, not indistinguishable from quiet
|
|
381
|
-
return Holding()
|
|
382
|
-
return carrier(told)
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
def carrier(told: Delivery) -> Sink:
|
|
386
|
-
"""Build the one thing that carries reports out, and wire it to this process."""
|
|
399
|
+
def carrier(told: Delivery) -> "Deliverer":
|
|
400
|
+
"""Build the one thing that carries reports out, wired to nothing yet."""
|
|
387
401
|
# imported here, so a workload that never asked pays for none of it
|
|
388
402
|
from fluidattacks_agent.deliver import Deliverer # noqa: PLC0415
|
|
389
403
|
|
|
390
|
-
|
|
391
|
-
# before the flush handler, since exit handlers run in reverse order
|
|
392
|
-
atexit.register(held.parting)
|
|
393
|
-
held.registered()
|
|
394
|
-
return held
|
|
404
|
+
return Deliverer(told=told)
|
|
395
405
|
|
|
396
406
|
|
|
397
|
-
def _start(
|
|
407
|
+
def _start() -> Probe | None:
|
|
398
408
|
held = running()
|
|
399
409
|
if held is not None:
|
|
400
410
|
return held
|
|
401
411
|
if not switched_on():
|
|
402
412
|
return None
|
|
413
|
+
told = delivery()
|
|
414
|
+
if told is None:
|
|
415
|
+
# nowhere to deliver is nothing to observe for
|
|
416
|
+
return None
|
|
403
417
|
sites = site_dirs()
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
runs=RunObserver(sites=sites),
|
|
408
|
-
)
|
|
418
|
+
carrying = carrier(told)
|
|
419
|
+
probe = Probe(sink=carrying, loads=LoadObserver(sites=sites), runs=RunObserver(sites=sites))
|
|
420
|
+
carrying.fold = probe.fold
|
|
409
421
|
install(probe)
|
|
410
422
|
sys.addaudithook(watch(probe))
|
|
411
423
|
probe.monitoring = observe_runs(probe)
|
|
424
|
+
# last, because none of these can be taken back: everything above that a
|
|
425
|
+
# host could refuse has been refused or not by now. The carrier's parting
|
|
426
|
+
# goes before the flush, since exit handlers run in reverse order
|
|
427
|
+
atexit.register(carrying.parting)
|
|
428
|
+
carrying.registered()
|
|
429
|
+
if hasattr(os, "register_at_fork"):
|
|
430
|
+
os.register_at_fork(
|
|
431
|
+
before=probe.forking,
|
|
432
|
+
after_in_parent=probe.resumed,
|
|
433
|
+
after_in_child=probe.forked,
|
|
434
|
+
)
|
|
412
435
|
atexit.register(probe.flush)
|
|
436
|
+
# the clock starts with the probe, so a worker that sees little still
|
|
437
|
+
# reports what it saw when the period ends
|
|
438
|
+
carrying.rouse()
|
|
413
439
|
return probe
|
|
@@ -1,163 +0,0 @@
|
|
|
1
|
-
"""Leave a report where something can pick it up, or refuse and say which."""
|
|
2
|
-
|
|
3
|
-
import contextlib
|
|
4
|
-
import errno
|
|
5
|
-
import os
|
|
6
|
-
import stat
|
|
7
|
-
from dataclasses import dataclass, field
|
|
8
|
-
from typing import Final, Protocol
|
|
9
|
-
|
|
10
|
-
# the path is a contract with the agent that drains it
|
|
11
|
-
REPORT_DIR: Final = "/tmp/.watches-exec" # noqa: S108
|
|
12
|
-
REPORT_SUFFIX: Final = ".report"
|
|
13
|
-
STAGE_SUFFIX: Final = ".tmp"
|
|
14
|
-
|
|
15
|
-
# reports of its own, bytes any report holds, and entries it will walk to count
|
|
16
|
-
MAX_OUTSTANDING: Final = 64
|
|
17
|
-
MAX_OUTSTANDING_BYTES: Final = 8 << 20
|
|
18
|
-
MAX_ENTRIES: Final = 4096
|
|
19
|
-
|
|
20
|
-
SHARED: Final = 0o077
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
class Stalled(OSError):
|
|
24
|
-
"""Nothing was taken, and what was not taken is still owed."""
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
class Sink(Protocol):
|
|
28
|
-
"""Somewhere a report can go, which may refuse it and must never lie."""
|
|
29
|
-
|
|
30
|
-
def write(self, text: str) -> None: ...
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def ours(status: os.stat_result) -> bool:
|
|
34
|
-
"""Say whether what the status describes is ours alone to write."""
|
|
35
|
-
return status.st_uid == os.geteuid() and not status.st_mode & SHARED
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
@dataclass(frozen=True)
|
|
39
|
-
class Held:
|
|
40
|
-
"""What one directory entry costs, and whether the sink left it there."""
|
|
41
|
-
|
|
42
|
-
size: int
|
|
43
|
-
mine: bool
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def _held(entry: os.DirEntry[str], stem: str) -> Held:
|
|
47
|
-
if not entry.name.endswith(REPORT_SUFFIX):
|
|
48
|
-
return Held(size=0, mine=False)
|
|
49
|
-
try:
|
|
50
|
-
status = entry.stat(follow_symlinks=False)
|
|
51
|
-
except OSError:
|
|
52
|
-
return Held(size=0, mine=False)
|
|
53
|
-
if not stat.S_ISREG(status.st_mode):
|
|
54
|
-
return Held(size=0, mine=False)
|
|
55
|
-
return Held(size=status.st_size, mine=entry.name.startswith(stem))
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
class Holding:
|
|
59
|
-
"""A sink for a workload that asked for delivery, which nothing yet does."""
|
|
60
|
-
|
|
61
|
-
def write(self, text: str) -> None:
|
|
62
|
-
"""Take nothing, and be counted for it."""
|
|
63
|
-
raise Stalled(errno.ENOSYS, "nothing delivers yet", len(text))
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
@dataclass
|
|
67
|
-
class FileSink:
|
|
68
|
-
"""Reports left in a directory of our own for the agent to drain."""
|
|
69
|
-
|
|
70
|
-
directory: str | os.PathLike[str] = REPORT_DIR
|
|
71
|
-
written: int = 0
|
|
72
|
-
# so a report of ours cannot be named by anything that did not write it,
|
|
73
|
-
# and a reused pid never counts its predecessor's against itself
|
|
74
|
-
token: str = field(default_factory=lambda: os.urandom(8).hex())
|
|
75
|
-
|
|
76
|
-
def write(self, text: str) -> None:
|
|
77
|
-
"""Take a report, unless what is already here says to stop."""
|
|
78
|
-
directory = self._open_directory()
|
|
79
|
-
try:
|
|
80
|
-
# installed on its own nothing drains this, so the ceiling is ours
|
|
81
|
-
if self._piled(directory):
|
|
82
|
-
raise Stalled(errno.ENOSPC, "reports outstanding")
|
|
83
|
-
self._write_into(directory, text)
|
|
84
|
-
finally:
|
|
85
|
-
os.close(directory)
|
|
86
|
-
|
|
87
|
-
def _open_directory(self) -> int:
|
|
88
|
-
# every way this can fail is a flush that did not land, and one that
|
|
89
|
-
# said so as anything else was a mute nothing counted: a directory this
|
|
90
|
-
# cannot use costs an attacker one bit of its mode
|
|
91
|
-
try:
|
|
92
|
-
with contextlib.suppress(FileExistsError):
|
|
93
|
-
os.mkdir(self.directory, 0o700) # noqa: PTH102
|
|
94
|
-
handle = os.open(self.directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
95
|
-
except OSError as refused:
|
|
96
|
-
raise Stalled(errno.EACCES, "no directory", str(self.directory)) from refused
|
|
97
|
-
# the mode a directory was made with says nothing about one that was
|
|
98
|
-
# already there, and the descriptor is what the answer has to be about
|
|
99
|
-
if ours(os.fstat(handle)):
|
|
100
|
-
return handle
|
|
101
|
-
os.close(handle)
|
|
102
|
-
raise Stalled(errno.EACCES, "not ours", str(self.directory))
|
|
103
|
-
|
|
104
|
-
def _piled(self, directory: int) -> bool:
|
|
105
|
-
mine = 0
|
|
106
|
-
bytes_held = 0
|
|
107
|
-
walked = 0
|
|
108
|
-
with os.scandir(directory) as entries:
|
|
109
|
-
for entry in entries:
|
|
110
|
-
walked += 1
|
|
111
|
-
# too full to keep writing into, whoever filled it
|
|
112
|
-
if walked > MAX_ENTRIES:
|
|
113
|
-
return True
|
|
114
|
-
held = _held(entry, self._stem())
|
|
115
|
-
bytes_held += held.size
|
|
116
|
-
mine += int(held.mine)
|
|
117
|
-
if mine >= MAX_OUTSTANDING or bytes_held >= MAX_OUTSTANDING_BYTES:
|
|
118
|
-
return True
|
|
119
|
-
return False
|
|
120
|
-
|
|
121
|
-
def _stem(self) -> str:
|
|
122
|
-
return f"{os.getpid()}-{self.token}-"
|
|
123
|
-
|
|
124
|
-
def _write_into(self, directory: int, text: str) -> None:
|
|
125
|
-
payload = text.encode()
|
|
126
|
-
stem = f"{self._stem()}{self.written}"
|
|
127
|
-
staged = stem + STAGE_SUFFIX
|
|
128
|
-
# spent the moment it is tried: what sits under a name is not always
|
|
129
|
-
# ours to clear, and coming back to one forever is how one silences us
|
|
130
|
-
self.written += 1
|
|
131
|
-
try:
|
|
132
|
-
self._stage(directory, staged, payload)
|
|
133
|
-
os.replace(
|
|
134
|
-
staged,
|
|
135
|
-
stem + REPORT_SUFFIX,
|
|
136
|
-
src_dir_fd=directory,
|
|
137
|
-
dst_dir_fd=directory,
|
|
138
|
-
)
|
|
139
|
-
except OSError as refused:
|
|
140
|
-
# nothing will ever read what a failed attempt staged, and the name
|
|
141
|
-
# it holds counts against the entries this is willing to walk
|
|
142
|
-
with contextlib.suppress(OSError):
|
|
143
|
-
os.unlink(staged, dir_fd=directory)
|
|
144
|
-
raise Stalled(errno.ENOSPC, "not taken", staged) from refused
|
|
145
|
-
|
|
146
|
-
def _stage(self, directory: int, name: str, payload: bytes) -> None:
|
|
147
|
-
handle = self._create(directory, name)
|
|
148
|
-
try:
|
|
149
|
-
written = os.write(handle, payload)
|
|
150
|
-
finally:
|
|
151
|
-
os.close(handle)
|
|
152
|
-
# os.write does not raise on a partial write, and the statement is the
|
|
153
|
-
# last line, so a truncation takes it first
|
|
154
|
-
if written != len(payload):
|
|
155
|
-
raise OSError(errno.ENOSPC, "short write", name)
|
|
156
|
-
|
|
157
|
-
def _create(self, directory: int, name: str) -> int:
|
|
158
|
-
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
|
159
|
-
try:
|
|
160
|
-
return os.open(name, flags, 0o600, dir_fd=directory)
|
|
161
|
-
except FileExistsError:
|
|
162
|
-
os.unlink(name, dir_fd=directory)
|
|
163
|
-
return os.open(name, flags, 0o600, dir_fd=directory)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|