fluidattacks-agent 0.1.2__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/PKG-INFO +7 -8
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/README.md +6 -7
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/deliver.py +37 -33
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/post.py +10 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/settings.py +0 -9
- fluidattacks_agent-0.2.0/fluidattacks_agent/sink.py +16 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/startup.py +103 -103
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/pyproject.toml +1 -1
- fluidattacks_agent-0.1.2/fluidattacks_agent/sink.py +0 -163
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/.gitignore +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/__init__.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/batch.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/distributions.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/executions.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/gate.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/loads.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/observer.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/outbox.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/patience.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/report.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent/switch.py +0 -0
- {fluidattacks_agent-0.1.2 → fluidattacks_agent-0.2.0}/fluidattacks_agent.pth +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: fluidattacks-agent
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: In-process probe reporting what a Python workload imports and runs
|
|
5
5
|
Project-URL: Homepage, https://fluidattacks.com
|
|
6
6
|
Project-URL: Source, https://gitlab.com/fluidattacks/universe/-/tree/trunk/watches/agents/python
|
|
@@ -26,7 +26,7 @@ dependency inventory can say which of its findings are reachable at runtime and
|
|
|
26
26
|
which are not.
|
|
27
27
|
|
|
28
28
|
It is a library, not a service. It observes the interpreter it is installed in,
|
|
29
|
-
|
|
29
|
+
sends what it saw, and does nothing else. It takes no dependencies: the
|
|
30
30
|
standard library only, because it is installed into workloads we do not own.
|
|
31
31
|
|
|
32
32
|
## Installing
|
|
@@ -60,8 +60,8 @@ a workload's own first-party code produce no records.
|
|
|
60
60
|
|
|
61
61
|
## Where reports go
|
|
62
62
|
|
|
63
|
-
|
|
64
|
-
|
|
63
|
+
Naming an endpoint is what makes the probe observe at all. A workload that
|
|
64
|
+
names none starts nothing:
|
|
65
65
|
|
|
66
66
|
| | |
|
|
67
67
|
|---|---|
|
|
@@ -74,10 +74,9 @@ drain. Naming an endpoint sends them instead:
|
|
|
74
74
|
A file is preferred over a variable because a file can be mode 400, while an
|
|
75
75
|
environment variable is readable by any process of the same user.
|
|
76
76
|
|
|
77
|
-
An endpoint named without enough beside it to reach
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
up where nobody drains it.
|
|
77
|
+
An endpoint named without enough beside it to reach — no credential, or not
|
|
78
|
+
`https://` — is a misconfiguration, and the probe starts nothing rather than
|
|
79
|
+
observe what it could never deliver.
|
|
81
80
|
|
|
82
81
|
## What travels, and what does not
|
|
83
82
|
|
|
@@ -5,7 +5,7 @@ dependency inventory can say which of its findings are reachable at runtime and
|
|
|
5
5
|
which are not.
|
|
6
6
|
|
|
7
7
|
It is a library, not a service. It observes the interpreter it is installed in,
|
|
8
|
-
|
|
8
|
+
sends what it saw, and does nothing else. It takes no dependencies: the
|
|
9
9
|
standard library only, because it is installed into workloads we do not own.
|
|
10
10
|
|
|
11
11
|
## Installing
|
|
@@ -39,8 +39,8 @@ a workload's own first-party code produce no records.
|
|
|
39
39
|
|
|
40
40
|
## Where reports go
|
|
41
41
|
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
Naming an endpoint is what makes the probe observe at all. A workload that
|
|
43
|
+
names none starts nothing:
|
|
44
44
|
|
|
45
45
|
| | |
|
|
46
46
|
|---|---|
|
|
@@ -53,10 +53,9 @@ drain. Naming an endpoint sends them instead:
|
|
|
53
53
|
A file is preferred over a variable because a file can be mode 400, while an
|
|
54
54
|
environment variable is readable by any process of the same user.
|
|
55
55
|
|
|
56
|
-
An endpoint named without enough beside it to reach
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
up where nobody drains it.
|
|
56
|
+
An endpoint named without enough beside it to reach — no credential, or not
|
|
57
|
+
`https://` — is a misconfiguration, and the probe starts nothing rather than
|
|
58
|
+
observe what it could never deliver.
|
|
60
59
|
|
|
61
60
|
## What travels, and what does not
|
|
62
61
|
|
|
@@ -12,18 +12,18 @@ from typing import Final
|
|
|
12
12
|
from fluidattacks_agent.batch import Batch, Batcher
|
|
13
13
|
from fluidattacks_agent.outbox import Outbox
|
|
14
14
|
from fluidattacks_agent.patience import TOPMOST, fraction, splay, standoff
|
|
15
|
-
from fluidattacks_agent.post import TIMEOUT, Verdict, posted
|
|
15
|
+
from fluidattacks_agent.post import TIMEOUT, Verdict, posted, prepared
|
|
16
16
|
from fluidattacks_agent.settings import Delivery
|
|
17
|
-
from fluidattacks_agent.sink import Stalled
|
|
17
|
+
from fluidattacks_agent.sink import PARTING, Stalled
|
|
18
18
|
|
|
19
|
-
# how old the centre's picture may be
|
|
19
|
+
# how old the centre's picture may be: the period the window is folded on
|
|
20
20
|
INTERVAL: Final = 15.0
|
|
21
21
|
|
|
22
|
-
# what a workload's own exit may be delayed by while what is held goes out
|
|
23
|
-
PARTING: Final = 2.0
|
|
24
|
-
|
|
25
22
|
NAME: Final = "fluidattacks-agent"
|
|
26
23
|
|
|
24
|
+
# before any fold, so the first wait to end folds whatever the probe holds
|
|
25
|
+
NEVER: Final = float("-inf")
|
|
26
|
+
|
|
27
27
|
|
|
28
28
|
@dataclass
|
|
29
29
|
class Deliverer:
|
|
@@ -46,23 +46,27 @@ class Deliverer:
|
|
|
46
46
|
guard: threading.Lock = field(default_factory=threading.Lock, repr=False)
|
|
47
47
|
# one pass at a time: what is taken but not delivered lives in one place
|
|
48
48
|
passing: threading.Lock = field(default_factory=threading.Lock, repr=False)
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
# ends a wait every standoff, and a wait ending is not a period going by
|
|
54
|
-
reminded: float = 0.0
|
|
49
|
+
# what renders the probe's window into reports. Called here, on the
|
|
50
|
+
# carrier's own thread and clock, so a workload's threads render nothing
|
|
51
|
+
fold: Callable[[], None] = field(default=lambda: None, repr=False)
|
|
52
|
+
folded: float = NEVER
|
|
55
53
|
leaving: bool = False
|
|
56
54
|
delivered: int = 0
|
|
57
55
|
|
|
58
56
|
def write(self, text: str) -> None:
|
|
59
57
|
"""Take a report for delivery, or refuse and be counted for it."""
|
|
58
|
+
# here, inside the fold that made the report, so what a post imports is
|
|
59
|
+
# imported under the lock a fork waits out and not on the pass after
|
|
60
|
+
prepared()
|
|
60
61
|
if self.batcher.next(self.told, text, self.outbox.put) is None:
|
|
61
62
|
raise Stalled(errno.ENOSPC, "nothing taken", len(text))
|
|
62
|
-
|
|
63
|
+
# a fold of the carrier's own writes here, and its pass follows in the
|
|
64
|
+
# same turn: only another thread has a carrier to wake
|
|
65
|
+
if threading.current_thread() is not self.thread:
|
|
66
|
+
self.rouse()
|
|
63
67
|
|
|
64
|
-
def
|
|
65
|
-
"""
|
|
68
|
+
def rouse(self) -> None:
|
|
69
|
+
"""Start the carrier if there is none, and say there is something for it."""
|
|
66
70
|
with self.guard:
|
|
67
71
|
# a carrier that stopped takes delivery with it, and one is not
|
|
68
72
|
# worth starting for a process already on its way out
|
|
@@ -81,23 +85,17 @@ class Deliverer:
|
|
|
81
85
|
while not self.leaving:
|
|
82
86
|
# a thread that dies takes delivery with it and tells nobody
|
|
83
87
|
with contextlib.suppress(Exception):
|
|
84
|
-
|
|
88
|
+
self.woken.wait(self._waiting())
|
|
85
89
|
self.woken.clear()
|
|
86
|
-
|
|
90
|
+
now = time.monotonic()
|
|
91
|
+
if now < self.due:
|
|
87
92
|
continue
|
|
93
|
+
# on this clock and no other: a far end refusing and then taking
|
|
94
|
+
# ends a wait every standoff, and a wait ending is not a period
|
|
95
|
+
if now - self.folded >= self.interval:
|
|
96
|
+
self.folded = now
|
|
97
|
+
self.fold()
|
|
88
98
|
self.offering()
|
|
89
|
-
# a period, never a wake: a wake says there is something to
|
|
90
|
-
# carry, and asking for more on one made each flush bring the
|
|
91
|
-
# next. Nothing in hand, or the far end is still refusing
|
|
92
|
-
now = time.monotonic()
|
|
93
|
-
if (
|
|
94
|
-
not roused
|
|
95
|
-
and self.holding is None
|
|
96
|
-
and not self.leaving
|
|
97
|
-
and now - self.reminded >= self.interval
|
|
98
|
-
):
|
|
99
|
-
self.reminded = now
|
|
100
|
-
self.remind()
|
|
101
99
|
|
|
102
100
|
def parting(self) -> None:
|
|
103
101
|
"""Give whatever is held one bounded chance to travel, then give up."""
|
|
@@ -128,11 +126,15 @@ class Deliverer:
|
|
|
128
126
|
# every worker of a pre-forking server was born in the same second
|
|
129
127
|
self.due = 0.0
|
|
130
128
|
self.tries = 0
|
|
131
|
-
self.
|
|
129
|
+
self.folded = NEVER
|
|
132
130
|
# a child is not on its way out merely because its parent was
|
|
133
131
|
self.leaving = False
|
|
134
132
|
self.batcher.forked()
|
|
135
133
|
self.outbox.forked()
|
|
134
|
+
# the clock is this thread's, and the child has none until it is
|
|
135
|
+
# started again. Safe here because threading's own fork handler, which
|
|
136
|
+
# registered first, has already run
|
|
137
|
+
self.rouse()
|
|
136
138
|
|
|
137
139
|
def registered(self) -> None:
|
|
138
140
|
"""Ask the interpreter to say when this process has become two."""
|
|
@@ -173,6 +175,8 @@ class Deliverer:
|
|
|
173
175
|
self.due = time.monotonic() + standoff(self.tries, self.share())
|
|
174
176
|
|
|
175
177
|
def _waiting(self) -> float:
|
|
176
|
-
"""Wait out a standoff if there is one, and otherwise
|
|
177
|
-
|
|
178
|
-
|
|
178
|
+
"""Wait out a standoff if there is one, and otherwise until the next fold."""
|
|
179
|
+
now = time.monotonic()
|
|
180
|
+
if self.due > now:
|
|
181
|
+
return self.due - now
|
|
182
|
+
return max(0.0, self.folded + self.interval - now)
|
|
@@ -118,6 +118,16 @@ def targeted(path: str, query: str) -> str:
|
|
|
118
118
|
return f"{path or '/'}?{query}" if query else path or "/"
|
|
119
119
|
|
|
120
120
|
|
|
121
|
+
def prepared() -> None:
|
|
122
|
+
"""Import what a post needs, where a fork cannot catch an import half done."""
|
|
123
|
+
# a fork taken while another thread holds a module's import lock leaves
|
|
124
|
+
# that lock held in the child forever, so these are taken under the fold's
|
|
125
|
+
# own lock, which a fork waits out, and never first on the thread that posts
|
|
126
|
+
import http.client # noqa: F401, PLC0415
|
|
127
|
+
import ssl # noqa: F401, PLC0415
|
|
128
|
+
import urllib.parse # noqa: F401, PLC0415
|
|
129
|
+
|
|
130
|
+
|
|
121
131
|
def proven() -> "ssl.SSLContext":
|
|
122
132
|
"""Give a context the far end has to prove itself on, hostname included."""
|
|
123
133
|
import ssl # noqa: PLC0415
|
|
@@ -91,15 +91,6 @@ def _credential(value: str | None) -> Secret | None:
|
|
|
91
91
|
return Secret(named) if _readable(named) else None
|
|
92
92
|
|
|
93
93
|
|
|
94
|
-
def asked(environ: Mapping[str, str] | None = None) -> bool:
|
|
95
|
-
"""Say whether the workload asked for its reports to be delivered at all."""
|
|
96
|
-
try:
|
|
97
|
-
held = os.environ if environ is None else environ
|
|
98
|
-
return _setting(held, ENDPOINT) is not None or ENDPOINT in held
|
|
99
|
-
except Exception: # noqa: BLE001
|
|
100
|
-
return False
|
|
101
|
-
|
|
102
|
-
|
|
103
94
|
def delivery(environ: Mapping[str, str] | None = None) -> Delivery | None:
|
|
104
95
|
"""Read where reports go, or nothing at all if it cannot be read whole."""
|
|
105
96
|
try:
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Somewhere a report can go, and the refusal that says it did not."""
|
|
2
|
+
|
|
3
|
+
from typing import Final, Protocol
|
|
4
|
+
|
|
5
|
+
# what a workload's own exit may be delayed by while what is held goes out
|
|
6
|
+
PARTING: Final = 2.0
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Stalled(OSError):
|
|
10
|
+
"""Nothing was taken, and what was not taken is still owed."""
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Sink(Protocol):
|
|
14
|
+
"""Somewhere a report can go, which may refuse it and must never lie."""
|
|
15
|
+
|
|
16
|
+
def write(self, text: str) -> None: ...
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Start the probe inside a workload and
|
|
1
|
+
"""Start the probe inside a workload and hand its reports to what carries them."""
|
|
2
2
|
|
|
3
3
|
import _thread
|
|
4
4
|
import atexit
|
|
@@ -9,19 +9,19 @@ import time
|
|
|
9
9
|
from collections.abc import Callable
|
|
10
10
|
from dataclasses import dataclass, field, replace
|
|
11
11
|
from types import CodeType, ModuleType
|
|
12
|
-
from typing import Final
|
|
12
|
+
from typing import TYPE_CHECKING, Final
|
|
13
13
|
|
|
14
14
|
from fluidattacks_agent.distributions import Dist, distribution_map
|
|
15
15
|
from fluidattacks_agent.executions import TOOL, TOOL_NAME, RunObserver, alone, available
|
|
16
16
|
from fluidattacks_agent.loads import LoadObserver
|
|
17
17
|
from fluidattacks_agent.observer import ImportObserver, install
|
|
18
18
|
from fluidattacks_agent.report import Evidence, Record, Window, keyed, render
|
|
19
|
-
from fluidattacks_agent.settings import Delivery,
|
|
20
|
-
from fluidattacks_agent.sink import
|
|
19
|
+
from fluidattacks_agent.settings import Delivery, delivery
|
|
20
|
+
from fluidattacks_agent.sink import PARTING, Sink, Stalled
|
|
21
21
|
from fluidattacks_agent.switch import switched_on
|
|
22
22
|
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from fluidattacks_agent.deliver import Deliverer
|
|
25
25
|
|
|
26
26
|
# what a site directory is given so that installing the package starts the
|
|
27
27
|
# probe. The agent writes the same line when it injects one it did not install
|
|
@@ -33,6 +33,10 @@ PTH_LINE: Final = "import fluidattacks_agent.gate\n"
|
|
|
33
33
|
MS: Final = 1_000_000
|
|
34
34
|
SECOND: Final = 1000
|
|
35
35
|
|
|
36
|
+
# what a workload's fork may be delayed by while a fold under way ends: a fold
|
|
37
|
+
# imports, and a lock held across a fork is held in the child forever
|
|
38
|
+
FORKING: Final = 0.5
|
|
39
|
+
|
|
36
40
|
START: Final = time.monotonic_ns()
|
|
37
41
|
|
|
38
42
|
# the reader reads u= as milliseconds since the epoch. Taken once and advanced
|
|
@@ -47,9 +51,9 @@ def reached() -> int:
|
|
|
47
51
|
|
|
48
52
|
@dataclass
|
|
49
53
|
class Probe:
|
|
50
|
-
"""An observer whose reports are
|
|
54
|
+
"""An observer whose reports are handed on as the workload runs."""
|
|
51
55
|
|
|
52
|
-
sink: Sink
|
|
56
|
+
sink: Sink
|
|
53
57
|
observer: ImportObserver = field(default_factory=ImportObserver)
|
|
54
58
|
loads: LoadObserver = field(default_factory=LoadObserver)
|
|
55
59
|
runs: RunObserver = field(default_factory=RunObserver)
|
|
@@ -63,8 +67,13 @@ class Probe:
|
|
|
63
67
|
first: dict[tuple[str, str], int] = field(default_factory=dict)
|
|
64
68
|
last: dict[tuple[str, str], int] = field(default_factory=dict)
|
|
65
69
|
scanned: dict[str, Dist] | None = None
|
|
66
|
-
|
|
70
|
+
# set while a fold runs, with the thread running it: that thread's own
|
|
71
|
+
# reads and imports are the probe's, and every other thread's are the
|
|
72
|
+
# workload's, so a fold no longer costs anyone else their evidence
|
|
67
73
|
flushing: bool = False
|
|
74
|
+
folder: int = 0
|
|
75
|
+
# whether a fork in progress holds the gate, so the parent gives it back
|
|
76
|
+
pausing: bool = False
|
|
68
77
|
stalled: int = 0
|
|
69
78
|
charged: int = 0
|
|
70
79
|
# owed until a report states them: a flush can end after charging a record
|
|
@@ -72,16 +81,12 @@ class Probe:
|
|
|
72
81
|
refused: int = 0
|
|
73
82
|
undated: int = 0
|
|
74
83
|
monitoring: bool = False
|
|
75
|
-
looking: bool = False
|
|
76
84
|
# sampled when the events were armed, not when the report is written: the
|
|
77
85
|
# later reading governs the next window rather than the one reported
|
|
78
86
|
solitary: bool = False
|
|
79
|
-
#
|
|
80
|
-
#
|
|
81
|
-
|
|
82
|
-
# every thread sees overdue at once, so the flush is gated by a lock nothing
|
|
83
|
-
# waits on: a second caller finds it taken and returns. From _thread, which
|
|
84
|
-
# every interpreter start has already paid for, and threading is not
|
|
87
|
+
# one fold at a time: the carrier takes this without waiting, the exit with
|
|
88
|
+
# a bound. From _thread, which every interpreter start has already paid
|
|
89
|
+
# for, and threading is not
|
|
85
90
|
gate: _thread.LockType = field(default_factory=_thread.allocate_lock, repr=False)
|
|
86
91
|
|
|
87
92
|
def find_spec(
|
|
@@ -90,90 +95,92 @@ class Probe:
|
|
|
90
95
|
_path: object = None,
|
|
91
96
|
_target: ModuleType | None = None,
|
|
92
97
|
) -> None:
|
|
93
|
-
"""Note the lookup, and
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
98
|
+
"""Note the lookup, and claim nothing."""
|
|
99
|
+
with contextlib.suppress(Exception):
|
|
100
|
+
self._looked(fullname)
|
|
101
|
+
|
|
102
|
+
def _folding(self) -> bool:
|
|
103
|
+
# a fold of ours reads and imports on its own account, and none of that
|
|
104
|
+
# is the workload using anything. Its own thread only: the cost of
|
|
105
|
+
# asking is paid while a fold runs and at no other time
|
|
106
|
+
return self.flushing and _thread.get_ident() == self.folder
|
|
101
107
|
|
|
102
108
|
def _looked(self, fullname: str) -> None:
|
|
103
|
-
if self.
|
|
109
|
+
if self._folding():
|
|
104
110
|
return
|
|
105
111
|
module = self.observer.note(fullname)
|
|
106
|
-
|
|
112
|
+
if module is not None:
|
|
113
|
+
self._mark(Evidence.IMPORTED, module)
|
|
107
114
|
|
|
108
115
|
def read(self, path: str) -> None:
|
|
109
|
-
"""Note a file the workload opened
|
|
110
|
-
|
|
111
|
-
# is the workload using anything
|
|
112
|
-
if self.flushing:
|
|
116
|
+
"""Note a file the workload opened."""
|
|
117
|
+
if self._folding():
|
|
113
118
|
return
|
|
114
119
|
package = self.loads.note(path)
|
|
115
120
|
# tested here and not inside: the hook runs on every open the host
|
|
116
121
|
# makes, and all but a few are not ours to stamp
|
|
117
|
-
|
|
122
|
+
if package is not None:
|
|
123
|
+
self._mark(Evidence.READ, package)
|
|
118
124
|
|
|
119
125
|
def ran(self, filename: str, qualname: str) -> None:
|
|
120
|
-
"""Note code starting
|
|
121
|
-
if self.
|
|
126
|
+
"""Note code starting."""
|
|
127
|
+
if self._folding():
|
|
122
128
|
return
|
|
123
129
|
symbol = self.runs.note(filename, qualname)
|
|
124
|
-
|
|
130
|
+
if symbol is not None:
|
|
131
|
+
self._mark(Evidence.EXECUTED, symbol)
|
|
125
132
|
|
|
126
|
-
def _mark(self, evidence: Evidence, symbol: str) ->
|
|
127
|
-
"""Stamp a sighting
|
|
133
|
+
def _mark(self, evidence: Evidence, symbol: str) -> None:
|
|
134
|
+
"""Stamp a sighting: when its symbol was first reached, and this once."""
|
|
128
135
|
key = keyed(evidence, symbol)
|
|
129
136
|
seen = self.clock()
|
|
130
137
|
self.last[key] = seen
|
|
131
|
-
|
|
132
|
-
return 0
|
|
133
|
-
self.first[key] = seen
|
|
134
|
-
return 1
|
|
135
|
-
|
|
136
|
-
def remind(self) -> None:
|
|
137
|
-
"""Have the next sighting bring a flush, however few have piled up."""
|
|
138
|
-
# a flag and never a flush: the carrier's clock calls this from a
|
|
139
|
-
# thread of its own, and a flush runs on the workload's alone
|
|
140
|
-
self.overdue = True
|
|
141
|
-
|
|
142
|
-
def _pile(self, added: int) -> None:
|
|
143
|
-
# first sightings only: a flush re-arms every code object, so counting
|
|
144
|
-
# one seen again, or one with no room, made each flush bring the next
|
|
145
|
-
self.pending += added
|
|
146
|
-
if self.overdue or self.pending >= FLUSH_EVERY:
|
|
147
|
-
# cleared before the call, because monitoring reports the entry of
|
|
148
|
-
# flush itself and would otherwise pile up and call it again
|
|
149
|
-
self.pending = 0
|
|
150
|
-
self.flush()
|
|
138
|
+
self.first.setdefault(key, seen)
|
|
151
139
|
|
|
152
140
|
def flush(self) -> None:
|
|
153
|
-
"""Write what has not been reported,
|
|
154
|
-
|
|
141
|
+
"""Write what has not been reported, waiting out a fold under way."""
|
|
142
|
+
# the exit's: a fold under way is the last window being rendered, and
|
|
143
|
+
# what this renders after it is whatever that one had not
|
|
144
|
+
self._fold(self.gate.acquire(timeout=PARTING))
|
|
145
|
+
|
|
146
|
+
def fold(self) -> None:
|
|
147
|
+
"""Write what has not been reported, unless a fold is already under way."""
|
|
148
|
+
# the carrier's: nothing here waits, since the next period comes
|
|
149
|
+
self._fold(self.gate.acquire(blocking=False))
|
|
150
|
+
|
|
151
|
+
def _fold(self, acquired: bool) -> None:
|
|
152
|
+
if not acquired:
|
|
155
153
|
return
|
|
154
|
+
self.folder = _thread.get_ident()
|
|
156
155
|
self.flushing = True
|
|
157
|
-
# spent here, so a reminder given while this runs brings one more flush
|
|
158
|
-
self.overdue = False
|
|
159
156
|
try:
|
|
160
157
|
with contextlib.suppress(Exception):
|
|
161
158
|
if self._sighted():
|
|
162
159
|
self._write_fresh(self._installed())
|
|
163
160
|
finally:
|
|
164
|
-
self.pending = 0
|
|
165
161
|
self.flushing = False
|
|
166
162
|
self.gate.release()
|
|
167
163
|
self._rearm()
|
|
168
164
|
|
|
165
|
+
def forking(self) -> None:
|
|
166
|
+
"""Hold a fold under way until the fork is done, for as long as a fold takes."""
|
|
167
|
+
# bounded, because the fold may itself be waiting on an import the
|
|
168
|
+
# forking thread holds: past the bound the fork goes on as it would have
|
|
169
|
+
self.pausing = self.gate.acquire(timeout=FORKING)
|
|
170
|
+
|
|
171
|
+
def resumed(self) -> None:
|
|
172
|
+
"""Let the parent fold again once its fork is done."""
|
|
173
|
+
if self.pausing:
|
|
174
|
+
self.pausing = False
|
|
175
|
+
self.gate.release()
|
|
176
|
+
|
|
169
177
|
def forked(self) -> None:
|
|
170
|
-
"""Leave a child able to
|
|
178
|
+
"""Leave a child able to fold, whatever its parent was doing at the fork."""
|
|
171
179
|
# a lock another thread held at the fork is held in the child forever,
|
|
172
180
|
# and a flag it set stays set: the child has no thread left to clear it
|
|
173
181
|
self.gate = _thread.allocate_lock()
|
|
174
182
|
self.flushing = False
|
|
175
|
-
self.
|
|
176
|
-
self.pending = 0
|
|
183
|
+
self.pausing = False
|
|
177
184
|
|
|
178
185
|
def _owed(self) -> bool:
|
|
179
186
|
"""Say whether a count is waiting for a report that can state it."""
|
|
@@ -181,13 +188,15 @@ class Probe:
|
|
|
181
188
|
|
|
182
189
|
def _sighted(self) -> bool:
|
|
183
190
|
"""Say whether this window holds anything a report could carry."""
|
|
191
|
+
# what moved and not what is held, since only that can be a record; and
|
|
192
|
+
# a copy of it, since the workload's threads add to it meanwhile
|
|
184
193
|
return bool(
|
|
185
194
|
self._owed()
|
|
186
|
-
or self.loads.
|
|
187
|
-
or self.runs.
|
|
195
|
+
or self.loads.moved
|
|
196
|
+
or self.runs.moved
|
|
188
197
|
# a lookup that never became a module is dropped when the records are
|
|
189
198
|
# folded, so on its own it is not cause to walk any metadata
|
|
190
|
-
or any(name in sys.modules for name in self.observer.
|
|
199
|
+
or any(name in sys.modules for name in list(self.observer.moved))
|
|
191
200
|
or any(self._beyond().values())
|
|
192
201
|
or self.stalled != self.charged,
|
|
193
202
|
)
|
|
@@ -202,11 +211,6 @@ class Probe:
|
|
|
202
211
|
def _installed(self) -> dict[str, Dist]:
|
|
203
212
|
if self.scanned is not None:
|
|
204
213
|
return self.scanned
|
|
205
|
-
if self.looking:
|
|
206
|
-
# the walk imports, and a lookup of ours runs while the interpreter
|
|
207
|
-
# holds the lock every other import is waiting on. Not remembered,
|
|
208
|
-
# so the next flush outside one still reads it
|
|
209
|
-
return {}
|
|
210
214
|
try:
|
|
211
215
|
scanned = self.distributions()
|
|
212
216
|
except ImportError:
|
|
@@ -382,58 +386,54 @@ def running() -> Probe | None:
|
|
|
382
386
|
return None
|
|
383
387
|
|
|
384
388
|
|
|
385
|
-
def start(
|
|
389
|
+
def start() -> Probe | None:
|
|
386
390
|
"""Put the probe on the import path, unless one is there or it is unwanted."""
|
|
387
391
|
try:
|
|
388
|
-
return _start(
|
|
392
|
+
return _start()
|
|
389
393
|
except Exception: # noqa: BLE001
|
|
390
394
|
# every interpreter the workload runs reaches this line, and what runs
|
|
391
395
|
# it prints whatever escapes to the host's stderr and carries on
|
|
392
396
|
return None
|
|
393
397
|
|
|
394
398
|
|
|
395
|
-
def
|
|
396
|
-
"""
|
|
397
|
-
Give a workload what it asked for, and something loud if it asked badly.
|
|
398
|
-
|
|
399
|
-
``remind`` is what a carrier calls, off the workload's threads, once a
|
|
400
|
-
period has gone by with the far end in reach and nothing left to carry.
|
|
401
|
-
"""
|
|
402
|
-
if not asked():
|
|
403
|
-
return FileSink()
|
|
404
|
-
told = delivery()
|
|
405
|
-
if told is None:
|
|
406
|
-
# half configured must be loud, not indistinguishable from quiet
|
|
407
|
-
return Holding()
|
|
408
|
-
return carrier(told, remind)
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
def carrier(told: Delivery, remind: Callable[[], None]) -> Sink:
|
|
412
|
-
"""Build the one thing that carries reports out, and wire it to this process."""
|
|
399
|
+
def carrier(told: Delivery) -> "Deliverer":
|
|
400
|
+
"""Build the one thing that carries reports out, wired to nothing yet."""
|
|
413
401
|
# imported here, so a workload that never asked pays for none of it
|
|
414
402
|
from fluidattacks_agent.deliver import Deliverer # noqa: PLC0415
|
|
415
403
|
|
|
416
|
-
|
|
417
|
-
# before the flush handler, since exit handlers run in reverse order
|
|
418
|
-
atexit.register(held.parting)
|
|
419
|
-
held.registered()
|
|
420
|
-
return held
|
|
404
|
+
return Deliverer(told=told)
|
|
421
405
|
|
|
422
406
|
|
|
423
|
-
def _start(
|
|
407
|
+
def _start() -> Probe | None:
|
|
424
408
|
held = running()
|
|
425
409
|
if held is not None:
|
|
426
410
|
return held
|
|
427
411
|
if not switched_on():
|
|
428
412
|
return None
|
|
413
|
+
told = delivery()
|
|
414
|
+
if told is None:
|
|
415
|
+
# nowhere to deliver is nothing to observe for
|
|
416
|
+
return None
|
|
429
417
|
sites = site_dirs()
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
if hasattr(os, "register_at_fork"):
|
|
434
|
-
os.register_at_fork(after_in_child=probe.forked)
|
|
418
|
+
carrying = carrier(told)
|
|
419
|
+
probe = Probe(sink=carrying, loads=LoadObserver(sites=sites), runs=RunObserver(sites=sites))
|
|
420
|
+
carrying.fold = probe.fold
|
|
435
421
|
install(probe)
|
|
436
422
|
sys.addaudithook(watch(probe))
|
|
437
423
|
probe.monitoring = observe_runs(probe)
|
|
424
|
+
# last, because none of these can be taken back: everything above that a
|
|
425
|
+
# host could refuse has been refused or not by now. The carrier's parting
|
|
426
|
+
# goes before the flush, since exit handlers run in reverse order
|
|
427
|
+
atexit.register(carrying.parting)
|
|
428
|
+
carrying.registered()
|
|
429
|
+
if hasattr(os, "register_at_fork"):
|
|
430
|
+
os.register_at_fork(
|
|
431
|
+
before=probe.forking,
|
|
432
|
+
after_in_parent=probe.resumed,
|
|
433
|
+
after_in_child=probe.forked,
|
|
434
|
+
)
|
|
438
435
|
atexit.register(probe.flush)
|
|
436
|
+
# the clock starts with the probe, so a worker that sees little still
|
|
437
|
+
# reports what it saw when the period ends
|
|
438
|
+
carrying.rouse()
|
|
439
439
|
return probe
|
|
@@ -1,163 +0,0 @@
|
|
|
1
|
-
"""Leave a report where something can pick it up, or refuse and say which."""
|
|
2
|
-
|
|
3
|
-
import contextlib
|
|
4
|
-
import errno
|
|
5
|
-
import os
|
|
6
|
-
import stat
|
|
7
|
-
from dataclasses import dataclass, field
|
|
8
|
-
from typing import Final, Protocol
|
|
9
|
-
|
|
10
|
-
# the path is a contract with the agent that drains it
|
|
11
|
-
REPORT_DIR: Final = "/tmp/.watches-exec" # noqa: S108
|
|
12
|
-
REPORT_SUFFIX: Final = ".report"
|
|
13
|
-
STAGE_SUFFIX: Final = ".tmp"
|
|
14
|
-
|
|
15
|
-
# reports of its own, bytes any report holds, and entries it will walk to count
|
|
16
|
-
MAX_OUTSTANDING: Final = 64
|
|
17
|
-
MAX_OUTSTANDING_BYTES: Final = 8 << 20
|
|
18
|
-
MAX_ENTRIES: Final = 4096
|
|
19
|
-
|
|
20
|
-
SHARED: Final = 0o077
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
class Stalled(OSError):
|
|
24
|
-
"""Nothing was taken, and what was not taken is still owed."""
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
class Sink(Protocol):
|
|
28
|
-
"""Somewhere a report can go, which may refuse it and must never lie."""
|
|
29
|
-
|
|
30
|
-
def write(self, text: str) -> None: ...
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def ours(status: os.stat_result) -> bool:
|
|
34
|
-
"""Say whether what the status describes is ours alone to write."""
|
|
35
|
-
return status.st_uid == os.geteuid() and not status.st_mode & SHARED
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
@dataclass(frozen=True)
|
|
39
|
-
class Held:
|
|
40
|
-
"""What one directory entry costs, and whether the sink left it there."""
|
|
41
|
-
|
|
42
|
-
size: int
|
|
43
|
-
mine: bool
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def _held(entry: os.DirEntry[str], stem: str) -> Held:
|
|
47
|
-
if not entry.name.endswith(REPORT_SUFFIX):
|
|
48
|
-
return Held(size=0, mine=False)
|
|
49
|
-
try:
|
|
50
|
-
status = entry.stat(follow_symlinks=False)
|
|
51
|
-
except OSError:
|
|
52
|
-
return Held(size=0, mine=False)
|
|
53
|
-
if not stat.S_ISREG(status.st_mode):
|
|
54
|
-
return Held(size=0, mine=False)
|
|
55
|
-
return Held(size=status.st_size, mine=entry.name.startswith(stem))
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
class Holding:
|
|
59
|
-
"""A sink for a workload that asked for delivery, which nothing yet does."""
|
|
60
|
-
|
|
61
|
-
def write(self, text: str) -> None:
|
|
62
|
-
"""Take nothing, and be counted for it."""
|
|
63
|
-
raise Stalled(errno.ENOSYS, "nothing delivers yet", len(text))
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
@dataclass
|
|
67
|
-
class FileSink:
|
|
68
|
-
"""Reports left in a directory of our own for the agent to drain."""
|
|
69
|
-
|
|
70
|
-
directory: str | os.PathLike[str] = REPORT_DIR
|
|
71
|
-
written: int = 0
|
|
72
|
-
# so a report of ours cannot be named by anything that did not write it,
|
|
73
|
-
# and a reused pid never counts its predecessor's against itself
|
|
74
|
-
token: str = field(default_factory=lambda: os.urandom(8).hex())
|
|
75
|
-
|
|
76
|
-
def write(self, text: str) -> None:
|
|
77
|
-
"""Take a report, unless what is already here says to stop."""
|
|
78
|
-
directory = self._open_directory()
|
|
79
|
-
try:
|
|
80
|
-
# installed on its own nothing drains this, so the ceiling is ours
|
|
81
|
-
if self._piled(directory):
|
|
82
|
-
raise Stalled(errno.ENOSPC, "reports outstanding")
|
|
83
|
-
self._write_into(directory, text)
|
|
84
|
-
finally:
|
|
85
|
-
os.close(directory)
|
|
86
|
-
|
|
87
|
-
def _open_directory(self) -> int:
|
|
88
|
-
# every way this can fail is a flush that did not land, and one that
|
|
89
|
-
# said so as anything else was a mute nothing counted: a directory this
|
|
90
|
-
# cannot use costs an attacker one bit of its mode
|
|
91
|
-
try:
|
|
92
|
-
with contextlib.suppress(FileExistsError):
|
|
93
|
-
os.mkdir(self.directory, 0o700) # noqa: PTH102
|
|
94
|
-
handle = os.open(self.directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
95
|
-
except OSError as refused:
|
|
96
|
-
raise Stalled(errno.EACCES, "no directory", str(self.directory)) from refused
|
|
97
|
-
# the mode a directory was made with says nothing about one that was
|
|
98
|
-
# already there, and the descriptor is what the answer has to be about
|
|
99
|
-
if ours(os.fstat(handle)):
|
|
100
|
-
return handle
|
|
101
|
-
os.close(handle)
|
|
102
|
-
raise Stalled(errno.EACCES, "not ours", str(self.directory))
|
|
103
|
-
|
|
104
|
-
def _piled(self, directory: int) -> bool:
|
|
105
|
-
mine = 0
|
|
106
|
-
bytes_held = 0
|
|
107
|
-
walked = 0
|
|
108
|
-
with os.scandir(directory) as entries:
|
|
109
|
-
for entry in entries:
|
|
110
|
-
walked += 1
|
|
111
|
-
# too full to keep writing into, whoever filled it
|
|
112
|
-
if walked > MAX_ENTRIES:
|
|
113
|
-
return True
|
|
114
|
-
held = _held(entry, self._stem())
|
|
115
|
-
bytes_held += held.size
|
|
116
|
-
mine += int(held.mine)
|
|
117
|
-
if mine >= MAX_OUTSTANDING or bytes_held >= MAX_OUTSTANDING_BYTES:
|
|
118
|
-
return True
|
|
119
|
-
return False
|
|
120
|
-
|
|
121
|
-
def _stem(self) -> str:
|
|
122
|
-
return f"{os.getpid()}-{self.token}-"
|
|
123
|
-
|
|
124
|
-
def _write_into(self, directory: int, text: str) -> None:
|
|
125
|
-
payload = text.encode()
|
|
126
|
-
stem = f"{self._stem()}{self.written}"
|
|
127
|
-
staged = stem + STAGE_SUFFIX
|
|
128
|
-
# spent the moment it is tried: what sits under a name is not always
|
|
129
|
-
# ours to clear, and coming back to one forever is how one silences us
|
|
130
|
-
self.written += 1
|
|
131
|
-
try:
|
|
132
|
-
self._stage(directory, staged, payload)
|
|
133
|
-
os.replace(
|
|
134
|
-
staged,
|
|
135
|
-
stem + REPORT_SUFFIX,
|
|
136
|
-
src_dir_fd=directory,
|
|
137
|
-
dst_dir_fd=directory,
|
|
138
|
-
)
|
|
139
|
-
except OSError as refused:
|
|
140
|
-
# nothing will ever read what a failed attempt staged, and the name
|
|
141
|
-
# it holds counts against the entries this is willing to walk
|
|
142
|
-
with contextlib.suppress(OSError):
|
|
143
|
-
os.unlink(staged, dir_fd=directory)
|
|
144
|
-
raise Stalled(errno.ENOSPC, "not taken", staged) from refused
|
|
145
|
-
|
|
146
|
-
def _stage(self, directory: int, name: str, payload: bytes) -> None:
|
|
147
|
-
handle = self._create(directory, name)
|
|
148
|
-
try:
|
|
149
|
-
written = os.write(handle, payload)
|
|
150
|
-
finally:
|
|
151
|
-
os.close(handle)
|
|
152
|
-
# os.write does not raise on a partial write, and the statement is the
|
|
153
|
-
# last line, so a truncation takes it first
|
|
154
|
-
if written != len(payload):
|
|
155
|
-
raise OSError(errno.ENOSPC, "short write", name)
|
|
156
|
-
|
|
157
|
-
def _create(self, directory: int, name: str) -> int:
|
|
158
|
-
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
|
159
|
-
try:
|
|
160
|
-
return os.open(name, flags, 0o600, dir_fd=directory)
|
|
161
|
-
except FileExistsError:
|
|
162
|
-
os.unlink(name, dir_fd=directory)
|
|
163
|
-
return os.open(name, flags, 0o600, dir_fd=directory)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|