service-host-profiling 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- service_host_profiling/__init__.py +27 -0
- service_host_profiling/__main__.py +3 -0
- service_host_profiling/_python_collector.py +222 -0
- service_host_profiling/_switch.py +9 -0
- service_host_profiling/cli.py +72 -0
- service_host_profiling/collector.py +30 -0
- service_host_profiling/integrations/__init__.py +1 -0
- service_host_profiling/integrations/sglang.py +179 -0
- service_host_profiling/integrations/torch.py +111 -0
- service_host_profiling/parser.py +953 -0
- service_host_profiling-0.5.0.dist-info/METADATA +355 -0
- service_host_profiling-0.5.0.dist-info/RECORD +16 -0
- service_host_profiling-0.5.0.dist-info/WHEEL +5 -0
- service_host_profiling-0.5.0.dist-info/entry_points.txt +2 -0
- service_host_profiling-0.5.0.dist-info/licenses/LICENSE +124 -0
- service_host_profiling-0.5.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Low-overhead Python Host spans and independent offline trace export."""
|
|
2
|
+
|
|
3
|
+
__version__ = "0.5.0"
|
|
4
|
+
|
|
5
|
+
from ._switch import is_enabled
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"start", "stop", "span_start", "span_end", "profile", "is_enabled",
|
|
9
|
+
"attach_torch_profiler", "attach_sglang",
|
|
10
|
+
]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def __getattr__(name):
|
|
14
|
+
if name in {"start", "stop", "span_start", "span_end", "profile"}:
|
|
15
|
+
from . import collector
|
|
16
|
+
# Resolve lazy imports once. Repeated direct calls must not pay module
|
|
17
|
+
# __getattr__/import dispatch on every start and end.
|
|
18
|
+
value = getattr(collector, name)
|
|
19
|
+
globals()[name] = value
|
|
20
|
+
return value
|
|
21
|
+
if name == "attach_torch_profiler":
|
|
22
|
+
from .integrations.torch import attach_torch_profiler
|
|
23
|
+
return attach_torch_profiler
|
|
24
|
+
if name == "attach_sglang":
|
|
25
|
+
from .integrations.sglang import attach_sglang
|
|
26
|
+
return attach_sglang
|
|
27
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""Opt-in, per-thread Python recording; JSONL is written only at stop()."""
|
|
2
|
+
|
|
3
|
+
import functools
|
|
4
|
+
import inspect
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import socket
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
import uuid
|
|
12
|
+
|
|
13
|
+
_clock = time.perf_counter_ns if os.name == "nt" else time.monotonic_ns
|
|
14
|
+
_local = threading.local()
|
|
15
|
+
_lifecycle = threading.RLock()
|
|
16
|
+
_active = None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _calibration():
|
|
20
|
+
before = _clock()
|
|
21
|
+
epoch = time.time_ns()
|
|
22
|
+
after = _clock()
|
|
23
|
+
return (before + after) // 2, epoch, after - before
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class _Buffer:
|
|
27
|
+
__slots__ = ("session", "tid", "records", "active")
|
|
28
|
+
|
|
29
|
+
def __init__(self, session):
|
|
30
|
+
self.session = session
|
|
31
|
+
self.tid = threading.get_native_id()
|
|
32
|
+
self.records = []
|
|
33
|
+
self.active = True
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class _Session:
|
|
37
|
+
def __init__(self, output_dir, session_id):
|
|
38
|
+
self.pid = os.getpid()
|
|
39
|
+
root = Path(output_dir).resolve()
|
|
40
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
41
|
+
self.path = root / f"python_host_{self.pid}_{uuid.uuid4().hex}.jsonl"
|
|
42
|
+
self.session_id = session_id or self.path.stem
|
|
43
|
+
self.buffers = []
|
|
44
|
+
self.snapshots = []
|
|
45
|
+
self.registry = threading.Lock()
|
|
46
|
+
self.start = _calibration()
|
|
47
|
+
self.stop = None
|
|
48
|
+
|
|
49
|
+
def _accepted(self, record):
|
|
50
|
+
_, started, ended, _ = record
|
|
51
|
+
# Use the calibration bracket's upper bound, not its midpoint. A
|
|
52
|
+
# completion during stop calibration still belongs to the frozen data.
|
|
53
|
+
start_bound = self.start[0] - self.start[2] // 2
|
|
54
|
+
end_bound = self.stop[0] + (self.stop[2] + 1) // 2
|
|
55
|
+
return ended is not None and start_bound <= started <= ended <= end_bound
|
|
56
|
+
|
|
57
|
+
def _rows(self, tid, records):
|
|
58
|
+
for name, started, ended, attrs in records:
|
|
59
|
+
if not self._accepted((name, started, ended, attrs)):
|
|
60
|
+
continue
|
|
61
|
+
if not isinstance(name, str) or not name or "\0" in name:
|
|
62
|
+
raise ValueError("Span name must be a non-empty string without NUL")
|
|
63
|
+
if ended < started:
|
|
64
|
+
raise ValueError("Span end precedes start")
|
|
65
|
+
# Resolve deferred attributes only for accepted spans, after all
|
|
66
|
+
# thread buffers have been frozen. The recording path keeps the
|
|
67
|
+
# callable itself and never evaluates user code.
|
|
68
|
+
if callable(attrs):
|
|
69
|
+
attrs = attrs()
|
|
70
|
+
if attrs is not None and not isinstance(attrs, dict):
|
|
71
|
+
raise ValueError("Span attrs must be a JSON-serializable dict or None")
|
|
72
|
+
if attrs is not None:
|
|
73
|
+
attrs = {key: value() if callable(value) else value
|
|
74
|
+
for key, value in attrs.items()}
|
|
75
|
+
yield {"pid": self.pid, "tid": tid, "name": name,
|
|
76
|
+
"start_ns": started, "duration_ns": ended - started,
|
|
77
|
+
"args": attrs if attrs is not None else {}}
|
|
78
|
+
|
|
79
|
+
def dump(self):
|
|
80
|
+
written = sum(self._accepted(record) for _, records in self.snapshots for record in records)
|
|
81
|
+
dropped = sum(not self._accepted(record) for _, records in self.snapshots for record in records)
|
|
82
|
+
metadata = {"schema_version": 3, "session_id": self.session_id,
|
|
83
|
+
"pid": self.pid, "hostname": socket.gethostname(), "clock": "monotonic",
|
|
84
|
+
"start_monotonic_ns": self.start[0] - self.start[2] // 2,
|
|
85
|
+
"start_realtime_ns": self.start[1],
|
|
86
|
+
"clock_uncertainty_ns": self.start[2], "written_spans": written,
|
|
87
|
+
"dropped_spans": dropped,
|
|
88
|
+
"stop_monotonic_ns": self.stop[0] + (self.stop[2] + 1) // 2,
|
|
89
|
+
"stop_realtime_ns": self.stop[1], "complete": True}
|
|
90
|
+
pending = self.path.with_name(self.path.name + ".pending")
|
|
91
|
+
created = False
|
|
92
|
+
try:
|
|
93
|
+
with pending.open("x", encoding="utf-8", newline="\n") as output:
|
|
94
|
+
created = True
|
|
95
|
+
output.write(json.dumps({"type": "metadata", **metadata},
|
|
96
|
+
ensure_ascii=False, allow_nan=False,
|
|
97
|
+
separators=(",", ":")) + "\n")
|
|
98
|
+
span_id = 0
|
|
99
|
+
for tid, records in self.snapshots:
|
|
100
|
+
for row in self._rows(tid, records):
|
|
101
|
+
span_id += 1
|
|
102
|
+
output.write(json.dumps({"type": "span", "id": span_id, **row},
|
|
103
|
+
ensure_ascii=False, allow_nan=False,
|
|
104
|
+
separators=(",", ":")) + "\n")
|
|
105
|
+
output.flush()
|
|
106
|
+
os.fsync(output.fileno())
|
|
107
|
+
# Same-directory exclusive publication of already flushed bytes.
|
|
108
|
+
# Never overwrite an existing JSONL or remove somebody's pending.
|
|
109
|
+
os.link(pending, self.path)
|
|
110
|
+
finally:
|
|
111
|
+
if created:
|
|
112
|
+
pending.unlink(missing_ok=True)
|
|
113
|
+
for buffer in self.buffers:
|
|
114
|
+
buffer.records.clear()
|
|
115
|
+
buffer.session = None
|
|
116
|
+
self.snapshots.clear()
|
|
117
|
+
return self.path
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def start(output_dir, *, session_id=None):
|
|
121
|
+
"""Begin one process session. The returned path is complete only after stop()."""
|
|
122
|
+
global _active
|
|
123
|
+
with _lifecycle:
|
|
124
|
+
if _active is not None:
|
|
125
|
+
raise RuntimeError("Host collection is already active in this process")
|
|
126
|
+
if session_id is not None and (not isinstance(session_id, str) or not session_id):
|
|
127
|
+
raise ValueError("session_id must be a non-empty string")
|
|
128
|
+
_active = _Session(output_dir, session_id)
|
|
129
|
+
return _active.path
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _register_thread():
|
|
133
|
+
session = _active
|
|
134
|
+
if session is None:
|
|
135
|
+
return None
|
|
136
|
+
with session.registry:
|
|
137
|
+
if session is not _active:
|
|
138
|
+
return None
|
|
139
|
+
buffer = _Buffer(session)
|
|
140
|
+
session.buffers.append(buffer)
|
|
141
|
+
_local.buffer = buffer
|
|
142
|
+
return buffer
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def span_start(name, attrs=None):
|
|
146
|
+
"""Append [name, start_ns, end_ns, attrs]; return this thread's list span_id."""
|
|
147
|
+
buffer = getattr(_local, "buffer", None)
|
|
148
|
+
if buffer is None or not buffer.active:
|
|
149
|
+
buffer = _register_thread()
|
|
150
|
+
if buffer is None:
|
|
151
|
+
return -1
|
|
152
|
+
records = buffer.records
|
|
153
|
+
started = _clock()
|
|
154
|
+
# Stop can occur while the clock implementation runs. Recheck without a
|
|
155
|
+
# per-record lock; records begun outside the frozen window remain no-ops.
|
|
156
|
+
if not buffer.active:
|
|
157
|
+
return -1
|
|
158
|
+
span_id = len(records)
|
|
159
|
+
records.append([name, started, None, attrs])
|
|
160
|
+
return span_id
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def span_end(span_id):
|
|
164
|
+
"""Set only the end timestamp; an span_id is valid only in its original thread/session."""
|
|
165
|
+
if span_id == -1:
|
|
166
|
+
return
|
|
167
|
+
buffer = getattr(_local, "buffer", None)
|
|
168
|
+
if buffer is None or not buffer.active:
|
|
169
|
+
return
|
|
170
|
+
ended = _clock()
|
|
171
|
+
if buffer.active:
|
|
172
|
+
buffer.records[span_id][2] = ended
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def stop():
|
|
176
|
+
"""Freeze all thread buffers and synchronously publish one closed JSONL."""
|
|
177
|
+
global _active
|
|
178
|
+
with _lifecycle:
|
|
179
|
+
session = _active
|
|
180
|
+
if session is None:
|
|
181
|
+
raise RuntimeError("Host collection is not active")
|
|
182
|
+
with session.registry:
|
|
183
|
+
session.stop = _calibration()
|
|
184
|
+
_active = None
|
|
185
|
+
for buffer in session.buffers:
|
|
186
|
+
buffer.active = False
|
|
187
|
+
session.snapshots = [(buffer.tid, [tuple(record) for record in buffer.records])
|
|
188
|
+
for buffer in session.buffers]
|
|
189
|
+
return session.dump()
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def profile(name, attrs=None):
|
|
193
|
+
"""Decorate a synchronous function with the same start/end pair."""
|
|
194
|
+
def decorate(function):
|
|
195
|
+
if (inspect.iscoroutinefunction(function) or inspect.isgeneratorfunction(function)
|
|
196
|
+
or inspect.isasyncgenfunction(function)):
|
|
197
|
+
raise TypeError("profile supports synchronous functions only")
|
|
198
|
+
|
|
199
|
+
@functools.wraps(function)
|
|
200
|
+
def wrapped(*args, **kwargs):
|
|
201
|
+
span_id = span_start(name, attrs)
|
|
202
|
+
try:
|
|
203
|
+
return function(*args, **kwargs)
|
|
204
|
+
finally:
|
|
205
|
+
span_end(span_id)
|
|
206
|
+
|
|
207
|
+
return wrapped
|
|
208
|
+
|
|
209
|
+
return decorate
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _after_fork_child():
|
|
213
|
+
# Pure Python buffers must not be inherited as the child's active session,
|
|
214
|
+
# nor may inherited registry/lifecycle locks block its independent start.
|
|
215
|
+
global _active, _local, _lifecycle
|
|
216
|
+
_active = None
|
|
217
|
+
_local = threading.local()
|
|
218
|
+
_lifecycle = threading.RLock()
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
if hasattr(os, "register_at_fork"):
|
|
222
|
+
os.register_at_fork(after_in_child=_after_fork_child)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""One process-wide choice, made before collection functions are imported."""
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
ENABLED = os.environ.get("HOST_SERVICE_PROFILER_ENABLE") == "1"
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def is_enabled():
|
|
8
|
+
"""Return the import-time choice; changing the environment needs a restart."""
|
|
9
|
+
return ENABLED
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Offline export commands, usable without Torch or collection enabled."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def main(argv=None):
|
|
9
|
+
ap = argparse.ArgumentParser(prog="service-host-profiling")
|
|
10
|
+
commands = ap.add_subparsers(dest="command", required=True)
|
|
11
|
+
export = commands.add_parser("export", help="Export closed Host JSONL or historical SQLite files")
|
|
12
|
+
export.add_argument("db", nargs="+")
|
|
13
|
+
export.add_argument("-o", "--output", required=True)
|
|
14
|
+
directory = commands.add_parser("merge-dir", help="Update every rank trace under one Torch capture root")
|
|
15
|
+
directory.add_argument("host_dir", help="Host capture directory containing closed JSONL files")
|
|
16
|
+
directory.add_argument("--torch-dir", required=True, help="Whole Torch capture directory above all rank outputs")
|
|
17
|
+
directory.add_argument("--pid-map", help='JSON object mapping Host OS PID to Torch Host PID')
|
|
18
|
+
directory_clock = directory.add_mutually_exclusive_group()
|
|
19
|
+
directory_clock.add_argument("--clock-offset-ns", help='Integer ns for one trace or JSON {"host_pid":offset_ns} for multiple ranks; omit to derive it from the vendor metadata beside each trace')
|
|
20
|
+
directory_clock.add_argument("--clock-anchors-ns", help='JSON pair for one trace or JSON {"host_pid":[[host0,target0],[host1,target1]]}; overrides automatic derivation')
|
|
21
|
+
args = ap.parse_args(argv)
|
|
22
|
+
try:
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from .parser import export_chrome_trace, merge_torch_directory
|
|
25
|
+
if args.command == "export":
|
|
26
|
+
result = export_chrome_trace(args.db, args.output)
|
|
27
|
+
else:
|
|
28
|
+
mapping = json.loads(args.pid_map) if args.pid_map else None
|
|
29
|
+
if mapping is not None and not isinstance(mapping, dict):
|
|
30
|
+
raise ValueError("--pid-map must be a JSON object")
|
|
31
|
+
if mapping is not None:
|
|
32
|
+
converted = {}
|
|
33
|
+
for key, value in mapping.items():
|
|
34
|
+
if not key.isascii() or not key.isdecimal():
|
|
35
|
+
raise ValueError("--pid-map keys must be decimal OS PIDs")
|
|
36
|
+
pid = int(key)
|
|
37
|
+
if pid in converted:
|
|
38
|
+
raise ValueError("--pid-map contains duplicate PID spellings")
|
|
39
|
+
converted[pid] = value
|
|
40
|
+
mapping = converted
|
|
41
|
+
host_root = Path(args.host_dir).expanduser().resolve(strict=True)
|
|
42
|
+
if not host_root.is_dir():
|
|
43
|
+
raise ValueError("HOST_DIR must be a Host capture directory")
|
|
44
|
+
hosts = sorted(host_root.rglob("*.jsonl"))
|
|
45
|
+
if not hosts:
|
|
46
|
+
raise ValueError(f"No closed Host JSONL files found under {host_root}")
|
|
47
|
+
def clock_value(raw, label):
|
|
48
|
+
if raw is None:
|
|
49
|
+
return None
|
|
50
|
+
value = json.loads(raw)
|
|
51
|
+
if isinstance(value, dict):
|
|
52
|
+
converted = {}
|
|
53
|
+
for key, item in value.items():
|
|
54
|
+
if not key.isascii() or not key.isdecimal():
|
|
55
|
+
raise ValueError(f"{label} keys must be decimal Host OS PIDs")
|
|
56
|
+
converted[int(key)] = item
|
|
57
|
+
return converted
|
|
58
|
+
return value
|
|
59
|
+
offsets = clock_value(args.clock_offset_ns, "--clock-offset-ns")
|
|
60
|
+
anchors = clock_value(args.clock_anchors_ns, "--clock-anchors-ns")
|
|
61
|
+
result = merge_torch_directory(
|
|
62
|
+
hosts, args.torch_dir, pid_map=mapping,
|
|
63
|
+
clock_offset_ns=offsets, clock_anchors_ns=anchors)
|
|
64
|
+
if isinstance(result, list):
|
|
65
|
+
for path in result:
|
|
66
|
+
print(path)
|
|
67
|
+
else:
|
|
68
|
+
print(result)
|
|
69
|
+
return 0
|
|
70
|
+
except (ValueError, OSError, RuntimeError) as exc:
|
|
71
|
+
print(f"service-host-profiling: {exc}", file=sys.stderr)
|
|
72
|
+
return 1
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Bind enabled or disabled Python functions once at import time."""
|
|
2
|
+
from ._switch import ENABLED, is_enabled
|
|
3
|
+
|
|
4
|
+
if ENABLED:
|
|
5
|
+
from . import _python_collector as _implementation
|
|
6
|
+
from ._python_collector import start, stop, span_start, span_end, profile
|
|
7
|
+
|
|
8
|
+
def __getattr__(name):
|
|
9
|
+
# Integration lifecycle checks need current state, not a copied value.
|
|
10
|
+
return getattr(_implementation, name)
|
|
11
|
+
else:
|
|
12
|
+
_active = None
|
|
13
|
+
def span_start(name, attrs=None):
|
|
14
|
+
return -1
|
|
15
|
+
|
|
16
|
+
def span_end(span_id):
|
|
17
|
+
return None
|
|
18
|
+
|
|
19
|
+
def start(output_dir, *, session_id=None):
|
|
20
|
+
return None
|
|
21
|
+
|
|
22
|
+
def stop():
|
|
23
|
+
return None
|
|
24
|
+
|
|
25
|
+
def _identity(function):
|
|
26
|
+
return function
|
|
27
|
+
|
|
28
|
+
def profile(name, attrs=None):
|
|
29
|
+
# Returning the original callable eliminates a wrapper on every call.
|
|
30
|
+
return _identity
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Optional explicit integrations; no framework import at package import time."""
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Small explicit boundary for SGLang profile start/stop/export lifecycle.
|
|
2
|
+
|
|
3
|
+
Install calls in each scheduler/worker process for each actual capture. Nothing
|
|
4
|
+
imports or monkeypatches SGLang. Its own profiler still produces the Torch trace.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from contextlib import closing
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
import sqlite3
|
|
14
|
+
import tempfile
|
|
15
|
+
import time
|
|
16
|
+
|
|
17
|
+
from .torch import TorchBinding, attach_torch_profiler
|
|
18
|
+
from .._switch import ENABLED
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _clock_sample():
|
|
22
|
+
"""Bracket a Linux RAW sample in the database's MONOTONIC clock.
|
|
23
|
+
|
|
24
|
+
uncertainty_ns is the full bracket width, deliberately conservative.
|
|
25
|
+
Unsupported platforms record RAW=None; callers must calibrate explicitly.
|
|
26
|
+
"""
|
|
27
|
+
# Use the exact clock selected by the Python collector on this platform.
|
|
28
|
+
sample_ns = time.perf_counter_ns if os.name == "nt" else time.monotonic_ns
|
|
29
|
+
before = sample_ns()
|
|
30
|
+
raw_clock = getattr(time, "CLOCK_MONOTONIC_RAW", None)
|
|
31
|
+
raw = time.clock_gettime_ns(raw_clock) if raw_clock is not None else None
|
|
32
|
+
after = sample_ns()
|
|
33
|
+
return {"monotonic_ns": (before + after) // 2,
|
|
34
|
+
"monotonic_raw_ns": raw, "uncertainty_ns": after - before,
|
|
35
|
+
"sample_before_ns": before, "sample_after_ns": after}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _pid_namespace():
|
|
39
|
+
namespace = Path("/proc/self/ns/pid")
|
|
40
|
+
try:
|
|
41
|
+
return os.readlink(namespace)
|
|
42
|
+
except OSError:
|
|
43
|
+
# This describes the available identity rather than inventing a host PID.
|
|
44
|
+
return f"unavailable:{os.name}:process-local-pid"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _closed_metadata(path):
|
|
48
|
+
if path.suffix == ".jsonl":
|
|
49
|
+
with path.open("r", encoding="utf-8") as stream:
|
|
50
|
+
return json.loads(stream.readline())
|
|
51
|
+
with closing(sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)) as db:
|
|
52
|
+
return {key: json.loads(value) for key, value in
|
|
53
|
+
db.execute("SELECT key,value FROM metadata")}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _publish_manifest(path, document):
|
|
57
|
+
"""Publish one complete sidecar atomically without replacing any file."""
|
|
58
|
+
descriptor, temporary = tempfile.mkstemp(prefix=".pending-", suffix=".json", dir=path.parent)
|
|
59
|
+
pending = Path(temporary)
|
|
60
|
+
try:
|
|
61
|
+
with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as stream:
|
|
62
|
+
json.dump(document, stream, ensure_ascii=False, indent=2, allow_nan=False)
|
|
63
|
+
stream.write("\n")
|
|
64
|
+
stream.flush()
|
|
65
|
+
os.fsync(stream.fileno())
|
|
66
|
+
# A same-directory hardlink publishes fully written bytes exclusively;
|
|
67
|
+
# readers never observe an empty reservation or partially written JSON.
|
|
68
|
+
os.link(pending, path)
|
|
69
|
+
finally:
|
|
70
|
+
pending.unlink(missing_ok=True)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass
|
|
74
|
+
class SGLangCapture:
|
|
75
|
+
binding: TorchBinding
|
|
76
|
+
profile_id: str
|
|
77
|
+
rank: int | None = None
|
|
78
|
+
physical_device: int | None = None
|
|
79
|
+
source_version: object = None
|
|
80
|
+
clock_anchors: dict = field(default_factory=dict, init=False)
|
|
81
|
+
manifest_path: Path | None = field(default=None, init=False)
|
|
82
|
+
_stopped: bool = field(default=False, init=False)
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def db_path(self):
|
|
86
|
+
return self.binding.db_path
|
|
87
|
+
|
|
88
|
+
def start(self):
|
|
89
|
+
if not ENABLED:
|
|
90
|
+
return None
|
|
91
|
+
self.binding._check_process()
|
|
92
|
+
if self.binding.db_path is not None:
|
|
93
|
+
raise RuntimeError("Create a new capture for each profiling window")
|
|
94
|
+
self.clock_anchors["start_before"] = _clock_sample()
|
|
95
|
+
result = self.binding.start()
|
|
96
|
+
self.clock_anchors["start_after"] = _clock_sample()
|
|
97
|
+
return result
|
|
98
|
+
|
|
99
|
+
def stop(self):
|
|
100
|
+
"""Stop/drain this rank's whole process collector, then return its DB.
|
|
101
|
+
|
|
102
|
+
Collection is process scoped. This refuses to stop another binding's
|
|
103
|
+
active collector. Use this method for the four-anchor archive workflow;
|
|
104
|
+
Use this owning-capture stop before archive; external stop cannot supply
|
|
105
|
+
all four clock samples needed for the manifest workflow.
|
|
106
|
+
"""
|
|
107
|
+
if not ENABLED:
|
|
108
|
+
return None
|
|
109
|
+
self.binding._check_process()
|
|
110
|
+
if self._stopped:
|
|
111
|
+
return self.binding.db_path
|
|
112
|
+
from .. import collector
|
|
113
|
+
with collector._lifecycle:
|
|
114
|
+
active = collector._active
|
|
115
|
+
if active is None or self.binding.db_path is None or (
|
|
116
|
+
active.path.resolve() != self.binding.db_path.resolve() or
|
|
117
|
+
active.pid != self.binding.pid):
|
|
118
|
+
raise RuntimeError("This capture does not own the active process collector")
|
|
119
|
+
self.clock_anchors["stop_before"] = _clock_sample()
|
|
120
|
+
result = collector.stop()
|
|
121
|
+
self.clock_anchors["stop_after"] = _clock_sample()
|
|
122
|
+
if result.resolve() != self.binding.db_path.resolve():
|
|
123
|
+
raise RuntimeError("Stopped database does not match the owning capture")
|
|
124
|
+
self._stopped = True
|
|
125
|
+
return result
|
|
126
|
+
|
|
127
|
+
def archive(self, capture_root):
|
|
128
|
+
"""Archive the closed DB and publish its rank manifest without a trace.
|
|
129
|
+
|
|
130
|
+
capture_root is explicit and must be under the attached output root.
|
|
131
|
+
Call stop() first. The returned path is the closed database;
|
|
132
|
+
manifest_path identifies its exclusive, atomically published sidecar.
|
|
133
|
+
The manifest filesystem must support same-directory hardlinks.
|
|
134
|
+
"""
|
|
135
|
+
if not ENABLED:
|
|
136
|
+
return None
|
|
137
|
+
self.binding._check_process()
|
|
138
|
+
if not self._stopped:
|
|
139
|
+
raise RuntimeError("Call capture.stop() successfully before archive()")
|
|
140
|
+
root = Path(capture_root).resolve()
|
|
141
|
+
if not root.is_relative_to(self.binding.output_root):
|
|
142
|
+
raise ValueError("capture_root must be under the attached output root")
|
|
143
|
+
if root.is_relative_to(self.binding.output_root / ".host_trace"):
|
|
144
|
+
raise ValueError("capture_root cannot be the Host staging directory")
|
|
145
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
146
|
+
path = self.binding.archive(root)
|
|
147
|
+
if self.manifest_path is not None:
|
|
148
|
+
return path
|
|
149
|
+
metadata = _closed_metadata(path)
|
|
150
|
+
if metadata.get("complete") is not True:
|
|
151
|
+
raise RuntimeError("Only complete closed databases may be published")
|
|
152
|
+
label = "unranked" if self.rank is None else f"rank-{self.rank}"
|
|
153
|
+
manifest = path.parent / f"{label}-{self.binding.session_id}.closed.json"
|
|
154
|
+
document = {"schema_version": 1, "type": "host_rank_archive", "complete": True,
|
|
155
|
+
"profile_id": self.profile_id, "rank": self.rank,
|
|
156
|
+
"session_id": self.binding.session_id,
|
|
157
|
+
"db_relative_path": path.relative_to(root).as_posix(),
|
|
158
|
+
"db_sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
|
159
|
+
"hostname": metadata["hostname"], "pid": metadata["pid"],
|
|
160
|
+
"pid_namespace": _pid_namespace(), "physical_device": self.physical_device,
|
|
161
|
+
"source_version": self.source_version, "clock_anchors": self.clock_anchors}
|
|
162
|
+
_publish_manifest(manifest, document)
|
|
163
|
+
self.manifest_path = manifest
|
|
164
|
+
return path
|
|
165
|
+
|
|
166
|
+
def attach_sglang(output_dir, *, profile_id, rank=None, physical_device=None,
|
|
167
|
+
source_version=None):
|
|
168
|
+
"""Bind one SGLang capture in its owning process; collection starts explicitly."""
|
|
169
|
+
if not isinstance(profile_id, str) or not profile_id:
|
|
170
|
+
raise ValueError("profile_id must be a non-empty SGLang capture identifier")
|
|
171
|
+
if rank is not None and (isinstance(rank, bool) or not isinstance(rank, int) or rank < 0):
|
|
172
|
+
raise ValueError("rank must be a non-negative integer or None")
|
|
173
|
+
if physical_device is not None and (isinstance(physical_device, bool) or
|
|
174
|
+
not isinstance(physical_device, int) or physical_device < 0):
|
|
175
|
+
raise ValueError("physical_device must be a non-negative integer or None")
|
|
176
|
+
# Snapshot external provenance; retain opaque JSON without a config system.
|
|
177
|
+
provenance = json.loads(json.dumps(source_version, allow_nan=False))
|
|
178
|
+
binding = attach_torch_profiler(output_dir)
|
|
179
|
+
return SGLangCapture(binding, profile_id, rank, physical_device, provenance)
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""Explicit Host raw-data binding alongside an existing Torch profiler.
|
|
2
|
+
|
|
3
|
+
This module neither imports Torch nor starts, parses, or patches its profiler.
|
|
4
|
+
The caller supplies each real capture directory and an already exported trace.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
import os
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
import shutil
|
|
11
|
+
import tempfile
|
|
12
|
+
import uuid
|
|
13
|
+
|
|
14
|
+
from .._switch import ENABLED
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class TorchBinding:
|
|
19
|
+
output_root: Path
|
|
20
|
+
session_id: str
|
|
21
|
+
pid: int = field(default_factory=os.getpid)
|
|
22
|
+
db_path: Path | None = field(default=None, init=False)
|
|
23
|
+
session_dir: Path | None = field(default=None, init=False)
|
|
24
|
+
|
|
25
|
+
@property
|
|
26
|
+
def output_dir(self):
|
|
27
|
+
return self.output_root / ".host_trace" / self.session_id
|
|
28
|
+
|
|
29
|
+
def _check_process(self):
|
|
30
|
+
if os.getpid() != self.pid:
|
|
31
|
+
raise RuntimeError("TorchBinding cannot be reused after fork; attach in each worker")
|
|
32
|
+
|
|
33
|
+
def start(self):
|
|
34
|
+
"""Start Host collection before the concrete Torch directory exists."""
|
|
35
|
+
if not ENABLED:
|
|
36
|
+
return None
|
|
37
|
+
self._check_process()
|
|
38
|
+
if self.db_path is not None:
|
|
39
|
+
raise RuntimeError("This binding already owns a collection; create a new binding per capture")
|
|
40
|
+
from ..collector import start
|
|
41
|
+
self.db_path = Path(start(self.output_dir, session_id=self.session_id)).resolve()
|
|
42
|
+
return self.db_path
|
|
43
|
+
|
|
44
|
+
def archive(self, host_capture_dir):
|
|
45
|
+
"""Archive a stopped Host JSONL within the caller's Host-data root."""
|
|
46
|
+
if not ENABLED:
|
|
47
|
+
return None
|
|
48
|
+
self._check_process()
|
|
49
|
+
target_dir = Path(host_capture_dir).resolve(strict=True)
|
|
50
|
+
if not target_dir.is_dir() or not target_dir.is_relative_to(self.output_root):
|
|
51
|
+
raise ValueError("Host capture directory must be under the Host output_root")
|
|
52
|
+
if target_dir.is_relative_to(self.output_root / ".host_trace"):
|
|
53
|
+
raise ValueError("Supply the Host capture directory, not the staging directory")
|
|
54
|
+
source = self.db_path
|
|
55
|
+
if source is None:
|
|
56
|
+
raise RuntimeError("No Host data is bound; call binding.start() first")
|
|
57
|
+
source = source.resolve(strict=True)
|
|
58
|
+
if self.session_dir is not None:
|
|
59
|
+
if self.session_dir != target_dir or source != self.db_path:
|
|
60
|
+
raise ValueError("This session is already finalized to another location")
|
|
61
|
+
return self.db_path
|
|
62
|
+
if not source.is_relative_to(self.output_dir.resolve()):
|
|
63
|
+
raise ValueError("Host data must belong to this binding's staging directory")
|
|
64
|
+
# Validate every row/count, not only a header claiming complete=true.
|
|
65
|
+
from ..parser import _databases
|
|
66
|
+
database = _databases(source)[0]
|
|
67
|
+
for _ in database.spans():
|
|
68
|
+
pass
|
|
69
|
+
metadata = database.metadata
|
|
70
|
+
if metadata.get("session_id") != self.session_id or metadata.get("pid") != self.pid:
|
|
71
|
+
raise ValueError("Host data process/session does not match this binding")
|
|
72
|
+
if metadata.get("complete") is not True:
|
|
73
|
+
raise RuntimeError("Stop Host collection successfully before archiving")
|
|
74
|
+
if source.suffix == ".db":
|
|
75
|
+
wal = Path(str(source) + "-wal")
|
|
76
|
+
if wal.exists() and wal.stat().st_size:
|
|
77
|
+
raise RuntimeError("Host DB has an outstanding WAL; close/checkpoint the collector before archiving")
|
|
78
|
+
host_dir = target_dir / "HOST_TRACE"
|
|
79
|
+
host_dir.mkdir(exist_ok=True)
|
|
80
|
+
destination = host_dir / source.name
|
|
81
|
+
descriptor, temporary = tempfile.mkstemp(prefix=".archive-", suffix=".pending", dir=host_dir)
|
|
82
|
+
pending = Path(temporary)
|
|
83
|
+
try:
|
|
84
|
+
with os.fdopen(descriptor, "wb") as output:
|
|
85
|
+
with source.open("rb") as incoming:
|
|
86
|
+
shutil.copyfileobj(incoming, output)
|
|
87
|
+
output.flush()
|
|
88
|
+
os.fsync(output.fileno())
|
|
89
|
+
os.link(pending, destination)
|
|
90
|
+
except FileExistsError:
|
|
91
|
+
raise FileExistsError(f"Refusing to replace existing Host data: {destination}") from None
|
|
92
|
+
finally:
|
|
93
|
+
pending.unlink(missing_ok=True)
|
|
94
|
+
try:
|
|
95
|
+
source.unlink()
|
|
96
|
+
except BaseException:
|
|
97
|
+
destination.unlink(missing_ok=True)
|
|
98
|
+
raise
|
|
99
|
+
self.db_path = destination
|
|
100
|
+
self.session_dir = target_dir
|
|
101
|
+
return destination
|
|
102
|
+
|
|
103
|
+
def attach_torch_profiler(output_root, *, session_id=None):
|
|
104
|
+
"""Prepare an explicit per-worker, per-capture binding without starting Torch."""
|
|
105
|
+
root = Path(output_root).resolve()
|
|
106
|
+
token = session_id if session_id is not None else uuid.uuid4().hex
|
|
107
|
+
if not isinstance(token, str) or not token or len(token) > 128:
|
|
108
|
+
raise ValueError("session_id must be a non-empty string of at most 128 characters")
|
|
109
|
+
if any(not (c.isascii() and (c.isalnum() or c in "_-")) for c in token):
|
|
110
|
+
raise ValueError("session_id may contain only ASCII letters, digits, underscore and hyphen")
|
|
111
|
+
return TorchBinding(root, token)
|