pulseml 0.4.2__tar.gz → 0.4.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pulseml-0.4.2/src/pulseml.egg-info → pulseml-0.4.3}/PKG-INFO +50 -111
- {pulseml-0.4.2 → pulseml-0.4.3}/README.md +48 -109
- {pulseml-0.4.2 → pulseml-0.4.3}/pyproject.toml +3 -3
- {pulseml-0.4.2 → pulseml-0.4.3/src/pulseml.egg-info}/PKG-INFO +50 -111
- {pulseml-0.4.2 → pulseml-0.4.3}/LICENSE +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/setup.cfg +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/__init__.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/__main__.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/cli.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_about.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_app.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_approver.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_attach.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_backend.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_brain.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_cli.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_code.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_code_agent.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_code_rules.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_console.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_detect.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_experiments.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_headless.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_monitor.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_openrouter.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_pdf.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_project.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_ratelimit.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_settings.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_shapes.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_stream.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_supabase.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_terminal.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_trace.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_tui.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulse/pulse_ui.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulseml.egg-info/SOURCES.txt +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulseml.egg-info/dependency_links.txt +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulseml.egg-info/entry_points.txt +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulseml.egg-info/requires.txt +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/src/pulseml.egg-info/top_level.txt +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_app.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_approver.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_attach.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_audit_round.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_autonomy.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_brain.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_cached_login.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_checkin.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_cli_colors.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_code_rules.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_console.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_core_fixes_1011.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_detect.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_detect_semantics.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_detection_quality.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_detector_wiring.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_experiments.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_file_labels.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_fix_resume.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_full_context.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_headless.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_integration.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_message_tool.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_openrouter.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_projects.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_pulse_code_rules.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_ratelimit.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_remote_commands.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_settings.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_shapes.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_smoke_text.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_step_tracking.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_stream_monitor.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_syntax_repair.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_tui_modern.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_ui_choose_title.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_unattended_fixes.py +0 -0
- {pulseml-0.4.2 → pulseml-0.4.3}/tests/test_wiring_paths.py +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pulseml
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.3
|
|
4
4
|
Summary: A live ML training debugger for tracking, diagnosing, verifying, and patching training runs.
|
|
5
5
|
Author-email: Yash Patel <codeyash09@gmail.com>
|
|
6
6
|
License: Proprietary
|
|
7
|
-
Project-URL: Homepage, https://
|
|
7
|
+
Project-URL: Homepage, https://residualml.com/
|
|
8
8
|
Project-URL: Repository, https://github.com/codeyash09/PulseML
|
|
9
9
|
Project-URL: Issues, https://github.com/codeyash09/PulseML/issues
|
|
10
|
-
Project-URL: Documentation, https://
|
|
10
|
+
Project-URL: Documentation, https://residualml.com/docs
|
|
11
11
|
Keywords: machine-learning,deep-learning,debugging,ml-debugging,training,pytorch,tensorflow,numpy,cupy,jax,gradients,training-debugger
|
|
12
12
|
Classifier: Development Status :: 3 - Alpha
|
|
13
13
|
Classifier: Intended Audience :: Developers
|
|
@@ -69,35 +69,8 @@ dependencies separate from other projects.
|
|
|
69
69
|
- An AI provider is optional for monitoring. Pulse's AI agent setup can sign you in with
|
|
70
70
|
OpenRouter or use a provider API key.
|
|
71
71
|
|
|
72
|
-
### 1. Create and activate an environment
|
|
73
72
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
**Windows — PowerShell:**
|
|
77
|
-
|
|
78
|
-
```powershell
|
|
79
|
-
py -3 -m venv .venv
|
|
80
|
-
.\.venv\Scripts\Activate.ps1
|
|
81
|
-
```
|
|
82
|
-
|
|
83
|
-
If PowerShell activation is blocked by your execution policy, use Command Prompt instead:
|
|
84
|
-
|
|
85
|
-
```bat
|
|
86
|
-
py -3 -m venv .venv
|
|
87
|
-
.venv\Scripts\activate.bat
|
|
88
|
-
```
|
|
89
|
-
|
|
90
|
-
Or skip activation and use `.venv\Scripts\python.exe` in place of `python` in the
|
|
91
|
-
remaining commands.
|
|
92
|
-
|
|
93
|
-
**macOS / Linux:**
|
|
94
|
-
|
|
95
|
-
```bash
|
|
96
|
-
python3 -m venv .venv
|
|
97
|
-
source .venv/bin/activate
|
|
98
|
-
```
|
|
99
|
-
|
|
100
|
-
### 2. Install Pulse
|
|
73
|
+
### 1. Install Pulse
|
|
101
74
|
|
|
102
75
|
With the environment activated:
|
|
103
76
|
|
|
@@ -110,33 +83,23 @@ Using `python -m pip` ensures Pulse is installed for the Python interpreter sele
|
|
|
110
83
|
the active environment. If you already have a project environment, activate it and run
|
|
111
84
|
the second command there instead of creating a new one.
|
|
112
85
|
|
|
113
|
-
###
|
|
86
|
+
### 2. Verify and launch
|
|
114
87
|
|
|
115
88
|
```text
|
|
116
|
-
pulse --version
|
|
117
|
-
pulse --help
|
|
89
|
+
python -m pulse --version
|
|
90
|
+
python -m pulse --help
|
|
118
91
|
```
|
|
119
92
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
Start Pulse:
|
|
125
|
-
|
|
126
|
-
```text
|
|
127
|
-
pulse
|
|
128
|
-
```
|
|
93
|
+
`python -m pulse` works on Windows, macOS, and Linux, including when the environment's
|
|
94
|
+
Scripts directory is not on `PATH`. When the environment is activated, the shorter
|
|
95
|
+
`pulse` command is available too.
|
|
129
96
|
|
|
130
|
-
|
|
97
|
+
To open Pulse, run:
|
|
131
98
|
|
|
132
99
|
```text
|
|
133
|
-
|
|
100
|
+
python -m pulse
|
|
134
101
|
```
|
|
135
102
|
|
|
136
|
-
Replace `train.py` with your script path and include any script arguments in your request.
|
|
137
|
-
Pulse starts it with the active environment's Python; no source-code edits are needed.
|
|
138
|
-
This is the normal workflow for both single-GPU and distributed runs.
|
|
139
|
-
|
|
140
103
|
### AI provider configuration
|
|
141
104
|
|
|
142
105
|
Pulse supports cloud and local AI providers.
|
|
@@ -146,13 +109,9 @@ Pulse supports cloud and local AI providers.
|
|
|
146
109
|
OpenRouter serves every major model behind one account and has free models. Pulse can get
|
|
147
110
|
a key for you:
|
|
148
111
|
|
|
149
|
-
|
|
150
|
-
pulse
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
or just press Enter at any API key question. Pulse asks for a key during app setup and
|
|
154
|
-
when you switch agents, and every one of those offers sign-in: for an OpenRouter model
|
|
155
|
-
Enter starts it directly, and for any other provider Enter
|
|
112
|
+
or just press Enter at any API key question. Pulse asks for a key in `pulse run`,
|
|
113
|
+
`pulse code` and when you switch agents, and every one of those offers the
|
|
114
|
+
sign-in: for an OpenRouter model Enter starts it directly, and for any other provider Enter
|
|
156
115
|
offers to sign you in and switch to an OpenRouter model. The dashboard asks the same thing
|
|
157
116
|
in a dialog, and `pulse --model openrouter/<model>` offers it when there is no key.
|
|
158
117
|
Your browser opens on OpenRouter; sign in or sign up, click Authorize, and Pulse receives
|
|
@@ -260,20 +219,7 @@ pulse_monitor.Monitor ---stream---> pulse_brain.Brain
|
|
|
260
219
|
never blocks, never thinks thinks as long as it likes
|
|
261
220
|
```
|
|
262
221
|
|
|
263
|
-
Start
|
|
264
|
-
|
|
265
|
-
```text
|
|
266
|
-
pulse
|
|
267
|
-
```
|
|
268
|
-
|
|
269
|
-
The app launches runs with background streaming, so training does not wait for the
|
|
270
|
-
agent. To open a stream brain explicitly:
|
|
271
|
-
|
|
272
|
-
```text
|
|
273
|
-
python -m pulse.brain # newest session
|
|
274
|
-
python -m pulse.brain <session-dir> --model openrouter/deepseek/deepseek-v4.1-flash
|
|
275
|
-
python -m pulse.brain <session-dir> --once # one pass over a finished run
|
|
276
|
-
```
|
|
222
|
+
Start the script through Pulse; no source changes are needed:
|
|
277
223
|
|
|
278
224
|
### What the training process pays
|
|
279
225
|
|
|
@@ -594,14 +540,14 @@ rewrite git state, reach outside the project, start a background process or over
|
|
|
594
540
|
file. So that a long run never stops to wait for you, a model answers that y/N. By default
|
|
595
541
|
it is the agent's own model, asked separately: it sees only the command, why it was
|
|
596
542
|
flagged, the project folder and what the agent was working on -- never the agent's
|
|
597
|
-
reasoning -- so it judges the command on its own. To use a different model
|
|
598
|
-
`"approver": "<model>"` in `pulse_config.json`):
|
|
543
|
+
reasoning -- so it judges the command on its own. To use a different model:
|
|
599
544
|
|
|
600
545
|
```bash
|
|
601
|
-
|
|
546
|
+
pulse run --approver openrouter/anthropic/claude-sonnet-5 train.py
|
|
602
547
|
```
|
|
603
548
|
|
|
604
|
-
(
|
|
549
|
+
(or `PULSE_APPROVER=<model>`, or `"approver": "<model>"` in `pulse_config.json`). To be
|
|
550
|
+
asked yourself instead, use `--approver off` (or `"approver": "off"`). The same commands
|
|
605
551
|
are flagged either way; only who answers changes. The reviewer answers the other question
|
|
606
552
|
the agent otherwise stops for, too -- "apply this change?" for each edit it proposes --
|
|
607
553
|
judging the diff against your request; every applied change stays one `/undo` away. The approver answers APPROVE or DENY
|
|
@@ -674,40 +620,45 @@ Commands on an open run: `/findings`, `/curve <name>`, `/vars`, `/audit`, `/audi
|
|
|
674
620
|
watching). Everywhere: `/monitor`, `/run`, `/agent`, `/files`, `/add`, `/drop`, `/review`,
|
|
675
621
|
`/undo`, `/log`, `/cloud`, `/help`, `/exit`.
|
|
676
622
|
|
|
677
|
-
A run opened in the app -- started
|
|
678
|
-
|
|
679
|
-
snapshots, the detectors' findings and crashes as incidents,
|
|
623
|
+
A run opened in the app -- started with `/run`, picked with `/monitor`, or attached from
|
|
624
|
+
outside -- gets its own session on the web dashboard, as a run started with `pulse run`
|
|
625
|
+
does: its environment, metric snapshots, the detectors' findings and crashes as incidents,
|
|
680
626
|
the agent's turns about it, and its uptime, synced in the background while it is watched.
|
|
681
627
|
|
|
682
628
|
The agent can start, watch, restart and stop runs itself: "run it", "start training with
|
|
683
629
|
--epochs 3", "restart it with the fix" are requests it carries out with its `start_run`,
|
|
684
630
|
`run_status`, `restart_run` and `stop_run` tools (stopping asks you first). A request typed
|
|
685
|
-
at a paused
|
|
686
|
-
|
|
631
|
+
at a paused `pulse run` is likewise handled as what it is -- a question gets an answer, a
|
|
632
|
+
change you asked for gets made -- instead of being read as a bug report to diagnose.
|
|
687
633
|
|
|
688
634
|
`pulse watch <run>`, `pulse <script.py>` and `pulse attach --pid N` open the same app
|
|
689
635
|
directly on that run. On Windows it needs a console that understands escape sequences
|
|
690
636
|
(Windows Terminal, or the console of Windows 10 and later). Without a real terminal (a
|
|
691
637
|
pipe, CI), on a very small one, or with `PULSE_CLASSIC=1`, Pulse keeps the line-by-line
|
|
692
638
|
screens described below. `PULSE_STREAMING=0` turns off the streaming of the model's
|
|
693
|
-
replies.
|
|
639
|
+
replies. `pulse run train.py` -- training with Pulse inside the script's own terminal --
|
|
640
|
+
is unchanged.
|
|
694
641
|
|
|
695
642
|
---
|
|
696
643
|
|
|
697
644
|
## The `pulse` command
|
|
698
645
|
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
your script or choose an existing run to monitor.
|
|
646
|
+
Pulse does not have to be written into a script. Installing it puts a `pulse` command on
|
|
647
|
+
the path, which starts runs and attaches to ones already going.
|
|
702
648
|
|
|
703
649
|
```bash
|
|
704
|
-
pulse
|
|
650
|
+
pulse run --stream train.py # start train.py under Pulse, unmodified
|
|
651
|
+
pulse # the app; without a terminal, attach to the run on this machine
|
|
652
|
+
pulse train.py # attach to the run of that script
|
|
653
|
+
pulse sessions # list the runs Pulse knows about
|
|
705
654
|
```
|
|
706
655
|
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
656
|
+
`pulse run` starts a run; without it, a script name means the run that is **already
|
|
657
|
+
going**. If several runs match, Pulse lists them and asks which.
|
|
658
|
+
|
|
659
|
+
`--stream` puts the monitor in the training process and the agent in yours, so the run
|
|
660
|
+
keeps going whether or not anyone is watching it, and attaching costs the run nothing.
|
|
661
|
+
Without it the run is tracked in the terminal it was started from, as before.
|
|
711
662
|
|
|
712
663
|
### Watching a run you did not start under Pulse
|
|
713
664
|
|
|
@@ -723,7 +674,7 @@ sudo pulse attach --pid 12345 # when you already know the id
|
|
|
723
674
|
Two limits are worth knowing before relying on it. It reads **function locals**, so a
|
|
724
675
|
loop written at the top level of a script reports nothing — the values are module
|
|
725
676
|
globals, which cannot be read this way. And it **samples**, about once a second, so a
|
|
726
|
-
spike between two samples is not seen. A run
|
|
677
|
+
spike between two samples is not seen. A run started under `pulse run` has neither
|
|
727
678
|
problem, because Pulse is inside it.
|
|
728
679
|
|
|
729
680
|
If `sudo pulse` says `command not found`, sudo is the reason: it replaces `PATH` with its
|
|
@@ -771,33 +722,21 @@ Pulse can pause training for investigation, allowing you to inspect the current
|
|
|
771
722
|
|
|
772
723
|
## Quickstart
|
|
773
724
|
|
|
774
|
-
Start Pulse:
|
|
725
|
+
Start an existing training script under Pulse without editing it:
|
|
775
726
|
|
|
776
727
|
```text
|
|
777
|
-
pulse
|
|
728
|
+
python -m pulse run train.py
|
|
778
729
|
```
|
|
779
730
|
|
|
780
|
-
|
|
781
|
-
it with the active Python environment and instruments it in memory; your source files
|
|
782
|
-
are not modified. Use the same workflow for a single-GPU run or a distributed run.
|
|
783
|
-
|
|
784
|
-
### Distributed PyTorch (DDP and FSDP)
|
|
731
|
+
To keep training independent of the viewer and watch the run from another terminal, use:
|
|
785
732
|
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
each spawned worker in memory, so the training script still needs no Pulse-specific code
|
|
790
|
-
changes. The app launches runs in background-streaming mode; each worker is monitored in
|
|
791
|
-
its own stream session rather than as one consolidated rank-0 view. The underlying model
|
|
792
|
-
inspection recognizes DDP replicas and rank-local FSDP/DTensor shards.
|
|
733
|
+
```text
|
|
734
|
+
python -m pulse run --stream train.py
|
|
735
|
+
```
|
|
793
736
|
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
`
|
|
797
|
-
all-gather FSDP parameters or gradients just to inspect them; these statistics are
|
|
798
|
-
rank-local and are not presented as full-model norms. `SHAPETRACE`, `GRADCHECK`, and
|
|
799
|
-
`REPLAY` are guarded where a rank-0-only operation could otherwise hang or desynchronize
|
|
800
|
-
the distributed job.
|
|
737
|
+
Pulse launches the script with its existing source code and exposes the run through its
|
|
738
|
+
app and CLI. It instruments the script in memory; your files are not modified. Replace
|
|
739
|
+
`train.py` with your script path and put any script arguments after it.
|
|
801
740
|
|
|
802
741
|
You can then control tracking while the run is active:
|
|
803
742
|
|
|
@@ -878,7 +817,7 @@ LESS OVERHEAD
|
|
|
878
817
|
|
|
879
818
|
Rather than collecting everything continuously, Pulse lets you decide what information is worth monitoring.
|
|
880
819
|
|
|
881
|
-
Measured on a 20k-step loop started
|
|
820
|
+
Measured on a 20k-step loop started with `pulse run --stream`:
|
|
882
821
|
|
|
883
822
|
| Work on the training thread | single process | stream mode |
|
|
884
823
|
|---|---:|---:|
|
|
@@ -24,35 +24,8 @@ dependencies separate from other projects.
|
|
|
24
24
|
- An AI provider is optional for monitoring. Pulse's AI agent setup can sign you in with
|
|
25
25
|
OpenRouter or use a provider API key.
|
|
26
26
|
|
|
27
|
-
### 1. Create and activate an environment
|
|
28
27
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
**Windows — PowerShell:**
|
|
32
|
-
|
|
33
|
-
```powershell
|
|
34
|
-
py -3 -m venv .venv
|
|
35
|
-
.\.venv\Scripts\Activate.ps1
|
|
36
|
-
```
|
|
37
|
-
|
|
38
|
-
If PowerShell activation is blocked by your execution policy, use Command Prompt instead:
|
|
39
|
-
|
|
40
|
-
```bat
|
|
41
|
-
py -3 -m venv .venv
|
|
42
|
-
.venv\Scripts\activate.bat
|
|
43
|
-
```
|
|
44
|
-
|
|
45
|
-
Or skip activation and use `.venv\Scripts\python.exe` in place of `python` in the
|
|
46
|
-
remaining commands.
|
|
47
|
-
|
|
48
|
-
**macOS / Linux:**
|
|
49
|
-
|
|
50
|
-
```bash
|
|
51
|
-
python3 -m venv .venv
|
|
52
|
-
source .venv/bin/activate
|
|
53
|
-
```
|
|
54
|
-
|
|
55
|
-
### 2. Install Pulse
|
|
28
|
+
### 1. Install Pulse
|
|
56
29
|
|
|
57
30
|
With the environment activated:
|
|
58
31
|
|
|
@@ -65,33 +38,23 @@ Using `python -m pip` ensures Pulse is installed for the Python interpreter sele
|
|
|
65
38
|
the active environment. If you already have a project environment, activate it and run
|
|
66
39
|
the second command there instead of creating a new one.
|
|
67
40
|
|
|
68
|
-
###
|
|
41
|
+
### 2. Verify and launch
|
|
69
42
|
|
|
70
43
|
```text
|
|
71
|
-
pulse --version
|
|
72
|
-
pulse --help
|
|
44
|
+
python -m pulse --version
|
|
45
|
+
python -m pulse --help
|
|
73
46
|
```
|
|
74
47
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
Start Pulse:
|
|
80
|
-
|
|
81
|
-
```text
|
|
82
|
-
pulse
|
|
83
|
-
```
|
|
48
|
+
`python -m pulse` works on Windows, macOS, and Linux, including when the environment's
|
|
49
|
+
Scripts directory is not on `PATH`. When the environment is activated, the shorter
|
|
50
|
+
`pulse` command is available too.
|
|
84
51
|
|
|
85
|
-
|
|
52
|
+
To open Pulse, run:
|
|
86
53
|
|
|
87
54
|
```text
|
|
88
|
-
|
|
55
|
+
python -m pulse
|
|
89
56
|
```
|
|
90
57
|
|
|
91
|
-
Replace `train.py` with your script path and include any script arguments in your request.
|
|
92
|
-
Pulse starts it with the active environment's Python; no source-code edits are needed.
|
|
93
|
-
This is the normal workflow for both single-GPU and distributed runs.
|
|
94
|
-
|
|
95
58
|
### AI provider configuration
|
|
96
59
|
|
|
97
60
|
Pulse supports cloud and local AI providers.
|
|
@@ -101,13 +64,9 @@ Pulse supports cloud and local AI providers.
|
|
|
101
64
|
OpenRouter serves every major model behind one account and has free models. Pulse can get
|
|
102
65
|
a key for you:
|
|
103
66
|
|
|
104
|
-
|
|
105
|
-
pulse
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
or just press Enter at any API key question. Pulse asks for a key during app setup and
|
|
109
|
-
when you switch agents, and every one of those offers sign-in: for an OpenRouter model
|
|
110
|
-
Enter starts it directly, and for any other provider Enter
|
|
67
|
+
or just press Enter at any API key question. Pulse asks for a key in `pulse run`,
|
|
68
|
+
`pulse code` and when you switch agents, and every one of those offers the
|
|
69
|
+
sign-in: for an OpenRouter model Enter starts it directly, and for any other provider Enter
|
|
111
70
|
offers to sign you in and switch to an OpenRouter model. The dashboard asks the same thing
|
|
112
71
|
in a dialog, and `pulse --model openrouter/<model>` offers it when there is no key.
|
|
113
72
|
Your browser opens on OpenRouter; sign in or sign up, click Authorize, and Pulse receives
|
|
@@ -215,20 +174,7 @@ pulse_monitor.Monitor ---stream---> pulse_brain.Brain
|
|
|
215
174
|
never blocks, never thinks thinks as long as it likes
|
|
216
175
|
```
|
|
217
176
|
|
|
218
|
-
Start
|
|
219
|
-
|
|
220
|
-
```text
|
|
221
|
-
pulse
|
|
222
|
-
```
|
|
223
|
-
|
|
224
|
-
The app launches runs with background streaming, so training does not wait for the
|
|
225
|
-
agent. To open a stream brain explicitly:
|
|
226
|
-
|
|
227
|
-
```text
|
|
228
|
-
python -m pulse.brain # newest session
|
|
229
|
-
python -m pulse.brain <session-dir> --model openrouter/deepseek/deepseek-v4.1-flash
|
|
230
|
-
python -m pulse.brain <session-dir> --once # one pass over a finished run
|
|
231
|
-
```
|
|
177
|
+
Start the script through Pulse; no source changes are needed:
|
|
232
178
|
|
|
233
179
|
### What the training process pays
|
|
234
180
|
|
|
@@ -549,14 +495,14 @@ rewrite git state, reach outside the project, start a background process or over
|
|
|
549
495
|
file. So that a long run never stops to wait for you, a model answers that y/N. By default
|
|
550
496
|
it is the agent's own model, asked separately: it sees only the command, why it was
|
|
551
497
|
flagged, the project folder and what the agent was working on -- never the agent's
|
|
552
|
-
reasoning -- so it judges the command on its own. To use a different model
|
|
553
|
-
`"approver": "<model>"` in `pulse_config.json`):
|
|
498
|
+
reasoning -- so it judges the command on its own. To use a different model:
|
|
554
499
|
|
|
555
500
|
```bash
|
|
556
|
-
|
|
501
|
+
pulse run --approver openrouter/anthropic/claude-sonnet-5 train.py
|
|
557
502
|
```
|
|
558
503
|
|
|
559
|
-
(
|
|
504
|
+
(or `PULSE_APPROVER=<model>`, or `"approver": "<model>"` in `pulse_config.json`). To be
|
|
505
|
+
asked yourself instead, use `--approver off` (or `"approver": "off"`). The same commands
|
|
560
506
|
are flagged either way; only who answers changes. The reviewer answers the other question
|
|
561
507
|
the agent otherwise stops for, too -- "apply this change?" for each edit it proposes --
|
|
562
508
|
judging the diff against your request; every applied change stays one `/undo` away. The approver answers APPROVE or DENY
|
|
@@ -629,40 +575,45 @@ Commands on an open run: `/findings`, `/curve <name>`, `/vars`, `/audit`, `/audi
|
|
|
629
575
|
watching). Everywhere: `/monitor`, `/run`, `/agent`, `/files`, `/add`, `/drop`, `/review`,
|
|
630
576
|
`/undo`, `/log`, `/cloud`, `/help`, `/exit`.
|
|
631
577
|
|
|
632
|
-
A run opened in the app -- started
|
|
633
|
-
|
|
634
|
-
snapshots, the detectors' findings and crashes as incidents,
|
|
578
|
+
A run opened in the app -- started with `/run`, picked with `/monitor`, or attached from
|
|
579
|
+
outside -- gets its own session on the web dashboard, as a run started with `pulse run`
|
|
580
|
+
does: its environment, metric snapshots, the detectors' findings and crashes as incidents,
|
|
635
581
|
the agent's turns about it, and its uptime, synced in the background while it is watched.
|
|
636
582
|
|
|
637
583
|
The agent can start, watch, restart and stop runs itself: "run it", "start training with
|
|
638
584
|
--epochs 3", "restart it with the fix" are requests it carries out with its `start_run`,
|
|
639
585
|
`run_status`, `restart_run` and `stop_run` tools (stopping asks you first). A request typed
|
|
640
|
-
at a paused
|
|
641
|
-
|
|
586
|
+
at a paused `pulse run` is likewise handled as what it is -- a question gets an answer, a
|
|
587
|
+
change you asked for gets made -- instead of being read as a bug report to diagnose.
|
|
642
588
|
|
|
643
589
|
`pulse watch <run>`, `pulse <script.py>` and `pulse attach --pid N` open the same app
|
|
644
590
|
directly on that run. On Windows it needs a console that understands escape sequences
|
|
645
591
|
(Windows Terminal, or the console of Windows 10 and later). Without a real terminal (a
|
|
646
592
|
pipe, CI), on a very small one, or with `PULSE_CLASSIC=1`, Pulse keeps the line-by-line
|
|
647
593
|
screens described below. `PULSE_STREAMING=0` turns off the streaming of the model's
|
|
648
|
-
replies.
|
|
594
|
+
replies. `pulse run train.py` -- training with Pulse inside the script's own terminal --
|
|
595
|
+
is unchanged.
|
|
649
596
|
|
|
650
597
|
---
|
|
651
598
|
|
|
652
599
|
## The `pulse` command
|
|
653
600
|
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
your script or choose an existing run to monitor.
|
|
601
|
+
Pulse does not have to be written into a script. Installing it puts a `pulse` command on
|
|
602
|
+
the path, which starts runs and attaches to ones already going.
|
|
657
603
|
|
|
658
604
|
```bash
|
|
659
|
-
pulse
|
|
605
|
+
pulse run --stream train.py # start train.py under Pulse, unmodified
|
|
606
|
+
pulse # the app; without a terminal, attach to the run on this machine
|
|
607
|
+
pulse train.py # attach to the run of that script
|
|
608
|
+
pulse sessions # list the runs Pulse knows about
|
|
660
609
|
```
|
|
661
610
|
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
611
|
+
`pulse run` starts a run; without it, a script name means the run that is **already
|
|
612
|
+
going**. If several runs match, Pulse lists them and asks which.
|
|
613
|
+
|
|
614
|
+
`--stream` puts the monitor in the training process and the agent in yours, so the run
|
|
615
|
+
keeps going whether or not anyone is watching it, and attaching costs the run nothing.
|
|
616
|
+
Without it the run is tracked in the terminal it was started from, as before.
|
|
666
617
|
|
|
667
618
|
### Watching a run you did not start under Pulse
|
|
668
619
|
|
|
@@ -678,7 +629,7 @@ sudo pulse attach --pid 12345 # when you already know the id
|
|
|
678
629
|
Two limits are worth knowing before relying on it. It reads **function locals**, so a
|
|
679
630
|
loop written at the top level of a script reports nothing — the values are module
|
|
680
631
|
globals, which cannot be read this way. And it **samples**, about once a second, so a
|
|
681
|
-
spike between two samples is not seen. A run
|
|
632
|
+
spike between two samples is not seen. A run started under `pulse run` has neither
|
|
682
633
|
problem, because Pulse is inside it.
|
|
683
634
|
|
|
684
635
|
If `sudo pulse` says `command not found`, sudo is the reason: it replaces `PATH` with its
|
|
@@ -726,33 +677,21 @@ Pulse can pause training for investigation, allowing you to inspect the current
|
|
|
726
677
|
|
|
727
678
|
## Quickstart
|
|
728
679
|
|
|
729
|
-
Start Pulse:
|
|
680
|
+
Start an existing training script under Pulse without editing it:
|
|
730
681
|
|
|
731
682
|
```text
|
|
732
|
-
pulse
|
|
683
|
+
python -m pulse run train.py
|
|
733
684
|
```
|
|
734
685
|
|
|
735
|
-
|
|
736
|
-
it with the active Python environment and instruments it in memory; your source files
|
|
737
|
-
are not modified. Use the same workflow for a single-GPU run or a distributed run.
|
|
738
|
-
|
|
739
|
-
### Distributed PyTorch (DDP and FSDP)
|
|
686
|
+
To keep training independent of the viewer and watch the run from another terminal, use:
|
|
740
687
|
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
each spawned worker in memory, so the training script still needs no Pulse-specific code
|
|
745
|
-
changes. The app launches runs in background-streaming mode; each worker is monitored in
|
|
746
|
-
its own stream session rather than as one consolidated rank-0 view. The underlying model
|
|
747
|
-
inspection recognizes DDP replicas and rank-local FSDP/DTensor shards.
|
|
688
|
+
```text
|
|
689
|
+
python -m pulse run --stream train.py
|
|
690
|
+
```
|
|
748
691
|
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
`
|
|
752
|
-
all-gather FSDP parameters or gradients just to inspect them; these statistics are
|
|
753
|
-
rank-local and are not presented as full-model norms. `SHAPETRACE`, `GRADCHECK`, and
|
|
754
|
-
`REPLAY` are guarded where a rank-0-only operation could otherwise hang or desynchronize
|
|
755
|
-
the distributed job.
|
|
692
|
+
Pulse launches the script with its existing source code and exposes the run through its
|
|
693
|
+
app and CLI. It instruments the script in memory; your files are not modified. Replace
|
|
694
|
+
`train.py` with your script path and put any script arguments after it.
|
|
756
695
|
|
|
757
696
|
You can then control tracking while the run is active:
|
|
758
697
|
|
|
@@ -833,7 +772,7 @@ LESS OVERHEAD
|
|
|
833
772
|
|
|
834
773
|
Rather than collecting everything continuously, Pulse lets you decide what information is worth monitoring.
|
|
835
774
|
|
|
836
|
-
Measured on a 20k-step loop started
|
|
775
|
+
Measured on a 20k-step loop started with `pulse run --stream`:
|
|
837
776
|
|
|
838
777
|
| Work on the training thread | single process | stream mode |
|
|
839
778
|
|---|---:|---:|
|
|
@@ -988,4 +927,4 @@ Can that fix be verified?
|
|
|
988
927
|
|
|
989
928
|
Proprietary. See `LICENSE`.
|
|
990
929
|
|
|
991
|
-
Use of this software is governed by the terms in that file. Copying, redistribution, and reverse engineering are not permitted.
|
|
930
|
+
Use of this software is governed by the terms in that file. Copying, redistribution, and reverse engineering are not permitted.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pulseml"
|
|
7
|
-
version = "0.4.
|
|
7
|
+
version = "0.4.3"
|
|
8
8
|
description = "A live ML training debugger for tracking, diagnosing, verifying, and patching training runs."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -79,10 +79,10 @@ jax = [
|
|
|
79
79
|
]
|
|
80
80
|
|
|
81
81
|
[project.urls]
|
|
82
|
-
Homepage = "https://
|
|
82
|
+
Homepage = "https://residualml.com/"
|
|
83
83
|
Repository = "https://github.com/codeyash09/PulseML"
|
|
84
84
|
Issues = "https://github.com/codeyash09/PulseML/issues"
|
|
85
|
-
Documentation = "https://
|
|
85
|
+
Documentation = "https://residualml.com/docs"
|
|
86
86
|
|
|
87
87
|
[tool.setuptools.packages.find]
|
|
88
88
|
where = ["src"]
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pulseml
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.3
|
|
4
4
|
Summary: A live ML training debugger for tracking, diagnosing, verifying, and patching training runs.
|
|
5
5
|
Author-email: Yash Patel <codeyash09@gmail.com>
|
|
6
6
|
License: Proprietary
|
|
7
|
-
Project-URL: Homepage, https://
|
|
7
|
+
Project-URL: Homepage, https://residualml.com/
|
|
8
8
|
Project-URL: Repository, https://github.com/codeyash09/PulseML
|
|
9
9
|
Project-URL: Issues, https://github.com/codeyash09/PulseML/issues
|
|
10
|
-
Project-URL: Documentation, https://
|
|
10
|
+
Project-URL: Documentation, https://residualml.com/docs
|
|
11
11
|
Keywords: machine-learning,deep-learning,debugging,ml-debugging,training,pytorch,tensorflow,numpy,cupy,jax,gradients,training-debugger
|
|
12
12
|
Classifier: Development Status :: 3 - Alpha
|
|
13
13
|
Classifier: Intended Audience :: Developers
|
|
@@ -69,35 +69,8 @@ dependencies separate from other projects.
|
|
|
69
69
|
- An AI provider is optional for monitoring. Pulse's AI agent setup can sign you in with
|
|
70
70
|
OpenRouter or use a provider API key.
|
|
71
71
|
|
|
72
|
-
### 1. Create and activate an environment
|
|
73
72
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
**Windows — PowerShell:**
|
|
77
|
-
|
|
78
|
-
```powershell
|
|
79
|
-
py -3 -m venv .venv
|
|
80
|
-
.\.venv\Scripts\Activate.ps1
|
|
81
|
-
```
|
|
82
|
-
|
|
83
|
-
If PowerShell activation is blocked by your execution policy, use Command Prompt instead:
|
|
84
|
-
|
|
85
|
-
```bat
|
|
86
|
-
py -3 -m venv .venv
|
|
87
|
-
.venv\Scripts\activate.bat
|
|
88
|
-
```
|
|
89
|
-
|
|
90
|
-
Or skip activation and use `.venv\Scripts\python.exe` in place of `python` in the
|
|
91
|
-
remaining commands.
|
|
92
|
-
|
|
93
|
-
**macOS / Linux:**
|
|
94
|
-
|
|
95
|
-
```bash
|
|
96
|
-
python3 -m venv .venv
|
|
97
|
-
source .venv/bin/activate
|
|
98
|
-
```
|
|
99
|
-
|
|
100
|
-
### 2. Install Pulse
|
|
73
|
+
### 1. Install Pulse
|
|
101
74
|
|
|
102
75
|
With the environment activated:
|
|
103
76
|
|
|
@@ -110,33 +83,23 @@ Using `python -m pip` ensures Pulse is installed for the Python interpreter sele
|
|
|
110
83
|
the active environment. If you already have a project environment, activate it and run
|
|
111
84
|
the second command there instead of creating a new one.
|
|
112
85
|
|
|
113
|
-
###
|
|
86
|
+
### 2. Verify and launch
|
|
114
87
|
|
|
115
88
|
```text
|
|
116
|
-
pulse --version
|
|
117
|
-
pulse --help
|
|
89
|
+
python -m pulse --version
|
|
90
|
+
python -m pulse --help
|
|
118
91
|
```
|
|
119
92
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
Start Pulse:
|
|
125
|
-
|
|
126
|
-
```text
|
|
127
|
-
pulse
|
|
128
|
-
```
|
|
93
|
+
`python -m pulse` works on Windows, macOS, and Linux, including when the environment's
|
|
94
|
+
Scripts directory is not on `PATH`. When the environment is activated, the shorter
|
|
95
|
+
`pulse` command is available too.
|
|
129
96
|
|
|
130
|
-
|
|
97
|
+
To open Pulse, run:
|
|
131
98
|
|
|
132
99
|
```text
|
|
133
|
-
|
|
100
|
+
python -m pulse
|
|
134
101
|
```
|
|
135
102
|
|
|
136
|
-
Replace `train.py` with your script path and include any script arguments in your request.
|
|
137
|
-
Pulse starts it with the active environment's Python; no source-code edits are needed.
|
|
138
|
-
This is the normal workflow for both single-GPU and distributed runs.
|
|
139
|
-
|
|
140
103
|
### AI provider configuration
|
|
141
104
|
|
|
142
105
|
Pulse supports cloud and local AI providers.
|
|
@@ -146,13 +109,9 @@ Pulse supports cloud and local AI providers.
|
|
|
146
109
|
OpenRouter serves every major model behind one account and has free models. Pulse can get
|
|
147
110
|
a key for you:
|
|
148
111
|
|
|
149
|
-
|
|
150
|
-
pulse
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
or just press Enter at any API key question. Pulse asks for a key during app setup and
|
|
154
|
-
when you switch agents, and every one of those offers sign-in: for an OpenRouter model
|
|
155
|
-
Enter starts it directly, and for any other provider Enter
|
|
112
|
+
or just press Enter at any API key question. Pulse asks for a key in `pulse run`,
|
|
113
|
+
`pulse code` and when you switch agents, and every one of those offers the
|
|
114
|
+
sign-in: for an OpenRouter model Enter starts it directly, and for any other provider Enter
|
|
156
115
|
offers to sign you in and switch to an OpenRouter model. The dashboard asks the same thing
|
|
157
116
|
in a dialog, and `pulse --model openrouter/<model>` offers it when there is no key.
|
|
158
117
|
Your browser opens on OpenRouter; sign in or sign up, click Authorize, and Pulse receives
|
|
@@ -260,20 +219,7 @@ pulse_monitor.Monitor ---stream---> pulse_brain.Brain
|
|
|
260
219
|
never blocks, never thinks thinks as long as it likes
|
|
261
220
|
```
|
|
262
221
|
|
|
263
|
-
Start
|
|
264
|
-
|
|
265
|
-
```text
|
|
266
|
-
pulse
|
|
267
|
-
```
|
|
268
|
-
|
|
269
|
-
The app launches runs with background streaming, so training does not wait for the
|
|
270
|
-
agent. To open a stream brain explicitly:
|
|
271
|
-
|
|
272
|
-
```text
|
|
273
|
-
python -m pulse.brain # newest session
|
|
274
|
-
python -m pulse.brain <session-dir> --model openrouter/deepseek/deepseek-v4.1-flash
|
|
275
|
-
python -m pulse.brain <session-dir> --once # one pass over a finished run
|
|
276
|
-
```
|
|
222
|
+
Start the script through Pulse; no source changes are needed:
|
|
277
223
|
|
|
278
224
|
### What the training process pays
|
|
279
225
|
|
|
@@ -594,14 +540,14 @@ rewrite git state, reach outside the project, start a background process or over
|
|
|
594
540
|
file. So that a long run never stops to wait for you, a model answers that y/N. By default
|
|
595
541
|
it is the agent's own model, asked separately: it sees only the command, why it was
|
|
596
542
|
flagged, the project folder and what the agent was working on -- never the agent's
|
|
597
|
-
reasoning -- so it judges the command on its own. To use a different model
|
|
598
|
-
`"approver": "<model>"` in `pulse_config.json`):
|
|
543
|
+
reasoning -- so it judges the command on its own. To use a different model:
|
|
599
544
|
|
|
600
545
|
```bash
|
|
601
|
-
|
|
546
|
+
pulse run --approver openrouter/anthropic/claude-sonnet-5 train.py
|
|
602
547
|
```
|
|
603
548
|
|
|
604
|
-
(
|
|
549
|
+
(or `PULSE_APPROVER=<model>`, or `"approver": "<model>"` in `pulse_config.json`). To be
|
|
550
|
+
asked yourself instead, use `--approver off` (or `"approver": "off"`). The same commands
|
|
605
551
|
are flagged either way; only who answers changes. The reviewer answers the other question
|
|
606
552
|
the agent otherwise stops for, too -- "apply this change?" for each edit it proposes --
|
|
607
553
|
judging the diff against your request; every applied change stays one `/undo` away. The approver answers APPROVE or DENY
|
|
@@ -674,40 +620,45 @@ Commands on an open run: `/findings`, `/curve <name>`, `/vars`, `/audit`, `/audi
|
|
|
674
620
|
watching). Everywhere: `/monitor`, `/run`, `/agent`, `/files`, `/add`, `/drop`, `/review`,
|
|
675
621
|
`/undo`, `/log`, `/cloud`, `/help`, `/exit`.
|
|
676
622
|
|
|
677
|
-
A run opened in the app -- started
|
|
678
|
-
|
|
679
|
-
snapshots, the detectors' findings and crashes as incidents,
|
|
623
|
+
A run opened in the app -- started with `/run`, picked with `/monitor`, or attached from
|
|
624
|
+
outside -- gets its own session on the web dashboard, as a run started with `pulse run`
|
|
625
|
+
does: its environment, metric snapshots, the detectors' findings and crashes as incidents,
|
|
680
626
|
the agent's turns about it, and its uptime, synced in the background while it is watched.
|
|
681
627
|
|
|
682
628
|
The agent can start, watch, restart and stop runs itself: "run it", "start training with
|
|
683
629
|
--epochs 3", "restart it with the fix" are requests it carries out with its `start_run`,
|
|
684
630
|
`run_status`, `restart_run` and `stop_run` tools (stopping asks you first). A request typed
|
|
685
|
-
at a paused
|
|
686
|
-
|
|
631
|
+
at a paused `pulse run` is likewise handled as what it is -- a question gets an answer, a
|
|
632
|
+
change you asked for gets made -- instead of being read as a bug report to diagnose.
|
|
687
633
|
|
|
688
634
|
`pulse watch <run>`, `pulse <script.py>` and `pulse attach --pid N` open the same app
|
|
689
635
|
directly on that run. On Windows it needs a console that understands escape sequences
|
|
690
636
|
(Windows Terminal, or the console of Windows 10 and later). Without a real terminal (a
|
|
691
637
|
pipe, CI), on a very small one, or with `PULSE_CLASSIC=1`, Pulse keeps the line-by-line
|
|
692
638
|
screens described below. `PULSE_STREAMING=0` turns off the streaming of the model's
|
|
693
|
-
replies.
|
|
639
|
+
replies. `pulse run train.py` -- training with Pulse inside the script's own terminal --
|
|
640
|
+
is unchanged.
|
|
694
641
|
|
|
695
642
|
---
|
|
696
643
|
|
|
697
644
|
## The `pulse` command
|
|
698
645
|
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
your script or choose an existing run to monitor.
|
|
646
|
+
Pulse does not have to be written into a script. Installing it puts a `pulse` command on
|
|
647
|
+
the path, which starts runs and attaches to ones already going.
|
|
702
648
|
|
|
703
649
|
```bash
|
|
704
|
-
pulse
|
|
650
|
+
pulse run --stream train.py # start train.py under Pulse, unmodified
|
|
651
|
+
pulse # the app; without a terminal, attach to the run on this machine
|
|
652
|
+
pulse train.py # attach to the run of that script
|
|
653
|
+
pulse sessions # list the runs Pulse knows about
|
|
705
654
|
```
|
|
706
655
|
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
656
|
+
`pulse run` starts a run; without it, a script name means the run that is **already
|
|
657
|
+
going**. If several runs match, Pulse lists them and asks which.
|
|
658
|
+
|
|
659
|
+
`--stream` puts the monitor in the training process and the agent in yours, so the run
|
|
660
|
+
keeps going whether or not anyone is watching it, and attaching costs the run nothing.
|
|
661
|
+
Without it the run is tracked in the terminal it was started from, as before.
|
|
711
662
|
|
|
712
663
|
### Watching a run you did not start under Pulse
|
|
713
664
|
|
|
@@ -723,7 +674,7 @@ sudo pulse attach --pid 12345 # when you already know the id
|
|
|
723
674
|
Two limits are worth knowing before relying on it. It reads **function locals**, so a
|
|
724
675
|
loop written at the top level of a script reports nothing — the values are module
|
|
725
676
|
globals, which cannot be read this way. And it **samples**, about once a second, so a
|
|
726
|
-
spike between two samples is not seen. A run
|
|
677
|
+
spike between two samples is not seen. A run started under `pulse run` has neither
|
|
727
678
|
problem, because Pulse is inside it.
|
|
728
679
|
|
|
729
680
|
If `sudo pulse` says `command not found`, sudo is the reason: it replaces `PATH` with its
|
|
@@ -771,33 +722,21 @@ Pulse can pause training for investigation, allowing you to inspect the current
|
|
|
771
722
|
|
|
772
723
|
## Quickstart
|
|
773
724
|
|
|
774
|
-
Start Pulse:
|
|
725
|
+
Start an existing training script under Pulse without editing it:
|
|
775
726
|
|
|
776
727
|
```text
|
|
777
|
-
pulse
|
|
728
|
+
python -m pulse run train.py
|
|
778
729
|
```
|
|
779
730
|
|
|
780
|
-
|
|
781
|
-
it with the active Python environment and instruments it in memory; your source files
|
|
782
|
-
are not modified. Use the same workflow for a single-GPU run or a distributed run.
|
|
783
|
-
|
|
784
|
-
### Distributed PyTorch (DDP and FSDP)
|
|
731
|
+
To keep training independent of the viewer and watch the run from another terminal, use:
|
|
785
732
|
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
each spawned worker in memory, so the training script still needs no Pulse-specific code
|
|
790
|
-
changes. The app launches runs in background-streaming mode; each worker is monitored in
|
|
791
|
-
its own stream session rather than as one consolidated rank-0 view. The underlying model
|
|
792
|
-
inspection recognizes DDP replicas and rank-local FSDP/DTensor shards.
|
|
733
|
+
```text
|
|
734
|
+
python -m pulse run --stream train.py
|
|
735
|
+
```
|
|
793
736
|
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
`
|
|
797
|
-
all-gather FSDP parameters or gradients just to inspect them; these statistics are
|
|
798
|
-
rank-local and are not presented as full-model norms. `SHAPETRACE`, `GRADCHECK`, and
|
|
799
|
-
`REPLAY` are guarded where a rank-0-only operation could otherwise hang or desynchronize
|
|
800
|
-
the distributed job.
|
|
737
|
+
Pulse launches the script with its existing source code and exposes the run through its
|
|
738
|
+
app and CLI. It instruments the script in memory; your files are not modified. Replace
|
|
739
|
+
`train.py` with your script path and put any script arguments after it.
|
|
801
740
|
|
|
802
741
|
You can then control tracking while the run is active:
|
|
803
742
|
|
|
@@ -878,7 +817,7 @@ LESS OVERHEAD
|
|
|
878
817
|
|
|
879
818
|
Rather than collecting everything continuously, Pulse lets you decide what information is worth monitoring.
|
|
880
819
|
|
|
881
|
-
Measured on a 20k-step loop started
|
|
820
|
+
Measured on a 20k-step loop started with `pulse run --stream`:
|
|
882
821
|
|
|
883
822
|
| Work on the training thread | single process | stream mode |
|
|
884
823
|
|---|---:|---:|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|