de-shell 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {de_shell-0.2.0 → de_shell-0.3.0}/PKG-INFO +31 -14
- {de_shell-0.2.0 → de_shell-0.3.0}/README.md +25 -8
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/__init__.py +1 -1
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/backendProcess.ts +14 -56
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/pythonEnv.test.ts +16 -4
- de_shell-0.3.0/de_shell/js/main/stdoutDemux.test.ts +195 -0
- de_shell-0.3.0/de_shell/js/main/stdoutDemux.ts +167 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/renderer/FigureFrame.tsx +40 -16
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/renderer/figureBridge.test.ts +92 -1
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/renderer/figureBridge.ts +63 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/renderer/index.ts +1 -1
- de_shell-0.3.0/de_shell/js/renderer/sizeReporter.test.ts +76 -0
- de_shell-0.3.0/de_shell/js/renderer/sizeReporter.ts +29 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/testing/harness.cjs +35 -20
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/testing/harness.test.cjs +30 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/plotting/figure.py +111 -1
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell.egg-info/PKG-INFO +31 -14
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell.egg-info/SOURCES.txt +4 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell.egg-info/requires.txt +1 -1
- de_shell-0.3.0/pyproject.toml +157 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/tests/test_figure.py +82 -1
- de_shell-0.2.0/pyproject.toml +0 -83
- {de_shell-0.2.0 → de_shell-0.3.0}/LICENSE +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/actions/__init__.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/actions/context.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/actions/figure_registry.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/actions/lifecycle.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/actions/registry.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/actions/wizard.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/app.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/compute.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/debug_flags.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/ipc.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/__init__.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/__main__.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/backendProcess.test.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/config.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/dialogs.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/envProgress.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/errorReport.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/index.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/problemLog.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/pythonEnv.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/sentryEnvelope.test.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/sentryEnvelope.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/updater.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/updaterErrors.test.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/updaterErrors.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/main/window.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/package.json +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/preload/index.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/renderer/figureBridge.react.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/renderer/protocol.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/renderer/shellState.test.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/js/renderer/shellState.ts +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/log_stream.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/plotting/__init__.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/plotting/colormaps.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/plotting/selectors/__init__.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/plotting/selectors/utils.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/plotting/stream.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/process_guard.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/session.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/testing/__init__.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell/timing.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell.egg-info/dependency_links.txt +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/de_shell.egg-info/top_level.txt +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/setup.cfg +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/tests/test_actions.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/tests/test_boundary.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/tests/test_session_base.py +0 -0
- {de_shell-0.2.0 → de_shell-0.3.0}/tests/test_stream.py +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: de-shell
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: The Electron + Python desktop-app shell Direct Electron's apps (SpyDE, Ground Crew, Autopilot) are assembled from
|
|
5
5
|
Author-email: Carter Francis <cartsfrancis@gmail.com>
|
|
6
6
|
License: MIT
|
|
7
|
-
Project-URL: Homepage, https://github.com/
|
|
8
|
-
Project-URL: Repository, https://github.com/
|
|
9
|
-
Project-URL: Issues, https://github.com/
|
|
10
|
-
Project-URL: Changelog, https://github.com/
|
|
7
|
+
Project-URL: Homepage, https://github.com/directelectron/de-shell
|
|
8
|
+
Project-URL: Repository, https://github.com/directelectron/de-shell
|
|
9
|
+
Project-URL: Issues, https://github.com/directelectron/de-shell/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/directelectron/de-shell/blob/main/CHANGELOG.rst
|
|
11
11
|
Keywords: electron,desktop,anyplotlib,electron-microscopy,direct-electron
|
|
12
12
|
Classifier: Development Status :: 3 - Alpha
|
|
13
13
|
Classifier: Intended Audience :: Developers
|
|
@@ -26,7 +26,7 @@ Requires-Python: >=3.10
|
|
|
26
26
|
Description-Content-Type: text/markdown
|
|
27
27
|
License-File: LICENSE
|
|
28
28
|
Requires-Dist: numpy>=1.20.0
|
|
29
|
-
Requires-Dist: anyplotlib>=0.
|
|
29
|
+
Requires-Dist: anyplotlib>=0.10.0
|
|
30
30
|
Requires-Dist: pyyaml
|
|
31
31
|
Provides-Extra: tests
|
|
32
32
|
Requires-Dist: pytest>=3.6; extra == "tests"
|
|
@@ -36,7 +36,7 @@ Dynamic: license-file
|
|
|
36
36
|
# DE Shell
|
|
37
37
|
|
|
38
38
|
[](https://pypi.org/project/de-shell/)
|
|
39
|
-
[](https://github.com/directelectron/de-shell/actions/workflows/ci.yml)
|
|
40
40
|
[](LICENSE)
|
|
41
41
|
|
|
42
42
|
The substrate Direct Electron's desktop apps are assembled from: **SpyDE**
|
|
@@ -122,8 +122,10 @@ peer dependencies — react, electron, electron-updater, @playwright/test —
|
|
|
122
122
|
are the app's to declare; every app already does. The e2e specs take the
|
|
123
123
|
harness from `shell/testing/harness.cjs`.
|
|
124
124
|
|
|
125
|
-
|
|
126
|
-
|
|
125
|
+
All three apps are wired this way: Autopilot, SpyDE
|
|
126
|
+
([directelectron/spyde#152](https://github.com/directelectron/spyde/pull/152))
|
|
127
|
+
and Ground Crew ([directelectron/de_ground_crew#183](https://github.com/directelectron/de_ground_crew/pull/183)),
|
|
128
|
+
pinned to the `0.2` line.
|
|
127
129
|
|
|
128
130
|
## Developing it
|
|
129
131
|
|
|
@@ -148,8 +150,20 @@ tests, and builds the wheel and checks what it carries.
|
|
|
148
150
|
|
|
149
151
|
The version is written once, in `de_shell/__init__.py`. To release:
|
|
150
152
|
|
|
151
|
-
1.
|
|
152
|
-
|
|
153
|
+
1. Run **Prepare Release** from the Actions tab and pick the bump (`minor`,
|
|
154
|
+
`bugfix`, `major`, `pre-release`, or `finalize` to drop a `bN` suffix). It
|
|
155
|
+
bumps `__version__`, assembles `CHANGELOG.rst` from the news fragments in
|
|
156
|
+
`upcoming_changes/`, runs the pre-flight checks, and opens a release PR that
|
|
157
|
+
names the one tag that will pass.
|
|
158
|
+
2. Review and merge that PR, then tag the merge commit and push the tag — the
|
|
159
|
+
PR body has the exact commands.
|
|
160
|
+
|
|
161
|
+
The tag has to match `__version__` exactly; going through the workflow makes
|
|
162
|
+
them agree by construction, because the PR *is* the bump. To assemble the
|
|
163
|
+
changelog by hand instead, `uv tool run towncrier build --version X.Y.Z` does
|
|
164
|
+
the same thing — but stage `upcoming_changes/` with `git add -A`, because
|
|
165
|
+
towncrier deletes the fragments it consumed and a plain `git add CHANGELOG.rst`
|
|
166
|
+
leaves the deletions behind for the next release to re-publish.
|
|
153
167
|
|
|
154
168
|
`.github/workflows/publish.yml` builds the distributions, refuses a tag that
|
|
155
169
|
does not match `__version__`, and uploads to PyPI through trusted publishing
|
|
@@ -175,9 +189,9 @@ SpyDE commit the app copies were taken from:
|
|
|
175
189
|
panes on large stills), JSON emit that never writes bare `NaN`, the harness
|
|
176
190
|
hardening, and the unit tests for all of it.
|
|
177
191
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
192
|
+
* Ground Crew `main` @ 0382549 (0.2.1): the chunk-list stdout demuxer, the
|
|
193
|
+
size reporter and StrictMode-safe registration in `FigureFrame`, the figure
|
|
194
|
+
document's scroll pin, and the anyplotlib 0.8.0 floor.
|
|
181
195
|
|
|
182
196
|
## Rules
|
|
183
197
|
|
|
@@ -189,6 +203,9 @@ branch that has not merged.
|
|
|
189
203
|
* **The protocol is the contract.** `PLOTAPP:` JSON lines and `PLOTBIN:`
|
|
190
204
|
binary frames over the sidecar's stdio. Both halves of it live in this one
|
|
191
205
|
package on purpose; keep it that way.
|
|
206
|
+
* **Every pull request carries its own changelog entry**, as a news fragment
|
|
207
|
+
under [`upcoming_changes/`](upcoming_changes/README.rst) — one file per PR,
|
|
208
|
+
so two of them never conflict over the same lines of `CHANGELOG.rst`.
|
|
192
209
|
* **LF line endings**, enforced by `.gitattributes`.
|
|
193
210
|
|
|
194
211
|
## License
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# DE Shell
|
|
2
2
|
|
|
3
3
|
[](https://pypi.org/project/de-shell/)
|
|
4
|
-
[](https://github.com/directelectron/de-shell/actions/workflows/ci.yml)
|
|
5
5
|
[](LICENSE)
|
|
6
6
|
|
|
7
7
|
The substrate Direct Electron's desktop apps are assembled from: **SpyDE**
|
|
@@ -87,8 +87,10 @@ peer dependencies — react, electron, electron-updater, @playwright/test —
|
|
|
87
87
|
are the app's to declare; every app already does. The e2e specs take the
|
|
88
88
|
harness from `shell/testing/harness.cjs`.
|
|
89
89
|
|
|
90
|
-
|
|
91
|
-
|
|
90
|
+
All three apps are wired this way: Autopilot, SpyDE
|
|
91
|
+
([directelectron/spyde#152](https://github.com/directelectron/spyde/pull/152))
|
|
92
|
+
and Ground Crew ([directelectron/de_ground_crew#183](https://github.com/directelectron/de_ground_crew/pull/183)),
|
|
93
|
+
pinned to the `0.2` line.
|
|
92
94
|
|
|
93
95
|
## Developing it
|
|
94
96
|
|
|
@@ -113,8 +115,20 @@ tests, and builds the wheel and checks what it carries.
|
|
|
113
115
|
|
|
114
116
|
The version is written once, in `de_shell/__init__.py`. To release:
|
|
115
117
|
|
|
116
|
-
1.
|
|
117
|
-
|
|
118
|
+
1. Run **Prepare Release** from the Actions tab and pick the bump (`minor`,
|
|
119
|
+
`bugfix`, `major`, `pre-release`, or `finalize` to drop a `bN` suffix). It
|
|
120
|
+
bumps `__version__`, assembles `CHANGELOG.rst` from the news fragments in
|
|
121
|
+
`upcoming_changes/`, runs the pre-flight checks, and opens a release PR that
|
|
122
|
+
names the one tag that will pass.
|
|
123
|
+
2. Review and merge that PR, then tag the merge commit and push the tag — the
|
|
124
|
+
PR body has the exact commands.
|
|
125
|
+
|
|
126
|
+
The tag has to match `__version__` exactly; going through the workflow makes
|
|
127
|
+
them agree by construction, because the PR *is* the bump. To assemble the
|
|
128
|
+
changelog by hand instead, `uv tool run towncrier build --version X.Y.Z` does
|
|
129
|
+
the same thing — but stage `upcoming_changes/` with `git add -A`, because
|
|
130
|
+
towncrier deletes the fragments it consumed and a plain `git add CHANGELOG.rst`
|
|
131
|
+
leaves the deletions behind for the next release to re-publish.
|
|
118
132
|
|
|
119
133
|
`.github/workflows/publish.yml` builds the distributions, refuses a tag that
|
|
120
134
|
does not match `__version__`, and uploads to PyPI through trusted publishing
|
|
@@ -140,9 +154,9 @@ SpyDE commit the app copies were taken from:
|
|
|
140
154
|
panes on large stills), JSON emit that never writes bare `NaN`, the harness
|
|
141
155
|
hardening, and the unit tests for all of it.
|
|
142
156
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
157
|
+
* Ground Crew `main` @ 0382549 (0.2.1): the chunk-list stdout demuxer, the
|
|
158
|
+
size reporter and StrictMode-safe registration in `FigureFrame`, the figure
|
|
159
|
+
document's scroll pin, and the anyplotlib 0.8.0 floor.
|
|
146
160
|
|
|
147
161
|
## Rules
|
|
148
162
|
|
|
@@ -154,6 +168,9 @@ branch that has not merged.
|
|
|
154
168
|
* **The protocol is the contract.** `PLOTAPP:` JSON lines and `PLOTBIN:`
|
|
155
169
|
binary frames over the sidecar's stdio. Both halves of it live in this one
|
|
156
170
|
package on purpose; keep it that way.
|
|
171
|
+
* **Every pull request carries its own changelog entry**, as a news fragment
|
|
172
|
+
under [`upcoming_changes/`](upcoming_changes/README.rst) — one file per PR,
|
|
173
|
+
so two of them never conflict over the same lines of `CHANGELOG.rst`.
|
|
157
174
|
* **LF line endings**, enforced by `.gitattributes`.
|
|
158
175
|
|
|
159
176
|
## License
|
|
@@ -20,6 +20,6 @@ from __future__ import annotations
|
|
|
20
20
|
|
|
21
21
|
#: The package version — the one place it is written. pyproject.toml reads it
|
|
22
22
|
#: (dynamic version) and the release workflow refuses a tag that disagrees.
|
|
23
|
-
__version__ = "0.
|
|
23
|
+
__version__ = "0.3.0"
|
|
24
24
|
|
|
25
25
|
__all__ = ["ipc", "log_stream", "process_guard", "debug_flags", "compute"]
|
|
@@ -9,6 +9,7 @@ import process from 'process'
|
|
|
9
9
|
// Extension spelled out so node:test can load this module without a bundler
|
|
10
10
|
// (native type-stripping resolves relative imports literally).
|
|
11
11
|
import { shellConfig } from './config.ts'
|
|
12
|
+
import { createStdoutDemux } from './stdoutDemux.ts'
|
|
12
13
|
|
|
13
14
|
export interface BackendHandlers {
|
|
14
15
|
onMessage: (msg: Record<string, unknown>) => void
|
|
@@ -19,9 +20,6 @@ export interface BackendHandlers {
|
|
|
19
20
|
onBinary?: (header: Record<string, unknown>, payload: Buffer) => void
|
|
20
21
|
}
|
|
21
22
|
|
|
22
|
-
const PLOTBIN = Buffer.from('PLOTBIN:')
|
|
23
|
-
const NL = 0x0a
|
|
24
|
-
|
|
25
23
|
let proc: ChildProcess | null = null
|
|
26
24
|
let tickTimer: ReturnType<typeof setInterval> | null = null
|
|
27
25
|
|
|
@@ -95,60 +93,20 @@ export function startBackend(
|
|
|
95
93
|
if (!proc && tickTimer) { clearInterval(tickTimer); tickTimer = null }
|
|
96
94
|
}, 2000)
|
|
97
95
|
|
|
98
|
-
// Custom stdout demuxer: the stream interleaves text lines
|
|
99
|
-
// plain log output, both '\n'-terminated) with raw PLOTBIN
|
|
100
|
-
// (PLOTBIN:<hlen>:<plen>\n<header_json><payload>). readline
|
|
101
|
-
//
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
acc.subarray(0, PLOTBIN.length).equals(PLOTBIN)) {
|
|
111
|
-
const nl = acc.indexOf(NL)
|
|
112
|
-
if (nl < 0) break // prefix line incomplete
|
|
113
|
-
const prefix = acc.subarray(PLOTBIN.length, nl).toString('ascii')
|
|
114
|
-
const [hlenS, plenS] = prefix.split(':')
|
|
115
|
-
const hlen = parseInt(hlenS, 10), plen = parseInt(plenS, 10)
|
|
116
|
-
if (!(hlen >= 0) || !(plen >= 0)) { // malformed → drop the line
|
|
117
|
-
handlers.onStream(`[sidecar protocol] malformed PLOTBIN prefix: ${prefix}\n`, 'stderr')
|
|
118
|
-
acc = acc.subarray(nl + 1); continue
|
|
119
|
-
}
|
|
120
|
-
const bodyStart = nl + 1
|
|
121
|
-
const end = bodyStart + hlen + plen
|
|
122
|
-
if (acc.length < end) break // body not fully arrived yet
|
|
123
|
-
let header: Record<string, unknown> = {}
|
|
124
|
-
try {
|
|
125
|
-
header = JSON.parse(acc.subarray(bodyStart, bodyStart + hlen).toString('utf8'))
|
|
126
|
-
} catch { /* malformed header — still consume the frame */ }
|
|
127
|
-
// Copy the payload out so it survives `acc` being sliced/reused.
|
|
128
|
-
const payload = Buffer.from(acc.subarray(bodyStart + hlen, end))
|
|
129
|
-
acc = acc.subarray(end)
|
|
130
|
-
try { handlers.onBinary?.(header, payload) } catch { /* ignore */ }
|
|
131
|
-
continue
|
|
132
|
-
}
|
|
133
|
-
// Otherwise a text line up to the next '\n'.
|
|
134
|
-
const nl = acc.indexOf(NL)
|
|
135
|
-
if (nl < 0) break // line incomplete
|
|
136
|
-
const line = acc.subarray(0, nl).toString('utf8')
|
|
137
|
-
acc = acc.subarray(nl + 1)
|
|
138
|
-
if (line.startsWith('PLOTAPP:')) {
|
|
139
|
-
try {
|
|
140
|
-
handlers.onMessage(JSON.parse(line.slice(8)) as Record<string, unknown>)
|
|
141
|
-
} catch {
|
|
142
|
-
// Say so rather than swallow it: a truncated frame is how a backend
|
|
143
|
-
// bug presents, and silence turns it into "the UI just stopped".
|
|
144
|
-
handlers.onStream(`[sidecar protocol] malformed JSON message: ${line.slice(0, 200)}\n`, 'stderr')
|
|
145
|
-
}
|
|
146
|
-
} else if (line.trim()) {
|
|
147
|
-
rememberOutput(line)
|
|
148
|
-
handlers.onStream(line + '\n', 'stdout')
|
|
149
|
-
}
|
|
150
|
-
}
|
|
96
|
+
// Custom stdout demuxer (stdoutDemux.ts): the stream interleaves text lines
|
|
97
|
+
// (PLOTAPP: JSON and plain log output, both '\n'-terminated) with raw PLOTBIN
|
|
98
|
+
// binary frames (PLOTBIN:<hlen>:<plen>\n<header_json><payload>). readline
|
|
99
|
+
// can't carry binary, so the raw Buffer stream is parsed with a chunk-list
|
|
100
|
+
// accumulator — each byte is copied exactly once, into the emitted unit
|
|
101
|
+
// (the old whole-prefix Buffer.concat per chunk was O(N²/chunkSize) while a
|
|
102
|
+
// large frame streamed in: 11.9 s per 64 MB frame at 64 KiB chunks). Plain
|
|
103
|
+
// output is remembered for a problem report on the way past; the demuxer
|
|
104
|
+
// itself says so on a malformed message rather than swallowing it.
|
|
105
|
+
const demux = createStdoutDemux({
|
|
106
|
+
...handlers,
|
|
107
|
+
onStream: (text, kind) => { rememberOutput(text); handlers.onStream(text, kind) },
|
|
151
108
|
})
|
|
109
|
+
child.stdout!.on('data', (chunk: Buffer) => demux.push(chunk))
|
|
152
110
|
|
|
153
111
|
child.stderr!.on('data', (d: Buffer) => {
|
|
154
112
|
rememberOutput(d.toString())
|
|
@@ -10,9 +10,9 @@
|
|
|
10
10
|
* Run: `node --test src/pythonEnv.test.ts` (from packages/shell-main/), or via
|
|
11
11
|
* the `test:unit` npm script.
|
|
12
12
|
*/
|
|
13
|
-
import { test } from 'node:test'
|
|
13
|
+
import { test, after } from 'node:test'
|
|
14
14
|
import assert from 'node:assert/strict'
|
|
15
|
-
import { mkdtempSync, writeFileSync, mkdirSync } from 'fs'
|
|
15
|
+
import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from 'fs'
|
|
16
16
|
import { join } from 'path'
|
|
17
17
|
import { tmpdir } from 'os'
|
|
18
18
|
import { configureShell } from './config.ts'
|
|
@@ -24,17 +24,29 @@ configureShell({
|
|
|
24
24
|
pythonModule: 'testapp',
|
|
25
25
|
})
|
|
26
26
|
|
|
27
|
+
/** Every temp directory these tests make, removed when the file is done. */
|
|
28
|
+
const tempDirs: string[] = []
|
|
29
|
+
after(() => {
|
|
30
|
+
for (const dir of tempDirs) rmSync(dir, { recursive: true, force: true })
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
function tempDir(prefix: string): string {
|
|
34
|
+
const dir = mkdtempSync(join(tmpdir(), prefix))
|
|
35
|
+
tempDirs.push(dir)
|
|
36
|
+
return dir
|
|
37
|
+
}
|
|
38
|
+
|
|
27
39
|
/** A directory containing a uv stub under both spellings, so these tests do not
|
|
28
40
|
* fork on the host platform. */
|
|
29
41
|
function dirWithUv(): string {
|
|
30
|
-
const dir =
|
|
42
|
+
const dir = tempDir('uv-stub-')
|
|
31
43
|
writeFileSync(join(dir, 'uv'), '')
|
|
32
44
|
writeFileSync(join(dir, 'uv.exe'), '')
|
|
33
45
|
return dir
|
|
34
46
|
}
|
|
35
47
|
|
|
36
48
|
function emptyDir(): string {
|
|
37
|
-
return
|
|
49
|
+
return tempDir('uv-none-')
|
|
38
50
|
}
|
|
39
51
|
|
|
40
52
|
/** An env whose PATH and every fallback root point somewhere we control. */
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* stdoutDemux.test.ts — node:test unit tests for the backend stdout demuxer.
|
|
3
|
+
*
|
|
4
|
+
* The stream interleaves '\n'-terminated text lines (PLOTAPP: JSON + plain
|
|
5
|
+
* log output) with raw PLOTBIN binary frames
|
|
6
|
+
* (PLOTBIN:<hlen>:<plen>\n<header_json><payload>). The demuxer must produce
|
|
7
|
+
* the SAME event trace regardless of how the OS slices the stream into
|
|
8
|
+
* 'data' chunks — Windows delivers a single large child-stdout write as
|
|
9
|
+
* uniform 64 KiB chunks (measured 2026-08-15), and chunk boundaries can land
|
|
10
|
+
* mid-marker, mid-prefix, mid-header or mid-payload.
|
|
11
|
+
*
|
|
12
|
+
* Also pins the accumulator's cost shape: a 16 MB frame arriving as 64 KiB
|
|
13
|
+
* chunks must demux in linear time. The pre-fix Buffer.concat accumulator
|
|
14
|
+
* recopied the whole prefix per chunk — O(N²/chunk): measured 813 ms at
|
|
15
|
+
* 16 MB, 11.9 s at 64 MB (notes/runs/2026-08-15-perf-review-measurements.md
|
|
16
|
+
* §1) — so the 250 ms budget is ~3× under the old cost and ~38× over the
|
|
17
|
+
* chunk-list cost (6.6 ms measured); generous against CI timer noise on
|
|
18
|
+
* both sides.
|
|
19
|
+
*
|
|
20
|
+
* Run: `node --test src/stdoutDemux.test.ts` (from packages/shell-main/),
|
|
21
|
+
* or via the `test:unit` npm script.
|
|
22
|
+
*/
|
|
23
|
+
import { test } from 'node:test'
|
|
24
|
+
import assert from 'node:assert/strict'
|
|
25
|
+
import { createStdoutDemux } from './stdoutDemux.ts'
|
|
26
|
+
|
|
27
|
+
type Event =
|
|
28
|
+
| { kind: 'message'; msg: Record<string, unknown> }
|
|
29
|
+
| { kind: 'stream'; text: string }
|
|
30
|
+
| { kind: 'binary'; header: Record<string, unknown>; payload: string }
|
|
31
|
+
|
|
32
|
+
function collector() {
|
|
33
|
+
const events: Event[] = []
|
|
34
|
+
const demux = createStdoutDemux({
|
|
35
|
+
onMessage: (msg) => events.push({ kind: 'message', msg }),
|
|
36
|
+
onStream: (text) => events.push({ kind: 'stream', text }),
|
|
37
|
+
onBinary: (header, payload) =>
|
|
38
|
+
events.push({ kind: 'binary', header, payload: payload.toString('hex') }),
|
|
39
|
+
})
|
|
40
|
+
return { events, demux }
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function frame(header: Record<string, unknown>, payload: Buffer): Buffer {
|
|
44
|
+
const h = Buffer.from(JSON.stringify(header), 'utf8')
|
|
45
|
+
return Buffer.concat([
|
|
46
|
+
Buffer.from(`PLOTBIN:${h.length}:${payload.length}\n`, 'ascii'),
|
|
47
|
+
h,
|
|
48
|
+
payload,
|
|
49
|
+
])
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Feed `stream` in slices of `sizes` (cycled); return the event trace. */
|
|
53
|
+
function run(stream: Buffer, sizes: number[]): Event[] {
|
|
54
|
+
const { events, demux } = collector()
|
|
55
|
+
let pos = 0
|
|
56
|
+
let i = 0
|
|
57
|
+
while (pos < stream.length) {
|
|
58
|
+
const n = Math.min(sizes[i % sizes.length], stream.length - pos)
|
|
59
|
+
demux.push(stream.subarray(pos, pos + n))
|
|
60
|
+
pos += n
|
|
61
|
+
i++
|
|
62
|
+
}
|
|
63
|
+
return events
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function patternPayload(n: number): Buffer {
|
|
67
|
+
const p = Buffer.allocUnsafe(n)
|
|
68
|
+
for (let i = 0; i < n; i++) p[i] = i & 0xff // includes 0x0a bytes
|
|
69
|
+
return p
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// A representative interleaved stream: log line, PLOTAPP message, a binary
|
|
73
|
+
// frame whose payload contains newline bytes and marker-lookalike bytes,
|
|
74
|
+
// another log line, a second frame back-to-back with a third, final message.
|
|
75
|
+
function interleavedStream(): Buffer {
|
|
76
|
+
const nasty = Buffer.concat([
|
|
77
|
+
Buffer.from('\nPLOTBIN:9:9\nPLOTAPP:{}\n', 'ascii'), // lookalikes INSIDE payload
|
|
78
|
+
patternPayload(3000),
|
|
79
|
+
])
|
|
80
|
+
return Buffer.concat([
|
|
81
|
+
Buffer.from('starting up\n', 'utf8'),
|
|
82
|
+
Buffer.from('PLOTAPP:{"type":"state_update","key":"clim"}\n', 'utf8'),
|
|
83
|
+
frame({ fig_id: 'f1', key: 'image' }, nasty),
|
|
84
|
+
Buffer.from('mid-run log\n', 'utf8'),
|
|
85
|
+
frame({ fig_id: 'f2', key: 'image' }, patternPayload(1)),
|
|
86
|
+
frame({ fig_id: 'f3', key: 'spec' }, Buffer.alloc(0)),
|
|
87
|
+
Buffer.from('PLOTAPP:{"type":"done"}\n', 'utf8'),
|
|
88
|
+
])
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
test('one whole-stream chunk parses every unit in order', () => {
|
|
92
|
+
const events = run(interleavedStream(), [1 << 30])
|
|
93
|
+
assert.deepEqual(events.map((e) => e.kind),
|
|
94
|
+
['stream', 'message', 'binary', 'stream', 'binary', 'binary', 'message'])
|
|
95
|
+
const b1 = events[2] as Extract<Event, { kind: 'binary' }>
|
|
96
|
+
assert.deepEqual(b1.header, { fig_id: 'f1', key: 'image' })
|
|
97
|
+
const b3 = events[5] as Extract<Event, { kind: 'binary' }>
|
|
98
|
+
assert.equal(b3.payload, '')
|
|
99
|
+
assert.deepEqual((events[6] as Extract<Event, { kind: 'message' }>).msg,
|
|
100
|
+
{ type: 'done' })
|
|
101
|
+
})
|
|
102
|
+
|
|
103
|
+
test('the event trace is chunking-invariant (64 KiB, tiny, 1-byte, ragged)', () => {
|
|
104
|
+
const stream = interleavedStream()
|
|
105
|
+
const reference = run(stream, [1 << 30])
|
|
106
|
+
for (const sizes of [[65536], [7], [1], [3, 1, 40, 2, 1000, 1]]) {
|
|
107
|
+
assert.deepEqual(run(stream, sizes), reference,
|
|
108
|
+
`trace diverged for chunk sizes ${JSON.stringify(sizes)}`)
|
|
109
|
+
}
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
test('splits inside the marker, prefix, header and payload all reassemble', () => {
|
|
113
|
+
const f = frame({ fig_id: 'x' }, patternPayload(64))
|
|
114
|
+
const reference = run(f, [1 << 30])
|
|
115
|
+
// Split at every single boundary position of the frame — brute force
|
|
116
|
+
// covers mid-marker, mid-prefix, mid-header and mid-payload by exhaustion.
|
|
117
|
+
for (let cut = 1; cut < f.length; cut++) {
|
|
118
|
+
const { events, demux } = collector()
|
|
119
|
+
demux.push(f.subarray(0, cut))
|
|
120
|
+
demux.push(f.subarray(cut))
|
|
121
|
+
assert.deepEqual(events, reference, `split at byte ${cut} diverged`)
|
|
122
|
+
}
|
|
123
|
+
})
|
|
124
|
+
|
|
125
|
+
test('a malformed PLOTBIN prefix line is dropped and parsing resumes', () => {
|
|
126
|
+
const stream = Buffer.concat([
|
|
127
|
+
Buffer.from('PLOTBIN:not:numbers\n', 'ascii'),
|
|
128
|
+
Buffer.from('PLOTAPP:{"type":"after"}\n', 'utf8'),
|
|
129
|
+
])
|
|
130
|
+
for (const sizes of [[1 << 30], [1]]) {
|
|
131
|
+
const events = run(stream, sizes)
|
|
132
|
+
assert.deepEqual(events, [
|
|
133
|
+
{ kind: 'message', msg: { type: 'after' } },
|
|
134
|
+
])
|
|
135
|
+
}
|
|
136
|
+
})
|
|
137
|
+
|
|
138
|
+
test('a malformed header still consumes the frame and delivers the payload', () => {
|
|
139
|
+
const h = Buffer.from('{broken', 'utf8')
|
|
140
|
+
const payload = patternPayload(32)
|
|
141
|
+
const stream = Buffer.concat([
|
|
142
|
+
Buffer.from(`PLOTBIN:${h.length}:${payload.length}\n`, 'ascii'),
|
|
143
|
+
h, payload,
|
|
144
|
+
Buffer.from('PLOTAPP:{"type":"after"}\n', 'utf8'),
|
|
145
|
+
])
|
|
146
|
+
const events = run(stream, [5])
|
|
147
|
+
assert.deepEqual(events, [
|
|
148
|
+
{ kind: 'binary', header: {}, payload: payload.toString('hex') },
|
|
149
|
+
{ kind: 'message', msg: { type: 'after' } },
|
|
150
|
+
])
|
|
151
|
+
})
|
|
152
|
+
|
|
153
|
+
test('malformed PLOTAPP JSON is reported, not swallowed; blank lines emit nothing', () => {
|
|
154
|
+
// A truncated frame is how a backend bug presents, and silence turns it
|
|
155
|
+
// into "the UI just stopped" — so the line is dropped AND said so, on
|
|
156
|
+
// stderr, and the parse carries on to the next unit.
|
|
157
|
+
const stream = Buffer.from('PLOTAPP:{nope\n\n \nreal log\n', 'utf8')
|
|
158
|
+
const events = run(stream, [2])
|
|
159
|
+
assert.equal(events.length, 2)
|
|
160
|
+
assert.match((events[0] as { text?: string }).text ?? '',
|
|
161
|
+
/^\[sidecar protocol\] malformed JSON message: PLOTAPP:\{nope/)
|
|
162
|
+
assert.deepEqual(events[1], { kind: 'stream', text: 'real log\n' })
|
|
163
|
+
})
|
|
164
|
+
|
|
165
|
+
test('a handler that throws does not stop the parse', () => {
|
|
166
|
+
const seen: string[] = []
|
|
167
|
+
const demux = createStdoutDemux({
|
|
168
|
+
onMessage: (m) => { seen.push(`m:${(m as { type?: string }).type}`) },
|
|
169
|
+
onStream: () => { throw new Error('boom') },
|
|
170
|
+
onBinary: () => { throw new Error('boom') },
|
|
171
|
+
})
|
|
172
|
+
demux.push(Buffer.concat([
|
|
173
|
+
Buffer.from('log line\n', 'utf8'),
|
|
174
|
+
frame({ fig_id: 'f' }, patternPayload(8)),
|
|
175
|
+
Buffer.from('PLOTAPP:{"type":"after"}\n', 'utf8'),
|
|
176
|
+
]))
|
|
177
|
+
assert.deepEqual(seen, ['m:after'])
|
|
178
|
+
})
|
|
179
|
+
|
|
180
|
+
test('PERF PIN: a 16 MB frame in 64 KiB chunks demuxes in linear time', () => {
|
|
181
|
+
const payload = Buffer.allocUnsafe(16 * 1024 * 1024).fill(7)
|
|
182
|
+
const stream = frame({ fig_id: 'big', key: 'image' }, payload)
|
|
183
|
+
const { events, demux } = collector()
|
|
184
|
+
const t0 = process.hrtime.bigint()
|
|
185
|
+
for (let pos = 0; pos < stream.length; pos += 65536) {
|
|
186
|
+
demux.push(stream.subarray(pos, Math.min(pos + 65536, stream.length)))
|
|
187
|
+
}
|
|
188
|
+
const ms = Number(process.hrtime.bigint() - t0) / 1e6
|
|
189
|
+
assert.equal(events.length, 1)
|
|
190
|
+
assert.equal((events[0] as Extract<Event, { kind: 'binary' }>).payload.length,
|
|
191
|
+
payload.length * 2) // hex doubles
|
|
192
|
+
assert.ok(ms < 250,
|
|
193
|
+
`16 MB frame took ${ms.toFixed(0)} ms — the quadratic Buffer.concat ` +
|
|
194
|
+
`accumulator signature (measured 813 ms pre-fix, 6.6 ms chunk-list)`)
|
|
195
|
+
})
|