diffusers-workflow 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusers_workflow-0.4.0.dist-info/METADATA +318 -0
- diffusers_workflow-0.4.0.dist-info/RECORD +260 -0
- diffusers_workflow-0.4.0.dist-info/WHEEL +5 -0
- diffusers_workflow-0.4.0.dist-info/entry_points.txt +7 -0
- diffusers_workflow-0.4.0.dist-info/licenses/LICENSE +201 -0
- diffusers_workflow-0.4.0.dist-info/top_level.txt +2 -0
- dw/__init__.py +440 -0
- dw/adapter_compatibility.py +226 -0
- dw/arguments.py +1231 -0
- dw/assessment_rules.py +159 -0
- dw/assets.py +130 -0
- dw/cache_blocks.json +16 -0
- dw/cache_blocks.py +146 -0
- dw/community_pipelines/pipeline_flux_rf_inversion.py +1184 -0
- dw/content_types.py +150 -0
- dw/dissolve_frame_errors.py +121 -0
- dw/docs/ACCELERATION.md +352 -0
- dw/docs/AGENT_LOOP.md +95 -0
- dw/docs/DEPENDENCIES.md +91 -0
- dw/docs/IP_ADAPTER.md +109 -0
- dw/docs/LORAS.md +131 -0
- dw/docs/MCP.md +517 -0
- dw/docs/PROMPT_WEIGHTING.md +78 -0
- dw/docs/QUANTIZATION.md +230 -0
- dw/docs/RECIPES_24GB.md +201 -0
- dw/docs/RELEASING.md +195 -0
- dw/docs/REMOTE.md +140 -0
- dw/docs/REPL_COMMANDS.md +121 -0
- dw/docs/REPL_WORKER_GUIDE.md +51 -0
- dw/docs/SECURITY.md +272 -0
- dw/docs/SECURITY_QUICKREF.md +112 -0
- dw/docs/SERVER.md +679 -0
- dw/docs/TASKS.md +1741 -0
- dw/docs/TESTING.md +71 -0
- dw/docs/WORKFLOW_GUIDE.md +2038 -0
- dw/docs/WORKSPACES.md +316 -0
- dw/download_watch.py +335 -0
- dw/elision.py +306 -0
- dw/events.py +275 -0
- dw/for_each.py +409 -0
- dw/host_memory.py +258 -0
- dw/host_memory_projection.py +230 -0
- dw/hub_cache.py +432 -0
- dw/introspection.py +1228 -0
- dw/kernel_availability.py +208 -0
- dw/locations.py +599 -0
- dw/log_setup.py +45 -0
- dw/loudness.py +82 -0
- dw/media_audio.py +217 -0
- dw/media_frames.py +367 -0
- dw/media_info.py +297 -0
- dw/pipeline_processors/chain.py +821 -0
- dw/pipeline_processors/config_objects.py +237 -0
- dw/pipeline_processors/pipeline.py +2297 -0
- dw/pipeline_processors/remote.py +46 -0
- dw/plan.py +920 -0
- dw/previous_results.py +411 -0
- dw/probe_paths.py +59 -0
- dw/prompt_schema.json +48 -0
- dw/prompt_weighting.py +378 -0
- dw/prompts.py +159 -0
- dw/realize.py +250 -0
- dw/reference_limits.py +215 -0
- dw/reference_names.py +125 -0
- dw/repl.py +338 -0
- dw/repl_commands.py +836 -0
- dw/repl_worker.py +159 -0
- dw/result.py +1720 -0
- dw/result_fps.py +82 -0
- dw/run.py +162 -0
- dw/runs.py +768 -0
- dw/scalar_result_validation.py +97 -0
- dw/schema.py +283 -0
- dw/security.py +1038 -0
- dw/select_validation.py +115 -0
- dw/serve.py +277 -0
- dw/server/__init__.py +2 -0
- dw/server/app.py +4586 -0
- dw/server/assess.py +132 -0
- dw/server/catalog_shape.py +487 -0
- dw/server/enhancers.py +129 -0
- dw/server/exports.py +480 -0
- dw/server/guides.py +257 -0
- dw/server/jobs.py +1561 -0
- dw/server/mcp_mount.py +95 -0
- dw/server/netinfo.py +124 -0
- dw/server/observed_cost.py +379 -0
- dw/server/sysinfo.py +71 -0
- dw/server/ui/assets/abap-08VXUWAP.js +1 -0
- dw/server/ui/assets/apex-BWPQTe0t.js +1 -0
- dw/server/ui/assets/azcli-Bc_sGQ0U.js +1 -0
- dw/server/ui/assets/bat-i0X4ZdIN.js +1 -0
- dw/server/ui/assets/bicep-B5-_aFwp.js +2 -0
- dw/server/ui/assets/cameligo-DMUM7wLl.js +1 -0
- dw/server/ui/assets/clojure-Cm7r79vr.js +1 -0
- dw/server/ui/assets/codicon-Brq4_Ui5.ttf +0 -0
- dw/server/ui/assets/coffee-Ba7i2nA0.js +1 -0
- dw/server/ui/assets/cpp-C7h46wYY.js +1 -0
- dw/server/ui/assets/csharp-BKxtCVv1.js +1 -0
- dw/server/ui/assets/csp-bTuwJoIa.js +1 -0
- dw/server/ui/assets/css-DIMkf-bt.js +3 -0
- dw/server/ui/assets/css.worker-B3ciXF_0.js +93 -0
- dw/server/ui/assets/cssMode-CPznxfY8.js +1 -0
- dw/server/ui/assets/cypher-CVaqCwHa.js +1 -0
- dw/server/ui/assets/dart-onAF5SnQ.js +1 -0
- dw/server/ui/assets/dockerfile-DZFCIeNp.js +1 -0
- dw/server/ui/assets/ecl-D05T4iGw.js +1 -0
- dw/server/ui/assets/editor-jjEx9u7D.css +1 -0
- dw/server/ui/assets/editor.api-CpWcotrd.js +847 -0
- dw/server/ui/assets/editor.worker-q-txB4vs.js +30 -0
- dw/server/ui/assets/elixir-6RTg0lbw.js +1 -0
- dw/server/ui/assets/flow9-C5_-GSwl.js +1 -0
- dw/server/ui/assets/freemarker2-CXtRM8N4.js +3 -0
- dw/server/ui/assets/fsharp-C8Ef5oNN.js +1 -0
- dw/server/ui/assets/go-C-y9NEjX.js +1 -0
- dw/server/ui/assets/graphql-fmXr3nnJ.js +1 -0
- dw/server/ui/assets/handlebars-N7x-6NMY.js +1 -0
- dw/server/ui/assets/hcl-CpzslTdj.js +1 -0
- dw/server/ui/assets/html-PhsdjHSr.js +1 -0
- dw/server/ui/assets/html.worker-C93Ht9o9.js +506 -0
- dw/server/ui/assets/htmlMode-Dgj0SEok.js +1 -0
- dw/server/ui/assets/index-3Vw6WAPW.css +1 -0
- dw/server/ui/assets/index-DgrYhQd9.js +43 -0
- dw/server/ui/assets/ini-sBoK_t0W.js +1 -0
- dw/server/ui/assets/java-BEtHBSE6.js +1 -0
- dw/server/ui/assets/javascript-BJqN9Qhv.js +1 -0
- dw/server/ui/assets/json.worker-B2V3pomh.js +62 -0
- dw/server/ui/assets/jsonMode-DbM4SWSv.js +7 -0
- dw/server/ui/assets/julia-Bri6UV-V.js +1 -0
- dw/server/ui/assets/kotlin-BOotOW0E.js +1 -0
- dw/server/ui/assets/less-B9JPFI3C.js +2 -0
- dw/server/ui/assets/lexon-CfSJPG6W.js +1 -0
- dw/server/ui/assets/liquid-BWr8lEc4.js +1 -0
- dw/server/ui/assets/lspLanguageFeatures-C1iGuDyZ.js +4 -0
- dw/server/ui/assets/lua-CsQS60Ue.js +1 -0
- dw/server/ui/assets/m3-D-oSqn_W.js +1 -0
- dw/server/ui/assets/markdown-Cimd5fb3.js +1 -0
- dw/server/ui/assets/mdx-DAdMi_0p.js +1 -0
- dw/server/ui/assets/mips-CIPQ_RoX.js +1 -0
- dw/server/ui/assets/monaco--ixms01u.css +1 -0
- dw/server/ui/assets/monaco-BGCeEqaw.js +56 -0
- dw/server/ui/assets/msdax-DauUninz.js +1 -0
- dw/server/ui/assets/mysql-SOo6toE5.js +1 -0
- dw/server/ui/assets/objective-c-FvmIjYaQ.js +1 -0
- dw/server/ui/assets/pascal-DrH0SRf2.js +1 -0
- dw/server/ui/assets/pascaligo-D-ptJ9y-.js +1 -0
- dw/server/ui/assets/perl-oz_6vUea.js +1 -0
- dw/server/ui/assets/pgsql-DTj74zXo.js +1 -0
- dw/server/ui/assets/php-nr791fC2.js +1 -0
- dw/server/ui/assets/pla-CopQ2nXW.js +1 -0
- dw/server/ui/assets/postiats-43DmfD33.js +1 -0
- dw/server/ui/assets/powerquery-D3hlyOfw.js +1 -0
- dw/server/ui/assets/powershell-DmHpPYUd.js +1 -0
- dw/server/ui/assets/protobuf-C531GsRP.js +2 -0
- dw/server/ui/assets/pug-Z5eAx3Zn.js +1 -0
- dw/server/ui/assets/python-Bcn70HdC.js +1 -0
- dw/server/ui/assets/qsharp-DkqhCAOL.js +1 -0
- dw/server/ui/assets/r-BwWrilGY.js +1 -0
- dw/server/ui/assets/razor-D1HmNnby.js +1 -0
- dw/server/ui/assets/redis-ClamHrr6.js +1 -0
- dw/server/ui/assets/redshift-DT7zqm-g.js +1 -0
- dw/server/ui/assets/restructuredtext-BYgofb2h.js +1 -0
- dw/server/ui/assets/ruby-DezsRK8O.js +1 -0
- dw/server/ui/assets/rust-DdL9SqIa.js +1 -0
- dw/server/ui/assets/sb-CcwsVR0C.js +1 -0
- dw/server/ui/assets/scala-DHpiXF5c.js +1 -0
- dw/server/ui/assets/scheme-BeGwcela.js +1 -0
- dw/server/ui/assets/scss-gp-XZpBa.js +3 -0
- dw/server/ui/assets/shell-CC2rA5mh.js +1 -0
- dw/server/ui/assets/solidity-BEEn4gHE.js +1 -0
- dw/server/ui/assets/sophia-CRfGWb83.js +1 -0
- dw/server/ui/assets/sparql-D_Lu-MrJ.js +1 -0
- dw/server/ui/assets/sql-NEE52Syq.js +1 -0
- dw/server/ui/assets/st-DbInun42.js +1 -0
- dw/server/ui/assets/swift-Bxkupp3x.js +1 -0
- dw/server/ui/assets/systemverilog-Bz4Y3fRF.js +1 -0
- dw/server/ui/assets/tcl-DISqw1ZD.js +1 -0
- dw/server/ui/assets/ts.worker-D7T1-Ig5.js +67738 -0
- dw/server/ui/assets/tsMode-D6u0XmOW.js +11 -0
- dw/server/ui/assets/twig-De2hgUGE.js +1 -0
- dw/server/ui/assets/typescript-BU6v-LMV.js +1 -0
- dw/server/ui/assets/typespec-B8J7ngcE.js +1 -0
- dw/server/ui/assets/vb-DV3o63ZY.js +1 -0
- dw/server/ui/assets/wgsl-DpFanUEy.js +298 -0
- dw/server/ui/assets/workers-Cn7cTUKr.js +1 -0
- dw/server/ui/assets/xml--0LP2Lwk.js +1 -0
- dw/server/ui/assets/yaml-mpBg9jnt.js +1 -0
- dw/server/ui/index.html +17 -0
- dw/server/updater.py +192 -0
- dw/settings.py +98 -0
- dw/shot_span_preflight.py +116 -0
- dw/shots.py +359 -0
- dw/slice_preflight.py +148 -0
- dw/step.py +187 -0
- dw/step_cache.py +442 -0
- dw/subfolders.py +107 -0
- dw/task_domains.py +307 -0
- dw/tasks/assess.py +826 -0
- dw/tasks/audio_transcription.py +88 -0
- dw/tasks/audio_utils.py +1862 -0
- dw/tasks/background_remover.py +43 -0
- dw/tasks/borders.py +113 -0
- dw/tasks/compose_text.py +74 -0
- dw/tasks/concat_videos.py +300 -0
- dw/tasks/depth_estimator.py +54 -0
- dw/tasks/diffusion_upscale.py +109 -0
- dw/tasks/dissolve_videos.py +342 -0
- dw/tasks/format_messages.py +24 -0
- dw/tasks/gather.py +173 -0
- dw/tasks/grade.py +97 -0
- dw/tasks/image_to_text.py +43 -0
- dw/tasks/image_utils.py +764 -0
- dw/tasks/interpolate_frames.py +252 -0
- dw/tasks/judge.py +68 -0
- dw/tasks/model_cache.py +55 -0
- dw/tasks/pair_audio.py +268 -0
- dw/tasks/qr_code.py +19 -0
- dw/tasks/restore_faces.py +175 -0
- dw/tasks/rife_model.py +192 -0
- dw/tasks/segment.py +121 -0
- dw/tasks/select.py +111 -0
- dw/tasks/speech_generation.py +228 -0
- dw/tasks/stabilize.py +129 -0
- dw/tasks/task.py +920 -0
- dw/tasks/tensor_image.py +57 -0
- dw/tasks/text_generation.py +169 -0
- dw/tasks/text_sections.py +80 -0
- dw/tasks/upscale.py +203 -0
- dw/tasks/video_utils.py +624 -0
- dw/tasks/zoe_depth.py +71 -0
- dw/teacache.py +381 -0
- dw/teacache_models.json +99 -0
- dw/test.py +29 -0
- dw/type_helpers.py +231 -0
- dw/validate.py +68 -0
- dw/variable_constraints.py +444 -0
- dw/variables.py +443 -0
- dw/video_extensions.py +141 -0
- dw/vram_estimate.py +116 -0
- dw/worker.py +764 -0
- dw/workflow.py +2007 -0
- dw/workflow_schema.json +1346 -0
- dw/workflow_sources.py +383 -0
- dw/workflows/h3_context_ir.json +57 -0
- dw/workflows/test.json +31 -0
- dw/workspace.py +730 -0
- dw_mcp/__init__.py +6 -0
- dw_mcp/__main__.py +133 -0
- dw_mcp/assets.py +336 -0
- dw_mcp/authoring.py +114 -0
- dw_mcp/catalog.py +360 -0
- dw_mcp/client.py +486 -0
- dw_mcp/diagnose.py +371 -0
- dw_mcp/exports.py +84 -0
- dw_mcp/guides.py +35 -0
- dw_mcp/media.py +638 -0
- dw_mcp/models.py +97 -0
- dw_mcp/prompts.py +104 -0
- dw_mcp/server.py +1343 -0
- dw_mcp/workspaces.py +212 -0
dw/docs/REMOTE.md
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# Using dw from another machine
|
|
2
|
+
|
|
3
|
+
`dw.serve` runs on the machine with the GPU; a browser and Claude Code on
|
|
4
|
+
your laptop use it over the network. Nothing is installed on the laptop.
|
|
5
|
+
|
|
6
|
+
## On the GPU box
|
|
7
|
+
|
|
8
|
+
1. Install as usual: `git clone …`, `bash ./install.sh`.
|
|
9
|
+
2. Make a token: `openssl rand -hex 32`. This one string is the only thing
|
|
10
|
+
between "on your network" and "can run workflows on your GPU" - keep it
|
|
11
|
+
long and random.
|
|
12
|
+
3. Run it bound to the network, with the token, with MCP:
|
|
13
|
+
|
|
14
|
+
DW_API_TOKEN=<token> dw-serve --host 0.0.0.0 --mcp
|
|
15
|
+
|
|
16
|
+
`--mcp` is refused outright (exit 2) on a non-loopback bind with no
|
|
17
|
+
token: an MCP endpoint can author and run workflows and has no
|
|
18
|
+
token-entry page in front of it the way the web UI does.
|
|
19
|
+
|
|
20
|
+
To keep it running across logouts and reboots, install the systemd unit
|
|
21
|
+
in [contrib/systemd](../contrib/systemd/README.md) instead.
|
|
22
|
+
4. Open port 8765 in the box's firewall for your LAN only (for example
|
|
23
|
+
`sudo ufw allow from 192.168.1.0/24 to any port 8765`).
|
|
24
|
+
|
|
25
|
+
Never pass `--trust-workflows` on a server other machines can reach: it
|
|
26
|
+
lets any workflow - including one an agent authored - execute arbitrary
|
|
27
|
+
Python. See [SECURITY.md](SECURITY.md#trust-model).
|
|
28
|
+
|
|
29
|
+
Check it from the laptop:
|
|
30
|
+
|
|
31
|
+
curl http://<box>:8765/api/health -H "Authorization: Bearer <token>"
|
|
32
|
+
{"status":"ok","version":"…","hostname":"gpu-box","device":"cuda","mcp":true,…}
|
|
33
|
+
|
|
34
|
+
`hostname` and `device` are there so you can tell which machine answered.
|
|
35
|
+
|
|
36
|
+
If an agent on this box will ever export a job or list gallery/asset URLs to
|
|
37
|
+
hand to a person who isn't at a terminal on the box itself, set
|
|
38
|
+
`DW_API_TOKEN` and also set `DW_PUBLIC_URL` to this server's origin (for
|
|
39
|
+
example `https://dw.example.com`, or `http://<box>:8765` with no proxy):
|
|
40
|
+
|
|
41
|
+
DW_API_TOKEN=<token> DW_PUBLIC_URL=https://dw.example.com dw-serve --host 0.0.0.0 --mcp
|
|
42
|
+
|
|
43
|
+
Without it, `export_job`, `list_gallery` and the upload/asset routes only
|
|
44
|
+
return paths relative to this server (`/exports/job-1.zip`) - correct for a
|
|
45
|
+
browser already pointed at the box, useless handed to someone who isn't.
|
|
46
|
+
With `DW_PUBLIC_URL` set (or the equivalent `public_url` setting), those
|
|
47
|
+
responses add an `absolute_url` / `absolute_zip_url` built from it; nothing
|
|
48
|
+
guesses this from request headers, so an unconfigured server omits the
|
|
49
|
+
field rather than composing a wrong origin.
|
|
50
|
+
|
|
51
|
+
## Browser
|
|
52
|
+
|
|
53
|
+
Open `http://<box>:8765`. Click the key icon next to the theme toggle,
|
|
54
|
+
paste the token once; it is kept in the browser's `localStorage` and sent
|
|
55
|
+
with every request.
|
|
56
|
+
|
|
57
|
+
## Claude Code, no local install
|
|
58
|
+
|
|
59
|
+
claude mcp add --transport http dw http://<box>:8765/mcp \
|
|
60
|
+
--header "Authorization: Bearer <token>"
|
|
61
|
+
|
|
62
|
+
Start a new Claude Code session; `/mcp` should list `dw` as connected.
|
|
63
|
+
Pick the scope with `-s user` to have it in every project.
|
|
64
|
+
|
|
65
|
+
`http://<box>:8765/mcp` and `http://<box>:8765/mcp/` are both answered, and
|
|
66
|
+
both require the token in an `Authorization: Bearer` header - the
|
|
67
|
+
`?token=...` allowance a few browser GET routes have does not extend to
|
|
68
|
+
`/mcp`.
|
|
69
|
+
|
|
70
|
+
Two things differ from the local stdio setup:
|
|
71
|
+
|
|
72
|
+
- `download_output` writes on the GPU box (where the MCP server runs), not
|
|
73
|
+
on your laptop, so an omitted destination is refused rather than dropped
|
|
74
|
+
loose in the workspace root - nothing on your laptop would find or
|
|
75
|
+
delete it there (#353). Pass an explicit destination inside the
|
|
76
|
+
workspace to save one anyway, or use `get_output_image` /
|
|
77
|
+
`get_output_text` to see a result, or open
|
|
78
|
+
`http://<box>:8765/outputs/<name>` in the browser.
|
|
79
|
+
- The connection is a plain HTTP call per tool invocation; there is no
|
|
80
|
+
subprocess to restart.
|
|
81
|
+
- `use_workspace`/`create_workspace` pin *this box's* one MCP client, shared
|
|
82
|
+
by every agent connected to it - this server is single-user, so there is
|
|
83
|
+
no per-session isolation. If you and another agent are both against the
|
|
84
|
+
same box, either one's `use_workspace` call can move what the other reads
|
|
85
|
+
and writes next (#298). Pass `workspace=` on each call that takes it
|
|
86
|
+
(`validate_workflow`, `run_workflow`, most read/media tools) instead of
|
|
87
|
+
relying on the session pin when that matters.
|
|
88
|
+
|
|
89
|
+
## Claude Code with a local install (stdio)
|
|
90
|
+
|
|
91
|
+
If the laptop also has `dw` installed, the stdio server works against a
|
|
92
|
+
remote box too:
|
|
93
|
+
|
|
94
|
+
claude mcp add dw -- /path/to/venv/bin/dw-mcp \
|
|
95
|
+
--url http://<box>:8765 --token <token>
|
|
96
|
+
|
|
97
|
+
`dw-mcp` refuses to start (exit 2) against a non-loopback URL without a
|
|
98
|
+
token, and makes one `GET /api/health` at startup so a wrong URL or token
|
|
99
|
+
is reported once, with a message, instead of as a 401 on every tool call.
|
|
100
|
+
Against a remote URL a failed probe is fatal - it is a misconfiguration you
|
|
101
|
+
have to fix. Against a loopback URL it only warns and serves anyway, since
|
|
102
|
+
there it usually means "dw.serve is not up yet", which the next tool call
|
|
103
|
+
reports for itself. `--no-probe` skips the check entirely.
|
|
104
|
+
|
|
105
|
+
## Beyond your LAN
|
|
106
|
+
|
|
107
|
+
Everything above is plaintext HTTP: the token and every prompt and result
|
|
108
|
+
are readable by anything on the network path. That is acceptable on a
|
|
109
|
+
network you control and nowhere else. **Do not port-forward 8765 on your
|
|
110
|
+
router.** Two ways to reach the box from outside:
|
|
111
|
+
|
|
112
|
+
**Tailscale (or another WireGuard overlay).** Install it on the box and the
|
|
113
|
+
laptop; use the box's Tailscale IP or MagicDNS name in every URL above.
|
|
114
|
+
Traffic is encrypted end to end, no certificates to manage, and `dw-serve`
|
|
115
|
+
can stay bound to the Tailscale interface (`--host 100.x.y.z`) rather than
|
|
116
|
+
`0.0.0.0`. This is the recommended option.
|
|
117
|
+
|
|
118
|
+
**A TLS-terminating reverse proxy.** Bind `dw-serve` back to loopback and
|
|
119
|
+
put Caddy in front of it with a real hostname:
|
|
120
|
+
|
|
121
|
+
# /etc/caddy/Caddyfile
|
|
122
|
+
dw.example.com {
|
|
123
|
+
reverse_proxy 127.0.0.1:8765
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
Caddy obtains and renews a certificate automatically. Use
|
|
127
|
+
`https://dw.example.com` (no port) in every URL above. `dw-serve`'s Origin
|
|
128
|
+
and Host checks work unchanged behind the proxy because Caddy forwards the
|
|
129
|
+
`Host` header as-is. nginx works the same way with `proxy_pass` and
|
|
130
|
+
`proxy_set_header Host $host;` plus your own certificate.
|
|
131
|
+
|
|
132
|
+
The token is still the only authentication in either setup; a proxy or a
|
|
133
|
+
VPN protects the transport, not the door.
|
|
134
|
+
|
|
135
|
+
## What is not here
|
|
136
|
+
|
|
137
|
+
- TLS inside `dw-serve` itself: a reverse proxy does it better.
|
|
138
|
+
- More than one token, or users: one shared secret, deliberately
|
|
139
|
+
([SERVER.md](SERVER.md#authentication)).
|
|
140
|
+
- Docker: `install.sh` on the box is the supported install.
|
dw/docs/REPL_COMMANDS.md
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# REPL Command Reference
|
|
2
|
+
|
|
3
|
+
The REPL uses hierarchical commands grouped by function. Use `?` after any command group to see its subcommands.
|
|
4
|
+
|
|
5
|
+
## Starting the REPL
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
python -m dw.repl
|
|
9
|
+
python -m dw.repl -l DEBUG # with debug logging
|
|
10
|
+
python -m dw.repl --trust-workflows # only for workflow files you trust - see docs/SECURITY.md
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
A workflow JSON file can execute arbitrary Python (`pre_load_modules`,
|
|
14
|
+
dotted `*_type`/`config_type` values). `--trust-workflows` is off by
|
|
15
|
+
default for the whole session; `workflow load`/`workflow run` on a
|
|
16
|
+
workflow that needs it without the flag fails with a clear error. See
|
|
17
|
+
[Trust model](SECURITY.md#trust-model).
|
|
18
|
+
|
|
19
|
+
## Commands
|
|
20
|
+
|
|
21
|
+
### workflow — Manage and run workflows
|
|
22
|
+
|
|
23
|
+
```text
|
|
24
|
+
workflow list List workflows under the workflow directory
|
|
25
|
+
workflow load <file> Load a workflow from JSON file
|
|
26
|
+
workflow reload Reload current workflow from disk
|
|
27
|
+
workflow status Show current workflow information
|
|
28
|
+
workflow run Execute the currently loaded workflow
|
|
29
|
+
workflow run <n>=<v> ... Set arguments and run in one line
|
|
30
|
+
workflow run ask <arg> Prompt for one argument's value, then run
|
|
31
|
+
workflow restart Restart the worker process (clears GPU cache)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
`workflow load` resolves names against the workflow directory (`./workflows`
|
|
35
|
+
by default), including subfolders - `workflow load models/flux-dev` loads
|
|
36
|
+
`workflows/models/flux-dev.json`. `workflow list` shows the available names.
|
|
37
|
+
|
|
38
|
+
`workflow run ask <arg>` prompts you interactively for `<arg>`'s value (the
|
|
39
|
+
value is not saved to shell/readline history) before running — a shortcut
|
|
40
|
+
for `arg set` followed by `workflow run`.
|
|
41
|
+
|
|
42
|
+
### arg — Set workflow variables
|
|
43
|
+
|
|
44
|
+
```text
|
|
45
|
+
arg show Show available variables and current values
|
|
46
|
+
arg set <name>=<value> Set a variable value
|
|
47
|
+
arg clear [<name>] Clear one variable, or all of them
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
### memory — Monitor GPU memory
|
|
51
|
+
|
|
52
|
+
```text
|
|
53
|
+
memory show Show current GPU memory usage
|
|
54
|
+
memory clear Clear GPU memory, cached models, and cached step results
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Step caching keeps each step's last `Result` in RAM for the life of the
|
|
58
|
+
worker process so a rerun with unchanged inputs can skip straight to it -
|
|
59
|
+
this trades some memory for speed, and holding it works against the
|
|
60
|
+
normal end-of-step release that frees a result once no later step needs
|
|
61
|
+
it. The cache is bounded to the 50 most recently used steps, and evicts
|
|
62
|
+
the least recently used one beyond that; `memory clear` is the escape
|
|
63
|
+
hatch: it drops every cached step result along with the GPU/model caches.
|
|
64
|
+
|
|
65
|
+
Caching is not REPL-only - it applies to any `Workflow.run` in the
|
|
66
|
+
process, including a job the server runs. Re-running a fixed-seed workflow
|
|
67
|
+
whose inputs did not change therefore completes almost instantly and
|
|
68
|
+
produces no new gallery entry, because the previous run's files are
|
|
69
|
+
reused. A cache entry is discarded if any of the files it names has been
|
|
70
|
+
deleted since, so a deleted output is regenerated rather than reported
|
|
71
|
+
again from cache. A workflow that names no `seed` draws a fresh one every
|
|
72
|
+
run, so nothing it does can hit - the cache is skipped entirely for it. A
|
|
73
|
+
reused step's manifest entry and its `step_end` event carry `"reused":
|
|
74
|
+
true`, which is how the server keeps a file credited to the job that
|
|
75
|
+
actually wrote it rather than to every later run that reused it. Cache
|
|
76
|
+
entries are keyed by the workflow's `id`, so renaming or copying a
|
|
77
|
+
workflow to a new `id` is a full cache miss.
|
|
78
|
+
|
|
79
|
+
### config — REPL settings
|
|
80
|
+
|
|
81
|
+
```text
|
|
82
|
+
config show Show all settings
|
|
83
|
+
config set output_dir=<path> Change output directory
|
|
84
|
+
config set log_level=DEBUG Change log level
|
|
85
|
+
config set workflow_dir=<path> Change default workflow directory
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### General
|
|
89
|
+
|
|
90
|
+
```text
|
|
91
|
+
help / ? Show all commands
|
|
92
|
+
exit / quit Exit the REPL
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
`run`, `load`, and `set` also work at the top level as shortcuts for
|
|
96
|
+
`workflow run`, `workflow load`, and `arg set`.
|
|
97
|
+
|
|
98
|
+
Ctrl+C during a run cancels it cooperatively and keeps the worker's models
|
|
99
|
+
cached; a second Ctrl+C stops the worker process itself.
|
|
100
|
+
|
|
101
|
+
## Typical Session
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
dw> workflow load models/flux-dev
|
|
105
|
+
dw> arg show
|
|
106
|
+
dw> arg set prompt="a majestic mountain landscape"
|
|
107
|
+
dw> workflow run
|
|
108
|
+
dw> arg set prompt="a serene beach at sunset"
|
|
109
|
+
dw> workflow run
|
|
110
|
+
dw> memory show
|
|
111
|
+
dw> exit
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## Quick Reference
|
|
115
|
+
|
|
116
|
+
```text
|
|
117
|
+
workflow ── list | load <file> | reload | status | run [<n>=<v> ... | ask <arg>] | restart
|
|
118
|
+
arg ── show | set <name>=<value> | clear [<name>]
|
|
119
|
+
memory ── show | clear
|
|
120
|
+
config ── show | set <name>=<value>
|
|
121
|
+
```
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# REPL Worker Guide
|
|
2
|
+
|
|
3
|
+
## Overview
|
|
4
|
+
|
|
5
|
+
The REPL uses a persistent worker subprocess to keep models loaded in GPU memory across runs. After the first run loads a model, subsequent runs skip the loading step entirely.
|
|
6
|
+
|
|
7
|
+
The worker uses the `spawn` multiprocessing start method, required for CUDA and MPS compatibility. This is configured automatically.
|
|
8
|
+
|
|
9
|
+
## Usage
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
python -m dw.repl
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
```text
|
|
16
|
+
dw> workflow load models/flux-dev
|
|
17
|
+
dw> arg set prompt="a cat wearing a hat"
|
|
18
|
+
dw> workflow run # first run — loads model (~30-60s)
|
|
19
|
+
dw> arg set prompt="a dog in a park"
|
|
20
|
+
dw> workflow run # instant start — model cached
|
|
21
|
+
dw> memory show # check GPU memory
|
|
22
|
+
dw> memory clear # free GPU memory
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## How It Works
|
|
26
|
+
|
|
27
|
+
The REPL process handles user input and validation. The worker process handles model loading, caching, and inference. They communicate via multiprocessing queues.
|
|
28
|
+
|
|
29
|
+
- **First `workflow run`**: Worker starts and loads the model
|
|
30
|
+
- **Subsequent runs**: Worker reuses cached models
|
|
31
|
+
- **Workflow file edited**: Worker detects the change (SHA256 hash) and reloads
|
|
32
|
+
- **`workflow load` (different file)**: The worker switches in place - it frees the old workflow's models before loading the new one
|
|
33
|
+
- **`workflow restart`**: Worker shuts down immediately; a fresh one starts on the next run
|
|
34
|
+
- **`memory clear`**: Frees GPU memory, models reload on next run
|
|
35
|
+
- **`exit`**: Worker shuts down gracefully
|
|
36
|
+
|
|
37
|
+
## Memory Management
|
|
38
|
+
|
|
39
|
+
The worker cleans up automatically between runs (garbage collection + GPU cache clearing). If memory grows unexpectedly, use `memory show` to check and `memory clear` to reset.
|
|
40
|
+
|
|
41
|
+
## Troubleshooting
|
|
42
|
+
|
|
43
|
+
**Worker crashes**: The REPL detects it and starts a fresh worker on the next `workflow run`. Error messages are shown in the REPL.
|
|
44
|
+
|
|
45
|
+
**Execution errors**: The worker stays alive (models cached) so you can fix the issue and re-run immediately.
|
|
46
|
+
|
|
47
|
+
**Long runs**: There is no execution timeout - a run waits as long as the worker is alive (liveness is polled every second, so a crashed worker is noticed immediately). Ctrl+C cancels the run in place, keeping models cached; a second Ctrl+C stops the worker.
|
|
48
|
+
|
|
49
|
+
**GPU out of memory**: Use `memory clear`, reduce model size, or check for other processes using the GPU.
|
|
50
|
+
|
|
51
|
+
**"Cannot re-initialize CUDA in forked subprocess"**: Use `python -m dw.repl` to start the REPL — don't import torch before the REPL sets the spawn method.
|
dw/docs/SECURITY.md
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
# Security
|
|
2
|
+
|
|
3
|
+
## Overview
|
|
4
|
+
|
|
5
|
+
diffusers-workflow validates all file paths, user inputs, and URLs to protect against path traversal, injection, and resource exhaustion. A command-argument sanitizer (`sanitize_command_args()`) is available for any future subprocess use, though `dw/` currently invokes no subprocess/shell commands.
|
|
6
|
+
|
|
7
|
+
## Security Module (`dw/security.py`)
|
|
8
|
+
|
|
9
|
+
### Path Validation
|
|
10
|
+
|
|
11
|
+
- `validate_path()` — Blocks `../` (anywhere in the path), `~/` (or `~\`), and paths rooted at `/dev/`, `/proc/`, `/sys/`. Rejects null bytes and overlong paths (> 4096 chars). Resolves to an absolute, realpath'd path. If `base_dir` is given, raises `PathTraversalError` when the resolved path falls outside it.
|
|
12
|
+
- `validate_workflow_path()` — `validate_path()` plus a required `.json` extension (via `validate_file_extension()`)
|
|
13
|
+
- `validate_output_path()` — `validate_path()` with `allow_create=True`, for directories/files that don't need to exist yet
|
|
14
|
+
- `validate_file_extension()` — Checks a path's extension against an allowed set (used internally by `validate_workflow_path()` and by `arguments.py` for media files)
|
|
15
|
+
|
|
16
|
+
### Input Validation
|
|
17
|
+
|
|
18
|
+
- `validate_variable_name()` — Alphanumeric, underscore, hyphen only (pattern: `^[a-zA-Z_][a-zA-Z0-9_-]*$`), max 100 chars
|
|
19
|
+
- `validate_string_input()` — Max length, no null bytes, no control characters other than tab/newline/CR. Every caller that checks a caller-supplied variable value (`dw/variables.py`, `dw/run.py`, the REPL) passes `MAX_VARIABLE_VALUE_LENGTH`, 20,000 characters; file names and paths pass their own, shorter caps, so the function's bare default of 1000 is not the limit anything is held to. A variable's *default*, written in the definition, is not capped separately: the author controls the file, and the whole file is capped at 50MB
|
|
20
|
+
- `validate_json_size()` — Limits JSON files to 50MB
|
|
21
|
+
- `validate_url()` — Scheme must be `http` or `https`; must have a non-empty domain (`netloc`); may not contain a backslash. `urllib.parse` and the HTTP client disagree on which host `http://169.254.169.254\@example.com/` names, so the host the check approved need not be the one dialed; a `\` that belongs in a path is written `%5C`
|
|
22
|
+
- `validate_constant_name()` — Guards `constant:` references before import: dotted-name pattern only, module must already be importable, and anything callable is refused
|
|
23
|
+
- `safe_join_path()` — Joins path components after rejecting any that contain `..`, `/`, or `\\`. Defined in `security.py` but not currently called elsewhere in `dw/`.
|
|
24
|
+
|
|
25
|
+
### Command Sanitization
|
|
26
|
+
|
|
27
|
+
- `sanitize_command_args()` — Rejects arguments containing shell metacharacters ( `` ` `` `$` `|` `&` `;` `>` `<` and newline/CR). It does **not** call `shlex.quote()` — with `shell=False`, argument list separation is handled safely by Python/the OS, so this function is a defense-in-depth check, not an escaping step.
|
|
28
|
+
- As of this writing, `dw/` does not invoke `subprocess` anywhere — the REPL's worker process (`dw/repl_worker.py`, `dw/worker.py`) is a `multiprocessing.Process` communicating over `multiprocessing.Queue`, not a shelled-out command. `sanitize_command_args()` is exercised by `tests/test_security.py` but is otherwise unused; it exists for any future code path that shells out.
|
|
29
|
+
|
|
30
|
+
## Trust model
|
|
31
|
+
|
|
32
|
+
**A workflow JSON file can execute arbitrary Python.** This is a deliberate
|
|
33
|
+
design choice, in the same spirit as ComfyUI custom nodes - the engine's
|
|
34
|
+
dynamic-import machinery is what lets a workflow name any diffusers
|
|
35
|
+
pipeline, scheduler, or quantization backend without dw shipping a bespoke
|
|
36
|
+
adapter for each one. But the same machinery means loading a workflow file
|
|
37
|
+
is not a passive data-load. Three things in a workflow JSON run
|
|
38
|
+
`importlib.import_module()` on a name the file supplies, which executes
|
|
39
|
+
that module's top-level code:
|
|
40
|
+
|
|
41
|
+
- **`pre_load_modules`** (`dw/pipeline_processors/pipeline.py`) - a list of
|
|
42
|
+
module names imported before the pipeline loads, for their import-time
|
|
43
|
+
registration side effects (`sdnq` registering its quantization method
|
|
44
|
+
with diffusers, for instance)
|
|
45
|
+
- **A dotted `*_type`/`*_dtype`/`dtype`/`config_type` value**
|
|
46
|
+
(`dw/type_helpers.py`, reached from `dw/arguments.py` and
|
|
47
|
+
`dw/pipeline_processors/config_objects.py`) - `"sdnq.SDNQConfig"` imports
|
|
48
|
+
`sdnq` and reads `SDNQConfig` off it; nothing stops the module part from
|
|
49
|
+
naming something with no legitimate reason to appear in a workflow
|
|
50
|
+
- **A `constant:`-prefixed reference** (`dw/type_helpers.py`'s
|
|
51
|
+
`load_constant_from_name`, reached from `dw/arguments.py`) - imports the
|
|
52
|
+
module the constant is declared in the same way, before reading the
|
|
53
|
+
attribute off it. `fetch_constant` refuses anything callable it finds,
|
|
54
|
+
but the import itself has already run by that point
|
|
55
|
+
|
|
56
|
+
**Treat an untrusted workflow file exactly like an untrusted Python
|
|
57
|
+
script.** Don't run one from a source you would not run a `.py` file from -
|
|
58
|
+
a random download, a link in an issue, an LLM-authored file you have not
|
|
59
|
+
read.
|
|
60
|
+
|
|
61
|
+
### `--trust-workflows`
|
|
62
|
+
|
|
63
|
+
`dw-run`, `dw-serve`, and `dw-repl` all take a `--trust-workflows` flag,
|
|
64
|
+
**off by default**. Untrusted (the default), `pre_load_modules` and any
|
|
65
|
+
dotted `*_type`/`*_dtype`/`dtype`/`config_type` value are refused unless
|
|
66
|
+
they resolve under a top-level package the tool already depends on for
|
|
67
|
+
exactly this purpose - the framework packages (`diffusers`, `torch`,
|
|
68
|
+
`torchvision`, `transformers`, `accelerate`, `peft`) and the quantization
|
|
69
|
+
backends `pyproject.toml` declares for `config_objects.py`'s dynamic
|
|
70
|
+
loading (`sdnq`, `torchao`, `optimum` for optimum-quanto, `gguf`,
|
|
71
|
+
`bitsandbytes`), plus `dw` itself (a workflow's `component_type` can name a
|
|
72
|
+
pipeline under `dw.community_pipelines`, which ships in this repo, not a
|
|
73
|
+
third party one). The refusal names exactly what triggered it and points
|
|
74
|
+
back at `--trust-workflows`. The bundled examples under `workflows/` all
|
|
75
|
+
stay inside this set and load untrusted; a
|
|
76
|
+
workflow that needs to reach outside it - a community pipeline module from
|
|
77
|
+
somewhere else, a custom scheduler package - needs `--trust-workflows`.
|
|
78
|
+
|
|
79
|
+
The package is not the whole check, because an allowed package holds
|
|
80
|
+
things other than classes and re-exports modules outside itself. Untrusted,
|
|
81
|
+
two more rules apply (`dw/type_helpers.py`):
|
|
82
|
+
|
|
83
|
+
- **A type reference must resolve to a class.** A `*_type`/`config_type`
|
|
84
|
+
value is constructed with the workflow's own arguments, so
|
|
85
|
+
`"torch.hub.load"` - in `torch`, and a function that fetches and runs a
|
|
86
|
+
GitHub repo's code - is refused as "not a class", as is a module or any
|
|
87
|
+
other object. Under a `dtype` or `*_dtype` key a `torch.dtype`
|
|
88
|
+
(`"torch.bfloat16"`) is accepted too, since that is data rather than
|
|
89
|
+
something called. A bare name (`"FluxPipeline"`) resolves against
|
|
90
|
+
`diffusers` and is held to the same rule. `validate_workflow` reports the
|
|
91
|
+
refusal at the key's path, for every key the run loads as a type
|
|
92
|
+
(`from_pretrained_arguments.torch_dtype` as much as `config_type`).
|
|
93
|
+
- **A `constant:` walk stays inside the package.** A dotted `constant:`
|
|
94
|
+
reference is gated like a type (the module it names is imported before
|
|
95
|
+
`fetch_constant` gets to refuse a callable, so the import itself is what
|
|
96
|
+
the gate has to stop), and then every step of the walk is checked: no
|
|
97
|
+
segment may start with `_`, checked before anything imports, and no module
|
|
98
|
+
the walk passes through may sit outside the allowed packages -
|
|
99
|
+
`constant:torch.os.environ` starts in `torch` and ends in the server's
|
|
100
|
+
environment, and is refused at `torch.os`. Reading a field off a value
|
|
101
|
+
declared in an allowed module still works
|
|
102
|
+
(`...ltx2.utils.GEMMA4_PROMPT_ENHANCEMENT_CONFIG.max_new_tokens`). A bare
|
|
103
|
+
name (`constant:SOME_NAME`) reads from `diffusers`. `validate_workflow`
|
|
104
|
+
resolves every literal `constant:` in a step the way the run does and
|
|
105
|
+
reports a refusal at its path (a variable's default at `variables.<name>`),
|
|
106
|
+
so nothing is queued to find out.
|
|
107
|
+
|
|
108
|
+
Every `*_type` and `constant:` in the bundled catalog satisfies both rules,
|
|
109
|
+
pinned by `tests/test_workflow_trust.py`'s catalog sweep. `--trust-workflows`
|
|
110
|
+
lifts both, as it lifts the package check.
|
|
111
|
+
|
|
112
|
+
Two `from_pretrained_arguments` keys are refused untrusted as well, for
|
|
113
|
+
every component and pipeline: `trust_remote_code` (runs the model repo's
|
|
114
|
+
own modeling code) and `custom_pipeline` (fetches and imports a pipeline
|
|
115
|
+
module from the Hub or a local path). Neither goes through an
|
|
116
|
+
`importlib` call of ours, so the dotted-name gate alone would leave
|
|
117
|
+
diffusers' remote-code paths open. No bundled catalog entry sets either
|
|
118
|
+
(`tests/test_catalog_structure.py` refuses one that does); a workflow that
|
|
119
|
+
needs them needs `--trust-workflows`.
|
|
120
|
+
|
|
121
|
+
### Where a workflow may read and reach (`dw/locations.py`)
|
|
122
|
+
|
|
123
|
+
Code execution is not the only thing a workflow file chooses. Its arguments
|
|
124
|
+
choose *locations* - which image to open, which URL to fetch, which
|
|
125
|
+
directory a glob expands over - and until 2026-09-13 each loader trusted the
|
|
126
|
+
one it was handed. A workflow could name `/usr/share/pixmaps/debian-logo.png`
|
|
127
|
+
as its `image` and get it decoded, glob `/usr/share/pixmaps/*.png` and get
|
|
128
|
+
every match republished verbatim as an output, or point an `image` at
|
|
129
|
+
`http://127.0.0.1:8765` and have the server fetch its own loopback.
|
|
130
|
+
`remote_text_encoder.url` was the worst of them: the request carries this
|
|
131
|
+
machine's HuggingFace token.
|
|
132
|
+
|
|
133
|
+
One policy now answers all of it, untrusted:
|
|
134
|
+
|
|
135
|
+
- **A path** must resolve inside a root this installation already works in -
|
|
136
|
+
the workflow file's own directory, the asset libraries on the search path,
|
|
137
|
+
the output root. `validate_path` already refuses `..`, so in practice this
|
|
138
|
+
closes the absolute path that pointed somewhere else entirely; a relative
|
|
139
|
+
one that climbs out is refused on its `..` segments, at validation time as
|
|
140
|
+
well as at the loader, so both spellings are answered at the same moment
|
|
141
|
+
rather than one of them three seconds into a queued job (#124). The remedy
|
|
142
|
+
for a file outside is to put it in the asset library and use an `asset:`
|
|
143
|
+
reference. Containment is checked **before** existence, so the refusal
|
|
144
|
+
cannot be used as a file-existence oracle.
|
|
145
|
+
- **A glob** is contained the same way, on the fixed directory its pattern
|
|
146
|
+
starts from, and every match is re-checked on its real path so a symlink
|
|
147
|
+
cannot carry the expansion out.
|
|
148
|
+
- **An `http(s)` URL** must not resolve to an address inside the deployment -
|
|
149
|
+
loopback, link-local (`169.254.0.0/16`, the cloud metadata address),
|
|
150
|
+
private ranges, and anything else that is not globally routable
|
|
151
|
+
(`is_global`). Checked after DNS resolution, not on the literal string.
|
|
152
|
+
That last rule covers `100.64.0.0/10`, the shared address space, which is
|
|
153
|
+
**Tailscale's tailnet range** (and Alibaba's metadata address): a workflow
|
|
154
|
+
that fetches media from another machine on your tailnet is refused unless
|
|
155
|
+
it runs under `--trust-workflows`.
|
|
156
|
+
- **Every redirect is re-checked.** A media fetch (`safe_get`) never lets the
|
|
157
|
+
HTTP client follow a redirect on its own: it follows at most 5 hops
|
|
158
|
+
(`MAX_MEDIA_REDIRECTS`), and each `Location` passes the same scheme and
|
|
159
|
+
host policy before it is dialed. A public URL answering `302` to
|
|
160
|
+
`http://127.0.0.1:8765/api/server` is refused with the target named, and
|
|
161
|
+
nothing is fetched from it. Images are decoded from the fetched bytes and
|
|
162
|
+
still go through diffusers' `load_image`, so EXIF orientation and RGB
|
|
163
|
+
conversion are unchanged.
|
|
164
|
+
- **`remote_text_encoder.url`** is https-only, and the HuggingFace token is
|
|
165
|
+
attached only for `huggingface.co`, `huggingface.cloud` and `hf.space`. An
|
|
166
|
+
endpoint elsewhere is still reachable; it just does not get the credential.
|
|
167
|
+
- **`model_name`** must be a Hub repo id or a path inside a root - the same
|
|
168
|
+
shape check `download_model` has always applied to `repo_id`. A URL is
|
|
169
|
+
neither, and is refused as such rather than resolving into the workflow's
|
|
170
|
+
own directory as a path-shaped name (#117).
|
|
171
|
+
|
|
172
|
+
Enforced twice: `location_errors` runs inside `validation_errors`, so
|
|
173
|
+
`validate_workflow` refuses before a model load is spent on the run, and the
|
|
174
|
+
loaders call the same functions for a location that arrives through a
|
|
175
|
+
variable or a previous result. All of it yields to `--trust-workflows`.
|
|
176
|
+
|
|
177
|
+
`GET /api/server` reports `trust_workflows`, so a client can read the posture
|
|
178
|
+
it is running against rather than infer it.
|
|
179
|
+
|
|
180
|
+
`--trust-workflows` is a blanket, process-wide choice - it is not scoped
|
|
181
|
+
per-workflow or per-request. A `dw-serve` instance that accepts jobs from
|
|
182
|
+
anything other than yourself (including an MCP client - see below) should
|
|
183
|
+
be run without it.
|
|
184
|
+
|
|
185
|
+
### MCP-authored workflows (M3)
|
|
186
|
+
|
|
187
|
+
The MCP server's `save_workflow` and `run_workflow` tools let an LLM write
|
|
188
|
+
and then execute a workflow through `dw.serve` - `save_workflow` writes a
|
|
189
|
+
JSON file into the workflow directory, `run_workflow` queues it (or an
|
|
190
|
+
inline definition) as a job. Nothing in `dw_mcp/` inspects what a workflow
|
|
191
|
+
it saves or runs actually contains. What protects a server used this way
|
|
192
|
+
is exactly the mechanism above: `dw-serve`'s own `--trust-workflows`
|
|
193
|
+
default is untrusted, so an MCP-authored or MCP-submitted workflow gets
|
|
194
|
+
the same code-execution gate a workflow from any other untrusted source
|
|
195
|
+
does, with no code change needed in `dw_mcp` itself. Running `dw-serve
|
|
196
|
+
--trust-workflows` removes that gate for every job the server accepts,
|
|
197
|
+
MCP-submitted or not - see the blanket-choice note just above.
|
|
198
|
+
|
|
199
|
+
## Integration Points
|
|
200
|
+
|
|
201
|
+
| Entry Point | What's Validated |
|
|
202
|
+
|-------------|-----------------|
|
|
203
|
+
| `workflow.py` | Workflow file paths, JSON size, output directories, sub-workflow paths |
|
|
204
|
+
| `run.py`, `validate.py` | CLI arguments, variable names and values |
|
|
205
|
+
| `repl.py`, `repl_commands.py` | Interactive command arguments — paths, workflow paths, output paths, variable names/values |
|
|
206
|
+
| `arguments.py` | Image/video/audio URLs, file paths, file extensions (`validate_media_location`, `fetch_image`, `fetch_video`) |
|
|
207
|
+
| `tasks/gather.py` | URLs passed to the `gather` task |
|
|
208
|
+
| `result.py` | Output directories and filenames |
|
|
209
|
+
| `server/app.py`, `server/jobs.py` | Every HTTP-supplied path — workflow files confined to the workflow directory, gallery files to the output directory, inline-workflow `base_dir`, `Origin`-header guard on every request |
|
|
210
|
+
|
|
211
|
+
## MCP Server
|
|
212
|
+
|
|
213
|
+
`dw_mcp/` introduces no new file access and no authentication of its own. It
|
|
214
|
+
is an HTTP client of a running `dw.serve`: every path a tool touches (a
|
|
215
|
+
workflow name, a gallery file, a job id) is sent to the REST API as-is and
|
|
216
|
+
validated there, exactly as it would be for a browser request from the web
|
|
217
|
+
UI. A remote `dw.serve` is allowed only with a token — see
|
|
218
|
+
[MCP Server](MCP.md#security) and [REMOTE.md](REMOTE.md).
|
|
219
|
+
|
|
220
|
+
The two exceptions are `download_output`, which writes a local file for the
|
|
221
|
+
MCP client, and `upload_asset(file_path=...)`, which reads one — neither goes
|
|
222
|
+
through the API for that half of its work. Both turn on whose machine "local"
|
|
223
|
+
is. Over a **stdio `dw-mcp`** it is the user's own, so both act for the local
|
|
224
|
+
user the way a shell redirect would: `download_output` writes anywhere the
|
|
225
|
+
process may (a full path, a directory, or the working directory by default,
|
|
226
|
+
`~` expanded) and `upload_asset` reads anything it may. Over **`dw.serve
|
|
227
|
+
--mcp`** it is the operator's box, which the caller never chose, so both are
|
|
228
|
+
confined there: `download_output`'s `destination` to the workspace (#113) and
|
|
229
|
+
`upload_asset`'s `file_path` to the directories the server works in — its
|
|
230
|
+
workspace, workflows, assets, outputs and prompts (#138). `upload_asset`'s
|
|
231
|
+
refusal is ordered ahead of the existence and extension checks so it cannot
|
|
232
|
+
be used as a path-existence oracle. A `..` path segment in `destination` is
|
|
233
|
+
refused regardless, and an existing file at the resolved path is left alone
|
|
234
|
+
unless the caller passes `overwrite=True`.
|
|
235
|
+
|
|
236
|
+
## Exception Hierarchy
|
|
237
|
+
|
|
238
|
+
```text
|
|
239
|
+
SecurityError
|
|
240
|
+
PathTraversalError — path traversal attempt
|
|
241
|
+
InvalidInputError — input validation failure
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
## Rules
|
|
245
|
+
|
|
246
|
+
- Always validate paths before file operations
|
|
247
|
+
- Use `validate_url()` before loading remote resources
|
|
248
|
+
- If a subprocess is ever introduced, use `shell=False` and pass args through `sanitize_command_args()`
|
|
249
|
+
- Never use dynamic code execution (`eval`/`exec`) or shell interpretation
|
|
250
|
+
|
|
251
|
+
## Protected Against
|
|
252
|
+
|
|
253
|
+
- **Path traversal** — Cannot access files outside allowed directories
|
|
254
|
+
- **Command injection** — No shell interpretation is used anywhere in `dw/`; `sanitize_command_args()` is available as a guard should a subprocess call be added
|
|
255
|
+
- **Resource exhaustion** — File size limits prevent memory exhaustion
|
|
256
|
+
- **Decompression bombs** — an image a caller names is decoded at no more than `MAX_DECODE_PIXELS` (50M; an 8K frame is 33M), checked after `Image.open` and before any decode: `get_output_image` (crops included) refuses it, the gallery thumbnail answers 413, and embedded metadata is read from the PNG header chunks without decoding. Video and audio decode are not limited
|
|
257
|
+
- **Malicious URLs** — Only http/https schemes allowed, and an untrusted
|
|
258
|
+
workflow may not name a host inside the deployment (SSRF)
|
|
259
|
+
- **Arbitrary file read through a media argument** — a location a workflow
|
|
260
|
+
supplies is confined to the roots it may read (`dw/locations.py`)
|
|
261
|
+
- **Script on the UI's origin through an output** — a step's
|
|
262
|
+
`result.content_type` may not be `text/html` or `text/xml` (compared
|
|
263
|
+
without case or parameters): validation refuses it at
|
|
264
|
+
`steps[i].result.content_type` and the writer refuses it again
|
|
265
|
+
(`dw/content_types.py`). A file of an active type that reaches `/outputs`
|
|
266
|
+
or `/inputs` anyway is served with `Content-Security-Policy: sandbox`
|
|
267
|
+
|
|
268
|
+
## Testing
|
|
269
|
+
|
|
270
|
+
```bash
|
|
271
|
+
pytest tests/test_security.py tests/test_locations.py tests/test_workflow_trust.py -v
|
|
272
|
+
```
|