diffusers-workflow 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusers_workflow-0.4.0.dist-info/METADATA +318 -0
- diffusers_workflow-0.4.0.dist-info/RECORD +260 -0
- diffusers_workflow-0.4.0.dist-info/WHEEL +5 -0
- diffusers_workflow-0.4.0.dist-info/entry_points.txt +7 -0
- diffusers_workflow-0.4.0.dist-info/licenses/LICENSE +201 -0
- diffusers_workflow-0.4.0.dist-info/top_level.txt +2 -0
- dw/__init__.py +440 -0
- dw/adapter_compatibility.py +226 -0
- dw/arguments.py +1231 -0
- dw/assessment_rules.py +159 -0
- dw/assets.py +130 -0
- dw/cache_blocks.json +16 -0
- dw/cache_blocks.py +146 -0
- dw/community_pipelines/pipeline_flux_rf_inversion.py +1184 -0
- dw/content_types.py +150 -0
- dw/dissolve_frame_errors.py +121 -0
- dw/docs/ACCELERATION.md +352 -0
- dw/docs/AGENT_LOOP.md +95 -0
- dw/docs/DEPENDENCIES.md +91 -0
- dw/docs/IP_ADAPTER.md +109 -0
- dw/docs/LORAS.md +131 -0
- dw/docs/MCP.md +517 -0
- dw/docs/PROMPT_WEIGHTING.md +78 -0
- dw/docs/QUANTIZATION.md +230 -0
- dw/docs/RECIPES_24GB.md +201 -0
- dw/docs/RELEASING.md +195 -0
- dw/docs/REMOTE.md +140 -0
- dw/docs/REPL_COMMANDS.md +121 -0
- dw/docs/REPL_WORKER_GUIDE.md +51 -0
- dw/docs/SECURITY.md +272 -0
- dw/docs/SECURITY_QUICKREF.md +112 -0
- dw/docs/SERVER.md +679 -0
- dw/docs/TASKS.md +1741 -0
- dw/docs/TESTING.md +71 -0
- dw/docs/WORKFLOW_GUIDE.md +2038 -0
- dw/docs/WORKSPACES.md +316 -0
- dw/download_watch.py +335 -0
- dw/elision.py +306 -0
- dw/events.py +275 -0
- dw/for_each.py +409 -0
- dw/host_memory.py +258 -0
- dw/host_memory_projection.py +230 -0
- dw/hub_cache.py +432 -0
- dw/introspection.py +1228 -0
- dw/kernel_availability.py +208 -0
- dw/locations.py +599 -0
- dw/log_setup.py +45 -0
- dw/loudness.py +82 -0
- dw/media_audio.py +217 -0
- dw/media_frames.py +367 -0
- dw/media_info.py +297 -0
- dw/pipeline_processors/chain.py +821 -0
- dw/pipeline_processors/config_objects.py +237 -0
- dw/pipeline_processors/pipeline.py +2297 -0
- dw/pipeline_processors/remote.py +46 -0
- dw/plan.py +920 -0
- dw/previous_results.py +411 -0
- dw/probe_paths.py +59 -0
- dw/prompt_schema.json +48 -0
- dw/prompt_weighting.py +378 -0
- dw/prompts.py +159 -0
- dw/realize.py +250 -0
- dw/reference_limits.py +215 -0
- dw/reference_names.py +125 -0
- dw/repl.py +338 -0
- dw/repl_commands.py +836 -0
- dw/repl_worker.py +159 -0
- dw/result.py +1720 -0
- dw/result_fps.py +82 -0
- dw/run.py +162 -0
- dw/runs.py +768 -0
- dw/scalar_result_validation.py +97 -0
- dw/schema.py +283 -0
- dw/security.py +1038 -0
- dw/select_validation.py +115 -0
- dw/serve.py +277 -0
- dw/server/__init__.py +2 -0
- dw/server/app.py +4586 -0
- dw/server/assess.py +132 -0
- dw/server/catalog_shape.py +487 -0
- dw/server/enhancers.py +129 -0
- dw/server/exports.py +480 -0
- dw/server/guides.py +257 -0
- dw/server/jobs.py +1561 -0
- dw/server/mcp_mount.py +95 -0
- dw/server/netinfo.py +124 -0
- dw/server/observed_cost.py +379 -0
- dw/server/sysinfo.py +71 -0
- dw/server/ui/assets/abap-08VXUWAP.js +1 -0
- dw/server/ui/assets/apex-BWPQTe0t.js +1 -0
- dw/server/ui/assets/azcli-Bc_sGQ0U.js +1 -0
- dw/server/ui/assets/bat-i0X4ZdIN.js +1 -0
- dw/server/ui/assets/bicep-B5-_aFwp.js +2 -0
- dw/server/ui/assets/cameligo-DMUM7wLl.js +1 -0
- dw/server/ui/assets/clojure-Cm7r79vr.js +1 -0
- dw/server/ui/assets/codicon-Brq4_Ui5.ttf +0 -0
- dw/server/ui/assets/coffee-Ba7i2nA0.js +1 -0
- dw/server/ui/assets/cpp-C7h46wYY.js +1 -0
- dw/server/ui/assets/csharp-BKxtCVv1.js +1 -0
- dw/server/ui/assets/csp-bTuwJoIa.js +1 -0
- dw/server/ui/assets/css-DIMkf-bt.js +3 -0
- dw/server/ui/assets/css.worker-B3ciXF_0.js +93 -0
- dw/server/ui/assets/cssMode-CPznxfY8.js +1 -0
- dw/server/ui/assets/cypher-CVaqCwHa.js +1 -0
- dw/server/ui/assets/dart-onAF5SnQ.js +1 -0
- dw/server/ui/assets/dockerfile-DZFCIeNp.js +1 -0
- dw/server/ui/assets/ecl-D05T4iGw.js +1 -0
- dw/server/ui/assets/editor-jjEx9u7D.css +1 -0
- dw/server/ui/assets/editor.api-CpWcotrd.js +847 -0
- dw/server/ui/assets/editor.worker-q-txB4vs.js +30 -0
- dw/server/ui/assets/elixir-6RTg0lbw.js +1 -0
- dw/server/ui/assets/flow9-C5_-GSwl.js +1 -0
- dw/server/ui/assets/freemarker2-CXtRM8N4.js +3 -0
- dw/server/ui/assets/fsharp-C8Ef5oNN.js +1 -0
- dw/server/ui/assets/go-C-y9NEjX.js +1 -0
- dw/server/ui/assets/graphql-fmXr3nnJ.js +1 -0
- dw/server/ui/assets/handlebars-N7x-6NMY.js +1 -0
- dw/server/ui/assets/hcl-CpzslTdj.js +1 -0
- dw/server/ui/assets/html-PhsdjHSr.js +1 -0
- dw/server/ui/assets/html.worker-C93Ht9o9.js +506 -0
- dw/server/ui/assets/htmlMode-Dgj0SEok.js +1 -0
- dw/server/ui/assets/index-3Vw6WAPW.css +1 -0
- dw/server/ui/assets/index-DgrYhQd9.js +43 -0
- dw/server/ui/assets/ini-sBoK_t0W.js +1 -0
- dw/server/ui/assets/java-BEtHBSE6.js +1 -0
- dw/server/ui/assets/javascript-BJqN9Qhv.js +1 -0
- dw/server/ui/assets/json.worker-B2V3pomh.js +62 -0
- dw/server/ui/assets/jsonMode-DbM4SWSv.js +7 -0
- dw/server/ui/assets/julia-Bri6UV-V.js +1 -0
- dw/server/ui/assets/kotlin-BOotOW0E.js +1 -0
- dw/server/ui/assets/less-B9JPFI3C.js +2 -0
- dw/server/ui/assets/lexon-CfSJPG6W.js +1 -0
- dw/server/ui/assets/liquid-BWr8lEc4.js +1 -0
- dw/server/ui/assets/lspLanguageFeatures-C1iGuDyZ.js +4 -0
- dw/server/ui/assets/lua-CsQS60Ue.js +1 -0
- dw/server/ui/assets/m3-D-oSqn_W.js +1 -0
- dw/server/ui/assets/markdown-Cimd5fb3.js +1 -0
- dw/server/ui/assets/mdx-DAdMi_0p.js +1 -0
- dw/server/ui/assets/mips-CIPQ_RoX.js +1 -0
- dw/server/ui/assets/monaco--ixms01u.css +1 -0
- dw/server/ui/assets/monaco-BGCeEqaw.js +56 -0
- dw/server/ui/assets/msdax-DauUninz.js +1 -0
- dw/server/ui/assets/mysql-SOo6toE5.js +1 -0
- dw/server/ui/assets/objective-c-FvmIjYaQ.js +1 -0
- dw/server/ui/assets/pascal-DrH0SRf2.js +1 -0
- dw/server/ui/assets/pascaligo-D-ptJ9y-.js +1 -0
- dw/server/ui/assets/perl-oz_6vUea.js +1 -0
- dw/server/ui/assets/pgsql-DTj74zXo.js +1 -0
- dw/server/ui/assets/php-nr791fC2.js +1 -0
- dw/server/ui/assets/pla-CopQ2nXW.js +1 -0
- dw/server/ui/assets/postiats-43DmfD33.js +1 -0
- dw/server/ui/assets/powerquery-D3hlyOfw.js +1 -0
- dw/server/ui/assets/powershell-DmHpPYUd.js +1 -0
- dw/server/ui/assets/protobuf-C531GsRP.js +2 -0
- dw/server/ui/assets/pug-Z5eAx3Zn.js +1 -0
- dw/server/ui/assets/python-Bcn70HdC.js +1 -0
- dw/server/ui/assets/qsharp-DkqhCAOL.js +1 -0
- dw/server/ui/assets/r-BwWrilGY.js +1 -0
- dw/server/ui/assets/razor-D1HmNnby.js +1 -0
- dw/server/ui/assets/redis-ClamHrr6.js +1 -0
- dw/server/ui/assets/redshift-DT7zqm-g.js +1 -0
- dw/server/ui/assets/restructuredtext-BYgofb2h.js +1 -0
- dw/server/ui/assets/ruby-DezsRK8O.js +1 -0
- dw/server/ui/assets/rust-DdL9SqIa.js +1 -0
- dw/server/ui/assets/sb-CcwsVR0C.js +1 -0
- dw/server/ui/assets/scala-DHpiXF5c.js +1 -0
- dw/server/ui/assets/scheme-BeGwcela.js +1 -0
- dw/server/ui/assets/scss-gp-XZpBa.js +3 -0
- dw/server/ui/assets/shell-CC2rA5mh.js +1 -0
- dw/server/ui/assets/solidity-BEEn4gHE.js +1 -0
- dw/server/ui/assets/sophia-CRfGWb83.js +1 -0
- dw/server/ui/assets/sparql-D_Lu-MrJ.js +1 -0
- dw/server/ui/assets/sql-NEE52Syq.js +1 -0
- dw/server/ui/assets/st-DbInun42.js +1 -0
- dw/server/ui/assets/swift-Bxkupp3x.js +1 -0
- dw/server/ui/assets/systemverilog-Bz4Y3fRF.js +1 -0
- dw/server/ui/assets/tcl-DISqw1ZD.js +1 -0
- dw/server/ui/assets/ts.worker-D7T1-Ig5.js +67738 -0
- dw/server/ui/assets/tsMode-D6u0XmOW.js +11 -0
- dw/server/ui/assets/twig-De2hgUGE.js +1 -0
- dw/server/ui/assets/typescript-BU6v-LMV.js +1 -0
- dw/server/ui/assets/typespec-B8J7ngcE.js +1 -0
- dw/server/ui/assets/vb-DV3o63ZY.js +1 -0
- dw/server/ui/assets/wgsl-DpFanUEy.js +298 -0
- dw/server/ui/assets/workers-Cn7cTUKr.js +1 -0
- dw/server/ui/assets/xml--0LP2Lwk.js +1 -0
- dw/server/ui/assets/yaml-mpBg9jnt.js +1 -0
- dw/server/ui/index.html +17 -0
- dw/server/updater.py +192 -0
- dw/settings.py +98 -0
- dw/shot_span_preflight.py +116 -0
- dw/shots.py +359 -0
- dw/slice_preflight.py +148 -0
- dw/step.py +187 -0
- dw/step_cache.py +442 -0
- dw/subfolders.py +107 -0
- dw/task_domains.py +307 -0
- dw/tasks/assess.py +826 -0
- dw/tasks/audio_transcription.py +88 -0
- dw/tasks/audio_utils.py +1862 -0
- dw/tasks/background_remover.py +43 -0
- dw/tasks/borders.py +113 -0
- dw/tasks/compose_text.py +74 -0
- dw/tasks/concat_videos.py +300 -0
- dw/tasks/depth_estimator.py +54 -0
- dw/tasks/diffusion_upscale.py +109 -0
- dw/tasks/dissolve_videos.py +342 -0
- dw/tasks/format_messages.py +24 -0
- dw/tasks/gather.py +173 -0
- dw/tasks/grade.py +97 -0
- dw/tasks/image_to_text.py +43 -0
- dw/tasks/image_utils.py +764 -0
- dw/tasks/interpolate_frames.py +252 -0
- dw/tasks/judge.py +68 -0
- dw/tasks/model_cache.py +55 -0
- dw/tasks/pair_audio.py +268 -0
- dw/tasks/qr_code.py +19 -0
- dw/tasks/restore_faces.py +175 -0
- dw/tasks/rife_model.py +192 -0
- dw/tasks/segment.py +121 -0
- dw/tasks/select.py +111 -0
- dw/tasks/speech_generation.py +228 -0
- dw/tasks/stabilize.py +129 -0
- dw/tasks/task.py +920 -0
- dw/tasks/tensor_image.py +57 -0
- dw/tasks/text_generation.py +169 -0
- dw/tasks/text_sections.py +80 -0
- dw/tasks/upscale.py +203 -0
- dw/tasks/video_utils.py +624 -0
- dw/tasks/zoe_depth.py +71 -0
- dw/teacache.py +381 -0
- dw/teacache_models.json +99 -0
- dw/test.py +29 -0
- dw/type_helpers.py +231 -0
- dw/validate.py +68 -0
- dw/variable_constraints.py +444 -0
- dw/variables.py +443 -0
- dw/video_extensions.py +141 -0
- dw/vram_estimate.py +116 -0
- dw/worker.py +764 -0
- dw/workflow.py +2007 -0
- dw/workflow_schema.json +1346 -0
- dw/workflow_sources.py +383 -0
- dw/workflows/h3_context_ir.json +57 -0
- dw/workflows/test.json +31 -0
- dw/workspace.py +730 -0
- dw_mcp/__init__.py +6 -0
- dw_mcp/__main__.py +133 -0
- dw_mcp/assets.py +336 -0
- dw_mcp/authoring.py +114 -0
- dw_mcp/catalog.py +360 -0
- dw_mcp/client.py +486 -0
- dw_mcp/diagnose.py +371 -0
- dw_mcp/exports.py +84 -0
- dw_mcp/guides.py +35 -0
- dw_mcp/media.py +638 -0
- dw_mcp/models.py +97 -0
- dw_mcp/prompts.py +104 -0
- dw_mcp/server.py +1343 -0
- dw_mcp/workspaces.py +212 -0
dw/server/app.py
ADDED
|
@@ -0,0 +1,4586 @@
|
|
|
1
|
+
"""FastAPI application exposing the workflow engine.
|
|
2
|
+
|
|
3
|
+
All state lives in the JobManager; this module is routing, validation and
|
|
4
|
+
SSE framing. Everything path-shaped goes through dw.security validators.
|
|
5
|
+
Interactive API docs are served at /docs (OpenAPI at /openapi.json).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import base64
|
|
10
|
+
import mimetypes
|
|
11
|
+
import shutil
|
|
12
|
+
import io
|
|
13
|
+
import zipfile
|
|
14
|
+
import tempfile
|
|
15
|
+
import copy
|
|
16
|
+
import json
|
|
17
|
+
import re
|
|
18
|
+
import uuid
|
|
19
|
+
import asyncio
|
|
20
|
+
import logging
|
|
21
|
+
import secrets
|
|
22
|
+
from contextlib import asynccontextmanager
|
|
23
|
+
from datetime import datetime
|
|
24
|
+
from urllib.parse import quote, urlparse
|
|
25
|
+
from typing import Any, Dict, List, Optional, Union
|
|
26
|
+
|
|
27
|
+
from fastapi import Depends, FastAPI, HTTPException, Query, Request
|
|
28
|
+
from fastapi.concurrency import run_in_threadpool
|
|
29
|
+
from fastapi.responses import StreamingResponse, JSONResponse, Response, FileResponse
|
|
30
|
+
from fastapi.staticfiles import StaticFiles
|
|
31
|
+
from pydantic import BaseModel, Field
|
|
32
|
+
from starlette.routing import Match, Route
|
|
33
|
+
from starlette.background import BackgroundTask
|
|
34
|
+
|
|
35
|
+
from ..security import (
|
|
36
|
+
MAX_DECODE_PIXELS,
|
|
37
|
+
contained,
|
|
38
|
+
validate_asset_reference,
|
|
39
|
+
validate_path,
|
|
40
|
+
validate_output_path,
|
|
41
|
+
validate_prompt_reference,
|
|
42
|
+
ALLOWED_IMAGE_EXTENSIONS,
|
|
43
|
+
ALLOWED_VIDEO_EXTENSIONS,
|
|
44
|
+
validate_commit_hash,
|
|
45
|
+
InvalidInputError,
|
|
46
|
+
PathTraversalError,
|
|
47
|
+
SecurityError,
|
|
48
|
+
workflows_are_trusted,
|
|
49
|
+
)
|
|
50
|
+
from ..introspection import (
|
|
51
|
+
describe_class,
|
|
52
|
+
list_classes,
|
|
53
|
+
list_pipelines,
|
|
54
|
+
describe_pipeline,
|
|
55
|
+
list_tasks,
|
|
56
|
+
describe_task,
|
|
57
|
+
workflow_argument_warnings,
|
|
58
|
+
)
|
|
59
|
+
from ..events import select_kinds
|
|
60
|
+
from ..for_each import entry_field_warnings
|
|
61
|
+
from ..schema import (
|
|
62
|
+
load_schema,
|
|
63
|
+
schema_section,
|
|
64
|
+
validate_data,
|
|
65
|
+
format_validation_errors,
|
|
66
|
+
SchemaSectionError,
|
|
67
|
+
)
|
|
68
|
+
from ..prompts import (
|
|
69
|
+
PROMPT_PREFIX,
|
|
70
|
+
RESERVED_TEXT_PREFIXES,
|
|
71
|
+
resolve_prompt_reference,
|
|
72
|
+
)
|
|
73
|
+
from ..assets import ASSET_PREFIX, is_asset_reference, resolve_asset_reference
|
|
74
|
+
from ..variable_constraints import constraint_errors, constraint_warnings
|
|
75
|
+
from .observed_cost import ObservedCosts, declared_drivers
|
|
76
|
+
from ..variables import argument_errors
|
|
77
|
+
from ..workflow import Workflow, workflow_from_definition, workflow_from_file
|
|
78
|
+
from .enhancers import build_enhance_workflow, preset_descriptions
|
|
79
|
+
from .exports import export_directory, export_job
|
|
80
|
+
from .assess import assess, unknown_probe
|
|
81
|
+
from ..result import read_embedded_metadata
|
|
82
|
+
from ..media_info import probe_media
|
|
83
|
+
from ..media_audio import (
|
|
84
|
+
MAX_INLINE_AUDIO_BYTES,
|
|
85
|
+
NoSoundtrack,
|
|
86
|
+
audio_shape,
|
|
87
|
+
extract_audio,
|
|
88
|
+
media_duration,
|
|
89
|
+
projected_wav_base64_size,
|
|
90
|
+
)
|
|
91
|
+
from ..media_frames import (
|
|
92
|
+
contact_sheet,
|
|
93
|
+
frames_at,
|
|
94
|
+
resolve_crop_box,
|
|
95
|
+
seam_tiles,
|
|
96
|
+
video_shape,
|
|
97
|
+
)
|
|
98
|
+
from ..hub_cache import scan_models, delete_model, DownloadManager
|
|
99
|
+
from ..host_memory_projection import CEILING_FRACTION, host_memory_warnings
|
|
100
|
+
from ..plan import build_plan, gate_warnings, unseeded_cache_warnings
|
|
101
|
+
from ..runs import (
|
|
102
|
+
MANIFEST_FILE_NAME,
|
|
103
|
+
OUTPUT_PREFIX,
|
|
104
|
+
REALIZED_FILE_NAME,
|
|
105
|
+
is_output_reference,
|
|
106
|
+
is_run_id,
|
|
107
|
+
record_kept_shots,
|
|
108
|
+
record_run_versions,
|
|
109
|
+
resolve_output_reference,
|
|
110
|
+
run_versions,
|
|
111
|
+
recorded_shots,
|
|
112
|
+
shots_beside,
|
|
113
|
+
split_run_path,
|
|
114
|
+
)
|
|
115
|
+
from ..workspace import (
|
|
116
|
+
ASSETS_SUBDIR,
|
|
117
|
+
DEFAULT_WORKSPACE_NAME,
|
|
118
|
+
PROMPTS_SUBDIR,
|
|
119
|
+
ConfiguredWorkspace,
|
|
120
|
+
NotAWorkspaceError,
|
|
121
|
+
Workspace,
|
|
122
|
+
_holds_a_workspace,
|
|
123
|
+
create_workspace,
|
|
124
|
+
delete_workspace,
|
|
125
|
+
example_libraries,
|
|
126
|
+
forget_workspace_usage,
|
|
127
|
+
named_workspace,
|
|
128
|
+
workspace_contents,
|
|
129
|
+
workspace_names,
|
|
130
|
+
workspace_usage,
|
|
131
|
+
)
|
|
132
|
+
from ..workflow_sources import (
|
|
133
|
+
COMMON_ORIGIN,
|
|
134
|
+
EXAMPLES_ORIGIN,
|
|
135
|
+
WORKSPACE_ORIGIN,
|
|
136
|
+
find_workflow,
|
|
137
|
+
listing,
|
|
138
|
+
resolve_in_source,
|
|
139
|
+
resolve_sub_workflow,
|
|
140
|
+
source_for_path,
|
|
141
|
+
suggest_workflow_names,
|
|
142
|
+
workflow_names,
|
|
143
|
+
workflow_sources,
|
|
144
|
+
writable_source,
|
|
145
|
+
SubWorkflowNotFound,
|
|
146
|
+
)
|
|
147
|
+
from .jobs import (
|
|
148
|
+
ACK_BOOLEAN,
|
|
149
|
+
ACK_BOUND,
|
|
150
|
+
ACK_NONE,
|
|
151
|
+
JobManager,
|
|
152
|
+
MAX_PERSISTED_EVENTS,
|
|
153
|
+
QUEUED,
|
|
154
|
+
RUNNING,
|
|
155
|
+
TERMINAL_STATES,
|
|
156
|
+
)
|
|
157
|
+
from .netinfo import local_addresses
|
|
158
|
+
from .updater import DiffusersUpdater
|
|
159
|
+
from .sysinfo import runtime_info
|
|
160
|
+
from .catalog_shape import derive_catalog_metadata, project_listing
|
|
161
|
+
from . import guides
|
|
162
|
+
from .guides import GuideError
|
|
163
|
+
from .. import settings
|
|
164
|
+
|
|
165
|
+
logger = logging.getLogger("dw")
|
|
166
|
+
|
|
167
|
+
# How long one SSE poll waits for a new event before checking liveness
|
|
168
|
+
SSE_POLL_SECONDS = 1.0
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class AcknowledgedCost(BaseModel):
|
|
172
|
+
"""A cost acknowledgement bound to the plan a validate call answered
|
|
173
|
+
with (#85): the server refuses to queue a run whose plan no longer
|
|
174
|
+
matches it. `minutes` is recorded, never compared."""
|
|
175
|
+
|
|
176
|
+
fingerprint: str = Field(description="plan.fingerprint from POST /api/validate")
|
|
177
|
+
minutes: Optional[float] = Field(
|
|
178
|
+
default=None, description="plan.estimate.minutes, recorded on the job"
|
|
179
|
+
)
|
|
180
|
+
downloads: List[str] = Field(
|
|
181
|
+
default_factory=list,
|
|
182
|
+
description="The repos in plan.downloads_required that were acknowledged",
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
ACKNOWLEDGED_COST_FIELD = Field(
|
|
187
|
+
default=None,
|
|
188
|
+
description="Cost acknowledgement: true (recorded), or an object "
|
|
189
|
+
"{fingerprint, minutes, downloads} bound to the plan validate answered "
|
|
190
|
+
"with - then the run is refused with 409 if its plan changed",
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
class JobRequest(BaseModel):
|
|
195
|
+
workflow_path: Optional[str] = Field(
|
|
196
|
+
default=None, description="Path to a workflow JSON file on the server"
|
|
197
|
+
)
|
|
198
|
+
workflow: Optional[Dict[str, Any]] = Field(
|
|
199
|
+
default=None, description="Inline workflow definition"
|
|
200
|
+
)
|
|
201
|
+
arguments: Dict[str, Any] = Field(
|
|
202
|
+
default_factory=dict, description="Workflow variable overrides"
|
|
203
|
+
)
|
|
204
|
+
base_dir: Optional[str] = Field(
|
|
205
|
+
default=None,
|
|
206
|
+
description="Directory relative paths in an inline workflow resolve against",
|
|
207
|
+
)
|
|
208
|
+
workspace: Optional[str] = Field(
|
|
209
|
+
default=None,
|
|
210
|
+
description="Which workspace to run or resolve in; the default when omitted",
|
|
211
|
+
)
|
|
212
|
+
acknowledged_cost: Optional[Union[bool, AcknowledgedCost]] = ACKNOWLEDGED_COST_FIELD
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
# What a run directory holds besides its outputs - the files a run writes
|
|
216
|
+
# about itself. A run whose directory holds nothing else is an orphan
|
|
217
|
+
# (see _iter_orphan_runs, #170) whatever shape its output would have had.
|
|
218
|
+
# job.json is what an export bundle writes, listed defensively.
|
|
219
|
+
RUN_BOOKKEEPING_FILES = frozenset({MANIFEST_FILE_NAME, REALIZED_FILE_NAME, "job.json"})
|
|
220
|
+
|
|
221
|
+
# What each workflow produces and takes, for listing cards - cached by mtime
|
|
222
|
+
_workflow_detail_cache = {}
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _prune_detail_cache(cache, directory, names):
|
|
226
|
+
"""Forget files a listing no longer names - a long-lived server that
|
|
227
|
+
creates and deletes scratch files would otherwise grow the cache forever.
|
|
228
|
+
|
|
229
|
+
`names` are relative names under `directory`.
|
|
230
|
+
"""
|
|
231
|
+
live = {os.path.join(directory, f"{name}.json") for name in names}
|
|
232
|
+
for stale in [path for path in cache if path not in live]:
|
|
233
|
+
del cache[stale]
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _prune_missing(cache):
|
|
237
|
+
"""Forget cached files that are gone from disk.
|
|
238
|
+
|
|
239
|
+
Pruning by what one listing named would be wrong here: the workflow
|
|
240
|
+
cache is shared by every workspace, and a listing only ever sees one
|
|
241
|
+
workspace's search path, so anything cached for another workspace would
|
|
242
|
+
be thrown away and re-parsed on the next switch. Existence is the test
|
|
243
|
+
that holds for all of them at once.
|
|
244
|
+
"""
|
|
245
|
+
for stale in [path for path in cache if not os.path.exists(path)]:
|
|
246
|
+
del cache[stale]
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def collect_prompt_references(value):
|
|
250
|
+
"""Every stored-prompt name a definition references, at any depth - so
|
|
251
|
+
deleting a prompt can warn which workflows would break."""
|
|
252
|
+
references = set()
|
|
253
|
+
if isinstance(value, str):
|
|
254
|
+
if value.startswith(PROMPT_PREFIX):
|
|
255
|
+
references.add(value.removeprefix(PROMPT_PREFIX).strip())
|
|
256
|
+
elif isinstance(value, dict):
|
|
257
|
+
for item in value.values():
|
|
258
|
+
references |= collect_prompt_references(item)
|
|
259
|
+
elif isinstance(value, list):
|
|
260
|
+
for item in value:
|
|
261
|
+
references |= collect_prompt_references(item)
|
|
262
|
+
return references
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _catalog_name_from_root(path, root):
|
|
266
|
+
"""The listing name a resolved workflow path has under a root.
|
|
267
|
+
|
|
268
|
+
None when the path is not under the root after all - a name that does
|
|
269
|
+
not name an entry is worse than no name for anything that later joins
|
|
270
|
+
on it.
|
|
271
|
+
"""
|
|
272
|
+
if root is None:
|
|
273
|
+
return None
|
|
274
|
+
relative = os.path.relpath(path, root)
|
|
275
|
+
if relative.startswith(".."):
|
|
276
|
+
return None
|
|
277
|
+
return os.path.splitext(relative)[0].replace(os.sep, "/")
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def catalog_name_for(path, source):
|
|
281
|
+
"""The listing name a resolved workflow path has within its source.
|
|
282
|
+
|
|
283
|
+
None when the run came from an inline definition, or when the path is
|
|
284
|
+
not under the source root after all - a name that does not name an
|
|
285
|
+
entry is worse than no name for anything that later joins on it.
|
|
286
|
+
"""
|
|
287
|
+
if source is None:
|
|
288
|
+
return None
|
|
289
|
+
return _catalog_name_from_root(path, source.root)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def attach_observed(details, observed_costs, workspace_name=None):
|
|
293
|
+
"""Fold this box's own history into each detail, as `observed`.
|
|
294
|
+
|
|
295
|
+
Separate from `workflow_details` because that cache is keyed on a file's
|
|
296
|
+
mtime and this figure changes when no file has: a job finishing moves
|
|
297
|
+
every number here. A detail carries `cost_drivers` and the defaults they
|
|
298
|
+
take, which is everything the aggregate needs - the file is not read a
|
|
299
|
+
second time.
|
|
300
|
+
|
|
301
|
+
A detail's own `writable` says whether its entry is this workspace's own
|
|
302
|
+
copy or a shared catalog one (#274): only the former is scoped to
|
|
303
|
+
`workspace_name`, so two workspaces' saves of the same name do not leak
|
|
304
|
+
into each other's figure, while a template or example still pools every
|
|
305
|
+
workspace's runs of it, matching #154.
|
|
306
|
+
"""
|
|
307
|
+
if observed_costs is None or not observed_costs.refresh():
|
|
308
|
+
return details
|
|
309
|
+
for name, detail in details.items():
|
|
310
|
+
drivers = detail.get("cost_drivers") or {}
|
|
311
|
+
# The shape `observed_for` reads: the drivers with their defaults,
|
|
312
|
+
# and the variable names, which is what the no-drivers fallback
|
|
313
|
+
# (default-arguments-only runs) compares a job's arguments against
|
|
314
|
+
surrogate = {
|
|
315
|
+
"cost_drivers": sorted(drivers),
|
|
316
|
+
"variables": {
|
|
317
|
+
**{variable: None for variable in detail.get("variable_names") or []},
|
|
318
|
+
**drivers,
|
|
319
|
+
},
|
|
320
|
+
}
|
|
321
|
+
workspace = workspace_name if detail.get("writable") else None
|
|
322
|
+
observed = observed_costs.observed(
|
|
323
|
+
name, surrogate, fresh=False, workspace=workspace
|
|
324
|
+
)
|
|
325
|
+
if observed:
|
|
326
|
+
detail["observed"] = observed
|
|
327
|
+
return details
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def workflow_details(sources_by_name):
|
|
331
|
+
"""Per-workflow card metadata: output kinds, step and variable counts,
|
|
332
|
+
and the variable names themselves - enough for an agent to pick a
|
|
333
|
+
workflow and know what to pass it without fetching each candidate, and,
|
|
334
|
+
for a list-driven workflow, what an entry of each list carries. The
|
|
335
|
+
names but not their defaults: across the workflows on disk the defaults
|
|
336
|
+
are an order of magnitude more payload, on a listing the UI reloads.
|
|
337
|
+
|
|
338
|
+
Takes the name -> source mapping the search path produced, so each
|
|
339
|
+
entry also says where it came from and whether it can be written to -
|
|
340
|
+
what a client needs to decide between offering save and offering
|
|
341
|
+
save-a-copy.
|
|
342
|
+
"""
|
|
343
|
+
details = {}
|
|
344
|
+
for name, source in sources_by_name.items():
|
|
345
|
+
path = os.path.join(source.root, f"{name}.json")
|
|
346
|
+
try:
|
|
347
|
+
mtime = os.path.getmtime(path)
|
|
348
|
+
except OSError:
|
|
349
|
+
continue
|
|
350
|
+
cached = _workflow_detail_cache.get(path)
|
|
351
|
+
if cached and cached[0] == mtime:
|
|
352
|
+
# The cached detail is placement-free; the origin and writability
|
|
353
|
+
# are the source's, and a warm cache must still carry them or a
|
|
354
|
+
# second listing loses the fields a client decides save-vs-copy on
|
|
355
|
+
details[name] = {
|
|
356
|
+
**cached[1],
|
|
357
|
+
"origin": source.origin,
|
|
358
|
+
"writable": source.writable,
|
|
359
|
+
}
|
|
360
|
+
continue
|
|
361
|
+
try:
|
|
362
|
+
with open(path, "r") as file:
|
|
363
|
+
definition = json.load(file)
|
|
364
|
+
kinds = sorted(
|
|
365
|
+
{
|
|
366
|
+
step["result"]["content_type"].split("/")[0]
|
|
367
|
+
for step in definition.get("steps", [])
|
|
368
|
+
if isinstance(step.get("result"), dict)
|
|
369
|
+
and "content_type" in step["result"]
|
|
370
|
+
}
|
|
371
|
+
)
|
|
372
|
+
variables = definition.get("variables", {}) or {}
|
|
373
|
+
metadata = derive_catalog_metadata(definition)
|
|
374
|
+
cost = definition.get("cost")
|
|
375
|
+
detail = {
|
|
376
|
+
"kinds": kinds,
|
|
377
|
+
"steps": len(definition.get("steps", [])),
|
|
378
|
+
"variables": len(variables),
|
|
379
|
+
"variable_names": sorted(variables),
|
|
380
|
+
"description": str(definition.get("description", "") or ""),
|
|
381
|
+
# Empty for a template; a catalog name for a model config, which
|
|
382
|
+
# is what lets a client show the two as different kinds of thing
|
|
383
|
+
"configures": str(definition.get("configures", "") or ""),
|
|
384
|
+
"prompt_refs": sorted(collect_prompt_references(definition)),
|
|
385
|
+
"shape": metadata["shape"],
|
|
386
|
+
"traits": metadata["traits"],
|
|
387
|
+
"summary": metadata["summary"],
|
|
388
|
+
"lists": metadata["lists"],
|
|
389
|
+
# What a variable's value is allowed to be, so the rule is
|
|
390
|
+
# read rather than guessed at (#96)
|
|
391
|
+
"constraints": definition.get("variable_constraints") or {},
|
|
392
|
+
# The variables the author says move this workflow's cost,
|
|
393
|
+
# with what they default to - what buckets this box's own
|
|
394
|
+
# runs into comparable ones (#93). Carried here so an
|
|
395
|
+
# observed figure needs no second read of the file
|
|
396
|
+
"cost_drivers": {
|
|
397
|
+
name: (definition.get("variables") or {}).get(name)
|
|
398
|
+
for name in declared_drivers(definition)
|
|
399
|
+
},
|
|
400
|
+
"cost": cost if isinstance(cost, list) and cost else None,
|
|
401
|
+
}
|
|
402
|
+
except Exception:
|
|
403
|
+
detail = {
|
|
404
|
+
"kinds": [],
|
|
405
|
+
"steps": 0,
|
|
406
|
+
"variables": 0,
|
|
407
|
+
"variable_names": [],
|
|
408
|
+
"description": "",
|
|
409
|
+
"prompt_refs": [],
|
|
410
|
+
"shape": "utility",
|
|
411
|
+
"traits": [],
|
|
412
|
+
"summary": "",
|
|
413
|
+
"lists": {},
|
|
414
|
+
"constraints": {},
|
|
415
|
+
"cost_drivers": {},
|
|
416
|
+
"cost": None,
|
|
417
|
+
}
|
|
418
|
+
_workflow_detail_cache[path] = (mtime, detail)
|
|
419
|
+
# Cached by content, not by placement: the same file listed from a
|
|
420
|
+
# different source keeps its parsed detail and gets fresh origins
|
|
421
|
+
details[name] = {
|
|
422
|
+
**detail,
|
|
423
|
+
"origin": source.origin,
|
|
424
|
+
"writable": source.writable,
|
|
425
|
+
}
|
|
426
|
+
_prune_missing(_workflow_detail_cache)
|
|
427
|
+
# A model config names its template as a catalog name. Resolve it here,
|
|
428
|
+
# where the whole listing is in hand, so a badge is a link to a real card
|
|
429
|
+
# rather than a string - and say which name did not resolve. A config
|
|
430
|
+
# also takes its shape and traits from the template: what it makes is
|
|
431
|
+
# the template's business, what it costs is its own. Entries can be the
|
|
432
|
+
# very dict cached above (a cache hit skips the copy at the origin
|
|
433
|
+
# merge), so copy before mutating - otherwise a stale "not found yet"
|
|
434
|
+
# verdict would stick in the cache and outlive the typo once the
|
|
435
|
+
# template it names is added.
|
|
436
|
+
for name, detail in details.items():
|
|
437
|
+
named = detail.get("configures", "")
|
|
438
|
+
if not named:
|
|
439
|
+
continue
|
|
440
|
+
detail = dict(detail)
|
|
441
|
+
template = details.get(named)
|
|
442
|
+
if template is None:
|
|
443
|
+
detail["configures_missing"] = named
|
|
444
|
+
detail["configures"] = ""
|
|
445
|
+
else:
|
|
446
|
+
detail["shape"] = template["shape"]
|
|
447
|
+
detail["traits"] = list(template["traits"])
|
|
448
|
+
details[name] = detail
|
|
449
|
+
return details
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _write_bytes(path, data):
|
|
453
|
+
with open(path, "wb") as f:
|
|
454
|
+
f.write(data)
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _unknown_workflow_detail(sources, name):
|
|
458
|
+
"""'Unknown workflow: x', with a '- did you mean ...?' pointer when the
|
|
459
|
+
catalog holds something `name` could be short for or a typo of (#397) -
|
|
460
|
+
otherwise a caller has to spend a list_workflows call and guess the
|
|
461
|
+
right shape/traits to find the entry it already knows by its short
|
|
462
|
+
name."""
|
|
463
|
+
detail = f"Unknown workflow: {name}"
|
|
464
|
+
suggestions = suggest_workflow_names(sources, name)
|
|
465
|
+
if len(suggestions) == 1:
|
|
466
|
+
detail += f" - did you mean {suggestions[0]}?"
|
|
467
|
+
elif suggestions:
|
|
468
|
+
detail += f" - did you mean one of: {', '.join(suggestions)}?"
|
|
469
|
+
return detail
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def resolve_readable_workflow(sources, name):
|
|
473
|
+
"""The path a name has anywhere on the search path, and its source.
|
|
474
|
+
|
|
475
|
+
Reads span every root - the workspace's own workflows, any examples
|
|
476
|
+
directory, and the packaged builtins - front to back, so a workspace
|
|
477
|
+
copy shadows the example it came from.
|
|
478
|
+
"""
|
|
479
|
+
path, source = find_workflow(sources, name)
|
|
480
|
+
if path is None:
|
|
481
|
+
raise HTTPException(
|
|
482
|
+
status_code=404, detail=_unknown_workflow_detail(sources, name)
|
|
483
|
+
)
|
|
484
|
+
return path, source
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def resolve_writable_workflow(sources, name):
|
|
488
|
+
"""Where a save goes: always the writable source, whatever the name
|
|
489
|
+
currently resolves to.
|
|
490
|
+
|
|
491
|
+
Saving a workflow opened from an example is not an overwrite of that
|
|
492
|
+
example - it is a copy into the user's own library, which is what makes
|
|
493
|
+
the read-only roots safe to browse and edit from.
|
|
494
|
+
"""
|
|
495
|
+
source = writable_source(sources)
|
|
496
|
+
if source is None:
|
|
497
|
+
raise HTTPException(
|
|
498
|
+
status_code=409, detail="This server has no writable workflow directory"
|
|
499
|
+
)
|
|
500
|
+
path = resolve_in_source(source, name, allow_create=True)
|
|
501
|
+
if path is None:
|
|
502
|
+
raise HTTPException(status_code=404, detail=f"Unknown workflow: {name}")
|
|
503
|
+
return path, source
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def resolve_workflow_reference(workflow_path, sources):
|
|
507
|
+
"""A submitted workflow_path, resolved to a file on disk, and the source
|
|
508
|
+
it lives in - the same search path the /api/workflows CRUD routes read
|
|
509
|
+
from, spanning every root rather than confining to one, since a run of
|
|
510
|
+
an example is a read and reads are not confined to the writable root.
|
|
511
|
+
|
|
512
|
+
Tried as a stored workflow name first - exactly what /api/workflows
|
|
513
|
+
hands out, with or without .json and nested names included - so an
|
|
514
|
+
agent can run what a listing gave it. A relative or absolute path that
|
|
515
|
+
already names a file under one of the sources resolves the same way:
|
|
516
|
+
os.path.abspath handles a path relative to the server's cwd, and
|
|
517
|
+
source_for_path holds it to that source's containment check.
|
|
518
|
+
|
|
519
|
+
Anything that resolves under no source - an unknown name, a traversal
|
|
520
|
+
attempt, or a real file elsewhere on disk - is rejected with 400,
|
|
521
|
+
rather than silently opened: a workflow_path is not a general
|
|
522
|
+
filesystem path.
|
|
523
|
+
|
|
524
|
+
Returns (None, None) when workflow_path itself is None - an inline
|
|
525
|
+
workflow submission names no path to resolve.
|
|
526
|
+
"""
|
|
527
|
+
if workflow_path is None:
|
|
528
|
+
return None, None
|
|
529
|
+
path, source = find_workflow(sources, workflow_path)
|
|
530
|
+
if path is not None:
|
|
531
|
+
return path, source
|
|
532
|
+
candidate = os.path.abspath(workflow_path)
|
|
533
|
+
source = source_for_path(sources, candidate)
|
|
534
|
+
if source is not None:
|
|
535
|
+
# The containment check re-applied to the path this returns, rather
|
|
536
|
+
# than trusted from source_for_path's answer about it - and applied
|
|
537
|
+
# before anything asks the filesystem about the path, so a
|
|
538
|
+
# workflow_path outside every source cannot be used to find out
|
|
539
|
+
# whether a file exists there
|
|
540
|
+
try:
|
|
541
|
+
confined = validate_path(candidate, source.root, allow_create=False)
|
|
542
|
+
except SecurityError:
|
|
543
|
+
confined = None
|
|
544
|
+
if confined is not None and os.path.isfile(confined):
|
|
545
|
+
return confined, source
|
|
546
|
+
detail = f"workflow_path must name a workflow the server can reach: {workflow_path}"
|
|
547
|
+
suggestions = suggest_workflow_names(sources, workflow_path)
|
|
548
|
+
if len(suggestions) == 1:
|
|
549
|
+
detail += f" - did you mean {suggestions[0]}?"
|
|
550
|
+
elif suggestions:
|
|
551
|
+
detail += f" - did you mean one of: {', '.join(suggestions)}?"
|
|
552
|
+
raise HTTPException(status_code=400, detail=detail)
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
# What each prompt says about itself, for listing cards - cached by mtime
|
|
556
|
+
_prompt_detail_cache = {}
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def prompt_details(paths):
|
|
560
|
+
"""Per-prompt card metadata: description, intended model, tags - and
|
|
561
|
+
the text itself, which the editors show as the tooltip wherever a
|
|
562
|
+
prompt: reference stands in for it.
|
|
563
|
+
|
|
564
|
+
Keyed by path rather than by name under one directory: the prompt
|
|
565
|
+
library is a search path now, and two roots can hold the same name.
|
|
566
|
+
"""
|
|
567
|
+
details = {}
|
|
568
|
+
for name, path in paths.items():
|
|
569
|
+
try:
|
|
570
|
+
mtime = os.path.getmtime(path)
|
|
571
|
+
except OSError:
|
|
572
|
+
continue
|
|
573
|
+
cached = _prompt_detail_cache.get(path)
|
|
574
|
+
if cached and cached[0] == mtime:
|
|
575
|
+
details[name] = cached[1]
|
|
576
|
+
continue
|
|
577
|
+
try:
|
|
578
|
+
with open(path, "r") as file:
|
|
579
|
+
definition = json.load(file)
|
|
580
|
+
detail = {
|
|
581
|
+
"description": str(definition.get("description", "") or ""),
|
|
582
|
+
"intended_model": str(definition.get("intended_model", "") or ""),
|
|
583
|
+
"tags": [str(tag) for tag in definition.get("tags", []) or []],
|
|
584
|
+
"text": str(definition.get("text", "") or ""),
|
|
585
|
+
}
|
|
586
|
+
except Exception:
|
|
587
|
+
detail = {"description": "", "intended_model": "", "tags": [], "text": ""}
|
|
588
|
+
_prompt_detail_cache[path] = (mtime, detail)
|
|
589
|
+
details[name] = detail
|
|
590
|
+
# By existence, not by what this listing named: the cache spans every
|
|
591
|
+
# root on the search path, and one listing shows only the names that
|
|
592
|
+
# were not shadowed
|
|
593
|
+
_prune_missing(_prompt_detail_cache)
|
|
594
|
+
return details
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
def _matching_prompts(details, tag, intended_model):
|
|
598
|
+
"""The prompt names matching the filters, or None when no filter was given.
|
|
599
|
+
|
|
600
|
+
Case-insensitive and exact per value: a `tags` entry or the whole
|
|
601
|
+
`intended_model`, never a substring - `minimax-music` must not match
|
|
602
|
+
`minimax-music3`, which is the confusion the one-spelling-per-family
|
|
603
|
+
rule exists to prevent.
|
|
604
|
+
"""
|
|
605
|
+
if tag is None and intended_model is None:
|
|
606
|
+
return None
|
|
607
|
+
wanted_tag = tag.lower() if tag is not None else None
|
|
608
|
+
wanted_model = intended_model.lower() if intended_model is not None else None
|
|
609
|
+
matches = set()
|
|
610
|
+
for name, detail in details.items():
|
|
611
|
+
if wanted_tag is not None and wanted_tag not in {
|
|
612
|
+
str(each).lower() for each in detail.get("tags") or []
|
|
613
|
+
}:
|
|
614
|
+
continue
|
|
615
|
+
if (
|
|
616
|
+
wanted_model is not None
|
|
617
|
+
and str(detail.get("intended_model") or "").lower() != wanted_model
|
|
618
|
+
):
|
|
619
|
+
continue
|
|
620
|
+
matches.add(name)
|
|
621
|
+
return matches
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
def resolve_prompt_name(prompt_dir, name, allow_create=False):
|
|
625
|
+
"""The on-disk path for a prompt name, confined to prompt_dir.
|
|
626
|
+
|
|
627
|
+
The name is held to the same rule 'prompt:' references enforce - a save
|
|
628
|
+
the API accepted but no workflow could ever reference would be a trap. A
|
|
629
|
+
save is told what is wrong with the name; a read just misses."""
|
|
630
|
+
bare = name.removesuffix(".json")
|
|
631
|
+
try:
|
|
632
|
+
validate_prompt_reference(bare)
|
|
633
|
+
except InvalidInputError as e:
|
|
634
|
+
status = 400 if allow_create else 404
|
|
635
|
+
raise HTTPException(status_code=status, detail=str(e))
|
|
636
|
+
try:
|
|
637
|
+
return validate_path(
|
|
638
|
+
os.path.join(prompt_dir, f"{bare}.json"),
|
|
639
|
+
prompt_dir,
|
|
640
|
+
allow_create=allow_create,
|
|
641
|
+
)
|
|
642
|
+
except SecurityError as e:
|
|
643
|
+
raise HTTPException(status_code=404, detail=f"Unknown prompt: {e}")
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
def default_ui_dir():
|
|
647
|
+
"""Where the built SPA lives: ui/dist in a checkout (the copy npm just
|
|
648
|
+
built), else the copy packaged into the wheel at dw/server/ui, else None."""
|
|
649
|
+
here = os.path.dirname(os.path.abspath(__file__))
|
|
650
|
+
candidates = [
|
|
651
|
+
os.path.join(os.path.dirname(os.path.dirname(here)), "ui", "dist"),
|
|
652
|
+
os.path.join(here, "ui"),
|
|
653
|
+
]
|
|
654
|
+
for candidate in candidates:
|
|
655
|
+
if os.path.isfile(os.path.join(candidate, "index.html")):
|
|
656
|
+
return candidate
|
|
657
|
+
return None
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
def _historical_log_note(stored):
|
|
661
|
+
"""What a restored job's event page has to admit about itself.
|
|
662
|
+
|
|
663
|
+
History keeps only the last MAX_PERSISTED_EVENTS of a run, so a page can
|
|
664
|
+
be complete as a page and still be missing the start of the job. The
|
|
665
|
+
first stored event's seq is the direct signal: anything above zero means
|
|
666
|
+
the head was dropped at record time. Length is not the signal - a job
|
|
667
|
+
that emitted exactly MAX_PERSISTED_EVENTS events lost nothing.
|
|
668
|
+
"""
|
|
669
|
+
if not stored:
|
|
670
|
+
return "This job kept no event log - events were not retained with job history."
|
|
671
|
+
if stored[0].get("seq", 0) > 0:
|
|
672
|
+
return (
|
|
673
|
+
f"Only the last {MAX_PERSISTED_EVENTS} events of this job were "
|
|
674
|
+
f"retained; everything before seq {stored[0]['seq']} was dropped "
|
|
675
|
+
"when the job was recorded."
|
|
676
|
+
)
|
|
677
|
+
return None
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
# Host header values a locally-bound server accepts by default, regardless
|
|
681
|
+
# of what --host is configured to - a loopback request always presents one
|
|
682
|
+
# of these regardless of the server's own bind address.
|
|
683
|
+
LOOPBACK_HOSTS = {"localhost", "127.0.0.1", "::1"}
|
|
684
|
+
# Bind addresses that mean "every interface" - a request never carries one
|
|
685
|
+
# of these as its Host, so they define no allowlist
|
|
686
|
+
WILDCARD_HOSTS = {"0.0.0.0", "::", ""}
|
|
687
|
+
# Where the MCP endpoint is mounted when --mcp is given (see the mcp block
|
|
688
|
+
# at the bottom of create_app) - the Server page quotes it in the command
|
|
689
|
+
# it tells you to run on the other machine
|
|
690
|
+
MCP_PATH = "/mcp"
|
|
691
|
+
|
|
692
|
+
# Types a browser renders as a document, where script runs: /outputs and
|
|
693
|
+
# /inputs serve these under a CSP sandbox (_sandbox_active_content)
|
|
694
|
+
ACTIVE_DOCUMENT_TYPES = frozenset(
|
|
695
|
+
{
|
|
696
|
+
"text/html",
|
|
697
|
+
"application/xhtml+xml",
|
|
698
|
+
"text/xml",
|
|
699
|
+
"application/xml",
|
|
700
|
+
"image/svg+xml",
|
|
701
|
+
}
|
|
702
|
+
)
|
|
703
|
+
|
|
704
|
+
|
|
705
|
+
def query_token_ok(fn):
|
|
706
|
+
"""Mark a GET endpoint as one a browser loads without being able to set
|
|
707
|
+
headers (EventSource, an <img> tag, an <a download> navigation) - only
|
|
708
|
+
routes carrying this marker accept the bearer token as a ?token= query
|
|
709
|
+
param. Matched by the actual route at request time, not by a path
|
|
710
|
+
suffix, so a resource that merely happens to be named "download" or
|
|
711
|
+
"thumbnail" does not inherit the allowance."""
|
|
712
|
+
fn.query_token_ok = True
|
|
713
|
+
return fn
|
|
714
|
+
|
|
715
|
+
|
|
716
|
+
def _matched_route(request: Request):
|
|
717
|
+
"""Resolve the Route (if any) that will handle this request. Runs in
|
|
718
|
+
middleware, before routing has attached anything to request.scope, so
|
|
719
|
+
routes are matched by hand against request.app.router.routes. Skips
|
|
720
|
+
non-Route entries (the SPA static Mount) and routes with no endpoint.
|
|
721
|
+
|
|
722
|
+
A HEAD request path-matches a GET-only route as Match.PARTIAL (method
|
|
723
|
+
mismatch) rather than Match.FULL, since this route is declared with
|
|
724
|
+
methods=["GET"] and nothing here adds HEAD to it - but a HEAD request
|
|
725
|
+
is still the same header-less browser load a GET would be, so it is
|
|
726
|
+
treated the same for the query-token allowance."""
|
|
727
|
+
method = request.scope.get("method")
|
|
728
|
+
for route in request.app.router.routes:
|
|
729
|
+
if not isinstance(route, Route) or route.endpoint is None:
|
|
730
|
+
continue
|
|
731
|
+
match, _ = route.matches(request.scope)
|
|
732
|
+
if match == Match.FULL:
|
|
733
|
+
return route
|
|
734
|
+
if (
|
|
735
|
+
match == Match.PARTIAL
|
|
736
|
+
and method == "HEAD"
|
|
737
|
+
and route.methods
|
|
738
|
+
and "GET" in route.methods
|
|
739
|
+
):
|
|
740
|
+
return route
|
|
741
|
+
return None
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
def create_app(
|
|
745
|
+
workflow_dir="./workflows",
|
|
746
|
+
output_dir="./outputs",
|
|
747
|
+
log_level="INFO",
|
|
748
|
+
job_manager=None,
|
|
749
|
+
ui_dir=None,
|
|
750
|
+
download_manager=None,
|
|
751
|
+
diffusers_updater=None,
|
|
752
|
+
prompt_dir="./prompts",
|
|
753
|
+
asset_dir=None,
|
|
754
|
+
examples_dirs=None,
|
|
755
|
+
workspace=None,
|
|
756
|
+
host="127.0.0.1",
|
|
757
|
+
token=None,
|
|
758
|
+
mcp=False,
|
|
759
|
+
port=8765,
|
|
760
|
+
):
|
|
761
|
+
"""Build the application. A caller (tests) can inject a JobManager.
|
|
762
|
+
|
|
763
|
+
`host` is the address the server is bound to (informational here - it
|
|
764
|
+
is added to the Host-header allowlist alongside the loopback names, so
|
|
765
|
+
a deployment bound to one specific non-loopback address still accepts
|
|
766
|
+
its own requests). `token`, if given, is a static bearer token required
|
|
767
|
+
on every /api/* request - see require_bearer_token below.
|
|
768
|
+
"""
|
|
769
|
+
manager = job_manager or JobManager(
|
|
770
|
+
output_dir, log_level=log_level, workflow_dir=workflow_dir
|
|
771
|
+
)
|
|
772
|
+
# An injected manager must confine jobs to the same workflow_dir the
|
|
773
|
+
# routes do, or /api/validate and /api/jobs would enforce different
|
|
774
|
+
# boundaries
|
|
775
|
+
if manager.workflow_dir is None:
|
|
776
|
+
manager.workflow_dir = workflow_dir
|
|
777
|
+
elif manager.workflow_dir != workflow_dir:
|
|
778
|
+
raise ValueError(
|
|
779
|
+
"job_manager.workflow_dir must match the app's workflow_dir: "
|
|
780
|
+
f"{manager.workflow_dir!r} != {workflow_dir!r}"
|
|
781
|
+
)
|
|
782
|
+
|
|
783
|
+
mcp_asgi = mcp_server = mcp_client = None
|
|
784
|
+
if mcp:
|
|
785
|
+
from .mcp_mount import build_mcp_app
|
|
786
|
+
|
|
787
|
+
mcp_asgi, mcp_server, mcp_client = build_mcp_app(
|
|
788
|
+
host=host, port=port, token=token
|
|
789
|
+
)
|
|
790
|
+
|
|
791
|
+
@asynccontextmanager
|
|
792
|
+
async def lifespan(app):
|
|
793
|
+
if mcp_server is None:
|
|
794
|
+
yield
|
|
795
|
+
else:
|
|
796
|
+
# the SDK's session manager is the mounted app's own lifespan,
|
|
797
|
+
# which Starlette does not run for a sub-app
|
|
798
|
+
try:
|
|
799
|
+
async with mcp_server.session_manager.run():
|
|
800
|
+
yield
|
|
801
|
+
finally:
|
|
802
|
+
# the client owns a connection pool; a session manager that
|
|
803
|
+
# fails to start must not leak it
|
|
804
|
+
mcp_client.close()
|
|
805
|
+
manager.shutdown()
|
|
806
|
+
|
|
807
|
+
app = FastAPI(
|
|
808
|
+
title="diffusers-workflow",
|
|
809
|
+
description="Declarative diffusers workflows over HTTP: queue a job, "
|
|
810
|
+
"stream its progress, fetch what it saved.",
|
|
811
|
+
lifespan=lifespan,
|
|
812
|
+
)
|
|
813
|
+
app.state.job_manager = manager
|
|
814
|
+
# This box's own job history as a cost, recomputed when the jobs table
|
|
815
|
+
# moves rather than when a file does - a job landing changes every
|
|
816
|
+
# figure and changes no workflow file (#93)
|
|
817
|
+
app.state.observed_costs = ObservedCosts(getattr(manager, "history", None))
|
|
818
|
+
app.state.workflow_dir = workflow_dir
|
|
819
|
+
# The search path: the writable directory first, then read-only roots -
|
|
820
|
+
# any --examples-dir, then the packaged builtins. Reads span all of it,
|
|
821
|
+
# saves only ever reach the front
|
|
822
|
+
app.state.workflow_sources = workflow_sources(workflow_dir, examples_dirs)
|
|
823
|
+
app.state.prompt_dir = prompt_dir
|
|
824
|
+
# The read-only libraries the --examples-dir trees bring with them: an
|
|
825
|
+
# example workflow references the prompts and assets that live beside
|
|
826
|
+
# its tree, not the ones in this workspace. They are searched after the
|
|
827
|
+
# workspace's own and never written to - a save of an example prompt
|
|
828
|
+
# lands in the workspace, the way saving an example workflow does
|
|
829
|
+
_example_libraries = example_libraries(examples_dirs)
|
|
830
|
+
app.state.example_prompt_dirs = _example_libraries[PROMPTS_SUBDIR]
|
|
831
|
+
app.state.example_asset_dirs = _example_libraries[ASSETS_SUBDIR]
|
|
832
|
+
# Where uploads land and 'asset:' references resolve. None when the
|
|
833
|
+
# caller configured no asset library: uploads then fall back to the
|
|
834
|
+
# output directory's uploads/ subfolder, as they did before there was one
|
|
835
|
+
app.state.asset_dir = os.path.abspath(asset_dir) if asset_dir else None
|
|
836
|
+
# The workspace the three directories above default to folders of, for a
|
|
837
|
+
# client that wants to name the root rather than reason about the parts.
|
|
838
|
+
# None when the caller resolved no workspace (a test building an app
|
|
839
|
+
# around three explicit directories)
|
|
840
|
+
app.state.workspace = os.path.abspath(workspace) if workspace else None
|
|
841
|
+
# The root that holds named workspaces. Its own folders are the default
|
|
842
|
+
# workspace - which is what the three directories above already point at,
|
|
843
|
+
# so a server given individual directory overrides simply has one
|
|
844
|
+
# workspace and no others
|
|
845
|
+
app.state.workspace_root = (
|
|
846
|
+
Workspace(app.state.workspace, "flag") if app.state.workspace else None
|
|
847
|
+
)
|
|
848
|
+
# The default workspace itself, as a Workspace: its four folders are the
|
|
849
|
+
# configured directories above, not '<root>/workflows' and friends - a
|
|
850
|
+
# caller can override any one of them individually (--workflow-dir,
|
|
851
|
+
# etc), so they cannot be derived from a root the way a named
|
|
852
|
+
# workspace's folders are
|
|
853
|
+
app.state.default_workspace = ConfiguredWorkspace(
|
|
854
|
+
workflows=app.state.workflow_dir,
|
|
855
|
+
assets=app.state.asset_dir,
|
|
856
|
+
outputs=manager.output_dir,
|
|
857
|
+
prompts=app.state.prompt_dir,
|
|
858
|
+
root=app.state.workspace,
|
|
859
|
+
)
|
|
860
|
+
app.state.mcp_mounted = mcp_asgi is not None
|
|
861
|
+
# A StaticFiles instance per output/asset root, built lazily and reused -
|
|
862
|
+
# a mount is bound to one directory at startup, but a named workspace's
|
|
863
|
+
# root does not exist yet then. Keeping the instance around (rather than
|
|
864
|
+
# building one per request) is what makes /outputs and /inputs answer
|
|
865
|
+
# ETag/If-None-Match with 304 and Range with 206 the way a real mount
|
|
866
|
+
# does, instead of the plain FileResponse this replaced always resending
|
|
867
|
+
# the whole file
|
|
868
|
+
app.state.static_files_by_root = {}
|
|
869
|
+
|
|
870
|
+
wildcard_bind = host in WILDCARD_HOSTS
|
|
871
|
+
allowed_hosts = set(LOOPBACK_HOSTS)
|
|
872
|
+
if host and not wildcard_bind:
|
|
873
|
+
allowed_hosts.add(host.lower())
|
|
874
|
+
|
|
875
|
+
@app.middleware("http")
|
|
876
|
+
async def reject_foreign_origins(request, call_next):
|
|
877
|
+
"""Refuse browser cross-origin requests - a drive-by web page must
|
|
878
|
+
not be able to queue jobs on this server. Requests without an
|
|
879
|
+
Origin header (curl, scripts, same-origin GETs) pass.
|
|
880
|
+
|
|
881
|
+
An Origin is accepted when its hostname is a loopback name, the
|
|
882
|
+
configured bind host, or the hostname the request itself was
|
|
883
|
+
addressed to (same-origin). The last clause is what lets a browser
|
|
884
|
+
on another machine use a `--host 0.0.0.0` server by its LAN IP or
|
|
885
|
+
hostname - and it stays safe against DNS rebinding, where the
|
|
886
|
+
attacker's page carries its own Origin while Host is whatever
|
|
887
|
+
resolved: the two differ, so the request is refused. Scheme and
|
|
888
|
+
port are ignored, matching the Host check: a TLS-terminating proxy
|
|
889
|
+
forwards Host unchanged while the browser's Origin is https."""
|
|
890
|
+
origin = request.headers.get("origin")
|
|
891
|
+
if origin:
|
|
892
|
+
try:
|
|
893
|
+
origin_host = (urlparse(origin).hostname or "").lower()
|
|
894
|
+
except ValueError:
|
|
895
|
+
# urlparse raises on a bracketed host that is not IPv6
|
|
896
|
+
# ('http://[::1].evil.example'): refused like any other
|
|
897
|
+
# foreign Origin rather than escaping as a 500
|
|
898
|
+
return JSONResponse(
|
|
899
|
+
status_code=403,
|
|
900
|
+
content={"detail": "Cross-origin requests are not allowed"},
|
|
901
|
+
)
|
|
902
|
+
request_host = (request.url.hostname or "").lower()
|
|
903
|
+
# origin_host must be non-empty for the same-origin clause:
|
|
904
|
+
# `Origin: null` (a sandboxed iframe, a file:// page) parses to
|
|
905
|
+
# no hostname and would otherwise match a request whose Host
|
|
906
|
+
# carries none either
|
|
907
|
+
if origin_host not in allowed_hosts and not (
|
|
908
|
+
origin_host and origin_host == request_host
|
|
909
|
+
):
|
|
910
|
+
return JSONResponse(
|
|
911
|
+
status_code=403,
|
|
912
|
+
content={"detail": "Cross-origin requests are not allowed"},
|
|
913
|
+
)
|
|
914
|
+
return await call_next(request)
|
|
915
|
+
|
|
916
|
+
# Defense-in-depth for requests that carry no Origin at all (curl,
|
|
917
|
+
# scripts, the MCP client) and so skip the check above entirely: a
|
|
918
|
+
# request that arrived on this port but claims to be addressed to some
|
|
919
|
+
# unrelated public domain is rejected. This does not stop DNS rebinding
|
|
920
|
+
# by itself (the Origin check already does, since a browser's Origin
|
|
921
|
+
# header reflects the real requesting origin regardless of DNS) - it
|
|
922
|
+
# only closes the gap for non-browser clients that never send Origin.
|
|
923
|
+
# A wildcard bind is reached by whatever address the machine has - a LAN
|
|
924
|
+
# IP, a hostname - never by the bind string itself, so there is no
|
|
925
|
+
# allowlist to build; the Host check is skipped for it.
|
|
926
|
+
@app.middleware("http")
|
|
927
|
+
async def reject_foreign_hosts(request, call_next):
|
|
928
|
+
hostname = request.url.hostname
|
|
929
|
+
if (
|
|
930
|
+
not wildcard_bind
|
|
931
|
+
and hostname is not None
|
|
932
|
+
and hostname.lower() not in allowed_hosts
|
|
933
|
+
):
|
|
934
|
+
return JSONResponse(
|
|
935
|
+
status_code=400,
|
|
936
|
+
content={"detail": "Unrecognized Host header"},
|
|
937
|
+
)
|
|
938
|
+
return await call_next(request)
|
|
939
|
+
|
|
940
|
+
@app.middleware("http")
|
|
941
|
+
async def require_bearer_token(request: Request, call_next):
|
|
942
|
+
"""Static bearer-token auth (opt-in via --token / DW_API_TOKEN).
|
|
943
|
+
Only /api/* is gated - the UI's own static files and /outputs (an
|
|
944
|
+
<img>/<script> tag cannot attach an Authorization header anyway)
|
|
945
|
+
stay reachable so the page can load far enough to let a user enter
|
|
946
|
+
the token in the first place. EventSource cannot set custom headers
|
|
947
|
+
either, and neither can the <img> tags the gallery grid loads its
|
|
948
|
+
thumbnails through nor the <a download> navigations the download
|
|
949
|
+
buttons make, so those GET routes additionally accept the token as a
|
|
950
|
+
`token` query parameter - a documented trade-off, not a header-auth
|
|
951
|
+
peer."""
|
|
952
|
+
if not token:
|
|
953
|
+
return await call_next(request)
|
|
954
|
+
path = request.url.path
|
|
955
|
+
if not (path.startswith("/api/") or path == "/mcp" or path.startswith("/mcp/")):
|
|
956
|
+
return await call_next(request)
|
|
957
|
+
provided = None
|
|
958
|
+
auth = request.headers.get("authorization", "")
|
|
959
|
+
if auth.lower().startswith("bearer "):
|
|
960
|
+
provided = auth[len("bearer ") :].strip()
|
|
961
|
+
# GET/HEAD only, and only on a route explicitly marked
|
|
962
|
+
# query_token_ok - matched against the real route (see
|
|
963
|
+
# _matched_route), not by a path suffix, so a resource that
|
|
964
|
+
# happens to be named "download" or "thumbnail" does not inherit
|
|
965
|
+
# the allowance meant for the real routes.
|
|
966
|
+
if provided is None and request.method in ("GET", "HEAD"):
|
|
967
|
+
route = _matched_route(request)
|
|
968
|
+
if route is not None and getattr(route.endpoint, "query_token_ok", False):
|
|
969
|
+
provided = request.query_params.get("token")
|
|
970
|
+
# compared as bytes: compare_digest refuses non-ASCII str
|
|
971
|
+
if provided is None or not secrets.compare_digest(
|
|
972
|
+
provided.encode("utf-8"), token.encode("utf-8")
|
|
973
|
+
):
|
|
974
|
+
return JSONResponse(
|
|
975
|
+
status_code=401,
|
|
976
|
+
content={"detail": "Missing or invalid bearer token"},
|
|
977
|
+
)
|
|
978
|
+
return await call_next(request)
|
|
979
|
+
|
|
980
|
+
# Added last, so it is the outermost middleware and its headers land on
|
|
981
|
+
# every response - including the 400/401/403 answers the checks above
|
|
982
|
+
# return without reaching a route. nosniff stops a browser reading an
|
|
983
|
+
# output as a type other than the one it was served as; DENY stops any
|
|
984
|
+
# other site framing the UI to click its buttons (#407)
|
|
985
|
+
@app.middleware("http")
|
|
986
|
+
async def browser_headers(request, call_next):
|
|
987
|
+
response = await call_next(request)
|
|
988
|
+
response.headers.setdefault("X-Content-Type-Options", "nosniff")
|
|
989
|
+
response.headers.setdefault("X-Frame-Options", "DENY")
|
|
990
|
+
return response
|
|
991
|
+
|
|
992
|
+
# -------------------------------------------------------- workspace lookup
|
|
993
|
+
|
|
994
|
+
def _workspace_root():
|
|
995
|
+
root = app.state.workspace_root
|
|
996
|
+
if root is None:
|
|
997
|
+
raise HTTPException(
|
|
998
|
+
status_code=409,
|
|
999
|
+
detail="This server has no workspace root - it was started "
|
|
1000
|
+
"with individual directory overrides, so it has one "
|
|
1001
|
+
"workspace and cannot create others",
|
|
1002
|
+
)
|
|
1003
|
+
return root
|
|
1004
|
+
|
|
1005
|
+
def _workspace_for(name):
|
|
1006
|
+
"""The Workspace a request names.
|
|
1007
|
+
|
|
1008
|
+
No name, or the default name, is the server's own configuration -
|
|
1009
|
+
the directories it was started with - so every call that predates
|
|
1010
|
+
workspaces keeps working unchanged. A named one resolves under the
|
|
1011
|
+
root, and must already exist: creating a workspace by mentioning it
|
|
1012
|
+
would turn a typo into a directory. Checked by looking at the one
|
|
1013
|
+
candidate directory rather than listing the whole root - this runs
|
|
1014
|
+
on every gallery thumbnail request.
|
|
1015
|
+
"""
|
|
1016
|
+
if not name or name == DEFAULT_WORKSPACE_NAME:
|
|
1017
|
+
return app.state.default_workspace
|
|
1018
|
+
root = _workspace_root()
|
|
1019
|
+
try:
|
|
1020
|
+
selected = named_workspace(root, name)
|
|
1021
|
+
except SecurityError as e:
|
|
1022
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
1023
|
+
if not _holds_a_workspace(selected.root):
|
|
1024
|
+
raise HTTPException(status_code=404, detail=f"No such workspace: {name}")
|
|
1025
|
+
return selected
|
|
1026
|
+
|
|
1027
|
+
def selected_workspace(workspace: Optional[str] = None) -> Workspace:
|
|
1028
|
+
"""FastAPI dependency form of _workspace_for, reading the name from
|
|
1029
|
+
the `?workspace=` query parameter every scoped route already takes -
|
|
1030
|
+
used as `ws: Workspace = Depends(selected_workspace)`."""
|
|
1031
|
+
return _workspace_for(workspace)
|
|
1032
|
+
|
|
1033
|
+
def _sources_for(ws):
|
|
1034
|
+
"""The workflow search path of one workspace: its own workflows
|
|
1035
|
+
first, then the same read-only roots every workspace shares."""
|
|
1036
|
+
return workflow_sources(ws.workflows, examples_dirs)
|
|
1037
|
+
|
|
1038
|
+
# ------------------------------------------------------------------ jobs
|
|
1039
|
+
|
|
1040
|
+
def _acknowledgement_form(value):
|
|
1041
|
+
"""none | boolean | bound - classified once, here, so the check and
|
|
1042
|
+
the record agree (#85)."""
|
|
1043
|
+
if isinstance(value, AcknowledgedCost):
|
|
1044
|
+
return ACK_BOUND
|
|
1045
|
+
return ACK_BOOLEAN if value is True else ACK_NONE
|
|
1046
|
+
|
|
1047
|
+
def _check_bound_acknowledgement(candidate, arguments, acknowledged, workspace):
|
|
1048
|
+
"""Refuse with 409 when the run `candidate` + `arguments` will
|
|
1049
|
+
execute is not the one `acknowledged` was bound to: a different
|
|
1050
|
+
fingerprint, or a download the caller did not acknowledge. The body
|
|
1051
|
+
carries the current plan so the agent re-quotes from it without a
|
|
1052
|
+
second validate call. A plan that cannot be built is a refusal too -
|
|
1053
|
+
never a silent pass (#85).
|
|
1054
|
+
"""
|
|
1055
|
+
record = acknowledged.model_dump()
|
|
1056
|
+
|
|
1057
|
+
def refuse(message, reason, plan):
|
|
1058
|
+
raise HTTPException(
|
|
1059
|
+
status_code=409,
|
|
1060
|
+
detail={
|
|
1061
|
+
"message": message,
|
|
1062
|
+
"reason": reason,
|
|
1063
|
+
"acknowledged": record,
|
|
1064
|
+
"plan": plan,
|
|
1065
|
+
},
|
|
1066
|
+
)
|
|
1067
|
+
|
|
1068
|
+
try:
|
|
1069
|
+
from .. import get_device, get_device_type
|
|
1070
|
+
|
|
1071
|
+
current = build_plan(
|
|
1072
|
+
candidate,
|
|
1073
|
+
arguments,
|
|
1074
|
+
device=get_device_type(get_device()),
|
|
1075
|
+
prompt_dir=workspace.prompts,
|
|
1076
|
+
lookup_sizes=False,
|
|
1077
|
+
)
|
|
1078
|
+
except Exception:
|
|
1079
|
+
logger.exception("Plan could not be built for a bound acknowledgement")
|
|
1080
|
+
refuse(
|
|
1081
|
+
"The run could not be planned, so a bound acknowledgement "
|
|
1082
|
+
"cannot be checked; acknowledge with true or validate again",
|
|
1083
|
+
"unplannable",
|
|
1084
|
+
None,
|
|
1085
|
+
)
|
|
1086
|
+
current["workspace"] = workspace.name
|
|
1087
|
+
current["output_dir"] = workspace.outputs
|
|
1088
|
+
if current["fingerprint"] != acknowledged.fingerprint:
|
|
1089
|
+
refuse(
|
|
1090
|
+
"The run's shape changed since it was acknowledged: the "
|
|
1091
|
+
"workflow or its arguments differ from what was validated.",
|
|
1092
|
+
"fingerprint",
|
|
1093
|
+
current,
|
|
1094
|
+
)
|
|
1095
|
+
missing = [
|
|
1096
|
+
entry["repo"]
|
|
1097
|
+
for entry in current["downloads_required"]
|
|
1098
|
+
if entry.get("repo") and entry["repo"] not in acknowledged.downloads
|
|
1099
|
+
]
|
|
1100
|
+
if missing:
|
|
1101
|
+
refuse(
|
|
1102
|
+
"The run's shape changed since it was acknowledged: it now "
|
|
1103
|
+
f"has to download {', '.join(missing)} first",
|
|
1104
|
+
"downloads",
|
|
1105
|
+
current,
|
|
1106
|
+
)
|
|
1107
|
+
|
|
1108
|
+
def _candidate_for(
|
|
1109
|
+
workflow_path, workflow, base_dir, output_dir, workflow_dir, arguments
|
|
1110
|
+
):
|
|
1111
|
+
"""The Workflow a job spec names, built and checked as submit() will
|
|
1112
|
+
build and check it - schema first, then the caller's arguments -
|
|
1113
|
+
so a bound acknowledgement never turns the caller's 400 into a 409
|
|
1114
|
+
telling them to acknowledge with true and find out.
|
|
1115
|
+
|
|
1116
|
+
Raises what submit() raises (ValueError, SecurityError, ...), which
|
|
1117
|
+
the routes already answer as 400.
|
|
1118
|
+
"""
|
|
1119
|
+
if workflow_path is not None:
|
|
1120
|
+
candidate = workflow_from_file(workflow_path, output_dir, workflow_dir)
|
|
1121
|
+
else:
|
|
1122
|
+
candidate = workflow_from_definition(
|
|
1123
|
+
copy.deepcopy(workflow), output_dir, base_dir, workflow_dir
|
|
1124
|
+
)
|
|
1125
|
+
# Checked against the caller's arguments, not the document alone -
|
|
1126
|
+
# validate_workflow's candidate.validation_errors(arguments=...) is
|
|
1127
|
+
# what catches a content_type (or reference_name, video_extension,
|
|
1128
|
+
# ...) that only becomes active once a 'variable:' resolves; a bare
|
|
1129
|
+
# candidate.validate() checked the document with no arguments and so
|
|
1130
|
+
# queued a job validate_workflow had already refused for the same
|
|
1131
|
+
# call (#414)
|
|
1132
|
+
problems = candidate.validation_errors(arguments=arguments)
|
|
1133
|
+
problems += argument_errors(candidate.workflow_definition, arguments)
|
|
1134
|
+
# A value outside a rule the workflow declares, refused before the
|
|
1135
|
+
# job id rather than after the weights are loaded (#96)
|
|
1136
|
+
problems += constraint_errors(
|
|
1137
|
+
candidate.workflow_definition, arguments, supplied=set(arguments or {})
|
|
1138
|
+
)
|
|
1139
|
+
if problems:
|
|
1140
|
+
raise ValueError(
|
|
1141
|
+
"; ".join(
|
|
1142
|
+
f"{problem['path']}: {problem['message']}" for problem in problems
|
|
1143
|
+
)
|
|
1144
|
+
)
|
|
1145
|
+
return candidate
|
|
1146
|
+
|
|
1147
|
+
@app.post("/api/jobs", status_code=201)
|
|
1148
|
+
def submit_job(request: JobRequest, ws: Workspace = Depends(selected_workspace)):
|
|
1149
|
+
"""Queue a workflow. The workspace it runs in comes from the body or,
|
|
1150
|
+
for a client that scopes every call the same way, the query string -
|
|
1151
|
+
the body wins when both are given."""
|
|
1152
|
+
try:
|
|
1153
|
+
workspace = _workspace_for(request.workspace or ws.name)
|
|
1154
|
+
sources = _sources_for(workspace)
|
|
1155
|
+
resolved, source = resolve_workflow_reference(
|
|
1156
|
+
request.workflow_path, sources
|
|
1157
|
+
)
|
|
1158
|
+
# Built unconditionally - both the reference check below and a
|
|
1159
|
+
# bound acknowledgement (further down) need the definition a run
|
|
1160
|
+
# would actually use, and resolve_workflow_reference already
|
|
1161
|
+
# returns (None, None) for an inline definition, which
|
|
1162
|
+
# _candidate_for handles the same way _candidate_for always has
|
|
1163
|
+
candidate = _candidate_for(
|
|
1164
|
+
resolved,
|
|
1165
|
+
request.workflow,
|
|
1166
|
+
request.base_dir,
|
|
1167
|
+
workspace.outputs,
|
|
1168
|
+
source.root if source else workspace.workflows,
|
|
1169
|
+
request.arguments,
|
|
1170
|
+
)
|
|
1171
|
+
# The same reference check POST /api/validate makes, because a
|
|
1172
|
+
# caller who skipped the free pre-flight should still not get a
|
|
1173
|
+
# job id for an argument that cannot resolve. The name half of
|
|
1174
|
+
# this check lives in JobManager.submit, where the definition is
|
|
1175
|
+
# loaded; this half needs the workspace's search path, which is
|
|
1176
|
+
# here - which is why a bad 'asset:' used to queue and die on the
|
|
1177
|
+
# first step while a bad variable name was refused outright
|
|
1178
|
+
reference_problems = _argument_reference_errors(
|
|
1179
|
+
candidate.workflow_definition, request.arguments, workspace
|
|
1180
|
+
)
|
|
1181
|
+
if reference_problems:
|
|
1182
|
+
raise ValueError(
|
|
1183
|
+
"; ".join(
|
|
1184
|
+
f"{problem['path']}: {problem['message']}"
|
|
1185
|
+
for problem in reference_problems
|
|
1186
|
+
)
|
|
1187
|
+
)
|
|
1188
|
+
# A bound acknowledgement is checked against the plan this
|
|
1189
|
+
# request would run - before anything is queued, since a refusal
|
|
1190
|
+
# is free here and costs a job id anywhere later (#85)
|
|
1191
|
+
form = _acknowledgement_form(request.acknowledged_cost)
|
|
1192
|
+
if form == ACK_BOUND:
|
|
1193
|
+
_check_bound_acknowledgement(
|
|
1194
|
+
candidate, request.arguments, request.acknowledged_cost, workspace
|
|
1195
|
+
)
|
|
1196
|
+
job = manager.submit(
|
|
1197
|
+
workflow_path=resolved,
|
|
1198
|
+
workflow=request.workflow,
|
|
1199
|
+
arguments=request.arguments,
|
|
1200
|
+
base_dir=request.base_dir,
|
|
1201
|
+
# The root this run is confined to: the source the workflow
|
|
1202
|
+
# came from, so an example runs where it lives while an
|
|
1203
|
+
# inline definition stays held to this workspace's own
|
|
1204
|
+
# workflows
|
|
1205
|
+
workflow_dir=source.root if source else workspace.workflows,
|
|
1206
|
+
# The roots this job runs against, so it stays in its
|
|
1207
|
+
# workspace however many others the server serves meanwhile
|
|
1208
|
+
output_dir=workspace.outputs,
|
|
1209
|
+
asset_dir=workspace.assets,
|
|
1210
|
+
workspace=workspace.name,
|
|
1211
|
+
# The listing name, when the request came as one - what a
|
|
1212
|
+
# later runtime-by-workflow report joins on. Derived from
|
|
1213
|
+
# the resolved path rather than echoing what was asked
|
|
1214
|
+
# for, so 'Basic', 'Basic.json' and an absolute path
|
|
1215
|
+
# inside the source all record the one catalog name
|
|
1216
|
+
catalog_name=catalog_name_for(resolved, source),
|
|
1217
|
+
acknowledged=form,
|
|
1218
|
+
acknowledged_cost=(
|
|
1219
|
+
request.acknowledged_cost.model_dump()
|
|
1220
|
+
if form == ACK_BOUND
|
|
1221
|
+
else None
|
|
1222
|
+
),
|
|
1223
|
+
)
|
|
1224
|
+
except HTTPException:
|
|
1225
|
+
raise
|
|
1226
|
+
except Exception as e:
|
|
1227
|
+
# workflow_from_file / validate / the security layer all raise for
|
|
1228
|
+
# bad requests - every failure here is the client's fault
|
|
1229
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
1230
|
+
return manager.describe(job)
|
|
1231
|
+
|
|
1232
|
+
@app.get("/api/jobs")
|
|
1233
|
+
def list_jobs(
|
|
1234
|
+
workspace: Optional[str] = None,
|
|
1235
|
+
status: Optional[str] = None,
|
|
1236
|
+
limit: Optional[int] = None,
|
|
1237
|
+
):
|
|
1238
|
+
"""All jobs by default - a plain filter, not `selected_workspace`,
|
|
1239
|
+
since the jobs list spans every workspace the server holds unless a
|
|
1240
|
+
caller asks to narrow it.
|
|
1241
|
+
|
|
1242
|
+
`status` narrows to one state or a comma-separated set of them.
|
|
1243
|
+
`limit` keeps the newest N, and `total` always reports how many
|
|
1244
|
+
matched before the cut, so a caller can tell a bounded answer from a
|
|
1245
|
+
complete one. The default is still every matching job, oldest first -
|
|
1246
|
+
what the web UI polls."""
|
|
1247
|
+
statuses = [part.strip() for part in status.split(",")] if status else None
|
|
1248
|
+
statuses = [part for part in statuses if part] if statuses else None
|
|
1249
|
+
if statuses:
|
|
1250
|
+
unknown = [
|
|
1251
|
+
state
|
|
1252
|
+
for state in statuses
|
|
1253
|
+
if state not in (QUEUED, RUNNING, *TERMINAL_STATES)
|
|
1254
|
+
]
|
|
1255
|
+
if unknown:
|
|
1256
|
+
raise HTTPException(
|
|
1257
|
+
status_code=400,
|
|
1258
|
+
detail=f"Unknown job status {', '.join(unknown)} - one of "
|
|
1259
|
+
f"{', '.join((QUEUED, RUNNING, *TERMINAL_STATES))}",
|
|
1260
|
+
)
|
|
1261
|
+
jobs = manager.list(workspace=workspace, statuses=statuses)
|
|
1262
|
+
total = len(jobs)
|
|
1263
|
+
if limit is not None:
|
|
1264
|
+
if limit < 0:
|
|
1265
|
+
raise HTTPException(
|
|
1266
|
+
status_code=400, detail="limit must not be negative"
|
|
1267
|
+
)
|
|
1268
|
+
# the newest are the interesting ones, and the list is oldest
|
|
1269
|
+
# first - so the cut comes off the front, not the back. max(0, ...)
|
|
1270
|
+
# because a limit above what matched is no cut at all: a bare
|
|
1271
|
+
# negative start would be read from the end instead, and answer a
|
|
1272
|
+
# limit of 12 against 9 matching jobs with the last 3 of them
|
|
1273
|
+
jobs = jobs[max(0, len(jobs) - limit) :] if limit else []
|
|
1274
|
+
return {"jobs": jobs, "total": total}
|
|
1275
|
+
|
|
1276
|
+
@app.get("/api/jobs/{job_id}")
|
|
1277
|
+
def get_job(job_id: str):
|
|
1278
|
+
job = manager.get(job_id)
|
|
1279
|
+
if job is None:
|
|
1280
|
+
raise HTTPException(status_code=404, detail="Unknown job")
|
|
1281
|
+
# a historical job is already a detail dict; a live one renders itself
|
|
1282
|
+
return job if isinstance(job, dict) else manager.describe(job)
|
|
1283
|
+
|
|
1284
|
+
@app.get("/api/jobs/{job_id}/workflow")
|
|
1285
|
+
def get_job_workflow(job_id: str):
|
|
1286
|
+
"""The workflow this job ran, for the read-only graph on the job page
|
|
1287
|
+
and for `get_job_workflow` over MCP.
|
|
1288
|
+
|
|
1289
|
+
`realized: true` means every mutable input is pinned - the copy the
|
|
1290
|
+
run itself wrote. `false` means the job predates run tracking (or its
|
|
1291
|
+
run directory is gone) and this is the definition as submitted. 404
|
|
1292
|
+
when neither is readable - the job itself still is."""
|
|
1293
|
+
if manager.get(job_id) is None:
|
|
1294
|
+
raise HTTPException(status_code=404, detail="Unknown job")
|
|
1295
|
+
realized = manager.realized(job_id)
|
|
1296
|
+
definition = realized if realized is not None else manager.definition(job_id)
|
|
1297
|
+
if definition is None:
|
|
1298
|
+
raise HTTPException(
|
|
1299
|
+
status_code=404, detail="No workflow definition for this job"
|
|
1300
|
+
)
|
|
1301
|
+
return {
|
|
1302
|
+
"id": job_id,
|
|
1303
|
+
"definition": definition,
|
|
1304
|
+
"realized": realized is not None,
|
|
1305
|
+
# Which variable a new-seed rerun would draw into, or null when
|
|
1306
|
+
# there is none - read from the workflow as written, since the
|
|
1307
|
+
# realized copy above has its seed pinned to the integer it used
|
|
1308
|
+
"seed_variable": manager.seed_variable(job_id),
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
class RerunRequest(BaseModel):
|
|
1312
|
+
new_seed: bool = Field(
|
|
1313
|
+
default=False,
|
|
1314
|
+
description="Draw a fresh seed into the workflow's seed variable. "
|
|
1315
|
+
"Without it a rerun repeats the original arguments exactly, which "
|
|
1316
|
+
"the step cache serves from the earlier run - the same seed and "
|
|
1317
|
+
"inputs would produce the same files.",
|
|
1318
|
+
)
|
|
1319
|
+
acknowledged_cost: Optional[Union[bool, AcknowledgedCost]] = (
|
|
1320
|
+
ACKNOWLEDGED_COST_FIELD
|
|
1321
|
+
)
|
|
1322
|
+
|
|
1323
|
+
@app.post("/api/jobs/{job_id}/rerun", status_code=201)
|
|
1324
|
+
def rerun_job(job_id: str, body: RerunRequest = RerunRequest()):
|
|
1325
|
+
"""Queue a fresh job from a previous job's stored spec. Takes
|
|
1326
|
+
`acknowledged_cost` as POST /api/jobs does; a bound one is checked
|
|
1327
|
+
against the stored spec's plan - the fresh seed of `new_seed` does
|
|
1328
|
+
not change a fingerprint."""
|
|
1329
|
+
form = _acknowledgement_form(body.acknowledged_cost)
|
|
1330
|
+
if form == ACK_BOUND:
|
|
1331
|
+
prepared = manager.rerun_spec(job_id)
|
|
1332
|
+
if prepared is None:
|
|
1333
|
+
raise HTTPException(status_code=404, detail="Unknown job")
|
|
1334
|
+
spec, arguments = prepared
|
|
1335
|
+
try:
|
|
1336
|
+
candidate = _candidate_for(
|
|
1337
|
+
spec.get("workflow_path"),
|
|
1338
|
+
spec.get("workflow"),
|
|
1339
|
+
spec.get("base_dir"),
|
|
1340
|
+
spec.get("output_dir") or manager.output_dir,
|
|
1341
|
+
spec.get("workflow_dir"),
|
|
1342
|
+
arguments,
|
|
1343
|
+
)
|
|
1344
|
+
except Exception as e:
|
|
1345
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
1346
|
+
_check_bound_acknowledgement(
|
|
1347
|
+
candidate,
|
|
1348
|
+
arguments,
|
|
1349
|
+
body.acknowledged_cost,
|
|
1350
|
+
_workspace_for(spec.get("workspace")),
|
|
1351
|
+
)
|
|
1352
|
+
try:
|
|
1353
|
+
job = manager.rerun(
|
|
1354
|
+
job_id,
|
|
1355
|
+
new_seed=body.new_seed,
|
|
1356
|
+
acknowledged=form,
|
|
1357
|
+
acknowledged_cost=(
|
|
1358
|
+
body.acknowledged_cost.model_dump() if form == ACK_BOUND else None
|
|
1359
|
+
),
|
|
1360
|
+
)
|
|
1361
|
+
except HTTPException:
|
|
1362
|
+
raise
|
|
1363
|
+
except Exception as e:
|
|
1364
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
1365
|
+
if job is None:
|
|
1366
|
+
raise HTTPException(status_code=404, detail="Unknown job")
|
|
1367
|
+
return manager.describe(job)
|
|
1368
|
+
|
|
1369
|
+
@app.post("/api/jobs/{job_id}/export", status_code=201)
|
|
1370
|
+
def export_job_route(
|
|
1371
|
+
job_id: str,
|
|
1372
|
+
overwrite: bool = False,
|
|
1373
|
+
ws: Workspace = Depends(selected_workspace),
|
|
1374
|
+
):
|
|
1375
|
+
"""Gather one finished job into '<workspace>/exports/<job id>/': the
|
|
1376
|
+
workflow it ran, the run's manifest, the job row, the media it used
|
|
1377
|
+
and the media it made, plus a README. 404 for an unknown job, 409 for
|
|
1378
|
+
one still running or for an export that already exists without
|
|
1379
|
+
`overwrite`.
|
|
1380
|
+
|
|
1381
|
+
The three JSON files come back inline as well as on disk - the
|
|
1382
|
+
directory is on the server, and a client on another machine has no
|
|
1383
|
+
other way to read them without fetching the zip."""
|
|
1384
|
+
try:
|
|
1385
|
+
summary = export_job(
|
|
1386
|
+
manager,
|
|
1387
|
+
job_id,
|
|
1388
|
+
ws.root,
|
|
1389
|
+
_asset_roots_for_job(job_id, ws),
|
|
1390
|
+
overwrite=overwrite,
|
|
1391
|
+
)
|
|
1392
|
+
except FileExistsError as e:
|
|
1393
|
+
raise HTTPException(status_code=409, detail=str(e))
|
|
1394
|
+
except ValueError as e:
|
|
1395
|
+
message = str(e)
|
|
1396
|
+
if message.startswith("Unknown job"):
|
|
1397
|
+
raise HTTPException(status_code=404, detail=message)
|
|
1398
|
+
raise HTTPException(status_code=409, detail=message)
|
|
1399
|
+
body = summary.as_dict()
|
|
1400
|
+
zip_path = f"/exports/{quote(job_id)}.zip"
|
|
1401
|
+
body["zip_url"] = _served_url(zip_path, ws)
|
|
1402
|
+
absolute_zip_url = _absolute_served_url(zip_path, ws)
|
|
1403
|
+
if absolute_zip_url is not None:
|
|
1404
|
+
body["absolute_zip_url"] = absolute_zip_url
|
|
1405
|
+
# Same rule get_server_info's field states (#353): whether the zip
|
|
1406
|
+
# URL above needs a bearer token an MCP-only agent has no way to
|
|
1407
|
+
# attach itself, which is what tells the caller whether to fetch it
|
|
1408
|
+
# or hand it to the person.
|
|
1409
|
+
body["auth_required"] = bool(token)
|
|
1410
|
+
for key, name in (
|
|
1411
|
+
("workflow", "workflow.json"),
|
|
1412
|
+
("manifest", "manifest.json"),
|
|
1413
|
+
("job", "job.json"),
|
|
1414
|
+
):
|
|
1415
|
+
try:
|
|
1416
|
+
with open(os.path.join(summary.directory, name), "r") as file:
|
|
1417
|
+
body[key] = json.load(file)
|
|
1418
|
+
except (OSError, ValueError):
|
|
1419
|
+
body[key] = None
|
|
1420
|
+
return body
|
|
1421
|
+
|
|
1422
|
+
class MoveRequest(BaseModel):
|
|
1423
|
+
direction: str = Field(description="up, down, front, or back")
|
|
1424
|
+
|
|
1425
|
+
@app.post("/api/jobs/{job_id}/move")
|
|
1426
|
+
def move_job(job_id: str, body: MoveRequest):
|
|
1427
|
+
"""Reorder a queued job. 409 once it is running or finished -
|
|
1428
|
+
only the waiting portion of the queue can be rearranged."""
|
|
1429
|
+
if manager.get(job_id) is None:
|
|
1430
|
+
raise HTTPException(status_code=404, detail="Unknown job")
|
|
1431
|
+
try:
|
|
1432
|
+
order = manager.move(job_id, body.direction)
|
|
1433
|
+
except ValueError as e:
|
|
1434
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
1435
|
+
if order is None:
|
|
1436
|
+
raise HTTPException(
|
|
1437
|
+
status_code=409, detail="Job is not queued - only queued jobs move"
|
|
1438
|
+
)
|
|
1439
|
+
return {"id": job_id, "queue": order}
|
|
1440
|
+
|
|
1441
|
+
@app.post("/api/jobs/{job_id}/cancel")
|
|
1442
|
+
def cancel_job(job_id: str):
|
|
1443
|
+
status = manager.cancel(job_id)
|
|
1444
|
+
if status is None:
|
|
1445
|
+
raise HTTPException(status_code=404, detail="Unknown job")
|
|
1446
|
+
return {"id": job_id, "status": status}
|
|
1447
|
+
|
|
1448
|
+
@app.get("/api/jobs/{job_id}/events")
|
|
1449
|
+
@query_token_ok
|
|
1450
|
+
async def job_events(request: Request, job_id: str, after: int = -1):
|
|
1451
|
+
"""Server-sent events: every progress event from `after` (exclusive)
|
|
1452
|
+
until the job reaches a terminal state. Reconnect with the last seen
|
|
1453
|
+
seq (or let EventSource send Last-Event-ID) to resume without loss."""
|
|
1454
|
+
job = manager.get(job_id)
|
|
1455
|
+
if job is None:
|
|
1456
|
+
raise HTTPException(status_code=404, detail="Unknown job")
|
|
1457
|
+
if isinstance(job, dict):
|
|
1458
|
+
# historical jobs carry no event log - an immediately-closed
|
|
1459
|
+
# stream lets clients treat them uniformly
|
|
1460
|
+
return StreamingResponse(iter(()), media_type="text/event-stream")
|
|
1461
|
+
|
|
1462
|
+
last_event_id = request.headers.get("last-event-id")
|
|
1463
|
+
if last_event_id is not None:
|
|
1464
|
+
try:
|
|
1465
|
+
after = max(after, int(last_event_id))
|
|
1466
|
+
except ValueError:
|
|
1467
|
+
pass
|
|
1468
|
+
|
|
1469
|
+
async def stream():
|
|
1470
|
+
last_seq = after
|
|
1471
|
+
while True:
|
|
1472
|
+
events = job.events_after(last_seq)
|
|
1473
|
+
for event in events:
|
|
1474
|
+
last_seq = event["seq"]
|
|
1475
|
+
yield f"id: {event['seq']}\ndata: {json.dumps(event)}\n\n"
|
|
1476
|
+
if job.status in TERMINAL_STATES and not job.events_after(last_seq):
|
|
1477
|
+
return
|
|
1478
|
+
await asyncio.to_thread(job.wait_for_event, last_seq, SSE_POLL_SECONDS)
|
|
1479
|
+
|
|
1480
|
+
return StreamingResponse(
|
|
1481
|
+
stream(),
|
|
1482
|
+
media_type="text/event-stream",
|
|
1483
|
+
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
|
1484
|
+
)
|
|
1485
|
+
|
|
1486
|
+
@app.get("/api/jobs/{job_id}/event-log")
|
|
1487
|
+
def job_event_log(
|
|
1488
|
+
job_id: str,
|
|
1489
|
+
after: int = -1,
|
|
1490
|
+
limit: int = 200,
|
|
1491
|
+
kinds: list[str] | None = Query(None),
|
|
1492
|
+
):
|
|
1493
|
+
"""Job events as one JSON page rather than a stream, for clients that
|
|
1494
|
+
poll instead of holding a connection open (the MCP server). `after` is
|
|
1495
|
+
exclusive, matching the SSE route's parameter of the same name.
|
|
1496
|
+
`kinds` restricts the page to events whose `event` or `kind` is one
|
|
1497
|
+
of the named values (e.g. `log`, `warning`, `phase_stall`) - a consumer confirming what a step applied wants
|
|
1498
|
+
those two and not the `memory`/bookkeeping events that otherwise
|
|
1499
|
+
dominate the payload."""
|
|
1500
|
+
job = manager.get(job_id)
|
|
1501
|
+
if job is None:
|
|
1502
|
+
raise HTTPException(status_code=404, detail="Unknown job")
|
|
1503
|
+
limit = max(1, min(limit, 1000))
|
|
1504
|
+
if isinstance(job, dict):
|
|
1505
|
+
# A job restored from sqlite: history persists a bounded tail of
|
|
1506
|
+
# its events, so it can still explain itself after a restart
|
|
1507
|
+
status = job.get("status")
|
|
1508
|
+
stored = manager.history.events_for(job_id) or []
|
|
1509
|
+
pending = [event for event in stored if event.get("seq", -1) > after]
|
|
1510
|
+
note = _historical_log_note(stored)
|
|
1511
|
+
else:
|
|
1512
|
+
status = job.status
|
|
1513
|
+
pending = job.events_after(after)
|
|
1514
|
+
note = None
|
|
1515
|
+
pending = select_kinds(pending, kinds)
|
|
1516
|
+
page = pending[:limit]
|
|
1517
|
+
return {
|
|
1518
|
+
"id": job_id,
|
|
1519
|
+
"status": status,
|
|
1520
|
+
"events": page,
|
|
1521
|
+
"last_seq": page[-1]["seq"] if page else max(after, -1),
|
|
1522
|
+
# this page is cut short; `note` covers what record time dropped
|
|
1523
|
+
"truncated": len(pending) > len(page),
|
|
1524
|
+
"note": note,
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
# ---------------------------------------------------------- introspection
|
|
1528
|
+
|
|
1529
|
+
@app.get("/api/pipelines")
|
|
1530
|
+
def pipelines():
|
|
1531
|
+
"""Every pipeline class the installed diffusers exports."""
|
|
1532
|
+
return {"pipelines": list_pipelines()}
|
|
1533
|
+
|
|
1534
|
+
@app.get("/api/pipelines/{name}")
|
|
1535
|
+
def pipeline_description(name: str):
|
|
1536
|
+
"""A pipeline's __call__ argument schema, for form generation."""
|
|
1537
|
+
try:
|
|
1538
|
+
return describe_pipeline(name)
|
|
1539
|
+
except ValueError as e:
|
|
1540
|
+
raise HTTPException(status_code=404, detail=str(e))
|
|
1541
|
+
except Exception as e:
|
|
1542
|
+
# A pipeline whose import fails on this install (missing extra
|
|
1543
|
+
# dependency) is absent, not a server error
|
|
1544
|
+
raise HTTPException(status_code=404, detail=f"Could not load {name}: {e}")
|
|
1545
|
+
|
|
1546
|
+
@app.get("/api/tasks")
|
|
1547
|
+
def tasks():
|
|
1548
|
+
"""Every task command a workflow's task step can name."""
|
|
1549
|
+
return list_tasks()
|
|
1550
|
+
|
|
1551
|
+
@app.get("/api/tasks/{command}")
|
|
1552
|
+
def get_task(command: str):
|
|
1553
|
+
"""A task command's argument schema - the registered implementation
|
|
1554
|
+
function's real signature, in the same shape as a class description."""
|
|
1555
|
+
try:
|
|
1556
|
+
return describe_task(command)
|
|
1557
|
+
except ValueError as e:
|
|
1558
|
+
raise HTTPException(status_code=404, detail=str(e))
|
|
1559
|
+
|
|
1560
|
+
@app.get("/api/classes")
|
|
1561
|
+
def classes(kind: str):
|
|
1562
|
+
"""Class names of one kind (pipelines, models, schedulers,
|
|
1563
|
+
quantization) - the pickers' data source."""
|
|
1564
|
+
try:
|
|
1565
|
+
return {"kind": kind, "classes": list_classes(kind)}
|
|
1566
|
+
except ValueError as e:
|
|
1567
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
1568
|
+
|
|
1569
|
+
@app.get("/api/classes/{name:path}")
|
|
1570
|
+
def class_description(name: str, target: str = "init"):
|
|
1571
|
+
"""A class's argument schema: target=call reads __call__, init reads
|
|
1572
|
+
__init__, load reads from_pretrained plus the curated loading knobs."""
|
|
1573
|
+
try:
|
|
1574
|
+
return describe_class(name, target=target)
|
|
1575
|
+
except ValueError as e:
|
|
1576
|
+
raise HTTPException(status_code=404, detail=str(e))
|
|
1577
|
+
except Exception as e:
|
|
1578
|
+
raise HTTPException(status_code=404, detail=f"Could not load {name}: {e}")
|
|
1579
|
+
|
|
1580
|
+
@app.get("/api/schema")
|
|
1581
|
+
def workflow_schema(section: Optional[str] = None):
|
|
1582
|
+
"""The workflow JSON schema, for schema-aware JSON editing.
|
|
1583
|
+
|
|
1584
|
+
`?section=` answers one part of it - `steps`, `pipelines`, `tasks`,
|
|
1585
|
+
`result`, `variables` or `configuration` - as
|
|
1586
|
+
`{section, sections, elsewhere, schema}`, for a reader that wants
|
|
1587
|
+
the shape of a result block and not 36 KB of quantization configs
|
|
1588
|
+
(#101). Additive: the no-argument call is the whole schema, as it
|
|
1589
|
+
was. An unknown section is a 404 naming the ones that exist."""
|
|
1590
|
+
schema = load_schema("workflow")
|
|
1591
|
+
if section is None:
|
|
1592
|
+
return JSONResponse(schema)
|
|
1593
|
+
try:
|
|
1594
|
+
return JSONResponse(schema_section(schema, section))
|
|
1595
|
+
except SchemaSectionError as e:
|
|
1596
|
+
raise HTTPException(status_code=404, detail=str(e))
|
|
1597
|
+
|
|
1598
|
+
# ------------------------------------------------------------ guides
|
|
1599
|
+
|
|
1600
|
+
@app.get("/api/guides")
|
|
1601
|
+
def list_guides():
|
|
1602
|
+
"""The documentation that bears on choosing a capability: each
|
|
1603
|
+
guide's name, what it covers, and its section headings. Served by
|
|
1604
|
+
the engine rather than read from an MCP client's install, so the
|
|
1605
|
+
guides an agent reads are the guides for the engine it drives."""
|
|
1606
|
+
return guides.list_guides()
|
|
1607
|
+
|
|
1608
|
+
@app.get("/api/guides/{name}")
|
|
1609
|
+
def get_guide(name: str, section: Optional[str] = None):
|
|
1610
|
+
"""One guide from /api/guides, whole or one section of it. A
|
|
1611
|
+
section name is matched loosely - case and punctuation dropped -
|
|
1612
|
+
so a heading copied approximately still resolves. An unknown name
|
|
1613
|
+
or section is a 404 whose detail lists what exists."""
|
|
1614
|
+
try:
|
|
1615
|
+
return guides.get_guide(name, section=section)
|
|
1616
|
+
except GuideError as e:
|
|
1617
|
+
raise HTTPException(status_code=404, detail=str(e))
|
|
1618
|
+
|
|
1619
|
+
def _argument_reference_errors(definition, arguments, ws):
|
|
1620
|
+
"""The 'asset:', 'prompt:' and 'output:' references that name nothing
|
|
1621
|
+
this workspace can reach, in the values a run would actually use -
|
|
1622
|
+
the caller's `arguments`, plus every declared `variables` default
|
|
1623
|
+
the caller did not override.
|
|
1624
|
+
|
|
1625
|
+
A stored default is exactly as much a promise as a caller's value:
|
|
1626
|
+
`validate_workflow(name="templates/ltx2/reference-sheet")` with no
|
|
1627
|
+
arguments at all used to answer valid because only `arguments` was
|
|
1628
|
+
checked, while the same call with the stored default handed back
|
|
1629
|
+
explicitly answered invalid - one run, two verdicts (#166). Reported
|
|
1630
|
+
at `variables.<name>` so the message still says whether the caller
|
|
1631
|
+
wrote the bad reference or merely didn't override one.
|
|
1632
|
+
|
|
1633
|
+
Resolved through the engine's own resolvers over the roots this
|
|
1634
|
+
workspace searches, so validation agrees with what the run would
|
|
1635
|
+
find - an asset that exists in another workspace is a miss here for
|
|
1636
|
+
the same reason it would be a miss there. Only the reference is
|
|
1637
|
+
resolved, never loaded: the point is to answer before any bytes move.
|
|
1638
|
+
"""
|
|
1639
|
+
|
|
1640
|
+
def over_roots(roots, resolve):
|
|
1641
|
+
"""Resolve against each root in turn, and on a total miss raise
|
|
1642
|
+
the *first* root's error rather than the last.
|
|
1643
|
+
|
|
1644
|
+
The resolvers name the path they searched in their message, and
|
|
1645
|
+
the first root is the workspace's own library plus the read-only
|
|
1646
|
+
fallbacks the environment pins - which is the path a run would
|
|
1647
|
+
report. The last root's message would name an examples directory
|
|
1648
|
+
and leave out the workspace, reading as though the library the
|
|
1649
|
+
caller works in was never looked in."""
|
|
1650
|
+
first = None
|
|
1651
|
+
for root in roots:
|
|
1652
|
+
try:
|
|
1653
|
+
return resolve(root)
|
|
1654
|
+
except Exception as e:
|
|
1655
|
+
first = first or e
|
|
1656
|
+
# `roots` is never empty here: the asset branch answers an empty
|
|
1657
|
+
# search path itself, and the prompt path always holds the
|
|
1658
|
+
# server's own library. Re-raising None would be a TypeError
|
|
1659
|
+
raise first
|
|
1660
|
+
|
|
1661
|
+
def _string_leaves(value, path):
|
|
1662
|
+
"""Every string in `value`, paired with the path it sits at.
|
|
1663
|
+
|
|
1664
|
+
`value` is walked the way a for_each entry is - a list or dict
|
|
1665
|
+
of arbitrary nesting - so a reference inside `shots[2].
|
|
1666
|
+
references[1].from_file` is found the same as one at the
|
|
1667
|
+
argument's own top level."""
|
|
1668
|
+
if isinstance(value, str):
|
|
1669
|
+
yield path, value
|
|
1670
|
+
elif isinstance(value, list):
|
|
1671
|
+
for i, item in enumerate(value):
|
|
1672
|
+
yield from _string_leaves(item, f"{path}[{i}]")
|
|
1673
|
+
elif isinstance(value, dict):
|
|
1674
|
+
for key, item in value.items():
|
|
1675
|
+
yield from _string_leaves(item, f"{path}.{key}")
|
|
1676
|
+
|
|
1677
|
+
if arguments is not None and not isinstance(arguments, dict):
|
|
1678
|
+
return []
|
|
1679
|
+
supplied = arguments if isinstance(arguments, dict) else {}
|
|
1680
|
+
|
|
1681
|
+
effective = []
|
|
1682
|
+
declared = definition.get("variables") if isinstance(definition, dict) else None
|
|
1683
|
+
if isinstance(declared, dict):
|
|
1684
|
+
for name, value in declared.items():
|
|
1685
|
+
if name not in supplied:
|
|
1686
|
+
effective.append((f"variables.{name}", value))
|
|
1687
|
+
for name, value in supplied.items():
|
|
1688
|
+
effective.append((f"arguments.{name}", value))
|
|
1689
|
+
|
|
1690
|
+
errors = []
|
|
1691
|
+
for base_path, value in effective:
|
|
1692
|
+
for path, leaf in _string_leaves(value, base_path):
|
|
1693
|
+
try:
|
|
1694
|
+
if is_asset_reference(leaf):
|
|
1695
|
+
roots = _resolution_roots(ws)
|
|
1696
|
+
if not roots:
|
|
1697
|
+
# A server configured with no asset library has
|
|
1698
|
+
# no root to fail against: over_roots would
|
|
1699
|
+
# re-raise its "first error", which is None,
|
|
1700
|
+
# and the caller would read a TypeError about
|
|
1701
|
+
# BaseException in place of a verdict
|
|
1702
|
+
name = leaf.removeprefix(ASSET_PREFIX).strip()
|
|
1703
|
+
raise ValueError(
|
|
1704
|
+
f"Unknown asset {name!r}: "
|
|
1705
|
+
"this workspace has no asset library"
|
|
1706
|
+
)
|
|
1707
|
+
over_roots(
|
|
1708
|
+
roots,
|
|
1709
|
+
lambda root: resolve_asset_reference(leaf, asset_dir=root),
|
|
1710
|
+
)
|
|
1711
|
+
elif leaf.startswith(PROMPT_PREFIX):
|
|
1712
|
+
over_roots(
|
|
1713
|
+
_prompt_roots(),
|
|
1714
|
+
lambda root: resolve_prompt_reference(
|
|
1715
|
+
leaf, prompt_dir=root
|
|
1716
|
+
),
|
|
1717
|
+
)
|
|
1718
|
+
elif is_output_reference(leaf):
|
|
1719
|
+
resolve_output_reference(leaf, root=ws.outputs)
|
|
1720
|
+
except Exception as e:
|
|
1721
|
+
# Every resolver here raises with a message written for
|
|
1722
|
+
# the person who wrote the reference - a traversal
|
|
1723
|
+
# refusal from the security layer included
|
|
1724
|
+
errors.append({"path": path, "message": str(e)})
|
|
1725
|
+
return errors
|
|
1726
|
+
|
|
1727
|
+
def _probe_command_for(candidate, request, workspace, workflow_dir):
|
|
1728
|
+
"""The execute-shaped command a cache probe of this validate request
|
|
1729
|
+
needs - the same fields _run_job sends, so the worker loads the
|
|
1730
|
+
workflow exactly as a job would."""
|
|
1731
|
+
command = {
|
|
1732
|
+
"arguments": request.arguments,
|
|
1733
|
+
"output_dir": workspace.outputs,
|
|
1734
|
+
"workflow_dir": workflow_dir,
|
|
1735
|
+
}
|
|
1736
|
+
if workspace.assets:
|
|
1737
|
+
command["asset_dir"] = workspace.assets
|
|
1738
|
+
if request.workflow_path is not None:
|
|
1739
|
+
command["workflow_path"] = candidate.file_spec
|
|
1740
|
+
else:
|
|
1741
|
+
command["workflow"] = request.workflow
|
|
1742
|
+
command["base_dir"] = os.path.dirname(candidate.file_spec)
|
|
1743
|
+
return command
|
|
1744
|
+
|
|
1745
|
+
@app.post("/api/validate")
|
|
1746
|
+
def validate_workflow(
|
|
1747
|
+
request: JobRequest,
|
|
1748
|
+
ws: Workspace = Depends(selected_workspace),
|
|
1749
|
+
sizes: bool = Query(
|
|
1750
|
+
True,
|
|
1751
|
+
description="Ask the hub how large each missing model is; false "
|
|
1752
|
+
"skips the network for a faster answer",
|
|
1753
|
+
),
|
|
1754
|
+
):
|
|
1755
|
+
"""Schema-validate a workflow and check its pipeline arguments
|
|
1756
|
+
against real signatures, without queuing anything. Give either an
|
|
1757
|
+
inline workflow or a workflow_path - a path on the server or a
|
|
1758
|
+
stored workflow name from /api/workflows. The workspace it resolves
|
|
1759
|
+
in comes from the body or the query string, body first. A valid
|
|
1760
|
+
answer also carries a plan: the fingerprint of the work these
|
|
1761
|
+
arguments produce, the step count, the list lengths, the model
|
|
1762
|
+
repos not in the cache, and an estimate from the workflow's cost
|
|
1763
|
+
block."""
|
|
1764
|
+
if (request.workflow is None) == (request.workflow_path is None):
|
|
1765
|
+
raise HTTPException(
|
|
1766
|
+
status_code=400,
|
|
1767
|
+
detail="Provide exactly one of workflow or workflow_path",
|
|
1768
|
+
)
|
|
1769
|
+
try:
|
|
1770
|
+
workspace = _workspace_for(request.workspace or ws.name)
|
|
1771
|
+
if request.workflow_path is not None:
|
|
1772
|
+
# Built from the file so relative paths inside it resolve
|
|
1773
|
+
# against its own directory, exactly as a run would
|
|
1774
|
+
sources = _sources_for(workspace)
|
|
1775
|
+
resolved, source = resolve_workflow_reference(
|
|
1776
|
+
request.workflow_path, sources
|
|
1777
|
+
)
|
|
1778
|
+
# Confined to the source it came from, not to the writable
|
|
1779
|
+
# root - an example is read where it lives
|
|
1780
|
+
source_root = source.root if source else workspace.workflows
|
|
1781
|
+
candidate = workflow_from_file(resolved, workspace.outputs, source_root)
|
|
1782
|
+
definition = candidate.workflow_definition
|
|
1783
|
+
# The listing name the job history is keyed on, so the plan
|
|
1784
|
+
# can quote what this box's own runs of it took (#154)
|
|
1785
|
+
catalog_name = catalog_name_for(resolved, source)
|
|
1786
|
+
else:
|
|
1787
|
+
definition = request.workflow
|
|
1788
|
+
source_root = workspace.workflows
|
|
1789
|
+
# An inline definition has no catalog name, so no history
|
|
1790
|
+
catalog_name = None
|
|
1791
|
+
candidate = workflow_from_definition(
|
|
1792
|
+
copy.deepcopy(request.workflow),
|
|
1793
|
+
workspace.outputs,
|
|
1794
|
+
request.base_dir,
|
|
1795
|
+
source_root,
|
|
1796
|
+
)
|
|
1797
|
+
except HTTPException:
|
|
1798
|
+
raise
|
|
1799
|
+
except SecurityError as e:
|
|
1800
|
+
# Messages the security layer writes itself - safe to surface
|
|
1801
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
1802
|
+
except Exception:
|
|
1803
|
+
# Anything else could carry internals in its message; the log
|
|
1804
|
+
# keeps the detail, the client gets the category
|
|
1805
|
+
logger.exception("Workflow could not be constructed for validation")
|
|
1806
|
+
raise HTTPException(
|
|
1807
|
+
status_code=400,
|
|
1808
|
+
detail="Workflow could not be constructed - the server log "
|
|
1809
|
+
"has the detail",
|
|
1810
|
+
)
|
|
1811
|
+
# `arguments` defaults to `{}` on the model (JobRequest is shared
|
|
1812
|
+
# with run_workflow, which needs a dict), so an omitted field and an
|
|
1813
|
+
# explicit `{}` are otherwise indistinguishable here - and the two
|
|
1814
|
+
# mean different things: omitted is "check the document", explicit
|
|
1815
|
+
# is "check a run with these arguments" (#364). model_fields_set
|
|
1816
|
+
# tells them apart without changing the field's default for every
|
|
1817
|
+
# other caller of validate_workflow.
|
|
1818
|
+
caller_arguments = (
|
|
1819
|
+
request.arguments if "arguments" in request.model_fields_set else None
|
|
1820
|
+
)
|
|
1821
|
+
try:
|
|
1822
|
+
# The caller's list is the one a for_each expands over, so the
|
|
1823
|
+
# pre-flight checks the step set that will actually run
|
|
1824
|
+
errors = candidate.validation_errors(arguments=caller_arguments)
|
|
1825
|
+
except Exception:
|
|
1826
|
+
# An error here is not the schema's verdict on the workflow -
|
|
1827
|
+
# validation_errors() reports that by returning it. It is the
|
|
1828
|
+
# validator itself failing, and its message could carry
|
|
1829
|
+
# internals, so the log keeps the detail and the client is told
|
|
1830
|
+
# the category, as above
|
|
1831
|
+
logger.exception("Workflow could not be validated")
|
|
1832
|
+
detail = (
|
|
1833
|
+
"The workflow could not be validated - the server log has the detail"
|
|
1834
|
+
)
|
|
1835
|
+
return {
|
|
1836
|
+
"valid": False,
|
|
1837
|
+
"error": detail,
|
|
1838
|
+
"errors": [{"path": None, "message": detail}],
|
|
1839
|
+
"warnings": [],
|
|
1840
|
+
}
|
|
1841
|
+
if errors:
|
|
1842
|
+
return {
|
|
1843
|
+
"valid": False,
|
|
1844
|
+
"error": format_validation_errors(errors),
|
|
1845
|
+
"errors": errors,
|
|
1846
|
+
"warnings": [],
|
|
1847
|
+
}
|
|
1848
|
+
# The arguments a caller is about to run with, checked the way the
|
|
1849
|
+
# run itself would check them: an undeclared name, a value that will
|
|
1850
|
+
# not coerce, an 'asset:'/'prompt:'/'output:' reference that names
|
|
1851
|
+
# nothing in this workspace. Without this the free pre-flight covers
|
|
1852
|
+
# every part of a run except the part the caller actually wrote
|
|
1853
|
+
argument_problems = argument_errors(definition, request.arguments)
|
|
1854
|
+
argument_problems += _argument_reference_errors(
|
|
1855
|
+
definition, request.arguments, workspace
|
|
1856
|
+
)
|
|
1857
|
+
if argument_problems:
|
|
1858
|
+
return {
|
|
1859
|
+
"valid": False,
|
|
1860
|
+
"error": format_validation_errors(argument_problems),
|
|
1861
|
+
"errors": argument_problems,
|
|
1862
|
+
"warnings": [],
|
|
1863
|
+
"checked_arguments": sorted(request.arguments or {}),
|
|
1864
|
+
}
|
|
1865
|
+
answer = {
|
|
1866
|
+
"valid": True,
|
|
1867
|
+
"error": None,
|
|
1868
|
+
"errors": [],
|
|
1869
|
+
"warnings": workflow_argument_warnings(definition, request.arguments)
|
|
1870
|
+
# A value a declared constraint will round up - the silent half
|
|
1871
|
+
# of #96: the run changed the caller's frame count and only the
|
|
1872
|
+
# server's log said so
|
|
1873
|
+
+ constraint_warnings(definition, request.arguments)
|
|
1874
|
+
+ entry_field_warnings(definition, request.arguments)
|
|
1875
|
+
# Why `plan.cached_steps` is 0 for a workflow with no seed - the
|
|
1876
|
+
# cache is off, not empty
|
|
1877
|
+
+ unseeded_cache_warnings(definition, request.arguments)
|
|
1878
|
+
# An adapter whose file name says nothing about which checkpoint
|
|
1879
|
+
# partition it was trained for: valid, since the name of a
|
|
1880
|
+
# future checkpoint cannot be predicted, but nothing at run time
|
|
1881
|
+
# would say it loaded onto the wrong one (#155)
|
|
1882
|
+
+ candidate.adapter_warnings(request.arguments)
|
|
1883
|
+
# A required task argument fed by variable:name where name's
|
|
1884
|
+
# default is null - a fine document, but a run left as-is would
|
|
1885
|
+
# fail; empty once the caller names any arguments, since that
|
|
1886
|
+
# condition is a hard error above instead (#364)
|
|
1887
|
+
+ candidate.null_variable_argument_warnings(caller_arguments)
|
|
1888
|
+
# An argument a sub-workflow step passes to a workflow that
|
|
1889
|
+
# declares no variable for it - dropped in silence at run time
|
|
1890
|
+
+ candidate.sub_workflow_warnings()
|
|
1891
|
+
# A slice_audio source whose real duration is already knowable
|
|
1892
|
+
# (an asset:/output: reference validate can already probe) and
|
|
1893
|
+
# whose requested slice reaches past it - zero-padded rather than
|
|
1894
|
+
# refused, but previously said only by the run itself (#402)
|
|
1895
|
+
+ candidate.slice_past_end_warnings(request.arguments)
|
|
1896
|
+
# An assessment probe's shots argument reaching past a
|
|
1897
|
+
# statically-knowable video's real frame count - silently
|
|
1898
|
+
# clipped rather than refused, but previously said only by the
|
|
1899
|
+
# run itself (#425)
|
|
1900
|
+
+ candidate.shot_span_warnings(request.arguments),
|
|
1901
|
+
}
|
|
1902
|
+
if request.arguments:
|
|
1903
|
+
# Naming what was checked is the difference between 'the stored
|
|
1904
|
+
# definition is valid' and 'the values you are about to pass are'
|
|
1905
|
+
answer["checked_arguments"] = sorted(request.arguments)
|
|
1906
|
+
# What the run will execute for these arguments, fingerprinted so
|
|
1907
|
+
# an acknowledgement can be bound to it (#85). Best effort: the
|
|
1908
|
+
# verdict above is the schema's and the planner may not change it
|
|
1909
|
+
try:
|
|
1910
|
+
from .. import get_device, get_device_type
|
|
1911
|
+
|
|
1912
|
+
command = _probe_command_for(candidate, request, workspace, source_root)
|
|
1913
|
+
|
|
1914
|
+
def observed_for_child(path, child_definition, arguments=None):
|
|
1915
|
+
"""A composed child's own observed figure, keyed by the
|
|
1916
|
+
catalog name it resolves to - so a parent with no figure of
|
|
1917
|
+
its own can quote what this box's runs of the *child* took
|
|
1918
|
+
rather than falling back to unknown (#268).
|
|
1919
|
+
|
|
1920
|
+
`arguments` are the composing step's own overrides - the
|
|
1921
|
+
same role `arguments` plays for the top-level `observed`
|
|
1922
|
+
callback - so a child whose composing step shifted a
|
|
1923
|
+
declared scalar `cost_driver` (#341) is bucketed against
|
|
1924
|
+
*that* value rather than always the child's stored
|
|
1925
|
+
defaults, which silently answered the default bucket's
|
|
1926
|
+
history for every override."""
|
|
1927
|
+
base_dir = (
|
|
1928
|
+
os.path.dirname(os.path.abspath(candidate.file_spec))
|
|
1929
|
+
if candidate.file_spec
|
|
1930
|
+
else None
|
|
1931
|
+
)
|
|
1932
|
+
try:
|
|
1933
|
+
child_path, child_root = resolve_sub_workflow(
|
|
1934
|
+
path, base_dir or ".", candidate.workflow_dir
|
|
1935
|
+
)
|
|
1936
|
+
except (SecurityError, OSError, ValueError, SubWorkflowNotFound):
|
|
1937
|
+
return None
|
|
1938
|
+
child_name = _catalog_name_from_root(child_path, child_root)
|
|
1939
|
+
if not child_name:
|
|
1940
|
+
return None
|
|
1941
|
+
# resolve_sub_workflow hands back a bare root string, not a
|
|
1942
|
+
# Source, so writability is inferred the way that root was
|
|
1943
|
+
# built: the workspace's own workflows/ is the writable one
|
|
1944
|
+
# (#274)
|
|
1945
|
+
child_workspace = (
|
|
1946
|
+
workspace.name if child_root == workspace.workflows else None
|
|
1947
|
+
)
|
|
1948
|
+
return _observed_for_name(
|
|
1949
|
+
child_name, child_definition, arguments, workspace=child_workspace
|
|
1950
|
+
)
|
|
1951
|
+
|
|
1952
|
+
answer["plan"] = build_plan(
|
|
1953
|
+
candidate,
|
|
1954
|
+
request.arguments,
|
|
1955
|
+
device=get_device_type(get_device()),
|
|
1956
|
+
prompt_dir=workspace.prompts,
|
|
1957
|
+
lookup_sizes=sizes,
|
|
1958
|
+
cache_probe=lambda arguments: manager.probe_cache(
|
|
1959
|
+
{**command, "arguments": arguments}
|
|
1960
|
+
),
|
|
1961
|
+
# What this box's own runs of this shape took, which is what
|
|
1962
|
+
# the estimate quotes ahead of a curated figure (#154) - the
|
|
1963
|
+
# same aggregate the listing reports, asked with the
|
|
1964
|
+
# caller's arguments rather than the defaults
|
|
1965
|
+
observed=(
|
|
1966
|
+
(
|
|
1967
|
+
lambda arguments: _observed_for_name(
|
|
1968
|
+
catalog_name,
|
|
1969
|
+
definition,
|
|
1970
|
+
arguments,
|
|
1971
|
+
workspace=workspace.name if source.writable else None,
|
|
1972
|
+
)
|
|
1973
|
+
)
|
|
1974
|
+
if catalog_name
|
|
1975
|
+
else None
|
|
1976
|
+
),
|
|
1977
|
+
observed_for_child=observed_for_child,
|
|
1978
|
+
)
|
|
1979
|
+
except Exception:
|
|
1980
|
+
logger.exception("Plan could not be built")
|
|
1981
|
+
answer["plan"] = None
|
|
1982
|
+
if answer["plan"]:
|
|
1983
|
+
# cached_steps is 0 both when nothing hit and when the probe ran
|
|
1984
|
+
# against the wrong workspace's output root (#184) - echoing
|
|
1985
|
+
# what it was actually probed against turns the second case
|
|
1986
|
+
# from a silent miss into something a caller can read
|
|
1987
|
+
answer["plan"]["workspace"] = workspace.name
|
|
1988
|
+
answer["plan"]["output_dir"] = workspace.outputs
|
|
1989
|
+
answer["warnings"] += gate_warnings(answer["plan"]["downloads_required"])
|
|
1990
|
+
if catalog_name:
|
|
1991
|
+
answer["warnings"] += _host_memory_warnings(
|
|
1992
|
+
catalog_name,
|
|
1993
|
+
definition,
|
|
1994
|
+
answer["plan"]["list_entries"],
|
|
1995
|
+
workspace=workspace.name if source.writable else None,
|
|
1996
|
+
)
|
|
1997
|
+
return answer
|
|
1998
|
+
|
|
1999
|
+
def _host_memory_warnings(name, definition, list_entries, *, workspace=None):
|
|
2000
|
+
"""Whether this box's own history says the requested list is
|
|
2001
|
+
projected to exceed host RAM (#243) - best effort, since a warning
|
|
2002
|
+
that 500s the free pre-flight would be worse than skipping it."""
|
|
2003
|
+
costs = getattr(app.state, "observed_costs", None)
|
|
2004
|
+
if costs is None:
|
|
2005
|
+
return []
|
|
2006
|
+
try:
|
|
2007
|
+
from ..host_memory import host_memory_stats
|
|
2008
|
+
|
|
2009
|
+
rows = costs.rows_for(name, workspace=workspace)
|
|
2010
|
+
ceiling_mb = (host_memory_stats().get("total_mb") or 0) * CEILING_FRACTION
|
|
2011
|
+
return host_memory_warnings(definition, list_entries, rows, ceiling_mb)
|
|
2012
|
+
except Exception:
|
|
2013
|
+
logger.debug("host memory projection failed for %s", name, exc_info=True)
|
|
2014
|
+
return []
|
|
2015
|
+
|
|
2016
|
+
# ------------------------------------------------------------ workspaces
|
|
2017
|
+
|
|
2018
|
+
class WorkspaceRequest(BaseModel):
|
|
2019
|
+
name: str = Field(description="Name for the new workspace")
|
|
2020
|
+
|
|
2021
|
+
@app.get("/api/workspaces")
|
|
2022
|
+
def list_workspaces():
|
|
2023
|
+
"""Every workspace on this server, the default first.
|
|
2024
|
+
|
|
2025
|
+
A workspace is a namespace, not a security boundary: the API token
|
|
2026
|
+
is all-or-nothing, so anything that can list these can reach all of
|
|
2027
|
+
them.
|
|
2028
|
+
"""
|
|
2029
|
+
root = app.state.workspace_root
|
|
2030
|
+
# workspace_names lists the whole root once; everything after the
|
|
2031
|
+
# first entry (always the default, see its docstring) is a named
|
|
2032
|
+
# workspace to describe individually
|
|
2033
|
+
names = workspace_names(root)[1:] if root else []
|
|
2034
|
+
listed = [app.state.default_workspace]
|
|
2035
|
+
for name in names:
|
|
2036
|
+
listed.append(named_workspace(root, name))
|
|
2037
|
+
described = []
|
|
2038
|
+
for space in listed:
|
|
2039
|
+
entry = space.describe()
|
|
2040
|
+
# Roughly how much disk it holds, cached for a minute inside
|
|
2041
|
+
# workspace_usage - a listing is a glance, and a job writing
|
|
2042
|
+
# into outputs moves the number continuously anyway
|
|
2043
|
+
entry["usage"] = workspace_usage(space)
|
|
2044
|
+
described.append(entry)
|
|
2045
|
+
return {
|
|
2046
|
+
"workspace_root": root.root if root else None,
|
|
2047
|
+
"default": DEFAULT_WORKSPACE_NAME,
|
|
2048
|
+
"workspaces": described,
|
|
2049
|
+
}
|
|
2050
|
+
|
|
2051
|
+
@app.post("/api/workspaces", status_code=201)
|
|
2052
|
+
def add_workspace(request: WorkspaceRequest):
|
|
2053
|
+
"""Create a workspace: its own workflows, assets and outputs, sharing
|
|
2054
|
+
this server's one prompt library."""
|
|
2055
|
+
root = _workspace_root()
|
|
2056
|
+
try:
|
|
2057
|
+
created = create_workspace(root, request.name)
|
|
2058
|
+
except SecurityError as e:
|
|
2059
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
2060
|
+
except FileExistsError as e:
|
|
2061
|
+
raise HTTPException(status_code=409, detail=str(e))
|
|
2062
|
+
forget_workspace_usage()
|
|
2063
|
+
logger.info(f"Created workspace {request.name} at {created.root}")
|
|
2064
|
+
return created.describe()
|
|
2065
|
+
|
|
2066
|
+
@app.delete("/api/workspaces/{name}")
|
|
2067
|
+
def remove_workspace(name: str, acknowledged: bool = False):
|
|
2068
|
+
"""Delete a workspace and everything in it.
|
|
2069
|
+
|
|
2070
|
+
Answers what it would remove and refuses until `acknowledged=true`:
|
|
2071
|
+
this deletes generated work, and a count is what makes it an
|
|
2072
|
+
informed choice rather than a surprise. The unacknowledged message
|
|
2073
|
+
names only what would be removed - how to proceed is left to the
|
|
2074
|
+
caller, since the MCP surface tells its own callers to acknowledge
|
|
2075
|
+
through a differently-named parameter (`acknowledged_cost`).
|
|
2076
|
+
"""
|
|
2077
|
+
root = _workspace_root()
|
|
2078
|
+
if name == DEFAULT_WORKSPACE_NAME:
|
|
2079
|
+
raise HTTPException(
|
|
2080
|
+
status_code=400,
|
|
2081
|
+
detail="The default workspace cannot be deleted - it is the "
|
|
2082
|
+
"workspace root itself, and holds the shared prompt library",
|
|
2083
|
+
)
|
|
2084
|
+
if name not in workspace_names(root):
|
|
2085
|
+
raise HTTPException(status_code=404, detail=f"No such workspace: {name}")
|
|
2086
|
+
|
|
2087
|
+
contents = workspace_contents(named_workspace(root, name))
|
|
2088
|
+
if not acknowledged:
|
|
2089
|
+
raise HTTPException(
|
|
2090
|
+
status_code=409,
|
|
2091
|
+
detail={
|
|
2092
|
+
"message": f"Deleting workspace '{name}' removes these "
|
|
2093
|
+
f"files permanently.",
|
|
2094
|
+
"contents": contents,
|
|
2095
|
+
},
|
|
2096
|
+
)
|
|
2097
|
+
with manager._lock:
|
|
2098
|
+
queued = [
|
|
2099
|
+
job
|
|
2100
|
+
for job in manager.jobs.values()
|
|
2101
|
+
if job.status not in TERMINAL_STATES
|
|
2102
|
+
and job.spec.get("workspace") == name
|
|
2103
|
+
]
|
|
2104
|
+
if queued:
|
|
2105
|
+
raise HTTPException(
|
|
2106
|
+
status_code=409,
|
|
2107
|
+
detail=f"Workspace '{name}' has {len(queued)} job(s) queued or "
|
|
2108
|
+
f"running - cancel them first",
|
|
2109
|
+
)
|
|
2110
|
+
try:
|
|
2111
|
+
delete_workspace(root, name)
|
|
2112
|
+
except NotAWorkspaceError as e:
|
|
2113
|
+
# The directory holds more than a workspace - refused outright,
|
|
2114
|
+
# since what else it holds is not the caller's to acknowledge away
|
|
2115
|
+
raise HTTPException(
|
|
2116
|
+
status_code=409, detail={"message": str(e), "entries": e.entries}
|
|
2117
|
+
)
|
|
2118
|
+
except (ValueError, FileNotFoundError) as e:
|
|
2119
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
2120
|
+
forget_workspace_usage()
|
|
2121
|
+
logger.info(f"Deleted workspace {name}")
|
|
2122
|
+
return {"name": name, "deleted": True, "contents": contents}
|
|
2123
|
+
|
|
2124
|
+
# ------------------------------------------------------------- workflows
|
|
2125
|
+
|
|
2126
|
+
# How much of a long variable default the variables route shows before
|
|
2127
|
+
# cutting it: enough to recognize a prompt by, far short of carrying one
|
|
2128
|
+
VARIABLE_VALUE_PREVIEW = 200
|
|
2129
|
+
|
|
2130
|
+
@app.get("/api/workflows")
|
|
2131
|
+
def list_workflows(
|
|
2132
|
+
ws: Workspace = Depends(selected_workspace),
|
|
2133
|
+
shape: Optional[str] = None,
|
|
2134
|
+
traits: Optional[str] = None,
|
|
2135
|
+
configures: Optional[str] = None,
|
|
2136
|
+
include_models: bool = False,
|
|
2137
|
+
view: Optional[str] = None,
|
|
2138
|
+
):
|
|
2139
|
+
"""Every workflow the search path offers, each detail saying which
|
|
2140
|
+
source it came from and whether it can be written to. 'workflow_dir'
|
|
2141
|
+
stays the writable one - what a save targets.
|
|
2142
|
+
|
|
2143
|
+
`shape`, `traits` (comma-separated, all must match) and `configures`
|
|
2144
|
+
narrow the listing; `view=compact` is the agent's view - summaries
|
|
2145
|
+
rather than descriptions, templates rather than model configs
|
|
2146
|
+
unless `include_models` asks for them. `workflows` always names
|
|
2147
|
+
exactly the entries `details` holds.
|
|
2148
|
+
"""
|
|
2149
|
+
sources = _sources_for(ws)
|
|
2150
|
+
found = listing(sources)
|
|
2151
|
+
try:
|
|
2152
|
+
details = project_listing(
|
|
2153
|
+
attach_observed(
|
|
2154
|
+
workflow_details(found),
|
|
2155
|
+
getattr(app.state, "observed_costs", None),
|
|
2156
|
+
ws.name,
|
|
2157
|
+
),
|
|
2158
|
+
shape=shape,
|
|
2159
|
+
traits=[t.strip() for t in (traits or "").split(",") if t.strip()],
|
|
2160
|
+
configures=configures,
|
|
2161
|
+
include_models=include_models,
|
|
2162
|
+
view=view,
|
|
2163
|
+
)
|
|
2164
|
+
except ValueError as e:
|
|
2165
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
2166
|
+
return {
|
|
2167
|
+
"workspace": ws.name,
|
|
2168
|
+
"workflow_dir": ws.workflows,
|
|
2169
|
+
"sources": [source.to_dict() for source in sources],
|
|
2170
|
+
"workflows": sorted(details),
|
|
2171
|
+
"details": details,
|
|
2172
|
+
# What a `cost` is, and so what a null one means. Curated:
|
|
2173
|
+
# figures a maintainer measured once on the devices named and
|
|
2174
|
+
# wrote into the workflow - nothing derives them from this
|
|
2175
|
+
# server's own job history, so null means nobody wrote one
|
|
2176
|
+
# down, not that the run is cheap or that this box has never
|
|
2177
|
+
# run it (#91). A detail's `observed` block, when present, is the
|
|
2178
|
+
# other kind of number: this box's own finished runs of that
|
|
2179
|
+
# workflow, derived rather than claimed, and never a substitute
|
|
2180
|
+
# for `cost` (#93)
|
|
2181
|
+
"cost_basis": "curated",
|
|
2182
|
+
}
|
|
2183
|
+
|
|
2184
|
+
@app.put("/api/workflows/{name:path}")
|
|
2185
|
+
def save_workflow(
|
|
2186
|
+
name: str, request: JobRequest, ws: Workspace = Depends(selected_workspace)
|
|
2187
|
+
):
|
|
2188
|
+
"""Write a workflow into the writable workflow directory. The
|
|
2189
|
+
definition must be schema-valid - the editor validates before saving,
|
|
2190
|
+
and a save that silently wrote a broken file would betray both.
|
|
2191
|
+
|
|
2192
|
+
A name that currently resolves to a read-only source (an example, a
|
|
2193
|
+
builtin) is not overwritten: the copy lands in the writable source
|
|
2194
|
+
and shadows it from then on.
|
|
2195
|
+
"""
|
|
2196
|
+
if request.workflow is None:
|
|
2197
|
+
raise HTTPException(
|
|
2198
|
+
status_code=400,
|
|
2199
|
+
detail='Provide the definition as {"workflow": {...}}',
|
|
2200
|
+
)
|
|
2201
|
+
path, _source = resolve_writable_workflow(_sources_for(ws), name)
|
|
2202
|
+
candidate = Workflow(
|
|
2203
|
+
copy.deepcopy(request.workflow),
|
|
2204
|
+
ws.outputs,
|
|
2205
|
+
path,
|
|
2206
|
+
ws.workflows,
|
|
2207
|
+
)
|
|
2208
|
+
try:
|
|
2209
|
+
candidate.validate()
|
|
2210
|
+
except Exception as e:
|
|
2211
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
2212
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
2213
|
+
with open(path, "w") as file:
|
|
2214
|
+
json.dump(request.workflow, file, indent=2)
|
|
2215
|
+
file.write("\n")
|
|
2216
|
+
logger.info(f"Saved workflow {name} to {path}")
|
|
2217
|
+
# What the catalog will say about it, so the author sees the match
|
|
2218
|
+
# it just created. An empty summary is a warning, never a refusal:
|
|
2219
|
+
# a workflow with no description still runs, it is just invisible
|
|
2220
|
+
# to shape-first discovery
|
|
2221
|
+
metadata = derive_catalog_metadata(request.workflow)
|
|
2222
|
+
warnings = list(workflow_argument_warnings(request.workflow))
|
|
2223
|
+
warnings += candidate.null_variable_argument_warnings()
|
|
2224
|
+
if not metadata["summary"]:
|
|
2225
|
+
warnings.append(
|
|
2226
|
+
"No summary: add a 'description' (its first sentence becomes "
|
|
2227
|
+
"the catalog summary) or a 'summary' so the listing can say "
|
|
2228
|
+
"what this workflow is for"
|
|
2229
|
+
)
|
|
2230
|
+
return {
|
|
2231
|
+
"name": name,
|
|
2232
|
+
"path": path,
|
|
2233
|
+
"warnings": warnings,
|
|
2234
|
+
"shape": metadata["shape"],
|
|
2235
|
+
"traits": metadata["traits"],
|
|
2236
|
+
"summary": metadata["summary"],
|
|
2237
|
+
}
|
|
2238
|
+
|
|
2239
|
+
@app.delete("/api/workflows/{name:path}")
|
|
2240
|
+
def delete_workflow(name: str, ws: Workspace = Depends(selected_workspace)):
|
|
2241
|
+
"""Remove a workflow file from the writable workflow directory.
|
|
2242
|
+
|
|
2243
|
+
A read-only source is refused rather than silently ignored: an
|
|
2244
|
+
example or a builtin is not the caller's to delete, and saying so
|
|
2245
|
+
is more useful than a 404 that reads like the file is missing.
|
|
2246
|
+
"""
|
|
2247
|
+
path, source = resolve_readable_workflow(_sources_for(ws), name)
|
|
2248
|
+
if not source.writable:
|
|
2249
|
+
raise HTTPException(
|
|
2250
|
+
status_code=403,
|
|
2251
|
+
detail=f"'{name}' comes from the read-only {source.origin} "
|
|
2252
|
+
f"directory {source.root} and cannot be deleted",
|
|
2253
|
+
)
|
|
2254
|
+
os.remove(path)
|
|
2255
|
+
logger.info(f"Deleted workflow {name} ({path})")
|
|
2256
|
+
forget_workspace_usage()
|
|
2257
|
+
# This identity's job history goes with it (#274) - otherwise a name
|
|
2258
|
+
# reused in this workspace, including by a regression cycle that
|
|
2259
|
+
# deletes and recreates the same workflow, would inherit the deleted
|
|
2260
|
+
# copy's observed figures and host-memory history
|
|
2261
|
+
manager.history.orphan_workflow_history(ws.name, name)
|
|
2262
|
+
return {"name": name, "deleted": True}
|
|
2263
|
+
|
|
2264
|
+
@app.get("/api/workflows/{name:path}/download")
|
|
2265
|
+
@query_token_ok
|
|
2266
|
+
def download_workflow(name: str, ws: Workspace = Depends(selected_workspace)):
|
|
2267
|
+
"""Serve a workflow definition as a forced download."""
|
|
2268
|
+
path, _source = resolve_readable_workflow(_sources_for(ws), name)
|
|
2269
|
+
return FileResponse(
|
|
2270
|
+
path, filename=os.path.basename(path), media_type="application/json"
|
|
2271
|
+
)
|
|
2272
|
+
|
|
2273
|
+
# Declared before the catch-all below, which would otherwise swallow
|
|
2274
|
+
# '<name>/variables' as a workflow called that
|
|
2275
|
+
@app.get("/api/workflows/{name:path}/variables")
|
|
2276
|
+
def get_workflow_variables(
|
|
2277
|
+
name: str, full: bool = False, ws: Workspace = Depends(selected_workspace)
|
|
2278
|
+
):
|
|
2279
|
+
"""A workflow's variables and the values they default to.
|
|
2280
|
+
|
|
2281
|
+
The listing says which variables a workflow has; confirming what one
|
|
2282
|
+
of them defaults to meant fetching the whole definition, quantization
|
|
2283
|
+
blocks and all, to read a single integer. This answers that question
|
|
2284
|
+
by itself.
|
|
2285
|
+
|
|
2286
|
+
Long strings - a shot's prompt runs to kilobytes, and a list-driven
|
|
2287
|
+
workflow's default list holds several - are cut to their first 200
|
|
2288
|
+
characters wherever they sit and named in `truncated`
|
|
2289
|
+
(`shots[0].prompt`), so the answer stays small for the numbers and
|
|
2290
|
+
names it is usually asked about; `full=true` returns them whole, and
|
|
2291
|
+
`GET /api/workflows/{name}` is still the definition itself.
|
|
2292
|
+
"""
|
|
2293
|
+
path, source = resolve_readable_workflow(_sources_for(ws), name)
|
|
2294
|
+
try:
|
|
2295
|
+
with open(path, "r") as file:
|
|
2296
|
+
definition = json.load(file)
|
|
2297
|
+
except (OSError, json.JSONDecodeError) as e:
|
|
2298
|
+
raise HTTPException(status_code=500, detail=f"Could not read workflow: {e}")
|
|
2299
|
+
|
|
2300
|
+
def preview(value, path):
|
|
2301
|
+
if isinstance(value, str) and len(value) > VARIABLE_VALUE_PREVIEW:
|
|
2302
|
+
truncated.append(path)
|
|
2303
|
+
return value[:VARIABLE_VALUE_PREVIEW]
|
|
2304
|
+
if isinstance(value, list):
|
|
2305
|
+
return [preview(item, f"{path}[{i}]") for i, item in enumerate(value)]
|
|
2306
|
+
if isinstance(value, dict):
|
|
2307
|
+
return {
|
|
2308
|
+
key: preview(item, f"{path}.{key}") for key, item in value.items()
|
|
2309
|
+
}
|
|
2310
|
+
return value
|
|
2311
|
+
|
|
2312
|
+
variables = definition.get("variables") or {}
|
|
2313
|
+
values, truncated = {}, []
|
|
2314
|
+
for variable, value in variables.items():
|
|
2315
|
+
values[variable] = value if full else preview(value, variable)
|
|
2316
|
+
answer = {
|
|
2317
|
+
"name": name,
|
|
2318
|
+
"variables": values,
|
|
2319
|
+
"truncated": truncated,
|
|
2320
|
+
"seed": definition.get("seed"),
|
|
2321
|
+
"origin": source.origin,
|
|
2322
|
+
}
|
|
2323
|
+
# The rule beside the default it constrains: a consumer reading
|
|
2324
|
+
# `num_frames: 124` with no range picked 61 and paid 138 s of
|
|
2325
|
+
# loading to be told the rule was 17n + 5 from 124 (#96)
|
|
2326
|
+
constraints = definition.get("variable_constraints")
|
|
2327
|
+
if isinstance(constraints, dict) and constraints:
|
|
2328
|
+
answer["constraints"] = constraints
|
|
2329
|
+
# What an entry of each list-driven variable carries, with any rule
|
|
2330
|
+
# that reaches one of its fields stated beside that field: a caller
|
|
2331
|
+
# reading what a `shots` entry takes reads the bound for
|
|
2332
|
+
# `num_frames` there, rather than having to match it to a key of
|
|
2333
|
+
# `constraints` that names no top-level variable (#145)
|
|
2334
|
+
lists = derive_catalog_metadata(definition).get("lists")
|
|
2335
|
+
if lists:
|
|
2336
|
+
answer["lists"] = lists
|
|
2337
|
+
# What this box's own runs of it actually took, beside the defaults
|
|
2338
|
+
# they were run with - derived, never the curated `cost` (#93)
|
|
2339
|
+
observed = _observed_for_name(
|
|
2340
|
+
name, definition, workspace=ws.name if source.writable else None
|
|
2341
|
+
)
|
|
2342
|
+
if observed:
|
|
2343
|
+
answer["observed"] = observed
|
|
2344
|
+
return answer
|
|
2345
|
+
|
|
2346
|
+
def _observed_for_name(name, definition, arguments=None, *, workspace=None):
|
|
2347
|
+
"""One workflow's `observed` block, from the same aggregate the
|
|
2348
|
+
listing uses - so the figure a caller reads in the listing and the
|
|
2349
|
+
one they read here are the same figure.
|
|
2350
|
+
|
|
2351
|
+
`arguments` narrow it to the bucket the run being planned falls in;
|
|
2352
|
+
without them it is the figure the stored defaults give, which is the
|
|
2353
|
+
listing's. `workspace` scopes it to one workspace's own writable copy
|
|
2354
|
+
(#274); omitted, it is a shared catalog entry's pooled figure (#154)."""
|
|
2355
|
+
costs = getattr(app.state, "observed_costs", None)
|
|
2356
|
+
return (
|
|
2357
|
+
costs.observed(name, definition, arguments, workspace=workspace)
|
|
2358
|
+
if costs
|
|
2359
|
+
else None
|
|
2360
|
+
)
|
|
2361
|
+
|
|
2362
|
+
@app.get("/api/workflows/{name:path}")
|
|
2363
|
+
def get_workflow(name: str, ws: Workspace = Depends(selected_workspace)):
|
|
2364
|
+
path, source = resolve_readable_workflow(_sources_for(ws), name)
|
|
2365
|
+
try:
|
|
2366
|
+
with open(path, "r") as file:
|
|
2367
|
+
definition = json.load(file)
|
|
2368
|
+
except (OSError, json.JSONDecodeError) as e:
|
|
2369
|
+
raise HTTPException(status_code=500, detail=f"Could not read workflow: {e}")
|
|
2370
|
+
# Which root it came from and whether a save would land here or
|
|
2371
|
+
# copy elsewhere - the editor reads these to offer save-in-place
|
|
2372
|
+
# only for a writable source, save-a-copy otherwise
|
|
2373
|
+
return JSONResponse(
|
|
2374
|
+
definition,
|
|
2375
|
+
headers={
|
|
2376
|
+
"X-Workflow-Origin": source.origin,
|
|
2377
|
+
"X-Workflow-Writable": "true" if source.writable else "false",
|
|
2378
|
+
},
|
|
2379
|
+
)
|
|
2380
|
+
|
|
2381
|
+
# --------------------------------------------------------------- prompts
|
|
2382
|
+
|
|
2383
|
+
class PromptRequest(BaseModel):
|
|
2384
|
+
prompt: Dict[str, Any] = Field(description="The prompt definition to save")
|
|
2385
|
+
|
|
2386
|
+
@app.get("/api/prompt-schema")
|
|
2387
|
+
def get_prompt_schema():
|
|
2388
|
+
"""The JSON schema for stored prompts - the editor's diagnostics.
|
|
2389
|
+
Its own path, so a prompt named 'schema' cannot shadow it."""
|
|
2390
|
+
return JSONResponse(load_schema("prompt"))
|
|
2391
|
+
|
|
2392
|
+
def referenceable(name):
|
|
2393
|
+
try:
|
|
2394
|
+
validate_prompt_reference(name)
|
|
2395
|
+
return True
|
|
2396
|
+
except InvalidInputError:
|
|
2397
|
+
return False
|
|
2398
|
+
|
|
2399
|
+
def _prompt_roots():
|
|
2400
|
+
"""The prompt search path: the library this server writes to, then
|
|
2401
|
+
the read-only ones an --examples-dir tree brought with it. A name in
|
|
2402
|
+
an earlier root shadows the same name later, as on the workflow
|
|
2403
|
+
search path."""
|
|
2404
|
+
roots = [app.state.prompt_dir]
|
|
2405
|
+
primary = os.path.abspath(app.state.prompt_dir)
|
|
2406
|
+
for root in app.state.example_prompt_dirs:
|
|
2407
|
+
if os.path.abspath(root) != primary:
|
|
2408
|
+
roots.append(root)
|
|
2409
|
+
return roots
|
|
2410
|
+
|
|
2411
|
+
def _find_prompt(name):
|
|
2412
|
+
"""(path, writable) for the first root on the search path that holds
|
|
2413
|
+
this name. 404s when no root does, the way resolve_prompt_name does
|
|
2414
|
+
for a name that cannot be referenced at all."""
|
|
2415
|
+
for index, root in enumerate(_prompt_roots()):
|
|
2416
|
+
try:
|
|
2417
|
+
# allow_create so a name that is simply absent from this root
|
|
2418
|
+
# is a miss to carry on from, rather than a 404 raised out of
|
|
2419
|
+
# the middle of the search
|
|
2420
|
+
path = resolve_prompt_name(root, name, allow_create=True)
|
|
2421
|
+
except HTTPException as error:
|
|
2422
|
+
# a name no workflow could reference is a miss too, not the
|
|
2423
|
+
# 400 a save would get for it
|
|
2424
|
+
raise HTTPException(status_code=404, detail=error.detail)
|
|
2425
|
+
if os.path.isfile(path):
|
|
2426
|
+
return path, index == 0
|
|
2427
|
+
raise HTTPException(status_code=404, detail=f"Unknown prompt: {name}")
|
|
2428
|
+
|
|
2429
|
+
@app.get("/api/prompts")
|
|
2430
|
+
def list_prompts(
|
|
2431
|
+
tag: str | None = None,
|
|
2432
|
+
intended_model: str | None = None,
|
|
2433
|
+
include_text: bool = True,
|
|
2434
|
+
):
|
|
2435
|
+
# A stray file too deep or oddly named can sit in the directory, but
|
|
2436
|
+
# no workflow could reference it - listing it would only invite that
|
|
2437
|
+
paths = {}
|
|
2438
|
+
origins = {}
|
|
2439
|
+
roots = _prompt_roots()
|
|
2440
|
+
for index, root in enumerate(roots):
|
|
2441
|
+
for name in workflow_names(root):
|
|
2442
|
+
if referenceable(name) and name not in paths:
|
|
2443
|
+
paths[name] = os.path.join(root, f"{name}.json")
|
|
2444
|
+
origins[name] = WORKSPACE_ORIGIN if index == 0 else EXAMPLES_ORIGIN
|
|
2445
|
+
details = prompt_details(paths)
|
|
2446
|
+
|
|
2447
|
+
# Narrowing happens after the details are read, since that is where a
|
|
2448
|
+
# prompt says what it is for, and it narrows every parallel key at
|
|
2449
|
+
# once: a `prompts` list and a `details` map that disagree is worse
|
|
2450
|
+
# than no filter at all
|
|
2451
|
+
wanted = _matching_prompts(details, tag, intended_model)
|
|
2452
|
+
if wanted is not None:
|
|
2453
|
+
details = {
|
|
2454
|
+
name: detail for name, detail in details.items() if name in wanted
|
|
2455
|
+
}
|
|
2456
|
+
# The three parallel keys agree by construction, filter or no filter.
|
|
2457
|
+
# `prompt_details` drops a path whose mtime it cannot read - the file
|
|
2458
|
+
# went away between the walk and the read - and listing a name that
|
|
2459
|
+
# carries no detail only tells a caller to go and get a 404.
|
|
2460
|
+
origins = {name: origin for name, origin in origins.items() if name in details}
|
|
2461
|
+
|
|
2462
|
+
# The MCP listing cannot carry 44 prompt bodies - it exceeds a client's
|
|
2463
|
+
# result cap and the listing becomes uncallable - but the editors read
|
|
2464
|
+
# `text` as the card fallback, so the omission is opt-in and the size
|
|
2465
|
+
# is reported in its place
|
|
2466
|
+
if not include_text:
|
|
2467
|
+
details = {
|
|
2468
|
+
name: {
|
|
2469
|
+
**{key: value for key, value in detail.items() if key != "text"},
|
|
2470
|
+
"text_chars": len(detail.get("text") or ""),
|
|
2471
|
+
}
|
|
2472
|
+
for name, detail in details.items()
|
|
2473
|
+
}
|
|
2474
|
+
|
|
2475
|
+
return {
|
|
2476
|
+
# The writable library, unchanged: what a save is written to,
|
|
2477
|
+
# and what a client that predates the search path expects
|
|
2478
|
+
"prompt_dir": app.state.prompt_dir,
|
|
2479
|
+
"prompt_dirs": roots,
|
|
2480
|
+
"prompts": sorted(details),
|
|
2481
|
+
"origins": origins,
|
|
2482
|
+
"details": details,
|
|
2483
|
+
}
|
|
2484
|
+
|
|
2485
|
+
@app.put("/api/prompts/{name:path}")
|
|
2486
|
+
def save_prompt(name: str, request: PromptRequest):
|
|
2487
|
+
"""Write a prompt into the prompt directory. Like a workflow save,
|
|
2488
|
+
the definition must be schema-valid before it lands on disk."""
|
|
2489
|
+
status, message = validate_data(request.prompt, load_schema("prompt"))
|
|
2490
|
+
if not status:
|
|
2491
|
+
raise HTTPException(status_code=400, detail=message)
|
|
2492
|
+
if str(request.prompt.get("text", "")).startswith(RESERVED_TEXT_PREFIXES):
|
|
2493
|
+
raise HTTPException(
|
|
2494
|
+
status_code=400,
|
|
2495
|
+
detail="A prompt's text may not itself begin with a reference "
|
|
2496
|
+
f"prefix ({', '.join(RESERVED_TEXT_PREFIXES)})",
|
|
2497
|
+
)
|
|
2498
|
+
path = resolve_prompt_name(app.state.prompt_dir, name, allow_create=True)
|
|
2499
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
2500
|
+
with open(path, "w") as file:
|
|
2501
|
+
json.dump(request.prompt, file, indent=2)
|
|
2502
|
+
file.write("\n")
|
|
2503
|
+
logger.info(f"Saved prompt {name} to {path}")
|
|
2504
|
+
return {"name": name, "path": path}
|
|
2505
|
+
|
|
2506
|
+
@app.delete("/api/prompts/{name:path}")
|
|
2507
|
+
def delete_prompt(name: str):
|
|
2508
|
+
"""Remove a prompt file from the prompt directory. A prompt that
|
|
2509
|
+
came from a read-only examples library is not this server's to
|
|
2510
|
+
delete - the same 403 a read-only workflow answers with."""
|
|
2511
|
+
path, writable = _find_prompt(name)
|
|
2512
|
+
if not writable:
|
|
2513
|
+
raise HTTPException(
|
|
2514
|
+
status_code=403,
|
|
2515
|
+
detail=f"Prompt {name} is read-only: it comes from an examples "
|
|
2516
|
+
f"library, not this workspace's prompt directory",
|
|
2517
|
+
)
|
|
2518
|
+
os.remove(path)
|
|
2519
|
+
logger.info(f"Deleted prompt {name} ({path})")
|
|
2520
|
+
forget_workspace_usage()
|
|
2521
|
+
return {"name": name, "deleted": True}
|
|
2522
|
+
|
|
2523
|
+
@app.get("/api/prompts/{name:path}/download")
|
|
2524
|
+
@query_token_ok
|
|
2525
|
+
def download_prompt(name: str):
|
|
2526
|
+
"""Serve a stored prompt as a forced download."""
|
|
2527
|
+
path, _ = _find_prompt(name)
|
|
2528
|
+
return FileResponse(
|
|
2529
|
+
path, filename=os.path.basename(path), media_type="application/json"
|
|
2530
|
+
)
|
|
2531
|
+
|
|
2532
|
+
@app.get("/api/prompts/{name:path}")
|
|
2533
|
+
def get_prompt(name: str):
|
|
2534
|
+
path, writable = _find_prompt(name)
|
|
2535
|
+
try:
|
|
2536
|
+
with open(path, "r") as file:
|
|
2537
|
+
# Which library it came from, the way a workflow carries its
|
|
2538
|
+
# source - the editor offers delete only for a prompt this
|
|
2539
|
+
# server owns, and save-a-copy for a read-only one
|
|
2540
|
+
return JSONResponse(
|
|
2541
|
+
json.load(file),
|
|
2542
|
+
headers={
|
|
2543
|
+
"X-Prompt-Origin": (
|
|
2544
|
+
WORKSPACE_ORIGIN if writable else EXAMPLES_ORIGIN
|
|
2545
|
+
),
|
|
2546
|
+
"X-Prompt-Writable": "true" if writable else "false",
|
|
2547
|
+
},
|
|
2548
|
+
)
|
|
2549
|
+
except (OSError, json.JSONDecodeError) as e:
|
|
2550
|
+
raise HTTPException(status_code=500, detail=f"Could not read prompt: {e}")
|
|
2551
|
+
|
|
2552
|
+
# ------------------------------------------------------------- enhancers
|
|
2553
|
+
|
|
2554
|
+
class EnhanceRequest(BaseModel):
|
|
2555
|
+
idea: str = Field(description="The idea to expand into a full prompt")
|
|
2556
|
+
preset: str = Field(default="h3", description="Enhancer preset key")
|
|
2557
|
+
model_name: Optional[str] = Field(
|
|
2558
|
+
default=None, description="LLM repo id; the preset's default when omitted"
|
|
2559
|
+
)
|
|
2560
|
+
device: Optional[str] = Field(
|
|
2561
|
+
default=None,
|
|
2562
|
+
description="Device for the language model; defaults to cpu, "
|
|
2563
|
+
"keeping VRAM free for generation",
|
|
2564
|
+
)
|
|
2565
|
+
|
|
2566
|
+
@app.get("/api/enhancers")
|
|
2567
|
+
def list_enhancers():
|
|
2568
|
+
return {"presets": preset_descriptions()}
|
|
2569
|
+
|
|
2570
|
+
@app.post("/api/enhance", status_code=201)
|
|
2571
|
+
def enhance(request: EnhanceRequest, ws: Workspace = Depends(selected_workspace)):
|
|
2572
|
+
"""Queue a prompt enhancement as an ordinary job. The enhanced text
|
|
2573
|
+
is the job's single manifest file once it succeeds.
|
|
2574
|
+
|
|
2575
|
+
Scoped like any other job: the caller reads the result back from the
|
|
2576
|
+
workspace it asked in, so this has to write there too."""
|
|
2577
|
+
try:
|
|
2578
|
+
definition = build_enhance_workflow(
|
|
2579
|
+
request.preset,
|
|
2580
|
+
request.idea,
|
|
2581
|
+
model_name=request.model_name,
|
|
2582
|
+
device=request.device,
|
|
2583
|
+
)
|
|
2584
|
+
job = manager.submit(
|
|
2585
|
+
workflow=definition,
|
|
2586
|
+
arguments={},
|
|
2587
|
+
workflow_dir=ws.workflows,
|
|
2588
|
+
output_dir=ws.outputs,
|
|
2589
|
+
asset_dir=ws.assets,
|
|
2590
|
+
workspace=ws.name,
|
|
2591
|
+
)
|
|
2592
|
+
except Exception as e:
|
|
2593
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
2594
|
+
return manager.describe(job)
|
|
2595
|
+
|
|
2596
|
+
# --------------------------------------------------------------- gallery
|
|
2597
|
+
|
|
2598
|
+
# Built from the security layer's allowlists so a new format is added
|
|
2599
|
+
# exactly once - the gallery had already drifted (.bmp, .mkv, .mov)
|
|
2600
|
+
from ..security import (
|
|
2601
|
+
ALLOWED_AUDIO_EXTENSIONS,
|
|
2602
|
+
)
|
|
2603
|
+
|
|
2604
|
+
MEDIA_KINDS = {
|
|
2605
|
+
**{ext: "image" for ext in ALLOWED_IMAGE_EXTENSIONS},
|
|
2606
|
+
**{ext: "video" for ext in ALLOWED_VIDEO_EXTENSIONS},
|
|
2607
|
+
**{ext: "audio" for ext in ALLOWED_AUDIO_EXTENSIONS},
|
|
2608
|
+
# Not in the security allowlists above (nothing loads a .txt back
|
|
2609
|
+
# into a pipeline, so it is not a path a run reads), but a
|
|
2610
|
+
# text-shape run's deliverable is a real output and belongs in the
|
|
2611
|
+
# gallery like any other kind (#238)
|
|
2612
|
+
".txt": "text",
|
|
2613
|
+
}
|
|
2614
|
+
|
|
2615
|
+
# The allowlist members that are not already-compressed containers -
|
|
2616
|
+
# everything else in MEDIA_KINDS deflates for about nothing, so it is
|
|
2617
|
+
# stored instead (see _zip_download)
|
|
2618
|
+
RAW_MEDIA_EXTENSIONS = {".bmp", ".wav"}
|
|
2619
|
+
|
|
2620
|
+
# Exposed for tests - the two sets _zip_download's compression policy
|
|
2621
|
+
# reads, so a test can assert the relationship without reaching into a
|
|
2622
|
+
# closure
|
|
2623
|
+
app.state.media_kinds = MEDIA_KINDS
|
|
2624
|
+
app.state.raw_media_extensions = RAW_MEDIA_EXTENSIONS
|
|
2625
|
+
|
|
2626
|
+
# Longest side of an on-demand gallery thumbnail, in pixels
|
|
2627
|
+
GALLERY_THUMBNAIL_MAX_DIM = 320
|
|
2628
|
+
|
|
2629
|
+
def _strip_output_prefix(name):
|
|
2630
|
+
"""A gallery name, accepting the way a workflow argument would
|
|
2631
|
+
reference it ('output:<name>', #356) as well as the bare form
|
|
2632
|
+
every gallery listing reports. `asset:` already gets this courtesy
|
|
2633
|
+
on this same endpoint family (`is_asset_reference` below); a caller
|
|
2634
|
+
who spelled a name by copying an `output:` reference used to be met
|
|
2635
|
+
with a wrong-looking "path does not exist" instead, because the
|
|
2636
|
+
prefix was joined straight into the path rather than stripped first.
|
|
2637
|
+
|
|
2638
|
+
Applied once, at the top of every route that takes a gallery
|
|
2639
|
+
`name`, so the rest of that route - job lookups, run-path parsing,
|
|
2640
|
+
the file it echoes back - sees the same bare name `_output_file`
|
|
2641
|
+
resolves, rather than resolving the file correctly while a sibling
|
|
2642
|
+
lookup keyed on the untouched string quietly misses.
|
|
2643
|
+
"""
|
|
2644
|
+
if is_output_reference(name):
|
|
2645
|
+
return name.removeprefix(OUTPUT_PREFIX).strip()
|
|
2646
|
+
return name
|
|
2647
|
+
|
|
2648
|
+
def _output_file(name, root=None):
|
|
2649
|
+
"""A file inside a workspace's output directory, or a 404 - never
|
|
2650
|
+
outside it."""
|
|
2651
|
+
root = root or manager.output_dir
|
|
2652
|
+
name = _strip_output_prefix(name)
|
|
2653
|
+
try:
|
|
2654
|
+
path = validate_path(
|
|
2655
|
+
os.path.join(root, name),
|
|
2656
|
+
root,
|
|
2657
|
+
allow_create=False,
|
|
2658
|
+
)
|
|
2659
|
+
except PathTraversalError:
|
|
2660
|
+
# PathTraversalError's own message can embed the resolved
|
|
2661
|
+
# *absolute* server path (dw/security.py validate_path, the
|
|
2662
|
+
# containment branch) - useful in a log, not in a response a
|
|
2663
|
+
# remote caller reads. Still say *why* it was refused, since a
|
|
2664
|
+
# caller needs to tell "this name would have escaped the
|
|
2665
|
+
# workspace" from "this name is simply wrong" (#310) - the
|
|
2666
|
+
# distinction #134 pinned and a later leak fix (#247) collapsed.
|
|
2667
|
+
raise HTTPException(
|
|
2668
|
+
status_code=404,
|
|
2669
|
+
detail=f"Unknown file: {name} - path contains a disallowed pattern",
|
|
2670
|
+
)
|
|
2671
|
+
except InvalidInputError:
|
|
2672
|
+
raise HTTPException(
|
|
2673
|
+
status_code=404, detail=f"Unknown file: {name} - path does not exist"
|
|
2674
|
+
)
|
|
2675
|
+
except SecurityError:
|
|
2676
|
+
raise HTTPException(status_code=404, detail=f"Unknown file: {name}")
|
|
2677
|
+
if not os.path.isfile(path):
|
|
2678
|
+
raise HTTPException(status_code=404, detail="Unknown file")
|
|
2679
|
+
return path
|
|
2680
|
+
|
|
2681
|
+
def _asset_in(name, roots):
|
|
2682
|
+
"""The file a bare asset name has in one of these roots, or a 404.
|
|
2683
|
+
|
|
2684
|
+
`_asset_file` with the search path already in hand, for a caller
|
|
2685
|
+
resolving many names against the one workspace: each call to
|
|
2686
|
+
`resolve_asset_reference` walks the pinned fallbacks on its own, so
|
|
2687
|
+
calling it once per root re-walked them all every time - the name is
|
|
2688
|
+
validated once here instead, and each root is then just a join and
|
|
2689
|
+
an isfile check.
|
|
2690
|
+
"""
|
|
2691
|
+
try:
|
|
2692
|
+
validate_asset_reference(name)
|
|
2693
|
+
except SecurityError as e:
|
|
2694
|
+
raise HTTPException(status_code=404, detail=str(e))
|
|
2695
|
+
for root in roots:
|
|
2696
|
+
candidate = os.path.join(root, name)
|
|
2697
|
+
if not os.path.isfile(candidate):
|
|
2698
|
+
continue
|
|
2699
|
+
try:
|
|
2700
|
+
return validate_path(candidate, root)
|
|
2701
|
+
except SecurityError:
|
|
2702
|
+
# A symlink under this root can still point outside it -
|
|
2703
|
+
# isfile follows the link and says yes, and validate_path
|
|
2704
|
+
# is what actually catches the escape. That's a miss for
|
|
2705
|
+
# this root, not a 500: fall through to the next one and,
|
|
2706
|
+
# on a total miss, the same 404 every other miss gets.
|
|
2707
|
+
continue
|
|
2708
|
+
if not roots:
|
|
2709
|
+
detail = f"Unknown asset {name!r}: this workspace has no asset library"
|
|
2710
|
+
else:
|
|
2711
|
+
detail = f"Unknown asset {name!r}: not found in {', '.join(roots)}"
|
|
2712
|
+
raise HTTPException(status_code=404, detail=detail)
|
|
2713
|
+
|
|
2714
|
+
def _asset_file(reference, ws):
|
|
2715
|
+
"""The file an 'asset:' reference names in this workspace, or a 404.
|
|
2716
|
+
|
|
2717
|
+
Looked for down the same search path a run resolves 'asset:' in
|
|
2718
|
+
(_asset_roots), so what the API can read is what a job would load.
|
|
2719
|
+
A miss names every root that was searched, so the caller sees
|
|
2720
|
+
their own workspace library among them rather than just the last
|
|
2721
|
+
(often an examples directory they never wrote to).
|
|
2722
|
+
"""
|
|
2723
|
+
return _asset_in(
|
|
2724
|
+
reference.removeprefix(ASSET_PREFIX).strip(),
|
|
2725
|
+
_resolution_roots(ws),
|
|
2726
|
+
)
|
|
2727
|
+
|
|
2728
|
+
def _static_files_for(root):
|
|
2729
|
+
"""The StaticFiles instance bound to one root, built on first use and
|
|
2730
|
+
cached on app.state - see the comment where the cache is created."""
|
|
2731
|
+
cache = app.state.static_files_by_root
|
|
2732
|
+
files = cache.get(root)
|
|
2733
|
+
if files is None:
|
|
2734
|
+
files = StaticFiles(directory=root)
|
|
2735
|
+
cache[root] = files
|
|
2736
|
+
return files
|
|
2737
|
+
|
|
2738
|
+
def _common_assets(ws):
|
|
2739
|
+
"""The library every workspace under this root shares, or None.
|
|
2740
|
+
|
|
2741
|
+
A recurring cast is not the property of the workspace that first
|
|
2742
|
+
uploaded it, and a fresh workspace could not see it at all - the
|
|
2743
|
+
prompt library has been shared from the start for the same reason.
|
|
2744
|
+
"""
|
|
2745
|
+
return getattr(ws, "common_assets", None)
|
|
2746
|
+
|
|
2747
|
+
def _asset_roots(ws):
|
|
2748
|
+
"""The asset search path of one workspace: its own library, then the
|
|
2749
|
+
one shared by every workspace under this root, then the read-only
|
|
2750
|
+
ones an --examples-dir tree brought with it. The same order 'asset:'
|
|
2751
|
+
resolves in (dw/assets.asset_search_path), so what the browser lists
|
|
2752
|
+
is what a job would load."""
|
|
2753
|
+
roots = []
|
|
2754
|
+
for root in [ws.assets, _common_assets(ws), *app.state.example_asset_dirs]:
|
|
2755
|
+
if not root:
|
|
2756
|
+
continue
|
|
2757
|
+
root = os.path.abspath(root)
|
|
2758
|
+
if root not in roots and os.path.isdir(root):
|
|
2759
|
+
roots.append(root)
|
|
2760
|
+
return roots
|
|
2761
|
+
|
|
2762
|
+
def _resolution_roots(ws):
|
|
2763
|
+
"""`_asset_roots(ws)`, falling back to the workspace's own (possibly
|
|
2764
|
+
nonexistent) library when the search path is empty.
|
|
2765
|
+
|
|
2766
|
+
A caller resolving a name still needs *somewhere* to fail against:
|
|
2767
|
+
with no root at all the 404 would name no directory, leaving the
|
|
2768
|
+
caller to guess where it looked. Naming the workspace's own
|
|
2769
|
+
directory keeps the failure pointing at the library the caller
|
|
2770
|
+
thinks they're working in, even when that library hasn't been
|
|
2771
|
+
created yet.
|
|
2772
|
+
|
|
2773
|
+
Never `[None]`: a server configured with no asset library at all has
|
|
2774
|
+
nothing to point at either, and `_asset_in` turns the resulting empty
|
|
2775
|
+
list into the "no asset library" 404 rather than joining `None`.
|
|
2776
|
+
"""
|
|
2777
|
+
roots = _asset_roots(ws)
|
|
2778
|
+
if roots:
|
|
2779
|
+
return roots
|
|
2780
|
+
return [os.path.abspath(ws.assets)] if ws.assets else []
|
|
2781
|
+
|
|
2782
|
+
def _asset_roots_for_job(job_id, ws):
|
|
2783
|
+
"""The asset search path a job's own run used, for export: its spec's
|
|
2784
|
+
`asset_dir` (or the historical row's), then the read-only example
|
|
2785
|
+
libraries an --examples-dir tree brought with it - the same shape
|
|
2786
|
+
`_asset_roots` builds for the selected workspace, but rooted at
|
|
2787
|
+
wherever the job actually ran rather than at the workspace the
|
|
2788
|
+
caller happens to be scoped to now. A job that ran in one workspace
|
|
2789
|
+
while the caller exports it scoped to another must still find its
|
|
2790
|
+
own 'asset:' files, not the other workspace's.
|
|
2791
|
+
|
|
2792
|
+
Falls back to `_asset_roots(ws)` when the job carries no asset_dir
|
|
2793
|
+
of its own - an inline-workflow job, or one recorded before this
|
|
2794
|
+
field existed."""
|
|
2795
|
+
job = manager.get(job_id)
|
|
2796
|
+
if job is None:
|
|
2797
|
+
return _asset_roots(ws)
|
|
2798
|
+
spec = (job.get("spec") or {}) if isinstance(job, dict) else job.spec
|
|
2799
|
+
asset_dir = spec.get("asset_dir")
|
|
2800
|
+
if not asset_dir:
|
|
2801
|
+
return _asset_roots(ws)
|
|
2802
|
+
roots = []
|
|
2803
|
+
for root in [asset_dir, _common_assets(ws), *app.state.example_asset_dirs]:
|
|
2804
|
+
if not root:
|
|
2805
|
+
continue
|
|
2806
|
+
root = os.path.abspath(root)
|
|
2807
|
+
if root not in roots and os.path.isdir(root):
|
|
2808
|
+
roots.append(root)
|
|
2809
|
+
return roots
|
|
2810
|
+
|
|
2811
|
+
def _served_url(path, ws, version=None):
|
|
2812
|
+
"""The URL a served file is reachable at: the default workspace's
|
|
2813
|
+
files keep the URL they have always had, a named one carries the
|
|
2814
|
+
same selector its API calls do, so one route serves both. 'v=' is
|
|
2815
|
+
cache-busting for a name reused by a rerun, not the workspace
|
|
2816
|
+
selector, so it always comes last."""
|
|
2817
|
+
url = path if ws.is_default else f"{path}?workspace={quote(ws.name)}"
|
|
2818
|
+
if version is None:
|
|
2819
|
+
return url
|
|
2820
|
+
separator = "&" if "?" in url else "?"
|
|
2821
|
+
return f"{url}{separator}v={version}"
|
|
2822
|
+
|
|
2823
|
+
def _absolute_served_url(path, ws, version=None):
|
|
2824
|
+
"""The same URL, made openable by a client with no other way to
|
|
2825
|
+
learn this server's origin (#353) - an MCP-only agent, which is
|
|
2826
|
+
never told a request's Host and must not guess one. `None` unless
|
|
2827
|
+
an operator has configured `public_url` (or `DW_PUBLIC_URL`):
|
|
2828
|
+
deriving an origin from request/forwarded headers would trust
|
|
2829
|
+
whatever the caller claims to be, so a caller gets nothing rather
|
|
2830
|
+
than a guess."""
|
|
2831
|
+
origin = os.environ.get("DW_PUBLIC_URL") or settings.public_url
|
|
2832
|
+
if not origin:
|
|
2833
|
+
return None
|
|
2834
|
+
return f"{origin.rstrip('/')}{_served_url(path, ws, version)}"
|
|
2835
|
+
|
|
2836
|
+
def _iter_gallery_files(root, group_runs=True):
|
|
2837
|
+
"""Every media file under a directory tree. Yields (relative_name,
|
|
2838
|
+
folder, subfolder, kind, path) - relative_name always uses '/' so
|
|
2839
|
+
it round-trips through a URL the same way on every platform.
|
|
2840
|
+
|
|
2841
|
+
With group_runs (the gallery's own use, over the output directory):
|
|
2842
|
+
recurses into the per-workflow subfolders (dw/workflow.py's
|
|
2843
|
+
effective_output_dir writes each run under '<workflow
|
|
2844
|
+
identity>/<run id>/', and mirrors a workflow's position under a
|
|
2845
|
+
'workflows' tree in the flat layout). The folder a file is grouped
|
|
2846
|
+
under is the identity - the run id is dropped, so a workflow run
|
|
2847
|
+
fifty times is one folder in the filter, not fifty - and whatever
|
|
2848
|
+
followed the run id is the subfolder, the part of the run a step's
|
|
2849
|
+
'result.subfolder' put it in ('final', 'intermediate'). A flat-layout
|
|
2850
|
+
path has no run id to anchor on, so its subfolder is '' and its
|
|
2851
|
+
whole directory is the folder, as it always was.
|
|
2852
|
+
|
|
2853
|
+
Without it (the asset library's use, which has no run ids to strip):
|
|
2854
|
+
folder is just the plain relative directory and subfolder is ''.
|
|
2855
|
+
|
|
2856
|
+
A file symlink resolving outside root is skipped: os.walk lists it
|
|
2857
|
+
among the names, and the entry would carry the target's size and
|
|
2858
|
+
mtime. A linked directory is never descended (os.walk's default)."""
|
|
2859
|
+
for current, _dirs, names in os.walk(root):
|
|
2860
|
+
rel_root = os.path.relpath(current, root)
|
|
2861
|
+
directory = "" if rel_root == "." else rel_root.replace(os.sep, "/")
|
|
2862
|
+
for name in names:
|
|
2863
|
+
extension = os.path.splitext(name)[1].lower()
|
|
2864
|
+
kind = MEDIA_KINDS.get(extension)
|
|
2865
|
+
if kind is None:
|
|
2866
|
+
continue
|
|
2867
|
+
path = os.path.join(current, name)
|
|
2868
|
+
if not contained(path, root):
|
|
2869
|
+
continue
|
|
2870
|
+
relative_name = name if not directory else f"{directory}/{name}"
|
|
2871
|
+
if group_runs:
|
|
2872
|
+
folder, run_id, subfolder = split_run_path(relative_name)
|
|
2873
|
+
else:
|
|
2874
|
+
folder, subfolder, run_id = directory, "", ""
|
|
2875
|
+
yield (
|
|
2876
|
+
relative_name,
|
|
2877
|
+
folder,
|
|
2878
|
+
subfolder,
|
|
2879
|
+
run_id,
|
|
2880
|
+
kind,
|
|
2881
|
+
path,
|
|
2882
|
+
)
|
|
2883
|
+
|
|
2884
|
+
def _gallery_entries(root, ws):
|
|
2885
|
+
entries = []
|
|
2886
|
+
try:
|
|
2887
|
+
files = list(_iter_gallery_files(root))
|
|
2888
|
+
except OSError:
|
|
2889
|
+
files = []
|
|
2890
|
+
# One read of each workflow's run ordinals per listing, not per file:
|
|
2891
|
+
# a run of fifty files would otherwise re-read the same manifests
|
|
2892
|
+
# fifty times
|
|
2893
|
+
versions_by_folder = {}
|
|
2894
|
+
|
|
2895
|
+
def _version(folder, run_id):
|
|
2896
|
+
if not run_id:
|
|
2897
|
+
return None
|
|
2898
|
+
if folder not in versions_by_folder:
|
|
2899
|
+
versions_by_folder[folder] = run_versions(os.path.join(root, folder))
|
|
2900
|
+
return versions_by_folder[folder].get(run_id)
|
|
2901
|
+
|
|
2902
|
+
for relative_name, folder, subfolder, run_id, kind, path in files:
|
|
2903
|
+
try:
|
|
2904
|
+
stat = os.stat(path)
|
|
2905
|
+
except OSError:
|
|
2906
|
+
continue
|
|
2907
|
+
# File names look like '{workflow}-{step}-{i}.{j}.{k}.ext'; every
|
|
2908
|
+
# artifact from one step shares the '{workflow}-{step}' prefix, so
|
|
2909
|
+
# the label keeps the full name including the extension rather
|
|
2910
|
+
# than truncating at the first dot - otherwise sibling outputs of
|
|
2911
|
+
# the same step would show identical, indistinguishable labels,
|
|
2912
|
+
# and a step that writes more than one kind of file (e.g. a still
|
|
2913
|
+
# plus a video) would lose the extension that tells them apart
|
|
2914
|
+
label = os.path.basename(relative_name)
|
|
2915
|
+
output_path = f"/outputs/{quote(relative_name)}"
|
|
2916
|
+
entry = {
|
|
2917
|
+
"name": relative_name,
|
|
2918
|
+
"folder": folder,
|
|
2919
|
+
"subfolder": subfolder,
|
|
2920
|
+
# Which run wrote it, and that run's ordinal among this
|
|
2921
|
+
# workflow's runs - the 'v4' a person sees in the grid
|
|
2922
|
+
# and an agent says out loud. Two runs write the same
|
|
2923
|
+
# basename, so `label` cannot tell them apart and
|
|
2924
|
+
# `name` is too long to quote. None under the flat
|
|
2925
|
+
# layout, which has no runs to number
|
|
2926
|
+
"run_id": run_id,
|
|
2927
|
+
"version": _version(folder, run_id),
|
|
2928
|
+
# Quoted (slashes kept literal): a name carrying '#', '?'
|
|
2929
|
+
# or '%' would otherwise break the src the gallery
|
|
2930
|
+
# renders it into. The mtime still rides along for cache
|
|
2931
|
+
# busting when a file's content changes without its name
|
|
2932
|
+
# changing (e.g. a manual overwrite outside the engine) -
|
|
2933
|
+
# normal reruns get a fresh name instead, see
|
|
2934
|
+
# dw/result.py's output_file_path
|
|
2935
|
+
"url": _served_url(output_path, ws, int(stat.st_mtime)),
|
|
2936
|
+
"kind": kind,
|
|
2937
|
+
"size": stat.st_size,
|
|
2938
|
+
"mtime": stat.st_mtime,
|
|
2939
|
+
"label": label,
|
|
2940
|
+
}
|
|
2941
|
+
absolute_url = _absolute_served_url(output_path, ws, int(stat.st_mtime))
|
|
2942
|
+
if absolute_url is not None:
|
|
2943
|
+
entry["absolute_url"] = absolute_url
|
|
2944
|
+
entries.append(entry)
|
|
2945
|
+
entries.sort(key=lambda e: e["mtime"], reverse=True)
|
|
2946
|
+
return entries
|
|
2947
|
+
|
|
2948
|
+
def _iter_orphan_runs(root):
|
|
2949
|
+
"""Run directories under `root` holding nothing but their own
|
|
2950
|
+
bookkeeping (RUN_BOOKKEEPING_FILES) - a run whose output was deleted
|
|
2951
|
+
before #134's by-name `delete_output`, or one that failed before
|
|
2952
|
+
writing anything. Yields (name, mtime) where `name` is the
|
|
2953
|
+
`<identity>/<run id>` string `delete_output` already accepts (#170).
|
|
2954
|
+
|
|
2955
|
+
By what is absent, not by extension: a `text`-shape run writes .txt
|
|
2956
|
+
and a `utility`-shape run may write nothing the gallery lists, and
|
|
2957
|
+
neither is junk. This call only lists; deciding whether an entry is
|
|
2958
|
+
junk stays a human/agent call before `delete_output` is invoked."""
|
|
2959
|
+
for current, dirs, _names in os.walk(root):
|
|
2960
|
+
if not is_run_id(os.path.basename(current)):
|
|
2961
|
+
continue
|
|
2962
|
+
# A run directory holds no run directories of its own
|
|
2963
|
+
dirs[:] = []
|
|
2964
|
+
# A dotfile is not output either: a .DS_Store Finder left behind
|
|
2965
|
+
# would otherwise make the run permanently non-orphan
|
|
2966
|
+
has_output = any(
|
|
2967
|
+
name not in RUN_BOOKKEEPING_FILES and not name.startswith(".")
|
|
2968
|
+
for _sub_current, _sub_dirs, sub_names in os.walk(current)
|
|
2969
|
+
for name in sub_names
|
|
2970
|
+
)
|
|
2971
|
+
if has_output:
|
|
2972
|
+
continue
|
|
2973
|
+
try:
|
|
2974
|
+
mtime = os.stat(current).st_mtime
|
|
2975
|
+
except OSError:
|
|
2976
|
+
continue
|
|
2977
|
+
name = os.path.relpath(current, root).replace(os.sep, "/")
|
|
2978
|
+
yield (name, mtime)
|
|
2979
|
+
|
|
2980
|
+
def _orphan_entries(root):
|
|
2981
|
+
entries = [
|
|
2982
|
+
{"name": name, "mtime": mtime} for name, mtime in _iter_orphan_runs(root)
|
|
2983
|
+
]
|
|
2984
|
+
entries.sort(key=lambda e: e["mtime"], reverse=True)
|
|
2985
|
+
return entries
|
|
2986
|
+
|
|
2987
|
+
@app.get("/api/gallery")
|
|
2988
|
+
def gallery(
|
|
2989
|
+
limit: int = 200,
|
|
2990
|
+
offset: int = 0,
|
|
2991
|
+
folder: Optional[str] = None,
|
|
2992
|
+
subfolder: Optional[str] = None,
|
|
2993
|
+
only_orphans: bool = False,
|
|
2994
|
+
version: Optional[int] = None,
|
|
2995
|
+
media: bool = False,
|
|
2996
|
+
ws: Workspace = Depends(selected_workspace),
|
|
2997
|
+
):
|
|
2998
|
+
"""A page of media files in the output directory, newest first.
|
|
2999
|
+
Stateless by design - the gallery survives server restarts because
|
|
3000
|
+
it reads the directory tree, not job history. 'folders' lists every
|
|
3001
|
+
distinct workflow folder present (over the whole directory, not just
|
|
3002
|
+
this page), for the UI's folder filter - a run id is not a folder of
|
|
3003
|
+
its own, so a workflow's runs group together; '' stands for files
|
|
3004
|
+
saved directly at the output root, and is itself always a member so
|
|
3005
|
+
that folder-less outputs stay selectable once anything is nested.
|
|
3006
|
+
'subfolders' is the other axis, over the whole directory the same
|
|
3007
|
+
way: the in-run subfolders steps wrote into ('final',
|
|
3008
|
+
'intermediate'), '' for files at a run's root. `folder` and
|
|
3009
|
+
`subfolder` filter independently and intersect when both are given.
|
|
3010
|
+
`version` narrows to the runs holding that ordinal - with `folder`,
|
|
3011
|
+
the one run "v4" names; without it, that run of every workflow.
|
|
3012
|
+
|
|
3013
|
+
`only_orphans=true` inverts the whole call: instead of media files,
|
|
3014
|
+
it returns run directories holding nothing but their own
|
|
3015
|
+
bookkeeping (manifest.json, workflow.json, job.json) as `runs`,
|
|
3016
|
+
each `{name, mtime}` - a run that wrote any file at all, a
|
|
3017
|
+
text-shape prompt or a utility's side output included, is not
|
|
3018
|
+
listed. `folder`/`subfolder` and the `folders`/`subfolders` facets
|
|
3019
|
+
do not apply in this mode, since an orphan run has no file to
|
|
3020
|
+
carry either. `name` is exactly what `DELETE /api/gallery/{name}`
|
|
3021
|
+
accepts, so listing and deleting an orphan is a two-call round
|
|
3022
|
+
trip (#170).
|
|
3023
|
+
|
|
3024
|
+
`media=true` adds `duration_seconds` to each audio/video entry,
|
|
3025
|
+
probed the same way `get_gallery_metadata` reports it - which two
|
|
3026
|
+
takes of the same workflow otherwise have no way to be told apart
|
|
3027
|
+
by, since size and mtime are misleading proxies for length (#356).
|
|
3028
|
+
Off by default and bounded by `limit`: only the page actually
|
|
3029
|
+
returned is probed, not the whole listing, so the cost of asking
|
|
3030
|
+
stays proportional to the page size rather than the library size."""
|
|
3031
|
+
if only_orphans:
|
|
3032
|
+
entries = _orphan_entries(ws.outputs)
|
|
3033
|
+
offset = max(0, offset)
|
|
3034
|
+
limit = max(0, limit)
|
|
3035
|
+
page = entries[offset : offset + limit]
|
|
3036
|
+
return {
|
|
3037
|
+
"runs": page,
|
|
3038
|
+
"total": len(entries),
|
|
3039
|
+
"offset": offset,
|
|
3040
|
+
"limit": limit,
|
|
3041
|
+
"workspace": ws.name,
|
|
3042
|
+
}
|
|
3043
|
+
entries = _gallery_entries(ws.outputs, ws)
|
|
3044
|
+
folders = sorted({e["folder"] for e in entries} | {""})
|
|
3045
|
+
subfolders = sorted({e["subfolder"] for e in entries} | {""})
|
|
3046
|
+
if folder is not None:
|
|
3047
|
+
entries = [e for e in entries if e["folder"] == folder]
|
|
3048
|
+
if subfolder is not None:
|
|
3049
|
+
entries = [e for e in entries if e["subfolder"] == subfolder]
|
|
3050
|
+
if version is not None:
|
|
3051
|
+
entries = [e for e in entries if e["version"] == version]
|
|
3052
|
+
offset = max(0, offset)
|
|
3053
|
+
limit = max(0, limit)
|
|
3054
|
+
page = entries[offset : offset + limit]
|
|
3055
|
+
if media:
|
|
3056
|
+
for entry in page:
|
|
3057
|
+
if entry["kind"] not in ("audio", "video"):
|
|
3058
|
+
continue
|
|
3059
|
+
probed = probe_media(os.path.join(ws.outputs, entry["name"]))
|
|
3060
|
+
if probed is not None:
|
|
3061
|
+
entry["duration_seconds"] = probed.get("duration_seconds")
|
|
3062
|
+
return {
|
|
3063
|
+
"files": page,
|
|
3064
|
+
"total": len(entries),
|
|
3065
|
+
"offset": offset,
|
|
3066
|
+
"limit": limit,
|
|
3067
|
+
"folders": folders,
|
|
3068
|
+
"subfolders": subfolders,
|
|
3069
|
+
"workspace": ws.name,
|
|
3070
|
+
}
|
|
3071
|
+
|
|
3072
|
+
@app.get("/api/gallery/{name:path}/metadata")
|
|
3073
|
+
def gallery_metadata(
|
|
3074
|
+
name: str,
|
|
3075
|
+
envelope: bool = False,
|
|
3076
|
+
ws: Workspace = Depends(selected_workspace),
|
|
3077
|
+
):
|
|
3078
|
+
"""Generation metadata embedded in a saved image ('workflow' inside
|
|
3079
|
+
it is the full definition the editor can reopen), plus the job that
|
|
3080
|
+
produced the file when history remembers one, plus - for audio and
|
|
3081
|
+
video - what the file itself holds: duration, format and level,
|
|
3082
|
+
which is how an agent that cannot listen checks a track. Only an
|
|
3083
|
+
image embeds 'metadata' this way - it is always null for audio and
|
|
3084
|
+
video, since neither format has a slot this writer uses; recover
|
|
3085
|
+
the recipe from 'job' (GET /api/jobs/{id}/workflow) when one is
|
|
3086
|
+
known, or from nothing when it isn't (a kept asset has no job).
|
|
3087
|
+
|
|
3088
|
+
`envelope=true` adds the soundtrack's level second by second, which
|
|
3089
|
+
is what says *where* in a track something is - whether a shot is
|
|
3090
|
+
still voiced at its last frame, how deep the hole at a seam goes.
|
|
3091
|
+
Opt-in: a ten-minute track is 600 numbers, and the default call has
|
|
3092
|
+
to stay small.
|
|
3093
|
+
|
|
3094
|
+
`name` may also be an 'asset:' reference, and then it is the input
|
|
3095
|
+
asset of that name that is described rather than an output (#127).
|
|
3096
|
+
The numbers here - duration, frame count, fps, sample rate - are
|
|
3097
|
+
what decide whether a call will work at all, and for a file the
|
|
3098
|
+
caller is about to *consume* they were previously unobtainable:
|
|
3099
|
+
the only way to read a wav's length was to run a job that copied it
|
|
3100
|
+
into the output directory. `job` is null for an asset (nothing here
|
|
3101
|
+
produced it) and `source` says which of the two roots answered."""
|
|
3102
|
+
run_id, version = "", None
|
|
3103
|
+
name = _strip_output_prefix(name)
|
|
3104
|
+
if is_asset_reference(name):
|
|
3105
|
+
path = _asset_file(name, ws)
|
|
3106
|
+
source, job = "asset", None
|
|
3107
|
+
else:
|
|
3108
|
+
path = _output_file(name, ws.outputs)
|
|
3109
|
+
source = "output"
|
|
3110
|
+
try:
|
|
3111
|
+
# Scoped to this workspace: two workspaces can each write a
|
|
3112
|
+
# file with the same relative name, and an unscoped lookup
|
|
3113
|
+
# could attribute this one to the wrong workspace's job
|
|
3114
|
+
job = manager.history.job_for_file(name, workspace=ws.name)
|
|
3115
|
+
except Exception:
|
|
3116
|
+
job = None
|
|
3117
|
+
# Which run wrote it, and that run's ordinal - the same 'v4' the
|
|
3118
|
+
# listing reports. After "look at version 3" this is the next
|
|
3119
|
+
# call, so it confirms the right file was reached rather than
|
|
3120
|
+
# sending the caller back to the listing
|
|
3121
|
+
folder, run_id, _subfolder = split_run_path(name)
|
|
3122
|
+
if run_id:
|
|
3123
|
+
try:
|
|
3124
|
+
identity_dir = validate_path(
|
|
3125
|
+
os.path.join(ws.outputs, folder), ws.outputs
|
|
3126
|
+
)
|
|
3127
|
+
except SecurityError:
|
|
3128
|
+
identity_dir = None
|
|
3129
|
+
if identity_dir:
|
|
3130
|
+
version = run_versions(identity_dir).get(run_id)
|
|
3131
|
+
metadata = read_embedded_metadata(path)
|
|
3132
|
+
extension = os.path.splitext(path)[1].lower()
|
|
3133
|
+
media = (
|
|
3134
|
+
probe_media(path, envelope=envelope)
|
|
3135
|
+
if MEDIA_KINDS.get(extension) in ("audio", "video")
|
|
3136
|
+
else None
|
|
3137
|
+
)
|
|
3138
|
+
if media is not None and source == "output":
|
|
3139
|
+
# Where each shot of a joined video sits, as the run that wrote
|
|
3140
|
+
# it recorded (dw/shots.py) - null for a file not joined from shots
|
|
3141
|
+
media["shots"] = recorded_shots(ws.outputs, name)
|
|
3142
|
+
elif media is not None and source == "asset":
|
|
3143
|
+
# keep_output carries the source run's shots into a sidecar
|
|
3144
|
+
# manifest beside the asset (#393); a file kept before that fix,
|
|
3145
|
+
# or never joined from shots, has none
|
|
3146
|
+
media["shots"] = shots_beside(path)
|
|
3147
|
+
return {
|
|
3148
|
+
"name": name,
|
|
3149
|
+
"source": source,
|
|
3150
|
+
"metadata": metadata,
|
|
3151
|
+
"job": job,
|
|
3152
|
+
"run_id": run_id,
|
|
3153
|
+
"version": version,
|
|
3154
|
+
"media": media,
|
|
3155
|
+
}
|
|
3156
|
+
|
|
3157
|
+
@app.get("/api/gallery/{name:path}/assess")
|
|
3158
|
+
def gallery_assess(
|
|
3159
|
+
name: str,
|
|
3160
|
+
probe: Optional[str] = None,
|
|
3161
|
+
detail: bool = False,
|
|
3162
|
+
ws: Workspace = Depends(selected_workspace),
|
|
3163
|
+
):
|
|
3164
|
+
"""Measure a finished cut and say where to look (#388): every
|
|
3165
|
+
assessment probe that applies to the file, run here in the server
|
|
3166
|
+
process on one decode - a sync route, so it runs beside a GPU job
|
|
3167
|
+
rather than queueing behind it. Findings are places to look, not
|
|
3168
|
+
verdicts; nothing acts on one (dw/assessment_rules.py).
|
|
3169
|
+
|
|
3170
|
+
The default answer merges the probes' `findings`, `rules_applied`
|
|
3171
|
+
and `rules_skipped`, and names each probe the file cannot feed in
|
|
3172
|
+
`not_applicable` (a still, no soundtrack, no recorded shots);
|
|
3173
|
+
`detail=true` adds each probe's full answer under `probes`.
|
|
3174
|
+
`probe` names one - analyze_shots, analyze_seams or
|
|
3175
|
+
analyze_sync_drift - and answers with its full body. It is checked
|
|
3176
|
+
before the name is resolved. `name` may be an `asset:` reference,
|
|
3177
|
+
and then the shots are the ones keep_output carried beside it."""
|
|
3178
|
+
rejected = unknown_probe(probe)
|
|
3179
|
+
if rejected:
|
|
3180
|
+
raise HTTPException(status_code=400, detail=rejected)
|
|
3181
|
+
name = _strip_output_prefix(name)
|
|
3182
|
+
if is_asset_reference(name):
|
|
3183
|
+
path = _asset_file(name, ws)
|
|
3184
|
+
source, shots = "asset", shots_beside(path)
|
|
3185
|
+
else:
|
|
3186
|
+
path = _output_file(name, ws.outputs)
|
|
3187
|
+
source, shots = "output", recorded_shots(ws.outputs, name)
|
|
3188
|
+
kind = MEDIA_KINDS.get(os.path.splitext(path)[1].lower())
|
|
3189
|
+
try:
|
|
3190
|
+
body = assess(path, kind, shots, probe=probe, detail=detail)
|
|
3191
|
+
except (ValueError, OSError) as e:
|
|
3192
|
+
raise HTTPException(
|
|
3193
|
+
status_code=422, detail=f"{name} could not be read: {e}"
|
|
3194
|
+
)
|
|
3195
|
+
return {"name": name, "source": source, "kind": kind, **body}
|
|
3196
|
+
|
|
3197
|
+
@app.get("/api/gallery/{name:path}/audio")
|
|
3198
|
+
def gallery_audio(
|
|
3199
|
+
name: str,
|
|
3200
|
+
start: Optional[float] = None,
|
|
3201
|
+
duration: Optional[float] = None,
|
|
3202
|
+
ws: Workspace = Depends(selected_workspace),
|
|
3203
|
+
):
|
|
3204
|
+
"""The soundtrack of an output or asset, as WAV - a muxed video's
|
|
3205
|
+
track, which `get_output_audio` used to refuse outright, or an
|
|
3206
|
+
excerpt (`start` + `duration`, seconds) of a track too long to send
|
|
3207
|
+
whole (#193). An excerpt names itself in the response headers
|
|
3208
|
+
(`X-DW-Excerpt-Start`, `X-DW-Excerpt-Duration`) beside the whole
|
|
3209
|
+
track's `X-DW-Duration` - omitted only when a container carries no
|
|
3210
|
+
duration in its own header - so a cut is never silent (#204).
|
|
3211
|
+
|
|
3212
|
+
An audio-only file asked for whole is served as its own bytes in its
|
|
3213
|
+
own encoding - there is nothing to extract, and a transcode would
|
|
3214
|
+
change what the agent hears."""
|
|
3215
|
+
name = _strip_output_prefix(name)
|
|
3216
|
+
if is_asset_reference(name):
|
|
3217
|
+
path = _asset_file(name, ws)
|
|
3218
|
+
else:
|
|
3219
|
+
path = _output_file(name, ws.outputs)
|
|
3220
|
+
extension = os.path.splitext(path)[1].lower()
|
|
3221
|
+
kind = MEDIA_KINDS.get(extension)
|
|
3222
|
+
if kind not in ("audio", "video"):
|
|
3223
|
+
raise HTTPException(status_code=404, detail=f"{name} carries no soundtrack")
|
|
3224
|
+
|
|
3225
|
+
excerpt = start is not None or duration is not None
|
|
3226
|
+
if kind == "audio" and not excerpt:
|
|
3227
|
+
# The container's own header has the duration - reading it does
|
|
3228
|
+
# not decode a single frame, unlike probe_media (which measures
|
|
3229
|
+
# level and would pay for a full decode just for one number).
|
|
3230
|
+
headers = {}
|
|
3231
|
+
duration_seconds = media_duration(path)
|
|
3232
|
+
if duration_seconds is not None:
|
|
3233
|
+
headers["X-DW-Duration"] = str(duration_seconds)
|
|
3234
|
+
if extension == ".wav":
|
|
3235
|
+
# mimetypes says audio/x-wav on macOS, audio/vnd.wave from
|
|
3236
|
+
# Python 3.14's builtin table on a box with no system mime
|
|
3237
|
+
# file; an extract says audio/wav, and a whole WAV must not
|
|
3238
|
+
# read as a different kind
|
|
3239
|
+
media_type = "audio/wav"
|
|
3240
|
+
else:
|
|
3241
|
+
media_type = mimetypes.guess_type(path)[0] or "application/octet-stream"
|
|
3242
|
+
return FileResponse(path, media_type=media_type, headers=headers)
|
|
3243
|
+
|
|
3244
|
+
# A track over the cap is refused at the header, not after it has
|
|
3245
|
+
# been decoded and shipped: the MCP side would refuse the same bytes
|
|
3246
|
+
# for the same reason, having paid for all of them. An excerpt is
|
|
3247
|
+
# sized by its own span - `duration`, clipped to what is left of the
|
|
3248
|
+
# track after `start` - so a whole-length "excerpt" is not a way
|
|
3249
|
+
# around the gate.
|
|
3250
|
+
shape = audio_shape(path)
|
|
3251
|
+
if shape is not None and shape["duration_seconds"] is not None:
|
|
3252
|
+
span = shape["duration_seconds"]
|
|
3253
|
+
if excerpt and duration is not None:
|
|
3254
|
+
span = max(0.0, min(float(duration), span - float(start or 0.0)))
|
|
3255
|
+
projected = projected_wav_base64_size({**shape, "duration_seconds": span})
|
|
3256
|
+
if projected > MAX_INLINE_AUDIO_BYTES:
|
|
3257
|
+
what = (
|
|
3258
|
+
f"a {span:.1f}s excerpt of {name}"
|
|
3259
|
+
if excerpt
|
|
3260
|
+
else f"{name}'s whole soundtrack"
|
|
3261
|
+
)
|
|
3262
|
+
advice = (
|
|
3263
|
+
"Ask for a shorter `duration`"
|
|
3264
|
+
if excerpt
|
|
3265
|
+
else "Ask for an excerpt with `start` and `duration` (seconds)"
|
|
3266
|
+
)
|
|
3267
|
+
raise HTTPException(
|
|
3268
|
+
status_code=413,
|
|
3269
|
+
detail=(
|
|
3270
|
+
f"{what} would be {projected} bytes base64-encoded as WAV "
|
|
3271
|
+
f"- over the {MAX_INLINE_AUDIO_BYTES} byte limit for an "
|
|
3272
|
+
f"inline clip. {advice}, or download the file."
|
|
3273
|
+
),
|
|
3274
|
+
)
|
|
3275
|
+
|
|
3276
|
+
try:
|
|
3277
|
+
data, info = extract_audio(path, start=start, duration=duration)
|
|
3278
|
+
except NoSoundtrack:
|
|
3279
|
+
raise HTTPException(status_code=404, detail=f"{name} carries no soundtrack")
|
|
3280
|
+
except ValueError as e:
|
|
3281
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
3282
|
+
headers = {"X-DW-Duration": str(info["of_seconds"])}
|
|
3283
|
+
if info["excerpt"]:
|
|
3284
|
+
headers["X-DW-Excerpt-Start"] = str(info["start"])
|
|
3285
|
+
headers["X-DW-Excerpt-Duration"] = str(info["duration_seconds"])
|
|
3286
|
+
return Response(content=data, media_type="audio/wav", headers=headers)
|
|
3287
|
+
|
|
3288
|
+
FRAME_MIN_DIMENSION = 64
|
|
3289
|
+
# The most moments one `at` may name: each is a seek, a decode and a
|
|
3290
|
+
# PNG encode in the server process, and a contact sheet is the shape
|
|
3291
|
+
# for seeing more of a clip at once
|
|
3292
|
+
MAX_FRAME_MOMENTS = 32
|
|
3293
|
+
|
|
3294
|
+
@app.get("/api/gallery/{name:path}/frames")
|
|
3295
|
+
def gallery_frames(
|
|
3296
|
+
name: str,
|
|
3297
|
+
at: Optional[str] = None,
|
|
3298
|
+
count: Optional[int] = None,
|
|
3299
|
+
seams: Optional[str] = None,
|
|
3300
|
+
boundaries: Optional[str] = None,
|
|
3301
|
+
names: Optional[str] = None,
|
|
3302
|
+
max_dimension: int = 512,
|
|
3303
|
+
crop: Optional[str] = None,
|
|
3304
|
+
ws: Workspace = Depends(selected_workspace),
|
|
3305
|
+
):
|
|
3306
|
+
"""Frames of a video output or asset, as PNG tiles - the way an
|
|
3307
|
+
agent with no video content type sees what a run made (#193).
|
|
3308
|
+
Exactly one selector: `at` (a comma list of seconds or "frame:N"),
|
|
3309
|
+
`count` (an evenly spaced contact sheet, `frame_grid` without a
|
|
3310
|
+
workflow), or `seams` ("true", or a comma list of 1-based seam
|
|
3311
|
+
numbers) for the last frame before and first frame after each
|
|
3312
|
+
boundary, side by side. `boundaries` is the comma list of frame
|
|
3313
|
+
indexes each shot after the first starts at, and `names` the
|
|
3314
|
+
shots' names. Without `boundaries`, an output's seams are the shots
|
|
3315
|
+
its run's manifest recorded for it (a `concat_videos`,
|
|
3316
|
+
`dissolve_videos` or chained step), named as recorded unless `names`
|
|
3317
|
+
is given; a linked asset (`keep_output(shared=true)`) uses the same
|
|
3318
|
+
shots `get_gallery_metadata`'s `media.shots` reports for it, from the
|
|
3319
|
+
sidecar manifest kept beside it. A file with none recorded still
|
|
3320
|
+
needs `boundaries`.
|
|
3321
|
+
Tiles are downscaled to `max_dimension` on their longest side.
|
|
3322
|
+
`crop` is `x,y,width,height` in the video's own source pixels
|
|
3323
|
+
(`video_shape`'s `width`/`height`) - resolved once and cut from
|
|
3324
|
+
every sampled frame before any stamping, fitting or composing, so
|
|
3325
|
+
it names the same region whatever `max_dimension` downscales the
|
|
3326
|
+
result to."""
|
|
3327
|
+
name = _strip_output_prefix(name)
|
|
3328
|
+
if is_asset_reference(name):
|
|
3329
|
+
path = _asset_file(name, ws)
|
|
3330
|
+
else:
|
|
3331
|
+
path = _output_file(name, ws.outputs)
|
|
3332
|
+
if MEDIA_KINDS.get(os.path.splitext(path)[1].lower()) != "video":
|
|
3333
|
+
raise HTTPException(status_code=404, detail=f"{name} is not a video")
|
|
3334
|
+
|
|
3335
|
+
chosen = [
|
|
3336
|
+
key
|
|
3337
|
+
for key, value in (("at", at), ("count", count), ("seams", seams))
|
|
3338
|
+
if value
|
|
3339
|
+
]
|
|
3340
|
+
if len(chosen) != 1:
|
|
3341
|
+
raise HTTPException(
|
|
3342
|
+
status_code=400,
|
|
3343
|
+
detail="Pass exactly one of `at`, `count` or `seams`"
|
|
3344
|
+
+ (f" - got {', '.join(chosen)}" if chosen else ""),
|
|
3345
|
+
)
|
|
3346
|
+
# A floor on each *sub-tile* of a composite (contact sheet / seam
|
|
3347
|
+
# pair) - a caller asking for a small max_dimension still gets a
|
|
3348
|
+
# legible grid, which is then fit to max_dimension as a whole below.
|
|
3349
|
+
sub_tile_width = max(FRAME_MIN_DIMENSION, int(max_dimension))
|
|
3350
|
+
limit = max(1, int(max_dimension))
|
|
3351
|
+
|
|
3352
|
+
try:
|
|
3353
|
+
# Computed once and threaded through every selector below: each
|
|
3354
|
+
# of frames_at/contact_sheet/seam_tiles would otherwise call
|
|
3355
|
+
# video_shape itself, opening the container (and, lacking a
|
|
3356
|
+
# header frame count, decoding it whole to count) a second time
|
|
3357
|
+
# just to answer the same frame_count/fps/width/height (#193).
|
|
3358
|
+
shape = video_shape(path)
|
|
3359
|
+
crop_box = (
|
|
3360
|
+
resolve_crop_box(
|
|
3361
|
+
[c.strip() for c in crop.split(",")],
|
|
3362
|
+
shape["width"],
|
|
3363
|
+
shape["height"],
|
|
3364
|
+
)
|
|
3365
|
+
if crop
|
|
3366
|
+
else None
|
|
3367
|
+
)
|
|
3368
|
+
if at:
|
|
3369
|
+
moments = [
|
|
3370
|
+
m.strip() if m.strip().startswith("frame:") else float(m)
|
|
3371
|
+
for m in at.split(",")
|
|
3372
|
+
if m.strip()
|
|
3373
|
+
]
|
|
3374
|
+
if len(moments) > MAX_FRAME_MOMENTS:
|
|
3375
|
+
raise HTTPException(
|
|
3376
|
+
status_code=400,
|
|
3377
|
+
detail=f"`at` names {len(moments)} moments; the most is "
|
|
3378
|
+
f"{MAX_FRAME_MOMENTS} - ask for a contact sheet (`count`) "
|
|
3379
|
+
"to see more of the clip at once",
|
|
3380
|
+
)
|
|
3381
|
+
tiles = frames_at(path, moments, shape=shape, crop_box=crop_box)
|
|
3382
|
+
elif count:
|
|
3383
|
+
tiles = [
|
|
3384
|
+
contact_sheet(
|
|
3385
|
+
path,
|
|
3386
|
+
count,
|
|
3387
|
+
tile_width=sub_tile_width,
|
|
3388
|
+
shape=shape,
|
|
3389
|
+
crop_box=crop_box,
|
|
3390
|
+
)
|
|
3391
|
+
]
|
|
3392
|
+
else:
|
|
3393
|
+
recorded = (
|
|
3394
|
+
None
|
|
3395
|
+
if boundaries
|
|
3396
|
+
else shots_beside(path)
|
|
3397
|
+
if is_asset_reference(name)
|
|
3398
|
+
else recorded_shots(ws.outputs, name)
|
|
3399
|
+
)
|
|
3400
|
+
if recorded:
|
|
3401
|
+
# The file's own seams, from its run's manifest (or, for
|
|
3402
|
+
# a linked asset, the sidecar `record_kept_shots` wrote
|
|
3403
|
+
# beside it)
|
|
3404
|
+
starts = [shot["start_frame"] for shot in recorded[1:]]
|
|
3405
|
+
shot_names = (
|
|
3406
|
+
[n.strip() for n in names.split(",")]
|
|
3407
|
+
if names
|
|
3408
|
+
else [shot["name"] for shot in recorded]
|
|
3409
|
+
)
|
|
3410
|
+
elif not boundaries:
|
|
3411
|
+
raise HTTPException(
|
|
3412
|
+
status_code=400,
|
|
3413
|
+
detail="`seams` needs `boundaries`: the frame index each "
|
|
3414
|
+
"shot after the first starts at - this file's run "
|
|
3415
|
+
"recorded no shots for it",
|
|
3416
|
+
)
|
|
3417
|
+
else:
|
|
3418
|
+
starts = [int(b) for b in boundaries.split(",") if b.strip()]
|
|
3419
|
+
shot_names = (
|
|
3420
|
+
[n.strip() for n in names.split(",")] if names else None
|
|
3421
|
+
)
|
|
3422
|
+
wanted = (
|
|
3423
|
+
None
|
|
3424
|
+
if seams.lower() == "true"
|
|
3425
|
+
else {int(s) for s in seams.split(",") if s.strip()}
|
|
3426
|
+
)
|
|
3427
|
+
if wanted is not None and not wanted:
|
|
3428
|
+
raise HTTPException(
|
|
3429
|
+
status_code=400,
|
|
3430
|
+
detail="`seams` names no seam - pass `true` for every seam, "
|
|
3431
|
+
"or seam numbers from 1",
|
|
3432
|
+
)
|
|
3433
|
+
tiles = seam_tiles(
|
|
3434
|
+
path,
|
|
3435
|
+
starts,
|
|
3436
|
+
names=shot_names,
|
|
3437
|
+
tile_width=sub_tile_width,
|
|
3438
|
+
shape=shape,
|
|
3439
|
+
wanted=wanted,
|
|
3440
|
+
crop_box=crop_box,
|
|
3441
|
+
)
|
|
3442
|
+
except ValueError as e:
|
|
3443
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
3444
|
+
|
|
3445
|
+
return {
|
|
3446
|
+
"name": name,
|
|
3447
|
+
**shape,
|
|
3448
|
+
"tiles": [_encoded_tile(tile, limit) for tile in tiles],
|
|
3449
|
+
"crop": (
|
|
3450
|
+
[
|
|
3451
|
+
crop_box[0],
|
|
3452
|
+
crop_box[1],
|
|
3453
|
+
crop_box[2] - crop_box[0],
|
|
3454
|
+
crop_box[3] - crop_box[1],
|
|
3455
|
+
]
|
|
3456
|
+
if crop_box
|
|
3457
|
+
else None
|
|
3458
|
+
),
|
|
3459
|
+
}
|
|
3460
|
+
|
|
3461
|
+
def _encoded_tile(tile, limit):
|
|
3462
|
+
image = tile["image"]
|
|
3463
|
+
longest = max(image.width, image.height)
|
|
3464
|
+
if longest > limit:
|
|
3465
|
+
scale = limit / longest
|
|
3466
|
+
image = image.resize(
|
|
3467
|
+
(
|
|
3468
|
+
max(1, round(image.width * scale)),
|
|
3469
|
+
max(1, round(image.height * scale)),
|
|
3470
|
+
)
|
|
3471
|
+
)
|
|
3472
|
+
buffer = io.BytesIO()
|
|
3473
|
+
image.save(buffer, format="PNG")
|
|
3474
|
+
encoded = {key: value for key, value in tile.items() if key != "image"}
|
|
3475
|
+
encoded.update(
|
|
3476
|
+
{
|
|
3477
|
+
"data": base64.b64encode(buffer.getvalue()).decode("ascii"),
|
|
3478
|
+
"mime_type": "image/png",
|
|
3479
|
+
"width": image.width,
|
|
3480
|
+
"height": image.height,
|
|
3481
|
+
}
|
|
3482
|
+
)
|
|
3483
|
+
return encoded
|
|
3484
|
+
|
|
3485
|
+
@app.get("/api/gallery/{name:path}/thumbnail")
|
|
3486
|
+
@query_token_ok
|
|
3487
|
+
def gallery_thumbnail(
|
|
3488
|
+
name: str, request: Request, ws: Workspace = Depends(selected_workspace)
|
|
3489
|
+
):
|
|
3490
|
+
"""A small JPEG rendition of an image output, for the grid - the
|
|
3491
|
+
full-resolution file is only fetched for the detail/lightbox view.
|
|
3492
|
+
Generated on demand rather than cached to disk, so it never grows
|
|
3493
|
+
the output directory the gallery itself scans."""
|
|
3494
|
+
path = _output_file(name, ws.outputs)
|
|
3495
|
+
extension = os.path.splitext(path)[1].lower()
|
|
3496
|
+
if MEDIA_KINDS.get(extension) != "image":
|
|
3497
|
+
raise HTTPException(
|
|
3498
|
+
status_code=404, detail="Thumbnails are only generated for images"
|
|
3499
|
+
)
|
|
3500
|
+
# The file's mtime and size are the validator: the grid re-requests
|
|
3501
|
+
# every visible thumbnail on each visit, and a 304 skips the
|
|
3502
|
+
# decode/resize/encode; a rerun that overwrites the file changes it
|
|
3503
|
+
stat = os.stat(path)
|
|
3504
|
+
etag = f'"{stat.st_mtime_ns:x}-{stat.st_size:x}"'
|
|
3505
|
+
cache_headers = {"ETag": etag, "Cache-Control": "private, no-cache"}
|
|
3506
|
+
if request.headers.get("if-none-match") == etag:
|
|
3507
|
+
return Response(status_code=304, headers=cache_headers)
|
|
3508
|
+
try:
|
|
3509
|
+
from PIL import Image
|
|
3510
|
+
|
|
3511
|
+
with Image.open(path) as image:
|
|
3512
|
+
if image.width * image.height > MAX_DECODE_PIXELS:
|
|
3513
|
+
raise HTTPException(
|
|
3514
|
+
status_code=413,
|
|
3515
|
+
detail=f"{name} is {image.width}x{image.height}, more "
|
|
3516
|
+
f"than the {MAX_DECODE_PIXELS:,} pixels a thumbnail "
|
|
3517
|
+
"is decoded from",
|
|
3518
|
+
)
|
|
3519
|
+
# shrink first (JPEGs decode at reduced size via draft), then
|
|
3520
|
+
# convert - converting a full-resolution image only to
|
|
3521
|
+
# discard most of it is the expensive order
|
|
3522
|
+
image.draft(
|
|
3523
|
+
"RGB", (GALLERY_THUMBNAIL_MAX_DIM, GALLERY_THUMBNAIL_MAX_DIM)
|
|
3524
|
+
)
|
|
3525
|
+
image.thumbnail((GALLERY_THUMBNAIL_MAX_DIM, GALLERY_THUMBNAIL_MAX_DIM))
|
|
3526
|
+
image = image.convert("RGB")
|
|
3527
|
+
buffer = io.BytesIO()
|
|
3528
|
+
image.save(buffer, format="JPEG", quality=80)
|
|
3529
|
+
except Image.DecompressionBombError as e:
|
|
3530
|
+
# Pillow's own refusal, on open, of a header past twice its limit
|
|
3531
|
+
raise HTTPException(status_code=413, detail=str(e))
|
|
3532
|
+
except (OSError, ValueError) as e:
|
|
3533
|
+
# what PIL raises for an unreadable or corrupt file
|
|
3534
|
+
raise HTTPException(
|
|
3535
|
+
status_code=500, detail=f"Could not generate thumbnail: {e}"
|
|
3536
|
+
)
|
|
3537
|
+
return Response(
|
|
3538
|
+
content=buffer.getvalue(), media_type="image/jpeg", headers=cache_headers
|
|
3539
|
+
)
|
|
3540
|
+
|
|
3541
|
+
@app.get("/api/gallery/{name:path}/download")
|
|
3542
|
+
@query_token_ok
|
|
3543
|
+
def download_output(name: str, ws: Workspace = Depends(selected_workspace)):
|
|
3544
|
+
"""Serve one output file as a forced download rather than an inline view."""
|
|
3545
|
+
path = _output_file(name, ws.outputs)
|
|
3546
|
+
return FileResponse(path, filename=os.path.basename(name))
|
|
3547
|
+
|
|
3548
|
+
# A generous ceiling rather than a real limit - it exists so a
|
|
3549
|
+
# malformed client cannot ask the server to zip the whole directory
|
|
3550
|
+
MAX_ARCHIVE_FILES = 1000
|
|
3551
|
+
|
|
3552
|
+
class ArchiveRequest(BaseModel):
|
|
3553
|
+
names: list[str] = Field(min_length=1, max_length=MAX_ARCHIVE_FILES)
|
|
3554
|
+
|
|
3555
|
+
def _zip_download(entries, filename):
|
|
3556
|
+
"""Bundle (arcname, path) pairs into a zip and serve it as a download.
|
|
3557
|
+
|
|
3558
|
+
The archive is a temp file rather than memory - a selection of videos
|
|
3559
|
+
does not fit in RAM - unlinked once the response has been sent. The
|
|
3560
|
+
three routes that hand back a zip share this so the cleanup contract
|
|
3561
|
+
lives in one place: nothing has attached the background unlink while
|
|
3562
|
+
the archive is being written, so a failure there has to unlink on the
|
|
3563
|
+
way out or leak a half-written file into tmp.
|
|
3564
|
+
"""
|
|
3565
|
+
handle = tempfile.NamedTemporaryFile(suffix=".zip", delete=False)
|
|
3566
|
+
try:
|
|
3567
|
+
with handle:
|
|
3568
|
+
with zipfile.ZipFile(handle, "w", zipfile.ZIP_DEFLATED) as archive:
|
|
3569
|
+
for arcname, path in entries:
|
|
3570
|
+
# ZipFile.write follows a symlink and archives the
|
|
3571
|
+
# target's bytes; nothing the server writes is one
|
|
3572
|
+
if os.path.islink(path):
|
|
3573
|
+
continue
|
|
3574
|
+
extension = os.path.splitext(path)[1].lower()
|
|
3575
|
+
# A file in MEDIA_KINDS but not RAW_MEDIA_EXTENSIONS
|
|
3576
|
+
# is an already-compressed container - deflating it
|
|
3577
|
+
# buys about nothing for a full CPU pass the caller
|
|
3578
|
+
# waits through (the response doesn't start until the
|
|
3579
|
+
# temp file is complete), so it is stored instead.
|
|
3580
|
+
# Everything else - .json, .md, .txt, .bmp, .wav, an
|
|
3581
|
+
# unrecognized extension - deflates, including the
|
|
3582
|
+
# export zip's text files. ".txt" is in MEDIA_KINDS
|
|
3583
|
+
# (kind "text", #238) but is plain text, not an
|
|
3584
|
+
# already-compressed container, so it stays out of
|
|
3585
|
+
# this policy the same way .json and .md do
|
|
3586
|
+
kind = MEDIA_KINDS.get(extension)
|
|
3587
|
+
stored = (
|
|
3588
|
+
kind is not None
|
|
3589
|
+
and kind != "text"
|
|
3590
|
+
and extension not in RAW_MEDIA_EXTENSIONS
|
|
3591
|
+
)
|
|
3592
|
+
archive.write(
|
|
3593
|
+
path,
|
|
3594
|
+
arcname=arcname,
|
|
3595
|
+
compress_type=(
|
|
3596
|
+
zipfile.ZIP_STORED if stored else zipfile.ZIP_DEFLATED
|
|
3597
|
+
),
|
|
3598
|
+
)
|
|
3599
|
+
except BaseException:
|
|
3600
|
+
os.unlink(handle.name)
|
|
3601
|
+
raise
|
|
3602
|
+
|
|
3603
|
+
return FileResponse(
|
|
3604
|
+
handle.name,
|
|
3605
|
+
media_type="application/zip",
|
|
3606
|
+
filename=filename,
|
|
3607
|
+
background=BackgroundTask(os.unlink, handle.name),
|
|
3608
|
+
)
|
|
3609
|
+
|
|
3610
|
+
def _archive_selection(entries, kind):
|
|
3611
|
+
"""`_zip_download` plus the one tail the two archive routes shared:
|
|
3612
|
+
a timestamped `dw-<kind>s-*.zip` name and a log line naming the
|
|
3613
|
+
count. Logged after the archive is written, not before, so a write
|
|
3614
|
+
that fails partway (a bad path slipping past resolution, a full
|
|
3615
|
+
disk) doesn't log a success that didn't happen.
|
|
3616
|
+
"""
|
|
3617
|
+
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
3618
|
+
response = _zip_download(entries, f"dw-{kind}s-{stamp}.zip")
|
|
3619
|
+
logger.info(f"Archived {len(entries)} {kind} files")
|
|
3620
|
+
return response
|
|
3621
|
+
|
|
3622
|
+
@app.post("/api/gallery/archive")
|
|
3623
|
+
def archive_outputs(
|
|
3624
|
+
request: ArchiveRequest, ws: Workspace = Depends(selected_workspace)
|
|
3625
|
+
):
|
|
3626
|
+
"""Bundle a multi-file gallery selection into one zip. A browser
|
|
3627
|
+
cannot zip on its own and throttles a burst of single downloads, so
|
|
3628
|
+
the whole selection has to arrive as one file. Written to a temp
|
|
3629
|
+
file rather than memory - a selection of videos does not fit in
|
|
3630
|
+
RAM - and unlinked once the response has been sent."""
|
|
3631
|
+
# Resolved before anything is written, so a bad name in the
|
|
3632
|
+
# selection fails the request instead of yielding a partial zip
|
|
3633
|
+
# the gallery-relative name is the entry name, so a workflow's output
|
|
3634
|
+
# subfolders stay intact inside the download
|
|
3635
|
+
names = [_strip_output_prefix(name) for name in request.names]
|
|
3636
|
+
paths = [(name, _output_file(name, ws.outputs)) for name in names]
|
|
3637
|
+
|
|
3638
|
+
return _archive_selection(paths, "output")
|
|
3639
|
+
|
|
3640
|
+
# What a run directory holds besides its media: the engine writes them to
|
|
3641
|
+
# describe the run, and the gallery - which lists media - never shows them
|
|
3642
|
+
RUN_SIDECARS = (MANIFEST_FILE_NAME, REALIZED_FILE_NAME)
|
|
3643
|
+
|
|
3644
|
+
def _prune_empty_run_directory(name, root):
|
|
3645
|
+
"""Drop the run directory a just-deleted output belonged to, once no
|
|
3646
|
+
media is left in it.
|
|
3647
|
+
|
|
3648
|
+
A run writes `manifest.json` and `workflow.json` beside its files, and
|
|
3649
|
+
nothing in the gallery addresses either one. Deleting every output of a
|
|
3650
|
+
run therefore used to leave the directory behind forever: a consumer
|
|
3651
|
+
that removed everything it made still could not put a workspace back
|
|
3652
|
+
the way it found it, and nothing it could call would even show the
|
|
3653
|
+
residue (#134). Tying the sidecars' lifetime to the outputs they
|
|
3654
|
+
describe is what makes "delete what you made" true.
|
|
3655
|
+
|
|
3656
|
+
Only the sidecars may remain - any other leftover file means something
|
|
3657
|
+
is still there to describe, and the directory stays.
|
|
3658
|
+
|
|
3659
|
+
Returns:
|
|
3660
|
+
The run id swept, or None if nothing was
|
|
3661
|
+
"""
|
|
3662
|
+
identity, run_id, _ = split_run_path(name)
|
|
3663
|
+
if not run_id:
|
|
3664
|
+
# The flat layout writes no run directory and no sidecars
|
|
3665
|
+
return None
|
|
3666
|
+
relative = f"{identity}/{run_id}" if identity else run_id
|
|
3667
|
+
try:
|
|
3668
|
+
run_dir = validate_path(
|
|
3669
|
+
os.path.join(root, relative), root, allow_create=False
|
|
3670
|
+
)
|
|
3671
|
+
except SecurityError:
|
|
3672
|
+
return None
|
|
3673
|
+
if not os.path.isdir(run_dir):
|
|
3674
|
+
return None
|
|
3675
|
+
|
|
3676
|
+
for directory, _subdirectories, files in os.walk(run_dir):
|
|
3677
|
+
for file_name in files:
|
|
3678
|
+
if directory == run_dir and file_name in RUN_SIDECARS:
|
|
3679
|
+
continue
|
|
3680
|
+
return None
|
|
3681
|
+
|
|
3682
|
+
# Pin the siblings' numbers first: a run that predates versions is
|
|
3683
|
+
# ranked, and removing one ahead of it would renumber it
|
|
3684
|
+
record_run_versions(os.path.dirname(run_dir))
|
|
3685
|
+
shutil.rmtree(run_dir, ignore_errors=True)
|
|
3686
|
+
# And the identity folders above it, while they are empty - a swept
|
|
3687
|
+
# workspace should not keep one directory per workflow it once ran
|
|
3688
|
+
parent = os.path.dirname(run_dir)
|
|
3689
|
+
while os.path.normpath(parent) != os.path.normpath(root):
|
|
3690
|
+
try:
|
|
3691
|
+
os.rmdir(parent)
|
|
3692
|
+
except OSError:
|
|
3693
|
+
break
|
|
3694
|
+
parent = os.path.dirname(parent)
|
|
3695
|
+
logger.info(f"Swept empty run directory {relative}")
|
|
3696
|
+
return run_id
|
|
3697
|
+
|
|
3698
|
+
def _run_directory(name, root):
|
|
3699
|
+
"""The run directory `<identity>/<run id>` names, or None.
|
|
3700
|
+
|
|
3701
|
+
A run that failed before it wrote anything still has a directory and a
|
|
3702
|
+
manifest, and no gallery name addresses it - so the name of the
|
|
3703
|
+
directory itself is the only handle there can be (#134).
|
|
3704
|
+
"""
|
|
3705
|
+
parts = [part for part in (name or "").split("/") if part]
|
|
3706
|
+
if not parts or not is_run_id(parts[-1]):
|
|
3707
|
+
return None
|
|
3708
|
+
try:
|
|
3709
|
+
path = validate_path(
|
|
3710
|
+
os.path.join(root, "/".join(parts)), root, allow_create=False
|
|
3711
|
+
)
|
|
3712
|
+
except SecurityError:
|
|
3713
|
+
return None
|
|
3714
|
+
return path if os.path.isdir(path) else None
|
|
3715
|
+
|
|
3716
|
+
@app.delete("/api/gallery/{name:path}")
|
|
3717
|
+
def delete_output(name: str, ws: Workspace = Depends(selected_workspace)):
|
|
3718
|
+
"""Remove one file from the output directory.
|
|
3719
|
+
|
|
3720
|
+
When that was the last media file of its run, the run directory goes
|
|
3721
|
+
with it, sidecars included. `name` may also be a run directory
|
|
3722
|
+
(`<identity>/<run id>`), which removes the whole run - the only handle
|
|
3723
|
+
on a run that failed before it wrote any media (#134).
|
|
3724
|
+
"""
|
|
3725
|
+
name = _strip_output_prefix(name)
|
|
3726
|
+
run_dir = _run_directory(name, ws.outputs)
|
|
3727
|
+
if run_dir is not None:
|
|
3728
|
+
# As in _prune_empty_run_directory: pin the siblings' numbers
|
|
3729
|
+
# before one of them goes
|
|
3730
|
+
record_run_versions(os.path.dirname(run_dir))
|
|
3731
|
+
shutil.rmtree(run_dir, ignore_errors=True)
|
|
3732
|
+
parent = os.path.dirname(run_dir)
|
|
3733
|
+
while os.path.normpath(parent) != os.path.normpath(ws.outputs):
|
|
3734
|
+
try:
|
|
3735
|
+
os.rmdir(parent)
|
|
3736
|
+
except OSError:
|
|
3737
|
+
break
|
|
3738
|
+
parent = os.path.dirname(parent)
|
|
3739
|
+
logger.info(f"Deleted run directory {name}")
|
|
3740
|
+
forget_workspace_usage()
|
|
3741
|
+
return {
|
|
3742
|
+
"name": name,
|
|
3743
|
+
"deleted": True,
|
|
3744
|
+
"run_swept": os.path.basename(run_dir),
|
|
3745
|
+
}
|
|
3746
|
+
|
|
3747
|
+
path = _output_file(name, ws.outputs)
|
|
3748
|
+
os.remove(path)
|
|
3749
|
+
logger.info(f"Deleted output file {name}")
|
|
3750
|
+
swept = _prune_empty_run_directory(name, ws.outputs)
|
|
3751
|
+
forget_workspace_usage()
|
|
3752
|
+
return {"name": name, "deleted": True, "run_swept": swept}
|
|
3753
|
+
|
|
3754
|
+
# ---------------------------------------------------------------- uploads
|
|
3755
|
+
|
|
3756
|
+
UPLOADS_SUBDIR = "uploads"
|
|
3757
|
+
# Audio included: the asset library holds it and workflows read it (an
|
|
3758
|
+
# H3 audio reference is built from a .wav), so refusing it here would
|
|
3759
|
+
# leave one input kind with no way onto the machine
|
|
3760
|
+
ALLOWED_UPLOAD_EXTENSIONS = (
|
|
3761
|
+
ALLOWED_IMAGE_EXTENSIONS | ALLOWED_VIDEO_EXTENSIONS | ALLOWED_AUDIO_EXTENSIONS
|
|
3762
|
+
)
|
|
3763
|
+
MAX_UPLOAD_BYTES = 200 * 1024 * 1024 # 200MB - covers a short video clip
|
|
3764
|
+
|
|
3765
|
+
@app.post("/api/uploads", status_code=201)
|
|
3766
|
+
async def upload_media(
|
|
3767
|
+
request: Request,
|
|
3768
|
+
filename: str,
|
|
3769
|
+
asset_name: Optional[str] = None,
|
|
3770
|
+
shared: bool = False,
|
|
3771
|
+
ws: Workspace = Depends(selected_workspace),
|
|
3772
|
+
):
|
|
3773
|
+
"""Save a browser-picked image, video or audio file into the asset library's
|
|
3774
|
+
uploads/ subfolder and hand back the reference a workflow argument
|
|
3775
|
+
can carry.
|
|
3776
|
+
|
|
3777
|
+
An upload is input, so it belongs in the asset library rather than
|
|
3778
|
+
among generated output, and the reference handed back is
|
|
3779
|
+
'asset:uploads/<name>' - portable, and meaningful in a workflow that
|
|
3780
|
+
is saved and rerun later. A server with no asset library configured
|
|
3781
|
+
keeps the old behavior, writing to the output directory's uploads/
|
|
3782
|
+
and returning an absolute path. The body is the raw file bytes: no
|
|
3783
|
+
multipart parser dependency needed for a single-file upload.
|
|
3784
|
+
|
|
3785
|
+
`asset_name` stores it under a name of the caller's choosing -
|
|
3786
|
+
'cast/priya-voice.wav' rather than the random one a browser upload
|
|
3787
|
+
gets - which is what makes a recurring cast's references readable
|
|
3788
|
+
in every workflow that carries them. It may name a folder, is
|
|
3789
|
+
confined to the library the way `keep_output`'s is, and takes the
|
|
3790
|
+
uploaded file's extension when it has none of its own. Without it
|
|
3791
|
+
the name stays random, so two uploads of the same file never
|
|
3792
|
+
collide.
|
|
3793
|
+
|
|
3794
|
+
`shared` puts it in the library every workspace under this root
|
|
3795
|
+
shares rather than in this workspace's own - a recurring cast that
|
|
3796
|
+
episode four, in a workspace of its own, still has to reach.
|
|
3797
|
+
"""
|
|
3798
|
+
extension = os.path.splitext(os.path.basename(filename))[1].lower()
|
|
3799
|
+
if extension not in ALLOWED_UPLOAD_EXTENSIONS:
|
|
3800
|
+
raise HTTPException(
|
|
3801
|
+
status_code=400, detail=f"File extension not allowed: {extension}"
|
|
3802
|
+
)
|
|
3803
|
+
|
|
3804
|
+
# Refuse an oversized upload from its declared length, before
|
|
3805
|
+
# reading a single byte of it
|
|
3806
|
+
declared = request.headers.get("content-length")
|
|
3807
|
+
if declared and declared.isdigit() and int(declared) > MAX_UPLOAD_BYTES:
|
|
3808
|
+
raise HTTPException(
|
|
3809
|
+
status_code=413,
|
|
3810
|
+
detail=f"Upload too large: {declared} > {MAX_UPLOAD_BYTES}",
|
|
3811
|
+
)
|
|
3812
|
+
body = await request.body()
|
|
3813
|
+
if not body:
|
|
3814
|
+
raise HTTPException(status_code=400, detail="Empty upload")
|
|
3815
|
+
if len(body) > MAX_UPLOAD_BYTES:
|
|
3816
|
+
raise HTTPException(
|
|
3817
|
+
status_code=413,
|
|
3818
|
+
detail=f"Upload too large: {len(body)} > {MAX_UPLOAD_BYTES}",
|
|
3819
|
+
)
|
|
3820
|
+
|
|
3821
|
+
library = ws.assets or ws.outputs
|
|
3822
|
+
if shared:
|
|
3823
|
+
library = _common_assets(ws)
|
|
3824
|
+
if not library:
|
|
3825
|
+
raise HTTPException(
|
|
3826
|
+
status_code=409,
|
|
3827
|
+
detail="This server has no shared asset library - it was "
|
|
3828
|
+
"configured from loose directories rather than a workspace "
|
|
3829
|
+
"root, so there is nothing for an asset to be common to",
|
|
3830
|
+
)
|
|
3831
|
+
uploads_dir = os.path.join(library, UPLOADS_SUBDIR)
|
|
3832
|
+
name = f"{uuid.uuid4().hex}{extension}"
|
|
3833
|
+
if asset_name:
|
|
3834
|
+
name = asset_name
|
|
3835
|
+
if not os.path.splitext(name)[1]:
|
|
3836
|
+
name = f"{name}{extension}"
|
|
3837
|
+
try:
|
|
3838
|
+
# The same check the keep route makes: a name, possibly with
|
|
3839
|
+
# folders in it, that cannot climb out of the library
|
|
3840
|
+
name = validate_asset_reference(name)
|
|
3841
|
+
except SecurityError as e:
|
|
3842
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
3843
|
+
if os.path.splitext(name)[1].lower() != extension:
|
|
3844
|
+
raise HTTPException(
|
|
3845
|
+
status_code=400,
|
|
3846
|
+
detail=f"asset_name {asset_name!r} does not match the "
|
|
3847
|
+
f"uploaded file's kind ({extension})",
|
|
3848
|
+
)
|
|
3849
|
+
os.makedirs(uploads_dir, exist_ok=True)
|
|
3850
|
+
try:
|
|
3851
|
+
dest = validate_output_path(os.path.join(uploads_dir, name), uploads_dir)
|
|
3852
|
+
except SecurityError as e:
|
|
3853
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
3854
|
+
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
3855
|
+
|
|
3856
|
+
# Off the event loop: a 200 MB write would otherwise stall every SSE
|
|
3857
|
+
# stream and poll for its duration
|
|
3858
|
+
await run_in_threadpool(_write_bytes, dest, body)
|
|
3859
|
+
logger.info(f"Saved upload {filename!r} -> {dest}")
|
|
3860
|
+
if shared or ws.assets:
|
|
3861
|
+
path = f"/inputs/{UPLOADS_SUBDIR}/{quote(name)}"
|
|
3862
|
+
result = {
|
|
3863
|
+
"path": f"asset:{UPLOADS_SUBDIR}/{name}",
|
|
3864
|
+
"url": _served_url(path, ws),
|
|
3865
|
+
"shared": shared,
|
|
3866
|
+
}
|
|
3867
|
+
absolute_url = _absolute_served_url(path, ws)
|
|
3868
|
+
if absolute_url is not None:
|
|
3869
|
+
result["absolute_url"] = absolute_url
|
|
3870
|
+
return result
|
|
3871
|
+
path = f"/outputs/{UPLOADS_SUBDIR}/{quote(name)}"
|
|
3872
|
+
result = {
|
|
3873
|
+
"path": dest,
|
|
3874
|
+
"url": _served_url(path, ws),
|
|
3875
|
+
}
|
|
3876
|
+
absolute_url = _absolute_served_url(path, ws)
|
|
3877
|
+
if absolute_url is not None:
|
|
3878
|
+
result["absolute_url"] = absolute_url
|
|
3879
|
+
return result
|
|
3880
|
+
|
|
3881
|
+
def _asset_origin(ws, root):
|
|
3882
|
+
"""Which library an asset came from: this workspace's own, the one
|
|
3883
|
+
shared by every workspace under the root, or a read-only examples
|
|
3884
|
+
tree. A client that cannot tell them apart cannot say why deleting
|
|
3885
|
+
one answers 403.
|
|
3886
|
+
|
|
3887
|
+
By directory, never by position in the search path: the workspace's
|
|
3888
|
+
own library drops out of `_asset_roots` until it exists, and the
|
|
3889
|
+
examples tree that then sits first is still nobody's to write."""
|
|
3890
|
+
own = ws.assets
|
|
3891
|
+
if own and os.path.abspath(own) == root:
|
|
3892
|
+
return WORKSPACE_ORIGIN
|
|
3893
|
+
common = _common_assets(ws)
|
|
3894
|
+
if common and os.path.abspath(common) == root:
|
|
3895
|
+
return COMMON_ORIGIN
|
|
3896
|
+
return EXAMPLES_ORIGIN
|
|
3897
|
+
|
|
3898
|
+
@app.get("/api/assets")
|
|
3899
|
+
def list_assets(ws: Workspace = Depends(selected_workspace)):
|
|
3900
|
+
"""The asset library: the input media an 'asset:' reference names.
|
|
3901
|
+
|
|
3902
|
+
Reported by reference rather than by path - 'asset:uploads/x.png' is
|
|
3903
|
+
what a workflow argument carries, and a client that only ever sees
|
|
3904
|
+
references cannot accidentally write a path that means something
|
|
3905
|
+
else on another machine. Empty, not an error, on a server with no
|
|
3906
|
+
library configured: nothing is wrong, there is just nowhere for an
|
|
3907
|
+
asset to be.
|
|
3908
|
+
"""
|
|
3909
|
+
library = ws.assets
|
|
3910
|
+
roots = _asset_roots(ws)
|
|
3911
|
+
if not roots:
|
|
3912
|
+
return {
|
|
3913
|
+
"asset_dir": library,
|
|
3914
|
+
"asset_dirs": [],
|
|
3915
|
+
"assets": [],
|
|
3916
|
+
"folders": [],
|
|
3917
|
+
"libraries": [],
|
|
3918
|
+
"shadowed": [],
|
|
3919
|
+
}
|
|
3920
|
+
|
|
3921
|
+
libraries = [
|
|
3922
|
+
{
|
|
3923
|
+
"origin": (origin := _asset_origin(ws, root)),
|
|
3924
|
+
"dir": root,
|
|
3925
|
+
"writable": origin != EXAMPLES_ORIGIN,
|
|
3926
|
+
}
|
|
3927
|
+
for root in roots
|
|
3928
|
+
]
|
|
3929
|
+
|
|
3930
|
+
assets = []
|
|
3931
|
+
shadowed = []
|
|
3932
|
+
# Which origin first claimed a name, so a later root's same name can
|
|
3933
|
+
# be reported as shadowed rather than silently dropped
|
|
3934
|
+
seen = {}
|
|
3935
|
+
for root in roots:
|
|
3936
|
+
try:
|
|
3937
|
+
files = list(_iter_gallery_files(root, group_runs=False))
|
|
3938
|
+
except OSError:
|
|
3939
|
+
files = []
|
|
3940
|
+
origin = _asset_origin(ws, root)
|
|
3941
|
+
for relative, folder, _subfolder, _run_id, kind, path in files:
|
|
3942
|
+
try:
|
|
3943
|
+
stat = os.stat(path)
|
|
3944
|
+
except OSError:
|
|
3945
|
+
continue
|
|
3946
|
+
# A name in the workspace shadows the same name in an
|
|
3947
|
+
# examples library, exactly as 'asset:' resolution does
|
|
3948
|
+
if relative in seen:
|
|
3949
|
+
shadowed.append(
|
|
3950
|
+
{
|
|
3951
|
+
"name": relative,
|
|
3952
|
+
"reference": f"asset:{relative}",
|
|
3953
|
+
"folder": folder,
|
|
3954
|
+
"kind": kind,
|
|
3955
|
+
"size": stat.st_size,
|
|
3956
|
+
"mtime": stat.st_mtime,
|
|
3957
|
+
"origin": origin,
|
|
3958
|
+
"shadowed_by": seen[relative],
|
|
3959
|
+
}
|
|
3960
|
+
)
|
|
3961
|
+
continue
|
|
3962
|
+
seen[relative] = origin
|
|
3963
|
+
asset_path = f"/inputs/{quote(relative)}"
|
|
3964
|
+
asset_entry = {
|
|
3965
|
+
"name": relative,
|
|
3966
|
+
"reference": f"asset:{relative}",
|
|
3967
|
+
"folder": folder,
|
|
3968
|
+
"kind": kind,
|
|
3969
|
+
"size": stat.st_size,
|
|
3970
|
+
"mtime": stat.st_mtime,
|
|
3971
|
+
"origin": origin,
|
|
3972
|
+
# For the editor's own preview - fetchable the same
|
|
3973
|
+
# way an upload's URL is
|
|
3974
|
+
"url": _served_url(asset_path, ws),
|
|
3975
|
+
}
|
|
3976
|
+
absolute_url = _absolute_served_url(asset_path, ws)
|
|
3977
|
+
if absolute_url is not None:
|
|
3978
|
+
asset_entry["absolute_url"] = absolute_url
|
|
3979
|
+
assets.append(asset_entry)
|
|
3980
|
+
assets.sort(key=lambda entry: entry["mtime"], reverse=True)
|
|
3981
|
+
return {
|
|
3982
|
+
# The workspace's own library, unchanged: where an upload lands
|
|
3983
|
+
"asset_dir": library,
|
|
3984
|
+
"asset_dirs": [lib["dir"] for lib in libraries],
|
|
3985
|
+
"assets": assets,
|
|
3986
|
+
"folders": sorted({entry["folder"] for entry in assets} | {""}),
|
|
3987
|
+
"libraries": libraries,
|
|
3988
|
+
"shadowed": shadowed,
|
|
3989
|
+
}
|
|
3990
|
+
|
|
3991
|
+
class KeepRequest(BaseModel):
|
|
3992
|
+
name: str = Field(
|
|
3993
|
+
description="The generated file to keep, as the gallery names it"
|
|
3994
|
+
)
|
|
3995
|
+
asset_name: Optional[str] = Field(
|
|
3996
|
+
default=None,
|
|
3997
|
+
description="Name to keep it under in the asset library; its own "
|
|
3998
|
+
"file name when omitted. May name a folder; the kept file's "
|
|
3999
|
+
"extension is assumed when the name has none",
|
|
4000
|
+
)
|
|
4001
|
+
overwrite: bool = Field(
|
|
4002
|
+
default=False, description="Replace an asset already under that name"
|
|
4003
|
+
)
|
|
4004
|
+
shared: bool = Field(
|
|
4005
|
+
default=False,
|
|
4006
|
+
description="Keep it in the library every workspace under this "
|
|
4007
|
+
"root shares, rather than in this workspace's own",
|
|
4008
|
+
)
|
|
4009
|
+
|
|
4010
|
+
@app.post("/api/assets/keep", status_code=201)
|
|
4011
|
+
def keep_output_as_asset(
|
|
4012
|
+
request: KeepRequest, ws: Workspace = Depends(selected_workspace)
|
|
4013
|
+
):
|
|
4014
|
+
"""Keep a generated file as an input asset, under a stable name.
|
|
4015
|
+
|
|
4016
|
+
A run's files live under '<workflow>/<run id>/', which is the right
|
|
4017
|
+
place for them and the wrong name to build on: 'latest' moves, and a
|
|
4018
|
+
pinned run id breaks the moment outputs are pruned. Keeping one
|
|
4019
|
+
copies it into the workspace's asset library, where an 'asset:' name
|
|
4020
|
+
stays put - which is what turns a generated still or score into an
|
|
4021
|
+
input later workflows can rely on.
|
|
4022
|
+
|
|
4023
|
+
Within the workspace, so nothing crosses a namespace, and no bytes
|
|
4024
|
+
cross the network: a client that had to download and re-upload a
|
|
4025
|
+
multi-gigabyte video to reuse one frame would be paying for the
|
|
4026
|
+
round trip twice.
|
|
4027
|
+
"""
|
|
4028
|
+
library = _common_assets(ws) if request.shared else ws.assets
|
|
4029
|
+
if not library:
|
|
4030
|
+
raise HTTPException(
|
|
4031
|
+
status_code=409,
|
|
4032
|
+
detail=(
|
|
4033
|
+
"This server has no shared asset library"
|
|
4034
|
+
if request.shared
|
|
4035
|
+
else "This workspace has no asset library"
|
|
4036
|
+
),
|
|
4037
|
+
)
|
|
4038
|
+
|
|
4039
|
+
kept_name = _strip_output_prefix(request.name)
|
|
4040
|
+
source = _output_file(kept_name, ws.outputs)
|
|
4041
|
+
asset_name = request.asset_name or os.path.basename(kept_name)
|
|
4042
|
+
# The kept file's own extension when the name carries none, and a
|
|
4043
|
+
# refusal when it carries a contradicting one - exactly what the
|
|
4044
|
+
# upload route does with its `asset_name`. Without this a kept asset
|
|
4045
|
+
# could be written under an extensionless name, which the library
|
|
4046
|
+
# listing (which reads by kind) never shows again: the call reported
|
|
4047
|
+
# success and the asset was invisible (T014)
|
|
4048
|
+
extension = os.path.splitext(os.path.basename(kept_name))[1].lower()
|
|
4049
|
+
if not os.path.splitext(asset_name)[1]:
|
|
4050
|
+
asset_name = f"{asset_name}{extension}"
|
|
4051
|
+
elif os.path.splitext(asset_name)[1].lower() != extension:
|
|
4052
|
+
raise HTTPException(
|
|
4053
|
+
status_code=400,
|
|
4054
|
+
detail=f"asset_name {request.asset_name!r} does not match the "
|
|
4055
|
+
f"kept file's kind ({extension or 'no extension'})",
|
|
4056
|
+
)
|
|
4057
|
+
try:
|
|
4058
|
+
asset_name = validate_asset_reference(asset_name)
|
|
4059
|
+
destination = validate_path(os.path.join(library, asset_name), library)
|
|
4060
|
+
except SecurityError as e:
|
|
4061
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
4062
|
+
|
|
4063
|
+
if os.path.exists(destination) and not request.overwrite:
|
|
4064
|
+
raise HTTPException(
|
|
4065
|
+
status_code=409,
|
|
4066
|
+
detail=f"asset:{asset_name} already exists - pass overwrite=true "
|
|
4067
|
+
f"to replace it",
|
|
4068
|
+
)
|
|
4069
|
+
|
|
4070
|
+
os.makedirs(os.path.dirname(destination), exist_ok=True)
|
|
4071
|
+
if os.path.exists(destination):
|
|
4072
|
+
os.remove(destination)
|
|
4073
|
+
# A hard link first: keeping one frame of a multi-gigabyte render
|
|
4074
|
+
# should not cost another copy of it, and both names refer to the
|
|
4075
|
+
# same content anyway. Falls back to a copy when the link cannot be
|
|
4076
|
+
# made - a different filesystem, or one that has no links
|
|
4077
|
+
try:
|
|
4078
|
+
os.link(source, destination)
|
|
4079
|
+
linked = True
|
|
4080
|
+
except OSError:
|
|
4081
|
+
shutil.copy2(source, destination)
|
|
4082
|
+
linked = False
|
|
4083
|
+
|
|
4084
|
+
# The source run's shot boundaries - carrying bytes without them left
|
|
4085
|
+
# a kept multi-shot cut looking like one shot to every probe, with no
|
|
4086
|
+
# sign anything was missing (#393)
|
|
4087
|
+
record_kept_shots(
|
|
4088
|
+
os.path.dirname(destination),
|
|
4089
|
+
os.path.basename(destination),
|
|
4090
|
+
recorded_shots(ws.outputs, kept_name),
|
|
4091
|
+
)
|
|
4092
|
+
|
|
4093
|
+
logger.info(f"Kept output {request.name} as asset:{asset_name}")
|
|
4094
|
+
return {
|
|
4095
|
+
"reference": f"asset:{asset_name}",
|
|
4096
|
+
"name": asset_name,
|
|
4097
|
+
"path": destination,
|
|
4098
|
+
"linked": linked,
|
|
4099
|
+
"shared": bool(request.shared),
|
|
4100
|
+
}
|
|
4101
|
+
|
|
4102
|
+
@app.post("/api/assets/archive")
|
|
4103
|
+
def archive_assets(
|
|
4104
|
+
request: ArchiveRequest, ws: Workspace = Depends(selected_workspace)
|
|
4105
|
+
):
|
|
4106
|
+
"""Bundle a multi-file asset selection into one zip - the gallery's
|
|
4107
|
+
bulk download, for the input side of it.
|
|
4108
|
+
|
|
4109
|
+
Resolved down the same search path a run resolves 'asset:' in, so a
|
|
4110
|
+
selection spanning the workspace's own library, the shared one and
|
|
4111
|
+
an examples tree downloads as one archive; the library-relative name
|
|
4112
|
+
is the entry name, which is the name the 'asset:' reference carries.
|
|
4113
|
+
"""
|
|
4114
|
+
# The search path depends on the workspace, not on the name, so it is
|
|
4115
|
+
# built once rather than per name - each root's isdir check would
|
|
4116
|
+
# otherwise repeat once per name in the selection for no reason
|
|
4117
|
+
roots = _resolution_roots(ws)
|
|
4118
|
+
# Stripped and deduped before resolving, so "iris.png" and
|
|
4119
|
+
# "iris.png " (or a name repeated by an eager client) become the one
|
|
4120
|
+
# zip entry rather than a collision on write
|
|
4121
|
+
names = list(dict.fromkeys(n.strip() for n in request.names))
|
|
4122
|
+
# Resolved before anything is written, so a bad name in the
|
|
4123
|
+
# selection fails the request instead of yielding a partial zip
|
|
4124
|
+
paths = [(name, _asset_in(name, roots)) for name in names]
|
|
4125
|
+
|
|
4126
|
+
return _archive_selection(paths, "asset")
|
|
4127
|
+
|
|
4128
|
+
@app.delete("/api/assets/{name:path}")
|
|
4129
|
+
def delete_asset(name: str, ws: Workspace = Depends(selected_workspace)):
|
|
4130
|
+
"""Permanently remove one file from the asset library.
|
|
4131
|
+
|
|
4132
|
+
Deletes from whichever library on the search path holds it, the
|
|
4133
|
+
workspace's own first, so the name deleted is the name 'asset:'
|
|
4134
|
+
would have resolved to. An asset a read-only examples tree brought
|
|
4135
|
+
with it is not this server's to delete - the same 403 a read-only
|
|
4136
|
+
prompt or workflow answers with.
|
|
4137
|
+
|
|
4138
|
+
Not recoverable, and any workflow still carrying that 'asset:'
|
|
4139
|
+
reference stops loading. Without this, everything else that writes
|
|
4140
|
+
the library (uploads, keep) had no counterpart and a mistake could
|
|
4141
|
+
only be cleaned up on the box (T014).
|
|
4142
|
+
"""
|
|
4143
|
+
roots = _asset_roots(ws)
|
|
4144
|
+
if not roots:
|
|
4145
|
+
raise HTTPException(
|
|
4146
|
+
status_code=409, detail="This server has no asset library"
|
|
4147
|
+
)
|
|
4148
|
+
try:
|
|
4149
|
+
relative = validate_asset_reference(name)
|
|
4150
|
+
except SecurityError as e:
|
|
4151
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
4152
|
+
|
|
4153
|
+
for root in roots:
|
|
4154
|
+
try:
|
|
4155
|
+
path = validate_path(os.path.join(root, relative), root)
|
|
4156
|
+
except SecurityError:
|
|
4157
|
+
continue
|
|
4158
|
+
if not os.path.isfile(path):
|
|
4159
|
+
continue
|
|
4160
|
+
origin = _asset_origin(ws, root)
|
|
4161
|
+
if origin == EXAMPLES_ORIGIN:
|
|
4162
|
+
raise HTTPException(
|
|
4163
|
+
status_code=403,
|
|
4164
|
+
detail=f"asset:{relative} is read-only: it comes from an "
|
|
4165
|
+
f"examples library, not a library this server writes",
|
|
4166
|
+
)
|
|
4167
|
+
os.remove(path)
|
|
4168
|
+
logger.info(f"Deleted asset:{relative} ({path})")
|
|
4169
|
+
forget_workspace_usage()
|
|
4170
|
+
return {"name": relative, "deleted": True, "origin": origin}
|
|
4171
|
+
|
|
4172
|
+
raise HTTPException(status_code=404, detail=f"No such asset: {relative}")
|
|
4173
|
+
|
|
4174
|
+
# ----------------------------------------------------------------- models
|
|
4175
|
+
|
|
4176
|
+
@app.get("/api/models")
|
|
4177
|
+
def get_models():
|
|
4178
|
+
"""What the Hugging Face hub cache holds, largest repo first."""
|
|
4179
|
+
return scan_models()
|
|
4180
|
+
|
|
4181
|
+
downloads = download_manager or DownloadManager()
|
|
4182
|
+
|
|
4183
|
+
class DownloadRequest(BaseModel):
|
|
4184
|
+
repo_id: str = Field(description="Hub repo to download, e.g. org/model")
|
|
4185
|
+
|
|
4186
|
+
@app.post("/api/models/download", status_code=202)
|
|
4187
|
+
def start_download(body: DownloadRequest):
|
|
4188
|
+
"""Start a background snapshot download into the hub cache."""
|
|
4189
|
+
try:
|
|
4190
|
+
return downloads.start(body.repo_id)
|
|
4191
|
+
except ValueError as e:
|
|
4192
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
4193
|
+
|
|
4194
|
+
@app.get("/api/models/downloads")
|
|
4195
|
+
def list_downloads():
|
|
4196
|
+
return {"downloads": downloads.status_list()}
|
|
4197
|
+
|
|
4198
|
+
@app.post("/api/models/downloads/{download_id}/cancel")
|
|
4199
|
+
def cancel_download(download_id: str):
|
|
4200
|
+
"""Request cancellation; takes effect at the next progress tick.
|
|
4201
|
+
Partial files stay in the cache and resume on a retry."""
|
|
4202
|
+
status = downloads.cancel(download_id)
|
|
4203
|
+
if status is None:
|
|
4204
|
+
raise HTTPException(status_code=404, detail="Unknown download")
|
|
4205
|
+
return status
|
|
4206
|
+
|
|
4207
|
+
@app.delete("/api/models")
|
|
4208
|
+
def delete_cached_model(repo: str):
|
|
4209
|
+
"""Delete every cached revision of one repo from the hub cache.
|
|
4210
|
+
|
|
4211
|
+
Refused while a job is running or queued: the worker may be reading
|
|
4212
|
+
exactly the files a delete would remove out from under it."""
|
|
4213
|
+
if manager.is_busy():
|
|
4214
|
+
raise HTTPException(
|
|
4215
|
+
status_code=409,
|
|
4216
|
+
detail="A job is running or queued - deleting model files "
|
|
4217
|
+
"out from under it would corrupt the run",
|
|
4218
|
+
)
|
|
4219
|
+
if downloads.is_active():
|
|
4220
|
+
raise HTTPException(
|
|
4221
|
+
status_code=409,
|
|
4222
|
+
detail="A model download is in progress - deleting cache "
|
|
4223
|
+
"files while it writes them would corrupt both",
|
|
4224
|
+
)
|
|
4225
|
+
try:
|
|
4226
|
+
freed = delete_model(repo)
|
|
4227
|
+
except ValueError as e:
|
|
4228
|
+
raise HTTPException(status_code=404, detail=str(e))
|
|
4229
|
+
logger.info(f"Deleted {repo} from the hub cache ({freed} bytes)")
|
|
4230
|
+
return {"repo_id": repo, "deleted": True, "freed": freed}
|
|
4231
|
+
|
|
4232
|
+
# ------------------------------------------------------ diffusers update
|
|
4233
|
+
|
|
4234
|
+
updater = diffusers_updater or DiffusersUpdater()
|
|
4235
|
+
|
|
4236
|
+
@app.get("/api/system/diffusers")
|
|
4237
|
+
def diffusers_state():
|
|
4238
|
+
"""Installed diffusers version (with its git commit when installed
|
|
4239
|
+
from git) and the state of any update."""
|
|
4240
|
+
return updater.status()
|
|
4241
|
+
|
|
4242
|
+
class UpdateDiffusersRequest(BaseModel):
|
|
4243
|
+
commit: Optional[str] = Field(
|
|
4244
|
+
default=None,
|
|
4245
|
+
description="Git commit hash to pin the install to (7-40 hex "
|
|
4246
|
+
"characters) instead of tracking GitHub HEAD",
|
|
4247
|
+
)
|
|
4248
|
+
revert: bool = Field(
|
|
4249
|
+
default=False,
|
|
4250
|
+
description="Pin back to the known-good published release "
|
|
4251
|
+
"(pyproject.toml's diffusers floor) instead of installing from "
|
|
4252
|
+
"git. Mutually exclusive with commit.",
|
|
4253
|
+
)
|
|
4254
|
+
|
|
4255
|
+
@app.post("/api/system/diffusers/update", status_code=202)
|
|
4256
|
+
def update_diffusers(body: UpdateDiffusersRequest = UpdateDiffusersRequest()):
|
|
4257
|
+
"""Upgrade diffusers in the background: GitHub HEAD by default, a
|
|
4258
|
+
pinned commit when `commit` is given, or a revert to the last
|
|
4259
|
+
known-good published release when `revert` is true.
|
|
4260
|
+
|
|
4261
|
+
Refused while a job is running or queued: pip replacing package
|
|
4262
|
+
files under a loaded pipeline is the model-delete hazard in another
|
|
4263
|
+
form. On success the idle worker is shut down so the next job
|
|
4264
|
+
imports the new version."""
|
|
4265
|
+
if body.commit and body.revert:
|
|
4266
|
+
raise HTTPException(
|
|
4267
|
+
status_code=400,
|
|
4268
|
+
detail="commit and revert are mutually exclusive",
|
|
4269
|
+
)
|
|
4270
|
+
commit = None
|
|
4271
|
+
if body.commit:
|
|
4272
|
+
try:
|
|
4273
|
+
commit = validate_commit_hash(body.commit)
|
|
4274
|
+
except InvalidInputError as e:
|
|
4275
|
+
raise HTTPException(status_code=400, detail=str(e))
|
|
4276
|
+
if manager.is_busy():
|
|
4277
|
+
raise HTTPException(
|
|
4278
|
+
status_code=409,
|
|
4279
|
+
detail="A job is running or queued - updating diffusers "
|
|
4280
|
+
"underneath it could corrupt the run",
|
|
4281
|
+
)
|
|
4282
|
+
if downloads.is_active():
|
|
4283
|
+
raise HTTPException(
|
|
4284
|
+
status_code=409,
|
|
4285
|
+
detail="A model download is in progress - replacing package "
|
|
4286
|
+
"files while it runs could corrupt the download",
|
|
4287
|
+
)
|
|
4288
|
+
try:
|
|
4289
|
+
return updater.start(
|
|
4290
|
+
on_success=manager.restart_worker_if_idle,
|
|
4291
|
+
commit=commit,
|
|
4292
|
+
revert=body.revert,
|
|
4293
|
+
)
|
|
4294
|
+
except ValueError as e:
|
|
4295
|
+
raise HTTPException(status_code=409, detail=str(e))
|
|
4296
|
+
|
|
4297
|
+
# --------------------------------------------------------- memory/health
|
|
4298
|
+
|
|
4299
|
+
@app.get("/api/memory")
|
|
4300
|
+
def memory():
|
|
4301
|
+
try:
|
|
4302
|
+
return manager.memory_status()
|
|
4303
|
+
except Exception as e:
|
|
4304
|
+
raise HTTPException(status_code=503, detail=f"Worker unavailable: {e}")
|
|
4305
|
+
|
|
4306
|
+
@app.post("/api/memory/clear")
|
|
4307
|
+
def clear_memory():
|
|
4308
|
+
"""Drop every loaded pipeline and the step cache, freeing VRAM/RAM
|
|
4309
|
+
without waiting for the next job to evict one model for another.
|
|
4310
|
+
|
|
4311
|
+
Refused while a job is running or queued (409) rather than blocked -
|
|
4312
|
+
the queue is FIFO, so the caller should wait for the job to finish
|
|
4313
|
+
and retry instead of this call stalling until it does.
|
|
4314
|
+
|
|
4315
|
+
A server with no worker process resident answers `cleared` with a
|
|
4316
|
+
null `info` rather than a 503: the worker is on-demand, so its
|
|
4317
|
+
absence means there was nothing loaded to clear."""
|
|
4318
|
+
if manager.is_busy():
|
|
4319
|
+
raise HTTPException(
|
|
4320
|
+
status_code=409,
|
|
4321
|
+
detail="A job is running or queued - clearing memory out "
|
|
4322
|
+
"from under it would corrupt the run. Wait for it to finish.",
|
|
4323
|
+
)
|
|
4324
|
+
try:
|
|
4325
|
+
info = manager.clear_memory()
|
|
4326
|
+
except RuntimeError as e:
|
|
4327
|
+
raise HTTPException(status_code=503, detail=f"Worker unavailable: {e}")
|
|
4328
|
+
return {"cleared": True, "info": info}
|
|
4329
|
+
|
|
4330
|
+
@app.get("/api/health")
|
|
4331
|
+
def health():
|
|
4332
|
+
import socket
|
|
4333
|
+
|
|
4334
|
+
from .. import __version__, get_device, get_device_type
|
|
4335
|
+
|
|
4336
|
+
worker = manager.worker_manager
|
|
4337
|
+
return {
|
|
4338
|
+
"status": "ok",
|
|
4339
|
+
"version": __version__,
|
|
4340
|
+
# on-demand subprocess: false on an idle server that hasn't run
|
|
4341
|
+
# a job yet (or after a memory clear) is normal, not a fault -
|
|
4342
|
+
# it means no model process is currently resident, not that the
|
|
4343
|
+
# server is unhealthy (#206)
|
|
4344
|
+
"worker_alive": bool(
|
|
4345
|
+
worker.worker_active
|
|
4346
|
+
and worker.worker_process is not None
|
|
4347
|
+
and worker.worker_process.is_alive()
|
|
4348
|
+
),
|
|
4349
|
+
"current_job": manager._current_job_id,
|
|
4350
|
+
"queued": sum(1 for j in manager.list() if j["status"] == "queued"),
|
|
4351
|
+
# which machine answered - the thing a remote client cannot
|
|
4352
|
+
# otherwise tell apart from a stale tunnel pointed at nothing
|
|
4353
|
+
"hostname": socket.gethostname(),
|
|
4354
|
+
"device": get_device_type(get_device()),
|
|
4355
|
+
"mcp": bool(app.state.mcp_mounted),
|
|
4356
|
+
}
|
|
4357
|
+
|
|
4358
|
+
@app.get("/api/server")
|
|
4359
|
+
def server_info(ws: Workspace = Depends(selected_workspace)):
|
|
4360
|
+
"""How this server is reachable, for the UI's Server page: what it
|
|
4361
|
+
is bound to, whether a token is needed, whether MCP is mounted, and
|
|
4362
|
+
the addresses another machine could name it by.
|
|
4363
|
+
|
|
4364
|
+
No URL is composed here - the caller pairs an address with `port`
|
|
4365
|
+
and `mcp.path` - and the token itself is never reported in any
|
|
4366
|
+
form, only whether one is required. An interface enumeration
|
|
4367
|
+
failure is not a server failure: `addresses` comes back empty.
|
|
4368
|
+
|
|
4369
|
+
`directories` is scoped to the `?workspace=` a caller names (or the
|
|
4370
|
+
session's own pin, via `_scoped`) - a mounted `download_output`
|
|
4371
|
+
confines a write to *that* workspace's output tree, so reporting
|
|
4372
|
+
the server's own default here regardless of the selector sent a
|
|
4373
|
+
caller pinned elsewhere writing into `default` without any error (#389).
|
|
4374
|
+
"""
|
|
4375
|
+
import socket
|
|
4376
|
+
|
|
4377
|
+
from .. import __version__, get_device, get_device_type
|
|
4378
|
+
|
|
4379
|
+
try:
|
|
4380
|
+
addresses = local_addresses()
|
|
4381
|
+
except Exception:
|
|
4382
|
+
logger.debug("Could not enumerate local addresses", exc_info=True)
|
|
4383
|
+
addresses = []
|
|
4384
|
+
return {
|
|
4385
|
+
"hostname": socket.gethostname(),
|
|
4386
|
+
"version": __version__,
|
|
4387
|
+
"device": get_device_type(get_device()),
|
|
4388
|
+
"bind_host": host,
|
|
4389
|
+
"port": port,
|
|
4390
|
+
"wildcard_bind": wildcard_bind,
|
|
4391
|
+
"auth_required": bool(token),
|
|
4392
|
+
# The posture a security check has to know it is testing: with
|
|
4393
|
+
# this off, a workflow file is untrusted input - no arbitrary
|
|
4394
|
+
# imports, no remote code, no location outside the workspace's
|
|
4395
|
+
# roots. It is not a secret (the refusals name the flag), and
|
|
4396
|
+
# without it the posture could only be inferred from behavior
|
|
4397
|
+
# (#120)
|
|
4398
|
+
"trust_workflows": workflows_are_trusted(),
|
|
4399
|
+
"mcp": {"mounted": bool(app.state.mcp_mounted), "path": MCP_PATH},
|
|
4400
|
+
"addresses": addresses,
|
|
4401
|
+
# Python/torch/CUDA-driver/other-package versions - the detail
|
|
4402
|
+
# neither this route's own `version` field nor `get_health`
|
|
4403
|
+
# answers, e.g. whether bitsandbytes is even installed (#222)
|
|
4404
|
+
"runtime": runtime_info(),
|
|
4405
|
+
"directories": {
|
|
4406
|
+
# ws's properties are already absolute (Workspace and
|
|
4407
|
+
# ConfiguredWorkspace both resolve at construction). This
|
|
4408
|
+
# "workspace" is the root path a mounted download_output
|
|
4409
|
+
# confines a write to (dw_mcp/media.py's _remote_root) -
|
|
4410
|
+
# None for a default workspace configured from individual
|
|
4411
|
+
# directory overrides with no --workspace root, same as
|
|
4412
|
+
# before this route was workspace-aware
|
|
4413
|
+
"workspace": ws.root,
|
|
4414
|
+
"workflows": ws.workflows,
|
|
4415
|
+
"assets": ws.assets,
|
|
4416
|
+
"outputs": ws.outputs,
|
|
4417
|
+
"prompts": ws.prompts,
|
|
4418
|
+
},
|
|
4419
|
+
}
|
|
4420
|
+
|
|
4421
|
+
# ---------------------------------------------------------------- outputs
|
|
4422
|
+
|
|
4423
|
+
# ------------------------------------------------------------------ mcp
|
|
4424
|
+
|
|
4425
|
+
if mcp_asgi is not None:
|
|
4426
|
+
# One route rather than app.mount("/mcp", ...): Starlette's Mount
|
|
4427
|
+
# only matches paths *under* its prefix, so a bare POST /mcp - the
|
|
4428
|
+
# URL clients are configured with - would fall through to the SPA
|
|
4429
|
+
# catch-all below and come back 405. This matches /mcp and
|
|
4430
|
+
# anything under it; build_mcp_app's wrapper normalizes the path
|
|
4431
|
+
# for the SDK app's single route.
|
|
4432
|
+
# Exactly the two spellings require_bearer_token gates - a single
|
|
4433
|
+
# "/mcp{path:path}" route would also answer /mcpfoo, which the gate
|
|
4434
|
+
# does not cover.
|
|
4435
|
+
app.router.routes.append(Route("/mcp", endpoint=mcp_asgi, name="mcp"))
|
|
4436
|
+
app.router.routes.append(
|
|
4437
|
+
Route("/mcp/{sub_path:path}", endpoint=mcp_asgi, name="mcp_sub")
|
|
4438
|
+
)
|
|
4439
|
+
|
|
4440
|
+
# Generated files and input media, served as routes rather than static
|
|
4441
|
+
# mounts: a mount is bound to one directory at startup, and a workspace
|
|
4442
|
+
# can be created afterwards. Each handler delegates to a StaticFiles
|
|
4443
|
+
# instance for the workspace's own root (_static_files_for) rather than
|
|
4444
|
+
# a bare FileResponse - a FileResponse never answers 304 (no
|
|
4445
|
+
# If-None-Match handling), so every gallery load re-streamed the whole
|
|
4446
|
+
# file; going through StaticFiles.get_response restores ETag/
|
|
4447
|
+
# If-None-Match 304s, Range/206 and its own 404 handling, the way a real
|
|
4448
|
+
# mount always has.
|
|
4449
|
+
#
|
|
4450
|
+
# Ungated, as the mounts were, and for the same reason: an <img> or
|
|
4451
|
+
# <video> tag cannot attach an Authorization header. The auth middleware
|
|
4452
|
+
# only gates /api/, so these stay reachable exactly as before.
|
|
4453
|
+
#
|
|
4454
|
+
# '/inputs', not '/assets': Vite emits the SPA's own bundles under
|
|
4455
|
+
# /assets/, and serving the library there shadows them - the page loads
|
|
4456
|
+
# and then renders nothing, because its script and stylesheet 404. The
|
|
4457
|
+
# name is also the symmetric one, next to /outputs
|
|
4458
|
+
def _sandbox_active_content(response):
|
|
4459
|
+
"""Serve a document type under `Content-Security-Policy: sandbox`.
|
|
4460
|
+
|
|
4461
|
+
/outputs and /inputs share the UI's origin and need no token, so an
|
|
4462
|
+
.html, .xhtml, .xml or .svg file served as-is is a page whose script
|
|
4463
|
+
reads the token the UI keeps in localStorage. Validation refuses a
|
|
4464
|
+
workflow writing one (dw/content_types.py), but a planted file or a
|
|
4465
|
+
kept asset never passes through there. sandbox gives the document an
|
|
4466
|
+
opaque origin and no script, and still lets an image or a .txt show
|
|
4467
|
+
in the tab, which an attachment disposition would not. Set on the
|
|
4468
|
+
Response StaticFiles built, so its ETag/304 and Range/206 stand"""
|
|
4469
|
+
media_type = (
|
|
4470
|
+
response.headers.get("content-type", "").split(";")[0].strip().lower()
|
|
4471
|
+
)
|
|
4472
|
+
if media_type in ACTIVE_DOCUMENT_TYPES:
|
|
4473
|
+
response.headers["Content-Security-Policy"] = "sandbox"
|
|
4474
|
+
return response
|
|
4475
|
+
|
|
4476
|
+
@app.get("/outputs/{name:path}")
|
|
4477
|
+
async def output_file(
|
|
4478
|
+
name: str, request: Request, ws: Workspace = Depends(selected_workspace)
|
|
4479
|
+
):
|
|
4480
|
+
"""One generated file, from the workspace that made it - or, by an
|
|
4481
|
+
'asset:' reference, one file from its asset library (#445): every
|
|
4482
|
+
other route in this family (`get_gallery_metadata`, `/frames`,
|
|
4483
|
+
`/audio`, `/assess`) already accepts one, and this route answering a
|
|
4484
|
+
bare StaticFiles 404 for the same name gave no hint why."""
|
|
4485
|
+
name = _strip_output_prefix(name)
|
|
4486
|
+
if is_asset_reference(name):
|
|
4487
|
+
# This route is outside the token gate (the auth middleware
|
|
4488
|
+
# covers /api/ and /mcp only), so a miss must not carry
|
|
4489
|
+
# _asset_file's detail, which names every root searched by its
|
|
4490
|
+
# absolute server path. Keep the hint #445 added, without them.
|
|
4491
|
+
try:
|
|
4492
|
+
path = _asset_file(name, ws)
|
|
4493
|
+
except HTTPException as e:
|
|
4494
|
+
if e.status_code != 404:
|
|
4495
|
+
raise
|
|
4496
|
+
raise HTTPException(
|
|
4497
|
+
status_code=404,
|
|
4498
|
+
detail=f"Unknown asset {name!r}: not in this workspace's "
|
|
4499
|
+
"asset library (list_assets shows what is)",
|
|
4500
|
+
) from None
|
|
4501
|
+
files = _static_files_for(os.path.dirname(path))
|
|
4502
|
+
response = await files.get_response(os.path.basename(path), request.scope)
|
|
4503
|
+
else:
|
|
4504
|
+
files = _static_files_for(ws.outputs)
|
|
4505
|
+
response = await files.get_response(name, request.scope)
|
|
4506
|
+
return _sandbox_active_content(response)
|
|
4507
|
+
|
|
4508
|
+
@app.get("/inputs/{name:path}")
|
|
4509
|
+
async def input_file(
|
|
4510
|
+
name: str, request: Request, ws: Workspace = Depends(selected_workspace)
|
|
4511
|
+
):
|
|
4512
|
+
"""One file from the asset search path, for the editor's preview of
|
|
4513
|
+
an uploaded or chosen asset - the workspace's own library first,
|
|
4514
|
+
then any read-only examples library, so an example workflow's media
|
|
4515
|
+
previews the way an upload does."""
|
|
4516
|
+
roots = _asset_roots(ws)
|
|
4517
|
+
if not roots:
|
|
4518
|
+
raise HTTPException(status_code=404, detail="no asset library")
|
|
4519
|
+
for root in roots:
|
|
4520
|
+
try:
|
|
4521
|
+
candidate = validate_path(os.path.join(root, name), root)
|
|
4522
|
+
except SecurityError:
|
|
4523
|
+
continue
|
|
4524
|
+
if os.path.isfile(candidate):
|
|
4525
|
+
files = _static_files_for(root)
|
|
4526
|
+
return _sandbox_active_content(
|
|
4527
|
+
await files.get_response(name, request.scope)
|
|
4528
|
+
)
|
|
4529
|
+
# Nothing has it: let the workspace's own library answer, so the
|
|
4530
|
+
# 404 (and its headers) come from StaticFiles as they always did
|
|
4531
|
+
files = _static_files_for(roots[0])
|
|
4532
|
+
return await files.get_response(name, request.scope)
|
|
4533
|
+
|
|
4534
|
+
def _export_download_name(directory, job_id):
|
|
4535
|
+
"""'<workflow>-v4-<job id>.zip' when the exported manifest says which
|
|
4536
|
+
run it was, else '<job id>.zip'. Only the saved file's name: the
|
|
4537
|
+
URL and the entries inside keep the job id, so nothing that already
|
|
4538
|
+
names an export changes."""
|
|
4539
|
+
try:
|
|
4540
|
+
with open(os.path.join(directory, MANIFEST_FILE_NAME)) as file:
|
|
4541
|
+
manifest = json.load(file)
|
|
4542
|
+
except (OSError, ValueError):
|
|
4543
|
+
return f"{job_id}.zip"
|
|
4544
|
+
if not isinstance(manifest, dict):
|
|
4545
|
+
return f"{job_id}.zip"
|
|
4546
|
+
version = manifest.get("version")
|
|
4547
|
+
identity = (manifest.get("workflow") or {}).get("identity")
|
|
4548
|
+
if not isinstance(version, int) or isinstance(version, bool):
|
|
4549
|
+
return f"{job_id}.zip"
|
|
4550
|
+
if not isinstance(identity, str) or not identity:
|
|
4551
|
+
return f"v{version}-{job_id}.zip"
|
|
4552
|
+
slug = re.sub(r"[^A-Za-z0-9_.-]+", "-", identity).strip("-.")
|
|
4553
|
+
return f"{slug}-v{version}-{job_id}.zip" if slug else f"v{version}-{job_id}.zip"
|
|
4554
|
+
|
|
4555
|
+
# Ungated for the same reason the two above are: a download link cannot
|
|
4556
|
+
# attach an Authorization header either
|
|
4557
|
+
@app.get("/exports/{job_id}.zip")
|
|
4558
|
+
def export_zip(job_id: str, ws: Workspace = Depends(selected_workspace)):
|
|
4559
|
+
"""One job's export as a zip, built on request from the directory
|
|
4560
|
+
rather than kept as a second copy. Entries are named
|
|
4561
|
+
'<job id>/<relative path>', so unzipping anywhere gives the same tree
|
|
4562
|
+
the server holds."""
|
|
4563
|
+
try:
|
|
4564
|
+
directory = export_directory(ws.root, job_id)
|
|
4565
|
+
except SecurityError:
|
|
4566
|
+
raise HTTPException(status_code=404, detail="No export for this job")
|
|
4567
|
+
if not os.path.isdir(directory):
|
|
4568
|
+
raise HTTPException(status_code=404, detail="No export for this job")
|
|
4569
|
+
|
|
4570
|
+
entries = []
|
|
4571
|
+
for current, _dirs, names in os.walk(directory):
|
|
4572
|
+
for name in sorted(names):
|
|
4573
|
+
path = os.path.join(current, name)
|
|
4574
|
+
entry = os.path.relpath(path, directory).replace(os.sep, "/")
|
|
4575
|
+
entries.append((f"{job_id}/{entry}", path))
|
|
4576
|
+
return _zip_download(entries, _export_download_name(directory, job_id))
|
|
4577
|
+
|
|
4578
|
+
# ---------------------------------------------------------------- the UI
|
|
4579
|
+
|
|
4580
|
+
resolved_ui = ui_dir or default_ui_dir()
|
|
4581
|
+
if resolved_ui:
|
|
4582
|
+
# Mounted last so /api and /outputs keep precedence; html=True serves
|
|
4583
|
+
# index.html at /, and the SPA routes by hash so no fallback is needed
|
|
4584
|
+
app.mount("/", StaticFiles(directory=resolved_ui, html=True), name="ui")
|
|
4585
|
+
|
|
4586
|
+
return app
|