diffusers-workflow 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusers_workflow-0.4.0.dist-info/METADATA +318 -0
- diffusers_workflow-0.4.0.dist-info/RECORD +260 -0
- diffusers_workflow-0.4.0.dist-info/WHEEL +5 -0
- diffusers_workflow-0.4.0.dist-info/entry_points.txt +7 -0
- diffusers_workflow-0.4.0.dist-info/licenses/LICENSE +201 -0
- diffusers_workflow-0.4.0.dist-info/top_level.txt +2 -0
- dw/__init__.py +440 -0
- dw/adapter_compatibility.py +226 -0
- dw/arguments.py +1231 -0
- dw/assessment_rules.py +159 -0
- dw/assets.py +130 -0
- dw/cache_blocks.json +16 -0
- dw/cache_blocks.py +146 -0
- dw/community_pipelines/pipeline_flux_rf_inversion.py +1184 -0
- dw/content_types.py +150 -0
- dw/dissolve_frame_errors.py +121 -0
- dw/docs/ACCELERATION.md +352 -0
- dw/docs/AGENT_LOOP.md +95 -0
- dw/docs/DEPENDENCIES.md +91 -0
- dw/docs/IP_ADAPTER.md +109 -0
- dw/docs/LORAS.md +131 -0
- dw/docs/MCP.md +517 -0
- dw/docs/PROMPT_WEIGHTING.md +78 -0
- dw/docs/QUANTIZATION.md +230 -0
- dw/docs/RECIPES_24GB.md +201 -0
- dw/docs/RELEASING.md +195 -0
- dw/docs/REMOTE.md +140 -0
- dw/docs/REPL_COMMANDS.md +121 -0
- dw/docs/REPL_WORKER_GUIDE.md +51 -0
- dw/docs/SECURITY.md +272 -0
- dw/docs/SECURITY_QUICKREF.md +112 -0
- dw/docs/SERVER.md +679 -0
- dw/docs/TASKS.md +1741 -0
- dw/docs/TESTING.md +71 -0
- dw/docs/WORKFLOW_GUIDE.md +2038 -0
- dw/docs/WORKSPACES.md +316 -0
- dw/download_watch.py +335 -0
- dw/elision.py +306 -0
- dw/events.py +275 -0
- dw/for_each.py +409 -0
- dw/host_memory.py +258 -0
- dw/host_memory_projection.py +230 -0
- dw/hub_cache.py +432 -0
- dw/introspection.py +1228 -0
- dw/kernel_availability.py +208 -0
- dw/locations.py +599 -0
- dw/log_setup.py +45 -0
- dw/loudness.py +82 -0
- dw/media_audio.py +217 -0
- dw/media_frames.py +367 -0
- dw/media_info.py +297 -0
- dw/pipeline_processors/chain.py +821 -0
- dw/pipeline_processors/config_objects.py +237 -0
- dw/pipeline_processors/pipeline.py +2297 -0
- dw/pipeline_processors/remote.py +46 -0
- dw/plan.py +920 -0
- dw/previous_results.py +411 -0
- dw/probe_paths.py +59 -0
- dw/prompt_schema.json +48 -0
- dw/prompt_weighting.py +378 -0
- dw/prompts.py +159 -0
- dw/realize.py +250 -0
- dw/reference_limits.py +215 -0
- dw/reference_names.py +125 -0
- dw/repl.py +338 -0
- dw/repl_commands.py +836 -0
- dw/repl_worker.py +159 -0
- dw/result.py +1720 -0
- dw/result_fps.py +82 -0
- dw/run.py +162 -0
- dw/runs.py +768 -0
- dw/scalar_result_validation.py +97 -0
- dw/schema.py +283 -0
- dw/security.py +1038 -0
- dw/select_validation.py +115 -0
- dw/serve.py +277 -0
- dw/server/__init__.py +2 -0
- dw/server/app.py +4586 -0
- dw/server/assess.py +132 -0
- dw/server/catalog_shape.py +487 -0
- dw/server/enhancers.py +129 -0
- dw/server/exports.py +480 -0
- dw/server/guides.py +257 -0
- dw/server/jobs.py +1561 -0
- dw/server/mcp_mount.py +95 -0
- dw/server/netinfo.py +124 -0
- dw/server/observed_cost.py +379 -0
- dw/server/sysinfo.py +71 -0
- dw/server/ui/assets/abap-08VXUWAP.js +1 -0
- dw/server/ui/assets/apex-BWPQTe0t.js +1 -0
- dw/server/ui/assets/azcli-Bc_sGQ0U.js +1 -0
- dw/server/ui/assets/bat-i0X4ZdIN.js +1 -0
- dw/server/ui/assets/bicep-B5-_aFwp.js +2 -0
- dw/server/ui/assets/cameligo-DMUM7wLl.js +1 -0
- dw/server/ui/assets/clojure-Cm7r79vr.js +1 -0
- dw/server/ui/assets/codicon-Brq4_Ui5.ttf +0 -0
- dw/server/ui/assets/coffee-Ba7i2nA0.js +1 -0
- dw/server/ui/assets/cpp-C7h46wYY.js +1 -0
- dw/server/ui/assets/csharp-BKxtCVv1.js +1 -0
- dw/server/ui/assets/csp-bTuwJoIa.js +1 -0
- dw/server/ui/assets/css-DIMkf-bt.js +3 -0
- dw/server/ui/assets/css.worker-B3ciXF_0.js +93 -0
- dw/server/ui/assets/cssMode-CPznxfY8.js +1 -0
- dw/server/ui/assets/cypher-CVaqCwHa.js +1 -0
- dw/server/ui/assets/dart-onAF5SnQ.js +1 -0
- dw/server/ui/assets/dockerfile-DZFCIeNp.js +1 -0
- dw/server/ui/assets/ecl-D05T4iGw.js +1 -0
- dw/server/ui/assets/editor-jjEx9u7D.css +1 -0
- dw/server/ui/assets/editor.api-CpWcotrd.js +847 -0
- dw/server/ui/assets/editor.worker-q-txB4vs.js +30 -0
- dw/server/ui/assets/elixir-6RTg0lbw.js +1 -0
- dw/server/ui/assets/flow9-C5_-GSwl.js +1 -0
- dw/server/ui/assets/freemarker2-CXtRM8N4.js +3 -0
- dw/server/ui/assets/fsharp-C8Ef5oNN.js +1 -0
- dw/server/ui/assets/go-C-y9NEjX.js +1 -0
- dw/server/ui/assets/graphql-fmXr3nnJ.js +1 -0
- dw/server/ui/assets/handlebars-N7x-6NMY.js +1 -0
- dw/server/ui/assets/hcl-CpzslTdj.js +1 -0
- dw/server/ui/assets/html-PhsdjHSr.js +1 -0
- dw/server/ui/assets/html.worker-C93Ht9o9.js +506 -0
- dw/server/ui/assets/htmlMode-Dgj0SEok.js +1 -0
- dw/server/ui/assets/index-3Vw6WAPW.css +1 -0
- dw/server/ui/assets/index-DgrYhQd9.js +43 -0
- dw/server/ui/assets/ini-sBoK_t0W.js +1 -0
- dw/server/ui/assets/java-BEtHBSE6.js +1 -0
- dw/server/ui/assets/javascript-BJqN9Qhv.js +1 -0
- dw/server/ui/assets/json.worker-B2V3pomh.js +62 -0
- dw/server/ui/assets/jsonMode-DbM4SWSv.js +7 -0
- dw/server/ui/assets/julia-Bri6UV-V.js +1 -0
- dw/server/ui/assets/kotlin-BOotOW0E.js +1 -0
- dw/server/ui/assets/less-B9JPFI3C.js +2 -0
- dw/server/ui/assets/lexon-CfSJPG6W.js +1 -0
- dw/server/ui/assets/liquid-BWr8lEc4.js +1 -0
- dw/server/ui/assets/lspLanguageFeatures-C1iGuDyZ.js +4 -0
- dw/server/ui/assets/lua-CsQS60Ue.js +1 -0
- dw/server/ui/assets/m3-D-oSqn_W.js +1 -0
- dw/server/ui/assets/markdown-Cimd5fb3.js +1 -0
- dw/server/ui/assets/mdx-DAdMi_0p.js +1 -0
- dw/server/ui/assets/mips-CIPQ_RoX.js +1 -0
- dw/server/ui/assets/monaco--ixms01u.css +1 -0
- dw/server/ui/assets/monaco-BGCeEqaw.js +56 -0
- dw/server/ui/assets/msdax-DauUninz.js +1 -0
- dw/server/ui/assets/mysql-SOo6toE5.js +1 -0
- dw/server/ui/assets/objective-c-FvmIjYaQ.js +1 -0
- dw/server/ui/assets/pascal-DrH0SRf2.js +1 -0
- dw/server/ui/assets/pascaligo-D-ptJ9y-.js +1 -0
- dw/server/ui/assets/perl-oz_6vUea.js +1 -0
- dw/server/ui/assets/pgsql-DTj74zXo.js +1 -0
- dw/server/ui/assets/php-nr791fC2.js +1 -0
- dw/server/ui/assets/pla-CopQ2nXW.js +1 -0
- dw/server/ui/assets/postiats-43DmfD33.js +1 -0
- dw/server/ui/assets/powerquery-D3hlyOfw.js +1 -0
- dw/server/ui/assets/powershell-DmHpPYUd.js +1 -0
- dw/server/ui/assets/protobuf-C531GsRP.js +2 -0
- dw/server/ui/assets/pug-Z5eAx3Zn.js +1 -0
- dw/server/ui/assets/python-Bcn70HdC.js +1 -0
- dw/server/ui/assets/qsharp-DkqhCAOL.js +1 -0
- dw/server/ui/assets/r-BwWrilGY.js +1 -0
- dw/server/ui/assets/razor-D1HmNnby.js +1 -0
- dw/server/ui/assets/redis-ClamHrr6.js +1 -0
- dw/server/ui/assets/redshift-DT7zqm-g.js +1 -0
- dw/server/ui/assets/restructuredtext-BYgofb2h.js +1 -0
- dw/server/ui/assets/ruby-DezsRK8O.js +1 -0
- dw/server/ui/assets/rust-DdL9SqIa.js +1 -0
- dw/server/ui/assets/sb-CcwsVR0C.js +1 -0
- dw/server/ui/assets/scala-DHpiXF5c.js +1 -0
- dw/server/ui/assets/scheme-BeGwcela.js +1 -0
- dw/server/ui/assets/scss-gp-XZpBa.js +3 -0
- dw/server/ui/assets/shell-CC2rA5mh.js +1 -0
- dw/server/ui/assets/solidity-BEEn4gHE.js +1 -0
- dw/server/ui/assets/sophia-CRfGWb83.js +1 -0
- dw/server/ui/assets/sparql-D_Lu-MrJ.js +1 -0
- dw/server/ui/assets/sql-NEE52Syq.js +1 -0
- dw/server/ui/assets/st-DbInun42.js +1 -0
- dw/server/ui/assets/swift-Bxkupp3x.js +1 -0
- dw/server/ui/assets/systemverilog-Bz4Y3fRF.js +1 -0
- dw/server/ui/assets/tcl-DISqw1ZD.js +1 -0
- dw/server/ui/assets/ts.worker-D7T1-Ig5.js +67738 -0
- dw/server/ui/assets/tsMode-D6u0XmOW.js +11 -0
- dw/server/ui/assets/twig-De2hgUGE.js +1 -0
- dw/server/ui/assets/typescript-BU6v-LMV.js +1 -0
- dw/server/ui/assets/typespec-B8J7ngcE.js +1 -0
- dw/server/ui/assets/vb-DV3o63ZY.js +1 -0
- dw/server/ui/assets/wgsl-DpFanUEy.js +298 -0
- dw/server/ui/assets/workers-Cn7cTUKr.js +1 -0
- dw/server/ui/assets/xml--0LP2Lwk.js +1 -0
- dw/server/ui/assets/yaml-mpBg9jnt.js +1 -0
- dw/server/ui/index.html +17 -0
- dw/server/updater.py +192 -0
- dw/settings.py +98 -0
- dw/shot_span_preflight.py +116 -0
- dw/shots.py +359 -0
- dw/slice_preflight.py +148 -0
- dw/step.py +187 -0
- dw/step_cache.py +442 -0
- dw/subfolders.py +107 -0
- dw/task_domains.py +307 -0
- dw/tasks/assess.py +826 -0
- dw/tasks/audio_transcription.py +88 -0
- dw/tasks/audio_utils.py +1862 -0
- dw/tasks/background_remover.py +43 -0
- dw/tasks/borders.py +113 -0
- dw/tasks/compose_text.py +74 -0
- dw/tasks/concat_videos.py +300 -0
- dw/tasks/depth_estimator.py +54 -0
- dw/tasks/diffusion_upscale.py +109 -0
- dw/tasks/dissolve_videos.py +342 -0
- dw/tasks/format_messages.py +24 -0
- dw/tasks/gather.py +173 -0
- dw/tasks/grade.py +97 -0
- dw/tasks/image_to_text.py +43 -0
- dw/tasks/image_utils.py +764 -0
- dw/tasks/interpolate_frames.py +252 -0
- dw/tasks/judge.py +68 -0
- dw/tasks/model_cache.py +55 -0
- dw/tasks/pair_audio.py +268 -0
- dw/tasks/qr_code.py +19 -0
- dw/tasks/restore_faces.py +175 -0
- dw/tasks/rife_model.py +192 -0
- dw/tasks/segment.py +121 -0
- dw/tasks/select.py +111 -0
- dw/tasks/speech_generation.py +228 -0
- dw/tasks/stabilize.py +129 -0
- dw/tasks/task.py +920 -0
- dw/tasks/tensor_image.py +57 -0
- dw/tasks/text_generation.py +169 -0
- dw/tasks/text_sections.py +80 -0
- dw/tasks/upscale.py +203 -0
- dw/tasks/video_utils.py +624 -0
- dw/tasks/zoe_depth.py +71 -0
- dw/teacache.py +381 -0
- dw/teacache_models.json +99 -0
- dw/test.py +29 -0
- dw/type_helpers.py +231 -0
- dw/validate.py +68 -0
- dw/variable_constraints.py +444 -0
- dw/variables.py +443 -0
- dw/video_extensions.py +141 -0
- dw/vram_estimate.py +116 -0
- dw/worker.py +764 -0
- dw/workflow.py +2007 -0
- dw/workflow_schema.json +1346 -0
- dw/workflow_sources.py +383 -0
- dw/workflows/h3_context_ir.json +57 -0
- dw/workflows/test.json +31 -0
- dw/workspace.py +730 -0
- dw_mcp/__init__.py +6 -0
- dw_mcp/__main__.py +133 -0
- dw_mcp/assets.py +336 -0
- dw_mcp/authoring.py +114 -0
- dw_mcp/catalog.py +360 -0
- dw_mcp/client.py +486 -0
- dw_mcp/diagnose.py +371 -0
- dw_mcp/exports.py +84 -0
- dw_mcp/guides.py +35 -0
- dw_mcp/media.py +638 -0
- dw_mcp/models.py +97 -0
- dw_mcp/prompts.py +104 -0
- dw_mcp/server.py +1343 -0
- dw_mcp/workspaces.py +212 -0
dw/workspace.py
ADDED
|
@@ -0,0 +1,730 @@
|
|
|
1
|
+
"""The workspace: the directory holding what a user makes.
|
|
2
|
+
|
|
3
|
+
A workflow file, the assets it reads, the prompts it references and the files
|
|
4
|
+
it writes started life beside the code, which was right when the only entry
|
|
5
|
+
point was a CLI run from a checkout. It stops being right once an agent or a
|
|
6
|
+
remote client is the one authoring: generated media does not belong in the
|
|
7
|
+
source tree, and an agent's day-to-day workflows do not belong in the example
|
|
8
|
+
corpus this repository ships.
|
|
9
|
+
|
|
10
|
+
A workspace names one directory for all of it:
|
|
11
|
+
|
|
12
|
+
<workspace>/
|
|
13
|
+
workflows/ mine, writable
|
|
14
|
+
prompts/ the stored prompt library
|
|
15
|
+
assets/ input media
|
|
16
|
+
outputs/ generated files
|
|
17
|
+
|
|
18
|
+
Resolution order, first hit wins:
|
|
19
|
+
|
|
20
|
+
1. an explicit path (a --workspace flag)
|
|
21
|
+
2. the DW_WORKSPACE environment variable
|
|
22
|
+
3. 'workspace' in ~/.diffusers_helper/settings.json
|
|
23
|
+
4. the working directory, when it looks like a workspace - it holds any of
|
|
24
|
+
workflows/, prompts/ or outputs/
|
|
25
|
+
5. ~/diffusers-workspace
|
|
26
|
+
|
|
27
|
+
Rule 4 is what keeps a checkout working unchanged: the repository root holds
|
|
28
|
+
all three, so running from it resolves to it, and every default lands exactly
|
|
29
|
+
where it landed before there was a workspace at all. Only a working directory
|
|
30
|
+
with none of those markers falls through to the home workspace.
|
|
31
|
+
|
|
32
|
+
Nothing here creates a directory. Resolution is a pure question about paths -
|
|
33
|
+
an entry point that is about to write calls ensure() once it knows it needs to.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
import os
|
|
37
|
+
import time
|
|
38
|
+
from pathlib import Path
|
|
39
|
+
|
|
40
|
+
# Set by an entry point that resolved a workspace, so a spawned worker
|
|
41
|
+
# subprocess inherits the same answer - multiprocessing's 'spawn' start method
|
|
42
|
+
# launches a fresh interpreter that inherits os.environ, the way DW_PROMPT_DIR
|
|
43
|
+
# and DW_TRUST_WORKFLOWS already reach the worker
|
|
44
|
+
WORKSPACE_ENV_VAR = "DW_WORKSPACE"
|
|
45
|
+
|
|
46
|
+
# Carries alongside it how the workspace was chosen. Without it a workspace
|
|
47
|
+
# merely inferred from the working directory would come back to the worker
|
|
48
|
+
# looking like one the user named, and an inferred workspace deliberately
|
|
49
|
+
# yields to discovery that predates it - see Workspace.is_explicit
|
|
50
|
+
WORKSPACE_SOURCE_ENV_VAR = "DW_WORKSPACE_SOURCE"
|
|
51
|
+
|
|
52
|
+
# Deliberately not '~/diffusers-workflow': that is where this repository
|
|
53
|
+
# gets cloned, and a default that lands inside a checkout is the coupling
|
|
54
|
+
# workspaces exist to remove
|
|
55
|
+
DEFAULT_WORKSPACE = "~/diffusers-workspace"
|
|
56
|
+
|
|
57
|
+
WORKFLOWS_SUBDIR = "workflows"
|
|
58
|
+
PROMPTS_SUBDIR = "prompts"
|
|
59
|
+
ASSETS_SUBDIR = "assets"
|
|
60
|
+
OUTPUTS_SUBDIR = "outputs"
|
|
61
|
+
|
|
62
|
+
SUBDIRS = (WORKFLOWS_SUBDIR, PROMPTS_SUBDIR, ASSETS_SUBDIR, OUTPUTS_SUBDIR)
|
|
63
|
+
|
|
64
|
+
# The library shared by every workspace under one root:
|
|
65
|
+
# '<root>/common/assets'. The prompt library is shared because a 'prompt:'
|
|
66
|
+
# reference is shared by reference; assets are per workspace because an
|
|
67
|
+
# upload belongs to the work that made it - but a recurring cast is neither,
|
|
68
|
+
# and reaching it from a fresh workspace meant copying the files in
|
|
69
|
+
# (2026-09-11). A shared library sits on every workspace's asset search
|
|
70
|
+
# path, behind its own, so a workspace name still shadows a shared one and
|
|
71
|
+
# a write still lands in the workspace unless it says otherwise
|
|
72
|
+
COMMON_SUBDIR = "common"
|
|
73
|
+
|
|
74
|
+
# Where a job export lands: '<root>/exports/<job id>/'. Not a workspace
|
|
75
|
+
# folder - it is beside them, holding gathered copies rather than working
|
|
76
|
+
# content - but it is a name a workspace may not take, and workspace_names
|
|
77
|
+
# must not mistake it for one
|
|
78
|
+
EXPORTS_SUBDIR = "exports"
|
|
79
|
+
|
|
80
|
+
# What makes a directory recognizable as a workspace. assets/ is deliberately
|
|
81
|
+
# not a marker - a bare assets/ folder is a common thing to have lying around,
|
|
82
|
+
# where these three together say "content lives here"
|
|
83
|
+
MARKER_SUBDIRS = (WORKFLOWS_SUBDIR, PROMPTS_SUBDIR, OUTPUTS_SUBDIR)
|
|
84
|
+
|
|
85
|
+
# A server can hold several workspaces: the root's own workflows/assets/
|
|
86
|
+
# outputs are the default one, and a named workspace is a subdirectory of
|
|
87
|
+
# the root holding the same three. The prompt library is not among them -
|
|
88
|
+
# there is one, at the root, shared by every workspace, because a stored
|
|
89
|
+
# prompt is shared by reference and 'prompt:scenic' resolving to different
|
|
90
|
+
# text per workspace would break exactly that
|
|
91
|
+
DEFAULT_WORKSPACE_NAME = "default"
|
|
92
|
+
|
|
93
|
+
# Names a workspace cannot take, because the root's own folders already
|
|
94
|
+
# use them - its four content folders, and the exports gathered beside them
|
|
95
|
+
RESERVED_WORKSPACE_NAMES = SUBDIRS + (EXPORTS_SUBDIR, COMMON_SUBDIR)
|
|
96
|
+
|
|
97
|
+
# What a named workspace holds - prompts excluded, per above
|
|
98
|
+
NAMED_SUBDIRS = (WORKFLOWS_SUBDIR, ASSETS_SUBDIR, OUTPUTS_SUBDIR)
|
|
99
|
+
|
|
100
|
+
# How a workspace was chosen, in resolution order. Everything but the last two
|
|
101
|
+
# is an answer someone gave on purpose; see Workspace.is_explicit
|
|
102
|
+
FLAG = "flag"
|
|
103
|
+
ENVIRONMENT = "environment"
|
|
104
|
+
SETTINGS = "settings"
|
|
105
|
+
WORKING_DIRECTORY = "working directory"
|
|
106
|
+
DEFAULT = "default"
|
|
107
|
+
|
|
108
|
+
EXPLICIT_SOURCES = (FLAG, ENVIRONMENT, SETTINGS)
|
|
109
|
+
ALL_SOURCES = EXPLICIT_SOURCES + (WORKING_DIRECTORY, DEFAULT)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class Workspace:
|
|
113
|
+
"""A resolved workspace root and the four directories under it.
|
|
114
|
+
|
|
115
|
+
A named workspace is the same thing rooted one level down, with its
|
|
116
|
+
prompt library pointing back at the root's - `prompts_root` is what
|
|
117
|
+
carries that, and it is why a named workspace has three folders where
|
|
118
|
+
the default has four.
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
def __init__(
|
|
122
|
+
self,
|
|
123
|
+
root,
|
|
124
|
+
source,
|
|
125
|
+
name=DEFAULT_WORKSPACE_NAME,
|
|
126
|
+
prompts_root=None,
|
|
127
|
+
common_root=None,
|
|
128
|
+
):
|
|
129
|
+
self.root = os.path.abspath(os.path.expanduser(str(root)))
|
|
130
|
+
self.source = source
|
|
131
|
+
self.name = name
|
|
132
|
+
self._prompts_root = (
|
|
133
|
+
os.path.abspath(os.path.expanduser(str(prompts_root)))
|
|
134
|
+
if prompts_root
|
|
135
|
+
else None
|
|
136
|
+
)
|
|
137
|
+
# Like the prompt library, the shared one belongs to the root rather
|
|
138
|
+
# than to this workspace - a named workspace is handed the root's
|
|
139
|
+
self._common_root = (
|
|
140
|
+
os.path.abspath(os.path.expanduser(str(common_root)))
|
|
141
|
+
if common_root
|
|
142
|
+
else None
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
@property
|
|
146
|
+
def is_default(self):
|
|
147
|
+
"""Whether this is the root's own workspace rather than a named one."""
|
|
148
|
+
return self._prompts_root is None
|
|
149
|
+
|
|
150
|
+
@property
|
|
151
|
+
def is_explicit(self):
|
|
152
|
+
"""Whether someone named this workspace, rather than it being
|
|
153
|
+
inferred from the working directory or fallen back to.
|
|
154
|
+
|
|
155
|
+
Discovery that predates workspaces - the prompt library's walk up
|
|
156
|
+
from the workflow file - stays ahead of an inferred workspace and
|
|
157
|
+
behind a named one, so turning this on changes nothing for a caller
|
|
158
|
+
who has not asked for a workspace.
|
|
159
|
+
"""
|
|
160
|
+
return self.source in EXPLICIT_SOURCES
|
|
161
|
+
|
|
162
|
+
@property
|
|
163
|
+
def workflows(self):
|
|
164
|
+
return os.path.join(self.root, WORKFLOWS_SUBDIR)
|
|
165
|
+
|
|
166
|
+
@property
|
|
167
|
+
def prompts(self):
|
|
168
|
+
"""The shared library: a named workspace points back at the root's."""
|
|
169
|
+
return self._prompts_root or os.path.join(self.root, PROMPTS_SUBDIR)
|
|
170
|
+
|
|
171
|
+
@property
|
|
172
|
+
def assets(self):
|
|
173
|
+
return os.path.join(self.root, ASSETS_SUBDIR)
|
|
174
|
+
|
|
175
|
+
@property
|
|
176
|
+
def outputs(self):
|
|
177
|
+
return os.path.join(self.root, OUTPUTS_SUBDIR)
|
|
178
|
+
|
|
179
|
+
@property
|
|
180
|
+
def common_assets(self):
|
|
181
|
+
"""The asset library every workspace under this root shares."""
|
|
182
|
+
root = self._common_root or os.path.join(self.root, COMMON_SUBDIR)
|
|
183
|
+
return os.path.join(root, ASSETS_SUBDIR)
|
|
184
|
+
|
|
185
|
+
def ensure(self):
|
|
186
|
+
"""Create the workspace and its subdirectories if they are missing.
|
|
187
|
+
|
|
188
|
+
Called by an entry point that is about to write, not by resolution -
|
|
189
|
+
asking where the workspace is should never leave a directory behind.
|
|
190
|
+
"""
|
|
191
|
+
for subdir in NAMED_SUBDIRS if self._prompts_root else SUBDIRS:
|
|
192
|
+
Path(self.root, subdir).mkdir(parents=True, exist_ok=True)
|
|
193
|
+
if self._prompts_root:
|
|
194
|
+
Path(self._prompts_root).mkdir(parents=True, exist_ok=True)
|
|
195
|
+
return self
|
|
196
|
+
|
|
197
|
+
def describe(self):
|
|
198
|
+
"""What a client needs to name this workspace and its folders."""
|
|
199
|
+
return {
|
|
200
|
+
"name": self.name,
|
|
201
|
+
"default": self.is_default,
|
|
202
|
+
"root": self.root,
|
|
203
|
+
"workflows": self.workflows,
|
|
204
|
+
"assets": self.assets,
|
|
205
|
+
"outputs": self.outputs,
|
|
206
|
+
"prompts": self.prompts,
|
|
207
|
+
"common_assets": self.common_assets,
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
def __repr__(self):
|
|
211
|
+
return f"Workspace({self.root!r}, {self.name}, from {self.source})"
|
|
212
|
+
|
|
213
|
+
def __eq__(self, other):
|
|
214
|
+
return (
|
|
215
|
+
isinstance(other, Workspace)
|
|
216
|
+
and self.root == other.root
|
|
217
|
+
and self.source == other.source
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _has_subdirs(path, names, all_of):
|
|
222
|
+
"""Whether a directory holds some (any_of) or all (all_of) of these
|
|
223
|
+
subfolder names - the shared test behind looks_like_workspace's "any
|
|
224
|
+
marker is enough" and _holds_a_workspace's stricter "all three, or it
|
|
225
|
+
isn't a workspace"."""
|
|
226
|
+
test = all if all_of else any
|
|
227
|
+
return test(os.path.isdir(os.path.join(path, name)) for name in names)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def looks_like_workspace(path):
|
|
231
|
+
"""Whether a directory holds the subfolders that mark a workspace."""
|
|
232
|
+
return _has_subdirs(path, MARKER_SUBDIRS, all_of=False)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def resolve_workspace(explicit=None):
|
|
236
|
+
"""The workspace this process works in.
|
|
237
|
+
|
|
238
|
+
Args:
|
|
239
|
+
explicit: A path from a --workspace flag, when one was given
|
|
240
|
+
|
|
241
|
+
Returns:
|
|
242
|
+
A Workspace, which may not exist on disk yet
|
|
243
|
+
"""
|
|
244
|
+
if explicit:
|
|
245
|
+
return Workspace(explicit, FLAG)
|
|
246
|
+
|
|
247
|
+
from_environment = os.environ.get(WORKSPACE_ENV_VAR)
|
|
248
|
+
if from_environment:
|
|
249
|
+
source = os.environ.get(WORKSPACE_SOURCE_ENV_VAR)
|
|
250
|
+
return Workspace(
|
|
251
|
+
from_environment, source if source in ALL_SOURCES else ENVIRONMENT
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
# Imported here, not at module scope: dw.settings reads a file, and
|
|
255
|
+
# resolution is called from argument parsing on every entry point
|
|
256
|
+
from .settings import load_settings
|
|
257
|
+
|
|
258
|
+
from_settings = load_settings().workspace
|
|
259
|
+
if from_settings:
|
|
260
|
+
return Workspace(from_settings, SETTINGS)
|
|
261
|
+
|
|
262
|
+
working_directory = os.path.abspath(os.getcwd())
|
|
263
|
+
if looks_like_workspace(working_directory):
|
|
264
|
+
return Workspace(working_directory, WORKING_DIRECTORY)
|
|
265
|
+
|
|
266
|
+
return Workspace(DEFAULT_WORKSPACE, DEFAULT)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def set_workspace(workspace):
|
|
270
|
+
"""Pin a resolved workspace in the environment, so a spawned worker
|
|
271
|
+
subprocess and anything resolving later in this process agree with the
|
|
272
|
+
entry point that chose it.
|
|
273
|
+
|
|
274
|
+
Args:
|
|
275
|
+
workspace: The Workspace to pin, or a path
|
|
276
|
+
|
|
277
|
+
Returns:
|
|
278
|
+
The Workspace that was pinned
|
|
279
|
+
"""
|
|
280
|
+
if not isinstance(workspace, Workspace):
|
|
281
|
+
workspace = Workspace(workspace, FLAG)
|
|
282
|
+
os.environ[WORKSPACE_ENV_VAR] = workspace.root
|
|
283
|
+
os.environ[WORKSPACE_SOURCE_ENV_VAR] = workspace.source
|
|
284
|
+
return workspace
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
# Read-only libraries searched after the one a run writes to. A tree named
|
|
288
|
+
# by --examples-dir brings the prompts and assets its workflows reference
|
|
289
|
+
# along with it, and neither is reachable from the workspace's own library -
|
|
290
|
+
# the search path is what closes that. Carried in the environment, joined by
|
|
291
|
+
# os.pathsep, so a spawned worker inherits it the way it inherits
|
|
292
|
+
# DW_PROMPT_DIR and DW_ASSET_DIR
|
|
293
|
+
PROMPT_PATH_ENV_VAR = "DW_PROMPT_PATH"
|
|
294
|
+
ASSET_PATH_ENV_VAR = "DW_ASSET_PATH"
|
|
295
|
+
# The same idea for workflows, which a sub-workflow step names: a stored
|
|
296
|
+
# template lives in an examples tree the workspace's own workflows/ cannot
|
|
297
|
+
# reach, so composing one used to mean copying it in (#90)
|
|
298
|
+
WORKFLOW_PATH_ENV_VAR = "DW_WORKFLOW_PATH"
|
|
299
|
+
|
|
300
|
+
LIBRARY_PATH_ENV_VARS = {
|
|
301
|
+
PROMPTS_SUBDIR: PROMPT_PATH_ENV_VAR,
|
|
302
|
+
ASSETS_SUBDIR: ASSET_PATH_ENV_VAR,
|
|
303
|
+
WORKFLOWS_SUBDIR: WORKFLOW_PATH_ENV_VAR,
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def library_fallbacks(subdir, primary=None):
|
|
308
|
+
"""The read-only roots a library is searched in after its own, in order.
|
|
309
|
+
|
|
310
|
+
Args:
|
|
311
|
+
subdir: 'prompts' or 'assets'
|
|
312
|
+
primary: The library that is searched first, dropped from the result
|
|
313
|
+
when it also appears here - a checkout serving as both the
|
|
314
|
+
workspace and the examples tree has one library, not two
|
|
315
|
+
|
|
316
|
+
Returns:
|
|
317
|
+
A list of absolute paths, each an existing directory
|
|
318
|
+
"""
|
|
319
|
+
raw = os.environ.get(LIBRARY_PATH_ENV_VARS[subdir], "")
|
|
320
|
+
first = os.path.abspath(os.path.expanduser(str(primary))) if primary else None
|
|
321
|
+
roots = []
|
|
322
|
+
for entry in raw.split(os.pathsep):
|
|
323
|
+
if not entry.strip():
|
|
324
|
+
continue
|
|
325
|
+
root = os.path.abspath(os.path.expanduser(entry))
|
|
326
|
+
if root == first or root in roots or not os.path.isdir(root):
|
|
327
|
+
continue
|
|
328
|
+
roots.append(root)
|
|
329
|
+
return roots
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def set_library_fallbacks(subdir, roots):
|
|
333
|
+
"""Pin a library's read-only roots in the environment, so the worker
|
|
334
|
+
subprocess resolves a reference exactly as the entry point would."""
|
|
335
|
+
joined = os.pathsep.join(
|
|
336
|
+
os.path.abspath(os.path.expanduser(str(root))) for root in roots or []
|
|
337
|
+
)
|
|
338
|
+
if joined:
|
|
339
|
+
os.environ[LIBRARY_PATH_ENV_VARS[subdir]] = joined
|
|
340
|
+
else:
|
|
341
|
+
os.environ.pop(LIBRARY_PATH_ENV_VARS[subdir], None)
|
|
342
|
+
return joined
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def example_libraries(examples_dirs):
|
|
346
|
+
"""The prompt and asset libraries the trees named by --examples-dir
|
|
347
|
+
bring with them.
|
|
348
|
+
|
|
349
|
+
A workflows/ tree inside a checkout keeps the prompts and assets its
|
|
350
|
+
workflows reference beside it, so the sibling of each examples directory
|
|
351
|
+
is where they are; a directory that holds them itself - someone pointing
|
|
352
|
+
--examples-dir at a whole workspace - is checked first, and either can
|
|
353
|
+
be missing.
|
|
354
|
+
|
|
355
|
+
Args:
|
|
356
|
+
examples_dirs: The directories --examples-dir named, in order
|
|
357
|
+
|
|
358
|
+
Returns:
|
|
359
|
+
{'prompts': [roots], 'assets': [roots]}, deduplicated, in the order
|
|
360
|
+
the examples directories were given
|
|
361
|
+
"""
|
|
362
|
+
found = {PROMPTS_SUBDIR: [], ASSETS_SUBDIR: []}
|
|
363
|
+
for directory in examples_dirs or []:
|
|
364
|
+
root = os.path.abspath(os.path.expanduser(str(directory)))
|
|
365
|
+
for holder in (root, os.path.dirname(root)):
|
|
366
|
+
for subdir in found:
|
|
367
|
+
candidate = os.path.join(holder, subdir)
|
|
368
|
+
if os.path.isdir(candidate) and candidate not in found[subdir]:
|
|
369
|
+
found[subdir].append(candidate)
|
|
370
|
+
return found
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def discover_library(subdir, env_var, base_dir=None):
|
|
374
|
+
"""Where a shared library (prompts, assets) is rooted, by the precedence
|
|
375
|
+
get_prompt_dir and get_asset_dir both need: an environment variable names
|
|
376
|
+
it outright; then a workspace someone named (a --workspace flag,
|
|
377
|
+
DW_WORKSPACE, or the 'workspace' setting), whose <subdir>/ is the library
|
|
378
|
+
by definition; then <subdir>/ in the working directory when that exists;
|
|
379
|
+
then the walk from base_dir up toward the filesystem root, looking for
|
|
380
|
+
the <subdir>/ folder of whatever tree base_dir lives in; and finally the
|
|
381
|
+
workspace's <subdir>/ as the fallback.
|
|
382
|
+
|
|
383
|
+
The middle two steps predate workspaces and stay below the named-workspace
|
|
384
|
+
check on purpose - a workspace merely inferred from the working directory
|
|
385
|
+
or fallen back to must not preempt a library a workflow already reaches.
|
|
386
|
+
|
|
387
|
+
Args:
|
|
388
|
+
subdir: The library's folder name under a workspace ('prompts',
|
|
389
|
+
'assets')
|
|
390
|
+
env_var: The environment variable that names it explicitly
|
|
391
|
+
base_dir: The workflow file's directory, when one anchors the search
|
|
392
|
+
"""
|
|
393
|
+
explicit = os.environ.get(env_var)
|
|
394
|
+
if explicit:
|
|
395
|
+
return explicit
|
|
396
|
+
|
|
397
|
+
workspace = resolve_workspace()
|
|
398
|
+
if workspace.is_explicit:
|
|
399
|
+
return getattr(workspace, subdir)
|
|
400
|
+
|
|
401
|
+
working_directory_library = os.path.abspath(f"./{subdir}")
|
|
402
|
+
if os.path.isdir(working_directory_library):
|
|
403
|
+
return working_directory_library
|
|
404
|
+
if base_dir:
|
|
405
|
+
current = os.path.abspath(base_dir)
|
|
406
|
+
while True:
|
|
407
|
+
candidate = os.path.join(current, subdir)
|
|
408
|
+
if os.path.isdir(candidate):
|
|
409
|
+
return candidate
|
|
410
|
+
parent = os.path.dirname(current)
|
|
411
|
+
if parent == current:
|
|
412
|
+
break
|
|
413
|
+
current = parent
|
|
414
|
+
return getattr(workspace, subdir)
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _holds_a_workspace(path):
|
|
418
|
+
"""Whether a directory is a named workspace rather than some other
|
|
419
|
+
folder someone left at the root.
|
|
420
|
+
|
|
421
|
+
All three folders, not any one: create_workspace makes all three, and a
|
|
422
|
+
looser test claims too much - a checkout used as the workspace root has
|
|
423
|
+
dw/workflows/ (the packaged builtins) right there, and one folder would
|
|
424
|
+
make the source package a workspace, listed, browsable and deletable.
|
|
425
|
+
"""
|
|
426
|
+
return _has_subdirs(path, NAMED_SUBDIRS, all_of=True)
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _foreign_entries(path):
|
|
430
|
+
"""What a directory holds besides a workspace's own three folders and
|
|
431
|
+
the exports it may have gathered."""
|
|
432
|
+
ignored = NAMED_SUBDIRS + (EXPORTS_SUBDIR,)
|
|
433
|
+
try:
|
|
434
|
+
return sorted(entry for entry in os.listdir(path) if entry not in ignored)
|
|
435
|
+
except OSError:
|
|
436
|
+
return []
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def workspace_names(workspace):
|
|
440
|
+
"""Every workspace under this root, the default first.
|
|
441
|
+
|
|
442
|
+
The default is the root's own folders, so it is always present even on a
|
|
443
|
+
root that holds nothing else - there is always somewhere to work.
|
|
444
|
+
"""
|
|
445
|
+
names = []
|
|
446
|
+
try:
|
|
447
|
+
for entry in sorted(os.listdir(workspace.root)):
|
|
448
|
+
if entry in RESERVED_WORKSPACE_NAMES or entry.startswith("."):
|
|
449
|
+
continue
|
|
450
|
+
path = os.path.join(workspace.root, entry)
|
|
451
|
+
if os.path.isdir(path) and _holds_a_workspace(path):
|
|
452
|
+
names.append(entry)
|
|
453
|
+
except OSError:
|
|
454
|
+
names = []
|
|
455
|
+
return [DEFAULT_WORKSPACE_NAME] + names
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def named_workspace(workspace, name):
|
|
459
|
+
"""One workspace under this root, by name.
|
|
460
|
+
|
|
461
|
+
The default name resolves to the root's own workspace; any other name
|
|
462
|
+
resolves to '<root>/<name>', sharing the root's prompt library. The name
|
|
463
|
+
is validated before it is joined, so nothing here can leave the root.
|
|
464
|
+
"""
|
|
465
|
+
from .security import validate_workspace_name
|
|
466
|
+
|
|
467
|
+
if name is None or name == DEFAULT_WORKSPACE_NAME:
|
|
468
|
+
return workspace
|
|
469
|
+
name = validate_workspace_name(name)
|
|
470
|
+
return Workspace(
|
|
471
|
+
os.path.join(workspace.root, name),
|
|
472
|
+
workspace.source,
|
|
473
|
+
name=name,
|
|
474
|
+
prompts_root=os.path.join(workspace.root, PROMPTS_SUBDIR),
|
|
475
|
+
common_root=os.path.join(workspace.root, COMMON_SUBDIR),
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def create_workspace(workspace, name):
|
|
480
|
+
"""Make a new named workspace under this root.
|
|
481
|
+
|
|
482
|
+
Raises:
|
|
483
|
+
InvalidInputError: If the name is not one a workspace can take
|
|
484
|
+
FileExistsError: If a workspace of that name is already there
|
|
485
|
+
"""
|
|
486
|
+
from .security import validate_workspace_name
|
|
487
|
+
|
|
488
|
+
name = validate_workspace_name(name)
|
|
489
|
+
if name in workspace_names(workspace):
|
|
490
|
+
raise FileExistsError(f"Workspace '{name}' already exists")
|
|
491
|
+
return named_workspace(workspace, name).ensure()
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def _tree_usage(directory):
|
|
495
|
+
"""Files and bytes under one directory, as (files, bytes).
|
|
496
|
+
|
|
497
|
+
Walks with scandir and stats through the DirEntry, which reuses the stat
|
|
498
|
+
the directory read already did - the difference matters on an outputs
|
|
499
|
+
tree of thousands of generated files. Symlinks are counted as neither
|
|
500
|
+
file nor directory, so a link into a model cache cannot inflate the
|
|
501
|
+
number or send the walk outside the workspace. Anything unreadable is
|
|
502
|
+
skipped: this is a size to glance at, not an audit.
|
|
503
|
+
"""
|
|
504
|
+
files = 0
|
|
505
|
+
total = 0
|
|
506
|
+
stack = [directory]
|
|
507
|
+
while stack:
|
|
508
|
+
current = stack.pop()
|
|
509
|
+
try:
|
|
510
|
+
entries = list(os.scandir(current))
|
|
511
|
+
except OSError:
|
|
512
|
+
continue
|
|
513
|
+
for entry in entries:
|
|
514
|
+
try:
|
|
515
|
+
if entry.is_dir(follow_symlinks=False):
|
|
516
|
+
stack.append(entry.path)
|
|
517
|
+
elif entry.is_file(follow_symlinks=False):
|
|
518
|
+
total += entry.stat(follow_symlinks=False).st_size
|
|
519
|
+
files += 1
|
|
520
|
+
except OSError:
|
|
521
|
+
continue
|
|
522
|
+
return files, total
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def workspace_contents(workspace):
|
|
526
|
+
"""How much a workspace holds, for a client about to offer to delete it:
|
|
527
|
+
file counts and total bytes per folder. Counting is the point - the
|
|
528
|
+
number is what makes 'delete this workspace' an informed choice."""
|
|
529
|
+
summary = {}
|
|
530
|
+
for folder in NAMED_SUBDIRS:
|
|
531
|
+
files, total = _tree_usage(os.path.join(workspace.root, folder))
|
|
532
|
+
summary[folder] = {"files": files, "bytes": total}
|
|
533
|
+
return summary
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
# How long a computed workspace size stays good enough to answer with. The
|
|
537
|
+
# number is a glance at how much a workspace holds, not an accounting
|
|
538
|
+
# figure, and re-walking a large outputs tree for every listing would cost
|
|
539
|
+
# more than the precision is worth - a running job moves it continuously
|
|
540
|
+
# anyway
|
|
541
|
+
USAGE_CACHE_SECONDS = 60
|
|
542
|
+
|
|
543
|
+
# the folders counted (a tuple of paths) -> (monotonic time, usage)
|
|
544
|
+
_usage_cache = {}
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def _own_folders(workspace):
|
|
548
|
+
"""The folders whose bytes count as this workspace's own.
|
|
549
|
+
|
|
550
|
+
A workspace's four folder properties are the candidates, deduplicated,
|
|
551
|
+
minus any that is not actually inside its root: a named workspace's
|
|
552
|
+
prompt library belongs to the root and is shared by every workspace, so
|
|
553
|
+
counting it here would count it once per workspace. A workspace with no
|
|
554
|
+
root of its own (a server configured folder by folder, where each can
|
|
555
|
+
point anywhere) has nothing to exclude, and every folder it names is its
|
|
556
|
+
own by definition.
|
|
557
|
+
"""
|
|
558
|
+
folders = []
|
|
559
|
+
for path in (
|
|
560
|
+
workspace.workflows,
|
|
561
|
+
workspace.prompts,
|
|
562
|
+
workspace.assets,
|
|
563
|
+
workspace.outputs,
|
|
564
|
+
):
|
|
565
|
+
if not path or path in folders:
|
|
566
|
+
continue
|
|
567
|
+
if workspace.root and not _is_within(path, workspace.root):
|
|
568
|
+
continue
|
|
569
|
+
folders.append(path)
|
|
570
|
+
return folders
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def _is_within(path, root):
|
|
574
|
+
"""Whether a path is the root or sits under it."""
|
|
575
|
+
try:
|
|
576
|
+
return os.path.commonpath([path, root]) == root
|
|
577
|
+
except ValueError: # different drives on Windows
|
|
578
|
+
return False
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def workspace_usage(workspace, max_age=USAGE_CACHE_SECONDS):
|
|
582
|
+
"""Roughly how much disk a workspace occupies: files and total bytes.
|
|
583
|
+
|
|
584
|
+
Only the folders that are the workspace's own are counted, per
|
|
585
|
+
_own_folders - which is what keeps the shared prompt library from being
|
|
586
|
+
added to every workspace's total, and named workspaces from being
|
|
587
|
+
counted inside the default one (they sit beside its folders, not in
|
|
588
|
+
them).
|
|
589
|
+
"""
|
|
590
|
+
folders = _own_folders(workspace)
|
|
591
|
+
key = tuple(folders)
|
|
592
|
+
now = time.monotonic()
|
|
593
|
+
cached = _usage_cache.get(key)
|
|
594
|
+
if cached and now - cached[0] < max_age:
|
|
595
|
+
return cached[1]
|
|
596
|
+
files = 0
|
|
597
|
+
total = 0
|
|
598
|
+
for directory in folders:
|
|
599
|
+
if not os.path.isdir(directory):
|
|
600
|
+
continue
|
|
601
|
+
count, size = _tree_usage(directory)
|
|
602
|
+
files += count
|
|
603
|
+
total += size
|
|
604
|
+
usage = {"files": files, "bytes": total}
|
|
605
|
+
_usage_cache[key] = (now, usage)
|
|
606
|
+
return usage
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
def forget_workspace_usage():
|
|
610
|
+
"""Drop every cached size - after creating or deleting a workspace, or
|
|
611
|
+
after any deliberate delete of a workflow/prompt/asset/output, where a
|
|
612
|
+
stale answer would be visibly wrong rather than merely old (#177). A
|
|
613
|
+
job's own writes are not swept here - that churn is exactly what the TTL
|
|
614
|
+
is for."""
|
|
615
|
+
_usage_cache.clear()
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
def delete_workspace(workspace, name):
|
|
619
|
+
"""Remove a named workspace and everything in it.
|
|
620
|
+
|
|
621
|
+
The default workspace is the root itself and is never deletable - it
|
|
622
|
+
holds the shared prompt library, and there has to be somewhere to work.
|
|
623
|
+
|
|
624
|
+
A directory that holds anything besides the three workspace folders is
|
|
625
|
+
refused too: rmtree is unrecoverable, and a .git, an __init__.py or a
|
|
626
|
+
stray file says this is something that merely contains a workspace, not
|
|
627
|
+
one that is only a workspace.
|
|
628
|
+
|
|
629
|
+
Raises:
|
|
630
|
+
ValueError: For the default workspace
|
|
631
|
+
FileNotFoundError: If no workspace has that name
|
|
632
|
+
NotAWorkspaceError: If the directory holds more than a workspace
|
|
633
|
+
"""
|
|
634
|
+
import shutil
|
|
635
|
+
|
|
636
|
+
if name is None or name == DEFAULT_WORKSPACE_NAME:
|
|
637
|
+
raise ValueError("The default workspace cannot be deleted")
|
|
638
|
+
if name not in workspace_names(workspace):
|
|
639
|
+
raise FileNotFoundError(f"No such workspace: {name}")
|
|
640
|
+
target = named_workspace(workspace, name)
|
|
641
|
+
foreign = _foreign_entries(target.root)
|
|
642
|
+
if foreign:
|
|
643
|
+
raise NotAWorkspaceError(name, foreign)
|
|
644
|
+
shutil.rmtree(target.root)
|
|
645
|
+
return target
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
class ConfiguredWorkspace(Workspace):
|
|
649
|
+
"""The default workspace on a server started with individual directory
|
|
650
|
+
overrides (`--workflow-dir`, `--output-dir`, etc.), rather than a single
|
|
651
|
+
`--workspace` root.
|
|
652
|
+
|
|
653
|
+
Those overrides mean the default workspace's four folders are not
|
|
654
|
+
reliably `<root>/workflows` and friends - each can point anywhere - so
|
|
655
|
+
they cannot be derived the way Workspace derives them from a root. This
|
|
656
|
+
subclass instead takes the four directories directly and answers the
|
|
657
|
+
same four properties from them, `root` carrying whatever the caller has
|
|
658
|
+
(a workspace root when there is one, otherwise whatever describes the
|
|
659
|
+
configuration - possibly None) for `describe()` and logging only; it
|
|
660
|
+
plays no part in resolving the four folders below.
|
|
661
|
+
"""
|
|
662
|
+
|
|
663
|
+
def __init__(self, workflows, assets, outputs, prompts, root=None):
|
|
664
|
+
self.root = os.path.abspath(os.path.expanduser(str(root))) if root else None
|
|
665
|
+
self.source = DEFAULT
|
|
666
|
+
self.name = DEFAULT_WORKSPACE_NAME
|
|
667
|
+
self._workflows = os.path.abspath(os.path.expanduser(str(workflows)))
|
|
668
|
+
# assets is optional - a server configured with no asset library at
|
|
669
|
+
# all, same as Workspace's own callers see via app.state.asset_dir
|
|
670
|
+
self._assets = (
|
|
671
|
+
os.path.abspath(os.path.expanduser(str(assets))) if assets else None
|
|
672
|
+
)
|
|
673
|
+
self._outputs = os.path.abspath(os.path.expanduser(str(outputs)))
|
|
674
|
+
# prompts is optional too - a server configured with no prompt
|
|
675
|
+
# library at all (app.state.prompt_dir can be None)
|
|
676
|
+
self._prompts = (
|
|
677
|
+
os.path.abspath(os.path.expanduser(str(prompts))) if prompts else None
|
|
678
|
+
)
|
|
679
|
+
|
|
680
|
+
@property
|
|
681
|
+
def is_default(self):
|
|
682
|
+
return True
|
|
683
|
+
|
|
684
|
+
@property
|
|
685
|
+
def workflows(self):
|
|
686
|
+
return self._workflows
|
|
687
|
+
|
|
688
|
+
@property
|
|
689
|
+
def assets(self):
|
|
690
|
+
return self._assets
|
|
691
|
+
|
|
692
|
+
@property
|
|
693
|
+
def outputs(self):
|
|
694
|
+
return self._outputs
|
|
695
|
+
|
|
696
|
+
@property
|
|
697
|
+
def prompts(self):
|
|
698
|
+
return self._prompts
|
|
699
|
+
|
|
700
|
+
@property
|
|
701
|
+
def common_assets(self):
|
|
702
|
+
"""The shared library, when there is a root to hang it off. A server
|
|
703
|
+
configured from three loose directories has no root and so no shared
|
|
704
|
+
library - there is nothing for it to be common to."""
|
|
705
|
+
return (
|
|
706
|
+
os.path.join(self.root, COMMON_SUBDIR, ASSETS_SUBDIR) if self.root else None
|
|
707
|
+
)
|
|
708
|
+
|
|
709
|
+
def ensure(self):
|
|
710
|
+
"""Not implemented here: a ConfiguredWorkspace's folders were each
|
|
711
|
+
already resolved by the entry point that configured them, and each
|
|
712
|
+
has its own idea of who creates it (the CLI's own ensure() calls,
|
|
713
|
+
JobManager, etc.) - there is no single 'the workspace' to create."""
|
|
714
|
+
raise NotImplementedError(
|
|
715
|
+
"ConfiguredWorkspace folders are created by whatever configured "
|
|
716
|
+
"them, not by the workspace itself"
|
|
717
|
+
)
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
class NotAWorkspaceError(ValueError):
|
|
721
|
+
"""A directory named as a workspace holds more than a workspace does."""
|
|
722
|
+
|
|
723
|
+
def __init__(self, name, entries):
|
|
724
|
+
self.name = name
|
|
725
|
+
self.entries = entries
|
|
726
|
+
super().__init__(
|
|
727
|
+
f"'{name}' holds more than a workspace's own folders "
|
|
728
|
+
f"({', '.join(entries)}) and will not be deleted - remove those "
|
|
729
|
+
f"by hand if it really is a workspace"
|
|
730
|
+
)
|