diffusers-workflow 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusers_workflow-0.4.0.dist-info/METADATA +318 -0
- diffusers_workflow-0.4.0.dist-info/RECORD +260 -0
- diffusers_workflow-0.4.0.dist-info/WHEEL +5 -0
- diffusers_workflow-0.4.0.dist-info/entry_points.txt +7 -0
- diffusers_workflow-0.4.0.dist-info/licenses/LICENSE +201 -0
- diffusers_workflow-0.4.0.dist-info/top_level.txt +2 -0
- dw/__init__.py +440 -0
- dw/adapter_compatibility.py +226 -0
- dw/arguments.py +1231 -0
- dw/assessment_rules.py +159 -0
- dw/assets.py +130 -0
- dw/cache_blocks.json +16 -0
- dw/cache_blocks.py +146 -0
- dw/community_pipelines/pipeline_flux_rf_inversion.py +1184 -0
- dw/content_types.py +150 -0
- dw/dissolve_frame_errors.py +121 -0
- dw/docs/ACCELERATION.md +352 -0
- dw/docs/AGENT_LOOP.md +95 -0
- dw/docs/DEPENDENCIES.md +91 -0
- dw/docs/IP_ADAPTER.md +109 -0
- dw/docs/LORAS.md +131 -0
- dw/docs/MCP.md +517 -0
- dw/docs/PROMPT_WEIGHTING.md +78 -0
- dw/docs/QUANTIZATION.md +230 -0
- dw/docs/RECIPES_24GB.md +201 -0
- dw/docs/RELEASING.md +195 -0
- dw/docs/REMOTE.md +140 -0
- dw/docs/REPL_COMMANDS.md +121 -0
- dw/docs/REPL_WORKER_GUIDE.md +51 -0
- dw/docs/SECURITY.md +272 -0
- dw/docs/SECURITY_QUICKREF.md +112 -0
- dw/docs/SERVER.md +679 -0
- dw/docs/TASKS.md +1741 -0
- dw/docs/TESTING.md +71 -0
- dw/docs/WORKFLOW_GUIDE.md +2038 -0
- dw/docs/WORKSPACES.md +316 -0
- dw/download_watch.py +335 -0
- dw/elision.py +306 -0
- dw/events.py +275 -0
- dw/for_each.py +409 -0
- dw/host_memory.py +258 -0
- dw/host_memory_projection.py +230 -0
- dw/hub_cache.py +432 -0
- dw/introspection.py +1228 -0
- dw/kernel_availability.py +208 -0
- dw/locations.py +599 -0
- dw/log_setup.py +45 -0
- dw/loudness.py +82 -0
- dw/media_audio.py +217 -0
- dw/media_frames.py +367 -0
- dw/media_info.py +297 -0
- dw/pipeline_processors/chain.py +821 -0
- dw/pipeline_processors/config_objects.py +237 -0
- dw/pipeline_processors/pipeline.py +2297 -0
- dw/pipeline_processors/remote.py +46 -0
- dw/plan.py +920 -0
- dw/previous_results.py +411 -0
- dw/probe_paths.py +59 -0
- dw/prompt_schema.json +48 -0
- dw/prompt_weighting.py +378 -0
- dw/prompts.py +159 -0
- dw/realize.py +250 -0
- dw/reference_limits.py +215 -0
- dw/reference_names.py +125 -0
- dw/repl.py +338 -0
- dw/repl_commands.py +836 -0
- dw/repl_worker.py +159 -0
- dw/result.py +1720 -0
- dw/result_fps.py +82 -0
- dw/run.py +162 -0
- dw/runs.py +768 -0
- dw/scalar_result_validation.py +97 -0
- dw/schema.py +283 -0
- dw/security.py +1038 -0
- dw/select_validation.py +115 -0
- dw/serve.py +277 -0
- dw/server/__init__.py +2 -0
- dw/server/app.py +4586 -0
- dw/server/assess.py +132 -0
- dw/server/catalog_shape.py +487 -0
- dw/server/enhancers.py +129 -0
- dw/server/exports.py +480 -0
- dw/server/guides.py +257 -0
- dw/server/jobs.py +1561 -0
- dw/server/mcp_mount.py +95 -0
- dw/server/netinfo.py +124 -0
- dw/server/observed_cost.py +379 -0
- dw/server/sysinfo.py +71 -0
- dw/server/ui/assets/abap-08VXUWAP.js +1 -0
- dw/server/ui/assets/apex-BWPQTe0t.js +1 -0
- dw/server/ui/assets/azcli-Bc_sGQ0U.js +1 -0
- dw/server/ui/assets/bat-i0X4ZdIN.js +1 -0
- dw/server/ui/assets/bicep-B5-_aFwp.js +2 -0
- dw/server/ui/assets/cameligo-DMUM7wLl.js +1 -0
- dw/server/ui/assets/clojure-Cm7r79vr.js +1 -0
- dw/server/ui/assets/codicon-Brq4_Ui5.ttf +0 -0
- dw/server/ui/assets/coffee-Ba7i2nA0.js +1 -0
- dw/server/ui/assets/cpp-C7h46wYY.js +1 -0
- dw/server/ui/assets/csharp-BKxtCVv1.js +1 -0
- dw/server/ui/assets/csp-bTuwJoIa.js +1 -0
- dw/server/ui/assets/css-DIMkf-bt.js +3 -0
- dw/server/ui/assets/css.worker-B3ciXF_0.js +93 -0
- dw/server/ui/assets/cssMode-CPznxfY8.js +1 -0
- dw/server/ui/assets/cypher-CVaqCwHa.js +1 -0
- dw/server/ui/assets/dart-onAF5SnQ.js +1 -0
- dw/server/ui/assets/dockerfile-DZFCIeNp.js +1 -0
- dw/server/ui/assets/ecl-D05T4iGw.js +1 -0
- dw/server/ui/assets/editor-jjEx9u7D.css +1 -0
- dw/server/ui/assets/editor.api-CpWcotrd.js +847 -0
- dw/server/ui/assets/editor.worker-q-txB4vs.js +30 -0
- dw/server/ui/assets/elixir-6RTg0lbw.js +1 -0
- dw/server/ui/assets/flow9-C5_-GSwl.js +1 -0
- dw/server/ui/assets/freemarker2-CXtRM8N4.js +3 -0
- dw/server/ui/assets/fsharp-C8Ef5oNN.js +1 -0
- dw/server/ui/assets/go-C-y9NEjX.js +1 -0
- dw/server/ui/assets/graphql-fmXr3nnJ.js +1 -0
- dw/server/ui/assets/handlebars-N7x-6NMY.js +1 -0
- dw/server/ui/assets/hcl-CpzslTdj.js +1 -0
- dw/server/ui/assets/html-PhsdjHSr.js +1 -0
- dw/server/ui/assets/html.worker-C93Ht9o9.js +506 -0
- dw/server/ui/assets/htmlMode-Dgj0SEok.js +1 -0
- dw/server/ui/assets/index-3Vw6WAPW.css +1 -0
- dw/server/ui/assets/index-DgrYhQd9.js +43 -0
- dw/server/ui/assets/ini-sBoK_t0W.js +1 -0
- dw/server/ui/assets/java-BEtHBSE6.js +1 -0
- dw/server/ui/assets/javascript-BJqN9Qhv.js +1 -0
- dw/server/ui/assets/json.worker-B2V3pomh.js +62 -0
- dw/server/ui/assets/jsonMode-DbM4SWSv.js +7 -0
- dw/server/ui/assets/julia-Bri6UV-V.js +1 -0
- dw/server/ui/assets/kotlin-BOotOW0E.js +1 -0
- dw/server/ui/assets/less-B9JPFI3C.js +2 -0
- dw/server/ui/assets/lexon-CfSJPG6W.js +1 -0
- dw/server/ui/assets/liquid-BWr8lEc4.js +1 -0
- dw/server/ui/assets/lspLanguageFeatures-C1iGuDyZ.js +4 -0
- dw/server/ui/assets/lua-CsQS60Ue.js +1 -0
- dw/server/ui/assets/m3-D-oSqn_W.js +1 -0
- dw/server/ui/assets/markdown-Cimd5fb3.js +1 -0
- dw/server/ui/assets/mdx-DAdMi_0p.js +1 -0
- dw/server/ui/assets/mips-CIPQ_RoX.js +1 -0
- dw/server/ui/assets/monaco--ixms01u.css +1 -0
- dw/server/ui/assets/monaco-BGCeEqaw.js +56 -0
- dw/server/ui/assets/msdax-DauUninz.js +1 -0
- dw/server/ui/assets/mysql-SOo6toE5.js +1 -0
- dw/server/ui/assets/objective-c-FvmIjYaQ.js +1 -0
- dw/server/ui/assets/pascal-DrH0SRf2.js +1 -0
- dw/server/ui/assets/pascaligo-D-ptJ9y-.js +1 -0
- dw/server/ui/assets/perl-oz_6vUea.js +1 -0
- dw/server/ui/assets/pgsql-DTj74zXo.js +1 -0
- dw/server/ui/assets/php-nr791fC2.js +1 -0
- dw/server/ui/assets/pla-CopQ2nXW.js +1 -0
- dw/server/ui/assets/postiats-43DmfD33.js +1 -0
- dw/server/ui/assets/powerquery-D3hlyOfw.js +1 -0
- dw/server/ui/assets/powershell-DmHpPYUd.js +1 -0
- dw/server/ui/assets/protobuf-C531GsRP.js +2 -0
- dw/server/ui/assets/pug-Z5eAx3Zn.js +1 -0
- dw/server/ui/assets/python-Bcn70HdC.js +1 -0
- dw/server/ui/assets/qsharp-DkqhCAOL.js +1 -0
- dw/server/ui/assets/r-BwWrilGY.js +1 -0
- dw/server/ui/assets/razor-D1HmNnby.js +1 -0
- dw/server/ui/assets/redis-ClamHrr6.js +1 -0
- dw/server/ui/assets/redshift-DT7zqm-g.js +1 -0
- dw/server/ui/assets/restructuredtext-BYgofb2h.js +1 -0
- dw/server/ui/assets/ruby-DezsRK8O.js +1 -0
- dw/server/ui/assets/rust-DdL9SqIa.js +1 -0
- dw/server/ui/assets/sb-CcwsVR0C.js +1 -0
- dw/server/ui/assets/scala-DHpiXF5c.js +1 -0
- dw/server/ui/assets/scheme-BeGwcela.js +1 -0
- dw/server/ui/assets/scss-gp-XZpBa.js +3 -0
- dw/server/ui/assets/shell-CC2rA5mh.js +1 -0
- dw/server/ui/assets/solidity-BEEn4gHE.js +1 -0
- dw/server/ui/assets/sophia-CRfGWb83.js +1 -0
- dw/server/ui/assets/sparql-D_Lu-MrJ.js +1 -0
- dw/server/ui/assets/sql-NEE52Syq.js +1 -0
- dw/server/ui/assets/st-DbInun42.js +1 -0
- dw/server/ui/assets/swift-Bxkupp3x.js +1 -0
- dw/server/ui/assets/systemverilog-Bz4Y3fRF.js +1 -0
- dw/server/ui/assets/tcl-DISqw1ZD.js +1 -0
- dw/server/ui/assets/ts.worker-D7T1-Ig5.js +67738 -0
- dw/server/ui/assets/tsMode-D6u0XmOW.js +11 -0
- dw/server/ui/assets/twig-De2hgUGE.js +1 -0
- dw/server/ui/assets/typescript-BU6v-LMV.js +1 -0
- dw/server/ui/assets/typespec-B8J7ngcE.js +1 -0
- dw/server/ui/assets/vb-DV3o63ZY.js +1 -0
- dw/server/ui/assets/wgsl-DpFanUEy.js +298 -0
- dw/server/ui/assets/workers-Cn7cTUKr.js +1 -0
- dw/server/ui/assets/xml--0LP2Lwk.js +1 -0
- dw/server/ui/assets/yaml-mpBg9jnt.js +1 -0
- dw/server/ui/index.html +17 -0
- dw/server/updater.py +192 -0
- dw/settings.py +98 -0
- dw/shot_span_preflight.py +116 -0
- dw/shots.py +359 -0
- dw/slice_preflight.py +148 -0
- dw/step.py +187 -0
- dw/step_cache.py +442 -0
- dw/subfolders.py +107 -0
- dw/task_domains.py +307 -0
- dw/tasks/assess.py +826 -0
- dw/tasks/audio_transcription.py +88 -0
- dw/tasks/audio_utils.py +1862 -0
- dw/tasks/background_remover.py +43 -0
- dw/tasks/borders.py +113 -0
- dw/tasks/compose_text.py +74 -0
- dw/tasks/concat_videos.py +300 -0
- dw/tasks/depth_estimator.py +54 -0
- dw/tasks/diffusion_upscale.py +109 -0
- dw/tasks/dissolve_videos.py +342 -0
- dw/tasks/format_messages.py +24 -0
- dw/tasks/gather.py +173 -0
- dw/tasks/grade.py +97 -0
- dw/tasks/image_to_text.py +43 -0
- dw/tasks/image_utils.py +764 -0
- dw/tasks/interpolate_frames.py +252 -0
- dw/tasks/judge.py +68 -0
- dw/tasks/model_cache.py +55 -0
- dw/tasks/pair_audio.py +268 -0
- dw/tasks/qr_code.py +19 -0
- dw/tasks/restore_faces.py +175 -0
- dw/tasks/rife_model.py +192 -0
- dw/tasks/segment.py +121 -0
- dw/tasks/select.py +111 -0
- dw/tasks/speech_generation.py +228 -0
- dw/tasks/stabilize.py +129 -0
- dw/tasks/task.py +920 -0
- dw/tasks/tensor_image.py +57 -0
- dw/tasks/text_generation.py +169 -0
- dw/tasks/text_sections.py +80 -0
- dw/tasks/upscale.py +203 -0
- dw/tasks/video_utils.py +624 -0
- dw/tasks/zoe_depth.py +71 -0
- dw/teacache.py +381 -0
- dw/teacache_models.json +99 -0
- dw/test.py +29 -0
- dw/type_helpers.py +231 -0
- dw/validate.py +68 -0
- dw/variable_constraints.py +444 -0
- dw/variables.py +443 -0
- dw/video_extensions.py +141 -0
- dw/vram_estimate.py +116 -0
- dw/worker.py +764 -0
- dw/workflow.py +2007 -0
- dw/workflow_schema.json +1346 -0
- dw/workflow_sources.py +383 -0
- dw/workflows/h3_context_ir.json +57 -0
- dw/workflows/test.json +31 -0
- dw/workspace.py +730 -0
- dw_mcp/__init__.py +6 -0
- dw_mcp/__main__.py +133 -0
- dw_mcp/assets.py +336 -0
- dw_mcp/authoring.py +114 -0
- dw_mcp/catalog.py +360 -0
- dw_mcp/client.py +486 -0
- dw_mcp/diagnose.py +371 -0
- dw_mcp/exports.py +84 -0
- dw_mcp/guides.py +35 -0
- dw_mcp/media.py +638 -0
- dw_mcp/models.py +97 -0
- dw_mcp/prompts.py +104 -0
- dw_mcp/server.py +1343 -0
- dw_mcp/workspaces.py +212 -0
dw/result.py
ADDED
|
@@ -0,0 +1,1720 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import time
|
|
3
|
+
import numpy
|
|
4
|
+
import torch
|
|
5
|
+
import soundfile
|
|
6
|
+
import json
|
|
7
|
+
import mimetypes
|
|
8
|
+
import logging
|
|
9
|
+
from diffusers.utils import (
|
|
10
|
+
export_to_video,
|
|
11
|
+
export_to_gif,
|
|
12
|
+
encode_video,
|
|
13
|
+
is_av_available,
|
|
14
|
+
)
|
|
15
|
+
from collections.abc import Mapping
|
|
16
|
+
from .events import emit_log, emit_phase, emit_warning
|
|
17
|
+
from .security import (
|
|
18
|
+
MAX_DECODE_PIXELS,
|
|
19
|
+
SecurityError,
|
|
20
|
+
validate_file_base_name,
|
|
21
|
+
validate_output_path,
|
|
22
|
+
validate_string_input,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger("dw")
|
|
26
|
+
|
|
27
|
+
# Result saving constants
|
|
28
|
+
MAX_BASE_NAME_LENGTH = 200
|
|
29
|
+
DEFAULT_AUDIO_SAMPLE_RATE = 44100
|
|
30
|
+
# The rate a video is written at when neither the workflow nor the artifact
|
|
31
|
+
# says - a diffusers convention old enough that changing it would restate
|
|
32
|
+
# every existing workflow's output
|
|
33
|
+
DEFAULT_VIDEO_FPS = 8
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _artifact_size(artifact):
|
|
37
|
+
"""How much there is to write, said the way the thing itself counts -
|
|
38
|
+
frames for a video, samples for a waveform. Best effort: it is narration
|
|
39
|
+
beside a file name, so anything it cannot measure it does not mention."""
|
|
40
|
+
try:
|
|
41
|
+
frames = getattr(artifact, "frames", None)
|
|
42
|
+
if frames is not None:
|
|
43
|
+
return f"{len(frames)} frames"
|
|
44
|
+
if hasattr(artifact, "__len__") and not isinstance(artifact, (str, bytes)):
|
|
45
|
+
return f"{len(artifact)} frames"
|
|
46
|
+
except Exception:
|
|
47
|
+
pass
|
|
48
|
+
return ""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# A deliverable this close to full scale has no headroom left: an mp3 or AAC
|
|
52
|
+
# encode of it decodes above 0 dBFS and clips, which is why a track measured
|
|
53
|
+
# at +1.3 dBFS in the gallery can have been written from samples that never
|
|
54
|
+
# exceeded 1.0. Same ceiling `match_levels` holds a gain to
|
|
55
|
+
# (MATCH_CEILING_DBFS in dw/tasks/audio_utils.py)
|
|
56
|
+
HEADROOM_WARN_DBFS = -0.5
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _peak_dbfs(waveform):
|
|
60
|
+
"""The loudest sample of anything saveable as audio, in dBFS, or None
|
|
61
|
+
when it cannot be measured cheaply (no samples, a lazily-decoded
|
|
62
|
+
reader, a shape nothing here recognises)."""
|
|
63
|
+
try:
|
|
64
|
+
if hasattr(waveform, "detach"):
|
|
65
|
+
waveform = waveform.detach().float().cpu().numpy()
|
|
66
|
+
samples = numpy.asarray(waveform)
|
|
67
|
+
if samples.size == 0 or not numpy.issubdtype(samples.dtype, numpy.number):
|
|
68
|
+
return None
|
|
69
|
+
peak = float(numpy.abs(samples).max())
|
|
70
|
+
except Exception:
|
|
71
|
+
logger.debug("Could not measure the peak of a saved track", exc_info=True)
|
|
72
|
+
return None
|
|
73
|
+
if peak <= 0.0:
|
|
74
|
+
return None
|
|
75
|
+
return 20.0 * float(numpy.log10(peak))
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# Image formats that carry an alpha channel; every other image content type
|
|
79
|
+
# takes the picture flattened over white
|
|
80
|
+
ALPHA_CONTENT_TYPES = {"image/png", "image/webp", "image/gif", "image/tiff"}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def flatten_alpha_for(image, content_type, file_name):
|
|
84
|
+
"""The image as the content type can carry it: unchanged when it has no
|
|
85
|
+
alpha channel or the format keeps one, otherwise composited over white
|
|
86
|
+
and said out loud.
|
|
87
|
+
|
|
88
|
+
Pillow refuses to write mode RGBA as JPEG, and it refused after the whole
|
|
89
|
+
generation had run - Qwen-Image-2.1 decodes an alpha channel natively,
|
|
90
|
+
and every image template in the catalog asked for image/jpeg. A warning
|
|
91
|
+
rather than an error, because the run did what was asked and the file
|
|
92
|
+
is usable; the warning names the format that would have kept the alpha.
|
|
93
|
+
"""
|
|
94
|
+
mode = getattr(image, "mode", None)
|
|
95
|
+
if mode not in ("RGBA", "LA", "PA") or content_type in ALPHA_CONTENT_TYPES:
|
|
96
|
+
return image
|
|
97
|
+
from PIL import Image as PILImage
|
|
98
|
+
|
|
99
|
+
rgba = image.convert("RGBA")
|
|
100
|
+
flattened = PILImage.new("RGB", rgba.size, (255, 255, 255))
|
|
101
|
+
flattened.paste(rgba, mask=rgba.getchannel("A"))
|
|
102
|
+
if getattr(image, "info", None):
|
|
103
|
+
flattened.info.update(image.info)
|
|
104
|
+
emit_warning(
|
|
105
|
+
f"The image written to {file_name} had an alpha channel that "
|
|
106
|
+
f"{content_type} cannot carry - it was flattened over white. Set "
|
|
107
|
+
f"the step's result 'content_type' to 'image/png' to keep the "
|
|
108
|
+
f"transparency.",
|
|
109
|
+
kind="alpha_discarded",
|
|
110
|
+
file=file_name,
|
|
111
|
+
content_type=content_type,
|
|
112
|
+
)
|
|
113
|
+
return flattened
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def warn_without_headroom(waveform, file_name, emit=True, lossless=False):
|
|
117
|
+
"""Say when the soundtrack about to be written is at or over full scale.
|
|
118
|
+
|
|
119
|
+
A clipped deliverable is invisible to the consumer this server is built
|
|
120
|
+
for: the job succeeds, and an agent that cannot listen has `peak_dbfs`
|
|
121
|
+
and no rule to read it against - `get_gallery_metadata` teaches the
|
|
122
|
+
near-silent end of the range and said nothing about the other one (#158).
|
|
123
|
+
A warning rather than a change to the mix: what level a deliverable
|
|
124
|
+
should sit at is the workflow's to decide, and `normalize_audio` is the
|
|
125
|
+
step that decides it.
|
|
126
|
+
|
|
127
|
+
`emit=False` measures and returns the predicted peak without emitting
|
|
128
|
+
the warning - used for a video mux, where the caller holds the emission
|
|
129
|
+
until the post-encode probe (`warn_if_written_above_full_scale`) has a
|
|
130
|
+
ground-truth answer, and only falls back to this prediction if that
|
|
131
|
+
probe could not measure the written file at all (#174 amendment).
|
|
132
|
+
|
|
133
|
+
`lossless=True` is a plain wav/aiff/flac save: soundfile writes those as
|
|
134
|
+
integer PCM by default, which clips a sample outside [-1, 1] immediately
|
|
135
|
+
rather than merely risking it on some later lossy encode (#295) - the
|
|
136
|
+
message says so, and the caller does not suppress the post-write
|
|
137
|
+
ground-truth check for this file the way it does for an mp3/ogg/opus save.
|
|
138
|
+
"""
|
|
139
|
+
peak = _peak_dbfs(waveform)
|
|
140
|
+
if peak is None or peak < HEADROOM_WARN_DBFS:
|
|
141
|
+
return None
|
|
142
|
+
if emit:
|
|
143
|
+
risk = (
|
|
144
|
+
"the write itself clips samples above full scale to 0 dBFS - the "
|
|
145
|
+
"file just written is already clipped"
|
|
146
|
+
if lossless
|
|
147
|
+
else "an mp3 or AAC encode of it decodes above 0 dBFS and clips"
|
|
148
|
+
)
|
|
149
|
+
emit_warning(
|
|
150
|
+
f"The soundtrack written to {file_name} peaks at {peak:+.1f} dBFS, "
|
|
151
|
+
f"which leaves no headroom below full scale - {risk}. Add a "
|
|
152
|
+
f"'normalize_audio' step (peak_dbfs: -1) before the step that "
|
|
153
|
+
f"saves it, or 'match_levels' on the join that made it.",
|
|
154
|
+
kind="audio_no_headroom",
|
|
155
|
+
file=file_name,
|
|
156
|
+
peak_dbfs=round(peak, 2),
|
|
157
|
+
)
|
|
158
|
+
return peak
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# The written file, decoded, is the only measurement that is the consumer's
|
|
162
|
+
# own. `warn_without_headroom` measures the waveform handed to the writer,
|
|
163
|
+
# and the encoder is downstream of that: a track normalized to exactly
|
|
164
|
+
# -1.0 dBFS came back out of an AAC mux at +0.94, so the deliverable of a
|
|
165
|
+
# clean default run of `music-video` was above full scale and nothing
|
|
166
|
+
# warned, because the number the check read was -1.0 (#158, #159, #161)
|
|
167
|
+
CLIPPED_WARN_DBFS = 0.0
|
|
168
|
+
|
|
169
|
+
# Distinguishes "no info supplied, probe it yourself" from "already probed,
|
|
170
|
+
# and it came back empty/unmeasurable" - a caller that decoded the file once
|
|
171
|
+
# (save_artifact, sharing one pass between this and warn_if_written_near_silent)
|
|
172
|
+
# passes the dict (or None on failure) through instead of paying for a second
|
|
173
|
+
# full decode of the same file (#262 - two independent probes, each a full
|
|
174
|
+
# audio+video decode, doubled the 'saving' phase's wall clock)
|
|
175
|
+
_UNPROBED = object()
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _probe_written_media(output_path):
|
|
179
|
+
"""Decode the just-written file once. `None` on any failure to probe."""
|
|
180
|
+
try:
|
|
181
|
+
from .media_info import probe_media
|
|
182
|
+
|
|
183
|
+
return probe_media(output_path) or {}
|
|
184
|
+
except Exception:
|
|
185
|
+
logger.debug(
|
|
186
|
+
f"Could not measure the written level of {output_path}", exc_info=True
|
|
187
|
+
)
|
|
188
|
+
return None
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def warn_if_written_above_full_scale(
|
|
192
|
+
output_path, already_warned=False, info=_UNPROBED, lossless=False
|
|
193
|
+
):
|
|
194
|
+
"""Say when the file just written decodes above full scale.
|
|
195
|
+
|
|
196
|
+
The overshoot a lossy encode adds is material-dependent - about 0.1 dB
|
|
197
|
+
on the mp3s measured for #159 and about 1.9 dB on the AAC mux of the
|
|
198
|
+
same song - so no amount of headroom chosen up front can be known to be
|
|
199
|
+
enough. Reading the file back is what closes that: whatever the encoder
|
|
200
|
+
did, this is the number a consumer's decoder will see.
|
|
201
|
+
|
|
202
|
+
Silent when `warn_without_headroom` has already spoken for this file -
|
|
203
|
+
but only for a plain audio save into a lossy container (mp3/ogg/opus).
|
|
204
|
+
That suppression assumed the encoder only ever adds overshoot, which held
|
|
205
|
+
for the mp3s #159/#161 measured but is backwards for an H3 video mux:
|
|
206
|
+
#174 measured that family's AAC mux landing *under* full scale after
|
|
207
|
+
starting over it, so the pre-encode warning was right and the caller's
|
|
208
|
+
suppression hid the post-encode check that would have said so. It is
|
|
209
|
+
also wrong for a wav/aiff/flac save: soundfile's default integer PCM
|
|
210
|
+
subtype clips a sample outside [-1, 1] at write time, so the pre-write
|
|
211
|
+
prediction and the post-write ground truth are two different facts about
|
|
212
|
+
the same file rather than a duplicate of one, and both are worth reading
|
|
213
|
+
(#295). The caller decides which case applies (`content_type`), not this
|
|
214
|
+
function.
|
|
215
|
+
|
|
216
|
+
`info` lets a caller that already decoded the file (`_probe_written_media`)
|
|
217
|
+
hand the result in rather than have this probe it again. Best effort
|
|
218
|
+
either way: a file that will not probe is not a level problem, and a
|
|
219
|
+
deliverable that is already written is not worth failing a finished run
|
|
220
|
+
over.
|
|
221
|
+
"""
|
|
222
|
+
if already_warned:
|
|
223
|
+
return None
|
|
224
|
+
if info is _UNPROBED:
|
|
225
|
+
info = _probe_written_media(output_path)
|
|
226
|
+
if info is None:
|
|
227
|
+
return None
|
|
228
|
+
peak = info.get("peak_dbfs")
|
|
229
|
+
if peak is None or peak < CLIPPED_WARN_DBFS:
|
|
230
|
+
return peak
|
|
231
|
+
name = os.path.basename(output_path)
|
|
232
|
+
if lossless:
|
|
233
|
+
cause = (
|
|
234
|
+
"The write itself clipped it - there was no headroom left below full scale"
|
|
235
|
+
)
|
|
236
|
+
else:
|
|
237
|
+
cause = "The encode adds its own overshoot on top of the level it was handed"
|
|
238
|
+
emit_warning(
|
|
239
|
+
f"{name} decodes at {peak:+.2f} dBFS - above full scale, so it "
|
|
240
|
+
f"clips on playback. {cause}, so the fix is more headroom before "
|
|
241
|
+
f"the file is written: a 'normalize_audio' step at 'peak_dbfs: -3' "
|
|
242
|
+
f"ahead of the step that saves it. A mux into a video needs more "
|
|
243
|
+
f"of it than an audio file does.",
|
|
244
|
+
kind="audio_clipped",
|
|
245
|
+
file=name,
|
|
246
|
+
peak_dbfs=round(peak, 2),
|
|
247
|
+
)
|
|
248
|
+
return peak
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
# The mean level get_gallery_metadata's own hint text already teaches as
|
|
252
|
+
# near-silent (#158) - mirrored here as the check nothing was actually
|
|
253
|
+
# running: a succeeded job could hand back a track this quiet with
|
|
254
|
+
# warnings: [] (#261)
|
|
255
|
+
NEAR_SILENT_WARN_DBFS = -40.0
|
|
256
|
+
|
|
257
|
+
# Above this, a peak means "quiet but not empty" rather than "check for a
|
|
258
|
+
# defect" - s02's -18.5 dBFS peaks (an ambience-only shot) clear it, #261's
|
|
259
|
+
# -68.7 dBFS Bark clip and S-F077's -60 dBFS normalize do not (#358)
|
|
260
|
+
NEAR_SILENT_QUIET_NOT_EMPTY_DBFS = -30.0
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def warn_if_written_near_silent(
|
|
264
|
+
output_path, already_warned=False, info=_UNPROBED, source_already_quiet=False
|
|
265
|
+
):
|
|
266
|
+
"""Say when the file just written decodes as near-silent.
|
|
267
|
+
|
|
268
|
+
The other end of the range `warn_if_written_above_full_scale` guards:
|
|
269
|
+
`get_gallery_metadata` already taught mean_dbfs below -40 on a track
|
|
270
|
+
that should be full as a near-silent render, but nothing emitted a
|
|
271
|
+
warning for it at save time, so a job could succeed and hand back a
|
|
272
|
+
clip nobody could hear with `warnings: []` (#261).
|
|
273
|
+
|
|
274
|
+
`info` is the same shared-probe handoff `warn_if_written_above_full_scale`
|
|
275
|
+
takes. Best effort either way: a file that will not probe is not a
|
|
276
|
+
level problem, and a deliverable that is already written is not worth
|
|
277
|
+
failing a finished run over.
|
|
278
|
+
|
|
279
|
+
`source_already_quiet` is set by a pass-through task (slice_audio) whose
|
|
280
|
+
*source* material was already this quiet going in - a slice of room
|
|
281
|
+
tone is not a defect the slice introduced, and the warning's own wording
|
|
282
|
+
("check the step that generated it... an unintended near-zero gain
|
|
283
|
+
upstream") is aimed at a step that could plausibly have caused the
|
|
284
|
+
level, which a plain cut out of an already-quiet recording did not (#309).
|
|
285
|
+
|
|
286
|
+
The trigger stays mean-only (#358): a wordless, ambience-only shot (paws,
|
|
287
|
+
husks scraping, water) reads a low mean with real peaks - -54 dBFS mean,
|
|
288
|
+
-18 to -31 dBFS peaks - and split perfectly with dialogue presence, not
|
|
289
|
+
with silence, costing an investigation every run. The fix is the message,
|
|
290
|
+
not the gate: a peak above `NEAR_SILENT_QUIET_NOT_EMPTY_DBFS` says so
|
|
291
|
+
plainly rather than reusing the "check the step that generated it"
|
|
292
|
+
wording aimed at a genuinely empty render (#261's -68.7 dBFS, S-F077's
|
|
293
|
+
-60 dBFS).
|
|
294
|
+
"""
|
|
295
|
+
if already_warned or source_already_quiet:
|
|
296
|
+
return None
|
|
297
|
+
if info is _UNPROBED:
|
|
298
|
+
info = _probe_written_media(output_path)
|
|
299
|
+
if info is None:
|
|
300
|
+
return None
|
|
301
|
+
mean = info.get("mean_dbfs")
|
|
302
|
+
if mean is None or mean >= NEAR_SILENT_WARN_DBFS:
|
|
303
|
+
return mean
|
|
304
|
+
name = os.path.basename(output_path)
|
|
305
|
+
peak = info.get("peak_dbfs")
|
|
306
|
+
if peak is not None and peak >= NEAR_SILENT_QUIET_NOT_EMPTY_DBFS:
|
|
307
|
+
message = (
|
|
308
|
+
f"{name} decodes at a mean level of {mean:+.2f} dBFS but peaks "
|
|
309
|
+
f"at {peak:+.2f} dBFS: quiet overall, not empty. Expected for "
|
|
310
|
+
f"an ambience-only shot; a concern only if this was meant to "
|
|
311
|
+
f"carry speech or music."
|
|
312
|
+
)
|
|
313
|
+
else:
|
|
314
|
+
message = (
|
|
315
|
+
f"{name} decodes at a mean level of {mean:+.2f} dBFS - near-silent "
|
|
316
|
+
f"for a deliverable meant to be heard. Check the step that "
|
|
317
|
+
f"generated it: an empty or malformed prompt, a source model that "
|
|
318
|
+
f"produced no meaningful audio for this input, or an unintended "
|
|
319
|
+
f"near-zero gain upstream ('normalize_audio' or 'match_levels')."
|
|
320
|
+
)
|
|
321
|
+
fields = {"kind": "audio_near_silent", "file": name, "mean_dbfs": round(mean, 2)}
|
|
322
|
+
if peak is not None:
|
|
323
|
+
fields["peak_dbfs"] = round(peak, 2)
|
|
324
|
+
emit_warning(message, **fields)
|
|
325
|
+
return mean
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _file_size_mb(path):
|
|
329
|
+
try:
|
|
330
|
+
return os.path.getsize(path) / (1024 * 1024)
|
|
331
|
+
except OSError:
|
|
332
|
+
return 0.0
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def frames_for_encoding(frames):
|
|
336
|
+
"""Generated frames in the form `encode_video` encodes without first
|
|
337
|
+
inspecting them.
|
|
338
|
+
|
|
339
|
+
A pipeline that returns `output_type="np"` hands back float frames in
|
|
340
|
+
[0, 1], and diffusers' `encode_video` establishes that range with three
|
|
341
|
+
full-size temporaries - `np.zeros_like`, `np.ones_like` and the bool
|
|
342
|
+
mask - before converting. On a 121-frame 960x544 clip that is ~3 GB of
|
|
343
|
+
allocation and 16 s of wall clock on an idle box, against 2.4 s for the
|
|
344
|
+
encode itself, and it is the bulk of a 'saving' phase that ran for 53 s
|
|
345
|
+
with nothing else in it (#97, measured on lem 2026-09-14).
|
|
346
|
+
|
|
347
|
+
Converting here is a pass and a half and hands back a torch tensor,
|
|
348
|
+
which `encode_video` takes as given - so the check never runs. Frames
|
|
349
|
+
outside [0, 1] are left exactly as they were: that is the branch where
|
|
350
|
+
diffusers warns and treats them as pixel values already, and it is not
|
|
351
|
+
a path any pipeline here produces or that this can be tested against.
|
|
352
|
+
|
|
353
|
+
The source array is never written to - a later step may still read this
|
|
354
|
+
result through a `previous_result:` reference, and the step cache
|
|
355
|
+
retains it.
|
|
356
|
+
"""
|
|
357
|
+
if not isinstance(frames, numpy.ndarray) or frames.size == 0:
|
|
358
|
+
return frames
|
|
359
|
+
if not numpy.issubdtype(frames.dtype, numpy.floating):
|
|
360
|
+
return frames
|
|
361
|
+
if float(frames.min()) < 0.0 or float(frames.max()) > 1.0:
|
|
362
|
+
return frames
|
|
363
|
+
denormalized = numpy.empty(frames.shape, dtype=numpy.uint8)
|
|
364
|
+
# Frame by frame: the whole-array form allocates another copy the size
|
|
365
|
+
# of the video, which is the cost this exists to avoid
|
|
366
|
+
for index in range(frames.shape[0]):
|
|
367
|
+
denormalized[index] = numpy.round(frames[index] * 255.0)
|
|
368
|
+
return torch.from_numpy(denormalized)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def output_file_path(output_dir, file_name):
|
|
372
|
+
"""The path a result file is written to, confined to the output directory.
|
|
373
|
+
|
|
374
|
+
Every part of the name is workflow-supplied - the workflow id, the step
|
|
375
|
+
name, a result's file_base_name, and the keys of a dict artifact - and
|
|
376
|
+
they are concatenated into a file name. Without this a name carrying a
|
|
377
|
+
path separator would write outside the output directory, so the joined
|
|
378
|
+
path goes through the same validator every other path in the engine does.
|
|
379
|
+
|
|
380
|
+
A rerun that would otherwise produce a name already on disk gets a
|
|
381
|
+
'-2', '-3', ... counter instead of silently overwriting it - the same
|
|
382
|
+
guarantee ComfyUI's SaveImage node makes by scanning its output
|
|
383
|
+
directory before every write.
|
|
384
|
+
"""
|
|
385
|
+
candidate = validate_output_path(os.path.join(output_dir, file_name), output_dir)
|
|
386
|
+
return _dedupe_existing_path(candidate)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _dedupe_existing_path(path):
|
|
390
|
+
"""Append an incrementing counter before the extension until `path` is free."""
|
|
391
|
+
if not os.path.exists(path):
|
|
392
|
+
return path
|
|
393
|
+
|
|
394
|
+
base, ext = os.path.splitext(path)
|
|
395
|
+
counter = 2
|
|
396
|
+
while True:
|
|
397
|
+
candidate = f"{base}-{counter}{ext}"
|
|
398
|
+
if not os.path.exists(candidate):
|
|
399
|
+
return candidate
|
|
400
|
+
counter += 1
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
# Audio content types soundfile can write, mapped to their file extension and to any
|
|
404
|
+
# write arguments the extension alone does not imply. Opus has no extension of its own
|
|
405
|
+
# in libsndfile - it is a subtype of the ogg container.
|
|
406
|
+
AUDIO_FORMATS = {
|
|
407
|
+
"audio/wav": (".wav", {}),
|
|
408
|
+
"audio/x-wav": (".wav", {}),
|
|
409
|
+
"audio/aiff": (".aiff", {}),
|
|
410
|
+
"audio/flac": (".flac", {}),
|
|
411
|
+
"audio/x-flac": (".flac", {}),
|
|
412
|
+
"audio/mpeg": (".mp3", {}),
|
|
413
|
+
"audio/mp3": (".mp3", {}),
|
|
414
|
+
"audio/ogg": (".ogg", {}),
|
|
415
|
+
"audio/vorbis": (".ogg", {}),
|
|
416
|
+
"audio/opus": (".ogg", {"format": "OGG", "subtype": "OPUS"}),
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
# Formats whose write is a lossy re-encode, distinct from a wav/aiff/flac save:
|
|
420
|
+
# soundfile's default integer PCM subtype for those clips an out-of-range sample
|
|
421
|
+
# at write time, so there is no separate "encode" step for the pre-write warning
|
|
422
|
+
# to describe as a future risk (#295) - only these three still fit that framing
|
|
423
|
+
LOSSY_AUDIO_CONTENT_TYPES = {
|
|
424
|
+
"audio/mpeg",
|
|
425
|
+
"audio/mp3",
|
|
426
|
+
"audio/ogg",
|
|
427
|
+
"audio/vorbis",
|
|
428
|
+
"audio/opus",
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
# Result definition keys passed through to soundfile - encoding quality controls
|
|
432
|
+
AUDIO_WRITE_ARGUMENTS = ["subtype", "format", "compression_level", "bitrate_mode"]
|
|
433
|
+
|
|
434
|
+
# Audio is written in chunks of this many frames - see write_audio
|
|
435
|
+
AUDIO_WRITE_CHUNK_FRAMES = 1 << 20
|
|
436
|
+
|
|
437
|
+
# The only container encode_video writes - it always encodes h264 video
|
|
438
|
+
MUXED_VIDEO_CONTENT_TYPE = "video/mp4"
|
|
439
|
+
|
|
440
|
+
# Distinguishes "the artifact has no such attribute" from "it has one holding None" -
|
|
441
|
+
# an AudioVideo whose pipeline reported no sample rate carries exactly that
|
|
442
|
+
_NO_PROPERTY = object()
|
|
443
|
+
|
|
444
|
+
# The names a modular pipeline's outputs go by. Asked for more than one output it returns
|
|
445
|
+
# them in a dict rather than on a pipeline output object, so its videos and the soundtrack
|
|
446
|
+
# generated alongside them arrive keyed instead of as attributes. Every diffusers modular
|
|
447
|
+
# pipeline (minimax_h3, ltx2, ...) names these "videos"/"audio"/"sampling_rate" - kept as
|
|
448
|
+
# tuples, rather than plain strings, only so the lookup goes through first_item like the
|
|
449
|
+
# other key sets. "audio_sample_rate" is real too: it is the name dw's own
|
|
450
|
+
# attach_audio_sample_rate (pipeline_processors/pipeline.py) gives the rate when it
|
|
451
|
+
# attaches it to a non-modular output - a modular result carrying it under that name is
|
|
452
|
+
# tested (TestModularOutputs.test_audio_sample_rate_names_the_rate_too) and kept for it.
|
|
453
|
+
MODULAR_VIDEO_KEYS = ("videos",)
|
|
454
|
+
MODULAR_AUDIO_KEYS = ("audio",)
|
|
455
|
+
MODULAR_SAMPLE_RATE_KEYS = ("sampling_rate", "audio_sample_rate")
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
class AudioVideo:
|
|
459
|
+
"""A generated video together with the audio track generated alongside it.
|
|
460
|
+
|
|
461
|
+
Pipelines like LTX-2 return audio next to their frames. Keeping the two paired lets
|
|
462
|
+
the result mux them into one file instead of dropping the audio on the floor.
|
|
463
|
+
"""
|
|
464
|
+
|
|
465
|
+
def __init__(self, frames, audio, sample_rate, fps=None, shots=None):
|
|
466
|
+
"""
|
|
467
|
+
Args:
|
|
468
|
+
frames: The video, as PIL images or an array of frames
|
|
469
|
+
audio: Waveform for this video, shaped (channels, samples)
|
|
470
|
+
sample_rate: Sample rate of the waveform, or None if the pipeline did not report one
|
|
471
|
+
fps: Frame rate these frames are meant to play at, when something
|
|
472
|
+
knows it - a joined video's own rate, or the rate of the file
|
|
473
|
+
a task read. Carried for the same reason AudioTrack carries
|
|
474
|
+
its sample rate: `result.fps` defaults to 8, and a step that
|
|
475
|
+
joins 24 fps shots writing them at 8 is three times slow with
|
|
476
|
+
its audio still the right length (#84). A declared
|
|
477
|
+
`result.fps` still wins over this
|
|
478
|
+
shots: Where each input landed, for a video a step joined from
|
|
479
|
+
several - a list of shot records (dw/shots.py), or None for a
|
|
480
|
+
video that is one shot. Carried into the step's manifest
|
|
481
|
+
entry when the video is saved (#378)
|
|
482
|
+
"""
|
|
483
|
+
self.frames = frames
|
|
484
|
+
self.audio = audio
|
|
485
|
+
self.sample_rate = sample_rate
|
|
486
|
+
self.fps = fps
|
|
487
|
+
self.shots = shots
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
class AudioTrack:
|
|
491
|
+
"""A generated waveform together with the rate it was generated at.
|
|
492
|
+
|
|
493
|
+
A step that produces audio alone usually returns the waveform by itself, and
|
|
494
|
+
the workflow declares the rate - which is fine where the rate is a property of
|
|
495
|
+
the workflow (a slice of a file it named) rather than of the model. It is not
|
|
496
|
+
fine for a generated track: every text-to-speech model has its own rate, and a
|
|
497
|
+
declared 44100 against a 24 kHz model plays the speech fast without failing.
|
|
498
|
+
|
|
499
|
+
Carrying the rate with the waveform is what lets a workflow say nothing about
|
|
500
|
+
it. Everything downstream of audio already reads '.audio' and '.sample_rate'
|
|
501
|
+
off whatever it is handed - slice_audio, fade_audio, pair_audio and the H3
|
|
502
|
+
audio references all accept one of these - and a rate the workflow does declare
|
|
503
|
+
still wins over the one carried here.
|
|
504
|
+
"""
|
|
505
|
+
|
|
506
|
+
def __init__(self, audio, sample_rate, source_mean_dbfs=None):
|
|
507
|
+
"""
|
|
508
|
+
Args:
|
|
509
|
+
audio: The waveform, shaped (channels, samples)
|
|
510
|
+
sample_rate: Sample rate the waveform was generated at
|
|
511
|
+
source_mean_dbfs: The mean level of the material this track was
|
|
512
|
+
taken from, when a task (slice_audio) measured one before
|
|
513
|
+
cutting it down - lets a save skip the near-silent warning
|
|
514
|
+
for a slice whose source was already this quiet (#309)
|
|
515
|
+
"""
|
|
516
|
+
self.audio = audio
|
|
517
|
+
self.sample_rate = sample_rate
|
|
518
|
+
self.source_mean_dbfs = source_mean_dbfs
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
class Result:
|
|
522
|
+
"""Manages and stores results from workflow steps.
|
|
523
|
+
|
|
524
|
+
Handles result storage, artifact management, and file saving with support
|
|
525
|
+
for multiple content types including images, video, audio, and JSON.
|
|
526
|
+
"""
|
|
527
|
+
|
|
528
|
+
def __init__(self, result_definition, consumed_by_normalizer=False):
|
|
529
|
+
"""Initialize Result with configuration for how to handle/save results.
|
|
530
|
+
|
|
531
|
+
Args:
|
|
532
|
+
result_definition: Dict containing result configuration including:
|
|
533
|
+
- content_type: MIME type of the result
|
|
534
|
+
- save: Boolean indicating if result should be saved
|
|
535
|
+
- file_base_name: Base name for saved files
|
|
536
|
+
consumed_by_normalizer: Whether a later step resets this result's
|
|
537
|
+
level (normalize_audio/match_levels) before anything ships
|
|
538
|
+
it - when true, this save's own headroom is not a deliverable
|
|
539
|
+
concern (#286)
|
|
540
|
+
"""
|
|
541
|
+
self.result_definition = result_definition
|
|
542
|
+
self._consumed_by_normalizer = consumed_by_normalizer
|
|
543
|
+
self.result_list = []
|
|
544
|
+
self.metadata = None
|
|
545
|
+
self.saved_files = []
|
|
546
|
+
# The shots (dw/shots.py) of each saved file whose artifact carried
|
|
547
|
+
# any, keyed by the path in saved_files - plain data, so a step cache
|
|
548
|
+
# hit's stripped copy still reports them (#378)
|
|
549
|
+
self.saved_shots = {}
|
|
550
|
+
# Set when a select step's Selected wrapper flows through
|
|
551
|
+
# add_result - the winning position/score, replayable in the
|
|
552
|
+
# manifest and step_end alongside the unwrapped value (#119)
|
|
553
|
+
self.selected = None
|
|
554
|
+
# Whether the file currently being written already drew a headroom
|
|
555
|
+
# warning from the waveform it was handed, so the written-level
|
|
556
|
+
# check does not say the same thing twice (#161)
|
|
557
|
+
self._no_headroom_warned = False
|
|
558
|
+
# The predicted peak (dBFS) from the pre-encode headroom check, paired
|
|
559
|
+
# with _no_headroom_warned above; only ever set inside save_artifact,
|
|
560
|
+
# initialized here so a caller reading it before that runs gets None
|
|
561
|
+
# rather than an AttributeError.
|
|
562
|
+
self._predicted_peak_dbfs = None
|
|
563
|
+
# get_artifact_list(result) memoized by id(result) - see
|
|
564
|
+
# _artifacts_for. Keeps result_list itself untouched, so
|
|
565
|
+
# Result.retainable's attribute walk over result_list is unaffected.
|
|
566
|
+
self._artifact_cache = {}
|
|
567
|
+
logger.debug(f"Initialized Result with definition: {result_definition}")
|
|
568
|
+
|
|
569
|
+
def set_metadata(self, metadata):
|
|
570
|
+
"""Set metadata to embed in saved image artifacts.
|
|
571
|
+
|
|
572
|
+
Args:
|
|
573
|
+
metadata: Dict of generation parameters to embed
|
|
574
|
+
"""
|
|
575
|
+
self.metadata = metadata
|
|
576
|
+
|
|
577
|
+
def add_result(self, result):
|
|
578
|
+
"""Add one or more results to the result list.
|
|
579
|
+
|
|
580
|
+
Args:
|
|
581
|
+
result: Single result or list of results to store
|
|
582
|
+
"""
|
|
583
|
+
from .tasks.select import Selected
|
|
584
|
+
|
|
585
|
+
if isinstance(result, Selected):
|
|
586
|
+
self.selected = {"position": result.position, "score": result.score}
|
|
587
|
+
result = result.value
|
|
588
|
+
|
|
589
|
+
if isinstance(result, list):
|
|
590
|
+
logger.debug(f"Adding {len(result)} results to result list")
|
|
591
|
+
self.result_list.extend(result)
|
|
592
|
+
else:
|
|
593
|
+
if isinstance(result, str):
|
|
594
|
+
# Clean up string results by removing extra quotes and whitespace
|
|
595
|
+
result = result.strip().strip('"').strip()
|
|
596
|
+
logger.debug("Adding single result to result list")
|
|
597
|
+
self.result_list.append(result)
|
|
598
|
+
|
|
599
|
+
@property
|
|
600
|
+
def retainable(self):
|
|
601
|
+
"""Whether the step cache may keep this Result's result_list.
|
|
602
|
+
|
|
603
|
+
A chain step's save_segments spills each segment to disk and
|
|
604
|
+
replays it lazily through a SegmentedFrames; save() removes those
|
|
605
|
+
files once the video is written (SegmentedFrames.cleanup(), called
|
|
606
|
+
from save_audio_video below). A Result cached after that point would
|
|
607
|
+
point a later cache hit at files that are already gone, so a
|
|
608
|
+
result_list holding such an artifact is not retainable - checked by
|
|
609
|
+
duck-typed attribute rather than isinstance, so this module need not
|
|
610
|
+
import pipeline_processors.chain.
|
|
611
|
+
"""
|
|
612
|
+
for result in self.result_list:
|
|
613
|
+
frames = getattr(result, "frames", None)
|
|
614
|
+
if getattr(frames, "cleaned", False):
|
|
615
|
+
return False
|
|
616
|
+
return True
|
|
617
|
+
|
|
618
|
+
def get_artifacts(self):
|
|
619
|
+
"""Retrieve all artifacts from stored results.
|
|
620
|
+
|
|
621
|
+
Returns:
|
|
622
|
+
List of all artifacts from all results
|
|
623
|
+
"""
|
|
624
|
+
artifacts = []
|
|
625
|
+
for result in self.result_list:
|
|
626
|
+
artifacts.extend(self._artifacts_for(result))
|
|
627
|
+
|
|
628
|
+
logger.debug(f"Retrieved {len(artifacts)} artifacts from results")
|
|
629
|
+
return artifacts
|
|
630
|
+
|
|
631
|
+
def _artifacts_for(self, result):
|
|
632
|
+
"""get_artifact_list(result), memoized by identity of `result`.
|
|
633
|
+
|
|
634
|
+
save() extracts an in-memory pipeline output's artifacts and mutates
|
|
635
|
+
one in place to fit generated audio to the frame count (#197). A
|
|
636
|
+
later get_artifacts() call - a previous_result: consumer reading a
|
|
637
|
+
chained shot directly, bypassing any output: round trip - must see
|
|
638
|
+
that same fitted object rather than a fresh, unfitted extraction
|
|
639
|
+
from the same raw pipeline output, which get_artifact_list would
|
|
640
|
+
otherwise rebuild every time it runs.
|
|
641
|
+
"""
|
|
642
|
+
key = id(result)
|
|
643
|
+
if key not in self._artifact_cache:
|
|
644
|
+
self._artifact_cache[key] = get_artifact_list(result)
|
|
645
|
+
return self._artifact_cache[key]
|
|
646
|
+
|
|
647
|
+
def get_artifact_properties(self, property_name):
|
|
648
|
+
"""Extract specific properties from results.
|
|
649
|
+
|
|
650
|
+
A dict result is looked up by key; anything else by attribute, which is how
|
|
651
|
+
a step reaches into an artifact that is an object rather than a mapping -
|
|
652
|
+
the frames or the soundtrack of the AudioVideo a video-with-audio pipeline
|
|
653
|
+
produces, say, where the next step takes one of them on its own. Methods are
|
|
654
|
+
not properties: 'previous_result:step.index' on a list result names nothing
|
|
655
|
+
the workflow meant, so it fails rather than passing a bound method along.
|
|
656
|
+
|
|
657
|
+
Args:
|
|
658
|
+
property_name: Name of property to extract from results
|
|
659
|
+
|
|
660
|
+
Returns:
|
|
661
|
+
List of property values from results where property exists
|
|
662
|
+
|
|
663
|
+
Raises:
|
|
664
|
+
ValueError: If a result is neither a dict-like (Mapping) object with that
|
|
665
|
+
key nor an object carrying it as a data attribute - a plain string or
|
|
666
|
+
other scalar result has no properties to look up, and staying quiet
|
|
667
|
+
about that (or doing a membership/substring test instead of a key
|
|
668
|
+
lookup) would silently drop data or raise a confusing TypeError.
|
|
669
|
+
"""
|
|
670
|
+
values = []
|
|
671
|
+
for result in self.result_list:
|
|
672
|
+
if isinstance(result, Mapping):
|
|
673
|
+
if property_name in result:
|
|
674
|
+
values.append(result[property_name])
|
|
675
|
+
continue
|
|
676
|
+
|
|
677
|
+
value = getattr(result, property_name, _NO_PROPERTY)
|
|
678
|
+
# A string's every 'property' is a method, and so is most of a list's -
|
|
679
|
+
# the original loud failure for those is the useful answer
|
|
680
|
+
if value is _NO_PROPERTY or callable(value):
|
|
681
|
+
raise ValueError(
|
|
682
|
+
f"result has no property '{property_name}' "
|
|
683
|
+
f"(it is a {type(result).__name__}, not a dict)"
|
|
684
|
+
)
|
|
685
|
+
values.append(value)
|
|
686
|
+
|
|
687
|
+
logger.debug(f"Retrieved {len(values)} values for property: {property_name}")
|
|
688
|
+
return values
|
|
689
|
+
|
|
690
|
+
def save(self, output_dir, default_base_name):
|
|
691
|
+
"""Save results to files based on content type.
|
|
692
|
+
|
|
693
|
+
Args:
|
|
694
|
+
output_dir: Directory to save files in
|
|
695
|
+
default_base_name: Default name to use for files
|
|
696
|
+
|
|
697
|
+
Returns:
|
|
698
|
+
List of file paths written, in the order they were written - the
|
|
699
|
+
step's manifest. Empty when saving is disabled or nothing saved.
|
|
700
|
+
"""
|
|
701
|
+
try:
|
|
702
|
+
# Validate output directory
|
|
703
|
+
validated_output_dir = validate_output_path(output_dir, None)
|
|
704
|
+
validated_base_name = validate_string_input(
|
|
705
|
+
default_base_name, max_length=MAX_BASE_NAME_LENGTH
|
|
706
|
+
)
|
|
707
|
+
|
|
708
|
+
# Add directory check/creation
|
|
709
|
+
if not os.path.exists(validated_output_dir):
|
|
710
|
+
logger.debug(f"Creating output directory: {validated_output_dir}")
|
|
711
|
+
os.makedirs(validated_output_dir, exist_ok=True)
|
|
712
|
+
elif not os.path.isdir(validated_output_dir):
|
|
713
|
+
raise ValueError(
|
|
714
|
+
f"Output path exists but is not a directory: {validated_output_dir}"
|
|
715
|
+
)
|
|
716
|
+
except SecurityError as e:
|
|
717
|
+
logger.error(f"Security validation failed for output: {e}")
|
|
718
|
+
raise
|
|
719
|
+
except (OSError, PermissionError) as e:
|
|
720
|
+
logger.error(f"Failed to create output directory: {e}")
|
|
721
|
+
raise
|
|
722
|
+
|
|
723
|
+
# Check if saving is enabled and content type is specified
|
|
724
|
+
content_type = self.result_definition.get("content_type", None)
|
|
725
|
+
if not self.result_definition.get("save", True) or content_type is None:
|
|
726
|
+
logger.debug("Skipping save - disabled or no content type specified")
|
|
727
|
+
self.saved_files = []
|
|
728
|
+
self.saved_shots = {}
|
|
729
|
+
return self.saved_files
|
|
730
|
+
|
|
731
|
+
# Determine base filename with validation. A file_base_name *replaces*
|
|
732
|
+
# the derived name - it is set to get a name the caller can predict, and
|
|
733
|
+
# gluing it onto the name it was meant to replace made it neither (#100).
|
|
734
|
+
# What the derived name guaranteed - a distinct name per step - is then
|
|
735
|
+
# the caller's to keep; output_file_path's counter catches a collision.
|
|
736
|
+
file_base_name = validated_base_name
|
|
737
|
+
if "file_base_name" in self.result_definition:
|
|
738
|
+
file_base_name = validate_file_base_name(
|
|
739
|
+
validate_string_input(
|
|
740
|
+
self.result_definition["file_base_name"],
|
|
741
|
+
max_length=MAX_BASE_NAME_LENGTH,
|
|
742
|
+
)
|
|
743
|
+
)
|
|
744
|
+
|
|
745
|
+
# The same refusal validation makes, for a definition that reached
|
|
746
|
+
# the writer without it. Imported here: content_types imports this
|
|
747
|
+
# module for AUDIO_FORMATS
|
|
748
|
+
from .content_types import refuse_active_content_type
|
|
749
|
+
|
|
750
|
+
refuse_active_content_type(content_type)
|
|
751
|
+
|
|
752
|
+
# Get file extension for content type
|
|
753
|
+
extension = guess_extension(content_type)
|
|
754
|
+
logger.debug(
|
|
755
|
+
f"Saving with content type: {content_type}, extension: {extension}"
|
|
756
|
+
)
|
|
757
|
+
|
|
758
|
+
# Encoding a video here is minutes of work after the last denoise
|
|
759
|
+
# step, with the step's bar sitting full
|
|
760
|
+
emit_phase("saving", detail=content_type)
|
|
761
|
+
|
|
762
|
+
# Save each result, collecting the paths written as the step's manifest
|
|
763
|
+
saved_files = []
|
|
764
|
+
saved_shots = {}
|
|
765
|
+
for i, result in enumerate(self.result_list):
|
|
766
|
+
if content_type.endswith("json"):
|
|
767
|
+
# Handle JSON content type
|
|
768
|
+
output_path = output_file_path(
|
|
769
|
+
validated_output_dir, f"{file_base_name}-{i}{extension}"
|
|
770
|
+
)
|
|
771
|
+
logger.info(f"Saving JSON result to {output_path}")
|
|
772
|
+
with open(output_path, "w") as file:
|
|
773
|
+
file.write(json.dumps(result, indent=4))
|
|
774
|
+
saved_files.append(output_path)
|
|
775
|
+
else:
|
|
776
|
+
# Handle other content types
|
|
777
|
+
for j, artifact in enumerate(self._artifacts_for(result)):
|
|
778
|
+
paths = self.save_artifact(
|
|
779
|
+
validated_output_dir,
|
|
780
|
+
artifact,
|
|
781
|
+
f"{file_base_name}-{i}.{j}",
|
|
782
|
+
content_type,
|
|
783
|
+
extension,
|
|
784
|
+
)
|
|
785
|
+
shots = getattr(artifact, "shots", None)
|
|
786
|
+
if shots:
|
|
787
|
+
saved_shots.update((path, shots) for path in paths)
|
|
788
|
+
saved_files.extend(paths)
|
|
789
|
+
self.saved_files = saved_files
|
|
790
|
+
self.saved_shots = saved_shots
|
|
791
|
+
return saved_files
|
|
792
|
+
|
|
793
|
+
def save_artifact(
|
|
794
|
+
self, output_dir, artifact, file_base_name, content_type, extension
|
|
795
|
+
):
|
|
796
|
+
"""Save individual artifact to file based on its type.
|
|
797
|
+
|
|
798
|
+
Args:
|
|
799
|
+
output_dir: Directory to save file in, already validated by save()
|
|
800
|
+
artifact: The artifact to save
|
|
801
|
+
file_base_name: Base name for the file, derived from names save()
|
|
802
|
+
validated
|
|
803
|
+
content_type: MIME type of the content
|
|
804
|
+
extension: File extension to use
|
|
805
|
+
|
|
806
|
+
Returns:
|
|
807
|
+
List of file paths written.
|
|
808
|
+
"""
|
|
809
|
+
if artifact is None:
|
|
810
|
+
logger.warning(f"Skipping None artifact for {file_base_name}")
|
|
811
|
+
return []
|
|
812
|
+
|
|
813
|
+
if isinstance(artifact, (int, float, bool)):
|
|
814
|
+
# Static validation (dw/scalar_result_validation.py, #212) refuses
|
|
815
|
+
# a 'result' block on a command declared to return a scalar, but
|
|
816
|
+
# a command name reached only through a 'variable:' is literal
|
|
817
|
+
# only at run time and so invisible to that check - this is the
|
|
818
|
+
# same refusal for the one path that can still get here, naming
|
|
819
|
+
# the value rather than failing inside soundfile/PIL/open() with
|
|
820
|
+
# a bare TypeError after the step's own work is already done
|
|
821
|
+
raise ValueError(
|
|
822
|
+
f"'{file_base_name}' is a {type(artifact).__name__} ({artifact!r}), "
|
|
823
|
+
"not an artifact - a 'result' block cannot save it"
|
|
824
|
+
)
|
|
825
|
+
|
|
826
|
+
if isinstance(artifact, dict):
|
|
827
|
+
# Recursively save dictionary items
|
|
828
|
+
logger.debug(
|
|
829
|
+
f"Saving dictionary artifact with keys: {list(artifact.keys())}"
|
|
830
|
+
)
|
|
831
|
+
saved_files = []
|
|
832
|
+
for k, v in artifact.items():
|
|
833
|
+
saved_files.extend(
|
|
834
|
+
self.save_artifact(
|
|
835
|
+
output_dir,
|
|
836
|
+
v,
|
|
837
|
+
f"{file_base_name}-{k}",
|
|
838
|
+
content_type,
|
|
839
|
+
extension,
|
|
840
|
+
)
|
|
841
|
+
)
|
|
842
|
+
return saved_files
|
|
843
|
+
|
|
844
|
+
output_path = output_file_path(output_dir, f"{file_base_name}{extension}")
|
|
845
|
+
logger.info(f"Saving artifact to {output_path}")
|
|
846
|
+
# Writing one file is the whole of the 'saving' phase's wall clock,
|
|
847
|
+
# and on a video it is minutes of it with nothing else to report -
|
|
848
|
+
# the denoise counter is frozen at its last step and there is no
|
|
849
|
+
# further event until step_end, so a healthy run is indistinguishable
|
|
850
|
+
# from a hung one (#97). Name the file as it starts and report what
|
|
851
|
+
# it cost as it finishes, the same shape the modular block lead-in
|
|
852
|
+
# got in #95
|
|
853
|
+
emit_log(
|
|
854
|
+
f"writing {os.path.basename(output_path)}"
|
|
855
|
+
+ (f" ({_artifact_size(artifact)})" if _artifact_size(artifact) else ""),
|
|
856
|
+
file=os.path.basename(output_path),
|
|
857
|
+
content_type=content_type,
|
|
858
|
+
)
|
|
859
|
+
started = time.monotonic()
|
|
860
|
+
self._no_headroom_warned = False
|
|
861
|
+
self._predicted_peak_dbfs = None
|
|
862
|
+
|
|
863
|
+
try:
|
|
864
|
+
if content_type.startswith("video"):
|
|
865
|
+
if isinstance(artifact, AudioVideo):
|
|
866
|
+
self.save_audio_video(artifact, output_path, content_type)
|
|
867
|
+
else:
|
|
868
|
+
export_to_video(artifact, output_path, fps=self.video_fps(artifact))
|
|
869
|
+
elif content_type == "image/gif":
|
|
870
|
+
export_to_gif(artifact, output_path, fps=self.video_fps(artifact))
|
|
871
|
+
elif content_type.startswith("audio"):
|
|
872
|
+
waveforms = normalize_audio(artifact)
|
|
873
|
+
# Declared rate > the rate a generated track carries > default
|
|
874
|
+
declared_rate = self.result_definition.get("sample_rate")
|
|
875
|
+
carried_rate = getattr(artifact, "sample_rate", None)
|
|
876
|
+
# A template's 'result.sample_rate' relabels the file at save
|
|
877
|
+
# time exactly the way a task argument's 'sample_rate' does -
|
|
878
|
+
# and #180's guard only caught the argument, not this. A
|
|
879
|
+
# caller who passed the source's own correct rate as the
|
|
880
|
+
# argument (so the argument-level check is clean) still got
|
|
881
|
+
# the wrong file with `warnings: []` when the *result* block
|
|
882
|
+
# hardcoded a different rate (#205). Same warning either way.
|
|
883
|
+
if (
|
|
884
|
+
declared_rate is not None
|
|
885
|
+
and carried_rate is not None
|
|
886
|
+
and declared_rate != carried_rate
|
|
887
|
+
):
|
|
888
|
+
from .tasks.audio_utils import _warn_on_rate_override
|
|
889
|
+
|
|
890
|
+
_warn_on_rate_override("save_artifact", carried_rate, declared_rate)
|
|
891
|
+
sample_rate = declared_rate or carried_rate or DEFAULT_AUDIO_SAMPLE_RATE
|
|
892
|
+
# A batched waveform holds several songs - save each one separately
|
|
893
|
+
if len(waveforms) > 1:
|
|
894
|
+
saved_files = []
|
|
895
|
+
for k, waveform in enumerate(waveforms):
|
|
896
|
+
saved_files.extend(
|
|
897
|
+
self.save_artifact(
|
|
898
|
+
output_dir,
|
|
899
|
+
# Keep the rate the track carries across the
|
|
900
|
+
# recursion - a bare waveform would fall back
|
|
901
|
+
# to the default
|
|
902
|
+
(
|
|
903
|
+
AudioTrack(waveform.T, sample_rate)
|
|
904
|
+
if getattr(artifact, "sample_rate", None)
|
|
905
|
+
is not None
|
|
906
|
+
else waveform
|
|
907
|
+
),
|
|
908
|
+
f"{file_base_name}-{k}",
|
|
909
|
+
content_type,
|
|
910
|
+
extension,
|
|
911
|
+
)
|
|
912
|
+
)
|
|
913
|
+
return saved_files
|
|
914
|
+
self._no_headroom_warned = (
|
|
915
|
+
False
|
|
916
|
+
if self._consumed_by_normalizer
|
|
917
|
+
else warn_without_headroom(
|
|
918
|
+
waveforms[0],
|
|
919
|
+
os.path.basename(output_path),
|
|
920
|
+
lossless=content_type not in LOSSY_AUDIO_CONTENT_TYPES,
|
|
921
|
+
)
|
|
922
|
+
is not None
|
|
923
|
+
)
|
|
924
|
+
write_audio(
|
|
925
|
+
output_path,
|
|
926
|
+
waveforms[0],
|
|
927
|
+
sample_rate,
|
|
928
|
+
**self.get_audio_write_arguments(content_type),
|
|
929
|
+
)
|
|
930
|
+
elif content_type.endswith("json"):
|
|
931
|
+
with open(output_path, "w") as file:
|
|
932
|
+
file.write(json.dumps(artifact, indent=4))
|
|
933
|
+
elif content_type.startswith("text"):
|
|
934
|
+
with open(output_path, "w") as file:
|
|
935
|
+
file.write(artifact)
|
|
936
|
+
elif hasattr(artifact, "save"):
|
|
937
|
+
if content_type.startswith("image/"):
|
|
938
|
+
artifact = flatten_alpha_for(
|
|
939
|
+
artifact, content_type, os.path.basename(output_path)
|
|
940
|
+
)
|
|
941
|
+
if (
|
|
942
|
+
self.metadata is not None
|
|
943
|
+
and self.result_definition.get("embed_metadata", False)
|
|
944
|
+
and content_type.startswith("image/")
|
|
945
|
+
):
|
|
946
|
+
self._save_image_with_metadata(artifact, output_path, content_type)
|
|
947
|
+
else:
|
|
948
|
+
artifact.save(output_path)
|
|
949
|
+
else:
|
|
950
|
+
raise ValueError(
|
|
951
|
+
f"Content type {content_type} does not match result type {type(artifact)}"
|
|
952
|
+
)
|
|
953
|
+
except Exception as e:
|
|
954
|
+
logger.error(
|
|
955
|
+
f"Error saving artifact to {output_path}: {str(e)}", exc_info=True
|
|
956
|
+
)
|
|
957
|
+
raise
|
|
958
|
+
|
|
959
|
+
# The level of what was actually written, which is the only one the
|
|
960
|
+
# consumer will hear: the encode's own overshoot sits between the
|
|
961
|
+
# waveform `warn_without_headroom` measured and this (#161). Only a
|
|
962
|
+
# file that can carry a soundtrack, so an image never pays a probe.
|
|
963
|
+
# The pre-encode warning only suppresses this for a plain audio
|
|
964
|
+
# save - a video mux's overshoot is not reliably positive (#174),
|
|
965
|
+
# so a video always gets the ground-truth post-encode check
|
|
966
|
+
if content_type.startswith("audio") or content_type.startswith("video"):
|
|
967
|
+
# One decode pass shared between the two checks below (#262) -
|
|
968
|
+
# each used to probe the file independently, which for a video
|
|
969
|
+
# is a full audio+video decode and doubled the 'saving' phase's
|
|
970
|
+
# wall clock for no second answer
|
|
971
|
+
probed_info = _probe_written_media(output_path)
|
|
972
|
+
# The shot map (#426) was re-measured against the fitted
|
|
973
|
+
# in-memory track before this file was even encoded - a
|
|
974
|
+
# prediction, not the file's own ground truth. A lossy mux can
|
|
975
|
+
# still trim or pad past that (AAC's frame alignment cost the
|
|
976
|
+
# #426 repro 29-30 samples on top of what fitting alone
|
|
977
|
+
# accounted for), so once the file is probed the shots are
|
|
978
|
+
# re-measured again against what actually decodes from it - the
|
|
979
|
+
# same length assess.py's read_media() trims audio to
|
|
980
|
+
# (`audio_stream_seconds`, the audio *stream's* own reported
|
|
981
|
+
# duration - not the container's `duration_seconds`, which can
|
|
982
|
+
# disagree with it by a handful of samples on a lossy mux and
|
|
983
|
+
# would leave a residual overrun the probe still reports).
|
|
984
|
+
if (
|
|
985
|
+
content_type.startswith("video")
|
|
986
|
+
and getattr(artifact, "shots", None)
|
|
987
|
+
and probed_info
|
|
988
|
+
and probed_info.get("audio_stream_seconds") is not None
|
|
989
|
+
and probed_info.get("sample_rate")
|
|
990
|
+
):
|
|
991
|
+
from .shots import measured_num_samples
|
|
992
|
+
|
|
993
|
+
written_samples = int(
|
|
994
|
+
round(
|
|
995
|
+
probed_info["audio_stream_seconds"] * probed_info["sample_rate"]
|
|
996
|
+
)
|
|
997
|
+
)
|
|
998
|
+
measured_num_samples(artifact.shots, written_samples)
|
|
999
|
+
frame_count = len(getattr(artifact, "frames", []) or [])
|
|
1000
|
+
video_fps = self.video_fps(artifact)
|
|
1001
|
+
if frame_count and video_fps:
|
|
1002
|
+
expected_samples = int(
|
|
1003
|
+
round(frame_count / video_fps * probed_info["sample_rate"])
|
|
1004
|
+
)
|
|
1005
|
+
shortfall = expected_samples - written_samples
|
|
1006
|
+
# Only a residual the save-time fit (_fit_audio_to_frames)
|
|
1007
|
+
# would have padded: past its tolerance the track was
|
|
1008
|
+
# left at its own length, audio_video_length_mismatch
|
|
1009
|
+
# already names that gap, and "the mux trimmed it" would
|
|
1010
|
+
# misexplain it.
|
|
1011
|
+
from .tasks.video_utils import AUDIO_FIT_TOLERANCE_SECONDS
|
|
1012
|
+
|
|
1013
|
+
tolerance = AUDIO_FIT_TOLERANCE_SECONDS * probed_info["sample_rate"]
|
|
1014
|
+
if 0 < shortfall < probed_info["sample_rate"] / video_fps:
|
|
1015
|
+
# Under a frame: the encoder's alignment on every
|
|
1016
|
+
# joined deliverable, not something a caller can act
|
|
1017
|
+
# on - logged, with the shots already re-measured
|
|
1018
|
+
emit_log(
|
|
1019
|
+
f"{os.path.basename(output_path)}'s soundtrack "
|
|
1020
|
+
f"decodes {shortfall} sample(s) short of its "
|
|
1021
|
+
f"{frame_count}-frame grid after muxing; the shot "
|
|
1022
|
+
"map is measured against what it decodes to",
|
|
1023
|
+
file=os.path.basename(output_path),
|
|
1024
|
+
shortfall_samples=shortfall,
|
|
1025
|
+
)
|
|
1026
|
+
elif 0 < shortfall <= tolerance:
|
|
1027
|
+
emit_warning(
|
|
1028
|
+
f"{os.path.basename(output_path)}'s soundtrack decodes "
|
|
1029
|
+
f"{shortfall} sample(s) short of its {frame_count}-frame "
|
|
1030
|
+
f"grid after muxing, even though it was padded to the "
|
|
1031
|
+
f"grid before encoding - the mux itself (commonly AAC's "
|
|
1032
|
+
f"frame alignment) trimmed it further. The shot map has "
|
|
1033
|
+
f"been re-measured against what the file actually "
|
|
1034
|
+
f"decodes to, so it stays accurate, but a consumer "
|
|
1035
|
+
f"reading exact sample counts should expect this small "
|
|
1036
|
+
f"residual gap.",
|
|
1037
|
+
kind="joined_audio_short_after_mux",
|
|
1038
|
+
file=os.path.basename(output_path),
|
|
1039
|
+
shortfall_samples=shortfall,
|
|
1040
|
+
written_samples=written_samples,
|
|
1041
|
+
expected_samples=expected_samples,
|
|
1042
|
+
)
|
|
1043
|
+
written_peak = warn_if_written_above_full_scale(
|
|
1044
|
+
output_path,
|
|
1045
|
+
already_warned=(
|
|
1046
|
+
self._consumed_by_normalizer
|
|
1047
|
+
if content_type not in LOSSY_AUDIO_CONTENT_TYPES
|
|
1048
|
+
else (self._no_headroom_warned or self._consumed_by_normalizer)
|
|
1049
|
+
)
|
|
1050
|
+
if content_type.startswith("audio")
|
|
1051
|
+
else False,
|
|
1052
|
+
info=probed_info,
|
|
1053
|
+
lossless=content_type.startswith("audio")
|
|
1054
|
+
and content_type not in LOSSY_AUDIO_CONTENT_TYPES,
|
|
1055
|
+
)
|
|
1056
|
+
# A video's pre-encode prediction was held rather than emitted
|
|
1057
|
+
# (#174 amendment): the post-encode probe is the ground truth,
|
|
1058
|
+
# so a clean or genuinely-clipped result each get exactly one
|
|
1059
|
+
# answer - nothing here, or `audio_clipped` from the probe
|
|
1060
|
+
# itself. The only time the held prediction is worth anything is
|
|
1061
|
+
# when the probe could not measure the file at all, in which
|
|
1062
|
+
# case it is the one signal available and is surfaced late
|
|
1063
|
+
# rather than dropped silently
|
|
1064
|
+
#
|
|
1065
|
+
# Note the gap this leaves: a written peak between
|
|
1066
|
+
# HEADROOM_WARN_DBFS (around -0.5) and CLIPPED_WARN_DBFS (0.0)
|
|
1067
|
+
# produces no warning here - the post-encode probe only speaks
|
|
1068
|
+
# when the file is genuinely at or over full scale, so a mux
|
|
1069
|
+
# that predicted risk but measured merely close-but-clean says
|
|
1070
|
+
# nothing. That is the file's own ground truth, not a threshold
|
|
1071
|
+
# bug.
|
|
1072
|
+
if (
|
|
1073
|
+
content_type.startswith("video")
|
|
1074
|
+
and self._no_headroom_warned
|
|
1075
|
+
and written_peak is None
|
|
1076
|
+
):
|
|
1077
|
+
emit_warning(
|
|
1078
|
+
f"The soundtrack written to {os.path.basename(output_path)} "
|
|
1079
|
+
f"was predicted to peak at {self._predicted_peak_dbfs:+.1f} "
|
|
1080
|
+
f"dBFS before encoding, and the written file could not be "
|
|
1081
|
+
f"re-measured to confirm whether the mux corrected it - add "
|
|
1082
|
+
f"a 'normalize_audio' step (peak_dbfs: -1) before the step "
|
|
1083
|
+
f"that saves it, or 'match_levels' on the join that made it.",
|
|
1084
|
+
kind="audio_no_headroom",
|
|
1085
|
+
file=os.path.basename(output_path),
|
|
1086
|
+
peak_dbfs=round(self._predicted_peak_dbfs, 2),
|
|
1087
|
+
)
|
|
1088
|
+
source_mean_dbfs = getattr(artifact, "source_mean_dbfs", None)
|
|
1089
|
+
warn_if_written_near_silent(
|
|
1090
|
+
output_path,
|
|
1091
|
+
info=probed_info,
|
|
1092
|
+
source_already_quiet=(
|
|
1093
|
+
source_mean_dbfs is not None
|
|
1094
|
+
and source_mean_dbfs < NEAR_SILENT_WARN_DBFS
|
|
1095
|
+
),
|
|
1096
|
+
)
|
|
1097
|
+
|
|
1098
|
+
emit_log(
|
|
1099
|
+
f"wrote {os.path.basename(output_path)} in "
|
|
1100
|
+
f"{time.monotonic() - started:.1f}s ({_file_size_mb(output_path):.1f} MB)",
|
|
1101
|
+
file=os.path.basename(output_path),
|
|
1102
|
+
seconds=round(time.monotonic() - started, 1),
|
|
1103
|
+
)
|
|
1104
|
+
return [output_path]
|
|
1105
|
+
|
|
1106
|
+
def video_fps(self, artifact):
|
|
1107
|
+
"""The frame rate this video is written at.
|
|
1108
|
+
|
|
1109
|
+
Declared `result.fps` first, then the rate the artifact carries (a
|
|
1110
|
+
join's own rate, or the rate of the files it read), then 8.
|
|
1111
|
+
|
|
1112
|
+
The order matters more than it looks: `result.fps` and a task's own
|
|
1113
|
+
`fps` argument are separate knobs, and the one an author thinks to
|
|
1114
|
+
set is the task's. A step that told `concat_videos` its shots are 24
|
|
1115
|
+
fps and said nothing on `result` used to write them at 8 - the
|
|
1116
|
+
picture three times long against an audio track still the right
|
|
1117
|
+
length, with nothing said about it (#84). A workflow that does
|
|
1118
|
+
declare `result.fps` still wins, so writing at a rate other than the
|
|
1119
|
+
source's - a deliberate slow motion - stays available, and says so.
|
|
1120
|
+
"""
|
|
1121
|
+
declared = self.result_definition.get("fps")
|
|
1122
|
+
carried = getattr(artifact, "fps", None)
|
|
1123
|
+
if declared is None:
|
|
1124
|
+
return carried or DEFAULT_VIDEO_FPS
|
|
1125
|
+
if carried and abs(declared - carried) > 0.01:
|
|
1126
|
+
emit_warning(
|
|
1127
|
+
f"Writing video at {declared} fps, but the frames it was "
|
|
1128
|
+
f"given run at {carried} fps - the file will play "
|
|
1129
|
+
f"{declared / carried:.2g}x speed "
|
|
1130
|
+
f"({carried / declared:.2g} times as long). Drop 'fps' from "
|
|
1131
|
+
f"the step's result to keep the source rate",
|
|
1132
|
+
kind="fps_mismatch",
|
|
1133
|
+
declared_fps=declared,
|
|
1134
|
+
source_fps=carried,
|
|
1135
|
+
)
|
|
1136
|
+
return declared
|
|
1137
|
+
|
|
1138
|
+
def save_audio_video(self, artifact, output_path, content_type):
|
|
1139
|
+
"""Write a video and the audio generated with it into a single file.
|
|
1140
|
+
|
|
1141
|
+
encode_video muxes the two into an h264/mp4 file with PyAV. When PyAV is missing,
|
|
1142
|
+
the container is not mp4, or nothing told us the sample rate, the video is written
|
|
1143
|
+
on its own and the audio is dropped.
|
|
1144
|
+
|
|
1145
|
+
Args:
|
|
1146
|
+
artifact: AudioVideo holding the frames and their waveform
|
|
1147
|
+
output_path: Path of the file to write
|
|
1148
|
+
content_type: MIME type of the video being written
|
|
1149
|
+
"""
|
|
1150
|
+
fps = self.video_fps(artifact)
|
|
1151
|
+
# The pipeline reports the sample rate of what it generated - the result
|
|
1152
|
+
# definition can still override it
|
|
1153
|
+
sample_rate = self.result_definition.get(
|
|
1154
|
+
"audio_sample_rate", artifact.sample_rate
|
|
1155
|
+
)
|
|
1156
|
+
audio = artifact.audio
|
|
1157
|
+
# Frames generated in memory (a previous_result-chained shot, a modular
|
|
1158
|
+
# pipeline's own output) never went through _decode_audio_video, so a
|
|
1159
|
+
# codec-padding mismatch between the audio and the frame count survives
|
|
1160
|
+
# here instead of being trimmed there (#197). Segment-backed frames are
|
|
1161
|
+
# written by the chain pipeline processor, which already carries its own
|
|
1162
|
+
# fps and does its own segment-length accounting - left alone.
|
|
1163
|
+
#
|
|
1164
|
+
# Written back onto the artifact, not just the local used for muxing:
|
|
1165
|
+
# this same AudioVideo instance is what a later previous_result:
|
|
1166
|
+
# consumer (concat_videos, dissolve_videos) reads directly out of the
|
|
1167
|
+
# result store, so a local-only fit left the artifact's own audio
|
|
1168
|
+
# unfitted and a chain built from in-memory shots still drifted even
|
|
1169
|
+
# though each shot's own saved file was correct (#197 reopened).
|
|
1170
|
+
if (
|
|
1171
|
+
audio is not None
|
|
1172
|
+
and sample_rate is not None
|
|
1173
|
+
and fps
|
|
1174
|
+
and not hasattr(artifact.frames, "cleanup")
|
|
1175
|
+
):
|
|
1176
|
+
from .tasks.video_utils import _fit_audio_to_frames, _sample_axis
|
|
1177
|
+
|
|
1178
|
+
fitted = _fit_audio_to_frames(audio, len(artifact.frames), fps, sample_rate)
|
|
1179
|
+
axis = _sample_axis(audio)
|
|
1180
|
+
if (
|
|
1181
|
+
axis is not None
|
|
1182
|
+
and fitted.shape[axis] != audio.shape[axis]
|
|
1183
|
+
and artifact.shots
|
|
1184
|
+
):
|
|
1185
|
+
# The fit trims or pads to the frames' own duration - a shot
|
|
1186
|
+
# map measured against the pre-fit track (#426) now overruns
|
|
1187
|
+
# or falls short of what actually gets written, so it is
|
|
1188
|
+
# re-measured against the same length the mux will see
|
|
1189
|
+
from .shots import measured_num_samples
|
|
1190
|
+
|
|
1191
|
+
measured_num_samples(artifact.shots, fitted.shape[axis])
|
|
1192
|
+
audio = fitted
|
|
1193
|
+
artifact.audio = audio
|
|
1194
|
+
|
|
1195
|
+
# Segment-backed frames (a chained step with save_segments) replay from
|
|
1196
|
+
# disk one segment at a time, so the final video is streamed instead of
|
|
1197
|
+
# materialized - and the segment files are removed once it is written
|
|
1198
|
+
if hasattr(artifact.frames, "cleanup"):
|
|
1199
|
+
if content_type != MUXED_VIDEO_CONTENT_TYPE:
|
|
1200
|
+
raise ValueError(
|
|
1201
|
+
f"Segment-backed video can only be written as "
|
|
1202
|
+
f"{MUXED_VIDEO_CONTENT_TYPE}, not {content_type}"
|
|
1203
|
+
)
|
|
1204
|
+
if not is_av_available():
|
|
1205
|
+
raise ValueError(
|
|
1206
|
+
"Writing segment-backed video needs PyAV - install it "
|
|
1207
|
+
"with: pip install av"
|
|
1208
|
+
)
|
|
1209
|
+
|
|
1210
|
+
audio = None
|
|
1211
|
+
if artifact.audio is not None and sample_rate is not None:
|
|
1212
|
+
audio = as_audio_track(artifact.audio)
|
|
1213
|
+
self._predicted_peak_dbfs = warn_without_headroom(
|
|
1214
|
+
artifact.audio, os.path.basename(output_path), emit=False
|
|
1215
|
+
)
|
|
1216
|
+
self._no_headroom_warned = self._predicted_peak_dbfs is not None
|
|
1217
|
+
logger.debug(
|
|
1218
|
+
f"Streaming {len(artifact.frames)} segments into {output_path}"
|
|
1219
|
+
)
|
|
1220
|
+
encode_video(
|
|
1221
|
+
iter(artifact.frames),
|
|
1222
|
+
fps=fps,
|
|
1223
|
+
output_path=output_path,
|
|
1224
|
+
audio=audio,
|
|
1225
|
+
audio_sample_rate=sample_rate if audio is not None else None,
|
|
1226
|
+
video_chunks_number=len(artifact.frames),
|
|
1227
|
+
)
|
|
1228
|
+
artifact.frames.cleanup()
|
|
1229
|
+
return
|
|
1230
|
+
|
|
1231
|
+
reason = None
|
|
1232
|
+
if audio is None:
|
|
1233
|
+
reason = "the pipeline returned no audio"
|
|
1234
|
+
elif sample_rate is None:
|
|
1235
|
+
reason = "the audio sample rate is unknown"
|
|
1236
|
+
elif content_type != MUXED_VIDEO_CONTENT_TYPE:
|
|
1237
|
+
reason = f"audio can only be muxed into {MUXED_VIDEO_CONTENT_TYPE}"
|
|
1238
|
+
elif not is_av_available():
|
|
1239
|
+
reason = "PyAV is not installed - install it with: pip install av"
|
|
1240
|
+
|
|
1241
|
+
if reason is not None:
|
|
1242
|
+
# No audio at all is an expected shape - video-only chains and
|
|
1243
|
+
# concatenations - so it logs quietly; losing audio we do have warns
|
|
1244
|
+
log = logger.debug if audio is None else logger.warning
|
|
1245
|
+
log(f"Saving {output_path} without its audio because {reason}")
|
|
1246
|
+
export_to_video(artifact.frames, output_path, fps=fps)
|
|
1247
|
+
return
|
|
1248
|
+
|
|
1249
|
+
logger.debug(f"Muxing audio at {sample_rate}Hz into {output_path}")
|
|
1250
|
+
self._predicted_peak_dbfs = warn_without_headroom(
|
|
1251
|
+
audio, os.path.basename(output_path), emit=False
|
|
1252
|
+
)
|
|
1253
|
+
self._no_headroom_warned = self._predicted_peak_dbfs is not None
|
|
1254
|
+
encode_video(
|
|
1255
|
+
frames_for_encoding(artifact.frames),
|
|
1256
|
+
fps=fps,
|
|
1257
|
+
output_path=output_path,
|
|
1258
|
+
audio=as_audio_track(audio),
|
|
1259
|
+
audio_sample_rate=sample_rate,
|
|
1260
|
+
)
|
|
1261
|
+
|
|
1262
|
+
def get_audio_write_arguments(self, content_type):
|
|
1263
|
+
"""Collect the soundfile arguments for an audio content type.
|
|
1264
|
+
|
|
1265
|
+
The container comes from the content type and any encoding quality settings
|
|
1266
|
+
from the result definition.
|
|
1267
|
+
|
|
1268
|
+
Args:
|
|
1269
|
+
content_type: MIME type of the audio being written
|
|
1270
|
+
|
|
1271
|
+
Returns:
|
|
1272
|
+
Dict of keyword arguments for soundfile.write
|
|
1273
|
+
"""
|
|
1274
|
+
_, write_arguments = AUDIO_FORMATS.get(content_type, (None, {}))
|
|
1275
|
+
write_arguments = dict(write_arguments)
|
|
1276
|
+
|
|
1277
|
+
for argument_name in AUDIO_WRITE_ARGUMENTS:
|
|
1278
|
+
value = self.result_definition.get(argument_name, None)
|
|
1279
|
+
if value is not None:
|
|
1280
|
+
write_arguments[argument_name] = value
|
|
1281
|
+
|
|
1282
|
+
return write_arguments
|
|
1283
|
+
|
|
1284
|
+
def _save_image_with_metadata(self, image, output_path, content_type):
|
|
1285
|
+
"""Save an image with embedded generation metadata.
|
|
1286
|
+
|
|
1287
|
+
Args:
|
|
1288
|
+
image: PIL Image to save
|
|
1289
|
+
output_path: Path to save the image to
|
|
1290
|
+
content_type: MIME type of the image
|
|
1291
|
+
"""
|
|
1292
|
+
metadata_json = json.dumps(self.metadata, default=str)
|
|
1293
|
+
|
|
1294
|
+
if content_type == "image/png":
|
|
1295
|
+
from PIL.PngImagePlugin import PngInfo
|
|
1296
|
+
|
|
1297
|
+
png_info = PngInfo()
|
|
1298
|
+
png_info.add_text("parameters", metadata_json)
|
|
1299
|
+
image.save(output_path, pnginfo=png_info)
|
|
1300
|
+
logger.debug(f"Embedded PNG metadata in {output_path}")
|
|
1301
|
+
elif content_type in ("image/jpeg", "image/webp"):
|
|
1302
|
+
try:
|
|
1303
|
+
import piexif
|
|
1304
|
+
import piexif.helper
|
|
1305
|
+
|
|
1306
|
+
exif_dict = {"0th": {}, "Exif": {}, "GPS": {}, "1st": {}}
|
|
1307
|
+
if hasattr(image, "info") and "exif" in image.info:
|
|
1308
|
+
exif_dict = piexif.load(image.info["exif"])
|
|
1309
|
+
exif_dict["Exif"][piexif.ExifIFD.UserComment] = (
|
|
1310
|
+
piexif.helper.UserComment.dump(metadata_json)
|
|
1311
|
+
)
|
|
1312
|
+
exif_bytes = piexif.dump(exif_dict)
|
|
1313
|
+
image.save(output_path, exif=exif_bytes)
|
|
1314
|
+
logger.debug(f"Embedded EXIF metadata in {output_path}")
|
|
1315
|
+
except ImportError:
|
|
1316
|
+
logger.warning(
|
|
1317
|
+
"piexif not installed - saving without metadata. "
|
|
1318
|
+
"Install with: pip install piexif"
|
|
1319
|
+
)
|
|
1320
|
+
image.save(output_path)
|
|
1321
|
+
else:
|
|
1322
|
+
image.save(output_path)
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
def read_embedded_metadata(path):
|
|
1326
|
+
"""The generation metadata a saved image carries, or None.
|
|
1327
|
+
|
|
1328
|
+
The read-side mirror of _save_image_with_metadata: the 'parameters' PNG
|
|
1329
|
+
text chunk, or the EXIF UserComment for JPEG/WebP. Returns the parsed
|
|
1330
|
+
dict, or None when the file has no metadata this writer produced.
|
|
1331
|
+
"""
|
|
1332
|
+
try:
|
|
1333
|
+
from PIL import Image
|
|
1334
|
+
|
|
1335
|
+
with Image.open(path) as image:
|
|
1336
|
+
# image.text would load() the whole image to reach chunks after
|
|
1337
|
+
# IDAT; the writer puts its chunk before IDAT, where info holds it
|
|
1338
|
+
if image.width * image.height > MAX_DECODE_PIXELS:
|
|
1339
|
+
return None
|
|
1340
|
+
text = image.info.get("parameters")
|
|
1341
|
+
if text is None and "exif" in getattr(image, "info", {}):
|
|
1342
|
+
import piexif
|
|
1343
|
+
import piexif.helper
|
|
1344
|
+
|
|
1345
|
+
exif = piexif.load(image.info["exif"])
|
|
1346
|
+
comment = exif.get("Exif", {}).get(piexif.ExifIFD.UserComment)
|
|
1347
|
+
if comment:
|
|
1348
|
+
text = piexif.helper.UserComment.load(comment)
|
|
1349
|
+
if text is None:
|
|
1350
|
+
return None
|
|
1351
|
+
parsed = json.loads(text)
|
|
1352
|
+
return parsed if isinstance(parsed, dict) else None
|
|
1353
|
+
except Exception as e:
|
|
1354
|
+
logger.debug(f"No readable metadata in {path}: {e}")
|
|
1355
|
+
return None
|
|
1356
|
+
|
|
1357
|
+
|
|
1358
|
+
def _frames_from_attributes(result):
|
|
1359
|
+
"""The frames extractor for a pipeline output that carries `.frames` directly.
|
|
1360
|
+
|
|
1361
|
+
Some video pipelines (LTX-2) generate an audio track along with the frames, exposed
|
|
1362
|
+
as `.audio` and `.audio_sample_rate` attributes alongside `.frames`.
|
|
1363
|
+
"""
|
|
1364
|
+
return frames_with_audio(
|
|
1365
|
+
result.frames,
|
|
1366
|
+
getattr(result, "audio", None),
|
|
1367
|
+
getattr(result, "audio_sample_rate", None),
|
|
1368
|
+
)
|
|
1369
|
+
|
|
1370
|
+
|
|
1371
|
+
def _audios_from_attribute(result):
|
|
1372
|
+
"""Each `.audios` item as an artifact.
|
|
1373
|
+
|
|
1374
|
+
With the rate attach_audio_sample_rate recorded, each item becomes an
|
|
1375
|
+
AudioTrack carrying it, shaped (channels, samples); without one, the bare
|
|
1376
|
+
(samples, channels) array it always was, and the workflow's 'sample_rate'
|
|
1377
|
+
(or the default) applies at save.
|
|
1378
|
+
"""
|
|
1379
|
+
sample_rate = getattr(result, "audio_sample_rate", None)
|
|
1380
|
+
if sample_rate is None:
|
|
1381
|
+
return [as_waveform_array(audio) for audio in result.audios]
|
|
1382
|
+
return [
|
|
1383
|
+
AudioTrack(as_waveform_array(audio).T, int(sample_rate))
|
|
1384
|
+
for audio in result.audios
|
|
1385
|
+
]
|
|
1386
|
+
|
|
1387
|
+
|
|
1388
|
+
# Diffusers output fields get_artifact_list knows how to turn into artifacts, tried in
|
|
1389
|
+
# this order. A result is dispatched to the first field it has - images wins over frames
|
|
1390
|
+
# if a result somehow has both, matching the fixed hasattr chain this replaced. Supporting
|
|
1391
|
+
# a new diffusers output field (e.g. a standalone "depth" attribute) is one more entry
|
|
1392
|
+
# here, instead of another branch threaded through the chain.
|
|
1393
|
+
OUTPUT_FIELD_EXTRACTORS = [
|
|
1394
|
+
("images", lambda result: result.images),
|
|
1395
|
+
("image_embeds", lambda result: result.image_embeds),
|
|
1396
|
+
("image_embeddings", lambda result: result.image_embeddings),
|
|
1397
|
+
("frames", _frames_from_attributes),
|
|
1398
|
+
("audios", _audios_from_attribute),
|
|
1399
|
+
]
|
|
1400
|
+
|
|
1401
|
+
|
|
1402
|
+
def get_artifact_list(result):
|
|
1403
|
+
"""Extract list of artifacts from a result object.
|
|
1404
|
+
|
|
1405
|
+
Handles various result types including images, embeddings, frames, and audio.
|
|
1406
|
+
|
|
1407
|
+
Args:
|
|
1408
|
+
result: Result object to extract artifacts from
|
|
1409
|
+
|
|
1410
|
+
Returns:
|
|
1411
|
+
List of artifacts
|
|
1412
|
+
"""
|
|
1413
|
+
# Already a paired video and audio track - it has a .frames attribute of its own,
|
|
1414
|
+
# but the pair is one artifact, not something to run back through frame extraction
|
|
1415
|
+
if isinstance(result, AudioVideo):
|
|
1416
|
+
return [result]
|
|
1417
|
+
|
|
1418
|
+
for field_name, extract in OUTPUT_FIELD_EXTRACTORS:
|
|
1419
|
+
if hasattr(result, field_name):
|
|
1420
|
+
return extract(result)
|
|
1421
|
+
|
|
1422
|
+
if isinstance(result, dict):
|
|
1423
|
+
# A modular pipeline asked for several outputs returns them keyed
|
|
1424
|
+
artifacts = modular_artifacts(result)
|
|
1425
|
+
if artifacts is not None:
|
|
1426
|
+
return artifacts
|
|
1427
|
+
|
|
1428
|
+
if isinstance(result, list):
|
|
1429
|
+
return result
|
|
1430
|
+
|
|
1431
|
+
if hasattr(result, "to_tuple") or hasattr(result, "__dataclass_fields__"):
|
|
1432
|
+
# A diffusers output (BaseOutput subclasses have to_tuple; plain dataclasses
|
|
1433
|
+
# have __dataclass_fields__) whose fields matched none of the extractors above -
|
|
1434
|
+
# log what it actually looks like so the resulting content-type mismatch on save
|
|
1435
|
+
# is diagnosable instead of a bare "does not match result type" surprise.
|
|
1436
|
+
logger.warning(
|
|
1437
|
+
f"Don't know how to extract artifacts from a {type(result).__name__} - "
|
|
1438
|
+
f"treating it as a single artifact. Its fields are: {output_field_names(result)}"
|
|
1439
|
+
)
|
|
1440
|
+
|
|
1441
|
+
return [result]
|
|
1442
|
+
|
|
1443
|
+
|
|
1444
|
+
def output_field_names(result):
|
|
1445
|
+
"""Best-effort list of field names on a diffusers-style output object, for logging."""
|
|
1446
|
+
if hasattr(result, "keys"):
|
|
1447
|
+
return list(result.keys())
|
|
1448
|
+
if hasattr(result, "__dataclass_fields__"):
|
|
1449
|
+
return list(result.__dataclass_fields__.keys())
|
|
1450
|
+
return []
|
|
1451
|
+
|
|
1452
|
+
|
|
1453
|
+
def modular_artifacts(result):
|
|
1454
|
+
"""Extract the artifacts from the outputs a modular pipeline returns together.
|
|
1455
|
+
|
|
1456
|
+
Asked for several outputs - `"output": ["videos", "audio", "sampling_rate"]` - a
|
|
1457
|
+
modular pipeline returns them in a dict instead of on one output object. Pairing the
|
|
1458
|
+
videos with the audio generated alongside them here saves them the same way a video
|
|
1459
|
+
pipeline's own output is saved, muxed into a single file. Any other requested output
|
|
1460
|
+
- "images" or "latents", say - is not part of that pairing, so it is carried along as
|
|
1461
|
+
one extra dict artifact, saved key by key the same way any other dictionary result is.
|
|
1462
|
+
|
|
1463
|
+
Args:
|
|
1464
|
+
result: Dict of outputs returned by a modular pipeline
|
|
1465
|
+
|
|
1466
|
+
Returns:
|
|
1467
|
+
List of artifacts, or None when the outputs hold no video - those are saved one
|
|
1468
|
+
output at a time instead
|
|
1469
|
+
"""
|
|
1470
|
+
video_key, videos = first_item(result, MODULAR_VIDEO_KEYS)
|
|
1471
|
+
if videos is None:
|
|
1472
|
+
return None
|
|
1473
|
+
|
|
1474
|
+
consumed_keys = {video_key}
|
|
1475
|
+
|
|
1476
|
+
audio_key, audio = first_item(result, MODULAR_AUDIO_KEYS)
|
|
1477
|
+
sample_rate = None
|
|
1478
|
+
if audio is not None:
|
|
1479
|
+
consumed_keys.add(audio_key)
|
|
1480
|
+
rate_key, sample_rate = first_item(result, MODULAR_SAMPLE_RATE_KEYS)
|
|
1481
|
+
consumed_keys.add(rate_key)
|
|
1482
|
+
|
|
1483
|
+
artifacts = frames_with_audio(videos, audio, sample_rate)
|
|
1484
|
+
|
|
1485
|
+
# Keys the video/audio pairing above did not consume still need to be saved, not
|
|
1486
|
+
# dropped - carry them along as one extra artifact, saved key by key like any other
|
|
1487
|
+
# dictionary result
|
|
1488
|
+
leftovers = {
|
|
1489
|
+
key: value
|
|
1490
|
+
for key, value in result.items()
|
|
1491
|
+
if key not in consumed_keys and value is not None
|
|
1492
|
+
}
|
|
1493
|
+
if leftovers:
|
|
1494
|
+
artifacts = list(artifacts) + [leftovers]
|
|
1495
|
+
|
|
1496
|
+
return artifacts
|
|
1497
|
+
|
|
1498
|
+
|
|
1499
|
+
def first_item(values, keys):
|
|
1500
|
+
"""The key and value of the first of `keys` present in `values`.
|
|
1501
|
+
|
|
1502
|
+
Returns (None, None) when none of them are.
|
|
1503
|
+
"""
|
|
1504
|
+
for key in keys:
|
|
1505
|
+
value = values.get(key, None)
|
|
1506
|
+
if value is not None:
|
|
1507
|
+
return key, value
|
|
1508
|
+
|
|
1509
|
+
return None, None
|
|
1510
|
+
|
|
1511
|
+
|
|
1512
|
+
def frames_with_audio(frames, audio, sample_rate):
|
|
1513
|
+
"""Pair frames with the audio track generated alongside them, if there is one.
|
|
1514
|
+
|
|
1515
|
+
The one place that decides whether frames need pairing with audio at all - used by
|
|
1516
|
+
both routes a pipeline's frames-plus-audio output can take: attributes on a pipeline
|
|
1517
|
+
output object (`_frames_from_attributes`), and keys in the dict a modular pipeline
|
|
1518
|
+
returns (`modular_artifacts`). Frames without audio are returned unchanged; actually
|
|
1519
|
+
pairing them is `pair_audio_with_frames`'s job.
|
|
1520
|
+
|
|
1521
|
+
Args:
|
|
1522
|
+
frames: The generated video(s), one list of frames per generation
|
|
1523
|
+
audio: The generated waveform(s), or None if the pipeline produced no audio
|
|
1524
|
+
sample_rate: Sample rate of the waveform(s), or None if unknown
|
|
1525
|
+
|
|
1526
|
+
Returns:
|
|
1527
|
+
`frames` unchanged if `audio` is None, otherwise the list of AudioVideo pairs
|
|
1528
|
+
`pair_audio_with_frames` produces
|
|
1529
|
+
"""
|
|
1530
|
+
if audio is None:
|
|
1531
|
+
return frames
|
|
1532
|
+
return pair_audio_with_frames(frames, audio, sample_rate)
|
|
1533
|
+
|
|
1534
|
+
|
|
1535
|
+
def pair_audio_with_frames(videos, audio, sample_rate):
|
|
1536
|
+
"""Pair each generated video with its own audio track.
|
|
1537
|
+
|
|
1538
|
+
Both are batched - videos[i] and audio[i] belong to the same generation. Only the
|
|
1539
|
+
pipeline knows the sample rate its vocoder produced the audio at, so it comes along
|
|
1540
|
+
rather than being guessed at here.
|
|
1541
|
+
|
|
1542
|
+
Args:
|
|
1543
|
+
videos: The generated videos, one list of frames per generation
|
|
1544
|
+
audio: The generated waveforms, one per generation
|
|
1545
|
+
sample_rate: Sample rate of the waveforms, or None if the pipeline did not report one
|
|
1546
|
+
|
|
1547
|
+
Returns:
|
|
1548
|
+
List of AudioVideo artifacts, one per generated video
|
|
1549
|
+
"""
|
|
1550
|
+
return [
|
|
1551
|
+
AudioVideo(frames, audio[i] if i < len(audio) else None, sample_rate)
|
|
1552
|
+
for i, frames in enumerate(videos)
|
|
1553
|
+
]
|
|
1554
|
+
|
|
1555
|
+
|
|
1556
|
+
def as_waveform_array(audio):
|
|
1557
|
+
"""Transpose one batch item of a pipeline's `.audios` output to (samples, channels).
|
|
1558
|
+
|
|
1559
|
+
`.audios` is shaped (batch, channels, samples). Under the diffusers default
|
|
1560
|
+
`output_type='np'`, pipelines such as AudioLDM2 and StableAudio already call
|
|
1561
|
+
`.numpy()` before returning, so each item here is a numpy ndarray rather than a
|
|
1562
|
+
torch tensor - it has no `.float()`/`.cpu()` methods, only `.T`/`.astype()`.
|
|
1563
|
+
|
|
1564
|
+
Args:
|
|
1565
|
+
audio: One batch item, shaped (channels, samples), as a torch tensor or numpy array
|
|
1566
|
+
|
|
1567
|
+
Returns:
|
|
1568
|
+
Numpy float32 array shaped (samples, channels)
|
|
1569
|
+
"""
|
|
1570
|
+
if isinstance(audio, torch.Tensor):
|
|
1571
|
+
return audio.T.float().cpu().numpy()
|
|
1572
|
+
|
|
1573
|
+
return numpy.asarray(audio).T.astype(numpy.float32, copy=False)
|
|
1574
|
+
|
|
1575
|
+
|
|
1576
|
+
def as_audio_track(audio):
|
|
1577
|
+
"""Convert a generated waveform into the tensor encode_video expects.
|
|
1578
|
+
|
|
1579
|
+
encode_video wants a float torch tensor on the CPU shaped (channels, samples) -
|
|
1580
|
+
pipelines hand back bfloat16 tensors that are still on the GPU, or numpy arrays.
|
|
1581
|
+
|
|
1582
|
+
A mono track is duplicated into two channels, because the mp4 audio stream
|
|
1583
|
+
takes nothing else: diffusers' _write_audio refuses any other channel count
|
|
1584
|
+
with a raw tensor shape, and a mono voice track paired onto a picture is the
|
|
1585
|
+
ordinary case, not an edge one (#106). Duplicating one channel is lossless,
|
|
1586
|
+
but it is a change to what was handed in, so it warns.
|
|
1587
|
+
|
|
1588
|
+
Args:
|
|
1589
|
+
audio: Waveform as a torch tensor or numpy array
|
|
1590
|
+
|
|
1591
|
+
Returns:
|
|
1592
|
+
Float CPU torch tensor holding the waveform, with at least 2 channels
|
|
1593
|
+
"""
|
|
1594
|
+
if not isinstance(audio, torch.Tensor):
|
|
1595
|
+
audio = torch.from_numpy(numpy.asarray(audio))
|
|
1596
|
+
|
|
1597
|
+
audio = audio.detach().float().cpu()
|
|
1598
|
+
return _as_stereo(audio)
|
|
1599
|
+
|
|
1600
|
+
|
|
1601
|
+
def _as_stereo(audio):
|
|
1602
|
+
"""Duplicate a mono waveform into two channels, leaving anything else alone.
|
|
1603
|
+
|
|
1604
|
+
Orientation is read the way encode_video reads it: (samples,) and (1, samples)
|
|
1605
|
+
are mono, and so is (samples, 1) - a lone channel laid out samples-first.
|
|
1606
|
+
A 2-channel waveform in either orientation is already what the encoder takes
|
|
1607
|
+
and is handed back untouched, so a short stereo track is never mistaken for
|
|
1608
|
+
many channels of one sample.
|
|
1609
|
+
"""
|
|
1610
|
+
if audio.ndim == 2:
|
|
1611
|
+
if audio.shape[0] == 2 or audio.shape[1] == 2:
|
|
1612
|
+
return audio
|
|
1613
|
+
if audio.shape[0] == 1:
|
|
1614
|
+
audio = audio[0]
|
|
1615
|
+
elif audio.shape[1] == 1:
|
|
1616
|
+
audio = audio[:, 0]
|
|
1617
|
+
else:
|
|
1618
|
+
return audio
|
|
1619
|
+
elif audio.ndim != 1:
|
|
1620
|
+
return audio
|
|
1621
|
+
|
|
1622
|
+
emit_warning(
|
|
1623
|
+
f"Duplicating a mono audio track of {audio.shape[0]} samples into two "
|
|
1624
|
+
f"channels - an mp4 audio stream takes stereo and nothing else"
|
|
1625
|
+
)
|
|
1626
|
+
return audio.unsqueeze(0).repeat(2, 1)
|
|
1627
|
+
|
|
1628
|
+
|
|
1629
|
+
def write_audio(output_path, waveform, sample_rate, **write_arguments):
|
|
1630
|
+
"""Write a single waveform to disk.
|
|
1631
|
+
|
|
1632
|
+
soundfile.write() hands the whole waveform to libsndfile in one call, whose vorbis
|
|
1633
|
+
encoder segfaults past 2**21 frames - about 48 seconds of 44.1kHz audio. Writing in
|
|
1634
|
+
chunks avoids that and bounds the encoder's working set for long audio.
|
|
1635
|
+
|
|
1636
|
+
Args:
|
|
1637
|
+
output_path: Path of the file to write
|
|
1638
|
+
waveform: Numpy array shaped (samples,) or (samples, channels)
|
|
1639
|
+
sample_rate: Sample rate to record in the file
|
|
1640
|
+
write_arguments: Container and encoding arguments for soundfile
|
|
1641
|
+
"""
|
|
1642
|
+
channels = 1 if waveform.ndim == 1 else waveform.shape[1]
|
|
1643
|
+
|
|
1644
|
+
with soundfile.SoundFile(
|
|
1645
|
+
output_path,
|
|
1646
|
+
"w",
|
|
1647
|
+
samplerate=sample_rate,
|
|
1648
|
+
channels=channels,
|
|
1649
|
+
**write_arguments,
|
|
1650
|
+
) as audio_file:
|
|
1651
|
+
for start in range(0, len(waveform), AUDIO_WRITE_CHUNK_FRAMES):
|
|
1652
|
+
audio_file.write(waveform[start : start + AUDIO_WRITE_CHUNK_FRAMES])
|
|
1653
|
+
|
|
1654
|
+
|
|
1655
|
+
def normalize_audio(artifact):
|
|
1656
|
+
"""Convert an audio artifact into waveforms soundfile can write.
|
|
1657
|
+
|
|
1658
|
+
Pipelines return audio as torch tensors or numpy arrays, channels first and
|
|
1659
|
+
optionally batched. soundfile wants samples first, one waveform at a time.
|
|
1660
|
+
|
|
1661
|
+
Args:
|
|
1662
|
+
artifact: Audio waveform(s) as a torch tensor or numpy array, or anything
|
|
1663
|
+
carrying one as '.audio' - an AudioTrack, or a video whose soundtrack
|
|
1664
|
+
is what is being saved
|
|
1665
|
+
|
|
1666
|
+
Returns:
|
|
1667
|
+
List of numpy arrays shaped (samples,) or (samples, channels)
|
|
1668
|
+
"""
|
|
1669
|
+
if hasattr(artifact, "audio"):
|
|
1670
|
+
if artifact.audio is None:
|
|
1671
|
+
raise ValueError(
|
|
1672
|
+
f"Cannot save a {type(artifact).__name__} as audio - it carries no audio track"
|
|
1673
|
+
)
|
|
1674
|
+
artifact = artifact.audio
|
|
1675
|
+
|
|
1676
|
+
# Torch tensors may be on the GPU and in a dtype numpy does not understand
|
|
1677
|
+
if hasattr(artifact, "detach"):
|
|
1678
|
+
artifact = artifact.detach().float().cpu().numpy()
|
|
1679
|
+
|
|
1680
|
+
waveform = numpy.asarray(artifact)
|
|
1681
|
+
|
|
1682
|
+
if waveform.ndim == 1:
|
|
1683
|
+
return [waveform]
|
|
1684
|
+
|
|
1685
|
+
if waveform.ndim == 2:
|
|
1686
|
+
# Channels first - a waveform always has far more samples than channels
|
|
1687
|
+
if waveform.shape[0] < waveform.shape[1]:
|
|
1688
|
+
waveform = waveform.T
|
|
1689
|
+
return [waveform]
|
|
1690
|
+
|
|
1691
|
+
if waveform.ndim == 3:
|
|
1692
|
+
# (batch, channels, samples) - one waveform per batch item
|
|
1693
|
+
return [item.T for item in waveform]
|
|
1694
|
+
|
|
1695
|
+
raise ValueError(f"Cannot save audio with shape {waveform.shape}")
|
|
1696
|
+
|
|
1697
|
+
|
|
1698
|
+
def guess_extension(content_type):
|
|
1699
|
+
"""Determine file extension from MIME type.
|
|
1700
|
+
|
|
1701
|
+
Args:
|
|
1702
|
+
content_type: MIME type string
|
|
1703
|
+
|
|
1704
|
+
Returns:
|
|
1705
|
+
String containing file extension with leading dot
|
|
1706
|
+
"""
|
|
1707
|
+
if not content_type:
|
|
1708
|
+
logger.warning("No content type provided for extension guess")
|
|
1709
|
+
return ""
|
|
1710
|
+
|
|
1711
|
+
# Audio is looked up first - soundfile picks the container from the extension and
|
|
1712
|
+
# does not recognize every extension mimetypes suggests, such as '.oga' for ogg
|
|
1713
|
+
if content_type in AUDIO_FORMATS:
|
|
1714
|
+
return AUDIO_FORMATS[content_type][0]
|
|
1715
|
+
|
|
1716
|
+
ext = mimetypes.guess_extension(content_type)
|
|
1717
|
+
if ext is not None:
|
|
1718
|
+
return ext
|
|
1719
|
+
|
|
1720
|
+
return ""
|