diffusers-workflow 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusers_workflow-0.4.0.dist-info/METADATA +318 -0
- diffusers_workflow-0.4.0.dist-info/RECORD +260 -0
- diffusers_workflow-0.4.0.dist-info/WHEEL +5 -0
- diffusers_workflow-0.4.0.dist-info/entry_points.txt +7 -0
- diffusers_workflow-0.4.0.dist-info/licenses/LICENSE +201 -0
- diffusers_workflow-0.4.0.dist-info/top_level.txt +2 -0
- dw/__init__.py +440 -0
- dw/adapter_compatibility.py +226 -0
- dw/arguments.py +1231 -0
- dw/assessment_rules.py +159 -0
- dw/assets.py +130 -0
- dw/cache_blocks.json +16 -0
- dw/cache_blocks.py +146 -0
- dw/community_pipelines/pipeline_flux_rf_inversion.py +1184 -0
- dw/content_types.py +150 -0
- dw/dissolve_frame_errors.py +121 -0
- dw/docs/ACCELERATION.md +352 -0
- dw/docs/AGENT_LOOP.md +95 -0
- dw/docs/DEPENDENCIES.md +91 -0
- dw/docs/IP_ADAPTER.md +109 -0
- dw/docs/LORAS.md +131 -0
- dw/docs/MCP.md +517 -0
- dw/docs/PROMPT_WEIGHTING.md +78 -0
- dw/docs/QUANTIZATION.md +230 -0
- dw/docs/RECIPES_24GB.md +201 -0
- dw/docs/RELEASING.md +195 -0
- dw/docs/REMOTE.md +140 -0
- dw/docs/REPL_COMMANDS.md +121 -0
- dw/docs/REPL_WORKER_GUIDE.md +51 -0
- dw/docs/SECURITY.md +272 -0
- dw/docs/SECURITY_QUICKREF.md +112 -0
- dw/docs/SERVER.md +679 -0
- dw/docs/TASKS.md +1741 -0
- dw/docs/TESTING.md +71 -0
- dw/docs/WORKFLOW_GUIDE.md +2038 -0
- dw/docs/WORKSPACES.md +316 -0
- dw/download_watch.py +335 -0
- dw/elision.py +306 -0
- dw/events.py +275 -0
- dw/for_each.py +409 -0
- dw/host_memory.py +258 -0
- dw/host_memory_projection.py +230 -0
- dw/hub_cache.py +432 -0
- dw/introspection.py +1228 -0
- dw/kernel_availability.py +208 -0
- dw/locations.py +599 -0
- dw/log_setup.py +45 -0
- dw/loudness.py +82 -0
- dw/media_audio.py +217 -0
- dw/media_frames.py +367 -0
- dw/media_info.py +297 -0
- dw/pipeline_processors/chain.py +821 -0
- dw/pipeline_processors/config_objects.py +237 -0
- dw/pipeline_processors/pipeline.py +2297 -0
- dw/pipeline_processors/remote.py +46 -0
- dw/plan.py +920 -0
- dw/previous_results.py +411 -0
- dw/probe_paths.py +59 -0
- dw/prompt_schema.json +48 -0
- dw/prompt_weighting.py +378 -0
- dw/prompts.py +159 -0
- dw/realize.py +250 -0
- dw/reference_limits.py +215 -0
- dw/reference_names.py +125 -0
- dw/repl.py +338 -0
- dw/repl_commands.py +836 -0
- dw/repl_worker.py +159 -0
- dw/result.py +1720 -0
- dw/result_fps.py +82 -0
- dw/run.py +162 -0
- dw/runs.py +768 -0
- dw/scalar_result_validation.py +97 -0
- dw/schema.py +283 -0
- dw/security.py +1038 -0
- dw/select_validation.py +115 -0
- dw/serve.py +277 -0
- dw/server/__init__.py +2 -0
- dw/server/app.py +4586 -0
- dw/server/assess.py +132 -0
- dw/server/catalog_shape.py +487 -0
- dw/server/enhancers.py +129 -0
- dw/server/exports.py +480 -0
- dw/server/guides.py +257 -0
- dw/server/jobs.py +1561 -0
- dw/server/mcp_mount.py +95 -0
- dw/server/netinfo.py +124 -0
- dw/server/observed_cost.py +379 -0
- dw/server/sysinfo.py +71 -0
- dw/server/ui/assets/abap-08VXUWAP.js +1 -0
- dw/server/ui/assets/apex-BWPQTe0t.js +1 -0
- dw/server/ui/assets/azcli-Bc_sGQ0U.js +1 -0
- dw/server/ui/assets/bat-i0X4ZdIN.js +1 -0
- dw/server/ui/assets/bicep-B5-_aFwp.js +2 -0
- dw/server/ui/assets/cameligo-DMUM7wLl.js +1 -0
- dw/server/ui/assets/clojure-Cm7r79vr.js +1 -0
- dw/server/ui/assets/codicon-Brq4_Ui5.ttf +0 -0
- dw/server/ui/assets/coffee-Ba7i2nA0.js +1 -0
- dw/server/ui/assets/cpp-C7h46wYY.js +1 -0
- dw/server/ui/assets/csharp-BKxtCVv1.js +1 -0
- dw/server/ui/assets/csp-bTuwJoIa.js +1 -0
- dw/server/ui/assets/css-DIMkf-bt.js +3 -0
- dw/server/ui/assets/css.worker-B3ciXF_0.js +93 -0
- dw/server/ui/assets/cssMode-CPznxfY8.js +1 -0
- dw/server/ui/assets/cypher-CVaqCwHa.js +1 -0
- dw/server/ui/assets/dart-onAF5SnQ.js +1 -0
- dw/server/ui/assets/dockerfile-DZFCIeNp.js +1 -0
- dw/server/ui/assets/ecl-D05T4iGw.js +1 -0
- dw/server/ui/assets/editor-jjEx9u7D.css +1 -0
- dw/server/ui/assets/editor.api-CpWcotrd.js +847 -0
- dw/server/ui/assets/editor.worker-q-txB4vs.js +30 -0
- dw/server/ui/assets/elixir-6RTg0lbw.js +1 -0
- dw/server/ui/assets/flow9-C5_-GSwl.js +1 -0
- dw/server/ui/assets/freemarker2-CXtRM8N4.js +3 -0
- dw/server/ui/assets/fsharp-C8Ef5oNN.js +1 -0
- dw/server/ui/assets/go-C-y9NEjX.js +1 -0
- dw/server/ui/assets/graphql-fmXr3nnJ.js +1 -0
- dw/server/ui/assets/handlebars-N7x-6NMY.js +1 -0
- dw/server/ui/assets/hcl-CpzslTdj.js +1 -0
- dw/server/ui/assets/html-PhsdjHSr.js +1 -0
- dw/server/ui/assets/html.worker-C93Ht9o9.js +506 -0
- dw/server/ui/assets/htmlMode-Dgj0SEok.js +1 -0
- dw/server/ui/assets/index-3Vw6WAPW.css +1 -0
- dw/server/ui/assets/index-DgrYhQd9.js +43 -0
- dw/server/ui/assets/ini-sBoK_t0W.js +1 -0
- dw/server/ui/assets/java-BEtHBSE6.js +1 -0
- dw/server/ui/assets/javascript-BJqN9Qhv.js +1 -0
- dw/server/ui/assets/json.worker-B2V3pomh.js +62 -0
- dw/server/ui/assets/jsonMode-DbM4SWSv.js +7 -0
- dw/server/ui/assets/julia-Bri6UV-V.js +1 -0
- dw/server/ui/assets/kotlin-BOotOW0E.js +1 -0
- dw/server/ui/assets/less-B9JPFI3C.js +2 -0
- dw/server/ui/assets/lexon-CfSJPG6W.js +1 -0
- dw/server/ui/assets/liquid-BWr8lEc4.js +1 -0
- dw/server/ui/assets/lspLanguageFeatures-C1iGuDyZ.js +4 -0
- dw/server/ui/assets/lua-CsQS60Ue.js +1 -0
- dw/server/ui/assets/m3-D-oSqn_W.js +1 -0
- dw/server/ui/assets/markdown-Cimd5fb3.js +1 -0
- dw/server/ui/assets/mdx-DAdMi_0p.js +1 -0
- dw/server/ui/assets/mips-CIPQ_RoX.js +1 -0
- dw/server/ui/assets/monaco--ixms01u.css +1 -0
- dw/server/ui/assets/monaco-BGCeEqaw.js +56 -0
- dw/server/ui/assets/msdax-DauUninz.js +1 -0
- dw/server/ui/assets/mysql-SOo6toE5.js +1 -0
- dw/server/ui/assets/objective-c-FvmIjYaQ.js +1 -0
- dw/server/ui/assets/pascal-DrH0SRf2.js +1 -0
- dw/server/ui/assets/pascaligo-D-ptJ9y-.js +1 -0
- dw/server/ui/assets/perl-oz_6vUea.js +1 -0
- dw/server/ui/assets/pgsql-DTj74zXo.js +1 -0
- dw/server/ui/assets/php-nr791fC2.js +1 -0
- dw/server/ui/assets/pla-CopQ2nXW.js +1 -0
- dw/server/ui/assets/postiats-43DmfD33.js +1 -0
- dw/server/ui/assets/powerquery-D3hlyOfw.js +1 -0
- dw/server/ui/assets/powershell-DmHpPYUd.js +1 -0
- dw/server/ui/assets/protobuf-C531GsRP.js +2 -0
- dw/server/ui/assets/pug-Z5eAx3Zn.js +1 -0
- dw/server/ui/assets/python-Bcn70HdC.js +1 -0
- dw/server/ui/assets/qsharp-DkqhCAOL.js +1 -0
- dw/server/ui/assets/r-BwWrilGY.js +1 -0
- dw/server/ui/assets/razor-D1HmNnby.js +1 -0
- dw/server/ui/assets/redis-ClamHrr6.js +1 -0
- dw/server/ui/assets/redshift-DT7zqm-g.js +1 -0
- dw/server/ui/assets/restructuredtext-BYgofb2h.js +1 -0
- dw/server/ui/assets/ruby-DezsRK8O.js +1 -0
- dw/server/ui/assets/rust-DdL9SqIa.js +1 -0
- dw/server/ui/assets/sb-CcwsVR0C.js +1 -0
- dw/server/ui/assets/scala-DHpiXF5c.js +1 -0
- dw/server/ui/assets/scheme-BeGwcela.js +1 -0
- dw/server/ui/assets/scss-gp-XZpBa.js +3 -0
- dw/server/ui/assets/shell-CC2rA5mh.js +1 -0
- dw/server/ui/assets/solidity-BEEn4gHE.js +1 -0
- dw/server/ui/assets/sophia-CRfGWb83.js +1 -0
- dw/server/ui/assets/sparql-D_Lu-MrJ.js +1 -0
- dw/server/ui/assets/sql-NEE52Syq.js +1 -0
- dw/server/ui/assets/st-DbInun42.js +1 -0
- dw/server/ui/assets/swift-Bxkupp3x.js +1 -0
- dw/server/ui/assets/systemverilog-Bz4Y3fRF.js +1 -0
- dw/server/ui/assets/tcl-DISqw1ZD.js +1 -0
- dw/server/ui/assets/ts.worker-D7T1-Ig5.js +67738 -0
- dw/server/ui/assets/tsMode-D6u0XmOW.js +11 -0
- dw/server/ui/assets/twig-De2hgUGE.js +1 -0
- dw/server/ui/assets/typescript-BU6v-LMV.js +1 -0
- dw/server/ui/assets/typespec-B8J7ngcE.js +1 -0
- dw/server/ui/assets/vb-DV3o63ZY.js +1 -0
- dw/server/ui/assets/wgsl-DpFanUEy.js +298 -0
- dw/server/ui/assets/workers-Cn7cTUKr.js +1 -0
- dw/server/ui/assets/xml--0LP2Lwk.js +1 -0
- dw/server/ui/assets/yaml-mpBg9jnt.js +1 -0
- dw/server/ui/index.html +17 -0
- dw/server/updater.py +192 -0
- dw/settings.py +98 -0
- dw/shot_span_preflight.py +116 -0
- dw/shots.py +359 -0
- dw/slice_preflight.py +148 -0
- dw/step.py +187 -0
- dw/step_cache.py +442 -0
- dw/subfolders.py +107 -0
- dw/task_domains.py +307 -0
- dw/tasks/assess.py +826 -0
- dw/tasks/audio_transcription.py +88 -0
- dw/tasks/audio_utils.py +1862 -0
- dw/tasks/background_remover.py +43 -0
- dw/tasks/borders.py +113 -0
- dw/tasks/compose_text.py +74 -0
- dw/tasks/concat_videos.py +300 -0
- dw/tasks/depth_estimator.py +54 -0
- dw/tasks/diffusion_upscale.py +109 -0
- dw/tasks/dissolve_videos.py +342 -0
- dw/tasks/format_messages.py +24 -0
- dw/tasks/gather.py +173 -0
- dw/tasks/grade.py +97 -0
- dw/tasks/image_to_text.py +43 -0
- dw/tasks/image_utils.py +764 -0
- dw/tasks/interpolate_frames.py +252 -0
- dw/tasks/judge.py +68 -0
- dw/tasks/model_cache.py +55 -0
- dw/tasks/pair_audio.py +268 -0
- dw/tasks/qr_code.py +19 -0
- dw/tasks/restore_faces.py +175 -0
- dw/tasks/rife_model.py +192 -0
- dw/tasks/segment.py +121 -0
- dw/tasks/select.py +111 -0
- dw/tasks/speech_generation.py +228 -0
- dw/tasks/stabilize.py +129 -0
- dw/tasks/task.py +920 -0
- dw/tasks/tensor_image.py +57 -0
- dw/tasks/text_generation.py +169 -0
- dw/tasks/text_sections.py +80 -0
- dw/tasks/upscale.py +203 -0
- dw/tasks/video_utils.py +624 -0
- dw/tasks/zoe_depth.py +71 -0
- dw/teacache.py +381 -0
- dw/teacache_models.json +99 -0
- dw/test.py +29 -0
- dw/type_helpers.py +231 -0
- dw/validate.py +68 -0
- dw/variable_constraints.py +444 -0
- dw/variables.py +443 -0
- dw/video_extensions.py +141 -0
- dw/vram_estimate.py +116 -0
- dw/worker.py +764 -0
- dw/workflow.py +2007 -0
- dw/workflow_schema.json +1346 -0
- dw/workflow_sources.py +383 -0
- dw/workflows/h3_context_ir.json +57 -0
- dw/workflows/test.json +31 -0
- dw/workspace.py +730 -0
- dw_mcp/__init__.py +6 -0
- dw_mcp/__main__.py +133 -0
- dw_mcp/assets.py +336 -0
- dw_mcp/authoring.py +114 -0
- dw_mcp/catalog.py +360 -0
- dw_mcp/client.py +486 -0
- dw_mcp/diagnose.py +371 -0
- dw_mcp/exports.py +84 -0
- dw_mcp/guides.py +35 -0
- dw_mcp/media.py +638 -0
- dw_mcp/models.py +97 -0
- dw_mcp/prompts.py +104 -0
- dw_mcp/server.py +1343 -0
- dw_mcp/workspaces.py +212 -0
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
"""Join videos with cross-dissolves, and fade the whole from and to a colour.
|
|
2
|
+
|
|
3
|
+
`concat_videos` cuts: every seam is a hard edit, which is right for a scene
|
|
4
|
+
built from shots that each carry their own sound. A lyrical piece - a nature
|
|
5
|
+
film, a montage cut to a score - wants its shots to melt into one another
|
|
6
|
+
instead. Each seam here overlaps the tail of one video with the head of the
|
|
7
|
+
next by `dissolve_frames`, blending linearly across the overlap, so the result
|
|
8
|
+
is shorter than the plain sum by one overlap per seam; the soundtrack, when
|
|
9
|
+
every video carries one, is crossfaded over exactly the same span so it stays
|
|
10
|
+
in step with the picture. A `fade_in_frames` / `fade_out_frames` pair opens and
|
|
11
|
+
closes the piece on `fade_color`, which is what a film does instead of
|
|
12
|
+
starting on a full frame.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import logging
|
|
16
|
+
|
|
17
|
+
import numpy
|
|
18
|
+
from PIL import Image
|
|
19
|
+
|
|
20
|
+
from ..events import emit_log, emit_warning
|
|
21
|
+
from ..result import AudioVideo
|
|
22
|
+
from ..shots import measured_num_samples, nested_shots, shot_record
|
|
23
|
+
from .audio_utils import (
|
|
24
|
+
as_channels_samples,
|
|
25
|
+
crossfade_concat,
|
|
26
|
+
fit_audio_to_frames,
|
|
27
|
+
frames_to_samples,
|
|
28
|
+
match_levels as match_track_levels,
|
|
29
|
+
resample_waveform,
|
|
30
|
+
warn_on_level_spread,
|
|
31
|
+
)
|
|
32
|
+
from .concat_videos import video_names
|
|
33
|
+
from .video_utils import check_same_frame_size, frames_as_array, load_audio_video
|
|
34
|
+
|
|
35
|
+
logger = logging.getLogger("dw")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def dissolve_videos(
|
|
39
|
+
videos,
|
|
40
|
+
dissolve_frames=12,
|
|
41
|
+
fade_in_frames=0,
|
|
42
|
+
fade_out_frames=0,
|
|
43
|
+
fade_color=(0, 0, 0),
|
|
44
|
+
fps=None,
|
|
45
|
+
match_levels=None,
|
|
46
|
+
match_levels_dbfs=None,
|
|
47
|
+
sample_rate=None,
|
|
48
|
+
):
|
|
49
|
+
"""Task command: join videos with cross-dissolves at every seam.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
videos: The videos to join, in order - frame lists, frame arrays,
|
|
53
|
+
AudioVideos, or the path or URL of a video file. Give each video
|
|
54
|
+
its own entry, as with concat_videos. Soundtracks at different
|
|
55
|
+
sample rates are not a constraint - see `sample_rate` below
|
|
56
|
+
dissolve_frames: Frames of overlap at each seam. 0 is a hard cut
|
|
57
|
+
fade_in_frames: Frames over which the first video rises out of
|
|
58
|
+
`fade_color`
|
|
59
|
+
fade_out_frames: Frames over which the last video sinks into it
|
|
60
|
+
fade_color: The RGB colour the fades come from and go to
|
|
61
|
+
fps: Frame rate of the videos - required to crossfade audio at a
|
|
62
|
+
dissolve, and ignored when no video carries any
|
|
63
|
+
match_levels: Even the shots' loudness out before joining - "rms"
|
|
64
|
+
for perceived level, "peak" for the loudest sample. Off by
|
|
65
|
+
default; see concat_videos, which has the same pair. Left off, a
|
|
66
|
+
spread wide enough to hear is logged as a warning
|
|
67
|
+
match_levels_dbfs: The level match_levels moves every shot to -
|
|
68
|
+
defaults to -1 dBFS for "peak" and -20 dBFS for "rms". A shot
|
|
69
|
+
that would clip at the target is held at -0.5 dBFS peak instead,
|
|
70
|
+
reported as a match_levels_held warning, with a per-shot log
|
|
71
|
+
event naming the hold
|
|
72
|
+
sample_rate: The rate the joined soundtrack is at. Shots that come
|
|
73
|
+
from different sources routinely carry different rates, and
|
|
74
|
+
unlike a level jump that difference has no editorial meaning, so
|
|
75
|
+
by default the highest rate among the inputs is chosen and the
|
|
76
|
+
rest are resampled up to it, with a warning naming which - the
|
|
77
|
+
same conversion concat_videos does (#108, #287). Give this to
|
|
78
|
+
pin the target instead
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
One AudioVideo; its audio is None unless every input carries a track
|
|
82
|
+
|
|
83
|
+
Raises:
|
|
84
|
+
ValueError: If a video is too short to carry its share of the overlaps
|
|
85
|
+
"""
|
|
86
|
+
if not isinstance(videos, list) or not videos:
|
|
87
|
+
raise ValueError("dissolve_videos needs a non-empty list of videos")
|
|
88
|
+
if dissolve_frames < 0 or fade_in_frames < 0 or fade_out_frames < 0:
|
|
89
|
+
raise ValueError("dissolve_videos frame counts cannot be negative")
|
|
90
|
+
|
|
91
|
+
loaded = [load_audio_video(v) if isinstance(v, str) else v for v in videos]
|
|
92
|
+
clips = [frames_as_array(v).astype(numpy.float32) for v in loaded]
|
|
93
|
+
check_same_frame_size(clips, "dissolve_videos")
|
|
94
|
+
|
|
95
|
+
for index, clip in enumerate(clips):
|
|
96
|
+
seams = (index > 0) + (index < len(clips) - 1)
|
|
97
|
+
if len(clip) < seams * dissolve_frames:
|
|
98
|
+
raise ValueError(
|
|
99
|
+
f"video {index} has {len(clip)} frames, too few for its "
|
|
100
|
+
f"{seams} dissolve(s) of {dissolve_frames} frames"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
joined = clips[0]
|
|
104
|
+
# Where each clip's first frame landed - the start of its dissolve
|
|
105
|
+
frame_starts = [0]
|
|
106
|
+
for clip in clips[1:]:
|
|
107
|
+
frame_starts.append(len(joined) - dissolve_frames)
|
|
108
|
+
joined = _dissolve_join(joined, clip, dissolve_frames)
|
|
109
|
+
|
|
110
|
+
if fade_in_frames or fade_out_frames:
|
|
111
|
+
color = numpy.asarray(fade_color, dtype=numpy.float32)
|
|
112
|
+
total = len(joined)
|
|
113
|
+
fade_in = min(fade_in_frames, total)
|
|
114
|
+
fade_out = min(fade_out_frames, total - fade_in)
|
|
115
|
+
if fade_in:
|
|
116
|
+
weights = _ramp(fade_in, ascending=True)
|
|
117
|
+
joined[:fade_in] = _blend(color, joined[:fade_in], weights)
|
|
118
|
+
if fade_out:
|
|
119
|
+
weights = _ramp(fade_out, ascending=False)
|
|
120
|
+
joined[total - fade_out :] = _blend(
|
|
121
|
+
color, joined[total - fade_out :], weights
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
frames = [Image.fromarray(frame) for frame in joined.round().astype(numpy.uint8)]
|
|
125
|
+
written_fps = fps or next(
|
|
126
|
+
(v.fps for v in loaded if getattr(v, "fps", None)),
|
|
127
|
+
None,
|
|
128
|
+
)
|
|
129
|
+
audio, sample_rate = _dissolve_audio(
|
|
130
|
+
loaded,
|
|
131
|
+
dissolve_frames,
|
|
132
|
+
fps,
|
|
133
|
+
match_levels,
|
|
134
|
+
match_levels_dbfs,
|
|
135
|
+
sample_rate,
|
|
136
|
+
len(frames),
|
|
137
|
+
written_fps,
|
|
138
|
+
)
|
|
139
|
+
shots = _dissolve_shots(
|
|
140
|
+
loaded,
|
|
141
|
+
video_names(videos),
|
|
142
|
+
frame_starts,
|
|
143
|
+
len(frames),
|
|
144
|
+
written_fps,
|
|
145
|
+
audio,
|
|
146
|
+
sample_rate,
|
|
147
|
+
dissolve_frames,
|
|
148
|
+
)
|
|
149
|
+
logger.info(
|
|
150
|
+
f"Dissolved {len(clips)} videos into {len(frames)} frames "
|
|
151
|
+
f"({dissolve_frames}-frame seams)"
|
|
152
|
+
)
|
|
153
|
+
return AudioVideo(frames, audio, sample_rate, fps=written_fps, shots=shots)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _dissolve_shots(
|
|
157
|
+
videos,
|
|
158
|
+
names,
|
|
159
|
+
frame_starts,
|
|
160
|
+
total_frames,
|
|
161
|
+
fps,
|
|
162
|
+
audio,
|
|
163
|
+
sample_rate,
|
|
164
|
+
dissolve_frames,
|
|
165
|
+
):
|
|
166
|
+
"""One shot per video - or, for one that already carries its own, one per
|
|
167
|
+
inner shot - partitioning the dissolved picture and track.
|
|
168
|
+
|
|
169
|
+
A dissolve belongs to the shot coming in: each video's own frames map
|
|
170
|
+
onto the joined picture by the exact offset frame_starts[index] gives (a
|
|
171
|
+
dissolve blends in place rather than dropping frames, unlike
|
|
172
|
+
concat_videos' head trim), so an input that is itself an earlier join's
|
|
173
|
+
output keeps its inner seams rather than collapsing to one record
|
|
174
|
+
(#399). `overlap_frames` marks how much of the shot's own head - the
|
|
175
|
+
first inner one, when it nests - is blended with what came before. The
|
|
176
|
+
same dissolve also eats into the *tail* of the video before it: a
|
|
177
|
+
non-nested video's shot already stops at frame_starts[index + 1] (the
|
|
178
|
+
frame the next video's overlap starts blending at), so its recorded
|
|
179
|
+
num_frames excludes the overlap; a nested video's last inner shot must
|
|
180
|
+
be trimmed by the same dissolve_frames to keep that convention, or its
|
|
181
|
+
num_frames runs past where the next shot's start_frame picks up (#405).
|
|
182
|
+
|
|
183
|
+
Each shot's `start_sample` is *derived* from its frame offset
|
|
184
|
+
(frames_to_samples), the same rule pair_audio's remeasured_shots uses,
|
|
185
|
+
rather than read off where the crossfade actually landed: summing the
|
|
186
|
+
individually-rounded per-clip lengths a real dissolve measures does not
|
|
187
|
+
equal rounding the cumulative frame count in one step, so the two tools
|
|
188
|
+
disagreed by a sample on a shot whose frames never changed (#401). The
|
|
189
|
+
crossfade itself still blends the real, measured audio - only the
|
|
190
|
+
recorded seam position is derived, so it matches whatever a later
|
|
191
|
+
pair_audio recomputes for the same boundary. A nested video's own inner
|
|
192
|
+
shots get the same treatment (`nested_shots(..., fps=fps)`) rather than
|
|
193
|
+
rescaling their already-rounded stored `start_sample`, which would
|
|
194
|
+
compound the rounding across every join an input has passed through
|
|
195
|
+
(#405).
|
|
196
|
+
"""
|
|
197
|
+
total_samples = audio.shape[1] if audio is not None else None
|
|
198
|
+
shots = []
|
|
199
|
+
for index, (video, name) in enumerate(zip(videos, names)):
|
|
200
|
+
inner = getattr(video, "shots", None)
|
|
201
|
+
sample_offset = (
|
|
202
|
+
min(frames_to_samples(frame_starts[index], fps, sample_rate), total_samples)
|
|
203
|
+
if audio is not None and fps
|
|
204
|
+
else None
|
|
205
|
+
)
|
|
206
|
+
if inner:
|
|
207
|
+
nested = nested_shots(
|
|
208
|
+
inner,
|
|
209
|
+
frame_starts[index],
|
|
210
|
+
sample_offset,
|
|
211
|
+
getattr(video, "sample_rate", None),
|
|
212
|
+
sample_rate,
|
|
213
|
+
fps=fps,
|
|
214
|
+
)
|
|
215
|
+
if index and dissolve_frames and nested:
|
|
216
|
+
nested[0]["overlap_frames"] = dissolve_frames
|
|
217
|
+
if dissolve_frames and nested and index < len(videos) - 1:
|
|
218
|
+
nested[-1]["num_frames"] -= dissolve_frames
|
|
219
|
+
# Which input this shot came from - named_shots (dw/shots.py)
|
|
220
|
+
# uses it to place a step's override name on the right shot once
|
|
221
|
+
# an earlier input has nested more than one of its own (#432)
|
|
222
|
+
for shot in nested:
|
|
223
|
+
shot["source_index"] = index
|
|
224
|
+
shots.extend(nested)
|
|
225
|
+
else:
|
|
226
|
+
frame_end = (
|
|
227
|
+
frame_starts[index + 1]
|
|
228
|
+
if index + 1 < len(frame_starts)
|
|
229
|
+
else total_frames
|
|
230
|
+
)
|
|
231
|
+
shot = shot_record(
|
|
232
|
+
name,
|
|
233
|
+
frame_starts[index],
|
|
234
|
+
frame_end - frame_starts[index],
|
|
235
|
+
sample_offset,
|
|
236
|
+
)
|
|
237
|
+
if index and dissolve_frames:
|
|
238
|
+
shot["overlap_frames"] = dissolve_frames
|
|
239
|
+
shot["source_index"] = index
|
|
240
|
+
shots.append(shot)
|
|
241
|
+
measured_num_samples(shots, total_samples)
|
|
242
|
+
return shots
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _dissolve_join(previous, following, overlap):
|
|
246
|
+
"""Overlap the tail of `previous` with the head of `following`."""
|
|
247
|
+
if overlap == 0:
|
|
248
|
+
return numpy.concatenate([previous, following])
|
|
249
|
+
weights = _ramp(overlap, ascending=True)
|
|
250
|
+
blended = _blend(previous[-overlap:], following[:overlap], weights)
|
|
251
|
+
return numpy.concatenate([previous[:-overlap], blended, following[overlap:]])
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _ramp(count, ascending):
|
|
255
|
+
"""Blend weights that never sit on 0 or 1, so no frame is a bare copy of
|
|
256
|
+
either side - the seam's first frame already carries some of the incoming
|
|
257
|
+
picture and its last still carries some of the outgoing one."""
|
|
258
|
+
weights = (numpy.arange(count, dtype=numpy.float32) + 1) / (count + 1)
|
|
259
|
+
return weights if ascending else 1 - weights
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _blend(from_frames, to_frames, weights):
|
|
263
|
+
weights = weights.reshape(-1, 1, 1, 1)
|
|
264
|
+
return from_frames * (1 - weights) + to_frames * weights
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _dissolve_audio(
|
|
268
|
+
videos,
|
|
269
|
+
dissolve_frames,
|
|
270
|
+
fps,
|
|
271
|
+
match_levels=None,
|
|
272
|
+
match_levels_dbfs=None,
|
|
273
|
+
sample_rate=None,
|
|
274
|
+
total_frames=None,
|
|
275
|
+
written_fps=None,
|
|
276
|
+
):
|
|
277
|
+
"""Crossfade every video's track over the seams' own span."""
|
|
278
|
+
tracks = [v for v in videos if isinstance(v, AudioVideo) and v.audio is not None]
|
|
279
|
+
if len(tracks) != len(videos):
|
|
280
|
+
if tracks:
|
|
281
|
+
logger.warning(
|
|
282
|
+
"dissolve_videos: some videos carry no audio - the result is silent"
|
|
283
|
+
)
|
|
284
|
+
return None, None
|
|
285
|
+
if fps is None and dissolve_frames:
|
|
286
|
+
raise ValueError("dissolve_videos needs 'fps' to crossfade audio at a dissolve")
|
|
287
|
+
|
|
288
|
+
# Shots assembled from different sources disagree on rate routinely, and
|
|
289
|
+
# the disagreement carries no editorial meaning - so it is converted
|
|
290
|
+
# rather than refused, which is what made an agent invent a
|
|
291
|
+
# resample_audio step by hand for concat_videos before #108 (#287)
|
|
292
|
+
names = video_names(videos)
|
|
293
|
+
track_names = [
|
|
294
|
+
name
|
|
295
|
+
for name, video in zip(names, videos)
|
|
296
|
+
if isinstance(video, AudioVideo) and video.audio is not None
|
|
297
|
+
]
|
|
298
|
+
rates = {v.sample_rate for v in tracks}
|
|
299
|
+
sample_rate = sample_rate or max(rates)
|
|
300
|
+
waveforms = [as_channels_samples(v.audio) for v in tracks]
|
|
301
|
+
if len(rates) == 1 and next(iter(rates)) != sample_rate:
|
|
302
|
+
# The inputs agree and the caller pinned another rate: converting
|
|
303
|
+
# to what was asked for is not a decision made on its behalf (#453)
|
|
304
|
+
emit_log(
|
|
305
|
+
f"dissolve_videos: resampling every track from {next(iter(rates))} Hz "
|
|
306
|
+
f"to the requested {sample_rate} Hz",
|
|
307
|
+
command="dissolve_videos",
|
|
308
|
+
sample_rate=sample_rate,
|
|
309
|
+
)
|
|
310
|
+
elif len(rates) != 1:
|
|
311
|
+
per_track = {name: v.sample_rate for name, v in zip(track_names, tracks)}
|
|
312
|
+
emit_warning(
|
|
313
|
+
"dissolve_videos: videos carry audio at different sample rates ("
|
|
314
|
+
+ ", ".join(f"{name}: {rate} Hz" for name, rate in per_track.items())
|
|
315
|
+
+ f") - resampling them all to {sample_rate} Hz. Pass "
|
|
316
|
+
"'sample_rate' to pin a different target, or resample ahead of "
|
|
317
|
+
"this step with the 'resample_audio' task.",
|
|
318
|
+
kind="sample_rate_mismatch",
|
|
319
|
+
command="dissolve_videos",
|
|
320
|
+
sample_rate=sample_rate,
|
|
321
|
+
sample_rates=per_track,
|
|
322
|
+
)
|
|
323
|
+
waveforms = [
|
|
324
|
+
(
|
|
325
|
+
waveform
|
|
326
|
+
if video.sample_rate == sample_rate
|
|
327
|
+
else resample_waveform(waveform, video.sample_rate, sample_rate)
|
|
328
|
+
)
|
|
329
|
+
for video, waveform in zip(tracks, waveforms)
|
|
330
|
+
]
|
|
331
|
+
crossfade_ms = dissolve_frames / fps * 1000 if dissolve_frames else 0
|
|
332
|
+
if match_levels:
|
|
333
|
+
waveforms = match_track_levels(
|
|
334
|
+
waveforms, match_levels, match_levels_dbfs, "dissolve_videos"
|
|
335
|
+
)
|
|
336
|
+
else:
|
|
337
|
+
warn_on_level_spread(waveforms, "dissolve_videos")
|
|
338
|
+
joined = crossfade_concat(waveforms, sample_rate, crossfade_ms)
|
|
339
|
+
joined = fit_audio_to_frames(
|
|
340
|
+
joined, sample_rate, total_frames, written_fps, "dissolve_videos"
|
|
341
|
+
)
|
|
342
|
+
return joined, sample_rate
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
def format_chat_message(system_prompt, user_message):
|
|
2
|
+
return {
|
|
3
|
+
"text_inputs": [
|
|
4
|
+
{"role": "system", "content": system_prompt},
|
|
5
|
+
{
|
|
6
|
+
"role": "user",
|
|
7
|
+
"content": user_message,
|
|
8
|
+
},
|
|
9
|
+
]
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def batch_decode_post_process(processor, task, generated_ids):
|
|
14
|
+
generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
|
|
15
|
+
|
|
16
|
+
parsed_answer = processor.post_process_generation(generated_text, task=task)
|
|
17
|
+
|
|
18
|
+
return parsed_answer[task]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def get_dict_value(dict, key):
|
|
22
|
+
if key in dict:
|
|
23
|
+
return dict[key]
|
|
24
|
+
return None
|
dw/tasks/gather.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import glob as glob_lib
|
|
2
|
+
import io
|
|
3
|
+
import logging
|
|
4
|
+
from diffusers.utils import load_image
|
|
5
|
+
from PIL import Image
|
|
6
|
+
from ..arguments import fetch_image
|
|
7
|
+
from ..security import SecurityError
|
|
8
|
+
from ..locations import contained_matches, safe_get, validate_media_glob
|
|
9
|
+
from .video_utils import load_audio_video
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger("dw")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def gather_images(glob=None, urls=None):
|
|
15
|
+
"""
|
|
16
|
+
Gather images from local files and/or URLs.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
glob: Pattern for matching local image files (e.g., "images/*.png")
|
|
20
|
+
urls: List of URLs to download images from
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
List of loaded images
|
|
24
|
+
|
|
25
|
+
Raises:
|
|
26
|
+
ValueError: If no images are found
|
|
27
|
+
SecurityError: If validation fails
|
|
28
|
+
"""
|
|
29
|
+
if urls is None:
|
|
30
|
+
urls = []
|
|
31
|
+
images = []
|
|
32
|
+
|
|
33
|
+
# Load local images matching glob pattern
|
|
34
|
+
if glob is not None:
|
|
35
|
+
logger.debug(f"Searching for images matching pattern: {glob}")
|
|
36
|
+
# The pattern is a location like any other, and goes through the same
|
|
37
|
+
# policy: it may only expand inside the directories this workflow may
|
|
38
|
+
# read, and each match is re-checked because a wildcard can leave the
|
|
39
|
+
# tree through a symlink (dw/locations.py)
|
|
40
|
+
pattern = validate_media_glob(glob, what="the images glob")
|
|
41
|
+
# Sorted, because glob returns filesystem order: a numbered sequence
|
|
42
|
+
# of images gathered for concatenation has to come back in its own
|
|
43
|
+
# order, not in whatever order the directory happens to hold
|
|
44
|
+
image_paths = contained_matches(
|
|
45
|
+
sorted(glob_lib.glob(pattern)), what="the images glob"
|
|
46
|
+
)
|
|
47
|
+
logger.info(f"Found {len(image_paths)} local images")
|
|
48
|
+
|
|
49
|
+
for path in image_paths:
|
|
50
|
+
try:
|
|
51
|
+
logger.debug(f"Loading image from: {path}")
|
|
52
|
+
images.append(fetch_image(path))
|
|
53
|
+
except SecurityError:
|
|
54
|
+
raise
|
|
55
|
+
except Exception as e:
|
|
56
|
+
logger.error(
|
|
57
|
+
f"Failed to load image from {path}: {str(e)}", exc_info=True
|
|
58
|
+
)
|
|
59
|
+
raise
|
|
60
|
+
|
|
61
|
+
# Load images from URLs
|
|
62
|
+
for url in urls:
|
|
63
|
+
try:
|
|
64
|
+
logger.debug(f"Loading image from URL: {url}")
|
|
65
|
+
# safe_get re-checks every redirect, which load_image's own
|
|
66
|
+
# fetch does not; load_image still transposes and converts
|
|
67
|
+
response = safe_get(url, "a gathered image url", timeout=60)
|
|
68
|
+
images.append(load_image(Image.open(io.BytesIO(response.content))))
|
|
69
|
+
except SecurityError:
|
|
70
|
+
raise
|
|
71
|
+
except Exception as e:
|
|
72
|
+
logger.error(
|
|
73
|
+
f"Failed to load image from URL {url}: {str(e)}", exc_info=True
|
|
74
|
+
)
|
|
75
|
+
raise
|
|
76
|
+
|
|
77
|
+
# Validate that we found at least one image
|
|
78
|
+
if len(images) == 0:
|
|
79
|
+
error_msg = "No images found"
|
|
80
|
+
logger.error(error_msg)
|
|
81
|
+
raise ValueError(error_msg)
|
|
82
|
+
|
|
83
|
+
logger.debug(f"Successfully gathered {len(images)} images")
|
|
84
|
+
return images
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def gather_videos(glob=None, urls=None):
|
|
88
|
+
"""
|
|
89
|
+
Gather videos from local files and/or URLs, audio included. To join
|
|
90
|
+
videos rather than iterate over them, give their paths to concat_videos
|
|
91
|
+
directly instead of gathering them first.
|
|
92
|
+
|
|
93
|
+
Each video comes back as one artifact holding its frames and whatever
|
|
94
|
+
audio was muxed alongside them, so a step that references this one
|
|
95
|
+
iterates over videos rather than over frames - and a step meant to
|
|
96
|
+
consume all of them at once, as concat_videos is, would be fanned out
|
|
97
|
+
over them one at a time.
|
|
98
|
+
|
|
99
|
+
Args:
|
|
100
|
+
glob: Pattern for matching local video files (e.g., "videos/*.mp4")
|
|
101
|
+
urls: List of URLs to download videos from
|
|
102
|
+
|
|
103
|
+
Returns:
|
|
104
|
+
List of AudioVideo artifacts, one per gathered video
|
|
105
|
+
|
|
106
|
+
Raises:
|
|
107
|
+
ValueError: If no videos are found
|
|
108
|
+
SecurityError: If validation fails
|
|
109
|
+
"""
|
|
110
|
+
if urls is None:
|
|
111
|
+
urls = []
|
|
112
|
+
videos = []
|
|
113
|
+
|
|
114
|
+
# Load local videos matching glob pattern
|
|
115
|
+
if glob is not None:
|
|
116
|
+
logger.debug(f"Searching for videos matching pattern: {glob}")
|
|
117
|
+
# Same containment as gather_images - one policy, two tasks
|
|
118
|
+
pattern = validate_media_glob(glob, what="the videos glob")
|
|
119
|
+
# Sorted, because glob returns filesystem order: a numbered sequence
|
|
120
|
+
# of videos gathered for concatenation has to come back in its own
|
|
121
|
+
# order, not in whatever order the directory happens to hold
|
|
122
|
+
video_paths = contained_matches(
|
|
123
|
+
sorted(glob_lib.glob(pattern)), what="the videos glob"
|
|
124
|
+
)
|
|
125
|
+
logger.info(f"Found {len(video_paths)} local videos")
|
|
126
|
+
|
|
127
|
+
for path in video_paths:
|
|
128
|
+
try:
|
|
129
|
+
logger.debug(f"Loading video from: {path}")
|
|
130
|
+
videos.append(load_audio_video(path))
|
|
131
|
+
except SecurityError:
|
|
132
|
+
raise
|
|
133
|
+
except Exception as e:
|
|
134
|
+
logger.error(
|
|
135
|
+
f"Failed to load video from {path}: {str(e)}", exc_info=True
|
|
136
|
+
)
|
|
137
|
+
raise
|
|
138
|
+
|
|
139
|
+
# Load videos from URLs
|
|
140
|
+
for url in urls:
|
|
141
|
+
try:
|
|
142
|
+
logger.debug(f"Loading video from URL: {url}")
|
|
143
|
+
videos.append(load_audio_video(url))
|
|
144
|
+
except SecurityError:
|
|
145
|
+
raise
|
|
146
|
+
except Exception as e:
|
|
147
|
+
logger.error(
|
|
148
|
+
f"Failed to load video from URL {url}: {str(e)}", exc_info=True
|
|
149
|
+
)
|
|
150
|
+
raise
|
|
151
|
+
|
|
152
|
+
# Validate that we found at least one video
|
|
153
|
+
if len(videos) == 0:
|
|
154
|
+
error_msg = "No videos found"
|
|
155
|
+
logger.error(error_msg)
|
|
156
|
+
raise ValueError(error_msg)
|
|
157
|
+
|
|
158
|
+
logger.debug(f"Successfully gathered {len(videos)} videos")
|
|
159
|
+
return videos
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def gather_inputs(kwargs):
|
|
163
|
+
"""
|
|
164
|
+
Gather input arguments for passing to next task.
|
|
165
|
+
|
|
166
|
+
Args:
|
|
167
|
+
kwargs: Dictionary of input arguments
|
|
168
|
+
|
|
169
|
+
Returns:
|
|
170
|
+
Input arguments unchanged
|
|
171
|
+
"""
|
|
172
|
+
logger.debug(f"Gathering input arguments: {kwargs}")
|
|
173
|
+
return kwargs
|
dw/tasks/grade.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CPU colour grading: exposure, contrast, saturation, temperature/tint.
|
|
3
|
+
|
|
4
|
+
Pure numpy/PIL - no model, no GPU. A video is graded per frame by the
|
|
5
|
+
command handler (task.py's _per_frame), so this module only ever sees a
|
|
6
|
+
single PIL Image.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
from PIL import Image
|
|
11
|
+
|
|
12
|
+
# Luma weights (Rec. 709), used as the saturation pivot
|
|
13
|
+
_LUMA_WEIGHTS = np.array([0.2126, 0.7152, 0.0722], dtype=np.float32)
|
|
14
|
+
|
|
15
|
+
# Fraction of the full [0, 1] channel range shifted at |temperature| == 1.0 or
|
|
16
|
+
# |tint| == 1.0. Temperature moves the red and blue channels apart (warmer is
|
|
17
|
+
# more red, less blue); tint moves the green channel against them (more
|
|
18
|
+
# magenta is less green). Both are linear in their argument and independent
|
|
19
|
+
# of each other - this is the "documented scale" v1 promises rather than a
|
|
20
|
+
# physical colour-temperature-in-Kelvin model, which is more expensive to
|
|
21
|
+
# compute and no more useful to a generation pipeline's output.
|
|
22
|
+
_WHITE_BALANCE_STRENGTH = 0.15
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def grade_image(
|
|
26
|
+
media,
|
|
27
|
+
exposure=0.0,
|
|
28
|
+
contrast=1.0,
|
|
29
|
+
saturation=1.0,
|
|
30
|
+
temperature=0.0,
|
|
31
|
+
tint=0.0,
|
|
32
|
+
):
|
|
33
|
+
"""Adjust exposure, contrast, saturation and white balance of a single frame.
|
|
34
|
+
|
|
35
|
+
Every parameter is optional; omitting one leaves that adjustment at its
|
|
36
|
+
identity value, so calling with no arguments returns the input pixels
|
|
37
|
+
unchanged (within rounding). Adjustments apply in this order: exposure,
|
|
38
|
+
then contrast, then temperature/tint, then saturation.
|
|
39
|
+
|
|
40
|
+
Args:
|
|
41
|
+
media: PIL Image to grade. A video is dispatched to this one frame at
|
|
42
|
+
a time by the command handler (task.py's _per_frame), so this
|
|
43
|
+
function itself only ever sees a single frame.
|
|
44
|
+
exposure: Stops to brighten (positive) or darken (negative) by,
|
|
45
|
+
applied as a multiply of 2**exposure. 0.0 (default) is identity.
|
|
46
|
+
contrast: Multiplier applied around the mid grey point (0.5).
|
|
47
|
+
1.0 (default) is identity; above 1 increases contrast, below 1
|
|
48
|
+
(down to 0) flattens it.
|
|
49
|
+
saturation: Multiplier applied around each pixel's own luma
|
|
50
|
+
(Rec. 709 weights). 1.0 (default) is identity; 0.0 is greyscale.
|
|
51
|
+
temperature: Warm/cool white-balance shift from -1.0 (coolest, shifts
|
|
52
|
+
toward blue) to 1.0 (warmest, shifts toward red), linear, moving
|
|
53
|
+
the red and blue channels apart by up to 15% of the channel
|
|
54
|
+
range at |1.0|. 0.0 (default) is identity.
|
|
55
|
+
tint: Green/magenta white-balance shift from -1.0 (green) to 1.0
|
|
56
|
+
(magenta), linear, moving the green channel by up to 15% of the
|
|
57
|
+
channel range at |1.0| in the opposite direction to the shift's
|
|
58
|
+
sign. 0.0 (default) is identity.
|
|
59
|
+
|
|
60
|
+
Returns:
|
|
61
|
+
PIL Image, same size and mode as the input (graded). An alpha
|
|
62
|
+
channel, if the input has one, passes through untouched.
|
|
63
|
+
"""
|
|
64
|
+
alpha = None
|
|
65
|
+
if media.mode in ("RGBA", "LA"):
|
|
66
|
+
alpha = media.getchannel("A")
|
|
67
|
+
|
|
68
|
+
rgb = media.convert("RGB")
|
|
69
|
+
array = np.asarray(rgb, dtype=np.float32) / 255.0
|
|
70
|
+
|
|
71
|
+
if exposure != 0.0:
|
|
72
|
+
array = array * (2.0**exposure)
|
|
73
|
+
|
|
74
|
+
if contrast != 1.0:
|
|
75
|
+
array = (array - 0.5) * contrast + 0.5
|
|
76
|
+
|
|
77
|
+
if temperature != 0.0:
|
|
78
|
+
shift = temperature * _WHITE_BALANCE_STRENGTH
|
|
79
|
+
array[..., 0] += shift
|
|
80
|
+
array[..., 2] -= shift
|
|
81
|
+
|
|
82
|
+
if tint != 0.0:
|
|
83
|
+
shift = tint * _WHITE_BALANCE_STRENGTH
|
|
84
|
+
array[..., 1] -= shift
|
|
85
|
+
|
|
86
|
+
if saturation != 1.0:
|
|
87
|
+
luma = np.tensordot(array, _LUMA_WEIGHTS, axes=([-1], [0]))
|
|
88
|
+
array = luma[..., None] + (array - luma[..., None]) * saturation
|
|
89
|
+
|
|
90
|
+
array = np.clip(array * 255.0, 0, 255).astype(np.uint8)
|
|
91
|
+
graded = Image.fromarray(array, mode="RGB")
|
|
92
|
+
|
|
93
|
+
if alpha is not None:
|
|
94
|
+
graded = graded.convert("RGBA")
|
|
95
|
+
graded.putalpha(alpha)
|
|
96
|
+
|
|
97
|
+
return graded
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Image captioning via HuggingFace transformers.
|
|
3
|
+
|
|
4
|
+
Captioning is text generation that happens to be given a picture, so this
|
|
5
|
+
delegates to the text_generation task rather than keeping a second copy of
|
|
6
|
+
the pipeline handling. Transformers 5 removed the dedicated "image-to-text"
|
|
7
|
+
task this used to build, and with it the BLIP-style captioning models; the
|
|
8
|
+
work is done by vision-language models now.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import logging
|
|
12
|
+
from .text_generation import generate_text, _DEFAULT_VISION_MODEL
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger("dw")
|
|
15
|
+
|
|
16
|
+
_DEFAULT_MODEL = _DEFAULT_VISION_MODEL
|
|
17
|
+
_DEFAULT_PROMPT = "Describe this image."
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def image_to_text(image, device="cpu", **kwargs):
|
|
21
|
+
"""Generate a text caption for an image.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
image: PIL Image (or URL/path) to caption.
|
|
25
|
+
device: Target device ("cuda", "mps", "cpu").
|
|
26
|
+
**kwargs:
|
|
27
|
+
model_name: HuggingFace model ID of a vision-language model
|
|
28
|
+
(default: HuggingFaceTB/SmolVLM-256M-Instruct).
|
|
29
|
+
prompt: What to ask about the image (default: "Describe this
|
|
30
|
+
image."). Ask a narrower question for a narrower caption.
|
|
31
|
+
system_prompt: Optional system instruction for the model.
|
|
32
|
+
max_new_tokens: Max tokens to generate (default: 50).
|
|
33
|
+
|
|
34
|
+
Returns:
|
|
35
|
+
Caption string.
|
|
36
|
+
"""
|
|
37
|
+
prompt = kwargs.pop("prompt", _DEFAULT_PROMPT)
|
|
38
|
+
kwargs.setdefault("model_name", _DEFAULT_MODEL)
|
|
39
|
+
kwargs.setdefault("max_new_tokens", 50)
|
|
40
|
+
|
|
41
|
+
caption = generate_text(prompt, device=device, image=image, **kwargs)
|
|
42
|
+
logger.info(f"Caption: {caption[:100]}{'...' if len(caption) > 100 else ''}")
|
|
43
|
+
return caption
|