videobook 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/LICENSE +21 -0
- package/README.md +195 -2
- package/bin/videobook.js +7 -0
- package/dist/acestep/client.js +203 -0
- package/dist/acestep/config.js +62 -0
- package/dist/acestep/errors.js +8 -0
- package/dist/acestep/process.js +175 -0
- package/dist/alive-captions/analyze.js +117 -0
- package/dist/alive-captions/audio.js +179 -0
- package/dist/alive-captions/frames.js +59 -0
- package/dist/alive-captions/model.js +37 -0
- package/dist/alive-captions/schema.js +49 -0
- package/dist/alive-captions/track.js +9 -0
- package/dist/api-key.js +33 -0
- package/dist/app-state.js +56 -0
- package/dist/app.js +257 -0
- package/dist/archive-b2.js +332 -0
- package/dist/archive-store.js +169 -0
- package/dist/artifact-label.js +36 -0
- package/dist/artifact-refs.js +49 -0
- package/dist/asset-tag-extraction.js +169 -0
- package/dist/asset-tags.js +228 -0
- package/dist/asset-usage.js +23 -0
- package/dist/audio-cue.js +71 -0
- package/dist/audio-encoding.js +32 -0
- package/dist/audio-sync-request.js +1 -0
- package/dist/audio-sync.js +223 -0
- package/dist/audio-trim-range.js +31 -0
- package/dist/book-context.js +31 -0
- package/dist/book-manager.js +374 -0
- package/dist/catalog-gc.js +26 -0
- package/dist/chain-executor.js +815 -0
- package/dist/chain-generation.js +115 -0
- package/dist/chain-plan.js +746 -0
- package/dist/chain-step-review.js +60 -0
- package/dist/character-name.js +25 -0
- package/dist/chat-generation-routing.js +73 -0
- package/dist/chat-generation-settings.js +78 -0
- package/dist/chat-models.js +44 -0
- package/dist/chat-prompt-rewrite.js +67 -0
- package/dist/chat-router.js +628 -0
- package/dist/chat-view-actions.js +49 -0
- package/dist/cli/args.js +242 -0
- package/dist/cli/config-command.js +103 -0
- package/dist/cli/doctor-command.js +289 -0
- package/dist/cli/entry.js +2 -0
- package/dist/cli/key-report.js +16 -0
- package/dist/cli/main.js +104 -0
- package/dist/cli/mcp-command.js +88 -0
- package/dist/cli/mcp-relay.js +51 -0
- package/dist/cli/open-browser.js +14 -0
- package/dist/cli/run-command.js +231 -0
- package/dist/cli/serve-command.js +62 -0
- package/dist/cli/setup-command.js +60 -0
- package/dist/cli/stdio-guard.js +8 -0
- package/dist/cold-open-model.js +7 -0
- package/dist/cold-open.js +79 -0
- package/dist/command-reference.js +95 -0
- package/dist/content-store.js +74 -0
- package/dist/cost.js +8 -0
- package/dist/crop-geometry.js +23 -0
- package/dist/dancing-workflow.js +447 -0
- package/dist/data-home.js +44 -0
- package/dist/diagnostics.js +109 -0
- package/dist/dialog-quality.js +34 -0
- package/dist/downloads-dest.js +56 -0
- package/dist/editing-moments.js +194 -0
- package/dist/elevenlabs-audio-extension.js +66 -0
- package/dist/engine.js +54 -0
- package/dist/env-file.js +91 -0
- package/dist/env-secrets.js +148 -0
- package/dist/ffmpeg-binary.js +12 -0
- package/dist/ffmpeg.js +425 -0
- package/dist/ffprobe-binary.js +77 -0
- package/dist/forget-asset.js +216 -0
- package/dist/frame-utils.js +27 -0
- package/dist/generation-credit-error.js +11 -0
- package/dist/generation-input-mentions.js +117 -0
- package/dist/generation-input-policy.js +68 -0
- package/dist/generation-reference-mentions.js +97 -0
- package/dist/goal-characters.js +49 -0
- package/dist/goal-plan-v3.js +273 -0
- package/dist/goal-plan.js +220 -0
- package/dist/goal-planner-prompt.js +28 -0
- package/dist/goal-prompt-budget.js +4 -0
- package/dist/goal-reference-images.js +25 -0
- package/dist/goal-state.js +261 -0
- package/dist/goal-stills.js +153 -0
- package/dist/goal-store.js +144 -0
- package/dist/goal-styles.js +144 -0
- package/dist/goal-v3-runtime.js +157 -0
- package/dist/guides/character_schema_guide.md +70 -0
- package/dist/guides/gpt_image_2_5_prompting_guide.md +21 -0
- package/dist/guides/grok_imagine_prompting_guide.md +72 -0
- package/dist/guides/image_reproduction_guide.md +84 -0
- package/dist/guides/kling_3_0_prompting_guide.md +136 -0
- package/dist/guides/nano_banana_pro_prompting_guide.md +28 -0
- package/dist/guides/seedance_2_prompting_guide.md +172 -0
- package/dist/http-access.js +49 -0
- package/dist/http-byte-range.js +31 -0
- package/dist/hydration.js +56 -0
- package/dist/image-compress.js +77 -0
- package/dist/image-edit-default.js +1 -0
- package/dist/image-edit-models.js +41 -0
- package/dist/image-edit-settings.js +11 -0
- package/dist/image-region-edit.js +32 -0
- package/dist/image-subject-map.js +523 -0
- package/dist/image.js +46 -0
- package/dist/index.js +12 -0
- package/dist/job-deadline.js +60 -0
- package/dist/jobs/_alibaba-upload.js +30 -0
- package/dist/jobs/_audio-chunks.js +31 -0
- package/dist/jobs/_download-into-asset.js +15 -0
- package/dist/jobs/_fal-shared.js +22 -0
- package/dist/jobs/_fal-upload.js +35 -0
- package/dist/jobs/_image-edit-adapter.js +213 -0
- package/dist/jobs/_kie-poll.js +119 -0
- package/dist/jobs/_kie-upload.js +32 -0
- package/dist/jobs/_lease.js +163 -0
- package/dist/jobs/_persist-output.js +19 -0
- package/dist/jobs/_storage-retry.js +21 -0
- package/dist/jobs/_transcribe.js +245 -0
- package/dist/jobs/_translate-transcript.js +270 -0
- package/dist/jobs/analyze-alive-captions.js +10 -0
- package/dist/jobs/analyze-editing-moments.js +104 -0
- package/dist/jobs/analyze-jokes.js +143 -0
- package/dist/jobs/analyze-keyword-placement.js +155 -0
- package/dist/jobs/analyze-music.js +14 -0
- package/dist/jobs/analyze-video.js +282 -0
- package/dist/jobs/apply-cuts.js +83 -0
- package/dist/jobs/apply-sfx.js +107 -0
- package/dist/jobs/archive-clip.js +458 -0
- package/dist/jobs/assemble-goal.js +98 -0
- package/dist/jobs/audio-boost.js +71 -0
- package/dist/jobs/audio-mix.js +161 -0
- package/dist/jobs/backfill-asset-tags.js +144 -0
- package/dist/jobs/burn-watermark.js +316 -0
- package/dist/jobs/change-video-speed.js +176 -0
- package/dist/jobs/clean-video-audio.js +207 -0
- package/dist/jobs/cold-open.js +166 -0
- package/dist/jobs/combine-goal.js +108 -0
- package/dist/jobs/complete-provider-task.js +185 -0
- package/dist/jobs/copy-library-asset.js +33 -0
- package/dist/jobs/cost-estimate-assessment.js +113 -0
- package/dist/jobs/crop-image.js +47 -0
- package/dist/jobs/crop-video.js +71 -0
- package/dist/jobs/describe-image.js +181 -0
- package/dist/jobs/detect-scenes.js +33 -0
- package/dist/jobs/doctor-fix.js +100 -0
- package/dist/jobs/download-video.js +427 -0
- package/dist/jobs/enqueue-search-index.js +24 -0
- package/dist/jobs/even-split.js +122 -0
- package/dist/jobs/export-sequence.js +365 -0
- package/dist/jobs/extend-audio.js +229 -0
- package/dist/jobs/extract-audio.js +72 -0
- package/dist/jobs/extract-frame.js +58 -0
- package/dist/jobs/extract-notebook-audio.js +82 -0
- package/dist/jobs/extract-notebook-frame.js +93 -0
- package/dist/jobs/extract-video-clip.js +8 -0
- package/dist/jobs/extract-video-poster.js +55 -0
- package/dist/jobs/extract-waveform.js +25 -0
- package/dist/jobs/finish-goal.js +255 -0
- package/dist/jobs/forget-asset.js +11 -0
- package/dist/jobs/generate-acestep-cover.js +219 -0
- package/dist/jobs/generate-media.js +70 -0
- package/dist/jobs/image-subjects.js +1059 -0
- package/dist/jobs/index.js +18 -0
- package/dist/jobs/inject-first-frame.js +175 -0
- package/dist/jobs/isolate-vocals.js +69 -0
- package/dist/jobs/kie-paygate.js +177 -0
- package/dist/jobs/label-music-sections.js +5 -0
- package/dist/jobs/osg-placement.js +135 -0
- package/dist/jobs/pad-video.js +72 -0
- package/dist/jobs/plan-goal.js +97 -0
- package/dist/jobs/post-record.js +21 -0
- package/dist/jobs/post-to-x.js +21 -0
- package/dist/jobs/prepare-search-reference.js +190 -0
- package/dist/jobs/prepare-temporal-search.js +16 -0
- package/dist/jobs/process-audio.js +67 -0
- package/dist/jobs/process-image.js +58 -0
- package/dist/jobs/process-upload.js +166 -0
- package/dist/jobs/process-video.js +438 -0
- package/dist/jobs/register-all.js +161 -0
- package/dist/jobs/relayout-notebook.js +78 -0
- package/dist/jobs/remove-sound.js +299 -0
- package/dist/jobs/render-project.js +380 -0
- package/dist/jobs/render-qr-outro.js +138 -0
- package/dist/jobs/render-timeline.js +283 -0
- package/dist/jobs/repair-search-indexes.js +42 -0
- package/dist/jobs/replace-video-audio.js +64 -0
- package/dist/jobs/restore-archive-clip.js +53 -0
- package/dist/jobs/reverse-video.js +53 -0
- package/dist/jobs/rewrite-script.js +173 -0
- package/dist/jobs/run-notebook.js +3335 -0
- package/dist/jobs/semantic-index.js +797 -0
- package/dist/jobs/smooth-scene-concat.js +257 -0
- package/dist/jobs/splice-video.js +280 -0
- package/dist/jobs/split-video.js +127 -0
- package/dist/jobs/suggest-cuts.js +149 -0
- package/dist/jobs/suggest-sfx.js +135 -0
- package/dist/jobs/sync-video-to-audio.js +329 -0
- package/dist/jobs/tag-asset.js +266 -0
- package/dist/jobs/task-failure.js +16 -0
- package/dist/jobs/timewarp-video.js +152 -0
- package/dist/jobs/transcribe-video-audio.js +117 -0
- package/dist/jobs/transcribe.js +35 -0
- package/dist/jobs/translate-transcript.js +55 -0
- package/dist/jobs/trim-audio.js +133 -0
- package/dist/jobs/trim-video.js +291 -0
- package/dist/jobs/xai-paygate.js +26 -0
- package/dist/keyword-placement-cache.js +74 -0
- package/dist/lazy-render.js +142 -0
- package/dist/library-browse.js +78 -0
- package/dist/lightning-defaults.js +206 -0
- package/dist/load-env.js +12 -0
- package/dist/local-search-models.js +206 -0
- package/dist/local-upload.js +318 -0
- package/dist/logger.js +69 -0
- package/dist/media/adapters/alibaba.js +20 -0
- package/dist/media/adapters/elevenlabs.js +58 -0
- package/dist/media/adapters/fal.js +45 -0
- package/dist/media/adapters/index.js +26 -0
- package/dist/media/adapters/kie.js +120 -0
- package/dist/media/adapters/pending.js +21 -0
- package/dist/media/adapters/types.js +5 -0
- package/dist/media/adapters/xai.js +18 -0
- package/dist/media/catalog.js +236 -0
- package/dist/media/complete.js +191 -0
- package/dist/media/cost.js +45 -0
- package/dist/media/descriptor.js +157 -0
- package/dist/media/endpoints.js +11 -0
- package/dist/media/frame-options.js +18 -0
- package/dist/media/insert-video-timing.js +17 -0
- package/dist/media/json-schema.js +75 -0
- package/dist/media/legacy-task-types.js +129 -0
- package/dist/media/listing.js +24 -0
- package/dist/media/model-id.js +14 -0
- package/dist/media/preview.js +86 -0
- package/dist/media/prompt-dialects.js +65 -0
- package/dist/media/ref-fields.js +106 -0
- package/dist/media/refs.js +131 -0
- package/dist/media/request.js +188 -0
- package/dist/media/types.js +6 -0
- package/dist/media/wire.js +161 -0
- package/dist/media-library.js +535 -0
- package/dist/media-output-path.js +33 -0
- package/dist/multyculty-workflow.js +757 -0
- package/dist/notebook-action-catalog.js +608 -0
- package/dist/notebook-cell-outputs.js +726 -0
- package/dist/notebook-credit-retry.js +45 -0
- package/dist/notebook-fixtures.js +8 -0
- package/dist/notebook-generation-contract.js +77 -0
- package/dist/notebook-generation-controls.js +145 -0
- package/dist/notebook-goal-layout.js +40 -0
- package/dist/notebook-graph.js +37 -0
- package/dist/notebook-image-edit.js +84 -0
- package/dist/notebook-initial-generation-prompt.js +222 -0
- package/dist/notebook-live-analysis.js +941 -0
- package/dist/notebook-media-ops.js +579 -0
- package/dist/notebook-mutations.js +20 -0
- package/dist/notebook-placement.js +41 -0
- package/dist/notebook-provider-failure.js +87 -0
- package/dist/notebook-published-post.js +26 -0
- package/dist/notebook-reference-resolver.js +227 -0
- package/dist/notebook-relayout.js +270 -0
- package/dist/notebook-run-plan.js +519 -0
- package/dist/notebook-shots.js +162 -0
- package/dist/notebook-workflow.js +856 -0
- package/dist/one-password.js +200 -0
- package/dist/openai-intelligence.js +99 -0
- package/dist/original-media-file.js +12 -0
- package/dist/osg-assets.js +36 -0
- package/dist/osg-placement-cache.js +64 -0
- package/dist/osg-settings.js +27 -0
- package/dist/paid-tools.js +22 -0
- package/dist/platform-guidance.js +26 -0
- package/dist/posts/dropbox-auth.js +91 -0
- package/dist/posts/dropbox-upload.js +132 -0
- package/dist/posts/icloud-drive.js +134 -0
- package/dist/posts/icloud-status.js +64 -0
- package/dist/posts/post-contract.js +2 -0
- package/dist/posts/post-copy-format.js +43 -0
- package/dist/posts/post-copy.js +193 -0
- package/dist/posts/post-folder.js +18 -0
- package/dist/posts/post-metrics.js +119 -0
- package/dist/posts/post-store.js +340 -0
- package/dist/posts/x-post-target.js +43 -0
- package/dist/preview-payload.js +90 -0
- package/dist/prompt-surfaces.js +22 -0
- package/dist/provider-prompt-transform.js +77 -0
- package/dist/provider-status-link.js +21 -0
- package/dist/provider-task-record.js +78 -0
- package/dist/provider-task-status.js +47 -0
- package/dist/provider-usage.js +294 -0
- package/dist/queue.js +810 -0
- package/dist/reference-video-duration.js +17 -0
- package/dist/remotion/BlurFillVideo.js +51 -0
- package/dist/remotion/KeywordGraphicsOverlay.js +68 -0
- package/dist/remotion/OsgOverlay.js +30 -0
- package/dist/remotion/QrOutroComposition.js +47 -0
- package/dist/remotion/RenderComposition.js +11 -0
- package/dist/remotion/browser.js +59 -0
- package/dist/remotion/constants.js +35 -0
- package/dist/remotion/index.js +66 -0
- package/dist/remotion/keyword-graphics.js +151 -0
- package/dist/remotion/keyword-placement.js +182 -0
- package/dist/remotion/osg-spec.js +198 -0
- package/dist/remotion/public/qr-outro-impact.wav +0 -0
- package/dist/remotion/qr-outro-audio.js +1 -0
- package/dist/remotion/qr-outro-spec.js +211 -0
- package/dist/remotion/shared-bundle.js +37 -0
- package/dist/remotion/subtitles/AliveCaptions.js +83 -0
- package/dist/remotion/subtitles/Highlight.js +58 -0
- package/dist/remotion/subtitles/Impact.js +67 -0
- package/dist/remotion/subtitles/SubtitleOverlay.js +9 -0
- package/dist/remotion/subtitles/alive-layout.js +321 -0
- package/dist/remotion/subtitles/alive-types.js +17 -0
- package/dist/remotion/subtitles/chunker.js +68 -0
- package/dist/remotion/subtitles/segments.js +42 -0
- package/dist/remotion/subtitles/styles.js +4 -0
- package/dist/remotion/subtitles/types.js +1 -0
- package/dist/remotion/util.js +18 -0
- package/dist/remotion-cancellation.js +15 -0
- package/dist/rename-helpers.js +204 -0
- package/dist/rhythm/analyze.js +128 -0
- package/dist/rhythm/contract.js +14 -0
- package/dist/rhythm/draft-model.js +19 -0
- package/dist/rhythm/energy.js +133 -0
- package/dist/rhythm/fft.js +44 -0
- package/dist/rhythm/model.js +71 -0
- package/dist/rhythm/music-analysis.js +124 -0
- package/dist/rhythm/music-draft.js +261 -0
- package/dist/rhythm/onset.js +101 -0
- package/dist/rhythm/run-worker.js +51 -0
- package/dist/rhythm/score.js +47 -0
- package/dist/rhythm/section-labels.js +168 -0
- package/dist/rhythm/sequence-rhythm.js +67 -0
- package/dist/rhythm/synthesize.js +103 -0
- package/dist/rhythm/timing.js +98 -0
- package/dist/rhythm/track.js +320 -0
- package/dist/rhythm/visual-fit.js +4 -0
- package/dist/rhythm/worker.js +25 -0
- package/dist/scene-detect.js +200 -0
- package/dist/scene-transition.js +12 -0
- package/dist/scheduled-posts.js +200 -0
- package/dist/search-index-coverage.js +13 -0
- package/dist/search-index-media.js +71 -0
- package/dist/search-reference-store.js +60 -0
- package/dist/search-scoring.js +17 -0
- package/dist/serve.js +125 -0
- package/dist/server-lock.js +143 -0
- package/dist/server.js +42 -0
- package/dist/srt.js +16 -0
- package/dist/tag-backfill-state.js +121 -0
- package/dist/tag-settings.js +30 -0
- package/dist/tag-vocabulary.js +237 -0
- package/dist/test-queue-cleanup.js +1 -0
- package/dist/time-parse.js +50 -0
- package/dist/timeline-helpers.js +554 -0
- package/dist/timewarp.js +87 -0
- package/dist/tools/_image-inputs.js +9 -0
- package/dist/tools/_media-refs.js +84 -0
- package/dist/tools/_shared.js +59 -0
- package/dist/tools/_suggest-shared.js +279 -0
- package/dist/tools/add-timeline-audio.js +73 -0
- package/dist/tools/alive-captions.js +157 -0
- package/dist/tools/analyze-jokes.js +56 -0
- package/dist/tools/analyze-video.js +66 -0
- package/dist/tools/animate-keywords.js +217 -0
- package/dist/tools/api-status.js +42 -0
- package/dist/tools/application-commands.js +34 -0
- package/dist/tools/apply-cuts.js +112 -0
- package/dist/tools/apply-sfx.js +152 -0
- package/dist/tools/apply-watermark.js +670 -0
- package/dist/tools/archive-clip.js +74 -0
- package/dist/tools/asset-history.js +61 -0
- package/dist/tools/asset-manifest.js +24 -0
- package/dist/tools/asset-tags.js +151 -0
- package/dist/tools/assign-slot.js +97 -0
- package/dist/tools/audio-boost.js +107 -0
- package/dist/tools/audio-mix.js +121 -0
- package/dist/tools/book-usage.js +16 -0
- package/dist/tools/chain.js +136 -0
- package/dist/tools/change-video-speed.js +105 -0
- package/dist/tools/chat-history.js +5 -0
- package/dist/tools/chat.js +848 -0
- package/dist/tools/clean-video-audio.js +97 -0
- package/dist/tools/clear-failed-tasks.js +66 -0
- package/dist/tools/cold-open.js +114 -0
- package/dist/tools/command-plan.js +1680 -0
- package/dist/tools/commands.js +1666 -0
- package/dist/tools/concat-images.js +236 -0
- package/dist/tools/connect-service.js +55 -0
- package/dist/tools/copy-file-to-clipboard.js +58 -0
- package/dist/tools/create-character.js +114 -0
- package/dist/tools/create-qart.js +253 -0
- package/dist/tools/create-qr-outro.js +149 -0
- package/dist/tools/create-qrbtf.js +215 -0
- package/dist/tools/create-watermark.js +97 -0
- package/dist/tools/crop-asset.js +18 -0
- package/dist/tools/crop-image.js +120 -0
- package/dist/tools/crop-video.js +97 -0
- package/dist/tools/delete-asset.js +104 -0
- package/dist/tools/describe-image.js +69 -0
- package/dist/tools/detect-scenes.js +44 -0
- package/dist/tools/doctor.js +126 -0
- package/dist/tools/download-video.js +114 -0
- package/dist/tools/duplicate-asset.js +184 -0
- package/dist/tools/edit-timeline.js +59 -0
- package/dist/tools/editing-moments.js +105 -0
- package/dist/tools/estimate-cost.js +171 -0
- package/dist/tools/even-split.js +295 -0
- package/dist/tools/export-asset-file.js +22 -0
- package/dist/tools/export-asset.js +140 -0
- package/dist/tools/export-sequence.js +250 -0
- package/dist/tools/export-to-downloads.js +54 -0
- package/dist/tools/export-transcript-srt.js +79 -0
- package/dist/tools/extend-audio.js +100 -0
- package/dist/tools/extract-audio.js +174 -0
- package/dist/tools/extract-frame.js +108 -0
- package/dist/tools/extract-notebook-audio.js +173 -0
- package/dist/tools/extract-notebook-frame.js +160 -0
- package/dist/tools/extract-video-clip.js +101 -0
- package/dist/tools/generate-media.js +157 -0
- package/dist/tools/generation-payload.js +48 -0
- package/dist/tools/get-timeline.js +12 -0
- package/dist/tools/get-watermark-config.js +38 -0
- package/dist/tools/goal.js +644 -0
- package/dist/tools/image-subjects.js +711 -0
- package/dist/tools/index.js +323 -0
- package/dist/tools/inject-first-frame.js +154 -0
- package/dist/tools/invert-image.js +70 -0
- package/dist/tools/isolate-vocals.js +66 -0
- package/dist/tools/list-assets.js +98 -0
- package/dist/tools/list-cell-generations.js +225 -0
- package/dist/tools/list-elevenlabs-voices.js +163 -0
- package/dist/tools/log-wrapper.js +29 -0
- package/dist/tools/make-cover-song.js +95 -0
- package/dist/tools/mark-asset-seen.js +23 -0
- package/dist/tools/media-library.js +245 -0
- package/dist/tools/move-timeline-audio.js +35 -0
- package/dist/tools/music-analysis.js +60 -0
- package/dist/tools/music-draft.js +29 -0
- package/dist/tools/pad-video.js +87 -0
- package/dist/tools/post-record.js +83 -0
- package/dist/tools/post.js +168 -0
- package/dist/tools/prompt-history.js +164 -0
- package/dist/tools/provider-key-status.js +248 -0
- package/dist/tools/qrbtf-render.js +849 -0
- package/dist/tools/read-asset-metadata.js +26 -0
- package/dist/tools/read-file-at-commit.js +29 -0
- package/dist/tools/read-transcript.js +41 -0
- package/dist/tools/reconcile-project-work.js +1452 -0
- package/dist/tools/recover-asset.js +28 -0
- package/dist/tools/remove-sound.js +72 -0
- package/dist/tools/remove-timeline-audio.js +34 -0
- package/dist/tools/remove-timeline-slot.js +40 -0
- package/dist/tools/rename-all.js +434 -0
- package/dist/tools/rename-asset.js +34 -0
- package/dist/tools/render-settings.js +47 -0
- package/dist/tools/render-timeline.js +120 -0
- package/dist/tools/render.js +384 -0
- package/dist/tools/reorder-character-images.js +45 -0
- package/dist/tools/replace-video-audio.js +133 -0
- package/dist/tools/restore-archive-clips.js +209 -0
- package/dist/tools/restore-asset-version.js +37 -0
- package/dist/tools/retry-generation.js +54 -0
- package/dist/tools/reverse-video.js +74 -0
- package/dist/tools/rewrite-prompt.js +388 -0
- package/dist/tools/rewrite-script.js +67 -0
- package/dist/tools/schema-from-apicity.js +66 -0
- package/dist/tools/search-archive.js +150 -0
- package/dist/tools/semantic-search.js +1214 -0
- package/dist/tools/sequence-rhythm.js +43 -0
- package/dist/tools/set-character-metadata.js +84 -0
- package/dist/tools/set-env-secret.js +40 -0
- package/dist/tools/set-osg.js +143 -0
- package/dist/tools/show-folder.js +37 -0
- package/dist/tools/smooth-scene-concat.js +122 -0
- package/dist/tools/splice-video.js +149 -0
- package/dist/tools/split-video.js +202 -0
- package/dist/tools/suggest-cuts.js +45 -0
- package/dist/tools/suggest-sfx.js +43 -0
- package/dist/tools/sync-video-to-audio.js +115 -0
- package/dist/tools/tag-backfill.js +177 -0
- package/dist/tools/timewarp-video.js +85 -0
- package/dist/tools/tool-def.js +1 -0
- package/dist/tools/tool-json-schema.js +116 -0
- package/dist/tools/transcribe.js +90 -0
- package/dist/tools/translate-subtitles.js +59 -0
- package/dist/tools/trim-audio.js +139 -0
- package/dist/tools/trim-video.js +168 -0
- package/dist/tools/update-script.js +25 -0
- package/dist/tools/update-timeline-audio.js +40 -0
- package/dist/tools/upload-file.js +41 -0
- package/dist/tools/verify-x-mentions.js +141 -0
- package/dist/tools/video-editing-command-contract.js +291 -0
- package/dist/tools/video-source-reference.js +30 -0
- package/dist/tools/video-time-input.js +18 -0
- package/dist/tools/videobook-domain.js +2021 -0
- package/dist/tools/wait-for-asset-ready.js +143 -0
- package/dist/tools/workspace-actions.js +54 -0
- package/dist/transcript-projection.js +202 -0
- package/dist/transcript-srt.js +33 -0
- package/dist/upload.js +76 -0
- package/dist/usage-providers.js +25 -0
- package/dist/video-analysis-projection.js +46 -0
- package/dist/video-poster.js +33 -0
- package/dist/video-proxy.js +99 -0
- package/dist/vision-regions.js +111 -0
- package/dist/voice-actions.js +90 -0
- package/dist/voice-audio.js +55 -0
- package/dist/voice-config.js +122 -0
- package/dist/voice-continuation.js +54 -0
- package/dist/voice-contract.js +1 -0
- package/dist/voice-http.js +145 -0
- package/dist/voice-reply-progress.js +55 -0
- package/dist/voice-session.js +992 -0
- package/dist/watermark-badge.js +259 -0
- package/dist/watermark-config.js +111 -0
- package/dist/watermark-handle-position.js +55 -0
- package/dist/watermark-platform.js +15 -0
- package/dist/watermark-timing.js +16 -0
- package/dist/workspace-action-catalog.js +315 -0
- package/dist/workspace-action-contract.js +21 -0
- package/dist/workspace-action-groups.js +98 -0
- package/dist/workspace-action-surfaces.js +7 -0
- package/dist/x/account.js +88 -0
- package/dist/x/authenticated-handle.js +30 -0
- package/dist/x/oauth-flow.js +191 -0
- package/dist/x/post-contract.js +1 -0
- package/dist/x/post-metadata.js +44 -0
- package/dist/x/post-to-x.js +456 -0
- package/dist/x/published-post-cell.js +90 -0
- package/dist/x/x-tokens.js +69 -0
- package/dist/ytdlp-binary.js +65 -0
- package/dist/zod-slots.js +334 -0
- package/package.json +84 -4
- package/ui/assets/alive-captions-BWb5nLKj.css +1 -0
- package/ui/assets/alive-captions-DPEr1nKM.js +1 -0
- package/ui/assets/confirm-bodies-D8ouHXKy.js +1 -0
- package/ui/assets/confirm-bodies-DrMhaXC6.css +1 -0
- package/ui/assets/fullscreen-B7nw4YbE.js +1 -0
- package/ui/assets/goal-ui-DK-xTCK0.js +2 -0
- package/ui/assets/goal-ui-DXdUM2KF.css +1 -0
- package/ui/assets/index-DzPAnCUj.css +1 -0
- package/ui/assets/index-kS1905s5.js +37 -0
- package/ui/assets/inspector-MlzQbTVo.js +1 -0
- package/ui/assets/jobs-ui-CoinOWx0.js +1 -0
- package/ui/assets/music-video-Cbe80kmQ.js +1 -0
- package/ui/assets/music-video-DPaiKKEq.css +1 -0
- package/ui/assets/ui-Oirr9VOf.css +1 -0
- package/ui/assets/ui-xo_Y1yQg.js +2 -0
- package/ui/assets/vendor-D2rfiZ5j.css +1 -0
- package/ui/assets/vendor-DfTCWgKm.js +19 -0
- package/ui/assets/vendor-flow-BXbeWxW7.js +7 -0
- package/ui/assets/vendor-flow-BnuhLJ6X.css +1 -0
- package/ui/assets/vendor-icons-CGqhCIXV.js +1 -0
- package/ui/assets/vendor-mcp-BTlEJcHV.js +67 -0
- package/ui/assets/vendor-react-Dj7D1fD4.js +9 -0
- package/ui/assets/vendor-remotion-CH1HXrlc.js +95 -0
- package/ui/assets/videobook-Qlp9HQFu.js +20 -0
- package/ui/favicon.svg +8 -0
- package/ui/index.html +126 -0
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
import { createElevenLabs } from "../usage-providers.js";
|
|
2
|
+
import { findAudioFile } from "videobook-engine";
|
|
3
|
+
import { requireApiKey } from "../api-key.js";
|
|
4
|
+
import { enqueueAssetTagging } from "./tag-asset.js";
|
|
5
|
+
const ELEVENLABS_TIMEOUT_MS = 5 * 60_000;
|
|
6
|
+
function inferAudioMime(filename) {
|
|
7
|
+
const ext = filename.toLowerCase().split(".").pop() ?? "";
|
|
8
|
+
switch (ext) {
|
|
9
|
+
case "mp3":
|
|
10
|
+
return "audio/mpeg";
|
|
11
|
+
case "wav":
|
|
12
|
+
return "audio/wav";
|
|
13
|
+
case "m4a":
|
|
14
|
+
return "audio/mp4";
|
|
15
|
+
case "ogg":
|
|
16
|
+
return "audio/ogg";
|
|
17
|
+
case "weba":
|
|
18
|
+
return "audio/webm";
|
|
19
|
+
case "flac":
|
|
20
|
+
return "audio/flac";
|
|
21
|
+
case "mp4":
|
|
22
|
+
case "mov":
|
|
23
|
+
return "video/mp4";
|
|
24
|
+
default:
|
|
25
|
+
return "application/octet-stream";
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Transcribe an asset's audio with ElevenLabs scribe_v2 and write the result
|
|
30
|
+
* to `elevenlabs.json` in the asset directory. Lease-agnostic: the caller owns
|
|
31
|
+
* the asset lease (the transcribe job's local lease, or the render job's render
|
|
32
|
+
* lease when auto-transcribing). Throws on missing API key, unreadable audio,
|
|
33
|
+
* or a failed write so the caller can surface the failure.
|
|
34
|
+
*/
|
|
35
|
+
export async function transcribeAssetToElevenLabsJson(engine, assetId, audioFilename, options) {
|
|
36
|
+
const response = await requestElevenLabsTranscript(engine, assetId, audioFilename, options);
|
|
37
|
+
await writeElevenLabsTranscript(engine, assetId, response, options);
|
|
38
|
+
}
|
|
39
|
+
export async function requestElevenLabsTranscript(engine, assetId, audioFilename, options) {
|
|
40
|
+
const apiKey = requireApiKey("ELEVENLABS_API_KEY");
|
|
41
|
+
const readResult = await engine.files.read(assetId, audioFilename);
|
|
42
|
+
if (!readResult.ok) {
|
|
43
|
+
throw new Error(`Failed to read audio for ${assetId}: ${readResult.error.message}`);
|
|
44
|
+
}
|
|
45
|
+
const audioBytes = new Uint8Array(readResult.value);
|
|
46
|
+
const audioBlob = new Blob([audioBytes], { type: inferAudioMime(audioFilename) });
|
|
47
|
+
const provider = createElevenLabs({ apiKey, timeout: ELEVENLABS_TIMEOUT_MS });
|
|
48
|
+
const request = {
|
|
49
|
+
model_id: "scribe_v2",
|
|
50
|
+
file: audioBlob,
|
|
51
|
+
tag_audio_events: options.tagAudioEvents,
|
|
52
|
+
diarize: options.diarize,
|
|
53
|
+
...(options.languageCode ? { language_code: options.languageCode } : {}),
|
|
54
|
+
};
|
|
55
|
+
return provider.v1.speechToText(request);
|
|
56
|
+
}
|
|
57
|
+
export async function writeElevenLabsTranscript(engine, assetId, response, options) {
|
|
58
|
+
const transcriptionStr = JSON.stringify(response, null, 2);
|
|
59
|
+
const writeResult = await engine.files.write(assetId, "elevenlabs.json", transcriptionStr);
|
|
60
|
+
if (!writeResult.ok) {
|
|
61
|
+
throw new Error(`Failed to write transcription: ${writeResult.error.message}`);
|
|
62
|
+
}
|
|
63
|
+
await importElevenLabsTranscript(engine, assetId, response, options);
|
|
64
|
+
await engine.history.recordOperation("transcribe", assetId, {
|
|
65
|
+
...(options.languageCode ? { language_code: options.languageCode } : {}),
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
export async function importElevenLabsTranscript(engine, assetId, response, options) {
|
|
69
|
+
const source = normalizedTranscripts(response);
|
|
70
|
+
if (source.length === 0) {
|
|
71
|
+
throw new Error("ElevenLabs returned an acknowledgement without transcript content");
|
|
72
|
+
}
|
|
73
|
+
const streams = engine.streams.list(assetId);
|
|
74
|
+
if (!streams.ok) {
|
|
75
|
+
throw new Error(`Failed to read media streams for ${assetId}: ${streams.error.message}`);
|
|
76
|
+
}
|
|
77
|
+
const stream = streams.value.find((candidate) => candidate.kind === "audio");
|
|
78
|
+
if (!stream)
|
|
79
|
+
throw new Error(`No audio stream is registered for ${assetId}`);
|
|
80
|
+
const segments = transcriptSegments(source, stream);
|
|
81
|
+
const language = options.languageCode
|
|
82
|
+
?? [...source].sort((left, right) => right.language_probability - left.language_probability)[0]?.language_code
|
|
83
|
+
?? "und";
|
|
84
|
+
const imported = await engine.transcripts.import({
|
|
85
|
+
artifactId: assetId,
|
|
86
|
+
streamId: stream.streamId,
|
|
87
|
+
objectHash: stream.objectHash,
|
|
88
|
+
language,
|
|
89
|
+
provider: "elevenlabs",
|
|
90
|
+
model: "scribe_v2",
|
|
91
|
+
state: "current",
|
|
92
|
+
segments,
|
|
93
|
+
});
|
|
94
|
+
if (!imported.ok) {
|
|
95
|
+
throw new Error(`Failed to import transcript for ${assetId}: ${imported.error.message}`);
|
|
96
|
+
}
|
|
97
|
+
// Speech is the evidence audio tagging reads, and it can change what a video
|
|
98
|
+
// is grouped by too.
|
|
99
|
+
await enqueueAssetTagging(engine, assetId);
|
|
100
|
+
return imported.value;
|
|
101
|
+
}
|
|
102
|
+
function normalizedTranscripts(response) {
|
|
103
|
+
if ("words" in response)
|
|
104
|
+
return [response];
|
|
105
|
+
if ("transcripts" in response)
|
|
106
|
+
return response.transcripts;
|
|
107
|
+
return [];
|
|
108
|
+
}
|
|
109
|
+
function transcriptSegments(transcripts, stream) {
|
|
110
|
+
const timedWords = transcripts
|
|
111
|
+
.flatMap((transcript) => transcript.words.map((word) => ({
|
|
112
|
+
...word,
|
|
113
|
+
channelIndex: transcript.channel_index,
|
|
114
|
+
})))
|
|
115
|
+
.filter((word) => word.type !== "spacing"
|
|
116
|
+
&& Number.isFinite(word.start)
|
|
117
|
+
&& Number.isFinite(word.end)
|
|
118
|
+
&& word.end > word.start)
|
|
119
|
+
.sort((left, right) => left.start - right.start
|
|
120
|
+
|| left.channelIndex - right.channelIndex
|
|
121
|
+
|| left.end - right.end);
|
|
122
|
+
const drafts = [];
|
|
123
|
+
for (const word of timedWords) {
|
|
124
|
+
const kind = word.type === "audio_event"
|
|
125
|
+
? audioEventKind(word.text)
|
|
126
|
+
: "speech";
|
|
127
|
+
const speaker = speakerLabel(word);
|
|
128
|
+
const previous = drafts.at(-1);
|
|
129
|
+
const joinsPrevious = kind === "speech"
|
|
130
|
+
&& previous?.kind === "speech"
|
|
131
|
+
&& previous.speaker === speaker
|
|
132
|
+
&& previous.channelIndex === word.channelIndex
|
|
133
|
+
&& word.start - previous.end <= 1.25
|
|
134
|
+
&& word.end - previous.start <= 15;
|
|
135
|
+
if (joinsPrevious && previous) {
|
|
136
|
+
previous.end = Math.max(previous.end, word.end);
|
|
137
|
+
previous.words.push(word);
|
|
138
|
+
previous.text = joinedSpeech(previous.words);
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
drafts.push({
|
|
142
|
+
kind,
|
|
143
|
+
...(speaker ? { speaker } : {}),
|
|
144
|
+
channelIndex: word.channelIndex,
|
|
145
|
+
start: word.start,
|
|
146
|
+
end: word.end,
|
|
147
|
+
words: kind === "speech" ? [word] : [],
|
|
148
|
+
text: kind === "speech" ? word.text.trim() : word.text.trim(),
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
return drafts.map((draft, ordinal) => {
|
|
152
|
+
const startTick = secondsToTick(draft.start, stream);
|
|
153
|
+
const endTick = Math.max(startTick + 1, secondsToTick(draft.end, stream));
|
|
154
|
+
const confidenceValues = draft.words
|
|
155
|
+
.map(wordConfidence)
|
|
156
|
+
.filter((value) => value !== undefined);
|
|
157
|
+
const confidence = confidenceValues.length > 0
|
|
158
|
+
? confidenceValues.reduce((sum, value) => sum + value, 0)
|
|
159
|
+
/ confidenceValues.length
|
|
160
|
+
: undefined;
|
|
161
|
+
return {
|
|
162
|
+
ordinal,
|
|
163
|
+
range: {
|
|
164
|
+
streamId: stream.streamId,
|
|
165
|
+
objectHash: stream.objectHash,
|
|
166
|
+
startTick,
|
|
167
|
+
durationTicks: Math.min(stream.durationTicks, endTick) - startTick,
|
|
168
|
+
timeBase: stream.timeBase,
|
|
169
|
+
},
|
|
170
|
+
...(draft.speaker ? { speaker: draft.speaker } : {}),
|
|
171
|
+
text: draft.text,
|
|
172
|
+
...(confidence !== undefined ? { confidence } : {}),
|
|
173
|
+
kind: draft.kind,
|
|
174
|
+
words: draft.words.map((word, wordOrdinal) => {
|
|
175
|
+
const wordStartTick = secondsToTick(word.start, stream);
|
|
176
|
+
const wordEndTick = Math.max(wordStartTick + 1, secondsToTick(word.end, stream));
|
|
177
|
+
const wordConfidenceValue = wordConfidence(word);
|
|
178
|
+
return {
|
|
179
|
+
ordinal: wordOrdinal,
|
|
180
|
+
startTick: wordStartTick,
|
|
181
|
+
durationTicks: Math.min(stream.durationTicks, wordEndTick) - wordStartTick,
|
|
182
|
+
text: word.text.trim(),
|
|
183
|
+
...(wordConfidenceValue !== undefined
|
|
184
|
+
? { confidence: wordConfidenceValue }
|
|
185
|
+
: {}),
|
|
186
|
+
corrected: false,
|
|
187
|
+
};
|
|
188
|
+
}),
|
|
189
|
+
};
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
function secondsToTick(seconds, stream) {
|
|
193
|
+
const tick = Math.round(seconds * stream.timeBase.denominator / stream.timeBase.numerator);
|
|
194
|
+
return Math.max(0, Math.min(stream.durationTicks - 1, tick));
|
|
195
|
+
}
|
|
196
|
+
function speakerLabel(word) {
|
|
197
|
+
const channel = word.channelIndex > 0 ? `channel-${word.channelIndex + 1}` : "";
|
|
198
|
+
if (word.speaker_id && channel)
|
|
199
|
+
return `${channel}:${word.speaker_id}`;
|
|
200
|
+
return word.speaker_id ?? (channel || undefined);
|
|
201
|
+
}
|
|
202
|
+
function audioEventKind(text) {
|
|
203
|
+
const normalized = text.toLowerCase();
|
|
204
|
+
if (normalized.includes("music") || normalized.includes("sing"))
|
|
205
|
+
return "music";
|
|
206
|
+
if (normalized.includes("silence") || normalized.includes("pause"))
|
|
207
|
+
return "silence";
|
|
208
|
+
return "sound";
|
|
209
|
+
}
|
|
210
|
+
function joinedSpeech(words) {
|
|
211
|
+
return words
|
|
212
|
+
.map((word) => word.text.trim())
|
|
213
|
+
.filter(Boolean)
|
|
214
|
+
.join(" ")
|
|
215
|
+
.replace(/\s+([,.;:!?])/g, "$1");
|
|
216
|
+
}
|
|
217
|
+
function wordConfidence(word) {
|
|
218
|
+
if (word.logprob === undefined || !Number.isFinite(word.logprob))
|
|
219
|
+
return undefined;
|
|
220
|
+
return Math.max(0, Math.min(1, Math.exp(word.logprob)));
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Ensure an asset has ElevenLabs word-level timings (elevenlabs.json),
|
|
224
|
+
* transcribing inline if it's missing. Idempotent — returns immediately when the
|
|
225
|
+
* file already exists. Lease-agnostic: the caller owns the asset lease (a render
|
|
226
|
+
* job's render lease, a plan job's analyze lease). Throws on missing audio, a
|
|
227
|
+
* missing API key, or an API/write failure so the caller surfaces the cause.
|
|
228
|
+
*/
|
|
229
|
+
export async function ensureElevenLabsTranscript(engine, assetId) {
|
|
230
|
+
if ((await engine.files.read(assetId, "elevenlabs.json")).ok)
|
|
231
|
+
return;
|
|
232
|
+
const manifest = await engine.files.manifest(assetId);
|
|
233
|
+
if (!manifest.ok)
|
|
234
|
+
throw new Error(`auto-transcribe failed: ${manifest.error.message}`);
|
|
235
|
+
const artifact = engine.artifacts.get(assetId);
|
|
236
|
+
if (!artifact.ok)
|
|
237
|
+
throw new Error(`auto-transcribe failed: ${artifact.error.message}`);
|
|
238
|
+
const audioFile = findAudioFile(manifest.value.files, artifact.value.kind);
|
|
239
|
+
if (!audioFile)
|
|
240
|
+
throw new Error(`auto-transcribe failed: no audio file found in asset ${assetId}`);
|
|
241
|
+
await transcribeAssetToElevenLabsJson(engine, assetId, audioFile.name, {
|
|
242
|
+
tagAudioEvents: true,
|
|
243
|
+
diarize: true,
|
|
244
|
+
});
|
|
245
|
+
}
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { parseSegments } from "../remotion/subtitles/segments.js";
|
|
3
|
+
import { INTELLIGENCE_MODEL, firstResponseText, intelligenceRequest, } from "../openai-intelligence.js";
|
|
4
|
+
const TRANSLATION_MODEL = INTELLIGENCE_MODEL;
|
|
5
|
+
// Subtitle language setting: 'original' (use the source transcript) or an
|
|
6
|
+
// ISO 639-1 code naming the translation target.
|
|
7
|
+
export function isLanguageSetting(value) {
|
|
8
|
+
return typeof value === "string" && (value === "original" || /^[a-z]{2}$/.test(value));
|
|
9
|
+
}
|
|
10
|
+
export function translatedTranscriptFilename(language) {
|
|
11
|
+
return `elevenlabs.${language}.json`;
|
|
12
|
+
}
|
|
13
|
+
// ElevenLabs scribe reports language_code in ISO 639-3; the UI setting uses
|
|
14
|
+
// ISO 639-1. Covers the curated language list plus common detection results.
|
|
15
|
+
const ISO3_TO_ISO1 = {
|
|
16
|
+
eng: "en",
|
|
17
|
+
zho: "zh",
|
|
18
|
+
cmn: "zh",
|
|
19
|
+
yue: "zh",
|
|
20
|
+
spa: "es",
|
|
21
|
+
por: "pt",
|
|
22
|
+
fra: "fr",
|
|
23
|
+
deu: "de",
|
|
24
|
+
ita: "it",
|
|
25
|
+
nld: "nl",
|
|
26
|
+
pol: "pl",
|
|
27
|
+
rus: "ru",
|
|
28
|
+
tur: "tr",
|
|
29
|
+
ara: "ar",
|
|
30
|
+
hin: "hi",
|
|
31
|
+
ind: "id",
|
|
32
|
+
jpn: "ja",
|
|
33
|
+
kor: "ko",
|
|
34
|
+
vie: "vi",
|
|
35
|
+
tha: "th",
|
|
36
|
+
ukr: "uk",
|
|
37
|
+
swe: "sv",
|
|
38
|
+
dan: "da",
|
|
39
|
+
nor: "no",
|
|
40
|
+
fin: "fi",
|
|
41
|
+
ces: "cs",
|
|
42
|
+
ell: "el",
|
|
43
|
+
heb: "he",
|
|
44
|
+
hun: "hu",
|
|
45
|
+
ron: "ro",
|
|
46
|
+
};
|
|
47
|
+
const LANGUAGE_NAMES = {
|
|
48
|
+
en: "English",
|
|
49
|
+
zh: "Chinese",
|
|
50
|
+
es: "Spanish",
|
|
51
|
+
pt: "Portuguese",
|
|
52
|
+
fr: "French",
|
|
53
|
+
de: "German",
|
|
54
|
+
it: "Italian",
|
|
55
|
+
nl: "Dutch",
|
|
56
|
+
pl: "Polish",
|
|
57
|
+
ru: "Russian",
|
|
58
|
+
tr: "Turkish",
|
|
59
|
+
ar: "Arabic",
|
|
60
|
+
hi: "Hindi",
|
|
61
|
+
id: "Indonesian",
|
|
62
|
+
ja: "Japanese",
|
|
63
|
+
ko: "Korean",
|
|
64
|
+
vi: "Vietnamese",
|
|
65
|
+
th: "Thai",
|
|
66
|
+
uk: "Ukrainian",
|
|
67
|
+
};
|
|
68
|
+
export function iso1FromIso3(code) {
|
|
69
|
+
if (!code)
|
|
70
|
+
return null;
|
|
71
|
+
return ISO3_TO_ISO1[code.toLowerCase()] ?? null;
|
|
72
|
+
}
|
|
73
|
+
export function readLanguageCode(raw) {
|
|
74
|
+
if (typeof raw !== "object" || raw === null)
|
|
75
|
+
return null;
|
|
76
|
+
const code = raw.language_code;
|
|
77
|
+
return typeof code === "string" ? code : null;
|
|
78
|
+
}
|
|
79
|
+
// ── Sentence grouping ──
|
|
80
|
+
//
|
|
81
|
+
// The split exists only to preserve subtitle timing: each sentence carries the
|
|
82
|
+
// time window of its source words, and the translated sentence inherits it.
|
|
83
|
+
// Translation itself always sees the full transcript in one request.
|
|
84
|
+
const SENTENCE_END_RE = /[.!?。!?…]["'"』」)]*$/;
|
|
85
|
+
const MAX_SENTENCE_WORDS = 30;
|
|
86
|
+
const PAUSE_GAP_SECONDS = 0.8;
|
|
87
|
+
const CJK_RE = /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff66-\uff9f]/;
|
|
88
|
+
function joinWordTexts(texts) {
|
|
89
|
+
let joined = "";
|
|
90
|
+
for (const text of texts) {
|
|
91
|
+
if (joined.length === 0) {
|
|
92
|
+
joined = text;
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
const cjkBoundary = CJK_RE.test(joined.slice(-1)) && CJK_RE.test(text.charAt(0));
|
|
96
|
+
joined += cjkBoundary ? text : ` ${text}`;
|
|
97
|
+
}
|
|
98
|
+
return joined;
|
|
99
|
+
}
|
|
100
|
+
export function groupIntoSentences(words) {
|
|
101
|
+
const sentences = [];
|
|
102
|
+
let current = [];
|
|
103
|
+
const flush = () => {
|
|
104
|
+
const texts = current.map((w) => w.word.trim()).filter((t) => t.length > 0);
|
|
105
|
+
if (texts.length > 0) {
|
|
106
|
+
sentences.push({
|
|
107
|
+
start: current[0].start,
|
|
108
|
+
end: current[current.length - 1].end,
|
|
109
|
+
text: joinWordTexts(texts),
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
current = [];
|
|
113
|
+
};
|
|
114
|
+
for (let i = 0; i < words.length; i++) {
|
|
115
|
+
const word = words[i];
|
|
116
|
+
current.push(word);
|
|
117
|
+
const next = words[i + 1];
|
|
118
|
+
const sentenceEnd = SENTENCE_END_RE.test(word.word.trim());
|
|
119
|
+
const longPause = next !== undefined && next.start - word.end > PAUSE_GAP_SECONDS;
|
|
120
|
+
if (sentenceEnd || longPause || current.length >= MAX_SENTENCE_WORDS)
|
|
121
|
+
flush();
|
|
122
|
+
}
|
|
123
|
+
flush();
|
|
124
|
+
return sentences;
|
|
125
|
+
}
|
|
126
|
+
// ── Word-timing synthesis for translated text ──
|
|
127
|
+
function tokenizeForTiming(text) {
|
|
128
|
+
const tokens = [];
|
|
129
|
+
for (const rough of text.split(/\s+/)) {
|
|
130
|
+
if (rough.length === 0)
|
|
131
|
+
continue;
|
|
132
|
+
// CJK targets arrive as long unspaced runs; split into bigrams so
|
|
133
|
+
// word-highlight styles still animate.
|
|
134
|
+
if (rough.length > 2 && CJK_RE.test(rough)) {
|
|
135
|
+
for (let i = 0; i < rough.length; i += 2)
|
|
136
|
+
tokens.push(rough.slice(i, i + 2));
|
|
137
|
+
}
|
|
138
|
+
else {
|
|
139
|
+
tokens.push(rough);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return tokens;
|
|
143
|
+
}
|
|
144
|
+
export function synthesizeWordTimings(text, start, end) {
|
|
145
|
+
const tokens = tokenizeForTiming(text);
|
|
146
|
+
if (tokens.length === 0)
|
|
147
|
+
return [];
|
|
148
|
+
const weights = tokens.map((t) => Math.max(1, t.length));
|
|
149
|
+
const totalWeight = weights.reduce((sum, w) => sum + w, 0);
|
|
150
|
+
const duration = Math.max(0, end - start);
|
|
151
|
+
const words = [];
|
|
152
|
+
let consumed = 0;
|
|
153
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
154
|
+
const wordStart = start + (duration * consumed) / totalWeight;
|
|
155
|
+
consumed += weights[i];
|
|
156
|
+
const wordEnd = start + (duration * consumed) / totalWeight;
|
|
157
|
+
words.push({ text: tokens[i], start: wordStart, end: wordEnd, type: "word" });
|
|
158
|
+
}
|
|
159
|
+
return words;
|
|
160
|
+
}
|
|
161
|
+
// ── Cache invalidation hash ──
|
|
162
|
+
export function computeSourceTextHash(segments) {
|
|
163
|
+
const hash = createHash("sha1");
|
|
164
|
+
for (const segment of segments) {
|
|
165
|
+
if (segment.words.length > 0) {
|
|
166
|
+
for (const word of segment.words)
|
|
167
|
+
hash.update(`${word.word.trim()}\n`);
|
|
168
|
+
}
|
|
169
|
+
else {
|
|
170
|
+
hash.update(`${segment.text.trim()}\n`);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return hash.digest("hex").slice(0, 12);
|
|
174
|
+
}
|
|
175
|
+
// ── Translation ──
|
|
176
|
+
function buildInstructions(targetLanguage, lineCount) {
|
|
177
|
+
const languageName = LANGUAGE_NAMES[targetLanguage] ?? targetLanguage;
|
|
178
|
+
return ("You translate video subtitles. The user message contains the FULL transcript of one video " +
|
|
179
|
+
"as a JSON array of numbered lines, in spoken order. Translate the dialog as one coherent " +
|
|
180
|
+
`whole into ${languageName}: natural conversational phrasing, with names, pronouns, and ` +
|
|
181
|
+
"terminology kept consistent across lines.\n\n" +
|
|
182
|
+
`Return ONLY a JSON object {"translations": [...]} containing exactly ${lineCount} strings — ` +
|
|
183
|
+
"translation i corresponds to line i. Do not merge, split, reorder, or omit lines. " +
|
|
184
|
+
"No romanization, no notes, no text outside the JSON.");
|
|
185
|
+
}
|
|
186
|
+
async function requestTranslations(sentences, targetLanguage, responses, signal) {
|
|
187
|
+
const response = await responses(intelligenceRequest({
|
|
188
|
+
instructions: buildInstructions(targetLanguage, sentences.length),
|
|
189
|
+
input: [{
|
|
190
|
+
role: "user",
|
|
191
|
+
content: `TRANSCRIPT LINES (JSON):\n${JSON.stringify(sentences.map((s, i) => ({ i, text: s.text })))}`,
|
|
192
|
+
}],
|
|
193
|
+
text: {
|
|
194
|
+
format: {
|
|
195
|
+
type: "json_schema",
|
|
196
|
+
name: "translated_transcript",
|
|
197
|
+
schema: {
|
|
198
|
+
type: "object",
|
|
199
|
+
required: ["translations"],
|
|
200
|
+
additionalProperties: false,
|
|
201
|
+
properties: {
|
|
202
|
+
translations: {
|
|
203
|
+
type: "array",
|
|
204
|
+
minItems: sentences.length,
|
|
205
|
+
maxItems: sentences.length,
|
|
206
|
+
items: { type: "string" },
|
|
207
|
+
},
|
|
208
|
+
},
|
|
209
|
+
},
|
|
210
|
+
strict: true,
|
|
211
|
+
},
|
|
212
|
+
},
|
|
213
|
+
max_output_tokens: 4096,
|
|
214
|
+
}), signal);
|
|
215
|
+
const content = firstResponseText(response);
|
|
216
|
+
if (!content) {
|
|
217
|
+
throw new Error(`Translation provider returned no content (status=${response.status})`);
|
|
218
|
+
}
|
|
219
|
+
const parsed = JSON.parse(content);
|
|
220
|
+
if (!Array.isArray(parsed.translations)) {
|
|
221
|
+
throw new Error(`Translation model returned no translations array. Got keys: ${Object.keys(parsed).join(",")}`);
|
|
222
|
+
}
|
|
223
|
+
const translations = parsed.translations;
|
|
224
|
+
if (translations.length !== sentences.length || !translations.every((t) => typeof t === "string")) {
|
|
225
|
+
throw new Error(`Translation model returned ${translations.length} entries for ${sentences.length} lines`);
|
|
226
|
+
}
|
|
227
|
+
return translations.map((t) => t.trim());
|
|
228
|
+
}
|
|
229
|
+
function collectSentences(segments) {
|
|
230
|
+
const sentences = [];
|
|
231
|
+
for (const segment of segments) {
|
|
232
|
+
if (segment.words.length > 0) {
|
|
233
|
+
sentences.push(...groupIntoSentences(segment.words));
|
|
234
|
+
}
|
|
235
|
+
else if (segment.text.trim().length > 0) {
|
|
236
|
+
sentences.push({ start: segment.start, end: segment.end, text: segment.text.trim() });
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
return sentences;
|
|
240
|
+
}
|
|
241
|
+
export async function translateTranscript(raw, targetLanguage, responses, signal) {
|
|
242
|
+
const sourceSegments = parseSegments(raw);
|
|
243
|
+
const sentences = collectSentences(sourceSegments);
|
|
244
|
+
if (sentences.length === 0) {
|
|
245
|
+
throw new Error("Transcript contains no subtitle text to translate");
|
|
246
|
+
}
|
|
247
|
+
const translations = await requestTranslations(sentences, targetLanguage, responses, signal);
|
|
248
|
+
const segments = sentences.map((sentence, i) => ({
|
|
249
|
+
start: sentence.start,
|
|
250
|
+
end: sentence.end,
|
|
251
|
+
text: translations[i],
|
|
252
|
+
words: synthesizeWordTimings(translations[i], sentence.start, sentence.end),
|
|
253
|
+
}));
|
|
254
|
+
return {
|
|
255
|
+
sourceLanguageCode: readLanguageCode(raw),
|
|
256
|
+
targetLanguage,
|
|
257
|
+
model: TRANSLATION_MODEL,
|
|
258
|
+
translatedAt: new Date().toISOString(),
|
|
259
|
+
sourceTextHash: computeSourceTextHash(sourceSegments),
|
|
260
|
+
segments,
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
// True when an existing sidecar was produced from the current transcript and
|
|
264
|
+
// can be served as-is.
|
|
265
|
+
export function sidecarMatchesSource(sidecar, sourceSegments) {
|
|
266
|
+
if (typeof sidecar !== "object" || sidecar === null)
|
|
267
|
+
return false;
|
|
268
|
+
const hash = sidecar.sourceTextHash;
|
|
269
|
+
return typeof hash === "string" && hash === computeSourceTextHash(sourceSegments);
|
|
270
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { analyzeAliveCaptions } from "../alive-captions/analyze.js";
|
|
2
|
+
import { registerJobHandler } from "../queue.js";
|
|
3
|
+
import { withLockedLocalLease } from "./_lease.js";
|
|
4
|
+
const LOCK_DURATION_MS = 600_000;
|
|
5
|
+
export function registerAnalyzeAliveCaptionsJob(engine) {
|
|
6
|
+
registerJobHandler("analyze_alive_captions", async (payload, ctx) => {
|
|
7
|
+
const { assetId, assetDir, force = false } = payload;
|
|
8
|
+
return withLockedLocalLease(engine, assetId, assetDir, ctx, "analyze", { durationMs: LOCK_DURATION_MS, data: { state: "analyzing-alive-captions" } }, async () => ({ analysis: await analyzeAliveCaptions(engine, assetId, ctx.signal, force) }), { durationMs: LOCK_DURATION_MS, waitForActiveMs: 30_000 });
|
|
9
|
+
});
|
|
10
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { findPrimaryMediaFile } from "videobook-engine";
|
|
3
|
+
import { registerJobHandler } from "../queue.js";
|
|
4
|
+
import { FFMPEG } from "../ffmpeg.js";
|
|
5
|
+
import { MOMENTS_CHANGE_THRESHOLD, MOMENTS_EXTRACTOR_ID, MOMENTS_EXTRACTOR_VERSION, MOMENTS_MAX_SCAN_SECONDS, MOMENTS_MIN_MOMENT_SECONDS, MOMENTS_SHOT_THRESHOLD, editingMomentsCacheKey, editingMomentsId, rankedMoments, reuseContext, shotsFromScan, usableRanges, } from "../editing-moments.js";
|
|
6
|
+
import { DOWNSTREAM_LEASE_WAIT_MS, withLocalLease } from "./_lease.js";
|
|
7
|
+
export function registerEditingMomentsJob(engine) {
|
|
8
|
+
registerJobHandler("analyze_editing_moments", async (payload, ctx) => {
|
|
9
|
+
const { assetId, startSeconds, endSeconds } = payload;
|
|
10
|
+
return withLocalLease(engine, assetId, ctx, "analyze", async () => {
|
|
11
|
+
const artifact = engine.artifacts.get(assetId);
|
|
12
|
+
if (!artifact.ok)
|
|
13
|
+
throw new Error(artifact.error.message);
|
|
14
|
+
if (!["video", "final"].includes(artifact.value.kind))
|
|
15
|
+
throw new Error("Choose a video source to analyze");
|
|
16
|
+
const manifest = await engine.files.manifest(assetId);
|
|
17
|
+
if (!manifest.ok)
|
|
18
|
+
throw new Error(manifest.error.message);
|
|
19
|
+
const file = findPrimaryMediaFile(manifest.value.files, artifact.value.kind);
|
|
20
|
+
const streams = engine.streams.list(assetId);
|
|
21
|
+
if (!streams.ok)
|
|
22
|
+
throw new Error(streams.error.message);
|
|
23
|
+
const stream = streams.value.find((item) => item.kind === "video" && item.sourcePath === file?.name
|
|
24
|
+
&& item.objectHash === file?.objectHash);
|
|
25
|
+
if (!stream || !file)
|
|
26
|
+
throw new Error("Wait for the current video source to finish processing");
|
|
27
|
+
const durationSeconds = stream.durationTicks * stream.timeBase.numerator / stream.timeBase.denominator;
|
|
28
|
+
if (!(durationSeconds > 0) || durationSeconds > MOMENTS_MAX_SCAN_SECONDS) {
|
|
29
|
+
throw new Error(`Moment analysis supports video up to ${MOMENTS_MAX_SCAN_SECONDS / 60} minutes`);
|
|
30
|
+
}
|
|
31
|
+
const start = Math.min(Math.max(0, startSeconds), Math.max(0, durationSeconds - MOMENTS_MIN_MOMENT_SECONDS));
|
|
32
|
+
const end = endSeconds === null ? durationSeconds : Math.min(Math.max(start + MOMENTS_MIN_MOMENT_SECONDS, endSeconds), durationSeconds);
|
|
33
|
+
const momentsId = editingMomentsId({
|
|
34
|
+
objectHash: stream.objectHash, streamIndex: stream.streamIndex, startSeconds: start, endSeconds: end,
|
|
35
|
+
});
|
|
36
|
+
const scan = await scanSceneScores(`${manifest.value.path}/${file.name}`, start, end, ctx.signal);
|
|
37
|
+
const shots = shotsFromScan(scan, end - start).map((shot) => ({
|
|
38
|
+
...shot, startSeconds: shot.startSeconds + start, endSeconds: shot.endSeconds + start,
|
|
39
|
+
change: shot.change ? { ...shot.change, timeSeconds: shot.change.timeSeconds + start } : null,
|
|
40
|
+
}));
|
|
41
|
+
const moments = {
|
|
42
|
+
schemaVersion: 1,
|
|
43
|
+
momentsId,
|
|
44
|
+
artifactId: assetId,
|
|
45
|
+
objectHash: stream.objectHash,
|
|
46
|
+
streamId: stream.streamId,
|
|
47
|
+
durationSeconds,
|
|
48
|
+
extractor: {
|
|
49
|
+
id: MOMENTS_EXTRACTOR_ID,
|
|
50
|
+
version: MOMENTS_EXTRACTOR_VERSION,
|
|
51
|
+
params: {
|
|
52
|
+
shotThreshold: MOMENTS_SHOT_THRESHOLD,
|
|
53
|
+
changeThreshold: MOMENTS_CHANGE_THRESHOLD,
|
|
54
|
+
minMomentSeconds: MOMENTS_MIN_MOMENT_SECONDS,
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
analyzedRanges: [{ startSeconds: start, endSeconds: end }],
|
|
58
|
+
createdAt: Date.now(),
|
|
59
|
+
shots,
|
|
60
|
+
usable: usableRanges([{ startSeconds: start, endSeconds: end }], shots),
|
|
61
|
+
moments: rankedMoments(shots),
|
|
62
|
+
reused: await reuseContext(engine, assetId),
|
|
63
|
+
};
|
|
64
|
+
const saved = await engine.metadata.artifacts.write(assetId, editingMomentsCacheKey(momentsId), moments);
|
|
65
|
+
if (!saved.ok)
|
|
66
|
+
throw new Error(saved.error.message);
|
|
67
|
+
return { momentsId, shots: shots.length };
|
|
68
|
+
}, { waitForActiveMs: DOWNSTREAM_LEASE_WAIT_MS });
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
async function scanSceneScores(inputPath, startSeconds, endSeconds, signal) {
|
|
72
|
+
const stdout = await new Promise((resolve, reject) => {
|
|
73
|
+
execFile(FFMPEG, [
|
|
74
|
+
"-v", "error", "-i", inputPath,
|
|
75
|
+
...(startSeconds > 0 ? ["-ss", String(startSeconds)] : []),
|
|
76
|
+
"-to", String(endSeconds),
|
|
77
|
+
"-map", "0:v:0",
|
|
78
|
+
"-vf", "select='gte(scene,0)',metadata=print:file=-",
|
|
79
|
+
"-an", "-f", "null", "-",
|
|
80
|
+
], { encoding: "utf8", maxBuffer: 256 * 1024 * 1024, killSignal: "SIGKILL", signal }, (error, stdout) => {
|
|
81
|
+
if (error)
|
|
82
|
+
reject(new Error(`ffmpeg scene scan failed: ${error.message}`));
|
|
83
|
+
else
|
|
84
|
+
resolve(stdout);
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
const frames = [];
|
|
88
|
+
let pendingTime = null;
|
|
89
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
90
|
+
const time = /pts_time:([0-9]+(?:\.[0-9]+)?)/.exec(line);
|
|
91
|
+
if (time) {
|
|
92
|
+
pendingTime = Number.parseFloat(time[1]);
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
const score = /lavfi\.scene_score=([0-9]+(?:\.[0-9]+)?(?:e[+-][0-9]+)?)/.exec(line);
|
|
96
|
+
if (score && pendingTime !== null) {
|
|
97
|
+
frames.push({ timeSeconds: pendingTime, score: Number.parseFloat(score[1]) });
|
|
98
|
+
pendingTime = null;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
if (!frames.length)
|
|
102
|
+
throw new Error("The scene scan returned no frames; check the video source");
|
|
103
|
+
return frames;
|
|
104
|
+
}
|