videobook 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (565) hide show
  1. package/CHANGELOG.md +54 -0
  2. package/LICENSE +21 -0
  3. package/README.md +195 -2
  4. package/bin/videobook.js +7 -0
  5. package/dist/acestep/client.js +203 -0
  6. package/dist/acestep/config.js +62 -0
  7. package/dist/acestep/errors.js +8 -0
  8. package/dist/acestep/process.js +175 -0
  9. package/dist/alive-captions/analyze.js +117 -0
  10. package/dist/alive-captions/audio.js +179 -0
  11. package/dist/alive-captions/frames.js +59 -0
  12. package/dist/alive-captions/model.js +37 -0
  13. package/dist/alive-captions/schema.js +49 -0
  14. package/dist/alive-captions/track.js +9 -0
  15. package/dist/api-key.js +33 -0
  16. package/dist/app-state.js +56 -0
  17. package/dist/app.js +257 -0
  18. package/dist/archive-b2.js +332 -0
  19. package/dist/archive-store.js +169 -0
  20. package/dist/artifact-label.js +36 -0
  21. package/dist/artifact-refs.js +49 -0
  22. package/dist/asset-tag-extraction.js +169 -0
  23. package/dist/asset-tags.js +228 -0
  24. package/dist/asset-usage.js +23 -0
  25. package/dist/audio-cue.js +71 -0
  26. package/dist/audio-encoding.js +32 -0
  27. package/dist/audio-sync-request.js +1 -0
  28. package/dist/audio-sync.js +223 -0
  29. package/dist/audio-trim-range.js +31 -0
  30. package/dist/book-context.js +31 -0
  31. package/dist/book-manager.js +374 -0
  32. package/dist/catalog-gc.js +26 -0
  33. package/dist/chain-executor.js +815 -0
  34. package/dist/chain-generation.js +115 -0
  35. package/dist/chain-plan.js +746 -0
  36. package/dist/chain-step-review.js +60 -0
  37. package/dist/character-name.js +25 -0
  38. package/dist/chat-generation-routing.js +73 -0
  39. package/dist/chat-generation-settings.js +78 -0
  40. package/dist/chat-models.js +44 -0
  41. package/dist/chat-prompt-rewrite.js +67 -0
  42. package/dist/chat-router.js +628 -0
  43. package/dist/chat-view-actions.js +49 -0
  44. package/dist/cli/args.js +242 -0
  45. package/dist/cli/config-command.js +103 -0
  46. package/dist/cli/doctor-command.js +289 -0
  47. package/dist/cli/entry.js +2 -0
  48. package/dist/cli/key-report.js +16 -0
  49. package/dist/cli/main.js +104 -0
  50. package/dist/cli/mcp-command.js +88 -0
  51. package/dist/cli/mcp-relay.js +51 -0
  52. package/dist/cli/open-browser.js +14 -0
  53. package/dist/cli/run-command.js +231 -0
  54. package/dist/cli/serve-command.js +62 -0
  55. package/dist/cli/setup-command.js +60 -0
  56. package/dist/cli/stdio-guard.js +8 -0
  57. package/dist/cold-open-model.js +7 -0
  58. package/dist/cold-open.js +79 -0
  59. package/dist/command-reference.js +95 -0
  60. package/dist/content-store.js +74 -0
  61. package/dist/cost.js +8 -0
  62. package/dist/crop-geometry.js +23 -0
  63. package/dist/dancing-workflow.js +447 -0
  64. package/dist/data-home.js +44 -0
  65. package/dist/diagnostics.js +109 -0
  66. package/dist/dialog-quality.js +34 -0
  67. package/dist/downloads-dest.js +56 -0
  68. package/dist/editing-moments.js +194 -0
  69. package/dist/elevenlabs-audio-extension.js +66 -0
  70. package/dist/engine.js +54 -0
  71. package/dist/env-file.js +91 -0
  72. package/dist/env-secrets.js +148 -0
  73. package/dist/ffmpeg-binary.js +12 -0
  74. package/dist/ffmpeg.js +425 -0
  75. package/dist/ffprobe-binary.js +77 -0
  76. package/dist/forget-asset.js +216 -0
  77. package/dist/frame-utils.js +27 -0
  78. package/dist/generation-credit-error.js +11 -0
  79. package/dist/generation-input-mentions.js +117 -0
  80. package/dist/generation-input-policy.js +68 -0
  81. package/dist/generation-reference-mentions.js +97 -0
  82. package/dist/goal-characters.js +49 -0
  83. package/dist/goal-plan-v3.js +273 -0
  84. package/dist/goal-plan.js +220 -0
  85. package/dist/goal-planner-prompt.js +28 -0
  86. package/dist/goal-prompt-budget.js +4 -0
  87. package/dist/goal-reference-images.js +25 -0
  88. package/dist/goal-state.js +261 -0
  89. package/dist/goal-stills.js +153 -0
  90. package/dist/goal-store.js +144 -0
  91. package/dist/goal-styles.js +144 -0
  92. package/dist/goal-v3-runtime.js +157 -0
  93. package/dist/guides/character_schema_guide.md +70 -0
  94. package/dist/guides/gpt_image_2_5_prompting_guide.md +21 -0
  95. package/dist/guides/grok_imagine_prompting_guide.md +72 -0
  96. package/dist/guides/image_reproduction_guide.md +84 -0
  97. package/dist/guides/kling_3_0_prompting_guide.md +136 -0
  98. package/dist/guides/nano_banana_pro_prompting_guide.md +28 -0
  99. package/dist/guides/seedance_2_prompting_guide.md +172 -0
  100. package/dist/http-access.js +49 -0
  101. package/dist/http-byte-range.js +31 -0
  102. package/dist/hydration.js +56 -0
  103. package/dist/image-compress.js +77 -0
  104. package/dist/image-edit-default.js +1 -0
  105. package/dist/image-edit-models.js +41 -0
  106. package/dist/image-edit-settings.js +11 -0
  107. package/dist/image-region-edit.js +32 -0
  108. package/dist/image-subject-map.js +523 -0
  109. package/dist/image.js +46 -0
  110. package/dist/index.js +12 -0
  111. package/dist/job-deadline.js +60 -0
  112. package/dist/jobs/_alibaba-upload.js +30 -0
  113. package/dist/jobs/_audio-chunks.js +31 -0
  114. package/dist/jobs/_download-into-asset.js +15 -0
  115. package/dist/jobs/_fal-shared.js +22 -0
  116. package/dist/jobs/_fal-upload.js +35 -0
  117. package/dist/jobs/_image-edit-adapter.js +213 -0
  118. package/dist/jobs/_kie-poll.js +119 -0
  119. package/dist/jobs/_kie-upload.js +32 -0
  120. package/dist/jobs/_lease.js +163 -0
  121. package/dist/jobs/_persist-output.js +19 -0
  122. package/dist/jobs/_storage-retry.js +21 -0
  123. package/dist/jobs/_transcribe.js +245 -0
  124. package/dist/jobs/_translate-transcript.js +270 -0
  125. package/dist/jobs/analyze-alive-captions.js +10 -0
  126. package/dist/jobs/analyze-editing-moments.js +104 -0
  127. package/dist/jobs/analyze-jokes.js +143 -0
  128. package/dist/jobs/analyze-keyword-placement.js +155 -0
  129. package/dist/jobs/analyze-music.js +14 -0
  130. package/dist/jobs/analyze-video.js +282 -0
  131. package/dist/jobs/apply-cuts.js +83 -0
  132. package/dist/jobs/apply-sfx.js +107 -0
  133. package/dist/jobs/archive-clip.js +458 -0
  134. package/dist/jobs/assemble-goal.js +98 -0
  135. package/dist/jobs/audio-boost.js +71 -0
  136. package/dist/jobs/audio-mix.js +161 -0
  137. package/dist/jobs/backfill-asset-tags.js +144 -0
  138. package/dist/jobs/burn-watermark.js +316 -0
  139. package/dist/jobs/change-video-speed.js +176 -0
  140. package/dist/jobs/clean-video-audio.js +207 -0
  141. package/dist/jobs/cold-open.js +166 -0
  142. package/dist/jobs/combine-goal.js +108 -0
  143. package/dist/jobs/complete-provider-task.js +185 -0
  144. package/dist/jobs/copy-library-asset.js +33 -0
  145. package/dist/jobs/cost-estimate-assessment.js +113 -0
  146. package/dist/jobs/crop-image.js +47 -0
  147. package/dist/jobs/crop-video.js +71 -0
  148. package/dist/jobs/describe-image.js +181 -0
  149. package/dist/jobs/detect-scenes.js +33 -0
  150. package/dist/jobs/doctor-fix.js +100 -0
  151. package/dist/jobs/download-video.js +427 -0
  152. package/dist/jobs/enqueue-search-index.js +24 -0
  153. package/dist/jobs/even-split.js +122 -0
  154. package/dist/jobs/export-sequence.js +365 -0
  155. package/dist/jobs/extend-audio.js +229 -0
  156. package/dist/jobs/extract-audio.js +72 -0
  157. package/dist/jobs/extract-frame.js +58 -0
  158. package/dist/jobs/extract-notebook-audio.js +82 -0
  159. package/dist/jobs/extract-notebook-frame.js +93 -0
  160. package/dist/jobs/extract-video-clip.js +8 -0
  161. package/dist/jobs/extract-video-poster.js +55 -0
  162. package/dist/jobs/extract-waveform.js +25 -0
  163. package/dist/jobs/finish-goal.js +255 -0
  164. package/dist/jobs/forget-asset.js +11 -0
  165. package/dist/jobs/generate-acestep-cover.js +219 -0
  166. package/dist/jobs/generate-media.js +70 -0
  167. package/dist/jobs/image-subjects.js +1059 -0
  168. package/dist/jobs/index.js +18 -0
  169. package/dist/jobs/inject-first-frame.js +175 -0
  170. package/dist/jobs/isolate-vocals.js +69 -0
  171. package/dist/jobs/kie-paygate.js +177 -0
  172. package/dist/jobs/label-music-sections.js +5 -0
  173. package/dist/jobs/osg-placement.js +135 -0
  174. package/dist/jobs/pad-video.js +72 -0
  175. package/dist/jobs/plan-goal.js +97 -0
  176. package/dist/jobs/post-record.js +21 -0
  177. package/dist/jobs/post-to-x.js +21 -0
  178. package/dist/jobs/prepare-search-reference.js +190 -0
  179. package/dist/jobs/prepare-temporal-search.js +16 -0
  180. package/dist/jobs/process-audio.js +67 -0
  181. package/dist/jobs/process-image.js +58 -0
  182. package/dist/jobs/process-upload.js +166 -0
  183. package/dist/jobs/process-video.js +438 -0
  184. package/dist/jobs/register-all.js +161 -0
  185. package/dist/jobs/relayout-notebook.js +78 -0
  186. package/dist/jobs/remove-sound.js +299 -0
  187. package/dist/jobs/render-project.js +380 -0
  188. package/dist/jobs/render-qr-outro.js +138 -0
  189. package/dist/jobs/render-timeline.js +283 -0
  190. package/dist/jobs/repair-search-indexes.js +42 -0
  191. package/dist/jobs/replace-video-audio.js +64 -0
  192. package/dist/jobs/restore-archive-clip.js +53 -0
  193. package/dist/jobs/reverse-video.js +53 -0
  194. package/dist/jobs/rewrite-script.js +173 -0
  195. package/dist/jobs/run-notebook.js +3335 -0
  196. package/dist/jobs/semantic-index.js +797 -0
  197. package/dist/jobs/smooth-scene-concat.js +257 -0
  198. package/dist/jobs/splice-video.js +280 -0
  199. package/dist/jobs/split-video.js +127 -0
  200. package/dist/jobs/suggest-cuts.js +149 -0
  201. package/dist/jobs/suggest-sfx.js +135 -0
  202. package/dist/jobs/sync-video-to-audio.js +329 -0
  203. package/dist/jobs/tag-asset.js +266 -0
  204. package/dist/jobs/task-failure.js +16 -0
  205. package/dist/jobs/timewarp-video.js +152 -0
  206. package/dist/jobs/transcribe-video-audio.js +117 -0
  207. package/dist/jobs/transcribe.js +35 -0
  208. package/dist/jobs/translate-transcript.js +55 -0
  209. package/dist/jobs/trim-audio.js +133 -0
  210. package/dist/jobs/trim-video.js +291 -0
  211. package/dist/jobs/xai-paygate.js +26 -0
  212. package/dist/keyword-placement-cache.js +74 -0
  213. package/dist/lazy-render.js +142 -0
  214. package/dist/library-browse.js +78 -0
  215. package/dist/lightning-defaults.js +206 -0
  216. package/dist/load-env.js +12 -0
  217. package/dist/local-search-models.js +206 -0
  218. package/dist/local-upload.js +318 -0
  219. package/dist/logger.js +69 -0
  220. package/dist/media/adapters/alibaba.js +20 -0
  221. package/dist/media/adapters/elevenlabs.js +58 -0
  222. package/dist/media/adapters/fal.js +45 -0
  223. package/dist/media/adapters/index.js +26 -0
  224. package/dist/media/adapters/kie.js +120 -0
  225. package/dist/media/adapters/pending.js +21 -0
  226. package/dist/media/adapters/types.js +5 -0
  227. package/dist/media/adapters/xai.js +18 -0
  228. package/dist/media/catalog.js +236 -0
  229. package/dist/media/complete.js +191 -0
  230. package/dist/media/cost.js +45 -0
  231. package/dist/media/descriptor.js +157 -0
  232. package/dist/media/endpoints.js +11 -0
  233. package/dist/media/frame-options.js +18 -0
  234. package/dist/media/insert-video-timing.js +17 -0
  235. package/dist/media/json-schema.js +75 -0
  236. package/dist/media/legacy-task-types.js +129 -0
  237. package/dist/media/listing.js +24 -0
  238. package/dist/media/model-id.js +14 -0
  239. package/dist/media/preview.js +86 -0
  240. package/dist/media/prompt-dialects.js +65 -0
  241. package/dist/media/ref-fields.js +106 -0
  242. package/dist/media/refs.js +131 -0
  243. package/dist/media/request.js +188 -0
  244. package/dist/media/types.js +6 -0
  245. package/dist/media/wire.js +161 -0
  246. package/dist/media-library.js +535 -0
  247. package/dist/media-output-path.js +33 -0
  248. package/dist/multyculty-workflow.js +757 -0
  249. package/dist/notebook-action-catalog.js +608 -0
  250. package/dist/notebook-cell-outputs.js +726 -0
  251. package/dist/notebook-credit-retry.js +45 -0
  252. package/dist/notebook-fixtures.js +8 -0
  253. package/dist/notebook-generation-contract.js +77 -0
  254. package/dist/notebook-generation-controls.js +145 -0
  255. package/dist/notebook-goal-layout.js +40 -0
  256. package/dist/notebook-graph.js +37 -0
  257. package/dist/notebook-image-edit.js +84 -0
  258. package/dist/notebook-initial-generation-prompt.js +222 -0
  259. package/dist/notebook-live-analysis.js +941 -0
  260. package/dist/notebook-media-ops.js +579 -0
  261. package/dist/notebook-mutations.js +20 -0
  262. package/dist/notebook-placement.js +41 -0
  263. package/dist/notebook-provider-failure.js +87 -0
  264. package/dist/notebook-published-post.js +26 -0
  265. package/dist/notebook-reference-resolver.js +227 -0
  266. package/dist/notebook-relayout.js +270 -0
  267. package/dist/notebook-run-plan.js +519 -0
  268. package/dist/notebook-shots.js +162 -0
  269. package/dist/notebook-workflow.js +856 -0
  270. package/dist/one-password.js +200 -0
  271. package/dist/openai-intelligence.js +99 -0
  272. package/dist/original-media-file.js +12 -0
  273. package/dist/osg-assets.js +36 -0
  274. package/dist/osg-placement-cache.js +64 -0
  275. package/dist/osg-settings.js +27 -0
  276. package/dist/paid-tools.js +22 -0
  277. package/dist/platform-guidance.js +26 -0
  278. package/dist/posts/dropbox-auth.js +91 -0
  279. package/dist/posts/dropbox-upload.js +132 -0
  280. package/dist/posts/icloud-drive.js +134 -0
  281. package/dist/posts/icloud-status.js +64 -0
  282. package/dist/posts/post-contract.js +2 -0
  283. package/dist/posts/post-copy-format.js +43 -0
  284. package/dist/posts/post-copy.js +193 -0
  285. package/dist/posts/post-folder.js +18 -0
  286. package/dist/posts/post-metrics.js +119 -0
  287. package/dist/posts/post-store.js +340 -0
  288. package/dist/posts/x-post-target.js +43 -0
  289. package/dist/preview-payload.js +90 -0
  290. package/dist/prompt-surfaces.js +22 -0
  291. package/dist/provider-prompt-transform.js +77 -0
  292. package/dist/provider-status-link.js +21 -0
  293. package/dist/provider-task-record.js +78 -0
  294. package/dist/provider-task-status.js +47 -0
  295. package/dist/provider-usage.js +294 -0
  296. package/dist/queue.js +810 -0
  297. package/dist/reference-video-duration.js +17 -0
  298. package/dist/remotion/BlurFillVideo.js +51 -0
  299. package/dist/remotion/KeywordGraphicsOverlay.js +68 -0
  300. package/dist/remotion/OsgOverlay.js +30 -0
  301. package/dist/remotion/QrOutroComposition.js +47 -0
  302. package/dist/remotion/RenderComposition.js +11 -0
  303. package/dist/remotion/browser.js +59 -0
  304. package/dist/remotion/constants.js +35 -0
  305. package/dist/remotion/index.js +66 -0
  306. package/dist/remotion/keyword-graphics.js +151 -0
  307. package/dist/remotion/keyword-placement.js +182 -0
  308. package/dist/remotion/osg-spec.js +198 -0
  309. package/dist/remotion/public/qr-outro-impact.wav +0 -0
  310. package/dist/remotion/qr-outro-audio.js +1 -0
  311. package/dist/remotion/qr-outro-spec.js +211 -0
  312. package/dist/remotion/shared-bundle.js +37 -0
  313. package/dist/remotion/subtitles/AliveCaptions.js +83 -0
  314. package/dist/remotion/subtitles/Highlight.js +58 -0
  315. package/dist/remotion/subtitles/Impact.js +67 -0
  316. package/dist/remotion/subtitles/SubtitleOverlay.js +9 -0
  317. package/dist/remotion/subtitles/alive-layout.js +321 -0
  318. package/dist/remotion/subtitles/alive-types.js +17 -0
  319. package/dist/remotion/subtitles/chunker.js +68 -0
  320. package/dist/remotion/subtitles/segments.js +42 -0
  321. package/dist/remotion/subtitles/styles.js +4 -0
  322. package/dist/remotion/subtitles/types.js +1 -0
  323. package/dist/remotion/util.js +18 -0
  324. package/dist/remotion-cancellation.js +15 -0
  325. package/dist/rename-helpers.js +204 -0
  326. package/dist/rhythm/analyze.js +128 -0
  327. package/dist/rhythm/contract.js +14 -0
  328. package/dist/rhythm/draft-model.js +19 -0
  329. package/dist/rhythm/energy.js +133 -0
  330. package/dist/rhythm/fft.js +44 -0
  331. package/dist/rhythm/model.js +71 -0
  332. package/dist/rhythm/music-analysis.js +124 -0
  333. package/dist/rhythm/music-draft.js +261 -0
  334. package/dist/rhythm/onset.js +101 -0
  335. package/dist/rhythm/run-worker.js +51 -0
  336. package/dist/rhythm/score.js +47 -0
  337. package/dist/rhythm/section-labels.js +168 -0
  338. package/dist/rhythm/sequence-rhythm.js +67 -0
  339. package/dist/rhythm/synthesize.js +103 -0
  340. package/dist/rhythm/timing.js +98 -0
  341. package/dist/rhythm/track.js +320 -0
  342. package/dist/rhythm/visual-fit.js +4 -0
  343. package/dist/rhythm/worker.js +25 -0
  344. package/dist/scene-detect.js +200 -0
  345. package/dist/scene-transition.js +12 -0
  346. package/dist/scheduled-posts.js +200 -0
  347. package/dist/search-index-coverage.js +13 -0
  348. package/dist/search-index-media.js +71 -0
  349. package/dist/search-reference-store.js +60 -0
  350. package/dist/search-scoring.js +17 -0
  351. package/dist/serve.js +125 -0
  352. package/dist/server-lock.js +143 -0
  353. package/dist/server.js +42 -0
  354. package/dist/srt.js +16 -0
  355. package/dist/tag-backfill-state.js +121 -0
  356. package/dist/tag-settings.js +30 -0
  357. package/dist/tag-vocabulary.js +237 -0
  358. package/dist/test-queue-cleanup.js +1 -0
  359. package/dist/time-parse.js +50 -0
  360. package/dist/timeline-helpers.js +554 -0
  361. package/dist/timewarp.js +87 -0
  362. package/dist/tools/_image-inputs.js +9 -0
  363. package/dist/tools/_media-refs.js +84 -0
  364. package/dist/tools/_shared.js +59 -0
  365. package/dist/tools/_suggest-shared.js +279 -0
  366. package/dist/tools/add-timeline-audio.js +73 -0
  367. package/dist/tools/alive-captions.js +157 -0
  368. package/dist/tools/analyze-jokes.js +56 -0
  369. package/dist/tools/analyze-video.js +66 -0
  370. package/dist/tools/animate-keywords.js +217 -0
  371. package/dist/tools/api-status.js +42 -0
  372. package/dist/tools/application-commands.js +34 -0
  373. package/dist/tools/apply-cuts.js +112 -0
  374. package/dist/tools/apply-sfx.js +152 -0
  375. package/dist/tools/apply-watermark.js +670 -0
  376. package/dist/tools/archive-clip.js +74 -0
  377. package/dist/tools/asset-history.js +61 -0
  378. package/dist/tools/asset-manifest.js +24 -0
  379. package/dist/tools/asset-tags.js +151 -0
  380. package/dist/tools/assign-slot.js +97 -0
  381. package/dist/tools/audio-boost.js +107 -0
  382. package/dist/tools/audio-mix.js +121 -0
  383. package/dist/tools/book-usage.js +16 -0
  384. package/dist/tools/chain.js +136 -0
  385. package/dist/tools/change-video-speed.js +105 -0
  386. package/dist/tools/chat-history.js +5 -0
  387. package/dist/tools/chat.js +848 -0
  388. package/dist/tools/clean-video-audio.js +97 -0
  389. package/dist/tools/clear-failed-tasks.js +66 -0
  390. package/dist/tools/cold-open.js +114 -0
  391. package/dist/tools/command-plan.js +1680 -0
  392. package/dist/tools/commands.js +1666 -0
  393. package/dist/tools/concat-images.js +236 -0
  394. package/dist/tools/connect-service.js +55 -0
  395. package/dist/tools/copy-file-to-clipboard.js +58 -0
  396. package/dist/tools/create-character.js +114 -0
  397. package/dist/tools/create-qart.js +253 -0
  398. package/dist/tools/create-qr-outro.js +149 -0
  399. package/dist/tools/create-qrbtf.js +215 -0
  400. package/dist/tools/create-watermark.js +97 -0
  401. package/dist/tools/crop-asset.js +18 -0
  402. package/dist/tools/crop-image.js +120 -0
  403. package/dist/tools/crop-video.js +97 -0
  404. package/dist/tools/delete-asset.js +104 -0
  405. package/dist/tools/describe-image.js +69 -0
  406. package/dist/tools/detect-scenes.js +44 -0
  407. package/dist/tools/doctor.js +126 -0
  408. package/dist/tools/download-video.js +114 -0
  409. package/dist/tools/duplicate-asset.js +184 -0
  410. package/dist/tools/edit-timeline.js +59 -0
  411. package/dist/tools/editing-moments.js +105 -0
  412. package/dist/tools/estimate-cost.js +171 -0
  413. package/dist/tools/even-split.js +295 -0
  414. package/dist/tools/export-asset-file.js +22 -0
  415. package/dist/tools/export-asset.js +140 -0
  416. package/dist/tools/export-sequence.js +250 -0
  417. package/dist/tools/export-to-downloads.js +54 -0
  418. package/dist/tools/export-transcript-srt.js +79 -0
  419. package/dist/tools/extend-audio.js +100 -0
  420. package/dist/tools/extract-audio.js +174 -0
  421. package/dist/tools/extract-frame.js +108 -0
  422. package/dist/tools/extract-notebook-audio.js +173 -0
  423. package/dist/tools/extract-notebook-frame.js +160 -0
  424. package/dist/tools/extract-video-clip.js +101 -0
  425. package/dist/tools/generate-media.js +157 -0
  426. package/dist/tools/generation-payload.js +48 -0
  427. package/dist/tools/get-timeline.js +12 -0
  428. package/dist/tools/get-watermark-config.js +38 -0
  429. package/dist/tools/goal.js +644 -0
  430. package/dist/tools/image-subjects.js +711 -0
  431. package/dist/tools/index.js +323 -0
  432. package/dist/tools/inject-first-frame.js +154 -0
  433. package/dist/tools/invert-image.js +70 -0
  434. package/dist/tools/isolate-vocals.js +66 -0
  435. package/dist/tools/list-assets.js +98 -0
  436. package/dist/tools/list-cell-generations.js +225 -0
  437. package/dist/tools/list-elevenlabs-voices.js +163 -0
  438. package/dist/tools/log-wrapper.js +29 -0
  439. package/dist/tools/make-cover-song.js +95 -0
  440. package/dist/tools/mark-asset-seen.js +23 -0
  441. package/dist/tools/media-library.js +245 -0
  442. package/dist/tools/move-timeline-audio.js +35 -0
  443. package/dist/tools/music-analysis.js +60 -0
  444. package/dist/tools/music-draft.js +29 -0
  445. package/dist/tools/pad-video.js +87 -0
  446. package/dist/tools/post-record.js +83 -0
  447. package/dist/tools/post.js +168 -0
  448. package/dist/tools/prompt-history.js +164 -0
  449. package/dist/tools/provider-key-status.js +248 -0
  450. package/dist/tools/qrbtf-render.js +849 -0
  451. package/dist/tools/read-asset-metadata.js +26 -0
  452. package/dist/tools/read-file-at-commit.js +29 -0
  453. package/dist/tools/read-transcript.js +41 -0
  454. package/dist/tools/reconcile-project-work.js +1452 -0
  455. package/dist/tools/recover-asset.js +28 -0
  456. package/dist/tools/remove-sound.js +72 -0
  457. package/dist/tools/remove-timeline-audio.js +34 -0
  458. package/dist/tools/remove-timeline-slot.js +40 -0
  459. package/dist/tools/rename-all.js +434 -0
  460. package/dist/tools/rename-asset.js +34 -0
  461. package/dist/tools/render-settings.js +47 -0
  462. package/dist/tools/render-timeline.js +120 -0
  463. package/dist/tools/render.js +384 -0
  464. package/dist/tools/reorder-character-images.js +45 -0
  465. package/dist/tools/replace-video-audio.js +133 -0
  466. package/dist/tools/restore-archive-clips.js +209 -0
  467. package/dist/tools/restore-asset-version.js +37 -0
  468. package/dist/tools/retry-generation.js +54 -0
  469. package/dist/tools/reverse-video.js +74 -0
  470. package/dist/tools/rewrite-prompt.js +388 -0
  471. package/dist/tools/rewrite-script.js +67 -0
  472. package/dist/tools/schema-from-apicity.js +66 -0
  473. package/dist/tools/search-archive.js +150 -0
  474. package/dist/tools/semantic-search.js +1214 -0
  475. package/dist/tools/sequence-rhythm.js +43 -0
  476. package/dist/tools/set-character-metadata.js +84 -0
  477. package/dist/tools/set-env-secret.js +40 -0
  478. package/dist/tools/set-osg.js +143 -0
  479. package/dist/tools/show-folder.js +37 -0
  480. package/dist/tools/smooth-scene-concat.js +122 -0
  481. package/dist/tools/splice-video.js +149 -0
  482. package/dist/tools/split-video.js +202 -0
  483. package/dist/tools/suggest-cuts.js +45 -0
  484. package/dist/tools/suggest-sfx.js +43 -0
  485. package/dist/tools/sync-video-to-audio.js +115 -0
  486. package/dist/tools/tag-backfill.js +177 -0
  487. package/dist/tools/timewarp-video.js +85 -0
  488. package/dist/tools/tool-def.js +1 -0
  489. package/dist/tools/tool-json-schema.js +116 -0
  490. package/dist/tools/transcribe.js +90 -0
  491. package/dist/tools/translate-subtitles.js +59 -0
  492. package/dist/tools/trim-audio.js +139 -0
  493. package/dist/tools/trim-video.js +168 -0
  494. package/dist/tools/update-script.js +25 -0
  495. package/dist/tools/update-timeline-audio.js +40 -0
  496. package/dist/tools/upload-file.js +41 -0
  497. package/dist/tools/verify-x-mentions.js +141 -0
  498. package/dist/tools/video-editing-command-contract.js +291 -0
  499. package/dist/tools/video-source-reference.js +30 -0
  500. package/dist/tools/video-time-input.js +18 -0
  501. package/dist/tools/videobook-domain.js +2021 -0
  502. package/dist/tools/wait-for-asset-ready.js +143 -0
  503. package/dist/tools/workspace-actions.js +54 -0
  504. package/dist/transcript-projection.js +202 -0
  505. package/dist/transcript-srt.js +33 -0
  506. package/dist/upload.js +76 -0
  507. package/dist/usage-providers.js +25 -0
  508. package/dist/video-analysis-projection.js +46 -0
  509. package/dist/video-poster.js +33 -0
  510. package/dist/video-proxy.js +99 -0
  511. package/dist/vision-regions.js +111 -0
  512. package/dist/voice-actions.js +90 -0
  513. package/dist/voice-audio.js +55 -0
  514. package/dist/voice-config.js +122 -0
  515. package/dist/voice-continuation.js +54 -0
  516. package/dist/voice-contract.js +1 -0
  517. package/dist/voice-http.js +145 -0
  518. package/dist/voice-reply-progress.js +55 -0
  519. package/dist/voice-session.js +992 -0
  520. package/dist/watermark-badge.js +259 -0
  521. package/dist/watermark-config.js +111 -0
  522. package/dist/watermark-handle-position.js +55 -0
  523. package/dist/watermark-platform.js +15 -0
  524. package/dist/watermark-timing.js +16 -0
  525. package/dist/workspace-action-catalog.js +315 -0
  526. package/dist/workspace-action-contract.js +21 -0
  527. package/dist/workspace-action-groups.js +98 -0
  528. package/dist/workspace-action-surfaces.js +7 -0
  529. package/dist/x/account.js +88 -0
  530. package/dist/x/authenticated-handle.js +30 -0
  531. package/dist/x/oauth-flow.js +191 -0
  532. package/dist/x/post-contract.js +1 -0
  533. package/dist/x/post-metadata.js +44 -0
  534. package/dist/x/post-to-x.js +456 -0
  535. package/dist/x/published-post-cell.js +90 -0
  536. package/dist/x/x-tokens.js +69 -0
  537. package/dist/ytdlp-binary.js +65 -0
  538. package/dist/zod-slots.js +334 -0
  539. package/package.json +84 -4
  540. package/ui/assets/alive-captions-BWb5nLKj.css +1 -0
  541. package/ui/assets/alive-captions-DPEr1nKM.js +1 -0
  542. package/ui/assets/confirm-bodies-D8ouHXKy.js +1 -0
  543. package/ui/assets/confirm-bodies-DrMhaXC6.css +1 -0
  544. package/ui/assets/fullscreen-B7nw4YbE.js +1 -0
  545. package/ui/assets/goal-ui-DK-xTCK0.js +2 -0
  546. package/ui/assets/goal-ui-DXdUM2KF.css +1 -0
  547. package/ui/assets/index-DzPAnCUj.css +1 -0
  548. package/ui/assets/index-kS1905s5.js +37 -0
  549. package/ui/assets/inspector-MlzQbTVo.js +1 -0
  550. package/ui/assets/jobs-ui-CoinOWx0.js +1 -0
  551. package/ui/assets/music-video-Cbe80kmQ.js +1 -0
  552. package/ui/assets/music-video-DPaiKKEq.css +1 -0
  553. package/ui/assets/ui-Oirr9VOf.css +1 -0
  554. package/ui/assets/ui-xo_Y1yQg.js +2 -0
  555. package/ui/assets/vendor-D2rfiZ5j.css +1 -0
  556. package/ui/assets/vendor-DfTCWgKm.js +19 -0
  557. package/ui/assets/vendor-flow-BXbeWxW7.js +7 -0
  558. package/ui/assets/vendor-flow-BnuhLJ6X.css +1 -0
  559. package/ui/assets/vendor-icons-CGqhCIXV.js +1 -0
  560. package/ui/assets/vendor-mcp-BTlEJcHV.js +67 -0
  561. package/ui/assets/vendor-react-Dj7D1fD4.js +9 -0
  562. package/ui/assets/vendor-remotion-CH1HXrlc.js +95 -0
  563. package/ui/assets/videobook-Qlp9HQFu.js +20 -0
  564. package/ui/favicon.svg +8 -0
  565. package/ui/index.html +126 -0
@@ -0,0 +1,143 @@
1
+ import { z } from "zod";
2
+ import { registerJobHandler } from "../queue.js";
3
+ import { getApiKey } from "../api-key.js";
4
+ import { INTELLIGENCE_MODEL, buildIntelligenceResponses, firstResponseText, intelligenceRequest, } from "../openai-intelligence.js";
5
+ import { withLocalLease } from "./_lease.js";
6
+ const JOKE_TYPES = [
7
+ "pun",
8
+ "self_deprecating",
9
+ "observational",
10
+ "absurdist",
11
+ "callback",
12
+ "sarcasm",
13
+ "wordplay",
14
+ "misdirection",
15
+ "hyperbole",
16
+ "irony",
17
+ "anti_humor",
18
+ "other",
19
+ ];
20
+ const SYSTEM_PROMPT = `You are a comedy writer analyzing a transcript of a stand-up or monologue clip. Identify the distinct jokes in the bit.
21
+
22
+ For each joke:
23
+ - Classify its primary type, choosing exactly one value from the allowed set.
24
+ - Quote the joke verbatim from the transcript (the punchline plus the minimal setup needed to understand it).
25
+ - Give the start and end time in seconds, drawn from the word-timing table.
26
+ - Explain in one sentence why it works.
27
+
28
+ RULES:
29
+ - Do not invent jokes. Only use lines present in the transcript.
30
+ - start < end, and both must fall within the clip's word timings.
31
+ - Prefer a few clear, well-formed jokes over an exhaustive list.
32
+ - Output JSON only.`;
33
+ const JOKES_JSON_SCHEMA = {
34
+ type: "object",
35
+ required: ["jokes", "summary"],
36
+ additionalProperties: false,
37
+ properties: {
38
+ jokes: {
39
+ type: "array",
40
+ items: {
41
+ type: "object",
42
+ required: ["type", "text", "start", "end", "why"],
43
+ additionalProperties: false,
44
+ properties: {
45
+ type: { type: "string", enum: JOKE_TYPES },
46
+ text: { type: "string" },
47
+ start: { type: "number" },
48
+ end: { type: "number" },
49
+ why: { type: "string" },
50
+ },
51
+ },
52
+ },
53
+ summary: { type: "string" },
54
+ },
55
+ };
56
+ const JokeSchema = z.object({
57
+ type: z.enum(JOKE_TYPES),
58
+ text: z.string(),
59
+ start: z.number(),
60
+ end: z.number(),
61
+ why: z.string(),
62
+ });
63
+ const ModelOutputSchema = z.object({
64
+ jokes: z.array(JokeSchema),
65
+ summary: z.string(),
66
+ });
67
+ function spokenWords(words) {
68
+ return words.filter((w) => w.type === "word");
69
+ }
70
+ function buildJokesContext(eleven) {
71
+ const table = spokenWords(eleven.words)
72
+ .map((w) => `${w.start.toFixed(2)}-${w.end.toFixed(2)} ${w.text.trim()}`)
73
+ .join("\n");
74
+ return [
75
+ "TRANSCRIPT:",
76
+ eleven.text,
77
+ "",
78
+ "WORD TIMINGS (start-end seconds, one word per line):",
79
+ table,
80
+ ].join("\n");
81
+ }
82
+ export function registerAnalyzeJokesJob(engine) {
83
+ registerJobHandler("analyze_jokes", async (payload, ctx) => {
84
+ const { assetId } = payload;
85
+ return withLocalLease(engine, assetId, ctx, "analyze", async () => {
86
+ const apiKey = getApiKey("OPENAI_API_KEY");
87
+ if (!apiKey)
88
+ throw new Error("OPENAI_API_KEY environment variable is not set");
89
+ const read = await engine.files.read(assetId, "elevenlabs.json");
90
+ if (!read.ok) {
91
+ throw new Error(`No ElevenLabs transcript found for ${assetId} — run /transcribe @${assetId} first.`);
92
+ }
93
+ let eleven;
94
+ try {
95
+ eleven = JSON.parse(read.value.toString());
96
+ }
97
+ catch {
98
+ throw new Error(`elevenlabs.json for ${assetId} is not valid JSON`);
99
+ }
100
+ if (!Array.isArray(eleven.words) || spokenWords(eleven.words).length === 0) {
101
+ throw new Error(`No ElevenLabs transcript found for ${assetId} — run /transcribe @${assetId} first.`);
102
+ }
103
+ const responses = buildIntelligenceResponses(apiKey, { timeout: 240_000 });
104
+ const result = await responses(intelligenceRequest({
105
+ instructions: SYSTEM_PROMPT,
106
+ input: [{ role: "user", content: buildJokesContext(eleven) }],
107
+ text: {
108
+ format: {
109
+ type: "json_schema",
110
+ name: "analyze_jokes_result",
111
+ schema: JOKES_JSON_SCHEMA,
112
+ strict: true,
113
+ },
114
+ },
115
+ max_output_tokens: 4096,
116
+ }), ctx.signal);
117
+ const outputText = firstResponseText(result);
118
+ let parsed;
119
+ try {
120
+ parsed = ModelOutputSchema.parse(JSON.parse(outputText));
121
+ }
122
+ catch (err) {
123
+ const msg = err instanceof Error ? err.message : String(err);
124
+ throw new Error(`Model output failed schema validation: ${msg}`);
125
+ }
126
+ const final = {
127
+ jokes: parsed.jokes,
128
+ summary: parsed.summary,
129
+ based_on: {
130
+ transcript_source: "elevenlabs",
131
+ word_count: spokenWords(eleven.words).length,
132
+ },
133
+ model: INTELLIGENCE_MODEL,
134
+ asset_id: assetId,
135
+ };
136
+ const writeResult = await engine.files.write(assetId, ".analyze-jokes.json", JSON.stringify(final, null, 2));
137
+ if (!writeResult.ok) {
138
+ throw new Error(`Jokes analyzed but failed to persist: ${writeResult.error.message}`);
139
+ }
140
+ return { success: true, jokes: parsed.jokes.length };
141
+ });
142
+ });
143
+ }
@@ -0,0 +1,155 @@
1
+ import { mkdtemp, readFile, rm } from "node:fs/promises";
2
+ import { tmpdir } from "node:os";
3
+ import { join } from "node:path";
4
+ import sharp from "sharp";
5
+ import { getApiKey } from "../api-key.js";
6
+ import { runFfmpeg, probeVideo } from "../ffmpeg.js";
7
+ import { detectOccupiedRegions } from "../vision-regions.js";
8
+ import { writeKeywordPlacementAnalysis } from "../keyword-placement-cache.js";
9
+ import { registerJobHandler } from "../queue.js";
10
+ import { KEYWORD_FRAME_CACHE_FILENAME, KEYWORD_PLACEMENT_VERSION, keywordCueId, } from "../remotion/keyword-placement.js";
11
+ import { withLockedLocalLease } from "./_lease.js";
12
+ const LOCK_DURATION_MS = 300_000;
13
+ const CELL_WIDTH = 640;
14
+ const IMAGE_HEIGHT = 360;
15
+ const LABEL_HEIGHT = 48;
16
+ const CELL_HEIGHT = IMAGE_HEIGHT + LABEL_HEIGHT;
17
+ const CONTACT_SHEET_COLUMNS = 2;
18
+ const SYSTEM_PROMPT = `You are a precise video-layout analyst. Each attached image is a frame extracted at the exact midpoint of a spoken keyword.
19
+
20
+ For every frame, identify all occupied regions that cinematic keyword typography must not cover:
21
+ - every visible human face, including partial faces and faces in posters or screens
22
+ - every existing visible text block, subtitle, caption, title, logo, watermark, or graphic
23
+
24
+ Return one entry for every cue_id. Coordinates use a normalized image scale from 0 to 1000: x and y are the top-left corner, followed by width and height. Use tight boxes around the visible occupied content. Do not return boxes for bodies, clothing, scenery, microphones, or ordinary objects. Return an empty occupied_regions array only when none of the listed content is visible. Never omit, rename, or duplicate a cue_id.`;
25
+ function escapeXml(value) {
26
+ return value
27
+ .replaceAll("&", "&amp;")
28
+ .replaceAll("<", "&lt;")
29
+ .replaceAll(">", "&gt;")
30
+ .replaceAll('"', "&quot;")
31
+ .replaceAll("'", "&apos;");
32
+ }
33
+ async function createContactSheet(frames) {
34
+ const cells = await Promise.all(frames.map(async (frame) => {
35
+ const image = await sharp(frame.buffer)
36
+ .resize({
37
+ width: CELL_WIDTH,
38
+ height: IMAGE_HEIGHT,
39
+ fit: "contain",
40
+ background: "#080808",
41
+ })
42
+ .jpeg({ quality: 88 })
43
+ .toBuffer();
44
+ const label = Buffer.from(`<svg width="${CELL_WIDTH}" height="${LABEL_HEIGHT}" xmlns="http://www.w3.org/2000/svg"><rect width="100%" height="100%" fill="#080808"/><text x="18" y="32" fill="#ffffff" font-family="serif" font-size="22">${escapeXml(frame.text)} · ${frame.frameTime.toFixed(3)}s</text></svg>`);
45
+ return sharp({
46
+ create: {
47
+ width: CELL_WIDTH,
48
+ height: CELL_HEIGHT,
49
+ channels: 3,
50
+ background: "#080808",
51
+ },
52
+ }).composite([
53
+ { input: label, left: 0, top: 0 },
54
+ { input: image, left: 0, top: LABEL_HEIGHT },
55
+ ]).jpeg({ quality: 88 }).toBuffer();
56
+ }));
57
+ const rows = Math.ceil(cells.length / CONTACT_SHEET_COLUMNS);
58
+ return sharp({
59
+ create: {
60
+ width: CELL_WIDTH * CONTACT_SHEET_COLUMNS,
61
+ height: CELL_HEIGHT * rows,
62
+ channels: 3,
63
+ background: "#080808",
64
+ },
65
+ }).composite(cells.map((input, index) => ({
66
+ input,
67
+ left: index % CONTACT_SHEET_COLUMNS * CELL_WIDTH,
68
+ top: Math.floor(index / CONTACT_SHEET_COLUMNS) * CELL_HEIGHT,
69
+ }))).jpeg({ quality: 88 }).toBuffer();
70
+ }
71
+ export async function extractKeywordFrames(videoPath, graphics, duration, signal) {
72
+ const tempDir = await mkdtemp(join(tmpdir(), "videocity-keyword-frames-"));
73
+ try {
74
+ const frames = [];
75
+ for (let index = 0; index < graphics.length; index++) {
76
+ signal?.throwIfAborted();
77
+ const graphic = graphics[index];
78
+ const requestedTime = graphic.frameTime ?? graphic.start;
79
+ const frameTime = Math.max(0, Math.min(Math.max(0, duration - 0.05), requestedTime));
80
+ const framePath = join(tempDir, `${index.toString().padStart(2, "0")}.jpg`);
81
+ await runFfmpeg([
82
+ "-y",
83
+ "-ss", frameTime.toFixed(6),
84
+ "-i", videoPath,
85
+ "-frames:v", "1",
86
+ "-vf", "scale=1280:1280:force_original_aspect_ratio=decrease",
87
+ "-q:v", "2",
88
+ framePath,
89
+ ], { signal });
90
+ frames.push({
91
+ cueId: keywordCueId(graphic),
92
+ text: graphic.text,
93
+ frameTime,
94
+ buffer: await readFile(framePath),
95
+ });
96
+ }
97
+ return { frames, contactSheet: await createContactSheet(frames) };
98
+ }
99
+ finally {
100
+ await rm(tempDir, { recursive: true, force: true });
101
+ }
102
+ }
103
+ async function analyzeFrames(frames, apiKey, signal) {
104
+ return detectOccupiedRegions(frames.map((frame) => ({
105
+ id: frame.cueId,
106
+ label: `keyword=${frame.text}; timestamp=${frame.frameTime.toFixed(3)}s`,
107
+ buffer: frame.buffer,
108
+ })), {
109
+ apiKey,
110
+ signal,
111
+ instructions: SYSTEM_PROMPT,
112
+ schemaName: "keyword_placement_analysis",
113
+ errorLabel: "Keyword placement",
114
+ timeout: 240_000,
115
+ maxOutputTokens: 8192,
116
+ });
117
+ }
118
+ async function writeContactSheetIfChanged(engine, assetId, contactSheet) {
119
+ const existing = await engine.files.read(assetId, KEYWORD_FRAME_CACHE_FILENAME);
120
+ if (existing.ok && existing.value.equals(contactSheet))
121
+ return;
122
+ const write = await engine.files.write(assetId, KEYWORD_FRAME_CACHE_FILENAME, contactSheet);
123
+ if (!write.ok)
124
+ throw new Error(`Failed to persist keyword frame cache: ${write.error.message}`);
125
+ }
126
+ export function registerAnalyzeKeywordPlacementJob(engine) {
127
+ registerJobHandler("analyze_keyword_placement", async (payload, ctx) => {
128
+ const { assetId, assetDir, videoPath, fingerprint, graphics, } = payload;
129
+ return withLockedLocalLease(engine, assetId, assetDir, ctx, "analyze", { durationMs: LOCK_DURATION_MS, data: { state: "analyzing-keyword-placement" } }, async () => {
130
+ const apiKey = getApiKey("OPENAI_API_KEY");
131
+ if (!apiKey)
132
+ throw new Error("OPENAI_API_KEY is not set");
133
+ const probe = await probeVideo(videoPath, { signal: ctx.signal });
134
+ const extracted = await extractKeywordFrames(videoPath, graphics, probe.duration, ctx.signal);
135
+ const regions = await analyzeFrames(extracted.frames, apiKey, ctx.signal);
136
+ const analysis = {
137
+ version: KEYWORD_PLACEMENT_VERSION,
138
+ fingerprint,
139
+ sourceWidth: probe.width,
140
+ sourceHeight: probe.height,
141
+ frameCacheFilename: KEYWORD_FRAME_CACHE_FILENAME,
142
+ analyzedAt: new Date().toISOString(),
143
+ cues: extracted.frames.map((frame) => ({
144
+ cueId: frame.cueId,
145
+ text: frame.text,
146
+ frameTime: frame.frameTime,
147
+ regions: regions.get(frame.cueId) ?? [],
148
+ })),
149
+ };
150
+ await writeContactSheetIfChanged(engine, assetId, extracted.contactSheet);
151
+ await writeKeywordPlacementAnalysis(engine, assetId, analysis);
152
+ return { analysis };
153
+ }, { durationMs: LOCK_DURATION_MS, waitForActiveMs: 30_000 });
154
+ });
155
+ }
@@ -0,0 +1,14 @@
1
+ import { registerJobHandler, queue } from "../queue.js";
2
+ import { analyzeMusicSource } from "../rhythm/music-analysis.js";
3
+ export function registerAnalyzeMusicJob(engine) {
4
+ registerJobHandler("analyze_music", async (payload, context) => {
5
+ const result = await analyzeMusicSource(engine, String(payload.musicArtifactId), String(payload.analysisId), String(payload.streamId), context.signal);
6
+ if (payload.sections === true) {
7
+ await queue.enqueue(`music-sections-${payload.musicArtifactId}-${payload.analysisId}`, {
8
+ type: "label_music_sections", bookId: payload.bookId, artifactId: payload.musicArtifactId,
9
+ analysisId: payload.analysisId, streamId: payload.streamId,
10
+ });
11
+ }
12
+ return result;
13
+ });
14
+ }
@@ -0,0 +1,282 @@
1
+ import { join } from "node:path";
2
+ import { logger } from "../logger.js";
3
+ import { registerJobHandler, queue } from "../queue.js";
4
+ import { getApiKey } from "../api-key.js";
5
+ import { sampleEvenly } from "../frame-utils.js";
6
+ import { buildIntelligenceResponses, firstResponseText, intelligenceRequest, } from "../openai-intelligence.js";
7
+ import { DOWNSTREAM_LEASE_WAIT_MS, withLocalLease, } from "./_lease.js";
8
+ import { findAudioFile } from "videobook-engine";
9
+ import { readAndAssessDialog } from "../dialog-quality.js";
10
+ import { readAssetTranscript } from "../transcript-projection.js";
11
+ import { enqueueLazyTimelineRender } from "../lazy-render.js";
12
+ import { enqueueAssetTagging } from "./tag-asset.js";
13
+ import { frameVocabularyFor, MAX_AUTOMATIC_TAGS } from "../tag-vocabulary.js";
14
+ const MAX_FRAMES = 15;
15
+ const FALLBACK_ANALYSIS = {
16
+ summary: "", scenes: [], tags: [], mood: "",
17
+ subjects: [], actions: [], setting: "", textOnScreen: [],
18
+ burnedInSubtitles: null,
19
+ onScreenGraphic: null,
20
+ recommendedSubtitlePosition: "bottom",
21
+ recommendedOsgPosition: "top-right",
22
+ subjectPlacement: "center",
23
+ };
24
+ const SYSTEM_PROMPT = `You are a video analysis expert. Analyze the provided video frames, metadata, and transcription (if available) to produce a comprehensive JSON analysis.
25
+
26
+ Return a JSON object with these fields:
27
+ - "summary": A 2-3 sentence description of the video content
28
+ - "scenes": An array of scene descriptions, each with "timestamp" (approximate seconds), "description", and "tags" (array of strings)
29
+ - "tags": Top-level tags for the entire video (array of strings)
30
+ - "editorialTags": Up to ${MAX_AUTOMATIC_TAGS} {facet, label} records for grouping footage. Use only visible evidence: people (${frameVocabularyFor("people").join(", ")}), places (${frameVocabularyFor("places").join(", ")}), editing (${frameVocabularyFor("editing").join(", ")}). Return [] when unsupported. Never infer camera movement, sound, interview roles, names or exact geographic locations from these sparse frames. Never convert a negated claim or an incidental object into a tag.
31
+ - "mood": The overall mood/tone of the video
32
+ - "subjects": Array of main subjects/people/objects featured
33
+ - "subjectPlacement": Horizontal placement of the primary subject across the sampled frames: "left", "center", or "right"
34
+ - "actions": Array of key actions or events that occur
35
+ - "setting": Description of the environment/location
36
+ - "textOnScreen": Array of any visible text in the frames
37
+ - "burnedInSubtitles": Detect if subtitles or captions are already burned into the video frames. If detected, return an object with "detected": true, "position": one of "top", "middle", or "bottom" indicating where the text appears, and "confidence": "high", "medium", or "low". If no burned-in subtitles are found, return null. Look for persistent text overlays that appear across multiple frames in a consistent position — these are distinct from occasional on-screen titles or graphics.
38
+ - "onScreenGraphic": Detect if there is a watermark, logo, or channel graphic burned into the video. If detected, return an object with "detected": true, "position": one of "top-left", "top-right", "bottom-left", or "bottom-right" indicating the corner where it appears, "description": a brief description of what it is (e.g. "TikTok logo", "channel watermark"), and "confidence": "high", "medium", or "low". If none found, return null.
39
+ - "recommendedSubtitlePosition": Based on what is already on screen, recommend the best position ("top", "middle", or "bottom") for adding subtitle overlays that will not conflict with any detected burned-in subtitles or on-screen graphics. If burned-in subtitles are at the bottom, recommend "top". If nothing is detected, default to "bottom".
40
+ - "recommendedOsgPosition": Based on what is already on screen, recommend the best corner ("top-left", "top-right", "bottom-left", or "bottom-right") for placing an overlay graphic that will not conflict with any detected on-screen graphics. If nothing is detected, default to "top-right".`;
41
+ const ANALYZE_VIDEO_JSON_SCHEMA = {
42
+ type: "object",
43
+ required: [
44
+ "summary",
45
+ "scenes",
46
+ "tags",
47
+ "editorialTags",
48
+ "mood",
49
+ "subjects",
50
+ "subjectPlacement",
51
+ "actions",
52
+ "setting",
53
+ "textOnScreen",
54
+ "burnedInSubtitles",
55
+ "onScreenGraphic",
56
+ "recommendedSubtitlePosition",
57
+ "recommendedOsgPosition",
58
+ ],
59
+ additionalProperties: false,
60
+ properties: {
61
+ summary: { type: "string" },
62
+ scenes: {
63
+ type: "array",
64
+ items: {
65
+ type: "object",
66
+ required: ["timestamp", "description", "tags"],
67
+ additionalProperties: false,
68
+ properties: {
69
+ timestamp: { type: "number" },
70
+ description: { type: "string" },
71
+ tags: { type: "array", items: { type: "string" } },
72
+ },
73
+ },
74
+ },
75
+ tags: { type: "array", items: { type: "string" } },
76
+ editorialTags: {
77
+ type: "array", maxItems: MAX_AUTOMATIC_TAGS,
78
+ items: {
79
+ type: "object", additionalProperties: false, required: ["facet", "label"],
80
+ properties: { facet: { type: "string", enum: ["people", "places", "editing"] }, label: { type: "string" } },
81
+ },
82
+ },
83
+ mood: { type: "string" },
84
+ subjects: { type: "array", items: { type: "string" } },
85
+ subjectPlacement: { type: "string", enum: ["left", "center", "right"] },
86
+ actions: { type: "array", items: { type: "string" } },
87
+ setting: { type: "string" },
88
+ textOnScreen: { type: "array", items: { type: "string" } },
89
+ burnedInSubtitles: {
90
+ anyOf: [
91
+ {
92
+ type: "object",
93
+ required: ["detected", "position", "confidence"],
94
+ additionalProperties: false,
95
+ properties: {
96
+ detected: { type: "boolean" },
97
+ position: { type: "string", enum: ["top", "middle", "bottom"] },
98
+ confidence: { type: "string", enum: ["high", "medium", "low"] },
99
+ },
100
+ },
101
+ { type: "null" },
102
+ ],
103
+ },
104
+ onScreenGraphic: {
105
+ anyOf: [
106
+ {
107
+ type: "object",
108
+ required: ["detected", "position", "description", "confidence"],
109
+ additionalProperties: false,
110
+ properties: {
111
+ detected: { type: "boolean" },
112
+ position: {
113
+ type: "string",
114
+ enum: ["top-left", "top-right", "bottom-left", "bottom-right"],
115
+ },
116
+ description: { type: "string" },
117
+ confidence: { type: "string", enum: ["high", "medium", "low"] },
118
+ },
119
+ },
120
+ { type: "null" },
121
+ ],
122
+ },
123
+ recommendedSubtitlePosition: {
124
+ type: "string",
125
+ enum: ["top", "middle", "bottom"],
126
+ },
127
+ recommendedOsgPosition: {
128
+ type: "string",
129
+ enum: ["top-left", "top-right", "bottom-left", "bottom-right"],
130
+ },
131
+ },
132
+ };
133
+ function sampleFrames(frames) {
134
+ return sampleEvenly(frames, MAX_FRAMES);
135
+ }
136
+ export function registerAnalyzeVideoJob(engine) {
137
+ registerJobHandler("analyze_video", async (payload, ctx) => {
138
+ const { assetId, assetDir, bookId, hasTranscription, purpose, } = payload;
139
+ const result = await withLocalLease(engine, assetId, ctx, "analyze", async () => {
140
+ {
141
+ const metaResult = await engine.metadata.artifacts.read(assetId, "original");
142
+ const metadata = metaResult.ok ? metaResult.value : null;
143
+ const listResult = await engine.files.listSubdir(assetId, "original_frames");
144
+ const frameFiles = listResult.ok
145
+ ? listResult.value
146
+ .filter((f) => f.endsWith(".jpg"))
147
+ .sort((a, b) => parseFloat(a) - parseFloat(b))
148
+ : [];
149
+ const sampled = sampleFrames(frameFiles);
150
+ if (sampled.length === 0) {
151
+ throw new Error(`No frames to analyze for ${assetId}; process the video before analyzing it`);
152
+ }
153
+ const frameParts = [];
154
+ for (const file of sampled) {
155
+ const frameRead = await engine.files.read(assetId, `original_frames/${file}`);
156
+ if (!frameRead.ok)
157
+ throw new Error(frameRead.error.message);
158
+ frameParts.push({ type: "input_text", text: `Frame timestamp: ${parseFloat(file).toFixed(3)}s` }, {
159
+ type: "input_image",
160
+ image_url: `data:image/jpeg;base64,${Buffer.from(frameRead.value).toString("base64")}`,
161
+ detail: "high",
162
+ });
163
+ }
164
+ // Prefers ElevenLabs word cues, falls back to the plain ingest transcript —
165
+ // a manual /transcribe leaves only the former behind.
166
+ let transcriptionText = "";
167
+ if (hasTranscription) {
168
+ const transcript = await readAssetTranscript(engine, assetId);
169
+ if (transcript.kind === "timed" || transcript.kind === "plain") {
170
+ transcriptionText = transcript.text;
171
+ }
172
+ }
173
+ const contextParts = [];
174
+ if (metadata) {
175
+ const durationStr = typeof metadata.duration === "number" ? `${metadata.duration.toFixed(1)}s` : "unknown";
176
+ contextParts.push(`Video metadata: ${metadata.width}x${metadata.height}, duration ${durationStr}`);
177
+ }
178
+ if (transcriptionText) {
179
+ contextParts.push(`Transcription:\n${transcriptionText}`);
180
+ }
181
+ contextParts.push(`${sampled.length} frames from the video are attached below.`);
182
+ const userContent = [
183
+ { type: "input_text", text: contextParts.join("\n\n") },
184
+ ...frameParts,
185
+ ];
186
+ let analysis = FALLBACK_ANALYSIS;
187
+ const apiKey = getApiKey("OPENAI_API_KEY");
188
+ if (apiKey) {
189
+ try {
190
+ const responses = buildIntelligenceResponses(apiKey, { timeout: 180_000 });
191
+ const response = await responses(intelligenceRequest({
192
+ instructions: SYSTEM_PROMPT,
193
+ input: [{ role: "user", content: userContent }],
194
+ text: {
195
+ format: {
196
+ type: "json_schema",
197
+ name: "video_analysis",
198
+ schema: ANALYZE_VIDEO_JSON_SCHEMA,
199
+ strict: true,
200
+ },
201
+ },
202
+ max_output_tokens: 4096,
203
+ }), ctx.signal);
204
+ analysis = JSON.parse(firstResponseText(response));
205
+ }
206
+ catch (e) {
207
+ logger.error("video-analysis-failed", { assetId, error: e });
208
+ }
209
+ }
210
+ const jsonString = JSON.stringify(analysis, null, 2);
211
+ await engine.files.write(assetId, ".original.analysis.json", jsonString);
212
+ await queue.enqueue(`semantic-index-${assetId}-analysis-${Date.now()}`, {
213
+ type: "semantic_index",
214
+ assetId,
215
+ bookId,
216
+ });
217
+ const usedFallback = analysis === FALLBACK_ANALYSIS;
218
+ // A failed analysis wrote the empty fallback shape; tagging it would fence
219
+ // the asset against the next successful run, so only a real result is
220
+ // offered to the tagger.
221
+ if (!usedFallback) {
222
+ await enqueueAssetTagging(engine, assetId);
223
+ }
224
+ // Auto-transcribe trigger (replaces the whitelist-tool gate). Run only if
225
+ // the analysis says there's real dialog, ElevenLabs/Kie is configured, no
226
+ // prior failure or in-progress task, and we haven't already transcribed.
227
+ if (purpose !== "tagging") {
228
+ await maybeEnqueueAutoTranscribe(engine, assetId, assetDir, bookId);
229
+ }
230
+ return { success: true, fallback: usedFallback, analysis };
231
+ }
232
+ }, { waitForActiveMs: DOWNSTREAM_LEASE_WAIT_MS });
233
+ // Rendering follows analysis for assets on a timeline. A run that exists
234
+ // only to feed the tagger has no timeline behind it, so it renders nothing.
235
+ if (purpose !== "tagging") {
236
+ try {
237
+ await enqueueLazyTimelineRender(engine, bookId, assetId);
238
+ }
239
+ catch (err) {
240
+ void err;
241
+ }
242
+ }
243
+ return result;
244
+ });
245
+ }
246
+ async function maybeEnqueueAutoTranscribe(engine, assetId, assetDir, bookId) {
247
+ if (!getApiKey("ELEVENLABS_API_KEY"))
248
+ return;
249
+ const manifestResult = await engine.files.manifest(assetId);
250
+ if (!manifestResult.ok)
251
+ return;
252
+ const manifest = manifestResult.value;
253
+ const fileNames = new Set(manifest.files.map((f) => f.name));
254
+ if (fileNames.has("elevenlabs.json"))
255
+ return;
256
+ const errorRow = await engine.jobs.failures.read(assetId);
257
+ if (errorRow.ok && errorRow.value !== null)
258
+ return;
259
+ const dialog = await readAndAssessDialog(engine, assetId);
260
+ if (!dialog.hasDialog)
261
+ return;
262
+ const artifact = engine.artifacts.get(assetId);
263
+ if (!artifact.ok)
264
+ return;
265
+ const audioFile = findAudioFile(manifest.files, artifact.value.kind);
266
+ if (!audioFile)
267
+ return;
268
+ // Stable dedupe key, shared with sync's auto-transcribe: when this trigger
269
+ // races an overlapping sync poll, the queue collapses both onto one job
270
+ // instead of paying for two ElevenLabs transcriptions.
271
+ const jobId = `auto-transcribe-${assetId}`;
272
+ await queue.enqueue(jobId, {
273
+ type: "transcribe",
274
+ audioPath: join(manifest.path, audioFile.name),
275
+ audioFilename: audioFile.name,
276
+ tagAudioEvents: true,
277
+ diarize: true,
278
+ assetId,
279
+ assetDir,
280
+ bookId,
281
+ });
282
+ }