zilmate 1.9.9 → 1.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ad-creative/SKILL.md +362 -0
- package/.agents/skills/ad-creative/evals/evals.json +90 -0
- package/.agents/skills/ad-creative/references/generative-tools.md +637 -0
- package/.agents/skills/ad-creative/references/platform-specs.md +213 -0
- package/.agents/skills/ai-seo/SKILL.md +489 -0
- package/.agents/skills/ai-seo/evals/evals.json +90 -0
- package/.agents/skills/ai-seo/references/content-patterns.md +285 -0
- package/.agents/skills/ai-seo/references/content-types.md +71 -0
- package/.agents/skills/ai-seo/references/okf.md +104 -0
- package/.agents/skills/ai-seo/references/platform-ranking-factors.md +154 -0
- package/.agents/skills/clerk-backend-api/SKILL.md +426 -0
- package/.agents/skills/clerk-backend-api/evals/evals.json +87 -0
- package/.agents/skills/clerk-backend-api/scripts/api-specs-context.sh +30 -0
- package/.agents/skills/clerk-backend-api/scripts/execute-request.sh +88 -0
- package/.agents/skills/clerk-backend-api/scripts/extract-endpoint-detail.sh +165 -0
- package/.agents/skills/clerk-backend-api/scripts/extract-tag-endpoints.sh +208 -0
- package/.agents/skills/clerk-backend-api/scripts/extract-tags.js +14 -0
- package/.agents/skills/copy-editing/SKILL.md +457 -0
- package/.agents/skills/copy-editing/evals/evals.json +89 -0
- package/.agents/skills/copy-editing/references/checklist.md +66 -0
- package/.agents/skills/copy-editing/references/content-refresh.md +38 -0
- package/.agents/skills/copy-editing/references/plain-english-alternatives.md +394 -0
- package/.agents/skills/hyperframes/SKILL.md +163 -0
- package/.agents/skills/hyperframes-cli/SKILL.md +109 -0
- package/.agents/skills/hyperframes-cli/references/doctor-browser.md +45 -0
- package/.agents/skills/hyperframes-cli/references/init-and-scaffold.md +51 -0
- package/.agents/skills/hyperframes-cli/references/lambda.md +132 -0
- package/.agents/skills/hyperframes-cli/references/lint-validate-inspect.md +121 -0
- package/.agents/skills/hyperframes-cli/references/preview-render.md +109 -0
- package/.agents/skills/hyperframes-cli/references/upgrade-info-misc.md +75 -0
- package/.agents/skills/hyperframes-media/SKILL.md +97 -0
- package/.agents/skills/hyperframes-media/assets/sfx/CREDITS.md +35 -0
- package/.agents/skills/hyperframes-media/assets/sfx/manifest.json +97 -0
- package/.agents/skills/hyperframes-media/references/bgm.md +72 -0
- package/.agents/skills/hyperframes-media/references/captions/authoring.md +159 -0
- package/.agents/skills/hyperframes-media/references/captions/motion.md +87 -0
- package/.agents/skills/hyperframes-media/references/captions/transcript-handling.md +97 -0
- package/.agents/skills/hyperframes-media/references/remove-background.md +143 -0
- package/.agents/skills/hyperframes-media/references/requirements.md +29 -0
- package/.agents/skills/hyperframes-media/references/sfx.md +42 -0
- package/.agents/skills/hyperframes-media/references/transcribe.md +52 -0
- package/.agents/skills/hyperframes-media/references/tts-to-captions.md +24 -0
- package/.agents/skills/hyperframes-media/references/tts.md +135 -0
- package/.agents/skills/hyperframes-media/scripts/audio.mjs +282 -0
- package/.agents/skills/hyperframes-media/scripts/heygen-tts.mjs +121 -0
- package/.agents/skills/hyperframes-media/scripts/lib/bgm.mjs +235 -0
- package/.agents/skills/hyperframes-media/scripts/lib/heygen.mjs +131 -0
- package/.agents/skills/hyperframes-media/scripts/lib/sfx.mjs +128 -0
- package/.agents/skills/hyperframes-media/scripts/lib/tts.mjs +220 -0
- package/.agents/skills/hyperframes-media/scripts/lyria-recipe.py +128 -0
- package/.agents/skills/hyperframes-media/scripts/wait-bgm.mjs +167 -0
- package/.agents/skills/hyperframes-registry/SKILL.md +101 -0
- package/.agents/skills/hyperframes-registry/examples/add-block.md +51 -0
- package/.agents/skills/hyperframes-registry/examples/add-component.md +73 -0
- package/.agents/skills/hyperframes-registry/references/contributing.md +166 -0
- package/.agents/skills/hyperframes-registry/references/demo-html-pattern.md +54 -0
- package/.agents/skills/hyperframes-registry/references/discovery.md +215 -0
- package/.agents/skills/hyperframes-registry/references/install-locations.md +45 -0
- package/.agents/skills/hyperframes-registry/references/templates.md +417 -0
- package/.agents/skills/hyperframes-registry/references/wiring-blocks.md +61 -0
- package/.agents/skills/hyperframes-registry/references/wiring-components.md +39 -0
- package/.agents/skills/marketing-psychology/SKILL.md +455 -0
- package/.agents/skills/marketing-psychology/evals/evals.json +88 -0
- package/.agents/skills/mintlify/SKILL.md +328 -0
- package/.agents/skills/product-launch-video/SKILL.md +206 -0
- package/.agents/skills/product-launch-video/references/cut-catalog.md +220 -0
- package/.agents/skills/product-launch-video/references/motion-language.md +156 -0
- package/.agents/skills/product-launch-video/references/story-design.md +373 -0
- package/.agents/skills/product-launch-video/references/visual-design.md +126 -0
- package/.agents/skills/product-launch-video/scripts/assemble-index.mjs +570 -0
- package/.agents/skills/product-launch-video/scripts/audio.mjs +253 -0
- package/.agents/skills/product-launch-video/scripts/build-frame.mjs +536 -0
- package/.agents/skills/product-launch-video/scripts/captions.mjs +508 -0
- package/.agents/skills/product-launch-video/scripts/lib/assets.mjs +55 -0
- package/.agents/skills/product-launch-video/scripts/lib/dimensions.mjs +45 -0
- package/.agents/skills/product-launch-video/scripts/lib/storyboard.mjs +249 -0
- package/.agents/skills/product-launch-video/scripts/lib/tokens.mjs +204 -0
- package/.agents/skills/product-launch-video/scripts/lib/transition-registry.mjs +38 -0
- package/.agents/skills/product-launch-video/scripts/lib/transitions.json +71 -0
- package/.agents/skills/product-launch-video/scripts/stage-assets.mjs +39 -0
- package/.agents/skills/product-launch-video/scripts/transitions.mjs +320 -0
- package/.agents/skills/product-launch-video/sub-agents/frame-worker.md +74 -0
- package/.agents/skills/prompt-engineering-patterns/SKILL.md +144 -0
- package/.agents/skills/prompt-engineering-patterns/assets/few-shot-examples.json +106 -0
- package/.agents/skills/prompt-engineering-patterns/assets/prompt-template-library.md +264 -0
- package/.agents/skills/prompt-engineering-patterns/references/chain-of-thought.md +412 -0
- package/.agents/skills/prompt-engineering-patterns/references/details.md +338 -0
- package/.agents/skills/prompt-engineering-patterns/references/few-shot-learning.md +386 -0
- package/.agents/skills/prompt-engineering-patterns/references/prompt-optimization.md +428 -0
- package/.agents/skills/prompt-engineering-patterns/references/prompt-templates.md +484 -0
- package/.agents/skills/prompt-engineering-patterns/references/system-prompts.md +195 -0
- package/.agents/skills/prompt-engineering-patterns/scripts/optimize-prompt.py +279 -0
- package/.agents/skills/remotion-to-hyperframes/SKILL.md +127 -0
- package/.agents/skills/remotion-to-hyperframes/assets/.gitkeep +0 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/run.sh +249 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/README.md +41 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/expected.json +26 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/hf-src/index.html +61 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/remotion-src/package.json +14 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/remotion-src/remotion.config.ts +13 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/remotion-src/src/Root.tsx +13 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/remotion-src/src/TitleCard.tsx +34 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/remotion-src/src/index.ts +4 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/remotion-src/tsconfig.json +15 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/README.md +54 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/expected.json +29 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/hf-src/index.html +118 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/remotion-src/package.json +14 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/remotion-src/remotion.config.ts +13 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/remotion-src/src/MultiScene.tsx +105 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/remotion-src/src/Root.tsx +13 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/remotion-src/src/index.ts +4 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/remotion-src/tsconfig.json +15 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/setup.sh +34 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/README.md +85 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/expected.json +41 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/hf-src/index.html +269 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/package.json +15 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/remotion.config.ts +13 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/Root.tsx +27 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/Stargazed.tsx +37 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/components/AnimatedNumber.tsx +23 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/components/StatCard.tsx +59 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/components/UnderlinedText.tsx +47 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/index.ts +4 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/scenes/OutroScene.tsx +18 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/scenes/StatsScene.tsx +34 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/scenes/TitleScene.tsx +55 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/tsconfig.json +15 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/README.md +51 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/01-use-state.tsx +29 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/02-use-effect-deps.tsx +29 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/03-async-metadata.tsx +39 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/04-third-party-react.tsx +30 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/05-lambda-config.tsx +38 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/06-warnings-only.tsx +36 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/07-custom-hook.tsx +28 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/08-mixed.tsx +41 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/expected.json +85 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/validate.sh +110 -0
- package/.agents/skills/remotion-to-hyperframes/references/api-map.md +142 -0
- package/.agents/skills/remotion-to-hyperframes/references/escape-hatch.md +115 -0
- package/.agents/skills/remotion-to-hyperframes/references/eval.md +140 -0
- package/.agents/skills/remotion-to-hyperframes/references/fonts.md +112 -0
- package/.agents/skills/remotion-to-hyperframes/references/limitations.md +136 -0
- package/.agents/skills/remotion-to-hyperframes/references/lottie.md +121 -0
- package/.agents/skills/remotion-to-hyperframes/references/media.md +149 -0
- package/.agents/skills/remotion-to-hyperframes/references/parameters.md +167 -0
- package/.agents/skills/remotion-to-hyperframes/references/sequencing.md +195 -0
- package/.agents/skills/remotion-to-hyperframes/references/timing.md +165 -0
- package/.agents/skills/remotion-to-hyperframes/references/transitions.md +114 -0
- package/.agents/skills/remotion-to-hyperframes/scripts/.gitkeep +0 -0
- package/.agents/skills/remotion-to-hyperframes/scripts/frame_strip.sh +107 -0
- package/.agents/skills/remotion-to-hyperframes/scripts/lint_source.py +358 -0
- package/.agents/skills/remotion-to-hyperframes/scripts/render_diff.sh +103 -0
- package/.agents/skills/remotion-to-hyperframes/scripts/tests/fixtures/blocker.tsx +50 -0
- package/.agents/skills/remotion-to-hyperframes/scripts/tests/fixtures/clean.tsx +46 -0
- package/.agents/skills/remotion-to-hyperframes/scripts/tests/smoke.sh +90 -0
- package/.agents/skills/seo-audit/SKILL.md +497 -0
- package/.agents/skills/seo-audit/evals/evals.json +136 -0
- package/.agents/skills/seo-audit/references/ai-writing-detection.md +200 -0
- package/.agents/skills/seo-audit/references/international-seo.md +230 -0
- package/README.md +4 -2
- package/SDK.md +64 -1
- package/dist/agents/coding.agent.d.ts +558 -0
- package/dist/agents/coding.agent.d.ts.map +1 -1
- package/dist/agents/coding.agent.js +8 -0
- package/dist/agents/coding.agent.js.map +1 -1
- package/dist/agents/developer-helper.agent.d.ts +712 -0
- package/dist/agents/developer-helper.agent.d.ts.map +1 -1
- package/dist/agents/developer-helper.agent.js +6 -0
- package/dist/agents/developer-helper.agent.js.map +1 -1
- package/dist/agents/docs-research.agent.d.ts +8 -0
- package/dist/agents/docs-research.agent.d.ts.map +1 -1
- package/dist/agents/finance.agent.d.ts +8 -0
- package/dist/agents/finance.agent.d.ts.map +1 -1
- package/dist/agents/manager.d.ts +837 -2
- package/dist/agents/manager.d.ts.map +1 -1
- package/dist/agents/manager.js +28 -0
- package/dist/agents/manager.js.map +1 -1
- package/dist/agents/security.agent.d.ts +165 -3
- package/dist/agents/security.agent.d.ts.map +1 -1
- package/dist/agents/security.agent.js +3 -0
- package/dist/agents/security.agent.js.map +1 -1
- package/dist/agents/swarm/main.js +1 -1
- package/dist/agents/swarm/main.js.map +1 -1
- package/dist/agents/swarm/registry.d.ts.map +1 -1
- package/dist/agents/swarm/registry.js +232 -26
- package/dist/agents/swarm/registry.js.map +1 -1
- package/dist/cli/doctor.d.ts +14 -0
- package/dist/cli/doctor.d.ts.map +1 -1
- package/dist/cli/doctor.js +91 -0
- package/dist/cli/doctor.js.map +1 -1
- package/dist/cli/menu.d.ts.map +1 -1
- package/dist/cli/menu.js +102 -0
- package/dist/cli/menu.js.map +1 -1
- package/dist/cli/setup.d.ts +10 -0
- package/dist/cli/setup.d.ts.map +1 -1
- package/dist/cli/setup.js +178 -0
- package/dist/cli/setup.js.map +1 -1
- package/dist/cli/swarm.d.ts +1 -0
- package/dist/cli/swarm.d.ts.map +1 -1
- package/dist/cli/swarm.js +47 -8
- package/dist/cli/swarm.js.map +1 -1
- package/dist/cli/theme.d.ts +0 -24
- package/dist/cli/theme.d.ts.map +1 -1
- package/dist/cli/theme.js +6 -2
- package/dist/cli/theme.js.map +1 -1
- package/dist/config/env.d.ts +10 -0
- package/dist/config/env.d.ts.map +1 -1
- package/dist/config/env.js +38 -17
- package/dist/config/env.js.map +1 -1
- package/dist/index.js +96 -5
- package/dist/index.js.map +1 -1
- package/dist/jobs/trigger-orchestrator.d.ts +5 -1
- package/dist/jobs/trigger-orchestrator.d.ts.map +1 -1
- package/dist/jobs/trigger-orchestrator.js +29 -9
- package/dist/jobs/trigger-orchestrator.js.map +1 -1
- package/dist/jobs/trigger-router.d.ts.map +1 -1
- package/dist/jobs/trigger-router.js +3 -2
- package/dist/jobs/trigger-router.js.map +1 -1
- package/dist/memory/corporate-wiki.d.ts +15 -0
- package/dist/memory/corporate-wiki.d.ts.map +1 -0
- package/dist/memory/corporate-wiki.js +152 -0
- package/dist/memory/corporate-wiki.js.map +1 -0
- package/dist/memory/scratchpad.d.ts +2 -0
- package/dist/memory/scratchpad.d.ts.map +1 -1
- package/dist/memory/scratchpad.js +27 -0
- package/dist/memory/scratchpad.js.map +1 -1
- package/dist/observability/doctor.d.ts.map +1 -1
- package/dist/observability/doctor.js +12 -4
- package/dist/observability/doctor.js.map +1 -1
- package/dist/observability/traces.d.ts +72 -0
- package/dist/observability/traces.d.ts.map +1 -0
- package/dist/observability/traces.js +1134 -0
- package/dist/observability/traces.js.map +1 -0
- package/dist/runtime/swarm.d.ts +1 -1
- package/dist/runtime/swarm.d.ts.map +1 -1
- package/dist/runtime/swarm.js +54 -25
- package/dist/runtime/swarm.js.map +1 -1
- package/dist/safety/approvals.d.ts +2 -2
- package/dist/safety/approvals.d.ts.map +1 -1
- package/dist/safety/approvals.js +47 -1
- package/dist/safety/approvals.js.map +1 -1
- package/dist/safety/redaction.d.ts.map +1 -1
- package/dist/safety/redaction.js +10 -0
- package/dist/safety/redaction.js.map +1 -1
- package/dist/tools/cloud.tool.d.ts +394 -0
- package/dist/tools/cloud.tool.d.ts.map +1 -0
- package/dist/tools/cloud.tool.js +784 -0
- package/dist/tools/cloud.tool.js.map +1 -0
- package/dist/tools/computer-use.tool.d.ts +2 -2
- package/dist/tools/corporate-wiki.tool.d.ts +22 -0
- package/dist/tools/corporate-wiki.tool.d.ts.map +1 -0
- package/dist/tools/corporate-wiki.tool.js +43 -0
- package/dist/tools/corporate-wiki.tool.js.map +1 -0
- package/dist/tools/devops.tool.d.ts +162 -0
- package/dist/tools/devops.tool.d.ts.map +1 -0
- package/dist/tools/devops.tool.js +377 -0
- package/dist/tools/devops.tool.js.map +1 -0
- package/dist/tools/multimedia.tool.d.ts +126 -0
- package/dist/tools/multimedia.tool.d.ts.map +1 -0
- package/dist/tools/multimedia.tool.js +533 -0
- package/dist/tools/multimedia.tool.js.map +1 -0
- package/dist/tools/osint.tool.d.ts +6 -6
- package/dist/tools/sandbox-dev.tool.d.ts +15 -0
- package/dist/tools/sandbox-dev.tool.d.ts.map +1 -0
- package/dist/tools/sandbox-dev.tool.js +146 -0
- package/dist/tools/sandbox-dev.tool.js.map +1 -0
- package/dist/tools/scratchpad.tool.d.ts +8 -0
- package/dist/tools/scratchpad.tool.d.ts.map +1 -1
- package/dist/tools/scratchpad.tool.js +40 -1
- package/dist/tools/scratchpad.tool.js.map +1 -1
- package/dist/tools/setup-assistant.tool.d.ts +1 -1
- package/dist/tools/swarm-ops.tool.d.ts +14 -0
- package/dist/tools/swarm-ops.tool.d.ts.map +1 -1
- package/dist/tools/swarm-ops.tool.js +47 -0
- package/dist/tools/swarm-ops.tool.js.map +1 -1
- package/dist/tools/sysops.tool.d.ts +157 -0
- package/dist/tools/sysops.tool.d.ts.map +1 -0
- package/dist/tools/sysops.tool.js +643 -0
- package/dist/tools/sysops.tool.js.map +1 -0
- package/dist/voice/deepgram.d.ts.map +1 -1
- package/dist/voice/deepgram.js +30 -2
- package/dist/voice/deepgram.js.map +1 -1
- package/package.json +8 -1
- package/scripts/release-github.mjs +23 -16
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// audio.mjs — the shared HyperFrames audio engine. ONE implementation of TTS +
|
|
3
|
+
// BGM + SFX for every video workflow (product-launch, general-video, pr-to-video,
|
|
4
|
+
// …). Workflows do NOT vendor a copy: they write a neutral `audio_request.json`
|
|
5
|
+
// (a tiny per-workflow adapter maps their storyboard/scenes into it) and call:
|
|
6
|
+
//
|
|
7
|
+
// node <MEDIA_DIR>/scripts/audio.mjs --request ./audio_request.json --hyperframes . --out ./audio_meta.json
|
|
8
|
+
//
|
|
9
|
+
// The three capabilities degrade on ONE switch — whether HeyGen is configured
|
|
10
|
+
// (credential present, NOT the CLI). This mirrors the table in ../SKILL.md:
|
|
11
|
+
//
|
|
12
|
+
// TTS : HeyGen REST → ElevenLabs → Kokoro (CLI)
|
|
13
|
+
// BGM : HeyGen retrieve → (no credential) Lyria/MusicGen generate
|
|
14
|
+
// SFX : HeyGen retrieve → (no credential) bundled 21-file library
|
|
15
|
+
//
|
|
16
|
+
// ── audio_request.json (input) ────────────────────────────────────────────────
|
|
17
|
+
// {
|
|
18
|
+
// "provider": "auto", // auto|heygen|elevenlabs|kokoro (override: --provider)
|
|
19
|
+
// "lang": "en", "speed": 1.0,
|
|
20
|
+
// "lines": [ // one TTS unit each; id joins back to the caller's model
|
|
21
|
+
// { "id": "01", "text": "...", "sfx": ["whoosh", "ui click"] }
|
|
22
|
+
// ],
|
|
23
|
+
// "bgm": { "mode": "retrieve", // retrieve|generate|none (override: --bgm-mode / --no-bgm)
|
|
24
|
+
// "query": "calm cinematic underscore", // mood for retrieval
|
|
25
|
+
// "prompt": null, // full prompt for generation (else inferred)
|
|
26
|
+
// "blob": "...", "archetype": "...", "arc": "..." } // optional mood-inference hints
|
|
27
|
+
// }
|
|
28
|
+
//
|
|
29
|
+
// ── audio_meta.json (output, id-keyed) ───────────────────────────────────────
|
|
30
|
+
// { tts_provider, voice_id,
|
|
31
|
+
// bgm: { path, volume, mode, query?, duration_s? } | null,
|
|
32
|
+
// bgm_pending, bgm_provider, bgm_pid, bgm_log, bgm_mode, bgm_target_duration_s, …,
|
|
33
|
+
// voices: [ { id, path, duration_s, words: [{id,text,start,end}] } ],
|
|
34
|
+
// sfx: [ { id, name, file, source, offset_s, duration_s, volume } ],
|
|
35
|
+
// total_duration_s }
|
|
36
|
+
//
|
|
37
|
+
// --only tts,bgm,sfx runs a subset and MERGES into an existing --out (so a
|
|
38
|
+
// workflow can do TTS+BGM early, then SFX later once cues exist). When BGM uses
|
|
39
|
+
// the generate path it is spawned detached (bgm_pending:true) — run wait-bgm.mjs
|
|
40
|
+
// before assembling.
|
|
41
|
+
|
|
42
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
43
|
+
import { dirname, join, resolve } from "node:path";
|
|
44
|
+
import { fileURLToPath } from "node:url";
|
|
45
|
+
import { heygenAuthHeaders, heygenCredential, loadEnvFromDir } from "./lib/heygen.mjs";
|
|
46
|
+
import {
|
|
47
|
+
ffprobeDuration,
|
|
48
|
+
pickProvider,
|
|
49
|
+
resolveVoiceId,
|
|
50
|
+
synthesizeOne,
|
|
51
|
+
transcribeWav,
|
|
52
|
+
withWordIds,
|
|
53
|
+
} from "./lib/tts.mjs";
|
|
54
|
+
import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs";
|
|
55
|
+
import { resolveSfx } from "./lib/sfx.mjs";
|
|
56
|
+
|
|
57
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
58
|
+
const argv = process.argv.slice(2);
|
|
59
|
+
const flag = (name, def) => {
|
|
60
|
+
const i = argv.indexOf(`--${name}`);
|
|
61
|
+
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : def;
|
|
62
|
+
};
|
|
63
|
+
const has = (name) => argv.includes(`--${name}`);
|
|
64
|
+
const die = (m) => {
|
|
65
|
+
console.error(`✗ audio engine: ${m}`);
|
|
66
|
+
process.exit(1);
|
|
67
|
+
};
|
|
68
|
+
const r3 = (x) => Number(x.toFixed(3));
|
|
69
|
+
|
|
70
|
+
const hyperframesDir = resolve(flag("hyperframes", "."));
|
|
71
|
+
const requestPath = resolve(flag("request", join(hyperframesDir, "audio_request.json")));
|
|
72
|
+
const outPath = resolve(flag("out", join(hyperframesDir, "audio_meta.json")));
|
|
73
|
+
const sfxLibDir = resolve(flag("sfx-lib", join(HERE, "..", "assets", "sfx")));
|
|
74
|
+
const lyriaRecipe = resolve(flag("lyria-recipe", join(HERE, "lyria-recipe.py")));
|
|
75
|
+
const onlyArg = flag("only", "tts,bgm,sfx");
|
|
76
|
+
const only = new Set(
|
|
77
|
+
onlyArg
|
|
78
|
+
.split(",")
|
|
79
|
+
.map((s) => s.trim())
|
|
80
|
+
.filter(Boolean),
|
|
81
|
+
);
|
|
82
|
+
const providerOverride = flag("provider", null);
|
|
83
|
+
const bgmModeOverride = flag("bgm-mode", null);
|
|
84
|
+
const noBgm = has("no-bgm");
|
|
85
|
+
const voiceOverride = flag("voice", null);
|
|
86
|
+
const speedOverride = flag("speed", null);
|
|
87
|
+
const langOverride = flag("lang", null);
|
|
88
|
+
const seedSeconds = Number(flag("seed-seconds", "28")) || 28;
|
|
89
|
+
|
|
90
|
+
if (!existsSync(requestPath)) die(`audio_request.json not found at ${requestPath}`);
|
|
91
|
+
let request;
|
|
92
|
+
try {
|
|
93
|
+
request = JSON.parse(readFileSync(requestPath, "utf8"));
|
|
94
|
+
} catch (e) {
|
|
95
|
+
die(`audio_request.json parse: ${e.message}`);
|
|
96
|
+
}
|
|
97
|
+
const lines = Array.isArray(request.lines) ? request.lines : [];
|
|
98
|
+
const lang = langOverride || request.lang || "en";
|
|
99
|
+
const speed = Number(speedOverride ?? request.speed ?? 1.0) || 1.0;
|
|
100
|
+
|
|
101
|
+
// ── env + HeyGen availability (the single switch) ─────────────────────────────
|
|
102
|
+
loadEnvFromDir(hyperframesDir);
|
|
103
|
+
const heygenOK = heygenCredential() !== null;
|
|
104
|
+
const headers = heygenOK ? heygenAuthHeaders() : null;
|
|
105
|
+
|
|
106
|
+
// ── merge base: preserve sections not selected by --only ──────────────────────
|
|
107
|
+
const prev = existsSync(outPath) ? JSON.parse(readFileSync(outPath, "utf8")) : {};
|
|
108
|
+
const anomalies = [];
|
|
109
|
+
|
|
110
|
+
// ── TTS ───────────────────────────────────────────────────────────────────────
|
|
111
|
+
let voices = prev.voices ?? [];
|
|
112
|
+
let ttsProvider = prev.tts_provider ?? null;
|
|
113
|
+
let voiceId = prev.voice_id ?? null;
|
|
114
|
+
if (only.has("tts") && lines.length) {
|
|
115
|
+
try {
|
|
116
|
+
ttsProvider = pickProvider(
|
|
117
|
+
providerOverride || (request.provider === "auto" ? null : request.provider),
|
|
118
|
+
);
|
|
119
|
+
} catch (e) {
|
|
120
|
+
die(e.message);
|
|
121
|
+
}
|
|
122
|
+
voiceId = await resolveVoiceId({
|
|
123
|
+
provider: ttsProvider,
|
|
124
|
+
userVoice: voiceOverride || request.voice,
|
|
125
|
+
lang,
|
|
126
|
+
});
|
|
127
|
+
console.error(`· tts: ${ttsProvider} · voice ${voiceId} · ${lines.length} line(s)`);
|
|
128
|
+
const synthLine = async (line) => {
|
|
129
|
+
const id = String(line.id);
|
|
130
|
+
const text = String(line.text ?? "").trim();
|
|
131
|
+
if (!text) {
|
|
132
|
+
anomalies.push(`line ${id}: empty text — skipped`);
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
const rel = `assets/voice/${id}.wav`;
|
|
136
|
+
const abs = join(hyperframesDir, rel);
|
|
137
|
+
const { ok, words } = await synthesizeOne({
|
|
138
|
+
provider: ttsProvider,
|
|
139
|
+
text,
|
|
140
|
+
voiceId,
|
|
141
|
+
lang,
|
|
142
|
+
speed,
|
|
143
|
+
wavAbs: abs,
|
|
144
|
+
hyperframesDir,
|
|
145
|
+
});
|
|
146
|
+
if (!ok) {
|
|
147
|
+
anomalies.push(`line ${id}: TTS failed — omitted`);
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
let wordArr = words; // heygen: native; else transcribe
|
|
151
|
+
if (!wordArr) wordArr = await transcribeWav({ wavRel: rel, lang, hyperframesDir });
|
|
152
|
+
const dur = ffprobeDuration(abs);
|
|
153
|
+
if (!isFinite(dur) || dur <= 0) {
|
|
154
|
+
anomalies.push(`line ${id}: bad voice duration — omitted`);
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) };
|
|
158
|
+
};
|
|
159
|
+
const results = await Promise.all(lines.map(synthLine));
|
|
160
|
+
voices = results.filter(Boolean);
|
|
161
|
+
for (const v of voices)
|
|
162
|
+
console.error(` voice ${v.id}: ${v.path} (${v.duration_s}s, ${v.words.length} words)`);
|
|
163
|
+
}
|
|
164
|
+
const hasVoice = voices.length > 0;
|
|
165
|
+
const totalDuration = r3(voices.reduce((a, v) => a + (v.duration_s || 0), 0));
|
|
166
|
+
|
|
167
|
+
// ── BGM ─────────────────────────────────────────────────────────────────────
|
|
168
|
+
let bgm = prev.bgm ?? null;
|
|
169
|
+
const bgmFields = {
|
|
170
|
+
bgm_pending: prev.bgm_pending ?? false,
|
|
171
|
+
bgm_provider: prev.bgm_provider ?? null,
|
|
172
|
+
bgm_pid: prev.bgm_pid ?? null,
|
|
173
|
+
bgm_log: prev.bgm_log ?? null,
|
|
174
|
+
bgm_mode: prev.bgm_mode ?? null,
|
|
175
|
+
bgm_target_duration_s: prev.bgm_target_duration_s ?? null,
|
|
176
|
+
bgm_seed_duration_s: prev.bgm_seed_duration_s ?? null,
|
|
177
|
+
bgm_loop_count: prev.bgm_loop_count ?? null,
|
|
178
|
+
};
|
|
179
|
+
if (only.has("bgm")) {
|
|
180
|
+
bgm = null;
|
|
181
|
+
Object.keys(bgmFields).forEach((k) => (bgmFields[k] = k === "bgm_pending" ? false : null));
|
|
182
|
+
// Mode resolution. An EXPLICIT mode (flag or request.bgm.mode) is strict:
|
|
183
|
+
// "retrieve" means retrieve-or-nothing — it never silently starts a detached
|
|
184
|
+
// generate (a caller with no wait-bgm step, e.g. product-launch, must not get
|
|
185
|
+
// a pending job it can't await). Only the UNSET/auto default picks generate
|
|
186
|
+
// when HeyGen is absent.
|
|
187
|
+
const explicitMode = bgmModeOverride || request.bgm?.mode || null;
|
|
188
|
+
let mode = noBgm ? "none" : explicitMode || (heygenOK ? "retrieve" : "generate");
|
|
189
|
+
if (mode === "retrieve" && !heygenOK) {
|
|
190
|
+
anomalies.push(
|
|
191
|
+
"bgm: retrieve requires a HeyGen credential — skipped (no generate fallback for an explicit retrieve)",
|
|
192
|
+
);
|
|
193
|
+
mode = "none";
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (mode === "none") {
|
|
197
|
+
console.error(`· bgm: disabled`);
|
|
198
|
+
} else if (mode === "retrieve") {
|
|
199
|
+
try {
|
|
200
|
+
bgm = await retrieveBgm({ query: request.bgm?.query, headers, hyperframesDir, hasVoice });
|
|
201
|
+
if (bgm) {
|
|
202
|
+
bgmFields.bgm_provider = "heygen";
|
|
203
|
+
bgmFields.bgm_mode = "retrieve";
|
|
204
|
+
console.error(` bgm: ${bgm.path} (retrieve "${bgm.query}")`);
|
|
205
|
+
} else {
|
|
206
|
+
anomalies.push(`bgm: no music match for "${request.bgm?.query ?? ""}" — skipped`);
|
|
207
|
+
}
|
|
208
|
+
} catch (e) {
|
|
209
|
+
anomalies.push(`bgm retrieve failed: ${e.message} — skipped`);
|
|
210
|
+
}
|
|
211
|
+
} else {
|
|
212
|
+
// generate
|
|
213
|
+
const prompt = inferBgmPrompt({
|
|
214
|
+
userPrompt: request.bgm?.prompt,
|
|
215
|
+
blob: request.bgm?.blob || request.bgm?.query,
|
|
216
|
+
archetype: request.bgm?.archetype,
|
|
217
|
+
arc: request.bgm?.arc,
|
|
218
|
+
});
|
|
219
|
+
const gen = generateBgmDetached({
|
|
220
|
+
prompt,
|
|
221
|
+
durationS: totalDuration || 30,
|
|
222
|
+
hyperframesDir,
|
|
223
|
+
lyriaRecipe: existsSync(lyriaRecipe) ? lyriaRecipe : null,
|
|
224
|
+
seedSeconds,
|
|
225
|
+
hasVoice,
|
|
226
|
+
});
|
|
227
|
+
if (gen.disabled) {
|
|
228
|
+
anomalies.push(`bgm: ${gen.reason}`);
|
|
229
|
+
} else {
|
|
230
|
+
bgm = { path: gen.path, volume: gen.volume, mode: gen.mode, duration_s: null };
|
|
231
|
+
bgmFields.bgm_pending = true;
|
|
232
|
+
bgmFields.bgm_provider = gen.provider;
|
|
233
|
+
bgmFields.bgm_pid = gen.pid;
|
|
234
|
+
bgmFields.bgm_log = gen.log;
|
|
235
|
+
bgmFields.bgm_mode = gen.mode;
|
|
236
|
+
bgmFields.bgm_target_duration_s = gen.target_duration_s ?? null;
|
|
237
|
+
bgmFields.bgm_seed_duration_s = gen.seed_duration_s ?? null;
|
|
238
|
+
bgmFields.bgm_loop_count = gen.loop_count ?? null;
|
|
239
|
+
console.error(` bgm: launched ${gen.provider} (detached, pid ${gen.pid}) → ${gen.path}`);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// ── SFX ─────────────────────────────────────────────────────────────────────
|
|
245
|
+
let sfx = prev.sfx ?? [];
|
|
246
|
+
if (only.has("sfx")) {
|
|
247
|
+
const cues = lines.flatMap((l) =>
|
|
248
|
+
(Array.isArray(l.sfx) ? l.sfx : [])
|
|
249
|
+
.map((name) => ({ id: String(l.id), name: String(name).trim() }))
|
|
250
|
+
.filter((c) => c.name),
|
|
251
|
+
);
|
|
252
|
+
const res = await resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxLibDir });
|
|
253
|
+
sfx = res.sfx;
|
|
254
|
+
anomalies.push(...res.anomalies);
|
|
255
|
+
console.error(
|
|
256
|
+
`· sfx: ${sfx.length} cue(s) resolved (${heygenOK ? "heygen retrieval" : "bundled library"})`,
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
// ── write audio_meta.json ─────────────────────────────────────────────────────
|
|
261
|
+
const meta = {
|
|
262
|
+
tts_provider: ttsProvider,
|
|
263
|
+
voice_id: voiceId,
|
|
264
|
+
bgm,
|
|
265
|
+
...bgmFields,
|
|
266
|
+
voices,
|
|
267
|
+
sfx,
|
|
268
|
+
total_duration_s: totalDuration,
|
|
269
|
+
};
|
|
270
|
+
mkdirSync(dirname(outPath), { recursive: true });
|
|
271
|
+
writeFileSync(outPath, JSON.stringify(meta, null, 2));
|
|
272
|
+
|
|
273
|
+
console.log(`✓ audio engine → ${outPath}`);
|
|
274
|
+
console.log(` heygen: ${heygenOK ? "yes" : "no"} · ran: ${[...only].join(",")}`);
|
|
275
|
+
console.log(
|
|
276
|
+
` voices: ${voices.length} · bgm: ${bgm ? `${bgmFields.bgm_provider}${bgmFields.bgm_pending ? " (pending)" : ""}` : "none"} · sfx: ${sfx.length}`,
|
|
277
|
+
);
|
|
278
|
+
console.log(` total voice duration: ${totalDuration}s`);
|
|
279
|
+
if (anomalies.length) {
|
|
280
|
+
console.log(`\nanomalies (non-fatal):`);
|
|
281
|
+
for (const a of anomalies) console.log(` - ${a}`);
|
|
282
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Self-contained HeyGen TTS — single text in → one wav (+ optional words JSON)
|
|
3
|
+
// out. A thin CLI over lib/tts.mjs (the same code the audio engine uses), so the
|
|
4
|
+
// HeyGen REST call, starfish voice pick, mp3→wav transcode, and word-timestamp
|
|
5
|
+
// filtering live in exactly one place. Bypasses the `hyperframes` CLI, which in
|
|
6
|
+
// the published build is Kokoro-only.
|
|
7
|
+
//
|
|
8
|
+
// Usage:
|
|
9
|
+
// node heygen-tts.mjs "Text to speak" -o narration.wav [--words narration.words.json]
|
|
10
|
+
// node heygen-tts.mjs ./script.txt -o narration.wav --words narration.words.json
|
|
11
|
+
// node heygen-tts.mjs "Bonjour" -o fr.wav --lang fr --voice <id>
|
|
12
|
+
// node heygen-tts.mjs --list # list starfish voices and exit
|
|
13
|
+
//
|
|
14
|
+
// Flags: -o/--output (.wav → ffmpeg transcode; .mp3 → raw bytes), --words,
|
|
15
|
+
// --voice (starfish id), --speed, --lang, --list.
|
|
16
|
+
// Requires: $HEYGEN_API_KEY (or ~/.heygen) and ffmpeg for .wav output.
|
|
17
|
+
|
|
18
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
19
|
+
import { dirname, resolve } from "node:path";
|
|
20
|
+
import { heygenAuthHeaders, heygenJSON, loadEnvFromDir } from "./lib/heygen.mjs";
|
|
21
|
+
import { ffprobeDuration, resolveVoiceId, synthesizeOne, withWordIds } from "./lib/tts.mjs";
|
|
22
|
+
|
|
23
|
+
const argv = process.argv.slice(2);
|
|
24
|
+
function flag(name, def) {
|
|
25
|
+
const i = argv.indexOf(`--${name}`);
|
|
26
|
+
if (i < 0) return def;
|
|
27
|
+
if (i + 1 >= argv.length) return true;
|
|
28
|
+
const v = argv[i + 1];
|
|
29
|
+
return v.startsWith("--") ? true : v;
|
|
30
|
+
}
|
|
31
|
+
const die = (m) => {
|
|
32
|
+
console.error(`✗ heygen-tts: ${m}`);
|
|
33
|
+
process.exit(1);
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
// First arg that isn't a flag or the -o value is the text / .txt path.
|
|
37
|
+
const positional = (() => {
|
|
38
|
+
for (let i = 0; i < argv.length; i++) {
|
|
39
|
+
const a = argv[i];
|
|
40
|
+
if (a.startsWith("--")) {
|
|
41
|
+
const next = argv[i + 1];
|
|
42
|
+
if (next && !next.startsWith("--")) i++;
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
if (a === "-o") {
|
|
46
|
+
i++;
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
return a;
|
|
50
|
+
}
|
|
51
|
+
return null;
|
|
52
|
+
})();
|
|
53
|
+
|
|
54
|
+
const output = resolve(
|
|
55
|
+
(typeof flag("output") === "string" && flag("output")) ||
|
|
56
|
+
(argv.includes("-o") && argv[argv.indexOf("-o") + 1]) ||
|
|
57
|
+
"narration.wav",
|
|
58
|
+
);
|
|
59
|
+
const wordsPath = typeof flag("words") === "string" ? resolve(flag("words")) : null;
|
|
60
|
+
const userVoice = typeof flag("voice") === "string" ? flag("voice") : null;
|
|
61
|
+
const speedRaw = typeof flag("speed") === "string" ? Number(flag("speed")) : 1.0;
|
|
62
|
+
const speed = isFinite(speedRaw) && speedRaw > 0 ? speedRaw : 1.0;
|
|
63
|
+
const lang = typeof flag("lang") === "string" ? flag("lang") : "en";
|
|
64
|
+
const listOnly = flag("list") === true;
|
|
65
|
+
|
|
66
|
+
loadEnvFromDir(process.cwd());
|
|
67
|
+
let authHeaders;
|
|
68
|
+
try {
|
|
69
|
+
authHeaders = heygenAuthHeaders();
|
|
70
|
+
} catch (e) {
|
|
71
|
+
die(e.message);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// ---------- --list ----------
|
|
75
|
+
if (listOnly) {
|
|
76
|
+
const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
|
|
77
|
+
headers: authHeaders,
|
|
78
|
+
});
|
|
79
|
+
for (const v of payload.data ?? payload.voices ?? []) {
|
|
80
|
+
console.log(`${v.voice_id}\t${v.name}\t${v.language ?? ""}`);
|
|
81
|
+
}
|
|
82
|
+
process.exit(0);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// ---------- resolve text + voice ----------
|
|
86
|
+
if (!positional) die("no text given. Pass a string or a .txt path, or use --list.");
|
|
87
|
+
const text =
|
|
88
|
+
positional.endsWith(".txt") && existsSync(resolve(positional))
|
|
89
|
+
? readFileSync(resolve(positional), "utf8").trim()
|
|
90
|
+
: positional;
|
|
91
|
+
if (!text) die("input text is empty");
|
|
92
|
+
|
|
93
|
+
const voiceId = await resolveVoiceId({ provider: "heygen", userVoice, lang });
|
|
94
|
+
if (!userVoice) console.error(`· using voice ${voiceId}`);
|
|
95
|
+
|
|
96
|
+
// ---------- synthesize (shared engine code) ----------
|
|
97
|
+
const { ok, words } = await synthesizeOne({
|
|
98
|
+
provider: "heygen",
|
|
99
|
+
text,
|
|
100
|
+
voiceId,
|
|
101
|
+
lang,
|
|
102
|
+
speed,
|
|
103
|
+
wavAbs: output,
|
|
104
|
+
hyperframesDir: process.cwd(),
|
|
105
|
+
});
|
|
106
|
+
if (!ok) die("synthesis failed (HeyGen request/transcode error)");
|
|
107
|
+
|
|
108
|
+
let wordCount = 0;
|
|
109
|
+
if (wordsPath) {
|
|
110
|
+
if (words && words.length) {
|
|
111
|
+
mkdirSync(dirname(wordsPath), { recursive: true });
|
|
112
|
+
writeFileSync(wordsPath, JSON.stringify(withWordIds(words), null, 2));
|
|
113
|
+
wordCount = words.length;
|
|
114
|
+
} else {
|
|
115
|
+
console.error("⚠ no word_timestamps in response — run `hyperframes transcribe` instead");
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const dur = ffprobeDuration(output);
|
|
120
|
+
const durStr = isFinite(dur) ? ` (${dur.toFixed(2)}s)` : "";
|
|
121
|
+
console.log(`✓ ${output}${durStr}${wordCount ? ` · ${wordsPath} (${wordCount} words)` : ""}`);
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
// bgm.mjs — background music for the media audio engine. Two routes, gated the
|
|
2
|
+
// same way as TTS/SFX:
|
|
3
|
+
//
|
|
4
|
+
// retrieve (default when HeyGen is configured) — search HeyGen's music library
|
|
5
|
+
// by mood, download the top track. Synchronous. assets/bgm/track.mp3.
|
|
6
|
+
// generate (the alternative; the automatic choice when HeyGen is absent) —
|
|
7
|
+
// Lyria (cloud, $GEMINI_API_KEY/$GOOGLE_API_KEY + google-genai) preferred,
|
|
8
|
+
// else local MusicGen (facebook/musicgen-small via transformers). Spawned
|
|
9
|
+
// DETACHED so the engine can return while audio renders; the caller marks
|
|
10
|
+
// bgm_pending and runs wait-bgm.mjs before assembling. assets/bgm/track.wav.
|
|
11
|
+
//
|
|
12
|
+
// Missing/failed BGM never blocks a render.
|
|
13
|
+
|
|
14
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
15
|
+
import { existsSync, mkdirSync, openSync, closeSync } from "node:fs";
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { downloadTo, searchSounds } from "./heygen.mjs";
|
|
18
|
+
|
|
19
|
+
const r3 = (x) => Number(x.toFixed(3));
|
|
20
|
+
const lyriaKey = () => process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || "";
|
|
21
|
+
|
|
22
|
+
const BGM_PY_DEPS = ["transformers", "torch", "soundfile", "numpy"];
|
|
23
|
+
const BGM_PY_PROBE =
|
|
24
|
+
"import transformers, soundfile, torch, numpy; from transformers import MusicgenForConditionalGeneration";
|
|
25
|
+
const LYRIA_PY_DEPS = ["google-genai", "python-dotenv"];
|
|
26
|
+
const LYRIA_PY_PROBE = "import google.genai";
|
|
27
|
+
|
|
28
|
+
function pyOk(probe) {
|
|
29
|
+
return spawnSync("python3", ["-c", probe], { stdio: "ignore" }).status === 0;
|
|
30
|
+
}
|
|
31
|
+
function pipInstall(deps) {
|
|
32
|
+
return spawnSync("pip", ["install", "-q", ...deps], { stdio: "ignore" }).status === 0;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// ── retrieval (HeyGen music library) ──────────────────────────────────────────
|
|
36
|
+
export async function retrieveBgm({ query, headers, hyperframesDir, hasVoice }) {
|
|
37
|
+
const q = query || "calm cinematic underscore";
|
|
38
|
+
const results = await searchSounds(q, "music", headers, { limit: 5 });
|
|
39
|
+
if (!results.length) return null;
|
|
40
|
+
const top = results[0];
|
|
41
|
+
const rel = "assets/bgm/track.mp3";
|
|
42
|
+
await downloadTo(top.audio_url, join(hyperframesDir, rel));
|
|
43
|
+
return {
|
|
44
|
+
path: rel,
|
|
45
|
+
volume: hasVoice ? 0.8 : 0.9,
|
|
46
|
+
query: q,
|
|
47
|
+
mode: "retrieve",
|
|
48
|
+
duration_s: typeof top.duration === "number" ? r3(top.duration) : null,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// ── mood inference (for the generate path's prompt) ──────────────────────────
|
|
53
|
+
// Industry base → archetype shape → emotional-arc tiebreaker. Exported so a
|
|
54
|
+
// workflow adapter can build a rich prompt from its own narrative metadata; the
|
|
55
|
+
// engine also calls it when generate has only a plain mood query.
|
|
56
|
+
export function inferBgmPrompt({ blob = "", archetype = "", arc = "", userPrompt = "" } = {}) {
|
|
57
|
+
if (userPrompt) return userPrompt;
|
|
58
|
+
const b = String(blob).toLowerCase();
|
|
59
|
+
let base;
|
|
60
|
+
let bpm;
|
|
61
|
+
if (/\b(crypto|nft|web3|defi|token|blockchain|exchange|wallet|dao)\b/.test(b)) {
|
|
62
|
+
base = "atmospheric electronic, deep bass, futuristic synths, restrained percussion";
|
|
63
|
+
bpm = 100;
|
|
64
|
+
} else if (/\b(finance|fintech|bank|payment|invest|wealth|insurance|treasury)\b/.test(b)) {
|
|
65
|
+
base = "calm cinematic, soft strings, subtle piano, restrained percussion";
|
|
66
|
+
bpm = 92;
|
|
67
|
+
} else if (/\b(creative|agency|design|studio|art|brand|marketing|content)\b/.test(b)) {
|
|
68
|
+
base = "playful electronic, warm pads, light percussion";
|
|
69
|
+
bpm = 115;
|
|
70
|
+
} else {
|
|
71
|
+
base = "uplifting corporate tech, bright modern piano with synth pads";
|
|
72
|
+
bpm = 108;
|
|
73
|
+
}
|
|
74
|
+
const at = String(archetype).toLowerCase();
|
|
75
|
+
const ar = String(arc).toLowerCase();
|
|
76
|
+
if (/\bpas\b|pain.agitate|pain.+solve/.test(at))
|
|
77
|
+
return `${base}, starts with subtle tension then builds to resolution, BPM ${bpm}, transitions from MINOR to MAJOR`;
|
|
78
|
+
if (/\bbab\b|before.after|future.pac|vision/.test(at))
|
|
79
|
+
return `${base}, cinematic and aspirational, steady build with rising energy, BPM ${bpm}, MAJOR`;
|
|
80
|
+
if (/cascade|feature.benefit/.test(at))
|
|
81
|
+
return `${base}, energetic and driving, consistent momentum, BPM ${Math.min(bpm + 10, 128)}, MAJOR`;
|
|
82
|
+
if (/demo.loop|question.+answer/.test(at))
|
|
83
|
+
return `${base}, clean and focused, minimal arrangement, BPM ${Math.max(bpm - 8, 88)}`;
|
|
84
|
+
if (/frustrat|anxiety|overwhelm|tension/.test(ar) && /relief|excite|triumph/.test(ar))
|
|
85
|
+
return `${base}, builds from understated tension to uplifting resolution, BPM ${bpm}, MINOR to MAJOR`;
|
|
86
|
+
if (/excit|awe|power|triumph/.test(ar))
|
|
87
|
+
return `${base}, energetic and confident, BPM ${bpm}, MAJOR`;
|
|
88
|
+
if (/trust|ease|clarity|reassur/.test(ar))
|
|
89
|
+
return `${base}, warm and reassuring, BPM ${Math.max(bpm - 5, 85)}`;
|
|
90
|
+
return `${base}, BPM ${bpm}, MAJOR`;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// ── generation (Lyria → MusicGen, detached) ──────────────────────────────────
|
|
94
|
+
// Returns a bgmMeta the caller folds into audio_meta:
|
|
95
|
+
// { path, mode, volume, provider, pid, log, target_duration_s, seed_duration_s,
|
|
96
|
+
// loop_count, pending:true } on success, or { disabled:true, reason }.
|
|
97
|
+
export function generateBgmDetached({
|
|
98
|
+
prompt,
|
|
99
|
+
durationS,
|
|
100
|
+
hyperframesDir,
|
|
101
|
+
lyriaRecipe,
|
|
102
|
+
seedSeconds = 28,
|
|
103
|
+
hasVoice,
|
|
104
|
+
}) {
|
|
105
|
+
const rel = "assets/bgm/track.wav";
|
|
106
|
+
const abs = join(hyperframesDir, rel);
|
|
107
|
+
mkdirSync(join(hyperframesDir, "assets", "bgm"), { recursive: true });
|
|
108
|
+
const log = join(hyperframesDir, "assets", "bgm", `bgm-${Date.now()}.log`);
|
|
109
|
+
const targetS = Math.max(1, durationS);
|
|
110
|
+
const baseMeta = { path: rel, mode: null, volume: hasVoice ? 0.8 : 0.9, pending: true };
|
|
111
|
+
|
|
112
|
+
const lyriaConfigured = !!lyriaKey() && !!lyriaRecipe && existsSync(lyriaRecipe);
|
|
113
|
+
|
|
114
|
+
// Make a backend runnable: prefer Lyria when configured (install google-genai
|
|
115
|
+
// on demand), else ensure local MusicGen deps. Installs are synchronous here —
|
|
116
|
+
// generation itself is detached, so the engine still returns promptly.
|
|
117
|
+
if (lyriaConfigured && !pyOk(LYRIA_PY_PROBE)) pipInstall(LYRIA_PY_DEPS);
|
|
118
|
+
const useLyria = lyriaConfigured && pyOk(LYRIA_PY_PROBE);
|
|
119
|
+
if (!useLyria && !pyOk(BGM_PY_PROBE)) pipInstall(BGM_PY_DEPS);
|
|
120
|
+
|
|
121
|
+
const fd = openSync(log, "w");
|
|
122
|
+
if (useLyria) {
|
|
123
|
+
const proc = spawn(
|
|
124
|
+
"python3",
|
|
125
|
+
[lyriaRecipe, "--output", abs, "--duration", String(targetS), "--prompt", prompt],
|
|
126
|
+
{ detached: true, stdio: ["ignore", fd, fd] },
|
|
127
|
+
);
|
|
128
|
+
proc.unref();
|
|
129
|
+
closeSync(fd);
|
|
130
|
+
return {
|
|
131
|
+
...baseMeta,
|
|
132
|
+
mode: "detached-single",
|
|
133
|
+
provider: "lyria",
|
|
134
|
+
pid: proc.pid,
|
|
135
|
+
log,
|
|
136
|
+
target_duration_s: r3(targetS),
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
if (pyOk(BGM_PY_PROBE)) {
|
|
141
|
+
const seedS = Math.min(Math.max(seedSeconds, 10), 30);
|
|
142
|
+
const loops = targetS > seedS ? Math.ceil(targetS / seedS) : 1;
|
|
143
|
+
const script = musicgenScript({ prompt, abs, targetS, seedS });
|
|
144
|
+
const proc = spawn("python3", ["-c", script], { detached: true, stdio: ["ignore", fd, fd] });
|
|
145
|
+
proc.unref();
|
|
146
|
+
closeSync(fd);
|
|
147
|
+
return {
|
|
148
|
+
...baseMeta,
|
|
149
|
+
mode: targetS > seedS ? "detached-seed-loop" : "detached-seed-trim",
|
|
150
|
+
provider: "musicgen",
|
|
151
|
+
pid: proc.pid,
|
|
152
|
+
log,
|
|
153
|
+
target_duration_s: r3(targetS),
|
|
154
|
+
seed_duration_s: seedS,
|
|
155
|
+
loop_count: loops,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
closeSync(fd);
|
|
160
|
+
return {
|
|
161
|
+
disabled: true,
|
|
162
|
+
reason: lyriaConfigured
|
|
163
|
+
? `Lyria configured but google-genai uninstallable, and local MusicGen unavailable (pip install ${BGM_PY_DEPS.join(" ")})`
|
|
164
|
+
: `no Lyria key/recipe and local MusicGen deps unavailable (pip install ${BGM_PY_DEPS.join(" ")})`,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// Inline MusicGen: generate ONE seed clip (≤30s to stay under the decoder's
|
|
169
|
+
// positional limit), then trim it down or crossfade-loop it up to the target.
|
|
170
|
+
function musicgenScript({ prompt, abs, targetS, seedS }) {
|
|
171
|
+
return `
|
|
172
|
+
import math, os, sys, traceback
|
|
173
|
+
from pathlib import Path
|
|
174
|
+
import numpy as np
|
|
175
|
+
import soundfile as sf
|
|
176
|
+
from transformers import MusicgenForConditionalGeneration, AutoProcessor
|
|
177
|
+
|
|
178
|
+
prompt = ${JSON.stringify(prompt)}
|
|
179
|
+
out_path = ${JSON.stringify(abs)}
|
|
180
|
+
target_s = float(${targetS.toFixed(3)})
|
|
181
|
+
seed_s = float(${seedS.toFixed(3)})
|
|
182
|
+
token_rate = 50
|
|
183
|
+
crossfade_s = 0.3
|
|
184
|
+
|
|
185
|
+
def apply_fade(arr, sr, fade_in_s=0.08, fade_out_s=0.5):
|
|
186
|
+
n_in = min(int(round(fade_in_s * sr)), arr.shape[0] // 2)
|
|
187
|
+
n_out = min(int(round(fade_out_s * sr)), arr.shape[0] // 2)
|
|
188
|
+
if n_in > 1: arr[:n_in] *= np.linspace(0.0, 1.0, n_in, dtype="float32")
|
|
189
|
+
if n_out > 1: arr[-n_out:] *= np.linspace(1.0, 0.0, n_out, dtype="float32")
|
|
190
|
+
return arr
|
|
191
|
+
|
|
192
|
+
def loop_crossfade(seed, target_len, xf):
|
|
193
|
+
if seed.shape[0] >= target_len: return seed[:target_len]
|
|
194
|
+
xf = min(xf, seed.shape[0] // 2)
|
|
195
|
+
if xf < 1:
|
|
196
|
+
reps = int(math.ceil(target_len / seed.shape[0]))
|
|
197
|
+
return np.tile(seed, reps)[:target_len]
|
|
198
|
+
t = np.linspace(0.0, 1.0, xf, dtype="float32")
|
|
199
|
+
fade_out = np.cos(t * (math.pi / 2)); fade_in = np.sin(t * (math.pi / 2))
|
|
200
|
+
out = seed.copy()
|
|
201
|
+
while out.shape[0] < target_len:
|
|
202
|
+
tail = out[-xf:] * fade_out; head = seed[:xf] * fade_in
|
|
203
|
+
out = np.concatenate([out[:-xf], tail + head, seed[xf:]])
|
|
204
|
+
return out[:target_len]
|
|
205
|
+
|
|
206
|
+
try:
|
|
207
|
+
Path(os.path.dirname(out_path)).mkdir(parents=True, exist_ok=True)
|
|
208
|
+
processor = AutoProcessor.from_pretrained("facebook/musicgen-small")
|
|
209
|
+
model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")
|
|
210
|
+
model.eval()
|
|
211
|
+
sr = int(model.config.audio_encoder.sampling_rate)
|
|
212
|
+
gen_s = min(seed_s, target_s)
|
|
213
|
+
tokens = max(1, int(math.ceil(gen_s * token_rate)))
|
|
214
|
+
print(f"[musicgen] seed dur={gen_s:.2f}s tokens={tokens}", flush=True)
|
|
215
|
+
inputs = processor(text=[prompt], padding=True, return_tensors="pt")
|
|
216
|
+
audio = model.generate(**inputs, max_new_tokens=tokens)
|
|
217
|
+
seed = audio[0, 0].detach().cpu().numpy().astype("float32")
|
|
218
|
+
peak = float(np.max(np.abs(seed)))
|
|
219
|
+
if peak > 1e-6: seed = seed * (0.89 / peak)
|
|
220
|
+
want = max(1, int(round(target_s * sr)))
|
|
221
|
+
if seed.shape[0] >= want:
|
|
222
|
+
final = seed[:want].copy()
|
|
223
|
+
else:
|
|
224
|
+
final = loop_crossfade(seed, want, int(round(crossfade_s * sr)))
|
|
225
|
+
if final.shape[0] < want: final = np.pad(final, (0, want - final.shape[0]))
|
|
226
|
+
else: final = final[:want]
|
|
227
|
+
final = apply_fade(final, sr)
|
|
228
|
+
peak = float(np.max(np.abs(final)))
|
|
229
|
+
if peak > 1.0: final = final / peak
|
|
230
|
+
sf.write(out_path, final, sr)
|
|
231
|
+
print(f"[musicgen] wrote {out_path} samples={final.shape[0]} sr={sr}", flush=True)
|
|
232
|
+
except Exception:
|
|
233
|
+
traceback.print_exc(); sys.exit(1)
|
|
234
|
+
`;
|
|
235
|
+
}
|