zilmate 1.9.9 → 1.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ad-creative/SKILL.md +362 -0
- package/.agents/skills/ad-creative/evals/evals.json +90 -0
- package/.agents/skills/ad-creative/references/generative-tools.md +637 -0
- package/.agents/skills/ad-creative/references/platform-specs.md +213 -0
- package/.agents/skills/ai-seo/SKILL.md +489 -0
- package/.agents/skills/ai-seo/evals/evals.json +90 -0
- package/.agents/skills/ai-seo/references/content-patterns.md +285 -0
- package/.agents/skills/ai-seo/references/content-types.md +71 -0
- package/.agents/skills/ai-seo/references/okf.md +104 -0
- package/.agents/skills/ai-seo/references/platform-ranking-factors.md +154 -0
- package/.agents/skills/clerk-backend-api/SKILL.md +426 -0
- package/.agents/skills/clerk-backend-api/evals/evals.json +87 -0
- package/.agents/skills/clerk-backend-api/scripts/api-specs-context.sh +30 -0
- package/.agents/skills/clerk-backend-api/scripts/execute-request.sh +88 -0
- package/.agents/skills/clerk-backend-api/scripts/extract-endpoint-detail.sh +165 -0
- package/.agents/skills/clerk-backend-api/scripts/extract-tag-endpoints.sh +208 -0
- package/.agents/skills/clerk-backend-api/scripts/extract-tags.js +14 -0
- package/.agents/skills/copy-editing/SKILL.md +457 -0
- package/.agents/skills/copy-editing/evals/evals.json +89 -0
- package/.agents/skills/copy-editing/references/checklist.md +66 -0
- package/.agents/skills/copy-editing/references/content-refresh.md +38 -0
- package/.agents/skills/copy-editing/references/plain-english-alternatives.md +394 -0
- package/.agents/skills/hyperframes/SKILL.md +163 -0
- package/.agents/skills/hyperframes-cli/SKILL.md +109 -0
- package/.agents/skills/hyperframes-cli/references/doctor-browser.md +45 -0
- package/.agents/skills/hyperframes-cli/references/init-and-scaffold.md +51 -0
- package/.agents/skills/hyperframes-cli/references/lambda.md +132 -0
- package/.agents/skills/hyperframes-cli/references/lint-validate-inspect.md +121 -0
- package/.agents/skills/hyperframes-cli/references/preview-render.md +109 -0
- package/.agents/skills/hyperframes-cli/references/upgrade-info-misc.md +75 -0
- package/.agents/skills/hyperframes-media/SKILL.md +97 -0
- package/.agents/skills/hyperframes-media/assets/sfx/CREDITS.md +35 -0
- package/.agents/skills/hyperframes-media/assets/sfx/manifest.json +97 -0
- package/.agents/skills/hyperframes-media/references/bgm.md +72 -0
- package/.agents/skills/hyperframes-media/references/captions/authoring.md +159 -0
- package/.agents/skills/hyperframes-media/references/captions/motion.md +87 -0
- package/.agents/skills/hyperframes-media/references/captions/transcript-handling.md +97 -0
- package/.agents/skills/hyperframes-media/references/remove-background.md +143 -0
- package/.agents/skills/hyperframes-media/references/requirements.md +29 -0
- package/.agents/skills/hyperframes-media/references/sfx.md +42 -0
- package/.agents/skills/hyperframes-media/references/transcribe.md +52 -0
- package/.agents/skills/hyperframes-media/references/tts-to-captions.md +24 -0
- package/.agents/skills/hyperframes-media/references/tts.md +135 -0
- package/.agents/skills/hyperframes-media/scripts/audio.mjs +282 -0
- package/.agents/skills/hyperframes-media/scripts/heygen-tts.mjs +121 -0
- package/.agents/skills/hyperframes-media/scripts/lib/bgm.mjs +235 -0
- package/.agents/skills/hyperframes-media/scripts/lib/heygen.mjs +131 -0
- package/.agents/skills/hyperframes-media/scripts/lib/sfx.mjs +128 -0
- package/.agents/skills/hyperframes-media/scripts/lib/tts.mjs +220 -0
- package/.agents/skills/hyperframes-media/scripts/lyria-recipe.py +128 -0
- package/.agents/skills/hyperframes-media/scripts/wait-bgm.mjs +167 -0
- package/.agents/skills/hyperframes-registry/SKILL.md +101 -0
- package/.agents/skills/hyperframes-registry/examples/add-block.md +51 -0
- package/.agents/skills/hyperframes-registry/examples/add-component.md +73 -0
- package/.agents/skills/hyperframes-registry/references/contributing.md +166 -0
- package/.agents/skills/hyperframes-registry/references/demo-html-pattern.md +54 -0
- package/.agents/skills/hyperframes-registry/references/discovery.md +215 -0
- package/.agents/skills/hyperframes-registry/references/install-locations.md +45 -0
- package/.agents/skills/hyperframes-registry/references/templates.md +417 -0
- package/.agents/skills/hyperframes-registry/references/wiring-blocks.md +61 -0
- package/.agents/skills/hyperframes-registry/references/wiring-components.md +39 -0
- package/.agents/skills/marketing-psychology/SKILL.md +455 -0
- package/.agents/skills/marketing-psychology/evals/evals.json +88 -0
- package/.agents/skills/mintlify/SKILL.md +328 -0
- package/.agents/skills/product-launch-video/SKILL.md +206 -0
- package/.agents/skills/product-launch-video/references/cut-catalog.md +220 -0
- package/.agents/skills/product-launch-video/references/motion-language.md +156 -0
- package/.agents/skills/product-launch-video/references/story-design.md +373 -0
- package/.agents/skills/product-launch-video/references/visual-design.md +126 -0
- package/.agents/skills/product-launch-video/scripts/assemble-index.mjs +570 -0
- package/.agents/skills/product-launch-video/scripts/audio.mjs +253 -0
- package/.agents/skills/product-launch-video/scripts/build-frame.mjs +536 -0
- package/.agents/skills/product-launch-video/scripts/captions.mjs +508 -0
- package/.agents/skills/product-launch-video/scripts/lib/assets.mjs +55 -0
- package/.agents/skills/product-launch-video/scripts/lib/dimensions.mjs +45 -0
- package/.agents/skills/product-launch-video/scripts/lib/storyboard.mjs +249 -0
- package/.agents/skills/product-launch-video/scripts/lib/tokens.mjs +204 -0
- package/.agents/skills/product-launch-video/scripts/lib/transition-registry.mjs +38 -0
- package/.agents/skills/product-launch-video/scripts/lib/transitions.json +71 -0
- package/.agents/skills/product-launch-video/scripts/stage-assets.mjs +39 -0
- package/.agents/skills/product-launch-video/scripts/transitions.mjs +320 -0
- package/.agents/skills/product-launch-video/sub-agents/frame-worker.md +74 -0
- package/.agents/skills/prompt-engineering-patterns/SKILL.md +144 -0
- package/.agents/skills/prompt-engineering-patterns/assets/few-shot-examples.json +106 -0
- package/.agents/skills/prompt-engineering-patterns/assets/prompt-template-library.md +264 -0
- package/.agents/skills/prompt-engineering-patterns/references/chain-of-thought.md +412 -0
- package/.agents/skills/prompt-engineering-patterns/references/details.md +338 -0
- package/.agents/skills/prompt-engineering-patterns/references/few-shot-learning.md +386 -0
- package/.agents/skills/prompt-engineering-patterns/references/prompt-optimization.md +428 -0
- package/.agents/skills/prompt-engineering-patterns/references/prompt-templates.md +484 -0
- package/.agents/skills/prompt-engineering-patterns/references/system-prompts.md +195 -0
- package/.agents/skills/prompt-engineering-patterns/scripts/optimize-prompt.py +279 -0
- package/.agents/skills/remotion-to-hyperframes/SKILL.md +127 -0
- package/.agents/skills/remotion-to-hyperframes/assets/.gitkeep +0 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/run.sh +249 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/README.md +41 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/expected.json +26 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/hf-src/index.html +61 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/remotion-src/package.json +14 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/remotion-src/remotion.config.ts +13 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/remotion-src/src/Root.tsx +13 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/remotion-src/src/TitleCard.tsx +34 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/remotion-src/src/index.ts +4 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-1-title-card/remotion-src/tsconfig.json +15 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/README.md +54 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/expected.json +29 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/hf-src/index.html +118 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/remotion-src/package.json +14 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/remotion-src/remotion.config.ts +13 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/remotion-src/src/MultiScene.tsx +105 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/remotion-src/src/Root.tsx +13 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/remotion-src/src/index.ts +4 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/remotion-src/tsconfig.json +15 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-2-multi-scene/setup.sh +34 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/README.md +85 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/expected.json +41 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/hf-src/index.html +269 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/package.json +15 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/remotion.config.ts +13 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/Root.tsx +27 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/Stargazed.tsx +37 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/components/AnimatedNumber.tsx +23 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/components/StatCard.tsx +59 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/components/UnderlinedText.tsx +47 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/index.ts +4 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/scenes/OutroScene.tsx +18 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/scenes/StatsScene.tsx +34 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/src/scenes/TitleScene.tsx +55 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-3-data-driven/remotion-src/tsconfig.json +15 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/README.md +51 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/01-use-state.tsx +29 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/02-use-effect-deps.tsx +29 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/03-async-metadata.tsx +39 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/04-third-party-react.tsx +30 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/05-lambda-config.tsx +38 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/06-warnings-only.tsx +36 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/07-custom-hook.tsx +28 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/cases/08-mixed.tsx +41 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/expected.json +85 -0
- package/.agents/skills/remotion-to-hyperframes/assets/test-corpus/tier-4-escape-hatch/validate.sh +110 -0
- package/.agents/skills/remotion-to-hyperframes/references/api-map.md +142 -0
- package/.agents/skills/remotion-to-hyperframes/references/escape-hatch.md +115 -0
- package/.agents/skills/remotion-to-hyperframes/references/eval.md +140 -0
- package/.agents/skills/remotion-to-hyperframes/references/fonts.md +112 -0
- package/.agents/skills/remotion-to-hyperframes/references/limitations.md +136 -0
- package/.agents/skills/remotion-to-hyperframes/references/lottie.md +121 -0
- package/.agents/skills/remotion-to-hyperframes/references/media.md +149 -0
- package/.agents/skills/remotion-to-hyperframes/references/parameters.md +167 -0
- package/.agents/skills/remotion-to-hyperframes/references/sequencing.md +195 -0
- package/.agents/skills/remotion-to-hyperframes/references/timing.md +165 -0
- package/.agents/skills/remotion-to-hyperframes/references/transitions.md +114 -0
- package/.agents/skills/remotion-to-hyperframes/scripts/.gitkeep +0 -0
- package/.agents/skills/remotion-to-hyperframes/scripts/frame_strip.sh +107 -0
- package/.agents/skills/remotion-to-hyperframes/scripts/lint_source.py +358 -0
- package/.agents/skills/remotion-to-hyperframes/scripts/render_diff.sh +103 -0
- package/.agents/skills/remotion-to-hyperframes/scripts/tests/fixtures/blocker.tsx +50 -0
- package/.agents/skills/remotion-to-hyperframes/scripts/tests/fixtures/clean.tsx +46 -0
- package/.agents/skills/remotion-to-hyperframes/scripts/tests/smoke.sh +90 -0
- package/.agents/skills/seo-audit/SKILL.md +497 -0
- package/.agents/skills/seo-audit/evals/evals.json +136 -0
- package/.agents/skills/seo-audit/references/ai-writing-detection.md +200 -0
- package/.agents/skills/seo-audit/references/international-seo.md +230 -0
- package/README.md +4 -2
- package/SDK.md +64 -1
- package/dist/agents/coding.agent.d.ts +558 -0
- package/dist/agents/coding.agent.d.ts.map +1 -1
- package/dist/agents/coding.agent.js +8 -0
- package/dist/agents/coding.agent.js.map +1 -1
- package/dist/agents/developer-helper.agent.d.ts +712 -0
- package/dist/agents/developer-helper.agent.d.ts.map +1 -1
- package/dist/agents/developer-helper.agent.js +6 -0
- package/dist/agents/developer-helper.agent.js.map +1 -1
- package/dist/agents/docs-research.agent.d.ts +8 -0
- package/dist/agents/docs-research.agent.d.ts.map +1 -1
- package/dist/agents/finance.agent.d.ts +8 -0
- package/dist/agents/finance.agent.d.ts.map +1 -1
- package/dist/agents/manager.d.ts +837 -2
- package/dist/agents/manager.d.ts.map +1 -1
- package/dist/agents/manager.js +28 -0
- package/dist/agents/manager.js.map +1 -1
- package/dist/agents/security.agent.d.ts +165 -3
- package/dist/agents/security.agent.d.ts.map +1 -1
- package/dist/agents/security.agent.js +3 -0
- package/dist/agents/security.agent.js.map +1 -1
- package/dist/agents/swarm/main.js +1 -1
- package/dist/agents/swarm/main.js.map +1 -1
- package/dist/agents/swarm/registry.d.ts.map +1 -1
- package/dist/agents/swarm/registry.js +232 -26
- package/dist/agents/swarm/registry.js.map +1 -1
- package/dist/cli/doctor.d.ts +14 -0
- package/dist/cli/doctor.d.ts.map +1 -1
- package/dist/cli/doctor.js +91 -0
- package/dist/cli/doctor.js.map +1 -1
- package/dist/cli/menu.d.ts.map +1 -1
- package/dist/cli/menu.js +102 -0
- package/dist/cli/menu.js.map +1 -1
- package/dist/cli/setup.d.ts +10 -0
- package/dist/cli/setup.d.ts.map +1 -1
- package/dist/cli/setup.js +178 -0
- package/dist/cli/setup.js.map +1 -1
- package/dist/cli/swarm.d.ts +1 -0
- package/dist/cli/swarm.d.ts.map +1 -1
- package/dist/cli/swarm.js +47 -8
- package/dist/cli/swarm.js.map +1 -1
- package/dist/cli/theme.d.ts +0 -24
- package/dist/cli/theme.d.ts.map +1 -1
- package/dist/cli/theme.js +6 -2
- package/dist/cli/theme.js.map +1 -1
- package/dist/config/env.d.ts +10 -0
- package/dist/config/env.d.ts.map +1 -1
- package/dist/config/env.js +38 -17
- package/dist/config/env.js.map +1 -1
- package/dist/index.js +96 -5
- package/dist/index.js.map +1 -1
- package/dist/jobs/trigger-orchestrator.d.ts +5 -1
- package/dist/jobs/trigger-orchestrator.d.ts.map +1 -1
- package/dist/jobs/trigger-orchestrator.js +29 -9
- package/dist/jobs/trigger-orchestrator.js.map +1 -1
- package/dist/jobs/trigger-router.d.ts.map +1 -1
- package/dist/jobs/trigger-router.js +3 -2
- package/dist/jobs/trigger-router.js.map +1 -1
- package/dist/memory/corporate-wiki.d.ts +15 -0
- package/dist/memory/corporate-wiki.d.ts.map +1 -0
- package/dist/memory/corporate-wiki.js +152 -0
- package/dist/memory/corporate-wiki.js.map +1 -0
- package/dist/memory/scratchpad.d.ts +2 -0
- package/dist/memory/scratchpad.d.ts.map +1 -1
- package/dist/memory/scratchpad.js +27 -0
- package/dist/memory/scratchpad.js.map +1 -1
- package/dist/observability/doctor.d.ts.map +1 -1
- package/dist/observability/doctor.js +12 -4
- package/dist/observability/doctor.js.map +1 -1
- package/dist/observability/traces.d.ts +72 -0
- package/dist/observability/traces.d.ts.map +1 -0
- package/dist/observability/traces.js +1134 -0
- package/dist/observability/traces.js.map +1 -0
- package/dist/runtime/swarm.d.ts +1 -1
- package/dist/runtime/swarm.d.ts.map +1 -1
- package/dist/runtime/swarm.js +54 -25
- package/dist/runtime/swarm.js.map +1 -1
- package/dist/safety/approvals.d.ts +2 -2
- package/dist/safety/approvals.d.ts.map +1 -1
- package/dist/safety/approvals.js +47 -1
- package/dist/safety/approvals.js.map +1 -1
- package/dist/safety/redaction.d.ts.map +1 -1
- package/dist/safety/redaction.js +10 -0
- package/dist/safety/redaction.js.map +1 -1
- package/dist/tools/cloud.tool.d.ts +394 -0
- package/dist/tools/cloud.tool.d.ts.map +1 -0
- package/dist/tools/cloud.tool.js +784 -0
- package/dist/tools/cloud.tool.js.map +1 -0
- package/dist/tools/computer-use.tool.d.ts +2 -2
- package/dist/tools/corporate-wiki.tool.d.ts +22 -0
- package/dist/tools/corporate-wiki.tool.d.ts.map +1 -0
- package/dist/tools/corporate-wiki.tool.js +43 -0
- package/dist/tools/corporate-wiki.tool.js.map +1 -0
- package/dist/tools/devops.tool.d.ts +162 -0
- package/dist/tools/devops.tool.d.ts.map +1 -0
- package/dist/tools/devops.tool.js +377 -0
- package/dist/tools/devops.tool.js.map +1 -0
- package/dist/tools/multimedia.tool.d.ts +126 -0
- package/dist/tools/multimedia.tool.d.ts.map +1 -0
- package/dist/tools/multimedia.tool.js +533 -0
- package/dist/tools/multimedia.tool.js.map +1 -0
- package/dist/tools/osint.tool.d.ts +6 -6
- package/dist/tools/sandbox-dev.tool.d.ts +15 -0
- package/dist/tools/sandbox-dev.tool.d.ts.map +1 -0
- package/dist/tools/sandbox-dev.tool.js +146 -0
- package/dist/tools/sandbox-dev.tool.js.map +1 -0
- package/dist/tools/scratchpad.tool.d.ts +8 -0
- package/dist/tools/scratchpad.tool.d.ts.map +1 -1
- package/dist/tools/scratchpad.tool.js +40 -1
- package/dist/tools/scratchpad.tool.js.map +1 -1
- package/dist/tools/setup-assistant.tool.d.ts +1 -1
- package/dist/tools/swarm-ops.tool.d.ts +14 -0
- package/dist/tools/swarm-ops.tool.d.ts.map +1 -1
- package/dist/tools/swarm-ops.tool.js +47 -0
- package/dist/tools/swarm-ops.tool.js.map +1 -1
- package/dist/tools/sysops.tool.d.ts +157 -0
- package/dist/tools/sysops.tool.d.ts.map +1 -0
- package/dist/tools/sysops.tool.js +643 -0
- package/dist/tools/sysops.tool.js.map +1 -0
- package/dist/voice/deepgram.d.ts.map +1 -1
- package/dist/voice/deepgram.js +30 -2
- package/dist/voice/deepgram.js.map +1 -1
- package/package.json +8 -1
- package/scripts/release-github.mjs +23 -16
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
// heygen.mjs — vendored HeyGen REST helpers (auth + transport) for the audio
|
|
2
|
+
// pipeline. The credential resolver is copied from hyperframes-media's
|
|
3
|
+
// heygen-tts.mjs (and matches the hyperframes CLI auth): first usable source
|
|
4
|
+
// wins — $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY → a nearby .env → ~/.heygen/
|
|
5
|
+
// credentials (oauth → Bearer, else api_key → X-Api-Key; $HEYGEN_CONFIG_DIR
|
|
6
|
+
// overrides the dir). Vendored so the skill ships standalone. Pure node.
|
|
7
|
+
|
|
8
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
9
|
+
import { homedir } from "node:os";
|
|
10
|
+
import { dirname, join, resolve } from "node:path";
|
|
11
|
+
|
|
12
|
+
export const HEYGEN_BASE = "https://api.heygen.com/v3";
|
|
13
|
+
|
|
14
|
+
// Walk up ≤5 dirs from startDir; load the first .env (shell env always wins).
|
|
15
|
+
export function loadEnvFromDir(startDir) {
|
|
16
|
+
let dir = resolve(startDir);
|
|
17
|
+
for (let i = 0; i < 5; i++) {
|
|
18
|
+
const envPath = join(dir, ".env");
|
|
19
|
+
if (existsSync(envPath)) {
|
|
20
|
+
for (const raw of readFileSync(envPath, "utf8").split("\n")) {
|
|
21
|
+
let line = raw.trim();
|
|
22
|
+
if (!line || line.startsWith("#")) continue;
|
|
23
|
+
if (line.startsWith("export ")) line = line.slice(7).trim();
|
|
24
|
+
const eq = line.indexOf("=");
|
|
25
|
+
if (eq < 1) continue;
|
|
26
|
+
const key = line.slice(0, eq).trim();
|
|
27
|
+
let val = line.slice(eq + 1).trim();
|
|
28
|
+
if (val.startsWith('"') || val.startsWith("'")) {
|
|
29
|
+
const q = val[0];
|
|
30
|
+
const end = val.indexOf(q, 1);
|
|
31
|
+
val = end > 0 ? val.slice(1, end) : val.slice(1);
|
|
32
|
+
}
|
|
33
|
+
if (!(key in process.env)) process.env[key] = val;
|
|
34
|
+
}
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
const parent = dirname(dir);
|
|
38
|
+
if (parent === dir) break;
|
|
39
|
+
dir = parent;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// → { headers } | { expired: true } | null. Never throws.
|
|
44
|
+
export function heygenCredential() {
|
|
45
|
+
const envKey = process.env.HEYGEN_API_KEY || process.env.HYPERFRAMES_API_KEY;
|
|
46
|
+
if (envKey) return { headers: { "X-Api-Key": envKey } };
|
|
47
|
+
|
|
48
|
+
const file = join(process.env.HEYGEN_CONFIG_DIR || join(homedir(), ".heygen"), "credentials");
|
|
49
|
+
if (!existsSync(file)) return null;
|
|
50
|
+
const raw = readFileSync(file, "utf8").trim();
|
|
51
|
+
if (!raw) return null;
|
|
52
|
+
if (!raw.startsWith("{")) return { headers: { "X-Api-Key": raw } };
|
|
53
|
+
|
|
54
|
+
// A malformed credentials file (partial write / wrong shape) must degrade to
|
|
55
|
+
// "no credential", not crash the engine at startup — this function never throws.
|
|
56
|
+
let cred;
|
|
57
|
+
try {
|
|
58
|
+
cred = JSON.parse(raw);
|
|
59
|
+
} catch {
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
const oauth = cred.oauth;
|
|
63
|
+
if (oauth?.access_token) {
|
|
64
|
+
const expired = oauth.expires_at && new Date(oauth.expires_at).getTime() - 60_000 < Date.now();
|
|
65
|
+
if (!expired) return { headers: { Authorization: `Bearer ${oauth.access_token}` } };
|
|
66
|
+
if (!cred.api_key) return { expired: true };
|
|
67
|
+
}
|
|
68
|
+
if (cred.api_key) return { headers: { "X-Api-Key": cred.api_key } };
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// → auth headers object, or throw with a fix hint.
|
|
73
|
+
export function heygenAuthHeaders() {
|
|
74
|
+
const cred = heygenCredential();
|
|
75
|
+
if (cred?.headers) return cred.headers;
|
|
76
|
+
if (cred?.expired)
|
|
77
|
+
throw new Error(
|
|
78
|
+
"HeyGen OAuth token expired — run `npx hyperframes auth refresh` (or `npx hyperframes auth login`)",
|
|
79
|
+
);
|
|
80
|
+
throw new Error(
|
|
81
|
+
"no HeyGen credentials — set $HEYGEN_API_KEY, or run `npx hyperframes auth login` (writes ~/.heygen/credentials)",
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Authed JSON request against the v3 API; throws on a non-OK status.
|
|
86
|
+
export async function heygenJSON(path, { method = "GET", headers = {}, body } = {}) {
|
|
87
|
+
const opts = { method, headers: { ...headers } };
|
|
88
|
+
if (body !== undefined) {
|
|
89
|
+
opts.headers["Content-Type"] = "application/json";
|
|
90
|
+
opts.body = JSON.stringify(body);
|
|
91
|
+
}
|
|
92
|
+
const res = await fetch(`${HEYGEN_BASE}${path}`, opts);
|
|
93
|
+
if (!res.ok) {
|
|
94
|
+
const detail = await res.text().catch(() => "");
|
|
95
|
+
throw new Error(
|
|
96
|
+
`HeyGen ${method} ${path} → HTTP ${res.status}${detail ? `\n${detail.slice(0, 300)}` : ""}`,
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
return res.json();
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Download a (presigned) URL to destPath; returns byte length.
|
|
103
|
+
export async function downloadTo(url, destPath) {
|
|
104
|
+
const res = await fetch(url);
|
|
105
|
+
if (!res.ok) throw new Error(`download HTTP ${res.status}: ${String(url).slice(0, 80)}`);
|
|
106
|
+
const bytes = Buffer.from(await res.arrayBuffer());
|
|
107
|
+
mkdirSync(dirname(destPath), { recursive: true });
|
|
108
|
+
writeFileSync(destPath, bytes);
|
|
109
|
+
return bytes.length;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Retrieval search over HeyGen's audio catalog (NOT generation). type =
|
|
113
|
+
// "music" | "sound_effects". Returns the ranked results array (best first); each
|
|
114
|
+
// item has a presigned `audio_url` (+ `duration`, `description`, `name`, `score`).
|
|
115
|
+
// `query` is required (≥1 char, empty → HTTP 400) and `limit` is capped at 50.
|
|
116
|
+
// `minScore`: omit to use the server default (0.7). That default is TOO HIGH for
|
|
117
|
+
// sound_effects — good SFX hits score ~0.5–0.67, so callers wanting SFX should
|
|
118
|
+
// pass a lower floor (~0.4); music scores high and is fine at the default.
|
|
119
|
+
export async function searchSounds(query, type, headers, { limit = 5, minScore } = {}) {
|
|
120
|
+
const params = new URLSearchParams({ query, type, limit: String(limit) });
|
|
121
|
+
if (minScore != null) params.set("min_score", String(minScore));
|
|
122
|
+
const payload = await heygenJSON(`/audio/sounds?${params.toString()}`, { headers });
|
|
123
|
+
// `data` comes back as a ranked array (best first). Older responses keyed it by
|
|
124
|
+
// numeric index ("0","1",…); normalize both shapes to an array (empty → []).
|
|
125
|
+
const data = payload?.data ?? payload;
|
|
126
|
+
if (Array.isArray(data)) return data;
|
|
127
|
+
if (data && typeof data === "object") return Object.values(data);
|
|
128
|
+
throw new Error(
|
|
129
|
+
`unexpected /audio/sounds shape — top keys: ${Object.keys(payload ?? {}).join(", ")}`,
|
|
130
|
+
);
|
|
131
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
// sfx.mjs — sound effects for the media audio engine. Provider-gated (NOT a
|
|
2
|
+
// per-cue merge): the decision is made once, by whether HeyGen is configured —
|
|
3
|
+
// mirroring how TTS and BGM degrade.
|
|
4
|
+
//
|
|
5
|
+
// HeyGen credential present → retrieve EVERY cue from HeyGen's audio library
|
|
6
|
+
// (/v3/audio/sounds, type=sound_effects, min_score=0.4). The bundled
|
|
7
|
+
// library is NOT consulted.
|
|
8
|
+
// HeyGen credential absent → resolve cues against the bundled 21-file
|
|
9
|
+
// library (assets/sfx/manifest.json), copying matched files into the
|
|
10
|
+
// project. Offline, deterministic, free.
|
|
11
|
+
//
|
|
12
|
+
// A cue that matches nothing is skipped (recorded as an anomaly); SFX never
|
|
13
|
+
// blocks a render. Every cue sits at volume ~0.35, under voice + BGM.
|
|
14
|
+
|
|
15
|
+
import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { downloadTo, searchSounds } from "./heygen.mjs";
|
|
18
|
+
|
|
19
|
+
const SFX_VOLUME = 0.35;
|
|
20
|
+
const slug = (s) =>
|
|
21
|
+
s
|
|
22
|
+
.toLowerCase()
|
|
23
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
24
|
+
.replace(/^-+|-+$/g, "")
|
|
25
|
+
.slice(0, 40) || "x";
|
|
26
|
+
const r3 = (x) => Number(x.toFixed(3));
|
|
27
|
+
|
|
28
|
+
// cues: [{ id, name }] (id = the line/frame/scene the cue fires in). Returns
|
|
29
|
+
// { sfx: [{ id, name, file, source, offset_s, duration_s, volume }], anomalies }.
|
|
30
|
+
export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxLibDir }) {
|
|
31
|
+
const sfx = [];
|
|
32
|
+
const anomalies = [];
|
|
33
|
+
const destDir = join(hyperframesDir, "assets", "sfx");
|
|
34
|
+
|
|
35
|
+
// Dedupe identical (id,name) cues — the same effect named twice in one line
|
|
36
|
+
// downloads/copies once.
|
|
37
|
+
const seen = new Set();
|
|
38
|
+
const uniq = cues.filter((c) => {
|
|
39
|
+
const k = `${c.id}:${c.name}`;
|
|
40
|
+
if (seen.has(k)) return false;
|
|
41
|
+
seen.add(k);
|
|
42
|
+
return true;
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
if (heygenOK) {
|
|
46
|
+
for (const { id, name } of uniq) {
|
|
47
|
+
try {
|
|
48
|
+
// SFX hits score low (~0.5–0.67), below the API's default 0.7 which
|
|
49
|
+
// silently drops most named cues — floor to 0.4. (BGM/music score high
|
|
50
|
+
// and keep the default.)
|
|
51
|
+
const results = await searchSounds(name, "sound_effects", headers, {
|
|
52
|
+
limit: 3,
|
|
53
|
+
minScore: 0.4,
|
|
54
|
+
});
|
|
55
|
+
if (!results.length) {
|
|
56
|
+
anomalies.push(`sfx "${name}" (id ${id}): no HeyGen match — skipped`);
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
const top = results[0];
|
|
60
|
+
const file = `assets/sfx/${slug(name)}.mp3`;
|
|
61
|
+
await downloadTo(top.audio_url, join(hyperframesDir, file));
|
|
62
|
+
sfx.push({
|
|
63
|
+
id,
|
|
64
|
+
name,
|
|
65
|
+
file,
|
|
66
|
+
source: "heygen",
|
|
67
|
+
offset_s: 0,
|
|
68
|
+
duration_s: typeof top.duration === "number" ? r3(top.duration) : 1.0,
|
|
69
|
+
volume: SFX_VOLUME,
|
|
70
|
+
});
|
|
71
|
+
} catch (e) {
|
|
72
|
+
anomalies.push(`sfx "${name}" (id ${id}): retrieval failed — ${e.message}`);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return { sfx, anomalies };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// ── offline: bundled library ──
|
|
79
|
+
const manifestPath = join(sfxLibDir, "manifest.json");
|
|
80
|
+
if (!existsSync(manifestPath)) {
|
|
81
|
+
if (uniq.length)
|
|
82
|
+
anomalies.push(`no HeyGen credential and no SFX library at ${sfxLibDir} — all cues dropped`);
|
|
83
|
+
return { sfx, anomalies };
|
|
84
|
+
}
|
|
85
|
+
let manifest;
|
|
86
|
+
try {
|
|
87
|
+
manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
|
|
88
|
+
} catch (e) {
|
|
89
|
+
anomalies.push(`SFX manifest parse failed (${e.message}) — all cues dropped`);
|
|
90
|
+
return { sfx, anomalies };
|
|
91
|
+
}
|
|
92
|
+
// Build lookups: by manifest key, by file basename, and by slug of either, so
|
|
93
|
+
// a cue can name "whoosh", "whoosh.mp3", or "ui click" (→ slug match).
|
|
94
|
+
const byKey = new Map();
|
|
95
|
+
for (const [key, entry] of Object.entries(manifest)) {
|
|
96
|
+
if (!entry?.file || !isFinite(entry.duration)) continue;
|
|
97
|
+
const rec = { key, file: entry.file, duration: entry.duration };
|
|
98
|
+
byKey.set(key, rec);
|
|
99
|
+
byKey.set(entry.file, rec);
|
|
100
|
+
byKey.set(slug(key), rec);
|
|
101
|
+
byKey.set(slug(entry.file.replace(/\.\w+$/, "")), rec);
|
|
102
|
+
}
|
|
103
|
+
mkdirSync(destDir, { recursive: true });
|
|
104
|
+
for (const { id, name } of uniq) {
|
|
105
|
+
const hit = byKey.get(name) ?? byKey.get(slug(name));
|
|
106
|
+
if (!hit) {
|
|
107
|
+
const known = [...new Set([...byKey.values()].map((v) => v.key))].slice(0, 8).join(", ");
|
|
108
|
+
anomalies.push(
|
|
109
|
+
`sfx "${name}" (id ${id}): not in bundled library — skipped (have: ${known}…)`,
|
|
110
|
+
);
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
const src = join(sfxLibDir, hit.file);
|
|
114
|
+
const destRel = `assets/sfx/${hit.file}`;
|
|
115
|
+
const dest = join(hyperframesDir, destRel);
|
|
116
|
+
if (existsSync(src) && !existsSync(dest)) copyFileSync(src, dest);
|
|
117
|
+
sfx.push({
|
|
118
|
+
id,
|
|
119
|
+
name,
|
|
120
|
+
file: destRel,
|
|
121
|
+
source: "local",
|
|
122
|
+
offset_s: 0,
|
|
123
|
+
duration_s: r3(hit.duration),
|
|
124
|
+
volume: SFX_VOLUME,
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
return { sfx, anomalies };
|
|
128
|
+
}
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
// tts.mjs — multi-provider TTS for the media audio engine. The provider chain,
|
|
2
|
+
// auto-detected from env, is the one documented in ../SKILL.md:
|
|
3
|
+
//
|
|
4
|
+
// 1. HeyGen (Starfish) — $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY / ~/.heygen.
|
|
5
|
+
// Direct v3 REST (NOT `hyperframes tts`, which in the published build is
|
|
6
|
+
// Kokoro-only and silently ignores a HeyGen key). Returns word_timestamps
|
|
7
|
+
// in the same call, so no separate transcribe pass.
|
|
8
|
+
// 2. ElevenLabs — $ELEVENLABS_API_KEY + `pip install elevenlabs`. No
|
|
9
|
+
// word timings → caller chains transcribeWav().
|
|
10
|
+
// 3. Kokoro-82M (local) — always available, via the published `hyperframes tts`
|
|
11
|
+
// CLI. No word timings → caller chains transcribeWav().
|
|
12
|
+
//
|
|
13
|
+
// "HeyGen available" is decided by CREDENTIAL presence (heygenCredential), never
|
|
14
|
+
// by the CLI — see the note above.
|
|
15
|
+
|
|
16
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
17
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
18
|
+
import { tmpdir } from "node:os";
|
|
19
|
+
import { dirname, join } from "node:path";
|
|
20
|
+
import { heygenAuthHeaders, heygenCredential, heygenJSON } from "./heygen.mjs";
|
|
21
|
+
|
|
22
|
+
// ── provider detection ────────────────────────────────────────────────────────
|
|
23
|
+
export function heygenAvailable() {
|
|
24
|
+
return heygenCredential() !== null;
|
|
25
|
+
}
|
|
26
|
+
export function elevenlabsAvailable() {
|
|
27
|
+
if (!process.env.ELEVENLABS_API_KEY) return false;
|
|
28
|
+
const r = spawnSync("python3", ["-c", "import elevenlabs"], { stdio: "ignore" });
|
|
29
|
+
return r.status === 0;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// First available provider wins; an explicit choice is honored (and validated).
|
|
33
|
+
export function pickProvider(userProvider) {
|
|
34
|
+
if (userProvider) {
|
|
35
|
+
if (!["heygen", "elevenlabs", "kokoro"].includes(userProvider))
|
|
36
|
+
throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro)`);
|
|
37
|
+
if (userProvider === "heygen" && !heygenAvailable())
|
|
38
|
+
throw new Error(
|
|
39
|
+
"provider=heygen but no HeyGen credentials (set $HEYGEN_API_KEY or run `npx hyperframes auth login`)",
|
|
40
|
+
);
|
|
41
|
+
if (userProvider === "elevenlabs" && !process.env.ELEVENLABS_API_KEY)
|
|
42
|
+
throw new Error("provider=elevenlabs but $ELEVENLABS_API_KEY is not set");
|
|
43
|
+
return userProvider;
|
|
44
|
+
}
|
|
45
|
+
return heygenAvailable() ? "heygen" : elevenlabsAvailable() ? "elevenlabs" : "kokoro";
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// ── voice resolution ──────────────────────────────────────────────────────────
|
|
49
|
+
// HeyGen /v3/voices/speech only accepts STARFISH voice_ids; auto-pick the first
|
|
50
|
+
// English public starfish voice when none is pinned. ElevenLabs/Kokoro have
|
|
51
|
+
// their own defaults.
|
|
52
|
+
export async function resolveVoiceId({ provider, userVoice, lang = "en" }) {
|
|
53
|
+
if (userVoice) return userVoice;
|
|
54
|
+
if (provider === "elevenlabs") return "21m00Tcm4TlvDq8ikWAM"; // Rachel
|
|
55
|
+
if (provider === "kokoro") {
|
|
56
|
+
if (lang === "en") return "am_michael";
|
|
57
|
+
throw new Error("Kokoro non-English needs an explicit --voice (see references/tts.md)");
|
|
58
|
+
}
|
|
59
|
+
// heygen — pin a fixed English default so the choice is deterministic. The old
|
|
60
|
+
// "first English voice the API returns" drifts whenever HeyGen re-sorts the
|
|
61
|
+
// public catalog. Marcia (mature, low female). Override with --voice / request.voice.
|
|
62
|
+
if (lang === "en") return "05f19352e8f74b0392a8f411eba40de1"; // Marcia · English · female
|
|
63
|
+
// Non-English: no fixed default — fall back to the first matching catalog voice.
|
|
64
|
+
const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
|
|
65
|
+
headers: heygenAuthHeaders(),
|
|
66
|
+
});
|
|
67
|
+
const voices = payload.data ?? payload.voices ?? [];
|
|
68
|
+
const pick = voices.find((v) => v.language === "English") ?? voices[0];
|
|
69
|
+
if (!pick) throw new Error("no public starfish voice to default to — pass --voice");
|
|
70
|
+
return pick.voice_id;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// ── helpers ─────────────────────────────────────────────────────────────────
|
|
74
|
+
export function withWordIds(words) {
|
|
75
|
+
return (words ?? []).map((w, i) => ({ id: `w${i}`, text: w.text, start: w.start, end: w.end }));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function ffprobeDuration(absPath) {
|
|
79
|
+
const r = spawnSync(
|
|
80
|
+
"ffprobe",
|
|
81
|
+
["-v", "error", "-show_entries", "format=duration", "-of", "default=nw=1:nk=1", absPath],
|
|
82
|
+
{ encoding: "utf8" },
|
|
83
|
+
);
|
|
84
|
+
if (r.status !== 0) return NaN;
|
|
85
|
+
return parseFloat(String(r.stdout).trim());
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function spawnP(cmd, args, opts) {
|
|
89
|
+
return new Promise((resolve) => {
|
|
90
|
+
const p = spawn(cmd, args, { stdio: "ignore", ...opts });
|
|
91
|
+
p.on("exit", (code) => resolve({ status: code ?? -1 }));
|
|
92
|
+
p.on("error", () => resolve({ status: -1 }));
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// mp3/whatever bytes → wav 44.1k mono at destWav (ffmpeg detects true format).
|
|
97
|
+
function transcodeToWav(bytes, destWav) {
|
|
98
|
+
const td = mkdtempSync(join(tmpdir(), "hf-tts-"));
|
|
99
|
+
const tmp = join(td, "a.mp3");
|
|
100
|
+
writeFileSync(tmp, bytes);
|
|
101
|
+
mkdirSync(dirname(destWav), { recursive: true });
|
|
102
|
+
const ff = spawnSync(
|
|
103
|
+
"ffmpeg",
|
|
104
|
+
["-y", "-loglevel", "error", "-i", tmp, "-ar", "44100", "-ac", "1", destWav],
|
|
105
|
+
{ stdio: "ignore" },
|
|
106
|
+
);
|
|
107
|
+
rmSync(td, { recursive: true, force: true });
|
|
108
|
+
return ff.status === 0 && existsSync(destWav);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const ELEVENLABS_PY = `
|
|
112
|
+
import os, sys
|
|
113
|
+
from elevenlabs.client import ElevenLabs
|
|
114
|
+
from elevenlabs import save
|
|
115
|
+
client = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"])
|
|
116
|
+
text = open(sys.argv[1]).read()
|
|
117
|
+
audio = client.text_to_speech.convert(
|
|
118
|
+
text=text, voice_id=sys.argv[2],
|
|
119
|
+
model_id="eleven_multilingual_v2", output_format="mp3_44100_128",
|
|
120
|
+
)
|
|
121
|
+
save(audio, sys.argv[3])
|
|
122
|
+
`;
|
|
123
|
+
|
|
124
|
+
// ── synthesize one line ───────────────────────────────────────────────────────
|
|
125
|
+
// Writes wav at wavAbs. Returns { ok, words } — words is the raw
|
|
126
|
+
// [{text,start,end}] array for HeyGen (native), or null for ElevenLabs/Kokoro
|
|
127
|
+
// (caller must transcribeWav). Never throws; failures return { ok:false }.
|
|
128
|
+
export async function synthesizeOne({
|
|
129
|
+
provider,
|
|
130
|
+
text,
|
|
131
|
+
voiceId,
|
|
132
|
+
lang = "en",
|
|
133
|
+
speed = 1.0,
|
|
134
|
+
wavAbs,
|
|
135
|
+
hyperframesDir,
|
|
136
|
+
}) {
|
|
137
|
+
if (provider === "heygen") return synthesizeHeygen({ text, voiceId, lang, speed, wavAbs });
|
|
138
|
+
if (provider === "elevenlabs") {
|
|
139
|
+
const r = await spawnP(
|
|
140
|
+
"python3",
|
|
141
|
+
["-c", ELEVENLABS_PY, writeTmpText(text), voiceId, wavAbs],
|
|
142
|
+
{},
|
|
143
|
+
);
|
|
144
|
+
return { ok: r.status === 0 && existsSync(wavAbs), words: null };
|
|
145
|
+
}
|
|
146
|
+
// kokoro — via the published CLI; --output is relative to the project dir.
|
|
147
|
+
const wavRel = relTo(hyperframesDir, wavAbs);
|
|
148
|
+
const args = ["hyperframes", "tts", writeTmpText(text), "--voice", voiceId, "--output", wavRel];
|
|
149
|
+
if (lang !== "en") args.push("--lang", lang);
|
|
150
|
+
const r = await spawnP("npx", args, { cwd: hyperframesDir });
|
|
151
|
+
return { ok: r.status === 0 && existsSync(wavAbs), words: null };
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
async function synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }) {
|
|
155
|
+
try {
|
|
156
|
+
const body = { text, voice_id: voiceId, speed };
|
|
157
|
+
if (lang !== "en") body.language = lang;
|
|
158
|
+
const payload = await heygenJSON(`/voices/speech`, {
|
|
159
|
+
method: "POST",
|
|
160
|
+
headers: heygenAuthHeaders(),
|
|
161
|
+
body,
|
|
162
|
+
});
|
|
163
|
+
const inner = payload.data ?? payload;
|
|
164
|
+
if (!inner.audio_url) return { ok: false, words: null };
|
|
165
|
+
const res = await fetch(inner.audio_url);
|
|
166
|
+
if (!res.ok) return { ok: false, words: null };
|
|
167
|
+
const bytes = Buffer.from(await res.arrayBuffer());
|
|
168
|
+
// .wav output → transcode to 44.1k mono; .mp3 → raw bytes (no ffmpeg). The
|
|
169
|
+
// engine always asks for .wav; the standalone heygen-tts CLI may ask for .mp3.
|
|
170
|
+
if (wavAbs.endsWith(".wav")) {
|
|
171
|
+
if (!transcodeToWav(bytes, wavAbs)) return { ok: false, words: null };
|
|
172
|
+
} else {
|
|
173
|
+
mkdirSync(dirname(wavAbs), { recursive: true });
|
|
174
|
+
writeFileSync(wavAbs, bytes);
|
|
175
|
+
}
|
|
176
|
+
const words = Array.isArray(inner.word_timestamps)
|
|
177
|
+
? inner.word_timestamps
|
|
178
|
+
.filter((w) => w && typeof w.word === "string" && isFinite(w.start) && isFinite(w.end))
|
|
179
|
+
.filter((w) => !/^<.*>$/.test(w.word.trim())) // drop <start>/<end> sentinels
|
|
180
|
+
.map((w) => ({ text: w.word, start: w.start, end: w.end }))
|
|
181
|
+
: [];
|
|
182
|
+
return { ok: true, words };
|
|
183
|
+
} catch {
|
|
184
|
+
return { ok: false, words: null };
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// ElevenLabs/Kokoro have no word timings — run Whisper over the wav. Returns the
|
|
189
|
+
// flat [{id,text,start,end}] word array, or null. Each call uses a throwaway
|
|
190
|
+
// --dir so parallel scenes don't collide on transcript.json.
|
|
191
|
+
export async function transcribeWav({ wavRel, lang = "en", hyperframesDir }) {
|
|
192
|
+
const model = lang === "en" ? "small.en" : "small";
|
|
193
|
+
const td = mkdtempSync(join(tmpdir(), "hf-trans-"));
|
|
194
|
+
const args = ["hyperframes", "transcribe", wavRel, "--model", model, "--dir", td];
|
|
195
|
+
if (lang !== "en") args.push("--language", lang);
|
|
196
|
+
const r = await spawnP("npx", args, { cwd: hyperframesDir });
|
|
197
|
+
let words = null;
|
|
198
|
+
if (r.status === 0) {
|
|
199
|
+
const src = join(td, "transcript.json");
|
|
200
|
+
if (existsSync(src)) {
|
|
201
|
+
try {
|
|
202
|
+
const arr = JSON.parse(readFileSync(src, "utf8"));
|
|
203
|
+
if (Array.isArray(arr) && arr.length) words = arr;
|
|
204
|
+
} catch {}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
rmSync(td, { recursive: true, force: true });
|
|
208
|
+
return words;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// ── tiny local utils ──────────────────────────────────────────────────────────
|
|
212
|
+
function writeTmpText(text) {
|
|
213
|
+
const td = mkdtempSync(join(tmpdir(), "hf-txt-"));
|
|
214
|
+
const p = join(td, "line.txt");
|
|
215
|
+
writeFileSync(p, text);
|
|
216
|
+
return p;
|
|
217
|
+
}
|
|
218
|
+
function relTo(base, abs) {
|
|
219
|
+
return abs.startsWith(base + "/") ? abs.slice(base.length + 1) : abs;
|
|
220
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Generate BGM using Google Lyria RealTime API.
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
python lyria-recipe.py --output <path> --duration <seconds> [tuning flags]
|
|
6
|
+
|
|
7
|
+
Requires:
|
|
8
|
+
$GOOGLE_API_KEY or $GEMINI_API_KEY environment variable (treated as aliases).
|
|
9
|
+
pip install google-genai python-dotenv. audio.mjs Step 4b installs these on
|
|
10
|
+
demand when a key is set but google.genai is not importable; if that install
|
|
11
|
+
fails it falls back to local MusicGen rather than leaving the video with no BGM.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import argparse
|
|
17
|
+
import asyncio
|
|
18
|
+
import os
|
|
19
|
+
import sys
|
|
20
|
+
import wave
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
DEFAULT_PROMPT = "Uplifting corporate tech, bright and modern, gentle piano with synth pads"
|
|
24
|
+
SAMPLE_RATE = 48000
|
|
25
|
+
CHANNELS = 2
|
|
26
|
+
SAMPLE_WIDTH = 2 # 16-bit
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def parse_args() -> argparse.Namespace:
|
|
30
|
+
p = argparse.ArgumentParser(description="Generate BGM via Google Lyria RealTime.")
|
|
31
|
+
p.add_argument("--output", required=True, help="Output WAV path.")
|
|
32
|
+
p.add_argument("--duration", type=float, required=True, help="Target duration in seconds.")
|
|
33
|
+
p.add_argument("--prompt", default=DEFAULT_PROMPT, help="Mood / instrumentation prompt.")
|
|
34
|
+
p.add_argument("--negative-prompt", default=None, help="Styles to exclude (optional).")
|
|
35
|
+
p.add_argument("--bpm", type=int, default=110)
|
|
36
|
+
p.add_argument("--brightness", type=float, default=0.8, help="0-1, higher = brighter mood.")
|
|
37
|
+
p.add_argument("--density", type=float, default=0.5, help="0-1, higher = fuller mix.")
|
|
38
|
+
p.add_argument(
|
|
39
|
+
"--scale",
|
|
40
|
+
default="MAJOR",
|
|
41
|
+
help="MAJOR / MINOR / PENTATONIC / etc. — see google.genai.types.Scale. Pass empty string for none.",
|
|
42
|
+
)
|
|
43
|
+
return p.parse_args()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
async def generate_bgm(args: argparse.Namespace) -> dict:
|
|
47
|
+
from google import genai
|
|
48
|
+
from google.genai import types
|
|
49
|
+
|
|
50
|
+
api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY") or ""
|
|
51
|
+
if not api_key:
|
|
52
|
+
raise RuntimeError("Neither GOOGLE_API_KEY nor GEMINI_API_KEY is set.")
|
|
53
|
+
|
|
54
|
+
client = genai.Client(
|
|
55
|
+
api_key=api_key,
|
|
56
|
+
http_options={"api_version": "v1alpha"},
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
out_path = Path(args.output)
|
|
60
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
61
|
+
|
|
62
|
+
target_bytes = int(args.duration * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)
|
|
63
|
+
|
|
64
|
+
cfg: dict = {"bpm": args.bpm, "temperature": 1.0}
|
|
65
|
+
if args.density is not None:
|
|
66
|
+
cfg["density"] = args.density
|
|
67
|
+
if args.brightness is not None:
|
|
68
|
+
cfg["brightness"] = args.brightness
|
|
69
|
+
if args.scale:
|
|
70
|
+
scale_enum = getattr(types.Scale, args.scale, None)
|
|
71
|
+
if scale_enum:
|
|
72
|
+
cfg["scale"] = scale_enum
|
|
73
|
+
|
|
74
|
+
prompts = [types.WeightedPrompt(text=args.prompt, weight=1.0)]
|
|
75
|
+
if args.negative_prompt:
|
|
76
|
+
prompts.append(types.WeightedPrompt(text=args.negative_prompt, weight=-1.0))
|
|
77
|
+
|
|
78
|
+
buf = bytearray()
|
|
79
|
+
timeout = args.duration + 8
|
|
80
|
+
|
|
81
|
+
async with client.aio.live.music.connect(
|
|
82
|
+
model="models/lyria-realtime-exp",
|
|
83
|
+
) as session:
|
|
84
|
+
await session.set_weighted_prompts(prompts=prompts)
|
|
85
|
+
await session.set_music_generation_config(
|
|
86
|
+
config=types.LiveMusicGenerationConfig(**cfg),
|
|
87
|
+
)
|
|
88
|
+
await session.play()
|
|
89
|
+
|
|
90
|
+
async def collect():
|
|
91
|
+
while len(buf) < target_bytes:
|
|
92
|
+
async for msg in session.receive():
|
|
93
|
+
sc = msg.server_content
|
|
94
|
+
if sc and sc.audio_chunks:
|
|
95
|
+
for chunk in sc.audio_chunks:
|
|
96
|
+
buf.extend(chunk.data)
|
|
97
|
+
if len(buf) >= target_bytes:
|
|
98
|
+
return
|
|
99
|
+
await asyncio.sleep(1e-6)
|
|
100
|
+
|
|
101
|
+
try:
|
|
102
|
+
await asyncio.wait_for(collect(), timeout=timeout)
|
|
103
|
+
except TimeoutError:
|
|
104
|
+
print(f"Timeout after {timeout:.0f}s, collected {len(buf)} bytes", file=sys.stderr)
|
|
105
|
+
|
|
106
|
+
audio = bytes(buf[:target_bytes])
|
|
107
|
+
with wave.open(str(out_path), "wb") as wf:
|
|
108
|
+
wf.setnchannels(CHANNELS)
|
|
109
|
+
wf.setsampwidth(SAMPLE_WIDTH)
|
|
110
|
+
wf.setframerate(SAMPLE_RATE)
|
|
111
|
+
wf.writeframes(audio)
|
|
112
|
+
|
|
113
|
+
actual_duration = len(audio) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)
|
|
114
|
+
print(f"BGM: {out_path} ({actual_duration:.2f}s)")
|
|
115
|
+
return {"file": str(out_path), "duration_sec": round(actual_duration, 2)}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def main() -> None:
|
|
119
|
+
args = parse_args()
|
|
120
|
+
try:
|
|
121
|
+
asyncio.run(generate_bgm(args))
|
|
122
|
+
except RuntimeError as exc:
|
|
123
|
+
print(f"BGM generation failed: {exc}", file=sys.stderr)
|
|
124
|
+
sys.exit(1)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
if __name__ == "__main__":
|
|
128
|
+
main()
|