varg.ai-sdk 0.1.1 → 0.4.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +1 -1
- package/.env.example +3 -0
- package/.github/workflows/ci.yml +23 -0
- package/.husky/README.md +102 -0
- package/.husky/commit-msg +6 -0
- package/.husky/pre-commit +9 -0
- package/.husky/pre-push +6 -0
- package/.size-limit.json +8 -0
- package/.test-hooks.ts +5 -0
- package/CLAUDE.md +10 -3
- package/CONTRIBUTING.md +150 -0
- package/LICENSE.md +53 -0
- package/README.md +56 -209
- package/SKILLS.md +26 -10
- package/biome.json +7 -1
- package/bun.lock +1286 -0
- package/commitlint.config.js +22 -0
- package/docs/index.html +1130 -0
- package/docs/prompting.md +326 -0
- package/docs/react.md +834 -0
- package/docs/sdk.md +812 -0
- package/ffmpeg/CLAUDE.md +68 -0
- package/package.json +43 -10
- package/pipeline/cookbooks/scripts/animate-frames-parallel.ts +84 -0
- package/pipeline/cookbooks/scripts/combine-scenes.sh +53 -0
- package/pipeline/cookbooks/scripts/generate-frames-parallel.ts +99 -0
- package/pipeline/cookbooks/scripts/still-to-video.sh +37 -0
- package/pipeline/cookbooks/text-to-tiktok.md +669 -0
- package/pipeline/cookbooks/trendwatching.md +156 -0
- package/plan.md +281 -0
- package/scripts/.gitkeep +0 -0
- package/src/ai-sdk/cache.ts +142 -0
- package/src/ai-sdk/examples/cached-generation.ts +53 -0
- package/src/ai-sdk/examples/duet-scene-4.ts +53 -0
- package/src/ai-sdk/examples/duet-scene-5-audio.ts +32 -0
- package/src/ai-sdk/examples/duet-video.ts +56 -0
- package/src/ai-sdk/examples/editly-composition.ts +63 -0
- package/src/ai-sdk/examples/editly-test.ts +57 -0
- package/src/ai-sdk/examples/editly-video-test.ts +52 -0
- package/src/ai-sdk/examples/fal-lipsync.ts +43 -0
- package/src/ai-sdk/examples/higgsfield-image.ts +61 -0
- package/src/ai-sdk/examples/music-generation.ts +19 -0
- package/src/ai-sdk/examples/openai-sora.ts +34 -0
- package/src/ai-sdk/examples/replicate-bg-removal.ts +52 -0
- package/src/ai-sdk/examples/simpsons-scene.ts +61 -0
- package/src/ai-sdk/examples/talking-lion.ts +55 -0
- package/src/ai-sdk/examples/video-generation.ts +39 -0
- package/src/ai-sdk/examples/workflow-animated-girl.ts +104 -0
- package/src/ai-sdk/examples/workflow-before-after.ts +114 -0
- package/src/ai-sdk/examples/workflow-character-grid.ts +112 -0
- package/src/ai-sdk/examples/workflow-slideshow.ts +161 -0
- package/src/ai-sdk/file-cache.ts +112 -0
- package/src/ai-sdk/file.ts +238 -0
- package/src/ai-sdk/generate-element.ts +92 -0
- package/src/ai-sdk/generate-music.ts +46 -0
- package/src/ai-sdk/generate-video.ts +165 -0
- package/src/ai-sdk/index.ts +72 -0
- package/src/ai-sdk/music-model.ts +110 -0
- package/src/ai-sdk/providers/editly/editly.test.ts +1108 -0
- package/src/ai-sdk/providers/editly/ffmpeg.ts +60 -0
- package/src/ai-sdk/providers/editly/index.ts +817 -0
- package/src/ai-sdk/providers/editly/layers.ts +776 -0
- package/src/ai-sdk/providers/editly/plan.md +144 -0
- package/src/ai-sdk/providers/editly/types.ts +328 -0
- package/src/ai-sdk/providers/elevenlabs-provider.ts +255 -0
- package/src/ai-sdk/providers/fal-provider.ts +512 -0
- package/src/ai-sdk/providers/higgsfield.ts +379 -0
- package/src/ai-sdk/providers/openai.ts +251 -0
- package/src/ai-sdk/providers/replicate.ts +16 -0
- package/src/ai-sdk/video-model.ts +185 -0
- package/src/cli/commands/find.tsx +137 -0
- package/src/cli/commands/help.tsx +85 -0
- package/src/cli/commands/index.ts +6 -0
- package/src/cli/commands/list.tsx +238 -0
- package/src/cli/commands/render.tsx +71 -0
- package/src/cli/commands/run.tsx +511 -0
- package/src/cli/commands/which.tsx +253 -0
- package/src/cli/index.ts +114 -0
- package/src/cli/quiet.ts +44 -0
- package/src/cli/types.ts +32 -0
- package/src/cli/ui/components/Badge.tsx +29 -0
- package/src/cli/ui/components/DataTable.tsx +51 -0
- package/src/cli/ui/components/Header.tsx +23 -0
- package/src/cli/ui/components/HelpBlock.tsx +44 -0
- package/src/cli/ui/components/KeyValue.tsx +33 -0
- package/src/cli/ui/components/OptionRow.tsx +81 -0
- package/src/cli/ui/components/Separator.tsx +23 -0
- package/src/cli/ui/components/StatusBox.tsx +108 -0
- package/src/cli/ui/components/VargBox.tsx +51 -0
- package/src/cli/ui/components/VargProgress.tsx +36 -0
- package/src/cli/ui/components/VargSpinner.tsx +34 -0
- package/src/cli/ui/components/VargText.tsx +56 -0
- package/src/cli/ui/components/index.ts +19 -0
- package/src/cli/ui/index.ts +12 -0
- package/src/cli/ui/render.ts +35 -0
- package/src/cli/ui/theme.ts +63 -0
- package/src/cli/utils.ts +78 -0
- package/src/core/executor/executor.ts +201 -0
- package/src/core/executor/index.ts +13 -0
- package/src/core/executor/job.ts +214 -0
- package/src/core/executor/pipeline.ts +222 -0
- package/src/core/index.ts +11 -0
- package/src/core/registry/index.ts +9 -0
- package/src/core/registry/loader.ts +149 -0
- package/src/core/registry/registry.ts +221 -0
- package/src/core/registry/resolver.ts +206 -0
- package/src/core/schema/helpers.ts +134 -0
- package/src/core/schema/index.ts +8 -0
- package/src/core/schema/shared.ts +102 -0
- package/src/core/schema/types.ts +279 -0
- package/src/core/schema/validator.ts +92 -0
- package/src/definitions/actions/captions.ts +261 -0
- package/src/definitions/actions/edit.ts +298 -0
- package/src/definitions/actions/image.ts +125 -0
- package/src/definitions/actions/index.ts +114 -0
- package/src/definitions/actions/music.ts +205 -0
- package/src/definitions/actions/sync.ts +128 -0
- package/{action/transcribe/index.ts → src/definitions/actions/transcribe.ts} +58 -68
- package/src/definitions/actions/upload.ts +111 -0
- package/src/definitions/actions/video.ts +163 -0
- package/src/definitions/actions/voice.ts +119 -0
- package/src/definitions/index.ts +23 -0
- package/src/definitions/models/elevenlabs.ts +50 -0
- package/src/definitions/models/flux.ts +56 -0
- package/src/definitions/models/index.ts +36 -0
- package/src/definitions/models/kling.ts +56 -0
- package/src/definitions/models/llama.ts +54 -0
- package/src/definitions/models/nano-banana-pro.ts +102 -0
- package/src/definitions/models/sonauto.ts +68 -0
- package/src/definitions/models/soul.ts +65 -0
- package/src/definitions/models/wan.ts +54 -0
- package/src/definitions/models/whisper.ts +44 -0
- package/src/definitions/skills/index.ts +12 -0
- package/src/definitions/skills/talking-character.ts +87 -0
- package/src/definitions/skills/text-to-tiktok.ts +97 -0
- package/src/index.ts +118 -0
- package/src/providers/apify.ts +269 -0
- package/src/providers/base.ts +264 -0
- package/src/providers/elevenlabs.ts +217 -0
- package/src/providers/fal.ts +392 -0
- package/src/providers/ffmpeg.ts +544 -0
- package/src/providers/fireworks.ts +193 -0
- package/src/providers/groq.ts +149 -0
- package/src/providers/higgsfield.ts +145 -0
- package/src/providers/index.ts +143 -0
- package/src/providers/replicate.ts +147 -0
- package/src/providers/storage.ts +206 -0
- package/src/react/cli.ts +52 -0
- package/src/react/elements.ts +146 -0
- package/src/react/examples/branching.tsx +66 -0
- package/src/react/examples/captions-demo.tsx +37 -0
- package/src/react/examples/character-video.tsx +84 -0
- package/src/react/examples/grid.tsx +53 -0
- package/src/react/examples/layouts-demo.tsx +57 -0
- package/src/react/examples/madi.tsx +60 -0
- package/src/react/examples/music-test.tsx +35 -0
- package/src/react/examples/onlyfans-1m/workflow.tsx +88 -0
- package/src/react/examples/orange-portrait.tsx +41 -0
- package/src/react/examples/split-element-demo.tsx +60 -0
- package/src/react/examples/split-layout-demo.tsx +60 -0
- package/src/react/examples/split.tsx +41 -0
- package/src/react/examples/video-grid.tsx +46 -0
- package/src/react/index.ts +43 -0
- package/src/react/layouts/grid.tsx +28 -0
- package/src/react/layouts/index.ts +2 -0
- package/src/react/layouts/split.tsx +20 -0
- package/src/react/react.test.ts +309 -0
- package/src/react/render.ts +21 -0
- package/src/react/renderers/animate.ts +59 -0
- package/src/react/renderers/captions.ts +297 -0
- package/src/react/renderers/clip.ts +248 -0
- package/src/react/renderers/context.ts +17 -0
- package/src/react/renderers/image.ts +109 -0
- package/src/react/renderers/index.ts +22 -0
- package/src/react/renderers/music.ts +60 -0
- package/src/react/renderers/packshot.ts +84 -0
- package/src/react/renderers/progress.ts +173 -0
- package/src/react/renderers/render.ts +243 -0
- package/src/react/renderers/slider.ts +69 -0
- package/src/react/renderers/speech.ts +53 -0
- package/src/react/renderers/split.ts +91 -0
- package/src/react/renderers/subtitle.ts +16 -0
- package/src/react/renderers/swipe.ts +75 -0
- package/src/react/renderers/title.ts +17 -0
- package/src/react/renderers/utils.ts +124 -0
- package/src/react/renderers/video.ts +127 -0
- package/src/react/runtime/jsx-dev-runtime.ts +43 -0
- package/src/react/runtime/jsx-runtime.ts +35 -0
- package/src/react/types.ts +232 -0
- package/src/studio/index.ts +26 -0
- package/src/studio/scanner.ts +102 -0
- package/src/studio/server.ts +554 -0
- package/src/studio/stages.ts +251 -0
- package/src/studio/step-renderer.ts +279 -0
- package/src/studio/types.ts +60 -0
- package/src/studio/ui/cache.html +303 -0
- package/src/studio/ui/index.html +1820 -0
- package/src/tests/all.test.ts +509 -0
- package/src/tests/index.ts +33 -0
- package/src/tests/unit.test.ts +403 -0
- package/tsconfig.cli.json +8 -0
- package/tsconfig.json +21 -3
- package/TEST_RESULTS.md +0 -122
- package/action/captions/SKILL.md +0 -170
- package/action/captions/index.ts +0 -169
- package/action/edit/SKILL.md +0 -235
- package/action/edit/index.ts +0 -437
- package/action/image/SKILL.md +0 -140
- package/action/image/index.ts +0 -105
- package/action/sync/SKILL.md +0 -136
- package/action/sync/index.ts +0 -145
- package/action/transcribe/SKILL.md +0 -179
- package/action/video/SKILL.md +0 -116
- package/action/video/index.ts +0 -125
- package/action/voice/SKILL.md +0 -125
- package/action/voice/index.ts +0 -136
- package/cli/commands/find.ts +0 -58
- package/cli/commands/help.ts +0 -70
- package/cli/commands/list.ts +0 -49
- package/cli/commands/run.ts +0 -237
- package/cli/commands/which.ts +0 -66
- package/cli/discover.ts +0 -66
- package/cli/index.ts +0 -33
- package/cli/runner.ts +0 -65
- package/cli/types.ts +0 -49
- package/cli/ui.ts +0 -185
- package/index.ts +0 -75
- package/lib/README.md +0 -144
- package/lib/ai-sdk/fal.ts +0 -106
- package/lib/ai-sdk/replicate.ts +0 -107
- package/lib/elevenlabs.ts +0 -382
- package/lib/fal.ts +0 -467
- package/lib/ffmpeg.ts +0 -467
- package/lib/fireworks.ts +0 -235
- package/lib/groq.ts +0 -246
- package/lib/higgsfield.ts +0 -176
- package/lib/remotion/SKILL.md +0 -823
- package/lib/remotion/cli.ts +0 -115
- package/lib/remotion/functions.ts +0 -283
- package/lib/remotion/index.ts +0 -19
- package/lib/remotion/templates.ts +0 -73
- package/lib/replicate.ts +0 -304
- package/output.txt +0 -1
- package/test-import.ts +0 -7
- package/test-services.ts +0 -97
- package/utilities/s3.ts +0 -147
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# trendwatching pipeline
|
|
2
|
+
|
|
3
|
+
discover viral tiktok content for any topic or hashtag using apify scrapers
|
|
4
|
+
|
|
5
|
+
## overview
|
|
6
|
+
|
|
7
|
+
use this pipeline to:
|
|
8
|
+
- find trending videos for a specific topic/hashtag
|
|
9
|
+
- analyze engagement metrics (plays, likes, shares, comments)
|
|
10
|
+
- get video urls for inspiration or downloading
|
|
11
|
+
- identify top creators in a niche
|
|
12
|
+
|
|
13
|
+
## steps
|
|
14
|
+
|
|
15
|
+
### 1. search by hashtag (save to file)
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
# get 10 viral videos for a hashtag, save to output/
|
|
19
|
+
bun run lib/apify.ts run clockworks/tiktok-scraper '{"hashtags":["relationship"],"resultsPerPage":10}' tiktok-relationship.json
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
### 2. search multiple hashtags
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
# search multiple related hashtags
|
|
26
|
+
bun run lib/apify.ts run clockworks/tiktok-scraper '{"hashtags":["viral","trending","fyp"],"resultsPerPage":5}' tiktok-viral.json
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
### 3. scrape specific profiles
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
# get videos from specific creators
|
|
33
|
+
bun run lib/apify.ts run clockworks/tiktok-scraper '{"profiles":["@username"],"resultsPerPage":10}' creator-videos.json
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
### 4. get discover page trends
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
# scrape tiktok discover page for trending topics
|
|
40
|
+
bun run lib/apify.ts run clockworks/tiktok-discover-scraper '{"hashtags":["fitness"]}' fitness-trends.json
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
### 5. download videos from saved json
|
|
44
|
+
|
|
45
|
+
download all videos from a saved json file using yt-dlp:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
# download all videos to output/videos/
|
|
49
|
+
bun run lib/apify.ts download tiktok-relationship.json
|
|
50
|
+
|
|
51
|
+
# download to custom directory
|
|
52
|
+
bun run lib/apify.ts download tiktok-relationship.json output/my-videos
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
### 6. read saved results
|
|
56
|
+
|
|
57
|
+
results are saved to `output/<filename>.json` - you can read them later:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
# list all video urls
|
|
61
|
+
cat output/tiktok-relationship.json | jq -r '.[].webVideoUrl'
|
|
62
|
+
|
|
63
|
+
# get top videos by play count
|
|
64
|
+
cat output/tiktok-relationship.json | jq 'sort_by(-.playCount) | .[0:3] | .[] | {url: .webVideoUrl, plays: .playCount}'
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## output data
|
|
68
|
+
|
|
69
|
+
each video includes:
|
|
70
|
+
- `webVideoUrl` - tiktok video url
|
|
71
|
+
- `text` - video caption/description
|
|
72
|
+
- `playCount` - total views
|
|
73
|
+
- `diggCount` - likes
|
|
74
|
+
- `shareCount` - shares
|
|
75
|
+
- `commentCount` - comments
|
|
76
|
+
- `collectCount` - saves/bookmarks
|
|
77
|
+
- `authorMeta` - creator info (name, followers, etc.)
|
|
78
|
+
- `musicMeta` - audio/music info
|
|
79
|
+
- `hashtags` - all hashtags used
|
|
80
|
+
- `createTimeISO` - when posted
|
|
81
|
+
|
|
82
|
+
## example output
|
|
83
|
+
|
|
84
|
+
```json
|
|
85
|
+
{
|
|
86
|
+
"webVideoUrl": "https://www.tiktok.com/@user/video/123",
|
|
87
|
+
"text": "Not the tiny violin #prank #couples #relationship",
|
|
88
|
+
"playCount": 13200000,
|
|
89
|
+
"diggCount": 2900000,
|
|
90
|
+
"shareCount": 34700,
|
|
91
|
+
"commentCount": 11000,
|
|
92
|
+
"authorMeta": {
|
|
93
|
+
"name": "samandmonica",
|
|
94
|
+
"fans": 4000000
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## available scrapers
|
|
100
|
+
|
|
101
|
+
| actor | use case |
|
|
102
|
+
|-------|----------|
|
|
103
|
+
| `clockworks/tiktok-scraper` | general hashtag/profile scraping |
|
|
104
|
+
| `clockworks/tiktok-hashtag-scraper` | hashtag-specific scraping |
|
|
105
|
+
| `clockworks/tiktok-profile-scraper` | creator profile scraping |
|
|
106
|
+
| `clockworks/tiktok-discover-scraper` | trending topics & discover page |
|
|
107
|
+
| `clockworks/tiktok-video-scraper` | specific video urls |
|
|
108
|
+
| `clockworks/tiktok-comments-scraper` | video comments |
|
|
109
|
+
|
|
110
|
+
## pricing
|
|
111
|
+
|
|
112
|
+
apify pay-per-event pricing:
|
|
113
|
+
- actor start: $0.005
|
|
114
|
+
- per result: $0.003
|
|
115
|
+
- video download: $0.001 (optional)
|
|
116
|
+
|
|
117
|
+
example: 100 videos = ~$0.31
|
|
118
|
+
|
|
119
|
+
## complete workflow example
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
# 1. scrape viral relationship videos
|
|
123
|
+
bun run lib/apify.ts run clockworks/tiktok-scraper '{"hashtags":["relationship"],"resultsPerPage":5}' tiktok-relationship.json
|
|
124
|
+
|
|
125
|
+
# 2. check what we got
|
|
126
|
+
cat output/tiktok-relationship.json | jq -r '.[].webVideoUrl'
|
|
127
|
+
|
|
128
|
+
# 3. download all videos
|
|
129
|
+
bun run lib/apify.ts download tiktok-relationship.json
|
|
130
|
+
|
|
131
|
+
# 4. videos are now in output/videos/
|
|
132
|
+
ls output/videos/
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
## tips
|
|
136
|
+
|
|
137
|
+
- use `resultsPerPage` to limit results and costs
|
|
138
|
+
- combine multiple hashtags for broader search
|
|
139
|
+
- check `playCount` and `diggCount` ratio for engagement quality
|
|
140
|
+
- look at `createTimeISO` to find recent trending content
|
|
141
|
+
- save results to json to analyze patterns over time
|
|
142
|
+
- download videos for offline analysis or inspiration
|
|
143
|
+
|
|
144
|
+
## environment
|
|
145
|
+
|
|
146
|
+
requires `APIFY_TOKEN` in `.env`
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
APIFY_TOKEN=apify_api_xxx
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
requires `yt-dlp` for video downloads:
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
brew install yt-dlp
|
|
156
|
+
```
|
package/plan.md
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
# Photify AI - Concept 4: Animated Family Photos
|
|
2
|
+
|
|
3
|
+
## Overview
|
|
4
|
+
Create 15 video creatives showing "talking heads" emotionally reacting to animated vintage family photos. Each video uses the same 5 animated retro photos as background, with different characters wiping away tears.
|
|
5
|
+
|
|
6
|
+
## Deliverables per video
|
|
7
|
+
- 2 formats: 4x5 (1080x1350) + 9x16 (1080x1920)
|
|
8
|
+
- CapCut-style dynamic subtitles
|
|
9
|
+
- Background music/sfx
|
|
10
|
+
- Packshot at end: `media/Packshot_9_16.mp4`
|
|
11
|
+
|
|
12
|
+
## Assets Generated
|
|
13
|
+
|
|
14
|
+
### 1. Animated Retro Photos (5 total, shared across all videos)
|
|
15
|
+
| # | Description | Image | Video |
|
|
16
|
+
|---|-------------|-------|-------|
|
|
17
|
+
| 1 | Young grandparents sitting side by side, holding hands | `media/retro/01_grandparents.jpg` | `media/retro/01_grandparents.mp4` |
|
|
18
|
+
| 2 | Woman holding cat, cat rubbing against her hand | `media/retro/02_woman_cat.jpg` | `media/retro/02_woman_cat.mp4` |
|
|
19
|
+
| 3 | New Year by old tree, two kids in costumes | `media/retro/03_newyear.jpg` | `media/retro/03_newyear.mp4` |
|
|
20
|
+
| 4 | Old person on porch of old house | `media/retro/04_elderly_porch.jpg` | `media/retro/04_elderly_porch.mp4` |
|
|
21
|
+
| 5 | Family dinner gathering | `media/retro/05_family_dinner.jpg` | `media/retro/05_family_dinner.mp4` |
|
|
22
|
+
|
|
23
|
+
### 2. Talking Head Characters (15 total)
|
|
24
|
+
| # | Description | Portrait | Greenscreen |
|
|
25
|
+
|---|-------------|----------|-------------|
|
|
26
|
+
| 1 | Girl with long light brown hair (25-30) | `media/characters/01_light_brown_hair.jpg` | `media/characters/01_light_brown_hair_greenscreen.jpg` |
|
|
27
|
+
| 2 | Asian girl with short black hair and bangs | `media/characters/02_asian_bangs.jpg` | `media/characters/02_gs.jpg` |
|
|
28
|
+
| 3 | Girl with red curls and freckles | `media/characters/03_red_curls.jpg` | `media/characters/03_gs.jpg` |
|
|
29
|
+
| 4 | Girl with bright colored hair (pink/blue) | `media/characters/04_pink_hair.jpg` | `media/characters/04_gs.jpg` |
|
|
30
|
+
| 5 | Girl with bangs and long dark hair | `media/characters/05_dark_bangs.jpg` | `media/characters/05_gs.jpg` |
|
|
31
|
+
| 6 | Middle-aged woman with gray streaks | `media/characters/06_gray_streaks.jpg` | `media/characters/06_gs.jpg` |
|
|
32
|
+
| 7 | Girl with soft curls, warm skin tone | `media/characters/07_soft_curls.jpg` | `media/characters/07_gs.jpg` |
|
|
33
|
+
| 8 | Girl with shaved side, asymmetric cut | `media/characters/08_shaved_side.jpg` | `media/characters/08_gs.jpg` |
|
|
34
|
+
| 9 | Dark-skinned girl with afro curls | `media/characters/09_afro_curls.jpg` | `media/characters/09_gs.jpg` |
|
|
35
|
+
| 10 | Girl with long braids or dreads | `media/characters/10_long_braids.jpg` | `media/characters/10_gs.jpg` |
|
|
36
|
+
| 11 | Girl with glasses, bob haircut | `media/characters/11_glasses_bob.jpg` | `media/characters/11_gs.jpg` |
|
|
37
|
+
| 12 | Young girl with short cut, septum piercing | `media/characters/12_septum.jpg` | `media/characters/12_gs.jpg` |
|
|
38
|
+
| 13 | Girl with super long platinum white hair | `media/characters/13_platinum_hair.jpg` | `media/characters/13_gs.jpg` |
|
|
39
|
+
| 14 | Asian woman with neat haircut, glasses | `media/characters/14_asian_glasses.jpg` | `media/characters/14_gs.jpg` |
|
|
40
|
+
| 15 | Dark-skinned girl with wavy pink hair | `media/characters/15_pink_wig.jpg` | `media/characters/15_gs.jpg` |
|
|
41
|
+
|
|
42
|
+
### 3. Audio Assets
|
|
43
|
+
| Asset | File | Duration |
|
|
44
|
+
|-------|------|----------|
|
|
45
|
+
| Voice (Rachel) | `output/scene01/voice.mp3` | ~7 seconds |
|
|
46
|
+
| Captions SRT | `output/scene01/captions.srt` | 0-5 seconds |
|
|
47
|
+
|
|
48
|
+
### 4. Packshot
|
|
49
|
+
| Format | File | Duration |
|
|
50
|
+
|--------|------|----------|
|
|
51
|
+
| 9x16 | `media/Packshot_9_16.mp4` | 2.18s |
|
|
52
|
+
| 4x5 | `media/Packshot_4x5.mp4` | 2.18s (created from 9x16) |
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## Script
|
|
57
|
+
**Voice:** "Guys... I animated old family photos with Photify, and realized my great-grandma had the exact same smile as me."
|
|
58
|
+
|
|
59
|
+
**Action:** Character wipes tears (touched, not overly dramatic)
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## Captions SRT Format
|
|
64
|
+
```srt
|
|
65
|
+
1
|
|
66
|
+
00:00:00,000 --> 00:00:01,500
|
|
67
|
+
Guys...
|
|
68
|
+
|
|
69
|
+
2
|
|
70
|
+
00:00:01,500 --> 00:00:03,500
|
|
71
|
+
I animated old family photos
|
|
72
|
+
with Photify
|
|
73
|
+
|
|
74
|
+
3
|
|
75
|
+
00:00:03,500 --> 00:00:05,000
|
|
76
|
+
and realized my great-grandma
|
|
77
|
+
had the exact same smile as me
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## FINAL WORKING COMMANDS (Scene 01)
|
|
83
|
+
|
|
84
|
+
### Step 1: Generate Lipsync Video
|
|
85
|
+
```bash
|
|
86
|
+
bun run src/cli/index.ts run sync \
|
|
87
|
+
--image media/characters/01_light_brown_hair_greenscreen.jpg \
|
|
88
|
+
--audio output/scene01/voice.mp3 \
|
|
89
|
+
--prompt "woman speaking emotionally, warm expression" \
|
|
90
|
+
--duration 10 \
|
|
91
|
+
--resolution 720p \
|
|
92
|
+
--quiet
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
**Output:** `output/scene01/talking_head_v2.mp4` (10 seconds, greenscreen background)
|
|
96
|
+
|
|
97
|
+
### Step 2: Composite 9x16 (1080x1920)
|
|
98
|
+
```bash
|
|
99
|
+
ffmpeg -y \
|
|
100
|
+
-i media/retro/01_grandparents.mp4 \
|
|
101
|
+
-stream_loop 1 -i media/retro/01_grandparents.mp4 \
|
|
102
|
+
-i output/scene01/talking_head_v2.mp4 \
|
|
103
|
+
-filter_complex "\
|
|
104
|
+
[0:v][1:v]concat=n=2:v=1:a=0[bglong]; \
|
|
105
|
+
[bglong]scale=1080:1920:force_original_aspect_ratio=increase,crop=1080:1920,setsar=1[bg]; \
|
|
106
|
+
[2:v]chromakey=0x00ff00:0.3:0.1,crop=600:700:250:50,scale=420:-1[fg]; \
|
|
107
|
+
[bg][fg]overlay=(W-w)/2:H-h-650[out]" \
|
|
108
|
+
-map "[out]" -map 2:a -t 7 output/scene01/composite_9x16.mp4
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
**Key parameters for 9x16:**
|
|
112
|
+
- `scale=420:-1` - talking head width (420px, larger)
|
|
113
|
+
- `H-h-650` - position from bottom (650px up = upper-middle area)
|
|
114
|
+
- `crop=600:700:250:50` - crop talking head video (600x700, offset 250,50 to center on face)
|
|
115
|
+
|
|
116
|
+
### Step 3: Add Captions to 9x16
|
|
117
|
+
```bash
|
|
118
|
+
ffmpeg -y -i output/scene01/composite_9x16.mp4 \
|
|
119
|
+
-vf "subtitles=/Users/aleks/Github/varghq/sdk/output/scene01/captions.srt:force_style='FontSize=18,PrimaryColour=&H00FFFFFF,OutlineColour=&H00000000,Outline=2,Bold=1,Alignment=2,MarginV=15'" \
|
|
120
|
+
-c:a copy output/scene01/captioned_9x16.mp4
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
**Key parameters:**
|
|
124
|
+
- `FontSize=18` - smaller font (standard for social media)
|
|
125
|
+
- `MarginV=15` - 15px from bottom edge
|
|
126
|
+
- `Alignment=2` - bottom center
|
|
127
|
+
- **MUST use absolute path for subtitles file**
|
|
128
|
+
|
|
129
|
+
### Step 4: Add Packshot to 9x16
|
|
130
|
+
```bash
|
|
131
|
+
ffmpeg -y -i output/scene01/captioned_9x16.mp4 -i media/Packshot_9_16.mp4 \
|
|
132
|
+
-filter_complex "[0:v]fps=24[v0];[1:v]fps=24[v1];[v0][0:a][v1][1:a]concat=n=2:v=1:a=1[outv][outa]" \
|
|
133
|
+
-map "[outv]" -map "[outa]" output/scene01/final_9x16_v4.mp4
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
### Step 5: Composite 4x5 (1080x1350)
|
|
137
|
+
```bash
|
|
138
|
+
ffmpeg -y \
|
|
139
|
+
-i media/retro/01_grandparents.mp4 \
|
|
140
|
+
-stream_loop 1 -i media/retro/01_grandparents.mp4 \
|
|
141
|
+
-i output/scene01/talking_head_v2.mp4 \
|
|
142
|
+
-filter_complex "\
|
|
143
|
+
[0:v][1:v]concat=n=2:v=1:a=0[bglong]; \
|
|
144
|
+
[bglong]scale=1080:1350:force_original_aspect_ratio=increase,crop=1080:1350,setsar=1[bg]; \
|
|
145
|
+
[2:v]chromakey=0x00ff00:0.3:0.1,crop=600:700:250:50,scale=420:-1[fg]; \
|
|
146
|
+
[bg][fg]overlay=(W-w)/2:H-h-550[out]" \
|
|
147
|
+
-map "[out]" -map 2:a -t 7 output/scene01/composite_4x5.mp4
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
**Key parameters for 4x5:**
|
|
151
|
+
- `scale=420:-1` - same talking head width
|
|
152
|
+
- `H-h-550` - position from bottom (550px up for shorter frame)
|
|
153
|
+
- Background scales to 1080x1350 instead of 1920
|
|
154
|
+
|
|
155
|
+
### Step 6: Add Captions to 4x5
|
|
156
|
+
```bash
|
|
157
|
+
ffmpeg -y -i output/scene01/composite_4x5.mp4 \
|
|
158
|
+
-vf "subtitles=/Users/aleks/Github/varghq/sdk/output/scene01/captions.srt:force_style='FontSize=18,PrimaryColour=&H00FFFFFF,OutlineColour=&H00000000,Outline=2,Bold=1,Alignment=2,MarginV=15'" \
|
|
159
|
+
-c:a copy output/scene01/captioned_4x5.mp4
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
### Step 7: Create 4x5 Packshot (one-time)
|
|
163
|
+
```bash
|
|
164
|
+
ffmpeg -y -i media/Packshot_9_16.mp4 \
|
|
165
|
+
-vf "scale=1080:1350:force_original_aspect_ratio=increase,crop=1080:1350" \
|
|
166
|
+
-c:a copy media/Packshot_4x5.mp4
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
### Step 8: Add Packshot to 4x5
|
|
170
|
+
```bash
|
|
171
|
+
ffmpeg -y -i output/scene01/captioned_4x5.mp4 -i media/Packshot_4x5.mp4 \
|
|
172
|
+
-filter_complex "[0:v]fps=24[v0];[1:v]fps=24[v1];[v0][0:a][v1][1:a]concat=n=2:v=1:a=1[outv][outa]" \
|
|
173
|
+
-map "[outv]" -map "[outa]" output/scene01/final_4x5_v4.mp4
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
## CRITICAL LESSONS LEARNED
|
|
179
|
+
|
|
180
|
+
### 1. Subtitles Filter Issues
|
|
181
|
+
- **MUST use absolute path** for subtitle file: `subtitles=/full/path/to/captions.srt`
|
|
182
|
+
- Relative paths like `subtitles='output/scene01/captions.srt'` may silently fail
|
|
183
|
+
- MarginV is distance from BOTTOM edge when Alignment=2
|
|
184
|
+
|
|
185
|
+
### 2. Talking Head Positioning
|
|
186
|
+
- **9x16 (1920 height):** Use `H-h-650` to position character in upper-middle
|
|
187
|
+
- **4x5 (1350 height):** Use `H-h-550` for similar visual position
|
|
188
|
+
- Character should be ABOVE captions, not overlapping
|
|
189
|
+
- Larger head size (420px width) looks better than smaller (260-320px)
|
|
190
|
+
|
|
191
|
+
### 3. Caption Positioning
|
|
192
|
+
- **FontSize=18** is good for social media (not too big, not too small)
|
|
193
|
+
- **MarginV=15** places captions at very bottom
|
|
194
|
+
- Higher MarginV values push captions UP (e.g., MarginV=500 = 500px from bottom)
|
|
195
|
+
- For 4x5, use same MarginV as 9x16 since captions should be at bottom in both
|
|
196
|
+
|
|
197
|
+
### 4. Chromakey Settings
|
|
198
|
+
- `chromakey=0x00ff00:0.3:0.1` works well for bright green backgrounds
|
|
199
|
+
- 0x00ff00 = pure green
|
|
200
|
+
- 0.3 = similarity threshold
|
|
201
|
+
- 0.1 = blend/edge softness
|
|
202
|
+
|
|
203
|
+
### 5. Crop for Face Focus
|
|
204
|
+
- `crop=600:700:250:50` crops the lipsync video to focus on face
|
|
205
|
+
- Format: `crop=width:height:x_offset:y_offset`
|
|
206
|
+
- Adjust x_offset and y_offset based on face position in source video
|
|
207
|
+
|
|
208
|
+
### 6. Background Looping
|
|
209
|
+
- Use `-stream_loop 1` to loop short backgrounds
|
|
210
|
+
- Animated photos are ~5 seconds, need ~7 seconds for full audio
|
|
211
|
+
- Concat looped background before compositing
|
|
212
|
+
|
|
213
|
+
### 7. FPS Alignment for Concat
|
|
214
|
+
- Both videos must have same FPS for clean concat
|
|
215
|
+
- Use `fps=24` filter on both before concat
|
|
216
|
+
- Packshot may be 60fps, main content is 24fps
|
|
217
|
+
|
|
218
|
+
### 8. Color Format for ASS Styles
|
|
219
|
+
- Use `&H00FFFFFF` format (AABBGGRR in hex)
|
|
220
|
+
- `&H00FFFFFF` = white
|
|
221
|
+
- `&H00000000` = black
|
|
222
|
+
- The `00` prefix is alpha (00 = opaque)
|
|
223
|
+
|
|
224
|
+
---
|
|
225
|
+
|
|
226
|
+
## Scene 01 Final Output
|
|
227
|
+
|
|
228
|
+
| Format | File | Resolution | Duration |
|
|
229
|
+
|--------|------|------------|----------|
|
|
230
|
+
| 9x16 | `output/scene01/final_9x16_v4.mp4` | 1080x1920 | ~9.2s |
|
|
231
|
+
| 4x5 | `output/scene01/final_4x5_v4.mp4` | 1080x1350 | ~9.2s |
|
|
232
|
+
|
|
233
|
+
---
|
|
234
|
+
|
|
235
|
+
## Remaining Work
|
|
236
|
+
|
|
237
|
+
### Scenes 02-15
|
|
238
|
+
Each scene needs:
|
|
239
|
+
1. Generate lipsync video using greenscreen character image
|
|
240
|
+
2. Create composite (9x16 and 4x5)
|
|
241
|
+
3. Add captions
|
|
242
|
+
4. Add packshot
|
|
243
|
+
5. Export finals
|
|
244
|
+
|
|
245
|
+
### Batch Processing Template
|
|
246
|
+
```bash
|
|
247
|
+
# For each scene N (02-15):
|
|
248
|
+
SCENE="02"
|
|
249
|
+
CHAR_IMG="media/characters/02_gs.jpg"
|
|
250
|
+
BG_VIDEO="media/retro/02_woman_cat.mp4"
|
|
251
|
+
|
|
252
|
+
# Create scene folder
|
|
253
|
+
mkdir -p output/scene${SCENE}
|
|
254
|
+
cp output/scene01/voice.mp3 output/scene${SCENE}/
|
|
255
|
+
cp output/scene01/captions.srt output/scene${SCENE}/
|
|
256
|
+
|
|
257
|
+
# Generate lipsync
|
|
258
|
+
bun run src/cli/index.ts run sync \
|
|
259
|
+
--image ${CHAR_IMG} \
|
|
260
|
+
--audio output/scene${SCENE}/voice.mp3 \
|
|
261
|
+
--prompt "woman speaking emotionally, warm expression" \
|
|
262
|
+
--duration 10 \
|
|
263
|
+
--resolution 720p \
|
|
264
|
+
--output output/scene${SCENE}/talking_head.mp4
|
|
265
|
+
|
|
266
|
+
# Then run composite/caption/packshot commands as above
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
---
|
|
270
|
+
|
|
271
|
+
## Cost Tracking
|
|
272
|
+
|
|
273
|
+
| Item | Cost |
|
|
274
|
+
|------|------|
|
|
275
|
+
| Flux Pro images (5 retro + 15 chars + 15 greenscreen) | ~$0.80 |
|
|
276
|
+
| Kling video (5 animations) | ~$2.10 |
|
|
277
|
+
| Wan-25 lipsync (1 so far) | ~$0.50 |
|
|
278
|
+
| ElevenLabs TTS | ~$0.05 |
|
|
279
|
+
| **Total so far** | **~$3.45** |
|
|
280
|
+
|
|
281
|
+
Estimated for all 15 scenes: ~$10-12 total
|
package/scripts/.gitkeep
ADDED
|
File without changes
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
export interface CacheStorage {
|
|
2
|
+
get(key: string): Promise<unknown | undefined>;
|
|
3
|
+
set(key: string, value: unknown, ttl?: number): Promise<void>;
|
|
4
|
+
delete(key: string): Promise<void>;
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export interface WithCacheOptions {
|
|
8
|
+
ttl?: number | string;
|
|
9
|
+
storage?: CacheStorage;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
type CacheKeyDeps = (string | number | boolean | null | undefined)[];
|
|
13
|
+
|
|
14
|
+
type WithCacheKey<T> = Omit<T, "cacheKey"> & { cacheKey?: CacheKeyDeps };
|
|
15
|
+
|
|
16
|
+
type CachedFn<T, R> = (options: WithCacheKey<T>) => Promise<R>;
|
|
17
|
+
|
|
18
|
+
const memoryCache = new Map<string, { value: unknown; expires: number }>();
|
|
19
|
+
|
|
20
|
+
const defaultStorage: CacheStorage = {
|
|
21
|
+
async get(key: string) {
|
|
22
|
+
const entry = memoryCache.get(key);
|
|
23
|
+
if (!entry) return undefined;
|
|
24
|
+
if (entry.expires && Date.now() > entry.expires) {
|
|
25
|
+
memoryCache.delete(key);
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
return entry.value;
|
|
29
|
+
},
|
|
30
|
+
async set(key: string, value: unknown, ttl?: number) {
|
|
31
|
+
const expires = ttl ? Date.now() + ttl : 0;
|
|
32
|
+
memoryCache.set(key, { value, expires });
|
|
33
|
+
},
|
|
34
|
+
async delete(key: string) {
|
|
35
|
+
memoryCache.delete(key);
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
function parseTTL(ttl: number | string | undefined): number | undefined {
|
|
40
|
+
if (ttl === undefined) return undefined;
|
|
41
|
+
if (typeof ttl === "number") return ttl;
|
|
42
|
+
|
|
43
|
+
const match = ttl.match(/^(\d+)(s|m|h|d)$/);
|
|
44
|
+
if (!match) return undefined;
|
|
45
|
+
|
|
46
|
+
const value = Number.parseInt(match[1]!, 10);
|
|
47
|
+
const unit = match[2];
|
|
48
|
+
|
|
49
|
+
switch (unit) {
|
|
50
|
+
case "s":
|
|
51
|
+
return value * 1000;
|
|
52
|
+
case "m":
|
|
53
|
+
return value * 60 * 1000;
|
|
54
|
+
case "h":
|
|
55
|
+
return value * 60 * 60 * 1000;
|
|
56
|
+
case "d":
|
|
57
|
+
return value * 24 * 60 * 60 * 1000;
|
|
58
|
+
default:
|
|
59
|
+
return undefined;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function depsToKey(prefix: string, deps: CacheKeyDeps): string {
|
|
64
|
+
const depsStr = deps.map((d) => String(d ?? "")).join(":");
|
|
65
|
+
return prefix ? `${prefix}:${depsStr}` : depsStr;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function flatten(value: unknown): unknown {
|
|
69
|
+
if (value === null || value === undefined) return value;
|
|
70
|
+
if (value instanceof Uint8Array) return value;
|
|
71
|
+
if (Array.isArray(value)) return value.map(flatten);
|
|
72
|
+
if (typeof value === "object") {
|
|
73
|
+
const obj = value as Record<string, unknown>;
|
|
74
|
+
const result: Record<string, unknown> = {};
|
|
75
|
+
for (const key of Object.keys(obj)) {
|
|
76
|
+
result[key] = flatten(obj[key]);
|
|
77
|
+
}
|
|
78
|
+
const proto = Object.getPrototypeOf(obj);
|
|
79
|
+
if (proto && proto !== Object.prototype) {
|
|
80
|
+
const descriptors = Object.getOwnPropertyDescriptors(proto);
|
|
81
|
+
for (const [key, desc] of Object.entries(descriptors)) {
|
|
82
|
+
if (desc.get && key !== "constructor") {
|
|
83
|
+
result[key] = flatten(desc.get.call(obj));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return result;
|
|
88
|
+
}
|
|
89
|
+
return value;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Wrap an async function to add caching via `cacheKey` option.
|
|
94
|
+
*
|
|
95
|
+
* @example
|
|
96
|
+
* ```ts
|
|
97
|
+
* import { generateImage } from "ai";
|
|
98
|
+
* import { withCache } from "./cache";
|
|
99
|
+
*
|
|
100
|
+
* const generateImage_ = withCache(generateImage);
|
|
101
|
+
*
|
|
102
|
+
* const { images } = await generateImage_({
|
|
103
|
+
* model: fal.imageModel("flux-schnell"),
|
|
104
|
+
* prompt: "lion roaring",
|
|
105
|
+
* cacheKey: ["lion", take], // cache based on deps
|
|
106
|
+
* });
|
|
107
|
+
* ```
|
|
108
|
+
*/
|
|
109
|
+
const DEFAULT_TTL = "1h";
|
|
110
|
+
|
|
111
|
+
export function withCache<T extends object, R>(
|
|
112
|
+
fn: (options: T) => Promise<R>,
|
|
113
|
+
options: WithCacheOptions = {},
|
|
114
|
+
): CachedFn<T, R> {
|
|
115
|
+
const storage = options.storage ?? defaultStorage;
|
|
116
|
+
const ttl = parseTTL(options.ttl ?? DEFAULT_TTL);
|
|
117
|
+
const prefix = fn.name || "anonymous";
|
|
118
|
+
|
|
119
|
+
return async (opts: WithCacheKey<T>): Promise<R> => {
|
|
120
|
+
const { cacheKey, ...rest } = opts;
|
|
121
|
+
|
|
122
|
+
if (!cacheKey) {
|
|
123
|
+
return fn(rest as T);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const key = depsToKey(prefix, cacheKey);
|
|
127
|
+
const cached = await storage.get(key);
|
|
128
|
+
if (cached !== undefined) {
|
|
129
|
+
return cached as R;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const result = await fn(rest as T);
|
|
133
|
+
const flattened = flatten(result);
|
|
134
|
+
await storage.set(key, flattened, ttl);
|
|
135
|
+
|
|
136
|
+
return result;
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export function clearCache(): void {
|
|
141
|
+
memoryCache.clear();
|
|
142
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { generateImage as _generateImage } from "ai";
|
|
2
|
+
import {
|
|
3
|
+
generateVideo as _generateVideo,
|
|
4
|
+
fal,
|
|
5
|
+
fileCache,
|
|
6
|
+
withCache,
|
|
7
|
+
} from "../index";
|
|
8
|
+
|
|
9
|
+
const storage = fileCache({ dir: ".cache/ai" });
|
|
10
|
+
const generateImage = withCache(_generateImage, { storage });
|
|
11
|
+
const generateVideo = withCache(_generateVideo, { storage });
|
|
12
|
+
|
|
13
|
+
async function main() {
|
|
14
|
+
const take = 1;
|
|
15
|
+
|
|
16
|
+
// cached video generation
|
|
17
|
+
console.log("generating video...");
|
|
18
|
+
console.time("first");
|
|
19
|
+
const { video } = await generateVideo({
|
|
20
|
+
model: fal.videoModel("wan-2.5"),
|
|
21
|
+
prompt: "a cat playing piano",
|
|
22
|
+
duration: 5,
|
|
23
|
+
cacheKey: ["cat-piano", take],
|
|
24
|
+
});
|
|
25
|
+
console.timeEnd("first");
|
|
26
|
+
console.log(`video: ${video.uint8Array.byteLength} bytes`);
|
|
27
|
+
|
|
28
|
+
// same cacheKey = instant cache hit
|
|
29
|
+
console.log("\nsame cacheKey (from cache)...");
|
|
30
|
+
console.time("cached");
|
|
31
|
+
const { video: video2 } = await generateVideo({
|
|
32
|
+
model: fal.videoModel("wan-2.5"),
|
|
33
|
+
prompt: "a cat playing piano",
|
|
34
|
+
duration: 5,
|
|
35
|
+
cacheKey: ["cat-piano", take],
|
|
36
|
+
});
|
|
37
|
+
console.timeEnd("cached");
|
|
38
|
+
console.log(`video: ${video2.uint8Array.byteLength} bytes`);
|
|
39
|
+
|
|
40
|
+
// cached image generation
|
|
41
|
+
console.log("\ngenerating image...");
|
|
42
|
+
const { images } = await generateImage({
|
|
43
|
+
model: fal.imageModel("flux-schnell"),
|
|
44
|
+
prompt: "cyberpunk cityscape",
|
|
45
|
+
n: 1,
|
|
46
|
+
cacheKey: ["cyberpunk-city", 1],
|
|
47
|
+
});
|
|
48
|
+
console.log(`image: ${images[0]?.uint8Array.byteLength} bytes`);
|
|
49
|
+
|
|
50
|
+
console.log("\ndone!");
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
main().catch(console.error);
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { generateImage } from "ai";
|
|
2
|
+
import { File, fal, generateVideo } from "../index";
|
|
3
|
+
|
|
4
|
+
async function main() {
|
|
5
|
+
console.log("=== taisa solo closeup - scene 4 ===\n");
|
|
6
|
+
|
|
7
|
+
const referenceImages = [
|
|
8
|
+
File.fromPath("media/taisa/taisa.jpg"),
|
|
9
|
+
File.fromPath("output/original-frame-1m08s.png"),
|
|
10
|
+
];
|
|
11
|
+
|
|
12
|
+
const imageContents = await Promise.all(
|
|
13
|
+
referenceImages.map((f) => f.arrayBuffer()),
|
|
14
|
+
);
|
|
15
|
+
|
|
16
|
+
console.log("generating closeup frame with nano-banana-pro/edit...");
|
|
17
|
+
const { image } = await generateImage({
|
|
18
|
+
model: fal.imageModel("nano-banana-pro/edit"),
|
|
19
|
+
prompt: {
|
|
20
|
+
images: imageContents,
|
|
21
|
+
text: "Closeup portrait of dark-haired brunette woman Taisa singing passionately on concert stage, dramatic purple blue stage lighting, emotional expression, glamorous dress, beautiful face, professional concert hall background blurred",
|
|
22
|
+
},
|
|
23
|
+
aspectRatio: "16:9",
|
|
24
|
+
n: 1,
|
|
25
|
+
providerOptions: {
|
|
26
|
+
fal: { resolution: "1K" },
|
|
27
|
+
},
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
await Bun.write("output/duet-frame-4.png", image.uint8Array);
|
|
31
|
+
console.log(
|
|
32
|
+
`frame saved: output/duet-frame-4.png (${image.uint8Array.byteLength} bytes)\n`,
|
|
33
|
+
);
|
|
34
|
+
|
|
35
|
+
console.log("animating 10s with kling-v2.5...");
|
|
36
|
+
const { video } = await generateVideo({
|
|
37
|
+
model: fal.videoModel("kling-v2.5"),
|
|
38
|
+
prompt: {
|
|
39
|
+
images: [image.uint8Array],
|
|
40
|
+
text: "closeup of woman singing passionately, subtle head movements, lips moving as singing, natural breathing, blinking, emotional expressions, concert atmosphere with stage lighting",
|
|
41
|
+
},
|
|
42
|
+
duration: 10,
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
await Bun.write("output/duet-scene-4.mp4", video.uint8Array);
|
|
46
|
+
console.log(
|
|
47
|
+
`video saved: output/duet-scene-4.mp4 (${video.uint8Array.byteLength} bytes)`,
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
console.log("\ndone!");
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
main().catch(console.error);
|