varg.ai-sdk 0.1.0 → 0.4.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. package/.claude/settings.local.json +1 -1
  2. package/.env.example +3 -0
  3. package/.github/workflows/ci.yml +23 -0
  4. package/.husky/README.md +102 -0
  5. package/.husky/commit-msg +6 -0
  6. package/.husky/pre-commit +9 -0
  7. package/.husky/pre-push +6 -0
  8. package/.size-limit.json +8 -0
  9. package/.test-hooks.ts +5 -0
  10. package/CLAUDE.md +10 -3
  11. package/CONTRIBUTING.md +150 -0
  12. package/LICENSE.md +53 -0
  13. package/README.md +56 -209
  14. package/SKILLS.md +26 -10
  15. package/biome.json +7 -1
  16. package/bun.lock +1286 -0
  17. package/commitlint.config.js +22 -0
  18. package/docs/index.html +1130 -0
  19. package/docs/prompting.md +326 -0
  20. package/docs/react.md +834 -0
  21. package/docs/sdk.md +812 -0
  22. package/ffmpeg/CLAUDE.md +68 -0
  23. package/package.json +48 -8
  24. package/pipeline/cookbooks/scripts/animate-frames-parallel.ts +84 -0
  25. package/pipeline/cookbooks/scripts/combine-scenes.sh +53 -0
  26. package/pipeline/cookbooks/scripts/generate-frames-parallel.ts +99 -0
  27. package/pipeline/cookbooks/scripts/still-to-video.sh +37 -0
  28. package/pipeline/cookbooks/text-to-tiktok.md +669 -0
  29. package/pipeline/cookbooks/trendwatching.md +156 -0
  30. package/plan.md +281 -0
  31. package/scripts/.gitkeep +0 -0
  32. package/src/ai-sdk/cache.ts +142 -0
  33. package/src/ai-sdk/examples/cached-generation.ts +53 -0
  34. package/src/ai-sdk/examples/duet-scene-4.ts +53 -0
  35. package/src/ai-sdk/examples/duet-scene-5-audio.ts +32 -0
  36. package/src/ai-sdk/examples/duet-video.ts +56 -0
  37. package/src/ai-sdk/examples/editly-composition.ts +63 -0
  38. package/src/ai-sdk/examples/editly-test.ts +57 -0
  39. package/src/ai-sdk/examples/editly-video-test.ts +52 -0
  40. package/src/ai-sdk/examples/fal-lipsync.ts +43 -0
  41. package/src/ai-sdk/examples/higgsfield-image.ts +61 -0
  42. package/src/ai-sdk/examples/music-generation.ts +19 -0
  43. package/src/ai-sdk/examples/openai-sora.ts +34 -0
  44. package/src/ai-sdk/examples/replicate-bg-removal.ts +52 -0
  45. package/src/ai-sdk/examples/simpsons-scene.ts +61 -0
  46. package/src/ai-sdk/examples/talking-lion.ts +55 -0
  47. package/src/ai-sdk/examples/video-generation.ts +39 -0
  48. package/src/ai-sdk/examples/workflow-animated-girl.ts +104 -0
  49. package/src/ai-sdk/examples/workflow-before-after.ts +114 -0
  50. package/src/ai-sdk/examples/workflow-character-grid.ts +112 -0
  51. package/src/ai-sdk/examples/workflow-slideshow.ts +161 -0
  52. package/src/ai-sdk/file-cache.ts +112 -0
  53. package/src/ai-sdk/file.ts +238 -0
  54. package/src/ai-sdk/generate-element.ts +92 -0
  55. package/src/ai-sdk/generate-music.ts +46 -0
  56. package/src/ai-sdk/generate-video.ts +165 -0
  57. package/src/ai-sdk/index.ts +72 -0
  58. package/src/ai-sdk/music-model.ts +110 -0
  59. package/src/ai-sdk/providers/editly/editly.test.ts +1108 -0
  60. package/src/ai-sdk/providers/editly/ffmpeg.ts +60 -0
  61. package/src/ai-sdk/providers/editly/index.ts +817 -0
  62. package/src/ai-sdk/providers/editly/layers.ts +776 -0
  63. package/src/ai-sdk/providers/editly/plan.md +144 -0
  64. package/src/ai-sdk/providers/editly/types.ts +328 -0
  65. package/src/ai-sdk/providers/elevenlabs-provider.ts +255 -0
  66. package/src/ai-sdk/providers/fal-provider.ts +512 -0
  67. package/src/ai-sdk/providers/higgsfield.ts +379 -0
  68. package/src/ai-sdk/providers/openai.ts +251 -0
  69. package/src/ai-sdk/providers/replicate.ts +16 -0
  70. package/src/ai-sdk/video-model.ts +185 -0
  71. package/src/cli/commands/find.tsx +137 -0
  72. package/src/cli/commands/help.tsx +85 -0
  73. package/src/cli/commands/index.ts +6 -0
  74. package/src/cli/commands/list.tsx +238 -0
  75. package/src/cli/commands/render.tsx +71 -0
  76. package/src/cli/commands/run.tsx +511 -0
  77. package/src/cli/commands/which.tsx +253 -0
  78. package/src/cli/index.ts +114 -0
  79. package/src/cli/quiet.ts +44 -0
  80. package/src/cli/types.ts +32 -0
  81. package/src/cli/ui/components/Badge.tsx +29 -0
  82. package/src/cli/ui/components/DataTable.tsx +51 -0
  83. package/src/cli/ui/components/Header.tsx +23 -0
  84. package/src/cli/ui/components/HelpBlock.tsx +44 -0
  85. package/src/cli/ui/components/KeyValue.tsx +33 -0
  86. package/src/cli/ui/components/OptionRow.tsx +81 -0
  87. package/src/cli/ui/components/Separator.tsx +23 -0
  88. package/src/cli/ui/components/StatusBox.tsx +108 -0
  89. package/src/cli/ui/components/VargBox.tsx +51 -0
  90. package/src/cli/ui/components/VargProgress.tsx +36 -0
  91. package/src/cli/ui/components/VargSpinner.tsx +34 -0
  92. package/src/cli/ui/components/VargText.tsx +56 -0
  93. package/src/cli/ui/components/index.ts +19 -0
  94. package/src/cli/ui/index.ts +12 -0
  95. package/src/cli/ui/render.ts +35 -0
  96. package/src/cli/ui/theme.ts +63 -0
  97. package/src/cli/utils.ts +78 -0
  98. package/src/core/executor/executor.ts +201 -0
  99. package/src/core/executor/index.ts +13 -0
  100. package/src/core/executor/job.ts +214 -0
  101. package/src/core/executor/pipeline.ts +222 -0
  102. package/src/core/index.ts +11 -0
  103. package/src/core/registry/index.ts +9 -0
  104. package/src/core/registry/loader.ts +149 -0
  105. package/src/core/registry/registry.ts +221 -0
  106. package/src/core/registry/resolver.ts +206 -0
  107. package/src/core/schema/helpers.ts +134 -0
  108. package/src/core/schema/index.ts +8 -0
  109. package/src/core/schema/shared.ts +102 -0
  110. package/src/core/schema/types.ts +279 -0
  111. package/src/core/schema/validator.ts +92 -0
  112. package/src/definitions/actions/captions.ts +261 -0
  113. package/src/definitions/actions/edit.ts +298 -0
  114. package/src/definitions/actions/image.ts +125 -0
  115. package/src/definitions/actions/index.ts +114 -0
  116. package/src/definitions/actions/music.ts +205 -0
  117. package/src/definitions/actions/sync.ts +128 -0
  118. package/{action/transcribe/index.ts → src/definitions/actions/transcribe.ts} +63 -90
  119. package/src/definitions/actions/upload.ts +111 -0
  120. package/src/definitions/actions/video.ts +163 -0
  121. package/src/definitions/actions/voice.ts +119 -0
  122. package/src/definitions/index.ts +23 -0
  123. package/src/definitions/models/elevenlabs.ts +50 -0
  124. package/src/definitions/models/flux.ts +56 -0
  125. package/src/definitions/models/index.ts +36 -0
  126. package/src/definitions/models/kling.ts +56 -0
  127. package/src/definitions/models/llama.ts +54 -0
  128. package/src/definitions/models/nano-banana-pro.ts +102 -0
  129. package/src/definitions/models/sonauto.ts +68 -0
  130. package/src/definitions/models/soul.ts +65 -0
  131. package/src/definitions/models/wan.ts +54 -0
  132. package/src/definitions/models/whisper.ts +44 -0
  133. package/src/definitions/skills/index.ts +12 -0
  134. package/src/definitions/skills/talking-character.ts +87 -0
  135. package/src/definitions/skills/text-to-tiktok.ts +97 -0
  136. package/src/index.ts +118 -0
  137. package/src/providers/apify.ts +269 -0
  138. package/src/providers/base.ts +264 -0
  139. package/src/providers/elevenlabs.ts +217 -0
  140. package/src/providers/fal.ts +392 -0
  141. package/src/providers/ffmpeg.ts +544 -0
  142. package/src/providers/fireworks.ts +193 -0
  143. package/src/providers/groq.ts +149 -0
  144. package/src/providers/higgsfield.ts +145 -0
  145. package/src/providers/index.ts +143 -0
  146. package/src/providers/replicate.ts +147 -0
  147. package/src/providers/storage.ts +206 -0
  148. package/src/react/cli.ts +52 -0
  149. package/src/react/elements.ts +146 -0
  150. package/src/react/examples/branching.tsx +66 -0
  151. package/src/react/examples/captions-demo.tsx +37 -0
  152. package/src/react/examples/character-video.tsx +84 -0
  153. package/src/react/examples/grid.tsx +53 -0
  154. package/src/react/examples/layouts-demo.tsx +57 -0
  155. package/src/react/examples/madi.tsx +60 -0
  156. package/src/react/examples/music-test.tsx +35 -0
  157. package/src/react/examples/onlyfans-1m/workflow.tsx +88 -0
  158. package/src/react/examples/orange-portrait.tsx +41 -0
  159. package/src/react/examples/split-element-demo.tsx +60 -0
  160. package/src/react/examples/split-layout-demo.tsx +60 -0
  161. package/src/react/examples/split.tsx +41 -0
  162. package/src/react/examples/video-grid.tsx +46 -0
  163. package/src/react/index.ts +43 -0
  164. package/src/react/layouts/grid.tsx +28 -0
  165. package/src/react/layouts/index.ts +2 -0
  166. package/src/react/layouts/split.tsx +20 -0
  167. package/src/react/react.test.ts +309 -0
  168. package/src/react/render.ts +21 -0
  169. package/src/react/renderers/animate.ts +59 -0
  170. package/src/react/renderers/captions.ts +297 -0
  171. package/src/react/renderers/clip.ts +248 -0
  172. package/src/react/renderers/context.ts +17 -0
  173. package/src/react/renderers/image.ts +109 -0
  174. package/src/react/renderers/index.ts +22 -0
  175. package/src/react/renderers/music.ts +60 -0
  176. package/src/react/renderers/packshot.ts +84 -0
  177. package/src/react/renderers/progress.ts +173 -0
  178. package/src/react/renderers/render.ts +243 -0
  179. package/src/react/renderers/slider.ts +69 -0
  180. package/src/react/renderers/speech.ts +53 -0
  181. package/src/react/renderers/split.ts +91 -0
  182. package/src/react/renderers/subtitle.ts +16 -0
  183. package/src/react/renderers/swipe.ts +75 -0
  184. package/src/react/renderers/title.ts +17 -0
  185. package/src/react/renderers/utils.ts +124 -0
  186. package/src/react/renderers/video.ts +127 -0
  187. package/src/react/runtime/jsx-dev-runtime.ts +43 -0
  188. package/src/react/runtime/jsx-runtime.ts +35 -0
  189. package/src/react/types.ts +232 -0
  190. package/src/studio/index.ts +26 -0
  191. package/src/studio/scanner.ts +102 -0
  192. package/src/studio/server.ts +554 -0
  193. package/src/studio/stages.ts +251 -0
  194. package/src/studio/step-renderer.ts +279 -0
  195. package/src/studio/types.ts +60 -0
  196. package/src/studio/ui/cache.html +303 -0
  197. package/src/studio/ui/index.html +1820 -0
  198. package/src/tests/all.test.ts +509 -0
  199. package/src/tests/index.ts +33 -0
  200. package/src/tests/unit.test.ts +403 -0
  201. package/tsconfig.cli.json +8 -0
  202. package/tsconfig.json +21 -3
  203. package/TEST_RESULTS.md +0 -122
  204. package/action/captions/SKILL.md +0 -170
  205. package/action/captions/index.ts +0 -227
  206. package/action/edit/SKILL.md +0 -235
  207. package/action/edit/index.ts +0 -493
  208. package/action/image/SKILL.md +0 -140
  209. package/action/image/index.ts +0 -112
  210. package/action/sync/SKILL.md +0 -136
  211. package/action/sync/index.ts +0 -187
  212. package/action/transcribe/SKILL.md +0 -179
  213. package/action/video/SKILL.md +0 -116
  214. package/action/video/index.ts +0 -135
  215. package/action/voice/SKILL.md +0 -125
  216. package/action/voice/index.ts +0 -201
  217. package/index.ts +0 -38
  218. package/lib/README.md +0 -144
  219. package/lib/ai-sdk/fal.ts +0 -106
  220. package/lib/ai-sdk/replicate.ts +0 -107
  221. package/lib/elevenlabs.ts +0 -382
  222. package/lib/fal.ts +0 -478
  223. package/lib/ffmpeg.ts +0 -467
  224. package/lib/fireworks.ts +0 -235
  225. package/lib/groq.ts +0 -246
  226. package/lib/higgsfield.ts +0 -176
  227. package/lib/remotion/SKILL.md +0 -823
  228. package/lib/remotion/cli.ts +0 -115
  229. package/lib/remotion/functions.ts +0 -283
  230. package/lib/remotion/index.ts +0 -19
  231. package/lib/remotion/templates.ts +0 -73
  232. package/lib/replicate.ts +0 -304
  233. package/output.txt +0 -1
  234. package/test-import.ts +0 -7
  235. package/test-services.ts +0 -97
  236. package/utilities/s3.ts +0 -147
@@ -0,0 +1,156 @@
1
+ # trendwatching pipeline
2
+
3
+ discover viral tiktok content for any topic or hashtag using apify scrapers
4
+
5
+ ## overview
6
+
7
+ use this pipeline to:
8
+ - find trending videos for a specific topic/hashtag
9
+ - analyze engagement metrics (plays, likes, shares, comments)
10
+ - get video urls for inspiration or downloading
11
+ - identify top creators in a niche
12
+
13
+ ## steps
14
+
15
+ ### 1. search by hashtag (save to file)
16
+
17
+ ```bash
18
+ # get 10 viral videos for a hashtag, save to output/
19
+ bun run lib/apify.ts run clockworks/tiktok-scraper '{"hashtags":["relationship"],"resultsPerPage":10}' tiktok-relationship.json
20
+ ```
21
+
22
+ ### 2. search multiple hashtags
23
+
24
+ ```bash
25
+ # search multiple related hashtags
26
+ bun run lib/apify.ts run clockworks/tiktok-scraper '{"hashtags":["viral","trending","fyp"],"resultsPerPage":5}' tiktok-viral.json
27
+ ```
28
+
29
+ ### 3. scrape specific profiles
30
+
31
+ ```bash
32
+ # get videos from specific creators
33
+ bun run lib/apify.ts run clockworks/tiktok-scraper '{"profiles":["@username"],"resultsPerPage":10}' creator-videos.json
34
+ ```
35
+
36
+ ### 4. get discover page trends
37
+
38
+ ```bash
39
+ # scrape tiktok discover page for trending topics
40
+ bun run lib/apify.ts run clockworks/tiktok-discover-scraper '{"hashtags":["fitness"]}' fitness-trends.json
41
+ ```
42
+
43
+ ### 5. download videos from saved json
44
+
45
+ download all videos from a saved json file using yt-dlp:
46
+
47
+ ```bash
48
+ # download all videos to output/videos/
49
+ bun run lib/apify.ts download tiktok-relationship.json
50
+
51
+ # download to custom directory
52
+ bun run lib/apify.ts download tiktok-relationship.json output/my-videos
53
+ ```
54
+
55
+ ### 6. read saved results
56
+
57
+ results are saved to `output/<filename>.json` - you can read them later:
58
+
59
+ ```bash
60
+ # list all video urls
61
+ cat output/tiktok-relationship.json | jq -r '.[].webVideoUrl'
62
+
63
+ # get top videos by play count
64
+ cat output/tiktok-relationship.json | jq 'sort_by(-.playCount) | .[0:3] | .[] | {url: .webVideoUrl, plays: .playCount}'
65
+ ```
66
+
67
+ ## output data
68
+
69
+ each video includes:
70
+ - `webVideoUrl` - tiktok video url
71
+ - `text` - video caption/description
72
+ - `playCount` - total views
73
+ - `diggCount` - likes
74
+ - `shareCount` - shares
75
+ - `commentCount` - comments
76
+ - `collectCount` - saves/bookmarks
77
+ - `authorMeta` - creator info (name, followers, etc.)
78
+ - `musicMeta` - audio/music info
79
+ - `hashtags` - all hashtags used
80
+ - `createTimeISO` - when posted
81
+
82
+ ## example output
83
+
84
+ ```json
85
+ {
86
+ "webVideoUrl": "https://www.tiktok.com/@user/video/123",
87
+ "text": "Not the tiny violin #prank #couples #relationship",
88
+ "playCount": 13200000,
89
+ "diggCount": 2900000,
90
+ "shareCount": 34700,
91
+ "commentCount": 11000,
92
+ "authorMeta": {
93
+ "name": "samandmonica",
94
+ "fans": 4000000
95
+ }
96
+ }
97
+ ```
98
+
99
+ ## available scrapers
100
+
101
+ | actor | use case |
102
+ |-------|----------|
103
+ | `clockworks/tiktok-scraper` | general hashtag/profile scraping |
104
+ | `clockworks/tiktok-hashtag-scraper` | hashtag-specific scraping |
105
+ | `clockworks/tiktok-profile-scraper` | creator profile scraping |
106
+ | `clockworks/tiktok-discover-scraper` | trending topics & discover page |
107
+ | `clockworks/tiktok-video-scraper` | specific video urls |
108
+ | `clockworks/tiktok-comments-scraper` | video comments |
109
+
110
+ ## pricing
111
+
112
+ apify pay-per-event pricing:
113
+ - actor start: $0.005
114
+ - per result: $0.003
115
+ - video download: $0.001 (optional)
116
+
117
+ example: 100 videos = ~$0.31
118
+
119
+ ## complete workflow example
120
+
121
+ ```bash
122
+ # 1. scrape viral relationship videos
123
+ bun run lib/apify.ts run clockworks/tiktok-scraper '{"hashtags":["relationship"],"resultsPerPage":5}' tiktok-relationship.json
124
+
125
+ # 2. check what we got
126
+ cat output/tiktok-relationship.json | jq -r '.[].webVideoUrl'
127
+
128
+ # 3. download all videos
129
+ bun run lib/apify.ts download tiktok-relationship.json
130
+
131
+ # 4. videos are now in output/videos/
132
+ ls output/videos/
133
+ ```
134
+
135
+ ## tips
136
+
137
+ - use `resultsPerPage` to limit results and costs
138
+ - combine multiple hashtags for broader search
139
+ - check `playCount` and `diggCount` ratio for engagement quality
140
+ - look at `createTimeISO` to find recent trending content
141
+ - save results to json to analyze patterns over time
142
+ - download videos for offline analysis or inspiration
143
+
144
+ ## environment
145
+
146
+ requires `APIFY_TOKEN` in `.env`
147
+
148
+ ```bash
149
+ APIFY_TOKEN=apify_api_xxx
150
+ ```
151
+
152
+ requires `yt-dlp` for video downloads:
153
+
154
+ ```bash
155
+ brew install yt-dlp
156
+ ```
package/plan.md ADDED
@@ -0,0 +1,281 @@
1
+ # Photify AI - Concept 4: Animated Family Photos
2
+
3
+ ## Overview
4
+ Create 15 video creatives showing "talking heads" emotionally reacting to animated vintage family photos. Each video uses the same 5 animated retro photos as background, with different characters wiping away tears.
5
+
6
+ ## Deliverables per video
7
+ - 2 formats: 4x5 (1080x1350) + 9x16 (1080x1920)
8
+ - CapCut-style dynamic subtitles
9
+ - Background music/sfx
10
+ - Packshot at end: `media/Packshot_9_16.mp4`
11
+
12
+ ## Assets Generated
13
+
14
+ ### 1. Animated Retro Photos (5 total, shared across all videos)
15
+ | # | Description | Image | Video |
16
+ |---|-------------|-------|-------|
17
+ | 1 | Young grandparents sitting side by side, holding hands | `media/retro/01_grandparents.jpg` | `media/retro/01_grandparents.mp4` |
18
+ | 2 | Woman holding cat, cat rubbing against her hand | `media/retro/02_woman_cat.jpg` | `media/retro/02_woman_cat.mp4` |
19
+ | 3 | New Year by old tree, two kids in costumes | `media/retro/03_newyear.jpg` | `media/retro/03_newyear.mp4` |
20
+ | 4 | Old person on porch of old house | `media/retro/04_elderly_porch.jpg` | `media/retro/04_elderly_porch.mp4` |
21
+ | 5 | Family dinner gathering | `media/retro/05_family_dinner.jpg` | `media/retro/05_family_dinner.mp4` |
22
+
23
+ ### 2. Talking Head Characters (15 total)
24
+ | # | Description | Portrait | Greenscreen |
25
+ |---|-------------|----------|-------------|
26
+ | 1 | Girl with long light brown hair (25-30) | `media/characters/01_light_brown_hair.jpg` | `media/characters/01_light_brown_hair_greenscreen.jpg` |
27
+ | 2 | Asian girl with short black hair and bangs | `media/characters/02_asian_bangs.jpg` | `media/characters/02_gs.jpg` |
28
+ | 3 | Girl with red curls and freckles | `media/characters/03_red_curls.jpg` | `media/characters/03_gs.jpg` |
29
+ | 4 | Girl with bright colored hair (pink/blue) | `media/characters/04_pink_hair.jpg` | `media/characters/04_gs.jpg` |
30
+ | 5 | Girl with bangs and long dark hair | `media/characters/05_dark_bangs.jpg` | `media/characters/05_gs.jpg` |
31
+ | 6 | Middle-aged woman with gray streaks | `media/characters/06_gray_streaks.jpg` | `media/characters/06_gs.jpg` |
32
+ | 7 | Girl with soft curls, warm skin tone | `media/characters/07_soft_curls.jpg` | `media/characters/07_gs.jpg` |
33
+ | 8 | Girl with shaved side, asymmetric cut | `media/characters/08_shaved_side.jpg` | `media/characters/08_gs.jpg` |
34
+ | 9 | Dark-skinned girl with afro curls | `media/characters/09_afro_curls.jpg` | `media/characters/09_gs.jpg` |
35
+ | 10 | Girl with long braids or dreads | `media/characters/10_long_braids.jpg` | `media/characters/10_gs.jpg` |
36
+ | 11 | Girl with glasses, bob haircut | `media/characters/11_glasses_bob.jpg` | `media/characters/11_gs.jpg` |
37
+ | 12 | Young girl with short cut, septum piercing | `media/characters/12_septum.jpg` | `media/characters/12_gs.jpg` |
38
+ | 13 | Girl with super long platinum white hair | `media/characters/13_platinum_hair.jpg` | `media/characters/13_gs.jpg` |
39
+ | 14 | Asian woman with neat haircut, glasses | `media/characters/14_asian_glasses.jpg` | `media/characters/14_gs.jpg` |
40
+ | 15 | Dark-skinned girl with wavy pink hair | `media/characters/15_pink_wig.jpg` | `media/characters/15_gs.jpg` |
41
+
42
+ ### 3. Audio Assets
43
+ | Asset | File | Duration |
44
+ |-------|------|----------|
45
+ | Voice (Rachel) | `output/scene01/voice.mp3` | ~7 seconds |
46
+ | Captions SRT | `output/scene01/captions.srt` | 0-5 seconds |
47
+
48
+ ### 4. Packshot
49
+ | Format | File | Duration |
50
+ |--------|------|----------|
51
+ | 9x16 | `media/Packshot_9_16.mp4` | 2.18s |
52
+ | 4x5 | `media/Packshot_4x5.mp4` | 2.18s (created from 9x16) |
53
+
54
+ ---
55
+
56
+ ## Script
57
+ **Voice:** "Guys... I animated old family photos with Photify, and realized my great-grandma had the exact same smile as me."
58
+
59
+ **Action:** Character wipes tears (touched, not overly dramatic)
60
+
61
+ ---
62
+
63
+ ## Captions SRT Format
64
+ ```srt
65
+ 1
66
+ 00:00:00,000 --> 00:00:01,500
67
+ Guys...
68
+
69
+ 2
70
+ 00:00:01,500 --> 00:00:03,500
71
+ I animated old family photos
72
+ with Photify
73
+
74
+ 3
75
+ 00:00:03,500 --> 00:00:05,000
76
+ and realized my great-grandma
77
+ had the exact same smile as me
78
+ ```
79
+
80
+ ---
81
+
82
+ ## FINAL WORKING COMMANDS (Scene 01)
83
+
84
+ ### Step 1: Generate Lipsync Video
85
+ ```bash
86
+ bun run src/cli/index.ts run sync \
87
+ --image media/characters/01_light_brown_hair_greenscreen.jpg \
88
+ --audio output/scene01/voice.mp3 \
89
+ --prompt "woman speaking emotionally, warm expression" \
90
+ --duration 10 \
91
+ --resolution 720p \
92
+ --quiet
93
+ ```
94
+
95
+ **Output:** `output/scene01/talking_head_v2.mp4` (10 seconds, greenscreen background)
96
+
97
+ ### Step 2: Composite 9x16 (1080x1920)
98
+ ```bash
99
+ ffmpeg -y \
100
+ -i media/retro/01_grandparents.mp4 \
101
+ -stream_loop 1 -i media/retro/01_grandparents.mp4 \
102
+ -i output/scene01/talking_head_v2.mp4 \
103
+ -filter_complex "\
104
+ [0:v][1:v]concat=n=2:v=1:a=0[bglong]; \
105
+ [bglong]scale=1080:1920:force_original_aspect_ratio=increase,crop=1080:1920,setsar=1[bg]; \
106
+ [2:v]chromakey=0x00ff00:0.3:0.1,crop=600:700:250:50,scale=420:-1[fg]; \
107
+ [bg][fg]overlay=(W-w)/2:H-h-650[out]" \
108
+ -map "[out]" -map 2:a -t 7 output/scene01/composite_9x16.mp4
109
+ ```
110
+
111
+ **Key parameters for 9x16:**
112
+ - `scale=420:-1` - talking head width (420px, larger)
113
+ - `H-h-650` - position from bottom (650px up = upper-middle area)
114
+ - `crop=600:700:250:50` - crop talking head video (600x700, offset 250,50 to center on face)
115
+
116
+ ### Step 3: Add Captions to 9x16
117
+ ```bash
118
+ ffmpeg -y -i output/scene01/composite_9x16.mp4 \
119
+ -vf "subtitles=/Users/aleks/Github/varghq/sdk/output/scene01/captions.srt:force_style='FontSize=18,PrimaryColour=&H00FFFFFF,OutlineColour=&H00000000,Outline=2,Bold=1,Alignment=2,MarginV=15'" \
120
+ -c:a copy output/scene01/captioned_9x16.mp4
121
+ ```
122
+
123
+ **Key parameters:**
124
+ - `FontSize=18` - smaller font (standard for social media)
125
+ - `MarginV=15` - 15px from bottom edge
126
+ - `Alignment=2` - bottom center
127
+ - **MUST use absolute path for subtitles file**
128
+
129
+ ### Step 4: Add Packshot to 9x16
130
+ ```bash
131
+ ffmpeg -y -i output/scene01/captioned_9x16.mp4 -i media/Packshot_9_16.mp4 \
132
+ -filter_complex "[0:v]fps=24[v0];[1:v]fps=24[v1];[v0][0:a][v1][1:a]concat=n=2:v=1:a=1[outv][outa]" \
133
+ -map "[outv]" -map "[outa]" output/scene01/final_9x16_v4.mp4
134
+ ```
135
+
136
+ ### Step 5: Composite 4x5 (1080x1350)
137
+ ```bash
138
+ ffmpeg -y \
139
+ -i media/retro/01_grandparents.mp4 \
140
+ -stream_loop 1 -i media/retro/01_grandparents.mp4 \
141
+ -i output/scene01/talking_head_v2.mp4 \
142
+ -filter_complex "\
143
+ [0:v][1:v]concat=n=2:v=1:a=0[bglong]; \
144
+ [bglong]scale=1080:1350:force_original_aspect_ratio=increase,crop=1080:1350,setsar=1[bg]; \
145
+ [2:v]chromakey=0x00ff00:0.3:0.1,crop=600:700:250:50,scale=420:-1[fg]; \
146
+ [bg][fg]overlay=(W-w)/2:H-h-550[out]" \
147
+ -map "[out]" -map 2:a -t 7 output/scene01/composite_4x5.mp4
148
+ ```
149
+
150
+ **Key parameters for 4x5:**
151
+ - `scale=420:-1` - same talking head width
152
+ - `H-h-550` - position from bottom (550px up for shorter frame)
153
+ - Background scales to 1080x1350 instead of 1920
154
+
155
+ ### Step 6: Add Captions to 4x5
156
+ ```bash
157
+ ffmpeg -y -i output/scene01/composite_4x5.mp4 \
158
+ -vf "subtitles=/Users/aleks/Github/varghq/sdk/output/scene01/captions.srt:force_style='FontSize=18,PrimaryColour=&H00FFFFFF,OutlineColour=&H00000000,Outline=2,Bold=1,Alignment=2,MarginV=15'" \
159
+ -c:a copy output/scene01/captioned_4x5.mp4
160
+ ```
161
+
162
+ ### Step 7: Create 4x5 Packshot (one-time)
163
+ ```bash
164
+ ffmpeg -y -i media/Packshot_9_16.mp4 \
165
+ -vf "scale=1080:1350:force_original_aspect_ratio=increase,crop=1080:1350" \
166
+ -c:a copy media/Packshot_4x5.mp4
167
+ ```
168
+
169
+ ### Step 8: Add Packshot to 4x5
170
+ ```bash
171
+ ffmpeg -y -i output/scene01/captioned_4x5.mp4 -i media/Packshot_4x5.mp4 \
172
+ -filter_complex "[0:v]fps=24[v0];[1:v]fps=24[v1];[v0][0:a][v1][1:a]concat=n=2:v=1:a=1[outv][outa]" \
173
+ -map "[outv]" -map "[outa]" output/scene01/final_4x5_v4.mp4
174
+ ```
175
+
176
+ ---
177
+
178
+ ## CRITICAL LESSONS LEARNED
179
+
180
+ ### 1. Subtitles Filter Issues
181
+ - **MUST use absolute path** for subtitle file: `subtitles=/full/path/to/captions.srt`
182
+ - Relative paths like `subtitles='output/scene01/captions.srt'` may silently fail
183
+ - MarginV is distance from BOTTOM edge when Alignment=2
184
+
185
+ ### 2. Talking Head Positioning
186
+ - **9x16 (1920 height):** Use `H-h-650` to position character in upper-middle
187
+ - **4x5 (1350 height):** Use `H-h-550` for similar visual position
188
+ - Character should be ABOVE captions, not overlapping
189
+ - Larger head size (420px width) looks better than smaller (260-320px)
190
+
191
+ ### 3. Caption Positioning
192
+ - **FontSize=18** is good for social media (not too big, not too small)
193
+ - **MarginV=15** places captions at very bottom
194
+ - Higher MarginV values push captions UP (e.g., MarginV=500 = 500px from bottom)
195
+ - For 4x5, use same MarginV as 9x16 since captions should be at bottom in both
196
+
197
+ ### 4. Chromakey Settings
198
+ - `chromakey=0x00ff00:0.3:0.1` works well for bright green backgrounds
199
+ - 0x00ff00 = pure green
200
+ - 0.3 = similarity threshold
201
+ - 0.1 = blend/edge softness
202
+
203
+ ### 5. Crop for Face Focus
204
+ - `crop=600:700:250:50` crops the lipsync video to focus on face
205
+ - Format: `crop=width:height:x_offset:y_offset`
206
+ - Adjust x_offset and y_offset based on face position in source video
207
+
208
+ ### 6. Background Looping
209
+ - Use `-stream_loop 1` to loop short backgrounds
210
+ - Animated photos are ~5 seconds, need ~7 seconds for full audio
211
+ - Concat looped background before compositing
212
+
213
+ ### 7. FPS Alignment for Concat
214
+ - Both videos must have same FPS for clean concat
215
+ - Use `fps=24` filter on both before concat
216
+ - Packshot may be 60fps, main content is 24fps
217
+
218
+ ### 8. Color Format for ASS Styles
219
+ - Use `&H00FFFFFF` format (AABBGGRR in hex)
220
+ - `&H00FFFFFF` = white
221
+ - `&H00000000` = black
222
+ - The `00` prefix is alpha (00 = opaque)
223
+
224
+ ---
225
+
226
+ ## Scene 01 Final Output
227
+
228
+ | Format | File | Resolution | Duration |
229
+ |--------|------|------------|----------|
230
+ | 9x16 | `output/scene01/final_9x16_v4.mp4` | 1080x1920 | ~9.2s |
231
+ | 4x5 | `output/scene01/final_4x5_v4.mp4` | 1080x1350 | ~9.2s |
232
+
233
+ ---
234
+
235
+ ## Remaining Work
236
+
237
+ ### Scenes 02-15
238
+ Each scene needs:
239
+ 1. Generate lipsync video using greenscreen character image
240
+ 2. Create composite (9x16 and 4x5)
241
+ 3. Add captions
242
+ 4. Add packshot
243
+ 5. Export finals
244
+
245
+ ### Batch Processing Template
246
+ ```bash
247
+ # For each scene N (02-15):
248
+ SCENE="02"
249
+ CHAR_IMG="media/characters/02_gs.jpg"
250
+ BG_VIDEO="media/retro/02_woman_cat.mp4"
251
+
252
+ # Create scene folder
253
+ mkdir -p output/scene${SCENE}
254
+ cp output/scene01/voice.mp3 output/scene${SCENE}/
255
+ cp output/scene01/captions.srt output/scene${SCENE}/
256
+
257
+ # Generate lipsync
258
+ bun run src/cli/index.ts run sync \
259
+ --image ${CHAR_IMG} \
260
+ --audio output/scene${SCENE}/voice.mp3 \
261
+ --prompt "woman speaking emotionally, warm expression" \
262
+ --duration 10 \
263
+ --resolution 720p \
264
+ --output output/scene${SCENE}/talking_head.mp4
265
+
266
+ # Then run composite/caption/packshot commands as above
267
+ ```
268
+
269
+ ---
270
+
271
+ ## Cost Tracking
272
+
273
+ | Item | Cost |
274
+ |------|------|
275
+ | Flux Pro images (5 retro + 15 chars + 15 greenscreen) | ~$0.80 |
276
+ | Kling video (5 animations) | ~$2.10 |
277
+ | Wan-25 lipsync (1 so far) | ~$0.50 |
278
+ | ElevenLabs TTS | ~$0.05 |
279
+ | **Total so far** | **~$3.45** |
280
+
281
+ Estimated for all 15 scenes: ~$10-12 total
File without changes
@@ -0,0 +1,142 @@
1
+ export interface CacheStorage {
2
+ get(key: string): Promise<unknown | undefined>;
3
+ set(key: string, value: unknown, ttl?: number): Promise<void>;
4
+ delete(key: string): Promise<void>;
5
+ }
6
+
7
+ export interface WithCacheOptions {
8
+ ttl?: number | string;
9
+ storage?: CacheStorage;
10
+ }
11
+
12
+ type CacheKeyDeps = (string | number | boolean | null | undefined)[];
13
+
14
+ type WithCacheKey<T> = Omit<T, "cacheKey"> & { cacheKey?: CacheKeyDeps };
15
+
16
+ type CachedFn<T, R> = (options: WithCacheKey<T>) => Promise<R>;
17
+
18
+ const memoryCache = new Map<string, { value: unknown; expires: number }>();
19
+
20
+ const defaultStorage: CacheStorage = {
21
+ async get(key: string) {
22
+ const entry = memoryCache.get(key);
23
+ if (!entry) return undefined;
24
+ if (entry.expires && Date.now() > entry.expires) {
25
+ memoryCache.delete(key);
26
+ return undefined;
27
+ }
28
+ return entry.value;
29
+ },
30
+ async set(key: string, value: unknown, ttl?: number) {
31
+ const expires = ttl ? Date.now() + ttl : 0;
32
+ memoryCache.set(key, { value, expires });
33
+ },
34
+ async delete(key: string) {
35
+ memoryCache.delete(key);
36
+ },
37
+ };
38
+
39
+ function parseTTL(ttl: number | string | undefined): number | undefined {
40
+ if (ttl === undefined) return undefined;
41
+ if (typeof ttl === "number") return ttl;
42
+
43
+ const match = ttl.match(/^(\d+)(s|m|h|d)$/);
44
+ if (!match) return undefined;
45
+
46
+ const value = Number.parseInt(match[1]!, 10);
47
+ const unit = match[2];
48
+
49
+ switch (unit) {
50
+ case "s":
51
+ return value * 1000;
52
+ case "m":
53
+ return value * 60 * 1000;
54
+ case "h":
55
+ return value * 60 * 60 * 1000;
56
+ case "d":
57
+ return value * 24 * 60 * 60 * 1000;
58
+ default:
59
+ return undefined;
60
+ }
61
+ }
62
+
63
+ function depsToKey(prefix: string, deps: CacheKeyDeps): string {
64
+ const depsStr = deps.map((d) => String(d ?? "")).join(":");
65
+ return prefix ? `${prefix}:${depsStr}` : depsStr;
66
+ }
67
+
68
+ function flatten(value: unknown): unknown {
69
+ if (value === null || value === undefined) return value;
70
+ if (value instanceof Uint8Array) return value;
71
+ if (Array.isArray(value)) return value.map(flatten);
72
+ if (typeof value === "object") {
73
+ const obj = value as Record<string, unknown>;
74
+ const result: Record<string, unknown> = {};
75
+ for (const key of Object.keys(obj)) {
76
+ result[key] = flatten(obj[key]);
77
+ }
78
+ const proto = Object.getPrototypeOf(obj);
79
+ if (proto && proto !== Object.prototype) {
80
+ const descriptors = Object.getOwnPropertyDescriptors(proto);
81
+ for (const [key, desc] of Object.entries(descriptors)) {
82
+ if (desc.get && key !== "constructor") {
83
+ result[key] = flatten(desc.get.call(obj));
84
+ }
85
+ }
86
+ }
87
+ return result;
88
+ }
89
+ return value;
90
+ }
91
+
92
+ /**
93
+ * Wrap an async function to add caching via `cacheKey` option.
94
+ *
95
+ * @example
96
+ * ```ts
97
+ * import { generateImage } from "ai";
98
+ * import { withCache } from "./cache";
99
+ *
100
+ * const generateImage_ = withCache(generateImage);
101
+ *
102
+ * const { images } = await generateImage_({
103
+ * model: fal.imageModel("flux-schnell"),
104
+ * prompt: "lion roaring",
105
+ * cacheKey: ["lion", take], // cache based on deps
106
+ * });
107
+ * ```
108
+ */
109
+ const DEFAULT_TTL = "1h";
110
+
111
+ export function withCache<T extends object, R>(
112
+ fn: (options: T) => Promise<R>,
113
+ options: WithCacheOptions = {},
114
+ ): CachedFn<T, R> {
115
+ const storage = options.storage ?? defaultStorage;
116
+ const ttl = parseTTL(options.ttl ?? DEFAULT_TTL);
117
+ const prefix = fn.name || "anonymous";
118
+
119
+ return async (opts: WithCacheKey<T>): Promise<R> => {
120
+ const { cacheKey, ...rest } = opts;
121
+
122
+ if (!cacheKey) {
123
+ return fn(rest as T);
124
+ }
125
+
126
+ const key = depsToKey(prefix, cacheKey);
127
+ const cached = await storage.get(key);
128
+ if (cached !== undefined) {
129
+ return cached as R;
130
+ }
131
+
132
+ const result = await fn(rest as T);
133
+ const flattened = flatten(result);
134
+ await storage.set(key, flattened, ttl);
135
+
136
+ return result;
137
+ };
138
+ }
139
+
140
+ export function clearCache(): void {
141
+ memoryCache.clear();
142
+ }
@@ -0,0 +1,53 @@
1
+ import { generateImage as _generateImage } from "ai";
2
+ import {
3
+ generateVideo as _generateVideo,
4
+ fal,
5
+ fileCache,
6
+ withCache,
7
+ } from "../index";
8
+
9
+ const storage = fileCache({ dir: ".cache/ai" });
10
+ const generateImage = withCache(_generateImage, { storage });
11
+ const generateVideo = withCache(_generateVideo, { storage });
12
+
13
+ async function main() {
14
+ const take = 1;
15
+
16
+ // cached video generation
17
+ console.log("generating video...");
18
+ console.time("first");
19
+ const { video } = await generateVideo({
20
+ model: fal.videoModel("wan-2.5"),
21
+ prompt: "a cat playing piano",
22
+ duration: 5,
23
+ cacheKey: ["cat-piano", take],
24
+ });
25
+ console.timeEnd("first");
26
+ console.log(`video: ${video.uint8Array.byteLength} bytes`);
27
+
28
+ // same cacheKey = instant cache hit
29
+ console.log("\nsame cacheKey (from cache)...");
30
+ console.time("cached");
31
+ const { video: video2 } = await generateVideo({
32
+ model: fal.videoModel("wan-2.5"),
33
+ prompt: "a cat playing piano",
34
+ duration: 5,
35
+ cacheKey: ["cat-piano", take],
36
+ });
37
+ console.timeEnd("cached");
38
+ console.log(`video: ${video2.uint8Array.byteLength} bytes`);
39
+
40
+ // cached image generation
41
+ console.log("\ngenerating image...");
42
+ const { images } = await generateImage({
43
+ model: fal.imageModel("flux-schnell"),
44
+ prompt: "cyberpunk cityscape",
45
+ n: 1,
46
+ cacheKey: ["cyberpunk-city", 1],
47
+ });
48
+ console.log(`image: ${images[0]?.uint8Array.byteLength} bytes`);
49
+
50
+ console.log("\ndone!");
51
+ }
52
+
53
+ main().catch(console.error);
@@ -0,0 +1,53 @@
1
+ import { generateImage } from "ai";
2
+ import { File, fal, generateVideo } from "../index";
3
+
4
+ async function main() {
5
+ console.log("=== taisa solo closeup - scene 4 ===\n");
6
+
7
+ const referenceImages = [
8
+ File.fromPath("media/taisa/taisa.jpg"),
9
+ File.fromPath("output/original-frame-1m08s.png"),
10
+ ];
11
+
12
+ const imageContents = await Promise.all(
13
+ referenceImages.map((f) => f.arrayBuffer()),
14
+ );
15
+
16
+ console.log("generating closeup frame with nano-banana-pro/edit...");
17
+ const { image } = await generateImage({
18
+ model: fal.imageModel("nano-banana-pro/edit"),
19
+ prompt: {
20
+ images: imageContents,
21
+ text: "Closeup portrait of dark-haired brunette woman Taisa singing passionately on concert stage, dramatic purple blue stage lighting, emotional expression, glamorous dress, beautiful face, professional concert hall background blurred",
22
+ },
23
+ aspectRatio: "16:9",
24
+ n: 1,
25
+ providerOptions: {
26
+ fal: { resolution: "1K" },
27
+ },
28
+ });
29
+
30
+ await Bun.write("output/duet-frame-4.png", image.uint8Array);
31
+ console.log(
32
+ `frame saved: output/duet-frame-4.png (${image.uint8Array.byteLength} bytes)\n`,
33
+ );
34
+
35
+ console.log("animating 10s with kling-v2.5...");
36
+ const { video } = await generateVideo({
37
+ model: fal.videoModel("kling-v2.5"),
38
+ prompt: {
39
+ images: [image.uint8Array],
40
+ text: "closeup of woman singing passionately, subtle head movements, lips moving as singing, natural breathing, blinking, emotional expressions, concert atmosphere with stage lighting",
41
+ },
42
+ duration: 10,
43
+ });
44
+
45
+ await Bun.write("output/duet-scene-4.mp4", video.uint8Array);
46
+ console.log(
47
+ `video saved: output/duet-scene-4.mp4 (${video.uint8Array.byteLength} bytes)`,
48
+ );
49
+
50
+ console.log("\ndone!");
51
+ }
52
+
53
+ main().catch(console.error);