scrollcraft 2.0.5 → 2.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -21
- package/dist/cli/fal-service.js +1 -1
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +148 -72
- package/dist/cli/processor.d.ts +1 -0
- package/dist/cli/processor.js +13 -6
- package/dist/core/CoreEngine.d.ts +31 -13
- package/dist/core/index.d.ts +3 -0
- package/dist/core/scrollcraft.umd.min.js +1 -1
- package/dist/core/types.d.ts +6 -76
- package/dist/core/types.js +1 -1
- package/dist/pipeline/browser-driver.d.ts +31 -0
- package/dist/pipeline/browser-driver.js +176 -0
- package/dist/pipeline/fal-service.d.ts +15 -0
- package/dist/pipeline/fal-service.js +101 -0
- package/dist/pipeline/index.d.ts +21 -0
- package/dist/pipeline/index.js +223 -0
- package/dist/pipeline/node-driver.d.ts +18 -0
- package/dist/pipeline/node-driver.js +108 -0
- package/dist/pipeline/types.d.ts +43 -0
- package/dist/pipeline/types.js +2 -0
- package/dist/react/ScrollCraftProvider.d.ts +9 -8
- package/dist/react/index.js +1 -1
- package/package.json +20 -19
package/dist/core/types.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* SCROLLCRAFT
|
|
2
|
+
* SCROLLCRAFT - DECLARATIVE SCHEMA
|
|
3
3
|
*
|
|
4
4
|
* This file defines the core data structures that allow an AI Agent
|
|
5
5
|
* to describe a scroll experience in one step.
|
|
@@ -11,12 +11,12 @@ export interface ProjectConfiguration {
|
|
|
11
11
|
timeline: TimelineDefinition;
|
|
12
12
|
}
|
|
13
13
|
export interface ProjectSettings {
|
|
14
|
-
fps: number;
|
|
15
14
|
baseResolution: {
|
|
16
15
|
width: number;
|
|
17
16
|
height: number;
|
|
18
17
|
};
|
|
19
18
|
scrollMode: 'vh' | 'px';
|
|
19
|
+
basePath?: string;
|
|
20
20
|
}
|
|
21
21
|
/**
|
|
22
22
|
* ASSET SYSTEM
|
|
@@ -32,8 +32,11 @@ export interface AssetVariant {
|
|
|
32
32
|
path: string;
|
|
33
33
|
aspectRatio: string;
|
|
34
34
|
frameCount: number;
|
|
35
|
+
width: number;
|
|
36
|
+
height: number;
|
|
37
|
+
orientation: 'portrait' | 'landscape';
|
|
35
38
|
hasDepthMap?: boolean;
|
|
36
|
-
|
|
39
|
+
subjects?: string[];
|
|
37
40
|
}
|
|
38
41
|
export interface SubjectFrameData {
|
|
39
42
|
frame: number;
|
|
@@ -97,76 +100,3 @@ export interface LayerAnimation {
|
|
|
97
100
|
end: number;
|
|
98
101
|
easing?: string;
|
|
99
102
|
}
|
|
100
|
-
/**
|
|
101
|
-
* LEGACY TYPES (V1 Compatibility)
|
|
102
|
-
* These are required for existing modules to compile.
|
|
103
|
-
*/
|
|
104
|
-
export interface BlockInstanceInterface {
|
|
105
|
-
destroy(): void;
|
|
106
|
-
resize(params: {
|
|
107
|
-
wiWidth: number;
|
|
108
|
-
wiHeight: number;
|
|
109
|
-
}): void;
|
|
110
|
-
}
|
|
111
|
-
export type MediaGroupPositionAndSize = {
|
|
112
|
-
bgSize?: 'contain' | 'cover' | 'custom';
|
|
113
|
-
bgPosition?: {
|
|
114
|
-
x: number;
|
|
115
|
-
y: number;
|
|
116
|
-
};
|
|
117
|
-
};
|
|
118
|
-
export type ImagesUrlListArray = Array<string | {
|
|
119
|
-
i: string;
|
|
120
|
-
dur?: number;
|
|
121
|
-
}>;
|
|
122
|
-
export type ImageGroupUrlList = MediaGroupPositionAndSize & {
|
|
123
|
-
type: 'urlList';
|
|
124
|
-
prefix?: string;
|
|
125
|
-
suffix?: string;
|
|
126
|
-
images: ImagesUrlListArray;
|
|
127
|
-
duration?: number;
|
|
128
|
-
};
|
|
129
|
-
export type ImageGroupGap = {
|
|
130
|
-
type: 'gap';
|
|
131
|
-
duration: number;
|
|
132
|
-
};
|
|
133
|
-
export type VideoGroupUrl = MediaGroupPositionAndSize & {
|
|
134
|
-
type: 'urlVideo';
|
|
135
|
-
video?: string;
|
|
136
|
-
duration?: number;
|
|
137
|
-
};
|
|
138
|
-
export type ImageGroupWp = MediaGroupPositionAndSize & {
|
|
139
|
-
type: 'wpMedia';
|
|
140
|
-
images: number[];
|
|
141
|
-
duration?: number;
|
|
142
|
-
};
|
|
143
|
-
export type VideoGroupWp = MediaGroupPositionAndSize & {
|
|
144
|
-
type: 'wpVideo';
|
|
145
|
-
video?: number;
|
|
146
|
-
duration?: number;
|
|
147
|
-
};
|
|
148
|
-
export type MediaGroup = ImageGroupGap | ImageGroupUrlList | VideoGroupUrl | ImageGroupWp | VideoGroupWp;
|
|
149
|
-
export type MediaGroups = MediaGroup[];
|
|
150
|
-
export type AttributesCanvas2d = {
|
|
151
|
-
mediaGroups: MediaGroups;
|
|
152
|
-
markers: boolean;
|
|
153
|
-
scrub: number;
|
|
154
|
-
triggerStart: number;
|
|
155
|
-
triggerEnd: number;
|
|
156
|
-
pin: boolean;
|
|
157
|
-
pinSpacing: boolean;
|
|
158
|
-
};
|
|
159
|
-
export type AttributesScene = {
|
|
160
|
-
scenePosition: 'flow' | 'sticky' | 'fixed';
|
|
161
|
-
stickyType: 'css' | 'js';
|
|
162
|
-
height: number;
|
|
163
|
-
heightUnit: string;
|
|
164
|
-
duration: number;
|
|
165
|
-
durationUnit: string;
|
|
166
|
-
top: number;
|
|
167
|
-
topUnit: string;
|
|
168
|
-
markers: boolean;
|
|
169
|
-
scrub: number;
|
|
170
|
-
triggerStart: number;
|
|
171
|
-
triggerEnd: number;
|
|
172
|
-
};
|
package/dist/core/types.js
CHANGED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { IPipelineDriver, VariantConfig } from './types';
|
|
2
|
+
export declare class BrowserDriver implements IPipelineDriver {
|
|
3
|
+
private files;
|
|
4
|
+
constructor();
|
|
5
|
+
readFile(path: string): Promise<Uint8Array>;
|
|
6
|
+
writeFile(path: string, data: Uint8Array | string): Promise<void>;
|
|
7
|
+
mkdir(path: string): Promise<void>;
|
|
8
|
+
exists(path: string): Promise<boolean>;
|
|
9
|
+
readdir(dirPath: string): Promise<string[]>;
|
|
10
|
+
remove(path: string): Promise<void>;
|
|
11
|
+
join(...parts: string[]): string;
|
|
12
|
+
resolve(...parts: string[]): string;
|
|
13
|
+
/**
|
|
14
|
+
* EXTRACT FRAMES (via ffmpeg.wasm)
|
|
15
|
+
* Note: Requires SharedArrayBuffer & specific Headers if using multithreading.
|
|
16
|
+
*/
|
|
17
|
+
extractFrames(videoSource: string | File | Blob, outputDir: string, onProgress?: (percent: number) => void): Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* PROCESS IMAGE (via Canvas API)
|
|
20
|
+
* High-performance resizing and cropping using the browser's hardware-accelerated Canvas.
|
|
21
|
+
*/
|
|
22
|
+
processImage(input: Uint8Array | string, config: VariantConfig, options?: {
|
|
23
|
+
grayscale?: boolean;
|
|
24
|
+
blur?: number;
|
|
25
|
+
}): Promise<Uint8Array>;
|
|
26
|
+
/**
|
|
27
|
+
* ZIP PROJECT (via JSZip)
|
|
28
|
+
* Bundles all processed assets into a single file for upload or download.
|
|
29
|
+
*/
|
|
30
|
+
zipProject(outDir: string): Promise<Uint8Array>;
|
|
31
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.BrowserDriver = void 0;
|
|
37
|
+
class BrowserDriver {
|
|
38
|
+
files = new Map();
|
|
39
|
+
constructor() {
|
|
40
|
+
console.log('🌐 BrowserDriver initialized');
|
|
41
|
+
}
|
|
42
|
+
async readFile(path) {
|
|
43
|
+
const data = this.files.get(path);
|
|
44
|
+
if (!data)
|
|
45
|
+
throw new Error(`File not found: ${path}`);
|
|
46
|
+
if (typeof data === 'string')
|
|
47
|
+
return new TextEncoder().encode(data);
|
|
48
|
+
return data;
|
|
49
|
+
}
|
|
50
|
+
async writeFile(path, data) {
|
|
51
|
+
this.files.set(path, data);
|
|
52
|
+
}
|
|
53
|
+
async mkdir(path) {
|
|
54
|
+
// Virtual folders - no-op for simple Map implementation
|
|
55
|
+
}
|
|
56
|
+
async exists(path) {
|
|
57
|
+
return this.files.has(path);
|
|
58
|
+
}
|
|
59
|
+
async readdir(dirPath) {
|
|
60
|
+
const results = [];
|
|
61
|
+
for (const key of this.files.keys()) {
|
|
62
|
+
if (key.startsWith(dirPath)) {
|
|
63
|
+
// Simple relative path extraction
|
|
64
|
+
const relative = key.replace(dirPath, '').replace(/^[\\\/]/, '');
|
|
65
|
+
if (relative && !relative.includes('/') && !relative.includes('\\')) {
|
|
66
|
+
results.push(relative);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return results;
|
|
71
|
+
}
|
|
72
|
+
async remove(path) {
|
|
73
|
+
this.files.delete(path);
|
|
74
|
+
}
|
|
75
|
+
join(...parts) {
|
|
76
|
+
return parts.join('/').replace(/\/+/g, '/');
|
|
77
|
+
}
|
|
78
|
+
resolve(...parts) {
|
|
79
|
+
return this.join(...parts);
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* EXTRACT FRAMES (via ffmpeg.wasm)
|
|
83
|
+
* Note: Requires SharedArrayBuffer & specific Headers if using multithreading.
|
|
84
|
+
*/
|
|
85
|
+
async extractFrames(videoSource, outputDir, onProgress) {
|
|
86
|
+
try {
|
|
87
|
+
// Dynamic import to keep core bundle small
|
|
88
|
+
const { FFmpeg } = await Promise.resolve().then(() => __importStar(require('@ffmpeg/ffmpeg')));
|
|
89
|
+
const { fetchFile, toBlobURL } = await Promise.resolve().then(() => __importStar(require('@ffmpeg/util')));
|
|
90
|
+
const ffmpeg = new FFmpeg();
|
|
91
|
+
ffmpeg.on('progress', ({ progress }) => {
|
|
92
|
+
if (onProgress)
|
|
93
|
+
onProgress(Math.round(progress * 100));
|
|
94
|
+
});
|
|
95
|
+
// Load FFmpeg WASM
|
|
96
|
+
// You'll need to provide the correct URL for the core/worker files in your WP plugin
|
|
97
|
+
await ffmpeg.load({
|
|
98
|
+
coreURL: await toBlobURL(`https://unpkg.com/@ffmpeg/core@0.12.6/dist/esm/ffmpeg-core.js`, 'text/javascript'),
|
|
99
|
+
wasmURL: await toBlobURL(`https://unpkg.com/@ffmpeg/core@0.12.6/dist/esm/ffmpeg-core.wasm`, 'application/wasm'),
|
|
100
|
+
});
|
|
101
|
+
const inputName = 'input.mp4';
|
|
102
|
+
await ffmpeg.writeFile(inputName, await fetchFile(videoSource));
|
|
103
|
+
// Extract as PNGs/WebPs (WebP might be faster if supported in the WASM build)
|
|
104
|
+
await ffmpeg.exec(['-i', inputName, `${outputDir}/frame_%04d.png`]);
|
|
105
|
+
// Move files from FFmpeg VFS to our Map FS
|
|
106
|
+
const files = await ffmpeg.listDir(outputDir);
|
|
107
|
+
for (const file of files) {
|
|
108
|
+
if (file.name.startsWith('frame_')) {
|
|
109
|
+
const data = await ffmpeg.readFile(`${outputDir}/${file.name}`);
|
|
110
|
+
await this.writeFile(this.join(outputDir, file.name), data);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
await ffmpeg.terminate();
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
console.error('FFmpeg WASM Error:', err);
|
|
117
|
+
throw new Error('Failed to extract frames in browser. Did you enable SharedArrayBuffer headers?');
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* PROCESS IMAGE (via Canvas API)
|
|
122
|
+
* High-performance resizing and cropping using the browser's hardware-accelerated Canvas.
|
|
123
|
+
*/
|
|
124
|
+
async processImage(input, config, options = {}) {
|
|
125
|
+
// 1. Load image into a bitmap
|
|
126
|
+
let blob;
|
|
127
|
+
if (typeof input === 'string') {
|
|
128
|
+
const data = await this.readFile(input);
|
|
129
|
+
blob = new Blob([data]);
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
blob = new Blob([input]);
|
|
133
|
+
}
|
|
134
|
+
const img = await createImageBitmap(blob);
|
|
135
|
+
// 2. Setup Canvas
|
|
136
|
+
const canvas = new OffscreenCanvas(config.width, config.height);
|
|
137
|
+
const ctx = canvas.getContext('2d');
|
|
138
|
+
if (!ctx)
|
|
139
|
+
throw new Error('Could not get Canvas context');
|
|
140
|
+
// 3. Smart Crop Logic (simplified to cover/center)
|
|
141
|
+
const scale = Math.max(config.width / img.width, config.height / img.height);
|
|
142
|
+
const x = (config.width - img.width * scale) / 2;
|
|
143
|
+
const y = (config.height - img.height * scale) / 2;
|
|
144
|
+
// Apply filters
|
|
145
|
+
let filters = '';
|
|
146
|
+
if (options.grayscale)
|
|
147
|
+
filters += 'grayscale(100%) ';
|
|
148
|
+
if (options.blur)
|
|
149
|
+
filters += `blur(${options.blur}px) `;
|
|
150
|
+
if (filters)
|
|
151
|
+
ctx.filter = filters.trim();
|
|
152
|
+
ctx.drawImage(img, x, y, img.width * scale, img.height * scale);
|
|
153
|
+
// 4. Encode to WebP
|
|
154
|
+
const outputBlob = await canvas.convertToBlob({
|
|
155
|
+
type: 'image/webp',
|
|
156
|
+
quality: 0.8
|
|
157
|
+
});
|
|
158
|
+
return new Uint8Array(await outputBlob.arrayBuffer());
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* ZIP PROJECT (via JSZip)
|
|
162
|
+
* Bundles all processed assets into a single file for upload or download.
|
|
163
|
+
*/
|
|
164
|
+
async zipProject(outDir) {
|
|
165
|
+
const { default: JSZip } = await Promise.resolve().then(() => __importStar(require('jszip')));
|
|
166
|
+
const zip = new JSZip();
|
|
167
|
+
for (const [path, data] of this.files.entries()) {
|
|
168
|
+
if (path.startsWith(outDir)) {
|
|
169
|
+
const relativePath = path.replace(outDir, '').replace(/^[\\\/]/, '');
|
|
170
|
+
zip.file(relativePath, data);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return await zip.generateAsync({ type: 'uint8array' });
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
exports.BrowserDriver = BrowserDriver;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { IPipelineDriver } from './types';
|
|
2
|
+
import { SubjectFrameData } from '../core/types';
|
|
3
|
+
export interface FalOptions {
|
|
4
|
+
apiKey?: string;
|
|
5
|
+
proxyUrl?: string;
|
|
6
|
+
}
|
|
7
|
+
export declare class FalService {
|
|
8
|
+
private options;
|
|
9
|
+
constructor(options?: FalOptions);
|
|
10
|
+
private getAuthHeaders;
|
|
11
|
+
trackSubject(input: string | File | Blob, driver: IPipelineDriver, prompt?: string): Promise<SubjectFrameData[]>;
|
|
12
|
+
generateDepthMap(input: string | File | Blob, driver: IPipelineDriver): Promise<string>;
|
|
13
|
+
private uploadFile;
|
|
14
|
+
private mapBoxesToTrackingData;
|
|
15
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FalService = void 0;
|
|
4
|
+
const client_1 = require("@fal-ai/client");
|
|
5
|
+
class FalService {
|
|
6
|
+
options;
|
|
7
|
+
constructor(options = {}) {
|
|
8
|
+
this.options = options;
|
|
9
|
+
// In node, we might fallback to process.env if not provided
|
|
10
|
+
const key = options.apiKey || (typeof process !== 'undefined' ? process.env?.FAL_KEY : '');
|
|
11
|
+
if (!key && !options.proxyUrl) {
|
|
12
|
+
// Don't throw yet, only when a cloud method is called
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
getAuthHeaders() {
|
|
16
|
+
return {}; // fal-ai/client handles FAL_KEY from env or we can set it
|
|
17
|
+
}
|
|
18
|
+
async trackSubject(input, driver, prompt = "main subject") {
|
|
19
|
+
let videoUrl;
|
|
20
|
+
if (typeof input === 'string') {
|
|
21
|
+
// Local path or URL
|
|
22
|
+
if (await driver.exists(input)) {
|
|
23
|
+
videoUrl = await this.uploadFile(input, driver);
|
|
24
|
+
}
|
|
25
|
+
else {
|
|
26
|
+
videoUrl = input;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
else {
|
|
30
|
+
// File or Blob
|
|
31
|
+
videoUrl = await client_1.fal.storage.upload(input);
|
|
32
|
+
}
|
|
33
|
+
console.log(`🤖 AI is tracking "${prompt}" via SAM 3...`);
|
|
34
|
+
const result = await client_1.fal.subscribe("fal-ai/sam-3/video-rle", {
|
|
35
|
+
input: {
|
|
36
|
+
video_url: videoUrl,
|
|
37
|
+
prompt: prompt,
|
|
38
|
+
},
|
|
39
|
+
logs: true,
|
|
40
|
+
});
|
|
41
|
+
const payload = result.data || result;
|
|
42
|
+
const boxes = payload.boxes;
|
|
43
|
+
if (!boxes || !Array.isArray(boxes) || boxes.length === 0) {
|
|
44
|
+
throw new Error(`AI tracking returned no data.`);
|
|
45
|
+
}
|
|
46
|
+
return this.mapBoxesToTrackingData(boxes);
|
|
47
|
+
}
|
|
48
|
+
async generateDepthMap(input, driver) {
|
|
49
|
+
let videoUrl;
|
|
50
|
+
if (typeof input === 'string') {
|
|
51
|
+
if (await driver.exists(input)) {
|
|
52
|
+
videoUrl = await this.uploadFile(input, driver);
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
videoUrl = input;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
videoUrl = await client_1.fal.storage.upload(input);
|
|
60
|
+
}
|
|
61
|
+
console.log(`🤖 AI is generating Depth Map...`);
|
|
62
|
+
const result = await client_1.fal.subscribe("fal-ai/video-depth-anything", {
|
|
63
|
+
input: {
|
|
64
|
+
video_url: videoUrl,
|
|
65
|
+
model_size: "VDA-Base",
|
|
66
|
+
},
|
|
67
|
+
logs: true
|
|
68
|
+
});
|
|
69
|
+
const payload = result.data || result;
|
|
70
|
+
if (!payload.video || !payload.video.url) {
|
|
71
|
+
throw new Error(`AI Depth Map generation failed.`);
|
|
72
|
+
}
|
|
73
|
+
return payload.video.url;
|
|
74
|
+
}
|
|
75
|
+
async uploadFile(filePath, driver) {
|
|
76
|
+
const data = await driver.readFile(filePath);
|
|
77
|
+
return await client_1.fal.storage.upload(new Blob([data]));
|
|
78
|
+
}
|
|
79
|
+
mapBoxesToTrackingData(boxes) {
|
|
80
|
+
let lastKnown = { x: 0.5, y: 0.5, scale: 0 };
|
|
81
|
+
return boxes.map((frameBoxes, i) => {
|
|
82
|
+
if (frameBoxes && Array.isArray(frameBoxes)) {
|
|
83
|
+
let box = null;
|
|
84
|
+
if (typeof frameBoxes[0] === 'number' && frameBoxes.length >= 4) {
|
|
85
|
+
box = frameBoxes;
|
|
86
|
+
}
|
|
87
|
+
else if (Array.isArray(frameBoxes[0]) && frameBoxes[0].length >= 4) {
|
|
88
|
+
box = frameBoxes[0];
|
|
89
|
+
}
|
|
90
|
+
else if (typeof frameBoxes[0] === 'object' && frameBoxes[0].box_2d) {
|
|
91
|
+
box = frameBoxes[0].box_2d;
|
|
92
|
+
}
|
|
93
|
+
if (box) {
|
|
94
|
+
lastKnown = { x: box[0], y: box[1], scale: box[2] * box[3] };
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return { frame: i, ...lastKnown };
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
exports.FalService = FalService;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { PipelineOptions, CreateCommandOptions } from './types';
|
|
2
|
+
import { ProjectConfiguration, AssetVariant } from '../core/types';
|
|
3
|
+
export declare class AssetPipeline {
|
|
4
|
+
private driver;
|
|
5
|
+
private options;
|
|
6
|
+
private fal;
|
|
7
|
+
constructor(options?: PipelineOptions);
|
|
8
|
+
/**
|
|
9
|
+
* INITIALIZE DRIVER
|
|
10
|
+
* Detects environment and loads the appropriate driver dynamically.
|
|
11
|
+
*/
|
|
12
|
+
init(): Promise<void>;
|
|
13
|
+
private report;
|
|
14
|
+
/**
|
|
15
|
+
* THE MAIN ORCHESTRATOR
|
|
16
|
+
*/
|
|
17
|
+
create(opts: CreateCommandOptions): Promise<ProjectConfiguration | Uint8Array<ArrayBufferLike>>;
|
|
18
|
+
private normalizeVariants;
|
|
19
|
+
private processVariants;
|
|
20
|
+
saveConfig(variants: AssetVariant[], outDir: string): Promise<ProjectConfiguration>;
|
|
21
|
+
}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.AssetPipeline = void 0;
|
|
37
|
+
const fal_service_1 = require("./fal-service");
|
|
38
|
+
class AssetPipeline {
|
|
39
|
+
driver;
|
|
40
|
+
options;
|
|
41
|
+
fal;
|
|
42
|
+
constructor(options = {}) {
|
|
43
|
+
this.options = options;
|
|
44
|
+
this.fal = new fal_service_1.FalService({ apiKey: options.apiKey, proxyUrl: options.proxyUrl });
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* INITIALIZE DRIVER
|
|
48
|
+
* Detects environment and loads the appropriate driver dynamically.
|
|
49
|
+
*/
|
|
50
|
+
async init() {
|
|
51
|
+
if (this.driver)
|
|
52
|
+
return;
|
|
53
|
+
if (typeof window !== 'undefined') {
|
|
54
|
+
// Browser Environment
|
|
55
|
+
try {
|
|
56
|
+
// @ts-ignore - Assuming BrowserDriver will exist in same folder
|
|
57
|
+
const { BrowserDriver } = await Promise.resolve().then(() => __importStar(require('./browser-driver')));
|
|
58
|
+
this.driver = new BrowserDriver();
|
|
59
|
+
}
|
|
60
|
+
catch (e) {
|
|
61
|
+
throw new Error('Could not load BrowserDriver. Ensure @scrollcraft/pipeline/browser is available.');
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
// Node Environment
|
|
66
|
+
try {
|
|
67
|
+
const { NodeDriver } = await Promise.resolve().then(() => __importStar(require('./node-driver')));
|
|
68
|
+
this.driver = new NodeDriver();
|
|
69
|
+
}
|
|
70
|
+
catch (e) {
|
|
71
|
+
throw new Error('Could not load NodeDriver. Ensure @scrollcraft/pipeline/node (sharp, ffmpeg) is installed.');
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
report(step, percent, message) {
|
|
76
|
+
if (this.options.onProgress) {
|
|
77
|
+
this.options.onProgress({ step, percent, message });
|
|
78
|
+
}
|
|
79
|
+
console.log(`[${step}] ${percent}% - ${message}`);
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* THE MAIN ORCHESTRATOR
|
|
83
|
+
*/
|
|
84
|
+
async create(opts) {
|
|
85
|
+
await this.init();
|
|
86
|
+
const { input, name, track, hasDepth, step = 1 } = opts;
|
|
87
|
+
const outDir = this.driver.resolve(name);
|
|
88
|
+
const tempDir = this.driver.join(outDir, '.temp-frames');
|
|
89
|
+
this.report('initializing', 0, `Creating project: ${name}`);
|
|
90
|
+
await this.driver.mkdir(outDir);
|
|
91
|
+
await this.driver.mkdir(tempDir);
|
|
92
|
+
// 1. FRAME EXTRACTION
|
|
93
|
+
this.report('extracting', 10, 'Extracting frames from source...');
|
|
94
|
+
await this.driver.extractFrames(input, tempDir);
|
|
95
|
+
// 2. AI TRACKING & DEPTH
|
|
96
|
+
let trackingData = [];
|
|
97
|
+
let isDepthActive = false;
|
|
98
|
+
if (track || hasDepth) {
|
|
99
|
+
this.report('tracking', 30, 'Performing AI analysis...');
|
|
100
|
+
if (track) {
|
|
101
|
+
trackingData = await this.fal.trackSubject(input, this.driver, track);
|
|
102
|
+
}
|
|
103
|
+
if (hasDepth) {
|
|
104
|
+
this.report('depth', 40, 'Generating depth maps...');
|
|
105
|
+
const depthVideoUrl = await this.fal.generateDepthMap(input, this.driver);
|
|
106
|
+
// Download and extract depth frames
|
|
107
|
+
const response = await fetch(depthVideoUrl);
|
|
108
|
+
const buffer = await response.arrayBuffer();
|
|
109
|
+
const depthPath = this.driver.join(tempDir, 'depth_video.mp4');
|
|
110
|
+
await this.driver.writeFile(depthPath, new Uint8Array(buffer));
|
|
111
|
+
await this.driver.extractFrames(depthPath, tempDir); // Note: needs to handle prefix
|
|
112
|
+
isDepthActive = true;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
// Default tracking if none
|
|
116
|
+
if (trackingData.length === 0) {
|
|
117
|
+
const files = await this.driver.readdir(tempDir);
|
|
118
|
+
const frameFiles = files.filter(f => f.startsWith('frame_'));
|
|
119
|
+
trackingData = frameFiles.map((_, i) => ({ frame: i, x: 0.5, y: 0.5, scale: 0 }));
|
|
120
|
+
}
|
|
121
|
+
// 3. VARIANT GENERATION
|
|
122
|
+
this.report('processing', 60, 'Generating optimized variants...');
|
|
123
|
+
const variants = await this.processVariants(tempDir, trackingData, {
|
|
124
|
+
step,
|
|
125
|
+
hasDepth: isDepthActive,
|
|
126
|
+
variants: this.normalizeVariants(opts.variants),
|
|
127
|
+
outDir
|
|
128
|
+
});
|
|
129
|
+
// 4. SAVE CONFIG
|
|
130
|
+
this.report('saving', 90, 'Finalizing project configuration...');
|
|
131
|
+
const config = await this.saveConfig(variants, outDir);
|
|
132
|
+
// Cleanup
|
|
133
|
+
await this.driver.remove(tempDir);
|
|
134
|
+
this.report('saving', 100, 'Project ready!');
|
|
135
|
+
if (opts.outputZip && this.driver.zipProject) {
|
|
136
|
+
return await this.driver.zipProject(outDir);
|
|
137
|
+
}
|
|
138
|
+
return config;
|
|
139
|
+
}
|
|
140
|
+
normalizeVariants(v) {
|
|
141
|
+
if (v.length === 0)
|
|
142
|
+
return [];
|
|
143
|
+
if (typeof v[0] === 'object')
|
|
144
|
+
return v;
|
|
145
|
+
// Convert numbers to baseline portrait/landscape pairs
|
|
146
|
+
const res = v;
|
|
147
|
+
const normalized = [];
|
|
148
|
+
res.forEach(r => {
|
|
149
|
+
normalized.push({
|
|
150
|
+
id: `${r}p_p`, width: r, height: Math.round(r * (16 / 9)),
|
|
151
|
+
orientation: 'portrait', aspectRatio: '9:16', media: '(orientation: portrait)'
|
|
152
|
+
});
|
|
153
|
+
normalized.push({
|
|
154
|
+
id: `${r}p_l`, width: Math.round(r * (16 / 9)), height: r,
|
|
155
|
+
orientation: 'landscape', aspectRatio: '16:9', media: '(orientation: landscape)'
|
|
156
|
+
});
|
|
157
|
+
});
|
|
158
|
+
return normalized;
|
|
159
|
+
}
|
|
160
|
+
async processVariants(tempDir, trackingData, options) {
|
|
161
|
+
const { step, outDir } = options;
|
|
162
|
+
const allFiles = await this.driver.readdir(tempDir);
|
|
163
|
+
const allFrames = allFiles.filter(f => f.startsWith('frame_')).sort((a, b) => a.localeCompare(b, undefined, { numeric: true }));
|
|
164
|
+
const framesToProcess = allFrames.filter((_, i) => i % step === 0);
|
|
165
|
+
const assetVariants = [];
|
|
166
|
+
for (const config of options.variants) {
|
|
167
|
+
const variantDir = this.driver.join(outDir, config.id);
|
|
168
|
+
await this.driver.mkdir(variantDir);
|
|
169
|
+
const variantTracking = [];
|
|
170
|
+
for (let i = 0; i < framesToProcess.length; i++) {
|
|
171
|
+
const originalIndex = i * step;
|
|
172
|
+
const frameName = framesToProcess[i];
|
|
173
|
+
const framePath = this.driver.join(tempDir, frameName);
|
|
174
|
+
const targetPath = this.driver.join(variantDir, `index_${i}.webp`);
|
|
175
|
+
const subject = trackingData.find(f => f.frame === originalIndex) || { frame: originalIndex, x: 0.5, y: 0.5, scale: 0 };
|
|
176
|
+
const imageBuffer = await this.driver.processImage(framePath, config, {});
|
|
177
|
+
await this.driver.writeFile(targetPath, imageBuffer);
|
|
178
|
+
if (options.hasDepth) {
|
|
179
|
+
const numStr = frameName.match(/(\d+)/)?.[1] || "";
|
|
180
|
+
const depthFile = allFiles.find(f => f.startsWith('depth_') && f.includes(numStr));
|
|
181
|
+
if (depthFile) {
|
|
182
|
+
const depthPath = this.driver.join(tempDir, depthFile);
|
|
183
|
+
const depthBuffer = await this.driver.processImage(depthPath, config, { grayscale: true, blur: 2 });
|
|
184
|
+
await this.driver.writeFile(this.driver.join(variantDir, `index_${i}_depth.webp`), depthBuffer);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
variantTracking.push({ ...subject, frame: i });
|
|
188
|
+
}
|
|
189
|
+
await this.driver.writeFile(this.driver.join(variantDir, '000_tracking-main.json'), JSON.stringify(variantTracking, null, 2));
|
|
190
|
+
assetVariants.push({
|
|
191
|
+
id: config.id,
|
|
192
|
+
media: config.media,
|
|
193
|
+
width: config.width,
|
|
194
|
+
height: config.height,
|
|
195
|
+
orientation: config.orientation,
|
|
196
|
+
aspectRatio: config.aspectRatio,
|
|
197
|
+
path: `./${config.id}`,
|
|
198
|
+
frameCount: framesToProcess.length,
|
|
199
|
+
hasDepthMap: options.hasDepth,
|
|
200
|
+
subjects: ['main']
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
return assetVariants;
|
|
204
|
+
}
|
|
205
|
+
async saveConfig(variants, outDir) {
|
|
206
|
+
const pkg = { version: '2.0.6' }; // In real app, import from package.json
|
|
207
|
+
const config = {
|
|
208
|
+
version: pkg.version,
|
|
209
|
+
settings: { baseResolution: { width: 1920, height: 1080 }, scrollMode: 'vh' },
|
|
210
|
+
assets: [{ id: "main-sequence", strategy: "adaptive", variants: variants }],
|
|
211
|
+
timeline: {
|
|
212
|
+
totalDuration: "300vh",
|
|
213
|
+
scenes: [{
|
|
214
|
+
id: "scene-1", assetId: "main-sequence", startProgress: 0, duration: 1,
|
|
215
|
+
assetRange: [0, variants[0].frameCount - 1], layers: []
|
|
216
|
+
}]
|
|
217
|
+
}
|
|
218
|
+
};
|
|
219
|
+
await this.driver.writeFile(this.driver.join(outDir, 'scrollcraft.json'), JSON.stringify(config, null, 2));
|
|
220
|
+
return config;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
exports.AssetPipeline = AssetPipeline;
|