varg.ai-sdk 0.1.1 → 0.4.0-alpha.1

package/src/providers/apify.ts

@@ -0,0 +1,269 @@
+/**
+ * Apify provider for running actors and web scraping
+ * Supports TikTok scraping, web scraping, and other Apify actors
+ */
+
+import { ApifyClient } from "apify-client";
+import type { JobStatusUpdate, ProviderConfig } from "../core/schema/types";
+import { BaseProvider } from "./base";
+
+export interface ApifyProviderConfig extends ProviderConfig {
+  /** Apify API token (defaults to APIFY_TOKEN env var) */
+  token?: string;
+}
+
+export interface RunActorOptions {
+  /** Actor ID in format "username/actor-name" */
+  actorId: string;
+  /** Input parameters for the actor */
+  input?: Record<string, unknown>;
+  /** Whether to wait for the actor to finish (default: true) */
+  waitForFinish?: boolean;
+}
+
+export interface ApifyRunResult {
+  /** Run ID */
+  runId: string;
+  /** Dataset ID containing results */
+  datasetId: string;
+  /** Run status */
+  status: string;
+  /** Result items (if waitForFinish was true) */
+  items?: unknown[];
+}
+
+export class ApifyProvider extends BaseProvider {
+  readonly name = "apify";
+  private client: ApifyClient;
+
+  constructor(config?: ApifyProviderConfig) {
+    super(config);
+    this.client = new ApifyClient({
+      token: config?.token || process.env.APIFY_TOKEN,
+    });
+  }
+
+  /**
+   * Submit an actor run job
+   * @param model - Actor ID (e.g., "clockworks/tiktok-scraper")
+   * @param inputs - Actor input parameters
+   */
+  async submit(
+    model: string,
+    inputs: Record<string, unknown>,
+    _config?: ProviderConfig,
+  ): Promise<string> {
+    console.log(`[apify] submitting actor: ${model}`);
+
+    const actor = this.client.actor(model);
+    const run = await actor.start(inputs);
+
+    console.log(`[apify] run started: ${run.id}`);
+    return run.id;
+  }
+
+  /**
+   * Get the status of an actor run
+   */
+  async getStatus(jobId: string): Promise<JobStatusUpdate> {
+    const run = await this.client.run(jobId).get();
+
+    if (!run) {
+      return { status: "failed", error: "Run not found" };
+    }
+
+    const statusMap: Record<string, JobStatusUpdate["status"]> = {
+      READY: "queued",
+      RUNNING: "processing",
+      SUCCEEDED: "completed",
+      FAILED: "failed",
+      ABORTED: "cancelled",
+      ABORTING: "cancelled",
+      TIMED_OUT: "failed",
+    };
+
+    return {
+      status: statusMap[run.status] ?? "processing",
+      output: run.defaultDatasetId,
+      error: run.status === "FAILED" ? "Actor run failed" : undefined,
+    };
+  }
+
+  /**
+   * Get the results of a completed actor run
+   */
+  async getResult(jobId: string): Promise<unknown> {
+    const run = await this.client.run(jobId).get();
+
+    if (!run?.defaultDatasetId) {
+      throw new Error("Run not found or has no dataset");
+    }
+
+    const { items } = await this.client
+      .dataset(run.defaultDatasetId)
+      .listItems();
+    return items;
+  }
+
+  /**
+   * Cancel a running actor
+   */
+  override async cancel(jobId: string): Promise<void> {
+    await this.client.run(jobId).abort();
+    console.log(`[apify] run aborted: ${jobId}`);
+  }
+
+  // ============================================================================
+  // High-level convenience methods
+  // ============================================================================
+
+  /**
+   * Run an actor and optionally wait for results
+   */
+  async runActor(options: RunActorOptions): Promise<ApifyRunResult> {
+    console.log(`[apify] running actor: ${options.actorId}`);
+
+    const actor = this.client.actor(options.actorId);
+
+    if (options.waitForFinish !== false) {
+      // call() waits for the run to finish
+      const run = await actor.call(options.input);
+      console.log(`[apify] run completed: ${run.id}`);
+
+      // Get results from dataset
+      const { items } = await this.client
+        .dataset(run.defaultDatasetId)
+        .listItems();
+      console.log(`[apify] retrieved ${items.length} items`);
+
+      return {
+        runId: run.id,
+        datasetId: run.defaultDatasetId,
+        status: run.status,
+        items,
+      };
+    }
+
+    // start() returns immediately
+    const run = await actor.start(options.input);
+    console.log(`[apify] run started: ${run.id}`);
+
+    return {
+      runId: run.id,
+      datasetId: run.defaultDatasetId,
+      status: run.status,
+    };
+  }
+
+  /**
+   * Get items from a dataset
+   */
+  async getDataset(datasetId: string): Promise<unknown[]> {
+    console.log(`[apify] fetching dataset: ${datasetId}`);
+    const { items } = await this.client.dataset(datasetId).listItems();
+    console.log(`[apify] retrieved ${items.length} items`);
+    return items;
+  }
+
+  /**
+   * Get run info
+   */
+  async getRunInfo(runId: string) {
+    console.log(`[apify] fetching run info: ${runId}`);
+    const run = await this.client.run(runId).get();
+    return run;
+  }
+
+  /**
+   * Wait for a run to finish
+   */
+  async waitForRun(runId: string) {
+    console.log(`[apify] waiting for run: ${runId}`);
+    const run = await this.client.run(runId).waitForFinish();
+    console.log(`[apify] run finished with status: ${run?.status}`);
+    return run;
+  }
+
+  /**
+   * Get value from key-value store
+   */
+  async getKeyValueStoreValue(storeId: string, key: string) {
+    console.log(`[apify] fetching key "${key}" from store: ${storeId}`);
+    const value = await this.client.keyValueStore(storeId).getRecord(key);
+    return value;
+  }
+
+  /**
+   * Download videos from scraped results using yt-dlp
+   */
+  async downloadVideos(
+    items: Array<{ webVideoUrl?: string }>,
+    outputDir = "output/videos",
+  ): Promise<string[]> {
+    const urls = items
+      .map((item) => item.webVideoUrl)
+      .filter((url): url is string => !!url);
+
+    console.log(`[apify] downloading ${urls.length} videos`);
+
+    // Create download dir if needed
+    await Bun.$`mkdir -p ${outputDir}`;
+
+    const downloaded: string[] = [];
+
+    for (const url of urls) {
+      console.log(`[apify] downloading: ${url}`);
+      try {
+        await Bun.$`yt-dlp -o "${outputDir}/%(id)s.%(ext)s" ${url}`;
+        downloaded.push(url);
+        console.log(`[apify] downloaded successfully`);
+      } catch (err) {
+        console.error(`[apify] failed to download ${url}:`, err);
+      }
+    }
+
+    console.log(`[apify] all downloads complete. saved to: ${outputDir}`);
+    return downloaded;
+  }
+}
+
+// Popular actors registry
+export const ACTORS = {
+  TIKTOK: {
+    SCRAPER: "clockworks/tiktok-scraper",
+    HASHTAG: "clockworks/tiktok-hashtag-scraper",
+    PROFILE: "clockworks/tiktok-profile-scraper",
+  },
+  WEB: {
+    SCRAPER: "apify/web-scraper",
+    CHEERIO: "apify/cheerio-scraper",
+    PUPPETEER: "apify/puppeteer-scraper",
+  },
+  SOCIAL: {
+    INSTAGRAM: "apify/instagram-scraper",
+    TWITTER: "apify/twitter-scraper",
+    YOUTUBE: "bernardo/youtube-scraper",
+  },
+};
+
+// Export singleton instance
+export const apifyProvider = new ApifyProvider();
+
+// Export convenience functions
+export const runActor = (options: RunActorOptions) =>
+  apifyProvider.runActor(options);
+
+export const getDataset = (datasetId: string) =>
+  apifyProvider.getDataset(datasetId);
+
+export const getRunInfo = (runId: string) => apifyProvider.getRunInfo(runId);
+
+export const waitForRun = (runId: string) => apifyProvider.waitForRun(runId);
+
+export const getKeyValueStoreValue = (storeId: string, key: string) =>
+  apifyProvider.getKeyValueStoreValue(storeId, key);
+
+export const downloadVideos = (
+  items: Array<{ webVideoUrl?: string }>,
+  outputDir?: string,
+) => apifyProvider.downloadVideos(items, outputDir);

package/src/providers/base.ts

@@ -0,0 +1,264 @@
+/**
+ * Base provider interface and abstract class
+ * All providers implement this interface for consistent execution
+ */
+
+import type {
+  JobStatusUpdate,
+  Provider,
+  ProviderConfig,
+} from "../core/schema/types";
+
+/**
+ * Abstract base class for providers
+ * Provides common functionality and enforces the Provider interface
+ */
+export abstract class BaseProvider implements Provider {
+  abstract readonly name: string;
+
+  protected config: ProviderConfig;
+
+  constructor(config: ProviderConfig = {}) {
+    this.config = {
+      timeout: 300000, // 5 minutes default
+      retries: 3,
+      ...config,
+    };
+  }
+
+  /**
+   * Submit a job to the provider
+   */
+  abstract submit(
+    model: string,
+    inputs: Record<string, unknown>,
+    config?: ProviderConfig,
+  ): Promise<string>;
+
+  /**
+   * Get the current status of a job
+   */
+  abstract getStatus(jobId: string): Promise<JobStatusUpdate>;
+
+  /**
+   * Get the result of a completed job
+   */
+  abstract getResult(jobId: string): Promise<unknown>;
+
+  /**
+   * Optional: Cancel a running job
+   */
+  async cancel?(jobId: string): Promise<void>;
+
+  /**
+   * Optional: Upload a file to the provider's storage
+   */
+  async uploadFile?(
+    file: File | Blob | ArrayBuffer,
+    filename?: string,
+  ): Promise<string>;
+
+  /**
+   * Helper: Wait for a job to complete with polling
+   */
+  protected async waitForCompletion(
+    jobId: string,
+    options: {
+      maxWait?: number;
+      pollInterval?: number;
+      onProgress?: (progress: number, logs?: string[]) => void;
+    } = {},
+  ): Promise<unknown> {
+    const { maxWait = this.config.timeout ?? 300000, pollInterval = 2000 } =
+      options;
+    const startTime = Date.now();
+    let currentInterval = pollInterval;
+
+    while (Date.now() - startTime < maxWait) {
+      const status = await this.getStatus(jobId);
+
+      if (options.onProgress && status.progress !== undefined) {
+        options.onProgress(status.progress, status.logs);
+      }
+
+      if (status.status === "completed") {
+        return status.output ?? (await this.getResult(jobId));
+      }
+
+      if (status.status === "failed") {
+        throw new Error(status.error ?? "Job failed");
+      }
+
+      if (status.status === "cancelled") {
+        throw new Error("Job was cancelled");
+      }
+
+      // Exponential backoff with cap
+      await this.sleep(currentInterval);
+      currentInterval = Math.min(currentInterval * 1.5, 10000);
+    }
+
+    throw new Error(`Job ${jobId} timed out after ${maxWait}ms`);
+  }
+
+  /**
+   * Helper: Sleep for a duration
+   */
+  protected sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Helper: Retry a function with exponential backoff
+   */
+  protected async retry<T>(
+    fn: () => Promise<T>,
+    options: { retries?: number; delay?: number } = {},
+  ): Promise<T> {
+    const { retries = this.config.retries ?? 3, delay = 1000 } = options;
+    let lastError: Error | undefined;
+
+    for (let attempt = 0; attempt <= retries; attempt++) {
+      try {
+        return await fn();
+      } catch (error) {
+        lastError = error instanceof Error ? error : new Error(String(error));
+        if (attempt < retries) {
+          await this.sleep(delay * 2 ** attempt);
+        }
+      }
+    }
+
+    throw lastError;
+  }
+}
+
+/**
+ * Provider registry for managing multiple providers
+ */
+export class ProviderRegistry {
+  private providers = new Map<string, Provider>();
+
+  /**
+   * Register a provider
+   */
+  register(provider: Provider): void {
+    this.providers.set(provider.name, provider);
+  }
+
+  /**
+   * Get a provider by name
+   */
+  get(name: string): Provider | undefined {
+    return this.providers.get(name);
+  }
+
+  /**
+   * Check if a provider exists
+   */
+  has(name: string): boolean {
+    return this.providers.has(name);
+  }
+
+  /**
+   * Get all registered providers
+   */
+  all(): Provider[] {
+    return Array.from(this.providers.values());
+  }
+
+  /**
+   * Get provider names
+   */
+  names(): string[] {
+    return Array.from(this.providers.keys());
+  }
+}
+
+/**
+ * Global provider registry instance
+ */
+export const providers = new ProviderRegistry();
+
+/**
+ * Helper type for provider-specific result structures
+ */
+export interface ProviderResult<T = unknown> {
+  data: T;
+  requestId?: string;
+  timing?: {
+    queueTime?: number;
+    processingTime?: number;
+    totalTime?: number;
+  };
+}
+
+/**
+ * Helper: Ensure a path is a URL (upload local files if needed)
+ */
+export async function ensureUrl(
+  pathOrUrl: string,
+  uploader: (file: ArrayBuffer) => Promise<string>,
+): Promise<string> {
+  // Already a URL
+  if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) {
+    return pathOrUrl;
+  }
+
+  // Local file - read and upload
+  const file = Bun.file(pathOrUrl);
+  if (!(await file.exists())) {
+    throw new Error(`Local file not found: ${pathOrUrl}`);
+  }
+
+  const buffer = await file.arrayBuffer();
+  return uploader(buffer);
+}
+
+/**
+ * Helper: Download a URL to a local file
+ */
+export async function downloadToFile(
+  url: string,
+  outputPath: string,
+): Promise<string> {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to download: ${response.statusText}`);
+  }
+
+  const buffer = await response.arrayBuffer();
+  await Bun.write(outputPath, buffer);
+  return outputPath;
+}
+
+/**
+ * Helper: Get file extension from URL or content type
+ */
+export function getExtension(url: string, contentType?: string): string {
+  // Try to get from URL
+  const urlPath = new URL(url).pathname;
+  const urlExt = urlPath.split(".").pop();
+  if (urlExt && urlExt.length <= 4) {
+    return urlExt;
+  }
+
+  // Try content type
+  if (contentType) {
+    const typeMap: Record<string, string> = {
+      "video/mp4": "mp4",
+      "video/webm": "webm",
+      "video/quicktime": "mov",
+      "image/jpeg": "jpg",
+      "image/png": "png",
+      "image/webp": "webp",
+      "image/gif": "gif",
+      "audio/mpeg": "mp3",
+      "audio/wav": "wav",
+      "audio/ogg": "ogg",
+    };
+    return typeMap[contentType] ?? "bin";
+  }
+
+  return "bin";
+}