pi-web-access 0.5.1 → 0.7.0

package/extract.ts

@@ -6,10 +6,16 @@ import { activityMonitor } from "./activity.js";
 import { extractRSCContent } from "./rsc-extract.js";
 import { extractPDFToMarkdown, isPDF } from "./pdf-extract.js";
 import { extractGitHub } from "./github-extract.js";
+import { isYouTubeURL, isYouTubeEnabled, extractYouTube, extractYouTubeFrame, extractYouTubeFrames, getYouTubeStreamInfo } from "./youtube-extract.js";
+import { extractWithUrlContext, extractWithGeminiWeb } from "./gemini-url-context.js";
+import { isVideoFile, extractVideo, extractVideoFrame, getLocalVideoDuration } from "./video-extract.js";
+import { formatSeconds } from "./utils.js";
 
 const DEFAULT_TIMEOUT_MS = 30000;
 const CONCURRENT_LIMIT = 3;
 
+const NON_RECOVERABLE_ERRORS = ["Unsupported content type", "Response too large"];
+
 const turndown = new TurndownService({
 	headingStyle: "atx",
 	codeBlockStyle: "fenced",
@@ -17,16 +23,102 @@ const turndown = new TurndownService({
 
 const fetchLimit = pLimit(CONCURRENT_LIMIT);
 
+export interface VideoFrame {
+	data: string;
+	mimeType: string;
+	timestamp: string;
+}
+
+export type FrameData = { data: string; mimeType: string };
+export type FrameResult = FrameData | { error: string };
+
 export interface ExtractedContent {
 	url: string;
 	title: string;
 	content: string;
 	error: string | null;
+	thumbnail?: { data: string; mimeType: string };
+	frames?: VideoFrame[];
+	duration?: number;
 }
 
 export interface ExtractOptions {
 	timeoutMs?: number;
 	forceClone?: boolean;
+	prompt?: string;
+	timestamp?: string;
+	frames?: number;
+}
+
+function parseTimestamp(ts: string): number | null {
+	const num = Number(ts);
+	if (!isNaN(num) && num >= 0) return Math.floor(num);
+	const parts = ts.split(":").map(Number);
+	if (parts.some(isNaN)) return null;
+	if (parts.length === 3) return Math.floor(parts[0] * 3600 + parts[1] * 60 + parts[2]);
+	if (parts.length === 2) return Math.floor(parts[0] * 60 + parts[1]);
+	return null;
+}
+
+type TimestampSpec = { type: "single"; seconds: number } | { type: "range"; start: number; end: number };
+
+function parseTimestampSpec(ts: string): TimestampSpec | null {
+	const dashIdx = ts.indexOf("-", 1);
+	if (dashIdx > 0) {
+		const start = parseTimestamp(ts.slice(0, dashIdx));
+		const end = parseTimestamp(ts.slice(dashIdx + 1));
+		if (start !== null && end !== null && end > start) return { type: "range", start, end };
+	}
+	const seconds = parseTimestamp(ts);
+	return seconds !== null ? { type: "single", seconds } : null;
+}
+
+const DEFAULT_RANGE_FRAMES = 6;
+const MIN_FRAME_INTERVAL = 5;
+
+function computeRangeTimestamps(start: number, end: number, maxFrames: number = DEFAULT_RANGE_FRAMES): number[] {
+	if (maxFrames <= 1) return [start];
+	const duration = end - start;
+	const idealInterval = duration / (maxFrames - 1);
+	if (idealInterval < MIN_FRAME_INTERVAL) {
+		const timestamps: number[] = [];
+		for (let t = start; t <= end && timestamps.length < maxFrames; t += MIN_FRAME_INTERVAL) {
+			timestamps.push(t);
+		}
+		return timestamps;
+	}
+	return Array.from({ length: maxFrames }, (_, i) => Math.round(start + i * idealInterval));
+}
+
+function buildFrameResult(
+	url: string, label: string, requestedCount: number,
+	frames: VideoFrame[], error: string | null, duration?: number,
+): ExtractedContent {
+	if (frames.length === 0) {
+		const msg = error ?? "Frame extraction failed";
+		return { url, title: `Frames ${label} (0/${requestedCount})`, content: msg, error: msg };
+	}
+	return {
+		url,
+		title: `Frames ${label} (${frames.length}/${requestedCount})`,
+		content: `${frames.length} frames extracted from ${label}`,
+		error: null,
+		frames,
+		duration,
+	};
+}
+
+async function extractLocalFrames(
+	filePath: string, timestamps: number[],
+): Promise<{ frames: VideoFrame[]; error: string | null }> {
+	const results = await Promise.all(timestamps.map(async (t) => {
+		const frame = await extractVideoFrame(filePath, t);
+		if ("error" in frame) return { error: frame.error };
+		return { ...frame, timestamp: formatSeconds(t) };
+	}));
+	const frames = results.filter((f): f is VideoFrame => "data" in f);
+	const firstError = results.find((f): f is { error: string } => "error" in f);
+	return { frames, error: frames.length === 0 && firstError ? firstError.error : null };
 }
 
 export async function extractContent(
@@ -34,11 +126,135 @@ export async function extractContent(
 	signal?: AbortSignal,
 	options?: ExtractOptions,
 ): Promise<ExtractedContent> {
-	const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
 	if (signal?.aborted) {
 		return { url, title: "", content: "", error: "Aborted" };
 	}
 
+	if (options?.frames && !options.timestamp) {
+		const frameCount = options.frames;
+		const ytInfo = isYouTubeURL(url);
+		if (ytInfo.isYouTube && ytInfo.videoId) {
+			const streamInfo = await getYouTubeStreamInfo(ytInfo.videoId);
+			if ("error" in streamInfo) {
+				return { url, title: "Frames", content: streamInfo.error, error: streamInfo.error };
+			}
+			if (streamInfo.duration === null) {
+				const error = "Cannot determine video duration. Use a timestamp range instead.";
+				return { url, title: "Frames", content: error, error };
+			}
+			const dur = Math.floor(streamInfo.duration);
+			const timestamps = computeRangeTimestamps(0, dur, frameCount);
+			const result = await extractYouTubeFrames(ytInfo.videoId, timestamps, streamInfo);
+			const label = `${formatSeconds(0)}-${formatSeconds(dur)}`;
+			return buildFrameResult(url, label, timestamps.length, result.frames, result.error, streamInfo.duration);
+		}
+
+		const videoInfo = isVideoFile(url);
+		if (videoInfo) {
+			const durationResult = await getLocalVideoDuration(videoInfo.absolutePath);
+			if (typeof durationResult !== "number") {
+				return { url, title: "Frames", content: durationResult.error, error: durationResult.error };
+			}
+			const dur = Math.floor(durationResult);
+			const timestamps = computeRangeTimestamps(0, dur, frameCount);
+			const result = await extractLocalFrames(videoInfo.absolutePath, timestamps);
+			const label = `${formatSeconds(0)}-${formatSeconds(dur)}`;
+			return buildFrameResult(url, label, timestamps.length, result.frames, result.error, durationResult);
+		}
+
+		return { url, title: "", content: "", error: "Frame extraction only works with YouTube and local video files" };
+	}
+
+	if (options?.timestamp) {
+		const spec = parseTimestampSpec(options.timestamp);
+		if (spec) {
+			const frameCount = options.frames;
+			const ytInfo = isYouTubeURL(url);
+			if (ytInfo.isYouTube && ytInfo.videoId) {
+				const streamInfo = await getYouTubeStreamInfo(ytInfo.videoId);
+				if ("error" in streamInfo) {
+					if (spec.type === "range") {
+						const label = `${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;
+						return { url, title: `Frames ${label}`, content: streamInfo.error, error: streamInfo.error };
+					}
+					if (frameCount) {
+						const end = spec.seconds + (frameCount - 1) * MIN_FRAME_INTERVAL;
+						const label = `${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;
+						return { url, title: `Frames ${label}`, content: streamInfo.error, error: streamInfo.error };
+					}
+					return { url, title: `Frame at ${options.timestamp}`, content: streamInfo.error, error: streamInfo.error };
+				}
+
+				if (spec.type === "range") {
+					const label = `${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;
+					if (streamInfo.duration !== null && spec.end > streamInfo.duration) {
+						const error = `Timestamp ${formatSeconds(spec.end)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;
+						return { url, title: `Frames ${label}`, content: error, error };
+					}
+					const timestamps = frameCount
+						? computeRangeTimestamps(spec.start, spec.end, frameCount)
+						: computeRangeTimestamps(spec.start, spec.end);
+					const result = await extractYouTubeFrames(ytInfo.videoId, timestamps, streamInfo);
+					return buildFrameResult(url, label, timestamps.length, result.frames, result.error, result.duration ?? undefined);
+				}
+
+				if (frameCount) {
+					const end = spec.seconds + (frameCount - 1) * MIN_FRAME_INTERVAL;
+					const label = `${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;
+					if (streamInfo.duration !== null && end > streamInfo.duration) {
+						const error = `Timestamp ${formatSeconds(end)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;
+						return { url, title: `Frames ${label}`, content: error, error };
+					}
+					const timestamps = computeRangeTimestamps(spec.seconds, end, frameCount);
+					const result = await extractYouTubeFrames(ytInfo.videoId, timestamps, streamInfo);
+					return buildFrameResult(url, label, timestamps.length, result.frames, result.error, result.duration ?? undefined);
+				}
+
+				if (streamInfo.duration !== null && spec.seconds > streamInfo.duration) {
+					const error = `Timestamp ${formatSeconds(spec.seconds)} exceeds video duration (${formatSeconds(Math.floor(streamInfo.duration))})`;
+					return { url, title: `Frame at ${options.timestamp}`, content: error, error };
+				}
+				const frame = await extractYouTubeFrame(ytInfo.videoId, spec.seconds, streamInfo);
+				if ("error" in frame) {
+					return { url, title: `Frame at ${options.timestamp}`, content: frame.error, error: frame.error };
+				}
+				return { url, title: `Frame at ${options.timestamp}`, content: `Video frame at ${options.timestamp}`, error: null, thumbnail: frame };
+			}
+
+			const videoInfo = isVideoFile(url);
+			if (videoInfo) {
+				if (spec.type === "range") {
+					const timestamps = frameCount
+						? computeRangeTimestamps(spec.start, spec.end, frameCount)
+						: computeRangeTimestamps(spec.start, spec.end);
+					const result = await extractLocalFrames(videoInfo.absolutePath, timestamps);
+					const label = `${formatSeconds(spec.start)}-${formatSeconds(spec.end)}`;
+					return buildFrameResult(url, label, timestamps.length, result.frames, result.error);
+				}
+
+				if (frameCount) {
+					const end = spec.seconds + (frameCount - 1) * MIN_FRAME_INTERVAL;
+					const timestamps = computeRangeTimestamps(spec.seconds, end, frameCount);
+					const result = await extractLocalFrames(videoInfo.absolutePath, timestamps);
+					const label = `${formatSeconds(spec.seconds)}-${formatSeconds(end)}`;
+					return buildFrameResult(url, label, timestamps.length, result.frames, result.error);
+				}
+
+				const frame = await extractVideoFrame(videoInfo.absolutePath, spec.seconds);
+				if ("error" in frame) {
+					return { url, title: `Frame at ${options.timestamp}`, content: frame.error, error: frame.error };
+				}
+				return { url, title: `Frame at ${options.timestamp}`, content: `Video frame at ${options.timestamp}`, error: null, thumbnail: frame };
+			}
+		}
+	}
+
+	const videoInfo = isVideoFile(url);
+	if (videoInfo) {
+		const result = await extractVideo(videoInfo, signal, options);
+		return result ?? { url, title: "", content: "", error: "Video analysis requires Gemini access. Either:\n  1. Sign into gemini.google.com in Chrome (free, uses cookies)\n  2. Set GEMINI_API_KEY in ~/.pi/web-search.json" };
+	}
+
 	try {
 		new URL(url);
 	} catch {
@@ -48,10 +264,39 @@ export async function extractContent(
 	try {
 		const ghResult = await extractGitHub(url, signal, options?.forceClone);
 		if (ghResult) return ghResult;
-	} catch {
-		// GitHub extraction failed unexpectedly, fall through to normal HTTP pipeline
+	} catch {}
+
+	const ytInfo = isYouTubeURL(url);
+	if (ytInfo.isYouTube && isYouTubeEnabled()) {
+		try {
+			const ytResult = await extractYouTube(url, signal, options?.prompt);
+			if (ytResult) return ytResult;
+		} catch {}
+		return {
+			url,
+			title: "",
+			content: "",
+			error: "Could not extract YouTube video content. Sign into Google in Chrome for automatic access, or set GEMINI_API_KEY.",
+		};
 	}
 
+	const httpResult = await extractViaHttp(url, signal, options);
+
+	if (!httpResult.error || signal?.aborted) return httpResult;
+	if (NON_RECOVERABLE_ERRORS.some(prefix => httpResult.error!.startsWith(prefix))) return httpResult;
+
+	const geminiResult = await extractWithUrlContext(url, signal)
+		?? await extractWithGeminiWeb(url, signal);
+
+	return geminiResult ?? httpResult;
+}
+
+async function extractViaHttp(
+	url: string,
+	signal?: AbortSignal,
+	options?: ExtractOptions,
+): Promise<ExtractedContent> {
+	const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
 	const activityId = activityMonitor.logStart({ type: "fetch", url });
 
 	const controller = new AbortController();
@@ -79,11 +324,10 @@ export async function extractContent(
 			};
 		}
 
-		// Check content length to avoid memory issues with huge responses
 		const contentLengthHeader = response.headers.get("content-length");
 		const contentType = response.headers.get("content-type") || "";
 		const isPDFContent = isPDF(url, contentType);
-		const maxResponseSize = isPDFContent ? 20 * 1024 * 1024 : 5 * 1024 * 1024; // 20MB for PDFs, 5MB otherwise
+		const maxResponseSize = isPDFContent ? 20 * 1024 * 1024 : 5 * 1024 * 1024;
 		if (contentLengthHeader) {
 			const contentLength = parseInt(contentLengthHeader, 10);
 			if (contentLength > maxResponseSize) {
@@ -97,7 +341,6 @@ export async function extractContent(
 			}
 		}
 
-		// Handle PDFs - extract and save to markdown file
 		if (isPDFContent) {
 			try {
 				const buffer = await response.arrayBuffer();
@@ -115,8 +358,7 @@ export async function extractContent(
 				return { url, title: "", content: "", error: `PDF extraction failed: ${message}` };
 			}
 		}
-		
-		// Reject binary/non-text content types
+
 		if (contentType.includes("application/octet-stream") ||
 			contentType.includes("image/") ||
 			contentType.includes("audio/") ||
@@ -130,35 +372,27 @@ export async function extractContent(
 				error: `Unsupported content type: ${contentType.split(";")[0]}`,
 			};
 		}
-		
-		// Return plain text directly without Readability
-		const urlHostname = new URL(url).hostname;
-		const isPlainText = contentType.includes("text/plain") || 
-			urlHostname === "raw.githubusercontent.com" ||
-			urlHostname === "gist.githubusercontent.com";
 
 		const text = await response.text();
+		const isHTML = contentType.includes("text/html") || contentType.includes("application/xhtml+xml");
 
-		if (isPlainText) {
+		if (!isHTML) {
 			activityMonitor.logComplete(activityId, response.status);
-			const urlPath = new URL(url).pathname;
-			const title = urlPath.split("/").pop() || url;
+			const title = extractTextTitle(text, url);
 			return { url, title, content: text, error: null };
 		}
 
 		const { document } = parseHTML(text);
-
 		const reader = new Readability(document as unknown as Document);
 		const article = reader.parse();
 
 		if (!article) {
-			// Fallback: Try extracting from RSC flight data (Next.js App Router)
 			const rscResult = extractRSCContent(text);
 			if (rscResult) {
 				activityMonitor.logComplete(activityId, response.status);
 				return { url, title: rscResult.title, content: rscResult.content, error: null };
 			}
-			
+
 			activityMonitor.logComplete(activityId, response.status);
 			return {
 				url,
@@ -169,14 +403,8 @@ export async function extractContent(
 		}
 
 		const markdown = turndown.turndown(article.content);
-
 		activityMonitor.logComplete(activityId, response.status);
-		return {
-			url,
-			title: article.title || "",
-			content: markdown,
-			error: null,
-		};
+		return { url, title: article.title || "", content: markdown, error: null };
 	} catch (err) {
 		const message = err instanceof Error ? err.message : String(err);
 		if (message.toLowerCase().includes("abort")) {
@@ -191,6 +419,17 @@ export async function extractContent(
 	}
 }
 
+export function extractHeadingTitle(text: string): string | null {
+	const match = text.match(/^#{1,2}\s+(.+)/m);
+	if (!match) return null;
+	const cleaned = match[1].replace(/\*+/g, "").trim();
+	return cleaned || null;
+}
+
+function extractTextTitle(text: string, url: string): string {
+	return extractHeadingTitle(text) ?? (new URL(url).pathname.split("/").pop() || url);
+}
+
 export async function fetchAllContent(
 	urls: string[],
 	signal?: AbortSignal,

package/gemini-api.ts

@@ -0,0 +1,103 @@
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+export const API_BASE = "https://generativelanguage.googleapis.com/v1beta";
+const CONFIG_PATH = join(homedir(), ".pi", "web-search.json");
+export const DEFAULT_MODEL = "gemini-2.5-flash";
+
+interface GeminiApiConfig {
+	geminiApiKey?: string;
+}
+
+let cachedConfig: GeminiApiConfig | null = null;
+
+function loadConfig(): GeminiApiConfig {
+	if (cachedConfig) return cachedConfig;
+	if (existsSync(CONFIG_PATH)) {
+		try {
+			cachedConfig = JSON.parse(readFileSync(CONFIG_PATH, "utf-8")) as GeminiApiConfig;
+			return cachedConfig;
+		} catch {}
+	}
+	cachedConfig = {};
+	return cachedConfig;
+}
+
+function withTimeout(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal {
+	const timeout = AbortSignal.timeout(timeoutMs);
+	return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+
+export function getApiKey(): string | null {
+	const envKey = process.env.GEMINI_API_KEY;
+	if (envKey) return envKey;
+	return loadConfig().geminiApiKey ?? null;
+}
+
+export function isGeminiApiAvailable(): boolean {
+	return getApiKey() !== null;
+}
+
+export interface GeminiApiOptions {
+	model?: string;
+	mimeType?: string;
+	signal?: AbortSignal;
+	timeoutMs?: number;
+}
+
+export async function queryGeminiApiWithVideo(
+	prompt: string,
+	videoUri: string,
+	options: GeminiApiOptions = {},
+): Promise<string> {
+	const apiKey = getApiKey();
+	if (!apiKey) throw new Error("GEMINI_API_KEY not configured");
+
+	const model = options.model ?? DEFAULT_MODEL;
+	const signal = withTimeout(options.signal, options.timeoutMs ?? 120000);
+	const url = `${API_BASE}/models/${model}:generateContent?key=${apiKey}`;
+
+	const fileData: Record<string, string> = { fileUri: videoUri };
+	if (options.mimeType) fileData.mimeType = options.mimeType;
+
+	const body = {
+		contents: [
+			{
+				parts: [
+					{ fileData },
+					{ text: prompt },
+				],
+			},
+		],
+	};
+
+	const res = await fetch(url, {
+		method: "POST",
+		headers: { "Content-Type": "application/json" },
+		body: JSON.stringify(body),
+		signal,
+	});
+
+	if (!res.ok) {
+		const errorText = await res.text();
+		throw new Error(`Gemini API error ${res.status}: ${errorText.slice(0, 300)}`);
+	}
+
+	const data = (await res.json()) as GenerateContentResponse;
+	const text = data.candidates?.[0]?.content?.parts
+		?.map((p) => p.text)
+		.filter(Boolean)
+		.join("\n");
+
+	if (!text) throw new Error("Gemini API returned empty response");
+	return text;
+}
+
+interface GenerateContentResponse {
+	candidates?: Array<{
+		content?: {
+			parts?: Array<{ text?: string }>;
+		};
+	}>;
+}