pi-multimodal-proxy 1.11.0 → 1.12.0

package/README.md

@@ -6,6 +6,13 @@ When images are sent, this extension routes them to a **vision-capable model**,
 
 When **video or audio files** are detected, they are routed to a **multimodal model** (default: Grok 4.3) that natively understands video content — transcribing speech with speaker diarization, describing visual scenes, reading on-screen text, and reasoning about the content — all in a single call.
 
+**YouTube links** are detected too: paste a URL (`youtube.com/watch?v=…`, `youtu.be/…`, `/shorts/…`, etc.) and the video is downloaded with [`yt-dlp`](https://github.com/yt-dlp/yt-dlp) and analyzed exactly like a local file.
+
+## What's new in 1.12.0
+
+- **YouTube video download** — paste a YouTube URL in your prompt and the extension downloads it via `yt-dlp` and analyzes it like any local video file. Gated by path detection (on by default); requires `yt-dlp` on your PATH. See [YouTube videos](#youtube-videos). After successful analysis you are prompted to optionally keep the downloaded file in your Downloads folder.
+- **Save downloaded videos** — after analysis completes, the extension asks whether to save a copy to `~/Downloads` (sanitised filename, opt-in per file).
+
 ## What's new in 1.11.0
 
 - **Global consent wildcard** — `/multimodal-proxy allowed-providers add *` (or `all`) grants consent for **all providers** globally, so you can consent once and never be prompted again. The wildcard (`*`) appears in the pre-consented list as "* (all providers)" and can be removed with `/multimodal-proxy allowed-providers remove *`. An explicit in-session `consent no` still beats the wildcard.
@@ -56,6 +63,19 @@ pi install npm:pi-multimodal-proxy
 
 > **Upgrading from pi-vision-proxy?** Just install the new package. Your existing config is automatically migrated from `~/.pi/agent/vision-proxy.json`. The `/vision-proxy` command still works.
 
+### YouTube videos
+
+Pasting a YouTube URL downloads the video with [`yt-dlp`](https://github.com/yt-dlp/yt-dlp) and analyzes it through the normal video pipeline. Install yt-dlp once:
+
+```bash
+winget install yt-dlp.yt-dlp        # Windows
+# or:  choco install yt-dlp
+# or:  brew install yt-dlp           # macOS
+# or:  pipx install yt-dlp           # any OS with Python
+```
+
+`ffmpeg` is also used (for duration probing and stream merging) — it usually ships alongside yt-dlp. Downloads are capped to ≤720p and rejected past the configured size limit (`PI_VISION_PROXY_MAX_VIDEO_BYTES`, default 200 MB). To disable URL auto-download, turn off path detection: `/multimodal-proxy path-detection off`.
+
 ## Modes
 
 | Mode | Behavior |

package/extensions/__tests__/internal.test.ts

@@ -35,6 +35,9 @@ import {
 	extractCandidateImagePaths,
 	extractCandidateVideoPaths,
 	extractCandidateAudioPaths,
+	extractCandidateMediaUrls,
+	canonicalYouTubeUrl,
+	youTubeVideoId,
 	extractDimensions,
 	fenceUntrusted,
 	findDescriptions,
@@ -964,6 +967,80 @@ describe("extractCandidateMediaPaths", () => {
 	});
 });
 
+describe("extractCandidateMediaUrls (YouTube)", () => {
+	it("extracts the video id from common URL forms", () => {
+		assert.equal(youTubeVideoId("https://www.youtube.com/watch?v=dQw4w9WgXcQ"), "dQw4w9WgXcQ");
+		assert.equal(youTubeVideoId("https://youtu.be/dQw4w9WgXcQ"), "dQw4w9WgXcQ");
+		assert.equal(youTubeVideoId("https://www.youtube.com/shorts/dQw4w9WgXcQ"), "dQw4w9WgXcQ");
+		assert.equal(youTubeVideoId("https://www.youtube.com/embed/dQw4w9WgXcQ"), "dQw4w9WgXcQ");
+		assert.equal(youTubeVideoId("https://www.youtube.com/live/dQw4w9WgXcQ"), "dQw4w9WgXcQ");
+		assert.equal(youTubeVideoId("https://m.youtube.com/watch?v=dQw4w9WgXcQ"), "dQw4w9WgXcQ");
+		assert.equal(youTubeVideoId("https://music.youtube.com/watch?v=dQw4w9WgXcQ"), "dQw4w9WgXcQ");
+	});
+
+	it("accepts scheme-less / bare URLs", () => {
+		assert.equal(youTubeVideoId("youtu.be/dQw4w9WgXcQ"), "dQw4w9WgXcQ");
+		assert.equal(youTubeVideoId("www.youtube.com/watch?v=dQw4w9WgXcQ"), "dQw4w9WgXcQ");
+	});
+
+	it("handles leading query params before v=", () => {
+		assert.equal(
+			youTubeVideoId("https://www.youtube.com/watch?app=desktop&v=dQw4w9WgXcQ&list=xyz"),
+			"dQw4w9WgXcQ",
+		);
+	});
+
+	it("returns null for non-YouTube URLs", () => {
+		assert.equal(youTubeVideoId("https://vimeo.com/123456"), null);
+		assert.equal(youTubeVideoId("https://example.com/watch?v=abc"), null);
+		assert.equal(youTubeVideoId("not a url at all"), null);
+	});
+
+	it("builds a canonical watch URL", () => {
+		assert.equal(
+			canonicalYouTubeUrl("https://youtu.be/dQw4w9WgXcQ?t=42"),
+			"https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+		);
+		assert.equal(canonicalYouTubeUrl("https://vimeo.com/123"), null);
+	});
+
+	it("detects YouTube URLs embedded in prompt text", () => {
+		assert.deepEqual(
+			extractCandidateMediaUrls("check this out https://www.youtube.com/watch?v=dQw4w9WgXcQ thanks"),
+			["https://www.youtube.com/watch?v=dQw4w9WgXcQ"],
+		);
+		assert.deepEqual(
+			extractCandidateMediaUrls("see youtu.be/abcdefghij and a shorts https://youtube.com/shorts/123456789ab"),
+			["youtu.be/abcdefghij", "https://youtube.com/shorts/123456789ab"],
+		);
+	});
+
+	it("captures trailing query/fragment so the whole URL can be stripped", () => {
+		assert.deepEqual(
+			extractCandidateMediaUrls("https://youtu.be/dQw4w9WgXcQ?t=42&feature=shared"),
+			["https://youtu.be/dQw4w9WgXcQ?t=42&feature=shared"],
+		);
+	});
+
+	it("does not swallow a trailing markdown link closer", () => {
+		assert.deepEqual(
+			extractCandidateMediaUrls("[video](https://www.youtube.com/watch?v=dQw4w9WgXcQ)"),
+			["https://www.youtube.com/watch?v=dQw4w9WgXcQ"],
+		);
+	});
+
+	it("deduplicates by video id across URL forms", () => {
+		const out = extractCandidateMediaUrls(
+			"https://youtu.be/dQw4w9WgXcQ and https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+		);
+		assert.deepEqual(out, ["https://youtu.be/dQw4w9WgXcQ"]);
+	});
+
+	it("ignores non-YouTube URLs", () => {
+		assert.deepEqual(extractCandidateMediaUrls("see https://vimeo.com/123456 and https://example.com"), []);
+	});
+});
+
 describe("stripMediaPaths", () => {
 	it("replaces media paths with placeholder", () => {
 		const mediaPath = "D:\\Downloads\\Rethinking Agents - Harness is All you Need_.mp4";

package/extensions/internal.ts

@@ -1382,6 +1382,64 @@ export function extractCandidateAudioPaths(text: string): string[] {
 	return extractCandidateMediaPaths(text, AUDIO_EXT_ALT);
 }
 
+// ── Media URL detection (YouTube) ──────────────────────────────────────────
+
+/**
+ * Match a YouTube video URL and capture its video id (group 1).
+ *
+ * Supports youtu.be/<id>, youtube.com/watch?v=<id> (with optional leading
+ * query params such as `app=desktop`), and the /shorts/, /embed/, /live/,
+ * and /v/ path forms, on www., m., and music. subdomains. The scheme is
+ * optional (a bare `youtu.be/<id>` is accepted). An optional trailing
+ * query/fragment is consumed so the full matched text can be stripped from
+ * the prompt. Build a `gi` instance from `.source` when scanning text.
+ */
+const YOUTUBE_URL_RE =
+	/(?:https?:\/\/)?(?:www\.|m\.|music\.)?(?:youtu\.be\/|youtube\.com\/(?:watch\?(?:.*?&)?v=|shorts\/|embed\/|live\/|v\/))([A-Za-z0-9_-]{6,})(?:[?&][^\s"'<>()\[\]]*)?/i;
+
+/**
+ * Extract the YouTube video id from a string. Returns null when the string
+ * does not contain a YouTube video URL.
+ */
+export function youTubeVideoId(s: string): string | null {
+	const m = s.match(YOUTUBE_URL_RE);
+	return m?.[1] ?? null;
+}
+
+/**
+ * Return the canonical `https://www.youtube.com/watch?v=<id>` form for a
+ * YouTube URL. Returns null when the input is not a YouTube video URL.
+ * yt-dlp only needs the canonical watch URL — timestamp/playlist params are
+ * not required to fetch the video.
+ */
+export function canonicalYouTubeUrl(s: string): string | null {
+	const id = youTubeVideoId(s);
+	return id ? `https://www.youtube.com/watch?v=${id}` : null;
+}
+
+/**
+ * Extract candidate YouTube video URLs from prompt text.
+ *
+ * Returns the matched substrings exactly as they appear so they can be
+ * stripped from the prompt, de-duplicated by video id. Pass each result to
+ * `canonicalYouTubeUrl()` to obtain the yt-dlp input. Non-YouTube URLs are
+ * ignored (other providers can be added later by extending the regex).
+ */
+export function extractCandidateMediaUrls(text: string): string[] {
+	const out: string[] = [];
+	const seen = new Set<string>();
+	const re = new RegExp(YOUTUBE_URL_RE.source, "gi");
+	let m: RegExpExecArray | null;
+	while ((m = re.exec(text)) !== null) {
+		const id = m[1];
+		if (id && !seen.has(id)) {
+			seen.add(id);
+			out.push(m[0]);
+		}
+	}
+	return out;
+}
+
 /**
  * Extract candidate image file paths from prompt text.
  * Matches `pi-clipboard-*` temp files and general paths ending with image extensions.

package/extensions/vision-proxy.ts

@@ -43,7 +43,7 @@
  */
 
 import { execFile } from "node:child_process";
-import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { copyFile, mkdtemp, readFile, readdir, rm } from "node:fs/promises";
 import os from "node:os";
 import { isAbsolute, join } from "node:path";
 import { promisify } from "node:util";
@@ -96,6 +96,9 @@ import {
 	extractCandidateImagePaths,
 	extractCandidateVideoPaths,
 	extractCandidateAudioPaths,
+	extractCandidateMediaUrls,
+	canonicalYouTubeUrl,
+	youTubeVideoId,
 	applyDefaultModelFallback,
 	applyRecallCompletion,
 	buildRecallItems,
@@ -929,6 +932,106 @@ async function extractAudioChunkToMp3(inputPath: string, outputPath: string, sta
 	], { windowsHide: true, timeout: 120_000 });
 }
 
+// ── yt-dlp: download media URLs (YouTube) ───────────────────────────────────
+
+/**
+ * Cache yt-dlp availability so a missing binary is reported once per session
+ * rather than every turn.
+ */
+let ytDlpAvailable: boolean | null = null;
+
+async function checkYtDlp(): Promise<boolean> {
+	if (ytDlpAvailable !== null) return ytDlpAvailable;
+	try {
+		await execFileAsync("yt-dlp", ["--version"], { windowsHide: true, timeout: 15_000 });
+		ytDlpAvailable = true;
+	} catch {
+		ytDlpAvailable = false;
+	}
+	return ytDlpAvailable;
+}
+
+interface DownloadedMedia {
+	/** Final on-disk path of the downloaded file. */
+	path: string;
+	/** Temp dir holding the file; the caller must remove it. */
+	tempDir: string;
+	/** Human-readable label (video title) for notifications / fences. */
+	title: string;
+}
+
+/**
+ * Download a media URL via yt-dlp into a fresh temp dir.
+ *
+ * Prefers an already-muxed mp4 at <=720p to keep the payload small for the
+ * video model; falls back to the best available stream merged to mp4. The
+ * post-read size guard (maxVideoFileBytes) still applies, so oversized
+ * downloads are rejected downstream with a clear "too-large" reason.
+ *
+ * Returns null (with a user-facing notification) when yt-dlp is missing or
+ * the download fails. The caller owns tempDir cleanup.
+ */
+async function downloadMediaUrl(
+	url: string,
+	signal: AbortSignal | undefined,
+	ctx: ExtensionContext,
+): Promise<DownloadedMedia | null> {
+	if (!(await checkYtDlp())) {
+		ctx.ui.notify(
+			"[multimodal-proxy] YouTube download skipped — yt-dlp not found. Install it (e.g. `winget install yt-dlp.yt-dlp` or `choco install yt-dlp`) and retry.",
+			"warning",
+		);
+		return null;
+	}
+
+	const tempDir = await mkdtemp(join(os.tmpdir(), "multimodal-proxy-ytdl-"));
+	try {
+		// NOTE on `--print after_move:filepath`: when every --print field is a
+		// pre-download metadata field (e.g. only "%(title)s"), yt-dlp skips the
+		// actual download — it can satisfy the print from metadata alone. Adding
+		// an after_move field forces the download (it is only known once the file
+		// is in its final location) and also hands us the exact output path.
+		const { stdout } = await execFileAsync(
+			"yt-dlp",
+			[
+				"--no-playlist",
+				"--no-warnings",
+				"--no-progress",
+				"-f", "best[ext=mp4][height<=720]/best[height<=720]/best",
+				"--merge-output-format", "mp4",
+				"-o", join(tempDir, "%(id)s.%(ext)s"),
+				"--print", "%(title)s",
+				"--print", "after_move:filepath",
+				url,
+			],
+			{ windowsHide: true, timeout: 240_000, maxBuffer: 4 * 1024 * 1024, signal },
+		);
+
+		const lines = stdout.split(/\r?\n/).map((l) => l.trim()).filter((l) => l.length > 0);
+		// The filepath line is the one that looks like an absolute path.
+		const pathLine = lines.find((l) => /^([a-zA-Z]:[\\/]|[\\/])/.test(l));
+		const title = (lines.find((l) => l !== pathLine) ?? "").trim() || url;
+
+		// Prefer the path yt-dlp reported; fall back to scanning the temp dir.
+		let path = pathLine ?? "";
+		if (!path) {
+			const entries = await readdir(tempDir);
+			const file = entries.find((f) => !f.endsWith(".part") && !f.endsWith(".ytdl"));
+			path = file ? join(tempDir, file) : "";
+		}
+		if (!path) {
+			ctx.ui.notify(`[multimodal-proxy] YouTube download produced no file for ${url}`, "warning");
+			return null;
+		}
+		return { path, tempDir, title };
+	} catch (err) {
+		if (signal?.aborted) return null;
+		const msg = err instanceof Error ? err.message : String(err);
+		ctx.ui.notify(`[multimodal-proxy] YouTube download failed for ${url}: ${msg}`, "warning");
+		return null;
+	}
+}
+
 async function analyzeVideoViaXaiStt(
 	mediaFile: { type: "image"; data: string; mimeType: string },
 	filename: string,
@@ -1566,6 +1669,52 @@ export default function (pi: ExtensionAPI) {
 				event.prompt = stripMediaPaths(event.prompt, acceptedMediaPaths);
 			}
 
+			// ── Detect & download media URLs (YouTube, etc.) ─────────────────
+			// Reuses the local-file pipeline: download to a temp dir, then read it
+			// like any other media file. Gated by path detection (on by default) —
+			// no separate switch. Requires yt-dlp on PATH (ffmpeg is already used
+			// elsewhere by this extension).
+			const downloadedTempDirs: string[] = [];
+			const downloadedFiles = new Map<string, { title: string; tempDir: string; filename: string; size: number }>();
+			const cleanupDownloads = async () => {
+				for (const d of downloadedTempDirs) {
+					try { await rm(d, { recursive: true, force: true }); } catch { /* ignore */ }
+				}
+				downloadedTempDirs.length = 0;
+			};
+			if (pathDetectionOn) {
+				const mediaUrls = extractCandidateMediaUrls(event.prompt);
+				const acceptedMediaUrls: string[] = [];
+				for (const url of mediaUrls) {
+					const watch = canonicalYouTubeUrl(url);
+					if (!watch) continue;
+					const id = youTubeVideoId(url) ?? "";
+					const dl = await withProgress(
+						ctx,
+						() => `Downloading YouTube ${id}…`,
+						`Downloading YouTube ${id}…`,
+						() => downloadMediaUrl(watch, ctx.signal, ctx),
+					);
+					if (!dl) continue;
+					downloadedTempDirs.push(dl.tempDir);
+					const r = await readMediaFileWithReason(dl.path, pathAccess);
+					if (r.media) {
+						const fname = r.filename ?? dl.title;
+						downloadedFiles.set(dl.path, { title: dl.title, tempDir: dl.tempDir, filename: fname, size: r.bytes ?? 0 });
+						mediaFiles.push({ file: r.media, filename: fname, path: dl.path });
+						acceptedMediaUrls.push(url);
+					} else if (r.reason && r.reason !== "not-a-media") {
+						ctx.ui.notify(
+							`[multimodal-proxy] Skipped YouTube ${id}: ${describeReadMediaReason(r.reason, r.bytes)}`,
+							"warning",
+						);
+					}
+				}
+				if (acceptedMediaUrls.length > 0) {
+					event.prompt = stripMediaPaths(event.prompt, acceptedMediaUrls);
+				}
+			}
+
 			// Inject loaded file-path images into the event so they reach the model
 			// regardless of whether vision-proxy stripping runs. Strip paths from the
 			// prompt text to avoid duplicate references.
@@ -1582,10 +1731,12 @@ export default function (pi: ExtensionAPI) {
 
 			// ── Handle video/audio files ─────────────────────────────────────
 			let videoDescriptionFence = "";
+			const videoResults: VideoAnalysisResult[] = [];
 			if (mediaFiles.length > 0 && config.mode !== "off") {
 				// Check consent for video provider
 				if (!(await ensureConsent({ ...config, provider: config.videoProvider }, ctx, entries, pi))) {
 					ctx.ui.notify("[multimodal-proxy] Video analysis skipped - no consent.", "warning");
+					await cleanupDownloads();
 					// Inject actionable message so the agent tells the user what to do
 					return {
 						systemPrompt:
@@ -1597,8 +1748,7 @@ export default function (pi: ExtensionAPI) {
 							")",
 					};
 				} else {
-					const videoResults: VideoAnalysisResult[] = [];
-					for (const [mi, mf] of mediaFiles.entries()) {
+				for (const [mi, mf] of mediaFiles.entries()) {
 						const label = () =>
 							mediaFiles.length > 1
 								? `Analyzing ${mf.filename} (${mi + 1}/${mediaFiles.length})…`
@@ -1653,6 +1803,28 @@ export default function (pi: ExtensionAPI) {
 					}
 				}
 			}
+			// Offer to save successfully analyzed downloaded videos
+			for (const [path, info] of downloadedFiles) {
+				const result = videoResults.find(r => r.filename === info.filename && r.description);
+				if (!result) continue;
+				const sizeMB = (info.size / (1024 * 1024)).toFixed(1);
+				try {
+					const save = await ctx.ui.confirm(
+						"Save downloaded video?",
+						`"${info.title}"\n\n${sizeMB} MB — save to your Downloads folder?`,
+					);
+					if (save) {
+						const safeTitle = info.title.replace(/[<>:"\/\\|?*]/g, "_").replace(/\s+/g, " ").trim().slice(0, 200);
+						const dest = join(os.homedir(), "Downloads", `${safeTitle}.mp4`);
+						await copyFile(path, dest);
+						ctx.ui.notify(`[multimodal-proxy] ✓ Saved "${safeTitle}.mp4" to Downloads`, "info");
+					}
+				} catch {
+					// Don't block the agent on save errors — temp files are
+					// cleaned below regardless.
+				}
+			}
+			await cleanupDownloads();
 
 			// ── Handle images (existing flow) ──────────────────────────────────
 			if (images.length === 0) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-multimodal-proxy",
-  "version": "1.11.0",
+  "version": "1.12.0",
   "description": "Automatic image, video and audio description for any model in Pi. Routes media to a multimodal model and injects descriptions into context.",
   "keywords": [
     "pi-package"