pi-web-access 0.4.5 → 0.5.0

package/CHANGELOG.md

@@ -4,6 +4,24 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+## [0.5.0] - 2026-02-01
+
+### Added
+- GitHub repository clone extraction for `fetch_content` -- detects GitHub code URLs, clones repos to `/tmp/pi-github-repos/`, and returns actual file contents plus local path for further exploration with `read` and `bash`
+- Lightweight API fallback for oversized repos (>350MB) and commit SHA URLs via `gh api`
+- Clone cache with concurrent request deduplication (second request awaits first's clone)
+- `forceClone` parameter on `fetch_content` to override the size threshold
+- Configurable via `~/.pi/web-search.json` under `githubClone` key (enabled, maxRepoSizeMB, cloneTimeoutSeconds, clonePath)
+- Falls back to `git clone` when `gh` CLI is unavailable (public repos only)
+- README: GitHub clone documentation with config, flow diagram, and limitations
+
+### Changed
+- Refactored `extractContent`/`fetchAllContent` signatures from positional `timeoutMs` to `ExtractOptions` object
+- Blob/tree fetch titles now include file path (e.g. `owner/repo - src/index.ts`) for better disambiguation in multi-URL results and TUI
+
+### Fixed
+- README: Activity monitor keybinding corrected from `Ctrl+Shift+O` to `Ctrl+Shift+W`
+
 ## [0.4.5] - 2026-02-01
 
 ### Changed

package/README.md

@@ -4,7 +4,7 @@
 
 # Pi Web Access
 
-An extension for [Pi coding agent](https://github.com/badlogic/pi-mono/) that gives Pi web capabilities: search via Perplexity AI, fetch and extract content from URLs, and read PDFs.
+An extension for [Pi coding agent](https://github.com/badlogic/pi-mono/) that gives Pi web capabilities: search via Perplexity AI, fetch and extract content from URLs, clone GitHub repos for local exploration, and read PDFs.
 
 ```typescript
 web_search({ query: "TypeScript best practices 2025" })
@@ -74,6 +74,25 @@ fetch_content({ url: "https://arxiv.org/pdf/1706.03762" })
 // → "PDF extracted and saved to: ~/Downloads/arxiv-170603762.md"
 ```
 
+**GitHub repos:** GitHub code URLs are automatically detected and cloned locally instead of scraping HTML. The agent gets actual file contents and a local path to explore with `read` and `bash`.
+
+```typescript
+// Clone a repo - returns structure + README
+fetch_content({ url: "https://github.com/owner/repo" })
+// → "Repository cloned to: /tmp/pi-github-repos/owner/repo"
+
+// Specific file - returns file contents
+fetch_content({ url: "https://github.com/owner/repo/blob/main/src/index.ts" })
+
+// Directory - returns listing
+fetch_content({ url: "https://github.com/owner/repo/tree/main/src" })
+
+// Force-clone a large repo that exceeds the size threshold
+fetch_content({ url: "https://github.com/big/repo", forceClone: true })
+```
+
+Repos over 350MB get a lightweight API-based view instead of a full clone. Commit SHA URLs are also handled via the API. Clones are cached for the session -- multiple files from the same repo share one clone, but clones are wiped on session change/shutdown and re-cloned as needed.
+
 **PDF handling:** When fetching a PDF URL, the extension extracts text and saves it as a markdown file in `~/Downloads/`. The agent can then use `read` to access specific sections without loading 200K+ chars into context.
 
 ### get_search_content
@@ -93,7 +112,7 @@ get_search_content({ responseId: "abc123", query: "original query" })
 
 ## Features
 
-### Activity Monitor (Ctrl+Shift+O)
+### Activity Monitor (Ctrl+Shift+W)
 
 Toggle live request/response activity:
 
@@ -129,25 +148,67 @@ Browse stored search results interactively.
 
 ## How It Works
 
+### fetch_content routing
+
+```
+fetch_content(url)
+       │
+       ├── github.com code URL? ──→ Clone repo (gh/git --depth 1)
+       │                                    │
+       │                            ┌───────┼───────┐
+       │                            ↓       ↓       ↓
+       │                          root    tree     blob
+       │                            ↓       ↓       ↓
+       │                         tree +   dir     file
+       │                         README  listing  contents
+       │                            │       │       │
+       │                            └───────┼───────┘
+       │                                    ↓
+       │                           Return content + local
+       │                           path for read/bash
+       │
+       ├── PDF? ──→ unpdf → Save to ~/Downloads/
+       │
+       ├── Plain text? ──→ Return directly
+       │
+       └── HTML ──→ Readability → Markdown
+                         │
+                    [if fails]
+                         ↓
+                    RSC Parser → Markdown
+```
+
+### web_search with includeContent
+
 ```
 Agent Request → Perplexity API → Synthesized Answer + Citations
                                          ↓
                               [if includeContent: true]
                                          ↓
                               Background Fetch (3 concurrent)
-                                         ↓
-                        ┌────────────────┼────────────────┐
-                        ↓                ↓                ↓
-                       PDF          HTML/Text          RSC
-                        ↓                ↓                ↓
-                   unpdf →        Readability →    RSC Parser →
-                 Save to file      Markdown          Markdown
-                        ↓                ↓                ↓
-                        └────────────────┼────────────────┘
+                              (uses same routing as above)
                                          ↓
                               Agent Notification (triggerTurn)
 ```
 
+## Configuration
+
+All config lives in `~/.pi/web-search.json`:
+
+```json
+{
+  "perplexityApiKey": "pplx-...",
+  "githubClone": {
+    "enabled": true,
+    "maxRepoSizeMB": 350,
+    "cloneTimeoutSeconds": 30,
+    "clonePath": "/tmp/pi-github-repos"
+  }
+}
+```
+
+All `githubClone` fields are optional with the defaults shown above. Set `"enabled": false` to disable GitHub cloning entirely and fall through to normal HTML extraction.
+
 ## Rate Limits
 
 - **Perplexity API**: 10 requests/minute (enforced client-side)
@@ -161,6 +222,8 @@ Agent Request → Perplexity API → Synthesized Answer + Citations
 | `index.ts` | Extension entry, tool definitions, commands, widget |
 | `perplexity.ts` | Perplexity API client, rate limiting |
 | `extract.ts` | URL fetching, content extraction routing |
+| `github-extract.ts` | GitHub URL parser, clone cache, content generation |
+| `github-api.ts` | GitHub API fallback for oversized repos and commit SHAs |
 | `pdf-extract.ts` | PDF text extraction, saves to markdown |
 | `rsc-extract.ts` | RSC flight data parser for Next.js pages |
 | `storage.ts` | Session-aware result storage |
@@ -173,4 +236,7 @@ Agent Request → Perplexity API → Synthesized Answer + Citations
 - PDFs are extracted as text (no OCR for scanned documents)
 - Max response size: 20MB for PDFs, 5MB for HTML
 - Max inline content: 30,000 chars per URL (larger content stored for retrieval via get_search_content)
+- GitHub cloning requires `gh` CLI for private repos (public repos fall back to `git clone`)
+- GitHub branch names with slashes (e.g. `feature/foo`) may resolve the wrong file path; the clone still succeeds and the agent can navigate manually
+- Non-code GitHub URLs (issues, PRs, wiki, etc.) fall through to normal Readability extraction
 - Requires Pi restart after config file changes

package/extract.ts

@@ -5,6 +5,7 @@ import pLimit from "p-limit";
 import { activityMonitor } from "./activity.js";
 import { extractRSCContent } from "./rsc-extract.js";
 import { extractPDFToMarkdown, isPDF } from "./pdf-extract.js";
+import { extractGitHub } from "./github-extract.js";
 
 const DEFAULT_TIMEOUT_MS = 30000;
 const CONCURRENT_LIMIT = 3;
@@ -23,11 +24,17 @@ export interface ExtractedContent {
 	error: string | null;
 }
 
+export interface ExtractOptions {
+	timeoutMs?: number;
+	forceClone?: boolean;
+}
+
 export async function extractContent(
 	url: string,
 	signal?: AbortSignal,
-	timeoutMs: number = DEFAULT_TIMEOUT_MS,
+	options?: ExtractOptions,
 ): Promise<ExtractedContent> {
+	const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
 	if (signal?.aborted) {
 		return { url, title: "", content: "", error: "Aborted" };
 	}
@@ -38,6 +45,13 @@ export async function extractContent(
 		return { url, title: "", content: "", error: "Invalid URL" };
 	}
 
+	try {
+		const ghResult = await extractGitHub(url, signal, options?.forceClone);
+		if (ghResult) return ghResult;
+	} catch {
+		// GitHub extraction failed unexpectedly, fall through to normal HTTP pipeline
+	}
+
 	const activityId = activityMonitor.logStart({ type: "fetch", url });
 
 	const controller = new AbortController();
@@ -127,22 +141,19 @@ export async function extractContent(
 
 		if (isPlainText) {
 			activityMonitor.logComplete(activityId, response.status);
-			const content = text;
-			// Extract filename from URL as title
 			const urlPath = new URL(url).pathname;
 			const title = urlPath.split("/").pop() || url;
-			return { url, title, content, error: null };
+			return { url, title, content: text, error: null };
 		}
 
-		const html = text;
-		const { document } = parseHTML(html);
+		const { document } = parseHTML(text);
 
 		const reader = new Readability(document as unknown as Document);
 		const article = reader.parse();
 
 		if (!article) {
 			// Fallback: Try extracting from RSC flight data (Next.js App Router)
-			const rscResult = extractRSCContent(html);
+			const rscResult = extractRSCContent(text);
 			if (rscResult) {
 				activityMonitor.logComplete(activityId, response.status);
 				return { url, title: rscResult.title, content: rscResult.content, error: null };
@@ -183,7 +194,7 @@ export async function extractContent(
 export async function fetchAllContent(
 	urls: string[],
 	signal?: AbortSignal,
-	timeoutMs?: number,
+	options?: ExtractOptions,
 ): Promise<ExtractedContent[]> {
-	return Promise.all(urls.map((url) => fetchLimit(() => extractContent(url, signal, timeoutMs))));
+	return Promise.all(urls.map((url) => fetchLimit(() => extractContent(url, signal, options))));
 }

package/github-api.ts

@@ -0,0 +1,195 @@
+import { execFile } from "node:child_process";
+import type { ExtractedContent } from "./extract.js";
+import type { GitHubUrlInfo } from "./github-extract.js";
+
+const MAX_TREE_ENTRIES = 200;
+const MAX_INLINE_FILE_CHARS = 100_000;
+
+let ghAvailable: boolean | null = null;
+let ghHintShown = false;
+
+export async function checkGhAvailable(): Promise<boolean> {
+	if (ghAvailable !== null) return ghAvailable;
+
+	return new Promise((resolve) => {
+		execFile("gh", ["--version"], { timeout: 5000 }, (err) => {
+			ghAvailable = !err;
+			resolve(ghAvailable);
+		});
+	});
+}
+
+export function showGhHint(): void {
+	if (!ghHintShown) {
+		ghHintShown = true;
+		console.error("[pi-web-access] Install `gh` CLI for better GitHub repo access including private repos.");
+	}
+}
+
+export async function checkRepoSize(owner: string, repo: string): Promise<number | null> {
+	if (!(await checkGhAvailable())) return null;
+
+	return new Promise((resolve) => {
+		execFile("gh", ["api", `repos/${owner}/${repo}`, "--jq", ".size"], { timeout: 10000 }, (err, stdout) => {
+			if (err) {
+				resolve(null);
+				return;
+			}
+			const kb = parseInt(stdout.trim(), 10);
+			resolve(Number.isNaN(kb) ? null : kb);
+		});
+	});
+}
+
+export async function getDefaultBranch(owner: string, repo: string): Promise<string | null> {
+	if (!(await checkGhAvailable())) return null;
+
+	return new Promise((resolve) => {
+		execFile("gh", ["api", `repos/${owner}/${repo}`, "--jq", ".default_branch"], { timeout: 10000 }, (err, stdout) => {
+			if (err) {
+				resolve(null);
+				return;
+			}
+			const branch = stdout.trim();
+			resolve(branch || null);
+		});
+	});
+}
+
+async function fetchTreeViaApi(owner: string, repo: string, ref: string): Promise<string | null> {
+	if (!(await checkGhAvailable())) return null;
+
+	return new Promise((resolve) => {
+		execFile(
+			"gh",
+			["api", `repos/${owner}/${repo}/git/trees/${ref}?recursive=1`, "--jq", ".tree[].path"],
+			{ timeout: 15000, maxBuffer: 5 * 1024 * 1024 },
+			(err, stdout) => {
+				if (err) {
+					resolve(null);
+					return;
+				}
+				const paths = stdout.trim().split("\n").filter(Boolean);
+				if (paths.length === 0) {
+					resolve(null);
+					return;
+				}
+				const truncated = paths.length > MAX_TREE_ENTRIES;
+				const display = paths.slice(0, MAX_TREE_ENTRIES).join("\n");
+				resolve(truncated ? display + `\n... (${paths.length} total entries)` : display);
+			},
+		);
+	});
+}
+
+async function fetchReadmeViaApi(owner: string, repo: string, ref: string): Promise<string | null> {
+	if (!(await checkGhAvailable())) return null;
+
+	return new Promise((resolve) => {
+		execFile(
+			"gh",
+			["api", `repos/${owner}/${repo}/readme?ref=${ref}`, "--jq", ".content"],
+			{ timeout: 10000 },
+			(err, stdout) => {
+				if (err) {
+					resolve(null);
+					return;
+				}
+				try {
+					const decoded = Buffer.from(stdout.trim(), "base64").toString("utf-8");
+					resolve(decoded.length > 8192 ? decoded.slice(0, 8192) + "\n\n[README truncated at 8K chars]" : decoded);
+				} catch {
+					resolve(null);
+				}
+			},
+		);
+	});
+}
+
+async function fetchFileViaApi(owner: string, repo: string, path: string, ref: string): Promise<string | null> {
+	if (!(await checkGhAvailable())) return null;
+
+	return new Promise((resolve) => {
+		execFile(
+			"gh",
+			["api", `repos/${owner}/${repo}/contents/${path}?ref=${ref}`, "--jq", ".content"],
+			{ timeout: 10000, maxBuffer: 2 * 1024 * 1024 },
+			(err, stdout) => {
+				if (err) {
+					resolve(null);
+					return;
+				}
+				try {
+					resolve(Buffer.from(stdout.trim(), "base64").toString("utf-8"));
+				} catch {
+					resolve(null);
+				}
+			},
+		);
+	});
+}
+
+export async function fetchViaApi(
+	url: string,
+	owner: string,
+	repo: string,
+	info: GitHubUrlInfo,
+	sizeNote?: string,
+): Promise<ExtractedContent | null> {
+	const ref = info.ref || (await getDefaultBranch(owner, repo));
+	if (!ref) return null;
+
+	const lines: string[] = [];
+	if (sizeNote) {
+		lines.push(sizeNote);
+		lines.push("");
+	}
+
+	if (info.type === "blob" && info.path) {
+		const content = await fetchFileViaApi(owner, repo, info.path, ref);
+		if (!content) return null;
+
+		lines.push(`## ${info.path}`);
+		if (content.length > MAX_INLINE_FILE_CHARS) {
+			lines.push(content.slice(0, MAX_INLINE_FILE_CHARS));
+			lines.push(`\n[File truncated at 100K chars]`);
+		} else {
+			lines.push(content);
+		}
+
+		return {
+			url,
+			title: `${owner}/${repo} - ${info.path}`,
+			content: lines.join("\n"),
+			error: null,
+		};
+	}
+
+	const [tree, readme] = await Promise.all([
+		fetchTreeViaApi(owner, repo, ref),
+		fetchReadmeViaApi(owner, repo, ref),
+	]);
+
+	if (!tree && !readme) return null;
+
+	if (tree) {
+		lines.push("## Structure");
+		lines.push(tree);
+		lines.push("");
+	}
+
+	if (readme) {
+		lines.push("## README.md");
+		lines.push(readme);
+		lines.push("");
+	}
+
+	lines.push("This is an API-only view. Clone the repo or use `read`/`bash` for deeper exploration.");
+
+	return {
+		url,
+		title: `${owner}/${repo}`,
+		content: lines.join("\n"),
+		error: null,
+	};
+}

package/github-extract.ts

@@ -0,0 +1,505 @@
+import { existsSync, readFileSync, rmSync, statSync, readdirSync, openSync, readSync, closeSync } from "node:fs";
+import { execFile } from "node:child_process";
+import { homedir } from "node:os";
+import { join, extname } from "node:path";
+import { activityMonitor } from "./activity.js";
+import type { ExtractedContent } from "./extract.js";
+import { checkGhAvailable, checkRepoSize, fetchViaApi, showGhHint } from "./github-api.js";
+
+const CONFIG_PATH = join(homedir(), ".pi", "web-search.json");
+
+const BINARY_EXTENSIONS = new Set([
+	".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".webp", ".svg", ".tiff", ".tif",
+	".mp3", ".mp4", ".avi", ".mov", ".mkv", ".flv", ".wmv", ".wav", ".ogg", ".webm", ".flac", ".aac",
+	".zip", ".tar", ".gz", ".bz2", ".xz", ".7z", ".rar", ".zst",
+	".exe", ".dll", ".so", ".dylib", ".bin", ".o", ".a", ".lib",
+	".woff", ".woff2", ".ttf", ".otf", ".eot",
+	".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
+	".sqlite", ".db", ".sqlite3",
+	".pyc", ".pyo", ".class", ".jar", ".war",
+	".iso", ".img", ".dmg",
+]);
+
+const NOISE_DIRS = new Set([
+	"node_modules", "vendor", ".next", "dist", "build", "__pycache__",
+	".venv", "venv", ".tox", ".mypy_cache", ".pytest_cache",
+	"target", ".gradle", ".idea", ".vscode",
+]);
+
+const MAX_INLINE_FILE_CHARS = 100_000;
+const MAX_TREE_ENTRIES = 200;
+
+export interface GitHubUrlInfo {
+	owner: string;
+	repo: string;
+	ref?: string;
+	refIsFullSha: boolean;
+	path?: string;
+	type: "root" | "blob" | "tree";
+}
+
+interface CachedClone {
+	localPath: string;
+	clonePromise: Promise<string | null>;
+}
+
+interface GitHubCloneConfig {
+	enabled: boolean;
+	maxRepoSizeMB: number;
+	cloneTimeoutSeconds: number;
+	clonePath: string;
+}
+
+const cloneCache = new Map<string, CachedClone>();
+
+let cachedConfig: GitHubCloneConfig | null = null;
+
+function loadGitHubConfig(): GitHubCloneConfig {
+	if (cachedConfig) return cachedConfig;
+
+	const defaults: GitHubCloneConfig = {
+		enabled: true,
+		maxRepoSizeMB: 350,
+		cloneTimeoutSeconds: 30,
+		clonePath: "/tmp/pi-github-repos",
+	};
+
+	try {
+		if (existsSync(CONFIG_PATH)) {
+			const raw = JSON.parse(readFileSync(CONFIG_PATH, "utf-8"));
+			const gc = raw.githubClone ?? {};
+			cachedConfig = {
+				enabled: gc.enabled ?? defaults.enabled,
+				maxRepoSizeMB: gc.maxRepoSizeMB ?? defaults.maxRepoSizeMB,
+				cloneTimeoutSeconds: gc.cloneTimeoutSeconds ?? defaults.cloneTimeoutSeconds,
+				clonePath: gc.clonePath ?? defaults.clonePath,
+			};
+			return cachedConfig;
+		}
+	} catch {
+		// ignore parse errors
+	}
+
+	cachedConfig = defaults;
+	return cachedConfig;
+}
+
+const NON_CODE_SEGMENTS = new Set([
+	"issues", "pull", "pulls", "discussions", "releases", "wiki",
+	"actions", "settings", "security", "projects", "graphs",
+	"compare", "commits", "tags", "branches", "stargazers",
+	"watchers", "network", "forks", "milestone", "labels",
+	"packages", "codespaces", "contribute", "community",
+	"sponsors", "invitations", "notifications", "insights",
+]);
+
+export function parseGitHubUrl(url: string): GitHubUrlInfo | null {
+	let parsed: URL;
+	try {
+		parsed = new URL(url);
+	} catch {
+		return null;
+	}
+
+	if (parsed.hostname !== "github.com") return null;
+
+	const segments = parsed.pathname.split("/").filter(Boolean);
+	if (segments.length < 2) return null;
+
+	const owner = segments[0];
+	const repo = segments[1].replace(/\.git$/, "");
+
+	if (NON_CODE_SEGMENTS.has(segments[2]?.toLowerCase())) return null;
+
+	if (segments.length === 2) {
+		return { owner, repo, refIsFullSha: false, type: "root" };
+	}
+
+	const action = segments[2];
+	if (action !== "blob" && action !== "tree") return null;
+	if (segments.length < 4) return null;
+
+	const ref = segments[3];
+	const refIsFullSha = /^[0-9a-f]{40}$/.test(ref);
+	const pathParts = segments.slice(4);
+	const path = pathParts.length > 0 ? pathParts.join("/") : "";
+
+	return {
+		owner,
+		repo,
+		ref,
+		refIsFullSha,
+		path,
+		type: action as "blob" | "tree",
+	};
+}
+
+function cacheKey(owner: string, repo: string, ref?: string): string {
+	return ref ? `${owner}/${repo}@${ref}` : `${owner}/${repo}`;
+}
+
+function cloneDir(config: GitHubCloneConfig, owner: string, repo: string, ref?: string): string {
+	const dirName = ref ? `${repo}@${ref}` : repo;
+	return join(config.clonePath, owner, dirName);
+}
+
+function execClone(args: string[], localPath: string, timeoutMs: number, signal?: AbortSignal): Promise<string | null> {
+	return new Promise((resolve) => {
+		const child = execFile(args[0], args.slice(1), { timeout: timeoutMs }, (err) => {
+			if (err) {
+				try {
+					rmSync(localPath, { recursive: true, force: true });
+				} catch { /* ignore */ }
+				resolve(null);
+				return;
+			}
+			resolve(localPath);
+		});
+
+		if (signal) {
+			const onAbort = () => child.kill();
+			signal.addEventListener("abort", onAbort, { once: true });
+			child.on("exit", () => signal.removeEventListener("abort", onAbort));
+		}
+	});
+}
+
+async function cloneRepo(
+	owner: string,
+	repo: string,
+	ref: string | undefined,
+	config: GitHubCloneConfig,
+	signal?: AbortSignal,
+): Promise<string | null> {
+	const localPath = cloneDir(config, owner, repo, ref);
+
+	try {
+		rmSync(localPath, { recursive: true, force: true });
+	} catch { /* ignore */ }
+
+	const timeoutMs = config.cloneTimeoutSeconds * 1000;
+	const hasGh = await checkGhAvailable();
+
+	if (hasGh) {
+		const args = ["gh", "repo", "clone", `${owner}/${repo}`, localPath, "--", "--depth", "1", "--single-branch"];
+		if (ref) args.push("--branch", ref);
+		return execClone(args, localPath, timeoutMs, signal);
+	}
+
+	showGhHint();
+
+	const gitUrl = `https://github.com/${owner}/${repo}.git`;
+	const args = ["git", "clone", "--depth", "1", "--single-branch"];
+	if (ref) args.push("--branch", ref);
+	args.push(gitUrl, localPath);
+	return execClone(args, localPath, timeoutMs, signal);
+}
+
+function isBinaryFile(filePath: string): boolean {
+	const ext = extname(filePath).toLowerCase();
+	if (BINARY_EXTENSIONS.has(ext)) return true;
+
+	let fd: number;
+	try {
+		fd = openSync(filePath, "r");
+	} catch {
+		return false;
+	}
+	try {
+		const buf = Buffer.alloc(512);
+		const bytesRead = readSync(fd, buf, 0, 512, 0);
+		for (let i = 0; i < bytesRead; i++) {
+			if (buf[i] === 0) return true;
+		}
+	} catch {
+		return false;
+	} finally {
+		closeSync(fd);
+	}
+
+	return false;
+}
+
+function formatFileSize(bytes: number): string {
+	if (bytes < 1024) return `${bytes} B`;
+	if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+	return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+
+function buildTree(rootPath: string): string {
+	const entries: string[] = [];
+
+	function walk(dir: string, relPath: string): void {
+		if (entries.length >= MAX_TREE_ENTRIES) return;
+
+		let items: string[];
+		try {
+			items = readdirSync(dir).sort();
+		} catch {
+			return;
+		}
+
+		for (const item of items) {
+			if (entries.length >= MAX_TREE_ENTRIES) return;
+			if (item === ".git") continue;
+
+			const fullPath = join(dir, item);
+			let stat;
+			try {
+				stat = statSync(fullPath);
+			} catch {
+				continue;
+			}
+
+			const rel = relPath ? `${relPath}/${item}` : item;
+
+			if (stat.isDirectory()) {
+				if (NOISE_DIRS.has(item)) {
+					entries.push(`${rel}/  [skipped]`);
+					continue;
+				}
+				entries.push(`${rel}/`);
+				walk(fullPath, rel);
+			} else {
+				entries.push(rel);
+			}
+		}
+	}
+
+	walk(rootPath, "");
+
+	if (entries.length >= MAX_TREE_ENTRIES) {
+		entries.push(`... (truncated at ${MAX_TREE_ENTRIES} entries)`);
+	}
+
+	return entries.join("\n");
+}
+
+function buildDirListing(rootPath: string, subPath: string): string {
+	const targetPath = join(rootPath, subPath);
+	const lines: string[] = [];
+
+	let items: string[];
+	try {
+		items = readdirSync(targetPath).sort();
+	} catch {
+		return "(directory not readable)";
+	}
+
+	for (const item of items) {
+		if (item === ".git") continue;
+		const fullPath = join(targetPath, item);
+		try {
+			const stat = statSync(fullPath);
+			if (stat.isDirectory()) {
+				lines.push(`  ${item}/`);
+			} else {
+				lines.push(`  ${item}  (${formatFileSize(stat.size)})`);
+			}
+		} catch {
+			lines.push(`  ${item}  (unreadable)`);
+		}
+	}
+
+	return lines.join("\n");
+}
+
+function readReadme(localPath: string): string | null {
+	const candidates = ["README.md", "readme.md", "README", "README.txt", "README.rst"];
+	for (const name of candidates) {
+		const readmePath = join(localPath, name);
+		if (existsSync(readmePath)) {
+			try {
+				const content = readFileSync(readmePath, "utf-8");
+				return content.length > 8192 ? content.slice(0, 8192) + "\n\n[README truncated at 8K chars]" : content;
+			} catch {
+				return null;
+			}
+		}
+	}
+	return null;
+}
+
+function generateContent(localPath: string, info: GitHubUrlInfo): string {
+	const lines: string[] = [];
+	lines.push(`Repository cloned to: ${localPath}`);
+	lines.push("");
+
+	if (info.type === "root") {
+		lines.push("## Structure");
+		lines.push(buildTree(localPath));
+		lines.push("");
+
+		const readme = readReadme(localPath);
+		if (readme) {
+			lines.push("## README.md");
+			lines.push(readme);
+			lines.push("");
+		}
+
+		lines.push("Use `read` and `bash` tools at the path above to explore further.");
+		return lines.join("\n");
+	}
+
+	if (info.type === "tree") {
+		const dirPath = info.path || "";
+		const fullDirPath = join(localPath, dirPath);
+
+		if (!existsSync(fullDirPath)) {
+			lines.push(`Path \`${dirPath}\` not found in clone. Showing repository root instead.`);
+			lines.push("");
+			lines.push("## Structure");
+			lines.push(buildTree(localPath));
+		} else {
+			lines.push(`## ${dirPath || "/"}`);
+			lines.push(buildDirListing(localPath, dirPath));
+		}
+
+		lines.push("");
+		lines.push("Use `read` and `bash` tools at the path above to explore further.");
+		return lines.join("\n");
+	}
+
+	if (info.type === "blob") {
+		const filePath = info.path || "";
+		const fullFilePath = join(localPath, filePath);
+
+		if (!existsSync(fullFilePath)) {
+			lines.push(`Path \`${filePath}\` not found in clone. Showing repository root instead.`);
+			lines.push("");
+			lines.push("## Structure");
+			lines.push(buildTree(localPath));
+			lines.push("");
+			lines.push("Use `read` and `bash` tools at the path above to explore further.");
+			return lines.join("\n");
+		}
+
+		const stat = statSync(fullFilePath);
+
+		if (stat.isDirectory()) {
+			lines.push(`## ${filePath || "/"}`);
+			lines.push(buildDirListing(localPath, filePath));
+			lines.push("");
+			lines.push("Use `read` and `bash` tools at the path above to explore further.");
+			return lines.join("\n");
+		}
+
+		if (isBinaryFile(fullFilePath)) {
+			const ext = extname(filePath).replace(".", "");
+			lines.push(`## ${filePath}`);
+			lines.push(`Binary file (${ext}, ${formatFileSize(stat.size)}). Use \`read\` or \`bash\` tools at the path above to inspect.`);
+			return lines.join("\n");
+		}
+
+		const content = readFileSync(fullFilePath, "utf-8");
+		lines.push(`## ${filePath}`);
+
+		if (content.length > MAX_INLINE_FILE_CHARS) {
+			lines.push(content.slice(0, MAX_INLINE_FILE_CHARS));
+			lines.push("");
+			lines.push(`[File truncated at 100K chars. Full file: ${fullFilePath}]`);
+		} else {
+			lines.push(content);
+		}
+
+		lines.push("");
+		lines.push("Use `read` and `bash` tools at the path above to explore further.");
+		return lines.join("\n");
+	}
+
+	return lines.join("\n");
+}
+
+async function awaitCachedClone(
+	cached: CachedClone,
+	url: string,
+	owner: string,
+	repo: string,
+	info: GitHubUrlInfo,
+	signal?: AbortSignal,
+): Promise<ExtractedContent | null> {
+	if (signal?.aborted) return fetchViaApi(url, owner, repo, info);
+	const result = await cached.clonePromise;
+	if (signal?.aborted) return fetchViaApi(url, owner, repo, info);
+	if (result) {
+		const content = generateContent(result, info);
+		const title = info.path ? `${owner}/${repo} - ${info.path}` : `${owner}/${repo}`;
+		return { url, title, content, error: null };
+	}
+	return fetchViaApi(url, owner, repo, info);
+}
+
+export async function extractGitHub(
+	url: string,
+	signal?: AbortSignal,
+	forceClone?: boolean,
+): Promise<ExtractedContent | null> {
+	const info = parseGitHubUrl(url);
+	if (!info) return null;
+
+	const config = loadGitHubConfig();
+	if (!config.enabled) return null;
+
+	const { owner, repo } = info;
+	const key = cacheKey(owner, repo, info.ref);
+
+	const cached = cloneCache.get(key);
+	if (cached) return awaitCachedClone(cached, url, owner, repo, info, signal);
+
+	if (info.refIsFullSha) {
+		const sizeNote = `Note: Commit SHA URLs use the GitHub API instead of cloning.`;
+		return fetchViaApi(url, owner, repo, info, sizeNote);
+	}
+
+	const activityId = activityMonitor.logStart({ type: "fetch", url: `github.com/${owner}/${repo}` });
+
+	if (!forceClone) {
+		const sizeKB = await checkRepoSize(owner, repo);
+		if (sizeKB !== null) {
+			const sizeMB = sizeKB / 1024;
+			if (sizeMB > config.maxRepoSizeMB) {
+				activityMonitor.logComplete(activityId, 200);
+				const sizeNote =
+					`Note: Repository is ${Math.round(sizeMB)}MB (threshold: ${config.maxRepoSizeMB}MB). ` +
+					`Showing API-fetched content instead of full clone. Ask the user if they'd like to clone the full repo -- ` +
+					`if yes, call fetch_content again with the same URL and add forceClone: true to the params.`;
+				return fetchViaApi(url, owner, repo, info, sizeNote);
+			}
+		}
+	}
+
+	// Re-check: another concurrent caller may have started a clone while we awaited the size check
+	const cachedAfterSizeCheck = cloneCache.get(key);
+	if (cachedAfterSizeCheck) return awaitCachedClone(cachedAfterSizeCheck, url, owner, repo, info, signal);
+
+	const clonePromise = cloneRepo(owner, repo, info.ref, config, signal);
+	const localPath = cloneDir(config, owner, repo, info.ref);
+	cloneCache.set(key, { localPath, clonePromise });
+
+	const result = await clonePromise;
+
+	if (!result) {
+		cloneCache.delete(key);
+		activityMonitor.logError(activityId, "clone failed");
+
+		const apiFallback = await fetchViaApi(url, owner, repo, info);
+		if (apiFallback) return apiFallback;
+
+		return null;
+	}
+
+	activityMonitor.logComplete(activityId, 200);
+	const content = generateContent(result, info);
+	const title = info.path ? `${owner}/${repo} - ${info.path}` : `${owner}/${repo}`;
+	return { url, title, content, error: null };
+}
+
+export function clearCloneCache(): void {
+	for (const entry of cloneCache.values()) {
+		try {
+			rmSync(entry.localPath, { recursive: true, force: true });
+		} catch { /* ignore */ }
+	}
+	cloneCache.clear();
+	cachedConfig = null;
+}

package/index.ts

@@ -3,6 +3,7 @@ import { Key, Text, truncateToWidth } from "@mariozechner/pi-tui";
 import { Type } from "@sinclair/typebox";
 import { StringEnum } from "@mariozechner/pi-ai";
 import { fetchAllContent, type ExtractedContent } from "./extract.js";
+import { clearCloneCache } from "./github-extract.js";
 import { searchWithPerplexity, type SearchResult } from "./perplexity.js";
 import {
 	clearResults,
@@ -111,6 +112,7 @@ function formatEntryLine(
 
 function handleSessionChange(ctx: ExtensionContext): void {
 	abortPendingFetches();
+	clearCloneCache();
 	sessionActive = true;
 	restoreFromSession(ctx);
 	// Unsubscribe before clear() to avoid callback with stale ctx
@@ -148,6 +150,7 @@ export default function (pi: ExtensionAPI) {
 	pi.on("session_shutdown", () => {
 		sessionActive = false;
 		abortPendingFetches();
+		clearCloneCache();
 		clearResults();
 		// Unsubscribe before clear() to avoid callback with stale ctx
 		widgetUnsubscribe?.();
@@ -393,6 +396,9 @@ export default function (pi: ExtensionAPI) {
 		parameters: Type.Object({
 			url: Type.Optional(Type.String({ description: "Single URL to fetch" })),
 			urls: Type.Optional(Type.Array(Type.String(), { description: "Multiple URLs (parallel)" })),
+			forceClone: Type.Optional(Type.Boolean({
+				description: "Force cloning large GitHub repositories that exceed the size threshold",
+			})),
 		}),
 
 		async execute(_toolCallId, params, signal, onUpdate, _ctx) {
@@ -409,7 +415,9 @@ export default function (pi: ExtensionAPI) {
 				details: { phase: "fetch", progress: 0 },
 			});
 
-			const fetchResults = await fetchAllContent(urlList, signal);
+			const fetchResults = await fetchAllContent(urlList, signal, {
+				forceClone: params.forceClone,
+			});
 			const successful = fetchResults.filter((r) => !r.error).length;
 			const totalChars = fetchResults.reduce((sum, r) => sum + r.content.length, 0);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-web-access",
-  "version": "0.4.5",
+  "version": "0.5.0",
   "type": "module",
   "keywords": ["pi-package", "pi", "pi-coding-agent", "extension", "web-search", "perplexity", "fetch", "scraping"],
   "dependencies": {