pi-harness-runtime 1.1.80 → 1.1.82

package/harness/e2e/browser-scrapers/codex-browser-scraper.ts

@@ -0,0 +1,343 @@
+/**
+ * Codex Browser Scraper — Scrapes chatgpt.com/codex/cloud/settings/analytics
+ *
+ * Uses Playwright with Chrome profile to access the authenticated page
+ * and extract usage/quota data.
+ */
+
+import { BaseBrowserScraper } from "./base-browser-scraper.js";
+
+export interface CodexQuotaData {
+	provider: "openai-codex";
+	/** Plan type (e.g., "pro", "plus") */
+	planType?: string;
+	/** Current usage percentage (0-100) */
+	usagePercent: number;
+	/** Total quota amount */
+	totalQuota?: string;
+	/** Used quota amount */
+	usedQuota?: string;
+	/** Remaining quota */
+	remainingQuota?: string;
+	/** Reset date/time */
+	resetAt?: string;
+	/** Usage period description */
+	period?: string;
+	/** Raw data for debugging */
+	rawData?: Record<string, unknown>;
+	/** Timestamp */
+	scrapedAt: string;
+}
+
+interface CodexScraperConfig {
+	provider?: string;
+	url?: string;
+	headless?: boolean;
+	timeout?: number;
+	viewport?: { width: number; height: number };
+	slowMo?: number;
+}
+
+/**
+ * Codex Browser Scraper
+ *
+ * Scrapes usage data from chatgpt.com/codex/cloud/settings/analytics
+ */
+export class CodexBrowserScraper extends BaseBrowserScraper<CodexQuotaData> {
+	constructor(config: Partial<CodexScraperConfig> = {}) {
+		super({
+			provider: "openai-codex",
+			url: config.url ?? "https://chatgpt.com/codex/cloud/settings/analytics",
+			headless: config.headless ?? true,
+			timeout: config.timeout ?? 30000,
+			viewport: config.viewport,
+			slowMo: config.slowMo,
+		});
+	}
+
+	protected async createBrowser(): Promise<void> {
+		const { chromium } = await import("playwright");
+		const profileDir = this.getProfileDir();
+
+		this.browser = await chromium.launch({
+			headless: this.config.headless,
+			slowMo: this.config.slowMo,
+			args: [
+				`--user-data-dir=${profileDir}`,
+				"--disable-blink-features=AutomationControlled",
+			],
+		});
+
+		this.context = await this.browser.newContext({
+			viewport: this.config.viewport,
+		});
+
+		this.page = await this.context.newPage();
+	}
+
+	private getProfileDir(): string {
+		const { join } = require("node:path");
+		const { homedir } = require("node:os");
+		return join(homedir(), ".pi-harness-runtime", "browser-profiles", "codex");
+	}
+
+	protected async isLoginPage(): Promise<boolean> {
+		if (!this.page) return false;
+
+		// Check for common login indicators
+		const loginSelectors = [
+			'button[data-testid="sign-in"]',
+			'button:has-text("Sign in")',
+			'input[name="email"]',
+			'input[placeholder*="email" i]',
+			'[data-testid="login"]',
+		];
+
+		for (const selector of loginSelectors) {
+				try {
+					const element = await this.page.$(selector);
+					if (element != null) {
+					const text = await element.textContent();
+					if (text && /sign\s*in|log\s*in/i.test(text)) {
+						return true;
+					}
+				}
+			} catch {
+				// Selector might not exist, continue
+			}
+		}
+
+		// Check URL for login-related paths
+		const url = this.page.url();
+		if (/signin|login|auth/i.test(url)) {
+			return true;
+		}
+
+		return false;
+	}
+
+	protected async extractData(): Promise<CodexQuotaData | null> {
+		if (!this.page) return null;
+
+		try {
+			// Wait for the analytics page to load
+			await this.page.waitForTimeout(3000);
+
+			// Try multiple selectors for usage data
+			const usageData = await this.extractUsageData();
+			if (!usageData) {
+				return null;
+			}
+
+			return {
+				provider: "openai-codex",
+				...usageData,
+				scrapedAt: new Date().toISOString(),
+			};
+		} catch (error) {
+			console.error(`[codex] Extraction error: ${error}`);
+			return null;
+		}
+	}
+
+	private async extractUsageData(): Promise<Omit<CodexQuotaData, "provider" | "scrapedAt"> | null> {
+		if (!this.page) return null;
+
+		// Strategy 1: Look for percentage display
+		const percentSelectors = [
+			// Common patterns for usage percentage
+			'[class*="usage"] [class*="percent"]',
+			'[data-testid*="usage"] [class*="progress"]',
+			'[class*="quota"] [class*="percent"]',
+			// Text content patterns
+			'text=/\\d+%/',
+			// SVG progress circles
+			'svg[class*="progress"]',
+			// Heading patterns
+			'h1:has-text("Usage") + *',
+			'h2:has-text("Usage") + *',
+		];
+
+		for (const selector of percentSelectors) {
+				try {
+					const element = await this.page.$(selector);
+					if (element != null) {
+					const text = await element.textContent();
+					const percentMatch = text?.match(/(\d+(?:\.\d+)?)\s*%/);
+					if (percentMatch) {
+						const usagePercent = parseFloat(percentMatch[1]);
+						if (!isNaN(usagePercent)) {
+							// Try to get additional data
+							const additionalData = await this.extractAdditionalData();
+							return {
+								usagePercent,
+								...additionalData,
+							};
+						}
+					}
+				}
+			} catch {
+				// Continue to next selector
+			}
+		}
+
+		// Strategy 2: Look for numbers in heading/title
+		const headingData = await this.extractFromHeadings();
+		if (headingData) {
+			return headingData;
+		}
+
+		// Strategy 3: Look for chart/graph data (often contains raw numbers)
+		const chartData = await this.extractChartData();
+		if (chartData) {
+			return chartData;
+		}
+
+		return null;
+	}
+
+	private async extractAdditionalData(): Promise<Partial<CodexQuotaData>> {
+		const result: Partial<CodexQuotaData> = {};
+
+		// Try to extract plan type
+		const planSelectors = [
+			'[class*="plan"]',
+			'[data-testid*="plan"]',
+			'text=/Pro|Plus|Team|Enterprise/i',
+		];
+
+		for (const selector of planSelectors) {
+			try {
+				if (!this.page) continue;
+				const text = await this.page.locator(selector).first().textContent({ timeout: 2000 }).catch(() => null);
+				if (text && /Pro|Plus|Team|Enterprise/i.test(text)) {
+					result.planType = text.trim();
+					break;
+				}
+			} catch {
+				// Continue
+			}
+		}
+
+		// Try to extract quota amounts (e.g., "1.5 GB of 5 GB used")
+		const quotaText = await this.extractText('text=/\\d+.*\\/(?:of\\s+)?\\d+/');
+		if (quotaText) {
+			const match = quotaText.match(/([\d,.]+)\s*(?:GB|MB|TB)?\s*(?:\/|of)\s*([\d,.]+)\s*(?:GB|MB|TB)?/i);
+			if (match) {
+				result.usedQuota = match[1];
+				result.totalQuota = match[2];
+				result.remainingQuota = String(parseFloat(match[2]) - parseFloat(match[1]));
+			}
+		}
+
+		// Try to extract reset date
+		const resetSelectors = [
+			'text=/resets?\\s+(at|on|in)/i',
+			'[class*="reset"]',
+			'[class*="renews"]',
+		];
+
+		for (const selector of resetSelectors) {
+			try {
+				if (!this.page) continue;
+				const text = await this.page.locator(selector).first().textContent({ timeout: 2000 }).catch(() => null);
+				if (text) {
+					result.resetAt = text.trim();
+					break;
+				}
+			} catch {
+				// Continue
+			}
+		}
+
+		return result;
+	}
+
+	private async extractFromHeadings(): Promise<Omit<CodexQuotaData, "provider" | "scrapedAt"> | null> {
+		if (!this.page) return null;
+
+		// Get all headings and look for ones containing numbers
+		const headings = await this.page.$$("h1, h2, h3, h4");
+		for (const heading of headings) {
+			const text = await heading.textContent();
+			if (text) {
+				const percentMatch = text.match(/(\d+(?:\.\d+)?)\s*%/);
+				if (percentMatch) {
+					return {
+						usagePercent: parseFloat(percentMatch[1]),
+						period: text.replace(percentMatch[0], "").trim() || undefined,
+					};
+				}
+			}
+		}
+
+		return null;
+	}
+
+	private async extractChartData(): Promise<Omit<CodexQuotaData, "provider" | "scrapedAt"> | null> {
+		if (!this.page) return null;
+
+		// Look for canvas or SVG charts
+		const charts = await this.page.$$("canvas, svg[class*='chart'], svg[class*='graph']");
+		if (charts.length > 0) {
+			// Try to get data from aria-labels or data attributes
+			for (const chart of charts) {
+				const ariaLabel = await chart.getAttribute("aria-label");
+				if (ariaLabel) {
+					const percentMatch = ariaLabel.match(/(\d+(?:\.\d+)?)\s*%/);
+					if (percentMatch) {
+						return {
+							usagePercent: parseFloat(percentMatch[1]),
+						};
+					}
+				}
+			}
+
+			// Try to extract from JavaScript data
+			// eslint-disable-next-line @typescript-eslint/no-explicit-any
+			const chartData = await this.page.evaluate(() => {
+				// eslint-disable-next-line @typescript-eslint/no-explicit-any
+			const win = (globalThis as any);
+			const doc = win.document;
+			if (!doc) return null;
+			const canvases = doc.querySelectorAll("canvas");
+			const foundResult: { percent: number } | null = null;
+			for (const canvas of canvases) {
+				const parent = canvas.parentElement;
+				if (parent) {
+					const attrNames: string[] = Array.from(parent.getAttributeNames());
+					for (const attrName of attrNames) {
+						if (attrName.startsWith("data-") || attrName.includes("chart")) {
+							const value = parent.getAttribute(attrName);
+							const match = value?.match(/(\d+(?:\.\d+)?)\s*%/);
+							if (match) {
+								return { percent: parseFloat(match[1]) };
+							}
+						}
+					}
+				}
+			}
+			return foundResult;
+		});
+
+			if (chartData?.percent) {
+				return { usagePercent: chartData.percent };
+			}
+		}
+
+		return null;
+	}
+}
+
+/**
+ * Quick scrape function
+ */
+export async function scrapeCodexQuota(): Promise<CodexQuotaData | null> {
+	const scraper = new CodexBrowserScraper({ headless: true });
+	try {
+		const result = await scraper.scrape();
+		return result.data ?? null;
+	} finally {
+		await scraper.cleanup();
+	}
+}

package/harness/e2e/browser-scrapers/index.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Browser Scrapers — Browser-based quota scraping via Playwright
+ *
+ * Authentication strategies by provider:
+ *
+ * | Provider  | Auth Method          | Cookies Work? | Notes                    |
+ * |-----------|----------------------|---------------|--------------------------|
+ * | MiniMax   | Browser cookies      | Yes           | Direct browser scraping  |
+ * | ChatGPT   | OAuth tokens        | No            | Use ~/.codex/auth.json   |
+ * | Codex     | OAuth tokens        | No            | Use ~/.codex/auth.json   |
+ * | Z.ai      | Browser cookies     | Unknown       | Try browser profile      |
+ *
+ * For providers where browser cookies don't work for automation:
+ * - Use API tokens (OAuth, API keys)
+ * - Export cookies from DevTools (may not work due to HttpOnly, Secure flags)
+ * - Manual authentication via headful browser with user interaction
+ */
+export { ChromeProfileManager, getChromeProfileManager, detectChromeUserDataDir, detectChromeBinary, type ChromeProfileConfig, type BrowserProfile, } from "./chrome-profile-manager.js";
+export { BaseBrowserScraper, type BrowserScraperConfig, type QuotaScrapingResult, } from "./base-browser-scraper.js";
+export { CodexBrowserScraper, scrapeCodexQuota, type CodexQuotaData } from "./codex-browser-scraper.js";
+export { ZaiBrowserScraper, scrapeZaiQuota, checkZaiCookies, type ZaiQuotaData } from "./zai-browser-scraper.js";
+/**
+ * All available browser scrapers
+ */
+export declare const BROWSER_SCRAPERS: {
+    readonly "openai-codex": () => Promise<typeof import("./codex-browser-scraper.js").CodexBrowserScraper>;
+    readonly "z-ai": () => Promise<typeof import("./zai-browser-scraper.js").ZaiBrowserScraper>;
+};
+export type BrowserScraperProvider = keyof typeof BROWSER_SCRAPERS;
+//# sourceMappingURL=index.d.ts.map

package/harness/e2e/browser-scrapers/index.ts

@@ -0,0 +1,48 @@
+/**
+ * Browser Scrapers — Browser-based quota scraping via Playwright
+ *
+ * Authentication strategies by provider:
+ *
+ * | Provider  | Auth Method          | Cookies Work? | Notes                    |
+ * |-----------|----------------------|---------------|--------------------------|
+ * | MiniMax   | Browser cookies      | Yes           | Direct browser scraping  |
+ * | ChatGPT   | OAuth tokens        | No            | Use ~/.codex/auth.json   |
+ * | Codex     | OAuth tokens        | No            | Use ~/.codex/auth.json   |
+ * | Z.ai      | Browser cookies     | Unknown       | Try browser profile      |
+ *
+ * For providers where browser cookies don't work for automation:
+ * - Use API tokens (OAuth, API keys)
+ * - Export cookies from DevTools (may not work due to HttpOnly, Secure flags)
+ * - Manual authentication via headful browser with user interaction
+ */
+
+// Chrome profile management for persistent sessions
+export {
+	ChromeProfileManager,
+	getChromeProfileManager,
+	detectChromeUserDataDir,
+	detectChromeBinary,
+	type ChromeProfileConfig,
+	type BrowserProfile,
+} from "./chrome-profile-manager.js";
+
+// Base scraper class
+export {
+	BaseBrowserScraper,
+	type BrowserScraperConfig,
+	type QuotaScrapingResult,
+} from "./base-browser-scraper.js";
+
+// Provider-specific scrapers
+export { CodexBrowserScraper, scrapeCodexQuota, type CodexQuotaData } from "./codex-browser-scraper.js";
+export { ZaiBrowserScraper, scrapeZaiQuota, checkZaiCookies, type ZaiQuotaData } from "./zai-browser-scraper.js";
+
+/**
+ * All available browser scrapers
+ */
+export const BROWSER_SCRAPERS = {
+	"openai-codex": () => import("./codex-browser-scraper.js").then((m) => m.CodexBrowserScraper),
+	"z-ai": () => import("./zai-browser-scraper.js").then((m) => m.ZaiBrowserScraper),
+} as const;
+
+export type BrowserScraperProvider = keyof typeof BROWSER_SCRAPERS;

package/harness/e2e/browser-scrapers/zai-browser-scraper.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Z.ai Browser Scraper — Scrapes z.ai usage data
+ *
+ * Authentication approach:
+ * - Z.ai uses cookies from browser login (different from ChatGPT)
+ * - Cookies from ~/.config/google-chrome or browser export need to be tested
+ * - Fallback: Manual cookie export from browser DevTools
+ *
+ * Note: If cookies don't work for browser automation (like Codex),
+ * we may need to find alternative auth methods (OAuth, API keys, etc.)
+ */
+import { BaseBrowserScraper } from "./base-browser-scraper.js";
+export interface ZaiQuotaData {
+    provider: "z-ai";
+    /** Plan type (e.g., "Pro", "Team", "Enterprise") */
+    planType?: string;
+    /** Current usage percentage (0-100) */
+    usagePercent: number;
+    /** Total quota (e.g., "5 GB") */
+    totalQuota?: string;
+    /** Used quota (e.g., "1.5 GB") */
+    usedQuota?: string;
+    /** Remaining quota */
+    remainingQuota?: string;
+    /** Reset date/time */
+    resetAt?: string;
+    /** Billing period */
+    period?: string;
+    /** Timestamp */
+    scrapedAt: string;
+}
+interface ZaiScraperConfig {
+    provider?: string;
+    url?: string;
+    headless?: boolean;
+    timeout?: number;
+    viewport?: {
+        width: number;
+        height: number;
+    };
+    slowMo?: number;
+    /** Path to cookies file (Netscape format) */
+    cookiesFile?: string;
+}
+/**
+ * Z.ai Browser Scraper
+ */
+export declare class ZaiBrowserScraper extends BaseBrowserScraper<ZaiQuotaData> {
+    private cookiesFile?;
+    constructor(config?: Partial<ZaiScraperConfig>);
+    /**
+     * Find available cookies file
+     */
+    private findCookiesFile;
+    protected createBrowser(): Promise<void>;
+    /**
+     * Load cookies from file and convert to headers (fallback auth method)
+     * Note: This won't work for all sites - some require full cookie jar
+     */
+    private loadCookiesAsHeaders;
+    protected isLoginPage(): Promise<boolean>;
+    protected extractData(): Promise<ZaiQuotaData | null>;
+    private extractUsagePercent;
+    private extractPlanType;
+    private extractQuotaData;
+    private extractResetDate;
+}
+/**
+ * Quick scrape function
+ */
+export declare function scrapeZaiQuota(cookiesFile?: string): Promise<ZaiQuotaData | null>;
+/**
+ * Check if z.ai cookies are available
+ */
+export declare function checkZaiCookies(): {
+    available: boolean;
+    path?: string;
+};
+export {};
+//# sourceMappingURL=zai-browser-scraper.d.ts.map