pi-harness-runtime 1.1.81 → 1.1.83

package/harness/browser-scrapers/base-browser-scraper.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Base Browser Scraper — Abstract class for browser-based quota scraping
+ *
+ * Uses Playwright with Chrome profiles to scrape JavaScript-rendered pages
+ * that require authentication. Similar approach to google-flow-mcp.
+ */
+import type { Page, Browser, BrowserContext } from "playwright";
+export interface BrowserScraperConfig {
+    /** Headless mode (default: true) */
+    headless?: boolean;
+    /** Page load timeout in ms (default: 30000) */
+    timeout?: number;
+    /** Viewport dimensions */
+    viewport?: {
+        width: number;
+        height: number;
+    };
+    /** Slow down operations by ms (for debugging) */
+    slowMo?: number;
+    /** Provider name for logging */
+    provider: string;
+    /** URL to scrape */
+    url: string;
+}
+export interface QuotaScrapingResult<T> {
+    /** Whether scraping was successful */
+    success: boolean;
+    /** The scraped data */
+    data?: T;
+    /** Error message if failed */
+    error?: string;
+    /** Timestamp of scrape */
+    scrapedAt: string;
+    /** Screenshot path if debug screenshot was taken */
+    screenshotPath?: string;
+}
+/**
+ * Abstract base class for browser-based quota scrapers
+ */
+export declare abstract class BaseBrowserScraper<T> {
+    protected config: Required<BrowserScraperConfig>;
+    protected browser: Browser | null;
+    protected context: BrowserContext | null;
+    protected page: Page | null;
+    protected screenshotsEnabled: boolean;
+    protected screenshotsDir: string;
+    constructor(config: BrowserScraperConfig);
+    /**
+     * Initialize Playwright and launch browser
+     */
+    protected abstract createBrowser(): Promise<void>;
+    /**
+     * Take a debug screenshot
+     */
+    takeScreenshot(name: string): Promise<string | undefined>;
+    /**
+     * Wait for a selector with retry logic
+     */
+    protected waitForSelector(selector: string, options?: {
+        timeout?: number;
+        state?: "attached" | "detached" | "visible" | "hidden";
+    }): Promise<boolean>;
+    /**
+     * Extract text content from page
+     */
+    protected extractText(selector: string): Promise<string | null>;
+    /**
+     * Extract number from text (handles various formats like "1.5 GB", "$10.00", etc.)
+     */
+    protected extractNumber(text: string | null, pattern?: RegExp): number | null;
+    /**
+     * Main scraping method — navigates and extracts data
+     */
+    scrape(): Promise<QuotaScrapingResult<T>>;
+    /**
+     * Check if current page is a login page
+     */
+    protected abstract isLoginPage(): Promise<boolean>;
+    /**
+     * Extract quota data from the page
+     */
+    protected abstract extractData(): Promise<T | null>;
+    /**
+     * Cleanup browser resources
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Enable screenshot capture for debugging
+     */
+    enableScreenshots(dir?: string): void;
+}

package/harness/browser-scrapers/chrome-profile-manager.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Chrome Profile Manager — Browser-based quota scraping via Playwright
+ *
+ * Key concepts from google-flow-mcp:
+ * - Uses Playwright to connect to existing Chrome profile with stored cookies
+ * - Maintains persistent session across runs
+ * - Bypasses anti-bot measures by launching Chrome directly
+ *
+ * This enables scraping JavaScript-rendered pages that require authentication:
+ * - chatgpt.com/codex/cloud/settings/analytics
+ * - z.ai/manage-apikey/coding-plan/personal/usage
+ * - etc.
+ */
+export interface ChromeProfileConfig {
+    /** Provider name (e.g., "codex", "z-ai") */
+    provider: string;
+    /** Path to Chrome user data directory (auto-detected if not provided) */
+    userDataDir?: string;
+    /** Profile directory name within userDataDir */
+    profileName?: string;
+    /** Path to browser binary (auto-detected if not provided) */
+    binaryPath?: string;
+    /** Arguments to pass to Chrome */
+    extraArgs?: string[];
+    /** Headless mode (default: true for scraping) */
+    headless?: boolean;
+    /** Launch timeout in ms */
+    launchTimeout?: number;
+}
+export interface BrowserProfile {
+    /** Path to the profile directory */
+    profilePath: string;
+    /** Provider name */
+    provider: string;
+    /** Whether the profile exists */
+    exists: boolean;
+    /** Last used timestamp */
+    lastUsed?: number;
+}
+/** Auto-detect Chrome user data directory based on OS */
+export declare function detectChromeUserDataDir(): string | null;
+/** Auto-detect Chrome binary path */
+export declare function detectChromeBinary(): string | null;
+/**
+ * Chrome Profile Manager for browser-based scraping
+ *
+ * Manages Chrome profiles for different providers to maintain
+ * authenticated sessions across scraping runs.
+ */
+export declare class ChromeProfileManager {
+    private profilesDir;
+    private profiles;
+    constructor(profilesDir?: string);
+    private ensureProfilesDir;
+    private getProfileMetaPath;
+    private loadProfiles;
+    /**
+     * Get profile path for a provider
+     */
+    getProfilePath(provider: string): string;
+    /**
+     * Ensure profile directory exists
+     */
+    ensureProfile(provider: string): BrowserProfile;
+    private saveProfileMeta;
+    /**
+     * List all managed profiles
+     */
+    listProfiles(): BrowserProfile[];
+    /**
+     * Delete a profile
+     */
+    deleteProfile(provider: string): Promise<void>;
+    /**
+     * Get Chrome launch arguments for anti-bot evasion
+     * Based on google-flow-mcp approach
+     */
+    getLaunchArgs(config: ChromeProfileConfig): string[];
+}
+export declare function getChromeProfileManager(): ChromeProfileManager;

package/harness/browser-scrapers/codex-browser-scraper.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Codex Browser Scraper — Scrapes chatgpt.com/codex/cloud/settings/analytics
+ *
+ * Uses Playwright with Chrome profile to access the authenticated page
+ * and extract usage/quota data.
+ */
+import { BaseBrowserScraper } from "./base-browser-scraper.js";
+export interface CodexQuotaData {
+    provider: "openai-codex";
+    /** Plan type (e.g., "pro", "plus") */
+    planType?: string;
+    /** Current usage percentage (0-100) */
+    usagePercent: number;
+    /** Total quota amount */
+    totalQuota?: string;
+    /** Used quota amount */
+    usedQuota?: string;
+    /** Remaining quota */
+    remainingQuota?: string;
+    /** Reset date/time */
+    resetAt?: string;
+    /** Usage period description */
+    period?: string;
+    /** Raw data for debugging */
+    rawData?: Record<string, unknown>;
+    /** Timestamp */
+    scrapedAt: string;
+}
+interface CodexScraperConfig {
+    provider?: string;
+    url?: string;
+    headless?: boolean;
+    timeout?: number;
+    viewport?: {
+        width: number;
+        height: number;
+    };
+    slowMo?: number;
+}
+/**
+ * Codex Browser Scraper
+ *
+ * Scrapes usage data from chatgpt.com/codex/cloud/settings/analytics
+ */
+export declare class CodexBrowserScraper extends BaseBrowserScraper<CodexQuotaData> {
+    constructor(config?: Partial<CodexScraperConfig>);
+    protected createBrowser(): Promise<void>;
+    private getProfileDir;
+    protected isLoginPage(): Promise<boolean>;
+    protected extractData(): Promise<CodexQuotaData | null>;
+    private extractUsageData;
+    private extractAdditionalData;
+    private extractFromHeadings;
+    private extractChartData;
+}
+/**
+ * Quick scrape function
+ */
+export declare function scrapeCodexQuota(): Promise<CodexQuotaData | null>;
+export {};

package/harness/browser-scrapers/index.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Browser Scrapers — Browser-based quota scraping via Playwright
+ *
+ * Authentication strategies by provider:
+ *
+ * | Provider  | Auth Method          | Cookies Work? | Notes                    |
+ * |-----------|----------------------|---------------|--------------------------|
+ * | MiniMax   | Browser cookies      | Yes           | Direct browser scraping  |
+ * | ChatGPT   | OAuth tokens        | No            | Use ~/.codex/auth.json   |
+ * | Codex     | OAuth tokens        | No            | Use ~/.codex/auth.json   |
+ * | Z.ai      | Browser cookies     | Unknown       | Try browser profile      |
+ *
+ * For providers where browser cookies don't work for automation:
+ * - Use API tokens (OAuth, API keys)
+ * - Export cookies from DevTools (may not work due to HttpOnly, Secure flags)
+ * - Manual authentication via headful browser with user interaction
+ */
+export { ChromeProfileManager, getChromeProfileManager, detectChromeUserDataDir, detectChromeBinary, type ChromeProfileConfig, type BrowserProfile, } from "./chrome-profile-manager.js";
+export { BaseBrowserScraper, type BrowserScraperConfig, type QuotaScrapingResult, } from "./base-browser-scraper.js";
+export { CodexBrowserScraper, scrapeCodexQuota, type CodexQuotaData } from "./codex-browser-scraper.js";
+export { ZaiBrowserScraper, scrapeZaiQuota, checkZaiCookies, type ZaiQuotaData } from "./zai-browser-scraper.js";
+/**
+ * All available browser scrapers
+ */
+export declare const BROWSER_SCRAPERS: {
+    readonly "openai-codex": () => Promise<typeof import("./codex-browser-scraper.js").CodexBrowserScraper>;
+    readonly "z-ai": () => Promise<typeof import("./zai-browser-scraper.js").ZaiBrowserScraper>;
+};
+export type BrowserScraperProvider = keyof typeof BROWSER_SCRAPERS;

package/harness/browser-scrapers/zai-browser-scraper.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Z.ai Browser Scraper — Scrapes z.ai usage data
+ *
+ * Authentication approach:
+ * - Z.ai uses cookies from browser login (different from ChatGPT)
+ * - Cookies from ~/.config/google-chrome or browser export need to be tested
+ * - Fallback: Manual cookie export from browser DevTools
+ *
+ * Note: If cookies don't work for browser automation (like Codex),
+ * we may need to find alternative auth methods (OAuth, API keys, etc.)
+ */
+import { BaseBrowserScraper } from "./base-browser-scraper.js";
+export interface ZaiQuotaData {
+    provider: "z-ai";
+    /** Plan type (e.g., "Pro", "Team", "Enterprise") */
+    planType?: string;
+    /** Current usage percentage (0-100) */
+    usagePercent: number;
+    /** Total quota (e.g., "5 GB") */
+    totalQuota?: string;
+    /** Used quota (e.g., "1.5 GB") */
+    usedQuota?: string;
+    /** Remaining quota */
+    remainingQuota?: string;
+    /** Reset date/time */
+    resetAt?: string;
+    /** Billing period */
+    period?: string;
+    /** Timestamp */
+    scrapedAt: string;
+}
+interface ZaiScraperConfig {
+    provider?: string;
+    url?: string;
+    headless?: boolean;
+    timeout?: number;
+    viewport?: {
+        width: number;
+        height: number;
+    };
+    slowMo?: number;
+    /** Path to cookies file (Netscape format) */
+    cookiesFile?: string;
+}
+/**
+ * Z.ai Browser Scraper
+ */
+export declare class ZaiBrowserScraper extends BaseBrowserScraper<ZaiQuotaData> {
+    private cookiesFile?;
+    constructor(config?: Partial<ZaiScraperConfig>);
+    /**
+     * Find available cookies file
+     */
+    private findCookiesFile;
+    protected createBrowser(): Promise<void>;
+    /**
+     * Load cookies from file and convert to headers (fallback auth method)
+     * Note: This won't work for all sites - some require full cookie jar
+     */
+    private loadCookiesAsHeaders;
+    protected isLoginPage(): Promise<boolean>;
+    protected extractData(): Promise<ZaiQuotaData | null>;
+    private extractUsagePercent;
+    private extractPlanType;
+    private extractQuotaData;
+    private extractResetDate;
+}
+/**
+ * Quick scrape function
+ */
+export declare function scrapeZaiQuota(cookiesFile?: string): Promise<ZaiQuotaData | null>;
+/**
+ * Check if z.ai cookies are available
+ */
+export declare function checkZaiCookies(): {
+    available: boolean;
+    path?: string;
+};
+export {};

package/harness/e2e/browser-scrapers/base-browser-scraper.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Base Browser Scraper — Abstract class for browser-based quota scraping
+ *
+ * Uses Playwright with Chrome profiles to scrape JavaScript-rendered pages
+ * that require authentication. Similar approach to google-flow-mcp.
+ */
+import type { Page, Browser, BrowserContext } from "playwright";
+export interface BrowserScraperConfig {
+    /** Headless mode (default: true) */
+    headless?: boolean;
+    /** Page load timeout in ms (default: 30000) */
+    timeout?: number;
+    /** Viewport dimensions */
+    viewport?: {
+        width: number;
+        height: number;
+    };
+    /** Slow down operations by ms (for debugging) */
+    slowMo?: number;
+    /** Provider name for logging */
+    provider: string;
+    /** URL to scrape */
+    url: string;
+}
+export interface QuotaScrapingResult<T> {
+    /** Whether scraping was successful */
+    success: boolean;
+    /** The scraped data */
+    data?: T;
+    /** Error message if failed */
+    error?: string;
+    /** Timestamp of scrape */
+    scrapedAt: string;
+    /** Screenshot path if debug screenshot was taken */
+    screenshotPath?: string;
+}
+/**
+ * Abstract base class for browser-based quota scrapers
+ */
+export declare abstract class BaseBrowserScraper<T> {
+    protected config: Required<BrowserScraperConfig>;
+    protected browser: Browser | null;
+    protected context: BrowserContext | null;
+    protected page: Page | null;
+    protected screenshotsEnabled: boolean;
+    protected screenshotsDir: string;
+    constructor(config: BrowserScraperConfig);
+    /**
+     * Initialize Playwright and launch browser
+     */
+    protected abstract createBrowser(): Promise<void>;
+    /**
+     * Take a debug screenshot
+     */
+    takeScreenshot(name: string): Promise<string | undefined>;
+    /**
+     * Wait for a selector with retry logic
+     */
+    protected waitForSelector(selector: string, options?: {
+        timeout?: number;
+        state?: "attached" | "detached" | "visible" | "hidden";
+    }): Promise<boolean>;
+    /**
+     * Extract text content from page
+     */
+    protected extractText(selector: string): Promise<string | null>;
+    /**
+     * Extract number from text (handles various formats like "1.5 GB", "$10.00", etc.)
+     */
+    protected extractNumber(text: string | null, pattern?: RegExp): number | null;
+    /**
+     * Main scraping method — navigates and extracts data
+     */
+    scrape(): Promise<QuotaScrapingResult<T>>;
+    /**
+     * Check if current page is a login page
+     */
+    protected abstract isLoginPage(): Promise<boolean>;
+    /**
+     * Extract quota data from the page
+     */
+    protected abstract extractData(): Promise<T | null>;
+    /**
+     * Cleanup browser resources
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Enable screenshot capture for debugging
+     */
+    enableScreenshots(dir?: string): void;
+}
+//# sourceMappingURL=base-browser-scraper.d.ts.map

package/harness/e2e/browser-scrapers/base-browser-scraper.ts

@@ -0,0 +1,244 @@
+/**
+ * Base Browser Scraper — Abstract class for browser-based quota scraping
+ *
+ * Uses Playwright with Chrome profiles to scrape JavaScript-rendered pages
+ * that require authentication. Similar approach to google-flow-mcp.
+ */
+
+import type { Page, Browser, BrowserContext } from "playwright";
+
+export interface BrowserScraperConfig {
+	/** Headless mode (default: true) */
+	headless?: boolean;
+	/** Page load timeout in ms (default: 30000) */
+	timeout?: number;
+	/** Viewport dimensions */
+	viewport?: { width: number; height: number };
+	/** Slow down operations by ms (for debugging) */
+	slowMo?: number;
+	/** Provider name for logging */
+	provider: string;
+	/** URL to scrape */
+	url: string;
+}
+
+export interface QuotaScrapingResult<T> {
+	/** Whether scraping was successful */
+	success: boolean;
+	/** The scraped data */
+	data?: T;
+	/** Error message if failed */
+	error?: string;
+	/** Timestamp of scrape */
+	scrapedAt: string;
+	/** Screenshot path if debug screenshot was taken */
+	screenshotPath?: string;
+}
+
+/**
+ * Abstract base class for browser-based quota scrapers
+ */
+export abstract class BaseBrowserScraper<T> {
+	protected config: Required<BrowserScraperConfig>;
+	protected browser: Browser | null = null;
+	protected context: BrowserContext | null = null;
+	protected page: Page | null = null;
+	protected screenshotsEnabled = false;
+	protected screenshotsDir = "/tmp/pi-harness-screenshots";
+
+	constructor(config: BrowserScraperConfig) {
+		this.config = {
+			headless: config.headless ?? true,
+			timeout: config.timeout ?? 30000,
+			viewport: config.viewport ?? { width: 1280, height: 720 },
+			slowMo: config.slowMo ?? 0,
+			provider: config.provider,
+			url: config.url,
+		};
+	}
+
+	/**
+	 * Initialize Playwright and launch browser
+	 */
+	protected abstract createBrowser(): Promise<void>;
+
+	/**
+	 * Take a debug screenshot
+	 */
+	async takeScreenshot(name: string): Promise<string | undefined> {
+		if (!this.page) return undefined;
+
+		try {
+			const { existsSync, mkdirSync } = require("node:fs");
+			const path = require("node:path");
+
+			if (!existsSync(this.screenshotsDir)) {
+				mkdirSync(this.screenshotsDir, { recursive: true });
+			}
+
+			const filePath = path.join(
+				this.screenshotsDir,
+				`${this.config.provider}-${name}-${Date.now()}.png`,
+			);
+
+			await this.page.screenshot({ path: filePath, fullPage: true });
+			console.log(`[${this.config.provider}] Screenshot saved: ${filePath}`);
+			return filePath;
+		} catch {
+			return undefined;
+		}
+	}
+
+	/**
+	 * Wait for a selector with retry logic
+	 */
+	protected async waitForSelector(
+		selector: string,
+		options?: { timeout?: number; state?: "attached" | "detached" | "visible" | "hidden" },
+	): Promise<boolean> {
+		if (!this.page) return false;
+
+		try {
+			await this.page.waitForSelector(selector, {
+				timeout: options?.timeout ?? this.config.timeout,
+				state: options?.state ?? "visible",
+			});
+			return true;
+		} catch {
+			return false;
+		}
+	}
+
+	/**
+	 * Extract text content from page
+	 */
+	protected async extractText(selector: string): Promise<string | null> {
+		if (!this.page) return null;
+
+		try {
+			const element = await this.page.$(selector);
+			if (!element) return null;
+			return await element.textContent();
+		} catch {
+			return null;
+		}
+	}
+
+	/**
+	 * Extract number from text (handles various formats like "1.5 GB", "$10.00", etc.)
+	 */
+	protected extractNumber(text: string | null, pattern?: RegExp): number | null {
+		if (!text) return null;
+
+		const cleaned = pattern ? text.match(pattern)?.[0] ?? text : text;
+		const match = cleaned.match(/[\d,.]+/);
+		if (!match) return null;
+
+		const num = parseFloat(match[0].replace(/,/g, ""));
+		return isNaN(num) ? null : num;
+	}
+
+	/**
+	 * Main scraping method — navigates and extracts data
+	 */
+	async scrape(): Promise<QuotaScrapingResult<T>> {
+		const result: QuotaScrapingResult<T> = {
+			success: false,
+			scrapedAt: new Date().toISOString(),
+		};
+
+		try {
+			// Initialize browser
+			await this.createBrowser();
+
+			if (!this.page) {
+				result.error = "Failed to create browser page";
+				return result;
+			}
+
+			// Navigate to URL
+			console.log(`[${this.config.provider}] Navigating to ${this.config.url}`);
+			await this.page.goto(this.config.url, {
+				waitUntil: "networkidle",
+				timeout: this.config.timeout,
+			});
+
+			// Wait for page to settle
+			await this.page.waitForTimeout(2000);
+
+			// Take screenshot for debugging if enabled
+			if (this.screenshotsEnabled) {
+				await this.takeScreenshot("initial-load");
+			}
+
+			// Check if login is required
+			if (await this.isLoginPage()) {
+				result.error = "Login required - browser session not authenticated";
+				await this.takeScreenshot("login-required");
+				return result;
+			}
+
+			// Extract data using provider-specific logic
+			const data = await this.extractData();
+			if (!data) {
+				result.error = "Failed to extract quota data from page";
+				await this.takeScreenshot("extraction-failed");
+				return result;
+			}
+
+			result.success = true;
+			result.data = data;
+
+		} catch (error) {
+			result.error = error instanceof Error ? error.message : String(error);
+			await this.takeScreenshot("error");
+
+		} finally {
+			await this.cleanup();
+		}
+
+		return result;
+	}
+
+	/**
+	 * Check if current page is a login page
+	 */
+	protected abstract isLoginPage(): Promise<boolean>;
+
+	/**
+	 * Extract quota data from the page
+	 */
+	protected abstract extractData(): Promise<T | null>;
+
+	/**
+	 * Cleanup browser resources
+	 */
+	async cleanup(): Promise<void> {
+		try {
+			if (this.page) {
+				await this.page.close();
+				this.page = null;
+			}
+			if (this.context) {
+				await this.context.close();
+				this.context = null;
+			}
+			if (this.browser) {
+				await this.browser.close();
+				this.browser = null;
+			}
+		} catch {
+			// Ignore cleanup errors
+		}
+	}
+
+	/**
+	 * Enable screenshot capture for debugging
+	 */
+	enableScreenshots(dir?: string): void {
+		this.screenshotsEnabled = true;
+		if (dir) {
+			this.screenshotsDir = dir;
+		}
+	}
+}