pi-harness-runtime 1.1.80 → 1.1.82

package/harness/e2e/browser-scrapers/zai-browser-scraper.ts

@@ -0,0 +1,392 @@
+/**
+ * Z.ai Browser Scraper — Scrapes z.ai usage data
+ *
+ * Authentication approach:
+ * - Z.ai uses cookies from browser login (different from ChatGPT)
+ * - Cookies from ~/.config/google-chrome or browser export need to be tested
+ * - Fallback: Manual cookie export from browser DevTools
+ *
+ * Note: If cookies don't work for browser automation (like Codex),
+ * we may need to find alternative auth methods (OAuth, API keys, etc.)
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+
+import { BaseBrowserScraper, type BrowserScraperConfig } from "./base-browser-scraper.js";
+
+export interface ZaiQuotaData {
+	provider: "z-ai";
+	/** Plan type (e.g., "Pro", "Team", "Enterprise") */
+	planType?: string;
+	/** Current usage percentage (0-100) */
+	usagePercent: number;
+	/** Total quota (e.g., "5 GB") */
+	totalQuota?: string;
+	/** Used quota (e.g., "1.5 GB") */
+	usedQuota?: string;
+	/** Remaining quota */
+	remainingQuota?: string;
+	/** Reset date/time */
+	resetAt?: string;
+	/** Billing period */
+	period?: string;
+	/** Timestamp */
+	scrapedAt: string;
+}
+
+interface ZaiScraperConfig {
+	provider?: string;
+	url?: string;
+	headless?: boolean;
+	timeout?: number;
+	viewport?: { width: number; height: number };
+	slowMo?: number;
+	/** Path to cookies file (Netscape format) */
+	cookiesFile?: string;
+}
+
+// Known cookie file locations to try
+const COOKIE_LOCATIONS = [
+	// Chrome Linux
+	join(homedir(), ".config", "google-chrome", "Default", "Cookies"),
+	join(homedir(), ".config", "google-chrome", "Default", "Network", "Cookies"),
+	join(homedir(), ".config", "chromium", "Default", "Cookies"),
+	// Netscape cookie format exports
+	join(homedir(), ".pi-harness-runtime", "cookies", "zai-cookies.txt"),
+	join(homedir(), ".config", "zai-cookies.txt"),
+];
+
+/**
+ * Z.ai Browser Scraper
+ */
+export class ZaiBrowserScraper extends BaseBrowserScraper<ZaiQuotaData> {
+	private cookiesFile?: string;
+
+	constructor(config: Partial<ZaiScraperConfig> = {}) {
+		super({
+			provider: "z-ai",
+			url: config.url ?? "https://z.ai/manage-apikey/coding-plan/personal/usage",
+			headless: config.headless ?? true,
+			timeout: config.timeout ?? 30000,
+			viewport: config.viewport,
+			slowMo: config.slowMo,
+		});
+
+		this.cookiesFile = config.cookiesFile ?? this.findCookiesFile();
+	}
+
+	/**
+	 * Find available cookies file
+	 */
+	private findCookiesFile(): string | undefined {
+		for (const location of COOKIE_LOCATIONS) {
+			if (existsSync(location)) {
+				return location;
+			}
+		}
+		return undefined;
+	}
+
+	protected async createBrowser(): Promise<void> {
+		const { chromium } = await import("playwright");
+
+		const profileDir = join(homedir(), ".pi-harness-runtime", "browser-profiles", "zai");
+
+		// Launch with anti-detection args
+		this.browser = await chromium.launch({
+			headless: this.config.headless,
+			slowMo: this.config.slowMo,
+			args: [
+				`--user-data-dir=${profileDir}`,
+				"--disable-blink-features=AutomationControlled",
+				"--no-first-run",
+				"--no-default-browser-check",
+				"--disable-extensions",
+			],
+		});
+
+		this.context = await this.browser.newContext({
+			viewport: this.config.viewport,
+			// If we have a cookies file, import it
+			...(this.cookiesFile && existsSync(this.cookiesFile)
+				? { extraHTTPHeaders: await this.loadCookiesAsHeaders() }
+				: {}),
+		});
+
+		this.page = await this.context.newPage();
+	}
+
+	/**
+	 * Load cookies from file and convert to headers (fallback auth method)
+	 * Note: This won't work for all sites - some require full cookie jar
+	 */
+	private async loadCookiesAsHeaders(): Promise<Record<string, string>> {
+		if (!this.cookiesFile || !existsSync(this.cookiesFile)) {
+			return {};
+		}
+
+		try {
+			// Try to parse as Netscape format
+			const content = readFileSync(this.cookiesFile, "utf-8");
+			const lines = content.split("\n").filter((l) => !l.startsWith("#") && l.trim());
+
+			// Extract session-relevant cookies
+			const cookies: Record<string, string> = {};
+			for (const line of lines) {
+				const parts = line.split("\t");
+				if (parts.length >= 7) {
+					const domain = parts[0];
+					const name = parts[5];
+					const value = parts[6];
+
+					// Only include z.ai cookies
+					if (domain.includes("z.ai") || domain.includes("zcloud")) {
+						cookies[name] = value;
+					}
+				}
+			}
+
+			// Return as Cookie header
+			const cookieHeader = Object.entries(cookies)
+				.map(([k, v]) => `${k}=${v}`)
+				.join("; ");
+
+			return cookieHeader ? { Cookie: cookieHeader } : {};
+		} catch {
+			return {};
+		}
+	}
+
+	protected async isLoginPage(): Promise<boolean> {
+		if (!this.page) return false;
+
+		const url = this.page.url();
+
+		// Check for login/ auth URLs
+		if (/signin|login|auth|oauth/i.test(url)) {
+			return true;
+		}
+
+		// Check for common login indicators
+		const loginSelectors = [
+			'button:has-text("Sign in")',
+			'button:has-text("Login")',
+			'input[type="email"]',
+			'input[placeholder*="email" i]',
+			'form[action*="login" i]',
+		];
+
+		for (const selector of loginSelectors) {
+			try {
+				const element = await this.page.$(selector);
+				if (element != null) {
+					return true;
+				}
+			} catch {
+				// Selector might not exist
+			}
+		}
+
+		return false;
+	}
+
+	protected async extractData(): Promise<ZaiQuotaData | null> {
+		if (!this.page) return null;
+
+		try {
+			// Wait for page to fully load
+			await this.page.waitForTimeout(3000);
+
+			// Extract usage percentage
+			const usagePercent = await this.extractUsagePercent();
+			if (usagePercent === null) {
+				console.log("[z.ai] Could not extract usage percentage");
+				return null;
+			}
+
+			// Extract additional data
+			const planType = await this.extractPlanType();
+			const quotaData = await this.extractQuotaData();
+			const resetAt = await this.extractResetDate();
+
+			return {
+				provider: "z-ai",
+				usagePercent,
+				planType,
+				...quotaData,
+				resetAt,
+				scrapedAt: new Date().toISOString(),
+			};
+		} catch (error) {
+			console.error(`[z.ai] Extraction error: ${error}`);
+			return null;
+		}
+	}
+
+	private async extractUsagePercent(): Promise<number | null> {
+		if (!this.page) return null;
+
+		// Try various selectors for usage percentage
+		const selectors = [
+			// Percentage text
+			'text=/\\d+%/',
+			// Progress bar/circle with percentage
+			"[class*='progress'] [class*='text']",
+			"[class*='usage'] [class*='percent']",
+			"[class*='quota'] [class*='percent']",
+			// Heading near percentage
+			"h1:has-text('Usage') + div",
+			"h2:has-text('Usage') + div",
+			// Data attributes
+			"[data-usage]",
+			"[data-percent]",
+			"[data-quota]",
+		];
+
+		for (const selector of selectors) {
+			try {
+				const element = await this.page.$(selector);
+				if (element) {
+					const text = await element.textContent();
+					const match = text?.match(/(\d+(?:\.\d+)?)\s*%/);
+					if (match) {
+						const percent = parseFloat(match[1]);
+						if (!isNaN(percent) && percent >= 0 && percent <= 100) {
+							return percent;
+						}
+					}
+				}
+			} catch {
+				// Continue to next selector
+			}
+		}
+
+		return null;
+	}
+
+	private async extractPlanType(): Promise<string | undefined> {
+		if (!this.page) return undefined;
+
+		const selectors = [
+			"[class*='plan']",
+			"[class*='tier']",
+			'text=/Pro|Team|Enterprise|Starter/i',
+		];
+
+		for (const selector of selectors) {
+			try {
+				const element = await this.page.$(selector);
+				if (element) {
+					const text = await element.textContent();
+					if (text && /Pro|Team|Enterprise|Starter/i.test(text)) {
+						return text.trim();
+					}
+				}
+			} catch {
+				// Continue
+			}
+		}
+
+		return undefined;
+	}
+
+	private async extractQuotaData(): Promise<{
+		totalQuota?: string;
+		usedQuota?: string;
+		remainingQuota?: string;
+	}> {
+		if (!this.page) return {};
+
+		const result: { totalQuota?: string; usedQuota?: string; remainingQuota?: string } = {};
+
+		// Try to find quota amounts like "1.5 GB / 5 GB"
+		const selectors = [
+			'text=/\\d+.*(?:GB|MB|TB).*(?:\\/|of).*(?:GB|MB|TB)/i',
+			"[class*='quota'] [class*='amount']",
+			"[class*='usage'] [class*='amount']",
+		];
+
+		for (const selector of selectors) {
+			try {
+				const element = await this.page.$(selector);
+				if (element) {
+					const text = await element.textContent();
+					if (text) {
+						// Parse "1.5 GB / 5 GB" format
+						const match = text.match(/([\d,.]+)\s*(GB|MB|TB)\s*(?:\/|of)\s*([\d,.]+)\s*(GB|MB|TB)/i);
+						if (match) {
+							result.usedQuota = `${match[1]} ${match[2]}`;
+							result.totalQuota = `${match[3]} ${match[4]}`;
+							// Calculate remaining
+							const used = parseFloat(match[1].replace(/,/g, ""));
+							const total = parseFloat(match[3].replace(/,/g, ""));
+							if (!isNaN(used) && !isNaN(total)) {
+								const remaining = total - used;
+								result.remainingQuota = `${remaining.toFixed(2)} ${match[4]}`;
+							}
+							break;
+						}
+					}
+				}
+			} catch {
+				// Continue
+			}
+		}
+
+		return result;
+	}
+
+	private async extractResetDate(): Promise<string | undefined> {
+		if (!this.page) return undefined;
+
+		const selectors = [
+			'text=/resets?(?:\\s+(?:at|on|in))?\\s*/i',
+			"[class*='reset']",
+			"[class*='renews']",
+			"[class*='billing'] [class*='date']",
+		];
+
+		for (const selector of selectors) {
+			try {
+				const element = await this.page.$(selector);
+				if (element) {
+					const text = await element.textContent();
+					if (text && /\d{4}|resets?|renews|billing/i.test(text)) {
+						return text.trim();
+					}
+				}
+			} catch {
+				// Continue
+			}
+		}
+
+		return undefined;
+	}
+}
+
+/**
+ * Quick scrape function
+ */
+export async function scrapeZaiQuota(cookiesFile?: string): Promise<ZaiQuotaData | null> {
+	const scraper = new ZaiBrowserScraper({ cookiesFile });
+	try {
+		const result = await scraper.scrape();
+		return result.data ?? null;
+	} finally {
+		await scraper.cleanup();
+	}
+}
+
+/**
+ * Check if z.ai cookies are available
+ */
+export function checkZaiCookies(): { available: boolean; path?: string } {
+	for (const location of COOKIE_LOCATIONS) {
+		if (existsSync(location)) {
+			return { available: true, path: location };
+		}
+	}
+	return { available: false };
+}

package/harness/e2e/glm-quota-scraper.ts

@@ -44,6 +44,18 @@ export interface GLMQuotaData {
 	hourlyCalls: number[];
 	/** Hourly token usage */
 	hourlyTokens: number[];
+	/** 5h window usage percentage (0-100) */
+	h5UsedPct: number;
+	/** When the 5h window resets (human-readable) */
+	h5ResetsAt: string;
+	/** Epoch ms when the 5h window resets */
+	h5ResetsAtEpoch: number;
+	/** Weekly usage percentage (0-100) */
+	weeklyUsedPct: number;
+	/** When the weekly window resets (human-readable) */
+	weeklyResetsAt: string;
+	/** Epoch ms when the weekly window resets */
+	weeklyResetsAtEpoch: number;
 	/** Raw API response for diagnostics */
 	apiEndpoint?: string;
 	/** Timestamp of scrape */
@@ -141,17 +153,63 @@ export class GLMQuotaScraper {
 		quiet: boolean;
 	};
 
-	constructor(config: GLMScraperConfig = {}) {
-		// Resolve API key: direct value > file > null
-		const apiKey =
-			config.apiKey ?? loadApiKey(config.apiKeyFile ?? DEFAULT_API_KEY_FILE);
+		constructor(config: GLMScraperConfig = {}) {
+			// Resolve API key: direct value > file > null
+			const apiKey =
+				config.apiKey ?? loadApiKey(config.apiKeyFile ?? DEFAULT_API_KEY_FILE);
 
-		this.config = {
-			apiKeyFile: config.apiKeyFile ?? DEFAULT_API_KEY_FILE,
-			apiKey: apiKey ?? null,
-			quiet: config.quiet ?? false,
-		};
-	}
+			this.config = {
+				apiKeyFile: config.apiKeyFile ?? DEFAULT_API_KEY_FILE,
+				apiKey: apiKey ?? null,
+				quiet: config.quiet ?? false,
+			};
+		}
+
+		/**
+		 * Calculate 5h and weekly usage from hourly data
+		 */
+			private calculateHourlyUsage(
+			timestamps: string[],
+			tokens: number[],
+			_calls: number[],
+		): { h5Tokens: number; weeklyTokens: number } {
+			const now = new Date();
+			const fiveHoursAgo = new Date(now.getTime() - 5 * 60 * 60 * 1000);
+			const weekAgo = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000);
+
+			let h5Tokens = 0;
+			let weeklyTokens = 0;
+
+			for (let i = 0; i < timestamps.length; i++) {
+				const ts = timestamps[i];
+				if (!ts) continue;
+
+				// Parse timestamp: "2026-09-14 00:00"
+				const hourTime = new Date(ts.replace(" ", "T") + ":00");
+
+				if (hourTime >= fiveHoursAgo) {
+					h5Tokens += tokens[i] ?? 0;
+				}
+
+				if (hourTime >= weekAgo) {
+					weeklyTokens += tokens[i] ?? 0;
+				}
+			}
+
+			return { h5Tokens, weeklyTokens };
+		}
+
+		/**
+		 * Find the index of the last hour with usage
+		 */
+		private findLastUsageIndex(timestamps: string[], tokens: number[]): number {
+			for (let i = timestamps.length - 1; i >= 0; i--) {
+				if ((tokens[i] ?? 0) > 0) {
+					return i;
+				}
+			}
+			return -1;
+		}
 
 	/**
 	 * Set API key directly
@@ -211,8 +269,9 @@ export class GLMQuotaScraper {
 		const end = endDate ?? new Date();
 		const start = startDate ?? new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
 
-		const startStr = start.toISOString().replace("T", " ").replace("Z", "");
-		const endStr = end.toISOString().replace("T", " ").replace("Z", "");
+		// Format: yyyy-MM-dd HH:mm:ss (no milliseconds)
+		const startStr = start.toISOString().replace("T", " ").replace("Z", "").replace(/\.\d+$/, "");
+		const endStr = end.toISOString().replace("T", " ").replace("Z", "").replace(/\.\d+$/, "");
 
 		const url = `${API_BASE_URL}${USAGE_ENDPOINT}?startTime=${encodeURIComponent(startStr)}&endTime=${encodeURIComponent(endStr)}`;
 
@@ -272,6 +331,33 @@ export class GLMQuotaScraper {
 			const totalUsage = d.totalUsage ?? {};
 			const modelSummary = d.modelSummaryList?.[0];
 
+			// Calculate 5h and weekly usage from hourly data
+			const { h5Tokens, weeklyTokens } = this.calculateHourlyUsage(d.x_time ?? [], d.tokensUsage ?? [], d.modelCallCount ?? []);
+
+			// GLM Lite plan limits (Legacy Plan V2):
+			// Based on browser comparison: 20% weekly = 29.6M tokens
+			// So weekly quota ≈ 148M tokens, 5h quota ≈ 30M tokens
+			const H5_TOKEN_LIMIT = parseInt(process.env.GLM_5H_TOKEN_LIMIT ?? "30000000", 10); // 30M tokens per 5h
+			const WEEKLY_TOKEN_LIMIT = parseInt(process.env.GLM_WEEKLY_TOKEN_LIMIT ?? "148000000", 10); // 148M tokens per week
+
+			const h5UsedPct = Math.min(100, (h5Tokens / H5_TOKEN_LIMIT) * 100);
+			const weeklyUsedPct = Math.min(100, (weeklyTokens / WEEKLY_TOKEN_LIMIT) * 100);
+
+			// Calculate reset times
+			const now = new Date();
+			const h5ResetDurationMs = 5 * 60 * 60 * 1000; // 5 hours
+			const weeklyResetDurationMs = 7 * 24 * 60 * 60 * 1000; // 7 days
+
+			// Find last usage time to calculate when 5h window will reset
+			const lastUsageIdx = this.findLastUsageIndex(d.x_time ?? [], d.tokensUsage ?? []);
+			const lastUsageTime = lastUsageIdx >= 0 && d.x_time ? new Date(d.x_time[lastUsageIdx].replace(" ", "T") + ":00") : now;
+			const h5ResetAt = new Date(lastUsageTime.getTime() + h5ResetDurationMs);
+			const weeklyResetAt = new Date(now.getTime() + weeklyResetDurationMs);
+
+			// Calculate seconds until reset
+			const h5SecondsUntilReset = Math.max(0, Math.floor((h5ResetAt.getTime() - now.getTime()) / 1000));
+			const weeklySecondsUntilReset = Math.max(0, Math.floor((weeklyResetAt.getTime() - now.getTime()) / 1000));
+
 			const result: GLMQuotaData = {
 				provider: "glm",
 				totalCalls: totalUsage.totalModelCallCount ?? 0,
@@ -280,15 +366,20 @@ export class GLMQuotaScraper {
 				hourlyTimestamps: d.x_time ?? [],
 				hourlyCalls: d.modelCallCount ?? [],
 				hourlyTokens: d.tokensUsage ?? [],
+				h5UsedPct,
+				h5ResetsAt: formatRemainsSeconds(h5SecondsUntilReset),
+				h5ResetsAtEpoch: h5ResetAt.getTime(),
+				weeklyUsedPct,
+				weeklyResetsAt: formatRemainsSeconds(weeklySecondsUntilReset),
+				weeklyResetsAtEpoch: weeklyResetAt.getTime(),
 				apiEndpoint: url,
 				scrapedAt: new Date().toISOString(),
 			};
 
 			if (!this.config.quiet) {
 				console.log(
-					`[DEBUG GLMQuotaScraper] Total calls: ${result.totalCalls}, ` +
-						`Total tokens: ${result.totalTokens.toLocaleString()}, ` +
-						`Model: ${result.modelName}`,
+					`[DEBUG GLMQuotaScraper] 5h: ${h5UsedPct.toFixed(1)}% (${(h5Tokens/1e6).toFixed(1)}M tokens), ` +
+						`Week: ${weeklyUsedPct.toFixed(1)}% (${(weeklyTokens/1e6).toFixed(1)}M tokens)`,
 				);
 			}
 

package/harness/e2e/index.ts

@@ -42,3 +42,32 @@ export type {
 	PlaywrightRunnerConfig,
 	QuotaPageData,
 } from "./playwright-runner.js";
+
+// Unified quota management
+export {
+	UnifiedQuotaManager,
+	createUnifiedQuotaManagerFromEnv,
+	type QuotaData,
+	type Provider,
+} from "./unified-quota-manager.js";
+
+// Individual scrapers for direct access
+export { MiniMaxQuotaManager, type MiniMaxQuotaData } from "./minimax-quota-scraper.js";
+export { GLMQuotaManager, GLMQuotaScraper, type GLMQuotaData, type GLMSubscriptionData, parseGLMErrorResetTime, parseMinimaxOverloadResetTime } from "./glm-quota-scraper.js";
+export { ChatGPTQuotaManager, ChatGPTQuotaScraper, type ChatGPTQuotaData } from "./chatgpt-quota-scraper.js";
+export { OpenAIQuotaManager, OpenAIQuotaScraper, type OpenAIQuotaData } from "./openai-quota-scraper.js";
+
+// Browser-based scrapers
+export {
+	ChromeProfileManager,
+	getChromeProfileManager,
+	BaseBrowserScraper,
+	CodexBrowserScraper,
+	ZaiBrowserScraper,
+	scrapeCodexQuota,
+	scrapeZaiQuota,
+	checkZaiCookies,
+	type BrowserScraperConfig,
+	type CodexQuotaData,
+	type ZaiQuotaData,
+} from "./browser-scrapers/index.js";