webpeel 0.3.5 → 0.3.6

package/dist/cache.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Local response cache for WebPeel CLI
+ *
+ * Caches fetch results in ~/.webpeel/cache/ with TTL support.
+ * Cache key is a hash of URL + relevant options.
+ */
+/**
+ * Parse a TTL string like "5m", "1h", "30s", "1d" into milliseconds
+ */
+export declare function parseTTL(ttl: string): number;
+/**
+ * Get a cached result if it exists and hasn't expired
+ */
+export declare function getCache(url: string, options?: Record<string, any>): any | null;
+/**
+ * Store a result in the cache
+ */
+export declare function setCache(url: string, result: any, ttlMs: number, options?: Record<string, any>): void;
+/**
+ * Clear expired cache entries (or all entries)
+ */
+export declare function clearCache(all?: boolean): number;
+/**
+ * Get cache stats
+ */
+export declare function cacheStats(): {
+    entries: number;
+    sizeBytes: number;
+    dir: string;
+};
+//# sourceMappingURL=cache.d.ts.map

package/dist/cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAiBH;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAc5C;AAiBD;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,GAAG,GAAG,IAAI,CAoB/E;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAerG;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,GAAG,UAAQ,GAAG,MAAM,CAyB9C;AAED;;GAEG;AACH,wBAAgB,UAAU,IAAI;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAchF"}

package/dist/cache.js

@@ -0,0 +1,132 @@
+/**
+ * Local response cache for WebPeel CLI
+ *
+ * Caches fetch results in ~/.webpeel/cache/ with TTL support.
+ * Cache key is a hash of URL + relevant options.
+ */
+import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, unlinkSync, statSync } from 'fs';
+import { homedir } from 'os';
+import { join } from 'path';
+import { createHash } from 'crypto';
+const CACHE_DIR = join(homedir(), '.webpeel', 'cache');
+/**
+ * Parse a TTL string like "5m", "1h", "30s", "1d" into milliseconds
+ */
+export function parseTTL(ttl) {
+    const match = ttl.match(/^(\d+)(s|m|h|d)$/);
+    if (!match) {
+        throw new Error(`Invalid TTL format: "${ttl}". Use: 30s, 5m, 1h, 1d`);
+    }
+    const value = parseInt(match[1]);
+    const unit = match[2];
+    switch (unit) {
+        case 's': return value * 1000;
+        case 'm': return value * 60 * 1000;
+        case 'h': return value * 60 * 60 * 1000;
+        case 'd': return value * 24 * 60 * 60 * 1000;
+        default: throw new Error(`Unknown TTL unit: ${unit}`);
+    }
+}
+/**
+ * Generate a cache key from URL + options
+ */
+function cacheKey(url, options) {
+    const relevant = {
+        url,
+        render: options?.render || false,
+        stealth: options?.stealth || false,
+        selector: options?.selector || null,
+        format: options?.format || 'markdown',
+    };
+    const hash = createHash('sha256').update(JSON.stringify(relevant)).digest('hex').slice(0, 16);
+    return hash;
+}
+/**
+ * Get a cached result if it exists and hasn't expired
+ */
+export function getCache(url, options) {
+    const key = cacheKey(url, options);
+    const filePath = join(CACHE_DIR, `${key}.json`);
+    if (!existsSync(filePath))
+        return null;
+    try {
+        const entry = JSON.parse(readFileSync(filePath, 'utf-8'));
+        const age = Date.now() - entry.cachedAt;
+        if (age > entry.ttlMs) {
+            // Expired — delete and return null
+            try {
+                unlinkSync(filePath);
+            }
+            catch { }
+            return null;
+        }
+        return entry.result;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Store a result in the cache
+ */
+export function setCache(url, result, ttlMs, options) {
+    if (!existsSync(CACHE_DIR)) {
+        mkdirSync(CACHE_DIR, { recursive: true });
+    }
+    const key = cacheKey(url, options);
+    const entry = {
+        url,
+        result,
+        cachedAt: Date.now(),
+        ttlMs,
+        options: options ? JSON.stringify(options) : undefined,
+    };
+    writeFileSync(join(CACHE_DIR, `${key}.json`), JSON.stringify(entry));
+}
+/**
+ * Clear expired cache entries (or all entries)
+ */
+export function clearCache(all = false) {
+    if (!existsSync(CACHE_DIR))
+        return 0;
+    const files = readdirSync(CACHE_DIR).filter(f => f.endsWith('.json'));
+    let cleared = 0;
+    for (const file of files) {
+        const filePath = join(CACHE_DIR, file);
+        try {
+            if (all) {
+                unlinkSync(filePath);
+                cleared++;
+            }
+            else {
+                const entry = JSON.parse(readFileSync(filePath, 'utf-8'));
+                if (Date.now() - entry.cachedAt > entry.ttlMs) {
+                    unlinkSync(filePath);
+                    cleared++;
+                }
+            }
+        }
+        catch {
+            // Skip corrupt files
+        }
+    }
+    return cleared;
+}
+/**
+ * Get cache stats
+ */
+export function cacheStats() {
+    if (!existsSync(CACHE_DIR))
+        return { entries: 0, sizeBytes: 0, dir: CACHE_DIR };
+    const files = readdirSync(CACHE_DIR).filter(f => f.endsWith('.json'));
+    let sizeBytes = 0;
+    for (const file of files) {
+        try {
+            const stat = statSync(join(CACHE_DIR, file));
+            sizeBytes += stat.size;
+        }
+        catch { }
+    }
+    return { entries: files.length, sizeBytes, dir: CACHE_DIR };
+}
+//# sourceMappingURL=cache.js.map

package/dist/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AAC3G,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;AAUvD;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IAC5C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,yBAAyB,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACtB,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,IAAI,CAAC;QAC9B,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC;QACnC,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QACxC,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAC7C,OAAO,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,QAAQ,CAAC,GAAW,EAAE,OAA6B;IAC1D,MAAM,QAAQ,GAAG;QACf,GAAG;QACH,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,KAAK;QAChC,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,KAAK;QAClC,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,IAAI;QACnC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,UAAU;KACtC,CAAC;IACF,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9F,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW,EAAE,OAA6B;IACjE,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;IAEhD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,KAAK,GAAe,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC;QAExC,IAAI,GAAG,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YACtB,mCAAmC;YACnC,IAAI,CAAC;gBAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;YACtC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW,EAAE,MAAW,EAAE,KAAa,EAAE,OAA6B;IAC7F,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACnC,MAAM,KAAK,GAAe;QACxB,GAAG;QACH,MAAM;QACN,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;QACpB,KAAK;QACL,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;KACvD,CAAC;IAEF,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AACvE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,GAAG,GAAG,KAAK;IACpC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,CAAC,CAAC;IAErC,MAAM,KAAK,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACvC,IAAI,CAAC;YACH,IAAI,GAAG,EAAE,CAAC;gBACR,UAAU,CAAC,QAAQ,CAAC,CAAC;gBACrB,OAAO,EAAE,CAAC;YACZ,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,GAAe,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;gBACtE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;oBAC9C,UAAU,CAAC,QAAQ,CAAC,CAAC;oBACrB,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qBAAqB;QACvB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU;IACxB,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;IAEhF,MAAM,KAAK,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7C,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;IACZ,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;AAC9D,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "MIT",
@@ -37,7 +37,11 @@
     "dist/cli-auth.js",
     "dist/cli-auth.js.map",
     "dist/cli-auth.d.ts",
-    "dist/cli-auth.d.ts.map"
+    "dist/cli-auth.d.ts.map",
+    "dist/cache.js",
+    "dist/cache.js.map",
+    "dist/cache.d.ts",
+    "dist/cache.d.ts.map"
   ],
   "scripts": {
     "build": "tsc",