unbrowse 1.1.3 → 1.1.5

package/dist/cli.js

@@ -615,24 +615,28 @@ function buildEntityIndex(items) {
 function detectEntityIndex(data) {
   if (data == null || typeof data !== "object")
     return null;
-  const obj = data;
-  const candidates = [];
-  if (Array.isArray(obj.included))
-    candidates.push(obj.included);
-  if (obj.data && typeof obj.data === "object") {
-    const d = obj.data;
-    if (Array.isArray(d.included))
-      candidates.push(d.included);
-  }
-  for (const arr of candidates) {
+  let best = null;
+  const check = (arr) => {
     if (arr.length < 2)
-      continue;
-    const sample = arr.slice(0, 5);
+      return;
+    const sample = arr.slice(0, 10);
     const withUrn = sample.filter((i) => i != null && typeof i === "object" && typeof i.entityUrn === "string").length;
-    if (withUrn >= sample.length * 0.5)
-      return buildEntityIndex(arr);
+    if (withUrn >= sample.length * 0.5 && (!best || arr.length > best.length)) {
+      best = arr;
+    }
+  };
+  const obj = data;
+  for (const val of Object.values(obj)) {
+    if (Array.isArray(val)) {
+      check(val);
+    } else if (val != null && typeof val === "object" && !Array.isArray(val)) {
+      for (const nested of Object.values(val)) {
+        if (Array.isArray(nested))
+          check(nested);
+      }
+    }
   }
-  return null;
+  return best ? buildEntityIndex(best) : null;
 }
 function resolvePath(obj, path5, entityIndex) {
   if (!path5 || obj == null)
@@ -658,7 +662,7 @@ function resolvePath(obj, path5, entityIndex) {
     }
     const rec = cur;
     let val = rec[seg];
-    if (val === undefined && entityIndex) {
+    if (val == null && entityIndex) {
       const ref = rec[`*${seg}`];
       if (typeof ref === "string") {
         val = entityIndex.get(ref);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "1.1.3",
+  "version": "1.1.5",
   "description": "Reverse-engineer any website into reusable API skills. npm CLI + local engine.",
   "type": "module",
   "bin": {
@@ -14,6 +14,7 @@
   ],
   "scripts": {
     "prepack": "bun ../../scripts/sync-skill-md.ts --check && node scripts/prepare-pack.mjs",
+    "prepublishOnly": "node scripts/assert-release-flow.mjs",
     "start": "bun src/index.ts",
     "dev": "bun --watch src/index.ts"
   },

package/runtime-src/api/routes.ts

@@ -18,6 +18,125 @@ const BETA_API_URL = process.env.UNBROWSE_BACKEND_URL || "https://beta-api.unbro
 
 const TRACES_DIR = process.env.TRACES_DIR ?? join(process.cwd(), "traces");
 
+// ── /v1/stats cache ──────────────────────────────────────────────────
+let statsCache: { data: unknown; ts: number } | null = null;
+const STATS_CACHE_TTL = 5 * 60 * 1000; // 5 minutes
+
+async function fetchStats() {
+  if (statsCache && Date.now() - statsCache.ts < STATS_CACHE_TTL) {
+    return statsCache.data;
+  }
+
+  const npmPoint = (pkg: string, range: string) =>
+    fetch(`https://api.npmjs.org/downloads/point/${range}/${pkg}`)
+      .then(r => r.json() as Promise<{ downloads?: number }>);
+
+  const npmRange = (pkg: string) =>
+      .then(r => r.json() as Promise<{ downloads?: Array<{ day: string; downloads: number }> }>);
+      .then(r => r.json() as Promise<{ downloads?: number; downloads?: Array<{ day: string; downloads: number }> }>);
+
+  const externalCalls: Promise<unknown>[] = [
+    npmPoint("unbrowse", "last-month"),
+    npmPoint("@getfoundry/unbrowse-openclaw", "last-month"),
+    npmPoint("unbrowse", "1970-01-01:2099-12-31"),
+    npmPoint("@getfoundry/unbrowse-openclaw", "1970-01-01:2099-12-31"),
+    npmRange("unbrowse"),
+    npmRange("@getfoundry/unbrowse-openclaw"),
+    fetch("https://api.github.com/repos/anthropic-ai/unbrowse", {
+      headers: { "User-Agent": "unbrowse-stats" },
+    }).then(r => r.json() as Promise<Record<string, unknown>>),
+  ];
+
+  // Only call Unkey analytics if the key is available as an env var
+  const unkeyAnalyticsKey = process.env.UNKEY_ANALYTICS_KEY;
+  if (unkeyAnalyticsKey) {
+    externalCalls.push(
+      fetch("https://api.unkey.com/v2/analytics.getVerifications", {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${unkeyAnalyticsKey}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({ apiId: "api_2bUScBc8U6JNsXLrhfHwfqzXHJDi" }),
+      }).then(r => r.json() as Promise<unknown>),
+    );
+  }
+
+  const [
+    unbrowse30d, plugin30d,
+    unbrowseAll, pluginAll,
+    unbrowseDaily, pluginDaily,
+    github,
+    ...rest
+  ] = await Promise.allSettled(externalCalls);
+  const unkey = rest[0]; // may be undefined if no key
+
+  const val = <T>(r: PromiseSettledResult<T> | undefined): T | null =>
+    r?.status === "fulfilled" ? r.value : null;
+
+  // npm numbers
+  const u30 = val(unbrowse30d)?.downloads ?? null;
+  const p30 = val(plugin30d)?.downloads ?? null;
+  const uAll = val(unbrowseAll)?.downloads ?? null;
+  const pAll = val(pluginAll)?.downloads ?? null;
+
+  // daily breakdown — merge the two packages by day
+  const uDays = val(unbrowseDaily)?.downloads ?? [];
+  const pDays = val(pluginDaily)?.downloads ?? [];
+  const dayMap = new Map<string, { unbrowse: number; plugin: number }>();
+  for (const d of uDays) dayMap.set(d.day, { unbrowse: d.downloads, plugin: 0 });
+  for (const d of pDays) {
+    const entry = dayMap.get(d.day);
+    if (entry) entry.plugin = d.downloads;
+    else dayMap.set(d.day, { unbrowse: 0, plugin: d.downloads });
+  }
+  const daily = [...dayMap.entries()]
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([day, v]) => ({ day, unbrowse: v.unbrowse, plugin: v.plugin, total: v.unbrowse + v.plugin }));
+
+  // github
+  const gh = val(github);
+  const githubData = gh && typeof gh === "object"
+    ? {
+        stars: (gh as Record<string, number>).stargazers_count ?? null,
+        forks: (gh as Record<string, number>).forks_count ?? null,
+        open_issues: (gh as Record<string, number>).open_issues_count ?? null,
+        watchers: (gh as Record<string, number>).watchers_count ?? null,
+      }
+    : { stars: null, forks: null, open_issues: null, watchers: null };
+
+  // unkey
+  let agentsData: { total_api_calls_30d: number | null; note?: string } = {
+    total_api_calls_30d: null,
+    note: "unkey analytics unavailable",
+  };
+  const uk = val(unkey);
+  if (uk && Array.isArray(uk)) {
+    const total = (uk as Array<{ total?: number }>).reduce((s, v) => s + (v.total ?? 0), 0);
+    agentsData = { total_api_calls_30d: total };
+  } else if (uk && typeof uk === "object" && (uk as Record<string, unknown>).total != null) {
+    agentsData = { total_api_calls_30d: (uk as Record<string, number>).total };
+  }
+
+  const data = {
+    npm: {
+      unbrowse: { last_30d: u30, all_time: uAll },
+      openclaw_plugin: { last_30d: p30, all_time: pAll },
+      combined: {
+        last_30d: u30 != null && p30 != null ? u30 + p30 : (u30 ?? p30),
+        all_time: uAll != null && pAll != null ? uAll + pAll : (uAll ?? pAll),
+      },
+      daily,
+    },
+    github: githubData,
+    agents: agentsData,
+    fetched_at: new Date().toISOString(),
+  };
+
+  statsCache = { data, ts: Date.now() };
+  return data;
+}
+
 export async function registerRoutes(app: FastifyInstance) {
   const clientScopeFor = (req: { headers: Record<string, unknown>; id: string }) =>
     (typeof req.headers["x-unbrowse-client-id"] === "string" && req.headers["x-unbrowse-client-id"].trim())
@@ -26,7 +145,7 @@ export async function registerRoutes(app: FastifyInstance) {
 
   // Auth gate: block all routes except /health when no API key is configured
   app.addHook("onRequest", async (req, reply) => {
-    if (req.url === "/health") return;
+    if (req.url === "/health" || req.url === "/v1/stats") return;
 
     const key = getApiKey();
     if (!key) {
@@ -299,6 +418,16 @@ export async function registerRoutes(app: FastifyInstance) {
     }
   });
 
+  // GET /v1/stats — public, no auth required
+  app.get("/v1/stats", async (_req, reply) => {
+    try {
+      const data = await fetchStats();
+      return reply.send(data);
+    } catch (err) {
+      return reply.code(500).send({ error: (err as Error).message });
+    }
+  });
+
   // GET /health
   app.get("/health", async (_req, reply) => reply.send({ status: "ok", trace_version: TRACE_VERSION, code_hash: CODE_HASH, git_sha: GIT_SHA }));
 

package/runtime-src/cli.ts

@@ -118,28 +118,38 @@ function buildEntityIndex(items: unknown[]): Map<string, unknown> {
   return index;
 }
 
-/** Detect if an object contains a normalized entity array and build the index. */
+/** Detect if an object contains a normalized entity array and build the index.
+ *  Searches all top-level and one-level-nested arrays for entityUrn-keyed items,
+ *  picking the largest qualifying array. Works for any normalized API shape. */
 function detectEntityIndex(data: unknown): Map<string, unknown> | null {
   if (data == null || typeof data !== "object") return null;
-  const obj = data as Record<string, unknown>;
 
-  // Check common locations: { included: [...] }, { data: { included: [...] } }
-  const candidates: unknown[][] = [];
-  if (Array.isArray(obj.included)) candidates.push(obj.included);
-  if (obj.data && typeof obj.data === "object") {
-    const d = obj.data as Record<string, unknown>;
-    if (Array.isArray(d.included)) candidates.push(d.included);
-  }
+  let best: unknown[] | null = null;
 
-  for (const arr of candidates) {
-    if (arr.length < 2) continue;
-    const sample = arr.slice(0, 5);
+  const check = (arr: unknown[]) => {
+    if (arr.length < 2) return;
+    const sample = arr.slice(0, 10);
     const withUrn = sample.filter(
       (i) => i != null && typeof i === "object" && typeof (i as Record<string, unknown>).entityUrn === "string"
     ).length;
-    if (withUrn >= sample.length * 0.5) return buildEntityIndex(arr);
+    if (withUrn >= sample.length * 0.5 && (!best || arr.length > best.length)) {
+      best = arr;
+    }
+  };
+
+  const obj = data as Record<string, unknown>;
+  for (const val of Object.values(obj)) {
+    if (Array.isArray(val)) {
+      check(val);
+    } else if (val != null && typeof val === "object" && !Array.isArray(val)) {
+      // One level deep: { data: { included: [...] } }, { response: { entities: [...] } }, etc.
+      for (const nested of Object.values(val as Record<string, unknown>)) {
+        if (Array.isArray(nested)) check(nested);
+      }
+    }
   }
-  return null;
+
+  return best ? buildEntityIndex(best) : null;
 }
 
 /** Resolve a dot-path like "data.items[].name" against an object.
@@ -165,8 +175,10 @@ function resolvePath(obj: unknown, path: string, entityIndex?: Map<string, unkno
     const rec = cur as Record<string, unknown>;
     let val = rec[seg];
 
-    // URN reference resolution: if direct lookup fails, check for "*key" reference
-    if (val === undefined && entityIndex) {
+    // URN reference resolution: if direct lookup fails (or is null), check for "*key" reference.
+    // Normalized APIs (LinkedIn Voyager, Facebook Graph) set inline fields to null when
+    // the value is stored as a URN reference: e.g. socialDetail: null + *socialDetail: "urn:li:..."
+    if (val == null && entityIndex) {
       const ref = rec[`*${seg}`];
       if (typeof ref === "string") {
         val = entityIndex.get(ref);