productfoo 0.0.0-stage → 1.0.1

package/dist-public/src/social/reddit.js

@@ -0,0 +1,125 @@
+// Reddit adapter — the one non-trivial port. Bare reddit.com/*.json rate-limits
+// hard (429/403), which is why last30days ships several strategies.
+//
+// HONEST LIMITATION (found by the dependency watcher, 2026-07-12): Arctic Shift
+// CANNOT serve a global keyword search — its API answers
+//   "'query' query parameter requires one of: author, subreddit"
+// so an arctic call with only a query returns HTTP 400, always. It was in this
+// chain and had never once worked; the fallback masked it. It is removed rather
+// than left as a strategy that silently fails. (It does work when scoped to a
+// subreddit — see OQ: use it for subreddit-scoped search, which would restore an
+// off-reddit.com fallback.)
+//
+// So the chain is Reddit Atom (.rss) → Reddit public JSON. Both live on
+// reddit.com and therefore throttle TOGETHER: there is currently no off-domain
+// fallback for global keyword search. Resilience here is weaker than a
+// multi-host chain, and the spec says so rather than implying otherwise. A
+// strategy that throws or 429s drops to the next; the source is only `excluded`
+// if all fail (R-001 done_when).
+//
+// Reddit asks clients to send a descriptive User-Agent; we do (it is correct
+// practice, though it does not by itself defeat rate limiting).
+import { resolveOptions, settleStatus, withinWindow } from "./types.js";
+import { parseAtomEntries } from "./xml.js";
+function postToItem(p) {
+    if (!p.id || !p.title)
+        return null;
+    const createdSec = typeof p.created_utc === "number" ? p.created_utc : 0;
+    return {
+        source: "reddit",
+        id: `reddit:${p.id}`,
+        title: p.title,
+        url: p.permalink ? `https://www.reddit.com${p.permalink}` : p.url || "",
+        author: p.author ?? null,
+        createdAt: new Date(createdSec * 1000).toISOString(),
+        score: typeof p.score === "number" ? p.score : null,
+        snippet: p.selftext ? p.selftext.slice(0, 280) : null,
+    };
+}
+function filterAndCap(items, o) {
+    return items
+        .filter((it) => it !== null && Boolean(it.url))
+        .filter((it) => withinWindow(it.createdAt, o.now, o.days))
+        .slice(0, o.limit);
+}
+// Reddit asks for a unique, descriptive User-Agent on API-ish requests.
+const REDDIT_UA = "foo-research/0.1 (+https://github.com/rwo572/productfoo)";
+async function redditRss(query, o) {
+    const url = `https://www.reddit.com/search.rss?q=${encodeURIComponent(query)}&sort=relevance&t=month`;
+    const res = await o.fetchImpl(url, {
+        headers: { Accept: "application/atom+xml", "User-Agent": REDDIT_UA },
+        signal: o.signal,
+    });
+    if (!res.ok)
+        throw new Error(`HTTP ${res.status}`);
+    const xml = await res.text();
+    const items = parseAtomEntries(xml).map((e) => {
+        if (!e.title || !e.link)
+            return null;
+        // Reddit Atom ids look like "t3_abcdef"; fall back to the link.
+        const id = e.id ? e.id.replace(/^.*?(t3_[a-z0-9]+).*$/i, "$1") : e.link;
+        return {
+            source: "reddit",
+            id: `reddit:${id}`,
+            title: e.title,
+            url: e.link,
+            author: e.author,
+            createdAt: e.published ?? new Date(0).toISOString(),
+            score: null, // the Atom feed carries no engagement score
+            snippet: e.summary ? e.summary.slice(0, 280) : null,
+        };
+    });
+    return filterAndCap(items, o);
+}
+async function redditPublicJson(query, o) {
+    const url = `https://www.reddit.com/search.json?q=${encodeURIComponent(query)}` +
+        `&sort=relevance&t=month&limit=${o.limit}`;
+    const res = await o.fetchImpl(url, {
+        headers: { Accept: "application/json", "User-Agent": REDDIT_UA },
+        signal: o.signal,
+    });
+    if (!res.ok)
+        throw new Error(`HTTP ${res.status}`);
+    const body = (await res.json());
+    const children = body.data?.children ?? [];
+    return filterAndCap(children.map((c) => (c.data ? postToItem(c.data) : null)), o);
+}
+// arctic is deliberately absent: its API cannot do a global keyword search, so
+// it returned HTTP 400 on every call (see the header note). Both remaining
+// strategies are on reddit.com and throttle together — a known, recorded
+// resilience gap, not a silent one.
+const STRATEGIES = [
+    { name: "rss", run: redditRss },
+    { name: "public", run: redditPublicJson },
+];
+export async function fetchReddit(query, opts = {}) {
+    const o = resolveOptions(opts);
+    const tried = [];
+    for (const strategy of STRATEGIES) {
+        try {
+            const items = await strategy.run(query, o);
+            if (items.length > 0) {
+                const prior = tried.length ? ` (after ${tried.join(", ")})` : "";
+                return {
+                    source: "reddit",
+                    status: settleStatus("available", items.length),
+                    reason: `Reddit via ${strategy.name}: ${items.length} post(s) within ${o.days}d${prior}.`,
+                    items,
+                };
+            }
+            tried.push(`${strategy.name}: empty`);
+        }
+        catch (err) {
+            tried.push(`${strategy.name}: ${err.message}`);
+        }
+    }
+    // Every strategy either failed or returned nothing. If any strategy actually
+    // ran without error we degraded (no signal); if all threw we exclude.
+    const allErrored = tried.every((t) => !t.endsWith("empty"));
+    return {
+        source: "reddit",
+        status: allErrored ? "excluded" : "degraded",
+        reason: `Reddit returned no usable posts — ${tried.join("; ")}.`,
+        items: [],
+    };
+}

package/dist-public/src/social/types.d.ts

@@ -0,0 +1,73 @@
+export type SocialSource = "reddit" | "hackernews" | "arxiv" | "github" | "youtube";
+export interface YtVideo {
+    id: string;
+    title: string;
+    url: string;
+    uploader: string | null;
+    uploadDate: string;
+    viewCount: number | null;
+}
+export interface YtDlp {
+    search(query: string, n: number): Promise<YtVideo[]>;
+    transcript?(videoId: string): Promise<string | null>;
+}
+export type AdapterStatus = "available" | "empty" | "degraded" | "excluded";
+export interface SocialItem {
+    source: SocialSource;
+    id: string;
+    title: string;
+    url: string;
+    author: string | null;
+    createdAt: string;
+    score: number | null;
+    snippet: string | null;
+}
+export interface AdapterResult {
+    source: SocialSource;
+    status: AdapterStatus;
+    reason: string;
+    items: SocialItem[];
+}
+export interface HttpResponse {
+    ok: boolean;
+    status: number;
+    json(): Promise<unknown>;
+    text(): Promise<string>;
+}
+export type FetchLike = (url: string, init?: {
+    headers?: Record<string, string>;
+    signal?: AbortSignal;
+}) => Promise<HttpResponse>;
+export interface AdapterOptions {
+    days?: number;
+    limit?: number;
+    now?: Date;
+    fetchImpl?: FetchLike;
+    githubToken?: string;
+    ytdlp?: YtDlp;
+    timeoutMs?: number;
+}
+export declare const DEFAULT_DAYS = 30;
+export declare const DEFAULT_LIMIT = 25;
+export declare const DEFAULT_FETCH_TIMEOUT_MS = 30000;
+export interface ResolvedOptions {
+    days: number;
+    limit: number;
+    now: Date;
+    fetchImpl: FetchLike;
+    githubToken?: string;
+    signal: AbortSignal;
+}
+export declare function resolveOptions(opts?: AdapterOptions): ResolvedOptions;
+/**
+ * The honest-zero rule, in one place: a source that returned no items did not
+ * contribute evidence, so it cannot report `available` — whatever it thinks.
+ *
+ * Applied at BOTH layers on purpose. Each adapter settles its own status through
+ * this, so an adapter is honest when called directly; and `gatherSocial` re-applies
+ * it, so a future adapter that forgets cannot reintroduce the lie. `degraded` and
+ * `excluded` already carry their own meaning and pass through untouched.
+ */
+export declare function settleStatus(status: AdapterStatus, itemCount: number): AdapterStatus;
+export declare function windowStartMs(now: Date, days: number): number;
+export declare function withinWindow(iso: string, now: Date, days: number): boolean;

package/dist-public/src/social/types.js

@@ -0,0 +1,61 @@
+// foo social adapters — the keyless free-tier port of last30days into productfoo's
+// `social` SourceClass (specs/research-social-last30d-v0, R-001). This module is
+// the *fetch* layer: it is deliberately separate from research.ts (the
+// deterministic, fetch-free spine) so the spine keeps its "never fetches"
+// invariant. The agent invokes these adapters at gather time; foo synthesizes.
+//
+// Every adapter is offline-testable: network access is a single injected
+// `FetchLike` (default = global fetch), so unit tests pass fixtures and no live
+// I/O runs in the test suite — the same posture as src/sourceIngress.ts.
+//
+// Every HTTP fetch is also time-bounded: `resolveOptions` hands each adapter one
+// abort signal (default ceiling `DEFAULT_FETCH_TIMEOUT_MS`, overridable with
+// `timeoutMs` for tests and slow-host tuning) that the adapter passes on every
+// call, so one hung endpoint can no longer stall the whole gather.
+export const DEFAULT_DAYS = 30;
+export const DEFAULT_LIMIT = 25;
+// The HTTP fetch ceiling. Before this existed a hung endpoint stalled the whole
+// gather for minutes, bounded only by undici's defaults (#774).
+export const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
+const globalFetch = (url, init) => 
+// eslint-disable-next-line no-undef
+fetch(url, {
+    ...init,
+    // Default ceiling for direct callers; the adapters pass the resolved signal
+    // explicitly so the `timeoutMs` override reaches every call.
+    signal: init?.signal ?? AbortSignal.timeout(DEFAULT_FETCH_TIMEOUT_MS),
+});
+export function resolveOptions(opts = {}) {
+    return {
+        days: opts.days ?? DEFAULT_DAYS,
+        limit: opts.limit ?? DEFAULT_LIMIT,
+        now: opts.now ?? new Date(),
+        fetchImpl: opts.fetchImpl ?? globalFetch,
+        githubToken: opts.githubToken,
+        signal: AbortSignal.timeout(opts.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS),
+    };
+}
+/**
+ * The honest-zero rule, in one place: a source that returned no items did not
+ * contribute evidence, so it cannot report `available` — whatever it thinks.
+ *
+ * Applied at BOTH layers on purpose. Each adapter settles its own status through
+ * this, so an adapter is honest when called directly; and `gatherSocial` re-applies
+ * it, so a future adapter that forgets cannot reintroduce the lie. `degraded` and
+ * `excluded` already carry their own meaning and pass through untouched.
+ */
+export function settleStatus(status, itemCount) {
+    return status === "available" && itemCount === 0 ? "empty" : status;
+}
+// The oldest timestamp (ms) an item may carry to fall inside the recency window.
+export function windowStartMs(now, days) {
+    return now.getTime() - days * 24 * 60 * 60 * 1000;
+}
+// True when `iso` parses and is at or after the window start. Unparseable dates
+// are treated as out-of-window (defensive: never let a garbage date leak in).
+export function withinWindow(iso, now, days) {
+    const t = Date.parse(iso);
+    if (Number.isNaN(t))
+        return false;
+    return t >= windowStartMs(now, days) && t <= now.getTime() + 60_000;
+}

package/dist-public/src/social/xml.d.ts

@@ -0,0 +1,10 @@
+export interface AtomEntry {
+    id: string | null;
+    title: string | null;
+    link: string | null;
+    published: string | null;
+    author: string | null;
+    summary: string | null;
+}
+export declare function unescapeXml(value: string | null): string | null;
+export declare function parseAtomEntries(xml: string): AtomEntry[];

package/dist-public/src/social/xml.js

@@ -0,0 +1,49 @@
+// A minimal, dependency-free Atom reader — just enough to parse the two Atom
+// feeds this module consumes (Reddit search.rss and the arXiv query API). It is
+// deliberately lenient (regex over well-formed, regular feeds) rather than a
+// general XML parser; fixtures in the test suite pin the shapes it must handle.
+function firstTag(block, name) {
+    const m = block.match(new RegExp(`<${name}\\b[^>]*>([\\s\\S]*?)</${name}>`, "i"));
+    return m ? m[1].trim() : null;
+}
+function linkHref(block) {
+    // Prefer an alternate/self <link href="…">; fall back to a text <link>…</link>.
+    const href = block.match(/<link\b[^>]*\bhref="([^"]+)"/i);
+    if (href)
+        return href[1];
+    return firstTag(block, "link");
+}
+function authorName(block) {
+    const author = block.match(/<author\b[^>]*>([\s\S]*?)<\/author>/i);
+    if (!author)
+        return null;
+    return firstTag(author[1], "name");
+}
+export function unescapeXml(value) {
+    if (value == null)
+        return null;
+    return value
+        .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/&apos;/g, "'")
+        .replace(/&amp;/g, "&")
+        .replace(/<[^>]+>/g, "") // strip any inline markup left in titles/summaries
+        .trim();
+}
+export function parseAtomEntries(xml) {
+    const blocks = xml.split(/<entry\b/i).slice(1);
+    return blocks.map((raw) => {
+        const block = raw.split(/<\/entry>/i)[0];
+        return {
+            id: firstTag(block, "id"),
+            title: unescapeXml(firstTag(block, "title")),
+            link: linkHref(block),
+            published: firstTag(block, "published") ?? firstTag(block, "updated"),
+            author: authorName(block),
+            summary: unescapeXml(firstTag(block, "summary")),
+        };
+    });
+}

package/dist-public/src/social/youtube.d.ts

@@ -0,0 +1,7 @@
+import type { AdapterOptions, AdapterResult, YtDlp } from "./types.js";
+export declare class MissingBinaryError extends Error {
+    constructor(message?: string);
+}
+export declare const defaultYtDlp: YtDlp;
+export declare function parseVtt(vtt: string): string;
+export declare function fetchYouTube(query: string, opts?: AdapterOptions): Promise<AdapterResult>;

package/dist-public/src/social/youtube.js

@@ -0,0 +1,217 @@
+// YouTube adapter — keyless, but via the yt-dlp binary rather than an HTTP API
+// (upstream last30days does the same: "no API keys needed — just have yt-dlp
+// installed"). yt-dlp is a *dependency*, not a secret, so YouTube stays a
+// free-tier source (D-004). When yt-dlp is absent the source degrades honestly
+// and the run continues — it never crashes (R-003 done_when).
+//
+// V0 covers recency-scoped search with view-count engagement. Transcript
+// extraction (a bounded per-video enrichment) is a follow-on — see OQ-5 — kept
+// out of V0 because it needs on-disk subtitle handling that would ship untested.
+//
+// The yt-dlp boundary is injected (types.ts YtDlp) so the adapter's recency and
+// status logic is unit-tested with a fake and no binary runs in the suite.
+import { execFile } from "node:child_process";
+import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+import { resolveOptions, settleStatus, withinWindow } from "./types.js";
+const execFileAsync = promisify(execFile);
+// How many top videos to enrich with a transcript per run — bounded so a run
+// never fans out an unbounded number of subtitle fetches.
+const TRANSCRIPT_LIMIT = 3;
+const TRANSCRIPT_MAX_CHARS = 500;
+// Hard ceiling on each yt-dlp process. A shell-out has no fetch signal to
+// respect, so without this a hung yt-dlp stalled the whole gather for minutes
+// (#774). Generous — a ytsearch fans out several upstream requests — but a
+// bound: the child is killed and the adapter reports `excluded` like any other
+// failure.
+const YTDLP_TIMEOUT_MS = 120_000;
+// Thrown by a YtDlp implementation when the binary is not installed, so the
+// adapter can distinguish "not set up" (degrade + how-to) from "ran and failed".
+export class MissingBinaryError extends Error {
+    constructor(message = "yt-dlp is not installed") {
+        super(message);
+        this.name = "MissingBinaryError";
+    }
+}
+// Default implementation: shells out to yt-dlp and streams one JSON object per
+// result. `ytsearchN:` is yt-dlp's search selector; --dump-json emits full
+// metadata (upload_date, view_count) per video.
+export const defaultYtDlp = {
+    async search(query, n) {
+        try {
+            const { stdout } = await execFileAsync("yt-dlp", ["--dump-json", "--no-warnings", `ytsearch${n}:${query}`], { maxBuffer: 32 * 1024 * 1024, timeout: YTDLP_TIMEOUT_MS });
+            return stdout
+                .split("\n")
+                .map((line) => line.trim())
+                .filter(Boolean)
+                .map((line) => {
+                try {
+                    const v = JSON.parse(line);
+                    if (!v.id || !v.title)
+                        return null;
+                    return {
+                        id: v.id,
+                        title: v.title,
+                        url: v.webpage_url || v.url || `https://www.youtube.com/watch?v=${v.id}`,
+                        uploader: v.uploader ?? v.channel ?? null,
+                        uploadDate: v.upload_date ?? "",
+                        viewCount: typeof v.view_count === "number" ? v.view_count : null,
+                    };
+                }
+                catch {
+                    return null;
+                }
+            })
+                .filter((v) => v !== null);
+        }
+        catch (err) {
+            const code = err.code;
+            if (code === "ENOENT")
+                throw new MissingBinaryError();
+            if (err.killed) {
+                throw new Error(`yt-dlp did not finish within ${YTDLP_TIMEOUT_MS / 1000}s (process killed)`);
+            }
+            throw err;
+        }
+    },
+    // Best-effort auto-subtitle fetch. yt-dlp writes subtitles to disk, so we use
+    // an isolated temp dir, read the VTT back, and always clean up. Any failure
+    // (no captions, network, missing binary) yields null — a transcript is
+    // enrichment, never a reason to fail the item.
+    async transcript(videoId) {
+        let dir;
+        try {
+            dir = await mkdtemp(path.join(tmpdir(), "foo-yt-"));
+            const url = `https://www.youtube.com/watch?v=${videoId}`;
+            await execFileAsync("yt-dlp", [
+                "--skip-download",
+                "--write-auto-subs",
+                "--sub-lang",
+                "en",
+                "--sub-format",
+                "vtt",
+                "--no-warnings",
+                "-o",
+                path.join(dir, "%(id)s.%(ext)s"),
+                url,
+            ], { maxBuffer: 32 * 1024 * 1024, timeout: YTDLP_TIMEOUT_MS });
+            const files = await readdir(dir);
+            const vttFile = files.find((f) => f.endsWith(".vtt"));
+            if (!vttFile)
+                return null;
+            const text = parseVtt(await readFile(path.join(dir, vttFile), "utf8"));
+            return text || null;
+        }
+        catch {
+            return null;
+        }
+        finally {
+            if (dir)
+                await rm(dir, { recursive: true, force: true }).catch(() => { });
+        }
+    },
+};
+// Reduce a WebVTT caption file to plain running text: drop the header, cue
+// indices, timestamp lines, and inline tags, and collapse the duplicate lines
+// that auto-generated rolling captions produce.
+export function parseVtt(vtt) {
+    const out = [];
+    for (const raw of vtt.split(/\r?\n/)) {
+        const line = raw.trim();
+        if (!line || line === "WEBVTT")
+            continue;
+        if (line.startsWith("Kind:") || line.startsWith("Language:"))
+            continue;
+        if (line.includes("-->"))
+            continue; // timestamp cue
+        if (/^\d+$/.test(line))
+            continue; // cue index
+        const text = line
+            .replace(/<[^>]+>/g, "")
+            .replace(/&nbsp;/g, " ")
+            .trim();
+        if (!text || out[out.length - 1] === text)
+            continue; // dedupe rolling captions
+        out.push(text);
+    }
+    return out.join(" ");
+}
+// yt-dlp's upload_date is "YYYYMMDD"; normalize to an ISO instant so the shared
+// recency filter can parse it. Pass through anything already ISO-ish.
+function normalizeDate(raw) {
+    const m = /^(\d{4})(\d{2})(\d{2})$/.exec(raw);
+    if (m)
+        return `${m[1]}-${m[2]}-${m[3]}T00:00:00Z`;
+    return raw;
+}
+function toItem(v) {
+    if (!v.uploadDate)
+        return null; // no date → cannot place in the window
+    return {
+        source: "youtube",
+        id: `youtube:${v.id}`,
+        title: v.title,
+        url: v.url,
+        author: v.uploader,
+        createdAt: normalizeDate(v.uploadDate),
+        score: v.viewCount,
+        snippet: null,
+    };
+}
+export async function fetchYouTube(query, opts = {}) {
+    const { days, limit, now } = resolveOptions(opts);
+    const ytdlp = opts.ytdlp ?? defaultYtDlp;
+    try {
+        const videos = await ytdlp.search(query, limit);
+        const items = videos
+            .map(toItem)
+            .filter((it) => it !== null)
+            .filter((it) => withinWindow(it.createdAt, now, days))
+            .slice(0, limit);
+        if (items.length === 0) {
+            return {
+                source: "youtube",
+                status: "degraded",
+                reason: `yt-dlp ran but found no videos within ${days}d.`,
+                items,
+            };
+        }
+        // Best-effort transcript enrichment for the top few videos (bounded).
+        let transcribed = 0;
+        if (ytdlp.transcript) {
+            const fetchTranscript = ytdlp.transcript.bind(ytdlp);
+            await Promise.all(items.slice(0, TRANSCRIPT_LIMIT).map(async (it) => {
+                const text = await fetchTranscript(it.id.replace(/^youtube:/, ""));
+                if (text) {
+                    it.snippet = text.slice(0, TRANSCRIPT_MAX_CHARS);
+                    transcribed += 1;
+                }
+            }));
+        }
+        return {
+            source: "youtube",
+            status: settleStatus("available", items.length),
+            reason: `yt-dlp returned ${items.length} video(s) within ${days}d` +
+                (transcribed > 0 ? `, ${transcribed} with transcript.` : "."),
+            items,
+        };
+    }
+    catch (err) {
+        if (err instanceof MissingBinaryError) {
+            return {
+                source: "youtube",
+                status: "excluded",
+                reason: "yt-dlp is not installed; install it (e.g. brew install yt-dlp) to enable YouTube.",
+                items: [],
+            };
+        }
+        return {
+            source: "youtube",
+            status: "excluded",
+            reason: `yt-dlp failed: ${err.message}`,
+            items: [],
+        };
+    }
+}