phronesis 1.0.0

package/src/ingest.js

@@ -0,0 +1,965 @@
+import { createHash } from 'node:crypto';
+import { lookup as dnsLookup } from 'node:dns/promises';
+import { constants as fsConstants } from 'node:fs';
+import { request as httpRequest } from 'node:http';
+import { request as httpsRequest } from 'node:https';
+import { isIP } from 'node:net';
+import { TextDecoder } from 'node:util';
+import { basename, extname, isAbsolute, join, relative, resolve, sep } from 'node:path';
+import { open, readdir, realpath, stat } from 'node:fs/promises';
+import { brotliDecompressSync, gunzipSync, inflateRawSync, inflateSync } from 'node:zlib';
+import { assertRawDestination, resolveInstall, stageRawSource, slugify } from '@phronesis/core';
+
+const TRUST_TIERS = ['trusted', 'untrusted'];
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+const FETCH_TIMEOUT_MS = 15_000;
+const MAX_RESPONSE_BYTES = 10 * 1024 * 1024;
+const MAX_LOCAL_FILE_BYTES = 10 * 1024 * 1024;
+const NOFOLLOW_OPEN_FLAGS =
+  typeof fsConstants.O_NOFOLLOW === 'number' ? fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW : fsConstants.O_RDONLY;
+const STRUCTURAL_HTML_TAG_RE =
+  /<!doctype\b|<\/?\s*(html|head|body|main|article|section|header|footer|nav|aside|div|p|br|h[1-6]|ul|ol|li|blockquote|pre|table|thead|tbody|tfoot|tr|td|th|script|style|noscript|title|meta|link|img|a|span|strong|em)\b[^>]*>/i;
+
+function sha256Short(input, length = 16) {
+  return createHash('sha256').update(input).digest('hex').slice(0, length);
+}
+
+function sanitizeUrlForProvenance(raw) {
+  const url = new URL(raw);
+  url.search = '';
+  url.hash = '';
+  return url.toString();
+}
+
+function isHttpUrl(value) {
+  try {
+    const url = new URL(value);
+    return url.protocol === 'http:' || url.protocol === 'https:';
+  } catch {
+    return false;
+  }
+}
+
+function isExplicitUrl(value) {
+  return /^[a-z][a-z0-9+.-]*:\/\//i.test(String(value || ''));
+}
+
+function htmlEntityDecode(text) {
+  const named = {
+    amp: '&',
+    apos: "'",
+    gt: '>',
+    lt: '<',
+    nbsp: ' ',
+    quot: '"',
+  };
+  return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, entity) => {
+    const key = entity.toLowerCase();
+    if (Object.hasOwn(named, key)) return named[key];
+    const codePoint = key.startsWith('#x') ? parseInt(key.slice(2), 16) : key.startsWith('#') ? parseInt(key.slice(1), 10) : NaN;
+    if (Number.isInteger(codePoint) && codePoint >= 0 && codePoint <= 0x10ffff) return String.fromCodePoint(codePoint);
+    return m;
+  });
+}
+
+function stripRawTextElement(text, tagName) {
+  const openRe = new RegExp(`<\\s*${tagName}\\b[^>]*>`, 'gi');
+  const closeRe = new RegExp(`<\\/\\s*${tagName}\\s*>`, 'gi');
+  const nextTagRe = /<[^>]+>/g;
+  let output = '';
+  let cursor = 0;
+
+  while (cursor < text.length) {
+    openRe.lastIndex = cursor;
+    const open = openRe.exec(text);
+    if (!open) return output + text.slice(cursor);
+    output += text.slice(cursor, open.index);
+
+    closeRe.lastIndex = openRe.lastIndex;
+    const close = closeRe.exec(text);
+    if (!close) return output;
+    cursor = closeRe.lastIndex;
+
+    while (cursor < text.length) {
+      nextTagRe.lastIndex = cursor;
+      const nextTag = nextTagRe.exec(text);
+      if (!nextTag) break;
+      closeRe.lastIndex = cursor;
+      const nextClose = closeRe.exec(text);
+      if (!nextClose || nextClose.index !== nextTag.index) break;
+      cursor = closeRe.lastIndex;
+    }
+  }
+  return output;
+}
+
+function stripDangerousHtmlBlocks(text) {
+  let stripped = text;
+  for (const tagName of ['script', 'style', 'noscript']) {
+    stripped = stripRawTextElement(stripped, tagName);
+  }
+  return stripped;
+}
+
+export function stripHtmlToMarkdown(html) {
+  const withoutRawBlocks = stripDangerousHtmlBlocks(String(html));
+  const decoded = stripDangerousHtmlBlocks(htmlEntityDecode(withoutRawBlocks));
+  return decoded
+    .replace(/<!--[\s\S]*?-->/g, '')
+    .replace(/<!doctype[^>]*>/gi, '')
+    .replace(/<h1\b[^>]*>([\s\S]*?)<\/h1>/gi, '\n\n# $1\n\n')
+    .replace(/<h2\b[^>]*>([\s\S]*?)<\/h2>/gi, '\n\n## $1\n\n')
+    .replace(/<h3\b[^>]*>([\s\S]*?)<\/h3>/gi, '\n\n### $1\n\n')
+    .replace(/<li\b[^>]*>/gi, '\n- ')
+    .replace(/<\/(p|div|section|article|header|footer|main|li|ul|ol|blockquote|pre|tr)>/gi, '\n')
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<[^>]+>/g, '')
+    .replace(/[<>]/g, (char) => (char === '<' ? '&lt;' : '&gt;'))
+    .replace(/[ \t]+\n/g, '\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+function firstMeaningfulLine(body) {
+  const line = body.split(/\r?\n/).find((l) => l.trim());
+  return line ? line.replace(/^#+\s*/, '').trim() : '';
+}
+
+function localFileSlug(file) {
+  return slugify(basename(file, extname(file))) || 'local-file';
+}
+
+function urlSlug(raw) {
+  const url = new URL(raw);
+  const last = url.pathname.split('/').filter(Boolean).pop();
+  return slugify(last ? last.replace(/\.[A-Za-z0-9]+$/, '') : url.hostname) || 'url-source';
+}
+
+function looksLikeHtml({ body, contentType = '' }) {
+  return /\bhtml\b/i.test(contentType) || STRUCTURAL_HTML_TAG_RE.test(body);
+}
+
+function readStream(stream) {
+  return new Promise((resolvePromise, reject) => {
+    let data = '';
+    stream.setEncoding('utf8');
+    stream.on('data', (chunk) => {
+      data += chunk;
+    });
+    stream.on('end', () => resolvePromise(data));
+    stream.on('error', reject);
+  });
+}
+
+function privateIpv4(address) {
+  const parts = address.split('.').map((p) => Number(p));
+  if (parts.length !== 4 || parts.some((p) => !Number.isInteger(p) || p < 0 || p > 255)) return true;
+  const [a, b, c, d] = parts;
+  return (
+    a === 0 ||
+    a === 10 ||
+    a === 127 ||
+    (a === 169 && b === 254) ||
+    (a === 172 && b >= 16 && b <= 31) ||
+    (a === 192 && b === 0 && c === 0) ||
+    (a === 192 && b === 168) ||
+    (a === 198 && (b === 18 || b === 19)) ||
+    (a === 100 && b >= 64 && b <= 127) ||
+    (a === 192 && b === 0 && c === 2) ||
+    (a === 198 && b === 51 && c === 100) ||
+    (a === 203 && b === 0 && c === 113) ||
+    (a >= 224 && a <= 255) ||
+    (a === 255 && b === 255 && c === 255 && d === 255)
+  );
+}
+
+function mappedIpv4FromIpv6(address) {
+  if (!address.startsWith('::ffff:')) return null;
+  const tail = address.slice('::ffff:'.length);
+  if (isIP(tail) === 4) return tail;
+  const pieces = tail.split(':');
+  if (pieces.length !== 2) return null;
+  const words = pieces.map((piece) => parseInt(piece, 16));
+  if (words.some((word) => !Number.isInteger(word) || word < 0 || word > 0xffff)) return null;
+  return `${words[0] >> 8}.${words[0] & 0xff}.${words[1] >> 8}.${words[1] & 0xff}`;
+}
+
+function embeddedMappedIpv4(hextets) {
+  if (
+    hextets?.length === 8 &&
+    hextets[0] === 0 &&
+    hextets[1] === 0 &&
+    hextets[2] === 0 &&
+    hextets[3] === 0 &&
+    hextets[4] === 0 &&
+    hextets[5] === 0xffff
+  ) {
+    return `${hextets[6] >> 8}.${hextets[6] & 0xff}.${hextets[7] >> 8}.${hextets[7] & 0xff}`;
+  }
+  return null;
+}
+
+function embeddedCompatibleIpv4(hextets) {
+  if (
+    hextets?.length === 8 &&
+    hextets[0] === 0 &&
+    hextets[1] === 0 &&
+    hextets[2] === 0 &&
+    hextets[3] === 0 &&
+    hextets[4] === 0 &&
+    hextets[5] === 0
+  ) {
+    return `${hextets[6] >> 8}.${hextets[6] & 0xff}.${hextets[7] >> 8}.${hextets[7] & 0xff}`;
+  }
+  return null;
+}
+
+function ipv6Hextets(address) {
+  let input = address;
+  if (input.includes('.')) {
+    const lastColon = input.lastIndexOf(':');
+    const ipv4 = input.slice(lastColon + 1);
+    if (isIP(ipv4) !== 4) return null;
+    const octets = ipv4.split('.').map((part) => Number(part));
+    const ipv4Words = [(octets[0] << 8) | octets[1], (octets[2] << 8) | octets[3]];
+    input = `${input.slice(0, lastColon)}:${ipv4Words.map((word) => word.toString(16)).join(':')}`;
+  }
+  const halves = input.split('::');
+  if (halves.length > 2) return null;
+  const left = halves[0] ? halves[0].split(':') : [];
+  const right = halves.length === 2 && halves[1] ? halves[1].split(':') : [];
+  const missing = 8 - left.length - right.length;
+  if (missing < 0 || (halves.length === 1 && missing !== 0)) return null;
+  const parts = halves.length === 2 ? [...left, ...Array(missing).fill('0'), ...right] : [...left, ...right];
+  if (parts.length !== 8) return null;
+  const parsed = parts.map((part) => parseInt(part || '0', 16));
+  if (parsed.some((part) => !Number.isInteger(part) || part < 0 || part > 0xffff)) return null;
+  return parsed;
+}
+
+function embeddedNat64Ipv4(hextets) {
+  if (
+    hextets?.length === 8 &&
+    hextets[0] === 0x0064 &&
+    hextets[1] === 0xff9b &&
+    hextets[2] === 0 &&
+    hextets[3] === 0 &&
+    hextets[4] === 0 &&
+    hextets[5] === 0
+  ) {
+    return `${hextets[6] >> 8}.${hextets[6] & 0xff}.${hextets[7] >> 8}.${hextets[7] & 0xff}`;
+  }
+  return null;
+}
+
+function privateIp(address) {
+  const normalized = address.replace(/^\[|\]$/g, '').replace(/%.*$/, '').toLowerCase();
+  const mappedIpv4 = mappedIpv4FromIpv6(normalized);
+  if (mappedIpv4) return privateIpv4(mappedIpv4);
+  if (isIP(normalized) === 4) return privateIpv4(normalized);
+  if (isIP(normalized) === 6) {
+    const hextets = ipv6Hextets(normalized);
+    const expandedMappedIpv4 = embeddedMappedIpv4(hextets);
+    if (expandedMappedIpv4) return privateIpv4(expandedMappedIpv4);
+    const compatibleIpv4 = embeddedCompatibleIpv4(hextets);
+    if (compatibleIpv4) return privateIpv4(compatibleIpv4);
+    const nat64Ipv4 = embeddedNat64Ipv4(hextets);
+    if (nat64Ipv4) return privateIpv4(nat64Ipv4);
+    const firstHextet = hextets?.[0] ?? parseInt(normalized.split(':').find((part) => part.length > 0) || '0', 16);
+    return (
+      normalized === '::' ||
+      normalized === '::1' ||
+      (firstHextet & 0xfe00) === 0xfc00 ||
+      (firstHextet & 0xffc0) === 0xfe80 ||
+      (firstHextet & 0xffc0) === 0xfec0 ||
+      (firstHextet & 0xff00) === 0xff00 ||
+      (firstHextet === 0x2001 && hextets?.[1] === 0x0db8)
+    );
+  }
+  return false;
+}
+
+async function resolveFetchablePublicUrl(url, lookupImpl, { allowPrivateNetworkForTest = false } = {}) {
+  if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+    throw new Error(`URL protocol "${url.protocol}" is not supported (expected http or https).`);
+  }
+  const host = url.hostname.replace(/^\[|\]$/g, '').toLowerCase();
+  if (!allowPrivateNetworkForTest && (host === 'localhost' || host.endsWith('.localhost') || privateIp(host))) {
+    throw new Error(`refusing to fetch internal or loopback URL host "${url.hostname}".`);
+  }
+  let addresses;
+  try {
+    addresses = await lookupImpl(url.hostname, { all: true, verbatim: true });
+  } catch (error) {
+    throw new Error(`could not resolve URL host "${url.hostname}": ${error.message}`);
+  }
+  if (!Array.isArray(addresses) || addresses.length === 0) {
+    throw new Error(`could not resolve URL host "${url.hostname}" to an address.`);
+  }
+  for (const entry of addresses) {
+    if (!allowPrivateNetworkForTest && privateIp(entry.address)) {
+      throw new Error(`refusing to fetch URL host "${url.hostname}" because it resolves to internal address ${entry.address}.`);
+    }
+  }
+  return addresses[0];
+}
+
+function responseHeader(headers, name) {
+  const value = headers?.[name.toLowerCase()];
+  if (Array.isArray(value)) return value.join(', ');
+  return value ?? null;
+}
+
+function declaredContentLength(headers) {
+  const raw = responseHeader(headers, 'content-length');
+  if (!raw) return null;
+  const parts = String(raw)
+    .split(',')
+    .map((part) => part.trim())
+    .filter(Boolean);
+  if (parts.length === 0) return null;
+  if (!parts.every((part) => /^\d+$/.test(part))) {
+    throw new Error(`URL fetch returned malformed Content-Length "${raw}".`);
+  }
+  if (!parts.every((part) => part === parts[0])) {
+    throw new Error(`URL fetch returned conflicting Content-Length "${raw}".`);
+  }
+  return Number(parts[0]);
+}
+
+function contentEncodingChain(headers) {
+  const raw = responseHeader(headers, 'content-encoding');
+  return raw
+    ? raw
+        .split(',')
+        .map((part) => part.trim().toLowerCase())
+        .filter(Boolean)
+    : [];
+}
+
+function inflateDeflate(buffer, maxBytes) {
+  try {
+    return inflateSync(buffer, { maxOutputLength: maxBytes });
+  } catch (error) {
+    if (error?.code !== 'Z_DATA_ERROR') throw error;
+    return inflateRawSync(buffer, { maxOutputLength: maxBytes });
+  }
+}
+
+function decompressBody(buffer, encodings, maxBytes) {
+  let decoded = buffer;
+  for (const encoding of [...encodings].reverse()) {
+    if (encoding === 'identity') continue;
+    try {
+      if (encoding === 'gzip' || encoding === 'x-gzip') decoded = gunzipSync(decoded, { maxOutputLength: maxBytes });
+      else if (encoding === 'deflate') decoded = inflateDeflate(decoded, maxBytes);
+      else if (encoding === 'br') decoded = brotliDecompressSync(decoded, { maxOutputLength: maxBytes });
+      else throw new Error(`URL fetch returned unsupported content-encoding "${encoding}".`);
+    } catch (error) {
+      if (error?.code === 'ERR_BUFFER_TOO_LARGE') {
+        throw new Error(`URL fetch decoded response exceeded ${maxBytes} bytes.`);
+      }
+      throw error;
+    }
+    if (decoded.length > maxBytes) throw new Error(`URL fetch decoded response exceeded ${maxBytes} bytes.`);
+  }
+  return decoded;
+}
+
+function responseCharset(headers) {
+  const contentType = responseHeader(headers, 'content-type') || '';
+  const match = contentType.match(/;\s*charset=([^;]+)/i);
+  const charset = (match?.[1] || 'utf-8').trim().replace(/^"|"$/g, '').toLowerCase();
+  if (charset === 'utf8') return 'utf-8';
+  if (charset === 'latin1') return 'iso-8859-1';
+  return charset;
+}
+
+function decodeBody(buffer, headers, maxBytes) {
+  const decompressed = decompressBody(buffer, contentEncodingChain(headers), maxBytes);
+  const charset = responseCharset(headers);
+  try {
+    return new TextDecoder(charset, { fatal: false }).decode(decompressed);
+  } catch (error) {
+    throw new Error(`URL fetch returned unsupported charset "${charset}": ${error.message}`);
+  }
+}
+
+function toReportPath(path) {
+  return String(path).split(sep).join('/');
+}
+
+function sortedDirents(entries) {
+  return [...entries].sort((a, b) => {
+    if (a.name < b.name) return -1;
+    if (a.name > b.name) return 1;
+    return 0;
+  });
+}
+
+function reportRecord(path, reason) {
+  return { path: toReportPath(path), reason };
+}
+
+function addSkip(report, path, reason) {
+  report.skipped.push(reportRecord(path, reason));
+}
+
+function addFailure(report, path, reason) {
+  report.failed.push(reportRecord(path, reason));
+}
+
+function isSameOrInside(parent, child) {
+  const rel = relative(parent, child);
+  return rel === '' || (rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel));
+}
+
+async function realpathOrResolve(path) {
+  try {
+    return await realpath(path);
+  } catch {
+    return resolve(path);
+  }
+}
+
+async function isPhronesisInstallRoot(dir) {
+  try {
+    const marker = await stat(join(dir, '.phronesis', 'registry.json'));
+    return marker.isFile();
+  } catch {
+    return false;
+  }
+}
+
+function addDeduped(report, path, existingPath) {
+  report.deduped += 1;
+  report.deduped_items.push({ path: toReportPath(path), existing_path: toReportPath(existingPath) });
+}
+
+function hasConnectorBridgeInput({ sourceConnector, sourceExternalId, sourceTimestamp }) {
+  return sourceConnector !== undefined || sourceExternalId !== undefined || sourceTimestamp !== undefined;
+}
+
+function requireNonEmptyOption(value, flag) {
+  if (typeof value !== 'string' || value.trim() === '') throw new Error(`connector bridge ingest requires ${flag} <value>.`);
+}
+
+function connectorBridgeProvenance({ sourceConnector, sourceExternalId, sourceTimestamp }) {
+  requireNonEmptyOption(sourceConnector, '--source-connector');
+  requireNonEmptyOption(sourceExternalId, '--source-external-id');
+  requireNonEmptyOption(sourceTimestamp, '--source-timestamp');
+  return {
+    source_connector: sourceConnector,
+    source_external_id: sourceExternalId,
+    source_timestamp: sourceTimestamp,
+  };
+}
+
+function slugFromTitle({ title, slug }) {
+  if (typeof slug === 'string' && slug.trim()) return slug.trim();
+  return slugify(title);
+}
+
+function singleSourceReportPath(options) {
+  if (options.paste || options.source === '-') return '-';
+  return options.source || 'source';
+}
+
+function decodeStrictUtf8(buffer) {
+  if (buffer.includes(0)) return { error: 'binary' };
+  try {
+    return { raw: new TextDecoder('utf-8', { fatal: true }).decode(buffer) };
+  } catch {
+    return { error: 'invalid-utf8' };
+  }
+}
+
+async function readLocalFileCapped(absolute, maxBytes, { followSymlinks = true } = {}) {
+  const flags = followSymlinks ? 'r' : NOFOLLOW_OPEN_FLAGS;
+  const handle = await open(absolute, flags);
+  const chunks = [];
+  let total = 0;
+  try {
+    while (true) {
+      const remainingBeforeProbe = maxBytes + 1 - total;
+      const buffer = Buffer.alloc(Math.min(64 * 1024, Math.max(1, remainingBeforeProbe)));
+      const { bytesRead } = await handle.read(buffer, 0, buffer.length, null);
+      if (bytesRead === 0) break;
+      total += bytesRead;
+      if (total > maxBytes) {
+        return { skipped: `oversized: more than ${maxBytes} bytes` };
+      }
+      chunks.push(buffer.subarray(0, bytesRead));
+    }
+  } finally {
+    await handle.close().catch(() => {});
+  }
+  return { buffer: Buffer.concat(chunks, total) };
+}
+
+async function collectBulkFiles({ root, dir = root, report, excludedRoot }) {
+  const resolvedDir = await realpathOrResolve(dir);
+  if ((excludedRoot && isSameOrInside(excludedRoot, resolvedDir)) || (await isPhronesisInstallRoot(dir))) {
+    addSkip(report, relative(root, dir) || '.', 'install-root');
+    return [];
+  }
+
+  let entries;
+  try {
+    entries = await readdir(dir, { withFileTypes: true });
+  } catch (error) {
+    addSkip(report, relative(root, dir) || '.', `unreadable: ${error.message}`);
+    return [];
+  }
+
+  const files = [];
+  for (const entry of sortedDirents(entries)) {
+    const absolute = join(dir, entry.name);
+    const sourcePath = relative(root, absolute);
+    if (entry.name.startsWith('.')) {
+      addSkip(report, sourcePath, 'hidden');
+      continue;
+    }
+    if (entry.isSymbolicLink()) {
+      addSkip(report, sourcePath, 'non-regular');
+      continue;
+    }
+    if (entry.isDirectory()) {
+      files.push(...(await collectBulkFiles({ root, dir: absolute, report, excludedRoot })));
+      continue;
+    }
+    if (!entry.isFile()) {
+      addSkip(report, sourcePath, 'non-regular');
+      continue;
+    }
+    files.push({ absolute, sourcePath: toReportPath(sourcePath) });
+  }
+  return files;
+}
+
+function prepareLocalFileSource({ absolute, info, raw, trustTier, now, sourcePath }) {
+  const fileBase = basename(absolute);
+  const isHtml = ['.html', '.htm'].includes(extname(fileBase).toLowerCase()) || looksLikeHtml({ body: raw });
+  const body = isHtml ? stripHtmlToMarkdown(raw) : raw;
+  if (body.trim() === '') throw new Error('empty');
+  return {
+    body,
+    slug: localFileSlug(fileBase),
+    provenance: {
+      source_connector: 'manual',
+      // Basename-only parity with single-file ingest is the dedup contract; source_path is traceability, not identity.
+      source_external_id: `file:${fileBase}:${sha256Short(raw)}`,
+      source_timestamp: info.mtime.toISOString(),
+      source_file: fileBase,
+      source_path: sourcePath,
+      content_type: 'file',
+      retrieval_tool: 'local-file',
+      trust_tier: trustTier,
+      operator_affirmed_trusted: trustTier === 'trusted' ? true : undefined,
+      fetched_at: now.toISOString(),
+    },
+  };
+}
+
+async function sourceFromBulkFile({ absolute, sourcePath, trustTier, now, maxLocalFileBytes }) {
+  let info;
+  try {
+    info = await stat(absolute);
+  } catch (error) {
+    return { skipped: `unreadable: ${error.message}` };
+  }
+  if (!info.isFile()) return { skipped: 'non-regular' };
+  if (info.size > maxLocalFileBytes) return { skipped: `oversized: ${info.size} bytes exceeds ${maxLocalFileBytes}` };
+
+  let read;
+  try {
+    read = await readLocalFileCapped(absolute, maxLocalFileBytes, { followSymlinks: false });
+  } catch (error) {
+    return { skipped: `unreadable: ${error.message}` };
+  }
+  if (read.skipped) return { skipped: read.skipped };
+  const buffer = read.buffer;
+
+  const decoded = decodeStrictUtf8(buffer);
+  if (decoded.error) return { skipped: decoded.error };
+
+  try {
+    return {
+      source: prepareLocalFileSource({
+        absolute,
+        info,
+        raw: decoded.raw,
+        trustTier,
+        now,
+        sourcePath,
+      }),
+    };
+  } catch (error) {
+    return { skipped: error.message || 'invalid' };
+  }
+}
+
+async function ingestDirectory({ absolute, installRoot, domain, lane, trustTier, now, maxLocalFileBytes = MAX_LOCAL_FILE_BYTES }) {
+  await assertRawDestination({ installRoot, domain, lane });
+
+  const report = { staged: 0, deduped: 0, deduped_items: [], skipped: [], failed: [] };
+  const files = (await collectBulkFiles({ root: absolute, report, excludedRoot: await realpathOrResolve(installRoot) })).sort((a, b) => {
+    if (a.sourcePath < b.sourcePath) return -1;
+    if (a.sourcePath > b.sourcePath) return 1;
+    return 0;
+  });
+  // The flattened sort makes the retained source_path deterministic when same-basename/same-byte copies dedupe.
+  for (const file of files) {
+    const prepared = await sourceFromBulkFile({
+      absolute: file.absolute,
+      sourcePath: file.sourcePath,
+      trustTier,
+      now,
+      maxLocalFileBytes,
+    });
+    if (prepared.skipped) {
+      addSkip(report, file.sourcePath, prepared.skipped);
+      continue;
+    }
+    try {
+      const result = await stageRawSource({
+        installRoot,
+        domain,
+        lane,
+        slug: prepared.source.slug,
+        ext: 'md',
+        body: prepared.source.body,
+        provenance: prepared.source.provenance,
+        now,
+      });
+      if (result.deduped) addDeduped(report, file.sourcePath, result.path);
+      else if (result.written) report.staged += 1;
+    } catch (error) {
+      addFailure(report, file.sourcePath, error.message);
+    }
+  }
+  return { ok: true, bulk: true, report };
+}
+
+function fetchPinnedUrl(
+  rawUrl,
+  { headers = {}, pinnedAddress, pinnedFamily, timeoutMs = FETCH_TIMEOUT_MS, maxBytes = MAX_RESPONSE_BYTES } = {},
+) {
+  const url = new URL(rawUrl);
+  if (!pinnedAddress) throw new Error(`cannot fetch "${url.hostname}" without a pinned address.`);
+  const client = url.protocol === 'https:' ? httpsRequest : httpRequest;
+  const port = url.port || (url.protocol === 'https:' ? 443 : 80);
+  const host = url.host;
+
+  return new Promise((resolvePromise, reject) => {
+    const request = client(
+      {
+        protocol: url.protocol,
+        hostname: pinnedAddress,
+        family: pinnedFamily,
+        port,
+        method: 'GET',
+        path: `${url.pathname || '/'}${url.search}`,
+        headers: { ...headers, host },
+        servername: url.hostname,
+        timeout: timeoutMs,
+      },
+      (response) => {
+        const chunks = [];
+        let received = 0;
+        let failed = false;
+        const fail = (error) => {
+          if (failed) return;
+          failed = true;
+          reject(error);
+          response.destroy();
+          request.destroy();
+        };
+        response.on('data', (chunk) => {
+          received += chunk.length;
+          if (received > maxBytes) {
+            fail(new Error(`URL fetch response exceeded ${maxBytes} bytes.`));
+            return;
+          }
+          chunks.push(chunk);
+        });
+        response.on('aborted', () => fail(new Error('URL fetch response ended before the declared body was received.')));
+        response.on('error', fail);
+        response.on('end', () => {
+          if (failed) return;
+          const status = Number(response.statusCode || 0);
+          let contentLength;
+          try {
+            contentLength = declaredContentLength(response.headers);
+          } catch (error) {
+            reject(error);
+            return;
+          }
+          if (contentLength !== null && received !== contentLength) {
+            reject(new Error(`URL fetch response was truncated (${received}/${contentLength} bytes).`));
+            return;
+          }
+          let decodedText;
+          try {
+            decodedText = decodeBody(Buffer.concat(chunks), response.headers, maxBytes);
+          } catch (error) {
+            reject(error);
+            return;
+          }
+          resolvePromise({
+            status,
+            ok: status >= 200 && status < 300,
+            headers: { get: (name) => responseHeader(response.headers, name) },
+            text: async () => decodedText,
+          });
+        });
+      },
+    );
+    request.on('timeout', () => request.destroy(new Error(`URL fetch timed out after ${timeoutMs}ms.`)));
+    request.on('error', reject);
+    request.end();
+  });
+}
+
+async function fetchText(rawUrl, { fetchImpl, lookupImpl, allowPrivateNetworkForTest = false, maxResponseBytes = MAX_RESPONSE_BYTES }) {
+  let url = new URL(rawUrl);
+  for (let redirect = 0; redirect <= 5; redirect += 1) {
+    const resolved = await resolveFetchablePublicUrl(url, lookupImpl, { allowPrivateNetworkForTest });
+    const response = await (fetchImpl || fetchPinnedUrl)(url.toString(), {
+      redirect: 'manual',
+      headers: {
+        'user-agent': 'phronesis-ingest-cli/0.0.0',
+        'accept-encoding': 'gzip, deflate, br',
+        connection: 'close',
+      },
+      pinnedAddress: resolved.address,
+      pinnedFamily: resolved.family,
+      timeoutMs: FETCH_TIMEOUT_MS,
+      maxBytes: maxResponseBytes,
+    });
+    const status = Number(response.status || 0);
+    if (REDIRECT_STATUSES.has(status)) {
+      const location = response.headers?.get?.('location');
+      if (!location) throw new Error(`URL returned redirect ${status} without a Location header.`);
+      url = new URL(location, url);
+      continue;
+    }
+    if (!response.ok) throw new Error(`URL fetch failed with HTTP ${status}.`);
+    return {
+      body: await response.text(),
+      contentType: response.headers?.get?.('content-type') || '',
+      finalUrl: url.toString(),
+    };
+  }
+  throw new Error('URL fetch followed too many redirects.');
+}
+
+async function sourceFromPaste({ stdin, now, title, slug }) {
+  const body = await readStream(stdin);
+  if (body.trim() === '') throw new Error('paste input was empty; ingest wrote nothing.');
+  const derivedSlug = slugFromTitle({ title, slug }) || slugify(firstMeaningfulLine(body)) || 'pasted-source';
+  const id = `operator-paste:${sha256Short(body)}`;
+  return {
+    body,
+    slug: derivedSlug,
+    provenance: {
+      source_connector: 'manual',
+      source_external_id: id,
+      source_timestamp: now.toISOString(),
+      source_url: `operator-paste:${derivedSlug}`,
+      content_type: 'paste',
+      retrieval_tool: 'operator-paste',
+      trust_tier: 'untrusted',
+    },
+  };
+}
+
+async function sourceFromFile({ source, trustTier, now, cwd, absolute: resolvedAbsolute, info: resolvedInfo, maxLocalFileBytes }) {
+  const absolute = resolvedAbsolute || resolve(cwd, source);
+  const info =
+    resolvedInfo ||
+    (await stat(absolute).catch((error) => {
+      throw new Error(`could not read local file "${source}": ${error.message}`);
+    }));
+  if (!info.isFile()) throw new Error(`local source "${source}" is not a file.`);
+  if (info.size > maxLocalFileBytes) {
+    throw new Error(`local source "${source}" exceeds ${maxLocalFileBytes} bytes.`);
+  }
+  let read;
+  try {
+    read = await readLocalFileCapped(absolute, maxLocalFileBytes);
+  } catch (error) {
+    throw new Error(`could not read local file "${source}": ${error.message}`);
+  }
+  if (read.skipped) throw new Error(`local source "${source}" ${read.skipped}.`);
+  const raw = read.buffer.toString('utf8');
+  try {
+    return prepareLocalFileSource({ absolute, info, raw, trustTier, now });
+  } catch (error) {
+    if (error.message === 'empty') throw new Error(`local source "${source}" did not produce non-empty text.`);
+    throw error;
+  }
+}
+
+async function sourceFromUrl({ source, now, fetchImpl, lookupImpl, allowPrivateNetworkForTest, maxResponseBytes }) {
+  const fetched = await fetchText(source, { fetchImpl, lookupImpl, allowPrivateNetworkForTest, maxResponseBytes });
+  const body = looksLikeHtml(fetched) ? stripHtmlToMarkdown(fetched.body) : fetched.body.trim();
+  if (body.trim() === '') throw new Error('URL fetch produced empty text; ingest wrote nothing.');
+  const clean = sanitizeUrlForProvenance(fetched.finalUrl);
+  return {
+    body,
+    slug: urlSlug(clean),
+    provenance: {
+      source_connector: 'manual',
+      source_external_id: `url:${clean}`,
+      source_timestamp: now.toISOString(),
+      source_url: clean,
+      content_type: 'article',
+      retrieval_tool: 'web-fetch',
+      trust_tier: 'untrusted',
+      fetched_at: now.toISOString(),
+    },
+  };
+}
+
+export async function ingestSource({
+  source,
+  paste = false,
+  trust,
+  lane = 'inbox',
+  domain,
+  dir,
+  cwd = process.cwd(),
+  stdin = process.stdin,
+  fetchImpl,
+  lookupImpl = dnsLookup,
+  allowPrivateNetworkForTest = false,
+  maxResponseBytes = MAX_RESPONSE_BYTES,
+  maxLocalFileBytes = MAX_LOCAL_FILE_BYTES,
+  sourceConnector,
+  sourceExternalId,
+  sourceTimestamp,
+  title,
+  slug,
+  now = new Date(),
+} = {}) {
+  const installed = resolveInstall({ dir, cwd });
+  if (installed.error) throw new Error(installed.error);
+  if (typeof domain !== 'string' || domain.trim() === '') throw new Error('ingest requires --domain <slug>.');
+  const trustTier = trust || 'untrusted';
+  if (!TRUST_TIERS.includes(trustTier)) throw new Error(`--trust must be trusted | untrusted (got "${trust}").`);
+  const connectorBridge = hasConnectorBridgeInput({ sourceConnector, sourceExternalId, sourceTimestamp });
+  let bridgeProvenance = null;
+  if (connectorBridge) {
+    bridgeProvenance = connectorBridgeProvenance({ sourceConnector, sourceExternalId, sourceTimestamp });
+    if (source !== '-') throw new Error('connector bridge ingest requires - as the stdin source.');
+    if (trustTier !== 'untrusted') throw new Error('connector bridge ingest is always untrusted; --trust trusted is refused.');
+    if (paste) throw new Error('connector bridge ingest uses - for stdin; --paste is the legacy manual paste mode.');
+  }
+  if (source === '-' && (typeof title !== 'string' || title.trim() === '')) {
+    throw new Error('stdin ingest requires --title <title>; --slug is only an optional exact filename override.');
+  }
+
+  let prepared;
+  if (paste || source === '-') {
+    if (source && source !== '-') throw new Error('--paste does not accept a source argument; pass content on stdin.');
+    if (trustTier === 'trusted') throw new Error('--trust trusted is only allowed for local-file ingest.');
+    prepared = await sourceFromPaste({ stdin, now, title, slug });
+  } else if (isHttpUrl(source || '')) {
+    if (trustTier === 'trusted') throw new Error('--trust trusted is refused for URL ingest; fetched sources are always untrusted.');
+    prepared = await sourceFromUrl({ source, now, fetchImpl, lookupImpl, allowPrivateNetworkForTest, maxResponseBytes });
+  } else if (isExplicitUrl(source)) {
+    const url = new URL(source);
+    throw new Error(`URL protocol "${url.protocol}" is not supported (expected http or https).`);
+  } else if (source) {
+    const absolute = resolve(cwd, source);
+    const info = await stat(absolute).catch((error) => {
+      throw new Error(`could not read local file "${source}": ${error.message}`);
+    });
+    if (info.isDirectory()) {
+      if (connectorBridge) {
+        throw new Error('connector bridge provenance flags are single-source only; use one ingest call per bridge item.');
+      }
+      return ingestDirectory({
+        absolute,
+        installRoot: installed.root,
+        domain,
+        lane,
+        trustTier,
+        now,
+        maxLocalFileBytes,
+      });
+    }
+    prepared = await sourceFromFile({ source, trustTier, now, cwd, absolute, info, maxLocalFileBytes });
+  } else {
+    throw new Error('ingest requires a <url|file|dir> source, or --paste / - for stdin.');
+  }
+
+  if (connectorBridge) {
+    prepared = {
+      body: prepared.body,
+      slug: slugFromTitle({ title, slug }) || prepared.slug,
+      provenance: bridgeProvenance,
+    };
+  }
+
+  return stageRawSource({
+    installRoot: installed.root,
+    domain,
+    lane,
+    slug: prepared.slug,
+    ext: 'md',
+    body: prepared.body,
+    provenance: prepared.provenance,
+    provenanceMode: connectorBridge ? 'connector-bridge' : 'manual',
+    now,
+  });
+}
+
+export async function runIngest(options = {}) {
+  try {
+    const result = await ingestSource(options);
+    if (options.json) {
+      const report = result.bulk
+        ? result.report
+        : {
+            staged: result.deduped ? 0 : 1,
+            deduped: result.deduped ? 1 : 0,
+            deduped_items: result.deduped ? [{ path: toReportPath(singleSourceReportPath(options)), existing_path: toReportPath(result.path) }] : [],
+            skipped: [],
+            failed: [],
+          };
+      console.log(JSON.stringify(report, null, 2));
+      process.exitCode = 0;
+      return result;
+    }
+    if (result.bulk) {
+      console.log(`Staged ${result.report.staged} source(s); deduped ${result.report.deduped} existing source(s).`);
+      if (result.report.skipped.length) {
+        console.log(`Skipped ${result.report.skipped.length}:`);
+        for (const item of result.report.skipped) console.log(`  - ${item.path}: ${item.reason}`);
+      }
+      if (result.report.deduped_items.length) {
+        console.log(`Deduped ${result.report.deduped_items.length}:`);
+        for (const item of result.report.deduped_items) console.log(`  - ${item.path}: already staged at ${item.existing_path}`);
+      }
+      if (result.report.failed.length) {
+        console.log(`Failed ${result.report.failed.length}:`);
+        for (const item of result.report.failed) console.log(`  - ${item.path}: ${item.reason}`);
+      }
+      console.log('Now run compile for these raw sources.');
+      process.exitCode = 0;
+      return result;
+    }
+    const status = result.deduped ? 'Already staged' : 'Staged 1 source';
+    console.log(`${status} to ${result.path}`);
+    console.log(`trust_tier: ${result.deduped ? 'unchanged' : options.trust || 'untrusted'}`);
+    console.log('Now run compile for this raw source.');
+    process.exitCode = 0;
+    return result;
+  } catch (error) {
+    console.error(`ingest: ${error.message}`);
+    process.exitCode = 1;
+    return null;
+  }
+}