weftens 0.0.1 → 0.1.1

package/agent-cores/content-ops/src/inventory.mjs

@@ -0,0 +1,343 @@
+// Deterministic content-operations engine. No LLM.
+//
+// Input: rows from a CMS export (url, title, published date, last-updated,
+// word count, pageviews, meta description). Output: a per-URL inventory with a
+// keep / update / retire decision and the evidence for it, a metadata-gap list,
+// and a migration-readiness QA summary.
+//
+// Every decision is a pure function of the row's numbers against fixed thresholds,
+// plus a fixed asOf date (defaults to the latest date in the export, so results
+// never depend on the wall clock — same input always yields the same output).
+
+import { parseExport } from './parse.mjs';
+
+export const DEFAULT_THRESHOLDS = Object.freeze({
+  staleDays: 365,        // not touched in a year → aging
+  veryStaleDays: 730,    // not touched in two years → decayed
+  lowTrafficViews: 50,   // at or below this over the reporting window → low value
+  highTrafficViews: 500, // at or above this → protect it, never retire
+  thinWords: 300,        // below this → thin content (ROT candidate)
+  titleMin: 10,
+  titleMax: 65,
+  descMin: 50,
+  descMax: 160,
+});
+
+// --- field normalization: map an export's own headers onto the canonical schema ---
+
+const FIELD_ALIASES = {
+  url: ['url', 'link', 'address', 'page', 'permalink', 'loc'],
+  title: ['title', 'page title', 'name', 'heading', 'h1'],
+  published: ['published', 'published date', 'publish date', 'publishdate', 'published_date', 'date', 'created', 'created date', 'first published'],
+  updated: ['updated', 'last updated', 'last-updated', 'lastupdated', 'last_updated', 'modified', 'last modified', 'lastmod', 'updated date'],
+  wordCount: ['word count', 'wordcount', 'words', 'word_count', 'length'],
+  pageviews: ['pageviews', 'page views', 'views', 'traffic', 'sessions', 'visits', 'page_views'],
+  metaDescription: ['meta description', 'metadescription', 'meta_description', 'description', 'meta desc', 'seo description', 'excerpt'],
+};
+
+function buildHeaderMap(sampleRow) {
+  const map = {};
+  const keys = Object.keys(sampleRow);
+  const lowered = keys.map((k) => k.trim().toLowerCase());
+  for (const [canonical, aliases] of Object.entries(FIELD_ALIASES)) {
+    for (const alias of aliases) {
+      const idx = lowered.indexOf(alias);
+      if (idx !== -1) {
+        map[canonical] = keys[idx];
+        break;
+      }
+    }
+  }
+  return map;
+}
+
+function toNumber(v) {
+  if (v === null || v === undefined) return null;
+  const s = String(v).trim().replace(/,/g, '');
+  if (s === '') return null;
+  const n = Number(s);
+  return Number.isFinite(n) ? n : null;
+}
+
+// Parse a date to a UTC-midnight epoch (ms). Accepts YYYY-MM-DD and anything
+// Date can parse; returns null on failure so "missing" is distinguishable from 0.
+function toDateMs(v) {
+  if (v === null || v === undefined) return null;
+  const s = String(v).trim();
+  if (s === '') return null;
+  const iso = /^(\d{4})-(\d{2})-(\d{2})/.exec(s);
+  if (iso) {
+    return Date.UTC(Number(iso[1]), Number(iso[2]) - 1, Number(iso[3]));
+  }
+  const t = Date.parse(s);
+  return Number.isNaN(t) ? null : t;
+}
+
+const DAY = 86_400_000;
+const daysBetween = (aMs, bMs) => Math.round((aMs - bMs) / DAY);
+
+function cleanStr(v) {
+  if (v === null || v === undefined) return '';
+  return String(v).trim();
+}
+
+// --- the decision core ---
+
+function decide(item, asOfMs, T) {
+  const reasons = [];
+  const { pageviews, updatedMs, publishedMs, wordCount } = item;
+
+  // freshness is measured from last-update when present, else from publish
+  const freshMs = updatedMs ?? publishedMs;
+  const staleDays = freshMs !== null ? daysBetween(asOfMs, freshMs) : null;
+
+  const hasTraffic = pageviews !== null;
+  const hasFresh = staleDays !== null;
+
+  // If we can't judge value or freshness, we don't guess — we route to review.
+  if (!hasTraffic || !hasFresh) {
+    const missing = [];
+    if (!hasTraffic) missing.push('pageviews');
+    if (!hasFresh) missing.push('published/updated date');
+    reasons.push(`insufficient evidence to auto-decide (missing ${missing.join(' and ')})`);
+    return { decision: 'review', reasons, staleDays, confidence: 'low' };
+  }
+
+  const views = pageviews;
+  const words = wordCount; // may be null; only used in ROT branch guarded below
+
+  // 1) Protect valuable pages: high traffic is never retired.
+  if (views >= T.highTrafficViews) {
+    if (staleDays >= T.staleDays) {
+      reasons.push(`high traffic (${views} views) but not updated in ${staleDays} days → refresh, do not retire`);
+      return { decision: 'update', reasons, staleDays, confidence: 'high' };
+    }
+    reasons.push(`high traffic (${views} views) and fresh (updated ${staleDays} days ago)`);
+    return { decision: 'keep', reasons, staleDays, confidence: 'high' };
+  }
+
+  // 2) Low traffic: candidates for retirement or refresh.
+  if (views <= T.lowTrafficViews) {
+    if (staleDays >= T.veryStaleDays) {
+      reasons.push(`stale (${staleDays} days without update) and low traffic (${views} views) → ROT, retire with redirect`);
+      return { decision: 'retire', reasons, staleDays, confidence: 'high' };
+    }
+    if (words !== null && words < T.thinWords) {
+      reasons.push(`thin content (${words} words) and low traffic (${views} views) → ROT, retire with redirect`);
+      return { decision: 'retire', reasons, staleDays, confidence: 'medium' };
+    }
+    if (staleDays >= T.staleDays) {
+      reasons.push(`low traffic (${views} views) and aging (${staleDays} days) but not yet ROT → refresh and reassess`);
+      return { decision: 'update', reasons, staleDays, confidence: 'medium' };
+    }
+    reasons.push(`low traffic (${views} views) but recent (${staleDays} days) → keep and monitor`);
+    return { decision: 'keep', reasons, staleDays, confidence: 'low' };
+  }
+
+  // 3) Mid traffic: keep unless it has decayed past very-stale.
+  if (staleDays >= T.veryStaleDays) {
+    reasons.push(`moderate traffic (${views} views) but decayed (${staleDays} days without update) → refresh`);
+    return { decision: 'update', reasons, staleDays, confidence: 'medium' };
+  }
+  reasons.push(`moderate traffic (${views} views) and acceptable freshness (${staleDays} days)`);
+  return { decision: 'keep', reasons, staleDays, confidence: 'medium' };
+}
+
+// --- metadata gap checks (mechanical, per item) ---
+
+function metadataGaps(item, T) {
+  const gaps = [];
+  if (item.title === '') gaps.push({ field: 'title', issue: 'missing', severity: 'fail' });
+  else if (item.title.length < T.titleMin) gaps.push({ field: 'title', issue: `too short (${item.title.length} chars, aim ${T.titleMin}-${T.titleMax})`, severity: 'warn' });
+  else if (item.title.length > T.titleMax) gaps.push({ field: 'title', issue: `too long (${item.title.length} chars, aim ${T.titleMin}-${T.titleMax})`, severity: 'warn' });
+
+  if (item.metaDescription === '') gaps.push({ field: 'metaDescription', issue: 'missing', severity: 'warn' });
+  else if (item.metaDescription.length < T.descMin) gaps.push({ field: 'metaDescription', issue: `too short (${item.metaDescription.length} chars, aim ${T.descMin}-${T.descMax})`, severity: 'warn' });
+  else if (item.metaDescription.length > T.descMax) gaps.push({ field: 'metaDescription', issue: `too long (${item.metaDescription.length} chars, aim ${T.descMin}-${T.descMax})`, severity: 'warn' });
+
+  if (item.url === '') gaps.push({ field: 'url', issue: 'missing', severity: 'fail' });
+
+  // data-completeness gaps that block an evidence-based decision
+  if (item.publishedMs === null && item.updatedMs === null) gaps.push({ field: 'date', issue: 'no published or updated date', severity: 'fail' });
+  if (item.pageviews === null) gaps.push({ field: 'pageviews', issue: 'missing traffic data', severity: 'warn' });
+  if (item.wordCount === null) gaps.push({ field: 'wordCount', issue: 'missing word count', severity: 'info' });
+
+  return gaps;
+}
+
+// --- main entry ---
+
+export function buildInventory(rows, options = {}) {
+  const T = { ...DEFAULT_THRESHOLDS, ...(options.thresholds || {}) };
+  const arr = Array.isArray(rows) ? rows : [];
+
+  if (arr.length === 0) {
+    return {
+      asOf: options.asOf ?? null,
+      thresholds: T,
+      counts: { total: 0, keep: 0, update: 0, retire: 0, review: 0 },
+      unmappedFields: [],
+      headerMap: {},
+      items: [],
+      metadataGaps: [],
+      duplicates: { titles: [], urls: [] },
+      migrationReadiness: {
+        totalUrls: 0,
+        redirectsRequired: 0,
+        redirectLedger: [],
+        itemsWithGaps: 0,
+        blockingGaps: 0,
+        fieldCoverage: {},
+        readinessScore: 0,
+        blockers: ['export is empty — nothing to inventory'],
+      },
+    };
+  }
+
+  const headerMap = buildHeaderMap(arr[0]);
+
+  // normalize each row to the canonical schema
+  const normalized = arr.map((raw, idx) => {
+    const g = (canonical) => (headerMap[canonical] !== undefined ? raw[headerMap[canonical]] : undefined);
+    return {
+      rowIndex: idx,
+      url: cleanStr(g('url')),
+      title: cleanStr(g('title')),
+      metaDescription: cleanStr(g('metaDescription')),
+      publishedMs: toDateMs(g('published')),
+      updatedMs: toDateMs(g('updated')),
+      wordCount: toNumber(g('wordCount')),
+      pageviews: toNumber(g('pageviews')),
+    };
+  });
+
+  // asOf: explicit option, else the latest date seen anywhere in the export.
+  let asOfMs = options.asOf ? toDateMs(options.asOf) : null;
+  if (asOfMs === null) {
+    for (const it of normalized) {
+      for (const ms of [it.publishedMs, it.updatedMs]) {
+        if (ms !== null && (asOfMs === null || ms > asOfMs)) asOfMs = ms;
+      }
+    }
+  }
+  if (asOfMs === null) asOfMs = Date.UTC(1970, 0, 1); // no dates at all; decisions will route to review
+
+  // duplicate detection (a taxonomy/QA signal migrations must resolve)
+  const titleIndex = new Map();
+  const urlIndex = new Map();
+  for (const it of normalized) {
+    if (it.title) {
+      const k = it.title.toLowerCase();
+      (titleIndex.get(k) || titleIndex.set(k, []).get(k)).push(it.url || `(row ${it.rowIndex})`);
+    }
+    if (it.url) {
+      const k = it.url.toLowerCase();
+      (urlIndex.get(k) || urlIndex.set(k, []).get(k)).push(it.rowIndex);
+    }
+  }
+  const dupTitles = [...titleIndex.entries()].filter(([, v]) => v.length > 1).map(([title, urls]) => ({ title, count: urls.length, urls }));
+  const dupUrls = [...urlIndex.entries()].filter(([, v]) => v.length > 1).map(([url, rows2]) => ({ url, count: rows2.length, rows: rows2 }));
+
+  // per-item decision + gaps
+  const items = normalized.map((it) => {
+    const { decision, reasons, staleDays, confidence } = decide(it, asOfMs, T);
+    const gaps = metadataGaps(it, T);
+    return {
+      url: it.url,
+      title: it.title,
+      decision,
+      confidence,
+      reasons,
+      staleDays,
+      pageviews: it.pageviews,
+      wordCount: it.wordCount,
+      metadataGaps: gaps,
+      redirectTarget: decision === 'retire' ? null : undefined, // null = decision needed, not omission
+    };
+  });
+
+  const counts = { total: items.length, keep: 0, update: 0, retire: 0, review: 0 };
+  for (const it of items) counts[it.decision] += 1;
+
+  // flat metadata-gap list for quick triage, ordered fail → warn → info
+  const sevOrder = { fail: 0, warn: 1, info: 2 };
+  const flatGaps = [];
+  for (const it of items) {
+    for (const g of it.metadataGaps) flatGaps.push({ url: it.url || '(no url)', ...g });
+  }
+  flatGaps.sort((a, b) => sevOrder[a.severity] - sevOrder[b.severity]);
+
+  // migration readiness
+  const redirectLedger = items
+    .filter((it) => it.decision === 'retire')
+    .map((it) => ({ from: it.url || '(no url)', to: null, status: 301, resolved: false, note: 'redirect target UNRESOLVED — a decision, not an omission' }));
+
+  const itemsWithGaps = items.filter((it) => it.metadataGaps.length > 0).length;
+  const blockingGaps = flatGaps.filter((g) => g.severity === 'fail').length;
+
+  const fieldCoverage = fieldCoverageReport(normalized);
+
+  const blockers = [];
+  if (redirectLedger.length > 0) blockers.push(`${redirectLedger.length} retire decision(s) need a redirect target before launch`);
+  if (blockingGaps > 0) blockers.push(`${blockingGaps} blocking metadata gap(s) (missing url/title/date)`);
+  if (dupUrls.length > 0) blockers.push(`${dupUrls.length} duplicate URL(s) in the export`);
+  if (counts.review > 0) blockers.push(`${counts.review} item(s) lack the data to auto-decide (manual review)`);
+  for (const [field, cov] of Object.entries(fieldCoverage)) {
+    if (cov.present === 0) blockers.push(`source field "${field}" is absent from the export — no destination to map`);
+  }
+
+  // readiness score: start at 100, subtract weighted penalties, floor at 0.
+  const penalty =
+    redirectLedger.length * 2 +
+    blockingGaps * 3 +
+    dupUrls.length * 4 +
+    counts.review * 2 +
+    (flatGaps.length - blockingGaps) * 1;
+  const readinessScore = Math.max(0, Math.min(100, 100 - Math.round((penalty / Math.max(1, items.length * 6)) * 100)));
+
+  // unmapped source headers (a field with no canonical home is a mapping decision)
+  const mappedHeaders = new Set(Object.values(headerMap));
+  const unmappedFields = Object.keys(arr[0]).filter((h) => !mappedHeaders.has(h));
+
+  return {
+    asOf: new Date(asOfMs).toISOString().slice(0, 10),
+    thresholds: T,
+    counts,
+    headerMap,
+    unmappedFields,
+    items,
+    metadataGaps: flatGaps,
+    duplicates: { titles: dupTitles, urls: dupUrls },
+    migrationReadiness: {
+      totalUrls: items.length,
+      redirectsRequired: redirectLedger.length,
+      redirectLedger,
+      itemsWithGaps,
+      blockingGaps,
+      fieldCoverage,
+      readinessScore,
+      blockers,
+    },
+  };
+}
+
+function fieldCoverageReport(normalized) {
+  const fields = ['url', 'title', 'metaDescription', 'publishedMs', 'updatedMs', 'wordCount', 'pageviews'];
+  const report = {};
+  for (const f of fields) {
+    let present = 0;
+    for (const it of normalized) {
+      const v = it[f];
+      const has = f === 'publishedMs' || f === 'updatedMs' || f === 'wordCount' || f === 'pageviews' ? v !== null : v !== '';
+      if (has) present += 1;
+    }
+    report[f] = { present, total: normalized.length, pct: Math.round((present / normalized.length) * 100) };
+  }
+  return report;
+}
+
+// Convenience: parse raw text and build the inventory in one call.
+export function inventoryFromText(text, hint = null, options = {}) {
+  const rows = parseExport(text, hint);
+  return buildInventory(rows, options);
+}

package/agent-cores/content-ops/src/parse.mjs

@@ -0,0 +1,121 @@
+// Dependency-free parsing for CMS exports. Two input shapes:
+//   - CSV (RFC-4180-ish: quoted fields, escaped quotes "", commas/newlines inside quotes)
+//   - JSON (an array of row objects, or { rows: [...] } / { items: [...] })
+//
+// Both normalize to an array of raw row objects keyed by the export's own headers.
+// Field-name normalization to the canonical schema happens in inventory.mjs.
+
+// --- CSV ---
+
+// State-machine parser. Handles quoted fields, "" escapes, and CR/LF inside quotes.
+// Returns an array of arrays (rows of string cells).
+export function parseCsv(text) {
+  const rows = [];
+  let row = [];
+  let field = '';
+  let inQuotes = false;
+  let i = 0;
+  const n = text.length;
+  let started = false; // did we see any char on this line?
+
+  const pushField = () => {
+    row.push(field);
+    field = '';
+  };
+  const pushRow = () => {
+    row.push(field);
+    field = '';
+    rows.push(row);
+    row = [];
+    started = false;
+  };
+
+  while (i < n) {
+    const c = text[i];
+
+    if (inQuotes) {
+      if (c === '"') {
+        if (text[i + 1] === '"') {
+          field += '"';
+          i += 2;
+          continue;
+        }
+        inQuotes = false;
+        i += 1;
+        continue;
+      }
+      field += c;
+      i += 1;
+      continue;
+    }
+
+    if (c === '"') {
+      inQuotes = true;
+      started = true;
+      i += 1;
+      continue;
+    }
+    if (c === ',') {
+      pushField();
+      started = true;
+      i += 1;
+      continue;
+    }
+    if (c === '\r') {
+      // swallow CR; handle the LF (or lone CR) as the line break
+      if (text[i + 1] === '\n') i += 1;
+      pushRow();
+      i += 1;
+      continue;
+    }
+    if (c === '\n') {
+      pushRow();
+      i += 1;
+      continue;
+    }
+    field += c;
+    started = true;
+    i += 1;
+  }
+
+  // flush trailing field/row if the file doesn't end in a newline
+  if (field.length > 0 || row.length > 0 || started) pushRow();
+
+  return rows;
+}
+
+// CSV text -> array of row objects keyed by header. Blank lines are dropped.
+export function csvToRecords(text) {
+  const rows = parseCsv(text).filter((r) => !(r.length === 1 && r[0].trim() === ''));
+  if (rows.length === 0) return [];
+  const headers = rows[0].map((h) => h.trim());
+  const records = [];
+  for (let r = 1; r < rows.length; r += 1) {
+    const cells = rows[r];
+    const obj = {};
+    for (let c = 0; c < headers.length; c += 1) {
+      obj[headers[c]] = cells[c] !== undefined ? cells[c] : '';
+    }
+    records.push(obj);
+  }
+  return records;
+}
+
+// --- dispatch ---
+
+// Auto-detect JSON vs CSV from the text and return raw row objects.
+export function parseExport(text, hint = null) {
+  const trimmed = text.replace(/^/, '').trimStart(); // strip BOM
+  const looksJson = hint === 'json' || (hint !== 'csv' && (trimmed.startsWith('[') || trimmed.startsWith('{')));
+
+  if (looksJson) {
+    const data = JSON.parse(trimmed);
+    if (Array.isArray(data)) return data;
+    if (data && Array.isArray(data.rows)) return data.rows;
+    if (data && Array.isArray(data.items)) return data.items;
+    if (data && Array.isArray(data.records)) return data.records;
+    throw new Error('JSON export must be an array, or an object with a rows/items/records array.');
+  }
+
+  return csvToRecords(text);
+}

package/agent-cores/content-ops/test/fixtures/cms-export.csv

@@ -0,0 +1,13 @@
+url,title,published,last-updated,word count,pageviews,meta description,author
+https://acme.example/old-news-2019,Old News From 2019,2019-01-10,2019-01-15,800,12,An announcement from early 2019 that has not been touched since.,J. Smith
+https://acme.example/tags/misc,Miscellaneous Tag Archive Page,2025-12-20,2026-01-01,90,8,,A. Bot
+https://acme.example/pricing-guide,Pricing Guide,2022-06-01,2024-03-01,2400,4200,"This is a deliberately overlong meta description written to exceed the recommended one hundred and sixty character ceiling so the engine flags it as too long for a snippet.",M. Lee
+https://acme.example/,Acme — Home,2020-01-01,2026-06-15,650,9000,"The homepage of Acme, updated recently and carrying the most traffic on the whole site.",Team
+https://acme.example/blog/mid-article,A Mid-Traffic Article Worth Refreshing,2021-04-01,2025-05-01,1200,30,A long-form article with modest traffic that has aged past a year without a refresh.,J. Smith
+https://acme.example/blog/new-post,A Freshly Published Post,2026-06-18,2026-06-20,900,20,A brand new post that has not accumulated traffic yet but is current.,A. Bot
+https://acme.example/docs/reference,The Complete and Exhaustive Reference Documentation for Every Single Public API Endpoint,2019-05-01,2023-01-01,5000,200,Long-lived reference doc with steady moderate traffic that has decayed past two years.,M. Lee
+https://acme.example/guide,Complete Guide,2024-01-01,2026-03-01,1800,300,A moderately trafficked guide kept current within the last few months of the reporting window.,Team
+https://acme.example/reports/quarterly,Quarterly Report Archive,2025-01-01,2025-01-01,700,,An archived report where the traffic column was never populated by the analytics export.,J. Smith
+https://acme.example/team/jane-doe,Jane Doe,2026-04-01,2026-05-01,400,600,,Team
+https://acme.example/guide-legacy,Complete Guide,2023-10-01,2026-02-01,1500,150,A second page sharing the exact same title as the primary guide — a taxonomy collision.,Team
+https://acme.example/orphan,Orphaned Page With No Dates,,,200,5,A page whose export row is missing both the published and the last-updated date fields.,A. Bot

package/agent-cores/content-ops/test/fixtures/cms-export.json

@@ -0,0 +1,31 @@
+{
+  "rows": [
+    {
+      "url": "https://acme.example/legacy-faq",
+      "title": "Legacy FAQ",
+      "published": "2018-03-01",
+      "lastUpdated": "2018-09-01",
+      "wordCount": 1100,
+      "pageviews": 7,
+      "metaDescription": "An old FAQ nobody has updated in years and almost nobody reads."
+    },
+    {
+      "url": "https://acme.example/flagship",
+      "title": "Flagship Product Overview",
+      "published": "2021-01-01",
+      "lastUpdated": "2023-06-01",
+      "wordCount": 2200,
+      "pageviews": 8800,
+      "metaDescription": "The overview page for the flagship product, high traffic but overdue for a refresh."
+    },
+    {
+      "url": "https://acme.example/whats-new",
+      "title": "What's New",
+      "published": "2026-05-01",
+      "lastUpdated": "2026-05-15",
+      "wordCount": 500,
+      "pageviews": 640,
+      "metaDescription": "The most recent changelog entry with strong traffic and a current date."
+    }
+  ]
+}