weftens 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +138 -12
- package/agent-cores/amazon-ppc/package.json +17 -0
- package/agent-cores/amazon-ppc/src/cli.mjs +66 -0
- package/agent-cores/amazon-ppc/src/optimize.mjs +272 -0
- package/agent-cores/amazon-ppc/src/parse.mjs +153 -0
- package/agent-cores/amazon-ppc/test/fixtures/search-term-report.csv +8 -0
- package/agent-cores/amazon-ppc/test/fixtures/search-term-report.json +5 -0
- package/agent-cores/amazon-ppc/test/optimize.test.mjs +161 -0
- package/agent-cores/content-ops/package.json +17 -0
- package/agent-cores/content-ops/src/cli.mjs +51 -0
- package/agent-cores/content-ops/src/inventory.mjs +343 -0
- package/agent-cores/content-ops/src/parse.mjs +121 -0
- package/agent-cores/content-ops/test/fixtures/cms-export.csv +13 -0
- package/agent-cores/content-ops/test/fixtures/cms-export.json +31 -0
- package/agent-cores/content-ops/test/inventory.test.mjs +177 -0
- package/agent-cores/meta-ads/package.json +16 -0
- package/agent-cores/meta-ads/src/analyze.mjs +166 -0
- package/agent-cores/meta-ads/src/cli.mjs +39 -0
- package/agent-cores/meta-ads/src/parse.mjs +123 -0
- package/agent-cores/meta-ads/test/analyze.test.js +72 -0
- package/agent-cores/meta-ads/test/fixtures/meta-ads.csv +6 -0
- package/agent-cores/paid-search/package.json +17 -0
- package/agent-cores/paid-search/src/analyze.mjs +303 -0
- package/agent-cores/paid-search/src/cli.mjs +57 -0
- package/agent-cores/paid-search/src/parse.mjs +158 -0
- package/agent-cores/paid-search/test/analyze.test.mjs +120 -0
- package/agent-cores/paid-search/test/fixtures/campaigns.csv +7 -0
- package/agent-cores/paid-search/test/fixtures/sample.json +9 -0
- package/agent-cores/paid-search/test/fixtures/search-terms.csv +11 -0
- package/agent-cores/paid-search/test/parse.test.mjs +56 -0
- package/agent-cores/seo-content/package.json +17 -0
- package/agent-cores/seo-content/src/cli.mjs +41 -0
- package/agent-cores/seo-content/src/parse.mjs +141 -0
- package/agent-cores/seo-content/src/strategy.mjs +266 -0
- package/agent-cores/seo-content/test/fixtures/gsc-sample.csv +11 -0
- package/agent-cores/seo-content/test/parse.test.mjs +59 -0
- package/agent-cores/seo-content/test/strategy.test.mjs +108 -0
- package/agent-cores/seo-technical/package.json +17 -0
- package/agent-cores/seo-technical/src/audit.mjs +233 -0
- package/agent-cores/seo-technical/src/cli.mjs +48 -0
- package/agent-cores/seo-technical/src/fetch.mjs +51 -0
- package/agent-cores/seo-technical/test/audit.test.mjs +73 -0
- package/agent-cores/seo-technical/test/fixtures/sample.html +22 -0
- package/agent-cores/supply-verify/package.json +17 -0
- package/agent-cores/supply-verify/src/cli.mjs +44 -0
- package/agent-cores/supply-verify/src/verify.mjs +303 -0
- package/agent-cores/supply-verify/test/fixtures/placements.csv +8 -0
- package/agent-cores/supply-verify/test/fixtures/placements.json +4 -0
- package/agent-cores/supply-verify/test/verify.test.mjs +148 -0
- package/agents/agent2060.md +99 -0
- package/agents/amazon-ppc.md +42 -0
- package/agents/claims-content.md +35 -0
- package/agents/content-ops.md +46 -0
- package/agents/content-strategy.md +13 -0
- package/agents/disclosure-qc.md +13 -0
- package/agents/feed-optimizer.md +13 -0
- package/agents/growth-analyst.md +24 -0
- package/agents/hilbert.md +55 -0
- package/agents/ideator.md +13 -0
- package/agents/lippmann.md +68 -0
- package/agents/paid-search.md +35 -0
- package/agents/pio.md +34 -0
- package/agents/researcher.md +13 -0
- package/agents/richy.md +72 -0
- package/agents/script-editor.md +13 -0
- package/agents/seo-content.md +57 -0
- package/agents/seo-technical.md +53 -0
- package/agents/showrunner.md +13 -0
- package/agents/social-voice.md +38 -0
- package/agents/supply-verify.md +38 -0
- package/agents/video-editor.md +13 -0
- package/agents/writer.md +45 -0
- package/bin/weftens.js +226 -0
- package/content-org/.preflight/telemetry.jsonl +6 -0
- package/content-org/briefs/BRIEFS.md +47 -0
- package/content-org/config.mjs +78 -0
- package/content-org/cost.mjs +36 -0
- package/content-org/lead.js +64 -0
- package/content-org/metrics.mjs +166 -0
- package/content-org/orchestrator.mjs +246 -0
- package/content-org/package.json +14 -0
- package/content-org/producer.mjs +59 -0
- package/content-org/publish.mjs +34 -0
- package/content-org/review.mjs +32 -0
- package/content-org/studio/README.md +27 -0
- package/content-org/studio/assemble.sh +48 -0
- package/content-org/test.mjs +110 -0
- package/index.js +32 -0
- package/package.json +49 -10
- package/providers.json.example +9 -0
- package/src/agent-router.js +124 -0
- package/src/agent.js +123 -0
- package/src/invoke.js +129 -0
- package/src/pio-client.js +142 -0
- package/src/provider.js +81 -0
- package/src/registry.js +160 -0
- package/src/richy-client.js +216 -0
- package/src/router.js +34 -0
- package/src/synthesize.js +111 -0
- package/src/weftens.js +61 -0
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
[
|
|
2
|
+
{ "Customer Search Term": "garlic press", "Campaign Name": "Auto - Garlic Press", "Match Type": "auto", "Impressions": 5000, "Clicks": 40, "Spend": 20.0, "7 Day Total Sales": 0, "7 Day Total Orders (#)": 0 },
|
|
3
|
+
{ "Customer Search Term": "stainless garlic press", "Campaign Name": "Auto - Garlic Press", "Match Type": "auto", "Impressions": 3200, "Clicks": 30, "Spend": 15.0, "7 Day Total Sales": 120.0, "7 Day Total Orders (#)": 8 },
|
|
4
|
+
{ "Customer Search Term": "professional garlic press", "Campaign Name": "Exact - Pro Press", "Match Type": "exact", "Impressions": 6000, "Clicks": 50, "Spend": 10.0, "7 Day Total Sales": 200.0, "7 Day Total Orders (#)": 15 }
|
|
5
|
+
]
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
// Offline tests — run with `node --test`, zero install. Asserts the optimizer
|
|
2
|
+
// computes real, specific proposals from a known search-term report with seeded
|
|
3
|
+
// issues. If these pass, the core is doing deterministic arithmetic, not echoing.
|
|
4
|
+
|
|
5
|
+
import { test } from 'node:test';
|
|
6
|
+
import assert from 'node:assert/strict';
|
|
7
|
+
import { readFileSync } from 'node:fs';
|
|
8
|
+
import { fileURLToPath } from 'node:url';
|
|
9
|
+
import { dirname, join } from 'node:path';
|
|
10
|
+
import { optimize } from '../src/optimize.mjs';
|
|
11
|
+
import { normalizeRows, parseDelimited } from '../src/parse.mjs';
|
|
12
|
+
|
|
13
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
14
|
+
const csv = readFileSync(join(__dirname, 'fixtures', 'search-term-report.csv'), 'utf8');
|
|
15
|
+
const jsonText = readFileSync(join(__dirname, 'fixtures', 'search-term-report.json'), 'utf8');
|
|
16
|
+
|
|
17
|
+
const CFG = { targetAcos: 25, margin: 35 };
|
|
18
|
+
const run = () => optimize(csv, CFG);
|
|
19
|
+
const findNeg = (r, term) => r.negatives.find((n) => n.term === term);
|
|
20
|
+
const findPromo = (r, term) => r.promotions.find((p) => p.term === term);
|
|
21
|
+
const findRebid = (r, term) => r.rebids.find((b) => b.term === term);
|
|
22
|
+
|
|
23
|
+
test('CSV parser strips $/%/commas and normalizes messy Amazon headers', () => {
|
|
24
|
+
const rows = normalizeRows(csv);
|
|
25
|
+
assert.equal(rows.length, 7);
|
|
26
|
+
const r = rows.find((x) => x.searchTerm === 'garlic mincer tool');
|
|
27
|
+
assert.equal(r.spend, 22.0);
|
|
28
|
+
assert.equal(r.sales, 40.0);
|
|
29
|
+
assert.equal(r.orders, 2);
|
|
30
|
+
assert.equal(r.clicks, 25);
|
|
31
|
+
assert.equal(r.cpc, 0.88);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test('CSV tokenizer handles a quoted field containing a comma', () => {
|
|
35
|
+
const rows = parseDelimited('a,b\n"hello, world",2\n');
|
|
36
|
+
assert.deepEqual(rows[1], ['hello, world', '2']);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test('negates zero-order waste terms and estimates spend stopped', () => {
|
|
40
|
+
const r = run();
|
|
41
|
+
assert.equal(r.negatives.length, 2);
|
|
42
|
+
const gp = findNeg(r, 'garlic press');
|
|
43
|
+
assert.ok(gp, 'garlic press negated');
|
|
44
|
+
assert.equal(gp.spendStopped, 20.0);
|
|
45
|
+
assert.match(gp.reason, /40 clicks, 0 orders/);
|
|
46
|
+
assert.ok(findNeg(r, 'cheap kitchen gadget'), '12-click zero-order term negated');
|
|
47
|
+
assert.equal(r.summary.estSpendStopped, 26.5);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test('does NOT negate a low-signal term (3 clicks, $1.50, 0 orders) — it holds', () => {
|
|
51
|
+
const r = run();
|
|
52
|
+
assert.equal(findNeg(r, 'garlic'), undefined);
|
|
53
|
+
const held = r.held.find((h) => h.term === 'garlic');
|
|
54
|
+
assert.ok(held && held.action === 'hold', 'low-signal term held, not negated');
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test('promotes converting non-exact winners with the margin math attached', () => {
|
|
58
|
+
const r = run();
|
|
59
|
+
assert.equal(r.promotions.length, 2);
|
|
60
|
+
const p = findPromo(r, 'stainless garlic press');
|
|
61
|
+
assert.ok(p, 'winner promoted');
|
|
62
|
+
assert.equal(p.newMatchType, 'exact');
|
|
63
|
+
assert.equal(p.acos, 12.5);
|
|
64
|
+
// rev/click $4.00 * 25% target = $1.00 ideal, capped to 1.5x CPC($0.50) = $0.75
|
|
65
|
+
assert.equal(p.startingBid, 0.75);
|
|
66
|
+
assert.equal(p.capped, true);
|
|
67
|
+
assert.equal(p.negateInSource, true);
|
|
68
|
+
assert.match(p.marginMath, /\$4 rev\/click x 25% target ACOS = \$1 ideal bid/);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test('promotions are ranked by sales (biggest winner first)', () => {
|
|
72
|
+
const r = run();
|
|
73
|
+
assert.equal(r.promotions[0].term, 'stainless garlic press'); // $120
|
|
74
|
+
assert.equal(r.promotions[1].term, 'rocker garlic press'); // $90
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test('does NOT promote a high-ACOS term (55% > 25% target) — it re-bids down', () => {
|
|
78
|
+
const r = run();
|
|
79
|
+
assert.equal(findPromo(r, 'garlic mincer tool'), undefined);
|
|
80
|
+
const b = findRebid(r, 'garlic mincer tool');
|
|
81
|
+
assert.ok(b, 'high-ACOS converting term re-bid instead of promoted');
|
|
82
|
+
assert.equal(b.direction, 'down');
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test('re-bids DOWN an overpaying term, capped at -50%, flagged as losing vs break-even', () => {
|
|
86
|
+
const r = run();
|
|
87
|
+
const b = findRebid(r, 'garlic mincer tool');
|
|
88
|
+
assert.equal(b.oldBid, 0.88);
|
|
89
|
+
// ideal $0.40 but -50% cut floor is $0.44 -> capped
|
|
90
|
+
assert.equal(b.newBid, 0.44);
|
|
91
|
+
assert.equal(b.capped, true);
|
|
92
|
+
assert.match(b.why, /losing: ACOS above 35% break-even/);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test('re-bids UP an underpriced converter, capped at +100%', () => {
|
|
96
|
+
const r = run();
|
|
97
|
+
const b = findRebid(r, 'professional garlic press');
|
|
98
|
+
assert.ok(b && b.direction === 'up');
|
|
99
|
+
assert.equal(b.oldBid, 0.2);
|
|
100
|
+
// ideal $1.00 but +100% ceiling is $0.40 -> capped
|
|
101
|
+
assert.equal(b.newBid, 0.4);
|
|
102
|
+
assert.equal(b.capped, true);
|
|
103
|
+
assert.equal(b.changePct, 100);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test('re-bids ranked by magnitude of change', () => {
|
|
107
|
+
const r = run();
|
|
108
|
+
assert.equal(r.rebids[0].term, 'professional garlic press'); // +100%
|
|
109
|
+
assert.equal(r.rebids[1].term, 'garlic mincer tool'); // -50%
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test('computes correct account totals and blended ACOS', () => {
|
|
113
|
+
const r = run();
|
|
114
|
+
assert.equal(r.totals.terms, 7);
|
|
115
|
+
assert.equal(r.totals.clicks, 178);
|
|
116
|
+
assert.equal(r.totals.spend, 84.0);
|
|
117
|
+
assert.equal(r.totals.sales, 450.0);
|
|
118
|
+
assert.equal(r.totals.orders, 30);
|
|
119
|
+
assert.equal(r.totals.acos, 18.67); // 84/450
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test('inventory gate suppresses scaling on a low-inventory campaign', () => {
|
|
123
|
+
const r = optimize(csv, { ...CFG, lowInventoryCampaigns: ['Auto - Garlic Press'] });
|
|
124
|
+
// stainless + rocker garlic press were promotions from that auto campaign
|
|
125
|
+
assert.equal(findPromo(r, 'stainless garlic press'), undefined);
|
|
126
|
+
assert.equal(findPromo(r, 'rocker garlic press'), undefined);
|
|
127
|
+
const held = r.held.find((h) => h.term === 'stainless garlic press');
|
|
128
|
+
assert.ok(held && held.action === 'promote-suppressed', 'promotion suppressed by inventory gate');
|
|
129
|
+
assert.match(held.reason, /low inventory/);
|
|
130
|
+
// negatives from the same campaign still fire (a stock-out does not un-waste spend)
|
|
131
|
+
assert.ok(findNeg(r, 'garlic press'), 'negatives unaffected by inventory gate');
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test('margin-only config sets target below break-even (0.75 x margin)', () => {
|
|
135
|
+
const r = optimize(csv, { margin: 40 }); // no explicit targetAcos
|
|
136
|
+
assert.equal(r.config.margin, 40);
|
|
137
|
+
assert.equal(r.config.targetAcos, 30); // 40 * 0.75
|
|
138
|
+
assert.equal(r.config.breakEvenAcos, 40);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
test('deterministic: same input -> byte-identical output', () => {
|
|
142
|
+
const a = JSON.stringify(optimize(csv, CFG));
|
|
143
|
+
const b = JSON.stringify(optimize(csv, CFG));
|
|
144
|
+
assert.equal(a, b);
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
test('JSON report input parses through the same path as CSV', () => {
|
|
148
|
+
const r = optimize(jsonText, CFG);
|
|
149
|
+
assert.equal(r.totals.terms, 3);
|
|
150
|
+
assert.ok(findNeg(r, 'garlic press'));
|
|
151
|
+
assert.ok(findPromo(r, 'stainless garlic press'));
|
|
152
|
+
assert.ok(findRebid(r, 'professional garlic press'));
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test('every proposal is propose-only and carries its numbers', () => {
|
|
156
|
+
const r = run();
|
|
157
|
+
for (const n of r.negatives) assert.ok(typeof n.spendStopped === 'number' && n.reason);
|
|
158
|
+
for (const p of r.promotions) assert.ok(typeof p.startingBid === 'number' && p.marginMath);
|
|
159
|
+
for (const b of r.rebids) assert.ok(b.oldBid >= 0 && b.newBid >= 0 && b.why);
|
|
160
|
+
assert.match(r.summary.headline, /PROPOSE-ONLY/);
|
|
161
|
+
});
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@agent-cores/content-ops",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"private": true,
|
|
5
|
+
"type": "module",
|
|
6
|
+
"description": "Content-operations engine. Deterministic: parse a CMS export, compute keep/update/retire decisions with evidence, metadata gaps, and migration-readiness QA. No LLM in the core.",
|
|
7
|
+
"bin": {
|
|
8
|
+
"content-ops": "src/cli.mjs"
|
|
9
|
+
},
|
|
10
|
+
"scripts": {
|
|
11
|
+
"test": "node --test",
|
|
12
|
+
"inventory": "node src/cli.mjs"
|
|
13
|
+
},
|
|
14
|
+
"engines": {
|
|
15
|
+
"node": ">=20"
|
|
16
|
+
}
|
|
17
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Real CLI.
|
|
3
|
+
// content-ops <file.csv|file.json> [--asof YYYY-MM-DD] [--pretty]
|
|
4
|
+
// Reads a CMS export from disk, computes the inventory, prints JSON to stdout
|
|
5
|
+
// (machine-readable) and a one-line summary to stderr.
|
|
6
|
+
|
|
7
|
+
import { readFileSync } from 'node:fs';
|
|
8
|
+
import { inventoryFromText } from './inventory.mjs';
|
|
9
|
+
|
|
10
|
+
function parseArgs(argv) {
|
|
11
|
+
const args = { _: [] };
|
|
12
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
13
|
+
const a = argv[i];
|
|
14
|
+
if (a === '--asof') args.asof = argv[++i];
|
|
15
|
+
else if (a === '--pretty') args.pretty = true;
|
|
16
|
+
else if (a === '--csv') args.hint = 'csv';
|
|
17
|
+
else if (a === '--json') args.hint = 'json';
|
|
18
|
+
else args._.push(a);
|
|
19
|
+
}
|
|
20
|
+
return args;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function main() {
|
|
24
|
+
const args = parseArgs(process.argv.slice(2));
|
|
25
|
+
const file = args._[0];
|
|
26
|
+
if (!file) {
|
|
27
|
+
console.error('Usage: content-ops <file.csv|file.json> [--asof YYYY-MM-DD] [--pretty]');
|
|
28
|
+
process.exitCode = 1;
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const hint = args.hint ?? (file.toLowerCase().endsWith('.json') ? 'json' : file.toLowerCase().endsWith('.csv') ? 'csv' : null);
|
|
33
|
+
const text = readFileSync(file, 'utf8');
|
|
34
|
+
const result = inventoryFromText(text, hint, { asOf: args.asof });
|
|
35
|
+
|
|
36
|
+
const c = result.counts;
|
|
37
|
+
const m = result.migrationReadiness;
|
|
38
|
+
console.error(
|
|
39
|
+
`content-ops: ${c.total} URLs (asOf ${result.asOf}) — ` +
|
|
40
|
+
`keep ${c.keep} / update ${c.update} / retire ${c.retire} / review ${c.review} — ` +
|
|
41
|
+
`${m.redirectsRequired} redirect(s) needed, readiness ${m.readinessScore}/100`,
|
|
42
|
+
);
|
|
43
|
+
console.log(JSON.stringify(result, null, args.pretty ? 2 : 0));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
try {
|
|
47
|
+
main();
|
|
48
|
+
} catch (err) {
|
|
49
|
+
console.error(err instanceof Error ? err.message : String(err));
|
|
50
|
+
process.exitCode = 1;
|
|
51
|
+
}
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
// Deterministic content-operations engine. No LLM.
|
|
2
|
+
//
|
|
3
|
+
// Input: rows from a CMS export (url, title, published date, last-updated,
|
|
4
|
+
// word count, pageviews, meta description). Output: a per-URL inventory with a
|
|
5
|
+
// keep / update / retire decision and the evidence for it, a metadata-gap list,
|
|
6
|
+
// and a migration-readiness QA summary.
|
|
7
|
+
//
|
|
8
|
+
// Every decision is a pure function of the row's numbers against fixed thresholds,
|
|
9
|
+
// plus a fixed asOf date (defaults to the latest date in the export, so results
|
|
10
|
+
// never depend on the wall clock — same input always yields the same output).
|
|
11
|
+
|
|
12
|
+
import { parseExport } from './parse.mjs';
|
|
13
|
+
|
|
14
|
+
export const DEFAULT_THRESHOLDS = Object.freeze({
|
|
15
|
+
staleDays: 365, // not touched in a year → aging
|
|
16
|
+
veryStaleDays: 730, // not touched in two years → decayed
|
|
17
|
+
lowTrafficViews: 50, // at or below this over the reporting window → low value
|
|
18
|
+
highTrafficViews: 500, // at or above this → protect it, never retire
|
|
19
|
+
thinWords: 300, // below this → thin content (ROT candidate)
|
|
20
|
+
titleMin: 10,
|
|
21
|
+
titleMax: 65,
|
|
22
|
+
descMin: 50,
|
|
23
|
+
descMax: 160,
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
// --- field normalization: map an export's own headers onto the canonical schema ---
|
|
27
|
+
|
|
28
|
+
const FIELD_ALIASES = {
|
|
29
|
+
url: ['url', 'link', 'address', 'page', 'permalink', 'loc'],
|
|
30
|
+
title: ['title', 'page title', 'name', 'heading', 'h1'],
|
|
31
|
+
published: ['published', 'published date', 'publish date', 'publishdate', 'published_date', 'date', 'created', 'created date', 'first published'],
|
|
32
|
+
updated: ['updated', 'last updated', 'last-updated', 'lastupdated', 'last_updated', 'modified', 'last modified', 'lastmod', 'updated date'],
|
|
33
|
+
wordCount: ['word count', 'wordcount', 'words', 'word_count', 'length'],
|
|
34
|
+
pageviews: ['pageviews', 'page views', 'views', 'traffic', 'sessions', 'visits', 'page_views'],
|
|
35
|
+
metaDescription: ['meta description', 'metadescription', 'meta_description', 'description', 'meta desc', 'seo description', 'excerpt'],
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
function buildHeaderMap(sampleRow) {
|
|
39
|
+
const map = {};
|
|
40
|
+
const keys = Object.keys(sampleRow);
|
|
41
|
+
const lowered = keys.map((k) => k.trim().toLowerCase());
|
|
42
|
+
for (const [canonical, aliases] of Object.entries(FIELD_ALIASES)) {
|
|
43
|
+
for (const alias of aliases) {
|
|
44
|
+
const idx = lowered.indexOf(alias);
|
|
45
|
+
if (idx !== -1) {
|
|
46
|
+
map[canonical] = keys[idx];
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return map;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function toNumber(v) {
|
|
55
|
+
if (v === null || v === undefined) return null;
|
|
56
|
+
const s = String(v).trim().replace(/,/g, '');
|
|
57
|
+
if (s === '') return null;
|
|
58
|
+
const n = Number(s);
|
|
59
|
+
return Number.isFinite(n) ? n : null;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Parse a date to a UTC-midnight epoch (ms). Accepts YYYY-MM-DD and anything
|
|
63
|
+
// Date can parse; returns null on failure so "missing" is distinguishable from 0.
|
|
64
|
+
function toDateMs(v) {
|
|
65
|
+
if (v === null || v === undefined) return null;
|
|
66
|
+
const s = String(v).trim();
|
|
67
|
+
if (s === '') return null;
|
|
68
|
+
const iso = /^(\d{4})-(\d{2})-(\d{2})/.exec(s);
|
|
69
|
+
if (iso) {
|
|
70
|
+
return Date.UTC(Number(iso[1]), Number(iso[2]) - 1, Number(iso[3]));
|
|
71
|
+
}
|
|
72
|
+
const t = Date.parse(s);
|
|
73
|
+
return Number.isNaN(t) ? null : t;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const DAY = 86_400_000;
|
|
77
|
+
const daysBetween = (aMs, bMs) => Math.round((aMs - bMs) / DAY);
|
|
78
|
+
|
|
79
|
+
function cleanStr(v) {
|
|
80
|
+
if (v === null || v === undefined) return '';
|
|
81
|
+
return String(v).trim();
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// --- the decision core ---
|
|
85
|
+
|
|
86
|
+
function decide(item, asOfMs, T) {
|
|
87
|
+
const reasons = [];
|
|
88
|
+
const { pageviews, updatedMs, publishedMs, wordCount } = item;
|
|
89
|
+
|
|
90
|
+
// freshness is measured from last-update when present, else from publish
|
|
91
|
+
const freshMs = updatedMs ?? publishedMs;
|
|
92
|
+
const staleDays = freshMs !== null ? daysBetween(asOfMs, freshMs) : null;
|
|
93
|
+
|
|
94
|
+
const hasTraffic = pageviews !== null;
|
|
95
|
+
const hasFresh = staleDays !== null;
|
|
96
|
+
|
|
97
|
+
// If we can't judge value or freshness, we don't guess — we route to review.
|
|
98
|
+
if (!hasTraffic || !hasFresh) {
|
|
99
|
+
const missing = [];
|
|
100
|
+
if (!hasTraffic) missing.push('pageviews');
|
|
101
|
+
if (!hasFresh) missing.push('published/updated date');
|
|
102
|
+
reasons.push(`insufficient evidence to auto-decide (missing ${missing.join(' and ')})`);
|
|
103
|
+
return { decision: 'review', reasons, staleDays, confidence: 'low' };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const views = pageviews;
|
|
107
|
+
const words = wordCount; // may be null; only used in ROT branch guarded below
|
|
108
|
+
|
|
109
|
+
// 1) Protect valuable pages: high traffic is never retired.
|
|
110
|
+
if (views >= T.highTrafficViews) {
|
|
111
|
+
if (staleDays >= T.staleDays) {
|
|
112
|
+
reasons.push(`high traffic (${views} views) but not updated in ${staleDays} days → refresh, do not retire`);
|
|
113
|
+
return { decision: 'update', reasons, staleDays, confidence: 'high' };
|
|
114
|
+
}
|
|
115
|
+
reasons.push(`high traffic (${views} views) and fresh (updated ${staleDays} days ago)`);
|
|
116
|
+
return { decision: 'keep', reasons, staleDays, confidence: 'high' };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// 2) Low traffic: candidates for retirement or refresh.
|
|
120
|
+
if (views <= T.lowTrafficViews) {
|
|
121
|
+
if (staleDays >= T.veryStaleDays) {
|
|
122
|
+
reasons.push(`stale (${staleDays} days without update) and low traffic (${views} views) → ROT, retire with redirect`);
|
|
123
|
+
return { decision: 'retire', reasons, staleDays, confidence: 'high' };
|
|
124
|
+
}
|
|
125
|
+
if (words !== null && words < T.thinWords) {
|
|
126
|
+
reasons.push(`thin content (${words} words) and low traffic (${views} views) → ROT, retire with redirect`);
|
|
127
|
+
return { decision: 'retire', reasons, staleDays, confidence: 'medium' };
|
|
128
|
+
}
|
|
129
|
+
if (staleDays >= T.staleDays) {
|
|
130
|
+
reasons.push(`low traffic (${views} views) and aging (${staleDays} days) but not yet ROT → refresh and reassess`);
|
|
131
|
+
return { decision: 'update', reasons, staleDays, confidence: 'medium' };
|
|
132
|
+
}
|
|
133
|
+
reasons.push(`low traffic (${views} views) but recent (${staleDays} days) → keep and monitor`);
|
|
134
|
+
return { decision: 'keep', reasons, staleDays, confidence: 'low' };
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// 3) Mid traffic: keep unless it has decayed past very-stale.
|
|
138
|
+
if (staleDays >= T.veryStaleDays) {
|
|
139
|
+
reasons.push(`moderate traffic (${views} views) but decayed (${staleDays} days without update) → refresh`);
|
|
140
|
+
return { decision: 'update', reasons, staleDays, confidence: 'medium' };
|
|
141
|
+
}
|
|
142
|
+
reasons.push(`moderate traffic (${views} views) and acceptable freshness (${staleDays} days)`);
|
|
143
|
+
return { decision: 'keep', reasons, staleDays, confidence: 'medium' };
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// --- metadata gap checks (mechanical, per item) ---
|
|
147
|
+
|
|
148
|
+
function metadataGaps(item, T) {
|
|
149
|
+
const gaps = [];
|
|
150
|
+
if (item.title === '') gaps.push({ field: 'title', issue: 'missing', severity: 'fail' });
|
|
151
|
+
else if (item.title.length < T.titleMin) gaps.push({ field: 'title', issue: `too short (${item.title.length} chars, aim ${T.titleMin}-${T.titleMax})`, severity: 'warn' });
|
|
152
|
+
else if (item.title.length > T.titleMax) gaps.push({ field: 'title', issue: `too long (${item.title.length} chars, aim ${T.titleMin}-${T.titleMax})`, severity: 'warn' });
|
|
153
|
+
|
|
154
|
+
if (item.metaDescription === '') gaps.push({ field: 'metaDescription', issue: 'missing', severity: 'warn' });
|
|
155
|
+
else if (item.metaDescription.length < T.descMin) gaps.push({ field: 'metaDescription', issue: `too short (${item.metaDescription.length} chars, aim ${T.descMin}-${T.descMax})`, severity: 'warn' });
|
|
156
|
+
else if (item.metaDescription.length > T.descMax) gaps.push({ field: 'metaDescription', issue: `too long (${item.metaDescription.length} chars, aim ${T.descMin}-${T.descMax})`, severity: 'warn' });
|
|
157
|
+
|
|
158
|
+
if (item.url === '') gaps.push({ field: 'url', issue: 'missing', severity: 'fail' });
|
|
159
|
+
|
|
160
|
+
// data-completeness gaps that block an evidence-based decision
|
|
161
|
+
if (item.publishedMs === null && item.updatedMs === null) gaps.push({ field: 'date', issue: 'no published or updated date', severity: 'fail' });
|
|
162
|
+
if (item.pageviews === null) gaps.push({ field: 'pageviews', issue: 'missing traffic data', severity: 'warn' });
|
|
163
|
+
if (item.wordCount === null) gaps.push({ field: 'wordCount', issue: 'missing word count', severity: 'info' });
|
|
164
|
+
|
|
165
|
+
return gaps;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// --- main entry ---
|
|
169
|
+
|
|
170
|
+
export function buildInventory(rows, options = {}) {
|
|
171
|
+
const T = { ...DEFAULT_THRESHOLDS, ...(options.thresholds || {}) };
|
|
172
|
+
const arr = Array.isArray(rows) ? rows : [];
|
|
173
|
+
|
|
174
|
+
if (arr.length === 0) {
|
|
175
|
+
return {
|
|
176
|
+
asOf: options.asOf ?? null,
|
|
177
|
+
thresholds: T,
|
|
178
|
+
counts: { total: 0, keep: 0, update: 0, retire: 0, review: 0 },
|
|
179
|
+
unmappedFields: [],
|
|
180
|
+
headerMap: {},
|
|
181
|
+
items: [],
|
|
182
|
+
metadataGaps: [],
|
|
183
|
+
duplicates: { titles: [], urls: [] },
|
|
184
|
+
migrationReadiness: {
|
|
185
|
+
totalUrls: 0,
|
|
186
|
+
redirectsRequired: 0,
|
|
187
|
+
redirectLedger: [],
|
|
188
|
+
itemsWithGaps: 0,
|
|
189
|
+
blockingGaps: 0,
|
|
190
|
+
fieldCoverage: {},
|
|
191
|
+
readinessScore: 0,
|
|
192
|
+
blockers: ['export is empty — nothing to inventory'],
|
|
193
|
+
},
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const headerMap = buildHeaderMap(arr[0]);
|
|
198
|
+
|
|
199
|
+
// normalize each row to the canonical schema
|
|
200
|
+
const normalized = arr.map((raw, idx) => {
|
|
201
|
+
const g = (canonical) => (headerMap[canonical] !== undefined ? raw[headerMap[canonical]] : undefined);
|
|
202
|
+
return {
|
|
203
|
+
rowIndex: idx,
|
|
204
|
+
url: cleanStr(g('url')),
|
|
205
|
+
title: cleanStr(g('title')),
|
|
206
|
+
metaDescription: cleanStr(g('metaDescription')),
|
|
207
|
+
publishedMs: toDateMs(g('published')),
|
|
208
|
+
updatedMs: toDateMs(g('updated')),
|
|
209
|
+
wordCount: toNumber(g('wordCount')),
|
|
210
|
+
pageviews: toNumber(g('pageviews')),
|
|
211
|
+
};
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
// asOf: explicit option, else the latest date seen anywhere in the export.
|
|
215
|
+
let asOfMs = options.asOf ? toDateMs(options.asOf) : null;
|
|
216
|
+
if (asOfMs === null) {
|
|
217
|
+
for (const it of normalized) {
|
|
218
|
+
for (const ms of [it.publishedMs, it.updatedMs]) {
|
|
219
|
+
if (ms !== null && (asOfMs === null || ms > asOfMs)) asOfMs = ms;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
if (asOfMs === null) asOfMs = Date.UTC(1970, 0, 1); // no dates at all; decisions will route to review
|
|
224
|
+
|
|
225
|
+
// duplicate detection (a taxonomy/QA signal migrations must resolve)
|
|
226
|
+
const titleIndex = new Map();
|
|
227
|
+
const urlIndex = new Map();
|
|
228
|
+
for (const it of normalized) {
|
|
229
|
+
if (it.title) {
|
|
230
|
+
const k = it.title.toLowerCase();
|
|
231
|
+
(titleIndex.get(k) || titleIndex.set(k, []).get(k)).push(it.url || `(row ${it.rowIndex})`);
|
|
232
|
+
}
|
|
233
|
+
if (it.url) {
|
|
234
|
+
const k = it.url.toLowerCase();
|
|
235
|
+
(urlIndex.get(k) || urlIndex.set(k, []).get(k)).push(it.rowIndex);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
const dupTitles = [...titleIndex.entries()].filter(([, v]) => v.length > 1).map(([title, urls]) => ({ title, count: urls.length, urls }));
|
|
239
|
+
const dupUrls = [...urlIndex.entries()].filter(([, v]) => v.length > 1).map(([url, rows2]) => ({ url, count: rows2.length, rows: rows2 }));
|
|
240
|
+
|
|
241
|
+
// per-item decision + gaps
|
|
242
|
+
const items = normalized.map((it) => {
|
|
243
|
+
const { decision, reasons, staleDays, confidence } = decide(it, asOfMs, T);
|
|
244
|
+
const gaps = metadataGaps(it, T);
|
|
245
|
+
return {
|
|
246
|
+
url: it.url,
|
|
247
|
+
title: it.title,
|
|
248
|
+
decision,
|
|
249
|
+
confidence,
|
|
250
|
+
reasons,
|
|
251
|
+
staleDays,
|
|
252
|
+
pageviews: it.pageviews,
|
|
253
|
+
wordCount: it.wordCount,
|
|
254
|
+
metadataGaps: gaps,
|
|
255
|
+
redirectTarget: decision === 'retire' ? null : undefined, // null = decision needed, not omission
|
|
256
|
+
};
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
const counts = { total: items.length, keep: 0, update: 0, retire: 0, review: 0 };
|
|
260
|
+
for (const it of items) counts[it.decision] += 1;
|
|
261
|
+
|
|
262
|
+
// flat metadata-gap list for quick triage, ordered fail → warn → info
|
|
263
|
+
const sevOrder = { fail: 0, warn: 1, info: 2 };
|
|
264
|
+
const flatGaps = [];
|
|
265
|
+
for (const it of items) {
|
|
266
|
+
for (const g of it.metadataGaps) flatGaps.push({ url: it.url || '(no url)', ...g });
|
|
267
|
+
}
|
|
268
|
+
flatGaps.sort((a, b) => sevOrder[a.severity] - sevOrder[b.severity]);
|
|
269
|
+
|
|
270
|
+
// migration readiness
|
|
271
|
+
const redirectLedger = items
|
|
272
|
+
.filter((it) => it.decision === 'retire')
|
|
273
|
+
.map((it) => ({ from: it.url || '(no url)', to: null, status: 301, resolved: false, note: 'redirect target UNRESOLVED — a decision, not an omission' }));
|
|
274
|
+
|
|
275
|
+
const itemsWithGaps = items.filter((it) => it.metadataGaps.length > 0).length;
|
|
276
|
+
const blockingGaps = flatGaps.filter((g) => g.severity === 'fail').length;
|
|
277
|
+
|
|
278
|
+
const fieldCoverage = fieldCoverageReport(normalized);
|
|
279
|
+
|
|
280
|
+
const blockers = [];
|
|
281
|
+
if (redirectLedger.length > 0) blockers.push(`${redirectLedger.length} retire decision(s) need a redirect target before launch`);
|
|
282
|
+
if (blockingGaps > 0) blockers.push(`${blockingGaps} blocking metadata gap(s) (missing url/title/date)`);
|
|
283
|
+
if (dupUrls.length > 0) blockers.push(`${dupUrls.length} duplicate URL(s) in the export`);
|
|
284
|
+
if (counts.review > 0) blockers.push(`${counts.review} item(s) lack the data to auto-decide (manual review)`);
|
|
285
|
+
for (const [field, cov] of Object.entries(fieldCoverage)) {
|
|
286
|
+
if (cov.present === 0) blockers.push(`source field "${field}" is absent from the export — no destination to map`);
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// readiness score: start at 100, subtract weighted penalties, floor at 0.
|
|
290
|
+
const penalty =
|
|
291
|
+
redirectLedger.length * 2 +
|
|
292
|
+
blockingGaps * 3 +
|
|
293
|
+
dupUrls.length * 4 +
|
|
294
|
+
counts.review * 2 +
|
|
295
|
+
(flatGaps.length - blockingGaps) * 1;
|
|
296
|
+
const readinessScore = Math.max(0, Math.min(100, 100 - Math.round((penalty / Math.max(1, items.length * 6)) * 100)));
|
|
297
|
+
|
|
298
|
+
// unmapped source headers (a field with no canonical home is a mapping decision)
|
|
299
|
+
const mappedHeaders = new Set(Object.values(headerMap));
|
|
300
|
+
const unmappedFields = Object.keys(arr[0]).filter((h) => !mappedHeaders.has(h));
|
|
301
|
+
|
|
302
|
+
return {
|
|
303
|
+
asOf: new Date(asOfMs).toISOString().slice(0, 10),
|
|
304
|
+
thresholds: T,
|
|
305
|
+
counts,
|
|
306
|
+
headerMap,
|
|
307
|
+
unmappedFields,
|
|
308
|
+
items,
|
|
309
|
+
metadataGaps: flatGaps,
|
|
310
|
+
duplicates: { titles: dupTitles, urls: dupUrls },
|
|
311
|
+
migrationReadiness: {
|
|
312
|
+
totalUrls: items.length,
|
|
313
|
+
redirectsRequired: redirectLedger.length,
|
|
314
|
+
redirectLedger,
|
|
315
|
+
itemsWithGaps,
|
|
316
|
+
blockingGaps,
|
|
317
|
+
fieldCoverage,
|
|
318
|
+
readinessScore,
|
|
319
|
+
blockers,
|
|
320
|
+
},
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function fieldCoverageReport(normalized) {
|
|
325
|
+
const fields = ['url', 'title', 'metaDescription', 'publishedMs', 'updatedMs', 'wordCount', 'pageviews'];
|
|
326
|
+
const report = {};
|
|
327
|
+
for (const f of fields) {
|
|
328
|
+
let present = 0;
|
|
329
|
+
for (const it of normalized) {
|
|
330
|
+
const v = it[f];
|
|
331
|
+
const has = f === 'publishedMs' || f === 'updatedMs' || f === 'wordCount' || f === 'pageviews' ? v !== null : v !== '';
|
|
332
|
+
if (has) present += 1;
|
|
333
|
+
}
|
|
334
|
+
report[f] = { present, total: normalized.length, pct: Math.round((present / normalized.length) * 100) };
|
|
335
|
+
}
|
|
336
|
+
return report;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
// Convenience: parse raw text and build the inventory in one call.
|
|
340
|
+
export function inventoryFromText(text, hint = null, options = {}) {
|
|
341
|
+
const rows = parseExport(text, hint);
|
|
342
|
+
return buildInventory(rows, options);
|
|
343
|
+
}
|