tadforce 0.3.0 → 0.4.1

package/src/commands/org.js

@@ -124,28 +124,92 @@ export async function retrieve(args) {
   const size = Number(args.batchSize || 12);
   const outDir = ctx.p(ctx.cfg.source.dir);
   fs.mkdirSync(outDir, { recursive: true });
+  // the CLI refuses to retrieve when a package directory of the project is missing
+  for (const d of (exists(path.join(ctx.project, 'sfdx-project.json')) ? JSON.parse(readText(path.join(ctx.project, 'sfdx-project.json'))).packageDirectories || [] : [])) fs.mkdirSync(path.join(ctx.project, d.path), { recursive: true });
   log.banner('retrieve');
   const failed = [];
-  const runBatch = async (batch, label) => {
+  const nameOf = (b) => /<name>([^<]+)<\/name>/.exec(b)[1];
+  const typeXml = (name, members) => `    <types>\n${members.map((m) => `        <members>${m}</members>`).join('\n')}\n        <name>${name}</name>\n    </types>`;
+  const runBatch = async (batch, label, minutes) => {
     const p = ctx.p('.snapshot', 'batch.xml');
     writeText(p, `<?xml version="1.0" encoding="UTF-8"?>\n<Package xmlns="http://soap.sforce.com/2006/04/metadata">\n${batch.join('\n')}\n    <version>${version}</version>\n</Package>\n`);
-    await sfJson(['project', 'retrieve', 'start', '--manifest', p, '--target-org', alias, '--output-dir', outDir, '--wait', String(args.wait || 60), '--ignore-conflicts'], { timeoutMs: 3600000, cwd: ctx.project });
-    log.ok(label);
+    await sfJson(['project', 'retrieve', 'start', '--manifest', p, '--target-org', alias, '--output-dir', outDir, '--wait', String(minutes), '--ignore-conflicts'], { timeoutMs: (minutes + 2) * 60000, cwd: ctx.project });
+    if (label) log.ok(label);
   };
-  for (let i = 0; i < types.length; i += size) {
-    const batch = types.slice(i, i + size);
-    const names = batch.map((b) => /<name>([^<]+)<\/name>/.exec(b)[1]);
+  // the CLI stopped waiting (« The client has timed out »): the job was too big for one retrieve
+  const timedOut = (e) => /timed out|MetadataTransferError/i.test(`${e.message} ${e.sf && e.sf.name}`);
+  // an error about the project itself fails every type the same way: stop at once, do not retry type by type
+  const projectError = (e) => /MissingPackageDirectory|InvalidProjectWorkspace|specified in sfdx-project\.json, does not exist|sfdx-project\.json/i.test(`${e.message} ${e.sf && e.sf.name}`);
+  // types the local Salesforce CLI does not know fail at once and never succeed: dropped, not retried
+  const unknownType = (e) => (/registry for id '([^']+)'|Missing metadata type definition[^']*'([^']+)'/.exec(e.message) || []).slice(1).find(Boolean);
+  // The types the document is built on come first, one by one, with up to 60 minutes each (--wait can only
+  // lengthen it). Too big for one retrieve (huge org, 39 MB / 10,000 files limit): its components are listed
+  // and retrieved by chunks of 100. The other types go by batches of 12 (30 minutes), then one by one (10 minutes).
+  const CORE = ['ApexClass', 'ApexTrigger', 'CustomObject', 'Flow', 'LightningComponentBundle', 'AuraDefinitionBundle', 'Profile', 'PermissionSet', 'PermissionSetGroup', 'FlexiPage', 'CustomApplication', 'CustomMetadata', 'OmniScript', 'OmniIntegrationProcedure', 'OmniDataTransform', 'OmniUiCard'];
+  // --core-wait shortens the first try (tests, impatient runs); the chunks always get the full 60 minutes
+  if (args.coreWait !== undefined && !/^\d+$/.test(String(args.coreWait))) throw new Error('--core-wait takes a number of minutes (0 or more).');
+  const coreWait = args.coreWait !== undefined ? Number(args.coreWait) : 60;
+  const chunkWait = 60;
+  const restWait = Math.min(Number(args.wait || 30), 30);
+  const byChunks = async (b, step) => {
+    const name = nameOf(b);
+    let listed;
+    try { listed = await sfJson(['org', 'list', 'metadata', '--metadata-type', name, '--target-org', alias], { timeoutMs: 600000 }); } catch (e) {
+      // a product this org does not have (OmniStudio, …): nothing to retrieve, as `*` would say
+      if (/INVALID_TYPE|Cannot use/i.test(e.message)) { log.info(`${step} ${name}: not used in this org`); return; }
+      throw e;
+    }
+    const explicit = [...b.matchAll(/<members>([^<]+)<\/members>/g)].map((m) => m[1]).filter((m) => m !== '*');
+    // the same components as `*`: everything but what installed packages bring (standard__ apps, force__ groups stay)
+    const own = (Array.isArray(listed) ? listed : []).filter((x) => !/^(installed|deprecated)/i.test(x.manageableState || ''));
+    const members = [...new Set([...own.map((x) => x.fullName), ...explicit])].sort();
+    if (!members.length) {
+      log.info(`${step} ${name}: nothing listed — one more try with the full wait`);
+      return runBatch([b], `${step} ${name}`, chunkWait);
+    }
+    log.info(`${step} ${name}: did not finish in ${coreWait} min — ${members.length} component(s), retrieved by chunks of 100`);
+    for (let i = 0; i < members.length; i += 100) await runBatch([typeXml(name, members.slice(i, i + 100))], `${step} ${name} ${Math.min(i + 100, members.length)}/${members.length}`, chunkWait);
+  };
+  const core = types.filter((b) => CORE.includes(nameOf(b)));
+  const rest = types.filter((b) => !CORE.includes(nameOf(b)));
+  const total = core.length + Math.ceil(rest.length / size);
+  let n = 0;
+  for (const b of core) {
+    const step = `[${++n}/${total}]`;
     try {
-      await runBatch(batch, `${names.join(', ')}`);
+      try { await runBatch([b], `${step} ${nameOf(b)}`, coreWait); } catch (e) { if (!timedOut(e)) throw e; await byChunks(b, step); }
     } catch (e) {
-      log.warn(`Batch failed (${e.message.slice(0, 160)}) — retrying type by type`);
-      for (const b of batch) {
-        const n = /<name>([^<]+)<\/name>/.exec(b)[1];
-        try { await runBatch([b], n); } catch (e2) { failed.push({ type: n, error: e2.message.slice(0, 300) }); log.warn(`${n}: skipped (${e2.message.slice(0, 120)})`); }
+      if (projectError(e)) throw new Error(`The Salesforce DX project cannot be used (${e.message.slice(0, 200)}). Fix sfdx-project.json or its package directories, then retrieve again.`);
+      const why = unknownType(e) ? 'not supported by this Salesforce CLI version' : e.message.slice(0, 300);
+      failed.push({ type: nameOf(b), error: why, core: true });
+      log.warn(`${step} ${nameOf(b)}: NOT retrieved (${why.slice(0, 120)}) — the document needs it`);
+    }
+  }
+  for (let i = 0; i < rest.length; i += size) {
+    let batch = rest.slice(i, i + size);
+    const step = `[${++n}/${total}]`;
+    for (let tries = 0; batch.length && tries < 3; tries++) {
+      try { await runBatch(batch, `${step} ${batch.map(nameOf).join(', ')}`, restWait); batch = []; } catch (e) {
+        if (projectError(e)) throw new Error(`The Salesforce DX project cannot be used (${e.message.slice(0, 200)}). Fix sfdx-project.json or its package directories, then retrieve again.`);
+        const bad = unknownType(e);
+        if (bad && batch.some((b) => nameOf(b) === bad)) {
+          failed.push({ type: bad, error: 'not supported by this Salesforce CLI version' });
+          log.warn(`${bad}: skipped (not supported by this Salesforce CLI version)`);
+          batch = batch.filter((b) => nameOf(b) !== bad);
+          continue;
+        }
+        log.warn(`${step} batch failed (${e.message.slice(0, 160)}) — retrying type by type`);
+        for (const b of batch) {
+          try { await runBatch([b], `${step} ${nameOf(b)}`, Math.min(10, restWait)); } catch (e2) { failed.push({ type: nameOf(b), error: e2.message.slice(0, 300) }); log.warn(`${nameOf(b)}: skipped (${e2.message.slice(0, 120)})`); }
+        }
+        batch = [];
       }
     }
   }
   writeJson(ctx.p('.snapshot', 'retrieve-report.json'), { failed });
+  // a type the document is built on is missing: not a ready snapshot
+  const lost = failed.filter((x) => x.core).map((x) => x.type);
+  if (lost.length) { log.fail(`Not retrieved, and the document needs them: ${lost.join(', ')}. Run the retrieve again (tadforce doctor --refresh).`); return 1; }
   log.ok(`Snapshot ready in ${path.relative(ctx.project, outDir)}${failed.length ? ` — ${failed.length} type(s) could not be retrieved (see .snapshot/retrieve-report.json)` : ''}.`);
   return 0;
 }

package/src/commands/spec.js

@@ -2,8 +2,8 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import YAML from 'yaml';
-import { exists, readText, readYaml } from '../util/fsx.js';
-import { loadWorkspace } from '../util/config.js';
+import { exists, readText, readYaml, walk, toPosix } from '../util/fsx.js';
+import { loadWorkspace, sourceFlags } from '../util/config.js';
 import { TEMPLATES } from '../util/paths.js';
 import { loadFacts } from '../facts/index.js';
 import { resolveConditionals, get } from '../build/directives.js';
@@ -33,13 +33,19 @@ export function specWithFacts(ctx, id) {
   const out = { id, ...s, factsExcerpt: {} };
   for (const p of s.facts || []) {
     if (p.startsWith('knowledge/')) {
+      // every file, sub-folders included (knowledge/inputs/specs/SFD.pdf), README excepted
       const dir = ctx.p(p);
-      out.factsExcerpt[p] = exists(dir) ? fs.readdirSync(dir) : [];
+      out.factsExcerpt[p] = exists(dir) ? walk(dir).map((x) => toPosix(path.relative(dir, x))).filter((x) => x !== 'README.md') : [];
       continue;
     }
     const v = get(facts, p) ?? get(facts.counts || {}, p);
     if (v !== undefined) out.factsExcerpt[p] = excerpt(v);
   }
+  // Functional specifications given by the project: evidence P2/P3 for the chapters that explain the why.
+  const projectSpecs = (ctx.cfg.inputs && ctx.cfg.inputs.specs) || [];
+  if (projectSpecs.length && /^(summary|purpose|context|functional|general)\./.test(id)) {
+    out.specifications = { read: projectSpecs.map((p) => toPosix(path.join('tadforce', p))), how: 'Read them before writing: they explain the purpose, the processes and the vocabulary. Cite them (« according to the specifications (DR-nn) ») — the metadata still decides what is actually built.' };
+  }
   if (s.modules) {
     const refs = s.modules.includes('all detected') ? (facts.modules && facts.modules.references) || [] : s.modules.filter((m) => (facts.modules && facts.modules.references || []).includes(m) || s.modules.length === 1);
     out.readModules = refs.map((r) => `skills/tadforce/references/modules/${r}.md`);
@@ -57,18 +63,25 @@ export async function spec(args) {
   return 0;
 }
 
-/** First section still to write (in reading order), among the sections active for this org. */
+// The order that works: the facts-heavy chapters first, the summary last (it is written from all the others).
+const ORDER = { tad: ['00', '06', '07', '08', '09', '10', '11', '12', '13', '05', '04', '03', '14', '15', '99', '02', '01'], axes: ['00', '03', '04', '05', '02', '99', '01'] };
+export function writingOrder(doc, files) {
+  const rank = (f) => { const i = (ORDER[doc] || []).indexOf(f.slice(0, 2)); return i < 0 ? 50 : i; };
+  return [...files].sort((a, b) => rank(a) - rank(b) || a.localeCompare(b));
+}
+
+/** First section still to write (in the recommended order), among the sections active for this org. */
 export async function next(args) {
   const ctx = loadWorkspace(args.cwd || process.cwd());
   const facts = loadFacts(ctx.ws);
   if (!facts) { log.fail('Run `tadforce facts` first.'); return 1; }
-  const scope = { modules: facts.modules, counts: facts.counts, org: facts.org, cfg: ctx.cfg, facts, glossary: ctx.glossary() };
+  const scope = { modules: facts.modules, counts: facts.counts, org: facts.org, cfg: ctx.cfg, facts, glossary: ctx.glossary(), source: sourceFlags(ctx.cfg) };
   const docs = !args.doc || args.doc === 'all' ? DOCS : [args.doc];
   let remaining = 0, first = null;
   for (const doc of docs) {
     const dir = ctx.p('src', doc);
     if (!exists(dir)) continue;
-    for (const f of fs.readdirSync(dir).filter((x) => x.endsWith('.md')).sort()) {
+    for (const f of writingOrder(doc, fs.readdirSync(dir).filter((x) => x.endsWith('.md')))) {
       const raw = readText(path.join(dir, f));
       const active = resolveConditionals(raw, scope);
       for (const m of active.matchAll(/<!--\s*tad:pending\s+([\w.-]+)\s*-->/g)) {

package/src/commands/update.js

@@ -61,20 +61,33 @@ function sectionLocations(lang) {
   return out;
 }
 
-/** Sections whose spec reads a facts area that changed. */
+// Sections whose spec names no facts are matched by their chapter.
+const CHAPTER_AREAS = { data: ['data'], application: ['apex', 'automation'], ui: ['ui'], integration: ['integration'], security: ['security'], reporting: ['misc'] };
+
+/**
+ * Sections whose spec reads a facts area that changed. A component change (« changed classes:Foo ») makes a
+ * section relevant; a moved count only matters where the spec reads that count.
+ */
 export function affectedSections(areas, specs, locations) {
   const hits = [];
   for (const [id, s] of Object.entries(specs)) {
     const reasons = [];
-    for (const p of s.facts || []) {
+    const paths = s.facts && s.facts.length ? s.facts : (CHAPTER_AREAS[id.split('.')[0]] || []);
+    for (const p of paths) {
       const fp = p.replace(/^counts\./, '');
-      for (const [area, why] of Object.entries(areas)) {
-        if (p === 'counts' || fp === area || fp.startsWith(area + '.') || area.startsWith(fp + '.') || (p === 'modules' && area === 'modules')) reasons.push(...why);
+      for (const [area, whys] of Object.entries(areas)) {
+        for (const why of whys) {
+          const isCount = why.startsWith('count ');
+          if (p === 'counts' ? isCount : fp === area || fp.startsWith(area + '.') || area.startsWith(fp + '.') || (p === 'modules' && area === 'modules')) reasons.push(why);
+        }
       }
     }
-    if (reasons.length) hits.push({ id, ...(locations[id] || {}), reasons: [...new Set(reasons)].slice(0, 12) });
+    if (!reasons.length) continue;
+    const unique = [...new Set(reasons)];
+    hits.push({ id, ...(locations[id] || {}), relevance: unique.some((r) => !r.startsWith('count ')) ? 'components' : 'counts', reasons: unique.slice(0, 12) });
   }
-  return hits;
+  // what changed in the components first: count-only sections usually need no rewrite
+  return hits.sort((a, b) => (a.relevance === b.relevance ? 0 : a.relevance === 'components' ? -1 : 1));
 }
 
 const nextVersion = (v) => { const m = /^(\d+)\.(\d+)$/.exec(String(v)); return m ? `${m[1]}.${Number(m[2]) + 1}` : `${v}.1`; };
@@ -86,25 +99,21 @@ export async function run(args) {
   if (!exists(before)) throw new Error('No previous facts: run `tadforce doctor` and `tadforce build` first.');
   say(() => log.banner('update'));
 
-  // 1. Archive what the current document was written from.
+  // 1. What the current version was written from (archived below, once something changed).
   const stamp = new Date().toISOString().slice(0, 10);
-  const archive = ctx.p('facts', 'history', stamp);
-  fs.mkdirSync(archive, { recursive: true });
   const oldFacts = readJson(before);
   const oldPrints = exists(ctx.p('facts', 'fingerprints.json')) ? readJson(ctx.p('facts', 'fingerprints.json')) : fingerprints(sourceDirs(ctx));
-  writeJson(path.join(archive, 'facts.json'), oldFacts);
-  writeJson(path.join(archive, 'fingerprints.json'), oldPrints);
-  say(() => log.ok(`Previous facts archived in ${path.relative(ctx.project, archive)}`));
 
   // 2. Read the org again (a fresh snapshot: deleted components must disappear too).
   const mode = ctx.cfg.source.mode || 'org';
-  const quiet = async (fn) => { if (!args.json) return fn(); const o = console.log; console.log = () => {}; try { return await fn(); } finally { console.log = o; } };
+  // in --json mode stdout carries only the report; progress goes to stderr
+  const quiet = async (fn) => { if (!args.json) return fn(); const o = console.log; console.log = (...a) => process.stderr.write(a.join(' ') + '\n'); try { return await fn(); } finally { console.log = o; } };
   if (mode !== 'local' && !args.skipRetrieve) {
     const snap = ctx.p(ctx.cfg.source.dir);
     if (exists(snap)) fs.rmSync(snap, { recursive: true, force: true });
     for (const fn of [org.manifest, org.retrieve]) { const code = await quiet(() => fn({ ...args, cwd: ctx.project })); if (code) return code; }
   }
-  await quiet(() => docs.facts({ ...args, cwd: ctx.project, org: mode === 'local' || args.skipRetrieve ? false : ctx.cfg.org.alias }));
+  await quiet(() => docs.facts({ ...args, json: false, cwd: ctx.project, org: mode === 'local' || args.skipRetrieve ? false : ctx.cfg.org.alias }));
   const ctx2 = loadWorkspace(ctx.project);
   const newPrints = fingerprints(sourceDirs(ctx2));
   writeJson(ctx2.p('facts', 'fingerprints.json'), newPrints);
@@ -122,33 +131,55 @@ export async function run(args) {
   for (const m of modulesAfter.filter((x) => !modulesBefore.includes(x))) note('modules', `new product detected: ${m}`);
   for (const m of modulesBefore.filter((x) => !modulesAfter.includes(x))) note('modules', `product no longer detected: ${m}`);
   const specs = readYaml(path.join(TEMPLATES, 'specs', 'sections.yml'), {}).sections || {};
-  const sections = affectedSections(areas, specs, sectionLocations(templateLang(ctx2.cfg.language)));
+  // sections still to write are not « to rewrite »: their first writing reads the new facts anyway
+  const srcText = (file) => (exists(ctx2.p(file)) ? readText(ctx2.p(file)) : '');
+  const sections = affectedSections(areas, specs, sectionLocations(templateLang(ctx2.cfg.language)))
+    .filter((x) => !x.file || !new RegExp(`<!--\\s*tad:pending\\s+${x.id.replace(/\./g, '\\.')}\\s*-->`).test(srcText(x.file)));
+  const upToDate = !(diff.added.length + diff.changed.length + diff.removed.length) && !Object.keys(counts).length;
+
+  // the facts of the previous version, kept once per version: never overwritten by a later run
+  const archive = ctx2.p('facts', 'history', `v${ctx2.cfg.document.version}`);
+  if (!upToDate && !exists(archive)) {
+    writeJson(path.join(archive, 'facts.json'), oldFacts);
+    writeJson(path.join(archive, 'fingerprints.json'), oldPrints);
+    say(() => log.ok(`Facts of version ${ctx2.cfg.document.version} archived in ${path.relative(ctx2.project, archive)}`));
+  }
 
   // 4. New version in the document control.
   const cfg = ctx2.cfg;
   const total = diff.added.length + diff.changed.length + diff.removed.length;
   const from = cfg.document.version;
   if (total && args.bump !== false) {
-    const by = [cfg.document.author, cfg.integrator.name].filter(Boolean).join(' — ') || cfg.client.name;
+    const by = [cfg.document.author, cfg.integrator.name].filter(Boolean).join(' — ') || '—';
     const history = cfg.document.history && cfg.document.history.length ? cfg.document.history : [{ version: from, date: cfg.document.date, author: by, purpose: cfg.language === 'fr' ? 'Version initiale' : 'Initial version' }];
     cfg.document.version = nextVersion(from);
     cfg.document.date = stamp;
-    const purpose = cfg.language === 'fr'
-      ? `Mise à jour : ${diff.added.length} composant(s) ajouté(s), ${diff.changed.length} modifié(s), ${diff.removed.length} supprimé(s)`
-      : `Update: ${diff.added.length} component(s) added, ${diff.changed.length} changed, ${diff.removed.length} removed`;
+    const fr = cfg.language === 'fr';
+    const parts = [[diff.added.length, fr ? 'ajouté(s)' : 'added'], [diff.changed.length, fr ? 'modifié(s)' : 'changed'], [diff.removed.length, fr ? 'supprimé(s)' : 'removed']].filter(([n]) => n).map(([n, w]) => `${n} ${w}`);
+    const purpose = `${fr ? 'Mise à jour, composants' : 'Update, components'} : ${parts.join(', ')}`.replace(/^Update, components :/, 'Update, components:');
     cfg.document.history = [...history, { version: cfg.document.version, date: stamp, author: by, purpose }];
     ctx2.save(cfg);
   }
-  const report = { from, to: cfg.document.version, changes: diff, counts, sections, next: sections.length ? 'rewrite each listed section from the new facts (keep its heading and anchor), then `tadforce build`' : 'tadforce build' };
-  writeJson(ctx2.p('facts', 'update.json'), report);
+  const report = { upToDate, from, to: cfg.document.version, changes: diff, counts, sections, next: upToDate ? null : sections.length ? 'rewrite the sections whose relevance is « components » from the new facts (keep heading and anchor); check the « counts » ones for numbers written in the text; then `npx tadforce build`' : `npx tadforce build --cwd "${ctx2.project}"` };
+  // a rerun before the rewrite and the build keeps the work to do of the previous update
+  const prevFile = ctx2.p('facts', 'update.json');
+  const prev = exists(prevFile) ? readJson(prevFile) : null;
+  if (upToDate && prev && !prev.built && prev.to === cfg.document.version && (prev.sections || []).length) {
+    const carried = { ...prev, carried: true, next: prev.next };
+    if (args.json) { console.log(JSON.stringify(carried, null, 2)); return 0; }
+    log.ok(`Nothing new since the update to ${prev.to}, whose ${prev.sections.length} section(s) are still to rewrite:`);
+    for (const s of prev.sections) console.log(`  ${c.cyan(s.id.padEnd(28))} ${c.dim((s.file || '') + (s.anchor ? '#' + s.anchor : ''))}`);
+    return 0;
+  }
+  writeJson(prevFile, report);
   if (args.json) { console.log(JSON.stringify(report, null, 2)); return 0; }
 
-  if (!total && !Object.keys(counts).length) { log.ok('Nothing changed since the last version. The document is up to date.'); return 0; }
+  if (upToDate) { log.ok('Nothing changed since the last version. The document is up to date.'); return 0; }
   log.ok(`${diff.added.length} added · ${diff.changed.length} changed · ${diff.removed.length} removed · ${Object.keys(counts).length} count(s) moved`);
   for (const [kind, list] of Object.entries(diff)) if (list.length) console.log(`  ${c.bold(kind)}: ${list.slice(0, 15).map((x) => `${x.folder}/${x.name}`).join(', ')}${list.length > 15 ? c.dim(` … +${list.length - 15}`) : ''}`);
   if (total) log.ok(`Version ${from} → ${cfg.document.version} (history updated)`);
   log.step(`${sections.length} section(s) to review:`);
-  for (const s of sections) console.log(`  ${c.cyan(s.id.padEnd(28))} ${c.dim((s.file || '') + (s.anchor ? '#' + s.anchor : ''))}  ${s.reasons.slice(0, 2).join('; ')}`);
+  for (const s of sections) console.log(`  ${s.relevance === 'components' ? c.cyan(s.id.padEnd(28)) : c.dim(s.id.padEnd(28))} ${c.dim((s.file || '') + (s.anchor ? '#' + s.anchor : ''))}  ${s.reasons.slice(0, 2).join('; ')}`);
   log.human('let your agent rewrite these sections (/tadforce-update), then `tadforce build`. Numbers and generated tables update by themselves.');
   return 0;
 }

package/src/facts/extractors/misc.js

@@ -25,6 +25,8 @@ export function extractMisc(roots) {
     genAi: count('genAiPlanners', '') + count('genAiPlannerBundles', '') + count('genAiPlugins', '') + count('genAiFunctions', '') + bundles(roots, 'aiAuthoringBundles').length,
     entitlementProcesses: count('entitlementProcesses', ''),
     dataStreams: count('dataStreamDefinitions', ''),
+    // every org has a PlatformEncryption.settings file: only switched-on options are evidence of Shield
+    platformEncryption: files(roots, 'settings', 'PlatformEncryption.settings-meta.xml').reduce((n, f) => n + (safeRead(f).match(/<enable\w*>true</g) || []).length, 0),
   };
 
   // Components documented by name only (routing, channels, sites, paths, rules, agents…)

package/src/facts/heuristics/exclusions.js

@@ -26,6 +26,7 @@ export const COLLECTIONS = [
   ['namedCredential', (f) => f.integration.namedCredentials],
   ['externalCredential', (f) => f.integration.externalCredentials],
   ['remoteSite', (f) => f.integration.remoteSites],
+  ['cspTrustedSite', (f) => f.integration.cspTrustedSites],
   ['connectedApp', (f) => f.integration.connectedApps],
   ['authProvider', (f) => f.integration.authProviders],
   ['report', (f) => f.misc.reports.names, null],
@@ -64,11 +65,21 @@ export function applyExclusions(facts, knowledge = {}) {
       }
     }
   }
+  // the tests of an excluded class leave with it (LegacyRateHelper → LegacyRateHelperTest, …_Test, …Tests)
+  const gone = new Set(applied.filter((a) => a.type === 'apex').map((a) => String(a.name).toLowerCase()));
+  const classes = facts.apex.classes;
+  for (let i = classes.length - 1; i >= 0; i--) {
+    const base = String(classes[i].name).toLowerCase().replace(/_?tests?$/, '');
+    if (classes[i].test && base !== String(classes[i].name).toLowerCase() && gone.has(base)) {
+      applied.push({ type: 'apex', name: classes[i].name, reason: `test of an excluded class`, feature: null });
+      classes.splice(i, 1);
+    }
+  }
   return applied.sort((a, b) => (a.type + a.name).localeCompare(b.type + b.name));
 }
 
 // Org-wide components (profiles, roles, queues, report folders…) are shared by every project: never scoped out.
-const SCOPED_TYPES = new Set(['apex', 'trigger', 'lwc', 'aura', 'vfPage', 'flexipage', 'app', 'tab', 'quickAction', 'flow', 'omniScript', 'integrationProcedure', 'dataMapper', 'flexCard', 'object', 'cmdt', 'permissionSet', 'permissionSetGroup', 'namedCredential', 'externalCredential', 'remoteSite', 'connectedApp', 'authProvider', 'staticResource']);
+const SCOPED_TYPES = new Set(['apex', 'trigger', 'lwc', 'aura', 'vfPage', 'flexipage', 'app', 'tab', 'quickAction', 'flow', 'omniScript', 'integrationProcedure', 'dataMapper', 'flexCard', 'object', 'cmdt', 'permissionSet', 'permissionSetGroup', 'namedCredential', 'externalCredential', 'remoteSite', 'cspTrustedSite', 'connectedApp', 'authProvider', 'staticResource']);
 
 /**
  * Keep only the part of the org a project owns (tadforce.config.json › scope.include, name patterns with `*`).
@@ -115,6 +126,7 @@ export function exclusionCandidates(facts, refs) {
     for (const item of list) {
       const name = keyOf(item, field);
       if (type === 'apex' && item.test) continue;
+      if (type === 'object' && !String(name).includes('__')) continue; // standard objects (AiEval…TestCase, Devops…Test) are Salesforce's
       const bad = nameWords(name).filter((w) => SUSPICIOUS.has(w));
       if (bad.length) add(type, name, `name suggests ${bad.join('/')}`);
     }

package/src/facts/heuristics/modules.js

@@ -35,9 +35,9 @@ const CLOUDS = [
   { id: 'netZero', name: 'Net Zero Cloud', ref: 'industries', objects: ['StnryAssetCrbnFtprnt', 'VehicleAssetCrbnFtprnt', 'Scope3CrbnFtprnt'] },
   { id: 'omnistudio', name: 'OmniStudio', ref: 'omnistudio', ns: ['omnistudio', 'vlocity_cmt', 'vlocity_ins', 'vlocity_ps'], test: (f) => Object.values(f.automation.omnistudio).some((l) => l.length > 0) },
   { id: 'dataCloud', name: 'Data Cloud', ref: 'data-cloud', test: (f) => f.misc.signals.dataStreams > 0 || f.data.objects.some((o) => /__(dlm|dll)$/.test(o.name)) || (f.misc.folders.dataStreamDefinitions || 0) + (f.misc.folders.dataSourceObjects || 0) > 0 },
-  { id: 'agentforce', name: 'Agentforce / Einstein', ref: 'agentforce-ai', test: (f) => f.misc.signals.bots + f.misc.signals.genAi > 0 || (f.misc.folders.genAiPromptTemplates || 0) + (f.misc.folders.mlPredictionDefinitions || 0) > 0 },
+  { id: 'agentforce', name: 'Agentforce / Einstein', ref: 'agentforce-ai', psg: /Agentforce|Copilot|EinsteinGPT|GenAI|Einstein_?Agent/i, test: (f) => f.misc.signals.bots + f.misc.signals.genAi > 0 || (f.misc.folders.genAiPromptTemplates || 0) + (f.misc.folders.mlPredictionDefinitions || 0) > 0 },
   { id: 'crmAnalytics', name: 'CRM Analytics', ref: 'analytics', ns: ['wave'], test: (f) => ['wave', 'waveDashboards', 'waveDataflows', 'waveRecipes', 'waveApplications', 'waveDatasets'].some((k) => f.misc.folders[k] > 0) },
-  { id: 'shield', name: 'Shield (encryption, event monitoring, audit)', ref: 'security-shield', test: (f) => ['transactionSecurityPolicies', 'eventSubscriptions'].some((k) => f.misc.folders[k] > 0) || f.misc.settings.some((s) => /PlatformEncryption|EventSettings|FieldHistoryRetention/.test(s)) },
+  { id: 'shield', name: 'Shield (encryption, event monitoring, audit)', ref: 'security-shield', test: (f) => ['transactionSecurityPolicies', 'eventSubscriptions'].some((k) => f.misc.folders[k] > 0) || (f.misc.signals.platformEncryption || 0) > 0 },
   { id: 'slack', name: 'Slack integration', ref: 'integration-platforms', ns: ['slackv2', 'slack'], test: (f) => (f.misc.folders.slackApps || 0) > 0 },
   { id: 'events', name: 'Platform Events / Change Data Capture', ref: 'platform-events', test: (f) => f.data.objects.some((o) => o.kind === 'platformEvent') || f.data.cdc.length > 0 },
 ];
@@ -53,7 +53,7 @@ function usedStandard(f) {
   return used;
 }
 
-export function detectModules(f) {
+export function detectModules(f, org = null) {
   const ns = new Set(f.namespaces.map((n) => n.namespace));
   const std = usedStandard(f);
   const clouds = [];
@@ -63,6 +63,8 @@ export function detectModules(f) {
     for (const n of c.ns || []) if (ns.has(n)) evidence.push(`namespace ${n}`);
     for (const o of c.objects || []) if (std.has(o) || f.data.objects.some((x) => x.name === o && x.kind === 'standard' && x.customFields > 0)) evidence.push(`object ${o}`);
     if (c.test && c.test(f)) evidence.push('metadata');
+    // users assigned to the product's permission set groups (aggregated org facts) prove it is used
+    for (const g of (org && org.psgAssignments) || []) if (c.psg && c.psg.test(String(g.group)) && g.users > 0) evidence.push(`permission set group ${g.group} (${g.users} users)`);
     const on = evidence.length > 0;
     flags[c.id] = on;
     if (on) clouds.push({ id: c.id, name: c.name, reference: c.ref, evidence: [...new Set(evidence)].slice(0, 6) });

package/src/facts/index.js

@@ -36,6 +36,7 @@ export function extractFacts(dirs, knowledge = {}, opts = {}) {
   const refs = references(roots, apex, ui, automation, integration);
   const facts = { schema: FACTS_SCHEMA, apex, data, automation, ui, security, integration, misc, namespaces };
 
+  const shipped = removeSalesforceShipped(facts);
   const outOfScope = applyScope(facts, knowledge.scope || {});
   const applied = applyExclusions(facts, knowledge.exclusions || {});
   // placements are recomputed so that excluded containers no longer count as usages
@@ -51,7 +52,8 @@ export function extractFacts(dirs, knowledge = {}, opts = {}) {
 
   facts.exclusions = { applied, candidates: exclusionCandidates(facts, refs) };
   if ((knowledge.scope?.include || []).length) facts.scope = { include: [...knowledge.scope.include], outOfScope: outOfScope.length };
-  facts.modules = detectModules(facts);
+  facts.modules = detectModules(facts, opts.org || null);
+  if (Object.keys(shipped).length) facts.salesforceShipped = shipped;
   // Products used without metadata traces are declared by a human in glossary.yml › products.
   for (const p of (knowledge.glossary && knowledge.glossary.products) || []) {
     if (!p || !p.name || facts.modules.clouds.some((c) => c.name === p.name)) continue;
@@ -74,6 +76,23 @@ export function extractFacts(dirs, knowledge = {}, opts = {}) {
   return facts;
 }
 
+// Components Salesforce ships in every org (standard apps, platform permission set groups…): not the project's architecture.
+const SHIPPED = /^(standard__|force__|sfdcInternalInt__|sf_devops)/;
+
+/** Set aside what Salesforce ships with the org; returns how many of each kind, by facts path. */
+function removeSalesforceShipped(f) {
+  const out = {};
+  // the utility bars of Salesforce's own apps go with them
+  const bars = new Set(f.ui.apps.filter((x) => SHIPPED.test(String(x.name)) && x.utilityBar).map((x) => x.utilityBar));
+  for (let i = f.ui.flexipages.length - 1; i >= 0; i--) if (bars.has(f.ui.flexipages[i].name)) { f.ui.flexipages.splice(i, 1); out['ui.utilityBars'] = (out['ui.utilityBars'] || 0) + 1; }
+  for (const [key, list] of [['ui.apps', f.ui.apps], ['ui.tabs', f.ui.tabs], ['ui.flexipages', f.ui.flexipages], ['security.permissionSets', f.security.permissionSets], ['security.permissionSetGroups', f.security.permissionSetGroups]]) {
+    let n = 0;
+    for (let i = list.length - 1; i >= 0; i--) if (SHIPPED.test(String(list[i].name))) { list.splice(i, 1); n++; }
+    if (n) out[key] = n;
+  }
+  return out;
+}
+
 /** Cross-references: where Apex classes, LWC/Aura and FlexiPages are used. */
 function references(roots, apex, ui, automation, integration) {
   const nonTest = apex.classes.filter((c) => !c.test);

package/src/sf/sf.js

@@ -1,5 +1,8 @@
 // Read-only wrapper around the Salesforce CLI. Every call goes through an allow-list:
 // TADForce never deploys, never deletes, never reads record-level data.
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
 import { spawn, spawnSync } from 'node:child_process';
 
 const ALLOWED = [
@@ -16,6 +19,8 @@ const ALLOWED = [
   ['package', 'installed', 'list'],
   ['config', 'get'],
 ];
+// Read-only commands that take positional arguments after their subcommand words (`sf config get target-org`).
+const WITH_POSITIONALS = [['config', 'get']];
 
 // sObjects whose rows describe the org itself (configuration, not business data).
 const METADATA_SOBJECTS = new Set([
@@ -30,7 +35,8 @@ export function isAllowed(args) {
   const cmd = [];
   for (const a of args) { if (a.startsWith('-')) break; cmd.push(a); }
   if (!cmd.length) return args[0] === '--version';
-  return ALLOWED.some((p) => p.length === cmd.length && p.every((w, i) => w === cmd[i]));
+  if (ALLOWED.some((p) => p.length === cmd.length && p.every((w, i) => w === cmd[i]))) return true;
+  return WITH_POSITIONALS.some((p) => cmd.length > p.length && p.every((w, i) => w === cmd[i]) && cmd.slice(p.length).every((a) => /^[\w-]+$/.test(a)));
 }
 
 const AGG = /^(count\(\s*(id)?\s*\)|max\(\s*(createddate|lastmodifieddate)\s*\)|min\(\s*createddate\s*\))(\s+\w+)?$/i;
@@ -66,8 +72,21 @@ function launch(args, opts) {
   return [[resolveSf(), ...args].map(quote).join(' '), [], { ...opts, shell: true }];
 }
 
-/** Run `sf ... --json` and return the parsed result (throws with the CLI message on failure). */
-export function sfJson(args, { timeoutMs = 600000, cwd } = {}) {
+// Network hiccups (DNS, resets) are retried; the CLI reports a failed DNS lookup as DomainNotFoundError.
+const TRANSIENT = /DomainNotFound|ENOTFOUND|EAI_AGAIN|ECONNRESET|ETIMEDOUT|socket hang up|getaddrinfo/i;
+export const isTransient = (e) => TRANSIENT.test(`${e && e.name} ${e && e.message} ${e && e.sf && e.sf.name}`);
+
+/** Run `sf ... --json` and return the parsed result (throws with the CLI message on failure). Transient network errors are retried twice. */
+export async function sfJson(args, opts = {}) {
+  for (let attempt = 0; ; attempt++) {
+    try { return await sfJsonOnce(args, opts); } catch (e) {
+      if (attempt >= 2 || !isTransient(e)) throw e;
+      await new Promise((r) => setTimeout(r, 1500 * (attempt + 1)));
+    }
+  }
+}
+
+function sfJsonOnce(args, { timeoutMs = 600000, cwd } = {}) {
   if (!isAllowed(args)) throw new Error(`TADForce refuses to run: sf ${args.join(' ')}`);
   if (args[0] === 'data' && args[1] === 'query') {
     const qi = args.findIndex((a) => a === '--query' || a === '-q');
@@ -105,12 +124,17 @@ export function sfInteractive(args) {
   return r.status === 0;
 }
 
+// `sf --version` takes seconds: its answer is kept a day (per CLI binary), outside any project.
+const VERSION_CACHE = path.join(os.tmpdir(), 'tadforce-sf-version.json');
 export function sfVersion() {
+  try { const c = JSON.parse(fs.readFileSync(VERSION_CACHE, 'utf8')); if (c.bin === resolveSf() && Date.now() - c.at < 86400000 && c.version) return c.version; } catch { /* no cache yet */ }
   const [cmd, argv, opts] = launch(['--version'], { encoding: 'utf8' });
   const r = spawnSync(cmd, argv, opts);
   if (r.status !== 0) return null;
   const m = /@salesforce\/cli\/(\d+\.\d+\.\d+)/.exec(r.stdout || '');
-  return m ? m[1] : (r.stdout || '').trim() || null;
+  const version = m ? m[1] : (r.stdout || '').trim() || null;
+  try { fs.writeFileSync(VERSION_CACHE, JSON.stringify({ bin: resolveSf(), version, at: Date.now() })); } catch { /* no writable tmp: no cache */ }
+  return version;
 }
 
 export function soql(query, alias, { tooling = false } = {}) {

package/src/util/agents.js

@@ -0,0 +1,35 @@
+// AI agents TADForce can hand over to once `doctor` is green: the agent writes the document.
+import { spawnSync } from 'node:child_process';
+
+const RUN = '/tadforce';
+const PROMPT = 'Use the tadforce skill: the prerequisites are green (tadforce doctor). Write every section of the Technical Architecture Document and the Improvement Axes, then build and validate them.';
+
+/** Command-line agents, in order of preference: the argument list that starts an interactive session on the task. */
+export const AGENTS = [
+  { id: 'claude', name: 'Claude Code', bin: 'claude', args: [RUN] },
+  { id: 'codex', name: 'Codex', bin: 'codex', args: [PROMPT] },
+  { id: 'gemini', name: 'Gemini CLI', bin: 'gemini', args: ['-i', PROMPT] },
+  { id: 'cursor', name: 'Cursor Agent', bin: 'cursor-agent', args: [PROMPT] },
+];
+
+const onPath = (bin) => spawnSync(process.platform === 'win32' ? 'where' : 'which', [bin], { encoding: 'utf8' }).status === 0;
+
+/** Agents installed on this machine. */
+export function installedAgents() {
+  return AGENTS.filter((a) => onPath(a.bin));
+}
+
+/** True when TADForce already runs inside an agent session: never start another agent from there. */
+export function insideAgent() {
+  const e = process.env;
+  return !!(e.CLAUDECODE || e.CLAUDE_CODE_ENTRYPOINT || e.CODEX_SANDBOX || e.CODEX_HOME && e.CODEX_SESSION || e.GEMINI_CLI || e.CURSOR_AGENT || e.TADFORCE_NO_LAUNCH);
+}
+
+/** Start the agent in this terminal, in the project folder. Returns its exit code. */
+export function launchAgent(agent, cwd) {
+  const quote = (a) => (/^[\w./-]+$/.test(a) ? a : `"${a.replace(/"/g, '\\"')}"`);
+  const r = process.platform === 'win32'
+    ? spawnSync([agent.bin, ...agent.args].map(quote).join(' '), [], { cwd, stdio: 'inherit', shell: true })
+    : spawnSync(agent.bin, agent.args, { cwd, stdio: 'inherit' });
+  return r.status ?? 1;
+}

package/src/util/config.js

@@ -65,7 +65,7 @@ export function findWorkspace(cwd = process.cwd()) {
 export function loadWorkspace(cwd) {
   const ws = findWorkspace(cwd);
   if (!ws) {
-    const err = new Error(`No TADForce workspace found under ${cwd}. Run \`npx tadforce init\` first.`);
+    const err = new Error(`No TADForce workspace found under ${cwd}. Run \`npx tadforce doctor\` in your Salesforce project first.`);
     err.code = 'NO_WORKSPACE';
     throw err;
   }
@@ -82,6 +82,9 @@ export function loadWorkspace(cwd) {
   };
 }
 
+/** Flags for the skeletons' conditions: `<!-- tad:if source.local -->`. */
+export const sourceFlags = (cfg) => ({ local: cfg.source.mode === 'local', org: cfg.source.mode !== 'local' });
+
 /** Metadata source directories: the retrieved snapshot and/or the project's package dirs (source.mode). */
 export function sourceDirs(wsCtx) {
   const mode = wsCtx.cfg.source.mode;

package/src/util/i18n.js

@@ -41,6 +41,16 @@ export function t(dict, key, vars = {}) {
   return s.replace(/\{(\w+)\}/g, (m, k) => (vars[k] !== undefined ? String(vars[k]) : m));
 }
 
+const locale = (lang) => (lang === 'fr' ? 'fr-FR' : lang === 'en' || !lang ? 'en-GB' : lang);
+
+/** 2026-10-30 → 30 October 2026 / 30 octobre 2026 / 30 de octubre de 2026. Anything else is kept as written. */
+export function formatDate(value, lang) {
+  const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(String(value || ''));
+  if (!m) return value;
+  const d = new Date(Date.UTC(+m[1], +m[2] - 1, +m[3]));
+  try { return new Intl.DateTimeFormat(locale(lang), { day: 'numeric', month: 'long', year: 'numeric', timeZone: 'UTC' }).format(d); } catch { return value; }
+}
+
 export function formatNumber(n, lang) {
   if (typeof n !== 'number' || !Number.isFinite(n)) return String(n);
   // fr-FR uses a narrow no-break space as thousands separator, as in the reference document.

package/src/validate/index.js

@@ -73,7 +73,8 @@ export function validateDocument(wsCtx, doc, { buildReport = null } = {}) {
       const pend = /<!--\s*tad:pending\s+([\w.-]+)\s*-->/.exec(line);
       // sections inside a false <!-- tad:if --> block are not part of this org's document
       if (pend && (!buildReport || buildReport.pending.includes(pend[1]))) add('error', 'pending', `Section "${pend[1]}" is not written yet.`, where);
-      if (/\bTODO\b|\bTBD\b|lorem ipsum/i.test(line)) add('error', 'todo', 'Placeholder text left in the document.', where);
+      // upper-case markers as whole words only: « Método », « todo » (Spanish « all ») are text, not placeholders
+      if (/(?<![\p{L}\p{N}_])(TODO|TBD|FIXME)(?![\p{L}\p{N}_])/u.test(line) || /lorem ipsum/i.test(line)) add('error', 'todo', 'Placeholder text left in the document.', where);
       for (const [id, re] of SECRET_PATTERNS) if (re.test(line)) add('error', 'secret', `Possible secret or org identifier (${id}).`, where);
       for (const a of avoid) if (new RegExp(`\\b${escapeRe(a.avoid)}\\b`, 'i').test(line)) add('error', 'terminology', `"${a.avoid}" must be written "${a.use}" (glossary).`, where);
       for (const fk of forbiddenFeatures) if (new RegExp(`\\b${escapeRe(fk.k)}\\b`, 'i').test(line)) add('error', 'decommissioned', `Mentions "${fk.k}" (decommissioned feature: ${fk.feature}).`, where);
@@ -91,7 +92,7 @@ export function validateDocument(wsCtx, doc, { buildReport = null } = {}) {
         if (/[*_]$/.test(id) || /^\*/.test(id)) continue; // naming patterns such as `NWB_*` or `*Service`
         if (!looksLikeComponent(id) || COMMON.has(id) || inventory.has(id)) continue;
         if (/^[\w]+\.[\w]+$/.test(id) && inventory.has(id.split('.')[1])) continue;
-        add('warn', 'unknown-component', `\`${id}\` is not in the metadata inventory (typo, other org, or package component?).`, where);
+        add('warn', 'unknown-component', `\`${id}\` is not in the metadata inventory (${(wsCtx.cfg.scope && wsCtx.cfg.scope.include || []).length ? 'out of the project scope, ' : ''}typo, other org, or package component?).`, where);
       }
     });
     if (inCode) add('error', 'unclosed-code', 'Unclosed code fence.', `src/${doc}/${f}`);