sdx-cli 0.3.1 → 0.3.2

package/README.md

@@ -180,6 +180,16 @@ Use overrides to:
 ### Canonical Root README Generation
 Generate a complete root `README.md` as the canonical onboarding and architecture overview for your org workspace.
 
+What `docs readme` now does:
+- traverses Markdown docs across repos in map scope (`README*`, `docs/**`, ADRs, runbooks),
+- infers service purpose, interfaces, async behavior, deployment cues, and operating notes,
+- combines that with map/contracts/architecture artifacts,
+- writes a clean narrative README (no SDX section marker blocks in output).
+
+For best results:
+- register local clones for repos you care about (`repo add`) so SDX can deeply scan docs,
+- set `GITHUB_TOKEN` to let SDX fetch Markdown docs for repos without local clones.
+
 ```bash
 # generate/update root README.md
 ./scripts/sdx docs readme --map platform-core
@@ -187,10 +197,10 @@ Generate a complete root `README.md` as the canonical onboarding and architectur
 # write to a different output file
 ./scripts/sdx docs readme --map platform-core --output ARCHITECTURE.md
 
-# check mode for CI (non-zero on stale sources, missing sources, or README drift)
+# check mode for CI (non-zero on stale/missing required artifacts or README drift)
 ./scripts/sdx docs readme --map platform-core --check
 
-# dry-run preview with unified diff and freshness summary
+# dry-run preview with unified diff + readiness summary
 ./scripts/sdx docs readme --map platform-core --dry-run
 
 # selective sections
@@ -230,10 +240,6 @@ Config capabilities:
 - custom intro text (`customIntro`)
 - stale threshold override in hours (`staleThresholdHours`, default `72`)
 
-Manual content preservation:
-- generated wrappers: `<!-- SDX:SECTION:<id>:START --> ... <!-- SDX:SECTION:<id>:END -->`
-- preserved manual blocks: `<!-- SDX:SECTION:<id>:MANUAL:START --> ... <!-- SDX:SECTION:<id>:MANUAL:END -->`
-
 CI automation example:
 - copy [`docs/examples/readme-refresh.yml`](./docs/examples/readme-refresh.yml) into your consumer workspace repo under `.github/workflows/`.
 - set repo/org variables:

package/dist/commands/docs/readme.js

@@ -24,7 +24,7 @@ class DocsReadmeCommand extends core_1.Command {
         const context = (0, project_1.loadProject)(process.cwd());
         const includeSections = (0, readme_1.parseReadmeSectionList)(flags.include);
         const excludeSections = (0, readme_1.parseReadmeSectionList)(flags.exclude);
-        const result = (0, readme_1.generateReadme)({
+        const result = await (0, readme_1.generateReadme)({
             mapId: flags.map,
             db: context.db,
             cwd: context.cwd,

package/dist/lib/github.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.fetchOrgRepos = fetchOrgRepos;
 exports.ensureOrgRepo = ensureOrgRepo;
+exports.fetchRepositoryMarkdownDocs = fetchRepositoryMarkdownDocs;
 async function fetchOrgRepos(org, token) {
     const { Octokit } = await import('@octokit/rest');
     const octokit = new Octokit({ auth: token });
@@ -50,3 +51,85 @@ async function ensureOrgRepo(org, repoName, token) {
         htmlUrl: created.data.html_url ?? undefined,
     };
 }
+function markdownPathPriority(filePath) {
+    const lower = filePath.toLowerCase();
+    if (lower === 'readme.md' || lower.endsWith('/readme.md') || lower.endsWith('/readme.mdx')) {
+        return 0;
+    }
+    if (lower.includes('/docs/architecture/') || lower.includes('/architecture/')) {
+        return 1;
+    }
+    if (lower.includes('/docs/api/') || lower.includes('/api/')) {
+        return 2;
+    }
+    if (lower.includes('/docs/')) {
+        return 3;
+    }
+    return 4;
+}
+function isIgnoredPath(filePath) {
+    const lower = filePath.toLowerCase();
+    return (lower.includes('/node_modules/') ||
+        lower.includes('/dist/') ||
+        lower.includes('/build/') ||
+        lower.includes('/.next/') ||
+        lower.includes('/coverage/') ||
+        lower.includes('/vendor/'));
+}
+function toBlobUrl(owner, repo, branch, filePath) {
+    const encodedPath = filePath
+        .split('/')
+        .map((part) => encodeURIComponent(part))
+        .join('/');
+    return `https://github.com/${owner}/${repo}/blob/${encodeURIComponent(branch)}/${encodedPath}`;
+}
+async function fetchRepositoryMarkdownDocs(options) {
+    const { Octokit } = await import('@octokit/rest');
+    const octokit = new Octokit({ auth: options.token });
+    const maxFiles = options.maxFiles ?? 30;
+    const maxBytesPerFile = options.maxBytesPerFile ?? 120_000;
+    const tree = await octokit.rest.git.getTree({
+        owner: options.owner,
+        repo: options.repo,
+        tree_sha: options.defaultBranch,
+        recursive: 'true',
+    });
+    const markdownFiles = (tree.data.tree ?? [])
+        .filter((entry) => Boolean(entry.path && entry.type))
+        .map((entry) => ({ path: entry.path, type: entry.type }))
+        .filter((entry) => entry.type === 'blob')
+        .map((entry) => entry.path)
+        .filter((filePath) => /\.(md|mdx)$/i.test(filePath))
+        .filter((filePath) => !isIgnoredPath(filePath))
+        .sort((a, b) => {
+        const priorityDelta = markdownPathPriority(a) - markdownPathPriority(b);
+        if (priorityDelta !== 0) {
+            return priorityDelta;
+        }
+        return a.localeCompare(b);
+    })
+        .slice(0, maxFiles);
+    const docs = [];
+    for (const filePath of markdownFiles) {
+        const response = await octokit.rest.repos.getContent({
+            owner: options.owner,
+            repo: options.repo,
+            path: filePath,
+            ref: options.defaultBranch,
+        });
+        if (Array.isArray(response.data) || response.data.type !== 'file' || !response.data.content) {
+            continue;
+        }
+        const raw = Buffer.from(response.data.content, 'base64').toString('utf8');
+        const body = raw.slice(0, maxBytesPerFile);
+        if (body.trim().length === 0) {
+            continue;
+        }
+        docs.push({
+            path: filePath,
+            body,
+            referenceUrl: toBlobUrl(options.owner, options.repo, options.defaultBranch, filePath),
+        });
+    }
+    return docs;
+}

package/dist/lib/readme.js

@@ -11,9 +11,12 @@ const node_path_1 = __importDefault(require("node:path"));
 const yaml_1 = __importDefault(require("yaml"));
 const zod_1 = require("zod");
 const architecture_1 = require("./architecture");
+const config_1 = require("./config");
 const constants_1 = require("./constants");
 const contracts_1 = require("./contracts");
+const fileScan_1 = require("./fileScan");
 const fs_1 = require("./fs");
+const github_1 = require("./github");
 const mapBuilder_1 = require("./mapBuilder");
 const repoRegistry_1 = require("./repoRegistry");
 const scope_1 = require("./scope");
@@ -84,6 +87,20 @@ const README_CONFIG_SCHEMA = zod_1.z.object({
     customIntro: zod_1.z.string().optional(),
     staleThresholdHours: zod_1.z.number().positive().optional(),
 });
+const EMPTY_REPO_INSIGHTS = {
+    summary: 'Unknown',
+    responsibilities: [],
+    interfaces: [],
+    asyncPatterns: [],
+    deployment: [],
+    runbooks: [],
+    localDevelopment: [],
+    security: [],
+    adrs: [],
+    dataStores: [],
+    glossary: [],
+    docReferences: [],
+};
 function normalizeRepoName(value) {
     return value.trim().replace(/^https?:\/\/github\.com\//i, '').replace(/\.git$/i, '').split('/').pop() ?? value.trim();
 }
@@ -256,6 +273,265 @@ function computeSnapshotTimestamp(sources, fallback) {
     }
     return sourceCandidates.reduce((latest, candidate) => (candidate.getTime() > latest.getTime() ? candidate : latest), sourceCandidates[0]).toISOString();
 }
+function markdownPriority(filePath) {
+    const lower = filePath.toLowerCase();
+    if (lower === 'readme.md' || lower.endsWith('/readme.md')) {
+        return 0;
+    }
+    if (lower.includes('/docs/architecture/') || lower.includes('/architecture/')) {
+        return 1;
+    }
+    if (lower.includes('/docs/api/') || lower.includes('/api/')) {
+        return 2;
+    }
+    if (lower.includes('/docs/')) {
+        return 3;
+    }
+    return 4;
+}
+function parseOwnerRepo(fullName) {
+    if (!fullName) {
+        return null;
+    }
+    const parts = fullName.split('/').map((part) => part.trim()).filter((part) => part.length > 0);
+    if (parts.length !== 2) {
+        return null;
+    }
+    return { owner: parts[0], repo: parts[1] };
+}
+function normalizeMarkdown(input) {
+    return input
+        .replace(/\r\n/g, '\n')
+        .replace(/```[\s\S]*?```/g, ' ')
+        .replace(/`([^`]+)`/g, '$1')
+        .replace(/\[([^\]]+)\]\(([^)]+)\)/g, '$1')
+        .replace(/[#>*_~\-]{1,}/g, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+function splitSentences(input) {
+    const cleaned = normalizeMarkdown(input);
+    if (cleaned.length === 0) {
+        return [];
+    }
+    return cleaned
+        .split(/(?<=[.!?])\s+/)
+        .map((sentence) => sentence.trim())
+        .filter((sentence) => sentence.length >= 20);
+}
+function markdownSections(content) {
+    const lines = content.replace(/\r\n/g, '\n').split('\n');
+    const out = [];
+    let currentHeading = 'root';
+    let buffer = [];
+    function flush() {
+        const body = buffer.join('\n').trim();
+        if (body.length > 0) {
+            out.push({ heading: currentHeading, body });
+        }
+        buffer = [];
+    }
+    for (const line of lines) {
+        const match = line.match(/^#{1,6}\s+(.+)$/);
+        if (match) {
+            flush();
+            currentHeading = match[1].trim().toLowerCase();
+            continue;
+        }
+        buffer.push(line);
+    }
+    flush();
+    return out;
+}
+function firstParagraph(content) {
+    const paragraphs = content
+        .replace(/\r\n/g, '\n')
+        .split(/\n\s*\n/)
+        .map((chunk) => normalizeMarkdown(chunk))
+        .filter((chunk) => chunk.length >= 20);
+    return paragraphs[0];
+}
+function topUnique(values, limit) {
+    const out = [];
+    for (const value of values) {
+        if (out.includes(value)) {
+            continue;
+        }
+        out.push(value);
+        if (out.length >= limit) {
+            break;
+        }
+    }
+    return out;
+}
+function collectLocalMarkdownDocs(repo, maxFiles = 180, maxChars = 180_000) {
+    if (!repo.localPath || !(0, fs_1.fileExists)(repo.localPath)) {
+        return [];
+    }
+    const candidates = (0, fileScan_1.listFilesRecursive)(repo.localPath)
+        .filter((candidate) => /\.(md|mdx)$/i.test(candidate))
+        .map((candidate) => node_path_1.default.relative(repo.localPath, candidate).split(node_path_1.default.sep).join('/'))
+        .sort((a, b) => {
+        const priorityDelta = markdownPriority(a) - markdownPriority(b);
+        if (priorityDelta !== 0) {
+            return priorityDelta;
+        }
+        return a.localeCompare(b);
+    });
+    const selected = candidates.slice(0, maxFiles);
+    const docs = [];
+    for (const relativePath of selected) {
+        const absolutePath = node_path_1.default.join(repo.localPath, relativePath);
+        const raw = (0, fs_1.safeReadText)(absolutePath);
+        const body = raw.slice(0, maxChars);
+        if (normalizeMarkdown(body).length === 0) {
+            continue;
+        }
+        docs.push({
+            path: relativePath,
+            body,
+            source: 'local',
+        });
+    }
+    return docs;
+}
+async function collectRemoteMarkdownDocs(repo, token) {
+    if (!token) {
+        return [];
+    }
+    const ownerRepo = parseOwnerRepo(repo.fullName);
+    if (!ownerRepo) {
+        return [];
+    }
+    try {
+        const files = await (0, github_1.fetchRepositoryMarkdownDocs)({
+            owner: ownerRepo.owner,
+            repo: ownerRepo.repo,
+            defaultBranch: repo.defaultBranch ?? 'main',
+            token,
+            maxFiles: 25,
+            maxBytesPerFile: 120_000,
+        });
+        return files.map((entry) => ({
+            path: entry.path,
+            body: entry.body,
+            source: 'remote',
+            referenceUrl: entry.referenceUrl,
+        }));
+    }
+    catch {
+        return [];
+    }
+}
+function inferStoresFromDocs(docs) {
+    const keywords = ['postgres', 'mysql', 'mongodb', 'dynamodb', 'redis', 'cassandra', 'sqlite', 'kafka', 'sqs', 'rabbitmq'];
+    const matches = [];
+    for (const doc of docs) {
+        const normalized = normalizeMarkdown(doc.body).toLowerCase();
+        for (const keyword of keywords) {
+            if (normalized.includes(keyword)) {
+                matches.push(keyword);
+            }
+        }
+    }
+    return topUnique(matches.sort((a, b) => a.localeCompare(b)), 6);
+}
+function collectByHeading(docs, matcher, limit) {
+    const collected = [];
+    for (const doc of docs) {
+        const sections = markdownSections(doc.body);
+        for (const section of sections) {
+            if (!matcher.test(section.heading)) {
+                continue;
+            }
+            collected.push(...splitSentences(section.body));
+        }
+    }
+    return topUnique(collected, limit);
+}
+function collectCommands(docs, limit) {
+    const commands = [];
+    for (const doc of docs) {
+        const matches = doc.body.match(/```(?:bash|sh|zsh|shell)?\n([\s\S]*?)```/gi) ?? [];
+        for (const block of matches) {
+            const body = block
+                .replace(/```(?:bash|sh|zsh|shell)?\n?/i, '')
+                .replace(/```$/i, '')
+                .split('\n')
+                .map((line) => line.trim())
+                .filter((line) => line.length > 0 && !line.startsWith('#'));
+            for (const line of body) {
+                commands.push(line);
+            }
+        }
+    }
+    return topUnique(commands, limit);
+}
+function extractGlossaryTerms(docs) {
+    const terms = [];
+    for (const doc of docs) {
+        const glossarySections = markdownSections(doc.body).filter((section) => /glossary|terms/.test(section.heading));
+        for (const section of glossarySections) {
+            for (const line of section.body.split('\n')) {
+                const match = line.match(/^\s*[-*]\s*\*\*([^*]+)\*\*:/);
+                if (match) {
+                    terms.push(match[1].trim());
+                }
+            }
+        }
+    }
+    return topUnique(terms.sort((a, b) => a.localeCompare(b)), 12);
+}
+async function buildRepoInsights(repo, token) {
+    const localDocs = collectLocalMarkdownDocs(repo);
+    const remoteDocs = localDocs.length > 0 ? [] : await collectRemoteMarkdownDocs(repo, token);
+    const docs = [...localDocs, ...remoteDocs];
+    const summary = firstParagraph(docs.find((doc) => /(^|\/)readme\.mdx?$/i.test(doc.path))?.body ?? '') ??
+        firstParagraph(docs[0]?.body ?? '') ??
+        'Unknown';
+    const responsibilities = collectByHeading(docs, /(overview|purpose|responsibilit|architecture|what .*does)/, 5);
+    const interfaces = collectByHeading(docs, /(api|endpoint|contract|schema|graphql|grpc|openapi)/, 5);
+    const asyncPatterns = collectByHeading(docs, /(event|async|queue|topic|stream|kafka|pubsub)/, 5);
+    const deployment = collectByHeading(docs, /(deploy|environment|infrastructure|release|production)/, 5);
+    const runbooks = collectByHeading(docs, /(runbook|incident|on.?call|troubleshoot|escalation)/, 5);
+    const security = collectByHeading(docs, /(security|auth|authorization|compliance|privacy|encryption|secret)/, 5);
+    const adrs = docs
+        .filter((doc) => /(\/|^)docs\/adr\/.+\.mdx?$/i.test(doc.path) || /(\/|^)adr\/.+\.mdx?$/i.test(doc.path))
+        .map((doc) => doc.referenceUrl ?? doc.path);
+    const localDevelopment = collectCommands(docs, 8);
+    const dataStores = inferStoresFromDocs(docs);
+    const glossary = extractGlossaryTerms(docs);
+    const docReferences = topUnique(docs
+        .map((doc) => doc.referenceUrl ?? doc.path)
+        .sort((a, b) => a.localeCompare(b)), 8);
+    return {
+        summary,
+        responsibilities,
+        interfaces,
+        asyncPatterns,
+        deployment,
+        runbooks,
+        localDevelopment,
+        security,
+        adrs,
+        dataStores,
+        glossary,
+        docReferences,
+    };
+}
+async function buildAllRepoInsights(selectedRepos, repoMap, token) {
+    const out = new Map();
+    await Promise.all(selectedRepos.map(async (repoName) => {
+        const repo = repoMap.get(repoName);
+        if (!repo) {
+            out.set(repoName, EMPTY_REPO_INSIGHTS);
+            return;
+        }
+        const insights = await buildRepoInsights(repo, token);
+        out.set(repoName, insights);
+    }));
+    return out;
+}
 function filterReposForReadme(scope, config) {
     const base = [...scope.effective];
     const include = new Set((config.repos?.include ?? []).map((value) => normalizeRepoName(value)));
@@ -606,6 +882,84 @@ function formatList(values) {
     }
     return values.join(', ');
 }
+function insightsForRepo(repoName, context) {
+    return context.repoInsights.get(repoName) ?? EMPTY_REPO_INSIGHTS;
+}
+function shorten(text, max = 180) {
+    const normalized = normalizeMarkdown(text);
+    if (normalized.length <= max) {
+        return normalized;
+    }
+    return `${normalized.slice(0, max - 1).trimEnd()}…`;
+}
+function formatLinks(links, limit = 3) {
+    if (links.length === 0) {
+        return 'Unknown';
+    }
+    return links
+        .slice(0, limit)
+        .map((value) => {
+        if (/^https?:\/\//.test(value)) {
+            return `[doc](${value})`;
+        }
+        return `\`${value}\``;
+    })
+        .join(', ');
+}
+function inferRuntimeFromInsights(insights) {
+    const corpus = [
+        insights.summary,
+        ...insights.responsibilities,
+        ...insights.interfaces,
+        ...insights.localDevelopment,
+    ]
+        .join(' ')
+        .toLowerCase();
+    if (corpus.includes('next.js') || corpus.includes('nextjs')) {
+        return 'Node.js (Next.js)';
+    }
+    if (corpus.includes('nestjs')) {
+        return 'Node.js (NestJS)';
+    }
+    if (corpus.includes('express')) {
+        return 'Node.js (Express)';
+    }
+    if (corpus.includes('fastapi')) {
+        return 'Python (FastAPI)';
+    }
+    if (corpus.includes('django')) {
+        return 'Python (Django)';
+    }
+    if (corpus.includes('spring boot') || corpus.includes('spring')) {
+        return 'JVM (Spring)';
+    }
+    if (corpus.includes('golang') || corpus.includes('go service') || corpus.includes('go microservice')) {
+        return 'Go';
+    }
+    if (corpus.includes('rust')) {
+        return 'Rust';
+    }
+    return undefined;
+}
+function inferDeployTargetFromInsights(insights) {
+    const corpus = [...insights.deployment, insights.summary].join(' ').toLowerCase();
+    if (corpus.includes('kubernetes') || corpus.includes('helm') || corpus.includes('k8s')) {
+        return 'Kubernetes';
+    }
+    if (corpus.includes('vercel')) {
+        return 'Vercel';
+    }
+    if (corpus.includes('ecs') || corpus.includes('fargate')) {
+        return 'AWS ECS/Fargate';
+    }
+    if (corpus.includes('lambda') || corpus.includes('serverless')) {
+        return 'Serverless';
+    }
+    if (corpus.includes('docker') || corpus.includes('container')) {
+        return 'Container';
+    }
+    return undefined;
+}
 function ownerForService(serviceId, context) {
     const overrides = context.config.ownerTeamOverrides ?? {};
     if (overrides[serviceId]) {
@@ -633,17 +987,20 @@ function criticalityForService(serviceId, context) {
 }
 function apiSurfaceForService(serviceId, context) {
     const serviceContracts = context.contracts.filter((record) => record.repo === serviceId);
-    if (serviceContracts.length === 0) {
-        return 'Unknown';
-    }
     const byType = new Map();
     for (const contract of serviceContracts) {
         byType.set(contract.type, (byType.get(contract.type) ?? 0) + 1);
     }
-    return [...byType.entries()]
+    const typed = [...byType.entries()]
         .sort((a, b) => a[0].localeCompare(b[0]))
-        .map(([type, count]) => `${type} (${count})`)
-        .join(', ');
+        .map(([type, count]) => `${type} (${count})`);
+    const insights = insightsForRepo(serviceId, context);
+    const inferred = topUnique([...insights.interfaces, ...insights.asyncPatterns].map((line) => shorten(line, 70)), 2);
+    const composed = [...typed, ...inferred];
+    if (composed.length === 0) {
+        return 'Unknown';
+    }
+    return composed.join(', ');
 }
 function dependenciesForService(serviceId, context) {
     const sourceId = `service:${serviceId}`;
@@ -684,79 +1041,41 @@ function serviceCatalog(context) {
         .sort((a, b) => a.localeCompare(b));
     return serviceIds.map((serviceId) => {
         const repo = context.repoMap.get(serviceId);
+        const insights = insightsForRepo(serviceId, context);
+        const runtime = repo ? inferRuntimeFramework(repo) : 'Unknown';
+        const runtimeFromDocs = inferRuntimeFromInsights(insights);
+        const deploy = repo ? inferDeployTarget(repo) : 'Unknown';
+        const deployFromDocs = inferDeployTargetFromInsights(insights);
+        const serviceDataStores = [...new Set([...datastoresForService(serviceId, context).split(', ').filter((entry) => entry !== 'Unknown'), ...insights.dataStores])]
+            .filter((entry) => entry.length > 0)
+            .sort((a, b) => a.localeCompare(b));
         return {
             serviceName: serviceId,
             repository: repo?.fullName ?? serviceId,
             ownerTeam: ownerForService(serviceId, context),
-            runtime: repo ? inferRuntimeFramework(repo) : 'Unknown',
+            runtime: runtime !== 'Unknown' ? runtime : runtimeFromDocs ?? 'Unknown',
             apiEventSurface: apiSurfaceForService(serviceId, context),
             dependencies: dependenciesForService(serviceId, context),
-            dataStores: datastoresForService(serviceId, context),
-            deployTarget: repo ? inferDeployTarget(repo) : 'Unknown',
+            dataStores: formatList(serviceDataStores),
+            deployTarget: deploy !== 'Unknown' ? deploy : deployFromDocs ?? 'Unknown',
             tier: criticalityForService(serviceId, context),
             status: statusForService(serviceId, context),
         };
     });
 }
-function resolveSectionSources(section, sources) {
-    const ids = new Set(section.sourceIds);
-    return sources
-        .filter((source) => ids.has(source.id))
-        .sort((a, b) => a.label.localeCompare(b.label));
-}
-function renderSourceBlock(sourceRefs) {
-    const lines = ['### Sources', ''];
-    if (sourceRefs.length === 0) {
-        lines.push('- Unknown');
-        lines.push('');
-        return lines;
-    }
-    for (const source of sourceRefs) {
-        const generated = source.generatedAt ?? 'Unknown';
-        const freshness = source.stale ? 'stale' : 'fresh';
-        const suffix = source.note ? ` (${source.note})` : '';
-        lines.push(`- ${source.label}: \`${source.path}\` (generated: ${generated}, ${freshness})${suffix}`);
-    }
-    lines.push('');
-    return lines;
-}
-function renderStaleWarning(sourceRefs) {
-    const stale = sourceRefs.filter((source) => source.required && (source.stale || !source.exists));
-    if (stale.length === 0) {
-        return [];
-    }
-    const lines = ['> [!WARNING]', '> Stale or missing source data detected for this section:', ...stale.map((source) => `> - ${source.label}`), ''];
-    return lines;
-}
-function defaultManualBlockText(sectionId) {
-    return `\nAdd team-specific notes for \`${sectionId}\` here.\n`;
+function sectionAnchor(title) {
+    return title
+        .toLowerCase()
+        .replace(/[^a-z0-9\s-]/g, '')
+        .trim()
+        .replace(/\s+/g, '-');
 }
-function extractManualBlocks(existingContent) {
-    const out = new Map();
-    for (const sectionId of exports.README_SECTION_ORDER) {
-        const escaped = sectionId.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-        const regex = new RegExp(`<!-- SDX:SECTION:${escaped}:MANUAL:START -->([\\s\\S]*?)<!-- SDX:SECTION:${escaped}:MANUAL:END -->`, 'm');
-        const match = existingContent.match(regex);
-        if (match) {
-            out.set(sectionId, match[1]);
-        }
-    }
-    return out;
-}
-function renderSection(section, sources, manualContent) {
+function renderSection(section) {
     const lines = [];
-    lines.push(`<!-- SDX:SECTION:${section.id}:START -->`);
     lines.push(`## ${section.title}`);
     lines.push('');
     lines.push(...section.body);
     lines.push('');
-    const sourceRefs = resolveSectionSources(section, sources);
-    lines.push(...renderStaleWarning(sourceRefs));
-    lines.push(...renderSourceBlock(sourceRefs));
-    const manualBody = manualContent ?? defaultManualBlockText(section.id);
-    lines.push(`<!-- SDX:SECTION:${section.id}:MANUAL:START -->${manualBody}<!-- SDX:SECTION:${section.id}:MANUAL:END -->`);
-    lines.push(`<!-- SDX:SECTION:${section.id}:END -->`);
-    lines.push('');
     return lines.join('\n');
 }
 function splitLines(input) {
@@ -874,7 +1193,7 @@ function unifiedDiff(oldText, newText, oldLabel, newLabel) {
     }
     return `${lines.join('\n')}\n`;
 }
-function buildReadmeContext(mapId, db, cwd, outputPath, config) {
+async function buildReadmeContext(mapId, db, cwd, outputPath, config) {
     const now = new Date();
     const threshold = config.staleThresholdHours ?? 72;
     const scope = (0, scope_1.loadScopeManifest)(mapId, cwd);
@@ -898,6 +1217,16 @@ function buildReadmeContext(mapId, db, cwd, outputPath, config) {
     sourceRefs.push(sourceFromFile('architecture-model', 'Architecture model', node_path_1.default.join(cwd, 'maps', mapId, 'architecture', 'model.json'), cwd, threshold, now, false));
     sourceRefs.push(sourceFromFile('architecture-validation', 'Architecture validation', node_path_1.default.join(cwd, 'maps', mapId, 'architecture', 'validation.json'), cwd, threshold, now, false));
     sourceRefs.push(sourceFromRepoSync(repos, selectedRepos, threshold, now));
+    let githubToken;
+    try {
+        const sdxConfig = (0, config_1.loadConfig)(cwd);
+        const tokenEnv = sdxConfig.github?.tokenEnv?.trim() || 'GITHUB_TOKEN';
+        githubToken = process.env[tokenEnv];
+    }
+    catch {
+        githubToken = process.env['GITHUB_TOKEN'];
+    }
+    const repoInsights = await buildAllRepoInsights(selectedRepos, repoMap, githubToken);
     return {
         cwd,
         mapId,
@@ -916,6 +1245,7 @@ function buildReadmeContext(mapId, db, cwd, outputPath, config) {
         sourceSnapshotAt: computeSnapshotTimestamp(sourceRefs, now),
         coreRequestPath: findCoreRequestPath(model, serviceMap),
         sourceRepoSyncAt: sourceRefs.find((source) => source.id === 'repo-sync')?.generatedAt,
+        repoInsights,
     };
 }
 function buildSections(context) {
@@ -931,18 +1261,10 @@ function buildSections(context) {
     };
     const catalogRows = serviceCatalog(context);
     const asyncContracts = context.contracts.filter((record) => record.type === 'asyncapi');
-    const repoRows = context.selectedRepos.map((repoName) => {
-        const repo = context.repoMap.get(repoName);
-        return {
-            name: repoName,
-            fullName: repo?.fullName ?? repoName,
-            owner: ownerForService(repoName, context),
-            source: repo?.source ?? 'Unknown',
-            branch: repo?.defaultBranch ?? 'Unknown',
-            localPath: repo?.localPath ?? 'Unknown',
-            domain: domainForRepo(repoName, context.config),
-        };
-    });
+    const allInsights = context.selectedRepos.map((repoName) => ({
+        repoName,
+        insights: insightsForRepo(repoName, context),
+    }));
     const datastoreNodes = context.architectureModel.nodes.filter((node) => node.type === 'datastore');
     const adrDir = node_path_1.default.join(context.cwd, 'docs', 'adr');
     const adrFiles = (0, fs_1.fileExists)(adrDir)
@@ -952,25 +1274,81 @@ function buildSections(context) {
             .map((entry) => entry.name)
             .sort((a, b) => a.localeCompare(b))
         : [];
+    const summaryHighlights = allInsights
+        .filter((entry) => entry.insights.summary !== 'Unknown')
+        .sort((a, b) => a.repoName.localeCompare(b.repoName))
+        .slice(0, 8)
+        .map((entry) => `- **${entry.repoName}**: ${shorten(entry.insights.summary, 220)}`);
     const coreFlowLines = context.coreRequestPath.length > 0
         ? context.coreRequestPath.map((edge) => {
             const from = edge.from.replace('service:', '');
             const to = edge.to.replace('service:', '');
-            return `- ${from} -> ${to} (confidence ${edge.provenance.confidence.toFixed(2)})`;
+            return `- ${from} -> ${to}`;
         })
         : ['- Unknown'];
+    const coreFlowServices = topUnique(context.coreRequestPath.flatMap((edge) => [edge.from.replace('service:', ''), edge.to.replace('service:', '')]), 8);
+    const coreFlowNotes = coreFlowServices.map((serviceId) => {
+        const insights = insightsForRepo(serviceId, context);
+        const line = insights.responsibilities[0] ?? insights.interfaces[0] ?? insights.summary;
+        return `- **${serviceId}**: ${shorten(line, 180)}`;
+    });
+    const asyncHighlights = topUnique(allInsights.flatMap((entry) => entry.insights.asyncPatterns.map((line) => `- **${entry.repoName}**: ${shorten(line, 180)}`)), 12);
+    const securityHighlights = topUnique(allInsights.flatMap((entry) => entry.insights.security.map((line) => `- **${entry.repoName}**: ${shorten(line, 180)}`)), 10);
+    const runbookHighlights = topUnique(allInsights.flatMap((entry) => entry.insights.runbooks.map((line) => `- **${entry.repoName}**: ${shorten(line, 180)}`)), 10);
+    const localDevCommands = topUnique(allInsights.flatMap((entry) => entry.insights.localDevelopment), 12);
+    const glossaryTerms = topUnique(allInsights.flatMap((entry) => entry.insights.glossary), 20).sort((a, b) => a.localeCompare(b));
+    const repoRows = context.selectedRepos
+        .map((repoName) => {
+        const repo = context.repoMap.get(repoName);
+        const insights = insightsForRepo(repoName, context);
+        return {
+            fullName: repo?.fullName ?? repoName,
+            owner: ownerForService(repoName, context),
+            domain: domainForRepo(repoName, context.config),
+            role: shorten(insights.responsibilities[0] ?? insights.summary, 140),
+            docs: formatLinks(insights.docReferences, 3),
+        };
+    })
+        .sort((a, b) => a.fullName.localeCompare(b.fullName));
+    const environmentRows = catalogRows.map((row) => {
+        const insights = insightsForRepo(row.serviceName, context);
+        const note = insights.deployment[0] ?? insights.runbooks[0] ?? 'Unknown';
+        return `| ${row.serviceName} | ${row.deployTarget} | ${row.runtime} | ${shorten(note, 140)} |`;
+    });
+    const contractsByRepo = new Map();
+    for (const contract of context.contracts) {
+        const entries = contractsByRepo.get(contract.repo) ?? [];
+        entries.push(contract);
+        contractsByRepo.set(contract.repo, entries);
+    }
+    const contractRows = context.selectedRepos.map((repoName) => {
+        const contracts = contractsByRepo.get(repoName) ?? [];
+        const summary = contracts.length > 0
+            ? contracts.map((contract) => `${contract.type}:${contract.path}`).slice(0, 4).join('<br/>')
+            : 'Unknown';
+        return `| ${repoName} | ${summary} |`;
+    });
+    const adrLinks = topUnique([
+        ...adrFiles.map((fileName) => `- [${fileName}](./docs/adr/${fileName})`),
+        ...allInsights.flatMap((entry) => entry.insights.adrs.map((candidate) => /^https?:\/\//.test(candidate)
+            ? `- [${entry.repoName} ADR](${candidate})`
+            : `- ${entry.repoName}: \`${candidate}\``)),
+    ], 30);
     const sectionById = {
         what_is_this_system: {
             id: 'what_is_this_system',
             title: SECTION_TITLES['what_is_this_system'],
             body: [
                 context.config.customIntro ??
-                    'This README is generated by SDX as the canonical architecture onboarding guide for this org workspace.',
+                    'SDX traverses repository documentation, contracts, and dependency signals to keep this architecture guide current.',
+                '',
+                `- Scope: \`${context.scope.org}\` org, map \`${context.mapId}\``,
+                `- Repositories in scope: ${context.selectedRepos.length}`,
+                `- Services identified: ${catalogRows.length}`,
                 '',
-                `- Organization: \`${context.scope.org}\``,
-                `- Map: \`${context.mapId}\``,
-                `- Repositories selected for this README: ${context.selectedRepos.length}`,
-                `- Services detected: ${catalogRows.length}`,
+                ...(summaryHighlights.length > 0
+                    ? ['### Service purpose highlights', '', ...summaryHighlights]
+                    : ['### Service purpose highlights', '', '- Unknown']),
             ],
             sourceIds: ['scope', 'repo-sync', 'service-map-json'],
         },
@@ -994,6 +1372,11 @@ function buildSections(context) {
                         ? `- [Optional C4 container](${links.optionalContainer})`
                         : '- Optional C4 container: Not available');
                 }
+                if (coreFlowLines.length > 0) {
+                    lines.push('');
+                    lines.push('### Primary request path');
+                    lines.push(...coreFlowLines);
+                }
                 return lines;
             })(),
             sourceIds: [
@@ -1015,6 +1398,9 @@ function buildSections(context) {
                 ...(catalogRows.length > 0
                     ? catalogRows.map((row) => `| ${row.serviceName} | ${row.repository} | ${row.ownerTeam} | ${row.runtime} | ${row.apiEventSurface} | ${row.dependencies} | ${row.dataStores} | ${row.deployTarget} | ${row.tier} | ${row.status} |`)
                     : ['| Unknown | Unknown | Unknown | Unknown | Unknown | Unknown | Unknown | Unknown | Unknown | Unknown |']),
+                '',
+                '### Service briefs',
+                ...(summaryHighlights.length > 0 ? summaryHighlights : ['- Unknown']),
             ],
             sourceIds: ['service-map-json', 'contracts-json', 'architecture-model', 'repo-sync'],
         },
@@ -1023,8 +1409,11 @@ function buildSections(context) {
             title: SECTION_TITLES['critical_flows'],
             body: [
                 `- Primary sequence diagram: [core-request-flow.mmd](${links.sequence})`,
-                '- Highest-confidence path:',
+                '- Current highest-confidence request chain:',
                 ...coreFlowLines,
+                '',
+                '### Service responsibilities in this flow',
+                ...(coreFlowNotes.length > 0 ? coreFlowNotes : ['- Unknown']),
             ],
             sourceIds: ['architecture-model', 'service-map-json', 'docs-dependencies', 'diagram-core-sequence'],
         },
@@ -1037,6 +1426,9 @@ function buildSections(context) {
                 ...(asyncContracts.length > 0
                     ? asyncContracts.map((record) => `| ${record.path} | ${record.repo} | ${record.version ?? 'Unknown'} | ${record.compatibilityStatus} | ${formatList(record.producers)} | ${formatList(record.consumers)} |`)
                     : ['| Unknown | Unknown | Unknown | Unknown | Unknown | Unknown |']),
+                '',
+                '### Async behavior from repository docs',
+                ...(asyncHighlights.length > 0 ? asyncHighlights : ['- Unknown']),
             ],
             sourceIds: ['contracts-json', 'architecture-model'],
         },
@@ -1044,11 +1436,9 @@ function buildSections(context) {
             id: 'contracts_index',
             title: SECTION_TITLES['contracts_index'],
             body: [
-                '| Repository | Type | Path | Version | Compatibility |',
-                '|---|---|---|---|---|',
-                ...(context.contracts.length > 0
-                    ? context.contracts.map((record) => `| ${record.repo} | ${record.type} | ${record.path} | ${record.version ?? 'Unknown'} | ${record.compatibilityStatus} |`)
-                    : ['| Unknown | Unknown | Unknown | Unknown | Unknown |']),
+                '| Repository | Contract surfaces |',
+                '|---|---|',
+                ...(contractRows.length > 0 ? contractRows : ['| Unknown | Unknown |']),
             ],
             sourceIds: ['contracts-json', 'contracts-md'],
         },
@@ -1056,11 +1446,11 @@ function buildSections(context) {
             id: 'repository_index',
             title: SECTION_TITLES['repository_index'],
             body: [
-                '| Repository | Owner/team | Domain | Source | Default branch | Local path |',
-                '|---|---|---|---|---|---|',
+                '| Repository | Owner/team | Domain | Role in system | Key docs |',
+                '|---|---|---|---|---|',
                 ...(repoRows.length > 0
-                    ? repoRows.map((row) => `| ${row.fullName} | ${row.owner} | ${row.domain} | ${row.source} | ${row.branch} | ${row.localPath.replace(/\|/g, '\\|')} |`)
-                    : ['| Unknown | Unknown | Unknown | Unknown | Unknown | Unknown |']),
+                    ? repoRows.map((row) => `| ${row.fullName} | ${row.owner} | ${row.domain} | ${row.role} | ${row.docs} |`)
+                    : ['| Unknown | Unknown | Unknown | Unknown | Unknown |']),
             ],
             sourceIds: ['scope', 'repo-sync'],
         },
@@ -1070,9 +1460,7 @@ function buildSections(context) {
             body: [
                 '| Service | Deploy target | Runtime/framework | Environment notes |',
                 '|---|---|---|---|',
-                ...(catalogRows.length > 0
-                    ? catalogRows.map((row) => `| ${row.serviceName} | ${row.deployTarget} | ${row.runtime} | ${row.deployTarget === 'Unknown' ? 'Unknown' : 'Validate env parity in deployment pipeline'} |`)
-                    : ['| Unknown | Unknown | Unknown | Unknown |']),
+                ...(environmentRows.length > 0 ? environmentRows : ['| Unknown | Unknown | Unknown | Unknown |']),
             ],
             sourceIds: ['service-map-json', 'repo-sync', 'architecture-model'],
         },
@@ -1100,11 +1488,13 @@ function buildSections(context) {
             id: 'security_compliance',
             title: SECTION_TITLES['security_compliance'],
             body: [
-                '- Authentication/authorization model: Unknown',
-                '- Data classification posture: Unknown',
-                '- Compliance scope (SOC2/PCI/HIPAA/etc.): Unknown',
-                '- Secret management baseline: Unknown',
-                '- Required action: populate this section via manual block with org security standards.',
+                ...(securityHighlights.length > 0
+                    ? securityHighlights
+                    : [
+                        '- Authentication and authorization approach: Unknown',
+                        '- Data classification and compliance posture: Unknown',
+                        '- Secret management and key handling: Unknown',
+                    ]),
             ],
             sourceIds: ['architecture-model', 'contracts-json'],
         },
@@ -1114,13 +1504,17 @@ function buildSections(context) {
             body: [
                 '```bash',
                 './scripts/sdx status',
+                `./scripts/sdx repo sync --org ${context.scope.org}`,
                 `./scripts/sdx map build ${context.mapId}`,
                 `./scripts/sdx contracts extract --map ${context.mapId}`,
-                `./scripts/sdx docs generate --map ${context.mapId}`,
+                `./scripts/sdx architecture generate --map ${context.mapId}`,
                 `./scripts/sdx docs readme --map ${context.mapId}`,
                 '```',
                 '',
-                '- Use `--check` in CI to enforce freshness and deterministic output.',
+                '### Commands found in service docs',
+                ...(localDevCommands.length > 0
+                    ? ['```bash', ...localDevCommands.map((command) => command.replace(/`/g, '')), '```']
+                    : ['- Unknown']),
             ],
             sourceIds: ['scope', 'service-map-json', 'contracts-json'],
         },
@@ -1128,32 +1522,25 @@ function buildSections(context) {
             id: 'runbooks_escalation',
             title: SECTION_TITLES['runbooks_escalation'],
             body: [
-                '- Runbook root: `docs/runbooks/` (Unknown if not present)',
-                '- Escalation path: Unknown',
-                '- Incident channel: Unknown',
-                '- Required action: populate escalation ownership in manual block.',
+                ...(runbookHighlights.length > 0
+                    ? runbookHighlights
+                    : ['- Runbooks and escalation notes are not explicitly documented in scanned repositories.']),
             ],
             sourceIds: ['architecture-model', 'repo-sync'],
         },
         adr_index: {
             id: 'adr_index',
             title: SECTION_TITLES['adr_index'],
-            body: [
-                ...(adrFiles.length > 0
-                    ? adrFiles.map((fileName) => `- [${fileName}](./docs/adr/${fileName})`)
-                    : ['- Unknown (no ADR markdown files found under `docs/adr/`)']),
-            ],
+            body: [...(adrLinks.length > 0 ? adrLinks : ['- Unknown'])],
             sourceIds: ['docs-architecture'],
         },
         glossary: {
             id: 'glossary',
             title: SECTION_TITLES['glossary'],
             body: [
-                '- **Service**: A deployable unit represented by a repository in the selected map scope.',
-                '- **Contract**: API/event interface artifact (OpenAPI, GraphQL, Protobuf, AsyncAPI).',
-                '- **Map**: A named SDX scope manifest that defines discovered/included/excluded repos.',
-                '- **Override**: Manual architecture hints in `maps/<map-id>/architecture-overrides.json`.',
-                '- **Unknown**: Field not currently derivable from SDX artifacts; requires manual completion.',
+                ...(glossaryTerms.length > 0
+                    ? glossaryTerms.map((term) => `- **${term}**: Refer to service-level docs for context.`)
+                    : ['- Unknown']),
             ],
             sourceIds: ['scope', 'service-map-json', 'contracts-json', 'architecture-model'],
         },
@@ -1161,33 +1548,32 @@ function buildSections(context) {
             id: 'changelog_metadata',
             title: SECTION_TITLES['changelog_metadata'],
             body: [
-                `- Generated timestamp: ${context.sourceSnapshotAt}`,
-                `- Map id: ${context.mapId}`,
-                `- Schema version: ${constants_1.SCHEMA_VERSION}`,
-                `- CLI version: ${(0, version_1.getCliPackageVersion)()}`,
-                `- Freshness threshold (hours): ${context.staleThresholdHours}`,
-                `- Repo sync baseline: ${context.sourceRepoSyncAt ?? 'Unknown'}`,
-                '- Source refs used:',
-                ...context.sources.map((source) => `  - ${source.label}: \`${source.path}\``),
+                `- Last generated: ${context.sourceSnapshotAt}`,
+                `- Source snapshot: ${context.sourceSnapshotAt}`,
+                `- Map: ${context.mapId}`,
+                `- Tooling: SDX CLI ${(0, version_1.getCliPackageVersion)()} (schema ${constants_1.SCHEMA_VERSION})`,
+                '- Run `./scripts/sdx docs readme --map <map-id> --check` in CI to verify freshness and drift.',
             ],
             sourceIds: context.sources.map((source) => source.id),
         },
     };
     return exports.README_SECTION_ORDER.map((sectionId) => sectionById[sectionId]);
 }
-function renderReadme(sections, context, existingContent) {
-    const manualBlocks = extractManualBlocks(existingContent);
+function renderReadme(sections, context) {
     const lines = [
-        '# SDX Organization Architecture Workspace',
+        `# ${context.scope.org} System Architecture`,
+        '',
+        `This README is the architecture entry point for the \`${context.scope.org}\` engineering organization.`,
+        '',
+        `It is generated by SDX from repository docs, contracts, and map artifacts using \`${context.mapId}\`.`,
         '',
-        `> Generated for org \`${context.scope.org}\` using map \`${context.mapId}\`.`,
+        '## Table of contents',
         '',
-        `> Source snapshot timestamp: ${context.sourceSnapshotAt}`,
+        ...sections.map((section) => `- [${section.title}](#${sectionAnchor(section.title)})`),
         '',
     ];
     for (const section of sections) {
-        const manual = manualBlocks.get(section.id);
-        lines.push(renderSection(section, context.sources, manual));
+        lines.push(renderSection(section));
     }
     return {
         content: `${lines.join('\n').trimEnd()}\n`,
@@ -1217,7 +1603,7 @@ function summarizeResult(outputPath, staleSources, missingSources, changed, chec
     }
     return lines.join('\n');
 }
-function generateReadme(options) {
+async function generateReadme(options) {
     const cwd = options.cwd ?? process.cwd();
     const outputPath = node_path_1.default.resolve(cwd, options.output ?? 'README.md');
     const includeSections = options.includeSections ?? [];
@@ -1227,7 +1613,7 @@ function generateReadme(options) {
     }
     const { config, sourcePath } = loadReadmeConfig(cwd);
     const selectedSections = selectSections(config, includeSections, excludeSections);
-    const context = buildReadmeContext(options.mapId, options.db, cwd, outputPath, config);
+    const context = await buildReadmeContext(options.mapId, options.db, cwd, outputPath, config);
     if (sourcePath) {
         context.sources.push(sourceFromFile('readme-config', 'README config', sourcePath, cwd, context.staleThresholdHours, context.now, false));
     }
@@ -1240,7 +1626,7 @@ function generateReadme(options) {
     context.sourceSnapshotAt = computeSnapshotTimestamp(context.sources, context.now);
     const orderedSections = buildSections(context).filter((section) => selectedSections.includes(section.id));
     const existingContent = (0, fs_1.safeReadText)(outputPath);
-    const rendered = renderReadme(orderedSections, context, existingContent);
+    const rendered = renderReadme(orderedSections, context);
     const { stale, missing, changed } = checkFailures(existingContent, rendered.content, rendered.sourceRefs);
     const shouldWrite = writeEnabled && changed;
     if (shouldWrite) {

package/dist/lib/repoRegistry.js

@@ -61,15 +61,20 @@ function upsertRepos(db, repos) {
 }
 function setLocalRepoPath(db, name, localPath, org) {
     const normalized = node_path_1.default.resolve(localPath);
+    const now = new Date().toISOString();
     const existing = db
         .prepare('SELECT * FROM repo_registry WHERE name = ?')
         .get(name);
     if (existing) {
-        db.prepare(`UPDATE repo_registry SET local_path = ?, source = CASE WHEN source='github' THEN 'hybrid' ELSE 'local' END WHERE name = ?`).run(normalized, name);
+        db.prepare(`UPDATE repo_registry
+       SET local_path = ?,
+           source = CASE WHEN source='github' THEN 'hybrid' ELSE 'local' END,
+           last_synced_at = ?
+       WHERE name = ?`).run(normalized, now, name);
     }
     else {
-        db.prepare(`INSERT INTO repo_registry (name, full_name, org, archived, fork, local_path, source)
-       VALUES (?, ?, ?, 0, 0, ?, 'local')`).run(name, org ? `${org}/${name}` : name, org ?? 'local', normalized);
+        db.prepare(`INSERT INTO repo_registry (name, full_name, org, archived, fork, local_path, source, last_synced_at)
+       VALUES (?, ?, ?, 0, 0, ?, 'local', ?)`).run(name, org ? `${org}/${name}` : name, org ?? 'local', normalized, now);
     }
     return getRepoByName(db, name);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sdx-cli",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "System Design Intelligence CLI",
   "type": "commonjs",
   "bin": {