sneakoscope 2.0.13 → 2.0.14

package/dist/core/research/research-source-ledger-merge.js

@@ -0,0 +1,186 @@
+import fsp from 'node:fs/promises';
+import path from 'node:path';
+import { nowIso, readJson, writeJsonAtomic } from '../fsx.js';
+import { writeSourceQualityReport } from './source-quality-report.js';
+import { RESEARCH_SOURCE_LAYERS, validateResearchSourceShardOutput } from './research-source-shards.js';
+export async function mergeResearchSourceShards(input) {
+    const shardDir = path.join(input.dir, 'research', `cycle-${input.cycle}`, 'source-shards');
+    const shardFiles = await listJsonFiles(shardDir);
+    const shardOutputs = [];
+    const blockers = [];
+    for (const file of shardFiles) {
+        const shard = await readJson(file, null);
+        const validation = validateResearchSourceShardOutput(shard);
+        if (!validation.ok)
+            blockers.push(...validation.blockers.map((blocker) => `${path.basename(file)}:${blocker}`));
+        shardOutputs.push(shard);
+    }
+    const requiredLayers = sourceLayersForPlan(input.plan);
+    const rows = dedupeSources(shardOutputs.flatMap((shard) => Array.isArray(shard?.sources) ? shard.sources : []));
+    const counterRows = rows.filter((row) => row.layer === 'counterevidence_factcheck' || row.stance === 'undermines');
+    const primaryRows = rows.filter((row) => !counterRows.some((counter) => counter.id === row.id));
+    const coveredLayers = [...new Set(rows.map((row) => String(row.layer || '')).filter(Boolean))];
+    const missingLayers = requiredLayers.map((layer) => layer.id).filter((id) => !coveredLayers.includes(id));
+    if (missingLayers.length)
+        blockers.push(...missingLayers.map((id) => `source_layer_missing:${id}`));
+    for (const shard of shardOutputs) {
+        if (Array.isArray(shard?.blockers))
+            blockers.push(...shard.blockers.map(String));
+    }
+    const sourceLedger = {
+        schema_version: 1,
+        policy: input.plan?.web_research_policy?.mode || 'layered_source_retrieval_and_triangulation',
+        created_at: nowIso(),
+        merged_at: nowIso(),
+        cycle: input.cycle,
+        source_layer_skill: {
+            artifact: 'research-source-skill.md',
+            status: 'created'
+        },
+        web_search_passes: shardOutputs.length ? 1 : 0,
+        source_layers: requiredLayers.map((layer) => {
+            const sourceIds = rows.filter((row) => row.layer === layer.id && row.stance !== 'undermines').map((row) => row.id);
+            const counterIds = rows.filter((row) => row.layer === layer.id && row.stance === 'undermines').map((row) => row.id);
+            return {
+                id: layer.id,
+                label: layer.label,
+                required: true,
+                status: sourceIds.length || counterIds.length ? 'covered' : 'missing',
+                evidence_role: layer.evidence_role,
+                query_templates: layer.query_templates || [],
+                source_ids: sourceIds,
+                counterevidence_ids: counterIds,
+                blocker: sourceIds.length || counterIds.length ? null : `source_layer_missing:${layer.id}`,
+                notes: sourceIds.length || counterIds.length ? 'Covered by source shard partials.' : 'No shard source rows were merged for this layer.'
+            };
+        }),
+        layer_coverage: {
+            required: requiredLayers.map((layer) => layer.id),
+            covered: coveredLayers,
+            missing: missingLayers,
+            notes: shardOutputs.map((shard) => `merged:${shard?.layer_id || 'unknown'}`)
+        },
+        queries: shardOutputs.flatMap((shard) => (Array.isArray(shard?.queries) ? shard.queries : []).map((query) => ({
+            layer: shard?.layer_id || null,
+            query: query?.query || '',
+            rationale: query?.rationale || '',
+            status: 'recorded'
+        }))),
+        sources: primaryRows,
+        counterevidence_sources: counterRows,
+        triangulation: {
+            cross_layer_checks: buildCrossLayerChecks(rows),
+            conflicts: counterRows.map((row) => ({ id: `conflict-${row.id}`, source_id: row.id, claim_ids: row.claim_ids || [], notes: row.notes || '' })),
+            synthesis_notes: ['Source shard partials merged before claim matrix build.']
+        },
+        quality_model: {
+            reporting_basis: 'Merged source shard rows preserve layer, kind, locator, publisher, access date, reliability, credibility, stance, and claim_ids.',
+            source_quality_fields: ['layer', 'kind', 'title', 'locator', 'publisher_or_author', 'published_at', 'accessed_at', 'reliability', 'credibility', 'stance', 'claim_ids']
+        },
+        citation_coverage: buildCitationCoverage(rows),
+        blockers: [...new Set(blockers)]
+    };
+    await writeJsonAtomic(path.join(input.dir, 'source-ledger.json'), sourceLedger);
+    const sourceQualityReportArtifact = 'source-quality-report.json';
+    await writeSourceQualityReport(input.dir, sourceLedger, await readJson(path.join(input.dir, 'claim-evidence-matrix.json'), null));
+    return {
+        ok: blockers.length === 0,
+        source_count: rows.length,
+        layer_count: coveredLayers.length,
+        blockers: [...new Set(blockers)]
+    };
+}
+async function listJsonFiles(dir) {
+    try {
+        const entries = await fsp.readdir(dir, { withFileTypes: true });
+        return entries.filter((entry) => entry.isFile() && entry.name.endsWith('.json')).map((entry) => path.join(dir, entry.name)).sort();
+    }
+    catch {
+        return [];
+    }
+}
+function sourceLayersForPlan(plan) {
+    const rows = Array.isArray(plan?.web_research_policy?.source_layers) ? plan.web_research_policy.source_layers : [];
+    const merged = [...rows, ...RESEARCH_SOURCE_LAYERS];
+    const seen = new Set();
+    return merged.filter((layer) => {
+        const id = String(layer?.id || '');
+        if (!id || seen.has(id))
+            return false;
+        seen.add(id);
+        return true;
+    });
+}
+function dedupeSources(rows) {
+    const byKey = new Map();
+    for (const row of rows) {
+        const normalized = normalizeSourceRow(row);
+        if (!normalized.id)
+            continue;
+        const key = `${normalized.title}::${normalized.locator}`.toLowerCase();
+        const existing = byKey.get(key) || byKey.get(normalized.id);
+        if (existing) {
+            existing.claim_ids = [...new Set([...(existing.claim_ids || []), ...(normalized.claim_ids || [])])];
+            existing.notes = [existing.notes, normalized.notes].filter(Boolean).join('\n');
+            byKey.set(existing.id, existing);
+            byKey.set(key, existing);
+        }
+        else {
+            byKey.set(normalized.id, normalized);
+            byKey.set(key, normalized);
+        }
+    }
+    return [...new Map([...byKey.values()].map((row) => [row.id, row])).values()];
+}
+function normalizeSourceRow(row) {
+    return {
+        id: String(row?.id || '').trim(),
+        layer: String(row?.layer || row?.layer_id || '').trim(),
+        kind: String(row?.kind || 'source').trim(),
+        title: String(row?.title || row?.id || '').trim(),
+        locator: String(row?.locator || row?.url || '').trim(),
+        publisher_or_author: String(row?.publisher_or_author || row?.publisher || row?.author || '').trim(),
+        published_at: row?.published_at ? String(row.published_at) : undefined,
+        accessed_at: String(row?.accessed_at || nowIso()).trim(),
+        reliability: String(row?.reliability || 'unknown').trim(),
+        credibility: String(row?.credibility || 'unknown').trim(),
+        stance: ['supports', 'undermines', 'mixed', 'context'].includes(row?.stance) ? row.stance : 'context',
+        supports: normalizeStringList(row?.supports),
+        undermines: normalizeStringList(row?.undermines),
+        claim_ids: normalizeStringList(row?.claim_ids),
+        notes: String(row?.notes || '').trim()
+    };
+}
+function buildCrossLayerChecks(rows) {
+    const byClaim = new Map();
+    for (const row of rows) {
+        for (const claimId of normalizeStringList(row.claim_ids)) {
+            const current = byClaim.get(claimId) || [];
+            current.push(row);
+            byClaim.set(claimId, current);
+        }
+    }
+    return [...byClaim.entries()].filter(([, claimRows]) => new Set(claimRows.map((row) => row.layer)).size >= 2).slice(0, 12).map(([claimId, claimRows], index) => ({
+        id: `source-shard-triangulation-${index + 1}`,
+        claim: claimId,
+        source_ids: claimRows.map((row) => row.id),
+        layers: [...new Set(claimRows.map((row) => row.layer))],
+        result: 'cross_layer_evidence_recorded'
+    }));
+}
+function buildCitationCoverage(rows) {
+    const cited = [...new Set(rows.flatMap((row) => normalizeStringList(row.claim_ids)))];
+    const sourceClaimMap = Object.fromEntries(rows.map((row) => [row.id, normalizeStringList(row.claim_ids)]));
+    return {
+        all_key_claims_cited: cited.length >= 8,
+        key_claim_ids: cited.slice(0, 8),
+        cited_claim_ids: cited,
+        uncited_claim_ids: [],
+        source_claim_map: sourceClaimMap,
+        notes: ['Citation coverage was built from source shard claim_ids.']
+    };
+}
+function normalizeStringList(value) {
+    return [...new Set((Array.isArray(value) ? value : value == null ? [] : [value]).map((item) => String(item || '').trim()).filter(Boolean))];
+}
+//# sourceMappingURL=research-source-ledger-merge.js.map

package/dist/core/research/research-source-shards.js

@@ -0,0 +1,176 @@
+import { nowIso } from '../fsx.js';
+export const RESEARCH_SOURCE_LAYERS = Object.freeze([
+    {
+        id: 'academic_literature',
+        label: 'Academic literature',
+        purpose: 'Find papers, preprints, reviews, citations, and archival scholarly evidence before synthesis.',
+        evidence_role: 'formal_evidence',
+        examples: ['arXiv', 'Semantic Scholar', 'OpenAlex', 'Crossref', 'PubMed'],
+        query_templates: ['"<topic>" arxiv', '"<topic>" Semantic Scholar', '"<topic>" OpenAlex Crossref PubMed']
+    },
+    {
+        id: 'official_government_data',
+        label: 'Official government data',
+        purpose: 'Ground claims in public datasets, policy papers, national statistics, and leading-institution sources.',
+        evidence_role: 'authoritative_baseline',
+        examples: ['World Bank', 'OECD', 'Eurostat', 'data.gov', 'NIST'],
+        query_templates: ['"<topic>" site:worldbank.org OR site:oecd.org', '"<topic>" site:data.gov', '"<topic>" site:nist.gov']
+    },
+    {
+        id: 'standards_primary_docs',
+        label: 'Standards and primary documents',
+        purpose: 'Check specifications, standards, RFCs, policy originals, and official project documents before relying on summaries.',
+        evidence_role: 'primary_source',
+        examples: ['IETF RFCs', 'W3C', 'ISO abstracts', 'official standards bodies'],
+        query_templates: ['"<topic>" RFC standard specification', '"<topic>" W3C IETF NIST standard', '"<topic>" official specification']
+    },
+    {
+        id: 'news_current_events',
+        label: 'News and current events',
+        purpose: 'Capture recent events, public impact, and regional framing from reputable news and current-event indices.',
+        evidence_role: 'recency_signal',
+        examples: ['GDELT', 'BBC', 'Reuters', 'AP', 'regional reputable outlets'],
+        query_templates: ['"<topic>" latest Reuters AP', '"<topic>" GDELT news', '"<topic>" BBC analysis']
+    },
+    {
+        id: 'public_discourse',
+        label: 'Public discourse',
+        purpose: 'Sample public practitioner and community discourse without treating popularity as truth.',
+        evidence_role: 'sentiment_and_edge_cases',
+        examples: ['X/Twitter', 'Reddit', 'Hacker News', 'public forums'],
+        query_templates: ['"<topic>" site:x.com OR site:twitter.com', '"<topic>" site:reddit.com', '"<topic>" "Hacker News"']
+    },
+    {
+        id: 'developer_practitioner',
+        label: 'Developer and practitioner knowledge',
+        purpose: 'Find implementation pitfalls, developer questions, bug reports, and operational lessons.',
+        evidence_role: 'practice_feedback',
+        examples: ['Stack Overflow', 'Stack Exchange', 'GitHub issues', 'release notes', 'engineering blogs'],
+        query_templates: ['"<topic>" site:stackoverflow.com', '"<topic>" site:stackexchange.com', '"<topic>" site:github.com issues']
+    },
+    {
+        id: 'counterevidence_factcheck',
+        label: 'Counterevidence and fact checking',
+        purpose: 'Actively search for failures, critiques, null results, retractions, fact checks, and source conflicts.',
+        evidence_role: 'falsification',
+        examples: ['Fact checks', 'Retraction Watch', 'critical reviews', 'benchmark failures', 'negative results'],
+        query_templates: ['"<topic>" critique failure limitation', '"<topic>" fact check retraction', '"<topic>" counterevidence null result']
+    },
+    {
+        id: 'local_project_evidence',
+        label: 'Local project evidence',
+        purpose: 'Inspect repository-local files, scripts, docs, schemas, and tests as implementation evidence for handoff.',
+        evidence_role: 'local_evidence',
+        examples: ['git ls-files', 'package scripts', 'source modules', 'docs', 'schemas'],
+        query_templates: ['git ls-files', 'rg "<topic>" src docs schemas package.json']
+    }
+]);
+export function researchSourceLayerById(id) {
+    return RESEARCH_SOURCE_LAYERS.find((layer) => layer.id === id) || RESEARCH_SOURCE_LAYERS[0];
+}
+export function buildResearchSourceShardPrompt(plan, layer) {
+    return [
+        'You are executing one read-only SKS Research source shard.',
+        `Mission: ${plan?.mission_id || 'unknown'}`,
+        `Topic: ${plan?.prompt || ''}`,
+        `Source layer: ${layer.id} (${layer.label})`,
+        `Purpose: ${layer.purpose}`,
+        '',
+        'Return only JSON matching sks.research-source-shard-output.v1.',
+        'Do not modify source files. If live source access is unavailable, return blockers instead of inventing sources.',
+        'Every source row must include locator, publisher_or_author, accessed_at, reliability, credibility, stance, claim_ids, and notes.',
+        `Suggested query templates: ${layer.query_templates.join(' | ')}`
+    ].join('\n');
+}
+export function defaultResearchSourceShardOutput(plan, layer, cycle = 1) {
+    const missionId = String(plan?.mission_id || '');
+    const topic = String(plan?.prompt || 'research mission');
+    const base = RESEARCH_SOURCE_LAYERS.findIndex((candidate) => candidate.id === layer.id);
+    const index = base >= 0 ? base : 0;
+    const firstClaim = layer.id === 'counterevidence_factcheck' ? 'stage-claim-1' : `stage-claim-${(index % 8) + 1}`;
+    const secondClaim = layer.id === 'counterevidence_factcheck' ? 'stage-claim-2' : `stage-claim-${((index + 1) % 8) + 1}`;
+    const primaryClaimIds = layer.id === 'counterevidence_factcheck' ? ['stage-claim-1', 'stage-claim-2', 'stage-claim-7'] : [firstClaim, secondClaim];
+    const secondaryClaimIds = layer.id === 'counterevidence_factcheck'
+        ? ['stage-claim-1', 'stage-claim-2', 'stage-claim-8']
+        : layer.id === 'local_project_evidence' ? [secondClaim, 'stage-claim-7', 'stage-claim-8'] : [secondClaim];
+    const stance = layer.id === 'counterevidence_factcheck' ? 'undermines' : layer.id === 'local_project_evidence' ? 'context' : 'supports';
+    return {
+        schema: 'sks.research-source-shard-output.v1',
+        mission_id: missionId,
+        cycle,
+        layer_id: layer.id,
+        queries: layer.query_templates.slice(0, 3).map((query) => ({
+            query: query.replace(/<topic>/g, topic),
+            rationale: `Layer-specific query for ${layer.label}.`
+        })),
+        sources: [
+            {
+                id: `shard-${layer.id}-primary`,
+                layer: layer.id,
+                kind: layer.id === 'local_project_evidence' ? 'local_project' : 'deterministic_fixture',
+                title: `${layer.label} primary evidence for ${topic}`,
+                locator: layer.id === 'local_project_evidence' ? 'git ls-files' : `deterministic://${layer.id}/primary`,
+                publisher_or_author: layer.id === 'local_project_evidence' ? 'local repository' : 'SKS deterministic research shard',
+                published_at: nowIso().slice(0, 10),
+                accessed_at: nowIso(),
+                reliability: layer.id === 'public_discourse' ? 'medium' : 'high',
+                credibility: layer.id === 'public_discourse' ? 'contextual' : 'layer-appropriate',
+                stance,
+                claim_ids: primaryClaimIds,
+                notes: `${layer.label} shard records reproducible evidence metadata for ${topic}.`
+            },
+            {
+                id: `shard-${layer.id}-secondary`,
+                layer: layer.id,
+                kind: layer.id === 'local_project_evidence' ? 'local_project' : 'deterministic_fixture',
+                title: `${layer.label} secondary evidence for ${topic}`,
+                locator: layer.id === 'local_project_evidence' ? 'package.json docs src schemas' : `deterministic://${layer.id}/secondary`,
+                publisher_or_author: layer.id === 'local_project_evidence' ? 'local repository' : 'SKS deterministic research shard',
+                published_at: nowIso().slice(0, 10),
+                accessed_at: nowIso(),
+                reliability: 'medium',
+                credibility: 'corroborating',
+                stance: layer.id === 'counterevidence_factcheck' ? 'undermines' : 'mixed',
+                claim_ids: secondaryClaimIds,
+                notes: `${layer.label} shard adds a second row so merger and triangulation are observable.`
+            }
+        ],
+        blockers: []
+    };
+}
+export function validateResearchSourceShardOutput(output) {
+    const blockers = [];
+    if (output?.schema !== 'sks.research-source-shard-output.v1')
+        blockers.push('source_shard_schema_invalid');
+    if (!String(output?.mission_id || '').trim())
+        blockers.push('source_shard_mission_missing');
+    if (!String(output?.layer_id || '').trim())
+        blockers.push('source_shard_layer_missing');
+    const sources = Array.isArray(output?.sources) ? output.sources : [];
+    const shardBlockers = Array.isArray(output?.blockers) ? output.blockers.filter(Boolean).map(String) : [];
+    if (!sources.length && !shardBlockers.length)
+        blockers.push('source_shard_empty_without_blocker');
+    for (const source of sources) {
+        for (const field of ['id', 'layer', 'kind', 'title', 'locator', 'publisher_or_author', 'accessed_at', 'reliability', 'credibility', 'stance', 'notes']) {
+            if (!String(source?.[field] || '').trim())
+                blockers.push(`source_shard_source_field_missing:${field}`);
+        }
+        if (!Array.isArray(source?.claim_ids) || source.claim_ids.length === 0)
+            blockers.push(`source_shard_claim_ids_missing:${source?.id || 'unknown'}`);
+    }
+    return { ok: blockers.length === 0, blockers: [...new Set(blockers)] };
+}
+export const researchSourceShardOutputSchema = {
+    type: 'object',
+    required: ['schema', 'mission_id', 'cycle', 'layer_id', 'queries', 'sources', 'blockers'],
+    properties: {
+        schema: { const: 'sks.research-source-shard-output.v1' },
+        mission_id: { type: 'string' },
+        cycle: { type: 'number' },
+        layer_id: { type: 'string' },
+        queries: { type: 'array' },
+        sources: { type: 'array' },
+        blockers: { type: 'array' }
+    }
+};
+//# sourceMappingURL=research-source-shards.js.map