pulseed 0.4.4 → 0.4.6

package/dist/platform/soil/importer.js

@@ -0,0 +1,145 @@
+import * as path from "node:path";
+import { z } from "zod";
+import { computeSoilChecksum } from "./checksum.js";
+import { createSoilConfig } from "./config.js";
+import { readJsonFileOrNull, writeJsonFileAtomic } from "../../base/utils/json-io.js";
+import { loadSoilManifest } from "./retriever.js";
+const OVERLAY_BEGIN = "<!-- soil:overlay-begin -->";
+const OVERLAY_END = "<!-- soil:overlay-end -->";
+const OVERLAY_QUEUE_FILE = "overlay-queue.json";
+export const SoilOverlayStatusSchema = z.enum(["candidate", "approved", "rejected", "superseded"]);
+export const SoilOverlayCandidateSchema = z.object({
+    overlay_id: z.string().min(1),
+    status: SoilOverlayStatusSchema,
+    soil_id: z.string().min(1),
+    relative_path: z.string().min(1),
+    target_ref: z.string().min(1),
+    content: z.string(),
+    content_hash: z.string().min(1),
+    detected_at: z.string().datetime(),
+    updated_at: z.string().datetime(),
+    decision_note: z.string().optional(),
+});
+export const SoilOverlayQueueSchema = z.object({
+    version: z.literal("soil-overlay-queue-v1"),
+    updated_at: z.string().datetime(),
+    overlays: z.array(SoilOverlayCandidateSchema),
+});
+function nowIso(clock) {
+    return (clock?.() ?? new Date()).toISOString();
+}
+function overlayQueuePath(rootDir) {
+    return path.join(rootDir, ".index", OVERLAY_QUEUE_FILE);
+}
+function overlayId(relativePath, index, content) {
+    return computeSoilChecksum({ relativePath, index, content }).replace(/^sha256:/, "overlay-");
+}
+function extractOverlayBlocks(body) {
+    const blocks = [];
+    let cursor = 0;
+    while (cursor < body.length) {
+        const begin = body.indexOf(OVERLAY_BEGIN, cursor);
+        if (begin < 0) {
+            break;
+        }
+        const contentStart = begin + OVERLAY_BEGIN.length;
+        const end = body.indexOf(OVERLAY_END, contentStart);
+        if (end < 0) {
+            break;
+        }
+        const content = body.slice(contentStart, end).trim();
+        if (content) {
+            blocks.push({ index: blocks.length, content });
+        }
+        cursor = end + OVERLAY_END.length;
+    }
+    return blocks;
+}
+export async function loadSoilOverlayQueue(configInput = {}) {
+    const config = createSoilConfig(configInput);
+    const raw = await readJsonFileOrNull(overlayQueuePath(config.rootDir));
+    const parsed = SoilOverlayQueueSchema.safeParse(raw);
+    return parsed.success
+        ? parsed.data
+        : { version: "soil-overlay-queue-v1", updated_at: new Date(0).toISOString(), overlays: [] };
+}
+export async function scanSoilOverlays(configInput = {}, options = {}) {
+    const config = createSoilConfig(configInput);
+    const detectedAt = nowIso(options.clock);
+    const manifest = await loadSoilManifest({ rootDir: config.rootDir, indexPath: config.indexPath });
+    const overlays = [];
+    for (const page of manifest.pages) {
+        const blocks = extractOverlayBlocks(page.body);
+        for (const block of blocks) {
+            const contentHash = computeSoilChecksum(block.content);
+            overlays.push(SoilOverlayCandidateSchema.parse({
+                overlay_id: overlayId(page.relativePath, block.index, block.content),
+                status: "candidate",
+                soil_id: page.soilId,
+                relative_path: page.relativePath,
+                target_ref: `${page.soilId}#overlay-${block.index}`,
+                content: block.content,
+                content_hash: contentHash,
+                detected_at: detectedAt,
+                updated_at: detectedAt,
+            }));
+        }
+    }
+    return overlays;
+}
+export async function scanAndStoreSoilOverlays(configInput = {}, options = {}) {
+    const config = createSoilConfig(configInput);
+    const now = nowIso(options.clock);
+    const existing = await loadSoilOverlayQueue(config);
+    const existingById = new Map(existing.overlays.map((overlay) => [overlay.overlay_id, overlay]));
+    const detected = await scanSoilOverlays(config, options);
+    const merged = detected.map((overlay) => {
+        const previous = existingById.get(overlay.overlay_id);
+        if (!previous) {
+            return overlay;
+        }
+        return SoilOverlayCandidateSchema.parse({
+            ...overlay,
+            status: previous.status,
+            detected_at: previous.detected_at,
+            updated_at: previous.updated_at,
+            decision_note: previous.decision_note,
+        });
+    });
+    const queue = SoilOverlayQueueSchema.parse({
+        version: "soil-overlay-queue-v1",
+        updated_at: now,
+        overlays: merged,
+    });
+    await writeJsonFileAtomic(overlayQueuePath(config.rootDir), queue);
+    return queue;
+}
+export async function updateSoilOverlayStatus(overlayId, status, configInput = {}, options = {}) {
+    const config = createSoilConfig(configInput);
+    const queue = await loadSoilOverlayQueue(config);
+    const now = nowIso(options.clock);
+    let found = false;
+    const overlays = queue.overlays.map((overlay) => {
+        if (overlay.overlay_id !== overlayId) {
+            return overlay;
+        }
+        found = true;
+        return SoilOverlayCandidateSchema.parse({
+            ...overlay,
+            status,
+            updated_at: now,
+            decision_note: options.decisionNote ?? overlay.decision_note,
+        });
+    });
+    if (!found) {
+        throw new Error(`Soil overlay not found: ${overlayId}`);
+    }
+    const next = SoilOverlayQueueSchema.parse({
+        version: "soil-overlay-queue-v1",
+        updated_at: now,
+        overlays,
+    });
+    await writeJsonFileAtomic(overlayQueuePath(config.rootDir), next);
+    return next;
+}
+//# sourceMappingURL=importer.js.map

package/dist/platform/soil/importer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"importer.js","sourceRoot":"","sources":["../../../src/platform/soil/importer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAwB,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AACtF,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAElD,MAAM,aAAa,GAAG,6BAA6B,CAAC;AACpD,MAAM,WAAW,GAAG,2BAA2B,CAAC;AAChD,MAAM,kBAAkB,GAAG,oBAAoB,CAAC;AAEhD,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC,CAAC;AAGnG,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,MAAM,EAAE,uBAAuB;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1B,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAChC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACrC,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,uBAAuB,CAAC;IAC3C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC;CAC9C,CAAC,CAAC;AAOH,SAAS,MAAM,CAAC,KAAkB;IAChC,OAAO,CAAC,KAAK,EAAE,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AACjD,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe;IACvC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,SAAS,CAAC,YAAoB,EAAE,KAAa,EAAE,OAAe;IACrE,OAAO,mBAAmB,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;AAC/F,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACxC,MAAM,MAAM,GAA8C,EAAE,CAAC;IAC7D,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,OAAO,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QAClD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,MAAM;QACR,CAAC;QACD,MAAM,YAAY,GAAG,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC;QAClD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;QACpD,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YACZ,MAAM;QACR,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACrD,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACjD,CAAC;QACD,MAAM,GAAG,GAAG,GAAG,WAAW,CAAC,MAAM,CAAC;IACpC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,cAA+B,EAAE;IAC1E,MAAM,MAAM,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,sBAAsB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,MAAM,CAAC,OAAO;QACnB,CAAC,CAAC,MAAM,CAAC,IAAI;QACb,CAAC,CAAC,EAAE,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;AAChG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,cAA+B,EAAE,EACjC,UAA6B,EAAE;IAE/B,MAAM,MAAM,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IAClG,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,WAAW,GAAG,mBAAmB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACvD,QAAQ,CAAC,IAAI,CAAC,0BAA0B,CAAC,KAAK,CAAC;gBAC7C,UAAU,EAAE,SAAS,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC;gBACpE,MAAM,EAAE,WAAW;gBACnB,OAAO,EAAE,IAAI,CAAC,MAAM;gBACpB,aAAa,EAAE,IAAI,CAAC,YAAY;gBAChC,UAAU,EAAE,GAAG,IAAI,CAAC,MAAM,YAAY,KAAK,CAAC,KAAK,EAAE;gBACnD,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,YAAY,EAAE,WAAW;gBACzB,WAAW,EAAE,UAAU;gBACvB,UAAU,EAAE,UAAU;aACvB,CAAC,CAAC,CAAC;QACN,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,cAA+B,EAAE,EACjC,UAA6B,EAAE;IAE/B,MAAM,MAAM,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC;IACpD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAChG,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QACtC,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACtD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,OAAO,0BAA0B,CAAC,KAAK,CAAC;YACtC,GAAG,OAAO;YACV,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,aAAa,EAAE,QAAQ,CAAC,aAAa;SACtC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,sBAAsB,CAAC,KAAK,CAAC;QACzC,OAAO,EAAE,uBAAuB;QAChC,UAAU,EAAE,GAAG;QACf,QAAQ,EAAE,MAAM;KACjB,CAAC,CAAC;IACH,MAAM,mBAAmB,CAAC,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC;IACnE,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,SAAiB,EACjB,MAAyB,EACzB,cAA+B,EAAE,EACjC,UAAyD,EAAE;IAE3D,MAAM,MAAM,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QAC9C,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,KAAK,GAAG,IAAI,CAAC;QACb,OAAO,0BAA0B,CAAC,KAAK,CAAC;YACtC,GAAG,OAAO;YACV,MAAM;YACN,UAAU,EAAE,GAAG;YACf,aAAa,EAAE,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,aAAa;SAC7D,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,EAAE,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,IAAI,GAAG,sBAAsB,CAAC,KAAK,CAAC;QACxC,OAAO,EAAE,uBAAuB;QAChC,UAAU,EAAE,GAAG;QACf,QAAQ;KACT,CAAC,CAAC;IACH,MAAM,mBAAmB,CAAC,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;IAClE,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/platform/soil/index-store.d.ts

@@ -0,0 +1,89 @@
+import { type SoilConfigInput } from "./config.js";
+import { type SoilPageStore } from "./retriever.js";
+import { type SoilGenerationWatermark, type SoilManualOverlay, type SoilSourceRef, type SoilSourceTruth } from "./types.js";
+export declare const SOIL_INDEX_STORAGE_FORMAT: "file-json-v1";
+export interface SoilIndexChunkSnapshot {
+    chunk_id: string;
+    soil_id: string;
+    relative_path: string;
+    absolute_path: string;
+    chunk_type: "heading" | "paragraph";
+    heading_path: string[];
+    heading?: string;
+    position: number;
+    text: string;
+    token_count: number;
+}
+export interface SoilIndexPageSnapshot {
+    soil_id: string;
+    relative_path: string;
+    absolute_path: string;
+    title: string;
+    kind: string;
+    route: string;
+    status: string;
+    source: string;
+    version: string;
+    created_at: string;
+    updated_at: string;
+    generated_at: string;
+    summary?: string;
+    source_refs: SoilSourceRef[];
+    generation_watermark: SoilGenerationWatermark;
+    stale: boolean;
+    manual_overlay: SoilManualOverlay;
+    source_truth?: SoilSourceTruth;
+    page_format_version?: string;
+    checksum?: string;
+    chunk_count: number;
+}
+export interface SoilIndexRetrievalRun {
+    query: string;
+    limit: number;
+    generated_at: string;
+    hit_count: number;
+}
+export interface SoilIndexSnapshot {
+    storage: typeof SOIL_INDEX_STORAGE_FORMAT;
+    root_dir: string;
+    index_path: string;
+    generated_at: string;
+    source_manifest_checksum: string;
+    page_count: number;
+    chunk_count: number;
+    pages: SoilIndexPageSnapshot[];
+    chunks: SoilIndexChunkSnapshot[];
+    retrieval_runs: SoilIndexRetrievalRun[];
+}
+export interface SoilIndexQueryHit {
+    soil_id: string;
+    relative_path: string;
+    absolute_path: string;
+    title: string;
+    kind: string;
+    route: string;
+    status: string;
+    summary?: string;
+    score: number;
+    snippet: string;
+    chunk_count: number;
+    checksum?: string;
+}
+export interface SoilIndexBuildOptions {
+    clock?: () => Date;
+    store?: SoilPageStore;
+}
+export interface SoilIndexFreshnessReport {
+    fresh: boolean;
+    reason: "missing-index" | "manifest-checksum-mismatch" | "page-count-mismatch" | "fresh";
+    indexPath: string;
+    indexedManifestChecksum?: string;
+    currentManifestChecksum?: string;
+    indexedPageCount?: number;
+    currentPageCount?: number;
+}
+export declare function rebuildSoilIndex(configInput?: SoilConfigInput, options?: SoilIndexBuildOptions): Promise<SoilIndexSnapshot>;
+export declare function loadSoilIndexSnapshot(configInput?: SoilConfigInput): Promise<SoilIndexSnapshot | null>;
+export declare function checkSoilIndexFresh(configInput?: SoilConfigInput, options?: Pick<SoilIndexBuildOptions, "store">): Promise<SoilIndexFreshnessReport>;
+export declare function querySoilIndexSnapshot(query: string, limit?: number, configInput?: SoilConfigInput): Promise<SoilIndexQueryHit[]>;
+//# sourceMappingURL=index-store.d.ts.map

package/dist/platform/soil/index-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index-store.d.ts","sourceRoot":"","sources":["../../../src/platform/soil/index-store.ts"],"names":[],"mappings":"AAKA,OAAO,EAAoB,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAyC,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC3F,OAAO,EAML,KAAK,uBAAuB,EAC5B,KAAK,iBAAiB,EACtB,KAAK,aAAa,EAClB,KAAK,eAAe,EACrB,MAAM,YAAY,CAAC;AAEpB,eAAO,MAAM,yBAAyB,EAAG,cAAuB,CAAC;AAEjE,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,SAAS,GAAG,WAAW,CAAC;IACpC,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,aAAa,EAAE,CAAC;IAC7B,oBAAoB,EAAE,uBAAuB,CAAC;IAC9C,KAAK,EAAE,OAAO,CAAC;IACf,cAAc,EAAE,iBAAiB,CAAC;IAClC,YAAY,CAAC,EAAE,eAAe,CAAC;IAC/B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,yBAAyB,CAAC;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,wBAAwB,EAAE,MAAM,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,MAAM,EAAE,sBAAsB,EAAE,CAAC;IACjC,cAAc,EAAE,qBAAqB,EAAE,CAAC;CACzC;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,CAAC,EAAE,MAAM,IAAI,CAAC;IACnB,KAAK,CAAC,EAAE,aAAa,CAAC;CACvB;AAED,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,eAAe,GAAG,4BAA4B,GAAG,qBAAqB,GAAG,OAAO,CAAC;IACzF,SAAS,EAAE,MAAM,CAAC;IAClB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAySD,wBAAsB,gBAAgB,CACpC,WAAW,GAAE,eAAoB,EACjC,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,iBAAiB,CAAC,CAwB5B;AAED,wBAAsB,qBAAqB,CACzC,WAAW,GAAE,eAAoB,GAChC,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAWnC;AAED,wBAAsB,mBAAmB,CACvC,WAAW,GAAE,eAAoB,EACjC,OAAO,GAAE,IAAI,CAAC,qBAAqB,EAAE,OAAO,CAAM,GACjD,OAAO,CAAC,wBAAwB,CAAC,CAkDnC;AAED,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,MAAM,EACb,KAAK,SAAK,EACV,WAAW,GAAE,eAAoB,GAChC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAoC9B"}

package/dist/platform/soil/index-store.js

@@ -0,0 +1,399 @@
+import * as fsp from "node:fs/promises";
+import * as path from "node:path";
+import { z } from "zod";
+import { readJsonFileOrNull, writeJsonFileAtomic } from "../../base/utils/json-io.js";
+import { computeSoilChecksum } from "./checksum.js";
+import { createSoilConfig } from "./config.js";
+import { loadSoilManifest } from "./retriever.js";
+import { SoilGenerationWatermarkSchema, SoilManualOverlaySchema, SoilPageFrontmatterSchema, SoilSourceRefSchema, SoilSourceTruthSchema, } from "./types.js";
+export const SOIL_INDEX_STORAGE_FORMAT = "file-json-v1";
+const ISO_SCHEMA = z.string().refine((value) => !Number.isNaN(Date.parse(value)), "Must be a valid ISO-8601 datetime string");
+const SoilIndexChunkSnapshotSchema = z.object({
+    chunk_id: z.string().min(1),
+    soil_id: z.string().min(1),
+    relative_path: z.string().min(1),
+    absolute_path: z.string().min(1),
+    chunk_type: z.enum(["heading", "paragraph"]),
+    heading_path: z.array(z.string().min(1)),
+    heading: z.string().min(1).optional(),
+    position: z.number().int().nonnegative(),
+    text: z.string(),
+    token_count: z.number().int().nonnegative(),
+});
+const SoilIndexPageSnapshotSchema = z.object({
+    soil_id: z.string().min(1),
+    relative_path: z.string().min(1),
+    absolute_path: z.string().min(1),
+    title: z.string().min(1),
+    kind: z.string().min(1),
+    route: z.string().min(1),
+    status: z.string().min(1),
+    source: z.string().min(1),
+    version: z.string().min(1),
+    created_at: ISO_SCHEMA,
+    updated_at: ISO_SCHEMA,
+    generated_at: ISO_SCHEMA,
+    summary: z.string().optional(),
+    source_refs: z.array(SoilSourceRefSchema).default([]),
+    generation_watermark: SoilGenerationWatermarkSchema,
+    stale: z.boolean(),
+    manual_overlay: SoilManualOverlaySchema,
+    source_truth: SoilSourceTruthSchema.optional(),
+    page_format_version: z.string().optional(),
+    checksum: z.string().optional(),
+    chunk_count: z.number().int().nonnegative(),
+});
+const SoilIndexRetrievalRunSchema = z.object({
+    query: z.string(),
+    limit: z.number().int(),
+    generated_at: ISO_SCHEMA,
+    hit_count: z.number().int().nonnegative(),
+});
+const SoilIndexSnapshotSchema = z.object({
+    storage: z.literal(SOIL_INDEX_STORAGE_FORMAT),
+    root_dir: z.string().min(1),
+    index_path: z.string().min(1),
+    generated_at: ISO_SCHEMA,
+    source_manifest_checksum: z.string().min(1),
+    page_count: z.number().int().nonnegative(),
+    chunk_count: z.number().int().nonnegative(),
+    pages: z.array(SoilIndexPageSnapshotSchema),
+    chunks: z.array(SoilIndexChunkSnapshotSchema),
+    retrieval_runs: z.array(SoilIndexRetrievalRunSchema).default([]),
+});
+function nowIso(clock) {
+    return (clock?.() ?? new Date()).toISOString();
+}
+function tokenize(value) {
+    return value
+        .toLowerCase()
+        .split(/[^a-z0-9._/-]+/g)
+        .map((token) => token.trim())
+        .filter((token) => token.length > 0);
+}
+function countOccurrences(haystack, needle) {
+    if (!needle) {
+        return 0;
+    }
+    let count = 0;
+    let index = 0;
+    while (index >= 0) {
+        index = haystack.indexOf(needle, index);
+        if (index >= 0) {
+            count += 1;
+            index += needle.length;
+        }
+    }
+    return count;
+}
+function stableSortPages(items) {
+    return [...items].sort((left, right) => left.relativePath.localeCompare(right.relativePath));
+}
+function stableSortChunks(chunks) {
+    return [...chunks].sort((left, right) => left.relative_path.localeCompare(right.relative_path) ||
+        left.position - right.position ||
+        left.chunk_type.localeCompare(right.chunk_type));
+}
+function withoutVolatileFields(page) {
+    return SoilPageFrontmatterSchema.parse({
+        ...page.frontmatter,
+        checksum: undefined,
+        generated_at: "1970-01-01T00:00:00.000Z",
+        updated_at: "1970-01-01T00:00:00.000Z",
+        generation_watermark: {
+            ...page.frontmatter.generation_watermark,
+            generated_at: "1970-01-01T00:00:00.000Z",
+        },
+    });
+}
+function buildPageChecksum(page) {
+    return computeSoilChecksum({
+        frontmatter: withoutVolatileFields(page),
+        body: page.body,
+    });
+}
+function buildPageChunks(page) {
+    const chunks = [];
+    const lines = page.body.split(/\r?\n/);
+    const headingPath = [];
+    const paragraphLines = [];
+    let position = 0;
+    const flushParagraph = () => {
+        const text = paragraphLines.join("\n").trim();
+        paragraphLines.length = 0;
+        if (!text) {
+            return;
+        }
+        const chunkPosition = position;
+        position += 1;
+        chunks.push({
+            chunk_id: `${page.relativePath}#paragraph-${chunkPosition}`,
+            soil_id: page.soilId,
+            relative_path: page.relativePath,
+            absolute_path: page.absolutePath,
+            chunk_type: "paragraph",
+            heading_path: [...headingPath],
+            heading: headingPath.at(-1),
+            position: chunkPosition,
+            text,
+            token_count: tokenize(text).length,
+        });
+    };
+    for (const line of lines) {
+        const headingMatch = /^(#{1,6})\s+(.*)$/.exec(line.trim());
+        if (headingMatch) {
+            flushParagraph();
+            const depth = headingMatch[1].length;
+            const heading = headingMatch[2].trim();
+            const nextHeadingPath = headingPath.slice(0, Math.max(0, depth - 1));
+            nextHeadingPath.push(heading);
+            headingPath.splice(0, headingPath.length, ...nextHeadingPath);
+            const chunkPosition = position;
+            position += 1;
+            chunks.push({
+                chunk_id: `${page.relativePath}#heading-${chunkPosition}`,
+                soil_id: page.soilId,
+                relative_path: page.relativePath,
+                absolute_path: page.absolutePath,
+                chunk_type: "heading",
+                heading_path: [...headingPath],
+                heading,
+                position: chunkPosition,
+                text: heading,
+                token_count: tokenize(heading).length,
+            });
+            continue;
+        }
+        if (line.trim() === "") {
+            flushParagraph();
+            continue;
+        }
+        paragraphLines.push(line);
+    }
+    flushParagraph();
+    return chunks;
+}
+function buildPageSnapshot(page) {
+    const checksum = buildPageChecksum(page);
+    const chunks = buildPageChunks(page);
+    return {
+        soil_id: page.soilId,
+        relative_path: page.relativePath,
+        absolute_path: page.absolutePath,
+        title: page.frontmatter.title,
+        kind: page.frontmatter.kind,
+        route: page.frontmatter.route,
+        status: page.frontmatter.status,
+        source: page.frontmatter.source,
+        version: page.frontmatter.version,
+        created_at: page.frontmatter.created_at,
+        updated_at: page.frontmatter.updated_at,
+        generated_at: page.frontmatter.generated_at,
+        summary: page.frontmatter.summary,
+        source_refs: page.frontmatter.source_refs,
+        generation_watermark: page.frontmatter.generation_watermark,
+        stale: page.frontmatter.stale,
+        manual_overlay: page.frontmatter.manual_overlay,
+        source_truth: page.frontmatter.source_truth,
+        page_format_version: page.frontmatter.page_format_version,
+        checksum,
+        chunk_count: chunks.length,
+    };
+}
+function buildSnapshotManifestChecksum(pages, chunks) {
+    return computeSoilChecksum({
+        pages: pages.map((page) => ({
+            soil_id: page.soil_id,
+            relative_path: page.relative_path,
+            checksum: page.checksum,
+            chunk_count: page.chunk_count,
+        })),
+        chunks: chunks.map((chunk) => ({
+            chunk_id: chunk.chunk_id,
+            soil_id: chunk.soil_id,
+            relative_path: chunk.relative_path,
+            position: chunk.position,
+            chunk_type: chunk.chunk_type,
+            token_count: chunk.token_count,
+        })),
+    });
+}
+function buildSearchText(page, chunks) {
+    return [
+        page.title,
+        page.summary,
+        page.kind,
+        page.route,
+        page.status,
+        page.source,
+        page.version,
+        page.soil_id,
+        page.relative_path,
+        JSON.stringify(page.source_refs),
+        JSON.stringify(page.generation_watermark),
+        JSON.stringify(page.manual_overlay),
+        JSON.stringify(page),
+        ...chunks.map((chunk) => chunk.text),
+    ]
+        .filter(Boolean)
+        .join("\n")
+        .toLowerCase();
+}
+function scorePage(queryTokens, page, chunks) {
+    if (queryTokens.length === 0) {
+        return 0;
+    }
+    const metadata = buildSearchText(page, chunks);
+    let score = 0;
+    for (const token of queryTokens) {
+        score += countOccurrences(page.title.toLowerCase(), token) * 8;
+        score += countOccurrences(page.summary?.toLowerCase() ?? "", token) * 5;
+        score += countOccurrences(metadata, token);
+        for (const chunk of chunks) {
+            score += countOccurrences(chunk.text.toLowerCase(), token) * 2;
+            score += countOccurrences(chunk.heading_path.join("\n").toLowerCase(), token);
+        }
+    }
+    return score;
+}
+function bestSnippet(queryTokens, page, chunks) {
+    const haystacks = [
+        page.title,
+        ...chunks.map((chunk) => `${chunk.heading ?? ""}\n${chunk.text}`),
+        page.summary,
+    ].filter((value) => Boolean(value));
+    for (const haystack of haystacks) {
+        if (!haystack) {
+            continue;
+        }
+        const lower = haystack.toLowerCase();
+        for (const token of queryTokens) {
+            const index = lower.indexOf(token);
+            if (index >= 0) {
+                const start = Math.max(0, index - 60);
+                const end = Math.min(haystack.length, index + token.length + 120);
+                return haystack.slice(start, end);
+            }
+        }
+    }
+    return page.summary ?? page.title;
+}
+export async function rebuildSoilIndex(configInput = {}, options = {}) {
+    const config = createSoilConfig(configInput);
+    const manifest = await loadSoilManifest({ rootDir: config.rootDir, indexPath: config.indexPath }, options.store);
+    const generatedAt = nowIso(options.clock);
+    const sortedPages = stableSortPages(manifest.pages);
+    const pages = sortedPages.map((page) => buildPageSnapshot(page));
+    const chunks = stableSortChunks(sortedPages.flatMap((page) => buildPageChunks(page)));
+    const snapshot = SoilIndexSnapshotSchema.parse({
+        storage: SOIL_INDEX_STORAGE_FORMAT,
+        root_dir: config.rootDir,
+        index_path: config.indexPath,
+        generated_at: generatedAt,
+        source_manifest_checksum: buildSnapshotManifestChecksum(pages, chunks),
+        page_count: pages.length,
+        chunk_count: chunks.length,
+        pages,
+        chunks,
+        retrieval_runs: [],
+    });
+    await fsp.mkdir(path.dirname(config.indexPath), { recursive: true });
+    await writeJsonFileAtomic(config.indexPath, snapshot);
+    return snapshot;
+}
+export async function loadSoilIndexSnapshot(configInput = {}) {
+    const config = createSoilConfig(configInput);
+    const raw = await readJsonFileOrNull(config.indexPath);
+    if (raw === null) {
+        return null;
+    }
+    const parsed = SoilIndexSnapshotSchema.safeParse(raw);
+    if (!parsed.success) {
+        return null;
+    }
+    return parsed.data;
+}
+export async function checkSoilIndexFresh(configInput = {}, options = {}) {
+    const config = createSoilConfig(configInput);
+    const snapshot = await loadSoilIndexSnapshot(config);
+    if (snapshot === null) {
+        return {
+            fresh: false,
+            reason: "missing-index",
+            indexPath: config.indexPath,
+        };
+    }
+    const manifest = await loadSoilManifest({ rootDir: config.rootDir, indexPath: config.indexPath }, options.store);
+    const sortedPages = stableSortPages(manifest.pages);
+    const pages = sortedPages.map((page) => buildPageSnapshot(page));
+    const chunks = stableSortChunks(sortedPages.flatMap((page) => buildPageChunks(page)));
+    const currentManifestChecksum = buildSnapshotManifestChecksum(pages, chunks);
+    if (snapshot.page_count !== pages.length) {
+        return {
+            fresh: false,
+            reason: "page-count-mismatch",
+            indexPath: config.indexPath,
+            indexedManifestChecksum: snapshot.source_manifest_checksum,
+            currentManifestChecksum,
+            indexedPageCount: snapshot.page_count,
+            currentPageCount: pages.length,
+        };
+    }
+    if (snapshot.source_manifest_checksum !== currentManifestChecksum) {
+        return {
+            fresh: false,
+            reason: "manifest-checksum-mismatch",
+            indexPath: config.indexPath,
+            indexedManifestChecksum: snapshot.source_manifest_checksum,
+            currentManifestChecksum,
+            indexedPageCount: snapshot.page_count,
+            currentPageCount: pages.length,
+        };
+    }
+    return {
+        fresh: true,
+        reason: "fresh",
+        indexPath: config.indexPath,
+        indexedManifestChecksum: snapshot.source_manifest_checksum,
+        currentManifestChecksum,
+        indexedPageCount: snapshot.page_count,
+        currentPageCount: pages.length,
+    };
+}
+export async function querySoilIndexSnapshot(query, limit = 10, configInput = {}) {
+    const snapshot = await loadSoilIndexSnapshot(configInput);
+    if (snapshot === null || limit <= 0) {
+        return [];
+    }
+    const queryTokens = tokenize(query);
+    const sortedPages = [...snapshot.pages].sort((left, right) => left.relative_path.localeCompare(right.relative_path));
+    const hits = sortedPages
+        .map((page) => {
+        const pageChunks = snapshot.chunks.filter((chunk) => chunk.soil_id === page.soil_id);
+        const score = scorePage(queryTokens, page, pageChunks);
+        return {
+            page,
+            chunks: pageChunks,
+            score,
+        };
+    })
+        .filter(({ score }) => queryTokens.length === 0 || score > 0)
+        .sort((left, right) => right.score - left.score || left.page.relative_path.localeCompare(right.page.relative_path))
+        .slice(0, limit)
+        .map(({ page, chunks, score }) => ({
+        soil_id: page.soil_id,
+        relative_path: page.relative_path,
+        absolute_path: page.absolute_path,
+        title: page.title,
+        kind: page.kind,
+        route: page.route,
+        status: page.status,
+        summary: page.summary,
+        score,
+        snippet: bestSnippet(queryTokens, page, chunks),
+        chunk_count: page.chunk_count,
+        checksum: page.checksum,
+    }));
+    return hits;
+}
+//# sourceMappingURL=index-store.js.map