praana 0.5.0

package/dist/context-engine/artifact-store.js

@@ -0,0 +1,272 @@
+import { createHash } from "node:crypto";
+import { createDefaultDistillerRegistry } from "../distillers/index.js";
+import { classifyContentType } from "./classify.js";
+import { buildPendingSummary } from "./distiller.js";
+import { evictStaleArtifacts, findArtifactByHash, getArtifactById, insertArtifact, insertDistillerStat, openContextEngineDb, touchArtifactAccess, updateArtifactSummary, } from "./db.js";
+import { buildArtifactCard, estimateTokens, } from "./summarize.js";
+function artifactIdFromHash(sha256) {
+    return `art_${sha256.slice(0, 12)}`;
+}
+function sha256(text) {
+    return createHash("sha256").update(text).digest("hex");
+}
+function sliceByLines(text, lineStart, lineEnd) {
+    const lines = text.split("\n");
+    const start = Math.max(1, lineStart ?? 1);
+    const end = Math.min(lines.length, lineEnd ?? lines.length);
+    if (start > end)
+        return "";
+    return lines.slice(start - 1, end).join("\n");
+}
+function extractJsonPath(text, jsonPath) {
+    const parsed = JSON.parse(text);
+    const parts = jsonPath.split(".").filter(Boolean);
+    let current = parsed;
+    for (const part of parts) {
+        if (current === null || current === undefined || typeof current !== "object") {
+            throw new Error(`Invalid jsonPath segment: ${part}`);
+        }
+        current = current[part];
+    }
+    if (typeof current === "string")
+        return current;
+    return JSON.stringify(current, null, 2);
+}
+function savingsPct(inputTokens, outputTokens) {
+    if (inputTokens <= 0)
+        return 0;
+    return Math.max(0, (1 - outputTokens / inputTokens) * 100);
+}
+function inferContentTypeFromTool(sourceTool, command) {
+    if (!command)
+        return null;
+    const normalized = command.trim();
+    if (!normalized)
+        return null;
+    if (sourceTool === "shell" &&
+        /(^|\s)(rg|grep|ag|ack)(\s|$)/.test(normalized)) {
+        return "search_results";
+    }
+    if (sourceTool === "shell" && /(^|\s)git\s+(diff|show)(\s|$)/.test(normalized)) {
+        return "diff";
+    }
+    if (sourceTool === "shell" &&
+        /(^|\s)(npm|pnpm|yarn|bun)\s+(run\s+)?(test|vitest|jest)(\s|$|:)/.test(normalized)) {
+        return "test_output";
+    }
+    if (sourceTool === "shell" &&
+        /(^|\s)(tsc|vue-tsc|npm\s+run\s+(build|typecheck)|pnpm\s+(build|typecheck))(\s|$)/.test(normalized)) {
+        return "build_output";
+    }
+    return null;
+}
+export class ArtifactStore {
+    db;
+    sessionId;
+    config;
+    distillers;
+    fileReadIndex = new Map();
+    pendingBackfills = [];
+    constructor(db, sessionId, config, distillers) {
+        this.db = db;
+        this.sessionId = sessionId;
+        this.config = config;
+        this.distillers = distillers;
+    }
+    static open(dbPath, sessionId, config, distillers = createDefaultDistillerRegistry()) {
+        const db = openContextEngineDb(dbPath);
+        return new ArtifactStore(db, sessionId, config, distillers);
+    }
+    close() {
+        this.db.close();
+    }
+    getDb() {
+        return this.db;
+    }
+    runEviction(currentTurn) {
+        return evictStaleArtifacts(this.db, currentTurn, this.config.artifact_ttl_turns);
+    }
+    async flushDeferredDistillation() {
+        const jobs = this.pendingBackfills.splice(0);
+        for (const job of jobs) {
+            let result;
+            try {
+                result = await job.backfill();
+            }
+            catch (err) {
+                result = {
+                    summary: `[compression failed: ${err.message ?? "unknown error"}]\n${buildPendingSummary()}`,
+                    distillerName: "failed-deferred",
+                    execTimeMs: 0,
+                    deferred: true,
+                };
+            }
+            updateArtifactSummary(this.db, job.artifactId, result.summary);
+            this.recordDistillerStat({
+                sourceTool: job.sourceTool,
+                contentType: job.contentType,
+                distiller: result.distillerName,
+                inputTokens: job.inputTokens,
+                outputTokens: estimateTokens(result.summary),
+                execTimeMs: Math.round(result.execTimeMs),
+                turn: job.turn,
+            });
+        }
+        return jobs.length;
+    }
+    ingestToolResult(input) {
+        const contentType = input.contentType ??
+            inferContentTypeFromTool(input.sourceTool, input.command) ??
+            classifyContentType(input.rawText);
+        const rawTokens = estimateTokens(input.rawText);
+        const inlineThreshold = this.config.artifact_inline_threshold;
+        if (contentType === "error" || rawTokens <= inlineThreshold) {
+            return { promptText: input.rawText, inlined: true };
+        }
+        const fileKey = this.fileReadKey(input.sourceTool, input.command);
+        if (fileKey) {
+            const existingId = this.fileReadIndex.get(fileKey);
+            if (existingId) {
+                const existing = getArtifactById(this.db, existingId);
+                if (existing) {
+                    touchArtifactAccess(this.db, existing.id, input.createdTurn);
+                    return {
+                        promptText: buildArtifactCard(existing.id, existing.sourceTool, existing.command, existing.rawTokens, existing.summary),
+                        artifactId: existing.id,
+                        inlined: false,
+                    };
+                }
+            }
+        }
+        const hash = sha256(input.rawText);
+        const deduped = findArtifactByHash(this.db, hash);
+        if (deduped) {
+            touchArtifactAccess(this.db, deduped.id, input.createdTurn);
+            if (fileKey)
+                this.fileReadIndex.set(fileKey, deduped.id);
+            return {
+                promptText: buildArtifactCard(deduped.id, deduped.sourceTool, deduped.command ?? input.command, deduped.rawTokens, deduped.summary),
+                artifactId: deduped.id,
+                inlined: false,
+            };
+        }
+        const intensity = this.distillers.selectIntensity(rawTokens, this.config.distiller.default_intensity);
+        const distilled = this.distillers.distillForIngestion(input.rawText, contentType, intensity);
+        let summary;
+        if ("backfill" in distilled) {
+            const deferred = distilled;
+            summary = deferred.pendingSummary;
+            const artifactId = artifactIdFromHash(hash);
+            this.pendingBackfills.push({
+                artifactId,
+                backfill: deferred.backfill,
+                sourceTool: input.sourceTool,
+                contentType,
+                inputTokens: rawTokens,
+                turn: input.createdTurn,
+            });
+        }
+        else {
+            const sync = distilled;
+            summary = sync.summary;
+            this.recordDistillerStat({
+                sourceTool: input.sourceTool,
+                contentType,
+                distiller: sync.distillerName,
+                inputTokens: rawTokens,
+                outputTokens: estimateTokens(sync.summary),
+                execTimeMs: Math.round(sync.execTimeMs),
+                turn: input.createdTurn,
+            });
+        }
+        const artifact = {
+            id: artifactIdFromHash(hash),
+            sha256: hash,
+            sessionId: this.sessionId,
+            sourceTool: input.sourceTool,
+            command: input.command,
+            createdTurn: input.createdTurn,
+            rawTokens,
+            rawText: input.rawText,
+            summary,
+            contentType,
+            lastAccessedTurn: input.createdTurn,
+            accessCount: 0,
+        };
+        insertArtifact(this.db, artifact);
+        if (fileKey)
+            this.fileReadIndex.set(fileKey, artifact.id);
+        return {
+            promptText: buildArtifactCard(artifact.id, artifact.sourceTool, artifact.command, artifact.rawTokens, artifact.summary),
+            artifactId: artifact.id,
+            inlined: false,
+        };
+    }
+    retrieve(id, currentTurn, options) {
+        const artifact = getArtifactById(this.db, id);
+        if (!artifact) {
+            return { ok: false, error: `Artifact ${id} not found` };
+        }
+        if (artifact.sessionId !== this.sessionId) {
+            return { ok: false, error: `Artifact ${id} belongs to another session` };
+        }
+        touchArtifactAccess(this.db, id, currentTurn);
+        let content = artifact.rawText;
+        try {
+            if (options?.jsonPath) {
+                content = extractJsonPath(content, options.jsonPath);
+            }
+            if (options?.lineStart !== undefined || options?.lineEnd !== undefined) {
+                content = sliceByLines(content, options.lineStart, options.lineEnd);
+            }
+            if (options?.grep) {
+                const re = new RegExp(options.grep, "m");
+                const matched = content.split("\n").filter((line) => re.test(line));
+                content = matched.join("\n");
+            }
+        }
+        catch (err) {
+            return {
+                ok: false,
+                error: err.message ?? "Failed to slice artifact content",
+            };
+        }
+        return { ok: true, content };
+    }
+    getArtifact(id) {
+        const artifact = getArtifactById(this.db, id);
+        if (!artifact || artifact.sessionId !== this.sessionId)
+            return null;
+        return artifact;
+    }
+    getSessionId() {
+        return this.sessionId;
+    }
+    countArtifacts() {
+        const row = this.db
+            .prepare("SELECT COUNT(*) AS count FROM context_artifacts WHERE session_id = ?")
+            .get(this.sessionId);
+        return row.count;
+    }
+    touchAccess(id, currentTurn) {
+        touchArtifactAccess(this.db, id, currentTurn);
+    }
+    recordDistillerStat(input) {
+        insertDistillerStat(this.db, {
+            sessionId: this.sessionId,
+            tool: input.sourceTool,
+            contentType: input.contentType,
+            distiller: input.distiller,
+            inputTokens: input.inputTokens,
+            outputTokens: input.outputTokens,
+            savingsPct: savingsPct(input.inputTokens, input.outputTokens),
+            execTimeMs: input.execTimeMs,
+            turn: input.turn,
+        });
+    }
+    fileReadKey(sourceTool, command) {
+        if (sourceTool !== "read_file" || !command)
+            return null;
+        return command;
+    }
+}

package/dist/context-engine/bm25.d.ts

@@ -0,0 +1,3 @@
+export declare function tokenize(text: string): string[];
+export declare function bm25Score(queryTokens: string[], docTokens: string[], avgDocLen: number, totalDocs: number, docFreq: Map<string, number>): number;
+export declare function bm25Relevance(query: string, content: string): number;

package/dist/context-engine/bm25.js

@@ -0,0 +1,32 @@
+export function tokenize(text) {
+    return text.toLowerCase().match(/[a-z0-9_./-]+/g) ?? [];
+}
+export function bm25Score(queryTokens, docTokens, avgDocLen, totalDocs, docFreq) {
+    const k1 = 1.5;
+    const b = 0.75;
+    const docLen = docTokens.length;
+    const tf = new Map();
+    for (const t of docTokens)
+        tf.set(t, (tf.get(t) ?? 0) + 1);
+    let score = 0;
+    for (const qt of queryTokens) {
+        const freq = tf.get(qt) ?? 0;
+        if (freq === 0)
+            continue;
+        const df = docFreq.get(qt) ?? 1;
+        const idf = Math.log(1 + (totalDocs - df + 0.5) / (df + 0.5));
+        const numerator = freq * (k1 + 1);
+        const denominator = freq + k1 * (1 - b + b * (docLen / avgDocLen));
+        score += idf * (numerator / denominator);
+    }
+    return score;
+}
+export function bm25Relevance(query, content) {
+    const queryTokens = tokenize(query);
+    if (queryTokens.length === 0)
+        return 0;
+    const docTokens = tokenize(content);
+    if (docTokens.length === 0)
+        return 0;
+    return bm25Score(queryTokens, docTokens, docTokens.length, 1, new Map());
+}

package/dist/context-engine/checkpoint.d.ts

@@ -0,0 +1,34 @@
+import type Database from "better-sqlite3";
+import type { ActivityEntry, CheckpointDraft, CheckpointState, SessionCheckpoint, TurnDigest } from "./types.js";
+type LegacyCheckpointState = CheckpointState & {
+    plan?: string;
+    volatile?: string;
+};
+export declare function createEmptyCheckpointState(): CheckpointState;
+export declare function normalizeCheckpointState(raw: Partial<LegacyCheckpointState>): CheckpointState;
+export declare function createEmptyCheckpoint(): SessionCheckpoint;
+export declare function reconcileCheckpoint(state: CheckpointState, digest: TurnDigest, draft: CheckpointDraft, turn: number): CheckpointState;
+export declare function replayCheckpointFromDigests(digests: TurnDigest[], activityEntries: ActivityEntry[]): CheckpointState;
+export declare function renderCheckpoint(checkpoint: SessionCheckpoint): string;
+export declare function renderContextSummary(checkpoint: SessionCheckpoint, stats?: {
+    artifactCount?: number;
+}): string;
+export declare function checkpointTokenEstimate(checkpoint: SessionCheckpoint): {
+    text: string;
+    tokens: number;
+};
+export declare class CheckpointStore {
+    private readonly db;
+    private readonly sessionId;
+    private checkpoint;
+    private constructor();
+    static open(db: Database.Database, sessionId: string): CheckpointStore;
+    reconcile(digest: TurnDigest, draft: CheckpointDraft, turn: number): void;
+    persist(): void;
+    getCheckpoint(): SessionCheckpoint;
+    render(): string;
+    renderContextSummary(stats?: {
+        artifactCount?: number;
+    }): string;
+}
+export {};