toolnet-memory 0.2.0

package/dist/src/session/codex/notify-installer.js

@@ -0,0 +1,135 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync, } from "node:fs";
+import { homedir, } from "node:os";
+import { dirname, join, } from "node:path";
+function stringsFromTomlArray(value) {
+    const result = [];
+    const regex = /"((?:\\.|[^"\\])*)"|'([^']*)'/g;
+    let match;
+    while ((match =
+        regex.exec(value))) {
+        const raw = match[1] ??
+            match[2] ??
+            "";
+        try {
+            result.push(match[1] !==
+                undefined
+                ? JSON.parse(`"${raw}"`)
+                : raw);
+        }
+        catch {
+            result.push(raw);
+        }
+    }
+    return result;
+}
+export function installCodexNotify(options = {}) {
+    const configFile = options.configFile ??
+        join(process.env
+            .CODEX_HOME ??
+            join(homedir(), ".codex"), "config.toml");
+    const previousFile = options.previousFile ??
+        join(homedir(), ".config", "toolnet-memory", "codex-notify-previous.json");
+    mkdirSync(dirname(configFile), {
+        recursive: true,
+    });
+    mkdirSync(dirname(previousFile), {
+        recursive: true,
+    });
+    let content = existsSync(configFile)
+        ? readFileSync(configFile, "utf8")
+        : "";
+    const binary = options.binary ??
+        "toolnet-memory";
+    const newLine = `notify = [${JSON.stringify(binary)}, "session:codex-notify"]`;
+    /*
+     * Root TOML keys occur before first table.
+     */
+    const lines = content.split("\n");
+    let firstTable = lines.findIndex(line => /^\s*\[/.test(line));
+    if (firstTable < 0) {
+        firstTable =
+            lines.length;
+    }
+    let notifyStart = -1;
+    let notifyEnd = -1;
+    for (let index = 0; index <
+        firstTable; index += 1) {
+        if (/^\s*notify\s*=/.test(lines[index])) {
+            notifyStart =
+                index;
+            notifyEnd =
+                index;
+            /*
+             * Handle multiline TOML array.
+             */
+            if (lines[index]
+                .includes("[") &&
+                !lines[index]
+                    .includes("]")) {
+                while (notifyEnd +
+                    1 <
+                    firstTable) {
+                    notifyEnd +=
+                        1;
+                    if (lines[notifyEnd].includes("]")) {
+                        break;
+                    }
+                }
+            }
+            break;
+        }
+    }
+    let previous = [];
+    if (notifyStart >=
+        0) {
+        const oldBlock = lines
+            .slice(notifyStart, notifyEnd +
+            1)
+            .join("\n");
+        previous =
+            stringsFromTomlArray(oldBlock);
+        lines.splice(notifyStart, notifyEnd -
+            notifyStart +
+            1, newLine);
+    }
+    else {
+        firstTable =
+            lines.findIndex(line => /^\s*\[/.test(line));
+        if (firstTable < 0) {
+            firstTable =
+                lines.length;
+        }
+        lines.splice(firstTable, 0, newLine);
+    }
+    const isToolNet = previous.length >=
+        2 &&
+        previous[previous.length -
+            1] ===
+            "session:codex-notify";
+    if (previous.length >
+        0 &&
+        !isToolNet) {
+        writeFileSync(previousFile, JSON.stringify(previous, null, 2) +
+            "\n", {
+            encoding: "utf8",
+            mode: 0o600,
+        });
+    }
+    content =
+        lines.join("\n");
+    if (!content.endsWith("\n")) {
+        content +=
+            "\n";
+    }
+    writeFileSync(configFile, content, {
+        encoding: "utf8",
+        mode: 0o600,
+    });
+    return {
+        configFile,
+        previousFile,
+        preservedPrevious: previous.length >
+            0 &&
+            !isToolNet,
+    };
+}

package/dist/src/session/codex/project-resolver.js

@@ -0,0 +1,25 @@
+import { existsSync, } from "node:fs";
+import { dirname, join, parse, resolve, } from "node:path";
+import { ProjectManager, } from "../../core/index.js";
+export function findCodexToolNetProject(cwd) {
+    let current = resolve(cwd);
+    const filesystemRoot = parse(current).root;
+    while (true) {
+        if (existsSync(join(current, ".toolnet", "project.json"))) {
+            return new ProjectManager()
+                .detect(current);
+        }
+        if (current ===
+            filesystemRoot) {
+            break;
+        }
+        const parent = dirname(current);
+        if (parent ===
+            current) {
+            break;
+        }
+        current =
+            parent;
+    }
+    return null;
+}

package/dist/src/session/codex/recovery.js

@@ -0,0 +1,27 @@
+import { inspectCodexRollout, listCodexRollouts, pathBelongsToProject, } from "./discovery.js";
+import { syncCodexSession, } from "./adapter.js";
+export async function recoverCodexProject(project, storage, limit = 100) {
+    const results = [];
+    for (const rolloutPath of listCodexRollouts()) {
+        const meta = inspectCodexRollout(rolloutPath);
+        if (!meta.threadId ||
+            !meta.cwd) {
+            continue;
+        }
+        if (!pathBelongsToProject(project.rootPath, meta.cwd)) {
+            continue;
+        }
+        results.push(await syncCodexSession({
+            project,
+            storage,
+            threadId: meta.threadId,
+            rolloutPath,
+            cwd: meta.cwd,
+        }));
+        if (results.length >=
+            limit) {
+            break;
+        }
+    }
+    return results;
+}

package/dist/src/session/codex/rollout.js

@@ -0,0 +1,186 @@
+import { closeSync, existsSync, openSync, readSync, statSync, } from "node:fs";
+function object(value) {
+    return (value &&
+        typeof value ===
+            "object" &&
+        !Array.isArray(value))
+        ? value
+        : {};
+}
+function timestamp(row) {
+    const raw = row.timestamp;
+    if (typeof raw ===
+        "string") {
+        const date = new Date(raw);
+        if (!Number.isNaN(date.getTime())) {
+            return date.toISOString();
+        }
+    }
+    return new Date()
+        .toISOString();
+}
+function classify(row) {
+    const outerType = typeof row.type ===
+        "string"
+        ? row.type
+            .toLowerCase()
+        : "";
+    const payload = object(row.payload);
+    const payloadType = typeof payload.type ===
+        "string"
+        ? payload.type
+            .toLowerCase()
+        : "";
+    const role = typeof payload.role ===
+        "string"
+        ? payload.role
+            .toLowerCase()
+        : undefined;
+    if (outerType ===
+        "session_meta") {
+        return {
+            type: "custom",
+        };
+    }
+    if (role ===
+        "user" ||
+        payloadType ===
+            "user_message") {
+        return {
+            type: "user_prompt",
+            role: "user",
+        };
+    }
+    if (role ===
+        "assistant" ||
+        payloadType ===
+            "agent_message" ||
+        payloadType ===
+            "assistant_message") {
+        return {
+            type: "assistant_message",
+            role: "assistant",
+        };
+    }
+    if (payloadType.includes("function_call") ||
+        payloadType.includes("tool_call") ||
+        payloadType.includes("exec_command_begin") ||
+        payloadType.includes("exec_command_start")) {
+        return {
+            type: "tool_call",
+        };
+    }
+    if (payloadType.includes("function_call_output") ||
+        payloadType.includes("tool_result") ||
+        payloadType.includes("exec_command_end") ||
+        payloadType.includes("exec_command_complete")) {
+        return {
+            type: "tool_result",
+        };
+    }
+    if (payloadType.includes("error")) {
+        return {
+            type: "error",
+        };
+    }
+    if (outerType.includes("compacted") ||
+        payloadType.includes("compacted")) {
+        return {
+            type: "session_compact",
+        };
+    }
+    return {
+        type: "custom",
+        role,
+    };
+}
+export function readCodexRollout(rolloutPath, offset = 0) {
+    if (!existsSync(rolloutPath)) {
+        return {
+            events: [],
+            nextOffset: offset,
+            reset: false,
+        };
+    }
+    const size = statSync(rolloutPath).size;
+    const reset = offset >
+        size;
+    const start = reset
+        ? 0
+        : offset;
+    if (start >= size) {
+        return {
+            events: [],
+            nextOffset: size,
+            reset,
+        };
+    }
+    const length = size -
+        start;
+    const buffer = Buffer.alloc(length);
+    const fd = openSync(rolloutPath, "r");
+    try {
+        readSync(fd, buffer, 0, length, start);
+    }
+    finally {
+        closeSync(fd);
+    }
+    const text = buffer.toString("utf8");
+    /*
+     * Codex may still be appending.
+     * Never consume half a JSONL line.
+     */
+    const lastNewline = text.lastIndexOf("\n");
+    if (lastNewline <
+        0) {
+        return {
+            events: [],
+            nextOffset: start,
+            reset,
+        };
+    }
+    const complete = text.slice(0, lastNewline +
+        1);
+    let localOffset = 0;
+    const events = [];
+    for (const line of complete.split("\n")) {
+        if (!line) {
+            continue;
+        }
+        const lineBytes = Buffer.byteLength(line +
+            "\n", "utf8");
+        const absoluteOffset = start +
+            localOffset;
+        localOffset +=
+            lineBytes;
+        let parsed;
+        try {
+            parsed =
+                JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        const row = object(parsed);
+        const classification = classify(row);
+        events.push({
+            type: classification.type,
+            timestamp: timestamp(row),
+            role: classification.role,
+            sourceEventId: `rollout:${absoluteOffset}`,
+            sourceSequence: absoluteOffset,
+            data: row,
+            provenance: {
+                source: "codex-rollout",
+                sourcePath: rolloutPath,
+                sourceOffset: absoluteOffset,
+            },
+        });
+    }
+    return {
+        events,
+        nextOffset: start +
+            Buffer.byteLength(complete, "utf8"),
+        reset,
+    };
+}

package/dist/src/session/core.js

@@ -0,0 +1,239 @@
+import { Sanitizer, } from "../security/sanitizer.js";
+import { createSessionIdentity, } from "./identity.js";
+import { RemoteSessionStore, } from "./store.js";
+import { SessionWal, } from "./wal.js";
+import { SessionMemoryLearner, } from "./learner/learner.js";
+import { WorkContinuityLearner, } from "../work-continuity/learner.js";
+import { SemanticWorkLearner, } from "../work-continuity/semantic-learner.js";
+import { SmartHandoffManager, } from "../work-continuity/handoff.js";
+export class SessionCore {
+    identity;
+    wal;
+    remote;
+    sanitizer = new Sanitizer();
+    learner;
+    continuity;
+    semantic;
+    handoff;
+    title;
+    metadata;
+    constructor(options) {
+        this.identity =
+            createSessionIdentity(options.project, options.agent, options.nativeSessionId);
+        this.title =
+            options.title;
+        this.metadata =
+            (this.sanitizer
+                .sanitizeValue(options.metadata ??
+                {}));
+        this.wal =
+            new SessionWal(this.identity);
+        this.remote =
+            new RemoteSessionStore(options.storage, options.maxEventsPerChunk ??
+                100, options.maxChunkBytes ??
+                512 * 1024);
+        this.learner =
+            new SessionMemoryLearner({
+                project: options.project,
+                storage: options.storage,
+                identity: this.identity,
+                wal: this.wal,
+            });
+        this.continuity =
+            new WorkContinuityLearner({
+                project: options.project,
+                storage: options.storage,
+                identity: this.identity,
+                wal: this.wal,
+            });
+        this.semantic =
+            new SemanticWorkLearner({
+                project: options.project,
+                storage: options.storage,
+                identity: this.identity,
+                wal: this.wal,
+            });
+        this.handoff =
+            new SmartHandoffManager({
+                project: options.project,
+                storage: options.storage,
+                identity: this.identity,
+            });
+    }
+    sanitizeEvent(event) {
+        const provenance = event.provenance
+            ? {
+                ...event.provenance,
+                metadata: this.sanitizer
+                    .sanitizeValue(event.provenance
+                    .metadata),
+            }
+            : undefined;
+        return {
+            ...event,
+            data: this.sanitizer
+                .sanitizeValue(event.data ??
+                {}),
+            provenance,
+        };
+    }
+    start(data = {}) {
+        const state = this.wal.loadState();
+        return this.record({
+            type: state.lastSequence ===
+                0
+                ? "session_start"
+                : "session_resume",
+            data,
+            provenance: {
+                source: this.identity.agent,
+            },
+        });
+    }
+    record(event) {
+        return this.wal.append([
+            this.sanitizeEvent(event),
+        ])[0];
+    }
+    recordMany(events) {
+        return this.wal.append(events.map(event => this.sanitizeEvent(event)));
+    }
+    setSourceCursor(source, value) {
+        this.wal
+            .setSourceCursor(source, value);
+    }
+    async flush() {
+        const pending = this.wal
+            .readPending();
+        const state = this.wal
+            .loadState();
+        const result = await this.remote.append(this.identity, pending.events, state.sourceCursors, {
+            title: this.title,
+            metadata: this.metadata,
+        });
+        if (pending.events.length >
+            0) {
+            const last = pending.events[pending.events.length -
+                1];
+            this.wal.markRemote(last.sequence, pending.endOffset);
+        }
+        /*
+         * Long-term learning is deliberately downstream
+         * of durable session persistence.
+         *
+         * A learner failure must NEVER destroy session capture.
+         * Cursor is not advanced on learner failure,
+         * so next flush retries.
+         */
+        if (process.env
+            .TOOLNET_SESSION_LEARNING !==
+            "0") {
+            try {
+                await this.learner
+                    .learnNew();
+            }
+            catch {
+                // Retry automatically on a later flush.
+            }
+        }
+        /*
+         * Work continuity is a separate projection from
+         * long-term memory.
+         *
+         * Failure here must never break session capture.
+         */
+        if (process.env
+            .TOOLNET_WORK_CONTINUITY !==
+            "0") {
+            try {
+                await this.continuity
+                    .learnNew();
+            }
+            catch {
+                // Immutable session WAL allows retry later.
+            }
+        }
+        /*
+         * Semantic continuity learns meaning, rationale,
+         * deliverables and completion criteria.
+         *
+         * It never invents missing rationale.
+         */
+        if (process.env
+            .TOOLNET_SEMANTIC_CONTINUITY !==
+            "0") {
+            try {
+                await this.semantic
+                    .learnNew();
+            }
+            catch {
+                // Derived projection: session capture must continue.
+            }
+        }
+        /*
+         * Smart Handoff checkpoint.
+         *
+         * We checkpoint on every durable flush, not only graceful
+         * session_end. This protects continuity when an agent hits
+         * token limit, terminal disconnect, crash, or user stops
+         * unexpectedly.
+         *
+         * Handoff IDs are state-digest based, so unchanged work
+         * does not create endless duplicate snapshots.
+         */
+        if (process.env
+            .TOOLNET_SMART_HANDOFF !==
+            "0" &&
+            pending.events.length >
+                0) {
+            try {
+                const lastEvent = pending.events[pending.events.length -
+                    1];
+                const explicitReason = [
+                    "session_idle",
+                    "session_end",
+                    "session_compact",
+                ].includes(lastEvent.type)
+                    ? lastEvent.type
+                    : "checkpoint";
+                await this.handoff
+                    .capture(explicitReason, lastEvent.sequence);
+            }
+            catch {
+                /*
+                 * Handoff is a derived projection.
+                 * It must never break durable session capture.
+                 */
+            }
+        }
+        return result;
+    }
+    async idle(data = {}) {
+        this.record({
+            type: "session_idle",
+            data,
+            provenance: {
+                source: this.identity.agent,
+            },
+        });
+        return this.flush();
+    }
+    async end(data = {}) {
+        this.record({
+            type: "session_end",
+            data,
+            provenance: {
+                source: this.identity.agent,
+            },
+        });
+        return this.flush();
+    }
+    status() {
+        return this.wal
+            .loadState();
+    }
+    recoverRemote() {
+        return this.remote
+            .recover(this.identity);
+    }
+}

package/dist/src/session/identity.js

@@ -0,0 +1,57 @@
+import { join, } from "node:path";
+import { sha256, } from "./utils.js";
+function safeSegment(value, fallback) {
+    const trimmed = value.trim();
+    const readable = trimmed
+        .replace(/\s+/g, "_")
+        .replace(/[^A-Za-z0-9._-]/g, "_")
+        .replace(/_+/g, "_")
+        .replace(/^\.+|\.+$/g, "");
+    const digest = sha256(trimmed).slice(0, 12);
+    if (!readable ||
+        readable === "." ||
+        readable === "..") {
+        return `${fallback}--${digest}`;
+    }
+    const limited = readable.slice(0, 100);
+    /*
+     * Preserve normal UUID/OpenCode IDs exactly.
+     * Add hash only when sanitization changed the value.
+     */
+    if (limited ===
+        trimmed &&
+        trimmed.length <=
+            100) {
+        return limited;
+    }
+    return `${limited.slice(0, 85)}--${digest}`;
+}
+export function createSessionIdentity(project, agent, nativeSessionId) {
+    const cleanAgent = agent.trim();
+    const cleanSessionId = nativeSessionId.trim();
+    if (!cleanAgent) {
+        throw new Error("Session agent is required");
+    }
+    if (!cleanSessionId) {
+        throw new Error("Native session ID is required");
+    }
+    const agentFolder = safeSegment(cleanAgent
+        .toLowerCase(), "agent");
+    const sessionFolder = safeSegment(cleanSessionId, "session");
+    return {
+        projectId: project.id,
+        projectName: project.name,
+        projectRoot: project.rootPath,
+        agent: cleanAgent,
+        nativeSessionId: cleanSessionId,
+        sessionKey: `${cleanAgent}:${cleanSessionId}`,
+        remotePrefix: [
+            "projects",
+            project.id,
+            "sessions",
+            agentFolder,
+            sessionFolder,
+        ].join("/"),
+        localDirectory: join(project.rootPath, ".toolnet", "sessions", agentFolder, sessionFolder),
+    };
+}

package/dist/src/session/index.js

@@ -0,0 +1,9 @@
+export * from "./types.js";
+export * from "./identity.js";
+export * from "./wal.js";
+export * from "./store.js";
+export * from "./core.js";
+export * from "./opencode/index.js";
+export * from "./agy/index.js";
+export * from "./codex/index.js";
+export * from "./learner/index.js";