toolfactory 0.1.1 → 0.2.0

package/dist/project/apply.js

@@ -8,6 +8,7 @@
 import { existsSync, lstatSync, mkdirSync, readFileSync, readlinkSync, rmSync, symlinkSync, writeFileSync, } from "node:fs";
 import { dirname, join } from "node:path";
 import { parse as parseToml, stringify as stringifyToml } from "smol-toml";
+import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
 import { LOCK_PATH, readLock, serializeLock, sha256 } from "./lock.js";
 function locate(text, region) {
     const start = text.indexOf(region.begin);
@@ -70,13 +71,156 @@ export function pickPatch(document, patch, owned = [], prefix = "") {
     }
     return out;
 }
+function valueAt(document, path) {
+    return path
+        .split(".")
+        .reduce((value, part) => (isRecord(value) ? value[part] : undefined), document);
+}
+function setAt(document, path, value) {
+    const parts = path.split(".");
+    let target = document;
+    for (const part of parts.slice(0, -1)) {
+        const next = target[part];
+        if (isRecord(next))
+            target = next;
+        else {
+            const created = {};
+            target[part] = created;
+            target = created;
+        }
+    }
+    target[parts.at(-1)] = value;
+}
+function omitKeyedArrays(patch, keyedArrays, prefix = "") {
+    const out = {};
+    for (const [key, value] of Object.entries(patch)) {
+        const path = prefix ? `${prefix}.${key}` : key;
+        if (path in keyedArrays)
+            continue;
+        out[key] = isRecord(value) ? omitKeyedArrays(value, keyedArrays, path) : value;
+    }
+    return out;
+}
+function keyedArrayEntries(value, id, path) {
+    if (!Array.isArray(value) ||
+        value.some((entry) => !isRecord(entry) || typeof entry[id] !== "string")) {
+        throw new Error(`${path} must be an array of objects with a string ${id} field.`);
+    }
+    return value;
+}
+function keyedArrayState(file) {
+    if (!file.keyedArrays || Object.keys(file.keyedArrays).length === 0)
+        return undefined;
+    return Object.fromEntries(Object.entries(file.keyedArrays).map(([path, id]) => [
+        path,
+        {
+            id,
+            values: keyedArrayEntries(valueAt(file.patch, path), id, path).map((entry) => entry[id]),
+            entries: Object.fromEntries(keyedArrayEntries(valueAt(file.patch, path), id, path).map((entry) => [
+                entry[id],
+                entry,
+            ])),
+        },
+    ]));
+}
+function removeProjectedLeaves(entry, previous, next) {
+    const out = { ...entry };
+    for (const [key, oldValue] of Object.entries(previous)) {
+        if (!(key in next))
+            delete out[key];
+        else if (isRecord(oldValue) && isRecord(next[key]) && isRecord(out[key]))
+            out[key] = removeProjectedLeaves(out[key], oldValue, next[key]);
+    }
+    return out;
+}
+function mergeKeyedArrays(document, file, previous) {
+    let changed = false;
+    for (const [path, id] of Object.entries(file.keyedArrays ?? {})) {
+        const desired = keyedArrayEntries(valueAt(file.patch, path), id, path);
+        const previousIds = new Set(previous.files[file.path]?.keyedArrays?.[path]?.values ?? []);
+        const current = valueAt(document, path);
+        const entries = current === undefined ? [] : keyedArrayEntries(current, id, path);
+        const desiredById = new Map(desired.map((entry) => [entry[id], entry]));
+        const retained = [];
+        const generated = [];
+        for (const entry of entries) {
+            const key = entry[id];
+            const replacement = desiredById.get(key);
+            if (previousIds.has(key)) {
+                if (replacement) {
+                    const previousEntry = previous.files[file.path]?.keyedArrays?.[path]?.entries?.[key];
+                    generated.push(deepMerge(previousEntry ? removeProjectedLeaves(entry, previousEntry, replacement) : entry, replacement));
+                }
+                continue;
+            }
+            // Migration from a formerly whole generated file: an identical entry is ours already.
+            if (replacement && JSON.stringify(entry) === JSON.stringify(replacement)) {
+                generated.push(entry);
+                continue;
+            }
+            if (replacement)
+                throw new Error(`${file.path} ${path} already has author entry ${id}=${key}.`);
+            retained.push(entry);
+        }
+        const next = [
+            ...retained,
+            ...generated,
+            // A generated row may have been deleted by hand. It remains owned, so put its desired
+            // replacement back even when the previous lock knows its id already.
+            ...desired.filter((entry) => !generated.some((current) => current[id] === entry[id])),
+        ];
+        if (JSON.stringify(entries) !== JSON.stringify(next))
+            changed = true;
+        setAt(document, path, next);
+    }
+    return changed;
+}
+/** Select only generated leaves from an entry, leaving authored extension fields out of drift. */
+function pickProjectedEntry(current, projected) {
+    if (!current)
+        return undefined;
+    const picked = {};
+    for (const [key, value] of Object.entries(projected)) {
+        const actual = current[key];
+        picked[key] = isRecord(value) && isRecord(actual) ? pickProjectedEntry(actual, value) : actual;
+    }
+    return picked;
+}
+function removeKeyedArrayEntries(document, keyedArrays) {
+    let changed = false;
+    for (const [path, state] of Object.entries(keyedArrays ?? {})) {
+        const current = valueAt(document, path);
+        if (!Array.isArray(current))
+            continue;
+        const next = current.filter((entry) => !isRecord(entry) || !state.values.includes(entry[state.id]));
+        if (next.length !== current.length) {
+            setAt(document, path, next);
+            changed = true;
+        }
+    }
+    return changed;
+}
+function pickMerge(document, file) {
+    const picked = pickPatch(document, file.patch, file.owned);
+    for (const [path, id] of Object.entries(file.keyedArrays ?? {})) {
+        const desired = keyedArrayEntries(valueAt(file.patch, path), id, path);
+        const current = Array.isArray(valueAt(document, path))
+            ? valueAt(document, path)
+            : [];
+        const byId = new Map(current.filter(isRecord).map((entry) => [entry[id], entry]));
+        setAt(picked, path, desired.map((entry) => pickProjectedEntry(byId.get(entry[id]), entry)));
+    }
+    return picked;
+}
 /** The dotted paths a patch writes: one per leaf value, and one per object it owns whole. */
-export function patchKeys(patch, owned = [], prefix = "") {
+export function patchKeys(patch, owned = [], keyedArrays = {}, prefix = "") {
     const keys = [];
     for (const [key, value] of Object.entries(patch)) {
         const path = prefix ? `${prefix}.${key}` : key;
+        if (path in keyedArrays)
+            continue;
         if (isRecord(value) && !owned.includes(path) && Object.keys(value).length > 0) {
-            keys.push(...patchKeys(value, owned, path));
+            keys.push(...patchKeys(value, owned, keyedArrays, path));
         }
         else {
             keys.push(path);
@@ -119,12 +263,14 @@ export function removeKeys(document, keys) {
     return removed;
 }
 function parseDocument(text, format) {
-    return (format === "toml" ? parseToml(text) : JSON.parse(text));
+    return (format === "toml" ? parseToml(text) : format === "yaml" ? parseYaml(text) : JSON.parse(text));
 }
 function serializeDocument(document, format) {
-    return format === "toml"
-        ? `${stringifyToml(document)}\n`
-        : `${JSON.stringify(document, null, 2)}\n`;
+    if (format === "toml")
+        return `${stringifyToml(document)}\n`;
+    if (format === "yaml")
+        return stringifyYaml(document);
+    return `${JSON.stringify(document, null, 2)}\n`;
 }
 /** The content toolfactory manages, as one string for hashing and comparison. */
 export function managedContent(file) {
@@ -152,11 +298,15 @@ function currentManagedContent(root, file) {
         return text;
     if (file.kind === "region")
         return extractRegions(text, file)?.join(" ");
-    return JSON.stringify(pickPatch(parseDocument(text, file.format), file.patch, file.owned));
+    return JSON.stringify(pickMerge(parseDocument(text, file.format), file));
 }
 /** The two structured formats merge files use; the extension is the only discriminator needed. */
 function documentFormat(path) {
-    return path.endsWith(".toml") ? "toml" : "json";
+    return path.endsWith(".toml")
+        ? "toml"
+        : path.endsWith(".yaml") || path.endsWith(".yml")
+            ? "yaml"
+            : "json";
 }
 /** The keys the lock recorded for a path that the current patch no longer writes. */
 function staleKeys(lock, path, current = []) {
@@ -169,6 +319,16 @@ function stranded(root, path, keys) {
     const document = parseDocument(readFileSync(join(root, path), "utf8"), documentFormat(path));
     return keys.some((key) => hasKey(document, key));
 }
+function keyedEntriesRemain(root, path, keyedArrays) {
+    if (!keyedArrays || !existsSync(join(root, path)))
+        return false;
+    const document = parseDocument(readFileSync(join(root, path), "utf8"), documentFormat(path));
+    return Object.entries(keyedArrays).some(([arrayPath, state]) => {
+        const current = valueAt(document, arrayPath);
+        return (Array.isArray(current) &&
+            current.some((entry) => isRecord(entry) && state.values.includes(entry[state.id])));
+    });
+}
 /** The marker pairs the lock recorded for a path that the current plan no longer writes. */
 function staleRegions(lock, path, current = []) {
     return (lock.files[path]?.regions ?? []).filter((region) => !current.some((planned) => planned.begin === region.begin));
@@ -219,7 +379,9 @@ export function check(root, plan, toolfactoryVersion) {
         if (planned.has(path) || entry?.state !== "generated" || !present(root, path))
             continue;
         // A merge file toolfactory stops writing loses its keys, not its existence.
-        if (entry.keys && !stranded(root, path, entry.keys))
+        if ((entry.keys || entry.keyedArrays) &&
+            !stranded(root, path, entry.keys ?? []) &&
+            !keyedEntriesRemain(root, path, entry.keyedArrays))
             continue;
         // A region file likewise loses its regions: once they are empty, nothing of ours is stranded.
         if (entry.regions && !filled(root, path, entry.regions))
@@ -240,25 +402,36 @@ function render(root, file, previous) {
             const stale = staleRegions(previous, file.path, file.regions);
             const replaced = replaceRegions(emptyRegions(text, stale) ?? text, file);
             if (replaced === undefined) {
+                const prior = previous.files[file.path];
+                // A generated full file can become a region file without forcing an author to recreate
+                // markers. Its lock hash proves every byte is still the old generator's, so replacing it
+                // cannot discard authored work. Manual or changed files still require `adopt`.
+                if (prior?.state === "generated" && sha256(text) === prior.sha256)
+                    return replaceRegions(file.template, file) ?? file.template;
                 throw new Error(`${file.path} exists but is missing a toolfactory region marker; restore the markers or run \`toolfactory adopt ${file.path}\`.`);
             }
             return replaced;
         }
         return replaceRegions(file.template, file) ?? file.template;
     }
-    if (!existsSync(path))
-        return serializeDocument(deepMerge({}, file.patch), file.format);
+    if (!existsSync(path)) {
+        const document = deepMerge({}, omitKeyedArrays(file.patch, file.keyedArrays ?? {}), file.owned);
+        mergeKeyedArrays(document, file, previous);
+        return serializeDocument(document, file.format);
+    }
     const text = readFileSync(path, "utf8");
     const document = parseDocument(text, file.format);
     // The patch's inverse first: keys a previous patch wrote and this one dropped are uninstalled.
-    const uninstalled = removeKeys(document, staleKeys(previous, file.path, patchKeys(file.patch, file.owned)));
+    const uninstalled = removeKeys(document, staleKeys(previous, file.path, patchKeys(file.patch, file.owned, file.keyedArrays)));
+    const arraysChanged = mergeKeyedArrays(document, file, previous);
     // A document that already carries the patch is left byte-for-byte alone, so a rebuild never
     // reserializes the author's file (and, for TOML, never drops their comments).
     if (!uninstalled &&
-        JSON.stringify(pickPatch(document, file.patch, file.owned)) === JSON.stringify(file.patch)) {
+        !arraysChanged &&
+        JSON.stringify(pickMerge(document, file)) === JSON.stringify(file.patch)) {
         return text;
     }
-    return serializeDocument(deepMerge(document, file.patch, file.owned), file.format);
+    return serializeDocument(deepMerge(document, omitKeyedArrays(file.patch, file.keyedArrays ?? {}), file.owned), file.format);
 }
 /** Write a plan to the tree and refresh the lock. */
 export function apply(root, plan, toolfactoryVersion) {
@@ -279,7 +452,8 @@ export function apply(root, plan, toolfactoryVersion) {
             result.manual.push(file.path);
             continue;
         }
-        const keys = file.kind === "merge" ? patchKeys(file.patch, file.owned) : undefined;
+        const keys = file.kind === "merge" ? patchKeys(file.patch, file.owned, file.keyedArrays) : undefined;
+        const keyedArrays = file.kind === "merge" ? keyedArrayState(file) : undefined;
         const regions = file.kind === "region" ? file.regions.map(({ begin, end }) => ({ begin, end })) : undefined;
         if (isLink(file)) {
             if (currentManagedContent(root, file) === managedContent(file)) {
@@ -308,6 +482,7 @@ export function apply(root, plan, toolfactoryVersion) {
             sha256: sha256(managedContent(file)),
             state: "generated",
             keys,
+            keyedArrays,
             regions,
         };
     }
@@ -320,11 +495,13 @@ export function apply(root, plan, toolfactoryVersion) {
         }
         if (!present(root, path))
             continue;
-        if (entry.keys) {
+        if (entry.keys || entry.keyedArrays) {
             // The inverse of a merge file is its keys: the author keeps the file and everything else in it.
             const format = documentFormat(path);
             const document = parseDocument(readFileSync(join(root, path), "utf8"), format);
-            if (removeKeys(document, entry.keys)) {
+            const removedKeys = removeKeys(document, entry.keys ?? []);
+            const removedEntries = removeKeyedArrayEntries(document, entry.keyedArrays);
+            if (removedKeys || removedEntries) {
                 writeFileSync(join(root, path), serializeDocument(document, format));
                 result.written.push(path);
             }

package/dist/project/apply.test.js

@@ -226,8 +226,111 @@ describe("apply / check — merge files", () => {
         expect(text).toContain('dependencies = [ "zod" ]');
         expect(check(root, plan, "0.1.0")).toEqual([]);
     });
+    it("merges YAML and keeps authored keyed-array entries while removing stale generated ones", () => {
+        const root = tmp();
+        const plan = (commands) => [
+            {
+                kind: "merge",
+                path: "extension.yaml",
+                format: "yaml",
+                patch: {
+                    name: "hello",
+                    commands: commands.map((command) => ({ command, title: command })),
+                },
+                keyedArrays: { commands: "command" },
+            },
+        ];
+        apply(root, plan(["hello.run", "hello.old"]), "0.1.0");
+        writeFileSync(join(root, "extension.yaml"), `${readFileSync(join(root, "extension.yaml"), "utf8")}commands:\n  - command: author.run\n    title: Author\n  - command: hello.run\n    title: hello.run\n  - command: hello.old\n    title: hello.old\n`);
+        // Recreate the document with a custom entry and the generated entries, as a hand-built host
+        // manifest would; the generated ids remain Toolfactory-owned from the lock.
+        writeFileSync(join(root, "extension.yaml"), "name: hello\ncommands:\n  - command: author.run\n    title: Author\n  - command: hello.run\n    title: hello.run\n  - command: hello.old\n    title: hello.old\n");
+        apply(root, plan(["hello.run", "hello.new"]), "0.1.0");
+        const text = readFileSync(join(root, "extension.yaml"), "utf8");
+        expect(text).toContain("author.run");
+        expect(text).toContain("hello.new");
+        expect(text).not.toContain("hello.old");
+        expect(check(root, plan(["hello.run", "hello.new"]), "0.1.0")).toEqual([]);
+        expect(() => apply(root, plan(["author.run"]), "0.1.0")).toThrow(/already has author entry/);
+    });
+    it("preserves unknown keyed-entry fields while removing stale projected leaves", () => {
+        const root = tmp();
+        const plan = (title, category) => [
+            {
+                kind: "merge",
+                path: "package.json",
+                format: "json",
+                patch: {
+                    contributes: {
+                        commands: [{ command: "hello.run", title, ...(category ? { category } : {}) }],
+                    },
+                },
+                keyedArrays: { "contributes.commands": "command" },
+            },
+        ];
+        apply(root, plan("Run", "Toolfactory"), "0.1.0");
+        const path = join(root, "package.json");
+        const document = JSON.parse(readFileSync(path, "utf8"));
+        document.contributes.commands[0].icon = "$(rocket)";
+        writeFileSync(path, JSON.stringify(document));
+        // Native command details remain authored: they are not drift merely because Toolfactory
+        // owns the command's id and projected leaves.
+        expect(check(root, plan("Run", "Toolfactory"), "0.1.0")).toEqual([]);
+        apply(root, plan("Run tool"), "0.1.0");
+        expect(JSON.parse(readFileSync(path, "utf8")).contributes.commands).toEqual([
+            { command: "hello.run", title: "Run tool", icon: "$(rocket)" },
+        ]);
+        expect(check(root, plan("Run tool"), "0.1.0")).toEqual([]);
+    });
+    it("upgrades an untouched generated full file to a region but protects changed files", () => {
+        const root = tmp();
+        const old = [{ kind: "file", path: "entry.ts", content: "old generated\n" }];
+        const region = {
+            kind: "region",
+            path: "entry.ts",
+            regions: [{ begin: "// tf:entry", end: "// /tf:entry", content: "new generated\n" }],
+            template: "// tf:entry\n// /tf:entry\n\n// author tail\n",
+        };
+        apply(root, old, "0.1.0");
+        apply(root, [region], "0.2.0");
+        expect(readFileSync(join(root, "entry.ts"), "utf8")).toContain("new generated");
+        const changed = tmp();
+        apply(changed, old, "0.1.0");
+        writeFileSync(join(changed, "entry.ts"), "author changed\n");
+        expect(() => apply(changed, [region], "0.2.0")).toThrow(/missing a toolfactory region marker/);
+    });
 });
 describe("apply / check — inverses and outputs", () => {
+    it("restores deleted keyed rows, leaves steady YAML bytes alone, and uninstalls keys and rows", () => {
+        const root = tmp();
+        const plan = [
+            {
+                kind: "merge",
+                path: "extension.yaml",
+                format: "yaml",
+                patch: {
+                    generated: true,
+                    commands: [{ command: "hello.run", title: "Run" }],
+                },
+                keyedArrays: { commands: "command" },
+            },
+        ];
+        apply(root, plan, "0.1.0");
+        writeFileSync(join(root, "extension.yaml"), "# author comment\nauthor: true\ngenerated: true\ncommands:\n  - command: author.run\n    title: Author\n");
+        // The prior lock owns hello.run, so removing it by hand does not turn it into an omission.
+        apply(root, plan, "0.1.0");
+        expect(readFileSync(join(root, "extension.yaml"), "utf8")).toContain("hello.run");
+        // Once current, no rewrite means YAML comments and formatting survive a rebuild.
+        const path = join(root, "extension.yaml");
+        writeFileSync(path, `# preserved comment\n${readFileSync(path, "utf8")}`);
+        const before = readFileSync(path, "utf8");
+        expect(apply(root, plan, "0.1.0").unchanged).toContain("extension.yaml");
+        expect(readFileSync(path, "utf8")).toBe(before);
+        apply(root, [], "0.1.0");
+        expect(readFileSync(path, "utf8")).toContain("author.run");
+        expect(readFileSync(path, "utf8")).not.toContain("hello.run");
+        expect(readFileSync(path, "utf8")).not.toContain("generated:");
+    });
     it("uninstalls the keys a patch stops writing, keeping every other key", () => {
         const root = tmp();
         mkdirSync(root, { recursive: true });

package/dist/project/gate.d.ts

@@ -1,3 +1,16 @@
+/**
+ * The gate and the release assets, as data.
+ *
+ * The gate is a command; CI is its projection. One list of steps is rendered three ways — the
+ * `test` job of `ci.yml`, the `gate`/`package` jobs of `release.yml`, and a shell script a local
+ * `toolfactory gate` / `toolfactory package` spawns — so "what green means" has exactly one
+ * definition whether or not the project ever sees GitHub.
+ *
+ * A step here is a command and nothing else. GitHub Actions `uses:` steps are runner provisioning
+ * (checkout, toolchains, caches) and stay in `src/surfaces/workflows.ts`, which wraps this list.
+ */
+import { type NativePackageId } from "../distribution/native.js";
+import { type SystemPackageId } from "../distribution/system.js";
 import { type PackageManager, type Project, type SurfaceId } from "../model.js";
 export interface GateStep {
     /** Step label: the CI step name, and the banner the shell script echoes. */
@@ -53,7 +66,9 @@ export declare function tagVersionAssert(project: Project): GateStep | undefined
  * the release workflow's `package` job, which uploads the directory as one artifact. Publishing
  * to a registry stays a CI concern; producing the artifacts never is.
  */
-export declare function packageSteps(project: Project): GateStep[];
+export declare function packageSteps(project: Project, options?: {
+    system?: boolean;
+}): GateStep[];
 /** npm's filename for the root package, including its flattened scope. */
 export declare function npmTarball(project: Project): string;
 /** The ClawHub tarball `npm pack ./hosts/openclaw` writes, by name, inside the release artifact. */
@@ -65,7 +80,7 @@ export declare function openclawTarball(project: Project): string;
  */
 export interface Registry {
     /** Stable id: the release job suffix, the `secrets status` row, the unpublish step name. */
-    id: "npm" | "pypi" | "mcp-registry" | "oci" | "clawhub-package" | "clawhub-skill" | "pages" | "chrome" | "firefox" | "edge" | "safari";
+    id: NativePackageId | SystemPackageId | "npm" | "pypi" | "mcp-registry" | "oci" | "clawhub-package" | "clawhub-skill" | "pages" | "chrome" | "firefox" | "edge" | "safari" | "vscode-marketplace" | "open-vsx";
     /** The surfaces whose selection publishes here; all must be selected. */
     surfaces: SurfaceId[];
     /** Environment names the publish leg consumes (repository scope); [] for OIDC/GITHUB_TOKEN legs. */