shadow-claw 1.34.5 → 1.36.0

package/src/cli/commands/agent.ts

@@ -21,6 +21,8 @@ import type {
   AgentRunResult,
   AgentModelOptions,
   AgentModelResult,
+  AgentImportOptions,
+  AgentImportResult,
 } from "../../worker/headless-types.js";
 
 import { bootstrapHeadlessAgent } from "./agent-bootstrap.js";
@@ -254,6 +256,8 @@ export async function runAgentTools(
   options: AgentToolsOptions & Record<string, any> = {},
 ): Promise<any> {
   const { db, core } = await bootstrapHeadlessAgent(options);
+  const groupId =
+    options.group || core.DEFAULT_SERVER_GROUP_ID || DEFAULT_SERVER_GROUP_ID;
 
   let profileToolNames: Set<string> | null = null;
   if (options.toolsProfile || options.profile) {
@@ -283,7 +287,21 @@ export async function runAgentTools(
     }
   }
 
-  const tools = (core.TOOL_DEFINITIONS || []).map((def) => ({
+  const allTools = [...(core.TOOL_DEFINITIONS || [])];
+  if (typeof core.loadDeclarativeTools === "function") {
+    try {
+      const decl = await core.loadDeclarativeTools(db, groupId);
+      if (decl && Array.isArray(decl.tools)) {
+        for (const dt of decl.tools) {
+          if (!allTools.some((t) => t.name === dt.name)) {
+            allTools.push(dt);
+          }
+        }
+      }
+    } catch {}
+  }
+
+  const tools = allTools.map((def) => ({
     name: def.name,
     description: def.description,
     headlessSafe: !BROWSER_ONLY_TOOLS.has(def.name),
@@ -310,11 +328,14 @@ export async function runAgentTool(
     options.group || core.DEFAULT_SERVER_GROUP_ID || DEFAULT_SERVER_GROUP_ID;
 
   // Find tool in built-in TOOL_DEFINITIONS or declarative tools
-  let toolDef = (core.TOOL_DEFINITIONS || []).find((t) => t.name === toolName);
+  let toolDef = (core.TOOL_DEFINITIONS || []).find(
+    (t: any) => t.name === toolName,
+  );
+
   if (!toolDef && typeof core.loadDeclarativeTools === "function") {
     try {
       const decl = await core.loadDeclarativeTools(db, groupId);
-      toolDef = decl.tools.find((t) => t.name === toolName);
+      toolDef = decl.tools.find((t: any) => t.name === toolName);
     } catch {}
   }
 
@@ -370,12 +391,10 @@ export async function runAgentTool(
 
     let invokeContext: any = undefined;
     try {
-      const resolved = await resolveAgentProvider(
-        db,
-        core,
-        workspaceDir,
-        options,
-      );
+      const resolved = await resolveAgentProvider(db, core, workspaceDir, {
+        ...options,
+        interactive: false,
+      });
       invokeContext = {
         db,
         provider: resolved.providerId,
@@ -471,7 +490,7 @@ export async function runAgentSkill(
     skill.execution?.type === "tools" &&
     Array.isArray(skill.execution.tools)
   ) {
-    const normalizedTools = skill.execution.tools.map((t) => ({
+    const normalizedTools = skill.execution.tools.map((t: any) => ({
       name: t.name || t.tool,
       input: t.input || t.arguments || {},
       suppressOutput: t.suppressOutput,
@@ -565,7 +584,8 @@ export async function resolveAgentProvider(
             process.env.CI &&
             !["0", "false"].includes(process.env.CI.toLowerCase()),
           );
-  const isInteractive = isTTY && !isCI && !options.quiet;
+  const isInteractive =
+    isTTY && !isCI && !options.quiet && options.interactive !== false;
 
   if (!providerId) {
     if (isInteractive) {
@@ -726,7 +746,7 @@ export async function resolveAgentProvider(
         const processSigintListener = () => handleSigint();
         process.on("SIGINT", processSigintListener);
 
-        const dataListener = (chunk) => {
+        const dataListener = (chunk: any) => {
           const str =
             typeof chunk === "string" ? chunk : chunk.toString("binary");
           if (str.includes("\x03") || str.includes("\x04")) {
@@ -883,11 +903,13 @@ async function resolveToolsAndProfile(
         ? options.tools
         : String(options.tools).split(",")
     )
-      .map((t) => String(t).trim())
+      .map((t: unknown) => String(t).trim())
       .filter(Boolean);
 
     enabledTools = requestedNames
-      .map((name) => allAvailableTools.find((t) => t.name === name))
+      .map((name: unknown) =>
+        allAvailableTools.find((t) => t.name === String(name)),
+      )
       .filter(Boolean);
     return { enabledTools, profileSystemPromptOverride };
   }
@@ -1098,7 +1120,7 @@ export async function runAgentRun(
   let hasStreamedChunks = false;
   const pendingWrites: Promise<void>[] = [];
 
-  core.setPostHandler(async (message) => {
+  core.setPostHandler(async (message: any) => {
     switch (message.type) {
       case "streaming-start":
         hasStreamedChunks = true;
@@ -1197,15 +1219,15 @@ export async function runAgentRun(
               groupId: taskGroupId,
               db,
             })
-            .then((res) => {
-              const resolvers = globalThis.pendingNativeAiResolvers;
+            .then((res: unknown) => {
+              const resolvers = (globalThis as any).pendingNativeAiResolvers;
               if (resolvers && resolvers[id]) {
                 resolvers[id].resolve(res);
                 delete resolvers[id];
               }
             })
-            .catch((err) => {
-              const resolvers = globalThis.pendingNativeAiResolvers;
+            .catch((err: unknown) => {
+              const resolvers = (globalThis as any).pendingNativeAiResolvers;
               if (resolvers && resolvers[id]) {
                 resolvers[id].reject(err);
                 delete resolvers[id];
@@ -1601,6 +1623,218 @@ export async function runAgentModel(
   return { success: false, error: errorMsg };
 }
 
+/**
+ * Import tools, skills, and companion scripts from a remote discovery manifest or site URL.
+ * @param {string} urlOrSite
+ * @param {import("../../src/worker/headless-types.js").AgentImportOptions} [options]
+ * @returns {Promise<import("../../src/worker/headless-types.js").AgentImportResult>}
+ */
+export async function runAgentImport(
+  urlOrSite: string,
+  options: AgentImportOptions & Record<string, any> = {},
+): Promise<AgentImportResult> {
+  const trimmedUrl = (urlOrSite || "").trim();
+  if (!trimmedUrl) {
+    const errorMsg =
+      "Error: Discovery URL or site URL required. Usage: shadow-claw agent import <url> [options]";
+    if (!options.quiet) {
+      console.error(errorMsg);
+    }
+    process.exitCode = 1;
+    return { success: false, error: errorMsg };
+  }
+
+  const { db, core } = await bootstrapHeadlessAgent(options);
+  const groupId =
+    options.group || core.DEFAULT_SERVER_GROUP_ID || DEFAULT_SERVER_GROUP_ID;
+
+  const fetchDiscoveryManifest =
+    core.fetchDiscoveryManifest ||
+    (await import("../../subsystems/tools/remote/discovery.js"))
+      .fetchDiscoveryManifest;
+  const importRemoteArtifacts =
+    core.importRemoteArtifacts ||
+    (await import("../../subsystems/tools/remote/importArtifacts.js"))
+      .importRemoteArtifacts;
+
+  const fetchFn = options.fetchFn || globalThis.fetch;
+
+  if (!options.quiet) {
+    console.log(`Fetching discovery manifest from ${trimmedUrl}...`);
+  }
+
+  let resolvedManifest: any;
+  try {
+    resolvedManifest = await fetchDiscoveryManifest(trimmedUrl, fetchFn);
+  } catch (err: any) {
+    const errorMsg = `Failed to fetch discovery manifest: ${err instanceof Error ? err.message : String(err)}`;
+    if (!options.quiet) {
+      console.error(`Error: ${errorMsg}`);
+    }
+    process.exitCode = 1;
+    return { success: false, error: errorMsg };
+  }
+
+  const { manifest, siteUrl, manifestUrl } = resolvedManifest;
+
+  const manifestTools = manifest.tools || [];
+  const manifestSkills = manifest.skills || [];
+  const manifestScripts = manifest.scripts || [];
+
+  if (!options.quiet) {
+    console.log(
+      `Discovered: ${manifest.name || "Remote Site"} (${manifestSkills.length} skill${manifestSkills.length === 1 ? "" : "s"}, ${manifestTools.length} tool${manifestTools.length === 1 ? "" : "s"}, ${manifestScripts.length} script${manifestScripts.length === 1 ? "" : "s"})`,
+    );
+  }
+
+  const allToolNames = manifestTools.map((t: any) => t.name);
+  const allSkillNames = manifestSkills.map((s: any) => s.name);
+  const allScriptNames = manifestScripts.map((s: any) => s.name);
+
+  let toolNames: string[];
+  let skillNames: string[];
+  let scriptNames: string[];
+
+  const hasExplicitFilter = Boolean(
+    options.tools || options.skills || options.scripts,
+  );
+
+  if (hasExplicitFilter && !options.all) {
+    const parseList = (val: any) => {
+      if (!val) return [];
+      if (Array.isArray(val))
+        return val.map((v) => String(v).trim()).filter(Boolean);
+      return String(val)
+        .split(",")
+        .map((v) => v.trim())
+        .filter(Boolean);
+    };
+    toolNames = parseList(options.tools);
+    skillNames = parseList(options.skills);
+    scriptNames = parseList(options.scripts);
+  } else {
+    toolNames = allToolNames;
+    skillNames = allSkillNames;
+    scriptNames = allScriptNames;
+  }
+
+  const autoEnable = options.autoEnable !== false;
+  const overwrite = Boolean(options.overwrite);
+
+  let importResult: any;
+  try {
+    importResult = await importRemoteArtifacts(
+      db,
+      groupId,
+      manifest,
+      { toolNames, skillNames, scriptNames },
+      {
+        autoEnable,
+        overwrite,
+        fetchFn,
+        siteSlug: options.siteSlug,
+        targetSubdir: options.targetSubdir,
+      },
+    );
+  } catch (err: any) {
+    const errorMsg = `Import failed: ${err instanceof Error ? err.message : String(err)}`;
+    if (!options.quiet) {
+      console.error(`Error: ${errorMsg}`);
+    }
+    process.exitCode = 1;
+    return { success: false, error: errorMsg };
+  }
+
+  const importedTools = importResult.tools.filter(
+    (t: any) => t.status === "imported",
+  );
+  const importedSkills = importResult.skills.filter(
+    (s: any) => s.status === "imported",
+  );
+  const importedScripts = importResult.scripts.filter(
+    (sc: any) => sc.status === "imported",
+  );
+
+  const skippedTools = importResult.tools.filter(
+    (t: any) => t.status === "skipped",
+  );
+  const skippedSkills = importResult.skills.filter(
+    (s: any) => s.status === "skipped",
+  );
+  const skippedScripts = importResult.scripts.filter(
+    (sc: any) => sc.status === "skipped",
+  );
+
+  const failedTools = importResult.tools.filter(
+    (t: any) => t.status === "failed",
+  );
+  const failedSkills = importResult.skills.filter(
+    (s: any) => s.status === "failed",
+  );
+  const failedScripts = importResult.scripts.filter(
+    (sc: any) => sc.status === "failed",
+  );
+
+  const importedCount =
+    importedTools.length + importedSkills.length + importedScripts.length;
+  const skippedCount =
+    skippedTools.length + skippedSkills.length + skippedScripts.length;
+  const failedCount =
+    failedTools.length + failedSkills.length + failedScripts.length;
+
+  if (!options.quiet) {
+    console.log("\nImporting artifacts:");
+    for (const item of [
+      ...importResult.tools,
+      ...importResult.skills,
+      ...importResult.scripts,
+    ]) {
+      const type = importResult.tools.includes(item)
+        ? "tool"
+        : importResult.skills.includes(item)
+          ? "skill"
+          : "script";
+      if (item.status === "imported") {
+        console.log(`  [${type}] ${item.name} -> ${item.path} (imported)`);
+      } else if (item.status === "skipped") {
+        console.log(
+          `  [${type}] ${item.name} -> ${item.path} (skipped, already exists - use --overwrite to replace)`,
+        );
+      } else {
+        console.log(
+          `  [${type}] ${item.name} (failed: ${item.error || "unknown error"})`,
+        );
+      }
+    }
+
+    if (failedCount > 0) {
+      console.log(
+        `\nCompleted with ${failedCount} error${failedCount === 1 ? "" : "s"}. Successfully imported ${importedCount} items, ${skippedCount} skipped.`,
+      );
+    } else {
+      console.log(
+        `\nSuccessfully imported ${importedCount} item${importedCount === 1 ? "" : "s"} (${importedTools.length} tool${importedTools.length === 1 ? "" : "s"}, ${importedSkills.length} skill${importedSkills.length === 1 ? "" : "s"}, ${importedScripts.length} script${importedScripts.length === 1 ? "" : "s"}).`,
+      );
+    }
+  }
+
+  const success = failedCount === 0 || importedCount > 0;
+  if (!success) {
+    process.exitCode = 1;
+  }
+
+  return {
+    success,
+    manifest,
+    siteUrl,
+    manifestUrl,
+    result: importResult,
+    importedCount,
+    skippedCount,
+    failedCount,
+  };
+}
+
 /**
  * Main dispatcher for the `agent` command group.
  */
@@ -1714,9 +1948,25 @@ export async function runAgentCommand(
       return await runAgentModel(subaction, modelId, actualOptions);
     }
 
+    case "import": {
+      const url = actualArgs[0];
+      if (!url) {
+        console.error(
+          "Error: Discovery URL or site URL required. Usage: shadow-claw agent import <url> [options]",
+        );
+        process.exitCode = 1;
+        return {
+          success: false,
+          error:
+            "Discovery URL or site URL required. Usage: shadow-claw agent import <url> [options]",
+        };
+      }
+      return await runAgentImport(url, actualOptions);
+    }
+
     default: {
       console.log(
-        "Usage: shadow-claw agent <init|model|skills|tools|tool|skill|run> [args...]",
+        "Usage: shadow-claw agent <init|model|skills|tools|tool|skill|import|run> [args...]",
       );
     }
   }

package/src/cli/commands/skills-index.test.ts

@@ -2,8 +2,11 @@ import fs from "node:fs";
 import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
-import { describe, it, expect, beforeEach, afterEach } from "@jest/globals";
-import { computeSha256, generateSkillsIndex } from "./skills-index.js";
+import {
+  computeSha256,
+  generateSkillsIndex,
+  runSkillsIndexCommand,
+} from "./skills-index.js";
 
 describe("skills-index command and generator", () => {
   let tempDir: string;
@@ -231,4 +234,83 @@ metadata:
       "../../.agents/skills/main/skill-creator/SKILL.md",
     );
   });
+
+  it("does not include bundled skills by default in runSkillsIndexCommand for an external directory", async () => {
+    const mockProject = path.join(tempDir, "external-project");
+    const toolDir = path.join(mockProject, ".agents", "tools", "main");
+    await mkdir(toolDir, { recursive: true });
+    await writeFile(
+      path.join(toolDir, "get_weather.json"),
+      JSON.stringify({ name: "get_weather", description: "Get weather" }),
+      "utf8",
+    );
+
+    const res = await runSkillsIndexCommand(mockProject, { write: true });
+    expect(res.tools).toHaveLength(1);
+    expect(res.tools[0].name).toBe("get_weather");
+    expect(res.skills).toHaveLength(0);
+
+    // Verify .agents/skills was not created with phantom skill-creator
+    const phantomSkillPath = path.join(
+      mockProject,
+      ".agents",
+      "skills",
+      "main",
+      "skill-creator",
+    );
+    expect(fs.existsSync(phantomSkillPath)).toBe(false);
+
+    // Verify written index.json matches
+    const written = JSON.parse(
+      await fs.promises.readFile(
+        path.join(mockProject, ".well-known", "agent-skills", "index.json"),
+        "utf8",
+      ),
+    );
+    expect(written.skills).toHaveLength(0);
+    expect(written.tools).toHaveLength(1);
+  });
+
+  it("materializes bundled skill files onto disk when bundled: true is passed to runSkillsIndexCommand", async () => {
+    const mockToolchain = path.join(tempDir, "custom-toolchain");
+    const bundledSkillDir = path.join(
+      mockToolchain,
+      ".agents",
+      "skills",
+      "main",
+      "skill-creator",
+    );
+    await mkdir(bundledSkillDir, { recursive: true });
+    await writeFile(
+      path.join(bundledSkillDir, "SKILL.md"),
+      "---\nname: skill-creator\ndescription: Bundled skill creator.\n---\n",
+      "utf8",
+    );
+
+    const mockProject = path.join(tempDir, "bundled-project");
+    const res = await runSkillsIndexCommand(mockProject, {
+      bundled: true,
+      toolchainRoot: mockToolchain,
+      write: true,
+    });
+
+    expect(res.skills).toHaveLength(1);
+    expect(res.skills[0].name).toBe("skill-creator");
+    expect(res.skills[0].url).toBe(
+      "../../.agents/skills/main/skill-creator/SKILL.md",
+    );
+
+    // Verify the file was actually materialized onto disk so the relative URL is not a 404
+    const materializedSkill = path.join(
+      mockProject,
+      ".agents",
+      "skills",
+      "main",
+      "skill-creator",
+      "SKILL.md",
+    );
+    expect(fs.existsSync(materializedSkill)).toBe(true);
+    const content = await fs.promises.readFile(materializedSkill, "utf8");
+    expect(content).toContain("skill-creator");
+  });
 });

package/src/cli/commands/skills-index.ts

@@ -6,7 +6,14 @@ import { getProjectRoot } from "../utils/resolve-project-root.js";
  */
 
 import crypto from "node:crypto";
-import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
+import {
+  cp,
+  mkdir,
+  readdir,
+  readFile,
+  stat,
+  writeFile,
+} from "node:fs/promises";
 import path from "node:path";
 import process from "node:process";
 import matter from "gray-matter";
@@ -381,6 +388,17 @@ export async function generateSkillsIndex(
         continue;
       }
 
+      // Materialize bundled skill files onto disk if writing is enabled and file does not yet exist
+      if (options.write !== false) {
+        const targetSkillFile = path.join(resolvedRoot, relFromRoot);
+        if (!(await pathExists(targetSkillFile))) {
+          const sourceSkillDir = path.dirname(fullSkillPath);
+          const targetSkillDir = path.dirname(targetSkillFile);
+          await mkdir(targetSkillDir, { recursive: true });
+          await cp(sourceSkillDir, targetSkillDir, { recursive: true });
+        }
+      }
+
       discoveredSkills.push({
         name: skillName,
         type: "skill-md",
@@ -465,7 +483,7 @@ export async function runSkillsIndexCommand(
   console.log(`Generating Agent Skills Discovery index for ${targetDir}...`);
 
   const result = await generateSkillsIndex(targetDir, {
-    includeBundled: options.bundled !== false,
+    includeBundled: Boolean(options.bundled),
     ...options,
     configPath: options.config || options.configPath,
   });

package/src/components/shadow-claw-tools/shadow-claw-tools.html

@@ -544,6 +544,13 @@
               >
                 🌱 Block Garden Knowledge Hub
               </button>
+              <button
+                class="tools__preset-btn"
+                data-url="https://xt-ml.github.io/shadow-claw-agent-cli-weather"
+                type="button"
+              >
+                🌦️ Weather Agent (Open-Meteo)
+              </button>
             </div>
           </div>
 

package/src/worker/headless-agent.ts

@@ -12,11 +12,14 @@ import {
 } from "./utils/handleInvoke.js";
 import { executeNodeTransformersCompletion } from "./tools/node-transformers-executor.js";
 import { executeNodeLlamafileCompletion } from "./tools/node-llamafile-executor.js";
+import { setHeadlessEvalExecutor } from "./utils/sandboxedEval.js";
+import { nativeEvalExecutor } from "./utils/native-eval-executor.js";
 
 // Automatically wire native OS process execution for headless CLI agent
 setHeadlessBashExecutor(nativeBashExecutor);
 setNodeTransformersCompletionExecutor(executeNodeTransformersCompletion);
 setNodeLlamafileCompletionExecutor(executeNodeLlamafileCompletion);
+setHeadlessEvalExecutor(nativeEvalExecutor);
 
 export { executeTool } from "./utils/executeTool.js";
 export { executeToolChain } from "./utils/toolChain.js";
@@ -52,6 +55,16 @@ export {
 export { TOOL_DEFINITIONS } from "../subsystems/tools/index.js";
 export { DEFAULT_BUILTIN_PROFILE } from "../subsystems/tools/builtin-profiles.js";
 export { loadDeclarativeTools } from "../subsystems/tools/declarative.js";
+export {
+  resolveDiscoveryUrl,
+  fetchDiscoveryManifest,
+} from "../subsystems/tools/remote/discovery.js";
+export {
+  importRemoteArtifacts,
+  importRemoteTool,
+  importRemoteSkill,
+  importRemoteScript,
+} from "../subsystems/tools/remote/importArtifacts.js";
 export {
   getProvider,
   getDefaultProvider,
@@ -81,4 +94,5 @@ export { executeNodeTransformersCompletion } from "./tools/node-transformers-exe
 export { createTransformersRuntimeService } from "../server/services/transformers-runtime.js";
 export { cleanupAllLlamafileProcesses } from "../server/services/llamafile-manager.js";
 export { nativeBashExecutor, setHeadlessBashExecutor };
+export { nativeEvalExecutor, setHeadlessEvalExecutor };
 export type * from "./headless-types.js";

package/src/worker/headless-types.ts

@@ -4,6 +4,7 @@
  */
 
 import type { ShadowClawDatabase } from "../db/types.js";
+import type { ImportResult } from "../subsystems/tools/remote/types.js";
 
 /**
  * Dedicated agent configuration section in shadow-claw.config.json.
@@ -235,3 +236,41 @@ export interface AgentModelResult {
   error?: string;
   [key: string]: unknown;
 }
+
+/**
+ * Options for runAgentImport.
+ */
+export interface AgentImportOptions {
+  workspace?: string;
+  group?: string;
+  databaseDir?: string;
+  cacheDir?: string;
+  tools?: string | string[];
+  skills?: string | string[];
+  scripts?: string | string[];
+  all?: boolean;
+  overwrite?: boolean;
+  autoEnable?: boolean;
+  targetSubdir?: string;
+  siteSlug?: string;
+  quiet?: boolean;
+  verbose?: boolean;
+  fetchFn?: typeof fetch;
+  [key: string]: unknown;
+}
+
+/**
+ * Result returned by runAgentImport.
+ */
+export interface AgentImportResult {
+  success: boolean;
+  manifest?: unknown;
+  siteUrl?: string;
+  manifestUrl?: string;
+  result?: ImportResult;
+  importedCount?: number;
+  skippedCount?: number;
+  failedCount?: number;
+  error?: string;
+  [key: string]: unknown;
+}

package/src/worker/utils/native-eval-executor.test.ts

@@ -0,0 +1,36 @@
+import { nativeEvalExecutor } from "./native-eval-executor.js";
+import { sandboxedEval, setHeadlessEvalExecutor } from "./sandboxedEval.js";
+
+describe("nativeEvalExecutor", () => {
+  it("evaluates javascript expressions in node:vm", async () => {
+    const res = await nativeEvalExecutor("return 2 + 2;", 5000, false);
+    expect(res).toEqual({ ok: true, value: 4 });
+  });
+
+  it("handles injected $PIPE_DATA and data parameter", async () => {
+    const res = await nativeEvalExecutor(
+      "const d = JSON.parse(data); return d.val * 3;",
+      5000,
+      false,
+      JSON.stringify({ val: 10 }),
+    );
+    expect(res).toEqual({ ok: true, value: 30 });
+  });
+
+  it("integrates with sandboxedEval when Worker is undefined", async () => {
+    setHeadlessEvalExecutor(nativeEvalExecutor);
+    const origWorker = (globalThis as any).Worker;
+    try {
+      delete (globalThis as any).Worker;
+      const res = await sandboxedEval("return 100 * 2;");
+      expect(res).toEqual({ ok: true, value: 200 });
+    } finally {
+      (globalThis as any).Worker = origWorker;
+    }
+  });
+
+  it("handles evaluation timeout gracefully", async () => {
+    const res = await nativeEvalExecutor("while(true) {}", 50, false);
+    expect(res.ok).toBe(false);
+  });
+});