tarsk 0.0.9 → 0.0.11

package/.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+      {
+        "type": "node",
+        "request": "launch",
+        "name": "Debug Tests",
+        "autoAttachChildProcesses": true,
+        "skipFiles": ["<node_internals>/**", "**/node_modules/**"],
+        "program": "${workspaceRoot}/node_modules/vitest/vitest.mjs",
+        "args": ["run", "${relativeFile}"],
+        "smartStep": true,
+        "console": "integratedTerminal"
+      }
+    ]
+  }

package/dist/.tarsk/.tools/books.js

@@ -1,5 +1,8 @@
-// Searches for books in the Project Gutenberg library based on provided search terms
-// @param @array @required searchTerms List of search terms to find books in the Gutenberg library (e.g. ['dickens', 'great'] to search for books by Dickens with 'great' in the title)
+/** 
+  * Searches for books in the Project Gutenberg library based on provided search terms
+  * @param @array @required searchTerms - List of search terms to find books in the Gutenberg library (e.g. ['dickens', 'great'] to search for books by Dickens with 'great' in the title)  
+  * @returns {Promise<Book[]>} A promise that resolves to an array of books matching the search terms
+ */
 export async function searchGutenbergBooks(searchTerms          )                  {
   const searchQuery = searchTerms.join(' ');
   const url = 'https://gutendex.com/books';
@@ -11,45 +14,3 @@ export async function searchGutenbergBooks(searchTerms          )
     authors: book.authors,
   }));
 }
-
-export default function tools() {
-  return booksTools;
-}
-
-// Automate via TS Morph?
-const booksTools = [
-  {
-    type: 'function',
-    function: {
-      name: 'searchGutenbergBooks',
-      description:
-        'Search for books in the Project Gutenberg library based on specified search terms',
-      parameters: {
-        type: 'object',
-        properties: {
-          searchTerms: { // Must match argument name in function
-            type: 'array',
-            items: {
-              type: 'string',
-            },
-            description:
-              "List of search terms to find books in the Gutenberg library (e.g. ['dickens', 'great'] to search for books by Dickens with 'great' in the title)",
-          },
-        },
-        required: ['searchTerms'],
-      },
-    },
-  },
-];
-
-;               
-             
-                
-                    
- 
-
-;                 
-                      
-                      
-               
- 

package/dist/.tarsk/.tools/books.js.definition.json

@@ -0,0 +1,24 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "searchGutenbergBooks",
+      "description": "Searches for books in the Project Gutenberg library based on provided search terms",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "searchTerms": {
+            "type": "array",
+            "description": "List of search terms to find books in the Gutenberg library (e.g. ['dickens', 'great'] to search for books by Dickens with 'great' in the title)",
+            "items": {
+              "type": "string"
+            }
+          }
+        },
+        "required": [
+          "searchTerms"
+        ]
+      }
+    }
+  }
+]

package/dist/.tarsk/.tools/books.test.js

@@ -1,4 +1,4 @@
-import {searchGutenbergBooks} from "./books";
+import {searchGutenbergBooks} from "./books.js";
 
 // ToDo - run all functions regardles of if its called test
 // ToDO - dynamically load properly

package/dist/.tarsk/.tools/dust.js

@@ -1,3 +1,8 @@
+/**
+ * Searches for burns based on provided search terms.
+ * @param {string[]} searchTerms - List of search terms to find burns.
+ * @returns {Promise<Burn[]>} A promise that resolves with an array of Burn objects.
+ */
 export async function searchBurns(searchTerms          )                  {
   const searchQuery = searchTerms.join(' ');
   const url = 'https://api.dust.events/data/festivals.json';
@@ -18,35 +23,6 @@ export async function searchBurns(searchTerms          )                  {
   }));
 }
 
-export default function tools() {
-  return burnTools;
-}
-
-const burnTools = [
-  {
-    type: 'function',
-    function: {
-      name: 'searchBurns',
-      description:
-        "Search for regional burning man events based on specified search terms",
-      parameters: {
-        type: 'object',
-        properties: {
-          searchTerms: { // Must match argument name in function
-            type: 'array',
-            items: {
-              type: 'string',
-            },
-            description:
-              "List of search terms (e.g. ['snrg', 'soak'] to search for regional burns with 'snrg' or 'soak' in the title",
-          },
-        },
-        required: ['searchTerms'],
-      },
-    },
-  },
-];
-
 ;                      
               
                

package/dist/.tarsk/.tools/dust.js.definition.json

@@ -0,0 +1,24 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "searchBurns",
+      "description": "Searches for burns based on provided search terms.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "searchTerms": {
+            "type": "array",
+            "description": "",
+            "items": {
+              "type": "string"
+            }
+          }
+        },
+        "required": [
+          "searchTerms"
+        ]
+      }
+    }
+  }
+]

package/dist/.tarsk/.tools/dust.test.js

@@ -1,4 +1,4 @@
-import { searchBurns } from './dust';
+import { searchBurns } from './dust.js';
 
 export async function test() {
     const burns = await searchBurns(['']);

package/dist/.tarsk/tools/books.test.ts

@@ -1,4 +1,4 @@
-import {searchGutenbergBooks} from "./books";
+import {searchGutenbergBooks} from "./books.js";
 
 // ToDo - run all functions regardles of if its called test
 // ToDO - dynamically load properly

package/dist/.tarsk/tools/books.ts

@@ -1,5 +1,8 @@
-// Searches for books in the Project Gutenberg library based on provided search terms
-// @param @array @required searchTerms List of search terms to find books in the Gutenberg library (e.g. ['dickens', 'great'] to search for books by Dickens with 'great' in the title)
+/** 
+  * Searches for books in the Project Gutenberg library based on provided search terms
+  * @param @array @required searchTerms - List of search terms to find books in the Gutenberg library (e.g. ['dickens', 'great'] to search for books by Dickens with 'great' in the title)  
+  * @returns {Promise<Book[]>} A promise that resolves to an array of books matching the search terms
+ */
 export async function searchGutenbergBooks(searchTerms: string[]): Promise<Book[]> {
   const searchQuery = searchTerms.join(' ');
   const url = 'https://gutendex.com/books';
@@ -11,45 +14,3 @@ export async function searchGutenbergBooks(searchTerms: string[]): Promise<Book[
     authors: book.authors,
   }));
 }
-
-export default function tools() {
-  return booksTools;
-}
-
-// Automate via TS Morph?
-const booksTools = [
-  {
-    type: 'function',
-    function: {
-      name: 'searchGutenbergBooks',
-      description:
-        'Search for books in the Project Gutenberg library based on specified search terms',
-      parameters: {
-        type: 'object',
-        properties: {
-          searchTerms: { // Must match argument name in function
-            type: 'array',
-            items: {
-              type: 'string',
-            },
-            description:
-              "List of search terms to find books in the Gutenberg library (e.g. ['dickens', 'great'] to search for books by Dickens with 'great' in the title)",
-          },
-        },
-        required: ['searchTerms'],
-      },
-    },
-  },
-];
-
-interface Book {
-  id: string;
-  title: string;
-  authors: Person[];
-}
-
-interface Person {
-  birth_year?: number;
-  death_year?: number;
-  name: string;
-}

package/dist/.tarsk/tools/dust.test.ts

@@ -1,4 +1,4 @@
-import { searchBurns } from './dust';
+import { searchBurns } from './dust.js';
 
 export async function test() {
     const burns = await searchBurns(['']);

package/dist/.tarsk/tools/dust.ts

@@ -1,3 +1,8 @@
+/**
+ * Searches for burns based on provided search terms.
+ * @param {string[]} searchTerms - List of search terms to find burns.
+ * @returns {Promise<Burn[]>} A promise that resolves with an array of Burn objects.
+ */
 export async function searchBurns(searchTerms: string[]): Promise<Burn[]> {
   const searchQuery = searchTerms.join(' ');
   const url = 'https://api.dust.events/data/festivals.json';
@@ -18,35 +23,6 @@ export async function searchBurns(searchTerms: string[]): Promise<Burn[]> {
   }));
 }
 
-export default function tools() {
-  return burnTools;
-}
-
-const burnTools = [
-  {
-    type: 'function',
-    function: {
-      name: 'searchBurns',
-      description:
-        "Search for regional burning man events based on specified search terms",
-      parameters: {
-        type: 'object',
-        properties: {
-          searchTerms: { // Must match argument name in function
-            type: 'array',
-            items: {
-              type: 'string',
-            },
-            description:
-              "List of search terms (e.g. ['snrg', 'soak'] to search for regional burns with 'snrg' or 'soak' in the title",
-          },
-        },
-        required: ['searchTerms'],
-      },
-    },
-  },
-];
-
 export interface Burn {
   name: string
   title: string

package/dist/api/tools.js

@@ -1,11 +1,12 @@
 import { existsSync, mkdirSync, promises, readdirSync, writeFileSync, } from "fs";
 import { extname, join } from "path";
-import { extensionLess, tarskFolder } from "./utils.js";
+import { extensionLess, isEmpty, tarskFolder } from "./utils.js";
 import { logError, logWrite } from "../log/log.js";
 import tsBlankSpace from "ts-blank-space";
 import { loadTools } from "../tools.js";
 import { callFunction } from "../agent/agent.js";
 import { readAsJSONIfExists } from "../utils/files.js";
+import ts, { SyntaxKind } from "typescript";
 export async function api_run_tool(c) {
     const tool = await c.req.json();
     logWrite({ type: "api_run_tool", args: { toolName: tool.name } });
@@ -56,6 +57,7 @@ export async function api_save_tool(c) {
     const filename = name.replace(/\s+/g, "_").toLowerCase();
     const srcPath = join(toolsSrcFolder(), `${filename}.ts`);
     const jsCodePath = join(toolsJSFolder(), `${filename}.js`);
+    const defPath = join(toolsJSFolder(), `${filename}.js.definition.json`);
     const testPath = join(toolsSrcFolder(), `${filename}.test.ts`);
     const jsTestPath = join(toolsJSFolder(), `${filename}.test.js`);
     const metaPath = join(toolsSrcFolder(), `${filename}.json`);
@@ -63,9 +65,16 @@ export async function api_save_tool(c) {
     if (currentMeta) {
         meta.revision = (meta.revision ?? 1) + 1;
     }
+    const missingData = {
+        missingParameters: [],
+        missingFunctions: []
+    };
+    const definition = await inspectCode(srcPath, missingData);
     // Save the Typescript
     writeFileSync(srcPath, code, "utf-8");
     writeFileSync(metaPath, JSON.stringify(meta, null, 2), "utf-8");
+    // Write the function definition
+    writeFileSync(defPath, JSON.stringify(definition, null, 2), "utf-8");
     // Save the Test
     writeFileSync(testPath, test, "utf-8");
     // Save the Javascript Code
@@ -73,7 +82,10 @@ export async function api_save_tool(c) {
     // Save the Javascript Test
     writeFileSync(jsTestPath, testCode, "utf-8");
     logWrite({ type: "api_save_tool", args: { test: testPath, code: srcPath, meta: metaPath } });
-    return c.json({ message: `Tool saved successfully (${jsCodePath})` });
+    return c.json({ message: `Tool saved successfully (${jsCodePath})`,
+        missingParameters: missingData.missingParameters,
+        missingFunctions: missingData.missingFunctions,
+    });
 }
 // Get the tool with code
 export async function api_get_tool(c) {
@@ -166,3 +178,110 @@ async function readOrEmpty(filename) {
     }
     return await promises.readFile(filename, "utf-8");
 }
+export async function inspectCode(filename, missingData) {
+    const code = await readOrEmpty(filename);
+    const tools = [];
+    const sourceFile = ts.createSourceFile('temp.ts', code, ts.ScriptTarget.Latest, true);
+    function visit(node) {
+        if ((ts.isFunctionDeclaration(node) || ts.isFunctionExpression(node) || ts.isArrowFunction(node)) && node.name) {
+            const parsed = parseJSDoc(node);
+            console.log("Parsed function", parsed);
+            const properties = {};
+            const required = [];
+            node.parameters.forEach(param => {
+                const isOptional = !!param.questionToken || !!param.initializer;
+                if (!isOptional) {
+                    required.push(param.name.getText());
+                }
+                let type = typeName(param.type?.getText());
+                if (param.type?.kind == SyntaxKind.TypeReference) {
+                    type = 'object';
+                }
+                parsed.params;
+                const hasDescription = Object.keys(parsed.params).includes(param.name.getText());
+                const description = hasDescription ? parsed.params[param.name.getText()] : '';
+                if (!hasDescription) {
+                    missingData.missingParameters.push(param.name.escapedText);
+                }
+                properties[param.name.getText()] = {
+                    type,
+                    description
+                };
+                if (type == 'array') {
+                    properties[param.name.getText()].items = {
+                        type: arrayType(param.type?.getText()),
+                    };
+                }
+            });
+            const functionDescription = parsed.description;
+            if (isEmpty(functionDescription)) {
+                missingData.missingFunctions.push(node.name.getText());
+            }
+            tools.push({
+                type: 'function',
+                function: {
+                    name: node.name?.text ?? '',
+                    description: functionDescription,
+                    parameters: {
+                        type: 'object',
+                        properties,
+                        required,
+                    }
+                }
+            });
+        }
+        ts.forEachChild(node, visit);
+    }
+    visit(sourceFile);
+    return tools;
+}
+function parseJSDoc(node) {
+    const functionName = node.name?.getText();
+    const jsDoc = ts.getJSDocCommentsAndTags(node);
+    const parsed = {
+        functionName,
+        description: "",
+        params: {},
+    };
+    let info = '';
+    for (const doc of jsDoc) {
+        if (ts.isJSDoc(doc)) {
+            if (doc.comment) {
+                parsed.description = doc.comment.toString();
+            }
+            if (doc.tags) {
+                for (const tag of doc.tags) {
+                    info += ' ' + tag.comment;
+                    //          if (ts.isJSDocPropertyTag(tag)) {
+                    if (tag.kind == 327) {
+                        let txt = tag.comment ? tag.comment.toString() : "";
+                        const lines = txt.split(' ');
+                        const paramName = lines[0];
+                        if (txt.startsWith(paramName)) {
+                            txt = txt.substring(paramName.length).trim();
+                            if (txt.startsWith('-')) {
+                                txt = txt.substring(1).trim();
+                            }
+                        }
+                        //parsed.params[paramName] = paramDesc;
+                        parsed.params[paramName] = txt;
+                    }
+                }
+            }
+        }
+    }
+    return parsed;
+}
+function typeName(name) {
+    if (!name)
+        return 'any';
+    if (name.endsWith('[]')) {
+        return 'array';
+    }
+    return name;
+}
+function arrayType(name) {
+    if (!name)
+        return 'any';
+    return name.replace('[]', '');
+}

package/dist/tools.js

@@ -19,24 +19,6 @@ export async function loadTools(modulePaths) {
     }
     return tools;
 }
-// async function importFromMemory(moduleCode: string) {
-//   const module = new Function(
-//     "exports",
-//     "module",
-//     "require",
-//     "__filename",
-//     "__dirname",
-//     moduleCode
-//   );
-//   const exports = {};
-//   const moduleObj = { exports: exports };
-//   module(exports, moduleObj, require, null, null);
-//   return moduleObj.exports;
-// }
-// async function importModule(code: string) {
-//   const myModule = await importFromMemory(code);
-//   //console.log(myModule.hello()); // Output: Hello from memory!
-// }
 function findMetaData(modulePath) {
     // Load the meta data: its in the toolsfolder not the .tools folder
     let metaPath = (getPathWithoutExtension(modulePath) + ".json").replace('.tools', 'tools');
@@ -61,9 +43,6 @@ async function inspectTool(modulePath) {
         path: modulePath,
         revision: undefined
     };
-    // When 'key' is no longer referenced and garbage collected,
-    // the module './myModule.js' associated with it in 'weakModules'
-    // will also be eligible for garbage collection.
     logStart({ type: "load_tool", args: { modulePath } }, `The modulePath is "${modulePath}"`);
     try {
         const meta = findMetaData(modulePath);
@@ -89,7 +68,7 @@ async function inspectTool(modulePath) {
         try {
             // This is the definition needed for the tool for the agent (TODO: Generate this)
             if (!isTest) {
-                result.definition = myModule.default();
+                result.definition = getDefinition(modulePath);
             }
         }
         catch (e) {
@@ -99,10 +78,14 @@ async function inspectTool(modulePath) {
     }
     catch (e) {
         logError({ type: "load_tool_error", args: e });
+        throw new Error(`load_tool_error ${modulePath}: ${e}`);
     }
     finally {
         logEnd("load_tool");
         logWrite({ type: 'load_tool', args: { result } });
     }
-    throw new Error(`Error loading tool from ${modulePath}`);
+}
+function getDefinition(modulePath) {
+    const filename = modulePath + '.definition.json';
+    return readAsJSONIfExists(filename);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tarsk",
-  "version": "0.0.9",
+  "version": "0.0.11",
   "author": "WebNative LLC",
   "description": "Tarsk is a AI tool available at https://tarsk.io",
   "license": "MIT",
@@ -10,6 +10,8 @@
     "start": "npm run build && npm link && tarsk",
     "restart": "npm unlink tarsk && rm -rf dist && npm run build && npm link && tarsk",
     "publish": "npm run build && npm publish",
+    "test": "vitest",
+    "test2": "bun test/test-get-definition.ts",
     "dev": "tsx watch src/index.ts"
   },
   "bin": {
@@ -23,6 +25,7 @@
   },
   "devDependencies": {
     "@types/node": "^20.11.17",
-    "tsx": "^4.7.1"
+    "tsx": "^4.7.1",
+    "vitest": "^3.1.2"
   }
 }

package/test/test-fn-expected-result.ts

@@ -0,0 +1,25 @@
+  // Automate via TS Morph?
+  const booksTools = [
+    {
+      type: 'function',
+      function: {
+        name: 'searchGutenbergBooks',
+        description:
+          'Search for books in the Project Gutenberg library based on specified search terms',
+        parameters: {
+          type: 'object',
+          properties: {
+            searchTerms: { // Must match argument name in function
+              type: 'array',
+              items: {
+                type: 'string',
+              },
+              description:
+                "List of search terms to find books in the Gutenberg library (e.g. ['dickens', 'great'] to search for books by Dickens with 'great' in the title)",
+            },
+          },
+          required: ['searchTerms'],
+        },
+      },
+    },
+  ];

package/test/test-fn.ts

@@ -0,0 +1,30 @@
+/** 
+  * Searches for books in the Project Gutenberg library based on provided search terms
+  * @param @array @required searchTerms - List of search terms to find books in the Gutenberg library (e.g. ['dickens', 'great'] to search for books by Dickens with 'great' in the title)  
+  * @param @object @required book - The book object to search for
+  * @returns {Promise<Book[]>} A promise that resolves to an array of books matching the search terms
+ */
+export async function searchGutenbergBooks(searchTerms: string[], count: number, book: Book): Promise<Book[]> {
+    const searchQuery = searchTerms ? searchTerms.join(' ') : '';
+    const url = 'https://gutendex.com/books';
+    const response = await fetch(`${url}?search=${searchQuery}`);
+    const data = await response.json();
+    return data.results.map((book: any) => ({
+      id: book.id,
+      title: book.title,
+      authors: book.authors,
+    }));
+  }
+  
+  interface Book {
+    id: string;
+    title: string;
+    authors: Person[];
+  }
+  
+  interface Person {
+    birth_year?: number;
+    death_year?: number;
+    name: string;
+  }
+  

package/test/test-fn2.ts

@@ -0,0 +1,55 @@
+/**
+ * Searches for burns based on provided search terms.
+ * @param {string[]} searchTerms - List of search terms to find burns.
+ * @returns {Promise<Burn[]>} A promise that resolves with an array of Burn objects.
+ */
+export async function searchBurns(searchTerms: string[]): Promise<Burn[]> {
+    const searchQuery = searchTerms.join(' ');
+    const url = 'https://api.dust.events/data/festivals.json';
+    const response = await fetch(`${url}`);
+    let data = await response.json();
+    data = data.filter((b) => b.active);
+    return data.map((burn: any) => ({
+      name: burn.name,
+      title: burn.title,
+      year: burn.year,
+      startDate: burn.start,
+      endDate: burn.end,
+      lat: burn.lat,
+      long: burn.long,
+      timeZone: burn.timeZone,
+      region: burn.region,
+      website: burn.website
+    }));
+  }
+  
+  export interface Burn {
+    name: string
+    title: string
+    year: string
+    active: boolean
+    id: string
+    uid: number
+    start: string
+    end: string
+    lat: any
+    long: any
+    imageUrl?: string
+    timeZone: string
+    mapDirection: number
+    mastodonHandle: string
+    rssFeed: string
+    inboxEmail: string
+    region: string
+    website: string
+    unknownDates: boolean
+    volunteeripateSubdomain: string
+    volunteeripateIdentifier: string
+    pin_size_multiplier: number
+    camp_registration: boolean
+    event_registration: boolean
+    pin: string
+    directions?: string
+  }
+  
+  

package/test/test-get-definition.spec.ts

@@ -0,0 +1,33 @@
+import { expect, test } from 'vitest'
+import { inspectCode, MissingData } from "../src/api/tools.js";
+import { Tool } from '../src/agent/interfaces.js';
+
+test('function definition generation works', async () => {
+    const missingData: MissingData = {
+        missingFunctions: [],
+        missingParameters: [],
+    };
+
+    const tools: Tool[] = await inspectCode('./test/test-fn.ts', missingData);
+    
+    // One comment is missing
+    expect(missingData.missingParameters.length).toBe(1);
+    // One comment is missing
+    expect(missingData.missingParameters[0]).toBe('count');
+    expect(tools[0].function.description).toBe('Searches for books in the Project Gutenberg library based on provided search terms');
+});
+
+test('function definition generation works style 2', async () => {
+    const missingData: MissingData = {
+        missingFunctions: [],
+        missingParameters: [],
+    };
+
+    const tools: Tool[] = await inspectCode('./test/test-fn2.ts', missingData);
+    
+    // One comment is missing
+    expect(missingData.missingParameters.length).toBe(1);
+    // One comment is missing
+    expect(missingData.missingParameters[0]).toBe('count');
+    expect(tools[0].function.description).toBe('');
+});

package/test/test-get-definition.ts

@@ -0,0 +1,9 @@
+import { inspectCode } from "../src/api/tools";
+
+async function test() {
+    const tools = await inspectCode('./test/test-fn.ts', { missingFunctions: [], missingParameters: [] });
+    const txt = JSON.stringify(tools, null, 2);
+    console.log();
+}
+
+test();