toolpack-sdk 1.0.0 → 1.1.0-SNAPSHOT

package/README.md

@@ -443,6 +443,117 @@ const sdk = await Toolpack.init({
 | `tools` | ToolDefinition[] | ✓ | Array of tool definitions |
 | `dependencies` | Record<string, string> | | npm dependencies (validated at load) |
 
+## Knowledge Module (RAG)
+
+The Knowledge module provides **Retrieval-Augmented Generation (RAG)** capabilities, enabling AI agents to search and retrieve relevant information from your documents during conversations.
+
+### Quick Start
+
+```typescript
+import {
+  Toolpack,
+  Knowledge,
+  MemoryProvider,
+  MarkdownSource,
+} from 'toolpack-sdk';
+
+// 1. Create a knowledge base from your documents
+const kb = await Knowledge.create({
+  provider: new MemoryProvider(),
+  source: new MarkdownSource('./docs'),  // Directory or glob pattern
+});
+
+// 2. Initialize Toolpack with knowledge
+const toolpack = await Toolpack.init({
+  provider: 'openai',
+  knowledge: kb,
+});
+
+// 3. AI automatically searches the knowledge base when needed
+const response = await toolpack.generate({
+  messages: [{ role: 'user', content: 'What does the documentation say about authentication?' }]
+});
+```
+
+When you provide a `knowledge` option, the SDK automatically:
+- Registers a `knowledge_search` tool
+- Adds it to `alwaysLoadedTools` for all modes
+- The AI calls it whenever it needs to retrieve information
+
+### Components
+
+| Component | Purpose | Options |
+|-----------|---------|---------|
+| **Sources** | Load and chunk content | `MarkdownSource`, `JSONSource`, `SQLiteTextSource` |
+| **Embedders** | Convert text to vectors | `OllamaEmbedder`, `OpenAIEmbedder`, `GeminiEmbedder` |
+| **Providers** | Store and query vectors | `MemoryProvider` |
+
+### Sources
+
+```typescript
+// Markdown files (auto-appends **/*.md if directory)
+new MarkdownSource('./docs')
+new MarkdownSource('./docs/**/*.md', { namespace: 'docs', watch: true })
+
+// JSON files
+new JSONSource('./data/products.json', {
+  chunkBy: 'item',
+  contentFields: ['name', 'description'],
+  metadataFields: ['id', 'category'],
+})
+
+// SQLite databases
+new SQLiteTextSource('./data/articles.db', {
+  table: 'articles',
+  contentColumns: ['title', 'body'],
+  metadataColumns: ['id', 'author'],
+})
+```
+
+### Embedders
+
+By default, `Knowledge.create()` auto-detects an available embedder:
+1. **Ollama** (if running locally)
+2. **OpenAI** (if `TOOLPACK_OPENAI_KEY` or `OPENAI_API_KEY` is set)
+
+```typescript
+// Explicit embedder
+const kb = await Knowledge.create({
+  provider: new MemoryProvider(),
+  source: new MarkdownSource('./docs'),
+  embedder: new OpenAIEmbedder({ model: 'text-embedding-3-small' }),
+});
+```
+
+### Direct Query
+
+```typescript
+const results = await kb.query('authentication flow', {
+  limit: 5,
+  threshold: 0.3,  // Minimum similarity (0-1)
+  filter: { type: 'documentation' },
+});
+
+for (const result of results) {
+  console.log(`Score: ${result.score}`);
+  console.log(`Content: ${result.chunk.content}`);
+}
+```
+
+### Multiple Sources
+
+```typescript
+const kb = await Knowledge.create({
+  provider: new MemoryProvider(),
+  sources: [
+    new MarkdownSource('./docs', { namespace: 'docs' }),
+    new JSONSource('./data/faq.json', { namespace: 'faq' }),
+  ],
+});
+```
+
+For complete documentation, see the [Knowledge Module Guide](./documentation/docs/guides/knowledge.md).
+
 ## Multimodal Support
 
 The SDK supports multimodal inputs (text + images) across all vision-capable providers. Images can be provided in three formats:
@@ -504,8 +615,7 @@ export GOOGLE_GENERATIVE_AI_KEY="AIza..."
 
 # SDK logging (override — prefer toolpack.config.json instead)
 export TOOLPACK_SDK_LOG_FILE="./toolpack.log"    # Log file path (also enables logging)
-export TOOLPACK_SDK_LOG_VERBOSE="true"             # Verbose logging (also enables logging)
-export TOOLPACK_SDK_TOOL_RESULT_MAX_CHARS="20000"  # Max chars per tool result
+export TOOLPACK_SDK_LOG_LEVEL="debug"            # Log level override (error, warn, info, debug, trace)
 ```
 
 ## Configuration Architecture
@@ -551,7 +661,7 @@ Create a `toolpack.config.json` in your project root:
   "logging": {
     "enabled": true,
     "filePath": "./toolpack.log",
-    "verbose": true
+    "level": "info"
   }
 }
 ```
@@ -560,7 +670,7 @@ Create a `toolpack.config.json` in your project root:
 |--------|---------|-------------|
 | `enabled` | `false` | Enable file logging |
 | `filePath` | `toolpack-sdk.log` | Log file path (relative to CWD) |
-| `verbose` | `false` | Include message previews and tool details |
+| `level` | `info` | Log level (`error`, `warn`, `info`, `debug`, `trace`) |
 
 ### Tools Configuration
 
@@ -573,6 +683,7 @@ Create a `toolpack.config.json` in your project root:
     "autoExecute": true,
     "maxToolRounds": 5,
     "toolChoicePolicy": "auto",
+    "resultMaxChars": 20000,
     "enabledTools": [],
     "enabledToolCategories": [],
     "additionalConfigurations": {

package/dist/client/index.d.ts

@@ -24,6 +24,7 @@ export declare class AIClient extends EventEmitter {
     private activeMode;
     private overrideSystemPrompt?;
     private disableBaseContext;
+    private toolResultMaxChars;
     constructor(config: AIClientConfig);
     /**
      * Register a new provider instance.

package/dist/client/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,eAAe,EAAmC,gBAAgB,EAAE,iBAAiB,EAAmC,MAAM,UAAU,CAAC;AAEzL,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,EAAE,WAAW,EAAiD,MAAM,mBAAmB,CAAC;AAC/F,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAsMpD,MAAM,WAAW,cAAc;IAC3B,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC3C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,qBAAa,QAAS,SAAQ,YAAY;IACtC,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,UAAU,CAAmB;IACrC,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,oBAAoB,CAAC,CAAS;IACtC,OAAO,CAAC,kBAAkB,CAAU;gBAExB,MAAM,EAAE,cAAc;IAmBlC;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,eAAe;IAIxD;;OAEG;IACH,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,eAAe;IAY3C;;OAEG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAOtC;;OAEG;IACH,eAAe,IAAI,YAAY,GAAG,SAAS;IAI3C;;;OAGG;IACH,YAAY,IAAI,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC;IAI5C;;OAEG;IACH,eAAe,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI;IAI7C;;OAEG;IACH,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;IAIzC;;OAEG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAIjD;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI,GAAG,IAAI;IAKtC;;OAEG;IACH,OAAO,IAAI,UAAU,GAAG,IAAI;IAI5B;;;OAGG;IACH,YAAY,IAAI,IAAI;IAOpB;;OAEG;IACH,uBAAuB,IAAI,IAAI;IAI/B;;;;OAIG;IACG,QAAQ,CAAC,OAAO,EAAE,iBAAiB,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAmP9F;;;;OAIG;IACI,MAAM,CAAC,OAAO,EAAE,iBAAiB,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,eAAe,CAAC;IAmQjG;;OAEG;IACG,KAAK,CAAC,OAAO,EAAE,gBAAgB,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;IASzF;;;OAGG;YACW,sBAAsB;IAyHpC;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAqB3B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAkC9B;;OAEG;IACH,OAAO,CAAC,0BAA0B;IA4BlC;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAmD9B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IA8C9B;;;OAGG;YACW,WAAW;IAsIzB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA0C3B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA4B3B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IA+BzB,OAAO,CAAC,SAAS;CAMpB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/client/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,eAAe,EAAmC,gBAAgB,EAAE,iBAAiB,EAAmC,MAAM,UAAU,CAAC;AAEzL,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,EAAE,WAAW,EAAiD,MAAM,mBAAmB,CAAC;AAC/F,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAgMpD,MAAM,WAAW,cAAc;IAC3B,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC3C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,qBAAa,QAAS,SAAQ,YAAY;IACtC,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,UAAU,CAAmB;IACrC,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,oBAAoB,CAAC,CAAS;IACtC,OAAO,CAAC,kBAAkB,CAAU;IACpC,OAAO,CAAC,kBAAkB,CAAS;gBAEvB,MAAM,EAAE,cAAc;IAqBlC;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,eAAe;IAIxD;;OAEG;IACH,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,eAAe;IAY3C;;OAEG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAOtC;;OAEG;IACH,eAAe,IAAI,YAAY,GAAG,SAAS;IAI3C;;;OAGG;IACH,YAAY,IAAI,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC;IAI5C;;OAEG;IACH,eAAe,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI;IAI7C;;OAEG;IACH,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;IAIzC;;OAEG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAIjD;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI,GAAG,IAAI;IAKtC;;OAEG;IACH,OAAO,IAAI,UAAU,GAAG,IAAI;IAI5B;;;OAGG;IACH,YAAY,IAAI,IAAI;IAOpB;;OAEG;IACH,uBAAuB,IAAI,IAAI;IAI/B;;;;OAIG;IACG,QAAQ,CAAC,OAAO,EAAE,iBAAiB,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAmP9F;;;;OAIG;IACI,MAAM,CAAC,OAAO,EAAE,iBAAiB,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,eAAe,CAAC;IAmQjG;;OAEG;IACG,KAAK,CAAC,OAAO,EAAE,gBAAgB,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;IASzF;;;OAGG;YACW,sBAAsB;IAgIpC;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAqB3B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAkC9B;;OAEG;IACH,OAAO,CAAC,0BAA0B;IA4BlC;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAmD9B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IA8C9B;;;OAGG;YACW,WAAW;IAsIzB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA0C3B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA4B3B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IA+BzB,OAAO,CAAC,SAAS;CAMpB"}