recursive-llm-ts 4.4.0 → 4.4.1

package/README.md

@@ -15,6 +15,7 @@ TypeScript/JavaScript package for [Recursive Language Models (RLM)](https://gith
 🔍 **Structured Outputs** - Extract typed data with Zod schemas and parallel execution
 🧠 **Meta-Agent Mode** - Automatically optimize queries for better results
 📊 **Observability** - OpenTelemetry tracing, Langfuse integration, and debug logging
+📁 **File Storage** - Process local directories or S3/MinIO/LocalStack buckets as LLM context
 
 ## Installation
 
@@ -281,6 +282,123 @@ LANGFUSE_SECRET_KEY=sk-...
 LANGFUSE_HOST=https://cloud.langfuse.com
 ```
 
+### File Storage Context
+
+Process files from local directories or S3-compatible storage as LLM context. Supports recursive directory traversal, filtering, and automatic context formatting.
+
+#### Local Files
+
+```typescript
+import { RLM } from 'recursive-llm-ts';
+
+const rlm = new RLM('gpt-4o-mini', {
+  api_key: process.env.OPENAI_API_KEY
+});
+
+// Process all TypeScript files in a directory
+const result = await rlm.completionFromFiles(
+  'Summarize the architecture of this codebase',
+  {
+    type: 'local',
+    path: '/path/to/project/src',
+    extensions: ['.ts', '.tsx'],
+    excludePatterns: ['*.test.ts', '*.spec.ts'],
+    maxFileSize: 100_000,  // Skip files over 100KB
+  }
+);
+
+console.log(result.result);
+console.log(`Files processed: ${result.fileStorage?.files.length}`);
+```
+
+#### S3 / Object Storage
+
+Works with AWS S3, MinIO, LocalStack, DigitalOcean Spaces, and Backblaze B2.
+
+```typescript
+// AWS S3 with explicit credentials
+const result = await rlm.completionFromFiles(
+  'What are the key findings in these reports?',
+  {
+    type: 's3',
+    path: 'my-bucket',          // Bucket name
+    prefix: 'reports/2024/',    // Folder prefix
+    extensions: ['.md', '.txt'],
+    credentials: {
+      accessKeyId: 'AKIA...',
+      secretAccessKey: '...',
+    },
+    region: 'us-west-2',
+  }
+);
+
+// S3 with environment variable credentials
+// Uses AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN
+const result2 = await rlm.completionFromFiles(
+  'Summarize these documents',
+  { type: 's3', path: 'my-bucket', prefix: 'docs/' }
+);
+
+// MinIO / LocalStack
+const result3 = await rlm.completionFromFiles(
+  'Analyze the data',
+  {
+    type: 's3',
+    path: 'local-bucket',
+    endpoint: 'http://localhost:9000',  // MinIO
+    credentials: { accessKeyId: 'minioadmin', secretAccessKey: 'minioadmin' },
+  }
+);
+```
+
+**Credential Resolution Order:**
+1. Explicit `credentials` in config
+2. Environment variables: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`
+3. AWS SDK default credential chain (IAM roles, `~/.aws/credentials`, ECS task role, etc.)
+
+#### Structured Extraction from Files
+
+```typescript
+import { z } from 'zod';
+
+const schema = z.object({
+  summary: z.string(),
+  mainTopics: z.array(z.string()),
+  sentiment: z.enum(['positive', 'negative', 'neutral']),
+});
+
+const result = await rlm.structuredCompletionFromFiles(
+  'Extract a summary, main topics, and overall sentiment',
+  { type: 'local', path: './docs', extensions: ['.md'] },
+  schema
+);
+
+console.log(result.result.mainTopics); // string[]
+```
+
+#### Preview Files Before Processing
+
+```typescript
+import { FileContextBuilder } from 'recursive-llm-ts';
+
+const builder = new FileContextBuilder({
+  type: 'local',
+  path: './src',
+  extensions: ['.ts'],
+  excludePatterns: ['node_modules/**'],
+});
+
+// List matching files without reading content
+const files = await builder.listMatchingFiles();
+console.log('Would process:', files);
+
+// Build full context
+const ctx = await builder.buildContext();
+console.log(`Total size: ${ctx.totalSize} bytes`);
+console.log(`Files included: ${ctx.files.length}`);
+console.log(`Files skipped: ${ctx.skipped.length}`);
+```
+
 ## API
 
 ### `RLM`
@@ -330,6 +448,21 @@ const result = await rlm.structuredCompletion('Analyze', doc, schema);
 // result.result is typed as { score: number, summary: string }
 ```
 
+#### `completionFromFiles(query: string, fileConfig: FileStorageConfig): Promise<RLMResult & { fileStorage?: FileStorageResult }>`
+
+Process a query using files from local or S3 storage as context.
+
+**Parameters:**
+- `query`: The question or task to perform
+- `fileConfig`: File storage configuration (local path or S3 bucket)
+
+**Returns:**
+- Result with `fileStorage` metadata (files included, skipped, total size)
+
+#### `structuredCompletionFromFiles<T>(query: string, fileConfig: FileStorageConfig, schema: ZodSchema<T>, options?): Promise<StructuredRLMResult<T> & { fileStorage?: FileStorageResult }>`
+
+Extract structured data from file-based context.
+
 #### `cleanup(): Promise<void>`
 
 Clean up the bridge and free resources.
@@ -423,6 +556,33 @@ interface TraceEvent {
   trace_id?: string;
   span_id?: string;
 }
+
+interface FileStorageConfig {
+  type: 'local' | 's3';         // Storage provider type
+  path: string;                  // Local directory path or S3 bucket name
+  prefix?: string;               // S3 key prefix (folder path)
+  region?: string;               // AWS region (default: AWS_REGION env or 'us-east-1')
+  credentials?: {                // S3 explicit credentials (optional)
+    accessKeyId: string;
+    secretAccessKey: string;
+    sessionToken?: string;
+  };
+  endpoint?: string;             // Custom S3 endpoint (MinIO, LocalStack, etc.)
+  forcePathStyle?: boolean;      // Force path-style S3 URLs (auto-enabled with endpoint)
+  extensions?: string[];         // File extensions to include (e.g., ['.ts', '.md'])
+  includePatterns?: string[];    // Glob patterns to include
+  excludePatterns?: string[];    // Glob patterns to exclude
+  maxFileSize?: number;          // Max individual file size in bytes (default: 1MB)
+  maxTotalSize?: number;         // Max total context size in bytes (default: 10MB)
+  maxFiles?: number;             // Max number of files (default: 1000)
+}
+
+interface FileStorageResult {
+  context: string;               // Built context string with file contents
+  files: Array<{ relativePath: string; size: number }>;
+  totalSize: number;
+  skipped: Array<{ relativePath: string; reason: string }>;
+}
 ```
 
 ## Environment Variables
@@ -441,6 +601,17 @@ const rlm = new RLM('gpt-4o-mini', {
 });
 ```
 
+### S3 File Storage
+
+When using S3 file storage without explicit credentials, these environment variables are checked:
+
+```bash
+export AWS_ACCESS_KEY_ID='your-access-key'
+export AWS_SECRET_ACCESS_KEY='your-secret-key'
+export AWS_SESSION_TOKEN='optional-session-token'
+export AWS_REGION='us-east-1'
+```
+
 ## Custom Providers
 
 The Go binary uses an **OpenAI-compatible chat completion API** and works seamlessly with

package/dist/bridge-interface.d.ts

@@ -49,6 +49,51 @@ export interface RLMConfig {
     debug?: boolean;
     [key: string]: any;
 }
+export interface FileStorageConfig {
+    /** Storage type: 'local' or 's3' */
+    type: 'local' | 's3';
+    /** For local: root directory path. For S3: bucket name */
+    path: string;
+    /** For S3: the prefix (folder path) within the bucket */
+    prefix?: string;
+    /** For S3: AWS region (falls back to AWS_REGION env var, then 'us-east-1') */
+    region?: string;
+    /**
+     * For S3: explicit credentials.
+     * Resolution order:
+     * 1. This field (explicit credentials)
+     * 2. Environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN
+     * 3. AWS SDK default credential chain (IAM role, ~/.aws/credentials, ECS task role, etc.)
+     */
+    credentials?: {
+        accessKeyId: string;
+        secretAccessKey: string;
+        sessionToken?: string;
+    };
+    /**
+     * For S3: custom endpoint URL.
+     * Use for S3-compatible services: MinIO, LocalStack, DigitalOcean Spaces, Backblaze B2.
+     * When set, forcePathStyle is automatically enabled.
+     */
+    endpoint?: string;
+    /**
+     * For S3: force path-style addressing (bucket in path, not subdomain).
+     * Automatically true when endpoint is set.
+     */
+    forcePathStyle?: boolean;
+    /** Glob patterns to include (e.g. ['*.ts', '*.md']) */
+    includePatterns?: string[];
+    /** Glob patterns to exclude (e.g. ['node_modules/**']) */
+    excludePatterns?: string[];
+    /** Maximum file size in bytes to include (default: 1MB) */
+    maxFileSize?: number;
+    /** Maximum total context size in bytes (default: 10MB) */
+    maxTotalSize?: number;
+    /** Maximum number of files to include (default: 1000) */
+    maxFiles?: number;
+    /** File extensions to include (e.g. ['.ts', '.md', '.txt']) */
+    extensions?: string[];
+}
 export interface PythonBridge {
     completion(model: string, query: string, context: string, rlmConfig: RLMConfig): Promise<RLMResult>;
     cleanup(): Promise<void>;

package/dist/file-storage.d.ts

@@ -0,0 +1,122 @@
+import { FileStorageConfig } from './bridge-interface';
+/**
+ * Custom error class for S3 storage operations.
+ * Provides actionable error messages for common failure modes.
+ */
+export declare class S3StorageError extends Error {
+    readonly code: string;
+    readonly bucket: string;
+    readonly key?: string;
+    readonly originalError?: Error;
+    constructor(opts: {
+        message: string;
+        code: string;
+        bucket: string;
+        key?: string;
+        originalError?: Error;
+    });
+}
+export { FileStorageConfig } from './bridge-interface';
+export interface FileEntry {
+    /** Relative path from the root of the storage source */
+    relativePath: string;
+    /** File content as a string */
+    content: string;
+    /** Size in bytes */
+    size: number;
+}
+export interface FileStorageResult {
+    /** The built context string containing all file contents */
+    context: string;
+    /** List of files that were included */
+    files: Array<{
+        relativePath: string;
+        size: number;
+    }>;
+    /** Total size of all file contents */
+    totalSize: number;
+    /** Files that were skipped and why */
+    skipped: Array<{
+        relativePath: string;
+        reason: string;
+    }>;
+}
+export interface FileStorageProvider {
+    listFiles(): Promise<string[]>;
+    readFile(relativePath: string): Promise<string>;
+    getFileSize(relativePath: string): Promise<number>;
+}
+export declare class LocalFileStorage implements FileStorageProvider {
+    private rootPath;
+    constructor(rootPath: string);
+    listFiles(): Promise<string[]>;
+    private walkDir;
+    readFile(relativePath: string): Promise<string>;
+    getFileSize(relativePath: string): Promise<number>;
+}
+/**
+ * S3FileStorage uses the AWS SDK v3 via dynamic import.
+ * If @aws-sdk/client-s3 is not installed, it throws a clear error.
+ *
+ * Supports:
+ * - AWS S3
+ * - MinIO (set endpoint to MinIO URL)
+ * - LocalStack (set endpoint to LocalStack URL, typically http://localhost:4566)
+ * - DigitalOcean Spaces, Backblaze B2, and other S3-compatible services
+ *
+ * Credential resolution order:
+ * 1. Explicit credentials in FileStorageConfig.credentials
+ * 2. Environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN
+ * 3. AWS SDK default credential chain (IAM role, ~/.aws/credentials, ECS task role, etc.)
+ */
+export declare class S3FileStorage implements FileStorageProvider {
+    private bucket;
+    private prefix;
+    private region;
+    private credentials?;
+    private endpoint?;
+    private forcePathStyle;
+    private s3Client;
+    constructor(config: FileStorageConfig);
+    /**
+     * Returns the credential source being used for debugging/logging.
+     */
+    getCredentialSource(): string;
+    private getClient;
+    private buildKey;
+    listFiles(): Promise<string[]>;
+    readFile(relativePath: string): Promise<string>;
+    getFileSize(relativePath: string): Promise<number>;
+}
+/**
+ * FileContextBuilder reads files from a storage provider, applies filters,
+ * and builds a structured context string for LLM consumption.
+ */
+export declare class FileContextBuilder {
+    private provider;
+    private config;
+    constructor(config: FileStorageConfig);
+    private createProvider;
+    /**
+     * Build a structured context string from all matching files.
+     * Files are formatted with clear delimiters so the LLM can reference them.
+     */
+    buildContext(): Promise<FileStorageResult>;
+    /**
+     * Build a file tree overview for the LLM to understand the folder structure.
+     */
+    private buildFileTree;
+    /**
+     * Format a single file's content with clear delimiters.
+     */
+    private formatFileBlock;
+    /**
+     * Get just the list of files without reading content.
+     * Useful for previewing what would be included.
+     */
+    listMatchingFiles(): Promise<string[]>;
+}
+/**
+ * Convenience function to build context from a file storage config.
+ */
+export declare function buildFileContext(config: FileStorageConfig): Promise<FileStorageResult>;