spine-framework 0.1.61 → 0.2.0

package/scripts/ingest-chunks.cjs

@@ -1,202 +0,0 @@
-#!/usr/bin/env node
-
-const fs = require('fs');
-const https = require('https');
-const querystring = require('querystring');
-
-// Configuration - use local Netlify Functions
-const NETLIFY_URL = 'http://localhost:8888';
-const SUPABASE_SERVICE_ROLE_KEY = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InV5b2t1aWlienR3YXNkcHJzb3YiLCJyb2xlIjoic2VydmljZV9yb2xlIiwiaWF0IjoxNzM2MTY3Njk5LCJleHAiOjIwNTE3NDM2OTl9.qKqJv8N-IY9q8o_XJtq2o5s3RfD2W4x7J8L9q8o_XJt';
-
-// Read chunks from file
-function readChunks(filePath) {
-  try {
-    const content = fs.readFileSync(filePath, 'utf8');
-    const data = JSON.parse(content);
-    return data.chunks || [];
-  } catch (error) {
-    console.error('Error reading chunks file:', error.message);
-    process.exit(1);
-  }
-}
-
-// Make HTTP request to local Netlify Function
-function makeRequest(functionName, data) {
-  return new Promise((resolve, reject) => {
-    const postData = JSON.stringify(data);
-    const url = `${NETLIFY_URL}/.netlify/functions/${functionName}`;
-    
-    console.log(`  🌐 Calling: ${url}`);
-    console.log(`  📦 Data size: ${postData.length} bytes`);
-
-    const options = {
-      hostname: 'localhost',
-      port: 8888,
-      path: `/.netlify/functions/${functionName}`,
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-        'Authorization': `Bearer ${SUPABASE_SERVICE_ROLE_KEY}`,
-        'Content-Length': Buffer.byteLength(postData)
-      }
-    };
-
-    const req = require('http').request(options, (res) => {
-      let responseData = '';
-      
-      res.on('data', (chunk) => {
-        responseData += chunk;
-      });
-      
-      res.on('end', () => {
-        console.log(`  📡 Response status: ${res.statusCode}`);
-        console.log(`  📄 Response body: ${responseData.substring(0, 200)}${responseData.length > 200 ? '...' : ''}`);
-        
-        try {
-          const parsed = JSON.parse(responseData);
-          resolve({ status: res.statusCode, data: parsed });
-        } catch (error) {
-          resolve({ status: res.statusCode, data: responseData });
-        }
-      });
-    });
-
-    req.on('error', (error) => {
-      console.log(`  ❌ Request error: ${error.message}`);
-      reject(error);
-    });
-
-    req.write(postData);
-    req.end();
-  });
-}
-
-// Ingest chunks in batches to avoid timeouts
-async function ingestChunks(chunks, batchSize = 10) {
-  const results = {
-    total: chunks.length,
-    processed: 0,
-    created: 0,
-    updated: 0,
-    errors: []
-  };
-
-  console.log(`Starting ingestion of ${chunks.length} chunks in batches of ${batchSize}...`);
-
-  for (let i = 0; i < chunks.length; i += batchSize) {
-    const batch = chunks.slice(i, i + batchSize);
-    const batchNum = Math.floor(i / batchSize) + 1;
-    const totalBatches = Math.ceil(chunks.length / batchSize);
-    
-    console.log(`Processing batch ${batchNum}/${totalBatches} (${batch.length} chunks)...`);
-    
-    try {
-      const response = await makeRequest('custom_kb-ingestion', {
-        chunks: batch,
-        force_update: false
-      });
-
-      if (response.status === 200) {
-        const result = response.data;
-        results.processed += batch.length;
-        results.created += result.items_created || 0;
-        results.updated += result.items_updated || 0;
-        
-        if (result.errors && result.errors.length > 0) {
-          results.errors.push(...result.errors);
-        }
-        
-        if (result.skipped && result.skipped.length > 0) {
-          console.log(`  Batch ${batchNum}: ${result.items_created || 0} created, ${result.items_updated || 0} updated, ${result.skipped?.length || 0} skipped`);
-        } else {
-          console.log(`  Batch ${batchNum}: ${result.items_created || 0} created, ${result.items_updated || 0} updated`);
-        }
-      } else {
-        const error = `HTTP ${response.status}: ${JSON.stringify(response.data)}`;
-        results.errors.push(`Batch ${batchNum}: ${error}`);
-        console.error(`  Batch ${batchNum} failed: ${error}`);
-      }
-    } catch (error) {
-      const batchError = `Batch ${batchNum}: ${error.message}`;
-      results.errors.push(batchError);
-      console.error(`  Batch ${batchNum} failed: ${error.message}`);
-    }
-
-    // Small delay between batches to avoid rate limiting
-    if (i + batchSize < chunks.length) {
-      await new Promise(resolve => setTimeout(resolve, 1000));
-    }
-  }
-
-  return results;
-}
-
-// Main function
-async function main() {
-  const args = process.argv.slice(2);
-  
-  if (args.length < 1) {
-    console.log('Usage: node ingest-chunks.js <chunks.json>');
-    console.log('Example: node ingest-chunks.js chunks.json');
-    process.exit(1);
-  }
-
-  const chunksFile = args[0];
-  
-  if (!fs.existsSync(chunksFile)) {
-    console.error(`Chunks file not found: ${chunksFile}`);
-    process.exit(1);
-  }
-
-  try {
-    const chunks = readChunks(chunksFile);
-    console.log(`Loaded ${chunks.length} chunks from ${chunksFile}`);
-    
-    // Show sample chunk
-    if (chunks.length > 0) {
-      console.log('\nSample chunk:');
-      console.log(`- ID: ${chunks[0].chunk_id}`);
-      console.log(`- Macro: ${chunks[0].macro}`);
-      console.log(`- Micro: ${chunks[0].micro}`);
-      console.log(`- File: ${chunks[0].metadata.file_path}`);
-    }
-
-    // Ask for confirmation
-    console.log('\nReady to ingest chunks into KB system.');
-    console.log('This will create KB articles and embeddings for each chunk.');
-    console.log('Continue? (y/N)');
-    
-    // For automation, we'll proceed without interactive confirmation
-    console.log('Proceeding with ingestion...\n');
-    
-    const results = await ingestChunks(chunks);
-    
-    console.log('\n=== Ingestion Complete ===');
-    console.log(`Total chunks: ${results.total}`);
-    console.log(`Processed: ${results.processed}`);
-    console.log(`Created: ${results.created}`);
-    console.log(`Updated: ${results.updated}`);
-    console.log(`Errors: ${results.errors.length}`);
-    
-    if (results.errors.length > 0) {
-      console.log('\nErrors:');
-      results.errors.slice(0, 10).forEach((error, i) => {
-        console.log(`${i + 1}. ${error}`);
-      });
-      
-      if (results.errors.length > 10) {
-        console.log(`... and ${results.errors.length - 10} more errors`);
-      }
-    }
-    
-    console.log('\nYou can now check the KB articles in your Spine instance!');
-    
-  } catch (error) {
-    console.error('Fatal error:', error.message);
-    process.exit(1);
-  }
-}
-
-if (require.main === module) {
-  main();
-}

package/scripts/kb-chunk-parser.cjs

@@ -1,312 +0,0 @@
-#!/usr/bin/env node
-
-const fs = require('fs');
-const path = require('path');
-
-class KBChunkParser {
-  constructor() {
-    this.chunks = [];
-    this.currentChunk = {};
-    this.inChunk = false;
-    this.currentFile = '';
-    this.currentLine = 0;
-  }
-
-  async parseDirectory(dirPath) {
-    this.chunks = [];
-    
-    const walkDir = (dir, fileList = []) => {
-      const files = fs.readdirSync(dir);
-      
-      for (const file of files) {
-        const filePath = path.join(dir, file);
-        const stat = fs.statSync(filePath);
-        
-        if (stat.isDirectory()) {
-          // Skip certain directories
-          if (!['node_modules', '.git', 'dist', 'src/v2-assembled'].includes(file)) {
-            walkDir(filePath, fileList);
-          }
-        } else if (file.match(/\.(ts|js|tsx|jsx)$/)) {
-          fileList.push(filePath);
-        }
-      }
-      
-      return fileList;
-    };
-    
-    const files = walkDir(dirPath);
-    
-    for (const filePath of files) {
-      await this.parseFile(filePath);
-    }
-    
-    return this.chunks;
-  }
-
-  async parseFile(filePath) {
-    this.currentFile = filePath;
-    this.currentLine = 0;
-    this.inChunk = false;
-    this.currentChunk = {};
-
-    try {
-      const content = fs.readFileSync(filePath, 'utf8');
-      const lines = content.split('\n');
-      
-      for (let i = 0; i < lines.length; i++) {
-        this.currentLine = i + 1;
-        const line = lines[i];
-        
-        if (line.includes('CHUNK_START:')) {
-          this.handleChunkStart(line);
-        } else if (line.includes('CHUNK_END:')) {
-          this.handleChunkEnd();
-        } else if (this.inChunk && this.currentChunk.code !== undefined) {
-          this.currentChunk.code += line + '\n';
-        }
-      }
-    } catch (error) {
-      console.error(`Error parsing file ${filePath}:`, error.message);
-    }
-  }
-
-  handleChunkStart(line) {
-    const match = line.match(/CHUNK_START:\s*(.+?)\s*─/);
-    if (!match) {
-      console.warn(`Invalid CHUNK_START format at line ${this.currentLine}: ${line}`);
-      return;
-    }
-
-    const identifier = match[1].trim();
-    this.inChunk = true;
-    this.currentChunk = {
-      identifier,
-      code: '',
-      metadata: {
-        chunk_id: '',
-        file_path: this.currentFile,
-        line_start: this.currentLine,
-        line_end: 0,
-        chunk_type: 'function',
-        purpose: '',
-        hash: '',
-        dependencies: [],
-        dependents: [],
-        source: {
-          source_type: 'core',
-          ref: this.currentFile,
-          line_start: this.currentLine,
-          line_end: 0
-        }
-      }
-    };
-  }
-
-  handleChunkEnd() {
-    if (!this.inChunk) {
-      console.warn(`CHUNK_END without CHUNK_START at line ${this.currentLine}`);
-      return;
-    }
-
-    const match = this.currentChunk.code?.match(/@chunk-id\s+(.+)/);
-    if (!match) {
-      console.warn(`Missing @chunk-id in chunk ${this.currentChunk.identifier}`);
-      this.inChunk = false;
-      return;
-    }
-
-    const chunkId = match[1].trim();
-    const hash = this.generateHash(this.currentChunk.code || '');
-    
-    // Parse JSDoc-style comments
-    const parsed = this.parseChunkDoc(this.currentChunk.code || '');
-    
-    const chunk = {
-      identifier: this.currentChunk.identifier,
-      chunk_id: chunkId,
-      version: '1.0.0',
-      hash,
-      macro: parsed.macro || '',
-      micro: parsed.micro || '',
-      inputs: parsed.inputs || {},
-      outputs: parsed.outputs || '',
-      depends_on: parsed.depends_on || [],
-      depended_by: parsed.depended_by || [],
-      side_effects: parsed.side_effects || [],
-      tags: parsed.tags || [],
-      code: (this.currentChunk.code || '').trim(),
-      metadata: {
-        ...this.currentChunk.metadata,
-        chunk_id: chunkId,
-        hash,
-        line_end: this.currentLine,
-        source: {
-          ...this.currentChunk.metadata.source,
-          line_end: this.currentLine
-        }
-      }
-    };
-
-    this.chunks.push(chunk);
-    this.inChunk = false;
-    this.currentChunk = {};
-  }
-
-  parseChunkDoc(code) {
-    const doc = {};
-    
-    // Extract @chunk-id
-    const chunkIdMatch = code.match(/@chunk-id\s+(.+)/);
-    if (chunkIdMatch) {
-      doc.chunk_id = chunkIdMatch[1].trim();
-    }
-
-    // Extract @version
-    const versionMatch = code.match(/@version\s+(.+)/);
-    if (versionMatch) {
-      doc.version = versionMatch[1].trim();
-    }
-
-    // Extract @hash
-    const hashMatch = code.match(/@hash\s+(.+)/);
-    if (hashMatch) {
-      doc.hash = hashMatch[1].trim();
-    }
-
-    // Extract @macro
-    const macroMatch = code.match(/@macro\s+(.+)/);
-    if (macroMatch) {
-      doc.macro = macroMatch[1].trim();
-    }
-
-    // Extract @micro
-    const microMatch = code.match(/@micro\s+(.+)/);
-    if (microMatch) {
-      doc.micro = microMatch[1].trim();
-    }
-
-    // Extract @inputs
-    const inputsMatch = code.match(/@inputs\s+(.+)/);
-    if (inputsMatch) {
-      doc.inputs = this.parseKeyValuePairs(inputsMatch[1].trim());
-    }
-
-    // Extract @outputs
-    const outputsMatch = code.match(/@outputs\s+(.+)/);
-    if (outputsMatch) {
-      doc.outputs = outputsMatch[1].trim();
-    }
-
-    // Extract @depends-on
-    const dependsOnMatch = code.match(/@depends-on\s+(.+)/);
-    if (dependsOnMatch) {
-      doc.depends_on = this.parseList(dependsOnMatch[1].trim());
-    }
-
-    // Extract @depended-by
-    const dependedByMatch = code.match(/@depended-by\s+(.+)/);
-    if (dependedByMatch) {
-      doc.depended_by = this.parseList(dependedByMatch[1].trim());
-    }
-
-    // Extract @side-effects
-    const sideEffectsMatch = code.match(/@side-effects\s+(.+)/);
-    if (sideEffectsMatch) {
-      doc.side_effects = this.parseList(sideEffectsMatch[1].trim());
-    }
-
-    // Extract @tags
-    const tagsMatch = code.match(/@tags\s+(.+)/);
-    if (tagsMatch) {
-      doc.tags = this.parseList(tagsMatch[1].trim());
-    }
-
-    return doc;
-  }
-
-  parseKeyValuePairs(str) {
-    const pairs = {};
-    const regex = /(\w+):\s*([^,]+)(?:,\s*|$)/g;
-    let match;
-    
-    while ((match = regex.exec(str)) !== null) {
-      pairs[match[1]] = match[2].trim();
-    }
-    
-    return pairs;
-  }
-
-  parseList(str) {
-    return str.split(',').map(s => s.trim()).filter(Boolean);
-  }
-
-  generateHash(content) {
-    // Simple hash function for demonstration
-    let hash = 0;
-    for (let i = 0; i < content.length; i++) {
-      const char = content.charCodeAt(i);
-      hash = ((hash << 5) - hash) + char;
-      hash = hash & hash; // Convert to 32-bit integer
-    }
-    return Math.abs(hash).toString(16);
-  }
-
-  async saveChunks(outputFile) {
-    const output = {
-      generated_at: new Date().toISOString(),
-      total_chunks: this.chunks.length,
-      chunks: this.chunks
-    };
-    
-    fs.writeFileSync(outputFile, JSON.stringify(output, null, 2));
-    console.log(`Saved ${this.chunks.length} chunks to ${outputFile}`);
-  }
-}
-
-// CLI usage
-async function main() {
-  const args = process.argv.slice(2);
-  
-  if (args.length < 2) {
-    console.log('Usage: node kb-chunk-parser.js <source-dir> <output-file>');
-    console.log('Example: node kb-chunk-parser.js v2-core/functions chunks.json');
-    process.exit(1);
-  }
-
-  const [sourceDir, outputFile] = args;
-  
-  if (!fs.existsSync(sourceDir)) {
-    console.error(`Source directory not found: ${sourceDir}`);
-    process.exit(1);
-  }
-
-  const parser = new KBChunkParser();
-  
-  try {
-    console.log(`Parsing chunks from ${sourceDir}...`);
-    const chunks = await parser.parseDirectory(sourceDir);
-    
-    console.log(`Found ${chunks.length} chunks`);
-    
-    await parser.saveChunks(outputFile);
-    
-    // Display summary
-    console.log('\nSummary:');
-    console.log(`- Functions: ${chunks.filter(c => c.metadata.chunk_type === 'function').length}`);
-    console.log(`- Classes: ${chunks.filter(c => c.metadata.chunk_type === 'class').length}`);
-    console.log(`- Interfaces: ${chunks.filter(c => c.metadata.chunk_type === 'interface').length}`);
-    console.log(`- Configs: ${chunks.filter(c => c.metadata.chunk_type === 'config').length}`);
-    console.log(`- Objects: ${chunks.filter(c => c.metadata.chunk_type === 'object').length}`);
-    
-  } catch (error) {
-    console.error('Error:', error.message);
-    process.exit(1);
-  }
-}
-
-if (require.main === module) {
-  main();
-}
-
-module.exports = { KBChunkParser };