snow-flow 2.8.9 → 2.9.4

package/dist/utils/ml-data-fetcher.js

@@ -0,0 +1,288 @@
+"use strict";
+/**
+ * Smart ML Data Fetcher
+ * Handles intelligent data fetching for ML training with batching and field discovery
+ * Prevents token limit errors and optimizes data retrieval
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MLDataFetcher = void 0;
+const logger_js_1 = require("./logger.js");
+const logger = new logger_js_1.Logger('MLDataFetcher');
+class MLDataFetcher {
+    constructor(operationsMCP) {
+        this.operationsMCP = operationsMCP;
+    }
+    /**
+     * Smart fetch with automatic field discovery and batching
+     */
+    async smartFetch(options) {
+        const { table, query = '', totalSamples = 1000, batchSize = 100, fields, discoverFields = true, includeContent = true } = options;
+        logger.info(`🧠 Smart ML data fetch for ${table} - Target: ${totalSamples} samples`);
+        // Step 1: Get total count
+        const countResult = await this.getRecordCount(table, query);
+        logger.info(`📊 Total available records: ${countResult}`);
+        // Step 2: Discover fields if needed
+        let fieldsToFetch = fields;
+        if (!fieldsToFetch && discoverFields) {
+            const discovery = await this.discoverFields(table, query);
+            fieldsToFetch = discovery.recommendedFields;
+            logger.info(`🔍 Discovered ${discovery.allFields.length} fields, using ${fieldsToFetch.length} for ML`);
+        }
+        // Step 3: Calculate optimal batching strategy
+        const actualTotal = Math.min(totalSamples, countResult);
+        const optimalBatchSize = this.calculateOptimalBatchSize(actualTotal, batchSize, fieldsToFetch?.length || 10);
+        const numBatches = Math.ceil(actualTotal / optimalBatchSize);
+        logger.info(`📦 Fetching ${actualTotal} records in ${numBatches} batches of ${optimalBatchSize}`);
+        // Step 4: Fetch data in batches
+        const allData = [];
+        for (let batch = 0; batch < numBatches; batch++) {
+            const offset = batch * optimalBatchSize;
+            const limit = Math.min(optimalBatchSize, actualTotal - offset);
+            logger.info(`  Batch ${batch + 1}/${numBatches}: Fetching ${limit} records (offset: ${offset})`);
+            try {
+                const batchData = await this.fetchBatch(table, query, limit, offset, fieldsToFetch, includeContent);
+                allData.push(...batchData);
+                // Small delay between batches to avoid overwhelming the API
+                if (batch < numBatches - 1) {
+                    await new Promise(resolve => setTimeout(resolve, 200)); // Increased from 100ms to 200ms
+                }
+            }
+            catch (error) {
+                if (error.message?.includes('exceeds maximum allowed tokens')) {
+                    // Reduce batch size and retry
+                    logger.warn(`⚠️ Token limit hit, reducing batch size and retrying...`);
+                    const smallerBatchSize = Math.floor(optimalBatchSize / 2);
+                    return this.smartFetch({
+                        ...options,
+                        batchSize: smallerBatchSize
+                    });
+                }
+                throw error;
+            }
+        }
+        return {
+            data: allData,
+            totalFetched: allData.length,
+            batchesProcessed: numBatches,
+            fields: fieldsToFetch || []
+        };
+    }
+    /**
+     * Get total count of records matching query
+     */
+    async getRecordCount(table, query) {
+        try {
+            const result = await this.operationsMCP.handleTool('snow_query_table', {
+                table,
+                query,
+                limit: 1,
+                include_content: false // Count only, no data
+            });
+            // Extract count from result
+            if (result?.content?.[0]?.text) {
+                const text = result.content[0].text;
+                const match = text.match(/Found (\d+) .* records/);
+                if (match) {
+                    return parseInt(match[1], 10);
+                }
+            }
+            return 0;
+        }
+        catch (error) {
+            logger.error('Failed to get record count:', error);
+            return 0;
+        }
+    }
+    /**
+     * Discover available fields by sampling a few records
+     */
+    async discoverFields(table, query) {
+        logger.info('🔍 Discovering fields from sample records...');
+        try {
+            // Fetch just 3 records with all fields to discover schema
+            const result = await this.operationsMCP.handleTool('snow_query_table', {
+                table,
+                query,
+                limit: 3,
+                include_content: true
+                // No fields specified = get all fields
+            });
+            const sampleData = this.extractDataFromResult(result);
+            if (sampleData.length === 0) {
+                logger.warn('No sample data available for field discovery');
+                return {
+                    allFields: [],
+                    recommendedFields: this.getDefaultFields(table),
+                    sampleData: []
+                };
+            }
+            // Extract all field names from sample
+            const allFields = Object.keys(sampleData[0] || {});
+            // Recommend fields for ML (exclude system fields and large text fields)
+            const recommendedFields = this.selectMLFields(allFields, sampleData, table);
+            return {
+                allFields,
+                recommendedFields,
+                sampleData
+            };
+        }
+        catch (error) {
+            logger.error('Field discovery failed:', error);
+            return {
+                allFields: [],
+                recommendedFields: this.getDefaultFields(table),
+                sampleData: []
+            };
+        }
+    }
+    /**
+     * Fetch a single batch of data
+     */
+    async fetchBatch(table, query, limit, offset, fields, includeContent = true) {
+        try {
+            // Build query with offset
+            const offsetQuery = query ? `${query}^ORDERBY${offset}` : `ORDERBY${offset}`;
+            const result = await this.operationsMCP.handleTool('snow_query_table', {
+                table,
+                query: offsetQuery,
+                limit,
+                fields,
+                include_content: includeContent
+            });
+            return this.extractDataFromResult(result);
+        }
+        catch (error) {
+            logger.error(`Failed to fetch batch (offset: ${offset}, limit: ${limit}):`, error);
+            return [];
+        }
+    }
+    /**
+     * Extract data from MCP tool result
+     */
+    extractDataFromResult(result) {
+        if (!result?.content?.[0]?.text) {
+            return [];
+        }
+        try {
+            const text = result.content[0].text;
+            // Try to parse as JSON first
+            if (text.includes('[') && text.includes(']')) {
+                const jsonMatch = text.match(/\[[\s\S]*\]/);
+                if (jsonMatch) {
+                    return JSON.parse(jsonMatch[0]);
+                }
+            }
+            // Try to extract from formatted output
+            const lines = text.split('\n');
+            const data = [];
+            let currentRecord = null;
+            for (const line of lines) {
+                if (line.includes('number:') || line.includes('sys_id:')) {
+                    if (currentRecord) {
+                        data.push(currentRecord);
+                    }
+                    currentRecord = {};
+                }
+                if (currentRecord && line.includes(':')) {
+                    const [key, ...valueParts] = line.split(':');
+                    const cleanKey = key.trim().replace(/^[-\s]+/, '');
+                    const value = valueParts.join(':').trim();
+                    if (cleanKey && value) {
+                        currentRecord[cleanKey] = value;
+                    }
+                }
+            }
+            if (currentRecord && Object.keys(currentRecord).length > 0) {
+                data.push(currentRecord);
+            }
+            return data;
+        }
+        catch (error) {
+            logger.error('Failed to extract data from result:', error);
+            return [];
+        }
+    }
+    /**
+     * Calculate optimal batch size based on data characteristics
+     */
+    calculateOptimalBatchSize(totalRecords, requestedBatchSize, numFields) {
+        // Estimate tokens per record (rough approximation)
+        const avgTokensPerField = 10; // Conservative estimate
+        const tokensPerRecord = numFields * avgTokensPerField;
+        const maxTokensPerBatch = 20000; // Leave buffer below 25000 limit
+        // Calculate max records per batch based on token limit
+        const maxRecordsPerBatch = Math.floor(maxTokensPerBatch / tokensPerRecord);
+        // Use the smaller of requested batch size and calculated max
+        const optimalSize = Math.min(requestedBatchSize, maxRecordsPerBatch);
+        // Ensure at least 10 records per batch but not more than total
+        return Math.max(10, Math.min(optimalSize, totalRecords));
+    }
+    /**
+     * Select appropriate fields for ML training
+     */
+    selectMLFields(allFields, sampleData, table) {
+        const excluded = new Set([
+            'sys_id', 'sys_created_on', 'sys_created_by', 'sys_updated_on', 'sys_updated_by',
+            'sys_mod_count', 'sys_tags', 'sys_package', 'sys_policy', 'sys_scope',
+            'sys_domain', 'sys_domain_path', 'sys_class_name'
+        ]);
+        const mlFields = allFields.filter(field => {
+            // Exclude system fields
+            if (excluded.has(field))
+                return false;
+            // Check if field has useful data in samples
+            const hasData = sampleData.some(record => {
+                const value = record[field];
+                return value && value !== 'null' && value !== '';
+            });
+            return hasData;
+        });
+        // Always include key fields for the table type
+        const keyFields = this.getKeyFields(table);
+        const combinedFields = [...new Set([...keyFields, ...mlFields])];
+        // Limit to 20 most relevant fields to avoid token issues
+        return combinedFields.slice(0, 20);
+    }
+    /**
+     * Get default fields for a table type
+     */
+    getDefaultFields(table) {
+        const fieldMap = {
+            incident: [
+                'number', 'short_description', 'description', 'category', 'subcategory',
+                'priority', 'urgency', 'impact', 'state', 'assignment_group',
+                'assigned_to', 'caller_id', 'opened_at', 'resolved_at'
+            ],
+            change_request: [
+                'number', 'short_description', 'description', 'type', 'category',
+                'priority', 'risk', 'impact', 'state', 'assignment_group',
+                'assigned_to', 'requested_by', 'start_date', 'end_date'
+            ],
+            problem: [
+                'number', 'short_description', 'description', 'category', 'subcategory',
+                'priority', 'urgency', 'impact', 'state', 'assignment_group',
+                'assigned_to', 'opened_at', 'known_error'
+            ],
+            sc_request: [
+                'number', 'short_description', 'description', 'request_state', 'approval',
+                'requested_for', 'requested_by', 'assignment_group', 'assigned_to',
+                'opened_at', 'closed_at'
+            ]
+        };
+        return fieldMap[table] || ['number', 'short_description', 'state', 'priority'];
+    }
+    /**
+     * Get key fields that should always be included
+     */
+    getKeyFields(table) {
+        const keyFieldMap = {
+            incident: ['number', 'short_description', 'category', 'priority', 'state'],
+            change_request: ['number', 'short_description', 'type', 'risk', 'state'],
+            problem: ['number', 'short_description', 'category', 'priority', 'state'],
+            sc_request: ['number', 'short_description', 'request_state', 'approval']
+        };
+        return keyFieldMap[table] || ['number', 'short_description', 'state'];
+    }
+}
+exports.MLDataFetcher = MLDataFetcher;
+//# sourceMappingURL=ml-data-fetcher.js.map

package/dist/utils/servicenow-client.js

@@ -13,10 +13,10 @@ const axios_1 = __importDefault(require("axios"));
 const https_1 = __importDefault(require("https"));
 const snow_oauth_1 = require("./snow-oauth");
 const action_type_cache_1 = require("./action-type-cache");
-const snow_flow_config_js_1 = require("../config/snow-flow-config.js");
 const widget_template_generator_js_1 = require("./widget-template-generator.js");
 const logger_1 = require("./logger");
 const unified_auth_store_js_1 = require("./unified-auth-store.js");
+const timeout_manager_js_1 = require("./timeout-manager.js");
 class ServiceNowClient {
     constructor() {
         this.credentials = null;
@@ -68,8 +68,10 @@ class ServiceNowClient {
             maxVersion: 'TLSv1.3',
             minVersion: 'TLSv1.2'
         });
+        // Use intelligent timeout based on operation type (default to TABLE_QUERY)
+        const defaultTimeout = (0, timeout_manager_js_1.getTimeoutConfig)(timeout_manager_js_1.OperationType.TABLE_QUERY).baseTimeout;
         this.client = axios_1.default.create({
-            timeout: snow_flow_config_js_1.snowFlowConfig.servicenow.timeout,
+            timeout: parseInt(process.env.SNOW_API_TIMEOUT || String(defaultTimeout)),
             headers: {
                 'Content-Type': 'application/json',
                 'Accept': 'application/json'
@@ -979,21 +981,33 @@ class ServiceNowClient {
     async searchRecords(table, query, limit = 10) {
         try {
             await this.ensureAuthenticated();
-            const response = await this.client.get(`${this.getBaseUrl()}/api/now/table/${table}`, {
-                params: {
-                    sysparm_query: query,
-                    sysparm_limit: limit
-                }
+            // Detect operation type for intelligent timeout
+            const operationType = (0, timeout_manager_js_1.detectOperationType)({
+                action: 'query',
+                table,
+                limit
             });
+            // Use retry wrapper with intelligent timeout
+            const result = await (0, timeout_manager_js_1.withRetry)(async () => {
+                const response = await this.client.get(`${this.getBaseUrl()}/api/now/table/${table}`, {
+                    params: {
+                        sysparm_query: query,
+                        sysparm_limit: limit
+                    },
+                    // Override timeout for this specific request
+                    timeout: (0, timeout_manager_js_1.getTimeoutConfig)(operationType).baseTimeout
+                });
+                return response;
+            }, operationType, `Search ${table} (${limit} records)`);
             return {
                 success: true,
                 data: {
-                    result: response.data.result || []
+                    result: result.data.result || []
                 }
             };
         }
         catch (error) {
-            console.error(`Failed to search records in ${table}:`, error);
+            this.logger.error(`Failed to search records in ${table}:`, error);
             return {
                 success: false,
                 error: error instanceof Error ? error.message : String(error)
@@ -1006,22 +1020,34 @@ class ServiceNowClient {
     async searchRecordsWithOffset(table, query, limit = 10, offset = 0) {
         try {
             await this.ensureAuthenticated();
-            const response = await this.client.get(`${this.getBaseUrl()}/api/now/table/${table}`, {
-                params: {
-                    sysparm_query: query,
-                    sysparm_limit: limit,
-                    sysparm_offset: offset
-                }
+            // Detect operation type for intelligent timeout
+            const operationType = (0, timeout_manager_js_1.detectOperationType)({
+                action: 'query',
+                table,
+                limit
             });
+            // Use retry wrapper with intelligent timeout
+            const result = await (0, timeout_manager_js_1.withRetry)(async () => {
+                const response = await this.client.get(`${this.getBaseUrl()}/api/now/table/${table}`, {
+                    params: {
+                        sysparm_query: query,
+                        sysparm_limit: limit,
+                        sysparm_offset: offset
+                    },
+                    // Override timeout for this specific request
+                    timeout: (0, timeout_manager_js_1.getTimeoutConfig)(operationType).baseTimeout
+                });
+                return response;
+            }, operationType, `Search ${table} with offset ${offset}`);
             return {
                 success: true,
                 data: {
-                    result: response.data.result || []
+                    result: result.data.result || []
                 }
             };
         }
         catch (error) {
-            console.error(`Failed to search records in ${table} with offset ${offset}:`, error);
+            this.logger.error(`Failed to search records in ${table} with offset ${offset}:`, error);
             return {
                 success: false,
                 error: error instanceof Error ? error.message : String(error)

package/dist/utils/timeout-manager.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Timeout Manager for Snow-Flow
+ * Provides intelligent timeout configuration and retry logic
+ */
+/**
+ * Operation types with their specific timeout requirements
+ */
+export declare enum OperationType {
+    SIMPLE_QUERY = "simple_query",
+    SINGLE_RECORD = "single_record",
+    HEALTH_CHECK = "health_check",
+    TABLE_QUERY = "table_query",
+    CREATE_RECORD = "create_record",
+    UPDATE_RECORD = "update_record",
+    DELETE_RECORD = "delete_record",
+    BATCH_OPERATION = "batch_operation",
+    BULK_QUERY = "bulk_query",
+    DEPLOYMENT = "deployment",
+    WORKFLOW_EXECUTION = "workflow_execution",
+    ML_TRAINING = "ml_training",
+    ML_BATCH_FETCH = "ml_batch_fetch",
+    ML_PREDICTION = "ml_prediction",
+    LARGE_EXPORT = "large_export",
+    MIGRATION = "migration",
+    FULL_SYNC = "full_sync"
+}
+/**
+ * Timeout configuration with intelligent defaults
+ */
+export interface TimeoutConfig {
+    baseTimeout: number;
+    maxTimeout: number;
+    retryCount: number;
+    backoffMultiplier: number;
+    jitterRange: number;
+}
+/**
+ * Get timeout configuration for operation type
+ */
+export declare function getTimeoutConfig(operationType: OperationType): TimeoutConfig;
+/**
+ * Calculate timeout with exponential backoff
+ */
+export declare function calculateTimeout(config: TimeoutConfig, attemptNumber: number): number;
+/**
+ * Retry wrapper with exponential backoff
+ */
+export declare function withRetry<T>(operation: () => Promise<T>, operationType: OperationType, operationName?: string): Promise<T>;
+/**
+ * Detect operation type from context
+ */
+export declare function detectOperationType(context: {
+    tool?: string;
+    table?: string;
+    action?: string;
+    limit?: number;
+}): OperationType;
+/**
+ * Get human-readable timeout description
+ */
+export declare function getTimeoutDescription(operationType: OperationType): string;
+//# sourceMappingURL=timeout-manager.d.ts.map