toon-formatter 2.2.1 → 2.3.0

package/src/json_formatter/index.js

@@ -7,6 +7,7 @@ import { yamlToJson, yamlToJsonSync, jsonToYaml, jsonToYamlSync } from './yaml.j
 import { xmlToJson, xmlToJsonSync, jsonToXml, jsonToXmlSync } from './xml.js';
 import { csvToJson, csvToJsonSync, jsonToCsv, jsonToCsvSync } from './csv.js';
 import { validateJsonString, validateJsonStringSync } from './validator.js';
+import { dataManager, dataManagerAsync, extractJsonFromString, extractXmlFromString, extractCsvFromString } from '../utils.js';
 
 export class JsonConverter {
     /**
@@ -110,8 +111,9 @@ export class JsonConverter {
      */
     toToon(jsonData, options = {}) {
         const { conversionMode = 'no_encryption' } = options;
+        const optimizedConverterFn = dataManager(jsonToToonSync, extractJsonFromString);
         return this._convertWithEncryption(
-            (data) => jsonToToonSync(data),
+            (data) => optimizedConverterFn(data),
             jsonData,
             conversionMode
         );
@@ -126,8 +128,9 @@ export class JsonConverter {
      */
     async toToonAsync(jsonData, options = {}) {
         const { conversionMode = 'no_encryption' } = options;
+        const optimizedConverterFn = dataManagerAsync(jsonToToon, extractJsonFromString);
         return this._convertWithEncryptionAsync(
-            async (data) => jsonToToon(data),
+            async (data) => optimizedConverterFn(data),
             jsonData,
             conversionMode
         );
@@ -178,8 +181,9 @@ export class JsonConverter {
      */
     toYaml(jsonData, options = {}) {
         const { conversionMode = 'no_encryption' } = options;
+        const optimizedConverterFn = dataManager(jsonToYamlSync, extractJsonFromString);
         return this._convertWithEncryption(
-            (data) => jsonToYamlSync(data),
+            (data) => optimizedConverterFn(data),
             jsonData,
             conversionMode
         );
@@ -194,8 +198,9 @@ export class JsonConverter {
      */
     async toYamlAsync(jsonData, options = {}) {
         const { conversionMode = 'no_encryption' } = options;
+        const optimizedConverterFn = dataManagerAsync(jsonToYaml, extractJsonFromString);
         return this._convertWithEncryptionAsync(
-            async (data) => jsonToYaml(data),
+            async (data) => optimizedConverterFn(data),
             jsonData,
             conversionMode
         );
@@ -212,8 +217,9 @@ export class JsonConverter {
      */
     fromXml(xmlString, options = {}) {
         const { conversionMode = 'no_encryption' } = options;
+        const optimizedConverterFn = dataManager(xmlToJsonSync, extractXmlFromString);
         return this._convertWithEncryption(
-            (data) => xmlToJsonSync(data),
+            (data) => optimizedConverterFn(data),
             xmlString,
             conversionMode
         );
@@ -228,8 +234,9 @@ export class JsonConverter {
      */
     async fromXmlAsync(xmlString, options = {}) {
         const { conversionMode = 'no_encryption' } = options;
+        const optimizedConverterFn = dataManagerAsync(xmlToJson, extractXmlFromString);
         return this._convertWithEncryptionAsync(
-            async (data) => xmlToJson(data),
+            async (data) => optimizedConverterFn(data),
             xmlString,
             conversionMode
         );
@@ -244,8 +251,9 @@ export class JsonConverter {
      */
     toXml(jsonData, options = {}) {
         const { conversionMode = 'no_encryption' } = options;
+        const optimizedConverterFn = dataManager(jsonToXmlSync, extractJsonFromString);
         return this._convertWithEncryption(
-            (data) => jsonToXmlSync(data),
+            (data) => optimizedConverterFn(data),
             jsonData,
             conversionMode
         );
@@ -260,8 +268,9 @@ export class JsonConverter {
      */
     async toXmlAsync(jsonData, options = {}) {
         const { conversionMode = 'no_encryption' } = options;
+        const optimizedConverterFn = dataManagerAsync(jsonToXml, extractJsonFromString);
         return this._convertWithEncryptionAsync(
-            async (data) => jsonToXml(data),
+            async (data) => optimizedConverterFn(data),
             jsonData,
             conversionMode
         );
@@ -278,8 +287,9 @@ export class JsonConverter {
      */
     fromCsv(csvString, options = {}) {
         const { conversionMode = 'no_encryption' } = options;
+        const optimizedConverterFn = dataManager(csvToJsonSync, extractCsvFromString);
         return this._convertWithEncryption(
-            (data) => csvToJsonSync(data),
+            (data) => optimizedConverterFn(data),
             csvString,
             conversionMode
         );
@@ -294,8 +304,9 @@ export class JsonConverter {
      */
     async fromCsvAsync(csvString, options = {}) {
         const { conversionMode = 'no_encryption' } = options;
+        const optimizedConverterFn = dataManagerAsync(csvToJson, extractCsvFromString);
         return this._convertWithEncryptionAsync(
-            async (data) => csvToJson(data),
+            async (data) => optimizedConverterFn(data),
             csvString,
             conversionMode
         );
@@ -310,8 +321,9 @@ export class JsonConverter {
      */
     toCsv(jsonData, options = {}) {
         const { conversionMode = 'no_encryption' } = options;
+        const optimizedConverterFn = dataManager(jsonToCsvSync, extractJsonFromString);
         return this._convertWithEncryption(
-            (data) => jsonToCsvSync(data),
+            (data) => optimizedConverterFn(data),
             jsonData,
             conversionMode
         );
@@ -326,8 +338,9 @@ export class JsonConverter {
      */
     async toCsvAsync(jsonData, options = {}) {
         const { conversionMode = 'no_encryption' } = options;
+        const optimizedConverterFn = dataManagerAsync(jsonToCsv, extractJsonFromString);
         return this._convertWithEncryptionAsync(
-            async (data) => jsonToCsv(data),
+            async (data) => optimizedConverterFn(data),
             jsonData,
             conversionMode
         );
@@ -383,7 +396,8 @@ export class JsonConverter {
      * @returns {string} TOON formatted string
      */
     static toToon(jsonData) {
-        return jsonToToonSync(jsonData);
+        const optimizedConverterFn = dataManager(jsonToToonSync, extractJsonFromString);
+        return optimizedConverterFn(jsonData);
     }
 
     /**
@@ -392,7 +406,8 @@ export class JsonConverter {
      * @returns {Promise<string>} TOON formatted string
      */
     static async toToonAsync(jsonData) {
-        return jsonToToon(jsonData);
+        const optimizedConverterFn = dataManagerAsync(jsonToToon, extractJsonFromString);
+        return optimizedConverterFn(jsonData);
     }
 
     /**
@@ -421,7 +436,8 @@ export class JsonConverter {
      * @returns {string} YAML formatted string
      */
     static toYaml(jsonData) {
-        return jsonToYamlSync(jsonData);
+        const optimizedConverterFn = dataManager(jsonToYamlSync, extractJsonFromString);
+        return optimizedConverterFn(jsonData);
     }
 
     /**
@@ -430,7 +446,8 @@ export class JsonConverter {
      * @returns {Promise<string>} YAML formatted string
      */
     static async toYamlAsync(jsonData) {
-        return jsonToYaml(jsonData);
+        const optimizedConverterFn = dataManagerAsync(jsonToYaml, extractJsonFromString);
+        return optimizedConverterFn(jsonData);
     }
 
     /**
@@ -439,7 +456,8 @@ export class JsonConverter {
      * @returns {Object|string} JSON object or string
      */
     static fromXml(xmlString) {
-        return xmlToJsonSync(xmlString);
+        const optimizedConverterFn = dataManager(xmlToJsonSync, extractXmlFromString);
+        return optimizedConverterFn(xmlString);
     }
 
     /**
@@ -448,7 +466,8 @@ export class JsonConverter {
      * @returns {Promise<Object|string>} JSON object or string
      */
     static async fromXmlAsync(xmlString) {
-        return xmlToJson(xmlString);
+        const optimizedConverterFn = dataManagerAsync(xmlToJson, extractXmlFromString);
+        return optimizedConverterFn(xmlString);
     }
 
     /**
@@ -457,7 +476,8 @@ export class JsonConverter {
      * @returns {string} XML formatted string
      */
     static toXml(jsonData) {
-        return jsonToXmlSync(jsonData);
+        const optimizedConverterFn = dataManager(jsonToXmlSync, extractJsonFromString);
+        return optimizedConverterFn(jsonData);
     }
 
     /**
@@ -466,7 +486,8 @@ export class JsonConverter {
      * @returns {Promise<string>} XML formatted string
      */
     static async toXmlAsync(jsonData) {
-        return jsonToXml(jsonData);
+        const optimizedConverterFn = dataManagerAsync(jsonToXml, extractJsonFromString);
+        return optimizedConverterFn(jsonData);
     }
 
     /**
@@ -475,7 +496,8 @@ export class JsonConverter {
      * @returns {Array<Object>|string} JSON object or string
      */
     static fromCsv(csvString) {
-        return csvToJsonSync(csvString);
+        const optimizedConverterFn = dataManager(csvToJsonSync, extractCsvFromString);
+        return optimizedConverterFn(csvString);
     }
 
     /**
@@ -484,7 +506,8 @@ export class JsonConverter {
      * @returns {Promise<Array<Object>|string>} JSON object or string
      */
     static async fromCsvAsync(csvString) {
-        return csvToJson(csvString);
+        const optimizedConverterFn = dataManagerAsync(csvToJson, extractCsvFromString);
+        return optimizedConverterFn(csvString);
     }
 
     /**
@@ -493,7 +516,8 @@ export class JsonConverter {
      * @returns {string} CSV formatted string
      */
     static toCsv(jsonData) {
-        return jsonToCsvSync(jsonData);
+        const optimizedConverterFn = dataManager(jsonToCsvSync, extractJsonFromString);
+        return optimizedConverterFn(jsonData);
     }
 
     /**
@@ -502,7 +526,8 @@ export class JsonConverter {
      * @returns {Promise<string>} CSV formatted string
      */
     static async toCsvAsync(jsonData) {
-        return jsonToCsv(jsonData);
+        const optimizedConverterFn = dataManagerAsync(jsonToCsv, extractJsonFromString);
+        return await optimizedConverterFn(jsonData);
     }
 
     /**

package/src/utils.js

@@ -2,6 +2,8 @@
  * Utility functions for TOON conversion
  */
 
+import { EXPENSIVE_WORDS } from './constants.js';
+
 /**
  * Encodes XML reserved characters to prevent parsing errors
  * @param {string} rawXmlString - Raw XML string
@@ -439,3 +441,303 @@ export function buildTag(key, value) {
         return `<${sanitizedKey}>${value}</${sanitizedKey}>`;
     }
 }
+
+// ============================================================================
+// SMART CODE OPTIMIZATION FUNCTIONS
+// ============================================================================
+
+/**
+ * Detects if a string is likely code.
+ * Uses heuristics including command patterns, keywords, and structure.
+ * @param {string} value - String to check
+ * @returns {boolean} True if the string appears to be code
+ */
+export function isCode(value) {
+    if (typeof value !== 'string' || value.length < 5) return false;
+
+    const trimmed = value.trim();
+
+    // Single-line command patterns
+    const isSingleLineCommand = (
+        /^(npm|yarn|pnpm|pip|pip3|brew|apt|gem|go|cargo|composer|mvn|gradle|dotnet|conda)\s+/.test(trimmed) ||
+        /^(git|docker|kubectl|curl|wget|ssh|scp|rsync|sudo)\s+/.test(trimmed) ||
+        /^(node|python|python3|ruby|java|go|rust)\s+/.test(trimmed) ||
+        trimmed.startsWith('$') ||
+        trimmed.startsWith('>') ||
+        trimmed.startsWith('#!')
+    );
+
+    if (isSingleLineCommand) return true;
+
+    // Multi-line code detection
+    const hasMultipleLines = /\n/.test(trimmed);
+    const hasCodePatterns = /import|require\(|function |const |let |var |class |def |async |=>|\[|\];|print\(|console\.log\(/.test(trimmed);
+    const startsWithShebang = trimmed.startsWith('#!');
+
+    return hasMultipleLines && (hasCodePatterns || startsWithShebang);
+}
+
+/**
+ * Extracts code blocks from text, separated by double newlines.
+ * @param {string} text - Text containing potential code blocks
+ * @returns {Array<{code: string, start: number, end: number}>} Array of code block objects
+ */
+export function extractCodeBlocks(text) {
+    if (typeof text !== 'string') return [];
+
+    const results = [];
+    let currentPos = 0;
+
+    while (true) {
+        const nextBreak = text.indexOf('\n\n', currentPos);
+        let chunk, chunkEnd, nextStart;
+
+        if (nextBreak !== -1) {
+            chunk = text.substring(currentPos, nextBreak);
+            chunkEnd = nextBreak;
+            nextStart = nextBreak + 2; // skip \n\n
+        } else {
+            chunk = text.substring(currentPos);
+            chunkEnd = text.length;
+            nextStart = text.length;
+        }
+
+        const cleanChunk = chunk.trim();
+
+        if (cleanChunk && isCode(cleanChunk)) {
+            results.push({
+                code: cleanChunk,
+                start: currentPos,
+                end: chunkEnd
+            });
+        }
+
+        currentPos = nextStart;
+        if (currentPos >= text.length) break;
+    }
+
+    return results;
+}
+
+/**
+ * Reduces a code block by removing comments and compressing whitespace.
+ * @param {string} codeBlock - Code to reduce
+ * @returns {string} Reduced code
+ */
+export function reduceCodeBlock(codeBlock) {
+    let reduced = codeBlock;
+
+    // Remove double newlines
+    reduced = reduced.replace(/\n\n/g, '\n');
+
+    // Remove single-line comments (# for Python, shell, etc.)
+    reduced = reduced.replace(/#.*/g, '');
+
+    // Remove single-line comments (// for JS, Java, etc.)
+    reduced = reduced.replace(/\/\/.*/g, '');
+
+    // Remove trailing whitespace from each line
+    reduced = reduced.replace(/\s*\n/g, '\n');
+
+    // Remove any remaining double newlines created by comment removal
+    reduced = reduced.replace(/\n\n/g, '\n');
+
+    return reduced.trim();
+}
+
+/**
+ * Replaces verbose phrases with token-efficient abbreviations.
+ * Case-insensitive replacement using EXPENSIVE_WORDS dictionary.
+ * @param {string} text - Text to optimize
+ * @returns {string} Optimized text
+ */
+export function alterExpensiveWords(text) {
+    if (typeof text !== 'string') return text;
+
+    const keys = Object.keys(EXPENSIVE_WORDS);
+    if (keys.length === 0) return text;
+
+    // Build regex pattern from all keys
+    const pattern = new RegExp(
+        '\\b(' + keys.map(k => k.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|') + ')\\b',
+        'gi'
+    );
+
+    return text.replace(pattern, (match) => {
+        return EXPENSIVE_WORDS[match.toLowerCase()] || match;
+    });
+}
+
+/**
+ * Higher-order function that wraps converters with Smart Code Optimization.
+ * 
+ * Preprocessing steps:
+ * 1. Extract code blocks and replace with placeholders
+ * 2. Extract data blocks (JSON/XML/CSV) and replace with placeholders
+ * 3. Apply expensive word replacements to remaining text
+ * 4. Convert data blocks using the converter function
+ * 5. Re-insert converted data blocks
+ * 6. Reduce and re-insert code blocks
+ * 
+ * @param {Function} converterFn - The converter function to wrap
+ * @param {Function} extractFn - Function to extract data blocks (e.g., extractJsonFromString)
+ * @returns {Function} Wrapped converter function
+ */
+export function dataManager(converterFn, extractFn) {
+    return function (data, ...args) {
+        if (typeof data !== 'string') {
+            return converterFn(data, ...args);
+        }
+
+        let processedData = data;
+
+        // Step 1: Extract code blocks and replace with placeholders
+        const codeBlocks = extractCodeBlocks(processedData);
+
+        // Replace from end to start to preserve indices
+        for (let i = codeBlocks.length - 1; i >= 0; i--) {
+            const block = codeBlocks[i];
+            processedData = processedData.substring(0, block.start) +
+                `#code#${i}#code#` +
+                processedData.substring(block.end);
+        }
+
+        // Step 2: Extract data blocks and replace with placeholders
+        const dataBlocks = [];
+        let iterationCount = 0;
+        const maxIterations = 100;
+
+        while (iterationCount < maxIterations) {
+            const block = extractFn(processedData);
+            if (!block) break;
+
+            dataBlocks.push(block);
+            processedData = processedData.replace(block, `#data#${iterationCount}#data#`);
+            iterationCount++;
+        }
+
+        // Step 3: Apply expensive word replacements to remaining text
+        processedData = alterExpensiveWords(processedData);
+
+        // Step 4: Compress double newlines
+        processedData = processedData.replace(/\n\n/g, '\n');
+
+        // Step 5: Convert and re-insert data blocks
+        let convertedData = processedData.trim();
+
+        if (dataBlocks.length > 0) {
+            // Special Case: If input was 100% one data block and no code blocks, return raw result
+            if (dataBlocks.length === 1 && convertedData === '#data#0#data#' && codeBlocks.length === 0) {
+                return converterFn(dataBlocks[0].trim(), ...args);
+            }
+
+            for (let i = dataBlocks.length - 1; i >= 0; i--) {
+                const block = dataBlocks[i];
+                let convertedBlock = converterFn(block.trim(), ...args);
+
+                // If the converter returned an object/array, stringify it for insertion
+                if (typeof convertedBlock !== 'string') {
+                    try {
+                        convertedBlock = JSON.stringify(convertedBlock);
+                    } catch (e) {
+                        convertedBlock = String(convertedBlock);
+                    }
+                }
+
+                convertedData = convertedData.replace(`#data#${i}#data#`, convertedBlock);
+            }
+        } else {
+            // No data blocks found, convert the whole thing
+            convertedData = converterFn(convertedData, ...args);
+        }
+
+        // Step 6: Reduce and re-insert code blocks
+        for (let i = codeBlocks.length - 1; i >= 0; i--) {
+            const reducedCode = reduceCodeBlock(codeBlocks[i].code);
+            convertedData = convertedData.replace(`#code#${i}#code#`, reducedCode);
+        }
+
+        return convertedData;
+    };
+}
+
+/**
+ * Asynchronous version of dataManager.
+ * 
+ * @param {Function} converterFn - The ASYNC converter function to wrap
+ * @param {Function} extractFn - Function to extract data blocks
+ * @returns {Function} Wrapped ASYNC converter function
+ */
+export function dataManagerAsync(converterFn, extractFn) {
+    return async function (data, ...args) {
+        if (typeof data !== 'string') {
+            return await converterFn(data, ...args);
+        }
+
+        let processedData = data;
+
+        // Step 1: Extract code blocks and replace with placeholders
+        const codeBlocks = extractCodeBlocks(processedData);
+        for (let i = codeBlocks.length - 1; i >= 0; i--) {
+            const block = codeBlocks[i];
+            processedData = processedData.substring(0, block.start) +
+                `#code#${i}#code#` +
+                processedData.substring(block.end);
+        }
+
+        // Step 2: Extract data blocks and replace with placeholders
+        const dataBlocks = [];
+        let iterationCount = 0;
+        const maxIterations = 100;
+
+        while (iterationCount < maxIterations) {
+            const block = extractFn(processedData);
+            if (!block) break;
+
+            dataBlocks.push(block);
+            processedData = processedData.replace(block, `#data#${iterationCount}#data#`);
+            iterationCount++;
+        }
+
+        // Step 3: Apply expensive word replacements
+        processedData = alterExpensiveWords(processedData);
+
+        // Step 4: Compress double newlines
+        processedData = processedData.replace(/\n\n/g, '\n');
+
+        // Step 5: Convert and re-insert data blocks
+        let convertedData = processedData.trim();
+
+        if (dataBlocks.length > 0) {
+            // Special Case: If input was 100% one data block and no code blocks, return raw result
+            if (dataBlocks.length === 1 && convertedData === '#data#0#data#' && codeBlocks.length === 0) {
+                return await converterFn(dataBlocks[0].trim(), ...args);
+            }
+
+            for (let i = dataBlocks.length - 1; i >= 0; i--) {
+                const block = dataBlocks[i];
+                let convertedBlock = await converterFn(block.trim(), ...args);
+
+                if (typeof convertedBlock !== 'string') {
+                    try {
+                        convertedBlock = JSON.stringify(convertedBlock);
+                    } catch (e) {
+                        convertedBlock = String(convertedBlock);
+                    }
+                }
+
+                convertedData = convertedData.replace(`#data#${i}#data#`, convertedBlock);
+            }
+        } else {
+            convertedData = await converterFn(convertedData, ...args);
+        }
+
+        // Step 6: Reduce and re-insert code blocks
+        for (let i = codeBlocks.length - 1; i >= 0; i--) {
+            const reducedCode = reduceCodeBlock(codeBlocks[i].code);
+            convertedData = convertedData.replace(`#code#${i}#code#`, reducedCode);
+        }
+
+        return convertedData;
+    };
+}