ultimate-jekyll-manager 0.0.38 → 0.0.40

package/README.md

@@ -76,6 +76,7 @@ npx uj audit
 ```bash
 GH_TOKEN=XXX \
 GITHUB_REPOSITORY=XXX \
+UJ_TRANSLATION_CACHE=true \
 npx uj translation
 ```
 <!-- Developing -->

package/dist/commands/audit.js

@@ -11,5 +11,5 @@ module.exports = async function (options) {
   // Log
   logger.log(`Starting audit...`);
 
-  await execute('UJ_FORCE_AUDIT=true bundle exec npm run gulp -- audit', { log: true })
+  await execute('UJ_AUDIT_FORCE=true bundle exec npm run gulp -- audit', { log: true })
 };

package/dist/commands/translation.js

@@ -11,5 +11,5 @@ module.exports = async function (options) {
   // Log
   logger.log(`Starting translation...`);
 
-  await execute('UJ_FORCE_TRANSLATION=true bundle exec npm run gulp -- translation', { log: true })
+  await execute('UJ_TRANSLATION_FORCE=true bundle exec npm run gulp -- translation', { log: true })
 };

package/dist/gulp/tasks/audit.js

@@ -35,8 +35,8 @@ async function audit(complete) {
   // Log
   logger.log('Starting...');
 
-  // Quit if NOT in build mode and UJ_FORCE_AUDIT is not true
-  if (!Manager.isBuildMode() && process.env.UJ_FORCE_AUDIT !== 'true') {
+  // Quit if NOT in build mode and UJ_AUDIT_FORCE is not true
+  if (!Manager.isBuildMode() && process.env.UJ_AUDIT_FORCE !== 'true') {
     logger.log('Skipping audit in development mode');
     return complete();
   }
@@ -72,10 +72,15 @@ async function validateFormat(file, content) {
     (async () => {
       try {
         // Format the content using Prettier
-        const formatted = await formatDocument(content, format, true);
+        const formatted = await formatDocument(content, format);
 
         // Save the formatted content back to the file
-        jetpack.write(file, formatted);
+        jetpack.write(file, formatted.content);
+
+        // Quit if there is an error
+        if (formatted.error) {
+          throw formatted.error;
+        }
 
         return { valid: true, messages: [] };
       } catch (e) {

package/dist/gulp/tasks/defaults.js

@@ -232,7 +232,7 @@ function defaultsWatcher(complete) {
   // Watch for changes
   watch(input, { delay: delay, dot: true }, defaults)
   .on('change', (path) => {
-    logger.log(`[watcher] File ${path} was changed`);
+    logger.log(`[watcher] File changed (${path})`);
   });
 
   // Complete

package/dist/gulp/tasks/developmentRebuild.js

@@ -91,7 +91,7 @@ function developmentRebuildWatcher(complete) {
   // Watch for changes
   watch(input, { delay: delay, dot: true }, developmentRebuild)
   .on('change', (path) => {
-    logger.log(`[watcher] File ${path} was changed`);
+    logger.log(`[watcher] File changed (${path})`);
   });
 
   // Complete

package/dist/gulp/tasks/distribute.js

@@ -102,7 +102,7 @@ function distributeWatcher(complete) {
   // Watch for changes
   watch(input, { delay: delay, dot: true }, distribute)
   .on('change', (path) => {
-    logger.log(`[watcher] File ${path} was changed`);
+    logger.log(`[watcher] File changed (${path})`);
   });
 
   // Complete

package/dist/gulp/tasks/imagemin.js

@@ -130,7 +130,7 @@ function imageminWatcher(complete) {
   // Watch for changes
   watch(input, { delay: delay, dot: true }, imagemin)
   .on('change', (path) => {
-    logger.log(`[watcher] File ${path} was changed`);
+    logger.log(`[watcher] File changed (${path})`);
   });
 
   // Complete

package/dist/gulp/tasks/jekyll.js

@@ -79,9 +79,6 @@ async function jekyll(complete) {
   // Run buildpost hook
   await hook('build:post', index);
 
-  // QUIT
-  // QUIT
-
   // Log
   logger.log('Finished!');
 
@@ -113,7 +110,7 @@ function jekyllWatcher(complete) {
   // Watch for changes
   watch(input, { delay: delay, dot: true }, jekyll)
   .on('change', (path) => {
-    logger.log(`[watcher] File ${path} was changed`);
+    logger.log(`[watcher] File changed (${path})`);
 
     // Check if changed file is a .rb file
     if (path.endsWith('.rb')) {

package/dist/gulp/tasks/sass.js

@@ -113,7 +113,7 @@ function sassWatcher(complete) {
   // Watch for changes
   watch(input, { delay: delay, dot: true }, sass)
   .on('change', (path) => {
-    logger.log(`[watcher] File ${path} was changed`);
+    logger.log(`[watcher] File changed (${path})`);
   });
 
   // Complete

package/dist/gulp/tasks/translation.js

@@ -1,7 +1,7 @@
 // Libraries
 const Manager = new (require('../../build.js'));
 const logger = Manager.logger('translation');
-const { series } = require('gulp');
+const { series, watch } = require('gulp');
 const glob = require('glob').globSync;
 const path = require('path');
 const fetch = require('wonderful-fetch');
@@ -9,7 +9,7 @@ const jetpack = require('fs-jetpack');
 const cheerio = require('cheerio');
 const crypto = require('crypto');
 const yaml = require('js-yaml');
-const { execute, wait } = require('node-powertools');
+const { execute, wait, template } = require('node-powertools');
 const { Octokit } = require('@octokit/rest')
 const AdmZip = require('adm-zip') // npm install adm-zip
 
@@ -38,13 +38,25 @@ const RECHECK_DAYS = 0;
 // const AI_MODEL = 'gpt-4.1-nano';
 const AI_MODEL = 'gpt-4.1-mini';
 const TRANSLATION_BRANCH = 'uj-translations';
-const LOUD = false;
-// const LOUD = true;
+// const LOUD = false;
+const LOUD = process.env.UJ_LOUD_LOGS === 'true';
 
 const TRANSLATION_DELAY_MS = 500; // wait between each translation
 const TRANSLATION_BATCH_SIZE = 25; // wait longer every N translations
 const TRANSLATION_BATCH_DELAY_MS = 10000; // longer wait after batch
 
+// Prompt
+const SYSTEM_PROMPT = `
+  You are a professional translator.
+  Translate the provided content, preserving all original formatting, HTML structure, metadata, and links.
+  Do not explain anything — just return the translated content.
+  The content is TAGGED with [0]...[/0], etc. to mark the text. You MUST KEEP THESE TAGS IN PLACE IN YOUR RESPONSE and OPEN ([0]) and CLOSE ([/0]) them PROPERLY.
+  DO NOT translate URLs or other non-text elements.
+  DO NOT translate the brand.
+  Brand: {brand}
+  Translate to {lang}
+`;
+
 // Variables
 let octokit;
 
@@ -61,11 +73,8 @@ async function translation(complete) {
   // Log
   logger.log('Starting...');
 
-  // Get ignored pages
-  const ignoredPages = getIgnoredPages();
-
-  // Quit if NOT in build mode and UJ_FORCE_TRANSLATION is not true
-  if (!Manager.isBuildMode() && process.env.UJ_FORCE_TRANSLATION !== 'true') {
+  // Quit if NOT in build mode and UJ_TRANSLATION_FORCE is not true
+  if (!Manager.isBuildMode() && process.env.UJ_TRANSLATION_FORCE !== 'true') {
     logger.log('Skipping translation in development mode');
     return complete();
   }
@@ -103,6 +112,34 @@ async function translation(complete) {
   return complete();
 };
 
+// TODO: Currently this does not work because it will run an infinite loop
+function translationWatcher(complete) {
+  // Quit if in build mode
+  if (Manager.isBuildMode()) {
+    logger.log('[watcher] Skipping watcher in build mode');
+    return complete();
+  }
+
+  // Log
+  logger.log('[watcher] Watching for changes...');
+
+  // Get ignored pages
+  const ignoredPages = getIgnoredPages();
+  const ignore = [
+    ...ignoredPages.files.map(key => `_site/${key}.html`),
+    ...ignoredPages.folders.map(folder => `_site/${folder}/**/*`)
+  ]
+
+  // Watch for changes
+  watch(input, { delay: delay, ...getGlobOptions(), }, translation)
+  .on('change', (path) => {
+    logger.log(`[watcher] File changed (${path})`);
+  });
+
+  // Complete
+  return complete();
+}
+
 // Default Task
 module.exports = series(translation);
 
@@ -110,7 +147,6 @@ module.exports = series(translation);
 async function processTranslation() {
   const enabled = config?.translation?.enabled !== false;
   const languages = config?.translation?.languages || [];
-  const ignoredPages = getIgnoredPages();
   const updatedFiles = new Set();
 
   // Quit if translation is disabled or no languages are configured
@@ -135,20 +171,20 @@ async function processTranslation() {
   // }
 
   // Get files
-  const allFiles = glob(input, {
-    nodir: true,
-    ignore: [
-      ...ignoredPages.files.map(key => `_site/${key}.html`),
-      ...ignoredPages.folders.map(folder => `_site/${folder}/**/*`)
-    ]
-  });
+  const allFiles = glob(input, getGlobOptions());
 
   // Log
   logger.log(`Translating ${allFiles.length} files for ${languages.length} supported languages: ${languages.join(', ')}`);
   // logger.log(allFiles);
 
   // Prepare meta caches per language
-  const metas = {};
+  const metas = {
+    global: {
+      skipped: new Set(),
+    }
+  };
+  const promptHash = crypto.createHash('sha256').update(SYSTEM_PROMPT).digest('hex');
+
   for (const lang of languages) {
     const metaPath = path.join(CACHE_DIR, lang, 'meta.json');
     let meta = {};
@@ -159,7 +195,16 @@ async function processTranslation() {
         logger.warn(`⚠️ Failed to parse meta for [${lang}], starting fresh`);
       }
     }
-    metas[lang] = { meta, path: metaPath, skipped: [] };
+
+    // Check if the promptHash matches; if not, invalidate the cache
+    if (meta.prompt?.hash !== promptHash) {
+      logger.warn(`⚠️ Prompt hash mismatch for [${lang}]. Invalidating cache.`);
+      meta = {};
+    }
+
+    // Store the current promptHash in the meta file
+    meta.prompt = { hash: promptHash };
+    metas[lang] = { meta, path: metaPath, skipped: new Set() };
   }
 
   // Track token usage
@@ -183,9 +228,8 @@ async function processTranslation() {
 
     // Skip all except the specified HTML file
     if (ujOnly && relativePath !== ujOnly) {
-      for (const lang of languages) {
-        metas[lang].skipped.push(`${relativePath} (UJ_TRANSLATION_ONLY set)`);
-      }
+      // Update to work with the new SET protocol
+      metas.global.skipped.add(`${relativePath} (UJ_TRANSLATION_ONLY set)`);
       continue;
     }
 
@@ -214,11 +258,16 @@ async function processTranslation() {
         const age = entry?.timestamp
           ? (Date.now() - new Date(entry.timestamp).getTime()) / (1000 * 60 * 60 * 24)
           : Infinity;
-        const useCached = entry && entry.hash === hash && (RECHECK_DAYS === 0 || age < RECHECK_DAYS);
+        const useCached = entry
+          && entry.hash === hash
+          && (RECHECK_DAYS === 0 || age < RECHECK_DAYS);
         const startTime = Date.now();
 
         // Check if we can use cached translation
-        if (useCached && jetpack.exists(cachePath)) {
+        if (
+          (useCached || process.env.UJ_TRANSLATION_CACHE === 'true')
+          && jetpack.exists(cachePath)
+        ) {
           translated = jetpack.read(cachePath);
           logger.log(`📦 Using cached translation for ${relativePath} [${lang}]`);
         } else {
@@ -260,6 +309,11 @@ async function processTranslation() {
         // Log result
         // console.log('---translated---', translated);
 
+        // Reset the DOM to avoid conflicts between languages
+        const $ = cheerio.load(originalHtml);
+        // Collect text nodes with tags
+        const textNodes = collectTextNodes($, { tag: true });
+
         // Replace original text nodes with translated versions
         textNodes.forEach((n, i) => {
           const regex = new RegExp(`\\[${i}\\](.*?)\\[/${i}\\]`, 's');
@@ -270,14 +324,22 @@ async function processTranslation() {
             return logger.warn(`⚠️ Could not find translated tag for index ${i}`);
           }
 
+          // Extract original leading and trailing whitespace
+          const originalText = n.text;
+          const leadingWhitespace = originalText.match(/^\s*/)?.[0] || '';
+          const trailingWhitespace = originalText.match(/\s*$/)?.[0] || '';
+
+          // Reapply the original whitespace to the translation
+          const adjustedTranslation = `${leadingWhitespace}${translation.trim()}${trailingWhitespace}`;
+
           if (n.type === 'data') {
-            n.reference.data = translation;
+            n.reference.data = adjustedTranslation;
           } else if (n.type === 'text') {
-            n.node.text(translation);
+            n.node.text(adjustedTranslation);
           } else if (n.type === 'attr') {
-            n.node.attr(n.attr, translation);
+            n.node.attr(n.attr, adjustedTranslation);
           }
-          if (LOUD) logger.log(`${i}: ${n.text} → ${translation}`);
+          if (LOUD) logger.log(`${i}: "${n.text.trim()}" → "${adjustedTranslation.trim()}"`);
         });
 
         // Rewrite links
@@ -294,7 +356,7 @@ async function processTranslation() {
         $('meta[property="og:url"]').attr('content', canonicalUrl);
 
         // Insert language tags on this translation
-        await insertLanguageTags($, languages, relativePath);
+        await insertLanguageTags($, languages, relativePath, outPath);
 
         // Insert language tags in original file
         await insertLanguageTags(cheerio.load(originalHtml), languages, relativePath, filePath);
@@ -305,7 +367,15 @@ async function processTranslation() {
         await insertLanguageTags(cheerio.load(sitemapXml, { xmlMode: true }), languages, relativePath, sitemapPath);
 
         // Save output
-        jetpack.write(outPath, await formatDocument($.html(), undefined, false));
+        // const formatted = await formatDocument($.html(), 'html');
+
+        // console.log('----relativePath', relativePath);
+        // console.log('----filePath', filePath);
+        // console.log('----outPath', outPath);
+        // console.log('----FORMATTED.ERROR', formatted.error);
+
+        // Write the translated file
+        // jetpack.write(outPath, formatted.content);
         // logger.log(`✅ Wrote: ${outPath}`);
 
         // Track updated files only if it's new or updated
@@ -334,13 +404,23 @@ async function processTranslation() {
     }
   }
 
+  // Log skipped files
+  logger.warn('🚫 Skipped files:');
+  let totalSkipped = 0;
+  for (const [lang, meta] of Object.entries(metas)) {
+    if (meta.skipped.size > 0) {
+      logger.warn(`  [${lang}] ${meta.skipped.size} skipped files:`);
+      meta.skipped.forEach(f => logger.warn(`    ${f}`));
+      totalSkipped += meta.skipped.size;
+    }
+  }
+  if (totalSkipped === 0) {
+    logger.warn('  NONE');
+  }
+
   // Save all updated meta files
   for (const lang of languages) {
     jetpack.write(metas[lang].path, metas[lang].meta);
-    if (metas[lang].skipped.length) {
-      logger.warn('🚫 Skipped files:');
-      metas[lang].skipped.forEach(f => logger.warn(f));
-    }
   }
 
   // Log total token usage
@@ -356,14 +436,11 @@ async function processTranslation() {
 }
 
 async function translateWithAPI(openAIKey, content, lang) {
-  // Prompt
-  const systemPrompt = `
-    You are a professional translator.
-    Translate the provided content, preserving all original formatting, HTML structure, metadata, and links.
-    Do not explain anything — just return the translated content.
-    The content is TAGGED with [0]...[/0], etc. to mark the text. You MUST KEEP THESE TAGS IN PLACE IN YOUR RESPONSE and OPEN ([0]) and CLOSE ([/0]) them PROPERLY.
-    Translate to ${lang}.
-  `;
+  const brand = config?.brand?.name || 'Unknown Brand';
+  const systemPrompt = template(SYSTEM_PROMPT, {
+    lang,
+    brand
+  });
 
   // Request
   const res = await fetch('https://api.openai.com/v1/chat/completions', {
@@ -521,9 +598,11 @@ async function insertLanguageTags($, languages, relativePath, filePath) {
 
   // Save the modified HTML back to the file if filePath
   if (filePath) {
-    // const format = isHtml ? 'html' : 'xml';
-    const format = 'html';
-    jetpack.write(filePath, await formatDocument($.html(), format));
+    const format = isHtml ? 'html' : 'xml';
+    const formatted = await formatDocument($.html(), format);
+
+    // Write the formatted content back to the file
+    jetpack.write(filePath, formatted.content);
   }
 }
 
@@ -597,6 +676,17 @@ function getIgnoredPages() {
   };
 }
 
+function getGlobOptions() {
+  const ignoredPages = getIgnoredPages();
+  return {
+    nodir: true,
+    ignore: [
+      ...ignoredPages.files.map(key => `_site/${key}.html`),
+      ...ignoredPages.folders.map(folder => `_site/${folder}/**/*`)
+    ]
+  }
+}
+
 // Git Sync: Pull
 async function fetchTranslationsBranch() {
   const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/')

package/dist/gulp/tasks/utils/collectTextNodes.js

@@ -21,14 +21,14 @@ const collectTextNodes = ($, options) => {
       const i = textNodes.length;
       const text = node.text().trim();
       if (text) {
+
+        // Push
         textNodes.push({
           node,
           type: 'text',
           attr: null,
           text,
           tagged: `[${i}]${text}[/${i}]`,
-          line: el.startIndex || 0, // Add line information
-          column: 0 // Column is not directly available, default to 0
         });
       }
       return;
@@ -51,14 +51,14 @@ const collectTextNodes = ($, options) => {
         const text = node.attr('content')?.trim();
         if (text) {
           const i = textNodes.length;
+
+          // Push
           textNodes.push({
             node,
             type: 'attr',
             attr: 'content',
             text,
             tagged: `[${i}]${text}[/${i}]`,
-            line: el.startIndex || 0, // Add line information
-            column: 0 // Column is not directly available, default to 0
           });
         }
       }
@@ -70,10 +70,15 @@ const collectTextNodes = ($, options) => {
       if (child.type === 'text' && child.data?.trim()) {
         const i = textNodes.length;
         const text = child.data
-          .replace(/^\s+/, '')
-          .replace(/\s+$/, '')
-          .replace(/\s+/g, ' ');
+          // Preserve a single leading whitespace if it exists
+          .replace(/^\s*(\s)\s*/, '$1')
+          // Preserve a single trailing whitespace if it exists
+          .replace(/\s*(\s)\s*$/, '$1')
+          // Normalize internal whitespace
+          .replace(/\s+/g, ' ')
+        // const text = trimPreserveOneCleanInner(child.data);
 
+        // Push
         textNodes.push({
           node,
           type: 'data',
@@ -81,8 +86,6 @@ const collectTextNodes = ($, options) => {
           reference: child,
           text,
           tagged: `[${i}]${text}[/${i}]`,
-          line: el.startIndex || 0, // Add line information
-          column: 0 // Column is not directly available, default to 0
         });
       }
     });
@@ -92,3 +95,12 @@ const collectTextNodes = ($, options) => {
 };
 
 module.exports = collectTextNodes;
+
+
+function trimPreserveOneCleanInner(str) {
+  const match = str.match(/^(\s*)(.*?)(\s*)$/);
+  const leading = match[1] ? match[1][0] || '' : '';
+  const content = match[2].replace(/\s+/g, ' ');
+  const trailing = match[3] ? match[3][0] || '' : '';
+  return leading + content + trailing;
+}

package/dist/gulp/tasks/utils/formatDocument.js

@@ -6,10 +6,9 @@ const path = require('path');
 // Load package
 const rootPathPackage = Manager.getRootPath('main');
 
-module.exports = async function formatHTML(content, format, throwError) {
+module.exports = async (content, format) => {
   // Set default format to 'html' if not provided
   format = format || 'html';
-  throwError = typeof throwError === 'undefined' ? true : throwError;
 
   // Setup Prettier options
   const options = {
@@ -30,16 +29,19 @@ module.exports = async function formatHTML(content, format, throwError) {
   return prettier
     .format(content, options)
     .then((formatted) => {
-      return removeMultipleNewlines(formatted);
+      return {
+        content: removeMultipleNewlines(formatted),
+        error: null,
+      };
     })
     .catch((e) => {
-      if (throwError) {
-        throw e;
-      }
-      return removeMultipleNewlines(content);
+      return {
+        content: removeMultipleNewlines(content),
+        error: e,
+      };
     });
 };
 
 function removeMultipleNewlines(content) {
-  return content.replace(/\n\s*\n+/g, '\n');
+  return content.replace(/\n\s*\n+/g, '\n').trim();
 }

package/dist/gulp/tasks/webpack.js

@@ -186,7 +186,7 @@ function webpackWatcher(complete) {
   watch(input, { delay: delay, dot: true }, webpack)
   .on('change', (path) => {
     // Log
-    logger.log(`[watcher] File ${path} was changed`);
+    logger.log(`[watcher] File changed (${path})`);
   });
 
   // Complete

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ultimate-jekyll-manager",
-  "version": "0.0.38",
+  "version": "0.0.40",
   "description": "Ultimate Jekyll dependency manager",
   "main": "dist/index.js",
   "exports": {