ultimate-jekyll-manager 1.3.11 → 1.4.0

package/dist/gulp/tasks/imagemin.js

@@ -4,6 +4,7 @@ const logger = Manager.logger('imagemin');
 const { src, dest, watch, series } = require('gulp');
 const glob = require('glob').globSync;
 const responsive = require('gulp-responsive-modern');
+const sharp = require('sharp');
 const path = require('path');
 const jetpack = require('fs-jetpack');
 const GitHubCache = require('./utils/github-cache');
@@ -15,6 +16,8 @@ const ujmConfig = Manager.getUJMConfig();
 // Settings
 const CACHE_DIR = '.temp/cache/imagemin';
 const CACHE_BRANCH = 'cache-uj-imagemin';
+const MAX_SOURCE_DIMENSION = 4096;
+const REWRITE_QUALITY = 80;
 
 // Variables
 let githubCache;
@@ -120,6 +123,15 @@ async function imagemin(complete) {
     githubCache.cleanDeletedFromMetadata(meta, files, rootPathProject);
   }
 
+  // Optionally rewrite oversized source images on disk (opt-in via UJ_IMAGEMIN_REWRITE_SOURCES=true).
+  // Caps longest dimension at MAX_SOURCE_DIMENSION so gulp-responsive-modern + sharp don't stall
+  // on huge inputs. Runs BEFORE determineFilesToProcess so cached-but-oversized images get
+  // rewritten too; the new on-disk content hashes differently than the stored meta hash, so
+  // determineFilesToProcess naturally picks the rewritten image up for re-optimization.
+  if (process.env.UJ_IMAGEMIN_REWRITE_SOURCES === 'true') {
+    await rewriteOversizedSources(files);
+  }
+
   // Determine what needs processing
   const { filesToProcess, validCachePaths } = await determineFilesToProcess(files, meta, githubCache, stats);
 
@@ -159,87 +171,101 @@ async function imagemin(complete) {
   // Progress counter
   let processedOutputs = 0;
 
-  // Process images
-  return src(filesToProcess, { base: 'src/assets/images' })
-    .pipe(responsive({
-      [`**/${RESPONSIVE_GLOB}`]: responsiveConfigs
-    }, {
-      quality: 80,
-      progressive: true,
-      withMetadata: false,
-      withoutEnlargement: false,
-      skipOnEnlargement: false,
-      errorOnUnusedImage: false,
-      passThroughUnused: true,
-    }))
-    .pipe(dest(output))
-    .on('data', (file) => {
-      // Progress tracking
-      processedOutputs++;
-      const relativePath = path.relative(path.join(rootPathProject, output), file.path);
-      logger.log(`🖼️  ${processedOutputs}/${expectedOutputs}: ${relativePath}`);
-
-      // Save to cache
-      const cachePath = path.join(CACHE_DIR, 'images', relativePath);
-      jetpack.copy(file.path, cachePath, { overwrite: true });
-
-      // Track size after optimization
-      const fileStats = jetpack.inspect(file.path);
-      if (fileStats) {
-        stats.sizeAfter += fileStats.size;
-      }
-    })
-    .on('finish', async () => {
-      // Calculate final statistics
-      stats.savedBytes = stats.sizeBefore - stats.sizeAfter;
-
-      // Calculate timing
-      const endTime = Date.now();
-      const elapsedMs = endTime - startTime;
-
-      // Log statistics
-      logImageStatistics(stats, startTime, endTime);
-
-      // Save metadata and push cache
-      if (githubCache && githubCache.hasCredentials()) {
-        githubCache.saveMetadata(metaPath, meta);
-
-        logger.log(`📊 Updating cache with ${stats.optimized} new optimizations and README stats...`);
-
-        // Collect all cache files to push (metadata will be auto-included)
-        const allCacheFiles = glob(path.join(CACHE_DIR, '**/*'), { nodir: true });
-
-        // Push to GitHub with atomic replacement
-        await githubCache.pushBranch(allCacheFiles, {
-          validFiles: validCachePaths,
-          stats: {
-            timestamp: new Date().toISOString(),
-            sourceCount: files.length,
-            cachedCount: allCacheFiles.length - 1,
-            processedNow: stats.optimized,
-            fromCache: stats.fromCache,
-            newlyProcessed: stats.optimized,
-            timing: {
-              startTime,
-              endTime,
-              elapsedMs
-            },
-            imageStats: {
-              totalImages: stats.totalImages,
-              optimized: stats.optimized,
-              skipped: stats.fromCache,
-              totalSizeBefore: stats.sizeBefore,
-              totalSizeAfter: stats.sizeAfter,
-              totalSaved: stats.savedBytes
-            },
-            details: `Optimized ${stats.optimized} images, ${stats.fromCache} from cache\n\n### Files from cache:\n${stats.cachedFiles.length > 0 ? stats.cachedFiles.map(f => `- ${f}`).join('\n') : 'None'}\n\n### Newly optimized files:\n${stats.optimizedFiles.length > 0 ? stats.optimizedFiles.map(f => `- ${f}`).join('\n') : 'None'}`
-          }
-        });
-      }
+  // Process images.
+  //
+  // CRITICAL: this function is `async`, which means returning a stream from it yields a
+  // Promise<Stream> to gulp — gulp resolves the task immediately on the Promise rather than
+  // waiting for the stream's 'finish' event. Downstream tasks (jekyll, audit, etc.) then start
+  // before imagemin has actually written its outputs to disk, and the build "succeeds" while
+  // silently shipping a partial _site/. We must explicitly await stream completion + cache push
+  // before returning, so gulp sees the real completion.
+  //
+  // This await only ever runs in build mode — dev mode short-circuits via `!Manager.isBuildMode()`
+  // above (so `npm start` never blocks on this), letting BrowserSync reload as images land later.
+  await new Promise((resolve, reject) => {
+    src(filesToProcess, { base: 'src/assets/images' })
+      .pipe(responsive({
+        [`**/${RESPONSIVE_GLOB}`]: responsiveConfigs
+      }, {
+        quality: 80,
+        progressive: true,
+        withMetadata: false,
+        withoutEnlargement: false,
+        skipOnEnlargement: false,
+        errorOnUnusedImage: false,
+        passThroughUnused: true,
+      }))
+      .on('error', reject)
+      .pipe(dest(output))
+      .on('error', reject)
+      .on('data', (file) => {
+        // Progress tracking
+        processedOutputs++;
+        const relativePath = path.relative(path.join(rootPathProject, output), file.path);
+        logger.log(`🖼️  ${processedOutputs}/${expectedOutputs}: ${relativePath}`);
+
+        // Save to cache
+        const cachePath = path.join(CACHE_DIR, 'images', relativePath);
+        jetpack.copy(file.path, cachePath, { overwrite: true });
+
+        // Track size after optimization
+        const fileStats = jetpack.inspect(file.path);
+        if (fileStats) {
+          stats.sizeAfter += fileStats.size;
+        }
+      })
+      .on('finish', resolve);
+  });
+
+  // Calculate final statistics
+  stats.savedBytes = stats.sizeBefore - stats.sizeAfter;
 
-      logger.log('✅ Finished!');
-      return complete();
+  // Calculate timing
+  const endTime = Date.now();
+  const elapsedMs = endTime - startTime;
+
+  // Log statistics
+  logImageStatistics(stats, startTime, endTime);
+
+  // Save metadata and push cache
+  if (githubCache && githubCache.hasCredentials()) {
+    githubCache.saveMetadata(metaPath, meta);
+
+    logger.log(`📊 Updating cache with ${stats.optimized} new optimizations and README stats...`);
+
+    // Collect all cache files to push (metadata will be auto-included)
+    const allCacheFiles = glob(path.join(CACHE_DIR, '**/*'), { nodir: true });
+
+    // Push to GitHub with atomic replacement
+    await githubCache.pushBranch(allCacheFiles, {
+      validFiles: validCachePaths,
+      stats: {
+        timestamp: new Date().toISOString(),
+        sourceCount: files.length,
+        cachedCount: allCacheFiles.length - 1,
+        processedNow: stats.optimized,
+        fromCache: stats.fromCache,
+        newlyProcessed: stats.optimized,
+        timing: {
+          startTime,
+          endTime,
+          elapsedMs
+        },
+        imageStats: {
+          totalImages: stats.totalImages,
+          optimized: stats.optimized,
+          skipped: stats.fromCache,
+          totalSizeBefore: stats.sizeBefore,
+          totalSizeAfter: stats.sizeAfter,
+          totalSaved: stats.savedBytes
+        },
+        details: `Optimized ${stats.optimized} images, ${stats.fromCache} from cache\n\n### Files from cache:\n${stats.cachedFiles.length > 0 ? stats.cachedFiles.map(f => `- ${f}`).join('\n') : 'None'}\n\n### Newly optimized files:\n${stats.optimizedFiles.length > 0 ? stats.optimizedFiles.map(f => `- ${f}`).join('\n') : 'None'}`
+      }
     });
+  }
+
+  logger.log('✅ Finished!');
+  return complete();
 }
 
 // Watcher task
@@ -273,6 +299,61 @@ module.exports = series(
 // Helper Functions
 // ============================================================================
 
+// Rewrite oversized source images in place, capping longest dimension at MAX_SOURCE_DIMENSION.
+// Only affects files whose decoded longest side exceeds the cap. Cache invalidation is implicit:
+// the new content hashes differently than the previously-cached entry, so determineFilesToProcess
+// will pick affected files up for re-optimization on its own.
+async function rewriteOversizedSources(files) {
+  const responsiveFiles = files.filter(f => RESPONSIVE_EXTENSIONS.has(path.extname(f).slice(1).toLowerCase()));
+  if (responsiveFiles.length === 0) {
+    return;
+  }
+
+  logger.log(`🔍 Checking ${responsiveFiles.length} source images for oversize (>${MAX_SOURCE_DIMENSION}px longest side)...`);
+
+  let rewritten = 0;
+  for (const file of responsiveFiles) {
+    const metadata = await sharp(file).metadata();
+    const longest = Math.max(metadata.width || 0, metadata.height || 0);
+
+    if (longest <= MAX_SOURCE_DIMENSION) {
+      continue;
+    }
+
+    const ext = path.extname(file).slice(1).toLowerCase();
+    const sizeBefore = jetpack.inspect(file)?.size || 0;
+
+    // Resize, encode to a buffer (sharp can't write back to its own input file directly), then overwrite.
+    const pipeline = sharp(file).resize({
+      width: MAX_SOURCE_DIMENSION,
+      height: MAX_SOURCE_DIMENSION,
+      fit: 'inside',
+      withoutEnlargement: true,
+    });
+
+    const buffer = ext === 'png'
+      ? await pipeline.png({ quality: REWRITE_QUALITY }).toBuffer()
+      : await pipeline.jpeg({ quality: REWRITE_QUALITY, progressive: true, mozjpeg: true }).toBuffer();
+
+    jetpack.write(file, buffer);
+    const sizeAfter = buffer.length;
+
+    // No cache bookkeeping needed: the rewritten file has new content, so the next
+    // determineFilesToProcess() call computes a different hash than the stored meta hash and
+    // naturally treats this image as needing reprocessing. Stored meta will be overwritten
+    // with the new hash when determineFilesToProcess() runs.
+
+    rewritten++;
+    logger.log(`✂️  Rewrote ${path.relative(rootPathProject, file)}: ${metadata.width}x${metadata.height} → max ${MAX_SOURCE_DIMENSION}px, ${formatBytes(sizeBefore)} → ${formatBytes(sizeAfter)}`);
+  }
+
+  if (rewritten === 0) {
+    logger.log(`✅ No oversized sources found (all within ${MAX_SOURCE_DIMENSION}px)`);
+  } else {
+    logger.log(`✂️  Rewrote ${rewritten} oversized source image(s)`);
+  }
+}
+
 // Build responsive configurations from PICTURE_SIZES
 function getResponsiveConfigs() {
   const configs = [];

package/dist/utils/attach-log-file.js

@@ -0,0 +1,78 @@
+// attachLogFile(name) — duplicate process.stdout + process.stderr writes to logs/<name>.log
+// in the consumer project root (process.cwd()).
+//
+// Mirrors EM's attach-log-file pattern so devs (and Claude) can `tail -f logs/dev.log` to see
+// every line of output a UJM session produces — gulp tasks, jekyll child, webpack, SCSS, the
+// works. ANSI color codes are stripped from the file output so it's grep-friendly; the
+// console continues to receive the original colored output unchanged.
+//
+// Skipped entirely when Manager.isServer() returns true — CI/cloud runs don't need a logs/
+// directory left behind in the workspace.
+//
+// Truncates fresh on each call (flags: 'w'), so a new `npm start` doesn't accumulate stale
+// lines from the previous run.
+//
+// Idempotent: calling twice with the same path just returns the existing stream.
+
+const fs = require('fs');
+const path = require('path');
+
+const ANSI_PATTERN = /\x1B\[[0-9;]*[a-zA-Z]/g;
+
+let activeStream = null;
+let activePath   = null;
+let originalStdoutWrite = null;
+let originalStderrWrite = null;
+
+function attachLogFile(name) {
+  // Skip on CI/cloud — controlled by UJ_IS_SERVER env var (set by workflows).
+  const Manager = require('../build.js');
+  if (Manager.isServer()) return null;
+
+  if (!name) return null;
+
+  const abs = path.resolve(process.cwd(), 'logs', `${name}.log`);
+
+  if (activeStream && activePath === abs) return activeStream;
+  if (activeStream) detach();
+
+  fs.mkdirSync(path.dirname(abs), { recursive: true });
+  const stream = fs.createWriteStream(abs, { flags: 'w' });
+
+  stream.write(`# ujm log — ${new Date().toISOString()} — pid=${process.pid}\n`);
+
+  originalStdoutWrite = process.stdout.write.bind(process.stdout);
+  originalStderrWrite = process.stderr.write.bind(process.stderr);
+
+  process.stdout.write = function (chunk, ...rest) {
+    try { stream.write(stripAnsi(String(chunk))); } catch (e) { /* ignore */ }
+    return originalStdoutWrite(chunk, ...rest);
+  };
+  process.stderr.write = function (chunk, ...rest) {
+    try { stream.write(stripAnsi(String(chunk))); } catch (e) { /* ignore */ }
+    return originalStderrWrite(chunk, ...rest);
+  };
+
+  activeStream = stream;
+  activePath   = abs;
+
+  return stream;
+}
+
+function detach() {
+  if (originalStdoutWrite) process.stdout.write = originalStdoutWrite;
+  if (originalStderrWrite) process.stderr.write = originalStderrWrite;
+  if (activeStream) activeStream.end();
+  activeStream = null;
+  activePath   = null;
+  originalStdoutWrite = null;
+  originalStderrWrite = null;
+}
+
+function stripAnsi(s) {
+  return String(s).replace(ANSI_PATTERN, '');
+}
+
+module.exports = attachLogFile;
+module.exports.detach    = detach;
+module.exports.stripAnsi = stripAnsi;

package/docs/images.md

@@ -40,3 +40,30 @@ When posts are created via BEM's `POST /admin/post` endpoint:
 2. Images are uploaded to `src/assets/images/blog/post-{id}/` on GitHub
 3. The body is rewritten to use `@post/{filename}` format
 4. Failed downloads are skipped (original external URL preserved)
+
+## Image Processing Pipeline (`imagemin` gulp task)
+
+The `imagemin` task (`src/gulp/tasks/imagemin.js`) processes every file under `src/assets/images/**/*.{jpg,jpeg,png,gif,svg,webp}` into `dist/assets/images/`. For raster formats supported by `gulp-responsive-modern` (`jpg`, `jpeg`, `png`), each source produces **8 outputs**: original + `1024px`, `640px`, `320px` variants, each as the source format **and** WebP.
+
+Outputs are content-addressed and cached on a dedicated branch (`cache-uj-imagemin`). Subsequent builds only reprocess images whose source hash changed.
+
+### Source size matters
+
+The responsive pipeline streams every input through `sharp` to generate variants. Sources with very large dimensions (tens of thousands of pixels) decode into hundreds of MB of raw pixel data per worker. In practice this can stall the underlying stream so quietly that gulp can't see the failure — the build "succeeds" but the affected images never land in `_site/`.
+
+**Recommendation:** Keep source images at sensible dimensions before they land in `src/assets/images/`. A 4096px longest-side cap at the upload step is plenty for blog hero images (the largest responsive variant is 1024px). If you ingest images via an automated pipeline (e.g. BEM's `admin/post` endpoint downloading external URLs), cap them there.
+
+### Cleanup for existing oversized sources: `UJ_IMAGEMIN_REWRITE_SOURCES`
+
+If oversized images already exist in your repo and you can't easily re-upload them, run the imagemin task once with the rewrite flag set:
+
+```bash
+UJ_IMAGEMIN_REWRITE_SOURCES=true npm run build
+```
+
+When set, the imagemin task — **before** it pipes images into `gulp-responsive-modern` — scans every file scheduled for processing and rewrites in place any whose longest dimension exceeds `4096px`. Rewrites use `sharp` with `fit: 'inside'` (preserves aspect ratio, never enlarges), JPEG quality 80 / `mozjpeg` / progressive, and PNG quality 80. Cache hashes for affected files are updated so the new content is the new cache key.
+
+**Notes:**
+- The flag is **opt-in by design** — running it commits real diffs to your source images. Intended for one-off cleanup runs, not for regular builds or CI.
+- Only files that the cache layer has decided need processing get checked. Already-cached images are untouched. To force-rewrite cached oversized images, also clear the cache (`npx mgr clean`) or delete the `cache-uj-imagemin` branch entries for the relevant files.
+- 4096px is a hardcoded cap (`MAX_SOURCE_DIMENSION` in `src/gulp/tasks/imagemin.js`). Not currently configurable per project — open a PR if you need this.

package/docs/local-development.md

@@ -2,6 +2,17 @@
 
 The local development server URL is stored in `.temp/_config_browsersync.yml` in the consuming project's root directory. Read this file to determine the correct URL for browsing and testing. By default, use `https://192.168.86.69:4000`.
 
+## Log Files
+
+UJM tees every line of stdout/stderr from the gulp pipeline into a log file in the consumer project root, so you can `tail -f` it or `grep` through it after a run:
+
+- `npm start` → `logs/dev.log`
+- `npm run build` (i.e. `UJ_BUILD_MODE=true`) → `logs/build.log`
+
+Both files **truncate fresh on each run** — the most recent session only. ANSI color codes are stripped from the file (so it's grep-friendly); the terminal continues to receive colored output unchanged. Captures everything that flows through stdout/stderr: `Manager.logger(...)` output, raw `console.log` calls, gulp task names, jekyll's child output, webpack output, the works.
+
+**Skipped on CI/cloud.** When `UJ_IS_SERVER=true` (set by GitHub Actions workflows and other server contexts), the tee is bypassed entirely — no `logs/` directory is written. Implementation: [src/utils/attach-log-file.js](../src/utils/attach-log-file.js), attached at the top of [src/gulp/main.js](../src/gulp/main.js).
+
 ## Connecting to Local Firebase Emulators
 
 Set the `FIREBASE_EMULATOR_CONNECT` environment variable to `true` to connect the frontend to local Firebase services (Auth, Firestore, Functions, etc.):

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ultimate-jekyll-manager",
-  "version": "1.3.11",
+  "version": "1.4.0",
   "description": "Ultimate Jekyll dependency manager",
   "main": "dist/index.js",
   "exports": {