retold-remote 0.0.4 → 0.0.6

package/source/server/RetoldRemote-EbookService.js

@@ -18,8 +18,7 @@ const libChildProcess = require('child_process');
 
 const _DefaultServiceConfiguration =
 {
-	"ContentPath": ".",
-	"CachePath": null
+	"ContentPath": "."
 };
 
 // Extensions that can be converted to EPUB by ebook-convert
@@ -60,16 +59,7 @@ class RetoldRemoteEbookService extends libFableServiceProviderBase
 
 		this.contentPath = libPath.resolve(this.options.ContentPath);
 
-		this.cachePath = this.options.CachePath
-			|| libPath.join(process.cwd(), 'dist', 'retold-cache', 'ebook-conversions');
-
-		// Ensure cache directory exists
-		if (!libFs.existsSync(this.cachePath))
-		{
-			libFs.mkdirSync(this.cachePath, { recursive: true });
-		}
-
-		this.fable.log.info(`Ebook Service: cache at ${this.cachePath}`);
+		this.fable.log.info('Ebook Service: using ParimeBinaryStorage (category: ebook-cache)');
 	}
 
 	/**
@@ -96,7 +86,7 @@ class RetoldRemoteEbookService extends libFableServiceProviderBase
 	{
 		let tmpInput = `${pAbsPath}:${pMtimeMs}`;
 		let tmpHash = libCrypto.createHash('sha256').update(tmpInput).digest('hex').substring(0, 16);
-		return libPath.join(this.cachePath, tmpHash);
+		return this.fable.ParimeBinaryStorage.resolvePath('ebook-cache', tmpHash);
 	}
 
 	/**
@@ -221,11 +211,12 @@ class RetoldRemoteEbookService extends libFableServiceProviderBase
 			return null;
 		}
 
-		let tmpPath = libPath.join(this.cachePath, pCacheKey, pFilename);
+		let tmpCacheDir = this.fable.ParimeBinaryStorage.resolvePath('ebook-cache', pCacheKey);
+		let tmpPath = libPath.join(tmpCacheDir, pFilename);
 
-		// Double-check it's under our cache dir
+		// Double-check it's under the storage root
 		let tmpResolved = libPath.resolve(tmpPath);
-		if (!tmpResolved.startsWith(this.cachePath))
+		if (!tmpResolved.startsWith(this.fable.ParimeBinaryStorage.storageRoot))
 		{
 			return null;
 		}

package/source/server/RetoldRemote-MediaService.js

@@ -18,15 +18,11 @@ const libUrl = require('url');
 const libToolDetector = require('./RetoldRemote-ToolDetector.js');
 const libThumbnailCache = require('./RetoldRemote-ThumbnailCache.js');
 
-const _ImageExtensions = { 'png': true, 'jpg': true, 'jpeg': true, 'gif': true, 'webp': true, 'svg': true, 'bmp': true, 'ico': true, 'avif': true, 'tiff': true, 'tif': true, 'heic': true, 'heif': true };
-const _VideoExtensions = { 'mp4': true, 'webm': true, 'mov': true, 'mkv': true, 'avi': true, 'wmv': true, 'flv': true, 'm4v': true, 'ogv': true };
-const _AudioExtensions = { 'mp3': true, 'wav': true, 'ogg': true, 'flac': true, 'aac': true, 'm4a': true, 'wma': true, 'oga': true };
-const _DocumentExtensions = { 'pdf': true, 'epub': true, 'mobi': true, 'doc': true, 'docx': true };
+const libExtensionMaps = require('../RetoldRemote-ExtensionMaps.js');
 
 const _DefaultServiceConfiguration =
 {
 	"ContentPath": ".",
-	"ThumbnailCachePath": null,
 	"APIRoutePrefix": "/api/media",
 	"DefaultThumbnailWidth": 200,
 	"DefaultThumbnailHeight": 200
@@ -51,12 +47,9 @@ class RetoldRemoteMediaService extends libFableServiceProviderBase
 
 		this.contentPath = libPath.resolve(this.options.ContentPath);
 
-		let tmpCachePath = this.options.ThumbnailCachePath
-			|| libPath.join(process.cwd(), 'dist', 'retold-cache', 'thumbnails');
-
 		this.toolDetector = new libToolDetector();
 		this.capabilities = this.toolDetector.detect();
-		this.thumbnailCache = new libThumbnailCache(tmpCachePath);
+		this.thumbnailCache = new libThumbnailCache(this.fable);
 		this.pathRegistry = this.options.PathRegistry || null;
 
 		this.fable.log.info(`Media Service: capabilities = ${JSON.stringify(this.capabilities)}`);
@@ -95,11 +88,7 @@ class RetoldRemoteMediaService extends libFableServiceProviderBase
 	 */
 	_getMediaCategory(pExtension)
 	{
-		if (_ImageExtensions[pExtension]) return 'image';
-		if (_VideoExtensions[pExtension]) return 'video';
-		if (_AudioExtensions[pExtension]) return 'audio';
-		if (_DocumentExtensions[pExtension]) return 'document';
-		return 'other';
+		return libExtensionMaps.getCategory(pExtension);
 	}
 
 	/**

package/source/server/RetoldRemote-ParimeCache.js

@@ -0,0 +1,349 @@
+/**
+ * Retold Remote -- Parime Cache Adapter
+ *
+ * Wraps parime's ParimeBinaryStorage as a unified cache layer for
+ * retold-remote.  Supports two modes:
+ *
+ *   1. Embedded (default): ParimeBinaryStorage accessed directly in-process.
+ *   2. Remote: If fable.settings.ParimeCacheServer is a URL, all cache
+ *      operations route through HTTP to a remote parime server.
+ *
+ * This adapter provides helper methods for cache key generation and
+ * passthrough access to the underlying storage.
+ */
+const libFableServiceProviderBase = require('fable-serviceproviderbase');
+const libCrypto = require('crypto');
+
+class RetoldRemoteParimeCache extends libFableServiceProviderBase
+{
+	constructor(pFable, pOptions, pServiceHash)
+	{
+		super(pFable, pOptions, pServiceHash);
+
+		this.serviceType = 'RetoldRemoteParimeCache';
+
+		this._remoteURL = (typeof(this.fable.settings.ParimeCacheServer) === 'string'
+			&& this.fable.settings.ParimeCacheServer.length > 0)
+			? this.fable.settings.ParimeCacheServer.replace(/\/+$/, '')
+			: null;
+	}
+
+	/**
+	 * Whether this adapter is operating in remote mode.
+	 *
+	 * @returns {boolean}
+	 */
+	get isRemote()
+	{
+		return this._remoteURL !== null;
+	}
+
+	/**
+	 * Build a cache key from an array of components.
+	 *
+	 * Components are joined with ':' and hashed with SHA-256.
+	 * Optionally truncate to pHashLength characters.
+	 *
+	 * @param {Array<string|number>} pComponents - Values to include in the key.
+	 * @param {number} [pHashLength] - Truncate the hash to this many hex chars.
+	 * @returns {string} The hex hash.
+	 */
+	buildCacheKey(pComponents, pHashLength)
+	{
+		let tmpInput = pComponents.join(':');
+		let tmpHash = libCrypto.createHash('sha256').update(tmpInput).digest('hex');
+		if (typeof(pHashLength) === 'number' && pHashLength > 0)
+		{
+			return tmpHash.substring(0, pHashLength);
+		}
+		return tmpHash;
+	}
+
+	/**
+	 * Get the underlying ParimeBinaryStorage service.
+	 *
+	 * @returns {object} The ParimeBinaryStorage instance.
+	 */
+	get storage()
+	{
+		return this.fable.ParimeBinaryStorage;
+	}
+
+	/**
+	 * Resolve a category + key to an absolute filesystem path.
+	 *
+	 * Only valid in embedded mode (not remote).
+	 *
+	 * @param {string} pCategory - The cache category.
+	 * @param {string} pKey - The cache key.
+	 * @returns {string} Absolute file path.
+	 */
+	resolvePath(pCategory, pKey)
+	{
+		return this.fable.ParimeBinaryStorage.resolvePath(pCategory, pKey);
+	}
+
+	/**
+	 * Check if a cached item exists.
+	 *
+	 * @param {string} pCategory - The cache category.
+	 * @param {string} pKey - The cache key.
+	 * @param {function} fCallback - Callback(pError, pExists).
+	 */
+	exists(pCategory, pKey, fCallback)
+	{
+		if (this._remoteURL)
+		{
+			return this._remoteExists(pCategory, pKey, fCallback);
+		}
+		return this.fable.ParimeBinaryStorage.exists(pCategory, pKey, fCallback);
+	}
+
+	/**
+	 * Read a cached item.
+	 *
+	 * @param {string} pCategory - The cache category.
+	 * @param {string} pKey - The cache key.
+	 * @param {function} fCallback - Callback(pError, pBuffer).
+	 */
+	read(pCategory, pKey, fCallback)
+	{
+		if (this._remoteURL)
+		{
+			return this._remoteRead(pCategory, pKey, fCallback);
+		}
+		return this.fable.ParimeBinaryStorage.read(pCategory, pKey, fCallback);
+	}
+
+	/**
+	 * Write a cached item.
+	 *
+	 * @param {string} pCategory - The cache category.
+	 * @param {string} pKey - The cache key.
+	 * @param {Buffer} pBuffer - The data to write.
+	 * @param {function} fCallback - Callback(pError).
+	 */
+	write(pCategory, pKey, pBuffer, fCallback)
+	{
+		if (this._remoteURL)
+		{
+			return this._remoteWrite(pCategory, pKey, pBuffer, fCallback);
+		}
+		return this.fable.ParimeBinaryStorage.write(pCategory, pKey, pBuffer, fCallback);
+	}
+
+	/**
+	 * Get a readable stream for a cached item.
+	 *
+	 * Only valid in embedded mode.
+	 *
+	 * @param {string} pCategory - The cache category.
+	 * @param {string} pKey - The cache key.
+	 * @param {object} [pOptions] - Stream options ({ start, end }).
+	 * @returns {ReadStream}
+	 */
+	readStream(pCategory, pKey, pOptions)
+	{
+		return this.fable.ParimeBinaryStorage.readStream(pCategory, pKey, pOptions);
+	}
+
+	/**
+	 * Get file stats for a cached item.
+	 *
+	 * @param {string} pCategory - The cache category.
+	 * @param {string} pKey - The cache key.
+	 * @param {function} fCallback - Callback(pError, pStats).
+	 */
+	stat(pCategory, pKey, fCallback)
+	{
+		if (this._remoteURL)
+		{
+			return this._remoteStat(pCategory, pKey, fCallback);
+		}
+		return this.fable.ParimeBinaryStorage.stat(pCategory, pKey, fCallback);
+	}
+
+	/**
+	 * Delete a cached item.
+	 *
+	 * @param {string} pCategory - The cache category.
+	 * @param {string} pKey - The cache key.
+	 * @param {function} fCallback - Callback(pError).
+	 */
+	delete(pCategory, pKey, fCallback)
+	{
+		if (this._remoteURL)
+		{
+			return this._remoteDelete(pCategory, pKey, fCallback);
+		}
+		return this.fable.ParimeBinaryStorage.delete(pCategory, pKey, fCallback);
+	}
+
+	// ── Remote mode helpers ──────────────────────────────────────
+
+	/**
+	 * @private
+	 */
+	_remoteExists(pCategory, pKey, fCallback)
+	{
+		let tmpURL = `${this._remoteURL}/1.0/Binary/${encodeURIComponent(pCategory)}/${pKey}/Stat`;
+		let libHTTP = require('http');
+		let tmpParsed = new URL(tmpURL);
+
+		let tmpReq = libHTTP.request(
+			{
+				hostname: tmpParsed.hostname,
+				port: tmpParsed.port,
+				path: tmpParsed.pathname,
+				method: 'GET'
+			},
+			(pResponse) =>
+			{
+				let tmpChunks = [];
+				pResponse.on('data', (pChunk) => { tmpChunks.push(pChunk); });
+				pResponse.on('end', () =>
+				{
+					return fCallback(null, pResponse.statusCode === 200);
+				});
+			});
+		tmpReq.on('error', (pError) => { return fCallback(pError); });
+		tmpReq.end();
+	}
+
+	/**
+	 * @private
+	 */
+	_remoteRead(pCategory, pKey, fCallback)
+	{
+		let tmpURL = `${this._remoteURL}/1.0/Binary/${encodeURIComponent(pCategory)}/${pKey}`;
+		let libHTTP = require('http');
+		let tmpParsed = new URL(tmpURL);
+
+		let tmpReq = libHTTP.request(
+			{
+				hostname: tmpParsed.hostname,
+				port: tmpParsed.port,
+				path: tmpParsed.pathname,
+				method: 'GET'
+			},
+			(pResponse) =>
+			{
+				if (pResponse.statusCode === 404)
+				{
+					pResponse.resume();
+					return fCallback(null, null);
+				}
+				let tmpChunks = [];
+				pResponse.on('data', (pChunk) => { tmpChunks.push(pChunk); });
+				pResponse.on('end', () =>
+				{
+					return fCallback(null, Buffer.concat(tmpChunks));
+				});
+			});
+		tmpReq.on('error', (pError) => { return fCallback(pError); });
+		tmpReq.end();
+	}
+
+	/**
+	 * @private
+	 */
+	_remoteWrite(pCategory, pKey, pBuffer, fCallback)
+	{
+		let tmpURL = `${this._remoteURL}/1.0/Binary/${encodeURIComponent(pCategory)}/${pKey}`;
+		let libHTTP = require('http');
+		let tmpParsed = new URL(tmpURL);
+
+		let tmpReq = libHTTP.request(
+			{
+				hostname: tmpParsed.hostname,
+				port: tmpParsed.port,
+				path: tmpParsed.pathname,
+				method: 'PUT',
+				headers: { 'Content-Type': 'application/octet-stream' }
+			},
+			(pResponse) =>
+			{
+				pResponse.resume();
+				pResponse.on('end', () =>
+				{
+					return fCallback(pResponse.statusCode >= 400
+						? new Error(`Remote write failed: ${pResponse.statusCode}`)
+						: null);
+				});
+			});
+		tmpReq.on('error', (pError) => { return fCallback(pError); });
+		tmpReq.write(pBuffer);
+		tmpReq.end();
+	}
+
+	/**
+	 * @private
+	 */
+	_remoteStat(pCategory, pKey, fCallback)
+	{
+		let tmpURL = `${this._remoteURL}/1.0/Binary/${encodeURIComponent(pCategory)}/${pKey}/Stat`;
+		let libHTTP = require('http');
+		let tmpParsed = new URL(tmpURL);
+
+		let tmpReq = libHTTP.request(
+			{
+				hostname: tmpParsed.hostname,
+				port: tmpParsed.port,
+				path: tmpParsed.pathname,
+				method: 'GET'
+			},
+			(pResponse) =>
+			{
+				let tmpChunks = [];
+				pResponse.on('data', (pChunk) => { tmpChunks.push(pChunk); });
+				pResponse.on('end', () =>
+				{
+					if (pResponse.statusCode === 404)
+					{
+						return fCallback(null, null);
+					}
+					try
+					{
+						let tmpBody = JSON.parse(Buffer.concat(tmpChunks).toString());
+						return fCallback(null, { size: tmpBody.Size });
+					}
+					catch (pError)
+					{
+						return fCallback(pError);
+					}
+				});
+			});
+		tmpReq.on('error', (pError) => { return fCallback(pError); });
+		tmpReq.end();
+	}
+
+	/**
+	 * @private
+	 */
+	_remoteDelete(pCategory, pKey, fCallback)
+	{
+		let tmpURL = `${this._remoteURL}/1.0/Binary/${encodeURIComponent(pCategory)}/${pKey}`;
+		let libHTTP = require('http');
+		let tmpParsed = new URL(tmpURL);
+
+		let tmpReq = libHTTP.request(
+			{
+				hostname: tmpParsed.hostname,
+				port: tmpParsed.port,
+				path: tmpParsed.pathname,
+				method: 'DELETE'
+			},
+			(pResponse) =>
+			{
+				pResponse.resume();
+				pResponse.on('end', () =>
+				{
+					return fCallback(null);
+				});
+			});
+		tmpReq.on('error', (pError) => { return fCallback(pError); });
+		tmpReq.end();
+	}
+}
+
+module.exports = RetoldRemoteParimeCache;

package/source/server/RetoldRemote-ThumbnailCache.js

@@ -1,45 +1,67 @@
 /**
  * Retold Remote -- Filesystem Thumbnail Cache
  *
- * Caches generated thumbnails as files in a hidden directory under the
- * content root.  Cache keys are derived from the file path, modification
- * time, and requested dimensions so that stale entries are automatically
- * invalidated when the source file changes.
+ * Caches generated thumbnails using Parime's ParimeBinaryStorage.
+ * Cache keys use a two-level structure: a folder hash derived from
+ * the source file's directory, and a file-specific hash derived from
+ * the filename, modification time, and requested dimensions.
+ *
+ * This means all thumbnails for images in the same folder are
+ * co-located under the same shard directory — browsing a gallery
+ * folder of 10,000 images creates entries in ONE shard directory
+ * rather than 10,000 separate shard paths.
+ *
+ * Stale entries are automatically invalidated when the source file
+ * changes (because mtime is part of the file-specific hash).
  */
 const libFs = require('fs');
 const libPath = require('path');
 const libCrypto = require('crypto');
 
+const _CATEGORY = 'thumbnails';
+
 class ThumbnailCache
 {
 	/**
-	 * @param {string} pCachePath - Absolute path to the cache directory
+	 * @param {object} pFable - The Fable instance (must have ParimeBinaryStorage wired)
 	 */
-	constructor(pCachePath)
+	constructor(pFable)
 	{
-		this._cachePath = pCachePath;
-
-		// Ensure the cache directory exists
-		if (!libFs.existsSync(this._cachePath))
-		{
-			libFs.mkdirSync(this._cachePath, { recursive: true });
-		}
+		this._fable = pFable;
+		this._storage = pFable.ParimeBinaryStorage;
 	}
 
 	/**
 	 * Build a cache key from the source file path, its mtime, and the
 	 * requested thumbnail dimensions.
 	 *
+	 * The key has two parts separated by a slash:
+	 *   {folderHash}/{fileHash}
+	 *
+	 * The folder hash groups all thumbnails from the same directory
+	 * together so they land in the same shard directory on disk.
+	 * The file hash uniquely identifies a particular file at a
+	 * particular mtime and dimension.
+	 *
 	 * @param {string} pFilePath   - Relative path to the source file
 	 * @param {number} pMtime      - Source file mtime (ms since epoch)
 	 * @param {number} pWidth      - Thumbnail width
 	 * @param {number} pHeight     - Thumbnail height
-	 * @returns {string} A hex hash suitable for use as a filename
+	 * @returns {string} A composite key "{folderHash}/{fileHash}"
 	 */
 	buildKey(pFilePath, pMtime, pWidth, pHeight)
 	{
-		let tmpInput = `${pFilePath}:${pMtime}:${pWidth}x${pHeight}`;
-		return libCrypto.createHash('sha256').update(tmpInput).digest('hex');
+		let tmpDir = libPath.dirname(pFilePath);
+		let tmpFile = libPath.basename(pFilePath);
+
+		// Folder hash — first 16 hex chars; used for sharding and co-location
+		let tmpFolderHash = libCrypto.createHash('sha256').update(tmpDir).digest('hex').substring(0, 16);
+
+		// File-specific hash — full 64 hex chars; unique per file + dimensions
+		let tmpFileInput = `${tmpFile}:${pMtime}:${pWidth}x${pHeight}`;
+		let tmpFileHash = libCrypto.createHash('sha256').update(tmpFileInput).digest('hex');
+
+		return `${tmpFolderHash}/${tmpFileHash}`;
 	}
 
 	/**
@@ -52,7 +74,8 @@ class ThumbnailCache
 	 */
 	get(pKey, pFormat)
 	{
-		let tmpPath = libPath.join(this._cachePath, `${pKey}.${pFormat || 'webp'}`);
+		let tmpFileKey = `${pKey}.${pFormat || 'webp'}`;
+		let tmpPath = this._storage.resolvePath(_CATEGORY, tmpFileKey);
 		if (libFs.existsSync(tmpPath))
 		{
 			return tmpPath;
@@ -70,19 +93,28 @@ class ThumbnailCache
 	 */
 	put(pKey, pBuffer, pFormat)
 	{
-		let tmpPath = libPath.join(this._cachePath, `${pKey}.${pFormat || 'webp'}`);
+		let tmpFileKey = `${pKey}.${pFormat || 'webp'}`;
+		let tmpPath = this._storage.resolvePath(_CATEGORY, tmpFileKey);
+
+		// Ensure parent directory exists (for sharded paths)
+		let tmpDir = libPath.dirname(tmpPath);
+		if (!libFs.existsSync(tmpDir))
+		{
+			libFs.mkdirSync(tmpDir, { recursive: true });
+		}
+
 		libFs.writeFileSync(tmpPath, pBuffer);
 		return tmpPath;
 	}
 
 	/**
-	 * Get the absolute path to the cache directory.
+	 * Get the cache category name.
 	 *
 	 * @returns {string}
 	 */
 	getCachePath()
 	{
-		return this._cachePath;
+		return this._storage.resolvePath(_CATEGORY, '');
 	}
 }
 

package/source/server/RetoldRemote-VideoFrameService.js

@@ -47,16 +47,7 @@ class RetoldRemoteVideoFrameService extends libFableServiceProviderBase
 
 		this.contentPath = libPath.resolve(this.options.ContentPath);
 
-		this.cachePath = this.options.CachePath
-			|| libPath.join(process.cwd(), 'dist', 'retold-cache', 'video-frames');
-
-		// Ensure cache directory exists
-		if (!libFs.existsSync(this.cachePath))
-		{
-			libFs.mkdirSync(this.cachePath, { recursive: true });
-		}
-
-		this.fable.log.info(`Video Frame Service: cache at ${this.cachePath}`);
+		this.fable.log.info('Video Frame Service: using ParimeBinaryStorage (category: video-frames)');
 	}
 
 	/**
@@ -75,7 +66,7 @@ class RetoldRemoteVideoFrameService extends libFableServiceProviderBase
 	{
 		let tmpInput = `${pAbsPath}:${pMtimeMs}:${pFrameCount}:${pWidth}x${pHeight}`;
 		let tmpHash = libCrypto.createHash('sha256').update(tmpInput).digest('hex').substring(0, 16);
-		return libPath.join(this.cachePath, tmpHash);
+		return this.fable.ParimeBinaryStorage.resolvePath('video-frames', tmpHash);
 	}
 
 	/**
@@ -398,7 +389,7 @@ class RetoldRemoteVideoFrameService extends libFableServiceProviderBase
 			return fCallback(new Error('Invalid cache key.'));
 		}
 
-		let tmpCacheDir = libPath.join(this.cachePath, pCacheKey);
+		let tmpCacheDir = this.fable.ParimeBinaryStorage.resolvePath('video-frames', pCacheKey);
 		if (!libFs.existsSync(tmpCacheDir))
 		{
 			return fCallback(new Error('Cache directory not found. Extract frames first.'));
@@ -466,11 +457,12 @@ class RetoldRemoteVideoFrameService extends libFableServiceProviderBase
 			return null;
 		}
 
-		let tmpPath = libPath.join(this.cachePath, pCacheKey, pFilename);
+		let tmpCacheDir = this.fable.ParimeBinaryStorage.resolvePath('video-frames', pCacheKey);
+		let tmpPath = libPath.join(tmpCacheDir, pFilename);
 
-		// Double-check it's under our cache dir
+		// Double-check it's under the storage root
 		let tmpResolved = libPath.resolve(tmpPath);
-		if (!tmpResolved.startsWith(this.cachePath))
+		if (!tmpResolved.startsWith(this.fable.ParimeBinaryStorage.storageRoot))
 		{
 			return null;
 		}