synthos 0.9.0 → 0.10.1

package/src/storage/FsStorageProvider.ts

@@ -0,0 +1,87 @@
+import * as fs from 'fs/promises';
+import { createReadStream } from 'fs';
+import path from 'path';
+import { StorageProvider } from './StorageProvider';
+
+/**
+ * Default StorageProvider backed by the local filesystem via `fs/promises`.
+ * Mirrors the existing logic in `files.ts`.
+ */
+export class FsStorageProvider implements StorageProvider {
+    // --- Text file operations ---
+
+    async checkIfExists(filePath: string): Promise<boolean> {
+        try {
+            await fs.access(filePath);
+            return true;
+        } catch {
+            return false;
+        }
+    }
+
+    async loadFile(filePath: string): Promise<string> {
+        return await fs.readFile(filePath, 'utf8');
+    }
+
+    async saveFile(filePath: string, content: string): Promise<void> {
+        await fs.writeFile(filePath, content, 'utf8');
+    }
+
+    async deleteFile(filePath: string): Promise<void> {
+        await fs.unlink(filePath);
+    }
+
+    // --- Binary file operations ---
+
+    async saveBuffer(filePath: string, data: Buffer): Promise<void> {
+        await fs.writeFile(filePath, data);
+    }
+
+    async loadBuffer(filePath: string): Promise<Buffer> {
+        return await fs.readFile(filePath);
+    }
+
+    async stat(filePath: string): Promise<{ size: number; isFile: boolean }> {
+        const s = await fs.stat(filePath);
+        return { size: s.size, isFile: s.isFile() };
+    }
+
+    createReadStream(filePath: string): NodeJS.ReadableStream {
+        return createReadStream(filePath);
+    }
+
+    // --- Directory operations ---
+
+    async listFiles(dirPath: string): Promise<string[]> {
+        return (await fs.readdir(dirPath)).filter(file => !file.startsWith('.') && file.includes('.'));
+    }
+
+    async listFolders(dirPath: string): Promise<string[]> {
+        const entries = await fs.readdir(dirPath, { withFileTypes: true });
+        return entries
+            .filter(entry => entry.isDirectory() && !entry.name.startsWith('.'))
+            .map(entry => entry.name);
+    }
+
+    async ensureFolderExists(dirPath: string): Promise<void> {
+        await fs.mkdir(dirPath, { recursive: true });
+    }
+
+    async deleteFolder(dirPath: string): Promise<void> {
+        await fs.rm(dirPath, { recursive: true });
+    }
+
+    async copyFolderRecursive(srcPath: string, destPath: string): Promise<void> {
+        await this.ensureFolderExists(destPath);
+        const entries = await fs.readdir(srcPath, { withFileTypes: true });
+        for (const entry of entries) {
+            const src = path.join(srcPath, entry.name);
+            const dest = path.join(destPath, entry.name);
+            if (entry.isDirectory()) {
+                await this.copyFolderRecursive(src, dest);
+            } else {
+                await fs.copyFile(src, dest);
+            }
+        }
+    }
+}

package/src/storage/StorageProvider.ts

@@ -0,0 +1,34 @@
+/**
+ * Abstraction over user-mutable storage (pages, settings, themes, data, uploads).
+ *
+ * All paths are absolute. Callers build paths with `path.join(config.pagesFolder, ...)`
+ * exactly as they do today — the provider simply executes the I/O against
+ * whatever backend it wraps (local filesystem, blob storage, etc.).
+ *
+ * Package/read-only content (required-pages, default-themes, static-files,
+ * service-connectors) stays on local filesystem and is NOT routed through
+ * this interface.
+ */
+export interface StorageProvider {
+    // --- Text file operations ---
+
+    checkIfExists(filePath: string): Promise<boolean>;
+    loadFile(filePath: string): Promise<string>;
+    saveFile(filePath: string, content: string): Promise<void>;
+    deleteFile(filePath: string): Promise<void>;
+
+    // --- Binary file operations (uploads / images) ---
+
+    saveBuffer(filePath: string, data: Buffer): Promise<void>;
+    loadBuffer(filePath: string): Promise<Buffer>;
+    stat(filePath: string): Promise<{ size: number; isFile: boolean }>;
+    createReadStream(filePath: string): NodeJS.ReadableStream;
+
+    // --- Directory operations ---
+
+    listFiles(dirPath: string): Promise<string[]>;
+    listFolders(dirPath: string): Promise<string[]>;
+    ensureFolderExists(dirPath: string): Promise<void>;
+    deleteFolder(dirPath: string): Promise<void>;
+    copyFolderRecursive(srcPath: string, destPath: string): Promise<void>;
+}

package/src/storage/index.ts

@@ -0,0 +1,2 @@
+export { StorageProvider } from './StorageProvider';
+export { FsStorageProvider } from './FsStorageProvider';

package/src/themes.ts

@@ -28,10 +28,29 @@ export function parseThemeFilename(filename: string): { name: string; version: n
 }
 
 /**
- * Find the CSS file for a theme by name in a folder.
+ * Find the CSS file for a theme by name in a user folder (via storageProvider).
  * Prefers the highest-versioned file (e.g. name.v2.css over name.css).
  */
-async function findThemeCssFile(folder: string, name: string): Promise<{ path: string; version: number } | undefined> {
+async function findUserThemeCssFile(config: SynthOSConfig, folder: string, name: string): Promise<{ path: string; version: number } | undefined> {
+    const sp = config.storageProvider;
+    if (!await sp.checkIfExists(folder)) return undefined;
+    const files = await sp.listFiles(folder);
+    let best: { path: string; version: number } | undefined;
+    for (const f of files) {
+        const parsed = parseThemeFilename(f);
+        if (parsed && parsed.name === name) {
+            if (!best || parsed.version > best.version) {
+                best = { path: path.join(folder, f), version: parsed.version };
+            }
+        }
+    }
+    return best;
+}
+
+/**
+ * Find the CSS file for a theme by name in a local-fs folder (package defaults).
+ */
+async function findDefaultThemeCssFile(folder: string, name: string): Promise<{ path: string; version: number } | undefined> {
     if (!await checkIfExists(folder)) return undefined;
     const files = await listFiles(folder);
     let best: { path: string; version: number } | undefined;
@@ -47,23 +66,26 @@ async function findThemeCssFile(folder: string, name: string): Promise<{ path: s
 }
 
 export async function loadThemeVersion(name: string, config: SynthOSConfig): Promise<number> {
-    const local = await findThemeCssFile(userThemesFolder(config), name);
+    const local = await findUserThemeCssFile(config, userThemesFolder(config), name);
     if (local) return local.version;
     for (const folder of config.defaultThemesFolders) {
-        const def = await findThemeCssFile(folder, name);
+        const def = await findDefaultThemeCssFile(folder, name);
         if (def) return def.version;
     }
     return 1;
 }
 
 export async function loadThemeInfo(name: string, config: SynthOSConfig): Promise<ThemeInfo | undefined> {
-    // Check user's local themes first, then fall back to package defaults
+    const sp = config.storageProvider;
+
+    // Check user's local themes first (user storage)
     const localPath = path.join(userThemesFolder(config), `${name}.json`);
-    if (await checkIfExists(localPath)) {
-        const raw = await loadFile(localPath);
+    if (await sp.checkIfExists(localPath)) {
+        const raw = await sp.loadFile(localPath);
         return raw ? JSON.parse(raw) : undefined;
     }
 
+    // Fall back to package defaults (local fs)
     const defaultPath = await findFileInFolders(config.defaultThemesFolders, `${name}.json`);
     if (defaultPath) {
         const raw = await loadFile(defaultPath);
@@ -74,15 +96,17 @@ export async function loadThemeInfo(name: string, config: SynthOSConfig): Promis
 }
 
 export async function loadTheme(name: string, config: SynthOSConfig): Promise<string | undefined> {
-    // Check user's local themes first, then fall back to package defaults
-    const local = await findThemeCssFile(userThemesFolder(config), name);
+    const sp = config.storageProvider;
+
+    // Check user's local themes first (user storage)
+    const local = await findUserThemeCssFile(config, userThemesFolder(config), name);
     if (local) {
-        return await loadFile(local.path);
+        return await sp.loadFile(local.path);
     }
 
-    // Search all default theme folders
+    // Search all default theme folders (local fs)
     for (const folder of config.defaultThemesFolders) {
-        const def = await findThemeCssFile(folder, name);
+        const def = await findDefaultThemeCssFile(folder, name);
         if (def) {
             return await loadFile(def.path);
         }
@@ -92,19 +116,20 @@ export async function loadTheme(name: string, config: SynthOSConfig): Promise<st
 }
 
 export async function listThemes(config: SynthOSConfig): Promise<string[]> {
+    const sp = config.storageProvider;
     const names = new Set<string>();
 
-    // Collect from user's local themes folder
+    // Collect from user's local themes folder (user storage)
     const localFolder = userThemesFolder(config);
-    if (await checkIfExists(localFolder)) {
-        const files = await listFiles(localFolder);
+    if (await sp.checkIfExists(localFolder)) {
+        const files = await sp.listFiles(localFolder);
         for (const f of files) {
             const parsed = parseThemeFilename(f);
             if (parsed) names.add(parsed.name);
         }
     }
 
-    // Collect from all default theme folders
+    // Collect from all default theme folders (local fs)
     const defaultFiles = await listFilesFromFolders(config.defaultThemesFolders);
     for (const f of defaultFiles) {
         const parsed = parseThemeFilename(f);
@@ -118,11 +143,12 @@ export async function listThemes(config: SynthOSConfig): Promise<string[]> {
  * Compare local theme versions against defaults and return themes that need upgrading.
  */
 export async function getOutdatedThemes(config: SynthOSConfig): Promise<string[]> {
+    const sp = config.storageProvider;
     const localFolder = userThemesFolder(config);
-    if (!await checkIfExists(localFolder)) return [];
+    if (!await sp.checkIfExists(localFolder)) return [];
 
     const defaultFiles = await listFilesFromFolders(config.defaultThemesFolders);
-    const localFiles = await listFiles(localFolder);
+    const localFiles = await sp.listFiles(localFolder);
 
     // Build maps: theme name → highest version
     const defaultVersions = new Map<string, number>();

package/tests/builders.spec.ts

@@ -46,7 +46,7 @@ describe('Anthropic Builder — Haiku Classifier Routing', () => {
 
             const builder = createAnthropicBuilder(tracker.fn, undefined, 'SynthOS', {
                 apiKey: 'test-key',
-                model: 'claude-sonnet-4-5-20250514',
+                model: 'claude-sonnet-4-6-20250514',
             });
 
             const result = await builder.run(currentPage, additionalSections, 'change heading', false);
@@ -113,7 +113,7 @@ describe('Anthropic Builder — Haiku Classifier Routing', () => {
             // Non-Opus model should just work without classification
             const builder = createBuilder('Anthropic', tracker.fn, undefined, 'SynthOS', {
                 apiKey: 'key',
-                model: 'claude-sonnet-4-5-20250514',
+                model: 'claude-sonnet-4-6-20250514',
             });
 
             const result = await builder.run(currentPage, additionalSections, 'test', false);

package/tests/pages.spec.ts

@@ -9,12 +9,31 @@ import {
     loadPageMetadata,
     savePageState,
     loadPageState,
-    updatePageState,
     listPages,
     deletePage,
     copyPage,
     PAGE_VERSION,
 } from '../src/pages';
+import { SynthOSConfig } from '../src/init';
+import { FsStorageProvider } from '../src/storage';
+
+/** Create a minimal SynthOSConfig for testing. */
+function makeConfig(pagesFolder: string): SynthOSConfig {
+    return {
+        localFolder: '.synthos',
+        pagesFolder,
+        requiredPagesFolders: [],
+        defaultPagesFolders: [],
+        defaultScriptsFolders: [],
+        defaultThemesFolders: [],
+        staticFilesFolders: [],
+        serviceConnectorsFolders: [],
+        requiredPages: [],
+        storageProvider: new FsStorageProvider(),
+        debug: false,
+        debugPageUpdates: false,
+    };
+}
 
 // ---------------------------------------------------------------------------
 // normalizePageName
@@ -123,15 +142,16 @@ describe('parseMetadata', () => {
 
 describe('page I/O', () => {
     let tmpDir: string;
-    let pagesFolder: string;
+    let config: SynthOSConfig;
     let fallbackFolder: string;
 
     beforeEach(async () => {
         tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'synthos-test-'));
-        pagesFolder = path.join(tmpDir, 'synthos');
+        const pagesFolder = path.join(tmpDir, 'synthos');
         fallbackFolder = path.join(tmpDir, 'fallback');
         await fs.mkdir(pagesFolder, { recursive: true });
         await fs.mkdir(fallbackFolder, { recursive: true });
+        config = makeConfig(pagesFolder);
     });
 
     afterEach(async () => {
@@ -152,8 +172,8 @@ describe('page I/O', () => {
                 pageVersion: 2,
                 mode: 'locked' as const,
             };
-            await savePageMetadata(pagesFolder, 'mypage', meta);
-            const loaded = await loadPageMetadata(pagesFolder, 'mypage');
+            await savePageMetadata(config, 'mypage', meta);
+            const loaded = await loadPageMetadata(config, 'mypage');
             assert.ok(loaded);
             assert.strictEqual(loaded.title, 'Test Page');
             assert.deepStrictEqual(loaded.categories, ['Tools', 'Dev']);
@@ -163,17 +183,19 @@ describe('page I/O', () => {
         });
 
         it('returns undefined when no metadata file exists', async () => {
-            const loaded = await loadPageMetadata(pagesFolder, 'nonexistent');
+            const loaded = await loadPageMetadata(config, 'nonexistent');
             assert.strictEqual(loaded, undefined);
         });
 
         it('falls back to fallbackFolder', async () => {
-            // Write a fallback JSON file
+            // Write a fallback JSON file in folder-based structure
+            const fallbackPageDir = path.join(fallbackFolder, 'system-page');
+            await fs.mkdir(fallbackPageDir, { recursive: true });
             await fs.writeFile(
-                path.join(fallbackFolder, 'system-page.json'),
+                path.join(fallbackPageDir, 'page.json'),
                 JSON.stringify({ title: 'System', categories: ['System'], pinned: true, pageVersion: 2, mode: 'locked' }),
             );
-            const loaded = await loadPageMetadata(pagesFolder, 'system-page', [fallbackFolder]);
+            const loaded = await loadPageMetadata(config, 'system-page', [fallbackFolder]);
             assert.ok(loaded);
             assert.strictEqual(loaded.title, 'System');
         });
@@ -184,89 +206,46 @@ describe('page I/O', () => {
     describe('savePageState / loadPageState', () => {
         it('roundtrips page HTML and creates metadata', async () => {
             const html = '<html><body>Hello</body></html>';
-            await savePageState(pagesFolder, 'testpage', html, 'Test Title', ['Cat']);
-            const loaded = await loadPageState(pagesFolder, 'testpage', true);
+            await savePageState(config, 'testpage', html, 'Test Title', ['Cat']);
+            const loaded = await loadPageState(config, 'testpage');
             assert.strictEqual(loaded, html);
 
             // Metadata should have been created
-            const meta = await loadPageMetadata(pagesFolder, 'testpage');
+            const meta = await loadPageMetadata(config, 'testpage');
             assert.ok(meta);
             assert.strictEqual(meta.pageVersion, PAGE_VERSION);
         });
 
         it('returns undefined for non-existent page', async () => {
-            const loaded = await loadPageState(pagesFolder, 'nope', false);
+            const loaded = await loadPageState(config, 'nope');
             assert.strictEqual(loaded, undefined);
         });
-
-        it('with reset=true re-reads from disk', async () => {
-            const html1 = '<html>v1</html>';
-            const html2 = '<html>v2</html>';
-            await savePageState(pagesFolder, 'resetpage', html1);
-            // First load
-            const first = await loadPageState(pagesFolder, 'resetpage', false);
-            assert.strictEqual(first, html1);
-
-            // Overwrite the file on disk directly
-            const pageHtmlPath = path.join(pagesFolder, 'pages', 'resetpage', 'page.html');
-            await fs.writeFile(pageHtmlPath, html2);
-
-            // Without reset, should return cached version
-            const cached = await loadPageState(pagesFolder, 'resetpage', false);
-            assert.strictEqual(cached, html1);
-
-            // With reset, should re-read
-            const refreshed = await loadPageState(pagesFolder, 'resetpage', true);
-            assert.strictEqual(refreshed, html2);
-        });
-    });
-
-    // -- updatePageState -----------------------------------------------------
-
-    describe('updatePageState', () => {
-        it('updates in-memory cache', async () => {
-            await savePageState(pagesFolder, 'cachepage', '<html>original</html>');
-            await loadPageState(pagesFolder, 'cachepage', false);
-
-            updatePageState('cachepage', '<html>updated</html>');
-
-            // Loading without reset should return updated cache
-            const loaded = await loadPageState(pagesFolder, 'cachepage', false);
-            assert.strictEqual(loaded, '<html>updated</html>');
-        });
     });
 
     // -- listPages -----------------------------------------------------------
 
     describe('listPages', () => {
         it('lists folder-based pages', async () => {
-            await savePageState(pagesFolder, 'alpha', '<html>A</html>');
-            await savePageState(pagesFolder, 'beta', '<html>B</html>');
-            const pages = await listPages(pagesFolder, [fallbackFolder]);
+            await savePageState(config, 'alpha', '<html>A</html>');
+            await savePageState(config, 'beta', '<html>B</html>');
+            const pages = await listPages(config, [fallbackFolder]);
             const names = pages.map(p => p.name);
             assert.ok(names.includes('alpha'));
             assert.ok(names.includes('beta'));
         });
 
         it('lists legacy flat HTML files', async () => {
-            await fs.writeFile(path.join(pagesFolder, 'legacy.html'), '<html>Legacy</html>');
-            const pages = await listPages(pagesFolder, [fallbackFolder]);
+            await fs.writeFile(path.join(config.pagesFolder, 'legacy.html'), '<html>Legacy</html>');
+            const pages = await listPages(config, [fallbackFolder]);
             const legacy = pages.find(p => p.name === 'legacy');
             assert.ok(legacy);
             assert.strictEqual(legacy.pageVersion, 1);
         });
 
-        it('lists fallback (required) pages', async () => {
-            await fs.writeFile(path.join(fallbackFolder, 'builder.html'), '<html>Builder</html>');
-            const pages = await listPages(pagesFolder, [fallbackFolder]);
-            const builder = pages.find(p => p.name === 'builder');
-            assert.ok(builder);
-        });
-
         it('returns pages sorted alphabetically', async () => {
-            await savePageState(pagesFolder, 'zebra', '<html></html>');
-            await savePageState(pagesFolder, 'apple', '<html></html>');
-            const pages = await listPages(pagesFolder, [fallbackFolder]);
+            await savePageState(config, 'zebra', '<html></html>');
+            await savePageState(config, 'apple', '<html></html>');
+            const pages = await listPages(config, [fallbackFolder]);
             const names = pages.map(p => p.name);
             const sortedNames = [...names].sort();
             assert.deepStrictEqual(names, sortedNames);
@@ -276,19 +255,18 @@ describe('page I/O', () => {
     // -- deletePage ----------------------------------------------------------
 
     describe('deletePage', () => {
-        it('removes folder-based page and clears cache', async () => {
-            await savePageState(pagesFolder, 'doomed', '<html>bye</html>');
-            await loadPageState(pagesFolder, 'doomed', false);
+        it('removes folder-based page', async () => {
+            await savePageState(config, 'doomed', '<html>bye</html>');
 
-            await deletePage(pagesFolder, 'doomed');
+            await deletePage(config, 'doomed');
 
             // Folder should be gone
-            const folderExists = await fs.access(path.join(pagesFolder, 'pages', 'doomed'))
+            const folderExists = await fs.access(path.join(config.pagesFolder, 'pages', 'doomed'))
                 .then(() => true).catch(() => false);
             assert.strictEqual(folderExists, false);
 
-            // Cache should be cleared — loading should return undefined
-            const loaded = await loadPageState(pagesFolder, 'doomed', true);
+            // Loading should return undefined
+            const loaded = await loadPageState(config, 'doomed');
             assert.strictEqual(loaded, undefined);
         });
     });
@@ -297,13 +275,13 @@ describe('page I/O', () => {
 
     describe('copyPage', () => {
         it('copies HTML and creates metadata with correct title/categories', async () => {
-            await savePageState(pagesFolder, 'source', '<html>Source</html>');
-            await copyPage(pagesFolder, 'source', 'target', 'Copied Page', ['Copy'], [fallbackFolder]);
+            await savePageState(config, 'source', '<html>Source</html>');
+            await copyPage(config, 'source', 'target', 'Copied Page', ['Copy'], [fallbackFolder]);
 
-            const html = await loadPageState(pagesFolder, 'target', true);
+            const html = await loadPageState(config, 'target');
             assert.strictEqual(html, '<html>Source</html>');
 
-            const meta = await loadPageMetadata(pagesFolder, 'target');
+            const meta = await loadPageMetadata(config, 'target');
             assert.ok(meta);
             assert.strictEqual(meta.title, 'Copied Page');
             assert.deepStrictEqual(meta.categories, ['Copy']);
@@ -312,17 +290,9 @@ describe('page I/O', () => {
 
         it('throws when source page does not exist', async () => {
             await assert.rejects(
-                () => copyPage(pagesFolder, 'ghost', 'target', 'T', [], [fallbackFolder]),
+                () => copyPage(config, 'ghost', 'target', 'T', [], [fallbackFolder]),
                 /Source page "ghost" not found/,
             );
         });
-
-        it('copies from required pages folder as fallback', async () => {
-            await fs.writeFile(path.join(fallbackFolder, 'system.html'), '<html>System</html>');
-            await copyPage(pagesFolder, 'system', 'mycopy', 'My Copy', ['User'], [fallbackFolder]);
-
-            const html = await loadPageState(pagesFolder, 'mycopy', true);
-            assert.strictEqual(html, '<html>System</html>');
-        });
     });
 });

package/tests/providers.spec.ts

@@ -39,7 +39,7 @@ describe('getProvider', () => {
 
 describe('detectProvider', () => {
     it('detects Anthropic from claude- prefix', () => {
-        const p = detectProvider('claude-sonnet-4-5');
+        const p = detectProvider('claude-sonnet-4-6');
         assert.strictEqual(p?.name, 'Anthropic');
     });