synthos 0.5.0 → 0.7.0

package/src/pages.ts

@@ -1,41 +1,174 @@
-import {checkIfExists, listFiles, loadFile, saveFile} from './files';
+import {checkIfExists, deleteFile, deleteFolder, ensureFolderExists, listFiles, listFolders, loadFile, saveFile} from './files';
 import path from 'path';
 
 // Page State Cache
 const _pages: { [name: string]: string } = {};
 
-export async function listPages(pagesFolder: string, fallbackPagesFolder: string): Promise<string[]> {
-    // Load all pages from primary pages folder
-    const all = (await listFiles(pagesFolder)).filter(file => file.endsWith('.html'));
+export const REQUIRED_PAGES = ['builder', 'pages', 'settings', 'apis', 'scripts'];
 
-    // Add pages from fallback pages folder that don't already exist
-    (await listFiles(fallbackPagesFolder)).forEach(file => {
-        if (file.endsWith('.html') && !all.includes(file)) {
-            all.push(file);
+export const PAGE_VERSION = 2;
+
+export interface PageInfo {
+    name: string;
+    title: string;
+    categories: string[];
+    pinned: boolean;
+    showInAll: boolean;    // true = visible in "All" filter, false = only in category filters
+    createdDate: string;   // ISO 8601, empty string if unknown
+    lastModified: string;  // ISO 8601, empty string if unknown
+    pageVersion: number;   // integer, 0 = pre-versioning
+    mode: 'unlocked' | 'locked';
+}
+
+export type PageMetadata = Omit<PageInfo, 'name'>;
+
+export async function loadPageMetadata(pagesFolder: string, name: string, fallbackFolder?: string): Promise<PageMetadata | undefined> {
+    // 1. Try user override: .synthos/pages/<name>/page.json
+    const metadataPath = path.join(pagesFolder, 'pages', name, 'page.json');
+    if (await checkIfExists(metadataPath)) {
+        try {
+            const raw = await loadFile(metadataPath);
+            const parsed = JSON.parse(raw);
+            return parseMetadata(parsed);
+        } catch {
+            // fall through
+        }
+    }
+
+    // 2. Try fallback: fallbackFolder/<name>.json
+    if (fallbackFolder) {
+        const fallbackPath = path.join(fallbackFolder, `${name}.json`);
+        if (await checkIfExists(fallbackPath)) {
+            try {
+                const raw = await loadFile(fallbackPath);
+                const parsed = JSON.parse(raw);
+                return parseMetadata(parsed);
+            } catch {
+                // fall through
+            }
+        }
+    }
+
+    return undefined;
+}
+
+export function parseMetadata(parsed: Record<string, unknown>): PageMetadata {
+    return {
+        title: typeof parsed.title === 'string' ? parsed.title : '',
+        categories: Array.isArray(parsed.categories) ? parsed.categories : [],
+        pinned: typeof parsed.pinned === 'boolean' ? parsed.pinned : false,
+        showInAll: typeof parsed.showInAll === 'boolean' ? parsed.showInAll : true,
+        createdDate: typeof parsed.createdDate === 'string' ? parsed.createdDate : '',
+        lastModified: typeof parsed.lastModified === 'string' ? parsed.lastModified : '',
+        pageVersion: typeof parsed.pageVersion === 'number' ? parsed.pageVersion
+            : typeof parsed.uxVersion === 'number' ? parsed.uxVersion : 0,
+        mode: parsed.mode === 'locked' ? 'locked' : 'unlocked',
+    };
+}
+
+export async function savePageMetadata(pagesFolder: string, name: string, metadata: PageMetadata): Promise<void> {
+    const pageFolder = path.join(pagesFolder, 'pages', name);
+    await ensureFolderExists(pageFolder);
+    await saveFile(path.join(pageFolder, 'page.json'), JSON.stringify(metadata, null, 4));
+}
+
+const DEFAULT_METADATA: PageMetadata = {
+    title: '',
+    categories: [],
+    pinned: false,
+    showInAll: true,
+    createdDate: '',
+    lastModified: '',
+    pageVersion: 0,
+    mode: 'unlocked',
+};
+
+export async function listPages(pagesFolder: string, fallbackPagesFolder: string): Promise<PageInfo[]> {
+    const pageMap = new Map<string, PageInfo>();
+
+    // Folder-based pages under pages/ subdirectory
+    const pagesSubdir = path.join(pagesFolder, 'pages');
+    if (await checkIfExists(pagesSubdir)) {
+        const folders = await listFolders(pagesSubdir);
+        for (const folder of folders) {
+            if (await checkIfExists(path.join(pagesSubdir, folder, 'page.html'))) {
+                const metadata = await loadPageMetadata(pagesFolder, folder);
+                pageMap.set(folder, {
+                    name: folder,
+                    ...(metadata ?? DEFAULT_METADATA),
+                });
+            }
+        }
+    }
+
+    // Legacy flat .html files in root (v1 pages)
+    const flatFiles = (await listFiles(pagesFolder)).filter(file => file.endsWith('.html'));
+    for (const file of flatFiles) {
+        const name = file.replace(/\.html$/, '');
+        if (!pageMap.has(name)) {
+            // Derive title: strip brackets, replace underscores with spaces, title-case
+            const stripped = name.replace(/^\[/, '').replace(/\]$/, '');
+            const title = stripped
+                .replace(/_/g, ' ')
+                .replace(/\b\w/g, c => c.toUpperCase());
+            // Assign category: bracketed names → "Builder", otherwise → "Page"
+            const hasBrackets = name.startsWith('[') && name.endsWith(']');
+            const categories = hasBrackets ? ['Builder'] : ['Page'];
+            pageMap.set(name, {
+                name,
+                title,
+                categories,
+                pinned: false,
+                showInAll: true,
+                createdDate: '',
+                lastModified: '',
+                pageVersion: 1,
+                mode: 'unlocked',
+            });
+        }
+    }
+
+    // Add pages from fallback (required) pages folder
+    const fallbackFiles = (await listFiles(fallbackPagesFolder)).filter(file => file.endsWith('.html'));
+    for (const file of fallbackFiles) {
+        const name = file.replace(/\.html$/, '');
+        if (!pageMap.has(name)) {
+            // System page not yet in map — check for user override, then fallback .json
+            const metadata = await loadPageMetadata(pagesFolder, name, fallbackPagesFolder);
+            pageMap.set(name, {
+                name,
+                title: metadata?.title ?? '',
+                categories: metadata?.categories ?? ['System'],
+                pinned: metadata?.pinned ?? true,
+                showInAll: metadata?.showInAll ?? true,
+                createdDate: metadata?.createdDate ?? '',
+                lastModified: metadata?.lastModified ?? '',
+                pageVersion: metadata?.pageVersion ?? 0,
+                mode: metadata?.mode ?? 'unlocked',
+            });
         }
-    });
+    }
 
-    // Remove .html and sort all files
-    const sorted = all.map(file => file.substring(0, file.length - 5)).sort();
-    
-    // Move [templates] to end of array
-    const pages = sorted.filter(page => !page.startsWith('['));
-    const templates = sorted.filter(page => page.startsWith('['));
-    pages.push(...templates);
+    // Sort alphabetically
+    const entries = Array.from(pageMap.values());
+    entries.sort((a, b) => a.name.localeCompare(b.name));
 
-    return pages;
+    return entries;
 }
 
 export async function loadPageState(pagesFolder: string, name: string, reset: boolean): Promise<string|undefined> {
     if (!_pages[name] || reset) {
-        // Check for file to exist
-        const filename = path.join(pagesFolder, `${name}.html`);
-        if (!(await checkIfExists(filename))) {
+        // Try folder-based path under pages/ first, then fall back to flat file
+        const folderPath = path.join(pagesFolder, 'pages', name, 'page.html');
+        const flatPath = path.join(pagesFolder, `${name}.html`);
+
+        if (await checkIfExists(folderPath)) {
+            _pages[name] = await loadFile(folderPath);
+        } else if (await checkIfExists(flatPath)) {
+            _pages[name] = await loadFile(flatPath);
+        } else {
             return undefined;
         }
-
-        // Load file
-        _pages[name] = await loadFile(filename);
     }
 
     return _pages[name];
@@ -45,11 +178,86 @@ export function normalizePageName(name: string|undefined): string|undefined {
     return typeof name == 'string' && name.length > 0 ? name.replace(/[^a-z0-9\-_\[\]\(\)\{\}@#\$%&]/gi, '_').toLowerCase() : undefined;
 }
 
-export async function savePageState(pagesFolder: string, name: string, content: string): Promise<void> {
+export async function savePageState(pagesFolder: string, name: string, content: string, title?: string, categories?: string[]): Promise<void> {
     _pages[name] = content;
-    await saveFile(path.join(pagesFolder, `${name}.html`), content);
+    const pageFolder = path.join(pagesFolder, 'pages', name);
+    await ensureFolderExists(pageFolder);
+    await saveFile(path.join(pageFolder, 'page.html'), content);
+
+    // Create page.json with full metadata if it doesn't exist
+    const metadataPath = path.join(pageFolder, 'page.json');
+    if (!(await checkIfExists(metadataPath))) {
+        const now = new Date().toISOString();
+        const metadata: PageMetadata = {
+            title: title ?? '',
+            categories: categories ?? [],
+            pinned: false,
+            showInAll: true,
+            createdDate: now,
+            lastModified: now,
+            pageVersion: PAGE_VERSION,
+            mode: 'unlocked',
+        };
+        await saveFile(metadataPath, JSON.stringify(metadata, null, 4));
+    }
 }
 
 export function updatePageState(name: string, content: string): void {
     _pages[name] = content;
-}
+}
+
+export async function deletePage(pagesFolder: string, name: string): Promise<void> {
+    // Delete folder-based page: <pagesFolder>/pages/<name>/
+    const folderPath = path.join(pagesFolder, 'pages', name);
+    if (await checkIfExists(folderPath)) {
+        await deleteFolder(folderPath);
+    }
+
+    // Delete legacy flat file: <pagesFolder>/<name>.html
+    const flatPath = path.join(pagesFolder, `${name}.html`);
+    if (await checkIfExists(flatPath)) {
+        await deleteFile(flatPath);
+    }
+
+    // Clear in-memory cache
+    delete _pages[name];
+}
+
+export async function copyPage(
+    pagesFolder: string,
+    sourceName: string,
+    targetName: string,
+    title: string,
+    categories: string[],
+    requiredPagesFolder: string
+): Promise<void> {
+    // Load source HTML from user folder, then try required folder as fallback
+    let html = await loadPageState(pagesFolder, sourceName, true);
+    if (!html) {
+        const requiredPath = path.join(requiredPagesFolder, `${sourceName}.html`);
+        if (await checkIfExists(requiredPath)) {
+            html = await loadFile(requiredPath);
+        }
+    }
+
+    if (!html) {
+        throw new Error(`Source page "${sourceName}" not found`);
+    }
+
+    // Save HTML to target (creates folder + page.html + page.json)
+    await savePageState(pagesFolder, targetName, html, title);
+
+    // Overwrite the generated metadata with provided title + categories
+    const now = new Date().toISOString();
+    const metadata: PageMetadata = {
+        title,
+        categories,
+        pinned: false,
+        showInAll: true,
+        createdDate: now,
+        lastModified: now,
+        pageVersion: PAGE_VERSION,
+        mode: 'unlocked',
+    };
+    await savePageMetadata(pagesFolder, targetName, metadata);
+}

package/src/scripts.ts

@@ -17,7 +17,7 @@ export interface ExecuteScriptResponse {
 export async function listScripts(pagesFolder: string): Promise<string> {
     if (!scriptsList) {
         let list: string[] = [];
-        const folder = path.join(pagesFolder, 'scripts');
+        const folder = path.join(pagesFolder, 'pages', 'scripts', 'scripts');
         if (await checkIfExists(folder)) {
             const files = (await listFiles(folder)).filter(f => f.endsWith('.json'));
             for (const file of files) {
@@ -52,7 +52,7 @@ export async function executeScript(args: ExecuteScriptArgs): Promise<AgentCompl
     const { pagesFolder, scriptId, variables } = args;
 
     // Load the script
-    const script = await loadFile(path.join(pagesFolder, `scripts`, `${scriptId}.json`));
+    const script = await loadFile(path.join(pagesFolder, 'pages', 'scripts', 'scripts', `${scriptId}.json`));
     if (!script) {
         return { completed: false, error: new Error(`Script not found: ${scriptId}`) };
     }

package/src/service/createCompletePrompt.ts

@@ -1,41 +1,37 @@
 import {anthropic, completePrompt, logCompletePrompt, openai} from 'agentm-core';
-import { loadSettings } from '../settings';
+import { getModelEntry, loadSettings } from '../settings';
+import { PROVIDERS } from '../models';
 
-export const availableModels = [
-    'claude-opus-4-5',
-    'Claude Sonnet 4.5',
-    'Claude Haiku 4.5',
-    'GPT-5.2',
-    'GPT-5 mini',
-    'GPT-5 nano'
-];
-
-export async function createCompletePrompt(pagesFolder: string, model?: string): Promise<completePrompt> {
+export async function createCompletePrompt(pagesFolder: string, use: 'builder' | 'chat', modelOverride?: string): Promise<completePrompt> {
     // Get configuration settings
     const settings = await loadSettings(pagesFolder);
-    if (!settings.serviceApiKey) {
-        throw new Error('OpenAI API key not configured');
+    const entry = getModelEntry(settings, use);
+
+    if (!entry.configuration.apiKey) {
+        throw new Error('API key not configured');
     }
 
     // Validate model
-    model = model || settings.model;
+    const model = modelOverride || entry.configuration.model;
     if (!model) {
         throw new Error('Model not configured');
     }
 
     // Create completion functions
     let modelInstance: completePrompt;
-    const apiKey = settings.serviceApiKey;
-    if (model.startsWith('claude-')) {
+    const apiKey = entry.configuration.apiKey;
+    if (entry.provider === 'Anthropic') {
         modelInstance = anthropic({apiKey, model});
     } else {
         modelInstance = openai({apiKey, model});
     }
 
     // Return new model instance
-    if (settings.logCompletions) {
+    if (entry.logCompletions) {
         return logCompletePrompt(modelInstance, true);
     } else {
         return modelInstance;
     }
-}
+}
+
+export { PROVIDERS };

package/src/service/debugLog.ts

@@ -0,0 +1,17 @@
+// ANSI color helpers — no external dependency
+export const reset = '\x1b[0m';
+export const cyan = (s: string): string => `\x1b[36m${s}${reset}`;
+export const yellow = (s: string): string => `\x1b[33m${s}${reset}`;
+export const green = (s: string): string => `\x1b[32m${s}${reset}`;
+export const red = (s: string): string => `\x1b[31m${s}${reset}`;
+export const dim = (s: string): string => `\x1b[2m${s}${reset}`;
+
+/** Format milliseconds as `X.XXXs` */
+export function formatTime(ms: number): string {
+    return `${(ms / 1000).toFixed(3)}s`;
+}
+
+/** Rough token estimate based on chars / 4 heuristic */
+export function estimateTokens(chars: number): number {
+    return Math.ceil(chars / 4);
+}

package/src/service/modelInstructions.ts

@@ -0,0 +1,14 @@
+import { ProviderName } from '../models';
+
+/**
+ * Returns provider-specific prompt instructions for how the model should
+ * format its change-list response.
+ */
+export function getModelInstructions(provider: ProviderName): string {
+    if (provider === 'Anthropic') {
+        return `Return ONLY the JSON array of change operations. Do not wrap it in markdown code fences or add any other text.`;
+    }
+
+    // OpenAI and other providers
+    return `Return ONLY the JSON array of change operations. Do not wrap it in markdown code fences or add any other text.`;
+}

package/src/service/requiresSettings.ts

@@ -1,7 +1,7 @@
-import { hasConfiguredSettings, loadSettings, Settings } from "../settings";
+import { hasConfiguredSettings, loadSettings, SettingsV2 } from "../settings";
 
 
-export async function requiresSettings(res: any, folder: string, cb: (settings: Settings) => Promise<void>) {
+export async function requiresSettings(res: any, folder: string, cb: (settings: SettingsV2) => Promise<void>) {
     try {
         // Ensure settings configured
         const isConfigured = await hasConfiguredSettings(folder);
@@ -19,4 +19,4 @@ export async function requiresSettings(res: any, folder: string, cb: (settings:
         console.error(err);
         res.status(500).send((err as Error).message);
     }
-}
+}

package/src/service/server.ts

@@ -3,10 +3,24 @@ import { usePageRoutes } from './usePageRoutes';
 import { useApiRoutes } from './useApiRoutes';
 import { SynthOSConfig } from '../init';
 import { useDataRoutes } from './useDataRoutes';
+import { useConnectorRoutes } from './useConnectorRoutes';
+import { cyan, yellow, formatTime } from './debugLog';
 
 export function server(config: SynthOSConfig): Application {
     const app = express();
-    
+
+    // Debug request-logging middleware
+    if (config.debug) {
+        app.use((req, res, next) => {
+            const start = Date.now();
+            res.on('finish', () => {
+                const ms = Date.now() - start;
+                console.log(`${cyan(req.method + ' ' + req.originalUrl)} → ${res.statusCode} ${yellow('(' + formatTime(ms) + ')')}`);
+            });
+            next();
+        });
+    }
+
     // Middleware to parse URL-encoded data (form data)
     app.use(express.urlencoded({ extended: true }));
 
@@ -19,8 +33,11 @@ export function server(config: SynthOSConfig): Application {
     // API routes
     useApiRoutes(config, app);
 
+    // Connector routes
+    useConnectorRoutes(config, app);
+
     // Data routes
     useDataRoutes(config, app);
-    
+
     return app;
 }