synthos 0.8.0 → 0.9.0

package/src/builders/fireworksai.ts

@@ -0,0 +1,59 @@
+import { completePrompt } from '../models';
+import { getTransformInstr } from '../service/transformPage';
+import { parseBuilderResponse } from './anthropic';
+import { Builder, BuilderResult, CHANGE_OPS_FORMAT_INSTRUCTION } from './types';
+
+/**
+ * Create a FireworksAI-tuned builder.
+ * Currently identical to the Anthropic builder — separate file enables
+ * future per-provider tuning.
+ */
+export function createFireworksAIBuilder(complete: completePrompt, userInstructions?: string, productName?: string): Builder {
+    const name = productName ?? 'SynthOS';
+
+    return {
+        async run(currentPage, additionalSections, userMessage, newBuild, _attachments?): Promise<BuilderResult> {
+            try {
+                // -- System message --
+                const systemParts: string[] = [
+                    `${currentPage.title}\n${currentPage.content}`,
+                ];
+                for (const section of additionalSections) {
+                    if (section.content) {
+                        systemParts.push(`${section.title}\n${section.content}`);
+                    }
+                }
+                const systemContent = systemParts.join('\n\n');
+
+                // -- User message --
+                const instructionParts: string[] = [];
+                if (userInstructions?.trim()) {
+                    instructionParts.push(userInstructions);
+                }
+                for (const section of additionalSections) {
+                    if (section.instructions?.trim()) {
+                        instructionParts.push(section.instructions);
+                    }
+                }
+                instructionParts.push(getTransformInstr(name));
+                instructionParts.push(CHANGE_OPS_FORMAT_INSTRUCTION);
+
+                const instructions = instructionParts.filter(s => s.trim() !== '').join('\n');
+                const promptContent = `<USER_MESSAGE>\n${userMessage}\n\n<INSTRUCTIONS>\n${instructions}`;
+
+                const result = await complete({
+                    system: { role: 'system', content: systemContent },
+                    prompt: { role: 'user', content: promptContent },
+                });
+
+                if (!result.completed) {
+                    return { kind: 'error', error: result.error ?? new Error('Model call failed') };
+                }
+
+                return parseBuilderResponse(result.value!);
+            } catch (err: unknown) {
+                return { kind: 'error', error: err instanceof Error ? err : new Error(String(err)) };
+            }
+        }
+    };
+}

package/src/builders/index.ts

@@ -0,0 +1,33 @@
+import { ProviderName, completePrompt } from '../models';
+import { createAnthropicBuilder, AnthropicBuilderOptions } from './anthropic';
+import { createOpenAIBuilder } from './openai';
+import { createFireworksAIBuilder } from './fireworksai';
+import { Builder } from './types';
+
+export { ContextSection, BuilderResult, Builder, Attachment } from './types';
+export { createAnthropicBuilder, AnthropicBuilderOptions } from './anthropic';
+export { createOpenAIBuilder } from './openai';
+export { createFireworksAIBuilder } from './fireworksai';
+export { parseBuilderResponse } from './anthropic';
+
+/**
+ * Factory that creates a provider-specific builder.
+ */
+export function createBuilder(
+    provider: ProviderName,
+    complete: completePrompt,
+    userInstructions?: string,
+    productName?: string,
+    options?: AnthropicBuilderOptions
+): Builder {
+    switch (provider) {
+        case 'Anthropic':
+            return createAnthropicBuilder(complete, userInstructions, productName, options);
+        case 'OpenAI':
+            return createOpenAIBuilder(complete, userInstructions, productName);
+        case 'FireworksAI':
+            return createFireworksAIBuilder(complete, userInstructions, productName);
+        default:
+            throw new Error(`Unknown provider: ${provider}`);
+    }
+}

package/src/builders/openai.ts

@@ -0,0 +1,89 @@
+import { completePrompt, ContentBlock } from '../models';
+import { getTransformInstr } from '../service/transformPage';
+import { parseBuilderResponse } from './anthropic';
+import { Builder, BuilderResult, OPENAI_CHANGE_OPS_SCHEMA } from './types';
+
+/**
+ * Create an OpenAI-tuned builder.
+ * Uses OpenAI structured outputs (json_schema) for reliable JSON responses.
+ */
+export function createOpenAIBuilder(complete: completePrompt, userInstructions?: string, productName?: string): Builder {
+    const name = productName ?? 'SynthOS';
+
+    return {
+        async run(currentPage, additionalSections, userMessage, newBuild, attachments?): Promise<BuilderResult> {
+            try {
+                // -- System message --
+                const systemParts: string[] = [
+                    `${currentPage.title}\n${currentPage.content}`,
+                ];
+                for (const section of additionalSections) {
+                    if (section.content) {
+                        systemParts.push(`${section.title}\n${section.content}`);
+                    }
+                }
+                const systemContent = systemParts.join('\n\n');
+
+                // -- User message --
+                const instructionParts: string[] = [];
+                if (userInstructions?.trim()) {
+                    instructionParts.push(userInstructions);
+                }
+                for (const section of additionalSections) {
+                    if (section.instructions?.trim()) {
+                        instructionParts.push(section.instructions);
+                    }
+                }
+                instructionParts.push(getTransformInstr(name));
+
+                const instructions = instructionParts.filter(s => s.trim() !== '').join('\n');
+                const promptText = `<USER_MESSAGE>\n${userMessage}\n\n<INSTRUCTIONS>\n${instructions}`;
+
+                // Build prompt content — multimodal when image attachments are present
+                const imageAttachments = (attachments ?? []).filter(a => a.mediaType.startsWith('image/'));
+                let promptContent: string | ContentBlock[];
+                if (imageAttachments.length > 0) {
+                    const blocks: ContentBlock[] = [{ type: 'text', text: promptText }];
+                    for (const att of imageAttachments) {
+                        blocks.push({ type: 'image', mediaType: att.mediaType, data: att.data });
+                    }
+                    promptContent = blocks;
+                } else {
+                    promptContent = promptText;
+                }
+
+                const result = await complete({
+                    system: { role: 'system', content: systemContent },
+                    prompt: { role: 'user', content: promptContent },
+                    jsonSchema: OPENAI_CHANGE_OPS_SCHEMA,
+                });
+
+                if (!result.completed) {
+                    return { kind: 'error', error: result.error ?? new Error('Model call failed') };
+                }
+
+                // Unwrap the { changes: [...] } envelope from structured output
+                return parseOpenAIResponse(result.value!);
+            } catch (err: unknown) {
+                return { kind: 'error', error: err instanceof Error ? err : new Error(String(err)) };
+            }
+        }
+    };
+}
+
+/**
+ * Parse the OpenAI structured output response.
+ * The schema wraps the array in { changes: [...] }, so unwrap before
+ * delegating to the shared parser.
+ */
+function parseOpenAIResponse(raw: string): BuilderResult {
+    try {
+        const parsed = JSON.parse(raw);
+        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed) && Array.isArray(parsed.changes)) {
+            return { kind: 'transforms', changes: parsed.changes };
+        }
+    } catch {
+        // fall through to shared parser
+    }
+    return parseBuilderResponse(raw);
+}

package/src/builders/types.ts

@@ -0,0 +1,261 @@
+import { ChangeList } from '../service/transformPage';
+
+// ---------------------------------------------------------------------------
+// Change operations output format — text instruction for non-structured builders
+// ---------------------------------------------------------------------------
+
+/**
+ * Text instruction that tells the model to return a JSON array of change operations.
+ * Append this to <INSTRUCTIONS> for builders that don't support structured outputs.
+ */
+export const CHANGE_OPS_FORMAT_INSTRUCTION = `Return a JSON array of change operations to apply to the page. Do NOT return the full HTML page.
+
+Each operation must be one of:
+{ "op": "update", "nodeId": "<data-node-id>", "html": "<new innerHTML>" }
+  — replaces the innerHTML of the target element
+
+{ "op": "replace", "nodeId": "<data-node-id>", "html": "<new outerHTML>" }
+  — replaces the entire element (outerHTML) with new markup
+
+{ "op": "delete", "nodeId": "<data-node-id>" }
+  — removes the element from the page
+
+{ "op": "insert", "parentId": "<data-node-id>", "position": "prepend"|"append"|"before"|"after", "html": "<new element HTML>" }
+  — inserts new HTML relative to the parent element
+
+{ "op": "style-element", "nodeId": "<data-node-id>", "style": "<css style string>" }
+  — sets the style attribute of the target element (must be unlocked)
+
+{ "op": "search-replace", "nodeId": "<data-node-id>", "search": "<exact text to find>", "replace": "<replacement text>" }
+  — finds exact text within a script/style block and replaces it. Use empty string for replace to delete.
+
+{ "op": "search-insert", "nodeId": "<data-node-id>", "after": "<exact text to find>", "content": "<new lines to insert>" }
+  — inserts new content immediately after the matched text in a script/style block.
+
+For partial edits to large scripts/styles, use search-replace or search-insert instead of
+replacing the entire block with update.
+Copy the search/after text exactly as it appears in the source.
+When making multiple edits to the same block, ensure each search string targets distinct text.
+To delete code, use search-replace with an empty replace string.
+
+Return ONLY the JSON array. Example:
+[
+  { "op": "update", "nodeId": "5", "html": "<p>Hello world</p>" },
+  { "op": "insert", "parentId": "3", "position": "append", "html": "<div class=\\"msg\\">New message</div>" }
+]`;
+
+// ---------------------------------------------------------------------------
+// Change operations JSON schema — for structured output (constrained decoding)
+// ---------------------------------------------------------------------------
+
+/**
+ * JSON schema matching the ChangeOp union type for Anthropic structured outputs.
+ * The top-level schema is an array of change operations.
+ */
+export const CHANGE_OPS_SCHEMA: Record<string, unknown> = {
+    type: 'array',
+    items: {
+        anyOf: [
+            {
+                type: 'object',
+                properties: {
+                    op: { type: 'string', const: 'update' },
+                    nodeId: { type: 'string' },
+                    html: { type: 'string' },
+                },
+                required: ['op', 'nodeId', 'html'],
+                additionalProperties: false,
+            },
+            {
+                type: 'object',
+                properties: {
+                    op: { type: 'string', const: 'replace' },
+                    nodeId: { type: 'string' },
+                    html: { type: 'string' },
+                },
+                required: ['op', 'nodeId', 'html'],
+                additionalProperties: false,
+            },
+            {
+                type: 'object',
+                properties: {
+                    op: { type: 'string', const: 'delete' },
+                    nodeId: { type: 'string' },
+                },
+                required: ['op', 'nodeId'],
+                additionalProperties: false,
+            },
+            {
+                type: 'object',
+                properties: {
+                    op: { type: 'string', const: 'insert' },
+                    parentId: { type: 'string' },
+                    position: { type: 'string', enum: ['prepend', 'append', 'before', 'after'] },
+                    html: { type: 'string' },
+                },
+                required: ['op', 'parentId', 'position', 'html'],
+                additionalProperties: false,
+            },
+            {
+                type: 'object',
+                properties: {
+                    op: { type: 'string', const: 'style-element' },
+                    nodeId: { type: 'string' },
+                    style: { type: 'string' },
+                },
+                required: ['op', 'nodeId', 'style'],
+                additionalProperties: false,
+            },
+            {
+                type: 'object',
+                properties: {
+                    op: { type: 'string', const: 'search-replace' },
+                    nodeId: { type: 'string' },
+                    search: { type: 'string' },
+                    replace: { type: 'string' },
+                },
+                required: ['op', 'nodeId', 'search', 'replace'],
+                additionalProperties: false,
+            },
+            {
+                type: 'object',
+                properties: {
+                    op: { type: 'string', const: 'search-insert' },
+                    nodeId: { type: 'string' },
+                    after: { type: 'string' },
+                    content: { type: 'string' },
+                },
+                required: ['op', 'nodeId', 'after', 'content'],
+                additionalProperties: false,
+            },
+        ],
+    },
+};
+
+/**
+ * Variant of CHANGE_OPS_SCHEMA that omits ranged write operations (search-replace, search-insert).
+ * Used for medium/hard/re-write changes where the model should replace full blocks instead of
+ * attempting partial edits.
+ */
+export const CHANGE_OPS_SCHEMA_NO_RANGED: Record<string, unknown> = {
+    type: 'array',
+    items: {
+        anyOf: [
+            {
+                type: 'object',
+                properties: {
+                    op: { type: 'string', const: 'update' },
+                    nodeId: { type: 'string' },
+                    html: { type: 'string' },
+                },
+                required: ['op', 'nodeId', 'html'],
+                additionalProperties: false,
+            },
+            {
+                type: 'object',
+                properties: {
+                    op: { type: 'string', const: 'replace' },
+                    nodeId: { type: 'string' },
+                    html: { type: 'string' },
+                },
+                required: ['op', 'nodeId', 'html'],
+                additionalProperties: false,
+            },
+            {
+                type: 'object',
+                properties: {
+                    op: { type: 'string', const: 'delete' },
+                    nodeId: { type: 'string' },
+                },
+                required: ['op', 'nodeId'],
+                additionalProperties: false,
+            },
+            {
+                type: 'object',
+                properties: {
+                    op: { type: 'string', const: 'insert' },
+                    parentId: { type: 'string' },
+                    position: { type: 'string', enum: ['prepend', 'append', 'before', 'after'] },
+                    html: { type: 'string' },
+                },
+                required: ['op', 'parentId', 'position', 'html'],
+                additionalProperties: false,
+            },
+            {
+                type: 'object',
+                properties: {
+                    op: { type: 'string', const: 'style-element' },
+                    nodeId: { type: 'string' },
+                    style: { type: 'string' },
+                },
+                required: ['op', 'nodeId', 'style'],
+                additionalProperties: false,
+            },
+        ],
+    },
+};
+
+/**
+ * Text instruction variant for no-ranged-writes mode.
+ * Tells the model to use `update` for full innerHTML replacement instead of partial edits.
+ */
+export const CHANGE_OPS_FORMAT_INSTRUCTION_NO_RANGED = `For script and style blocks, use "update" to replace the full innerHTML or "replace" to replace the full outerHTML. Do NOT use partial search-replace edits.`;
+
+/**
+ * OpenAI structured outputs require a root-level object.
+ * This wraps CHANGE_OPS_SCHEMA in { changes: [...] }.
+ */
+export const OPENAI_CHANGE_OPS_SCHEMA: Record<string, unknown> = {
+    type: 'object',
+    properties: {
+        changes: CHANGE_OPS_SCHEMA,
+    },
+    required: ['changes'],
+    additionalProperties: false,
+};
+
+// ---------------------------------------------------------------------------
+// Context sections — structured blocks passed to the builder
+// ---------------------------------------------------------------------------
+
+export interface ContextSection {
+    /** Section title, e.g. "<CURRENT_PAGE>", "<SERVER_SCRIPTS>" */
+    title: string;
+    /** The text body of this section */
+    content: string;
+    /** How the model should work with this section (appended to instructions) */
+    instructions: string;
+}
+
+// ---------------------------------------------------------------------------
+// Builder result — discriminated union returned by Builder.run()
+// ---------------------------------------------------------------------------
+
+export type BuilderResult =
+    | { kind: 'transforms'; changes: ChangeList }
+    | { kind: 'reply'; text: string }
+    | { kind: 'error'; error: Error };
+
+// ---------------------------------------------------------------------------
+// Attachments — images/files sent alongside the user message
+// ---------------------------------------------------------------------------
+
+export interface Attachment {
+    mediaType: string;
+    data: string;
+    name?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Builder interface
+// ---------------------------------------------------------------------------
+
+export interface Builder {
+    run(
+        currentPage: ContextSection,
+        additionalSections: ContextSection[],
+        userMessage: string,
+        newBuild: boolean,
+        attachments?: Attachment[]
+    ): Promise<BuilderResult>;
+}

package/src/connectors/index.ts

@@ -1,4 +1,4 @@
-export { CONNECTOR_REGISTRY } from './registry';
+export { loadConnectorRegistry, getConnectorRegistry } from './registry';
 export type {
     AuthStrategy,
     ConnectorField,

package/src/connectors/registry.ts

@@ -2,8 +2,6 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { ConnectorDefinition, ConnectorJson } from './types';
 
-const connectorsDir = __dirname;
-
 function loadConnectorJson(dir: string): ConnectorDefinition | null {
     const file = path.join(dir, 'connector.json');
     if (!fs.existsSync(file)) return null;
@@ -14,9 +12,31 @@ function loadConnectorJson(dir: string): ConnectorDefinition | null {
     };
 }
 
-export const CONNECTOR_REGISTRY: ConnectorDefinition[] = fs
-    .readdirSync(connectorsDir, { withFileTypes: true })
-    .filter(d => d.isDirectory())
-    .map(d => loadConnectorJson(path.join(connectorsDir, d.name)))
-    .filter((d): d is ConnectorDefinition => d !== null)
-    .sort((a, b) => a.name.localeCompare(b.name));
+export function loadConnectorRegistry(connectorsDirs: string[]): ConnectorDefinition[] {
+    const seen = new Set<string>();
+    const results: ConnectorDefinition[] = [];
+    for (const dir of connectorsDirs) {
+        if (!fs.existsSync(dir)) continue;
+        const entries = fs.readdirSync(dir, { withFileTypes: true })
+            .filter(d => d.isDirectory());
+        for (const d of entries) {
+            const def = loadConnectorJson(path.join(dir, d.name));
+            if (def && !seen.has(def.id)) {
+                seen.add(def.id);
+                results.push(def);
+            }
+        }
+    }
+    return results.sort((a, b) => a.name.localeCompare(b.name));
+}
+
+let _registry: ConnectorDefinition[] | undefined;
+let _registryKey: string | undefined;
+export function getConnectorRegistry(connectorsDirs?: string[]): ConnectorDefinition[] {
+    const key = connectorsDirs ? connectorsDirs.join('|') : undefined;
+    if (!_registry || (connectorsDirs && key !== _registryKey)) {
+        _registry = loadConnectorRegistry(connectorsDirs ?? [path.join(__dirname, '../../service-connectors')]);
+        _registryKey = key;
+    }
+    return _registry;
+}

package/src/customizer/Customizer.ts

@@ -0,0 +1,151 @@
+import { Application } from 'express';
+import { SynthOSConfig } from '../init';
+import { ContextSection } from '../builders/types';
+
+export type RouteInstaller = (config: SynthOSConfig, app: Application) => void;
+
+export class Customizer {
+    protected disabled: Set<string> = new Set();
+    protected extraRoutes: RouteInstaller[] = [];
+
+    /** Route hints for the LLM prompt (fork developers can add custom hints). */
+    protected routeHintEntries: { hints: string }[] = [];
+
+    /** Custom instructions appended to transformPage's instruction block. */
+    protected customTransformInstructions: string[] = [];
+
+    /** Custom context sections appended to the builder's additional sections. */
+    protected customContextSections: ContextSection[] = [];
+
+    // --- Local data folder ---
+    // Override in a derived class to change the local project folder name.
+
+    /** Name of the local data folder created in the user's project directory. */
+    get localFolder(): string {
+        return '.synthos';
+    }
+
+    // --- Content folder paths ---
+    // Override these in a derived class to point to your fork's folders.
+    // Each getter returns `string[]`. The sentinel value `'default'` is
+    // resolved by `createConfig()` to the built-in SynthOS folder for that
+    // getter. Forks can return additional paths alongside (or instead of)
+    // `'default'` to layer custom folders.
+
+    /** Folders containing built-in system pages (builder, settings, etc.) */
+    get requiredPagesFolders(): string[] {
+        return ['default'];
+    }
+
+    /** Folders containing starter page templates copied on init */
+    get defaultPagesFolders(): string[] {
+        return ['default'];
+    }
+
+    /** Folders containing theme CSS/JSON files */
+    get defaultThemesFolders(): string[] {
+        return ['default'];
+    }
+
+    /** Folders containing versioned static files (page-v2.js, helpers-v2.js, etc.) */
+    get staticFilesFolders(): string[] {
+        return ['default'];
+    }
+
+    /** Folders containing connector JSON definitions */
+    get serviceConnectorsFolders(): string[] {
+        return ['default'];
+    }
+
+    /** Folders containing default scripts copied on init */
+    get defaultScriptsFolders(): string[] {
+        return ['default'];
+    }
+
+    /** Route path for the "browse all pages/tabs" listing page.
+     *  Override in a derived class to change the redirect target for outdated pages. */
+    get tabsListRoute(): string {
+        return '/pages';
+    }
+
+    /** Product name used in LLM prompts and branding strings.
+     *  Override in a derived class for white-labeling. */
+    get productName(): string {
+        return 'SynthOS';
+    }
+
+    // --- Feature group control ---
+    // Built-in groups: 'pages', 'api', 'connectors', 'agents',
+    //   'data', 'brainstorm', 'search', 'scripts'
+
+    disable(...groups: string[]): this {
+        for (const g of groups) this.disabled.add(g);
+        return this;
+    }
+
+    enable(...groups: string[]): this {
+        for (const g of groups) this.disabled.delete(g);
+        return this;
+    }
+
+    isEnabled(group: string): boolean {
+        return !this.disabled.has(group);
+    }
+
+    // --- Custom routes ---
+
+    addRoutes(...installers: (RouteInstaller | { installer: RouteInstaller; hints: string })[]): this {
+        for (const entry of installers) {
+            if (typeof entry === 'function') {
+                this.extraRoutes.push(entry);
+            } else {
+                this.extraRoutes.push(entry.installer);
+                this.routeHintEntries.push({ hints: entry.hints });
+            }
+        }
+        return this;
+    }
+
+    getExtraRoutes(): RouteInstaller[] {
+        return this.extraRoutes;
+    }
+
+    // --- Route hints ---
+
+    /** Add route hints that will be shown to the LLM in transformPage. */
+    addRouteHints(...hints: string[]): this {
+        for (const h of hints) this.routeHintEntries.push({ hints: h });
+        return this;
+    }
+
+    /** Get all custom route hints. */
+    getRouteHints(): string[] {
+        return this.routeHintEntries.map(e => e.hints);
+    }
+
+    // --- Custom transform instructions ---
+
+    /** Add custom instructions for the transformPage LLM prompt. */
+    addTransformInstructions(...instructions: string[]): this {
+        this.customTransformInstructions.push(...instructions);
+        return this;
+    }
+
+    /** Get custom transform instructions. */
+    getTransformInstructions(): string[] {
+        return this.customTransformInstructions;
+    }
+
+    // --- Custom context sections ---
+
+    /** Add custom context sections for the builder. */
+    addContextSections(...sections: ContextSection[]): this {
+        this.customContextSections.push(...sections);
+        return this;
+    }
+
+    /** Get custom context sections. */
+    getContextSections(): ContextSection[] {
+        return this.customContextSections;
+    }
+}

package/src/customizer/index.ts

@@ -0,0 +1,5 @@
+export { Customizer, RouteInstaller } from './Customizer';
+import { Customizer } from './Customizer';
+
+// Default instance — replace with an App specific customizer
+export const customizer = new Customizer();