synthos 0.10.1 → 0.11.0

package/src/service/pageValidator.ts

@@ -0,0 +1,337 @@
+/**
+ * pageValidator.ts — Lightweight page validation for LLM-generated HTML.
+ *
+ * Catches common errors (syntax, missing DOM elements, bad synthos.* calls)
+ * before serving pages. No browser binary — static analysis only.
+ *
+ * Layers:
+ *   1. Script extraction + Acorn parse → syntax errors
+ *   2. DOM element ID inventory → missing element references
+ *   3. SynthOS API pattern checks → hallucinated method calls
+ */
+
+import * as acorn from 'acorn';
+import * as cheerio from 'cheerio';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface PageValidationError {
+    type: 'syntax-error' | 'missing-element' | 'unknown-api';
+    message: string;
+    source?: string;   // 'inline-script-N'
+    line?: number;
+    col?: number;
+}
+
+export interface PageValidationResult {
+    valid: boolean;
+    errors: PageValidationError[];
+    durationMs: number;
+}
+
+interface ScriptBlock {
+    index: number;
+    source: string;
+    startLine: number;
+    isModule: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Known injected script IDs — skip these during validation
+// ---------------------------------------------------------------------------
+
+const INJECTED_SCRIPT_IDS = new Set([
+    'page-bridge',                // legacy (replaced by shell-v3)
+    'page-helpers',               // legacy (replaced by v3 modules)
+    'page-info',
+    'page-script',                // legacy shell script
+    'synthos-error-capture',      // legacy error capture
+    // V3 module scripts
+    'shell-v3',
+    'server-v3',
+    'storage-v3',
+    'script-v3',
+    'connector-v3',
+    'agent-v3',
+]);
+
+// ---------------------------------------------------------------------------
+// Known SynthOS API surface (from helpers.v3.js)
+// ---------------------------------------------------------------------------
+
+const KNOWN_SYNTHOS_METHODS = new Set([
+    // data
+    'synthos.data',
+    'synthos.data.list',
+    'synthos.data.get',
+    'synthos.data.save',
+    'synthos.data.remove',
+    // files
+    'synthos.files',
+    'synthos.files.list',
+    'synthos.files.upload',
+    'synthos.files.url',
+    'synthos.files.remove',
+    // shared.data
+    'synthos.shared',
+    'synthos.shared.data',
+    'synthos.shared.data.list',
+    'synthos.shared.data.get',
+    'synthos.shared.data.save',
+    'synthos.shared.data.remove',
+    // shared.files
+    'synthos.shared.files',
+    'synthos.shared.files.list',
+    'synthos.shared.files.upload',
+    'synthos.shared.files.url',
+    'synthos.shared.files.remove',
+    // generate
+    'synthos.generate',
+    'synthos.generate.image',
+    'synthos.generate.completion',
+    // script
+    'synthos.script',
+    'synthos.script.run',
+    // page
+    'synthos.page',
+    'synthos.page.list',
+    'synthos.page.get',
+    'synthos.page.update',
+    'synthos.page.remove',
+    'synthos.page.ask',
+    // search
+    'synthos.search',
+    'synthos.search.web',
+    // connector
+    'synthos.connector',
+    'synthos.connector.call',
+    'synthos.connector.list',
+    // agent
+    'synthos.agent',
+    'synthos.agent.list',
+    'synthos.agent.send',
+    'synthos.agent.sendStream',
+    'synthos.agent.chat',
+    'synthos.agent.chat.send',
+    'synthos.agent.chat.sendStream',
+    'synthos.agent.chat.history',
+    'synthos.agent.chat.abort',
+    'synthos.agent.chat.clear',
+    'synthos.agent.isEnabled',
+    'synthos.agent.getCapabilities',
+    // shell
+    'synthos.shell',
+    'synthos.shell.navigate',
+    'synthos.shell.submitChat',
+    'synthos.shell.setDirty',
+    'synthos.shell.showLoading',
+    'synthos.shell.hideLoading',
+    'synthos.shell.focusChat',
+    'synthos.shell.openSaveModal',
+    'synthos.shell.toggleBuilder',
+    'synthos.shell.openBuilder',
+    'synthos.shell.closeBuilder',
+    'synthos.shell.on',
+]);
+
+// ---------------------------------------------------------------------------
+// Layer 1: Script Extraction + Acorn Parse
+// ---------------------------------------------------------------------------
+
+function estimateLineOffset(html: string, scriptHtml: string): number {
+    const idx = html.indexOf(scriptHtml);
+    if (idx === -1) return 0;
+    let line = 1;
+    for (let i = 0; i < idx; i++) {
+        if (html[i] === '\n') line++;
+    }
+    return line;
+}
+
+function extractScripts(html: string): ScriptBlock[] {
+    const $ = cheerio.load(html);
+    const scripts: ScriptBlock[] = [];
+    let index = 0;
+
+    $('script').each((_, el) => {
+        const $el = $(el);
+
+        // Skip external scripts (have src attribute)
+        if ($el.attr('src')) return;
+
+        // Skip known injected scripts
+        const id = $el.attr('id');
+        if (id && INJECTED_SCRIPT_IDS.has(id)) return;
+
+        const source = $el.text();
+        if (!source.trim()) return;
+
+        scripts.push({
+            index: index++,
+            source,
+            startLine: estimateLineOffset(html, $.html(el)!),
+            isModule: $el.attr('type') === 'module',
+        });
+    });
+
+    return scripts;
+}
+
+function parseSyntax(script: ScriptBlock): PageValidationError | null {
+    try {
+        acorn.parse(script.source, {
+            ecmaVersion: 'latest' as any,
+            sourceType: script.isModule ? 'module' : 'script',
+            allowAwaitOutsideFunction: true,
+        });
+        return null;
+    } catch (err: unknown) {
+        const e = err as { message?: string; loc?: { line: number; column: number } };
+        return {
+            type: 'syntax-error',
+            message: e.message ?? 'Unknown syntax error',
+            source: `inline-script-${script.index}`,
+            line: e.loc ? script.startLine + e.loc.line : undefined,
+            col: e.loc?.column,
+        };
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Layer 2: DOM Element ID Inventory
+// ---------------------------------------------------------------------------
+
+function buildIdInventory(html: string): Set<string> {
+    const $ = cheerio.load(html);
+    const ids = new Set<string>();
+    $('[id]').each((_, el) => {
+        const id = $(el).attr('id');
+        if (id) ids.add(id);
+    });
+    return ids;
+}
+
+/**
+ * Patterns that CREATE element IDs inside script source text. Scripts that
+ * build DOM via `innerHTML = \`...<input id="btnSaveAll">...\`` or assign
+ * `el.id = 'foo'` produce IDs at runtime that `buildIdInventory` misses.
+ * Seeding those IDs as "known" prevents false-positive missing-element
+ * errors for dynamically constructed UI.
+ */
+const ID_CREATE_PATTERNS = [
+    // id="foo" / id='foo' in string literals (innerHTML templates, etc.)
+    /\bid\s*=\s*["']([a-zA-Z_][\w-]*)["']/g,
+    // el.id = 'foo' / obj.id = "foo"
+    /\.id\s*=\s*["']([a-zA-Z_][\w-]*)["']/g,
+];
+
+function collectDynamicIds(scripts: ScriptBlock[]): Set<string> {
+    const ids = new Set<string>();
+    for (const script of scripts) {
+        for (const pattern of ID_CREATE_PATTERNS) {
+            pattern.lastIndex = 0;
+            let match;
+            while ((match = pattern.exec(script.source)) !== null) {
+                ids.add(match[1]);
+            }
+        }
+    }
+    return ids;
+}
+
+/** Patterns that reference element IDs in JS code. */
+const ID_REF_PATTERNS = [
+    /getElementById\(\s*['"]([^'"]+)['"]\s*\)/g,
+    /querySelector(?:All)?\(\s*['"]#([^'"]+)['"]\s*\)/g,
+    /\$\(\s*['"]#([^'"]+)['"]\s*\)/g,
+];
+
+function checkIdReferences(scripts: ScriptBlock[], ids: Set<string>): PageValidationError[] {
+    const errors: PageValidationError[] = [];
+
+    for (const script of scripts) {
+        for (const pattern of ID_REF_PATTERNS) {
+            pattern.lastIndex = 0;
+            let match;
+            while ((match = pattern.exec(script.source)) !== null) {
+                const refId = match[1];
+                if (!ids.has(refId)) {
+                    errors.push({
+                        type: 'missing-element',
+                        message: `Script references element '#${refId}' which does not exist in the page`,
+                        source: `inline-script-${script.index}`,
+                    });
+                }
+            }
+        }
+    }
+
+    return errors;
+}
+
+// ---------------------------------------------------------------------------
+// Layer 3: SynthOS API Pattern Checks
+// ---------------------------------------------------------------------------
+
+const SYNTHOS_CALL_PATTERN = /synthos(?:\.[\w]+)+/g;
+
+function checkSynthosApi(scripts: ScriptBlock[]): PageValidationError[] {
+    const errors: PageValidationError[] = [];
+    const seen = new Set<string>();
+
+    for (const script of scripts) {
+        SYNTHOS_CALL_PATTERN.lastIndex = 0;
+        let match;
+        while ((match = SYNTHOS_CALL_PATTERN.exec(script.source)) !== null) {
+            const call = match[0];
+            // Deduplicate within a single validation run
+            if (seen.has(call)) continue;
+
+            if (!KNOWN_SYNTHOS_METHODS.has(call)) {
+                seen.add(call);
+                errors.push({
+                    type: 'unknown-api',
+                    message: `Unknown SynthOS API call: ${call}`,
+                    source: `inline-script-${script.index}`,
+                });
+            }
+        }
+    }
+
+    return errors;
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+export function validatePage(html: string): PageValidationResult {
+    const start = Date.now();
+    const errors: PageValidationError[] = [];
+
+    // Layer 1: Extract and parse scripts
+    const scripts = extractScripts(html);
+    for (const script of scripts) {
+        const err = parseSyntax(script);
+        if (err) errors.push(err);
+    }
+
+    // Layer 2: DOM ID reference checks (only if no syntax errors)
+    if (errors.length === 0) {
+        const ids = buildIdInventory(html);
+        const dynamicIds = collectDynamicIds(scripts);
+        for (const id of dynamicIds) ids.add(id);
+        errors.push(...checkIdReferences(scripts, ids));
+    }
+
+    // Layer 3: SynthOS API pattern checks (always runs)
+    errors.push(...checkSynthosApi(scripts));
+
+    return {
+        valid: errors.length === 0,
+        errors,
+        durationMs: Date.now() - start,
+    };
+}

package/src/service/server.ts

@@ -4,6 +4,7 @@ import { useApiRoutes } from './useApiRoutes';
 import { SynthOSConfig } from '../init';
 import { useDataRoutes } from './useDataRoutes';
 import { useFileRoutes } from './useFileRoutes';
+import { useExtractRoutes } from './useExtractRoutes';
 import { useSharedDataRoutes } from './useSharedDataRoutes';
 import { useSharedFileRoutes } from './useSharedFileRoutes';
 import { useConnectorRoutes } from './useConnectorRoutes';
@@ -59,6 +60,9 @@ export function server(config: SynthOSConfig, customizer: Customizer = defaultCu
     // File routes
     if (customizer.isEnabled('files')) useFileRoutes(config, app);
 
+    // Extract routes — schema-typed file extraction via active chat model
+    if (customizer.isEnabled('extract')) useExtractRoutes(config, app);
+
     // Shared data routes
     if (customizer.isEnabled('shared-data')) useSharedDataRoutes(config, app);
 

package/src/service/sharedTableSchema.ts

@@ -0,0 +1,236 @@
+import path from 'path';
+import { SynthOSConfig } from '../init';
+
+/**
+ * Schema sidecar IO + additive-merge logic. Used by both shared-table and
+ * per-page-table routes — the only difference is the parent folder where the
+ * `<table>.schema.json` file lives.
+ *
+ * Spec: docs/specs/shared-tables-schema-sidecar.md
+ */
+
+export interface SchemaWrapper {
+    version: number;
+    schema: Record<string, unknown>;
+    createdAt: string;
+    updatedAt: string;
+    definedBy?: string;
+}
+
+export type MergeMode = 'additive' | 'replace';
+
+export interface MergeConflict {
+    field: string;
+    existing: unknown;
+    incoming: unknown;
+}
+
+const SCHEMA_VERSION = 1;
+
+/** Sidecar file path: `<parent>/<table>.schema.json`. */
+export function schemaFile(parent: string, table: string): string {
+    return path.join(parent, `${table}.schema.json`);
+}
+
+/** Load the wrapper. Returns undefined if missing or unparsable. */
+export async function loadSchema(config: SynthOSConfig, parent: string, table: string): Promise<SchemaWrapper | undefined> {
+    const sp = config.storageProvider;
+    const file = schemaFile(parent, table);
+    if (!await sp.checkIfExists(file)) return undefined;
+    try {
+        const parsed = JSON.parse(await sp.loadFile(file));
+        if (parsed && typeof parsed === 'object' && parsed.schema && typeof parsed.schema === 'object') {
+            return parsed as SchemaWrapper;
+        }
+    } catch {
+        // fall through
+    }
+    return undefined;
+}
+
+/** Persist a wrapper to disk. */
+export async function saveSchema(config: SynthOSConfig, parent: string, table: string, wrapper: SchemaWrapper): Promise<void> {
+    const sp = config.storageProvider;
+    await sp.ensureFolderExists(parent);
+    await sp.saveFile(schemaFile(parent, table), JSON.stringify(wrapper, null, 4));
+}
+
+/** Delete the schema sidecar. No-op if it doesn't exist. */
+export async function deleteSchema(config: SynthOSConfig, parent: string, table: string): Promise<void> {
+    const sp = config.storageProvider;
+    const file = schemaFile(parent, table);
+    if (await sp.checkIfExists(file)) {
+        await sp.deleteFile(file);
+    }
+}
+
+/**
+ * Apply `incoming` against `existing` per `mode`:
+ *   - 'replace'  → drop existing entirely; incoming wins.
+ *   - 'additive' → union of `properties` (existing wins on overlap if types match;
+ *                  conflicting types reported via `conflicts`); union of `required`.
+ *
+ * Returns the merged schema and any conflicts. On conflict, callers should
+ * surface them as a 409 and NOT persist the result.
+ */
+export function mergeSchema(
+    existing: Record<string, unknown> | undefined,
+    incoming: Record<string, unknown>,
+    mode: MergeMode,
+): { merged: Record<string, unknown>; conflicts: MergeConflict[] } {
+    if (mode === 'replace' || !existing) {
+        return { merged: incoming, conflicts: [] };
+    }
+
+    const conflicts: MergeConflict[] = [];
+
+    const existingProps = isPlainObject(existing.properties) ? existing.properties as Record<string, unknown> : {};
+    const incomingProps = isPlainObject(incoming.properties) ? incoming.properties as Record<string, unknown> : {};
+    const mergedProps: Record<string, unknown> = { ...existingProps };
+
+    for (const [field, def] of Object.entries(incomingProps)) {
+        if (!(field in existingProps)) {
+            mergedProps[field] = def;
+            continue;
+        }
+        const existingDef = existingProps[field];
+        if (typesAreCompatible(existingDef, def)) {
+            // Existing wins on overlap (preserves enum/required/format choices).
+            continue;
+        }
+        conflicts.push({ field, existing: existingDef, incoming: def });
+    }
+
+    if (conflicts.length > 0) {
+        return { merged: existing, conflicts };
+    }
+
+    const existingRequired = Array.isArray(existing.required) ? existing.required as unknown[] : [];
+    const incomingRequired = Array.isArray(incoming.required) ? incoming.required as unknown[] : [];
+    const mergedRequired = Array.from(new Set([...existingRequired, ...incomingRequired]
+        .filter(r => typeof r === 'string')));
+
+    const merged: Record<string, unknown> = {
+        ...existing,
+        properties: mergedProps,
+    };
+    if (mergedRequired.length > 0) {
+        merged.required = mergedRequired;
+    }
+    return { merged, conflicts: [] };
+}
+
+/**
+ * Validate the structural shape of an incoming JSON Schema. Strict-mode
+ * validation against the meta-schema is out of scope; we only check that the
+ * payload is a plain object so we don't persist garbage.
+ */
+export function isValidSchemaPayload(s: unknown): s is Record<string, unknown> {
+    return isPlainObject(s);
+}
+
+function isPlainObject(v: unknown): v is Record<string, unknown> {
+    return !!v && typeof v === 'object' && !Array.isArray(v);
+}
+
+/**
+ * Type-compatibility check used by additive merge. Two definitions are
+ * compatible when their `type` fields match (or both are absent). We
+ * intentionally tolerate other field differences (description, enum widening,
+ * format changes) — incoming-vs-existing field-level diffs aren't surfaced
+ * as conflicts in v1; only top-level type mismatches block the merge.
+ */
+function typesAreCompatible(existing: unknown, incoming: unknown): boolean {
+    if (!isPlainObject(existing) || !isPlainObject(incoming)) return existing === incoming;
+    const et = existing.type;
+    const it = incoming.type;
+    if (et === undefined && it === undefined) return true;
+    if (et === undefined || it === undefined) return true;
+    if (Array.isArray(et) || Array.isArray(it)) {
+        const ea = Array.isArray(et) ? et : [et];
+        const ia = Array.isArray(it) ? it : [it];
+        return ea.some(x => ia.includes(x));
+    }
+    return et === it;
+}
+
+/**
+ * Build a SchemaWrapper from a plain JSON Schema, using `now` for both
+ * timestamps when no existing wrapper is being preserved.
+ */
+export function newSchemaWrapper(schema: Record<string, unknown>, now: string, definedBy?: string): SchemaWrapper {
+    const wrapper: SchemaWrapper = {
+        version: SCHEMA_VERSION,
+        schema,
+        createdAt: now,
+        updatedAt: now,
+    };
+    if (definedBy) wrapper.definedBy = definedBy;
+    return wrapper;
+}
+
+/**
+ * Update an existing wrapper, preserving createdAt and bumping updatedAt.
+ */
+export function updateSchemaWrapper(
+    existing: SchemaWrapper,
+    schema: Record<string, unknown>,
+    now: string,
+    definedBy?: string,
+): SchemaWrapper {
+    const wrapper: SchemaWrapper = {
+        version: existing.version || SCHEMA_VERSION,
+        schema,
+        createdAt: existing.createdAt,
+        updatedAt: now,
+    };
+    if (definedBy) wrapper.definedBy = definedBy;
+    else if (existing.definedBy) wrapper.definedBy = existing.definedBy;
+    return wrapper;
+}
+
+/**
+ * Enumerate tables in a namespace folder. A "table" is either:
+ *   - a subfolder containing record files, OR
+ *   - a `<name>.schema.json` sidecar (table may be schemaless until first save).
+ *
+ * Returns one entry per unique table name, with hasSchema + recordCount.
+ *
+ * `reserved` filters out names that occupy the namespace for non-table use
+ * (e.g. `files`, the per-page uploads folder).
+ */
+export async function listTables(
+    config: SynthOSConfig,
+    parent: string,
+    reserved: ReadonlySet<string> = new Set(),
+): Promise<Array<{ name: string; hasSchema: boolean; recordCount: number }>> {
+    const sp = config.storageProvider;
+    if (!await sp.checkIfExists(parent)) return [];
+
+    const folders = await sp.listFolders(parent);
+    const files = await sp.listFiles(parent);
+
+    const known = new Map<string, { hasSchema: boolean; recordCount: number }>();
+    for (const folder of folders) {
+        if (reserved.has(folder)) continue;
+        const recs = (await sp.listFiles(path.join(parent, folder)))
+            .filter(f => f.endsWith('.json')).length;
+        known.set(folder, { hasSchema: false, recordCount: recs });
+    }
+    for (const file of files) {
+        const m = file.match(/^(.+)\.schema\.json$/);
+        if (!m) continue;
+        const name = m[1];
+        if (reserved.has(name)) continue;
+        const existing = known.get(name);
+        if (existing) {
+            existing.hasSchema = true;
+        } else {
+            known.set(name, { hasSchema: true, recordCount: 0 });
+        }
+    }
+
+    return Array.from(known.entries())
+        .map(([name, info]) => ({ name, ...info }))
+        .sort((a, b) => a.name.localeCompare(b.name));
+}