suemo 0.0.1 → 0.0.2

package/src/cognitive/consolidate.ts

@@ -1,5 +1,6 @@
 import type { Surreal } from 'surrealdb'
-import type { LLMConfig } from '../config.ts'
+import type { LLMConfig, SuemoConfig } from '../config.ts'
+import { getEmbedding } from '../embedding/index.ts'
 import { getLogger } from '../logger.ts'
 import type { ConsolidationRun, MemoryNode } from '../types.ts'
 
@@ -40,6 +41,7 @@ async function runNREM(
 	db: Surreal,
 	llm: LLMConfig,
 	similarityThreshold: number,
+	embeddingConfig: SuemoConfig['embedding'],
 ): Promise<{ nodesIn: number; nodesOut: number }> {
 	log.info('NREM phase starting')
 
@@ -75,12 +77,21 @@ async function runNREM(
 		// Find similar unassigned nodes using DB-side cosine
 		const similarResult = await db.query<[{ id: string; score: number }[]]>(
 			`
-      SELECT id, vector::similarity::cosine(embedding, $emb) AS score
-      FROM memory
-      WHERE consolidated = false
-        AND id != $self
-        AND (valid_until = NONE OR valid_until > time::now())
-      ORDER BY embedding <|10, 20|> $emb
+      LET $cand = (
+        SELECT id, embedding
+        FROM memory
+        WHERE consolidated = false
+          AND id != $self
+          AND (valid_until = NONE OR valid_until > time::now())
+          AND embedding <|10, 20|> $emb
+      );
+
+      SELECT id, score
+      FROM (
+        SELECT id, vector::similarity::cosine(embedding, $emb) AS score
+        FROM $cand
+      )
+      ORDER BY score DESC
       LIMIT 10
     `,
 			{ emb: node.embedding, self: node.id },
@@ -103,7 +114,10 @@ async function runNREM(
 	for (const cluster of clusters) {
 		if (cluster.length === 1) {
 			// Singleton — mark consolidated without compression
-			await db.query('UPDATE $id SET consolidated = true', { id: cluster[0]!.id })
+			await db.query(
+				'UPDATE <record<memory>>$id SET consolidated = true',
+				{ id: cluster[0]!.id },
+			)
 			nodesOut++
 			continue
 		}
@@ -122,13 +136,22 @@ async function runNREM(
 		} catch (e) {
 			log.error('LLM compression failed — marking cluster nodes individually', { error: String(e) })
 			for (const n of cluster) {
-				await db.query('UPDATE $id SET consolidated = true', { id: n.id })
+				await db.query(
+					'UPDATE <record<memory>>$id SET consolidated = true',
+					{ id: n.id },
+				)
 			}
 			nodesOut += cluster.length
 			continue
 		}
 
 		// Create the compressed summary node
+		const { clause: embeddingClause, param: embeddingParam } = await getEmbedding(summary, embeddingConfig)
+		log.debug('NREM summary embedding resolved', {
+			provider: embeddingConfig.provider,
+			hasVectorParam: Boolean(embeddingParam),
+			dimension: embeddingParam?.length,
+		})
 		const created = await db.query<[MemoryNode[]]>(
 			`
       CREATE memory CONTENT {
@@ -137,7 +160,7 @@ async function runNREM(
         summary:    $summary,
         tags:       $tags,
         scope:      $scope,
-        embedding:  fn::embed($summary),
+        embedding:  ${embeddingClause},
         confidence: 1.0,
         salience:   0.7,
         source:     'consolidation:nrem',
@@ -151,7 +174,8 @@ async function runNREM(
 			{
 				summary,
 				tags: [...new Set(cluster.flatMap((n) => n.tags))],
-				scope: cluster[0]!.scope ?? null,
+				...(cluster[0]!.scope !== null && cluster[0]!.scope !== undefined ? { scope: cluster[0]!.scope } : {}),
+				...(embeddingParam ? { embedding: embeddingParam } : {}),
 			},
 		)
 		const summaryNode = created[0]?.[0]
@@ -161,7 +185,7 @@ async function runNREM(
 		for (const n of cluster) {
 			await db.query(
 				`
-        UPDATE $id SET
+        UPDATE <record<memory>>$id SET
           consolidated      = true,
           consolidated_into = $into,
           updated_at        = time::now()
@@ -214,10 +238,13 @@ async function runREM(
 		const candidates = await db.query<[MemoryNode[]]>(
 			`
       SELECT * FROM memory
-      WHERE id != $self
-        AND consolidated = true
-        AND (valid_until = NONE OR valid_until > time::now())
-      ORDER BY embedding <|10, 40|> $emb
+      WHERE id IN (
+        SELECT VALUE id FROM memory
+        WHERE id != $self
+          AND consolidated = true
+          AND (valid_until = NONE OR valid_until > time::now())
+          AND embedding <|10, 40|> $emb
+      )
       LIMIT 10
     `,
 			{ self: node.id, emb: node.embedding },
@@ -274,7 +301,7 @@ async function runREM(
 
 		// Bump salience on newly connected node
 		await db.query(
-			'UPDATE $id SET salience = math::min(salience + 0.1, 1.0)',
+			'UPDATE <record<memory>>$id SET salience = math::min(salience + 0.1, 1.0)',
 			{ id: node.id },
 		)
 	}
@@ -288,6 +315,7 @@ export async function consolidate(
 		nremSimilarityThreshold?: number
 		remRelationThreshold?: number
 		llm: LLMConfig
+		embedding: SuemoConfig['embedding']
 	},
 ): Promise<ConsolidationRun> {
 	const phase = opts.nremOnly ? 'nrem' : 'full'
@@ -313,6 +341,7 @@ export async function consolidate(
 			db,
 			opts.llm,
 			opts.nremSimilarityThreshold ?? 0.85,
+			opts.embedding,
 		)
 
 		if (!opts.nremOnly) {
@@ -321,7 +350,7 @@ export async function consolidate(
 
 		await db.query(
 			`
-      UPDATE $id SET
+      UPDATE <record<consolidation_run>>$id SET
         completed_at = time::now(),
         status       = 'done',
         nodes_in     = $nodesIn,
@@ -337,7 +366,7 @@ export async function consolidate(
 		log.error('consolidate() failed', { error: errMsg })
 		await db.query(
 			`
-      UPDATE $id SET
+      UPDATE <record<consolidation_run>>$id SET
         completed_at = time::now(),
         status       = 'failed',
         error        = $err

package/src/cognitive/contradiction.ts

@@ -23,12 +23,21 @@ export async function detectContradiction(
 
 	const candidates = await db.query<[{ id: string; score: number }[]]>(
 		`
-    SELECT id, vector::similarity::cosine(embedding, $emb) AS score
-    FROM memory
-    WHERE kind = 'belief'
-      AND id != $self
-      AND (valid_until = NONE OR valid_until > time::now())
-    ORDER BY embedding <|3, 20|> $emb
+    LET $cand = (
+      SELECT id, embedding
+      FROM memory
+      WHERE kind = 'belief'
+        AND id != $self
+        AND (valid_until = NONE OR valid_until > time::now())
+        AND embedding <|3, 20|> $emb
+    );
+
+    SELECT id, score
+    FROM (
+      SELECT id, vector::similarity::cosine(embedding, $emb) AS score
+      FROM $cand
+    )
+    ORDER BY score DESC
     LIMIT 3
   `,
 		{ emb: newNode.embedding, self: newNode.id },
@@ -41,7 +50,10 @@ export async function detectContradiction(
 	if (best.score < similarityThreshold) return null
 
 	// Fetch full node
-	const result = await db.query<[MemoryNode[]]>('SELECT * FROM $id', { id: best.id })
+	const result = await db.query<[MemoryNode[]]>(
+		'SELECT * FROM <record<memory>>$id',
+		{ id: best.id },
+	)
 	const existing = result[0]?.[0]
 	if (!existing) return null
 

package/src/config.template.ts

@@ -0,0 +1,36 @@
+import { defineConfig } from 'suemo'
+
+export default defineConfig({
+	surreal: {
+		url: process.env.SUEMO_URL ?? 'ws://localhost:8000',
+		namespace: process.env.SUEMO_NS ?? 'suemo',
+		database: process.env.SUEMO_DB ?? 'suemo',
+		auth: {
+			user: process.env.SUEMO_USER ?? 'root',
+			pass: process.env.SUEMO_PASS ?? 'pass',
+		},
+	},
+	embedding: {
+		provider: 'stub',
+		dimension: 384,
+	},
+	consolidation: {
+		trigger: 'timer',
+		intervalMinutes: 30,
+		reactiveThreshold: 50,
+		nremSimilarityThreshold: 0.85,
+		remRelationThreshold: 0.4,
+		llm: {
+			url: process.env.SUEMO_LLM_URL ?? 'https://api.z.ai/api/coding/paas/v4',
+			model: process.env.SUEMO_LLM_MODEL ?? 'glm-4.7-flash',
+			apiKey: process.env.SUEMO_LLM_API_KEY!,
+		},
+	},
+	retrieval: {
+		weights: { vector: 0.5, bm25: 0.25, graph: 0.15, temporal: 0.1 },
+	},
+	mcp: {
+		port: Number(process.env.SUEMO_PORT) || 4242,
+		host: '127.0.0.1',
+	},
+})

package/src/config.ts

@@ -1,6 +1,7 @@
 import { existsSync } from 'node:fs'
 import { join, resolve } from 'node:path'
 import { pathToFileURL } from 'node:url'
+import { getLogger } from './logger.ts'
 
 // ── Config shape ──────────────────────────────────────────────────────────────
 
@@ -17,8 +18,9 @@ export interface SurrealTarget {
 }
 
 export type EmbeddingProvider =
+	| { provider: 'openai-compatible'; url: string; model: string; dimension: number; apiKey?: string }
 	| { provider: 'surreal'; dimension: number }
-	| { provider: 'stub'; dimension: number } // test-only
+	| { provider: 'stub'; dimension: number }
 
 export interface LLMConfig {
 	url: string // OpenAI-compatible endpoint
@@ -76,11 +78,21 @@ const CONFIG_CANDIDATES = [
 	'suemo.config.js',
 ]
 
-const HOME_CONFIG = join(
-	process.env.HOME ?? process.env.USERPROFILE ?? '~',
-	'.suemo',
-	'suemo.ts',
-)
+const log = getLogger(['suemo', 'config'])
+
+function getHomeConfigPath(): string | null {
+	const home = process.env.HOME ?? process.env.USERPROFILE
+	if (!home) return null
+	return join(home, '.suemo', 'suemo.ts')
+}
+
+function resolveConfigPath(p: string): string {
+	if (p.startsWith('~/')) {
+		const home = process.env.HOME ?? process.env.USERPROFILE
+		if (home) return resolve(home, p.slice(2))
+	}
+	return resolve(p)
+}
 
 export async function loadConfig(
 	cwd = process.cwd(),
@@ -88,23 +100,40 @@ export async function loadConfig(
 ): Promise<SuemoConfig> {
 	// 1. Explicit --config flag
 	if (overridePath) {
-		return importConfig(resolve(overridePath))
+		const resolved = resolveConfigPath(overridePath)
+		log.debug('Loading config from --config', { path: resolved })
+		return importConfig(resolved)
+	}
+	// 2. Environment override
+	const envConfigPath = process.env.SUEMO_CONFIG_PATH?.trim()
+	if (envConfigPath) {
+		const resolved = resolveConfigPath(envConfigPath)
+		log.debug('Loading config from SUEMO_CONFIG_PATH', { path: resolved })
+		return importConfig(resolved)
 	}
-	// 2. Project-local
+	// 3. Project-local
 	for (const name of CONFIG_CANDIDATES) {
 		const p = resolve(cwd, name)
-		if (existsSync(p)) return importConfig(p)
+		if (existsSync(p)) {
+			log.debug('Loading project-local config', { path: p })
+			return importConfig(p)
+		}
+	}
+	// 4. User-level
+	const homeConfig = getHomeConfigPath()
+	if (homeConfig && existsSync(homeConfig)) {
+		log.debug('Loading user-level config', { path: homeConfig })
+		return importConfig(homeConfig)
 	}
-	// 3. User-level
-	if (existsSync(HOME_CONFIG)) return importConfig(HOME_CONFIG)
 
 	throw new Error(
 		'No suemo config found.\n'
-			+ 'Run `suemo init` to create ~/.suemo/suemo.ts, or create suemo.config.ts in the project root.',
+			+ 'Run `suemo init config` to create ~/.suemo/suemo.ts, set SUEMO_CONFIG_PATH, or create suemo.config.ts in the project root.',
 	)
 }
 
 async function importConfig(path: string): Promise<SuemoConfig> {
+	log.debug('Importing config module', { path })
 	const mod = await import(pathToFileURL(path).href)
 	const cfg: unknown = mod.default ?? mod
 	if (!cfg || typeof cfg !== 'object') {

package/src/db/preflight.ts

@@ -12,18 +12,33 @@ export interface CompatibilityResult {
 	errors: string[]
 }
 
+export interface CompatibilityOptions {
+	/**
+	 * Whether missing fn::embed() should fail compatibility.
+	 * Keep true for runtime paths that execute vector/embed queries.
+	 */
+	requireEmbedding?: boolean
+	/**
+	 * Human-readable label for logging this preflight execution.
+	 */
+	context?: string
+}
+
 const MIN_MAJOR = 3
 
 export async function checkCompatibility(
 	db: Surreal,
+	options: CompatibilityOptions = {},
 ): Promise<CompatibilityResult> {
 	const errors: string[] = []
 	let surrealVersion = 'unknown'
 	let surrealkv = false
 	let retention_ok = false
 	let embedding = false
+	const requireEmbedding = options.requireEmbedding ?? true
+	const context = options.context ?? 'default'
 
-	log.info('Running preflight compatibility checks')
+	log.info('Running preflight compatibility checks', { requireEmbedding, context })
 
 	// ── Check 1: version string ───────────────────────────────────────────────
 	try {
@@ -41,6 +56,7 @@ export async function checkCompatibility(
 				)
 			}
 		}
+		log.debug('SurrealDB version check complete', { surrealVersion })
 	} catch (e) {
 		errors.push(`Cannot retrieve SurrealDB version: ${String(e)}`)
 	}
@@ -54,6 +70,7 @@ export async function checkCompatibility(
 		// if it was around then), not throw. An error means VERSION is unsupported.
 		await db.query("SELECT * FROM suemo_preflight:probe VERSION d'2020-01-01T00:00:00Z'")
 		surrealkv = true
+		log.debug('SurrealKV VERSION probe passed')
 	} catch (e: unknown) {
 		const msg = String(e).toLowerCase()
 		if (msg.includes('version') || msg.includes('surrealkv') || msg.includes('not supported')) {
@@ -83,6 +100,7 @@ export async function checkCompatibility(
 			await db.query('CREATE suemo_retention_probe:v SET t = time::now()')
 			await db.query(`SELECT * FROM suemo_retention_probe:v VERSION d'${ninetyDaysAgo}'`)
 			retention_ok = true
+			log.debug('Retention probe passed', { ninetyDaysAgo })
 		} catch (e: unknown) {
 			const msg = String(e).toLowerCase()
 			if (msg.includes('retention') || msg.includes('version')) {
@@ -104,20 +122,28 @@ export async function checkCompatibility(
 
 	// ── Check 4: fn::embed() resolves ────────────────────────────────────────
 	// We don't actually embed anything — we just check that the function exists.
-	// An "Unknown function" error means the embedding plugin is not configured.
+	// An "Unknown function" error means embedding runtime is unavailable.
 	try {
 		// fn::embed requires the ML module and a configured model.
 		// If it throws "No embedding model configured" that's acceptable — the
 		// function exists. If it throws "Unknown function 'fn::embed'" → not available.
 		await db.query(`RETURN fn::embed("suemo preflight test")`)
 		embedding = true
+		log.debug('Embedding function probe passed')
 	} catch (e: unknown) {
 		const msg = String(e).toLowerCase()
 		if (msg.includes('unknown function') || msg.includes('fn::embed')) {
-			errors.push(
-				'fn::embed() is not available. Configure an embedding model in SurrealDB, or '
-					+ "use embedding.provider = 'stub' in suemo config (test-only).",
-			)
+			embedding = false
+			if (requireEmbedding) {
+				errors.push(
+					'fn::embed() is not available in this SurrealDB database. Import/configure a SurrealML embedding model for this namespace/database, then retry.',
+				)
+			} else {
+				log.warn('Embedding function unavailable; continuing due to non-strict preflight mode', {
+					context,
+					error: String(e),
+				})
+			}
 		} else {
 			// Other error (e.g. no model configured but function exists) — treat as available
 			embedding = true

package/src/db/schema.surql

@@ -30,27 +30,30 @@ DEFINE FIELD OVERWRITE consolidated ON memory TYPE bool           DEFAULT false;
 
 -- REFERENCES: bidirectional record link (SurrealDB v3.0)
 DEFINE FIELD OVERWRITE consolidated_into ON memory
-  TYPE option<record<memory>> REFERENCES;
+  TYPE option<record<memory>> REFERENCE;
 
 -- FSRS spaced-repetition fields (all optional; populated on first recall())
 DEFINE FIELD OVERWRITE fsrs_stability   ON memory TYPE option<float>;
 DEFINE FIELD OVERWRITE fsrs_difficulty  ON memory TYPE option<float>;
 DEFINE FIELD OVERWRITE fsrs_next_review ON memory TYPE option<datetime>;
 
--- Vector index: HNSW, DEFER so init does not block on large existing datasets
+-- Vector index: HNSW (keep syntax compatible with current SurrealDB target)
 DEFINE INDEX OVERWRITE idx_memory_embedding
   ON memory FIELDS embedding
-  HNSW DIMENSION 1536 DIST COSINE
-  DEFER;
+  HNSW DIMENSION 384 DIST COSINE;
 
 -- Full-text index: BM25 over content + summary
 DEFINE ANALYZER OVERWRITE suemo_analyzer
   TOKENIZERS blank,class
   FILTERS lowercase,snowball(english);
 
-DEFINE INDEX OVERWRITE idx_memory_fts
-  ON memory FIELDS content, summary
-  SEARCH ANALYZER suemo_analyzer BM25;
+DEFINE INDEX OVERWRITE idx_memory_content_fts
+  ON memory FIELDS content
+  FULLTEXT ANALYZER suemo_analyzer BM25;
+
+DEFINE INDEX OVERWRITE idx_memory_summary_fts
+  ON memory FIELDS summary
+  FULLTEXT ANALYZER suemo_analyzer BM25;
 
 -- Compound indexes for timeline and filter queries
 DEFINE INDEX OVERWRITE idx_memory_scope_time
@@ -84,7 +87,7 @@ DEFINE FIELD OVERWRITE summary     ON episode TYPE option<string>;
 
 -- REFERENCES: bidirectional array of memory links
 DEFINE FIELD OVERWRITE memory_ids  ON episode
-  TYPE array<record<memory>> REFERENCES
+  TYPE array<record<memory>> REFERENCE
   DEFAULT [];
 
 -- ── consolidation_run ────────────────────────────────────────────────────────

package/src/db/schema.ts

@@ -11,12 +11,15 @@ const log = getLogger(['suemo', 'db', 'schema'])
 export async function runSchema(db: Surreal): Promise<void> {
 	log.info('Running schema migrations')
 	const statements = SCHEMA.split(/;\s*\n/).map((s) => s.trim()).filter(Boolean)
-	for (const stmt of statements) {
+	log.debug('Prepared schema statements', { count: statements.length, schemaBytes: SCHEMA.length })
+	for (const [index, stmt] of statements.entries()) {
 		try {
+			const snippet = stmt.length > 160 ? `${stmt.slice(0, 160)}…` : stmt
+			log.debug('Executing schema statement', { index, snippet })
 			await db.query(stmt)
-			log.debug('Schema statement OK', { stmt: stmt.slice(0, 60) })
+			log.debug('Schema statement OK', { index, stmt: stmt.slice(0, 60) })
 		} catch (e) {
-			log.error('Schema statement failed', { stmt, error: String(e) })
+			log.error('Schema statement failed', { index, stmt, error: String(e) })
 			throw e
 		}
 	}

package/src/embedding/index.ts

@@ -0,0 +1,52 @@
+import type { EmbeddingProvider } from '../config.ts'
+import { getLogger } from '../logger.ts'
+import { embedText } from './openai-compatible.ts'
+
+const log = getLogger(['suemo', 'embedding'])
+
+export interface EmbeddingResult {
+	clause: string
+	param?: number[]
+}
+
+export async function getEmbedding(
+	text: string,
+	config: EmbeddingProvider,
+): Promise<EmbeddingResult> {
+	log.debug('getEmbedding()', { provider: config.provider, dimension: config.dimension })
+
+	switch (config.provider) {
+		case 'surreal':
+			return { clause: 'fn::embed($content)' }
+
+		case 'openai-compatible': {
+			const vec = await embedText(text, config)
+			if (vec.length !== config.dimension) {
+				throw new Error(
+					`Embedding dimension mismatch: expected ${config.dimension}, got ${vec.length}`,
+				)
+			}
+			return { clause: '$embedding', param: vec }
+		}
+
+		case 'stub': {
+			const vec = new Array(config.dimension).fill(0)
+			log.debug('stub embedding: returning zero vector', { dimension: config.dimension })
+			return { clause: '$embedding', param: vec }
+		}
+	}
+}
+
+export function buildEmbeddingClause(config: EmbeddingProvider): string {
+	switch (config.provider) {
+		case 'surreal':
+			return 'fn::embed($content)'
+		case 'openai-compatible':
+		case 'stub':
+			return '$embedding'
+	}
+}
+
+export function getZeroVector(dimension: number): number[] {
+	return new Array(dimension).fill(0)
+}

package/src/embedding/openai-compatible.ts

@@ -0,0 +1,43 @@
+// src/embedding/openai_compatible.ts
+import { getLogger } from "../logger.ts";
+
+const log = getLogger(["suemo", "embedding", "openai-compatible"]);
+
+export interface OpenAICompatibleEmbeddingConfig {
+	url: string;       // e.g. http://127.0.0.1:8080/v1/embeddings
+	model: string;
+	dimension: number;
+	apiKey?: string;
+}
+
+export async function embedText(
+	text: string,
+	config: OpenAICompatibleEmbeddingConfig,
+): Promise<number[]> {
+	log.debug("embedText()", { textPreview: text.slice(0, 60) });
+
+	const res = await fetch(config.url, {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+			"Authorization": `Bearer ${config.apiKey ?? "local"}`,
+		},
+		body: JSON.stringify({ input: text, model: config.model }),
+	});
+
+	if (!res.ok) {
+		throw new Error(`Embedding server error ${res.status}: ${await res.text()}`);
+	}
+
+	const json = await res.json() as {
+		data: { embedding: number[]; index: number }[];
+	};
+
+	const vec = json.data[0]?.embedding;
+	if (!vec) throw new Error("Embedding server returned empty data");
+	if (vec.length !== config.dimension) {
+		throw new Error(`Dimension mismatch: expected ${config.dimension}, got ${vec.length}`);
+	}
+
+	return vec;
+}

package/src/goal.ts

@@ -1,4 +1,5 @@
 import type { Surreal } from 'surrealdb'
+import type { SuemoConfig } from './config.ts'
 import { getLogger } from './logger.ts'
 import { invalidate, observe } from './memory/write.ts'
 import type { MemoryNode } from './types.ts'
@@ -8,10 +9,11 @@ const log = getLogger(['suemo', 'goal'])
 export async function goalSet(
 	db: Surreal,
 	content: string,
+	config: SuemoConfig,
 	opts: { scope?: string; tags?: string[] } = {},
 ): Promise<MemoryNode> {
 	log.info('goalSet()', { content: content.slice(0, 60) })
-	return observe(db, { content, kind: 'goal', ...opts })
+	return observe(db, { content, kind: 'goal', ...opts }, config)
 }
 
 export async function goalResolve(db: Surreal, goalId: string): Promise<void> {