suemo 0.0.1 → 0.0.3

package/README.md

@@ -58,23 +58,24 @@ bun install
 **1. Create config**
 
 ```sh
-bun run src/cli/index.ts init
-# or, once linked: suemo init
+bun run src/cli/index.ts init config
+# or, once linked: suemo init config
 ```
 
 This writes `~/.suemo/suemo.ts`. Edit it with your SurrealDB URL, credentials, and LLM endpoint.
 
 **2. Apply schema**
 
-`suemo init` applies the schema automatically if `SURREAL_*` env vars are set. Otherwise:
+Apply schema after you set/edit namespace/database:
 
 ```sh
-SURREAL_URL=ws://localhost:8000 \
-SURREAL_USER=root \
-SURREAL_PASS=root \
-suemo init
+suemo init schema
 ```
 
+Pass `--yes` to skip schema confirmation in non-interactive flows.
+
+`suemo init` now shows guidance by default; run `init config` and `init schema` explicitly.
+
 **3. Store a memory**
 
 ```sh
@@ -107,8 +108,10 @@ Global flags (inherited by all commands):
   -d, --debug           Verbose debug logging
 
 Commands:
-  init                  Create config template and apply DB schema
-  serve                 Start the MCP server
+  init                  Show init subcommands and usage guidance
+  init config           Create/update ~/.suemo/suemo.ts
+  init schema           Apply DB schema from current config (with confirm)
+  serve                 Start the MCP server (HTTP or stdio)
   observe <content>     Store an observation
   believe <content>     Store a belief (triggers contradiction detection)
   query <input>         Hybrid semantic search
@@ -119,9 +122,11 @@ Commands:
   goal list             List active goals
   goal resolve <id>     Mark a goal achieved
   consolidate           Run NREM + REM consolidation pipeline
+  doctor embed          Diagnose fn::embed setup and print fix steps
   health                Memory health report
   health vitals         Last 10 consolidation runs + node counts
-  sync                  Push memories to a remote SurrealDB instance
+  health stats          Lightweight usage counters and timestamps
+	sync                  Sync memories with a remote SurrealDB instance
   export                Stream memories to JSONL on stdout
   import <file>         Import memories from a JSONL file
 ```
@@ -130,7 +135,16 @@ Commands:
 
 ## Config
 
-Config lives at `~/.suemo/suemo.ts` (user-level) or `suemo.config.ts` in the project root. Project-local takes precedence. Override with `--config <path>`.
+Config lives at `~/.suemo/suemo.ts` (user-level) or `suemo.config.ts` in the project root. Project-local takes precedence.
+
+Resolution order:
+
+1. `--config <path>`
+2. `SUEMO_CONFIG_PATH`
+3. project-local (`suemo.config.ts` / `suemo.config.js`)
+4. user-level (`~/.suemo/suemo.ts`)
+
+Set `SUEMO_DEBUG=1` (or `true`, `yes`, `on`) to enable debug logging globally for CLI commands.
 
 ```ts
 // ~/.suemo/suemo.ts
@@ -163,9 +177,56 @@ export default defineConfig({
 		weights: { vector: 0.5, bm25: 0.25, graph: 0.15, temporal: 0.1 },
 	},
 	mcp: { port: 4242, host: '127.0.0.1' },
+	sync: {
+		remotes: {
+			vps: {
+				url: process.env.SUEMO_SYNC_URL!,
+				namespace: process.env.SUEMO_SYNC_NS!,
+				database: process.env.SUEMO_SYNC_DB!,
+				auth: {
+					user: process.env.SUEMO_SYNC_USER!,
+					pass: process.env.SUEMO_SYNC_PASS!,
+				},
+			},
+		},
+		defaultRemote: 'vps',
+		auto: {
+			enabled: false,
+			intervalSeconds: 300,
+			direction: 'push',
+			remote: 'vps',
+			onWrite: false,
+			minWriteIntervalSeconds: 30,
+		},
+	},
 })
 ```
 
+### Sync config notes
+
+- `sync.remotes` supports multiple named remote instances.
+- `sync.defaultRemote` is used by `suemo sync` unless `--remote` is provided.
+- Legacy `sync.remote` (single target) is still supported for migration compatibility.
+- `sync.auto` powers background sync in `suemo serve` (HTTP and stdio):
+  - `enabled`: master switch
+  - `intervalSeconds`: timer cadence
+  - `direction`: `push | pull | both`
+  - `remote`: named target from `sync.remotes`
+  - `onWrite`: trigger sync after mutating MCP tools
+  - `minWriteIntervalSeconds`: throttle for write-triggered sync
+
+Manual sync examples:
+
+```sh
+suemo sync --direction both
+suemo sync --remote vps --direction pull
+suemo sync --remote vps --direction push --dry-run
+```
+
+Minimal local↔VPS smoke test guide:
+
+- `data/scenarios/sync-local-vps.md`
+
 Multiple agents on the same machine use separate config files that extend a shared base:
 
 ```ts
@@ -181,26 +242,46 @@ export default {
 suemo serve --config ~/.suemo/opencode.ts
 ```
 
+For local stdio transport (no network port), run:
+
+```sh
+suemo serve --stdio
+```
+
+If stdio startup fails with an embedding preflight error, run:
+
+```sh
+suemo doctor embed
+```
+
+This prints your active target (`url`, `namespace`, `database`) and step-by-step commands to import a model and define `fn::embed()` for that exact database.
+
 ---
 
 ## MCP Tools
 
-| Tool            | Description                                                         |
-| --------------- | ------------------------------------------------------------------- |
-| `observe`       | Store an observation, belief, question, or hypothesis               |
-| `believe`       | Store a belief — auto-detects and invalidates contradicting beliefs |
-| `invalidate`    | Soft-delete a node by ID (sets `valid_until`)                       |
-| `query`         | Hybrid semantic search (vector + BM25 + graph)                      |
-| `recall`        | Fetch a node and its 1-hop neighbourhood; ticks FSRS                |
-| `wander`        | Spreading-activation walk through the graph                         |
-| `timeline`      | Chronological memory slice with optional date range                 |
-| `episode_start` | Begin a bounded session window                                      |
-| `episode_end`   | Close a session, optionally with a summary                          |
-| `goal_set`      | Create a goal node                                                  |
-| `goal_resolve`  | Mark a goal achieved                                                |
-| `goal_list`     | List active (or all) goals                                          |
-| `consolidate`   | Manually trigger NREM + REM consolidation                           |
-| `health`        | Graph health report                                                 |
+| Tool                  | Description                                                         |
+| --------------------- | ------------------------------------------------------------------- |
+| `observe`             | Store an observation, belief, question, or hypothesis               |
+| `believe`             | Store a belief — auto-detects and invalidates contradicting beliefs |
+| `invalidate`          | Soft-delete a node by ID (sets `valid_until`)                       |
+| `query`               | Hybrid semantic search (vector + BM25 + graph)                      |
+| `recall`              | Fetch a node and its 1-hop neighbourhood; ticks FSRS                |
+| `wander`              | Spreading-activation walk through the graph                         |
+| `timeline`            | Chronological memory slice with optional date range                 |
+| `episode_start`       | Begin a bounded session window                                      |
+| `episode_end`         | Close a session, optionally with a summary                          |
+| `goal_set`            | Create a goal node                                                  |
+| `goal_resolve`        | Mark a goal achieved                                                |
+| `goal_list`           | List active (or all) goals                                          |
+| `upsert_by_key`       | Upsert a memory node by stable topic key                            |
+| `capture_prompt`      | Capture raw prompt and link derived observations                    |
+| `session_context_get` | Fetch open episode summary/context by session ID                    |
+| `session_context_set` | Update open episode summary/context by session ID                   |
+| `consolidate`         | Manually trigger NREM + REM consolidation                           |
+| `health`              | Graph health report                                                 |
+| `vitals`              | Last 10 consolidation runs + grouped node counts                    |
+| `stats`               | Lightweight usage stats and write/query counters                    |
 
 Agents never supply temporal fields (`valid_from`, `valid_until`). These are system-managed.
 
@@ -231,6 +312,25 @@ The CLI and MCP server are both thin shells. All logic lives in the domain layer
 
 ---
 
+## Giant end-to-end scenario (all tools / fields)
+
+For full-system stress testing (CLI + MCP + consolidation + relation kinds), use:
+
+- `data/scenarios/giant-scenario.md` — walkthrough + validation queries
+- `data/scenarios/run-giant-cli.sh` — comprehensive CLI flow
+- `data/scenarios/run-giant-mcp.mjs` — comprehensive stdio MCP flow
+
+Example:
+
+```sh
+./data/scenarios/run-giant-cli.sh ~/.suemo/suemo.ts giant-cli-main
+bun data/scenarios/run-giant-mcp.mjs --config ~/.suemo/suemo.ts --scope giant-mcp-main
+```
+
+These scenarios intentionally exercise all memory kinds and all relation kinds, and produce enough incident-like data to observe NREM/REM behavior.
+
+---
+
 ## Stack
 
 - **Runtime** — [Bun](https://bun.sh)

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "suemo",
-	"version": "0.0.1",
+	"version": "0.0.3",
 	"description": "Persistent semantic memory for AI agents — backed by SurrealDB.",
 	"author": {
 		"name": "Umar Alfarouk",

package/src/cli/commands/believe.ts

@@ -1,8 +1,10 @@
 import { loadConfig } from '../../config.ts'
 import { connect, disconnect } from '../../db/client.ts'
-import { initLogger } from '../../logger.ts'
+import { getLogger } from '../../logger.ts'
 import { believe } from '../../memory/write.ts'
-import { app } from '../shared.ts'
+import { app, initCliCommand } from '../shared.ts'
+
+const log = getLogger(['suemo', 'cli', 'believe'])
 
 export const believeCmd = app.sub('believe')
 	.meta({ description: 'Store a belief (triggers contradiction detection)' })
@@ -12,16 +14,24 @@ export const believeCmd = app.sub('believe')
 		confidence: { type: 'number', description: 'Confidence 0.0–1.0', default: 1.0 },
 	})
 	.run(async ({ args, flags }) => {
-		await initLogger({ level: flags.debug ? 'debug' : 'info' })
+		await initCliCommand('believe', { debug: flags.debug, config: flags.config })
 		const config = await loadConfig(process.cwd(), flags.config)
 		const db = await connect(config.surreal)
-		const { node, contradicted } = await believe(db, {
-			content: args.content,
-			scope: flags.scope,
-			confidence: flags.confidence,
-		})
-		await disconnect()
-		const out: Record<string, unknown> = { id: node.id, valid_from: node.valid_from }
-		if (contradicted) out.contradicted = contradicted.id
-		console.log(JSON.stringify(out, null, 2))
+		try {
+			log.debug('Running believe command', {
+				hasScope: Boolean(flags.scope),
+				confidence: flags.confidence,
+				contentLength: args.content.length,
+			})
+			const { node, contradicted } = await believe(db, {
+				content: args.content,
+				scope: flags.scope,
+				confidence: flags.confidence,
+			}, config)
+			const out: Record<string, unknown> = { id: node.id, valid_from: node.valid_from }
+			if (contradicted) out.contradicted = contradicted.id
+			console.log(JSON.stringify(out, null, 2))
+		} finally {
+			await disconnect()
+		}
 	})

package/src/cli/commands/consolidate.ts

@@ -1,8 +1,10 @@
 import { consolidate } from '../../cognitive/consolidate.ts'
 import { loadConfig } from '../../config.ts'
 import { connect, disconnect } from '../../db/client.ts'
-import { initLogger } from '../../logger.ts'
-import { app } from '../shared.ts'
+import { getLogger } from '../../logger.ts'
+import { app, initCliCommand } from '../shared.ts'
+
+const log = getLogger(['suemo', 'cli', 'consolidate'])
 
 export const consolidateCmd = app.sub('consolidate')
 	.meta({ description: 'Manually trigger memory consolidation (NREM + REM)' })
@@ -10,15 +12,20 @@ export const consolidateCmd = app.sub('consolidate')
 		'nrem-only': { type: 'boolean', description: 'Run only NREM (compression) phase', default: false },
 	})
 	.run(async ({ flags }) => {
-		await initLogger({ level: flags.debug ? 'debug' : 'info' })
+		await initCliCommand('consolidate', { debug: flags.debug, config: flags.config })
+		log.debug('Running consolidate command', { nremOnly: flags['nrem-only'] })
 		const config = await loadConfig(process.cwd(), flags.config)
 		const db = await connect(config.surreal)
-		const run = await consolidate(db, {
-			nremOnly: flags['nrem-only'],
-			nremSimilarityThreshold: config.consolidation.nremSimilarityThreshold,
-			remRelationThreshold: config.consolidation.remRelationThreshold,
-			llm: config.consolidation.llm,
-		})
-		await disconnect()
-		console.log(JSON.stringify(run, null, 2))
+		try {
+			const run = await consolidate(db, {
+				nremOnly: flags['nrem-only'],
+				nremSimilarityThreshold: config.consolidation.nremSimilarityThreshold,
+				remRelationThreshold: config.consolidation.remRelationThreshold,
+				llm: config.consolidation.llm,
+				embedding: config.embedding,
+			})
+			console.log(JSON.stringify(run, null, 2))
+		} finally {
+			await disconnect()
+		}
 	})

package/src/cli/commands/doctor.ts

@@ -0,0 +1,123 @@
+import type { Surreal } from 'surrealdb'
+import { loadConfig } from '../../config.ts'
+import { connect, disconnect } from '../../db/client.ts'
+import { checkCompatibility } from '../../db/preflight.ts'
+import { getLogger } from '../../logger.ts'
+import { app, initCliCommand } from '../shared.ts'
+
+const log = getLogger(['suemo', 'cli', 'doctor'])
+
+const doctor = app.sub('doctor')
+	.meta({ description: 'Diagnostics and setup guidance' })
+
+function toCliEndpoint(url: string): string {
+	if (url.startsWith('ws://')) return `http://${url.slice('ws://'.length)}`
+	if (url.startsWith('wss://')) return `https://${url.slice('wss://'.length)}`
+	return url
+}
+
+function printStatus(label: string, ok: boolean, detail?: string): void {
+	const prefix = ok ? '✓' : '✗'
+	if (detail) {
+		console.log(`${prefix} ${label}: ${detail}`)
+		return
+	}
+	console.log(`${prefix} ${label}`)
+}
+
+async function probeEmbed(db: Surreal): Promise<{ ok: boolean; error?: string }> {
+	try {
+		await db.query('RETURN fn::embed("suemo doctor embed probe")')
+		return { ok: true }
+	} catch (error) {
+		return { ok: false, error: String(error) }
+	}
+}
+
+async function detectModelNames(db: Surreal): Promise<string[]> {
+	try {
+		const result = await db.query<unknown[]>('INFO FOR DB')
+		const first = Array.isArray(result) ? result[0] : null
+		const row = Array.isArray(first) ? first[0] : first
+		if (!row || typeof row !== 'object') return []
+		const models = (row as { models?: unknown }).models
+		if (!models || typeof models !== 'object') return []
+		return Object.keys(models as Record<string, unknown>)
+	} catch {
+		return []
+	}
+}
+
+const doctorEmbedCmd = doctor.sub('embed')
+	.meta({ description: 'Diagnose fn::embed() and show setup steps' })
+	.run(async ({ flags }) => {
+		await initCliCommand('doctor embed', { debug: flags.debug, config: flags.config })
+
+		const config = await loadConfig(process.cwd(), flags.config)
+		const endpoint = toCliEndpoint(config.surreal.url)
+
+		console.log('Embedding diagnostics target:')
+		console.log(`  url: ${config.surreal.url}`)
+		console.log(`  namespace: ${config.surreal.namespace}`)
+		console.log(`  database: ${config.surreal.database}`)
+		console.log()
+
+		let db: Surreal | undefined
+		try {
+			db = await connect(config.surreal)
+
+			const compat = await checkCompatibility(db, {
+				requireEmbedding: false,
+				context: 'cli:doctor-embed',
+			})
+			const embedProbe = await probeEmbed(db)
+			const modelNames = await detectModelNames(db)
+
+			printStatus('SurrealDB version', compat.surrealVersion !== 'unknown', compat.surrealVersion)
+			printStatus('SurrealKV', compat.surrealkv)
+			printStatus('Retention >= 90d', compat.retention_ok)
+			printStatus('fn::embed()', embedProbe.ok, embedProbe.ok ? 'callable' : 'not callable')
+			printStatus(
+				'Imported models in DB',
+				modelNames.length > 0,
+				modelNames.length > 0 ? modelNames.join(', ') : 'none detected',
+			)
+
+			if (embedProbe.ok) {
+				console.log('\n✅ fn::embed() is configured. You should be able to run:')
+				console.log(`   suemo serve --stdio --config ${flags.config ?? '~/.suemo/suemo.ts'}`)
+				return
+			}
+
+			log.warn('fn::embed probe failed', { error: embedProbe.error ?? 'unknown' })
+			console.log('\nHow to set up fn::embed() (step-by-step):')
+			console.log('\n1) Ensure SurrealDB CLI exposes ML commands:')
+			console.log('   surreal ml --help')
+			console.log('\n2) Import a .surml embedding model into this exact NS/DB:')
+			console.log(
+				`   surreal ml import --conn ${endpoint} --user <USER> --pass <PASS> --ns ${config.surreal.namespace} --db ${config.surreal.database} path/to/your-model.surml`,
+			)
+			console.log('\n3) Define fn::embed wrapper to your imported model:')
+			console.log(
+				`   surreal sql --conn ${endpoint} --user <USER> --pass <PASS> --ns ${config.surreal.namespace} --db ${config.surreal.database} --query "DEFINE FUNCTION OVERWRITE fn::embed(\\$text: string) { RETURN ml::your_model<1.0.0>(\\$text); };"`,
+			)
+			console.log('\n4) Verify function now works:')
+			console.log(
+				`   surreal sql --conn ${endpoint} --user <USER> --pass <PASS> --ns ${config.surreal.namespace} --db ${config.surreal.database} --query 'RETURN fn::embed("suemo test");'`,
+			)
+			console.log('\n5) Start MCP stdio server:')
+			console.log(`   suemo serve --stdio --config ${flags.config ?? '~/.suemo/suemo.ts'}`)
+			console.log('\nIf step 4 still fails, rerun: suemo doctor embed --config ~/.suemo/suemo.ts --debug')
+		} finally {
+			if (db) await disconnect()
+		}
+	})
+
+export const doctorCmd = doctor
+	.command(doctorEmbedCmd)
+	.run(async ({ flags }) => {
+		await initCliCommand('doctor', { debug: flags.debug, config: flags.config })
+		console.log('Use one of:')
+		console.log('  suemo doctor embed')
+		console.log('\nRun `suemo doctor --help` for full details.')
+	})

package/src/cli/commands/export-import.ts

@@ -2,9 +2,11 @@ import { createReadStream } from 'node:fs'
 import { createInterface } from 'node:readline'
 import { loadConfig } from '../../config.ts'
 import { connect, disconnect } from '../../db/client.ts'
-import { initLogger } from '../../logger.ts'
+import { getLogger } from '../../logger.ts'
 import type { MemoryNode, Relation } from '../../types.ts'
-import { app } from '../shared.ts'
+import { app, initCliCommand } from '../shared.ts'
+
+const log = getLogger(['suemo', 'cli', 'export-import'])
 
 // ── export ─────────────────────────────────────────────────────────────────
 export const exportCmd = app.sub('export')
@@ -14,31 +16,36 @@ export const exportCmd = app.sub('export')
 		all: { type: 'boolean', description: 'Include invalidated nodes' },
 	})
 	.run(async ({ flags }) => {
-		await initLogger({ level: flags.debug ? 'debug' : 'info' })
+		await initCliCommand('export', { debug: flags.debug, config: flags.config })
+		log.debug('Running export command', {
+			hasScope: Boolean(flags.scope),
+			includeInvalidated: Boolean(flags.all),
+		})
 		const config = await loadConfig(process.cwd(), flags.config)
 		const db = await connect(config.surreal)
+		try {
+			const activeFilter = flags.all ? 'true' : '(valid_until = NONE OR valid_until > time::now())'
+			const scopeFilter = '($scope = NONE OR scope = $scope)'
 
-		const activeFilter = flags.all ? 'true' : '(valid_until = NONE OR valid_until > time::now())'
-		const scopeFilter = '($scope = NONE OR scope = $scope)'
-
-		const [nodesResult, relationsResult] = await Promise.all([
-			db.query<[MemoryNode[]]>(
-				`
+			const [nodesResult, relationsResult] = await Promise.all([
+				db.query<[MemoryNode[]]>(
+					`
         SELECT * FROM memory WHERE ${activeFilter} AND ${scopeFilter} ORDER BY created_at ASC
       `,
-				{ scope: flags.scope ?? null },
-			),
-			db.query<[Relation[]]>('SELECT * FROM relates_to ORDER BY created_at ASC'),
-		])
+					{ scope: flags.scope ?? null },
+				),
+				db.query<[Relation[]]>('SELECT * FROM relates_to ORDER BY created_at ASC'),
+			])
 
-		for (const node of nodesResult[0] ?? []) {
-			process.stdout.write(JSON.stringify({ _type: 'memory', ...node }) + '\n')
-		}
-		for (const rel of relationsResult[0] ?? []) {
-			process.stdout.write(JSON.stringify({ _type: 'relation', ...rel }) + '\n')
+			for (const node of nodesResult[0] ?? []) {
+				process.stdout.write(JSON.stringify({ _type: 'memory', ...node }) + '\n')
+			}
+			for (const rel of relationsResult[0] ?? []) {
+				process.stdout.write(JSON.stringify({ _type: 'relation', ...rel }) + '\n')
+			}
+		} finally {
+			await disconnect()
 		}
-
-		await disconnect()
 	})
 
 // ── import ──────────────────────────────────────────────────────────────────
@@ -46,7 +53,8 @@ export const importCmd = app.sub('import')
 	.meta({ description: 'Import memories from a JSONL file' })
 	.args([{ name: 'file', type: 'string', required: true }])
 	.run(async ({ args, flags }) => {
-		await initLogger({ level: flags.debug ? 'debug' : 'info' })
+		await initCliCommand('import', { debug: flags.debug, config: flags.config })
+		log.debug('Running import command', { file: args.file })
 		const config = await loadConfig(process.cwd(), flags.config)
 		const db = await connect(config.surreal)
 
@@ -56,36 +64,64 @@ export const importCmd = app.sub('import')
 		let skipped = 0
 		let errors = 0
 
-		for await (const line of rl) {
-			lineNum++
-			if (!line.trim()) continue
-			let row: Record<string, unknown>
-			try {
-				row = JSON.parse(line)
-			} catch {
-				console.error(`Line ${lineNum}: invalid JSON — stopping`)
-				break
-			}
+		try {
+			for await (const line of rl) {
+				lineNum++
+				if (!line.trim()) continue
+				let row: Record<string, unknown>
+				try {
+					row = JSON.parse(line)
+				} catch {
+					console.error(`Line ${lineNum}: invalid JSON — stopping`)
+					break
+				}
+
+				const type = row['_type']
+				delete row['_type']
 
-			const type = row['_type']
-			delete row['_type']
+				try {
+					if (type === 'memory') {
+						await db.query('INSERT IGNORE INTO memory $row', { row })
+						imported++
+					} else if (type === 'relation') {
+						const inId = typeof row['in'] === 'string' ? row['in'] : undefined
+						const outId = typeof row['out'] === 'string' ? row['out'] : undefined
+						const kind = typeof row['kind'] === 'string' ? row['kind'] : undefined
+						const strength = typeof row['strength'] === 'number' ? row['strength'] : undefined
 
-			try {
-				if (type === 'memory') {
-					await db.query('INSERT IGNORE INTO memory $row', { row })
-					imported++
-				} else if (type === 'relation') {
-					await db.query('INSERT IGNORE INTO relates_to $row', { row })
-					imported++
-				} else {
-					skipped++
+						if (!inId || !outId || !kind || strength === undefined) {
+							skipped++
+							continue
+						}
+
+						await db.query(
+							`
+							LET $inRec = type::record($in);
+							LET $outRec = type::record($out);
+							RELATE $inRec->relates_to->$outRec CONTENT {
+							  kind: $kind,
+							  strength: $strength
+							};
+						`,
+							{
+								in: inId,
+								out: outId,
+								kind,
+								strength,
+							},
+						)
+						imported++
+					} else {
+						skipped++
+					}
+				} catch (e) {
+					console.error(`Line ${lineNum}: insert failed — ${String(e)}`)
+					errors++
 				}
-			} catch (e) {
-				console.error(`Line ${lineNum}: insert failed — ${String(e)}`)
-				errors++
 			}
+		} finally {
+			rl.close()
+			await disconnect()
 		}
-
-		await disconnect()
 		console.log(JSON.stringify({ imported, skipped, errors, lines: lineNum }, null, 2))
 	})