pi-obsidian-memory 0.0.1 → 0.1.1

package/src/usecases/relevant-memory/types.ts

@@ -0,0 +1,106 @@
+import type { Static } from "typebox"
+
+import type {
+  AgentContext,
+  ChatModelReference,
+  CompletionPort,
+  JevClassifierPort,
+  ReadObservation,
+  RetrievalAction,
+} from "#src/services"
+import type { Succeeded } from "#src/shared"
+import type { MemoryResolutionRefusal } from "#src/usecases/resolve-memory"
+import type { LoadedChangesSchema } from "./schemas.ts"
+
+export type { BoolQuestion, DecisionPrompt } from "#src/services"
+
+export interface JevAccess {
+  available: true
+  classifier: JevClassifierPort
+}
+
+export interface JevRefusal {
+  available: false
+  reason: string
+}
+
+export type JevSource = (model: string) => Promise<JevAccess | JevRefusal>
+
+export interface LoadedStore {
+  all(): ReadObservation[]
+  has(note: ReadObservation): boolean
+  remember(note: ReadObservation): void
+  forget(note: ReadObservation): void
+  apply(changes: LoadedChanges): void
+}
+
+export interface RetrieveDeps {
+  complete: CompletionPort
+  jev: JevSource
+  loaded: LoadedStore
+}
+
+export interface SessionMemoryDeps {
+  complete: CompletionPort
+  loaded: LoadedStore
+}
+
+export interface RetrieveRequest {
+  message: string
+}
+
+export type LoadedChanges = Static<typeof LoadedChangesSchema>
+
+export interface RetrievedMemory {
+  text: string | null
+  notices: string[]
+  changes: LoadedChanges
+}
+
+export type CutoffSource = "config" | "typesafe" | "missing"
+
+export interface CutoffValue {
+  value: number | null
+  source: CutoffSource
+}
+
+export interface CutoffReport {
+  gate: CutoffValue
+  relevance: CutoffValue
+}
+
+export interface Refreshed {
+  blocks: string[]
+  changes: LoadedChanges
+}
+
+export type RetrieveMemoryResult = Succeeded<RetrievedMemory> | MemoryResolutionRefusal
+
+export interface AgentFound {
+  succeeded: true
+  output: ReadObservation[]
+}
+
+export interface AgentFailed {
+  succeeded: false
+  notice: string | null
+}
+
+export type AgentOutcome = AgentFound | AgentFailed
+
+export interface AgentRequest {
+  agent: AgentContext
+  query: string
+  model: ChatModelReference
+}
+
+export interface FinishDiscriminant {
+  action: "finish"
+}
+
+export type ReadStep = Exclude<RetrievalAction, FinishDiscriminant>
+
+export interface RankedNotes {
+  notes: ReadObservation[]
+  notice: string | null
+}

package/src/usecases/resolve-memory/index.ts

@@ -0,0 +1,2 @@
+export * from "./resolve-memory.ts"
+export * from "./types.ts"

package/src/usecases/resolve-memory/leased/index.ts

@@ -0,0 +1,2 @@
+export * from "./resolve.ts"
+export * from "./types.ts"

package/src/usecases/resolve-memory/leased/resolve.ts

@@ -0,0 +1,109 @@
+import {
+  type ConfigRefusal,
+  canonicalizeRegistration,
+  canonicalizeWorkingFolder,
+  type MemoryConfig,
+  type MemoryScope,
+  type PathPlatform,
+  readConfigSnapshot,
+  targetVault,
+} from "#src/services"
+import {
+  type MemoryOperationContext,
+  memoryOperationInterruption,
+  staleMemorySession,
+} from "#src/usecases/operation-context"
+import type { ResolvedMemory, ResolveMemoryResult, SelectMemoryVaultResult } from "./types.ts"
+
+export async function resolveLeasedMemory(context: MemoryOperationContext): Promise<ResolveMemoryResult> {
+  const interrupted = memoryOperationInterruption(context)
+  if (interrupted) return interrupted
+  const snapshot = await readConfigSnapshot({ path: context.configPath, platform: context.platform })
+  const stale = memoryOperationInterruption(context)
+  if (stale) return stale
+  if (!snapshot.succeeded) {
+    context.session.invalidate(context.stamp)
+    return { ...snapshot, reason: "config-unavailable", configPath: context.configPath }
+  }
+  if (snapshot.output.state === "missing") {
+    context.session.invalidate(context.stamp)
+    return {
+      succeeded: false,
+      reason: "not-configured",
+      configPath: context.configPath,
+      problems: ["Memory is not configured; provide a complete config file"],
+    }
+  }
+  if (!context.session.acceptConfiguration(context.stamp, snapshot.output.hash)) return staleMemorySession()
+  const parsed = snapshot.output.lexicalParse
+  if (!parsed.succeeded) return invalidMemoryConfiguration(context, parsed.problems)
+  return validateMemoryLocations(context, parsed.output, snapshot.output.hash)
+}
+
+async function validateMemoryLocations(
+  context: MemoryOperationContext,
+  config: MemoryConfig,
+  hash: string,
+): Promise<ResolveMemoryResult> {
+  const canonical = await canonicalizeRegistration({
+    config,
+    configPath: context.configPath,
+    indexPath: context.indexPath,
+    platform: context.platform,
+  })
+  const interrupted = memoryOperationInterruption(context)
+  if (interrupted) return interrupted
+  if (!canonical.succeeded) return invalidMemoryConfiguration(context, canonical.problems)
+  const workingFolder = await canonicalizeWorkingFolder(context.workingFolder)
+  const stale = memoryOperationInterruption(context)
+  if (stale) return stale
+  if (!workingFolder.succeeded) return invalidMemoryConfiguration(context, workingFolder.problems)
+  return resolvedMemory(
+    canonical.output.config,
+    hash,
+    workingFolder.output,
+    canonical.output.indexPath,
+    context.platform,
+  )
+}
+
+function resolvedMemory(
+  config: MemoryConfig,
+  configHash: string,
+  workingFolder: string,
+  indexPath: string,
+  platform: PathPlatform,
+): ResolveMemoryResult {
+  const global = targetVault(config, "global", workingFolder, platform)
+  if (!global.succeeded) throw new Error("Global target requires no project selection")
+  const projectTarget = targetVault(config, "project", workingFolder, platform)
+  const output: ResolvedMemory = {
+    config,
+    configHash,
+    workingFolder,
+    indexPath,
+    platform,
+    global: { scope: "global", vault: { ...global.output, limits: { ...config.limits } } },
+    project: null,
+    protectedRoots: [config.global.vault, ...config.projects.map((project) => project.vault)],
+  }
+  if (projectTarget.succeeded) {
+    output.project = { scope: "project", vault: { ...projectTarget.output, limits: { ...config.limits } } }
+  }
+  return { succeeded: true, output }
+}
+
+function invalidMemoryConfiguration(context: MemoryOperationContext, problems: string[]): ConfigRefusal {
+  return { succeeded: false, reason: "invalid-config", configPath: context.configPath, problems }
+}
+
+export function selectMemoryVault(memory: ResolvedMemory, scope: MemoryScope): SelectMemoryVaultResult {
+  if (scope === "global") return { succeeded: true, output: memory.global }
+  if (memory.project) return { succeeded: true, output: memory.project }
+  return {
+    succeeded: false,
+    reason: "no-project",
+    workingFolder: memory.workingFolder,
+    problems: ["The current folder has no registered project; use global memory instead"],
+  }
+}

package/src/usecases/resolve-memory/leased/types.ts

@@ -0,0 +1,35 @@
+import type {
+  ConfigRefusal,
+  MemoryConfig,
+  MemoryScope,
+  NamedVaultContext,
+  NoProjectRefusal,
+  PathPlatform,
+} from "#src/services"
+import type { Failed, Succeeded } from "#src/shared"
+import type { MemoryOperationRefusal } from "#src/usecases/operation-context"
+
+export interface ResolvedVault {
+  scope: MemoryScope
+  vault: NamedVaultContext
+}
+
+export interface ResolvedMemory {
+  config: MemoryConfig
+  configHash: string
+  global: ResolvedVault
+  project: ResolvedVault | null
+  protectedRoots: string[]
+  workingFolder: string
+  indexPath: string
+  platform: PathPlatform
+}
+
+export interface MemoryConfigurationUnavailable extends Failed {
+  reason: "config-unavailable"
+  configPath: string
+}
+
+export type MemoryResolutionRefusal = ConfigRefusal | MemoryConfigurationUnavailable | MemoryOperationRefusal
+export type ResolveMemoryResult = Succeeded<ResolvedMemory> | MemoryResolutionRefusal
+export type SelectMemoryVaultResult = Succeeded<ResolvedVault> | NoProjectRefusal

package/src/usecases/resolve-memory/resolve-memory.ts

@@ -0,0 +1,23 @@
+import {
+  captureMemoryOperationContext,
+  type MemoryOperationContext,
+  memoryAdmissionRefusal,
+  memoryOperationInterruption,
+} from "#src/usecases/operation-context"
+import { resolveLeasedMemory } from "#src/usecases/resolve-memory/leased"
+import type { ResolveMemoryResult } from "./types.ts"
+
+export async function resolveMemory(callerContext: MemoryOperationContext): Promise<ResolveMemoryResult> {
+  const context = captureMemoryOperationContext(callerContext)
+  const lease = await context.session.acquire(context.stamp, context.signal)
+  if (!lease) return memoryAdmissionRefusal(context)
+  try {
+    const result = await resolveLeasedMemory(context)
+    if (!result.succeeded) return result
+    const interrupted = memoryOperationInterruption(context)
+    if (interrupted) return interrupted
+    return result
+  } finally {
+    lease.release()
+  }
+}

package/src/usecases/resolve-memory/types.ts

@@ -0,0 +1,7 @@
+export type {
+  MemoryConfigurationUnavailable,
+  MemoryResolutionRefusal,
+  ResolvedMemory,
+  ResolvedVault,
+  ResolveMemoryResult,
+} from "#src/usecases/resolve-memory/leased"

package/src/usecases/search-memory/constants.ts

@@ -0,0 +1 @@
+export const SEMANTIC_PARTIAL_LINE = "notes are not embedded yet, so semantic results may be incomplete"

package/src/usecases/search-memory/index.ts

@@ -0,0 +1,3 @@
+export * from "./constants.ts"
+export * from "./search-memory.ts"
+export * from "./types.ts"

package/src/usecases/search-memory/search-memory.ts

@@ -0,0 +1,132 @@
+import {
+  closeIndex,
+  embeddingFingerprint,
+  type MemoryScope,
+  type OpenedIndex,
+  readVectorCoverage,
+  searchCatalog,
+} from "#src/services"
+import {
+  captureMemoryOperationContext,
+  type MemoryOperationContext,
+  memoryAdmissionRefusal,
+  memoryOperationInterruption,
+} from "#src/usecases/operation-context"
+import {
+  type ResolvedMemory,
+  type ResolvedVault,
+  resolveLeasedMemory,
+  selectMemoryVault,
+} from "#src/usecases/resolve-memory/leased"
+import { fuseSearchHits, prepareSemanticSearch, rankSemantically, semanticPortOf } from "#src/usecases/semantic"
+import { openVaultIndex, synchronizeOpenedMemory } from "#src/usecases/synchronize-memory/leased"
+import { SEMANTIC_PARTIAL_LINE } from "./constants.ts"
+import type {
+  SearchedMemory,
+  SearchMemoryRequest,
+  SearchMemoryResult,
+  VaultSearchFailure,
+  VaultSearchHits,
+  VaultSemantic,
+} from "./types.ts"
+
+export async function searchMemory(
+  input: SearchMemoryRequest,
+  callerContext: MemoryOperationContext,
+): Promise<SearchMemoryResult> {
+  const context = captureMemoryOperationContext(callerContext)
+  const request = { query: input.query, vault: input.vault }
+  const lease = await context.session.acquire(context.stamp, context.signal)
+  if (!lease) return memoryAdmissionRefusal(context)
+  try {
+    const memory = await resolveLeasedMemory(context)
+    if (!memory.succeeded) return memory
+    return await searchResolvedMemory(request, memory.output, context)
+  } finally {
+    lease.release()
+  }
+}
+
+async function searchResolvedMemory(
+  request: SearchMemoryRequest,
+  memory: ResolvedMemory,
+  context: MemoryOperationContext,
+): Promise<SearchMemoryResult> {
+  const port = await semanticPortOf(context, memory)
+  const prepared = await prepareSemanticSearch(port, request.query, context)
+  const semantic: VaultSemantic = { port, query: prepared.query }
+  const problems = [...prepared.problems]
+  const output: SearchedMemory = { vaults: [], projectMissing: false, semantic: { mode: "text-only", problems } }
+  if (prepared.query) output.semantic.mode = "fused"
+  for (const scope of scopesOf(request.vault)) {
+    const interrupted = memoryOperationInterruption(context)
+    if (interrupted) return interrupted
+    const target = selectMemoryVault(memory, scope)
+    if (!target.succeeded) {
+      output.projectMissing = true
+      continue
+    }
+    output.vaults.push(await searchLeasedVault(request.query, memory, target.output, context, semantic, problems))
+  }
+  const stale = memoryOperationInterruption(context)
+  if (stale) return stale
+  return { succeeded: true, output }
+}
+
+function scopesOf(vault: SearchMemoryRequest["vault"]): MemoryScope[] {
+  if (vault === "both") return ["global", "project"]
+  return [vault]
+}
+
+async function searchLeasedVault(
+  query: string,
+  memory: ResolvedMemory,
+  target: ResolvedVault,
+  context: MemoryOperationContext,
+  semantic: VaultSemantic,
+  problems: string[],
+): Promise<VaultSearchHits | VaultSearchFailure> {
+  const opened = openVaultIndex(memory, target)
+  if (!opened.succeeded) return { vault: target.scope, problems: opened.problems }
+  const warnings: string[] = []
+  let result: VaultSearchHits | VaultSearchFailure
+  try {
+    result = await searchSynchronizedVault(query, opened, memory, target, context, semantic, problems)
+  } finally {
+    const closed = closeIndex(opened.output)
+    if (!closed.succeeded) warnings.push(...closed.problems)
+  }
+  if ("hits" in result) result.warnings.push(...warnings)
+  else result.problems.push(...warnings)
+  return result
+}
+
+async function searchSynchronizedVault(
+  query: string,
+  opened: OpenedIndex,
+  memory: ResolvedMemory,
+  target: ResolvedVault,
+  context: MemoryOperationContext,
+  semantic: VaultSemantic,
+  problems: string[],
+): Promise<VaultSearchHits | VaultSearchFailure> {
+  const synchronized = await synchronizeOpenedMemory(opened, memory, target, context, semantic.port)
+  if (!synchronized.succeeded) return { vault: target.scope, problems: synchronized.problems }
+  const found = searchCatalog(opened.output, { query })
+  if (!found.succeeded) return { vault: target.scope, problems: found.problems }
+  const warnings = [...synchronized.output.warnings]
+  if (!semantic.query) return { vault: target.scope, hits: found.output, warnings }
+  const ranked = rankSemantically(opened.output, semantic.query, memory.config.embeddings.provider)
+  warnings.push(...ranked.problems)
+  const missing = missingVectors(opened, memory)
+  if (missing > 0) problems.push(`${target.scope}: ${missing} ${SEMANTIC_PARTIAL_LINE}`)
+  return { vault: target.scope, hits: fuseSearchHits(found.output, ranked), warnings }
+}
+
+function missingVectors(opened: OpenedIndex, memory: ResolvedMemory): number {
+  const coverage = readVectorCoverage(opened.output)
+  if (!coverage.succeeded) return 0
+  const { generation, embedded, current } = coverage.output
+  if (generation?.fingerprint !== embeddingFingerprint(memory.config.embeddings)) return current
+  return Math.max(0, current - embedded)
+}

package/src/usecases/search-memory/types.ts

@@ -0,0 +1,39 @@
+import type { CatalogHit, MemoryScope } from "#src/services"
+import type { Succeeded } from "#src/shared"
+import type { MemoryResolutionRefusal } from "#src/usecases/resolve-memory"
+import type { QueryVector } from "#src/usecases/semantic"
+import type { SemanticPort } from "#src/usecases/semantic-port"
+
+export interface SearchMemoryRequest {
+  query: string
+  vault: MemoryScope | "both"
+}
+
+export interface VaultSearchHits {
+  vault: MemoryScope
+  hits: CatalogHit[]
+  warnings: string[]
+}
+
+export interface VaultSearchFailure {
+  vault: MemoryScope
+  problems: string[]
+}
+
+export interface SemanticSummary {
+  mode: "fused" | "text-only"
+  problems: string[]
+}
+
+export interface SearchedMemory {
+  vaults: (VaultSearchHits | VaultSearchFailure)[]
+  projectMissing: boolean
+  semantic: SemanticSummary
+}
+
+export interface VaultSemantic {
+  port: SemanticPort | null
+  query: QueryVector | null
+}
+
+export type SearchMemoryResult = Succeeded<SearchedMemory> | MemoryResolutionRefusal

package/src/usecases/semantic/constants.ts

@@ -0,0 +1,7 @@
+export const EMBED_BATCH_SIZE = 32
+
+export const QUERY_KEY = "query"
+
+export const QUERY_NOT_EMBEDDED = "Semantic search is off: the query could not be embedded, so results use words only"
+
+export const OTHER_PROVIDER = "The embedder produced vectors for another provider; they were not stored"

package/src/usecases/semantic/index.ts

@@ -0,0 +1,5 @@
+export * from "./constants.ts"
+export * from "./prepare.ts"
+export * from "./rank.ts"
+export * from "./refresh-vectors.ts"
+export * from "./types.ts"

package/src/usecases/semantic/prepare.ts

@@ -0,0 +1,30 @@
+import type { MemoryOperationContext } from "#src/usecases/operation-context"
+import type { ResolvedMemory } from "#src/usecases/resolve-memory"
+import type { SemanticPort } from "#src/usecases/semantic-port"
+import { QUERY_KEY, QUERY_NOT_EMBEDDED } from "./constants.ts"
+import type { SemanticSearch } from "./types.ts"
+
+export async function semanticPortOf(
+  context: MemoryOperationContext,
+  memory: ResolvedMemory,
+): Promise<SemanticPort | null> {
+  if (!context.semantic) return null
+  return context.semantic(memory.config.embeddings)
+}
+
+export async function prepareSemanticSearch(
+  port: SemanticPort | null,
+  text: string,
+  context: MemoryOperationContext,
+): Promise<SemanticSearch> {
+  if (!port) return { query: null, problems: [] }
+  if (!port.available) return { query: null, problems: [`Semantic search is off: ${port.reason}`] }
+  const embedded = await port.embed(
+    { purpose: "query", inputs: [{ key: QUERY_KEY, text }] },
+    { signal: context.signal },
+  )
+  if (!embedded.succeeded) return { query: null, problems: [QUERY_NOT_EMBEDDED] }
+  const [item] = embedded.output.items
+  if (!item) return { query: null, problems: [QUERY_NOT_EMBEDDED] }
+  return { query: { fingerprint: embedded.output.fingerprint, vector: [...item.vector] }, problems: [] }
+}

package/src/usecases/semantic/rank.ts

@@ -0,0 +1,83 @@
+import {
+  type CatalogEntry,
+  type CatalogHit,
+  cosine,
+  fuseRankings,
+  type MemoryConfig,
+  MINIMUM_SIMILARITY,
+  readIndexInventory,
+  readVectorCache,
+  type ScoredPath,
+  type SearchIndex,
+  semanticRanking,
+  type VectorChunk,
+} from "#src/services"
+import type { QueryVector, SemanticRanked } from "./types.ts"
+
+export function rankSemantically(
+  index: SearchIndex,
+  query: QueryVector,
+  provider: MemoryConfig["embeddings"]["provider"],
+): SemanticRanked {
+  const inventory = readIndexInventory(index)
+  const cache = readVectorCache(index)
+  if (!inventory.succeeded) return { paths: [], hits: [], problems: inventory.problems }
+  if (!cache.succeeded) return { paths: [], hits: [], problems: cache.problems }
+  if (cache.output.generation?.fingerprint !== query.fingerprint) return { paths: [], hits: [], problems: [] }
+  const similarity = similarityByContent(query, cache.output.chunks)
+  const scored: ScoredPath[] = []
+  const hits = new Map<string, CatalogHit>()
+  for (const entry of inventory.output.entries) {
+    const hit = hitOf(entry, similarity)
+    if (!hit) continue
+    scored.push({ path: entry.path, similarity: hit.score })
+    hits.set(entry.path, hit)
+  }
+  const paths = semanticRanking(scored, MINIMUM_SIMILARITY[provider])
+  return { paths, hits: paths.flatMap((path) => listed(hits.get(path))), problems: [] }
+}
+
+function similarityByContent(query: QueryVector, chunks: VectorChunk[]): Map<string, number> {
+  const similarity = new Map<string, number>()
+  for (const chunk of chunks) {
+    const score = cosine(query.vector, chunk.vector)
+    const best = similarity.get(chunk.contentFingerprint)
+    if (typeof best !== "number" || score > best) similarity.set(chunk.contentFingerprint, score)
+  }
+  return similarity
+}
+
+function hitOf(entry: CatalogEntry, similarity: Map<string, number>): CatalogHit | null {
+  const { note } = entry
+  if (!note || note.archived || !entry.contentFingerprint) return null
+  const score = similarity.get(entry.contentFingerprint)
+  if (typeof score !== "number") return null
+  return {
+    path: entry.path,
+    id: note.id,
+    statement: note.statement,
+    type: note.type,
+    dontForget: note.dontForget,
+    needsFixing: entry.needsFixing,
+    score,
+  }
+}
+
+function listed(hit: CatalogHit | undefined): CatalogHit[] {
+  if (!hit) return []
+  return [hit]
+}
+
+export function fuseSearchHits(text: CatalogHit[], semantic: SemanticRanked): CatalogHit[] {
+  const byPath = new Map<string, CatalogHit>()
+  for (const hit of [...semantic.hits, ...text]) byPath.set(hit.path, hit)
+  const fused = fuseRankings(
+    text.map((hit) => hit.path),
+    semantic.paths,
+  )
+  return fused.flatMap((item) => {
+    const hit = byPath.get(item.path)
+    if (!hit) return []
+    return [{ ...hit, score: item.score }]
+  })
+}