pi-obsidian-memory 0.0.1 → 0.1.0

package/src/usecases/search-memory/types.ts

@@ -0,0 +1,39 @@
+import type { CatalogHit, MemoryScope } from "#src/services"
+import type { Succeeded } from "#src/shared"
+import type { MemoryResolutionRefusal } from "#src/usecases/resolve-memory"
+import type { QueryVector } from "#src/usecases/semantic"
+import type { SemanticPort } from "#src/usecases/semantic-port"
+
+export interface SearchMemoryRequest {
+  query: string
+  vault: MemoryScope | "both"
+}
+
+export interface VaultSearchHits {
+  vault: MemoryScope
+  hits: CatalogHit[]
+  warnings: string[]
+}
+
+export interface VaultSearchFailure {
+  vault: MemoryScope
+  problems: string[]
+}
+
+export interface SemanticSummary {
+  mode: "fused" | "text-only"
+  problems: string[]
+}
+
+export interface SearchedMemory {
+  vaults: (VaultSearchHits | VaultSearchFailure)[]
+  projectMissing: boolean
+  semantic: SemanticSummary
+}
+
+export interface VaultSemantic {
+  port: SemanticPort | null
+  query: QueryVector | null
+}
+
+export type SearchMemoryResult = Succeeded<SearchedMemory> | MemoryResolutionRefusal

package/src/usecases/semantic/constants.ts

@@ -0,0 +1,7 @@
+export const EMBED_BATCH_SIZE = 32
+
+export const QUERY_KEY = "query"
+
+export const QUERY_NOT_EMBEDDED = "Semantic search is off: the query could not be embedded, so results use words only"
+
+export const OTHER_PROVIDER = "The embedder produced vectors for another provider; they were not stored"

package/src/usecases/semantic/index.ts

@@ -0,0 +1,5 @@
+export * from "./constants.ts"
+export * from "./prepare.ts"
+export * from "./rank.ts"
+export * from "./refresh-vectors.ts"
+export * from "./types.ts"

package/src/usecases/semantic/prepare.ts

@@ -0,0 +1,30 @@
+import type { MemoryOperationContext } from "#src/usecases/operation-context"
+import type { ResolvedMemory } from "#src/usecases/resolve-memory"
+import type { SemanticPort } from "#src/usecases/semantic-port"
+import { QUERY_KEY, QUERY_NOT_EMBEDDED } from "./constants.ts"
+import type { SemanticSearch } from "./types.ts"
+
+export async function semanticPortOf(
+  context: MemoryOperationContext,
+  memory: ResolvedMemory,
+): Promise<SemanticPort | null> {
+  if (!context.semantic) return null
+  return context.semantic(memory.config.embeddings)
+}
+
+export async function prepareSemanticSearch(
+  port: SemanticPort | null,
+  text: string,
+  context: MemoryOperationContext,
+): Promise<SemanticSearch> {
+  if (!port) return { query: null, problems: [] }
+  if (!port.available) return { query: null, problems: [`Semantic search is off: ${port.reason}`] }
+  const embedded = await port.embed(
+    { purpose: "query", inputs: [{ key: QUERY_KEY, text }] },
+    { signal: context.signal },
+  )
+  if (!embedded.succeeded) return { query: null, problems: [QUERY_NOT_EMBEDDED] }
+  const [item] = embedded.output.items
+  if (!item) return { query: null, problems: [QUERY_NOT_EMBEDDED] }
+  return { query: { fingerprint: embedded.output.fingerprint, vector: [...item.vector] }, problems: [] }
+}

package/src/usecases/semantic/rank.ts

@@ -0,0 +1,83 @@
+import {
+  type CatalogEntry,
+  type CatalogHit,
+  cosine,
+  fuseRankings,
+  type MemoryConfig,
+  MINIMUM_SIMILARITY,
+  readIndexInventory,
+  readVectorCache,
+  type ScoredPath,
+  type SearchIndex,
+  semanticRanking,
+  type VectorChunk,
+} from "#src/services"
+import type { QueryVector, SemanticRanked } from "./types.ts"
+
+export function rankSemantically(
+  index: SearchIndex,
+  query: QueryVector,
+  provider: MemoryConfig["embeddings"]["provider"],
+): SemanticRanked {
+  const inventory = readIndexInventory(index)
+  const cache = readVectorCache(index)
+  if (!inventory.succeeded) return { paths: [], hits: [], problems: inventory.problems }
+  if (!cache.succeeded) return { paths: [], hits: [], problems: cache.problems }
+  if (cache.output.generation?.fingerprint !== query.fingerprint) return { paths: [], hits: [], problems: [] }
+  const similarity = similarityByContent(query, cache.output.chunks)
+  const scored: ScoredPath[] = []
+  const hits = new Map<string, CatalogHit>()
+  for (const entry of inventory.output.entries) {
+    const hit = hitOf(entry, similarity)
+    if (!hit) continue
+    scored.push({ path: entry.path, similarity: hit.score })
+    hits.set(entry.path, hit)
+  }
+  const paths = semanticRanking(scored, MINIMUM_SIMILARITY[provider])
+  return { paths, hits: paths.flatMap((path) => listed(hits.get(path))), problems: [] }
+}
+
+function similarityByContent(query: QueryVector, chunks: VectorChunk[]): Map<string, number> {
+  const similarity = new Map<string, number>()
+  for (const chunk of chunks) {
+    const score = cosine(query.vector, chunk.vector)
+    const best = similarity.get(chunk.contentFingerprint)
+    if (typeof best !== "number" || score > best) similarity.set(chunk.contentFingerprint, score)
+  }
+  return similarity
+}
+
+function hitOf(entry: CatalogEntry, similarity: Map<string, number>): CatalogHit | null {
+  const { note } = entry
+  if (!note || note.archived || !entry.contentFingerprint) return null
+  const score = similarity.get(entry.contentFingerprint)
+  if (typeof score !== "number") return null
+  return {
+    path: entry.path,
+    id: note.id,
+    statement: note.statement,
+    type: note.type,
+    dontForget: note.dontForget,
+    needsFixing: entry.needsFixing,
+    score,
+  }
+}
+
+function listed(hit: CatalogHit | undefined): CatalogHit[] {
+  if (!hit) return []
+  return [hit]
+}
+
+export function fuseSearchHits(text: CatalogHit[], semantic: SemanticRanked): CatalogHit[] {
+  const byPath = new Map<string, CatalogHit>()
+  for (const hit of [...semantic.hits, ...text]) byPath.set(hit.path, hit)
+  const fused = fuseRankings(
+    text.map((hit) => hit.path),
+    semantic.paths,
+  )
+  return fused.flatMap((item) => {
+    const hit = byPath.get(item.path)
+    if (!hit) return []
+    return [{ ...hit, score: item.score }]
+  })
+}

package/src/usecases/semantic/refresh-vectors.ts

@@ -0,0 +1,95 @@
+import {
+  type EmbeddingBatchRequest,
+  type EmbeddingInput,
+  embeddingFingerprint,
+  planVectorRefresh,
+  readIndexInventory,
+  readVectorCache,
+  type SearchIndex,
+  updateVectorCache,
+  type VectorCache,
+  type VectorChunk,
+  type VectorGeneration,
+  type VectorRefreshPlan,
+} from "#src/services"
+import { type MemoryOperationContext, memoryOperationInterruption } from "#src/usecases/operation-context"
+import type { ResolvedMemory } from "#src/usecases/resolve-memory"
+import type { SemanticAvailable, SemanticPort } from "#src/usecases/semantic-port"
+import { EMBED_BATCH_SIZE, OTHER_PROVIDER } from "./constants.ts"
+
+export async function refreshSemantic(
+  index: SearchIndex,
+  memory: ResolvedMemory,
+  port: SemanticPort | null,
+  context: MemoryOperationContext,
+): Promise<string[]> {
+  if (!port) return []
+  if (!port.available) return [`Semantic search is off: ${port.reason}`]
+  return refreshVectors(index, memory, port, context)
+}
+
+async function refreshVectors(
+  index: SearchIndex,
+  memory: ResolvedMemory,
+  port: SemanticAvailable,
+  context: MemoryOperationContext,
+): Promise<string[]> {
+  const inventory = readIndexInventory(index)
+  if (!inventory.succeeded) return inventory.problems
+  const cache = readVectorCache(index)
+  if (!cache.succeeded) return cache.problems
+  const { revision } = inventory.output
+  const fingerprint = embeddingFingerprint(memory.config.embeddings)
+  const plan = planVectorRefresh(inventory.output.entries, cache.output, fingerprint)
+  let stale = staleContent(cache.output, plan)
+  const inputs = inputsOf(plan)
+  if (inputs.length === 0) return removeStale(index, revision, cache.output.generation, fingerprint, stale)
+  for (let start = 0; start < inputs.length; start += EMBED_BATCH_SIZE) {
+    if (memoryOperationInterruption(context)) return []
+    const batch: EmbeddingBatchRequest = { purpose: "document", inputs: inputs.slice(start, start + EMBED_BATCH_SIZE) }
+    const embedded = await port.embed(batch, { signal: context.signal })
+    if (!embedded.succeeded) return [stoppedLine(embedded.status, start, inputs.length)]
+    if (embedded.output.fingerprint !== fingerprint) return [OTHER_PROVIDER]
+    const add: VectorChunk[] = embedded.output.items.map((item) => ({
+      contentFingerprint: item.key,
+      chunk: 0,
+      vector: [...item.vector],
+    }))
+    const added = new Set(add.map((chunk) => chunk.contentFingerprint))
+    const generation = { fingerprint, dimensions: embedded.output.dimensions }
+    const remove = stale.filter((entry) => !added.has(entry))
+    const written = updateVectorCache(index, { expectedRevision: revision, generation, add, remove })
+    if (!written.succeeded) return written.problems
+    stale = []
+  }
+  return []
+}
+
+function staleContent(cache: VectorCache, plan: VectorRefreshPlan): string[] {
+  const kept = new Set(plan.keep.map((chunk) => chunk.contentFingerprint))
+  return [...new Set(cache.chunks.map((chunk) => chunk.contentFingerprint))].filter((entry) => !kept.has(entry))
+}
+
+function inputsOf(plan: VectorRefreshPlan): EmbeddingInput[] {
+  return plan.embed.flatMap((entry) => {
+    if (!entry.note || !entry.contentFingerprint) return []
+    return [{ key: entry.contentFingerprint, text: `${entry.note.statement}\n\n${entry.note.body}` }]
+  })
+}
+
+function removeStale(
+  index: SearchIndex,
+  revision: number,
+  generation: VectorGeneration | null,
+  fingerprint: string,
+  stale: string[],
+): string[] {
+  if (stale.length === 0 || !generation || generation.fingerprint !== fingerprint) return []
+  const written = updateVectorCache(index, { expectedRevision: revision, generation, add: [], remove: stale })
+  if (written.succeeded) return []
+  return written.problems
+}
+
+function stoppedLine(status: string, done: number, total: number): string {
+  return `Embedding stopped (${status}); ${done} of ${total} notes were embedded before it stopped`
+}

package/src/usecases/semantic/types.ts

@@ -0,0 +1,17 @@
+import type { CatalogHit } from "#src/services"
+
+export interface QueryVector {
+  fingerprint: string
+  vector: number[]
+}
+
+export interface SemanticRanked {
+  paths: string[]
+  hits: CatalogHit[]
+  problems: string[]
+}
+
+export interface SemanticSearch {
+  query: QueryVector | null
+  problems: string[]
+}

package/src/usecases/semantic-port/index.ts

@@ -0,0 +1 @@
+export * from "./types.ts"

package/src/usecases/semantic-port/types.ts

@@ -0,0 +1,20 @@
+import type {
+  EmbeddingBatchRequest,
+  EmbeddingConfiguration,
+  EmbeddingOperationContext,
+  EmbeddingResult,
+} from "#src/services"
+
+export interface SemanticAvailable {
+  available: true
+  embed(request: EmbeddingBatchRequest, context: EmbeddingOperationContext): Promise<EmbeddingResult>
+}
+
+export interface SemanticUnavailable {
+  available: false
+  reason: string
+}
+
+export type SemanticPort = SemanticAvailable | SemanticUnavailable
+
+export type SemanticSource = (configuration: EmbeddingConfiguration) => Promise<SemanticPort>

package/src/usecases/semantic-provider/constants.ts

@@ -0,0 +1,6 @@
+import { checkLocalModel, createLocalSemantic } from "./local.ts"
+import type { LocalModelAccess } from "./types.ts"
+
+export const BYTES_PER_MEGABYTE = 1_000_000
+
+export const localModelAccess: LocalModelAccess = { check: checkLocalModel, create: createLocalSemantic }

package/src/usecases/semantic-provider/index.ts

@@ -0,0 +1,5 @@
+export * from "./constants.ts"
+export * from "./local.ts"
+export * from "./remote.ts"
+export * from "./size.ts"
+export * from "./types.ts"

package/src/usecases/semantic-provider/local.ts

@@ -0,0 +1,33 @@
+import { join } from "node:path"
+
+import {
+  createLocalEmbedder,
+  embeddingPolicy,
+  inspectModelAsset,
+  localEmbeddingConfiguration,
+  localModelAssets,
+} from "#src/services"
+import type { LocalModelStatus, LocalSemantic, LocalSemanticOptions } from "./types.ts"
+
+export function createLocalSemantic(options: LocalSemanticOptions): LocalSemantic {
+  const embedder = createLocalEmbedder({ cachePath: options.cachePath })
+  return {
+    port: { available: true, embed: (request, context) => embedder.embed(context, request) },
+    dispose: () => embedder.dispose(),
+  }
+}
+
+export async function checkLocalModel(cachePath: string, signal: AbortSignal): Promise<LocalModelStatus> {
+  const directory = join(
+    cachePath,
+    localEmbeddingConfiguration.model,
+    embeddingPolicy(localEmbeddingConfiguration).revision,
+  )
+  let status: LocalModelStatus = "cached"
+  for (const asset of localModelAssets) {
+    const inspected = await inspectModelAsset(join(directory, asset.path), asset, signal)
+    if (inspected === "invalid") return "invalid"
+    if (inspected === "missing") status = "missing"
+  }
+  return status
+}

package/src/usecases/semantic-provider/remote.ts

@@ -0,0 +1,10 @@
+import { embedGoogle, embedOpenAI } from "#src/services"
+import type { SemanticAvailable } from "#src/usecases/semantic-port"
+import type { RemoteProvider } from "./types.ts"
+
+export function createRemoteSemantic(provider: RemoteProvider, apiKey: string): SemanticAvailable {
+  if (provider === "openai") {
+    return { available: true, embed: (request, context) => embedOpenAI({ ...context, apiKey }, request) }
+  }
+  return { available: true, embed: (request, context) => embedGoogle({ ...context, apiKey }, request) }
+}

package/src/usecases/semantic-provider/size.ts

@@ -0,0 +1,10 @@
+import { localModelAssets } from "#src/services"
+import { BYTES_PER_MEGABYTE } from "./constants.ts"
+
+export function localModelDownloadBytes(): number {
+  return localModelAssets.reduce((sum, asset) => sum + asset.bytes, 0)
+}
+
+export function localModelDownloadMegabytes(): number {
+  return Math.round(localModelDownloadBytes() / BYTES_PER_MEGABYTE)
+}

package/src/usecases/semantic-provider/types.ts

@@ -0,0 +1,19 @@
+import type { SemanticAvailable } from "#src/usecases/semantic-port"
+
+export type LocalModelStatus = "cached" | "missing" | "invalid"
+
+export type RemoteProvider = "openai" | "google"
+
+export interface LocalSemanticOptions {
+  cachePath: string
+}
+
+export interface LocalSemantic {
+  port: SemanticAvailable
+  dispose(): Promise<void>
+}
+
+export interface LocalModelAccess {
+  check(cachePath: string, signal: AbortSignal): Promise<LocalModelStatus>
+  create(options: LocalSemanticOptions): LocalSemantic
+}

package/src/usecases/session/index.ts

@@ -0,0 +1,2 @@
+export * from "./state.ts"
+export * from "./types.ts"

package/src/usecases/session/lease.ts

@@ -0,0 +1,70 @@
+import type { Lease, LeaseWaiter } from "./types.ts"
+
+export class SessionLeases {
+  #accepting = true
+  #active = false
+  #waiting: LeaseWaiter[] = []
+  #drains: PromiseWithResolvers<void>[] = []
+
+  acquire(signal: AbortSignal): Promise<Lease | null> {
+    if (!this.#accepting || signal.aborted) return Promise.resolve(null)
+    const result = Promise.withResolvers<Lease | null>()
+    const waiter: LeaseWaiter = {
+      signal,
+      result,
+      cancel: () => this.#cancel(waiter),
+    }
+    this.#waiting.push(waiter)
+    signal.addEventListener("abort", waiter.cancel, { once: true })
+    this.#advance()
+    return result.promise
+  }
+
+  stopAdmissions(): void {
+    this.#accepting = false
+  }
+
+  drain(): Promise<void> {
+    if (!this.#active && this.#waiting.length === 0) return Promise.resolve()
+    const result = Promise.withResolvers<void>()
+    this.#drains.push(result)
+    return result.promise
+  }
+
+  #cancel(waiter: LeaseWaiter): void {
+    const index = this.#waiting.indexOf(waiter)
+    if (index === -1) return
+    this.#waiting.splice(index, 1)
+    waiter.signal.removeEventListener("abort", waiter.cancel)
+    waiter.result.resolve(null)
+    this.#advance()
+  }
+
+  #advance(): void {
+    if (this.#active) return
+    for (let waiter = this.#waiting.shift(); waiter; waiter = this.#waiting.shift()) {
+      waiter.signal.removeEventListener("abort", waiter.cancel)
+      if (waiter.signal.aborted) {
+        waiter.result.resolve(null)
+        continue
+      }
+      this.#active = true
+      waiter.result.resolve(this.#lease())
+      return
+    }
+    for (const drain of this.#drains) drain.resolve()
+    this.#drains = []
+  }
+
+  #lease(): Lease {
+    let released = false
+    return {
+      release: () => {
+        if (released) return
+        released = true
+        this.#active = false
+        this.#advance()
+      },
+    }
+  }
+}

package/src/usecases/session/receipts.ts

@@ -0,0 +1,46 @@
+import type { MemoryScope, ParsedVaultNote } from "#src/services"
+import type { DeliveredRead, MemoryAddress } from "./types.ts"
+
+export class GetReceipts {
+  #vaults = new Map<MemoryScope, Map<string, DeliveredRead>>()
+
+  remember(read: DeliveredRead): void {
+    let paths = this.#vaults.get(read.vault)
+    if (!paths) {
+      paths = new Map<string, DeliveredRead>()
+      this.#vaults.set(read.vault, paths)
+    }
+    paths.set(read.path, copyDeliveredRead(read))
+  }
+
+  get(address: MemoryAddress): DeliveredRead | null {
+    const read = this.#vaults.get(address.vault)?.get(address.path)
+    if (!read) return null
+    return copyDeliveredRead(read)
+  }
+
+  consume(address: MemoryAddress): void {
+    this.#vaults.get(address.vault)?.delete(address.path)
+  }
+
+  clear(): void {
+    this.#vaults.clear()
+  }
+}
+
+function copyDeliveredRead(read: DeliveredRead): DeliveredRead {
+  let frontmatter: ParsedVaultNote["frontmatter"] = null
+  if (read.analysis.frontmatter) frontmatter = { ...read.analysis.frontmatter }
+  return {
+    ...read,
+    stat: { ...read.stat },
+    analysis: {
+      ...read.analysis,
+      parsed: { ...read.analysis.parsed },
+      frontmatter,
+      frontmatterProblems: [...read.analysis.frontmatterProblems],
+      nameProblems: [...read.analysis.nameProblems],
+      bodyProblems: [...read.analysis.bodyProblems],
+    },
+  }
+}

package/src/usecases/session/state.ts

@@ -0,0 +1,106 @@
+import { SessionLeases } from "./lease.ts"
+import { GetReceipts } from "./receipts.ts"
+import type {
+  ConfigurationBinding,
+  DeliveredRead,
+  Lease,
+  MemoryAddress,
+  OperationStamp,
+  SessionIdentity,
+} from "./types.ts"
+
+export class MemorySession {
+  #stamp: OperationStamp
+  #controller = new AbortController()
+  #aborted = false
+  #configuration: ConfigurationBinding = { bound: false }
+  #leases: SessionLeases
+  #receipts = new GetReceipts()
+
+  constructor(identity: SessionIdentity, leases?: SessionLeases) {
+    this.#stamp = Object.freeze({ sessionId: identity.sessionId, branchId: identity.branchId, generation: 0 })
+    this.#leases = new SessionLeases()
+    if (leases) this.#leases = leases
+  }
+
+  sibling(identity: SessionIdentity): MemorySession {
+    return new MemorySession(identity, this.#leases)
+  }
+
+  stamp(): OperationStamp {
+    return this.#stamp
+  }
+
+  signal(stamp: OperationStamp): AbortSignal {
+    if (!this.isCurrent(stamp)) return AbortSignal.abort()
+    return this.#controller.signal
+  }
+
+  isCurrent(stamp: OperationStamp): boolean {
+    return !this.#aborted && stamp === this.#stamp
+  }
+
+  acquire(stamp: OperationStamp, callerSignal: AbortSignal): Promise<Lease | null> {
+    if (!this.isCurrent(stamp)) return Promise.resolve(null)
+    return this.#leases.acquire(AbortSignal.any([callerSignal, this.#controller.signal]))
+  }
+
+  acceptConfiguration(stamp: OperationStamp, hash: string): boolean {
+    if (!this.isCurrent(stamp)) return false
+    if (!this.#configuration.bound) {
+      this.#configuration = { bound: true, hash }
+      return true
+    }
+    if (this.#configuration.hash === hash) return true
+    this.invalidate(this.#stamp)
+    return false
+  }
+
+  rememberGet(stamp: OperationStamp, read: DeliveredRead): boolean {
+    if (!this.isCurrent(stamp)) return false
+    this.#receipts.remember(read)
+    return true
+  }
+
+  getReceipt(stamp: OperationStamp, address: MemoryAddress): DeliveredRead | null {
+    if (!this.isCurrent(stamp)) return null
+    return this.#receipts.get(address)
+  }
+
+  consumeReceipt(stamp: OperationStamp, address: MemoryAddress): void {
+    if (this.isCurrent(stamp)) this.#receipts.consume(address)
+  }
+
+  invalidate(identity: SessionIdentity): void {
+    const previous = this.#controller
+    this.#stamp = Object.freeze({
+      sessionId: identity.sessionId,
+      branchId: identity.branchId,
+      generation: this.#stamp.generation + 1,
+    })
+    this.#configuration = { bound: false }
+    this.#receipts.clear()
+    if (!this.#aborted) this.#controller = new AbortController()
+    previous.abort()
+  }
+
+  stopAdmissions(): void {
+    this.#leases.stopAdmissions()
+  }
+
+  abortPending(): void {
+    this.stopAdmissions()
+    this.#aborted = true
+    this.#configuration = { bound: false }
+    this.#receipts.clear()
+    this.#controller.abort()
+  }
+
+  drain(): Promise<void> {
+    return this.#leases.drain()
+  }
+}
+
+export function createMemorySession(identity: SessionIdentity): MemorySession {
+  return new MemorySession(identity)
+}

package/src/usecases/session/types.ts

@@ -0,0 +1,45 @@
+import type { InspectedVaultFile, MemoryScope, ParsedVaultNote } from "#src/services"
+
+export interface SessionIdentity {
+  readonly sessionId: string
+  readonly branchId: string | null
+}
+
+export interface OperationStamp extends SessionIdentity {
+  readonly generation: number
+}
+
+export interface MemoryAddress {
+  vault: MemoryScope
+  path: string
+}
+
+export interface DeliveredRead extends MemoryAddress, InspectedVaultFile {
+  canonicalVault: string
+  id: string | null
+  sourceCallId: string
+  analysis: ParsedVaultNote
+}
+
+export interface Lease {
+  release(): void
+}
+
+export interface BoundConfiguration {
+  bound: true
+  hash: string
+}
+
+export interface UnboundConfiguration {
+  bound: false
+}
+
+export type ConfigurationBinding = BoundConfiguration | UnboundConfiguration
+
+export type CancelLeaseWaiter = () => void
+
+export interface LeaseWaiter {
+  signal: AbortSignal
+  result: PromiseWithResolvers<Lease | null>
+  cancel: CancelLeaseWaiter
+}