pi-obsidian-memory 0.0.1 → 0.1.0

package/src/services/search-index/schema.ts

@@ -0,0 +1,42 @@
+import type { DatabaseSync } from "node:sqlite"
+
+import { INDEX_SCHEMA_VERSION, REBUILD_SCHEMA_SQL } from "./constants.ts"
+import { indexFailure } from "./refusals.ts"
+import type { SchemaResult, SearchIndexRefusal } from "./types.ts"
+
+export function initializeSchema(database: DatabaseSync): SchemaResult {
+  try {
+    database.exec("BEGIN IMMEDIATE")
+    const row = database.prepare("PRAGMA user_version").get()
+    if (!row || typeof row.user_version !== "number") {
+      return rollBackSchema(database, {
+        succeeded: false,
+        reason: "corrupted",
+        problems: ["the search index has no valid schema version"],
+      })
+    }
+
+    const rebuilt = row.user_version !== INDEX_SCHEMA_VERSION
+    if (rebuilt) {
+      database.exec(REBUILD_SCHEMA_SQL)
+    }
+    database.exec("INSERT INTO notes_fts(notes_fts, rank) VALUES ('integrity-check', 1)")
+    database.exec("COMMIT")
+    return { succeeded: true, output: rebuilt }
+  } catch (error) {
+    if (!(error instanceof Error)) throw error
+    return rollBackSchema(database, indexFailure(error))
+  }
+}
+
+function rollBackSchema(database: DatabaseSync, failure: SearchIndexRefusal): SearchIndexRefusal {
+  if (!database.isTransaction) return failure
+
+  try {
+    database.exec("ROLLBACK")
+  } catch (error) {
+    if (!(error instanceof Error)) throw error
+    failure.problems.push(`schema rollback failed: ${error.message}`)
+  }
+  return failure
+}

package/src/services/search-index/schemas.ts

@@ -0,0 +1,181 @@
+import Type from "typebox"
+
+import {
+  CatalogEntrySchema,
+  CatalogPathSchema,
+  CatalogStatSchema,
+  NoteIdSchema,
+  NoteTypeSchema,
+  VectorGenerationSchema,
+} from "#src/domain"
+
+export const SearchHitSchema = Type.Object(
+  {
+    id: NoteIdSchema,
+    folder: Type.String(),
+    statement: Type.String(),
+    type: NoteTypeSchema,
+    score: Type.Number(),
+  },
+  { additionalProperties: false },
+)
+
+export const IndexEntrySchema = Type.Object(
+  {
+    id: NoteIdSchema,
+    folder: Type.String(),
+    statement: Type.String(),
+    body: Type.String(),
+    type: NoteTypeSchema,
+    dontForget: Type.Boolean(),
+    archived: Type.Boolean(),
+    size: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    modifiedMs: Type.Number(),
+    hash: Type.String(),
+  },
+  { additionalProperties: false },
+)
+
+export const CatalogStateSchema = Type.Object(
+  {
+    revision: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    population: Type.Union([Type.Literal("ready"), Type.Literal("needs-population")]),
+  },
+  { additionalProperties: false },
+)
+
+export const CatalogChangesSchema = Type.Object(
+  {
+    expectedRevision: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    upserts: Type.Array(CatalogEntrySchema),
+    deletes: Type.Array(CatalogPathSchema),
+    statUpdates: Type.Array(
+      Type.Object({ path: CatalogPathSchema, stat: CatalogStatSchema }, { additionalProperties: false }),
+    ),
+    population: Type.Union([Type.Literal("ready"), Type.Literal("needs-population")]),
+  },
+  { additionalProperties: false },
+)
+
+export const CatalogRowSchema = Type.Object(
+  {
+    path: CatalogPathSchema,
+    id: Type.Union([NoteIdSchema, Type.Null()]),
+    type: Type.Union([NoteTypeSchema, Type.Null()]),
+    statement: Type.Union([Type.String(), Type.Null()]),
+    body: Type.Union([Type.String(), Type.Null()]),
+    dont_forget: Type.Union([Type.Literal(0), Type.Literal(1), Type.Null()]),
+    archived: Type.Union([Type.Literal(0), Type.Literal(1), Type.Null()]),
+    size: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    modified_ms: Type.Number(),
+    hash: Type.String({ minLength: 1 }),
+    content_fingerprint: Type.Union([Type.String({ minLength: 1 }), Type.Null()]),
+    validation_fingerprint: Type.String({ minLength: 1 }),
+    needs_fixing: Type.Union([Type.Literal(0), Type.Literal(1)]),
+  },
+  { additionalProperties: false },
+)
+
+export const CatalogHitSchema = Type.Object(
+  {
+    path: CatalogPathSchema,
+    id: NoteIdSchema,
+    type: NoteTypeSchema,
+    statement: Type.String(),
+    dontForget: Type.Boolean(),
+    needsFixing: Type.Boolean(),
+    score: Type.Number(),
+  },
+  { additionalProperties: false },
+)
+
+export const CatalogHitRowSchema = Type.Object(
+  {
+    path: CatalogPathSchema,
+    id: NoteIdSchema,
+    type: NoteTypeSchema,
+    statement: Type.String(),
+    dont_forget: Type.Union([Type.Literal(0), Type.Literal(1)]),
+    needs_fixing: Type.Union([Type.Literal(0), Type.Literal(1)]),
+    score: Type.Number(),
+  },
+  { additionalProperties: false },
+)
+
+export const VectorValuesSchema = Type.Array(Type.Number(), { minItems: 1 })
+
+export const VectorChunkSchema = Type.Object(
+  {
+    contentFingerprint: Type.String({ minLength: 1 }),
+    chunk: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    vector: VectorValuesSchema,
+  },
+  { additionalProperties: false },
+)
+
+export const VectorReplacementSchema = Type.Object(
+  {
+    expectedRevision: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    generation: VectorGenerationSchema,
+    chunks: Type.Array(VectorChunkSchema),
+  },
+  { additionalProperties: false },
+)
+
+export const VectorUpdateSchema = Type.Object(
+  {
+    expectedRevision: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    generation: VectorGenerationSchema,
+    add: Type.Array(VectorChunkSchema),
+    remove: Type.Array(Type.String({ minLength: 1 })),
+  },
+  { additionalProperties: false },
+)
+
+export const VectorRowSchema = Type.Object(
+  {
+    content_fingerprint: Type.String({ minLength: 1 }),
+    chunk: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    vector: Type.String(),
+  },
+  { additionalProperties: false },
+)
+
+export const SearchOptionsSchema = Type.Object(
+  {
+    includeArchived: Type.Boolean(),
+  },
+  { additionalProperties: false },
+)
+
+export const FolderIndexStateSchema = Type.Object(
+  {
+    folder: Type.String(),
+    fingerprint: Type.String({ minLength: 1 }),
+    indexSize: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    indexMtimeMs: Type.Number(),
+    indexCtimeMs: Type.Number(),
+    indexIno: Type.String({ minLength: 1 }),
+  },
+  { additionalProperties: false },
+)
+
+export const FolderIndexWriteSchema = Type.Object(
+  {
+    states: Type.Array(FolderIndexStateSchema),
+    keepFolders: Type.Array(Type.String()),
+  },
+  { additionalProperties: false },
+)
+
+export const FolderIndexRowSchema = Type.Object(
+  {
+    folder: Type.String(),
+    fingerprint: Type.String({ minLength: 1 }),
+    index_size: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    index_mtime_ms: Type.Number(),
+    index_ctime_ms: Type.Number(),
+    index_ino: Type.String({ minLength: 1 }),
+  },
+  { additionalProperties: false },
+)

package/src/services/search-index/search.ts

@@ -0,0 +1,5 @@
+import type { SearchIndex, SearchOptions, SearchResult } from "./types.ts"
+
+export function searchText(index: SearchIndex, query: string, options: SearchOptions): SearchResult {
+  return index.search(query, options)
+}

package/src/services/search-index/statements.ts

@@ -0,0 +1,18 @@
+import type { DatabaseSync } from "node:sqlite"
+
+export function prepareStatements(database: DatabaseSync) {
+  return {
+    byId: database.prepare("SELECT path FROM notes WHERE id = ?"),
+    move: database.prepare("UPDATE notes SET path = ? WHERE id = ?"),
+    folder: database.prepare("UPDATE notes SET folder = ? WHERE path = ?"),
+    remove: database.prepare("DELETE FROM notes WHERE id = :id"),
+    search: database.prepare(`
+      SELECT notes.id, notes.folder, notes.statement, notes.type,
+        -bm25(notes_fts, :statementWeight, :bodyWeight) AS score
+      FROM notes_fts
+      JOIN notes ON notes.rowid = notes_fts.rowid
+      WHERE notes_fts MATCH :match AND notes.id IS NOT NULL AND (notes.archived = 0 OR :includeArchived = 1)
+      ORDER BY score DESC, notes.id
+    `),
+  }
+}

package/src/services/search-index/storage.ts

@@ -0,0 +1,207 @@
+import type { DatabaseSync } from "node:sqlite"
+
+import type { CatalogChanges, VectorReplacement } from "#src/domain"
+import { readCatalogHits } from "./catalog-query.ts"
+import { prepareCatalogStatements } from "./catalog-statements.ts"
+import { closeFailedIndex } from "./cleanup.ts"
+import { prepareFolderIndexStatements } from "./folder-index-statements.ts"
+import { readStoredFolderIndexes, writeFolderIndexStates } from "./folder-index-storage.ts"
+import { readInventory } from "./inventory.ts"
+import { putLegacyEntry, removeLegacyEntry } from "./legacy.ts"
+import { readSearchHits } from "./query.ts"
+import { writeCatalog } from "./reconcile.ts"
+import { closedIndex, indexFailure } from "./refusals.ts"
+import { prepareStatements } from "./statements.ts"
+import type {
+  CatalogQuery,
+  CatalogSearchResult,
+  FolderIndexState,
+  FolderIndexStatesResult,
+  IndexEntry,
+  IndexWork,
+  IndexWriteResult,
+  InventoryResult,
+  SearchIndex,
+  SearchIndexRefusal,
+  SearchOptions,
+  SearchResult,
+  VectorCacheResult,
+  VectorCoverageResult,
+  VectorUpdate,
+} from "./types.ts"
+import { prepareVectorStatements } from "./vector-statements.ts"
+import { readVectorCoverage, readVectors, updateVectors, writeVectors } from "./vector-storage.ts"
+
+export class SqliteSearchIndex implements SearchIndex {
+  readonly #database: DatabaseSync
+  readonly #statements: ReturnType<typeof prepareStatements>
+  readonly #catalog: ReturnType<typeof prepareCatalogStatements>
+  readonly #vectors: ReturnType<typeof prepareVectorStatements>
+  readonly #folderIndexes: ReturnType<typeof prepareFolderIndexStatements>
+  #usable = true
+
+  constructor(database: DatabaseSync) {
+    this.#database = database
+    this.#statements = prepareStatements(database)
+    this.#catalog = prepareCatalogStatements(database)
+    this.#vectors = prepareVectorStatements(database)
+    this.#folderIndexes = prepareFolderIndexStatements(database)
+  }
+
+  put(entry: IndexEntry): IndexWriteResult {
+    return this.#write(() => putLegacyEntry(this.#statements, this.#catalog, entry))
+  }
+
+  remove(id: string): IndexWriteResult {
+    return this.#write(() => removeLegacyEntry(this.#statements, this.#catalog, id))
+  }
+
+  apply(changes: CatalogChanges): IndexWriteResult {
+    return this.#write(() => writeCatalog(this.#catalog, changes))
+  }
+
+  replaceVectors(request: VectorReplacement): IndexWriteResult {
+    return this.#write(() => writeVectors(this.#vectors, this.#catalog, request))
+  }
+
+  updateVectors(request: VectorUpdate): IndexWriteResult {
+    return this.#write(() => updateVectors(this.#vectors, this.#catalog, request))
+  }
+
+  vectorCoverage(): VectorCoverageResult {
+    if (!this.#usable || !this.#database.isOpen) return closedIndex()
+
+    try {
+      this.#database.exec("BEGIN")
+      const result = readVectorCoverage(this.#vectors)
+      this.#database.exec("COMMIT")
+      return result
+    } catch (error) {
+      if (!(error instanceof Error)) throw error
+      return this.#rollBack(indexFailure(error))
+    }
+  }
+
+  recordFolderIndexes(states: FolderIndexState[], keepFolders: string[]): IndexWriteResult {
+    return this.#write(() => writeFolderIndexStates(this.#folderIndexes, states, keepFolders))
+  }
+
+  folderIndexes(): FolderIndexStatesResult {
+    if (!this.#usable || !this.#database.isOpen) return closedIndex()
+
+    try {
+      this.#database.exec("BEGIN")
+      const result = readStoredFolderIndexes(this.#folderIndexes)
+      this.#database.exec("COMMIT")
+      return result
+    } catch (error) {
+      if (!(error instanceof Error)) throw error
+      return this.#rollBack(indexFailure(error))
+    }
+  }
+
+  inventory(): InventoryResult {
+    if (!this.#usable || !this.#database.isOpen) return closedIndex()
+
+    try {
+      this.#database.exec("BEGIN")
+      const result = readInventory(this.#catalog)
+      this.#database.exec("COMMIT")
+      return result
+    } catch (error) {
+      if (!(error instanceof Error)) throw error
+      return this.#rollBack(indexFailure(error))
+    }
+  }
+
+  vectors(): VectorCacheResult {
+    if (!this.#usable || !this.#database.isOpen) return closedIndex()
+
+    try {
+      this.#database.exec("BEGIN")
+      const result = readVectors(this.#vectors)
+      this.#database.exec("COMMIT")
+      return result
+    } catch (error) {
+      if (!(error instanceof Error)) throw error
+      if (error instanceof SyntaxError) {
+        return this.#rollBack({ succeeded: false, reason: "corrupted", problems: ["invalid cached vector JSON"] })
+      }
+      return this.#rollBack(indexFailure(error))
+    }
+  }
+
+  searchCatalog(request: CatalogQuery): CatalogSearchResult {
+    if (!this.#usable || !this.#database.isOpen) return closedIndex()
+
+    try {
+      return readCatalogHits(this.#catalog.search, request)
+    } catch (error) {
+      if (!(error instanceof Error)) throw error
+      return indexFailure(error)
+    }
+  }
+
+  search(query: string, options: SearchOptions): SearchResult {
+    if (!this.#usable || !this.#database.isOpen) return closedIndex()
+
+    try {
+      return readSearchHits(this.#statements.search, query, options)
+    } catch (error) {
+      if (!(error instanceof Error)) throw error
+      return indexFailure(error)
+    }
+  }
+
+  close(): IndexWriteResult {
+    if (!this.#database.isOpen) return { succeeded: true }
+
+    try {
+      this.#database.close()
+      return { succeeded: true }
+    } catch (error) {
+      if (!(error instanceof Error)) throw error
+      return indexFailure(error)
+    }
+  }
+
+  #write(work: IndexWork): IndexWriteResult {
+    if (!this.#usable || !this.#database.isOpen) return closedIndex()
+
+    try {
+      this.#database.exec("BEGIN IMMEDIATE")
+    } catch (error) {
+      if (!(error instanceof Error)) throw error
+      return indexFailure(error)
+    }
+
+    return this.#commit(work)
+  }
+
+  #commit(work: IndexWork): IndexWriteResult {
+    try {
+      const result = work()
+      if (!result.succeeded) return this.#rollBack(result)
+
+      this.#database.exec("COMMIT")
+      return result
+    } catch (error) {
+      if (!(error instanceof Error)) throw error
+      return this.#rollBack(indexFailure(error))
+    }
+  }
+
+  #rollBack(failure: SearchIndexRefusal): SearchIndexRefusal {
+    if (!this.#database.isTransaction) return failure
+
+    try {
+      this.#database.exec("ROLLBACK")
+    } catch (error) {
+      if (!(error instanceof Error)) throw error
+      this.#usable = false
+      failure.problems.push(`write rollback failed: ${error.message}`)
+      return closeFailedIndex(this.#database, failure)
+    }
+    return failure
+  }
+}

package/src/services/search-index/types.ts

@@ -0,0 +1,113 @@
+import type { Static } from "typebox"
+
+import type {
+  CatalogChanges,
+  CatalogSnapshot,
+  VectorCache,
+  VectorChunk,
+  VectorGeneration,
+  VectorReplacement,
+} from "#src/domain"
+import type { Done, Failed, Succeeded } from "#src/shared"
+import type {
+  CatalogHitSchema,
+  CatalogRowSchema,
+  FolderIndexRowSchema,
+  FolderIndexStateSchema,
+  IndexEntrySchema,
+  SearchHitSchema,
+  SearchOptionsSchema,
+} from "./schemas.ts"
+
+export type {
+  AliasFailureClassification,
+  CatalogChanges,
+  CatalogEntry,
+  CatalogPlan,
+  CatalogScan,
+  CatalogSnapshot,
+  SemanticContent,
+  VectorCache,
+  VectorChunk,
+  VectorGeneration,
+  VectorReplacement,
+} from "#src/domain"
+export type CatalogHit = Static<typeof CatalogHitSchema>
+export type CatalogRow = Static<typeof CatalogRowSchema>
+export type FolderIndexState = Static<typeof FolderIndexStateSchema>
+export type FolderIndexRow = Static<typeof FolderIndexRowSchema>
+export type FolderIndexStatesResult = Succeeded<FolderIndexState[]> | SearchIndexRefusal
+export type InventoryResult = Succeeded<CatalogSnapshot> | SearchIndexRefusal
+export type CatalogSearchResult = Succeeded<CatalogHit[]> | SearchIndexRefusal
+export type VectorCacheResult = Succeeded<VectorCache> | SearchIndexRefusal
+export type VectorCoverageResult = Succeeded<VectorCoverage> | SearchIndexRefusal
+
+export interface VectorUpdate {
+  expectedRevision: number
+  generation: VectorGeneration
+  add: VectorChunk[]
+  remove: string[]
+}
+
+export interface VectorCoverage {
+  generation: VectorGeneration | null
+  embedded: number
+  current: number
+}
+
+export interface IndexLocation {
+  path: string
+}
+
+export interface IndexReplacement {
+  path: string
+  temporaryPath: string
+}
+
+export interface CatalogQuery {
+  query: string
+}
+
+export interface CatalogWriteRow extends CatalogRow {
+  folder: string
+}
+
+export interface SearchIndex {
+  put(entry: IndexEntry): IndexWriteResult
+  remove(id: string): IndexWriteResult
+  search(query: string, options: SearchOptions): SearchResult
+  inventory(): InventoryResult
+  apply(changes: CatalogChanges): IndexWriteResult
+  searchCatalog(request: CatalogQuery): CatalogSearchResult
+  vectors(): VectorCacheResult
+  replaceVectors(request: VectorReplacement): IndexWriteResult
+  updateVectors(request: VectorUpdate): IndexWriteResult
+  vectorCoverage(): VectorCoverageResult
+  folderIndexes(): FolderIndexStatesResult
+  recordFolderIndexes(states: FolderIndexState[], keepFolders: string[]): IndexWriteResult
+  close(): IndexWriteResult
+}
+
+export type SearchHit = Static<typeof SearchHitSchema>
+
+export type IndexEntry = Static<typeof IndexEntrySchema>
+
+export type SearchOptions = Static<typeof SearchOptionsSchema>
+
+export interface SearchIndexRefusal extends Failed {
+  reason: "corrupted" | "unavailable" | "closed" | "invalid" | "conflict"
+}
+
+export interface OpenedIndex extends Succeeded<SearchIndex> {
+  rebuilt: boolean
+}
+
+export type OpenIndexResult = OpenedIndex | SearchIndexRefusal
+
+export type SearchResult = Succeeded<SearchHit[]> | SearchIndexRefusal
+
+export type IndexWriteResult = Done | SearchIndexRefusal
+
+export type SchemaResult = Succeeded<boolean> | SearchIndexRefusal
+
+export type IndexWork = () => IndexWriteResult

package/src/services/search-index/vector-statements.ts

@@ -0,0 +1,18 @@
+import type { DatabaseSync } from "node:sqlite"
+
+export function prepareVectorStatements(database: DatabaseSync) {
+  return {
+    generation: database.prepare("SELECT fingerprint, dimensions FROM vector_generation WHERE singleton = 1"),
+    chunks: database.prepare(
+      "SELECT content_fingerprint, chunk, vector FROM vector_chunks ORDER BY content_fingerprint, chunk",
+    ),
+    clearChunks: database.prepare("DELETE FROM vector_chunks"),
+    clearGeneration: database.prepare("DELETE FROM vector_generation"),
+    putGeneration: database.prepare("INSERT INTO vector_generation VALUES (1, ?, ?)"),
+    putChunk: database.prepare("INSERT INTO vector_chunks VALUES (?, ?, ?)"),
+    removeChunks: database.prepare("DELETE FROM vector_chunks WHERE content_fingerprint = ?"),
+    currentContent: database.prepare(`SELECT DISTINCT content_fingerprint FROM notes
+      WHERE id IS NOT NULL AND archived = 0 AND content_fingerprint IS NOT NULL`),
+    embeddedContent: database.prepare("SELECT COUNT(DISTINCT content_fingerprint) AS embedded FROM vector_chunks"),
+  }
+}