pi-obsidian-memory 0.0.1 → 0.1.1

package/src/services/search-index/legacy.ts

@@ -0,0 +1,64 @@
+import { Value } from "typebox/value"
+
+import { type CatalogEntry, semanticFingerprint } from "#src/domain"
+import type { prepareCatalogStatements } from "./catalog-statements.ts"
+import { validCatalogEntry } from "./inventory.ts"
+import { invalidCatalog, writeCatalogEntry } from "./reconcile.ts"
+import { IndexEntrySchema } from "./schemas.ts"
+import type { prepareStatements } from "./statements.ts"
+import type { IndexEntry, IndexWriteResult } from "./types.ts"
+
+export function putLegacyEntry(
+  statements: ReturnType<typeof prepareStatements>,
+  catalog: ReturnType<typeof prepareCatalogStatements>,
+  entry: IndexEntry,
+): IndexWriteResult {
+  if (!Value.Check(IndexEntrySchema, entry) || !Number.isFinite(entry.modifiedMs)) {
+    return invalidCatalog("the entry does not match the index schema")
+  }
+  if (statements.byId.all(entry.id).length > 1) {
+    return invalidCatalog("duplicate ids require a path-keyed catalog mutation")
+  }
+  const converted = legacyEntry(entry)
+  if (!validCatalogEntry(converted)) return invalidCatalog("the entry does not have a valid catalog path")
+  statements.move.run(converted.path, entry.id)
+  writeCatalogEntry(catalog, converted)
+  statements.folder.run(entry.folder, converted.path)
+  catalog.prune.run()
+  catalog.advance.run("needs-population")
+  return { succeeded: true }
+}
+
+export function removeLegacyEntry(
+  statements: ReturnType<typeof prepareStatements>,
+  catalog: ReturnType<typeof prepareCatalogStatements>,
+  id: string,
+): IndexWriteResult {
+  if (statements.byId.all(id).length > 1) return invalidCatalog("duplicate ids require a path-keyed catalog mutation")
+  statements.remove.run({ id })
+  catalog.prune.run()
+  catalog.advance.run("needs-population")
+  return { succeeded: true }
+}
+
+function legacyEntry(entry: IndexEntry): CatalogEntry {
+  let path = `${entry.statement}.md`
+  if (entry.folder !== "") path = `${entry.folder}/${path}`
+  if (entry.archived && path.split("/")[0]?.toLowerCase() !== "archive") path = `archive/${path}`
+  return {
+    path,
+    stat: { size: entry.size, mtimeMs: entry.modifiedMs },
+    rawHash: entry.hash,
+    contentFingerprint: semanticFingerprint(entry),
+    validationFingerprint: "legacy-unvalidated",
+    needsFixing: false,
+    note: {
+      id: entry.id,
+      statement: entry.statement,
+      body: entry.body,
+      type: entry.type,
+      dontForget: entry.dontForget,
+      archived: entry.archived,
+    },
+  }
+}

package/src/services/search-index/location.ts

@@ -0,0 +1,12 @@
+import { createHash } from "node:crypto"
+import { join } from "node:path"
+
+import { comparablePath, type PathPlatform } from "#src/domain"
+import { INDEX_FILE_EXTENSION, INDEX_FILE_NAME_LENGTH } from "./constants.ts"
+
+export function indexFileFor(indexFolder: string, vaultPath: string, platform: PathPlatform): string {
+  const vaultKey = comparablePath(vaultPath, platform)
+  const name = createHash("sha256").update(vaultKey).digest("hex").slice(0, INDEX_FILE_NAME_LENGTH)
+
+  return join(indexFolder, `${name}${INDEX_FILE_EXTENSION}`)
+}

package/src/services/search-index/open.ts

@@ -0,0 +1,46 @@
+import { lstatSync, mkdirSync } from "node:fs"
+import { dirname } from "node:path"
+import { DatabaseSync } from "node:sqlite"
+
+import { closeFailedIndex } from "./cleanup.ts"
+import { indexFailure } from "./refusals.ts"
+import { initializeSchema } from "./schema.ts"
+import { SqliteSearchIndex } from "./storage.ts"
+import type { OpenIndexResult } from "./types.ts"
+
+export function openIndex(path: string): OpenIndexResult {
+  try {
+    for (const suffix of ["", "-journal", "-wal", "-shm"]) {
+      if (lstatSync(`${path}${suffix}`, { throwIfNoEntry: false })?.isSymbolicLink()) {
+        return {
+          succeeded: false,
+          reason: "unavailable",
+          problems: [`Index path must not be a link: ${path}${suffix}`],
+        }
+      }
+    }
+    mkdirSync(dirname(path), { recursive: true })
+    return initializeIndex(new DatabaseSync(path))
+  } catch (error) {
+    if (!(error instanceof Error)) {
+      throw error
+    }
+    return indexFailure(error)
+  }
+}
+
+function initializeIndex(database: DatabaseSync): OpenIndexResult {
+  try {
+    const schema = initializeSchema(database)
+    if (!schema.succeeded) {
+      return closeFailedIndex(database, schema)
+    }
+
+    return { succeeded: true, output: new SqliteSearchIndex(database), rebuilt: schema.output }
+  } catch (error) {
+    if (!(error instanceof Error)) {
+      throw error
+    }
+    return closeFailedIndex(database, indexFailure(error))
+  }
+}

package/src/services/search-index/query.ts

@@ -0,0 +1,39 @@
+import type { StatementSync } from "node:sqlite"
+
+import { eng, removeStopwords } from "stopword"
+import { Value } from "typebox/value"
+
+import { BODY_WEIGHT, STATEMENT_WEIGHT, WORD_PATTERN } from "./constants.ts"
+import { SearchHitSchema, SearchOptionsSchema } from "./schemas.ts"
+import type { SearchHit, SearchOptions, SearchResult } from "./types.ts"
+
+export function readSearchHits(statement: StatementSync, query: string, options: SearchOptions): SearchResult {
+  if (!Value.Check(SearchOptionsSchema, options)) {
+    return { succeeded: false, reason: "invalid", problems: ["search options require a boolean includeArchived"] }
+  }
+
+  const terms = searchTermsOf(query)
+  if (terms.length === 0) return { succeeded: true, output: [] }
+
+  const rows = statement.all({
+    match: terms.map((term) => `"${term}"`).join(" OR "),
+    includeArchived: Number(options.includeArchived),
+    statementWeight: STATEMENT_WEIGHT,
+    bodyWeight: BODY_WEIGHT,
+  })
+  const hits: SearchHit[] = []
+  for (const row of rows) {
+    if (!Value.Check(SearchHitSchema, row) || !Number.isFinite(row.score)) {
+      return { succeeded: false, reason: "corrupted", problems: ["a search result does not match the index schema"] }
+    }
+    hits.push(row)
+  }
+  return { succeeded: true, output: hits }
+}
+
+export function searchTermsOf(query: string): string[] {
+  const words = query.toLowerCase().match(WORD_PATTERN)
+  if (!words) return []
+
+  return removeStopwords([...new Set(words)], eng)
+}

package/src/services/search-index/ranking.ts

@@ -0,0 +1,12 @@
+export {
+  cosine,
+  type EmbeddingProvider,
+  type FusedPath,
+  fuseRankings,
+  MINIMUM_SIMILARITY,
+  type MinimumSimilarities,
+  planVectorRefresh,
+  type ScoredPath,
+  semanticRanking,
+  type VectorRefreshPlan,
+} from "#src/domain"

package/src/services/search-index/reconcile.ts

@@ -0,0 +1,93 @@
+import { Value } from "typebox/value"
+
+import type { CatalogChanges, CatalogEntry } from "#src/domain"
+import { catalogFolder } from "#src/domain"
+import type { prepareCatalogStatements } from "./catalog-statements.ts"
+import { corruptCatalog, validCatalogEntry } from "./inventory.ts"
+import { CatalogChangesSchema, CatalogStateSchema } from "./schemas.ts"
+import type { CatalogWriteRow, IndexWriteResult, SearchIndexRefusal } from "./types.ts"
+
+export function writeCatalog(
+  statements: ReturnType<typeof prepareCatalogStatements>,
+  changes: CatalogChanges,
+): IndexWriteResult {
+  if (!validChanges(changes)) return invalidCatalog("the catalog changes are invalid or overlap")
+  const state = statements.state.get()
+  if (!Value.Check(CatalogStateSchema, state)) return corruptCatalog()
+  if (state.revision !== changes.expectedRevision) return staleCatalog()
+
+  for (const path of changes.deletes) statements.remove.run(path)
+  for (const entry of changes.upserts) writeCatalogEntry(statements, entry)
+  for (const update of changes.statUpdates) {
+    const result = statements.stat.run(update.stat.size, update.stat.mtimeMs, update.path)
+    if (result.changes !== 1) return invalidCatalog("stat updates require an existing path")
+  }
+  statements.prune.run()
+  statements.advance.run(changes.population)
+  return { succeeded: true }
+}
+
+function validChanges(changes: CatalogChanges): boolean {
+  const shape = {
+    expectedRevision: changes.expectedRevision,
+    upserts: changes.upserts,
+    deletes: changes.deletes,
+    statUpdates: changes.statUpdates,
+    population: changes.population,
+  }
+  if (!Value.Check(CatalogChangesSchema, shape)) return false
+  const paths = [
+    ...changes.deletes,
+    ...changes.upserts.map((entry) => entry.path),
+    ...changes.statUpdates.map((update) => update.path),
+  ]
+  return (
+    new Set(paths).size === paths.length &&
+    changes.upserts.every(validCatalogEntry) &&
+    changes.statUpdates.every((update) => Number.isFinite(update.stat.mtimeMs))
+  )
+}
+
+export function writeCatalogEntry(statements: ReturnType<typeof prepareCatalogStatements>, entry: CatalogEntry): void {
+  const row = emptyNoteRow(entry)
+  if (entry.note) {
+    row.id = entry.note.id
+    row.type = entry.note.type
+    row.statement = entry.note.statement
+    row.body = entry.note.body
+    row.dont_forget = 0
+    row.archived = 0
+    if (entry.note.dontForget) row.dont_forget = 1
+    if (entry.note.archived) row.archived = 1
+  }
+  statements.put.run({ ...row })
+}
+
+function emptyNoteRow(entry: CatalogEntry): CatalogWriteRow {
+  const row: CatalogWriteRow = {
+    path: entry.path,
+    folder: catalogFolder(entry.path),
+    id: null,
+    type: null,
+    statement: null,
+    body: null,
+    dont_forget: null,
+    archived: null,
+    size: entry.stat.size,
+    modified_ms: entry.stat.mtimeMs,
+    hash: entry.rawHash,
+    content_fingerprint: entry.contentFingerprint,
+    validation_fingerprint: entry.validationFingerprint,
+    needs_fixing: 0,
+  }
+  if (entry.needsFixing) row.needs_fixing = 1
+  return row
+}
+
+export function invalidCatalog(problem: string): IndexWriteResult {
+  return { succeeded: false, reason: "invalid", problems: [problem] }
+}
+
+export function staleCatalog(): SearchIndexRefusal {
+  return { succeeded: false, reason: "conflict", problems: ["the catalog changed since its inventory was read"] }
+}

package/src/services/search-index/recovery.ts

@@ -0,0 +1,56 @@
+import { randomUUID } from "node:crypto"
+import { renameSync, rmSync } from "node:fs"
+
+import { openIndex } from "./open.ts"
+import { indexFailure } from "./refusals.ts"
+import type { IndexLocation, IndexReplacement, OpenedIndex, OpenIndexResult } from "./types.ts"
+
+export function recoverIndex(location: IndexLocation): OpenIndexResult {
+  const opened = inspectExistingIndex(location)
+  if (opened.succeeded || opened.reason !== "corrupted") return opened
+
+  const temporaryPath = `${location.path}.rebuild-${randomUUID()}`
+  const replacement = installReplacement({ path: location.path, temporaryPath })
+  if (replacement.succeeded) return replacement
+
+  try {
+    rmSync(temporaryPath, { force: true })
+  } catch (error) {
+    if (!(error instanceof Error)) throw error
+    replacement.problems.push(`temporary index cleanup failed: ${error.message}`)
+  }
+  return replacement
+}
+
+function inspectExistingIndex(location: IndexLocation): OpenIndexResult {
+  const opened = openIndex(location.path)
+  if (!opened.succeeded) return opened
+  return validateOpenedIndex(opened)
+}
+
+function validateOpenedIndex(opened: OpenedIndex): OpenIndexResult {
+  const inventory = opened.output.inventory()
+  const vectors = opened.output.vectors()
+  if (inventory.succeeded && vectors.succeeded) return opened
+  const closed = opened.output.close()
+  if (!closed.succeeded) return closed
+  if (!inventory.succeeded) return inventory
+  if (!vectors.succeeded) return vectors
+  return opened
+}
+
+function installReplacement(request: IndexReplacement): OpenIndexResult {
+  try {
+    const temporary = openIndex(request.temporaryPath)
+    if (!temporary.succeeded) return temporary
+    const closed = temporary.output.close()
+    if (!closed.succeeded) return closed
+    renameSync(request.temporaryPath, request.path)
+    const opened = openIndex(request.path)
+    if (!opened.succeeded) return opened
+    return { ...opened, rebuilt: true }
+  } catch (error) {
+    if (!(error instanceof Error)) throw error
+    return indexFailure(error)
+  }
+}

package/src/services/search-index/refusals.ts

@@ -0,0 +1,21 @@
+import { SQLITE_CORRUPT, SQLITE_ERROR, SQLITE_NOTADB, SQLITE_PRIMARY_CODE_MASK } from "./constants.ts"
+import type { SearchIndexRefusal } from "./types.ts"
+
+export function indexFailure(error: Error): SearchIndexRefusal {
+  if ("errcode" in error && typeof error.errcode === "number") {
+    const primaryCode = error.errcode & SQLITE_PRIMARY_CODE_MASK
+    if (primaryCode === SQLITE_ERROR || primaryCode === SQLITE_CORRUPT || primaryCode === SQLITE_NOTADB) {
+      return { succeeded: false, reason: "corrupted", problems: [error.message] }
+    }
+  }
+
+  return { succeeded: false, reason: "unavailable", problems: [error.message] }
+}
+
+export function closedIndex(): SearchIndexRefusal {
+  return {
+    succeeded: false,
+    reason: "closed",
+    problems: ["the search index is closed or unusable; open a new connection"],
+  }
+}

package/src/services/search-index/schema.ts

@@ -0,0 +1,42 @@
+import type { DatabaseSync } from "node:sqlite"
+
+import { INDEX_SCHEMA_VERSION, REBUILD_SCHEMA_SQL } from "./constants.ts"
+import { indexFailure } from "./refusals.ts"
+import type { SchemaResult, SearchIndexRefusal } from "./types.ts"
+
+export function initializeSchema(database: DatabaseSync): SchemaResult {
+  try {
+    database.exec("BEGIN IMMEDIATE")
+    const row = database.prepare("PRAGMA user_version").get()
+    if (!row || typeof row.user_version !== "number") {
+      return rollBackSchema(database, {
+        succeeded: false,
+        reason: "corrupted",
+        problems: ["the search index has no valid schema version"],
+      })
+    }
+
+    const rebuilt = row.user_version !== INDEX_SCHEMA_VERSION
+    if (rebuilt) {
+      database.exec(REBUILD_SCHEMA_SQL)
+    }
+    database.exec("INSERT INTO notes_fts(notes_fts, rank) VALUES ('integrity-check', 1)")
+    database.exec("COMMIT")
+    return { succeeded: true, output: rebuilt }
+  } catch (error) {
+    if (!(error instanceof Error)) throw error
+    return rollBackSchema(database, indexFailure(error))
+  }
+}
+
+function rollBackSchema(database: DatabaseSync, failure: SearchIndexRefusal): SearchIndexRefusal {
+  if (!database.isTransaction) return failure
+
+  try {
+    database.exec("ROLLBACK")
+  } catch (error) {
+    if (!(error instanceof Error)) throw error
+    failure.problems.push(`schema rollback failed: ${error.message}`)
+  }
+  return failure
+}

package/src/services/search-index/schemas.ts

@@ -0,0 +1,181 @@
+import Type from "typebox"
+
+import {
+  CatalogEntrySchema,
+  CatalogPathSchema,
+  CatalogStatSchema,
+  NoteIdSchema,
+  NoteTypeSchema,
+  VectorGenerationSchema,
+} from "#src/domain"
+
+export const SearchHitSchema = Type.Object(
+  {
+    id: NoteIdSchema,
+    folder: Type.String(),
+    statement: Type.String(),
+    type: NoteTypeSchema,
+    score: Type.Number(),
+  },
+  { additionalProperties: false },
+)
+
+export const IndexEntrySchema = Type.Object(
+  {
+    id: NoteIdSchema,
+    folder: Type.String(),
+    statement: Type.String(),
+    body: Type.String(),
+    type: NoteTypeSchema,
+    dontForget: Type.Boolean(),
+    archived: Type.Boolean(),
+    size: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    modifiedMs: Type.Number(),
+    hash: Type.String(),
+  },
+  { additionalProperties: false },
+)
+
+export const CatalogStateSchema = Type.Object(
+  {
+    revision: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    population: Type.Union([Type.Literal("ready"), Type.Literal("needs-population")]),
+  },
+  { additionalProperties: false },
+)
+
+export const CatalogChangesSchema = Type.Object(
+  {
+    expectedRevision: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    upserts: Type.Array(CatalogEntrySchema),
+    deletes: Type.Array(CatalogPathSchema),
+    statUpdates: Type.Array(
+      Type.Object({ path: CatalogPathSchema, stat: CatalogStatSchema }, { additionalProperties: false }),
+    ),
+    population: Type.Union([Type.Literal("ready"), Type.Literal("needs-population")]),
+  },
+  { additionalProperties: false },
+)
+
+export const CatalogRowSchema = Type.Object(
+  {
+    path: CatalogPathSchema,
+    id: Type.Union([NoteIdSchema, Type.Null()]),
+    type: Type.Union([NoteTypeSchema, Type.Null()]),
+    statement: Type.Union([Type.String(), Type.Null()]),
+    body: Type.Union([Type.String(), Type.Null()]),
+    dont_forget: Type.Union([Type.Literal(0), Type.Literal(1), Type.Null()]),
+    archived: Type.Union([Type.Literal(0), Type.Literal(1), Type.Null()]),
+    size: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    modified_ms: Type.Number(),
+    hash: Type.String({ minLength: 1 }),
+    content_fingerprint: Type.Union([Type.String({ minLength: 1 }), Type.Null()]),
+    validation_fingerprint: Type.String({ minLength: 1 }),
+    needs_fixing: Type.Union([Type.Literal(0), Type.Literal(1)]),
+  },
+  { additionalProperties: false },
+)
+
+export const CatalogHitSchema = Type.Object(
+  {
+    path: CatalogPathSchema,
+    id: NoteIdSchema,
+    type: NoteTypeSchema,
+    statement: Type.String(),
+    dontForget: Type.Boolean(),
+    needsFixing: Type.Boolean(),
+    score: Type.Number(),
+  },
+  { additionalProperties: false },
+)
+
+export const CatalogHitRowSchema = Type.Object(
+  {
+    path: CatalogPathSchema,
+    id: NoteIdSchema,
+    type: NoteTypeSchema,
+    statement: Type.String(),
+    dont_forget: Type.Union([Type.Literal(0), Type.Literal(1)]),
+    needs_fixing: Type.Union([Type.Literal(0), Type.Literal(1)]),
+    score: Type.Number(),
+  },
+  { additionalProperties: false },
+)
+
+export const VectorValuesSchema = Type.Array(Type.Number(), { minItems: 1 })
+
+export const VectorChunkSchema = Type.Object(
+  {
+    contentFingerprint: Type.String({ minLength: 1 }),
+    chunk: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    vector: VectorValuesSchema,
+  },
+  { additionalProperties: false },
+)
+
+export const VectorReplacementSchema = Type.Object(
+  {
+    expectedRevision: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    generation: VectorGenerationSchema,
+    chunks: Type.Array(VectorChunkSchema),
+  },
+  { additionalProperties: false },
+)
+
+export const VectorUpdateSchema = Type.Object(
+  {
+    expectedRevision: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    generation: VectorGenerationSchema,
+    add: Type.Array(VectorChunkSchema),
+    remove: Type.Array(Type.String({ minLength: 1 })),
+  },
+  { additionalProperties: false },
+)
+
+export const VectorRowSchema = Type.Object(
+  {
+    content_fingerprint: Type.String({ minLength: 1 }),
+    chunk: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    vector: Type.String(),
+  },
+  { additionalProperties: false },
+)
+
+export const SearchOptionsSchema = Type.Object(
+  {
+    includeArchived: Type.Boolean(),
+  },
+  { additionalProperties: false },
+)
+
+export const FolderIndexStateSchema = Type.Object(
+  {
+    folder: Type.String(),
+    fingerprint: Type.String({ minLength: 1 }),
+    indexSize: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    indexMtimeMs: Type.Number(),
+    indexCtimeMs: Type.Number(),
+    indexIno: Type.String({ minLength: 1 }),
+  },
+  { additionalProperties: false },
+)
+
+export const FolderIndexWriteSchema = Type.Object(
+  {
+    states: Type.Array(FolderIndexStateSchema),
+    keepFolders: Type.Array(Type.String()),
+  },
+  { additionalProperties: false },
+)
+
+export const FolderIndexRowSchema = Type.Object(
+  {
+    folder: Type.String(),
+    fingerprint: Type.String({ minLength: 1 }),
+    index_size: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
+    index_mtime_ms: Type.Number(),
+    index_ctime_ms: Type.Number(),
+    index_ino: Type.String({ minLength: 1 }),
+  },
+  { additionalProperties: false },
+)

package/src/services/search-index/search.ts

@@ -0,0 +1,5 @@
+import type { SearchIndex, SearchOptions, SearchResult } from "./types.ts"
+
+export function searchText(index: SearchIndex, query: string, options: SearchOptions): SearchResult {
+  return index.search(query, options)
+}

package/src/services/search-index/statements.ts

@@ -0,0 +1,18 @@
+import type { DatabaseSync } from "node:sqlite"
+
+export function prepareStatements(database: DatabaseSync) {
+  return {
+    byId: database.prepare("SELECT path FROM notes WHERE id = ?"),
+    move: database.prepare("UPDATE notes SET path = ? WHERE id = ?"),
+    folder: database.prepare("UPDATE notes SET folder = ? WHERE path = ?"),
+    remove: database.prepare("DELETE FROM notes WHERE id = :id"),
+    search: database.prepare(`
+      SELECT notes.id, notes.folder, notes.statement, notes.type,
+        -bm25(notes_fts, :statementWeight, :bodyWeight) AS score
+      FROM notes_fts
+      JOIN notes ON notes.rowid = notes_fts.rowid
+      WHERE notes_fts MATCH :match AND notes.id IS NOT NULL AND (notes.archived = 0 OR :includeArchived = 1)
+      ORDER BY score DESC, notes.id
+    `),
+  }
+}