pi-obsidian-memory 0.0.1 → 0.1.0

package/src/domain/decision-model/rank.ts

@@ -0,0 +1,36 @@
+import type { CandidateScore } from "./types.ts"
+
+function validateProbability(probability: number): void {
+  if (!Number.isFinite(probability) || probability < 0 || probability > 1) {
+    throw new RangeError("Decision probability must be finite and between 0 and 1")
+  }
+}
+
+function validateCutoff(cutoff: number): void {
+  if (!Number.isFinite(cutoff) || cutoff < 0 || cutoff > 1) {
+    throw new RangeError("Decision cutoff must be finite and between 0 and 1")
+  }
+}
+
+export function shouldRetrieveMemory(probability: number, cutoff: number): boolean {
+  validateProbability(probability)
+  validateCutoff(cutoff)
+  return probability > cutoff
+}
+
+function compareScores(left: CandidateScore, right: CandidateScore): number {
+  const probabilityOrder = right.probability - left.probability
+  if (probabilityOrder !== 0) return probabilityOrder
+
+  for (const field of ["vault", "id", "revision"] as const) {
+    if (left.identity[field] < right.identity[field]) return -1
+    if (left.identity[field] > right.identity[field]) return 1
+  }
+  return 0
+}
+
+export function rankCandidates(scores: readonly CandidateScore[], cutoff: number): CandidateScore[] {
+  validateCutoff(cutoff)
+  for (const score of scores) validateProbability(score.probability)
+  return scores.filter((score) => score.probability >= cutoff).sort(compareScores)
+}

package/src/domain/decision-model/types.ts

@@ -0,0 +1,77 @@
+export interface CandidateIdentity {
+  vault: "global" | "project"
+  id: string
+  revision: string
+}
+
+export interface DecisionRequest {
+  message: string
+  project: string | null
+  folder: string
+}
+
+export interface CandidateContent {
+  statement: string
+  body: string
+}
+
+export interface CandidateDecisionRequest {
+  message: string
+  identity: CandidateIdentity
+  candidate: CandidateContent
+}
+
+export interface CandidateScore {
+  identity: CandidateIdentity
+  probability: number
+}
+
+export interface DecisionUsage {
+  inputTokens: number
+  outputTokens: number
+}
+
+export interface DecisionDiagnostics {
+  provider: "laya" | "typesafe"
+  requestedModel: string
+  reportedModel: string | null
+  usage: DecisionUsage | null
+}
+
+export interface RelevanceDecision {
+  probability: number
+  diagnostics: DecisionDiagnostics
+}
+
+export type DecisionFailureCode =
+  | "unsafe-endpoint"
+  | "invalid-settings"
+  | "unavailable"
+  | "http-error"
+  | "invalid-response"
+  | "routing-missing"
+  | "truncated"
+  | "collapsed-options"
+  | "aborted"
+
+export interface DecisionFailure {
+  succeeded: false
+  code: DecisionFailureCode
+  diagnostics: DecisionDiagnostics
+  httpStatus?: number
+  detail?: string
+}
+
+export interface DecisionSuccess {
+  succeeded: true
+  output: RelevanceDecision
+}
+
+export type DecisionResult = DecisionSuccess | DecisionFailure
+
+export type DecisionProvider = DecisionDiagnostics["provider"]
+
+export interface DecisionCutoffs {
+  gate: number
+  relevance: number
+}

package/src/domain/embeddings/constants.ts

@@ -0,0 +1,55 @@
+import type { EmbeddingConfiguration, EmbeddingPolicy, MinimumSimilarities } from "./types.ts"
+
+export const localEmbeddingConfiguration = {
+  provider: "local",
+  model: "Xenova/multilingual-e5-small",
+} as const satisfies EmbeddingConfiguration
+
+export const openAIEmbeddingConfiguration = {
+  provider: "openai",
+  model: "text-embedding-3-small",
+} as const satisfies EmbeddingConfiguration
+
+export const googleEmbeddingConfiguration = {
+  provider: "google",
+  model: "gemini-embedding-001",
+} as const satisfies EmbeddingConfiguration
+
+export const localEmbeddingPolicy = {
+  revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78",
+  dimensions: 384,
+  dtype: "cpu-q8-transformers-4.3.1",
+  pooling: "mean",
+  normalization: "l2",
+  queryPrefix: "query: ",
+  documentPrefix: "passage: ",
+  chunking: "v1-token-aware-512-special-prefix-inclusive-overlap32-mean-chunks-query-refuse",
+} as const satisfies EmbeddingPolicy
+
+export const openAIEmbeddingPolicy = {
+  revision: "provider-managed",
+  dimensions: 1536,
+  dtype: "float",
+  pooling: "provider",
+  normalization: "l2",
+  queryPrefix: "",
+  documentPrefix: "",
+  chunking: "v1-utf8-bytes4096-overlap128-mean-chunks-query-refuse",
+} as const satisfies EmbeddingPolicy
+
+export const googleEmbeddingPolicy = {
+  revision: "provider-managed",
+  dimensions: 768,
+  dtype: "float",
+  pooling: "provider",
+  normalization: "l2",
+  queryPrefix: "RETRIEVAL_QUERY",
+  documentPrefix: "RETRIEVAL_DOCUMENT",
+  chunking: "v1-utf8-bytes1024-overlap128-mean-chunks-query-refuse-autoTruncate-false",
+} as const satisfies EmbeddingPolicy
+
+export const MINIMUM_SIMILARITY = {
+  local: 0.8,
+  openai: null,
+  google: null,
+} as const satisfies MinimumSimilarities

package/src/domain/embeddings/identity.ts

@@ -0,0 +1,30 @@
+import { googleEmbeddingPolicy, localEmbeddingPolicy, openAIEmbeddingPolicy } from "./constants.ts"
+import type { EmbeddingConfiguration, EmbeddingPolicy } from "./types.ts"
+
+export function embeddingPolicy(configuration: EmbeddingConfiguration): EmbeddingPolicy {
+  switch (configuration.provider) {
+    case "local":
+      return localEmbeddingPolicy
+    case "openai":
+      return openAIEmbeddingPolicy
+    case "google":
+      return googleEmbeddingPolicy
+  }
+}
+
+export function embeddingFingerprint(configuration: EmbeddingConfiguration): string {
+  const policy = embeddingPolicy(configuration)
+  return JSON.stringify([
+    "embeddings-v1",
+    configuration.provider,
+    configuration.model,
+    policy.revision,
+    policy.dimensions,
+    policy.dtype,
+    policy.pooling,
+    policy.normalization,
+    policy.queryPrefix,
+    policy.documentPrefix,
+    policy.chunking,
+  ])
+}

package/src/domain/embeddings/index.ts

@@ -0,0 +1,5 @@
+export * from "./constants.ts"
+export * from "./identity.ts"
+export * from "./schemas.ts"
+export * from "./types.ts"
+export * from "./vectors.ts"

package/src/domain/embeddings/schemas.ts

@@ -0,0 +1,25 @@
+import Type from "typebox"
+
+export const EmbeddingConfigurationSchema = Type.Union([
+  Type.Object(
+    {
+      provider: Type.Literal("local"),
+      model: Type.Literal("Xenova/multilingual-e5-small"),
+    },
+    { additionalProperties: false },
+  ),
+  Type.Object(
+    {
+      provider: Type.Literal("openai"),
+      model: Type.Literal("text-embedding-3-small"),
+    },
+    { additionalProperties: false },
+  ),
+  Type.Object(
+    {
+      provider: Type.Literal("google"),
+      model: Type.Literal("gemini-embedding-001"),
+    },
+    { additionalProperties: false },
+  ),
+])

package/src/domain/embeddings/types.ts

@@ -0,0 +1,52 @@
+import type { Static } from "typebox"
+
+import type { Succeeded } from "#src/shared"
+import type { EmbeddingConfigurationSchema } from "./schemas.ts"
+
+export type EmbeddingConfiguration = Static<typeof EmbeddingConfigurationSchema>
+
+export interface EmbeddingInput {
+  key: string
+  text: string
+}
+
+export interface EmbeddingBatchRequest {
+  purpose: "query" | "document"
+  inputs: readonly EmbeddingInput[]
+}
+
+export interface EmbeddedInput {
+  key: string
+  vector: readonly number[]
+}
+
+export interface EmbeddingBatch {
+  fingerprint: string
+  dimensions: number
+  items: readonly EmbeddedInput[]
+}
+
+export type EmbeddingFailureStatus = "unavailable" | "input-too-long" | "invalid-input" | "invalid-response" | "aborted"
+
+export interface EmbeddingFailure {
+  succeeded: false
+  status: EmbeddingFailureStatus
+  cleanupFailed?: true
+}
+
+export type EmbeddingResult = Succeeded<EmbeddingBatch> | EmbeddingFailure
+
+export interface EmbeddingPolicy {
+  revision: string
+  dimensions: number
+  dtype: string
+  pooling: string
+  normalization: "l2"
+  queryPrefix: string
+  documentPrefix: string
+  chunking: string
+}
+
+export type EmbeddingProvider = EmbeddingConfiguration["provider"]
+
+export type MinimumSimilarities = Record<EmbeddingProvider, number | null>

package/src/domain/embeddings/vectors.ts

@@ -0,0 +1,53 @@
+export function normalizeVector(values: readonly number[], dimensions: number): number[] | null {
+  if (values.length !== dimensions || dimensions < 1 || values.some((value) => !Number.isFinite(value))) {
+    return null
+  }
+
+  const scale = values.reduce((largest, value) => Math.max(largest, Math.abs(value)), 0)
+  if (scale === 0) {
+    return null
+  }
+
+  const scaled = values.map((value) => value / scale)
+  const norm = Math.sqrt(scaled.reduce((sum, value) => sum + value * value, 0))
+  return scaled.map((value) => value / norm)
+}
+
+export function aggregateVectors(vectors: readonly (readonly number[])[], dimensions: number): number[] | null {
+  if (vectors.length === 0) {
+    return null
+  }
+
+  const sum = Array<number>(dimensions).fill(0)
+  for (const values of vectors) {
+    const vector = normalizeVector(values, dimensions)
+    if (!vector) {
+      return null
+    }
+    for (const [index, value] of vector.entries()) {
+      const previous = sum[index]
+      if (typeof previous !== "number") {
+        return null
+      }
+      sum[index] = previous + value / vectors.length
+    }
+  }
+  return normalizeVector(sum, dimensions)
+}
+
+export function cosine(left: readonly number[], right: readonly number[]): number {
+  if (left.length !== right.length) throw new RangeError("Vectors must have the same dimensions")
+
+  let dot = 0
+  let leftNorm = 0
+  let rightNorm = 0
+  for (const [index, value] of left.entries()) {
+    const other = right[index]
+    if (typeof other !== "number") throw new RangeError("Vectors must have the same dimensions")
+    dot += value * other
+    leftNorm += value * value
+    rightNorm += other * other
+  }
+  if (leftNorm === 0 || rightNorm === 0) throw new RangeError("Vectors must be nonzero")
+  return dot / Math.sqrt(leftNorm * rightNorm)
+}

package/src/domain/folder-index/constants.ts

@@ -0,0 +1,9 @@
+export const NOTE_LINE_SEPARATOR = " · "
+
+export const PROBLEM_SEPARATOR = "; "
+
+export const DONT_FORGET_LABEL = "don't forget"
+
+export const INDEX_TEXT_COLLATOR = new Intl.Collator("en", { sensitivity: "base" })
+
+export const FOLDER_INDEX_FORMAT = 1

package/src/domain/folder-index/display.ts

@@ -0,0 +1,22 @@
+export function displayIndexText(text: string): string {
+  if (/[<>[\]`]/.test(text)) {
+    return displayIndexName(text)
+  }
+
+  return text
+}
+
+export function displayIndexName(name: string): string {
+  if (name.length === 0) {
+    return "<code></code>"
+  }
+
+  const runs = [...name.matchAll(/`+/g)].map((match) => match[0].length)
+  const fence = "`".repeat(Math.max(0, ...runs) + 1)
+  const mixedEdgeSpaces = name.startsWith(" ") && name.endsWith(" ") && /[^ ]/.test(name)
+  if (name.startsWith("`") || name.endsWith("`") || mixedEdgeSpaces) {
+    return `${fence} ${name} ${fence}`
+  }
+
+  return `${fence}${name}${fence}`
+}

package/src/domain/folder-index/index.ts

@@ -0,0 +1,5 @@
+export * from "./constants.ts"
+export * from "./display.ts"
+export * from "./listing.ts"
+export * from "./render.ts"
+export * from "./types.ts"

package/src/domain/folder-index/listing.ts

@@ -0,0 +1,58 @@
+import { createHash } from "node:crypto"
+
+import type { VaultCatalogEntry, VaultEnumeration } from "#src/domain/vault-catalog"
+import type { FolderListingFingerprint } from "./types.ts"
+
+export function folderListingFingerprints(
+  enumeration: VaultEnumeration,
+  renderKey: string,
+): FolderListingFingerprint[] {
+  const entries = new Map(enumeration.entries.map((entry) => [entry.path, entry]))
+  return enumeration.completeDirectories.map((folder) => ({
+    folder,
+    fingerprint: fingerprintOf(enumeration, entries, folder, renderKey),
+  }))
+}
+
+function fingerprintOf(
+  enumeration: VaultEnumeration,
+  entries: Map<string, VaultCatalogEntry>,
+  folder: string,
+  renderKey: string,
+): string | null {
+  if (enumeration.failures.some((failure) => failure.path === folder || parentOf(failure.path) === folder)) {
+    return null
+  }
+
+  const children: string[] = []
+
+  for (const entry of enumeration.entries) {
+    if (parentOf(entry.path) !== folder) continue
+    if (entry.kind === "folder") children.push(JSON.stringify(["folder", nameOf(entry.path)]))
+    if (entry.kind === "note") children.push(JSON.stringify(["file", nameOf(entry.path), ...statOf(entry)]))
+  }
+
+  for (const alias of enumeration.aliases) {
+    if (parentOf(alias.path) !== folder) continue
+    const target = entries.get(alias.target)
+    if (!target || nameOf(alias.path).toLowerCase() === "index.md") return null
+    children.push(JSON.stringify(["alias", nameOf(alias.path), alias.target, ...statOf(target)]))
+  }
+
+  return createHash("sha256")
+    .update(JSON.stringify([renderKey, children.sort()]))
+    .digest("hex")
+}
+
+function parentOf(path: string): string {
+  return path.slice(0, Math.max(path.lastIndexOf("/"), 0))
+}
+
+function nameOf(path: string): string {
+  return path.slice(path.lastIndexOf("/") + 1)
+}
+
+function statOf(entry: VaultCatalogEntry): number[] {
+  const { size, mtimeMs, ctimeMs, device, inode } = entry.stat
+  return [size, mtimeMs, ctimeMs, device, inode]
+}

package/src/domain/folder-index/render.ts

@@ -0,0 +1,114 @@
+import type { Note } from "#src/domain/note"
+import { ARCHIVE_FOLDER, checkVaultPath, FOLDER_INDEX_NAME, joinVaultPath } from "#src/domain/vault-path"
+import { DONT_FORGET_LABEL, INDEX_TEXT_COLLATOR, NOTE_LINE_SEPARATOR, PROBLEM_SEPARATOR } from "./constants.ts"
+import { displayIndexName, displayIndexText } from "./display.ts"
+import type { FolderContents, InvalidNote, Titled } from "./types.ts"
+
+export function renderFolderIndex(folder: FolderContents): string {
+  const path = linkPathOf(folder)
+
+  const subjects = sortAlphabetically(folder.subjects).map((subject) => subjectLine(path, subject))
+  const notes = [...folder.notes].sort(byStatement).map((note) => noteLine(path, note))
+  const invalid = [...folder.invalid].sort(byStatement).map((note) => invalidLine(path, note))
+
+  const blocks = [
+    `# ${displayIndexText(titleOf(folder))}`,
+    ...section("Subjects", subjects),
+    ...section("Notes", notes),
+    ...section("Needs fixing", [...invalid, ...problemFolderLines(folder)]),
+    ...archiveSection(folder),
+  ]
+
+  return `${blocks.join("\n\n")}\n`
+}
+
+function titleOf(folder: FolderContents): string {
+  if (folder.kind === "root") {
+    return folder.vaultName
+  }
+
+  return folder.path
+}
+
+function linkPathOf(folder: FolderContents): string {
+  if (folder.kind === "root") {
+    return ""
+  }
+
+  return folder.path
+}
+
+function section(title: string, lines: string[]): string[] {
+  if (lines.length === 0) {
+    return []
+  }
+
+  return [`## ${title}`, lines.join("\n")]
+}
+
+function archiveSection(folder: FolderContents): string[] {
+  if (folder.kind === "root" && folder.hasArchive) {
+    return section("Archive", [subjectLine("", ARCHIVE_FOLDER)])
+  }
+
+  return []
+}
+
+function subjectLine(path: string, subject: string): string {
+  const target = joinVaultPath(path, subject, FOLDER_INDEX_NAME)
+  const problems = checkVaultPath(target)
+  if (problems.length > 0) {
+    return `- ${displayIndexName(subject)}${NOTE_LINE_SEPARATOR}${problems.map(displayIndexName).join(PROBLEM_SEPARATOR)}`
+  }
+
+  return `- [[${target}|${subject}]]`
+}
+
+function noteLine(path: string, note: Note): string {
+  const parts = [noteLink(path, note.statement), note.frontmatter.type]
+
+  if (note.frontmatter.dontForget) {
+    parts.push(DONT_FORGET_LABEL)
+  }
+
+  return `- ${parts.join(NOTE_LINE_SEPARATOR)}`
+}
+
+function invalidLine(path: string, note: InvalidNote): string {
+  const parts = [noteLink(path, note.statement), note.problems.map(displayIndexText).join(PROBLEM_SEPARATOR)]
+  return `- ${parts.join(NOTE_LINE_SEPARATOR)}`
+}
+
+function noteLink(path: string, statement: string): string {
+  const target = joinVaultPath(path, statement)
+  if (statement.length === 0 || checkVaultPath(target).length > 0) {
+    return displayIndexName(statement)
+  }
+
+  return `[[${target}|${statement}]]`
+}
+
+function problemFolderLines(folder: FolderContents): string[] {
+  if (!folder.problemFolders) {
+    return []
+  }
+
+  return [...folder.problemFolders]
+    .sort((first, second) => compareText(first.folder, second.folder))
+    .map(
+      (problem) =>
+        `- Folder ${displayIndexName(problem.folder)}${NOTE_LINE_SEPARATOR}${problem.problems.map(displayIndexText).join(PROBLEM_SEPARATOR)}`,
+    )
+}
+
+function sortAlphabetically(values: string[]): string[] {
+  return [...values].sort(compareText)
+}
+
+function byStatement(first: Titled, second: Titled): number {
+  return compareText(first.statement, second.statement)
+}
+
+function compareText(first: string, second: string): number {
+  return INDEX_TEXT_COLLATOR.compare(first, second)
+}

package/src/domain/folder-index/types.ts

@@ -0,0 +1,41 @@
+import type { Note } from "#src/domain/note"
+
+export interface Titled {
+  statement: string
+}
+
+export interface InvalidNote {
+  statement: string
+  problems: string[]
+}
+
+export interface ProblemFolder {
+  folder: string
+  problems: string[]
+}
+
+export interface SubjectFolder {
+  kind: "subject"
+  path: string
+  subjects: string[]
+  notes: Note[]
+  invalid: InvalidNote[]
+  problemFolders?: ProblemFolder[]
+}
+
+export interface RootFolder {
+  kind: "root"
+  vaultName: string
+  subjects: string[]
+  notes: Note[]
+  invalid: InvalidNote[]
+  hasArchive: boolean
+  problemFolders?: ProblemFolder[]
+}
+
+export type FolderContents = SubjectFolder | RootFolder
+
+export interface FolderListingFingerprint {
+  folder: string
+  fingerprint: string | null
+}

package/src/domain/index.ts

@@ -0,0 +1,22 @@
+export * from "./capture/index.ts"
+export * from "./command-flags/index.ts"
+export * from "./config/index.ts"
+export * from "./decision-model/index.ts"
+export * from "./embeddings/index.ts"
+export * from "./folder-index/index.ts"
+export * from "./lifecycle/index.ts"
+export * from "./loaded-memory/index.ts"
+export * from "./memory/index.ts"
+export * from "./memory-agent/index.ts"
+export * from "./names/index.ts"
+export * from "./note/index.ts"
+export * from "./path-platform/index.ts"
+export * from "./search-catalog/index.ts"
+export * from "./shell-edit/index.ts"
+export * from "./tool-flags/index.ts"
+export * from "./validation/index.ts"
+export * from "./vault-access/index.ts"
+export * from "./vault-assets/index.ts"
+export * from "./vault-catalog/index.ts"
+export * from "./vault-diagnosis/index.ts"
+export * from "./vault-path/index.ts"

package/src/domain/lifecycle/index.ts

@@ -0,0 +1,2 @@
+export * from "./lifecycle.ts"
+export * from "./types.ts"

package/src/domain/lifecycle/lifecycle.ts

@@ -0,0 +1,39 @@
+import type { Note } from "#src/domain/note"
+import { ARCHIVE_FOLDER, joinVaultPath } from "#src/domain/vault-path"
+import type { NoteChange, PlacedNote, Succession } from "./types.ts"
+
+export function updatedNote(note: Note, change: NoteChange, today: string): Note {
+  const { dontForget: _previous, ...kept } = note.frontmatter
+  const frontmatter = { ...kept, type: change.type, updated: today }
+  if (change.dontForget) return { ...note, frontmatter: { ...frontmatter, dontForget: true }, body: change.body }
+  return { ...note, frontmatter, body: change.body }
+}
+
+export function archiveNote(placed: PlacedNote, today: string): PlacedNote {
+  const { note } = placed
+
+  return {
+    folder: joinVaultPath(ARCHIVE_FOLDER, placed.folder),
+    note: { ...note, frontmatter: { ...note.frontmatter, archived: today } },
+  }
+}
+
+export function succeedNote(previous: PlacedNote, successor: PlacedNote, today: string): Succession {
+  const archived = archiveNote(previous, today)
+  const born = { ...successor.note.frontmatter, created: today, updated: today }
+
+  return {
+    archived: {
+      folder: archived.folder,
+      note: { ...archived.note, frontmatter: { ...archived.note.frontmatter, replacedBy: linkTo(successor) } },
+    },
+    successor: {
+      folder: successor.folder,
+      note: { ...successor.note, frontmatter: { ...born, replaces: linkTo(archived) } },
+    },
+  }
+}
+
+function linkTo(placed: PlacedNote): string {
+  return `[[${joinVaultPath(placed.folder, placed.note.statement)}]]`
+}

package/src/domain/lifecycle/types.ts

@@ -0,0 +1,17 @@
+import type { Note, NoteType } from "#src/domain/note"
+
+export interface NoteChange {
+  body: string
+  type: NoteType
+  dontForget: boolean
+}
+
+export interface PlacedNote {
+  folder: string
+  note: Note
+}
+
+export interface Succession {
+  archived: PlacedNote
+  successor: PlacedNote
+}