pi-obsidian-memory 0.0.1 → 0.1.1

package/src/domain/config/types.ts

@@ -0,0 +1,239 @@
+import type { Static, TSchema } from "typebox"
+
+import type { CaptureCadenceSettings } from "#src/domain/capture"
+import type { Absent, Failed, Succeeded } from "#src/shared"
+import type {
+  CaptureWhenSchema,
+  ChatModelReferenceSchema,
+  ConfigFieldObservationSchema,
+  MemoryConfigSchema,
+  ProjectEntrySchema,
+} from "./schemas.ts"
+
+export type MemoryConfig = Static<typeof MemoryConfigSchema>
+
+export type ProjectEntry = Static<typeof ProjectEntrySchema>
+
+export type LegacyCaptureWhen = Static<typeof CaptureWhenSchema>
+
+export type ParseConfigResult = Succeeded<MemoryConfig> | Failed
+
+export type ConfigFieldObservation = Static<typeof ConfigFieldObservationSchema>
+
+export interface ConfigObservationRule {
+  field: ConfigFieldObservation["field"]
+  index: number
+  path: string
+}
+
+export interface ConfigDocumentInspection {
+  config: Succeeded<MemoryConfig> | Absent
+  observations: ConfigFieldObservation[]
+  projectCount: number
+  shapeProblems: string[]
+  invalidTokenOrder: boolean
+}
+
+export interface ConfigJsonInspection extends ConfigDocumentInspection {
+  validJson: true
+}
+
+export interface ConfigSyntaxFailure {
+  validJson: false
+  message: string
+}
+
+export type ConfigInspection = ConfigJsonInspection | ConfigSyntaxFailure
+
+export interface ConfigAssessment {
+  lexicalParse: ParseConfigResult
+  repair: ConfigRepairDraft
+}
+
+export type ResolveProjectResult = Succeeded<ProjectEntry> | Absent
+
+export type ChatModelReference = Static<typeof ChatModelReferenceSchema>
+
+export interface ConfigObjectRule {
+  kind: "object"
+  path: string
+  keys: string[]
+  required: string[]
+}
+
+export interface ConfigValueRule {
+  kind: "value"
+  path: string
+  schema: TSchema
+}
+
+export type ConfigFieldRule = ConfigObjectRule | ConfigValueRule
+
+export interface ConfigRuleSelection {
+  projectCount: number
+  embeddings: string | null
+  decisionModel: string | null
+}
+
+export interface ConfigRegistrations {
+  global: Partial<MemoryConfig["global"]>
+  projects: Partial<ProjectEntry>[]
+}
+
+export interface CaptureFields extends Partial<CaptureCadenceSettings> {
+  model: Partial<ChatModelReference>
+}
+
+export interface MemoryAgentFields {
+  model: Partial<ChatModelReference>
+}
+
+export interface EmbeddingFields {
+  provider?: MemoryConfig["embeddings"]["provider"]
+  model?: MemoryConfig["embeddings"]["model"]
+}
+
+export interface DecisionFields {
+  provider?: MemoryConfig["decisionModel"]["provider"]
+  model?: MemoryConfig["decisionModel"]["model"]
+  endpoint?: string
+  maxTokens?: number
+  headMaxTokens?: number
+  gateThreshold?: number
+  relevanceThreshold?: number
+}
+
+export interface ConfigFields extends ConfigRegistrations {
+  limits: Partial<MemoryConfig["limits"]>
+  capture: CaptureFields
+  memoryAgent: MemoryAgentFields
+  context: Partial<MemoryConfig["context"]>
+  embeddings: EmbeddingFields
+  decisionModel: DecisionFields
+}
+
+export interface ConfigRepairDraft extends ConfigFields {
+  limits: MemoryConfig["limits"]
+  replacementRequired: boolean
+  problems: string[]
+}
+
+export interface SetupProposal {
+  config: ConfigFields
+  vault: string
+}
+
+export interface InitProposal {
+  config: MemoryConfig
+  folder: string
+  name: string
+  vault: string
+}
+
+export interface ConfigLocationPaths {
+  config: ConfigRegistrations
+  configPath: string
+}
+
+export interface ConfigInfrastructure extends ConfigLocationPaths {
+  indexPath: string
+}
+
+export interface VaultRegistration {
+  field: string
+  path: string
+}
+
+export type SetupDisclosure = "chat-models" | "jev" | "remote-embeddings"
+
+export interface CaptureModelChange {
+  kind: "captureModel"
+  model: ChatModelReference
+}
+
+export interface AgentModelChange {
+  kind: "agentModel"
+  model: ChatModelReference
+}
+
+export interface CaptureCountChange {
+  kind: "captureCount"
+  name: "nudgeInterval" | "nudgeToolCalls" | "flushMinTurns"
+  value: number
+}
+
+export interface CaptureSwitchChange {
+  kind: "captureSwitch"
+  name: "reviewEnabled" | "flushOnCompact" | "flushOnShutdown"
+  value: boolean
+}
+
+export interface EmbeddingsChange {
+  kind: "embeddings"
+  embeddings: MemoryConfig["embeddings"]
+}
+
+export interface JevDecisionChange {
+  kind: "decision"
+  provider: "typesafe"
+}
+
+export interface LayaDecisionChange {
+  kind: "decision"
+  provider: "laya"
+  endpoint: string
+}
+
+export interface ThresholdChange {
+  kind: "gateThreshold" | "relevanceThreshold"
+  value: number | null
+}
+
+export interface BudgetChange {
+  kind: "budgetChars"
+  value: number
+}
+
+export interface LimitChange {
+  kind: "limit"
+  name: keyof MemoryConfig["limits"]
+  value: number
+}
+
+export interface ProjectVaultChange {
+  kind: "projectVault"
+  name: string
+  vault: string
+}
+
+export interface RemoveProjectChange {
+  kind: "removeProject"
+  name: string
+}
+
+export type DecisionChange = JevDecisionChange | LayaDecisionChange
+
+export type SettingChange =
+  | CaptureModelChange
+  | AgentModelChange
+  | CaptureCountChange
+  | CaptureSwitchChange
+  | EmbeddingsChange
+  | DecisionChange
+  | ThresholdChange
+  | BudgetChange
+  | LimitChange
+  | ProjectVaultChange
+  | RemoveProjectChange
+
+export type SettingNumberKind = "threshold" | "count"
+
+export interface ConfigCandidate extends Omit<MemoryConfig, "decisionModel"> {
+  decisionModel: DecisionFields
+}
+
+export type NextConfig = Succeeded<ConfigCandidate> | Failed
+
+export type SettingNumberResult = Succeeded<number> | Failed
+
+export type ProjectReplacement = (project: ProjectEntry) => ProjectEntry | null

package/src/domain/decision-model/constants.ts

@@ -0,0 +1,7 @@
+import type { DecisionCutoffs, DecisionProvider } from "./types.ts"
+
+export const DEFAULT_CUTOFFS = new Map<DecisionProvider, Partial<DecisionCutoffs>>([
+  ["typesafe", { gate: 0.5, relevance: 0.5 }],
+])
+
+export const MAX_FAILURE_DETAIL_CHARS = 500

package/src/domain/decision-model/cutoffs.ts

@@ -0,0 +1,8 @@
+import { DEFAULT_CUTOFFS } from "./constants.ts"
+import type { DecisionCutoffs, DecisionProvider } from "./types.ts"
+
+export function defaultCutoffs(provider: DecisionProvider): Partial<DecisionCutoffs> {
+  const cutoffs = DEFAULT_CUTOFFS.get(provider)
+  if (cutoffs) return cutoffs
+  return {}
+}

package/src/domain/decision-model/endpoint.ts

@@ -0,0 +1,11 @@
+export function isLoopbackEndpoint(endpoint: string): boolean {
+  try {
+    const url = new URL(endpoint)
+    if (url.protocol !== "http:" && url.protocol !== "https:") return false
+    if (url.username || url.password || url.search || url.hash) return false
+    if (url.pathname !== "/v1/systemone") return false
+    return url.hostname === "127.0.0.1" || url.hostname === "[::1]"
+  } catch {
+    return false
+  }
+}

package/src/domain/decision-model/failure-detail.ts

@@ -0,0 +1,8 @@
+import { MAX_FAILURE_DETAIL_CHARS } from "./constants.ts"
+
+export function boundedFailureDetail(detail: string): string {
+  if (detail.length <= MAX_FAILURE_DETAIL_CHARS) return detail
+  const last = detail.charCodeAt(MAX_FAILURE_DETAIL_CHARS - 1)
+  if (last >= 0xd800 && last <= 0xdbff) return detail.slice(0, MAX_FAILURE_DETAIL_CHARS - 1)
+  return detail.slice(0, MAX_FAILURE_DETAIL_CHARS)
+}

package/src/domain/decision-model/index.ts

@@ -0,0 +1,6 @@
+export * from "./constants.ts"
+export * from "./cutoffs.ts"
+export * from "./endpoint.ts"
+export * from "./failure-detail.ts"
+export * from "./rank.ts"
+export * from "./types.ts"

package/src/domain/decision-model/rank.ts

@@ -0,0 +1,36 @@
+import type { CandidateScore } from "./types.ts"
+
+function validateProbability(probability: number): void {
+  if (!Number.isFinite(probability) || probability < 0 || probability > 1) {
+    throw new RangeError("Decision probability must be finite and between 0 and 1")
+  }
+}
+
+function validateCutoff(cutoff: number): void {
+  if (!Number.isFinite(cutoff) || cutoff < 0 || cutoff > 1) {
+    throw new RangeError("Decision cutoff must be finite and between 0 and 1")
+  }
+}
+
+export function shouldRetrieveMemory(probability: number, cutoff: number): boolean {
+  validateProbability(probability)
+  validateCutoff(cutoff)
+  return probability > cutoff
+}
+
+function compareScores(left: CandidateScore, right: CandidateScore): number {
+  const probabilityOrder = right.probability - left.probability
+  if (probabilityOrder !== 0) return probabilityOrder
+
+  for (const field of ["vault", "id", "revision"] as const) {
+    if (left.identity[field] < right.identity[field]) return -1
+    if (left.identity[field] > right.identity[field]) return 1
+  }
+  return 0
+}
+
+export function rankCandidates(scores: readonly CandidateScore[], cutoff: number): CandidateScore[] {
+  validateCutoff(cutoff)
+  for (const score of scores) validateProbability(score.probability)
+  return scores.filter((score) => score.probability >= cutoff).sort(compareScores)
+}

package/src/domain/decision-model/types.ts

@@ -0,0 +1,77 @@
+export interface CandidateIdentity {
+  vault: "global" | "project"
+  id: string
+  revision: string
+}
+
+export interface DecisionRequest {
+  message: string
+  project: string | null
+  folder: string
+}
+
+export interface CandidateContent {
+  statement: string
+  body: string
+}
+
+export interface CandidateDecisionRequest {
+  message: string
+  identity: CandidateIdentity
+  candidate: CandidateContent
+}
+
+export interface CandidateScore {
+  identity: CandidateIdentity
+  probability: number
+}
+
+export interface DecisionUsage {
+  inputTokens: number
+  outputTokens: number
+}
+
+export interface DecisionDiagnostics {
+  provider: "laya" | "typesafe"
+  requestedModel: string
+  reportedModel: string | null
+  usage: DecisionUsage | null
+}
+
+export interface RelevanceDecision {
+  probability: number
+  diagnostics: DecisionDiagnostics
+}
+
+export type DecisionFailureCode =
+  | "unsafe-endpoint"
+  | "invalid-settings"
+  | "unavailable"
+  | "http-error"
+  | "invalid-response"
+  | "routing-missing"
+  | "truncated"
+  | "collapsed-options"
+  | "aborted"
+
+export interface DecisionFailure {
+  succeeded: false
+  code: DecisionFailureCode
+  diagnostics: DecisionDiagnostics
+  httpStatus?: number
+  detail?: string
+}
+
+export interface DecisionSuccess {
+  succeeded: true
+  output: RelevanceDecision
+}
+
+export type DecisionResult = DecisionSuccess | DecisionFailure
+
+export type DecisionProvider = DecisionDiagnostics["provider"]
+
+export interface DecisionCutoffs {
+  gate: number
+  relevance: number
+}

package/src/domain/embeddings/constants.ts

@@ -0,0 +1,55 @@
+import type { EmbeddingConfiguration, EmbeddingPolicy, MinimumSimilarities } from "./types.ts"
+
+export const localEmbeddingConfiguration = {
+  provider: "local",
+  model: "Xenova/multilingual-e5-small",
+} as const satisfies EmbeddingConfiguration
+
+export const openAIEmbeddingConfiguration = {
+  provider: "openai",
+  model: "text-embedding-3-small",
+} as const satisfies EmbeddingConfiguration
+
+export const googleEmbeddingConfiguration = {
+  provider: "google",
+  model: "gemini-embedding-001",
+} as const satisfies EmbeddingConfiguration
+
+export const localEmbeddingPolicy = {
+  revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78",
+  dimensions: 384,
+  dtype: "cpu-q8-transformers-4.3.1",
+  pooling: "mean",
+  normalization: "l2",
+  queryPrefix: "query: ",
+  documentPrefix: "passage: ",
+  chunking: "v1-token-aware-512-special-prefix-inclusive-overlap32-mean-chunks-query-refuse",
+} as const satisfies EmbeddingPolicy
+
+export const openAIEmbeddingPolicy = {
+  revision: "provider-managed",
+  dimensions: 1536,
+  dtype: "float",
+  pooling: "provider",
+  normalization: "l2",
+  queryPrefix: "",
+  documentPrefix: "",
+  chunking: "v1-utf8-bytes4096-overlap128-mean-chunks-query-refuse",
+} as const satisfies EmbeddingPolicy
+
+export const googleEmbeddingPolicy = {
+  revision: "provider-managed",
+  dimensions: 768,
+  dtype: "float",
+  pooling: "provider",
+  normalization: "l2",
+  queryPrefix: "RETRIEVAL_QUERY",
+  documentPrefix: "RETRIEVAL_DOCUMENT",
+  chunking: "v1-utf8-bytes1024-overlap128-mean-chunks-query-refuse-autoTruncate-false",
+} as const satisfies EmbeddingPolicy
+
+export const MINIMUM_SIMILARITY = {
+  local: 0.8,
+  openai: null,
+  google: null,
+} as const satisfies MinimumSimilarities

package/src/domain/embeddings/identity.ts

@@ -0,0 +1,30 @@
+import { googleEmbeddingPolicy, localEmbeddingPolicy, openAIEmbeddingPolicy } from "./constants.ts"
+import type { EmbeddingConfiguration, EmbeddingPolicy } from "./types.ts"
+
+export function embeddingPolicy(configuration: EmbeddingConfiguration): EmbeddingPolicy {
+  switch (configuration.provider) {
+    case "local":
+      return localEmbeddingPolicy
+    case "openai":
+      return openAIEmbeddingPolicy
+    case "google":
+      return googleEmbeddingPolicy
+  }
+}
+
+export function embeddingFingerprint(configuration: EmbeddingConfiguration): string {
+  const policy = embeddingPolicy(configuration)
+  return JSON.stringify([
+    "embeddings-v1",
+    configuration.provider,
+    configuration.model,
+    policy.revision,
+    policy.dimensions,
+    policy.dtype,
+    policy.pooling,
+    policy.normalization,
+    policy.queryPrefix,
+    policy.documentPrefix,
+    policy.chunking,
+  ])
+}

package/src/domain/embeddings/index.ts

@@ -0,0 +1,5 @@
+export * from "./constants.ts"
+export * from "./identity.ts"
+export * from "./schemas.ts"
+export * from "./types.ts"
+export * from "./vectors.ts"

package/src/domain/embeddings/schemas.ts

@@ -0,0 +1,25 @@
+import Type from "typebox"
+
+export const EmbeddingConfigurationSchema = Type.Union([
+  Type.Object(
+    {
+      provider: Type.Literal("local"),
+      model: Type.Literal("Xenova/multilingual-e5-small"),
+    },
+    { additionalProperties: false },
+  ),
+  Type.Object(
+    {
+      provider: Type.Literal("openai"),
+      model: Type.Literal("text-embedding-3-small"),
+    },
+    { additionalProperties: false },
+  ),
+  Type.Object(
+    {
+      provider: Type.Literal("google"),
+      model: Type.Literal("gemini-embedding-001"),
+    },
+    { additionalProperties: false },
+  ),
+])

package/src/domain/embeddings/types.ts

@@ -0,0 +1,52 @@
+import type { Static } from "typebox"
+
+import type { Succeeded } from "#src/shared"
+import type { EmbeddingConfigurationSchema } from "./schemas.ts"
+
+export type EmbeddingConfiguration = Static<typeof EmbeddingConfigurationSchema>
+
+export interface EmbeddingInput {
+  key: string
+  text: string
+}
+
+export interface EmbeddingBatchRequest {
+  purpose: "query" | "document"
+  inputs: readonly EmbeddingInput[]
+}
+
+export interface EmbeddedInput {
+  key: string
+  vector: readonly number[]
+}
+
+export interface EmbeddingBatch {
+  fingerprint: string
+  dimensions: number
+  items: readonly EmbeddedInput[]
+}
+
+export type EmbeddingFailureStatus = "unavailable" | "input-too-long" | "invalid-input" | "invalid-response" | "aborted"
+
+export interface EmbeddingFailure {
+  succeeded: false
+  status: EmbeddingFailureStatus
+  cleanupFailed?: true
+}
+
+export type EmbeddingResult = Succeeded<EmbeddingBatch> | EmbeddingFailure
+
+export interface EmbeddingPolicy {
+  revision: string
+  dimensions: number
+  dtype: string
+  pooling: string
+  normalization: "l2"
+  queryPrefix: string
+  documentPrefix: string
+  chunking: string
+}
+
+export type EmbeddingProvider = EmbeddingConfiguration["provider"]
+
+export type MinimumSimilarities = Record<EmbeddingProvider, number | null>

package/src/domain/embeddings/vectors.ts

@@ -0,0 +1,53 @@
+export function normalizeVector(values: readonly number[], dimensions: number): number[] | null {
+  if (values.length !== dimensions || dimensions < 1 || values.some((value) => !Number.isFinite(value))) {
+    return null
+  }
+
+  const scale = values.reduce((largest, value) => Math.max(largest, Math.abs(value)), 0)
+  if (scale === 0) {
+    return null
+  }
+
+  const scaled = values.map((value) => value / scale)
+  const norm = Math.sqrt(scaled.reduce((sum, value) => sum + value * value, 0))
+  return scaled.map((value) => value / norm)
+}
+
+export function aggregateVectors(vectors: readonly (readonly number[])[], dimensions: number): number[] | null {
+  if (vectors.length === 0) {
+    return null
+  }
+
+  const sum = Array<number>(dimensions).fill(0)
+  for (const values of vectors) {
+    const vector = normalizeVector(values, dimensions)
+    if (!vector) {
+      return null
+    }
+    for (const [index, value] of vector.entries()) {
+      const previous = sum[index]
+      if (typeof previous !== "number") {
+        return null
+      }
+      sum[index] = previous + value / vectors.length
+    }
+  }
+  return normalizeVector(sum, dimensions)
+}
+
+export function cosine(left: readonly number[], right: readonly number[]): number {
+  if (left.length !== right.length) throw new RangeError("Vectors must have the same dimensions")
+
+  let dot = 0
+  let leftNorm = 0
+  let rightNorm = 0
+  for (const [index, value] of left.entries()) {
+    const other = right[index]
+    if (typeof other !== "number") throw new RangeError("Vectors must have the same dimensions")
+    dot += value * other
+    leftNorm += value * value
+    rightNorm += other * other
+  }
+  if (leftNorm === 0 || rightNorm === 0) throw new RangeError("Vectors must be nonzero")
+  return dot / Math.sqrt(leftNorm * rightNorm)
+}

package/src/domain/folder-index/constants.ts

@@ -0,0 +1,9 @@
+export const NOTE_LINE_SEPARATOR = " · "
+
+export const PROBLEM_SEPARATOR = "; "
+
+export const DONT_FORGET_LABEL = "don't forget"
+
+export const INDEX_TEXT_COLLATOR = new Intl.Collator("en", { sensitivity: "base" })
+
+export const FOLDER_INDEX_FORMAT = 1

package/src/domain/folder-index/display.ts

@@ -0,0 +1,22 @@
+export function displayIndexText(text: string): string {
+  if (/[<>[\]`]/.test(text)) {
+    return displayIndexName(text)
+  }
+
+  return text
+}
+
+export function displayIndexName(name: string): string {
+  if (name.length === 0) {
+    return "<code></code>"
+  }
+
+  const runs = [...name.matchAll(/`+/g)].map((match) => match[0].length)
+  const fence = "`".repeat(Math.max(0, ...runs) + 1)
+  const mixedEdgeSpaces = name.startsWith(" ") && name.endsWith(" ") && /[^ ]/.test(name)
+  if (name.startsWith("`") || name.endsWith("`") || mixedEdgeSpaces) {
+    return `${fence} ${name} ${fence}`
+  }
+
+  return `${fence}${name}${fence}`
+}

package/src/domain/folder-index/index.ts

@@ -0,0 +1,5 @@
+export * from "./constants.ts"
+export * from "./display.ts"
+export * from "./listing.ts"
+export * from "./render.ts"
+export * from "./types.ts"