pi-obsidian-memory 0.0.0 → 0.1.0

package/src/domain/note/frontmatter-fields.ts

@@ -0,0 +1,175 @@
+import type { Static, TSchema } from "typebox"
+import { Value } from "typebox/value"
+import { type Document, isAlias, isMap, isNode, isPair, isScalar, isSeq, type ParsedNode } from "yaml"
+
+import { describeNoteSchemaErrors } from "./schema-errors.ts"
+import { DontForgetSchema } from "./schemas.ts"
+import type { FieldNameResult, FieldReader, FieldResult, FrontmatterPairs, OptionalFieldResult } from "./types.ts"
+
+export function fieldNameOf(key: ParsedNode): FieldNameResult {
+  if (isScalar(key) && typeof key.value === "string") {
+    return { succeeded: true, output: key.value }
+  }
+
+  return { succeeded: false }
+}
+
+export function occurrencesOf(pairs: FrontmatterPairs, name: string): FrontmatterPairs {
+  return pairs.filter((pair) => isNamed(pair.key, name))
+}
+
+export function readRequiredField<Schema extends TSchema>(
+  document: Document.Parsed,
+  pairs: FrontmatterPairs,
+  name: string,
+  schema: Schema,
+): FieldResult<Static<Schema>> {
+  const field = readOptionalField(document, pairs, name, schema)
+  if (field.succeeded || "problems" in field) {
+    return field
+  }
+
+  return { succeeded: false, problems: [`${name}: is missing`] }
+}
+
+export function readOptionalField<Schema extends TSchema>(
+  document: Document.Parsed,
+  pairs: FrontmatterPairs,
+  name: string,
+  schema: Schema,
+): OptionalFieldResult<Static<Schema>> {
+  return readFieldOccurrences(pairs, name, (node) => readText(document, name, node, schema))
+}
+
+export function readDontForget(document: Document.Parsed, pairs: FrontmatterPairs): OptionalFieldResult<true> {
+  return readFieldOccurrences(pairs, "dontForget", (node) => readFlag(document, "dontForget", node))
+}
+
+function readFieldOccurrences<Value>(
+  pairs: FrontmatterPairs,
+  name: string,
+  read: FieldReader<Value>,
+): OptionalFieldResult<Value> {
+  const [first, ...others] = occurrencesOf(pairs, name)
+  if (!isPair(first)) {
+    return { succeeded: false }
+  }
+
+  if (others.length > 0) {
+    return repeatedField(
+      name,
+      [first, ...others].map((pair) => read(pair.value)),
+    )
+  }
+
+  return read(first.value)
+}
+
+export function shapeProblems(document: Document.Parsed, label: string, node: ParsedNode | null): string[] {
+  if (!isNode(node)) {
+    return [`${label}: has no value`]
+  }
+
+  if (isAlias(node)) {
+    return [`${label}: is an alias (*${node.source}); write the value itself`]
+  }
+
+  const tagged = tagProblems(document, label, node.tag)
+  if (tagged.length > 0) {
+    return tagged
+  }
+
+  if (isSeq(node)) {
+    return [`${label}: must be a single value, not a list`]
+  }
+
+  if (isMap(node)) {
+    return [`${label}: must be a single value, not a set of fields`]
+  }
+
+  return []
+}
+
+export function unknownValueProblems(document: Document.Parsed, label: string, node: ParsedNode | null): string[] {
+  const shape = shapeProblems(document, label, node)
+  if (shape.length > 0) {
+    return shape
+  }
+
+  if (!isScalar(node)) {
+    return [`${label}: has no value`]
+  }
+
+  const { value } = node
+  if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+    return []
+  }
+
+  return [`${label}: has no value`]
+}
+
+export function tagProblems(document: Document.Parsed, label: string, tag: string | undefined): string[] {
+  if (typeof tag !== "string") {
+    return []
+  }
+
+  return [`${label}: remove the tag ${document.directives.tagString(tag)}`]
+}
+
+export function repeatedProblem(name: string): string {
+  return `${name}: appears more than once`
+}
+
+function readText<Schema extends TSchema>(
+  document: Document.Parsed,
+  name: string,
+  node: ParsedNode | null,
+  schema: Schema,
+): FieldResult<Static<Schema>> {
+  const shape = shapeProblems(document, name, node)
+  if (shape.length > 0) {
+    return { succeeded: false, problems: shape }
+  }
+
+  if (!isScalar(node) || typeof node.value !== "string") {
+    return { succeeded: false, problems: [`${name}: must be text`] }
+  }
+
+  const text = node.value
+  if (Value.Check(schema, text)) {
+    return { succeeded: true, output: text }
+  }
+
+  return { succeeded: false, problems: describeNoteSchemaErrors(Value.Errors(schema, text), name) }
+}
+
+function readFlag(document: Document.Parsed, name: string, node: ParsedNode | null): FieldResult<true> {
+  const shape = shapeProblems(document, name, node)
+  if (shape.length > 0) {
+    return { succeeded: false, problems: shape }
+  }
+
+  if (isScalar(node) && typeof node.value === "boolean" && Value.Check(DontForgetSchema, node.value)) {
+    return { succeeded: true, output: node.value }
+  }
+
+  return { succeeded: false, problems: [`${name}: must be true (or left out)`] }
+}
+
+function repeatedField<Value>(name: string, readings: FieldResult<Value>[]): FieldResult<Value> {
+  const problems = readings.flatMap((reading) => problemsOf(reading))
+  return { succeeded: false, problems: [repeatedProblem(name), ...new Set(problems)] }
+}
+
+export function problemsOf<Value>(reading: OptionalFieldResult<Value>): string[] {
+  if ("problems" in reading) {
+    return reading.problems
+  }
+
+  return []
+}
+
+function isNamed(key: ParsedNode, name: string): boolean {
+  const fieldName = fieldNameOf(key)
+  return fieldName.succeeded && fieldName.output === name
+}

package/src/domain/note/frontmatter.ts

@@ -0,0 +1,138 @@
+import { type Document, isMap, type ParsedNode, parseDocument } from "yaml"
+
+import {
+  KNOWN_FRONTMATTER_FIELDS,
+  NAME_NOT_TEXT_PROBLEM,
+  NOT_FIELDS_PROBLEM,
+  REPEATED_FIELD_ERROR_CODE,
+} from "./constants.ts"
+import {
+  fieldNameOf,
+  occurrencesOf,
+  problemsOf,
+  readDontForget,
+  readOptionalField,
+  readRequiredField,
+  repeatedProblem,
+  tagProblems,
+  unknownValueProblems,
+} from "./frontmatter-fields.ts"
+import { NoteDateSchema, NoteIdSchema, NoteLinkSchema, NoteTypeSchema } from "./schemas.ts"
+import type { FrontmatterPairs, FrontmatterReadings, NoteFrontmatter, ReadFrontmatterResult } from "./types.ts"
+
+export function readFrontmatter(yaml: string): ReadFrontmatterResult {
+  const document = parseDocument(yaml)
+  const blockingErrors = document.errors.filter((error) => error.code !== REPEATED_FIELD_ERROR_CODE)
+  const yamlProblems = [...blockingErrors, ...document.warnings].map((issue) => `frontmatter: ${issue.message}`)
+  if (blockingErrors.length > 0) {
+    return { succeeded: false, problems: yamlProblems }
+  }
+
+  const root = document.contents
+  if (!isMap(root)) {
+    return { succeeded: false, problems: [...yamlProblems, NOT_FIELDS_PROBLEM] }
+  }
+
+  const pairs = root.items
+  const readings = readFields(document, pairs)
+  const problems = [
+    ...yamlProblems,
+    ...tagProblems(document, "frontmatter", root.tag),
+    ...nameProblems(document, pairs),
+    ...Object.values(readings).flatMap((field) => problemsOf<string | true>(field)),
+    ...unknownFieldProblems(document, pairs),
+  ]
+
+  return buildFrontmatter(readings, problems)
+}
+
+function readFields(document: Document.Parsed, pairs: FrontmatterPairs): FrontmatterReadings {
+  return {
+    id: readRequiredField(document, pairs, "id", NoteIdSchema),
+    type: readRequiredField(document, pairs, "type", NoteTypeSchema),
+    created: readRequiredField(document, pairs, "created", NoteDateSchema),
+    updated: readRequiredField(document, pairs, "updated", NoteDateSchema),
+    dontForget: readDontForget(document, pairs),
+    archived: readOptionalField(document, pairs, "archived", NoteDateSchema),
+    replaces: readOptionalField(document, pairs, "replaces", NoteLinkSchema),
+    replacedBy: readOptionalField(document, pairs, "replacedBy", NoteLinkSchema),
+  }
+}
+
+function buildFrontmatter(readings: FrontmatterReadings, problems: string[]): ReadFrontmatterResult {
+  const { id, type, created, updated, dontForget, archived, replaces, replacedBy } = readings
+  if (problems.length > 0 || !id.succeeded || !type.succeeded || !created.succeeded || !updated.succeeded) {
+    return { succeeded: false, problems }
+  }
+
+  const frontmatter: NoteFrontmatter = {
+    id: id.output,
+    type: type.output,
+    created: created.output,
+    updated: updated.output,
+  }
+
+  if (dontForget.succeeded) {
+    frontmatter.dontForget = dontForget.output
+  }
+
+  if (archived.succeeded) {
+    frontmatter.archived = archived.output
+  }
+
+  if (replaces.succeeded) {
+    frontmatter.replaces = replaces.output
+  }
+
+  if (replacedBy.succeeded) {
+    frontmatter.replacedBy = replacedBy.output
+  }
+
+  return { succeeded: true, output: frontmatter }
+}
+
+function nameProblems(document: Document.Parsed, pairs: FrontmatterPairs): string[] {
+  const hasNameThatIsNotText = pairs.some((pair) => !fieldNameOf(pair.key).succeeded)
+  const tagged = pairs.flatMap((pair) => nameTagProblems(document, pair.key))
+
+  if (hasNameThatIsNotText) {
+    return [NAME_NOT_TEXT_PROBLEM, ...tagged]
+  }
+
+  return tagged
+}
+
+function nameTagProblems(document: Document.Parsed, key: ParsedNode): string[] {
+  const name = fieldNameOf(key)
+  if (!name.succeeded) {
+    return []
+  }
+
+  return tagProblems(document, name.output, key.tag).map((problem) => `${problem} from the name`)
+}
+
+function unknownFieldProblems(document: Document.Parsed, pairs: FrontmatterPairs): string[] {
+  const names = pairs.flatMap((pair) => textNamesOf(pair.key))
+  const unknown = [...new Set(names)].filter((name) => !KNOWN_FRONTMATTER_FIELDS.includes(name))
+
+  return unknown.flatMap((name) => unknownProblems(document, occurrencesOf(pairs, name), name))
+}
+
+function unknownProblems(document: Document.Parsed, occurrences: FrontmatterPairs, name: string): string[] {
+  const shape = occurrences.flatMap((pair) => unknownValueProblems(document, name, pair.value))
+
+  if (occurrences.length > 1) {
+    return [repeatedProblem(name), ...new Set(shape), `${name}: not a known field`]
+  }
+
+  return [...shape, `${name}: not a known field`]
+}
+
+function textNamesOf(key: ParsedNode): string[] {
+  const name = fieldNameOf(key)
+  if (name.succeeded) {
+    return [name.output]
+  }
+
+  return []
+}

package/src/domain/note/index.ts

@@ -0,0 +1,9 @@
+export * from "./check.ts"
+export * from "./constants.ts"
+export * from "./frontmatter.ts"
+export * from "./parse.ts"
+export * from "./render.ts"
+export * from "./schema-errors.ts"
+export * from "./schemas.ts"
+export * from "./types.ts"
+export * from "./validate.ts"

package/src/domain/note/parse.ts

@@ -0,0 +1,41 @@
+import { FRONTMATTER_FENCE, LINE_BREAK } from "./constants.ts"
+import type { ParseNoteResult } from "./types.ts"
+
+export function parseNote(statement: string, text: string): ParseNoteResult {
+  const lines = text.replaceAll("\r\n", LINE_BREAK).split(LINE_BREAK)
+
+  if (lines[0] !== FRONTMATTER_FENCE) {
+    return { succeeded: false, problems: ["missing frontmatter"] }
+  }
+
+  const closingFence = lines.indexOf(FRONTMATTER_FENCE, 1)
+  if (closingFence < 0) {
+    return { succeeded: false, problems: ["frontmatter is not closed"] }
+  }
+
+  const frontmatter = lines.slice(1, closingFence).join(LINE_BREAK)
+  const afterFrontmatter = lines.slice(closingFence + 1).join(LINE_BREAK)
+
+  return { succeeded: true, output: { statement, frontmatter, body: withoutFraming(afterFrontmatter) } }
+}
+
+function withoutFraming(text: string): string {
+  const withoutBlankLine = withoutPrefix(text, LINE_BREAK)
+  return withoutSuffix(withoutBlankLine, LINE_BREAK)
+}
+
+function withoutPrefix(text: string, prefix: string): string {
+  if (text.startsWith(prefix)) {
+    return text.slice(prefix.length)
+  }
+
+  return text
+}
+
+function withoutSuffix(text: string, suffix: string): string {
+  if (text.endsWith(suffix)) {
+    return text.slice(0, -suffix.length)
+  }
+
+  return text
+}

package/src/domain/note/render.ts

@@ -0,0 +1,27 @@
+import { stringify } from "yaml"
+
+import { FRONTMATTER_FENCE, KNOWN_FRONTMATTER_FIELDS } from "./constants.ts"
+import { NoteFrontmatterSchema } from "./schemas.ts"
+import type { Note, NoteFrontmatter } from "./types.ts"
+
+export function renderNote(note: Note): string {
+  const frontmatter = stringify(orderedFrontmatter(note.frontmatter)).trimEnd()
+
+  return [FRONTMATTER_FENCE, frontmatter, FRONTMATTER_FENCE, "", note.body, ""].join("\n")
+}
+
+function orderedFrontmatter(frontmatter: NoteFrontmatter): Map<keyof NoteFrontmatter, string | true> {
+  const ordered = new Map<keyof NoteFrontmatter, string | true>()
+  for (const name of KNOWN_FRONTMATTER_FIELDS.filter(isFrontmatterField)) {
+    const value = frontmatter[name]
+    if (typeof value === "string" || typeof value === "boolean") {
+      ordered.set(name, value)
+    }
+  }
+
+  return ordered
+}
+
+function isFrontmatterField(name: string): name is keyof NoteFrontmatter {
+  return Object.hasOwn(NoteFrontmatterSchema.properties, name)
+}

package/src/domain/note/schema-errors.ts

@@ -0,0 +1,41 @@
+import type { TValidationError } from "typebox/error"
+
+import { describeSchemaErrorsWith } from "#src/domain/validation"
+import { NoteTypeSchema } from "./schemas.ts"
+
+export function describeNoteSchemaErrors(errors: readonly TValidationError[], label: string): string[] {
+  return describeSchemaErrorsWith(errors, label, (error) => noteMessage(error, label))
+}
+
+function noteMessage(error: TValidationError, label: string): string | null {
+  const field = fieldNameOf(error, label)
+  if (error.keyword === "pattern" && field === "id") {
+    return "must be a ULID"
+  }
+
+  if (error.keyword === "pattern" && (field === "replaces" || field === "replacedBy")) {
+    return "must be a wikilink without an alias, heading or block reference ([[folder/Statement]])"
+  }
+
+  if (error.keyword === "format" && error.params.format === "date") {
+    return "must be a date (YYYY-MM-DD)"
+  }
+
+  if (error.keyword === "enum") {
+    return `must be one of: ${NoteTypeSchema.enum.join(", ")}`
+  }
+
+  if (error.keyword === "const" && typeof error.params.allowedValue === "boolean" && error.params.allowedValue) {
+    return "must be true (or left out)"
+  }
+
+  return null
+}
+
+function fieldNameOf(error: TValidationError, label: string): string {
+  if (error.instancePath.length === 0) {
+    return label
+  }
+
+  return error.instancePath.slice(error.instancePath.lastIndexOf("/") + 1)
+}

package/src/domain/note/schemas.ts

@@ -0,0 +1,38 @@
+import Type from "typebox"
+
+export const NoteTypeSchema = Type.Enum([
+  "rule",
+  "procedure",
+  "constraint",
+  "fact",
+  "issue",
+  "question",
+  "decision",
+  "lesson",
+  "goal",
+  "risk",
+  "idea",
+  "reference",
+])
+
+export const NoteIdSchema = Type.String({ pattern: "^[0-7][0-9A-HJKMNP-TV-Z]{25}$" })
+
+export const NoteDateSchema = Type.String({ format: "date" })
+
+export const DontForgetSchema = Type.Literal(true)
+
+export const NoteLinkSchema = Type.String({ pattern: "^\\[\\[[^\\[\\]|#^]+\\]\\]$" })
+
+export const NoteFrontmatterSchema = Type.Object(
+  {
+    id: NoteIdSchema,
+    type: NoteTypeSchema,
+    created: NoteDateSchema,
+    updated: NoteDateSchema,
+    dontForget: Type.Optional(DontForgetSchema),
+    archived: Type.Optional(NoteDateSchema),
+    replaces: Type.Optional(NoteLinkSchema),
+    replacedBy: Type.Optional(NoteLinkSchema),
+  },
+  { additionalProperties: false },
+)

package/src/domain/note/types.ts

@@ -0,0 +1,53 @@
+import type { Static } from "typebox"
+import type { Pair, ParsedNode } from "yaml"
+
+import type { Absent, Failed, Succeeded } from "#src/shared"
+import type { NoteFrontmatterSchema, NoteTypeSchema } from "./schemas.ts"
+
+export type NoteType = Static<typeof NoteTypeSchema>
+
+export type NoteFrontmatter = Static<typeof NoteFrontmatterSchema>
+
+export interface Note {
+  statement: string
+  frontmatter: NoteFrontmatter
+  body: string
+}
+
+export interface NoteLimits {
+  maxBodyChars: number
+  maxStatementChars: number
+}
+
+export interface ParsedNote {
+  statement: string
+  frontmatter: string
+  body: string
+}
+
+export type ParseNoteResult = Succeeded<ParsedNote> | Failed
+
+export type ValidateNoteResult = Succeeded<Note> | Failed
+
+export type ReadFrontmatterResult = Succeeded<NoteFrontmatter> | Failed
+
+export type FieldResult<Value> = Succeeded<Value> | Failed
+
+export type OptionalFieldResult<Value> = Succeeded<Value> | Failed | Absent
+
+export type FieldNameResult = Succeeded<string> | Absent
+
+export type FrontmatterPairs = Pair<ParsedNode, ParsedNode | null>[]
+
+export type FieldReader<Value> = (node: ParsedNode | null) => FieldResult<Value>
+
+export interface FrontmatterReadings {
+  id: FieldResult<string>
+  type: FieldResult<NoteType>
+  created: FieldResult<string>
+  updated: FieldResult<string>
+  dontForget: OptionalFieldResult<true>
+  archived: OptionalFieldResult<string>
+  replaces: OptionalFieldResult<string>
+  replacedBy: OptionalFieldResult<string>
+}

package/src/domain/note/validate.ts

@@ -0,0 +1,19 @@
+import { checkNoteText } from "./check.ts"
+import { readFrontmatter } from "./frontmatter.ts"
+import type { NoteLimits, ParsedNote, ValidateNoteResult } from "./types.ts"
+
+export function validateNote(parsed: ParsedNote, limits: NoteLimits): ValidateNoteResult {
+  const { statement, body } = parsed
+  const frontmatter = readFrontmatter(parsed.frontmatter)
+  const textProblems = checkNoteText(statement, body, limits)
+
+  if (!frontmatter.succeeded) {
+    return { succeeded: false, problems: [...frontmatter.problems, ...textProblems] }
+  }
+
+  if (textProblems.length > 0) {
+    return { succeeded: false, problems: textProblems }
+  }
+
+  return { succeeded: true, output: { statement, frontmatter: frontmatter.output, body } }
+}

package/src/domain/path-platform/index.ts

@@ -0,0 +1,2 @@
+export * from "./path-module.ts"
+export * from "./types.ts"

package/src/domain/path-platform/path-module.ts

@@ -0,0 +1,11 @@
+import { posix, win32 } from "node:path"
+
+import type { PathPlatform } from "./types.ts"
+
+export function pathModule(platform: PathPlatform): typeof posix {
+  if (platform === "win32") {
+    return win32
+  }
+
+  return posix
+}

package/src/domain/path-platform/types.ts

@@ -0,0 +1 @@
+export type PathPlatform = "win32" | "darwin" | "linux"

package/src/domain/search-catalog/constants.ts

@@ -0,0 +1 @@
+export const standardRrfK = 60

package/src/domain/search-catalog/coverage.ts

@@ -0,0 +1,35 @@
+import type { CatalogScan } from "./types.ts"
+
+export function catalogFolder(path: string): string {
+  const slash = path.lastIndexOf("/")
+  if (slash < 0) return ""
+  return path.slice(0, slash)
+}
+
+export function retiredAliasPath(path: string, scan: CatalogScan): boolean {
+  if (scan.failedPaths.some((failed) => failed === "" || path === failed || path.startsWith(`${failed}/`))) return false
+  return (
+    scan.excludedAliasFiles.includes(path) ||
+    scan.untraversedAliasDirectories.some((alias) => path === alias || path.startsWith(`${alias}/`))
+  )
+}
+
+export function confirmedMissing(path: string, scan: CatalogScan): boolean {
+  if (scan.failedPaths.some((failed) => failed === "" || path === failed || path.startsWith(`${failed}/`))) return false
+  if (scan.completeDirectories.includes(path)) return true
+  if (scan.observedPaths.includes(path) || scan.entries.some((entry) => entry.path === path)) return false
+
+  let child = path
+  while (child !== "") {
+    const folder = catalogFolder(child)
+    if (scan.completeDirectories.includes(folder)) {
+      return (
+        !scan.observedPaths.some((seen) => seen === child || seen.startsWith(`${child}/`)) &&
+        !scan.entries.some((entry) => entry.path === child || entry.path.startsWith(`${child}/`)) &&
+        !scan.failedPaths.some((failed) => failed === child || failed.startsWith(`${child}/`))
+      )
+    }
+    child = folder
+  }
+  return false
+}

package/src/domain/search-catalog/dont-forget.ts

@@ -0,0 +1,10 @@
+import type { CatalogEntry, CatalogSnapshot } from "./types.ts"
+
+export function dontForgetCandidates(snapshot: CatalogSnapshot): string[] {
+  return snapshot.entries.filter(isLoadedEntry).map((entry) => entry.path)
+}
+
+function isLoadedEntry(entry: CatalogEntry): boolean {
+  if (!entry.note) return false
+  return entry.note.dontForget && !entry.note.archived
+}

package/src/domain/search-catalog/fingerprint.ts

@@ -0,0 +1,9 @@
+import { createHash } from "node:crypto"
+
+import type { SemanticContent } from "./types.ts"
+
+export function semanticFingerprint(content: SemanticContent): string {
+  return createHash("sha256")
+    .update(JSON.stringify([content.statement, content.body]))
+    .digest("hex")
+}

package/src/domain/search-catalog/index.ts

@@ -0,0 +1,10 @@
+export * from "./constants.ts"
+export * from "./coverage.ts"
+export * from "./dont-forget.ts"
+export * from "./fingerprint.ts"
+export * from "./inspection.ts"
+export * from "./plan.ts"
+export * from "./ranking.ts"
+export * from "./schemas.ts"
+export * from "./types.ts"
+export * from "./vector-refresh.ts"