portable-agent-layer 0.41.1 → 0.42.0

package/src/tools/knowledge/lib.ts

@@ -0,0 +1,493 @@
+/**
+ * Knowledge — markdown-per-entity store with typed relationships.
+ *
+ * Each entity lives at:
+ *   ~/.pal/memory/knowledge/<Domain>/<slug>.md
+ *
+ * Frontmatter schema (canonical fields):
+ *   title:    human label
+ *   type:     free-form sub-type (e.g. "person", "ai-lab")
+ *   tags:     string[]
+ *   created:  ISO timestamp
+ *   updated:  ISO timestamp
+ *   quality:  0-10
+ *   status:   seedling | budding | evergreen
+ *   related:  Array<{ slug, type }>  -- type ∈ RELATION_TYPES
+ *
+ * Any additional frontmatter keys (role, company, sentiment, etc.) are
+ * preserved verbatim so domain-specific extractors can store rich
+ * attributes without changing the core schema.
+ *
+ * Ported from PAI's KNOWLEDGE/ pattern (see
+ * Personal_AI_Infrastructure/Releases/v5.0.0/.claude/PAI/TOOLS/KnowledgeGraph.ts).
+ */
+
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  writeFileSync,
+} from "node:fs";
+import { resolve } from "node:path";
+import { ensureDir, paths } from "../../hooks/lib/paths";
+
+// --- Constants --------------------------------------------------------------
+
+export const DOMAINS = ["People", "Companies", "Ideas", "Research"] as const;
+export type Domain = (typeof DOMAINS)[number];
+
+const DEFAULT_TYPE_BY_DOMAIN: Record<Domain, string> = {
+  People: "person",
+  Companies: "company",
+  Ideas: "idea",
+  Research: "research",
+};
+
+export const RELATION_TYPES = [
+  "supports",
+  "contradicts",
+  "extends",
+  "part-of",
+  "instance-of",
+  "caused-by",
+  "preceded-by",
+  "related",
+] as const;
+export type RelationType = (typeof RELATION_TYPES)[number];
+
+export const STATUSES = ["seedling", "budding", "evergreen"] as const;
+export type Status = (typeof STATUSES)[number];
+
+const CANONICAL_KEYS = new Set([
+  "title",
+  "type",
+  "tags",
+  "created",
+  "updated",
+  "quality",
+  "status",
+  "related",
+]);
+
+// --- Types ------------------------------------------------------------------
+
+export interface Related {
+  slug: string;
+  type: RelationType;
+}
+
+export interface EntityFrontmatter {
+  title: string;
+  type: string;
+  tags: string[];
+  created: string;
+  updated: string;
+  quality: number;
+  status: Status;
+  related: Related[];
+  [key: string]: unknown;
+}
+
+export interface Entity {
+  domain: Domain;
+  slug: string;
+  frontmatter: EntityFrontmatter;
+  body: string;
+}
+
+// --- Slug -------------------------------------------------------------------
+
+/**
+ * Deterministic slug: NFKD-normalize, strip diacritics, lowercase, replace
+ * any run of non-alnum chars with a single dash, trim leading/trailing dashes.
+ */
+export function slugify(input: string): string {
+  const normalized = input.normalize("NFKD").replace(/[̀-ͯ]/g, "");
+  const kebab = normalized
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+  return kebab;
+}
+
+// --- YAML emitter (bounded schema) -----------------------------------------
+
+function emitScalar(v: unknown): string {
+  if (v === null || v === undefined) return '""';
+  if (typeof v === "number" || typeof v === "boolean") return String(v);
+  return JSON.stringify(String(v));
+}
+
+function emitStringArray(arr: readonly string[]): string {
+  if (arr.length === 0) return "[]";
+  return `[${arr.map((s) => JSON.stringify(s)).join(", ")}]`;
+}
+
+function emitFrontmatter(fm: EntityFrontmatter): string {
+  const lines: string[] = [
+    "---",
+    `title: ${emitScalar(fm.title)}`,
+    `type: ${emitScalar(fm.type)}`,
+    `tags: ${emitStringArray(fm.tags)}`,
+    `created: ${emitScalar(fm.created)}`,
+    `updated: ${emitScalar(fm.updated)}`,
+    `quality: ${fm.quality}`,
+    `status: ${emitScalar(fm.status)}`,
+  ];
+
+  if (fm.related.length === 0) {
+    lines.push("related: []");
+  } else {
+    lines.push("related:");
+    for (const r of fm.related) {
+      lines.push(`  - slug: ${emitScalar(r.slug)}`, `    type: ${emitScalar(r.type)}`);
+    }
+  }
+
+  for (const [k, v] of Object.entries(fm)) {
+    if (CANONICAL_KEYS.has(k)) continue;
+    if (v === undefined || v === null) continue;
+    if (Array.isArray(v)) {
+      if (v.every((x) => typeof x === "string")) {
+        lines.push(`${k}: ${emitStringArray(v as string[])}`);
+      }
+      // Other array shapes are out of scope in v1 — skipped silently.
+      continue;
+    }
+    if (typeof v === "object") {
+      // Nested objects out of scope in v1.
+      continue;
+    }
+    lines.push(`${k}: ${emitScalar(v)}`);
+  }
+  lines.push("---");
+  return lines.join("\n");
+}
+
+// --- YAML parser (bounded schema) ------------------------------------------
+
+function parseScalar(raw: string): unknown {
+  const trimmed = raw.trim();
+  if (trimmed === "") return "";
+  if (trimmed === "true") return true;
+  if (trimmed === "false") return false;
+  if (/^-?\d+(\.\d+)?$/.test(trimmed)) return Number(trimmed);
+  if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
+    try {
+      return JSON.parse(trimmed);
+    } catch {
+      return trimmed.slice(1, -1);
+    }
+  }
+  return trimmed;
+}
+
+function parseStringArray(raw: string): string[] {
+  const trimmed = raw.trim();
+  if (trimmed === "[]") return [];
+  if (!(trimmed.startsWith("[") && trimmed.endsWith("]"))) return [];
+  const inner = trimmed.slice(1, -1).trim();
+  if (inner === "") return [];
+  // Split on commas not inside quotes.
+  const out: string[] = [];
+  let buf = "";
+  let inStr = false;
+  let esc = false;
+  for (const ch of inner) {
+    if (esc) {
+      buf += ch;
+      esc = false;
+      continue;
+    }
+    if (ch === "\\" && inStr) {
+      buf += ch;
+      esc = true;
+      continue;
+    }
+    if (ch === '"') {
+      inStr = !inStr;
+      buf += ch;
+      continue;
+    }
+    if (ch === "," && !inStr) {
+      out.push(String(parseScalar(buf)));
+      buf = "";
+      continue;
+    }
+    buf += ch;
+  }
+  if (buf.trim() !== "") out.push(String(parseScalar(buf)));
+  return out;
+}
+
+interface SplitResult {
+  fm: string;
+  body: string;
+}
+
+function splitFrontmatter(raw: string): SplitResult {
+  const match = /^---\n([\s\S]*?)\n---\n?([\s\S]*)$/.exec(raw);
+  if (!match) return { fm: "", body: raw };
+  return { fm: match[1], body: match[2] };
+}
+
+function parseFrontmatter(fmText: string): EntityFrontmatter {
+  const out: Record<string, unknown> = {
+    title: "",
+    type: "",
+    tags: [],
+    created: "",
+    updated: "",
+    quality: 0,
+    status: "seedling",
+    related: [] as Related[],
+  };
+
+  const lines = fmText.split("\n");
+  let i = 0;
+  while (i < lines.length) {
+    const line = lines[i];
+    if (line.trim() === "") {
+      i++;
+      continue;
+    }
+
+    // Top-level keys start with no indentation.
+    if (line.startsWith(" ") || line.startsWith("\t") || line.startsWith("-")) {
+      i++;
+      continue;
+    }
+
+    const colonIdx = line.indexOf(":");
+    if (colonIdx < 0) {
+      i++;
+      continue;
+    }
+
+    const key = line.slice(0, colonIdx).trim();
+    const rawVal = line.slice(colonIdx + 1);
+
+    if (key === "tags") {
+      out.tags = parseStringArray(rawVal);
+      i++;
+      continue;
+    }
+
+    if (key === "related") {
+      const v = rawVal.trim();
+      if (v === "[]" || v === "") {
+        // Could be inline empty or block-style. Look ahead.
+        if (v === "[]") {
+          out.related = [];
+          i++;
+          continue;
+        }
+        const items: Related[] = [];
+        i++;
+        let current: Partial<Related> | null = null;
+        while (i < lines.length) {
+          const child = lines[i];
+          if (
+            child.trim() === "" ||
+            (!child.startsWith(" ") && !child.startsWith("\t"))
+          ) {
+            break;
+          }
+          const t = child.trim();
+          if (t.startsWith("- slug:")) {
+            if (current?.slug) {
+              items.push({
+                slug: current.slug,
+                type: (current.type ?? "related") as RelationType,
+              });
+            }
+            current = {
+              slug: String(parseScalar(t.slice("- slug:".length))),
+            };
+          } else if (t.startsWith("slug:") && current) {
+            current.slug = String(parseScalar(t.slice("slug:".length)));
+          } else if (t.startsWith("type:") && current) {
+            current.type = parseScalar(t.slice("type:".length)) as RelationType;
+          }
+          i++;
+        }
+        if (current?.slug) {
+          items.push({
+            slug: current.slug,
+            type: (current.type ?? "related") as RelationType,
+          });
+        }
+        out.related = items;
+        continue;
+      }
+      out.related = [];
+      i++;
+      continue;
+    }
+
+    // Inline string-array values.
+    const v = rawVal.trim();
+    if (v.startsWith("[") && v.endsWith("]")) {
+      out[key] = parseStringArray(v);
+    } else {
+      out[key] = parseScalar(v);
+    }
+    i++;
+  }
+
+  return out as EntityFrontmatter;
+}
+
+// --- Validation -------------------------------------------------------------
+
+export function validate(entity: Entity): void {
+  const fm = entity.frontmatter;
+  if (!fm.title || typeof fm.title !== "string") {
+    throw new Error(`knowledge: missing or invalid 'title' for ${entity.slug}`);
+  }
+  if (typeof fm.quality !== "number" || fm.quality < 0 || fm.quality > 10) {
+    throw new Error(`knowledge: 'quality' must be 0-10 for ${entity.slug}`);
+  }
+  if (!STATUSES.includes(fm.status)) {
+    throw new Error(
+      `knowledge: 'status' must be one of ${STATUSES.join("|")} for ${entity.slug}`
+    );
+  }
+  if (!DOMAINS.includes(entity.domain)) {
+    throw new Error(
+      `knowledge: 'domain' must be one of ${DOMAINS.join("|")} (got ${entity.domain})`
+    );
+  }
+  if (!Array.isArray(fm.tags) || fm.tags.some((t) => typeof t !== "string")) {
+    throw new Error(`knowledge: 'tags' must be string[] for ${entity.slug}`);
+  }
+  if (!Array.isArray(fm.related)) {
+    throw new Error(`knowledge: 'related' must be array for ${entity.slug}`);
+  }
+  for (const r of fm.related) {
+    if (!r.slug || !RELATION_TYPES.includes(r.type)) {
+      throw new Error(
+        `knowledge: invalid related entry on ${entity.slug}: ${JSON.stringify(r)}`
+      );
+    }
+  }
+}
+
+// --- Serialize / parse ------------------------------------------------------
+
+export function serialize(entity: Entity): string {
+  validate(entity);
+  const fmText = emitFrontmatter(entity.frontmatter);
+  const body = entity.body.endsWith("\n") ? entity.body : `${entity.body}\n`;
+  return `${fmText}\n\n${body}`;
+}
+
+export function parse(domain: Domain, slug: string, raw: string): Entity {
+  const { fm, body } = splitFrontmatter(raw);
+  const frontmatter = parseFrontmatter(fm);
+  return { domain, slug, frontmatter, body: body.replace(/^\n+/, "") };
+}
+
+// --- Filesystem -------------------------------------------------------------
+
+function domainDir(domain: Domain, rootDir?: string): string {
+  if (rootDir) {
+    const d = resolve(rootDir, domain);
+    if (!existsSync(d)) mkdirSync(d, { recursive: true });
+    return d;
+  }
+  return ensureDir(resolve(paths.knowledge(), domain));
+}
+
+/** @lintignore — consumed by Phase 2 graph layer (needs path-per-slug to read files) */
+export function entityPath(domain: Domain, slug: string, rootDir?: string): string {
+  return resolve(domainDir(domain, rootDir), `${slug}.md`);
+}
+
+export function exists(domain: Domain, slug: string, rootDir?: string): boolean {
+  return existsSync(entityPath(domain, slug, rootDir));
+}
+
+export function save(entity: Entity, rootDir?: string): void {
+  const p = entityPath(entity.domain, entity.slug, rootDir);
+  const tmp = `${p}.tmp`;
+  writeFileSync(tmp, serialize(entity), "utf-8");
+  renameSync(tmp, p);
+}
+
+export function load(domain: Domain, slug: string, rootDir?: string): Entity | null {
+  const p = entityPath(domain, slug, rootDir);
+  if (!existsSync(p)) return null;
+  return parse(domain, slug, readFileSync(p, "utf-8"));
+}
+
+export function list(domain?: Domain, rootDir?: string): Entity[] {
+  const target = domain ? [domain] : DOMAINS;
+  const out: Entity[] = [];
+  for (const d of target) {
+    const dir = domainDir(d, rootDir);
+    if (!existsSync(dir)) continue;
+    for (const entry of readdirSync(dir)) {
+      if (!entry.endsWith(".md") || entry.startsWith("_")) continue;
+      const slug = entry.slice(0, -3);
+      const e = load(d, slug, rootDir);
+      if (e) out.push(e);
+    }
+  }
+  return out;
+}
+
+// --- Create / get -----------------------------------------------------------
+
+export interface CreateInput {
+  domain: Domain;
+  name: string;
+  type?: string;
+  tags?: string[];
+  quality?: number;
+  status?: Status;
+  related?: Related[];
+  body?: string;
+  extra?: Record<string, unknown>;
+}
+
+/**
+ * Idempotent: if an entity already exists at the slug derived from `name`,
+ * return it untouched. Otherwise create a new file with sensible defaults
+ * (quality 5, status "seedling", today's ISO timestamps).
+ *
+ * Merging new attributes into an existing entity is intentionally NOT here —
+ * that lives in `ingest.ts`. Keeping `getOrCreate` idempotent makes it a safe
+ * storage primitive callable from anywhere.
+ */
+export function getOrCreate(input: CreateInput, rootDir?: string): Entity {
+  const slug = slugify(input.name);
+  if (!slug) {
+    throw new Error(`knowledge: cannot derive slug from name "${input.name}"`);
+  }
+  const existing = load(input.domain, slug, rootDir);
+  if (existing) return existing;
+
+  const now = new Date().toISOString();
+  const fm: EntityFrontmatter = {
+    title: input.name,
+    type: input.type ?? DEFAULT_TYPE_BY_DOMAIN[input.domain],
+    tags: input.tags ?? [],
+    created: now,
+    updated: now,
+    quality: input.quality ?? 5,
+    status: input.status ?? "seedling",
+    related: input.related ?? [],
+    ...(input.extra ?? {}),
+  };
+  const entity: Entity = {
+    domain: input.domain,
+    slug,
+    frontmatter: fm,
+    body: input.body ?? "",
+  };
+  save(entity, rootDir);
+  return entity;
+}

package/assets/skills/extract-entities/SKILL.md

@@ -1,62 +0,0 @@
----
-name: extract-entities
-description: Extract people and companies from content (articles, videos, URLs, pasted text). Use when identifying who and what organizations are mentioned in content.
-argument-hint: <content, URL, or pasted text>
----
-
-Extract people and companies from $ARGUMENTS:
-
-1. Read/fetch the content
-2. Extract ALL people and companies mentioned
-
-## People
-
-For each person, extract:
-- **name**: Full name
-- **role**: author | subject | mentioned | quoted | expert | interviewer | interviewee
-- **title**: Job title (null if unknown)
-- **company**: Company affiliation (null if unknown)
-- **social**: twitter (@handle), linkedin (URL), email, website — null if unknown
-- **context**: Why this person is mentioned and their relevance
-- **importance**: primary (central to content) | secondary (supporting) | minor (brief mention)
-
-## Companies
-
-For each company/organization, extract:
-- **name**: Official name
-- **domain**: Primary website domain (e.g. "anthropic.com", null if unknown)
-- **industry**: Classification (AI, security, fintech, healthcare, etc.)
-- **context**: How and why mentioned
-- **mentioned_as**: subject | source | example | competitor | partner | acquisition | product | other
-- **sentiment**: positive | neutral | negative | mixed
-
-## Output
-
-Return structured JSON:
-
-```json
-{
-  "people": [...],
-  "companies": [...]
-}
-```
-
-## Guidelines
-
-- Accuracy over quantity — use null for unknown fields, never guess
-- Include authors, subjects, quoted individuals, and anyone significantly mentioned
-- For research papers: all authors get "author" role
-- For interviews: distinguish interviewer vs interviewee
-- Universities and research institutions count as companies
-- Extract social handles from bios, signatures, or text body
-- Context fields should explain relevance, not just repeat the mention
-
-## Persistence
-
-After displaying results, ask the user if they want to save. When saving, pipe the JSON output through the entity-save tool which handles deduplication automatically:
-
-```bash
-echo '<the JSON output>' | bun ~/.pal/skills/extract-entities/tools/entity-save.ts -- --source "<URL or content origin>"
-```
-
-The tool deduplicates against the entity index (`memory/entities/entity-index.json`), assigns stable UUIDs, tracks occurrences, and reports what was new vs existing.

package/assets/skills/extract-entities/tools/entity-save.ts

@@ -1,110 +0,0 @@
-#!/usr/bin/env bun
-/**
- * Entity Save — Deduplicate and persist extracted entities.
- *
- * Accepts extracted people/companies JSON via stdin or --file,
- * deduplicates against the entity index, and saves.
- *
- * Usage:
- *   echo '{"people":[...],"companies":[...]}' | bun entity-save.ts -- --source "https://example.com"
- *   bun entity-save.ts -- --file /path/to/extracted.json --source "https://example.com"
- */
-
-import { readFileSync } from "node:fs";
-import { parseArgs } from "node:util";
-import { loadEntityIndex, processEntities } from "../../../../src/hooks/lib/entities";
-
-const { values } = parseArgs({
-  args: Bun.argv.slice(2),
-  options: {
-    source: { type: "string", short: "s", default: "manual" },
-    file: { type: "string", short: "f" },
-  },
-  strict: true,
-});
-
-const sourceId = values.source ?? "manual";
-
-let raw: string;
-if (values.file) {
-  raw = readFileSync(values.file, "utf-8");
-} else {
-  raw = await Bun.stdin.text();
-}
-
-if (!raw.trim()) {
-  console.error("Error: No input provided. Pipe JSON via stdin or use --file.");
-  process.exit(1);
-}
-
-let data: {
-  people: Array<Record<string, unknown>>;
-  companies: Array<Record<string, unknown>>;
-  links?: Array<Record<string, unknown>>;
-  sources?: Array<Record<string, unknown>>;
-};
-try {
-  data = JSON.parse(raw);
-} catch {
-  console.error("Error: Invalid JSON input.");
-  process.exit(1);
-}
-
-if (!Array.isArray(data.people) || !Array.isArray(data.companies)) {
-  console.error('Error: JSON must have "people" and "companies" arrays.');
-  process.exit(1);
-}
-data.links ??= [];
-data.sources ??= [];
-
-const before = loadEntityIndex();
-const counts = (idx: ReturnType<typeof loadEntityIndex>) => ({
-  people: Object.keys(idx.people).length,
-  companies: Object.keys(idx.companies).length,
-  links: Object.keys(idx.links).length,
-  sources: Object.keys(idx.sources).length,
-});
-const cb = counts(before);
-
-const result = processEntities(
-  {
-    people: data.people as Array<{ name: string; [key: string]: unknown }>,
-    companies: data.companies as Array<{
-      name: string;
-      domain: string | null;
-      [key: string]: unknown;
-    }>,
-    links: data.links as Array<{ url: string; [key: string]: unknown }>,
-    sources: data.sources as Array<{
-      url: string | null;
-      author: string | null;
-      publication: string | null;
-      [key: string]: unknown;
-    }>,
-  },
-  sourceId
-);
-
-const ca = counts(loadEntityIndex());
-
-console.log(
-  JSON.stringify(
-    {
-      saved: {
-        people: result.people.length,
-        companies: result.companies.length,
-        links: result.links.length,
-        sources: result.sources.length,
-      },
-      new: {
-        people: ca.people - cb.people,
-        companies: ca.companies - cb.companies,
-        links: ca.links - cb.links,
-        sources: ca.sources - cb.sources,
-      },
-      total: ca,
-    },
-    null,
-    2
-  )
-);