structured-context 0.9.2 → 0.10.0

package/dist/validate.js

@@ -0,0 +1,291 @@
+import { isAbsolute, relative, resolve } from 'node:path';
+import { getSpaceConfigDir, resolveSchema } from './config';
+import { readSpace } from './read/read-space';
+import { extractEntityInfo, loadSchema } from './schema/schema';
+import { validateGraph } from './schema/validate-graph';
+import { validateRules } from './schema/validate-rules';
+import { buildSpaceGraph } from './space-graph';
+/**
+ * Format AJV errors for better readability.
+ * Groups related errors and extracts helpful context like allowed values.
+ */
+export function formatErrors(errors, schema, schemaRefRegistry, nodeData) {
+    const formatted = [];
+    const seen = new Set();
+    // Group errors by instancePath
+    const byPath = new Map();
+    for (const err of errors) {
+        const path = err.instancePath || 'root';
+        if (!byPath.has(path)) {
+            byPath.set(path, []);
+        }
+        byPath.get(path).push(err);
+    }
+    for (const [path, pathErrors] of byPath) {
+        // Check if this is a oneOf failure at root - extract valid types from schema
+        const isRootOneOf = path === 'root' || path === '/type';
+        let hasOneOfContext = false;
+        if (isRootOneOf && pathErrors.length > 1) {
+            hasOneOfContext = Array.isArray(schema.oneOf);
+            if (hasOneOfContext) {
+                const entities = extractEntityInfo(schema, schemaRefRegistry);
+                const validTypes = entities.map((e) => e.type).sort();
+                if (validTypes.length > 0) {
+                    const actualValue = nodeData.type;
+                    // Only show type error if the actual type is NOT in the valid types list
+                    if (actualValue !== undefined && !validTypes.includes(String(actualValue))) {
+                        const message = `Invalid type "${actualValue}". Valid types are: ${validTypes.join(', ')}`;
+                        const key = `type:${validTypes.join(',')}`;
+                        if (!seen.has(key)) {
+                            seen.add(key);
+                            formatted.push({ message, dedupeKey: key });
+                        }
+                    }
+                }
+            }
+        }
+        // Handle individual errors
+        for (const err of pathErrors) {
+            // Skip individual type const/enum errors when in oneOf context (we handle it above)
+            if (hasOneOfContext &&
+                (err.keyword === 'const' || err.keyword === 'enum') &&
+                (err.instancePath === '' || err.instancePath === '/type')) {
+                continue;
+            }
+            const parts = err.instancePath.split('/').filter(Boolean);
+            const fieldName = parts.length > 0 ? parts[parts.length - 1] : 'root';
+            let message = err.message;
+            let key = `${err.instancePath}:${err.keyword}`;
+            // Enhance const errors
+            if (err.keyword === 'const' && err.params?.allowedValue !== undefined) {
+                const actual = err.data !== undefined ? `"${err.data}"` : 'missing value';
+                const expected = `"${err.params.allowedValue}"`;
+                message = `${fieldName}: expected ${expected}, got ${actual}`;
+                key = `${err.instancePath}:const:${err.params.allowedValue}`;
+            }
+            // Enhance enum errors
+            else if (err.keyword === 'enum' && err.params?.allowedValues && Array.isArray(err.params.allowedValues)) {
+                let actual = err.data !== undefined ? `"${err.data}"` : null;
+                // If err.data is undefined, try to get the value from nodeData using the field name
+                if (actual === null && fieldName !== 'root') {
+                    const actualValue = nodeData[fieldName];
+                    if (actualValue !== undefined) {
+                        actual = `"${actualValue}"`;
+                    }
+                }
+                if (actual === null) {
+                    actual = 'missing value';
+                }
+                const allowed = err.params.allowedValues.map((v) => `"${v}"`).join(', ');
+                message = `${fieldName}: ${actual} is not valid. Allowed: ${allowed}`;
+                key = `${err.instancePath}:enum:${err.params.allowedValues.join(',')}`;
+            }
+            // Generic message with path
+            else {
+                message = `${fieldName}: ${err.message}`;
+            }
+            if (!seen.has(key)) {
+                seen.add(key);
+                formatted.push({ message, dedupeKey: key });
+            }
+        }
+    }
+    return formatted;
+}
+/**
+ * Run full validation on a space and return a structured result.
+ * Does not output to console or call process.exit.
+ */
+export async function validateSpace(context) {
+    const { schema, schemaRefRegistry, schemaValidator } = context;
+    const metadata = schema.metadata;
+    const readResult = await readSpace(context);
+    const { nodes, parseIssues } = readResult;
+    // Pre-extract valid types for early type validation
+    const validTypes = Array.isArray(schema.oneOf) ? extractEntityInfo(schema, schemaRefRegistry).map((e) => e.type) : [];
+    const result = {
+        validCount: 0,
+        nodeErrorCount: 0,
+        nodeErrors: [],
+        refErrors: [],
+        duplicateErrors: [],
+        ruleViolations: [],
+        hierarchyViolations: [],
+        orphans: [],
+        parseIssues,
+    };
+    for (const node of nodes) {
+        // Early type validation - check before full AJV validation to prevent cascading errors
+        const nodeType = node.schemaData.type;
+        if (nodeType !== undefined && validTypes.length > 0 && !validTypes.includes(nodeType)) {
+            result.nodeErrorCount++;
+            result.nodeErrors.push({
+                file: node.label,
+                errors: [
+                    {
+                        instancePath: '/type',
+                        keyword: 'enum',
+                        message: `Invalid type "${nodeType}". Valid types are: ${validTypes.sort().join(', ')}`,
+                        params: { allowedValues: validTypes },
+                        schemaPath: '#/oneOf',
+                        data: nodeType,
+                    },
+                ],
+                nodeData: node.schemaData,
+            });
+            continue;
+        }
+        const valid = schemaValidator(node.schemaData);
+        if (valid) {
+            result.validCount++;
+        }
+        else {
+            result.nodeErrorCount++;
+            result.nodeErrors.push({
+                file: node.label,
+                errors: schemaValidator.errors || [],
+                nodeData: node.schemaData,
+            });
+        }
+    }
+    // Detect duplicate node keys (titles)
+    const titleToFiles = new Map();
+    for (const node of nodes) {
+        const title = node.title;
+        if (!titleToFiles.has(title)) {
+            titleToFiles.set(title, []);
+        }
+        titleToFiles.get(title).push(node.label);
+    }
+    for (const [title, files] of titleToFiles) {
+        if (files.length > 1) {
+            result.duplicateErrors.push({ title, files });
+        }
+    }
+    // Validate all hierarchy constraints (field references and structure)
+    const hierarchyValidation = validateGraph(nodes, metadata, readResult.unresolvedRefs);
+    result.refErrors.push(...hierarchyValidation.refErrors);
+    result.hierarchyViolations = [...hierarchyValidation.violations];
+    // Calculate orphan count (informational, not a validation error)
+    if (metadata.hierarchy) {
+        result.orphans = [...buildSpaceGraph(nodes, metadata.hierarchy.levels).orphans];
+    }
+    // Load and execute rules validation if schema defines rules
+    if (metadata.rules) {
+        result.ruleViolations = await validateRules(nodes, metadata.rules);
+    }
+    return result;
+}
+/** Find which space a file belongs to by checking directory containment. */
+function resolveFileSpace(filePath, config) {
+    const absFile = isAbsolute(filePath) ? filePath : resolve(process.cwd(), filePath);
+    for (const space of config.spaces) {
+        const absSpace = isAbsolute(space.path) ? space.path : resolve(process.cwd(), space.path);
+        // Trailing slash prevents prefix-match false positives (e.g. /foo matching /foobar/)
+        const spaceDir = absSpace.endsWith('/') ? absSpace : `${absSpace}/`;
+        if (absFile.startsWith(spaceDir) || absFile === absSpace) {
+            return { space, label: relative(absSpace, absFile) };
+        }
+    }
+    return null;
+}
+function buildContextForSpace(space, config) {
+    const resolvedSchemaPath = resolveSchema(config, space);
+    const { schema, schemaRefRegistry, schemaValidator } = loadSchema(resolvedSchemaPath);
+    const configDir = getSpaceConfigDir(space.name);
+    return { space, config, resolvedSchemaPath, schema, schemaRefRegistry, schemaValidator, configDir };
+}
+/**
+ * Validate a single file within its space.
+ *
+ * Reads the full space (required for graph correctness) but filters all reported
+ * errors to only those attributable to the target file.
+ *
+ * Returns a result object — does not output to console or call process.exit.
+ * If the file is not in any configured space, returns a result with `inSpace: false`.
+ */
+export async function validateFile(filePath, config) {
+    const resolved = resolveFileSpace(filePath, config);
+    if (!resolved) {
+        return {
+            file: filePath,
+            inSpace: false,
+            message: 'File does not belong to any configured space.',
+        };
+    }
+    const { space, label } = resolved;
+    const context = buildContextForSpace(space, config);
+    const { schema, schemaRefRegistry, schemaValidator } = context;
+    const metadata = schema.metadata;
+    const readResult = await readSpace(context);
+    const { nodes } = readResult;
+    // Pre-extract valid types for early type validation
+    const validTypes = Array.isArray(schema.oneOf) ? extractEntityInfo(schema, schemaRefRegistry).map((e) => e.type) : [];
+    const errors = {};
+    // Schema validation errors for this node
+    for (const node of nodes) {
+        if (node.label !== label)
+            continue;
+        // Early type validation - check before full AJV validation to prevent cascading errors
+        const nodeType = node.schemaData.type;
+        if (nodeType !== undefined && validTypes.length > 0 && !validTypes.includes(nodeType)) {
+            errors[`schema:type:${validTypes.join(',')}`] = {
+                kind: 'schema',
+                message: `Invalid type "${nodeType}". Valid types are: ${validTypes.sort().join(', ')}`,
+            };
+            continue;
+        }
+        const valid = schemaValidator(node.schemaData);
+        if (!valid) {
+            const formatted = formatErrors(schemaValidator.errors ?? [], schema, schemaRefRegistry, node.schemaData);
+            for (const { message, dedupeKey } of formatted) {
+                errors[`schema:${dedupeKey}`] = { kind: 'schema', message };
+            }
+        }
+    }
+    // Duplicate key errors — include if this file is one of the duplicates
+    const titleToFiles = new Map();
+    for (const node of nodes) {
+        if (!titleToFiles.has(node.title))
+            titleToFiles.set(node.title, []);
+        titleToFiles.get(node.title).push(node.label);
+    }
+    for (const [title, files] of titleToFiles) {
+        if (files.length > 1 && files.includes(label)) {
+            const others = files.filter((f) => f !== label);
+            errors[`duplicate:${title}`] = {
+                kind: 'duplicate',
+                message: `Duplicate title "${title}" also exists in: ${others.join(', ')}`,
+            };
+        }
+    }
+    // Broken links and hierarchy violations from this file
+    const hierarchyValidation = validateGraph(nodes, metadata, readResult.unresolvedRefs);
+    for (const { file, parent, error } of hierarchyValidation.refErrors) {
+        if (file === label) {
+            errors[`broken-link:${parent}`] = { kind: 'broken-link', message: `${parent} → ${error}` };
+        }
+    }
+    for (const v of hierarchyValidation.violations) {
+        if (v.file === label) {
+            errors[`hierarchy:${v.description}`] = { kind: 'hierarchy', message: v.description };
+        }
+    }
+    // Rule violations for this node
+    if (metadata.rules) {
+        const ruleViolations = await validateRules(nodes, metadata.rules);
+        for (const v of ruleViolations) {
+            if (v.file === label) {
+                errors[`rule:${v.ruleId}`] = { kind: 'rule', message: `[${v.ruleId}] ${v.description}` };
+            }
+        }
+    }
+    return {
+        file: isAbsolute(filePath) ? filePath : resolve(process.cwd(), filePath),
+        label,
+        space: space.name,
+        errors,
+        errorCount: Object.keys(errors).length,
+        inSpace: true,
+    };
+}

package/docs/schemas.md

@@ -258,8 +258,9 @@ Metadata is composed across the `$ref` graph with deterministic behavior:
 2. Apply root schema metadata last.
 
 Merge rules:
-- `hierarchy`: zero or one provider allowed. Multiple providers error.
-- `aliases`: shallow merged; later provider wins per key.
+- `hierarchy`: may be defined in partials; **last one wins**. This allows partials to define a default hierarchy that composing schemas can override.
+- `aliases`: shallow merged; later file wins per key.
+- `relationships`: collected from all files; order preserved.
 - `rules`: merged by `id`.
 - Duplicate rule `id` with different payload errors by default.
 - A later rule may replace an earlier one only with `"override": true`.
@@ -309,6 +310,8 @@ Imported rules are normalized into one executable flat list before validation.
 - Local partial `$id` values must not collide with bundled IDs.
 - `$ref` resolution is transitive across files.
 - Partials with no `$metadata` should prefer `$schema: "http://json-schema.org/draft-07/schema#"` so they validate standalone as plain JSON Schema fragments.
+- **Bundled partials as entity libraries**: `_sctx_base.json`, `_strategy_general.json`, `_knowledge_wiki.json`, and `_ost_strict.json` provide reusable entity definitions and metadata. Composing schemas can reference these via `$ref` rather than redefining common entity types.
+- **Partials can carry metadata**: Partials may include `$metadata` (hierarchy, aliases, relationships, rules). This makes them self-contained units that bundle both type definitions and behavioral metadata.
 
 ## Editor expectations
 
@@ -328,7 +331,7 @@ For schemas migrating from older metadata structure:
 3. Move `allowSkipLevels` under `hierarchy`.
 4. Convert grouped rule containers to flat `rules[]` with per-rule `category`.
 5. If duplicate rule IDs are intentional, mark later rules with `override: true`.
-6. Re-run `bunx structured-context schemas show --space <name>` and `validate` to confirm merged metadata/rules.
+6. Re-run `sctx schemas show --space <name>` and `validate` to confirm merged metadata/rules.
 
 ## JSON5 support
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "structured-context",
-  "version": "0.9.2",
-  "description": "Tools for working with Opportunity Solution Tree structures and other product management and strategy frameworks",
+  "version": "0.10.0",
+  "description": "Keep your content clean and your thinking clear with assistive tools for structured knowledge",
   "module": "./dist/index.js",
   "type": "module",
   "license": "MIT",
@@ -18,9 +18,13 @@
   ],
   "exports": {
     ".": "./dist/index.js",
-    "./plugin-api": {
-      "types": "./dist/plugin-api.d.ts",
-      "default": "./dist/plugin-api.js"
+    "./api": {
+      "types": "./dist/api.d.ts",
+      "default": "./dist/api.js"
+    },
+    "./types": {
+      "types": "./dist/types.d.ts",
+      "default": "./dist/types.js"
     }
   },
   "scripts": {
@@ -57,8 +61,10 @@
     "ajv": "^8.18.0",
     "chokidar": "^5.0.0",
     "commander": "^14.0.3",
+    "fast-glob": "^3.3.3",
     "gray-matter": "^4.0.3",
     "js-yaml": "^4.1.1",
+    "json5": "^2.2.3",
     "jsonata": "^2.1.0",
     "mdast-util-to-string": "^4.0.0",
     "remark-gfm": "^4.0.1",

package/schemas/_knowledge_wiki.json

@@ -0,0 +1,246 @@
+{
+  "$schema": "https://raw.githubusercontent.com/mindsocket/structured-context/main/schemas/generated/_structured_context_schema_meta.json",
+  "$id": "sctx://_knowledge_wiki",
+  // Shared entity definitions for knowledge wiki schemas.
+  // Provides aliases, relationships, rules, and entity type $defs.
+  // This partial has no hierarchy — wiki entities are flat with no required parent links.
+  "$metadata": {
+    "aliases": {
+      "source_summary": "source",
+      "study": "source",
+      "article": "source",
+      "paper": "source",
+      "research": "source",
+      "comparison": "synthesis",
+      "query": "synthesis"
+    },
+    "relationships": [
+      {
+        "parent": "source",
+        "type": "source",
+        "field": "sources",
+        "fieldOn": "child",
+        "multiple": true
+      },
+      {
+        "parent": "source",
+        "type": "synthesis",
+        "field": "sources",
+        "fieldOn": "child",
+        "multiple": true
+      },
+      {
+        "parent": "source",
+        "type": "entity",
+        "field": "sources",
+        "fieldOn": "child",
+        "multiple": true
+      },
+      {
+        "parent": "concept",
+        "type": "synthesis",
+        "field": "concepts",
+        "fieldOn": "child",
+        "multiple": true
+      }
+    ],
+    "rules": [
+      {
+        "id": "synthesis-references-sources",
+        "category": "coherence",
+        "description": "Syntheses should reference at least one contributing source in their sources array",
+        "type": "synthesis",
+        "check": "$exists(current.sources) and $count(current.sources) >= 1"
+      },
+      {
+        "id": "concept-has-summary",
+        "category": "best-practice",
+        "description": "Concept pages should have a short summary for index discoverability",
+        "type": "concept",
+        "check": "$exists(current.summary) and $length(current.summary) > 0"
+      }
+    ]
+  },
+  "$defs": {
+    "source": {
+      "type": "object",
+      "description": "A processed summary of a raw source (article, paper, book, video, thread, podcast, etc.). Captures key information from the source in structured form with provenance.",
+      "allOf": [{ "$ref": "sctx://_sctx_base#/$defs/baseNodeProps" }],
+      "properties": {
+        "type": { "const": "source" },
+        "summary": {
+          "$ref": "sctx://_sctx_base#/$defs/summary"
+        },
+        "url": {
+          "type": "string",
+          "description": "URL of the original source"
+        },
+        "local_copy": {
+          "$ref": "sctx://_sctx_base#/$defs/wikilink",
+          "description": "Wikilink to a local copy of the source (e.g. a saved PDF or clipped note)"
+        },
+        "author": {
+          "type": "string",
+          "description": "Author or creator of the source"
+        },
+        "book": {
+          "type": "string",
+          "description": "Book title and author if the source is a book"
+        },
+        "published_date": {
+          "type": "string",
+          "format": "date",
+          "description": "Original publication date"
+        },
+        "source_type": {
+          "type": "string",
+          "description": "Kind of source material. Free-form — common values include article, paper, book, video, podcast, thread, substack, tweet, website. Not enum-constrained to allow evolving vocabulary."
+        },
+        "sources": {
+          "type": "array",
+          "items": { "$ref": "sctx://_sctx_base#/$defs/wikilink" },
+          "description": "Wikilinks to other source pages that this one references or extends"
+        }
+      },
+      "required": ["type", "url"],
+      "additionalProperties": true,
+      "examples": [
+        {
+          "type": "source",
+          "author": "Joe Bloggs",
+          "url": "https://example.com/design-of-yesterday-things",
+          "source_type": "book",
+          "tags": ["design", "ux", "psychology"]
+        }
+      ]
+    },
+    "concept": {
+      "type": "object",
+      "description": "A concept page — defines and frames a specific idea, mechanism, or domain term. May draw from multiple sources. The building blocks of the wiki's knowledge graph.",
+      "allOf": [{ "$ref": "sctx://_sctx_base#/$defs/baseNodeProps" }],
+      "properties": {
+        "type": { "const": "concept" },
+        "summary": {
+          "$ref": "sctx://_sctx_base#/$defs/summary"
+        },
+        "sources": {
+          "type": "array",
+          "items": { "$ref": "sctx://_sctx_base#/$defs/wikilink" },
+          "description": "Wikilinks to source pages that contributed to this concept"
+        },
+        "related": {
+          "type": "array",
+          "items": { "$ref": "sctx://_sctx_base#/$defs/wikilink" },
+          "description": "Wikilinks to related concept or synthesis pages"
+        }
+      },
+      "required": ["type"],
+      "additionalProperties": true,
+      "examples": [
+        {
+          "type": "concept",
+          "summary": "A property of an object that signals how it can be used, first described by Gibson and popularised in design by Norman",
+          "tags": ["design", "ux"]
+        }
+      ]
+    },
+    "entity": {
+      "type": "object",
+      "description": "A concrete entity — a person, organisation, product, model, or other named thing. The building blocks of the wiki's knowledge graph alongside concepts. Entities are distinct from concepts: a concept frames an idea or mechanism; an entity is a specific, identifiable thing in the world.",
+      "allOf": [{ "$ref": "sctx://_sctx_base#/$defs/baseNodeProps" }],
+      "properties": {
+        "type": { "const": "entity" },
+        "summary": {
+          "$ref": "sctx://_sctx_base#/$defs/summary"
+        },
+        "sources": {
+          "type": "array",
+          "items": { "$ref": "sctx://_sctx_base#/$defs/wikilink" },
+          "description": "Wikilinks to source pages that contributed to this entity page"
+        },
+        "related": {
+          "type": "array",
+          "items": { "$ref": "sctx://_sctx_base#/$defs/wikilink" },
+          "description": "Wikilinks to related entity, concept, or synthesis pages"
+        }
+      },
+      "required": ["type"],
+      "additionalProperties": true,
+      "examples": [
+        {
+          "type": "entity",
+          "summary": "AI safety research lab founded by Dario Amodei and Daniela Amodei",
+          "tags": ["company", "ai", "safety"]
+        }
+      ]
+    },
+    "synthesis": {
+      "type": "object",
+      "description": "A synthesis page — connects multiple concepts and sources into a coherent model, framework, or thesis. The highest-value pages in a knowledge wiki. Must reference contributing sources.",
+      "allOf": [{ "$ref": "sctx://_sctx_base#/$defs/baseNodeProps" }],
+      "properties": {
+        "type": { "const": "synthesis" },
+        "summary": {
+          "$ref": "sctx://_sctx_base#/$defs/summary"
+        },
+        "sources": {
+          "type": "array",
+          "items": { "$ref": "sctx://_sctx_base#/$defs/wikilink" },
+          "description": "Wikilinks to source pages this synthesis draws from"
+        },
+        "concepts": {
+          "type": "array",
+          "items": { "$ref": "sctx://_sctx_base#/$defs/wikilink" },
+          "description": "Wikilinks to concept pages this synthesis connects"
+        }
+      },
+      "required": ["type"],
+      "additionalProperties": true,
+      "examples": [
+        {
+          "type": "synthesis",
+          "summary": "How Norman's affordance model and feedback principles combine to explain intuitive vs confusing interfaces",
+          "sources": ["[[The Design of Everyday Things - Don Norman]]"],
+          "concepts": ["[[Affordance]]", "[[Feedback loop]]"]
+        }
+      ]
+    },
+    "note": {
+      "type": "object",
+      "description": "Personal notes, journal entries, appointments, experiential records. Your own data about your journey — not sourced from external material.",
+      "allOf": [{ "$ref": "sctx://_sctx_base#/$defs/baseNodeProps" }],
+      "properties": {
+        "type": { "const": "note" },
+        "date": {
+          "type": "string",
+          "format": "date",
+          "description": "Date this entry was created or relates to"
+        }
+      },
+      "required": ["type"],
+      "additionalProperties": true,
+      "examples": [
+        {
+          "type": "note",
+          "date": "2024-03-15",
+          "tags": ["reading-log"]
+        }
+      ]
+    },
+    "index": {
+      "type": "object",
+      "description": "The wiki index — a catalog of all pages with links, summaries, and metadata. Updated by the agent on ingest. Used as the primary navigation entry point.",
+      "allOf": [{ "$ref": "sctx://_sctx_base#/$defs/baseNodeProps" }],
+      "properties": {
+        "type": { "const": "index" }
+      },
+      "required": ["type"],
+      "additionalProperties": true,
+      "examples": [
+        {
+          "type": "index"
+        }
+      ]
+    }
+  }
+}