studiograph 1.1.3 → 1.2.0-beta.10

package/dist/core/graph.js

@@ -4,8 +4,8 @@
  * Provides unified entity management across any repo type.
  * Integrates Git, Markdown, and Asset services.
  */
-import { join, basename } from 'path';
-import { existsSync, readdirSync, statSync } from 'fs';
+import { join, dirname } from 'path';
+import { existsSync, readdirSync, readFileSync, statSync, writeFileSync, mkdirSync } from 'fs';
 import { GitService } from '../services/git.js';
 import { MarkdownService } from '../services/markdown.js';
 import { AssetService } from '../services/assets/index.js';
@@ -54,33 +54,43 @@ export class BaseGraphManager {
     }
     /**
      * Get entity file path
+     *
+     * Checks three layouts:
+     *   1. Flat folder:       {repo}/{entityId}/main.md
+     *   2. Type subfolder:    {repo}/{entityType}/{entityId}/main.md
+     *   3. Collection (flat): {repo}/{entityId}.md
      */
     getEntityPath(entityType, entityId) {
-        // Entity files are stored in {entityType}/{entityId}.md
-        return join(this.repoPath, entityType, `${entityId}.md`);
-    }
-    /**
-     * Get entity folder path
-     */
-    getEntityFolder(entityType) {
-        return join(this.repoPath, entityType);
-    }
-    /**
-     * Public alias for getEntityPath (used internally by dataset methods)
-     */
-    getEntityPathPublic(entityType, entityId) {
-        return this.getEntityPath(entityType, entityId);
+        // 1. Flat folder layout (most common)
+        const folderPath = join(this.repoPath, entityId, 'main.md');
+        if (existsSync(folderPath))
+            return folderPath;
+        // 2. Type-based subfolder layout (e.g. projects/deck/sample-deck/main.md)
+        if (entityType) {
+            const typeFolderPath = join(this.repoPath, entityType, entityId, 'main.md');
+            if (existsSync(typeFolderPath))
+                return typeFolderPath;
+        }
+        // 3. Collection mode (flat file)
+        const flatPath = join(this.repoPath, `${entityId}.md`);
+        if (existsSync(flatPath))
+            return flatPath;
+        return folderPath; // default to folder layout for new entities
     }
     /**
      * Validate entity data against schema
      */
-    validateEntity(entityType, data) {
+    validateEntity(entityType, data, lenient) {
         // Use schemaRegistry when available (supports extended built-in schemas and
         // custom entity types).  Fall back to ENTITY_SCHEMAS, then EntityBaseSchema
         // for unregistered workspace-specific types.
-        const schema = this.schemaRegistry?.getSchema(entityType)
-            ?? ENTITY_SCHEMAS[entityType]
-            ?? EntityBaseSchema;
+        // In lenient mode, only validate base fields so entities missing type-specific
+        // required fields (e.g. name) are still discoverable.
+        const schema = lenient
+            ? EntityBaseSchema
+            : (this.schemaRegistry?.getSchema(entityType)
+                ?? ENTITY_SCHEMAS[entityType]
+                ?? EntityBaseSchema);
         try {
             // Add entity_type and preprocess YAML parsing artefacts:
             // - null → undefined (Zod .optional() doesn't accept null)
@@ -125,7 +135,7 @@ export class BaseGraphManager {
      * Create a new entity
      */
     async create(options) {
-        const { entityType, entityId, data, content = '', commitMessage } = options;
+        const { entityType, entityId, data, content = '', commitMessage, skipCommit } = options;
         // Validate data
         const validatedData = this.validateEntity(entityType, data);
         // Check if entity already exists
@@ -139,7 +149,7 @@ export class BaseGraphManager {
             content,
             wikilinks: this.markdownService.extractWikilinks(content, validatedData),
         });
-        if (this.gitService) {
+        if (this.gitService && !skipCommit) {
             const transaction = this.gitService.createTransaction();
             transaction.add({
                 operation: 'create',
@@ -165,8 +175,46 @@ export class BaseGraphManager {
             throw new Error(`Entity not found: ${entityType}/${entityId}`);
         }
         const document = this.markdownService.parseFile(entityPath);
-        // Validate frontmatter
-        const validatedData = this.validateEntity(entityType, document.frontmatter);
+        // Validate frontmatter — fall back to base schema, then raw data
+        // so app-defined entity types missing base fields are still readable
+        let validatedData;
+        try {
+            validatedData = this.validateEntity(entityType, document.frontmatter);
+        }
+        catch {
+            try {
+                validatedData = this.validateEntity(entityType, document.frontmatter, true);
+            }
+            catch {
+                // Last resort: use raw frontmatter (e.g. app-defined types without created_by)
+                validatedData = document.frontmatter;
+            }
+        }
+        return {
+            id: entityId,
+            path: entityPath,
+            entityType,
+            data: validatedData,
+            document,
+        };
+    }
+    /**
+     * Get an entity by ID only (type resolved from frontmatter)
+     */
+    getById(entityId) {
+        const entityPath = this.getEntityPath('', entityId);
+        if (!existsSync(entityPath)) {
+            throw new Error(`Entity not found: ${entityId}`);
+        }
+        const document = this.markdownService.parseFile(entityPath);
+        const entityType = document.frontmatter.entity_type ?? this.repoName;
+        let validatedData;
+        try {
+            validatedData = this.validateEntity(entityType, document.frontmatter);
+        }
+        catch {
+            validatedData = this.validateEntity(entityType, document.frontmatter, true);
+        }
         return {
             id: entityId,
             path: entityPath,
@@ -175,11 +223,98 @@ export class BaseGraphManager {
             document,
         };
     }
+    /**
+     * List files and folders alongside main.md in an entity folder.
+     * Returns a recursive tree. Collection-mode entities return [].
+     */
+    listEntityContents(entityType, entityId) {
+        const entityPath = this.getEntityPath(entityType, entityId);
+        if (!existsSync(entityPath))
+            return [];
+        // Collection-mode entities (flat .md files) have no sub-content
+        if (!entityPath.endsWith('main.md'))
+            return [];
+        const entityDir = dirname(entityPath);
+        const readDir = (dirPath) => {
+            const entries = [];
+            for (const name of readdirSync(dirPath)) {
+                if (name.startsWith('.') || name === 'main.md')
+                    continue;
+                const fullPath = join(dirPath, name);
+                try {
+                    const stat = statSync(fullPath);
+                    if (stat.isDirectory()) {
+                        const children = readDir(fullPath);
+                        // Count all files recursively in this directory
+                        const countFiles = (items) => items.reduce((n, item) => n + (item.type === 'file' ? 1 : countFiles(item.children ?? [])), 0);
+                        entries.push({ name, type: 'directory', count: countFiles(children), children });
+                    }
+                    else {
+                        entries.push({ name, type: 'file' });
+                    }
+                }
+                catch { /* skip unreadable */ }
+            }
+            return entries.sort((a, b) => {
+                // Directories first, then alphabetical
+                if (a.type !== b.type)
+                    return a.type === 'directory' ? -1 : 1;
+                return a.name.localeCompare(b.name);
+            });
+        };
+        return readDir(entityDir);
+    }
+    /**
+     * Read a sub-content file from within an entity folder.
+     * Returns the raw file content as a string, or null if not found.
+     */
+    readEntityContent(entityType, entityId, filePath) {
+        const entityPath = this.getEntityPath(entityType, entityId);
+        if (!existsSync(entityPath) || !entityPath.endsWith('main.md'))
+            return null;
+        const entityDir = dirname(entityPath);
+        const resolved = join(entityDir, filePath);
+        // Prevent path traversal
+        if (!resolved.startsWith(entityDir))
+            return null;
+        if (!existsSync(resolved))
+            return null;
+        try {
+            const stat = statSync(resolved);
+            if (!stat.isFile())
+                return null;
+            return readFileSync(resolved, 'utf-8');
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Write a sub-content file inside an entity folder.
+     * Creates parent directories as needed. Prevents path traversal.
+     */
+    writeEntityContent(entityType, entityId, filePath, content) {
+        const entityPath = this.getEntityPath(entityType, entityId);
+        if (!existsSync(entityPath) || !entityPath.endsWith('main.md')) {
+            throw new Error(`Entity not found or is collection-mode: ${entityType}/${entityId}`);
+        }
+        const entityDir = dirname(entityPath);
+        const resolved = join(entityDir, filePath);
+        // Prevent path traversal
+        if (!resolved.startsWith(entityDir)) {
+            throw new Error('Invalid file path: traversal not allowed');
+        }
+        const dir = dirname(resolved);
+        if (!existsSync(dir)) {
+            mkdirSync(dir, { recursive: true });
+        }
+        writeFileSync(resolved, content, 'utf-8');
+    }
     /**
      * Update an existing entity
      */
     async update(options) {
-        const { entityType, entityId, data, content, commitMessage } = options;
+        const { entityType, entityId, data, content, commitMessage, skipCommit } = options;
         // Get existing entity
         const existing = this.get(entityType, entityId);
         // Merge data (if provided) and add update metadata
@@ -192,8 +327,33 @@ export class BaseGraphManager {
                 updated_by: this.gitUser.id,
             }
             : existing.data;
-        // Validate merged data
-        const validatedData = this.validateEntity(entityType, mergedData);
+        // Validate merged data — use lenient validation when no new data is provided,
+        // since the existing entity may have been stored without all required fields.
+        // Raw fallback coerces Date objects to YYYY-MM-DD so gray-matter doesn't
+        // serialize them as full ISO timestamps.
+        let validatedData;
+        if (data) {
+            validatedData = this.validateEntity(entityType, mergedData);
+        }
+        else {
+            try {
+                validatedData = this.validateEntity(entityType, mergedData);
+            }
+            catch {
+                try {
+                    validatedData = this.validateEntity(entityType, mergedData, true);
+                }
+                catch {
+                    const coerced = Object.fromEntries(Object.entries(mergedData).map(([k, v]) => {
+                        if (v !== null && typeof v === 'object' && Object.prototype.toString.call(v) === '[object Date]') {
+                            return [k, v.toISOString().split('T')[0]];
+                        }
+                        return [k, v];
+                    }));
+                    validatedData = coerced;
+                }
+            }
+        }
         // Merge content (if provided)
         const mergedContent = content !== undefined ? content : existing.document.content;
         // Write updated markdown file
@@ -203,7 +363,7 @@ export class BaseGraphManager {
             content: mergedContent,
             wikilinks: this.markdownService.extractWikilinks(mergedContent, validatedData),
         });
-        if (this.gitService) {
+        if (this.gitService && !skipCommit) {
             const transaction = this.gitService.createTransaction();
             transaction.add({
                 operation: 'update',
@@ -226,9 +386,14 @@ export class BaseGraphManager {
     async delete(entityType, entityId, commitMessage) {
         // Verify entity exists
         const existing = this.get(entityType, entityId);
-        // Delete the file
-        const { unlinkSync } = await import('fs');
-        unlinkSync(existing.path);
+        // Delete the entity directory (derive from resolved path, not hardcoded layout)
+        const { rmSync } = await import('fs');
+        const entityDir = existing.path.endsWith('.md') && !existing.path.endsWith(`${entityId}.md`)
+            ? dirname(existing.path) // folder-mode: main.md → parent dir
+            : existing.path.endsWith(`${entityId}.md`)
+                ? existing.path // collection-mode: delete the single file
+                : join(this.repoPath, entityId);
+        rmSync(entityDir, { recursive: true, force: true });
         if (this.gitService) {
             const transaction = this.gitService.createTransaction();
             transaction.add({
@@ -247,73 +412,145 @@ export class BaseGraphManager {
      * List entities of a given type
      */
     list(entityType, limit) {
+        if (!existsSync(this.repoPath))
+            return [];
         const results = [];
-        // If entityType specified, search only that folder
-        if (entityType) {
-            const folder = this.getEntityFolder(entityType);
-            if (!existsSync(folder)) {
-                return [];
+        // Lenient validation for list(): try strict schema first, fall back to
+        // EntityBaseSchema, then raw data so app-defined types are still discovered.
+        const validateLenient = (type, data) => {
+            try {
+                return this.validateEntity(type, data);
             }
-            const files = readdirSync(folder)
-                .filter(f => f.endsWith('.md'))
-                .map(f => {
-                const entityId = basename(f, '.md');
+            catch {
                 try {
-                    return this.get(entityType, entityId);
+                    return this.validateEntity(type, { ...data, entity_type: type }, true);
                 }
                 catch {
-                    return null;
+                    return data;
                 }
-            })
-                .filter((e) => e !== null);
-            results.push(...files);
-        }
-        else {
-            // Search all entity type folders — registered types first, then any
-            // additional subdirectories present in the repo (for custom entity types).
-            // When a schemaRegistry is present, its custom extension types are also included.
-            const registeredTypes = new Set([
-                ...Object.keys(ENTITY_SCHEMAS),
-                ...(this.schemaRegistry?.getAllTypes() ?? []),
-            ]);
-            // Collect: all registered types + any actual subdirectories in the repo
-            const actualSubdirs = existsSync(this.repoPath)
-                ? readdirSync(this.repoPath).filter(d => {
-                    if (d.startsWith('.'))
-                        return false;
-                    try {
-                        return statSync(join(this.repoPath, d)).isDirectory();
-                    }
-                    catch {
-                        return false;
+            }
+        };
+        const seenIds = new Set();
+        // Helper to process a directory entry as a potential entity
+        const tryAddEntity = (dirPath, entry) => {
+            const mainPath = join(dirPath, entry, 'main.md');
+            if (!existsSync(mainPath))
+                return;
+            if (seenIds.has(entry))
+                return;
+            try {
+                const doc = this.markdownService.parseFile(mainPath);
+                const fmType = doc.frontmatter.entity_type;
+                const resolvedType = entityType ?? fmType ?? this.repoName;
+                if (entityType && fmType && fmType !== entityType)
+                    return;
+                const validatedData = validateLenient(resolvedType, doc.frontmatter);
+                // Count all sub-content files recursively (excluding main.md and dotfiles)
+                const entityDir = join(dirPath, entry);
+                const countFiles = (dir) => {
+                    let n = 0;
+                    for (const f of readdirSync(dir)) {
+                        if (f.startsWith('.'))
+                            continue;
+                        if (dir === entityDir && f === 'main.md')
+                            continue;
+                        const full = join(dir, f);
+                        try {
+                            if (statSync(full).isDirectory())
+                                n += countFiles(full);
+                            else
+                                n++;
+                        }
+                        catch { /* skip */ }
                     }
-                })
-                : [];
-            const allTypes = [...new Set([...registeredTypes, ...actualSubdirs])];
-            for (const type of allTypes) {
-                const folder = this.getEntityFolder(type);
-                if (!existsSync(folder)) {
+                    return n;
+                };
+                const fileCount = countFiles(entityDir);
+                seenIds.add(entry);
+                results.push({
+                    id: entry,
+                    path: mainPath,
+                    entityType: resolvedType,
+                    data: validatedData,
+                    document: doc,
+                    hasSubContent: fileCount > 0,
+                    subContentCount: fileCount,
+                });
+            }
+            catch { /* skip invalid */ }
+        };
+        for (const entry of readdirSync(this.repoPath)) {
+            if (entry.startsWith('.'))
+                continue;
+            const fullPath = join(this.repoPath, entry);
+            try {
+                if (!statSync(fullPath).isDirectory())
                     continue;
-                }
-                const files = readdirSync(folder)
-                    .filter(f => f.endsWith('.md'))
-                    .map(f => {
-                    const entityId = basename(f, '.md');
-                    try {
-                        return this.get(type, entityId);
-                    }
-                    catch {
-                        return null;
+            }
+            catch {
+                continue;
+            }
+            const mainPath = join(fullPath, 'main.md');
+            if (existsSync(mainPath)) {
+                // Direct entity folder (e.g. projects/acme-corp/main.md)
+                tryAddEntity(this.repoPath, entry);
+            }
+            else {
+                // Subfolder containing entities (e.g. projects/deck/ containing sample-deck/main.md)
+                try {
+                    for (const sub of readdirSync(fullPath)) {
+                        if (sub.startsWith('.'))
+                            continue;
+                        try {
+                            if (!statSync(join(fullPath, sub)).isDirectory())
+                                continue;
+                        }
+                        catch {
+                            continue;
+                        }
+                        tryAddEntity(fullPath, sub);
                     }
-                })
-                    .filter((e) => e !== null);
-                results.push(...files);
+                }
+                catch { /* skip unreadable */ }
             }
         }
-        // Apply limit
-        if (limit && limit > 0) {
-            return results.slice(0, limit);
+        // Also discover collection-mode entities (flat .md files at repo root)
+        for (const entry of readdirSync(this.repoPath)) {
+            if (entry.startsWith('.'))
+                continue;
+            if (!entry.endsWith('.md'))
+                continue;
+            const fullPath = join(this.repoPath, entry);
+            try {
+                if (!statSync(fullPath).isFile())
+                    continue;
+            }
+            catch {
+                continue;
+            }
+            const id = entry.replace(/\.md$/, '');
+            if (seenIds.has(id))
+                continue; // folder-mode entity takes precedence
+            try {
+                const doc = this.markdownService.parseFile(fullPath);
+                const fmType = doc.frontmatter.entity_type;
+                const resolvedType = entityType ?? fmType ?? this.repoName;
+                if (entityType && fmType && fmType !== entityType)
+                    continue;
+                const validatedData = validateLenient(resolvedType, doc.frontmatter);
+                seenIds.add(id);
+                results.push({
+                    id,
+                    path: fullPath,
+                    entityType: resolvedType,
+                    data: validatedData,
+                    document: doc,
+                });
+            }
+            catch { /* skip invalid */ }
         }
+        if (limit && limit > 0)
+            return results.slice(0, limit);
         return results;
     }
     /**
@@ -435,10 +672,11 @@ export class BaseGraphManager {
                 // Determine entity type from path
                 const relPath = found.replace(this.repoPath + '/', '');
                 const parts = relPath.split('/');
-                if (parts.length >= 2) {
-                    const linkedType = parts[0];
-                    const linkedId = basename(parts[1], '.md');
+                if (parts.length >= 2 && parts[parts.length - 1] === 'main.md') {
+                    const linkedId = parts[0];
                     try {
+                        const doc = this.markdownService.parseFile(found);
+                        const linkedType = doc.frontmatter.entity_type ?? this.repoName;
                         const linkedEntity = this.get(linkedType, linkedId);
                         related.push(linkedEntity);
                     }
@@ -460,10 +698,11 @@ export class BaseGraphManager {
             // Determine entity type from path
             const relPath = file.replace(this.repoPath + '/', '');
             const parts = relPath.split('/');
-            if (parts.length >= 2) {
-                const linkedType = parts[0];
-                const linkedId = basename(parts[1], '.md');
+            if (parts.length >= 2 && parts[parts.length - 1] === 'main.md') {
+                const linkedId = parts[0];
                 try {
+                    const doc = this.markdownService.parseFile(file);
+                    const linkedType = doc.frontmatter.entity_type ?? this.repoName;
                     const linkedEntity = this.get(linkedType, linkedId);
                     results.push(linkedEntity);
                 }
@@ -494,11 +733,12 @@ export class BaseGraphManager {
                 this.markdownService.updateWikilinks(file, oldId, newId);
                 const relPath = file.replace(this.repoPath + '/', '');
                 const parts = relPath.split('/');
-                if (parts.length >= 2) {
+                if (parts.length >= 2 && parts[parts.length - 1] === 'main.md') {
+                    const doc = this.markdownService.parseFile(file);
                     transaction.add({
                         operation: 'update',
-                        entityType: parts[0],
-                        entityId: basename(parts[1], '.md'),
+                        entityType: doc.frontmatter.entity_type ?? parts[0],
+                        entityId: parts[0],
                         filePath: file,
                     });
                 }
@@ -516,38 +756,38 @@ export class BaseGraphManager {
     /**
      * Upload asset for an entity
      */
-    async uploadAsset(repoName, entityType, entityId, filename, buffer) {
+    async uploadAsset(entityType, entityId, filename, buffer, keyPrefix) {
         if (!this.assetService) {
             throw new Error('Asset service is not configured for this graph');
         }
-        return await this.assetService.upload(repoName, entityType, entityId, filename, buffer);
+        return await this.assetService.upload(entityType, entityId, filename, buffer, keyPrefix);
     }
     /**
      * List assets for an entity
      */
-    async listAssets(repoName, entityType, entityId) {
+    async listAssets(entityType, entityId, keyPrefix) {
         if (!this.assetService) {
             throw new Error('Asset service is not configured for this graph');
         }
-        return await this.assetService.list(repoName, entityType, entityId);
+        return await this.assetService.list(entityType, entityId, keyPrefix);
     }
     /**
      * Delete an asset
      */
-    async deleteAsset(repoName, entityType, entityId, filename) {
+    async deleteAsset(entityType, entityId, filename, keyPrefix) {
         if (!this.assetService) {
             throw new Error('Asset service is not configured for this graph');
         }
-        return await this.assetService.delete(repoName, entityType, entityId, filename);
+        return await this.assetService.delete(entityType, entityId, filename, keyPrefix);
     }
     /**
      * Delete all assets for an entity (when entity is deleted)
      */
-    async deleteAllAssets(repoName, entityType, entityId) {
+    async deleteAllAssets(entityType, entityId, keyPrefix) {
         if (!this.assetService) {
             throw new Error('Asset service is not configured for this graph');
         }
-        return await this.assetService.deleteAll(repoName, entityType, entityId);
+        return await this.assetService.deleteAll(entityType, entityId, keyPrefix);
     }
     /**
      * Get dataset rows, optionally filtered
@@ -587,7 +827,7 @@ export class BaseGraphManager {
         // Stringify back to CSV
         const newContent = this.csvService.stringifyRows(allRows, schema);
         // Update the entity file (update content, bump updated_at)
-        const entityPath = this.getEntityPathPublic('dataset', entityId);
+        const entityPath = this.getEntityPath('dataset', entityId);
         const updatedData = {
             ...entity.data,
             updated_at: new Date().toISOString(),