studiograph 1.3.3 → 1.3.4

package/dist/agent/tools/sync-tools.js

@@ -1,10 +1,11 @@
 /**
- * Sync & Enrich Tools for Studiograph Agent
+ * Sync & Workspace Tools for Studiograph Agent
  *
- * These tools allow the agent to orchestrate sync and enrich operations:
- * - Run the sync pipeline, apply staging, commit
- * - Run LLM enrichment on entities
- * - Manage source configs
+ * These tools allow the agent to:
+ * - Sync external data into collections (collection-scoped sync)
+ * - Refresh individual synced entities from their source
+ * - Manage workspace git operations (commit, push, pull, status)
+ * - Inspect MCP connectors for sync configuration
  */
 import { Type as T } from '@sinclair/typebox';
 import { spawnSync } from 'child_process';
@@ -12,13 +13,13 @@ import { gitPush, gitPull } from '../../utils/git.js';
 import { existsSync } from 'fs';
 import { join } from 'path';
 import { homedir } from 'os';
-import { runSync } from '../../services/sync/sync-runner.js';
-import { commitStaging, applyStaging, commitRepos } from '../../services/sync/commit.js';
-import { runEnrichment } from '../../services/sync/enrichment.js';
+import { commitRepos } from '../../services/sync/commit.js';
 import { StagingManager } from '../../services/sync/staging.js';
 import { SourceConfigManager } from '../../services/sync/source-config.js';
 import { ConnectorManager } from '../../mcp/connector-manager.js';
 import { SOURCE_DEFINITIONS } from '../../services/sync/source-definitions/definitions.js';
+import { syncRepo, syncWorkspaceRule } from '../../services/sync/collection-sync.js';
+import { refreshEntity } from '../../services/sync/entity-refresh.js';
 import { wrapToolResult } from './graph-tools.js';
 import { VectorService } from '../../services/vector-service.js';
 import { BaseGraphManager } from '../../core/graph.js';
@@ -29,30 +30,36 @@ import { loadUserConfig } from '../../core/user-config.js';
 export function createSyncTools(workspacePath, workspaceConfig, registryRef) {
     const getRegistry = () => registryRef?.registry ?? null;
     return [
-        // Run sync pipeline
+        // ── Collection-scoped sync tools ─────────────────────────────────────────
+        // Sync a collection — runs collection-level rules AND workspace-level rules targeting it
         {
-            name: 'sync_run',
-            label: 'Run Sync',
-            description: 'Run the sync pipeline to extract data from external sources into staging. Review with sync_status before committing.',
+            name: 'sync_collection',
+            label: 'Sync Collection',
+            description: 'Pull new entities from external sources into a collection. Runs both collection-level sync rules and any workspace-level sync rules that target this collection. Creates new entities only — existing entities (matched by source_ref) are never updated. Always confirm with the user before calling.',
             parameters: T.Object({
-                sources: T.Optional(T.Array(T.String(), { description: 'Source names to sync (default: all enabled)' })),
-                dryRun: T.Optional(T.Boolean({ description: 'Preview without writing staging files' })),
-                incremental: T.Optional(T.Boolean({ description: 'Only sync records updated since last sync' })),
+                collection: T.String({ description: 'Collection name to sync' }),
             }),
             execute: async (toolCallId, params) => {
                 try {
                     const progress = [];
-                    const result = await runSync({
+                    const { collectionResults, workspaceResults } = await syncRepo({
                         workspacePath,
-                        sources: params.sources,
-                        dryRun: params.dryRun,
-                        incremental: params.incremental,
+                        collectionName: params.collection,
+                        workspaceConfig,
                         schemaExtensions: workspaceConfig.schema_extensions,
-                    }, (msg) => progress.push(msg));
+                        onProgress: (msg) => progress.push(msg),
+                    });
+                    const totalCreated = collectionResults.reduce((s, r) => s + r.created + r.derived, 0) +
+                        workspaceResults.reduce((s, r) => r.targets.reduce((ts, t) => ts + t.created + t.derived, 0) + s, 0);
+                    const totalSkipped = collectionResults.reduce((s, r) => s + r.skipped, 0) +
+                        workspaceResults.reduce((s, r) => r.targets.reduce((ts, t) => ts + t.skipped, 0) + s, 0);
                     return wrapToolResult({
                         success: true,
-                        result,
+                        collectionResults,
+                        workspaceResults,
+                        summary: `Created ${totalCreated}, skipped ${totalSkipped}`,
                         progress: progress.join('\n'),
+                        nextSteps: totalCreated > 0 ? 'Commit changes with workspace_commit' : undefined,
                     });
                 }
                 catch (error) {
@@ -63,20 +70,37 @@ export function createSyncTools(workspacePath, workspaceConfig, registryRef) {
                 }
             },
         },
-        // Check staging status
+        // Run all workspace-level sync rules
         {
-            name: 'sync_status',
-            label: 'Sync Status',
-            description: 'Check what is currently in staging (pending review). Shows counts of creates, updates, and skips by entity type.',
+            name: 'sync_workspace',
+            label: 'Sync Workspace',
+            description: 'Run all workspace-level sync rules defined in workspace.json. Each rule fetches from its source once and distributes records to multiple target collections. Use this instead of sync_collection when multiple collections share the same source. Always confirm with the user before calling.',
             parameters: T.Object({}),
-            execute: async (toolCallId, params) => {
+            execute: async (toolCallId, _params) => {
                 try {
-                    const staging = new StagingManager(workspacePath);
-                    const summary = staging.getSummary();
+                    const rules = workspaceConfig.sync ?? [];
+                    if (rules.length === 0) {
+                        return wrapToolResult({ success: true, results: [], summary: 'No workspace-level sync rules configured' });
+                    }
+                    const progress = [];
+                    const results = [];
+                    for (const rule of rules) {
+                        const result = await syncWorkspaceRule({
+                            workspacePath,
+                            rule,
+                            workspaceConfig,
+                            schemaExtensions: workspaceConfig.schema_extensions,
+                            onProgress: (msg) => progress.push(msg),
+                        });
+                        results.push(result);
+                    }
+                    const totalCreated = results.reduce((s, r) => r.targets.reduce((ts, t) => ts + t.created + t.derived, 0) + s, 0);
                     return wrapToolResult({
                         success: true,
-                        hasStaging: summary.total > 0,
-                        summary,
+                        results,
+                        summary: `Ran ${rules.length} workspace rule(s), created ${totalCreated} entities`,
+                        progress: progress.join('\n'),
+                        nextSteps: totalCreated > 0 ? 'Commit changes with workspace_commit' : undefined,
                     });
                 }
                 catch (error) {
@@ -87,23 +111,80 @@ export function createSyncTools(workspacePath, workspaceConfig, registryRef) {
                 }
             },
         },
-        // Apply staging to entity files (no git)
+        // Refresh a single synced entity
         {
-            name: 'sync_apply',
-            label: 'Apply Staging',
-            description: 'Apply staging files to the knowledge graph (writes entity files, does not git commit). Always confirm with the user before calling this.',
+            name: 'sync_refresh',
+            label: 'Refresh Synced Entity',
+            description: 'Refresh a single entity from its external source. Shows what fields have changed. Does NOT apply changes automatically — use the apply parameter or tell the user about the changes.',
             parameters: T.Object({
-                force: T.Optional(T.Boolean({ description: 'Force apply even with uncommitted changes in repos' })),
+                entity_id: T.String({ description: 'Entity ID to refresh' }),
+                collection: T.Optional(T.String({ description: 'Collection name (auto-detected if omitted)' })),
+                entity_type: T.Optional(T.String({ description: 'Entity type (auto-detected if omitted)' })),
+                apply: T.Optional(T.Boolean({ description: 'Apply changes immediately (default: false, just show diff)' })),
             }),
             execute: async (toolCallId, params) => {
                 try {
-                    const progress = [];
-                    const result = await applyStaging(workspacePath, (msg) => progress.push(msg), { force: params.force });
+                    let { collection, entity_type, entity_id, apply } = params;
+                    // Auto-detect collection and entity_type if not provided
+                    if (!collection || !entity_type) {
+                        for (const repo of workspaceConfig.repos) {
+                            try {
+                                const repoPath = join(workspacePath, repo.path);
+                                if (!existsSync(repoPath))
+                                    continue;
+                                const graph = new BaseGraphManager({
+                                    repoPath,
+                                    repoName: repo.name,
+                                    gitUser: { id: 'agent', name: 'Agent', email: 'agent@studiograph.local' },
+                                });
+                                const entities = graph.list();
+                                const match = entities.find(e => e.data?.entity_id === entity_id);
+                                if (match) {
+                                    collection = repo.name;
+                                    entity_type = match.entityType;
+                                    break;
+                                }
+                            }
+                            catch { /* skip */ }
+                        }
+                    }
+                    if (!collection || !entity_type) {
+                        return wrapToolResult({
+                            success: false,
+                            error: `Entity "${entity_id}" not found in any collection`,
+                        });
+                    }
+                    const result = await refreshEntity({
+                        workspacePath,
+                        collectionName: collection,
+                        entityType: entity_type,
+                        entityId: entity_id,
+                        workspaceConfig: workspaceConfig,
+                        schemaExtensions: workspaceConfig.schema_extensions,
+                        apply: apply ?? false,
+                    });
+                    if (result.diffs.length === 0 && !result.body_changed) {
+                        return wrapToolResult({
+                            success: true,
+                            message: 'Entity is already up to date with the source.',
+                            entity_id,
+                            source_ref: result.source_ref,
+                        });
+                    }
                     return wrapToolResult({
                         success: true,
-                        result: { created: result.created, updated: result.updated, skipped: result.skipped },
-                        progress: progress.join('\n'),
-                        nextSteps: 'Review changes with workspace_review, then commit with workspace_commit or studiograph commit',
+                        entity_id,
+                        source_ref: result.source_ref,
+                        applied: result.applied,
+                        changes: result.diffs.map(d => ({
+                            field: d.field,
+                            from: d.old_value,
+                            to: d.new_value,
+                        })),
+                        body_changed: result.body_changed,
+                        nextSteps: result.applied
+                            ? 'Changes applied. Commit with workspace_commit.'
+                            : 'Review the changes above and call sync_refresh with apply: true to apply them.',
                     });
                 }
                 catch (error) {
@@ -114,21 +195,72 @@ export function createSyncTools(workspacePath, workspaceConfig, registryRef) {
                 }
             },
         },
-        // Git commit across workspace repos
+        // Show sync status across collections
         {
-            name: 'workspace_commit',
-            label: 'Commit Changes',
-            description: 'Git commit entity changes across all workspace repos. Works after sync_apply, enrich_run, or manual edits. Always confirm with the user before calling this.',
+            name: 'sync_status',
+            label: 'Sync Status',
+            description: 'Show which collections have sync rules configured, their sources, and last sync time. Use this when users ask about sync configuration or what has been synced.',
             parameters: T.Object({
-                messagePrefix: T.Optional(T.String({ description: 'Commit message prefix (default: "update")' })),
+                collection: T.Optional(T.String({ description: 'Filter to a specific collection' })),
             }),
             execute: async (toolCallId, params) => {
                 try {
-                    const progress = [];
-                    await commitRepos(workspacePath, (msg) => progress.push(msg), { messagePrefix: params.messagePrefix ?? 'update' });
+                    const repos = params.collection
+                        ? workspaceConfig.repos.filter(r => r.name === params.collection)
+                        : workspaceConfig.repos;
+                    const collections = [];
+                    for (const repo of repos) {
+                        const syncRules = repo.sync;
+                        if (!syncRules || syncRules.length === 0)
+                            continue;
+                        let syncedCount = 0;
+                        let lastSync = null;
+                        try {
+                            const repoPath = join(workspacePath, repo.path);
+                            if (!existsSync(repoPath))
+                                continue;
+                            const graph = new BaseGraphManager({
+                                repoPath,
+                                repoName: repo.name,
+                                gitUser: { id: 'agent', name: 'Agent', email: 'agent@studiograph.local' },
+                            });
+                            for (const entity of graph.list()) {
+                                const syncedAt = entity.data?.synced_at;
+                                if (syncedAt) {
+                                    syncedCount++;
+                                    if (!lastSync || syncedAt > lastSync)
+                                        lastSync = syncedAt;
+                                }
+                            }
+                        }
+                        catch { /* skip */ }
+                        collections.push({ name: repo.name, syncRules, syncedCount, lastSync });
+                    }
+                    // Also list available source definitions
+                    const configMgr = new SourceConfigManager(workspacePath);
+                    const sources = configMgr.getAll().map(s => s.name);
+                    // Check for new-format too
+                    const { readdirSync } = await import('fs');
+                    const sourcesDir = join(workspacePath, '.studiograph', 'sources');
+                    if (existsSync(sourcesDir)) {
+                        for (const file of readdirSync(sourcesDir)) {
+                            if (!file.endsWith('.json'))
+                                continue;
+                            const name = file.replace('.json', '');
+                            if (!sources.includes(name)) {
+                                const def = configMgr.getDefinition(name);
+                                if (def)
+                                    sources.push(name);
+                            }
+                        }
+                    }
                     return wrapToolResult({
                         success: true,
-                        progress: progress.join('\n'),
+                        collections,
+                        availableSources: sources,
+                        message: collections.length > 0
+                            ? `${collections.length} collection(s) with sync rules`
+                            : 'No collections have sync rules configured.',
                     });
                 }
                 catch (error) {
@@ -139,22 +271,20 @@ export function createSyncTools(workspacePath, workspaceConfig, registryRef) {
                 }
             },
         },
-        // Commit staging to graph (deprecated — kept for backwards compatibility)
+        // Git commit across workspace repos
         {
-            name: 'sync_commit',
-            label: 'Commit Staging (deprecated)',
-            description: 'DEPRECATED: Use sync_apply then workspace_commit instead. Commits staging files and auto git-commits.',
+            name: 'workspace_commit',
+            label: 'Commit Changes',
+            description: 'Git commit entity changes across all workspace repos. Works after sync_apply, enrich_run, or manual edits. Always confirm with the user before calling this.',
             parameters: T.Object({
-                force: T.Optional(T.Boolean({ description: 'Force commit even with uncommitted changes in repos' })),
+                messagePrefix: T.Optional(T.String({ description: 'Commit message prefix (default: "update")' })),
             }),
             execute: async (toolCallId, params) => {
                 try {
                     const progress = [];
-                    progress.push('Note: sync_commit is deprecated. Use sync_apply then workspace_commit instead.');
-                    const result = await commitStaging(workspacePath, (msg) => progress.push(msg), { force: params.force, noGit: false });
+                    await commitRepos(workspacePath, (msg) => progress.push(msg), { messagePrefix: params.messagePrefix ?? 'update' });
                     return wrapToolResult({
                         success: true,
-                        result,
                         progress: progress.join('\n'),
                     });
                 }
@@ -166,31 +296,6 @@ export function createSyncTools(workspacePath, workspaceConfig, registryRef) {
                 }
             },
         },
-        // Abort staging
-        {
-            name: 'sync_abort',
-            label: 'Abort Staging',
-            description: 'Discard all staging files without committing.',
-            parameters: T.Object({}),
-            execute: async (toolCallId, params) => {
-                try {
-                    const staging = new StagingManager(workspacePath);
-                    const hadStaging = staging.hasStaging();
-                    staging.clear();
-                    return wrapToolResult({
-                        success: true,
-                        hadStaging,
-                        message: hadStaging ? 'Staging cleared.' : 'No staging files to clear.',
-                    });
-                }
-                catch (error) {
-                    return wrapToolResult({
-                        success: false,
-                        error: error instanceof Error ? error.message : 'Unknown error',
-                    });
-                }
-            },
-        },
         // List sources and connectors
         {
             name: 'sync_list_sources',
@@ -202,9 +307,9 @@ export function createSyncTools(workspacePath, workspaceConfig, registryRef) {
                     const configMgr = new SourceConfigManager(workspacePath);
                     const sources = configMgr.getAll();
                     // Load connectors but strip sensitive fields
-                    const connectors = ConnectorManager.loadConfigs().map(c => ({
+                    const connectors = ConnectorManager.loadConfigs(workspacePath).map(c => ({
                         name: c.name,
-                        type: c.type,
+                        url: c.url,
                         description: c.description,
                     }));
                     const availableDefinitions = Object.keys(SOURCE_DEFINITIONS);
@@ -227,11 +332,10 @@ export function createSyncTools(workspacePath, workspaceConfig, registryRef) {
         {
             name: 'sync_setup_source',
             label: 'Setup Sync Source',
-            description: 'Add or reconfigure a sync source. Uses default mappings from source definitions when available.',
+            description: 'Add or reconfigure a sync source using built-in source definitions. The collection assignment is managed via sync rules in collection settings, not here.',
             parameters: T.Object({
-                name: T.String({ description: 'Source name (lowercase, alphanumeric, hyphens/underscores)' }),
+                name: T.String({ description: 'Source name — must match a built-in source definition (e.g. "linear", "pipedrive", "granola", "asana")' }),
                 connector: T.Optional(T.String({ description: 'Connector name (defaults to source name)' })),
-                entity_mappings: T.Optional(T.Array(T.Object({}, { additionalProperties: true }), { description: 'Entity mappings (overrides defaults from source definition)' })),
                 force: T.Optional(T.Boolean({ description: 'Overwrite existing source config' })),
             }),
             execute: async (toolCallId, params) => {
@@ -247,7 +351,7 @@ export function createSyncTools(workspacePath, workspaceConfig, registryRef) {
                         });
                     }
                     // Verify connector exists
-                    const connectors = ConnectorManager.loadConfigs();
+                    const connectors = ConnectorManager.loadConfigs(workspacePath);
                     const connector = connectors.find(c => c.name === connectorName);
                     if (!connector) {
                         return wrapToolResult({
@@ -256,30 +360,27 @@ export function createSyncTools(workspacePath, workspaceConfig, registryRef) {
                             availableConnectors: connectors.map(c => c.name),
                         });
                     }
-                    // Build source config
+                    // Built-in definition required — don't allow custom configs
                     const definition = SOURCE_DEFINITIONS[params.name] ?? SOURCE_DEFINITIONS[connectorName];
-                    const mappings = params.entity_mappings ?? definition?.defaultMappings ?? [];
-                    const config = {
-                        name: params.name,
-                        connector: connectorName,
-                        enabled: true,
-                        added_at: new Date().toISOString(),
-                        entity_mappings: mappings,
-                    };
-                    // Validate
-                    const validation = configMgr.validate(config);
-                    if (!validation.valid) {
+                    if (!definition || !definition.entities || Object.keys(definition.entities).length === 0) {
                         return wrapToolResult({
                             success: false,
-                            error: 'Invalid source config',
-                            validationErrors: validation.errors,
+                            error: `No built-in source definition for "${params.name}". Available sources: ${Object.keys(SOURCE_DEFINITIONS).join(', ')}`,
                         });
                     }
-                    configMgr.save(config);
+                    // Save as new-format source definition
+                    const savedDef = {
+                        ...definition,
+                        name: params.name,
+                        enabled: true,
+                        added_at: new Date().toISOString(),
+                    };
+                    configMgr.saveDefinition(params.name, savedDef);
+                    const entityTypes = Object.keys(definition.entities);
                     return wrapToolResult({
                         success: true,
-                        message: `Source "${params.name}" configured with ${mappings.length} entity mapping(s).`,
-                        config,
+                        message: `Source "${params.name}" configured with ${entityTypes.length} entity type(s): ${entityTypes.join(', ')}. Assign it to a collection via sync rules in collection settings.`,
+                        definition: savedDef,
                     });
                 }
                 catch (error) {
@@ -573,80 +674,6 @@ export function createSyncTools(workspacePath, workspaceConfig, registryRef) {
                 }
             },
         },
-        // ── Enrich tools ──────────────────────────────────────────────────────────
-        // Run enrichment
-        {
-            name: 'enrich_run',
-            label: 'Enrich Entities',
-            description: 'Enrich entities with cross-source context via LLM. Writes changes in-place (no staging). Review with workspace_review, then commit with workspace_commit. Always confirm with the user before calling this.',
-            parameters: T.Object({
-                entityTypes: T.Optional(T.Array(T.String(), { description: 'Only enrich specific entity types' })),
-                dryRun: T.Optional(T.Boolean({ description: 'Preview without writing changes' })),
-                minRefs: T.Optional(T.Number({ description: 'Minimum context references required (default: 2)' })),
-                maxCompleteness: T.Optional(T.Number({ description: 'Maximum completeness to qualify (default: 0.6)' })),
-            }),
-            execute: async (toolCallId, params) => {
-                try {
-                    const progress = [];
-                    const result = await runEnrichment({
-                        workspacePath,
-                        entityTypes: params.entityTypes,
-                        dryRun: params.dryRun,
-                        minRefs: params.minRefs,
-                        maxCompleteness: params.maxCompleteness,
-                        schemaExtensions: workspaceConfig.schema_extensions,
-                    }, (msg) => progress.push(msg));
-                    return wrapToolResult({
-                        success: true,
-                        result,
-                        progress: progress.join('\n'),
-                        nextSteps: result.enriched > 0 && !params.dryRun
-                            ? 'Review changes with workspace_review, then commit with workspace_commit or studiograph commit'
-                            : undefined,
-                    });
-                }
-                catch (error) {
-                    return wrapToolResult({
-                        success: false,
-                        error: error instanceof Error ? error.message : 'Unknown error',
-                    });
-                }
-            },
-        },
-        // Abort enrichment (revert uncommitted changes)
-        {
-            name: 'enrich_abort',
-            label: 'Abort Enrichment',
-            description: 'Discard uncommitted enrichment changes by reverting all entity files (git checkout + git clean).',
-            parameters: T.Object({}),
-            execute: async (toolCallId, _params) => {
-                try {
-                    const repos = workspaceConfig.repos ?? [];
-                    const reverted = [];
-                    for (const repo of repos) {
-                        const repoPath = join(workspacePath, repo.path);
-                        if (!existsSync(repoPath))
-                            continue;
-                        spawnSync('git', ['checkout', '--', '.'], { cwd: repoPath, encoding: 'utf-8' });
-                        spawnSync('git', ['clean', '-fd'], { cwd: repoPath, encoding: 'utf-8' });
-                        reverted.push(repo.name);
-                    }
-                    return wrapToolResult({
-                        success: true,
-                        reverted,
-                        message: reverted.length > 0
-                            ? `Reverted changes in: ${reverted.join(', ')}`
-                            : 'No changes to revert.',
-                    });
-                }
-                catch (error) {
-                    return wrapToolResult({
-                        success: false,
-                        error: String(error instanceof Error ? error.message : error),
-                    });
-                }
-            },
-        },
         // Reset workspace (discard uncommitted changes)
         {
             name: 'workspace_reset',