studiograph 1.3.3 → 1.3.4

package/dist/services/sync/collection-sync.js

@@ -0,0 +1,726 @@
+/**
+ * Collection-Scoped Sync Runner
+ *
+ * Given a collection with sync rules, connects to sources via MCP,
+ * extracts records, derives secondary entities, deduplicates against
+ * existing entities, and writes new entities to the collection.
+ *
+ * Sync-once semantics: existing entities (matched by source_ref) are
+ * never updated during a pull. Use entity-refresh for single-entity updates.
+ */
+import { existsSync } from 'fs';
+import { join } from 'path';
+import { SyncMCPClient } from './mcp-client.js';
+import { StructuredExtractor, slugify, resolveFieldPath, applyTransform } from './structured-extractor.js';
+import { FrontmatterExtractor } from './frontmatter-extractor.js';
+import { SyncStateManager } from './sync-state.js';
+import { SourceConfigManager } from './source-config.js';
+import { BaseGraphManager } from '../../core/graph.js';
+import { SchemaRegistry } from '../../core/schema-registry.js';
+import { ConnectorManager, KNOWN_CONNECTORS } from '../../mcp/connector-manager.js';
+import { ServerOAuthProvider } from '../../mcp/server-oauth-provider.js';
+import { RESTClient } from './rest-client.js';
+import { SOURCE_DEFINITIONS } from './source-definitions/definitions.js';
+/** Create a DataFetcher — REST or MCP depending on source definition's adapter field. */
+function createDataFetcher(connectorConfig, definition, opts) {
+    // REST adapter — direct HTTP, no MCP
+    if (definition?.adapter === 'rest' && definition.rest) {
+        const headers = {};
+        if (connectorConfig.headers) {
+            for (const [k, v] of Object.entries(connectorConfig.headers)) {
+                headers[k] = ConnectorManager.resolveEnvVar(v);
+            }
+        }
+        // Fall back to OAuth tokens if no manual headers
+        if (!headers['Authorization'] && opts?.userId && opts?.authService) {
+            const oauthData = opts.authService.getOAuthData(opts.userId, connectorConfig.name);
+            if (oauthData?.tokens?.access_token) {
+                headers['Authorization'] = `Bearer ${oauthData.tokens.access_token}`;
+            }
+        }
+        // Resolve query-param auth (e.g. api_token for Pipedrive)
+        const queryAuth = {};
+        if (definition.rest.query_auth) {
+            for (const [k, v] of Object.entries(definition.rest.query_auth)) {
+                queryAuth[k] = ConnectorManager.resolveEnvVar(v);
+            }
+        }
+        return new RESTClient(definition.rest.base_url, headers, definition.entities, queryAuth);
+    }
+    // MCP adapter (default)
+    const known = KNOWN_CONNECTORS[connectorConfig.name];
+    const auth = connectorConfig.auth ?? known?.auth;
+    if (auth === 'oauth' && opts?.userId && opts?.authService && opts?.serverUrl) {
+        const provider = new ServerOAuthProvider(connectorConfig.name, opts.userId, opts.authService, opts.serverUrl, known?.oauth);
+        return new SyncMCPClient(connectorConfig, provider);
+    }
+    if (auth === 'oauth') {
+        throw new Error(`OAuth authorization required for "${connectorConfig.name}". Please authorize in Settings > Connectors.`);
+    }
+    return new SyncMCPClient(connectorConfig);
+}
+/**
+ * Run sync for a collection. Processes each sync rule in the collection's config.
+ */
+export async function syncCollection(options) {
+    const { workspacePath, collectionName, workspaceConfig, schemaExtensions, onProgress, userId, authService, serverUrl } = options;
+    const log = onProgress ?? (() => { });
+    // Find collection config
+    const repoConfig = workspaceConfig.repos.find(r => r.name === collectionName);
+    if (!repoConfig) {
+        throw new Error(`Collection "${collectionName}" not found in workspace config`);
+    }
+    const syncRules = repoConfig.sync;
+    if (!syncRules || syncRules.length === 0) {
+        throw new Error(`Collection "${collectionName}" has no sync rules configured`);
+    }
+    const results = [];
+    for (const rule of syncRules) {
+        log(`\nSyncing from source: ${rule.source}`);
+        try {
+            const result = await syncOneRule({
+                workspacePath,
+                repoConfig,
+                rule,
+                workspaceConfig,
+                schemaExtensions,
+                onProgress,
+                userId,
+                authService,
+                serverUrl,
+            });
+            results.push(result);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            log(`  Error: ${msg}`);
+            results.push({
+                collection: collectionName,
+                source: rule.source,
+                created: 0,
+                skipped: 0,
+                derived: 0,
+                errors: [msg],
+            });
+        }
+    }
+    return results;
+}
+/**
+ * Run all sync rules that apply to a collection:
+ * - Collection-level rules (if the collection has its own sync array)
+ * - Workspace-level rules that include this collection as a target
+ *
+ * This is the preferred entry point for per-collection sync — it handles both
+ * rule types without callers needing to know the distinction.
+ */
+export async function syncRepo(options) {
+    const { workspacePath, collectionName, workspaceConfig, schemaExtensions, onProgress, userId, authService, serverUrl } = options;
+    const log = onProgress ?? (() => { });
+    const collectionResults = [];
+    const workspaceResults = [];
+    // Run collection-level rules (if any)
+    const repoConfig = workspaceConfig.repos.find(r => r.name === collectionName);
+    const collectionRules = repoConfig?.sync;
+    if (collectionRules && collectionRules.length > 0) {
+        const results = await syncCollection({ workspacePath, collectionName, workspaceConfig, schemaExtensions, onProgress, userId, authService, serverUrl });
+        collectionResults.push(...results);
+    }
+    // Run workspace-level rules that target this collection
+    const workspaceRules = workspaceConfig.sync ?? [];
+    for (const rule of workspaceRules) {
+        const matchingTargets = rule.targets.filter(t => t.repo === collectionName);
+        if (matchingTargets.length === 0)
+            continue;
+        log(`\nSyncing workspace rule: ${rule.source} → ${collectionName}`);
+        try {
+            const result = await syncWorkspaceRule({
+                workspacePath,
+                rule: { ...rule, targets: matchingTargets },
+                workspaceConfig,
+                schemaExtensions,
+                onProgress,
+                userId,
+                authService,
+                serverUrl,
+            });
+            workspaceResults.push(result);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            log(`  Error: ${msg}`);
+            workspaceResults.push({
+                source: rule.source,
+                targets: matchingTargets.map(t => ({ repo: t.repo, created: 0, skipped: 0, derived: 0, errors: [msg] })),
+            });
+        }
+    }
+    if (collectionResults.length === 0 && workspaceResults.length === 0) {
+        throw new Error(`Collection "${collectionName}" has no sync rules configured (collection-level or workspace-level)`);
+    }
+    return { collectionResults, workspaceResults };
+}
+/**
+ * Run one workspace-level sync rule: fetch from source once, distribute to all targets.
+ * Each target receives a filtered subset of the extracted records.
+ */
+export async function syncWorkspaceRule(options) {
+    const { workspacePath, rule, workspaceConfig, schemaExtensions, onProgress, userId, authService, serverUrl } = options;
+    const log = onProgress ?? (() => { });
+    const result = { source: rule.source, targets: [] };
+    const configMgr = new SourceConfigManager(workspacePath);
+    const definition = configMgr.getDefinition(rule.source);
+    const oldConfig = !definition ? configMgr.get(rule.source) : null;
+    if (!definition && !oldConfig) {
+        throw new Error(`Source "${rule.source}" not found in .studiograph/sources/`);
+    }
+    const connectorName = definition?.connector ?? oldConfig.connector;
+    const connectorConfigs = ConnectorManager.loadConfigs(workspacePath);
+    const connectorConfig = connectorConfigs.find(c => c.name === connectorName);
+    if (!connectorConfig) {
+        throw new Error(`Connector "${connectorName}" not configured. Run: studiograph connector add ${connectorName}`);
+    }
+    const client = createDataFetcher(connectorConfig, definition, { userId, authService, serverUrl });
+    try {
+        log(`  Connecting to ${connectorName}${definition?.adapter === 'rest' ? ' (REST)' : ''}...`);
+        try {
+            await client.connect();
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            throw new Error(`Failed to connect to connector "${connectorName}" at ${connectorConfig.url}. (${msg})`);
+        }
+        log(`  Available tools: ${client.getTools().map(t => t.name).join(', ')}`);
+        const schemaRegistry = new SchemaRegistry(schemaExtensions);
+        const syncState = new SyncStateManager(workspacePath);
+        // ── Extract ALL records once (no target filter at this stage) ──────────────
+        let allRecords;
+        const filterFieldDefs = definition?.filter_fields ?? SOURCE_DEFINITIONS[connectorName]?.filter_fields ?? [];
+        if (definition) {
+            const primaryType = rule.entity_type ?? findPrimaryType(definition);
+            const primaryConfig = definition.entities[primaryType];
+            if (!primaryConfig) {
+                throw new Error(`Entity type "${primaryType}" not found in source definition`);
+            }
+            // Extract without any filter — filtering happens per-target below
+            const mapping = sourceEntityConfigToMapping(primaryType, primaryConfig, rule.source, '_workspace_', undefined);
+            const oldStyleConfig = {
+                name: rule.source,
+                connector: definition.connector,
+                enabled: true,
+                added_at: new Date().toISOString(),
+                entity_mappings: [mapping],
+            };
+            log(`  Extracting ${primaryType} records...`);
+            if (primaryConfig.extraction_mode === 'frontmatter') {
+                const extractor = new FrontmatterExtractor(client, oldStyleConfig);
+                allRecords = await extractor.extract(mapping, onProgress);
+            }
+            else {
+                const extractor = new StructuredExtractor(client, oldStyleConfig, syncState);
+                allRecords = await extractor.extract(mapping, false, onProgress);
+            }
+        }
+        else {
+            // Old-format source
+            const mapping = { ...oldConfig.entity_mappings[0], target_repo: '_workspace_' };
+            const extractor = new StructuredExtractor(client, oldConfig, syncState);
+            allRecords = await extractor.extract(mapping, false, onProgress);
+        }
+        log(`  Extracted ${allRecords.length} records (total)`);
+        // ── Write to each target ──────────────────────────────────────────────────
+        for (const target of rule.targets) {
+            const repoConfig = workspaceConfig.repos.find(r => r.name === target.repo);
+            if (!repoConfig) {
+                result.targets.push({ repo: target.repo, created: 0, skipped: 0, derived: 0, errors: [`Collection "${target.repo}" not found`] });
+                continue;
+            }
+            const repoPath = join(workspacePath, repoConfig.path);
+            if (!existsSync(repoPath)) {
+                result.targets.push({ repo: target.repo, created: 0, skipped: 0, derived: 0, errors: [`Collection path does not exist: ${repoPath}`] });
+                continue;
+            }
+            log(`  Target: ${target.repo}${target.filter ? ` (filtered)` : ' (all records)'}`);
+            const graph = new BaseGraphManager({
+                repoPath,
+                repoName: target.repo,
+                gitUser: { id: 'sync', name: 'Studiograph Sync', email: 'sync@studiograph.local' },
+                schemaRegistry,
+            });
+            const existingSourceRefs = buildSourceRefIndex(graph);
+            // Apply target-specific filter
+            let targetRecords = applyClientFilters(allRecords, target.filter, filterFieldDefs, log);
+            // Override target_repo on records before writing
+            targetRecords = targetRecords.map(r => ({ ...r, target_repo: target.repo }));
+            const targetResult = { repo: target.repo, created: 0, skipped: 0, derived: 0, errors: [] };
+            const writeResults = await writeEntities(targetRecords, rule.source, graph, existingSourceRefs, log);
+            targetResult.created += writeResults.created;
+            targetResult.skipped += writeResults.skipped;
+            targetResult.errors.push(...writeResults.errors);
+            // Derive secondary entities (if definition has derive_from configs)
+            if (definition) {
+                const primaryType = rule.entity_type ?? findPrimaryType(definition);
+                for (const [entityType, entityConfig] of Object.entries(definition.entities)) {
+                    if (entityType === primaryType || !entityConfig.derive_from)
+                        continue;
+                    log(`  Deriving ${entityType} entities for ${target.repo}...`);
+                    const derived = deriveEntities(targetRecords, entityType, entityConfig, rule.source);
+                    const existingDedupeValues = entityConfig.dedupe_field
+                        ? buildDedupeIndex(graph, entityType, entityConfig.dedupe_field)
+                        : new Map();
+                    const deduped = derived.filter(rec => {
+                        if (existingSourceRefs.has(rec.source_ref))
+                            return false;
+                        if (entityConfig.dedupe_field) {
+                            const val = rec.frontmatter[entityConfig.dedupe_field];
+                            if (val && existingDedupeValues.has(String(val).toLowerCase()))
+                                return false;
+                        }
+                        return true;
+                    });
+                    const derivedResults = await writeEntities(deduped, rule.source, graph, existingSourceRefs, log);
+                    targetResult.derived += derivedResults.created;
+                    targetResult.skipped += derivedResults.skipped;
+                    targetResult.errors.push(...derivedResults.errors);
+                }
+            }
+            result.targets.push(targetResult);
+        }
+    }
+    finally {
+        await client.close();
+    }
+    return result;
+}
+async function syncOneRule(options) {
+    const { workspacePath, repoConfig, rule, workspaceConfig, schemaExtensions, onProgress, userId, authService, serverUrl } = options;
+    const log = onProgress ?? (() => { });
+    const collectionName = repoConfig.name;
+    const configMgr = new SourceConfigManager(workspacePath);
+    const result = {
+        collection: collectionName,
+        source: rule.source,
+        created: 0,
+        skipped: 0,
+        derived: 0,
+        errors: [],
+    };
+    // Load source definition (try new format first, then old format)
+    const definition = configMgr.getDefinition(rule.source);
+    const oldConfig = !definition ? configMgr.get(rule.source) : null;
+    if (!definition && !oldConfig) {
+        throw new Error(`Source "${rule.source}" not found in .studiograph/sources/`);
+    }
+    // Resolve connector config
+    const connectorName = definition?.connector ?? oldConfig.connector;
+    const connectorConfigs = ConnectorManager.loadConfigs(workspacePath);
+    const connectorConfig = connectorConfigs.find(c => c.name === connectorName);
+    if (!connectorConfig) {
+        throw new Error(`Connector "${connectorName}" not configured. Run: studiograph connector add ${connectorName}`);
+    }
+    // Connect to source
+    const client = createDataFetcher(connectorConfig, definition, { userId, authService, serverUrl });
+    try {
+        log(`  Connecting to ${connectorName}${definition?.adapter === 'rest' ? ' (REST)' : ''}...`);
+        try {
+            await client.connect();
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            throw new Error(`Failed to connect to connector "${connectorName}" at ${connectorConfig.url}. (${msg})`);
+        }
+        log(`  Available tools: ${client.getTools().map(t => t.name).join(', ')}`);
+        // Build graph manager for the collection
+        const repoPath = join(workspacePath, repoConfig.path);
+        if (!existsSync(repoPath)) {
+            throw new Error(`Collection path does not exist: ${repoPath}`);
+        }
+        const schemaRegistry = new SchemaRegistry(schemaExtensions);
+        const graph = new BaseGraphManager({
+            repoPath,
+            repoName: collectionName,
+            gitUser: { id: 'sync', name: 'Studiograph Sync', email: 'sync@studiograph.local' },
+            schemaRegistry,
+        });
+        // Build index of existing source_refs in this collection
+        const existingSourceRefs = buildSourceRefIndex(graph);
+        log(`  ${existingSourceRefs.size} existing synced entities in collection`);
+        if (definition) {
+            // ── New-format source definition ──
+            await syncNewFormat({
+                definition,
+                sourceName: rule.source,
+                rule,
+                graph,
+                collectionName,
+                client,
+                existingSourceRefs,
+                schemaRegistry,
+                workspacePath,
+                schemaExtensions,
+                result,
+                onProgress,
+            });
+        }
+        else {
+            // ── Old-format source config (backwards compatibility) ──
+            await syncOldFormat({
+                config: oldConfig,
+                rule,
+                graph,
+                collectionName,
+                client,
+                existingSourceRefs,
+                schemaRegistry,
+                workspacePath,
+                schemaExtensions,
+                result,
+                onProgress,
+            });
+        }
+    }
+    finally {
+        await client.close();
+    }
+    return result;
+}
+async function syncNewFormat(options) {
+    const { definition, sourceName, rule, graph, collectionName, client, existingSourceRefs, schemaRegistry, workspacePath, schemaExtensions, result, onProgress } = options;
+    const log = onProgress ?? (() => { });
+    // Find primary entity config
+    const primaryType = rule.entity_type ?? findPrimaryType(definition);
+    const primaryConfig = definition.entities[primaryType];
+    if (!primaryConfig) {
+        throw new Error(`Entity type "${primaryType}" not found in source definition`);
+    }
+    // Convert to EntityMapping for the extractor
+    const mapping = sourceEntityConfigToMapping(primaryType, primaryConfig, sourceName, collectionName, rule.filter);
+    // Extract records
+    const syncState = new SyncStateManager(workspacePath);
+    const oldStyleConfig = {
+        name: sourceName,
+        connector: definition.connector,
+        enabled: true,
+        added_at: new Date().toISOString(),
+        entity_mappings: [mapping],
+    };
+    log(`  Extracting ${primaryType} records...`);
+    let records;
+    if (primaryConfig.extraction_mode === 'frontmatter') {
+        const extractor = new FrontmatterExtractor(client, oldStyleConfig);
+        records = await extractor.extract(mapping, onProgress);
+    }
+    else {
+        const extractor = new StructuredExtractor(client, oldStyleConfig, syncState);
+        records = await extractor.extract(mapping, false, onProgress);
+    }
+    log(`  Extracted ${records.length} records`);
+    // Apply client-side filters declared by the source definition
+    records = applyClientFilters(records, rule.filter, definition.filter_fields ?? [], log);
+    // Write primary entities
+    log(`  Writing ${records.length} records...`);
+    const writeResults = await writeEntities(records, sourceName, graph, existingSourceRefs, log);
+    result.created += writeResults.created;
+    result.skipped += writeResults.skipped;
+    result.errors.push(...writeResults.errors);
+    // Derive secondary entities
+    for (const [entityType, entityConfig] of Object.entries(definition.entities)) {
+        if (entityType === primaryType || !entityConfig.derive_from)
+            continue;
+        log(`  Deriving ${entityType} entities...`);
+        const derived = deriveEntities(records, entityType, entityConfig, sourceName);
+        log(`  Found ${derived.length} ${entityType} candidates`);
+        // Dedup derived entities against existing collection
+        const existingDedupeValues = entityConfig.dedupe_field
+            ? buildDedupeIndex(graph, entityType, entityConfig.dedupe_field)
+            : new Map();
+        const dedupedDerived = derived.filter(rec => {
+            // Skip if source_ref already exists
+            if (existingSourceRefs.has(rec.source_ref))
+                return false;
+            // Skip if dedupe field matches existing entity
+            if (entityConfig.dedupe_field) {
+                const dedupeVal = rec.frontmatter[entityConfig.dedupe_field];
+                if (dedupeVal && existingDedupeValues.has(String(dedupeVal).toLowerCase()))
+                    return false;
+            }
+            return true;
+        });
+        const derivedResults = await writeEntities(dedupedDerived, sourceName, graph, existingSourceRefs, log);
+        result.derived += derivedResults.created;
+        result.skipped += derivedResults.skipped;
+        result.errors.push(...derivedResults.errors);
+    }
+}
+async function syncOldFormat(options) {
+    const { config, rule, graph, collectionName, client, existingSourceRefs, schemaRegistry, workspacePath, result, onProgress } = options;
+    const log = onProgress ?? (() => { });
+    const syncState = new SyncStateManager(workspacePath);
+    for (const mapping of config.entity_mappings) {
+        // Override target_repo to write to this collection
+        const adjustedMapping = { ...mapping, target_repo: collectionName };
+        // Merge rule filter into list_params
+        if (rule.filter) {
+            const existing = adjustedMapping.list_params ?? {};
+            adjustedMapping.list_params = Array.isArray(existing)
+                ? existing.map(p => ({ ...p, ...rule.filter }))
+                : { ...existing, ...rule.filter };
+        }
+        log(`  Extracting ${mapping.entity_type} from ${mapping.source_type}...`);
+        let records;
+        if (adjustedMapping.extraction_mode === 'frontmatter') {
+            const extractor = new FrontmatterExtractor(client, config);
+            records = await extractor.extract(adjustedMapping, onProgress);
+        }
+        else {
+            const extractor = new StructuredExtractor(client, config, syncState);
+            records = await extractor.extract(adjustedMapping, false, onProgress);
+        }
+        log(`  Extracted ${records.length} records`);
+        // Apply client-side filters declared by the connector's source template
+        const templateDef = SOURCE_DEFINITIONS[config.connector];
+        records = applyClientFilters(records, rule.filter, templateDef?.filter_fields ?? [], log);
+        const writeResults = await writeEntities(records, config.name, graph, existingSourceRefs, log);
+        result.created += writeResults.created;
+        result.skipped += writeResults.skipped;
+        result.errors.push(...writeResults.errors);
+    }
+}
+async function writeEntities(records, sourceName, graph, existingSourceRefs, log) {
+    const results = { created: 0, skipped: 0, errors: [] };
+    for (const record of records) {
+        const sourceRef = `${sourceName}:${record.source_ref}`;
+        // Skip if this source_ref already exists
+        if (existingSourceRefs.has(sourceRef)) {
+            results.skipped++;
+            continue;
+        }
+        try {
+            const now = new Date().toISOString();
+            await graph.create({
+                entityType: record.entity_type,
+                entityId: record.entity_id,
+                data: {
+                    ...record.frontmatter,
+                    entity_type: record.entity_type,
+                    entity_id: record.entity_id,
+                    created_at: now,
+                    updated_at: now,
+                    created_by: 'sync',
+                    updated_by: 'sync',
+                    source_ref: sourceRef,
+                    sync_source: sourceName,
+                    synced_at: now,
+                },
+                content: record.content,
+                skipCommit: true,
+            });
+            existingSourceRefs.set(sourceRef, record.entity_id);
+            results.created++;
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            // Entity already exists (duplicate entity_id) — skip rather than fail
+            if (msg.includes('already exists')) {
+                results.skipped++;
+            }
+            else {
+                results.errors.push(`${record.entity_id}: ${msg}`);
+            }
+        }
+    }
+    if (results.errors.length > 0) {
+        log(`  Errors (first 5): ${results.errors.slice(0, 5).join('; ')}`);
+    }
+    log(`  Created ${results.created}, skipped ${results.skipped}`);
+    return results;
+}
+// ── Utility functions ────────────────────────────────────────────────────────
+/**
+ * Build a map of source_ref → entity_id for all entities in a collection.
+ */
+function buildSourceRefIndex(graph) {
+    const index = new Map();
+    try {
+        const entities = graph.list();
+        for (const entity of entities) {
+            const sourceRef = entity.data?.source_ref;
+            if (sourceRef && typeof sourceRef === 'string') {
+                index.set(sourceRef, entity.data.entity_id);
+            }
+        }
+    }
+    catch {
+        // Collection may be empty
+    }
+    return index;
+}
+/**
+ * Build a map of dedupe_field values for existing entities of a given type.
+ */
+function buildDedupeIndex(graph, entityType, dedupeField) {
+    const index = new Map();
+    try {
+        const entities = graph.list(entityType);
+        for (const entity of entities) {
+            const val = entity.data?.[dedupeField];
+            if (val && typeof val === 'string') {
+                index.set(val.toLowerCase(), entity.data.entity_id);
+            }
+        }
+    }
+    catch {
+        // Entity type folder may not exist yet
+    }
+    return index;
+}
+/**
+ * Find the primary entity type in a source definition.
+ */
+function findPrimaryType(def) {
+    const entries = Object.entries(def.entities);
+    const primary = entries.find(([_, config]) => config.primary);
+    if (primary)
+        return primary[0];
+    // If only one entity type, it's implicitly primary
+    if (entries.length === 1)
+        return entries[0][0];
+    throw new Error('Source definition has multiple entity types but none marked as primary');
+}
+/**
+ * Convert a new-format SourceEntityConfig to an old-format EntityMapping
+ * so existing extractors work unchanged.
+ */
+export function sourceEntityConfigToMapping(entityType, config, sourceName, targetRepo, filter) {
+    const mapping = {
+        source_type: config.source_type ?? entityType,
+        entity_type: entityType,
+        target_repo: targetRepo,
+        extraction_mode: config.extraction_mode ?? 'structured',
+        field_map: config.field_map,
+        status_map: config.status_map,
+        id_from: config.id_from,
+        content_from: config.content_from,
+        fetch_details: config.fetch_details,
+        list_limit: config.list_limit,
+        list_tool: config.list_tool,
+        read_tool: config.read_tool,
+        list_endpoint: config.list_endpoint,
+        detail_endpoint: config.detail_endpoint,
+        detail_params: config.detail_params,
+        page_size: config.page_size,
+        content_tool: config.content_tool,
+        directory_patterns: config.directory_patterns,
+    };
+    // Merge filter into list_params
+    if (filter || config.list_params) {
+        const baseParams = config.list_params ?? {};
+        if (filter) {
+            mapping.list_params = Array.isArray(baseParams)
+                ? baseParams.map(p => ({ ...p, ...filter }))
+                : { ...baseParams, ...filter };
+        }
+        else {
+            mapping.list_params = baseParams;
+        }
+    }
+    return mapping;
+}
+/**
+ * Apply client-side filters to extracted records based on source filter field definitions.
+ * Fields with mode='client' are matched against frontmatter after extraction.
+ */
+function applyClientFilters(records, filter, filterFieldDefs, log) {
+    if (!filter || filterFieldDefs.length === 0)
+        return records;
+    let result = records;
+    for (const fieldDef of filterFieldDefs) {
+        if (fieldDef.mode !== 'client')
+            continue;
+        const filterValue = filter[fieldDef.key];
+        if (!filterValue)
+            continue;
+        const keyword = String(filterValue).toLowerCase();
+        const before = result.length;
+        const checkFields = fieldDef.fields ?? ['title', 'name'];
+        // Normalize for comparison: strip wikilink brackets, replace hyphens with spaces
+        const normalize = (s) => s.toLowerCase().replace(/^\[\[|\]\]$/g, '').replace(/-/g, ' ');
+        const normalizedKeyword = normalize(keyword);
+        result = result.filter(rec => {
+            for (const field of checkFields) {
+                const val = rec.frontmatter[field];
+                if (val != null) {
+                    // Handle array values (e.g. folders: ["Ridge AI", "Other"])
+                    const values = Array.isArray(val) ? val.map(v => normalize(String(v))) : [normalize(String(val))];
+                    for (const str of values) {
+                        if (fieldDef.op === 'exact' ? str === normalizedKeyword : str.includes(normalizedKeyword))
+                            return true;
+                    }
+                }
+            }
+            return normalize(rec.entity_id).includes(normalizedKeyword);
+        });
+        log(`  Filter "${fieldDef.key}=${filterValue}": ${result.length} of ${before} match`);
+    }
+    return result;
+}
+/**
+ * Derive secondary entities from primary extracted records.
+ * Uses the derive_from field in the source entity config to find
+ * the source field containing the data to derive from.
+ */
+function deriveEntities(primaryRecords, entityType, config, sourceName) {
+    if (!config.derive_from)
+        return [];
+    const derived = [];
+    const seenIds = new Set();
+    for (const record of primaryRecords) {
+        const sourceData = record.frontmatter[config.derive_from];
+        if (!sourceData)
+            continue;
+        // Handle both single values and arrays
+        const items = Array.isArray(sourceData) ? sourceData : [sourceData];
+        for (const item of items) {
+            if (!item)
+                continue;
+            const frontmatter = { entity_type: entityType };
+            // Apply field map to the derived item
+            for (const [sourceField, target] of Object.entries(config.field_map)) {
+                const value = typeof item === 'object' ? resolveFieldPath(item, sourceField) : item;
+                if (value === undefined || value === null)
+                    continue;
+                if (typeof target === 'string') {
+                    frontmatter[target] = value;
+                }
+                else {
+                    const transform = target;
+                    frontmatter[transform.field] = applyTransform(value, transform.transform);
+                }
+            }
+            // Generate entity_id
+            const idField = config.id_from ?? 'name';
+            const idValue = frontmatter[idField] ?? frontmatter.name ?? frontmatter.email;
+            if (!idValue)
+                continue;
+            const entityId = slugify(String(idValue));
+            if (!entityId || seenIds.has(entityId))
+                continue;
+            seenIds.add(entityId);
+            frontmatter.entity_id = entityId;
+            // Generate synthetic source_ref
+            const sourceRef = `#${entityType}/${entityId}`;
+            derived.push({
+                entity_type: entityType,
+                entity_id: entityId,
+                target_repo: record.target_repo,
+                frontmatter,
+                content: '',
+                source_name: sourceName,
+                source_ref: sourceRef,
+                confidence: 0.7,
+            });
+        }
+    }
+    return derived;
+}
+//# sourceMappingURL=collection-sync.js.map