primitive-admin 1.1.0-alpha.31 → 1.1.0-alpha.32

package/dist/src/commands/sync.js

@@ -4,7 +4,7 @@ import { createHash } from "crypto";
 import * as TOML from "@iarna/toml";
 import { lookup as mimeLookup } from "mime-types";
 import { ApiClient, ApiError, ConflictError, SchemaRequiredError, OperationRefError, SchemaBreaksOpsError, SchemaHasUncheckableOpsError, TomlParseError, OpsExistError, } from "../lib/api-client.js";
-import { buildDatabaseTypeTomlData, detectExistingOperationForms, normalizeOperationFromToml, } from "../lib/toml-database-config.js";
+import { buildDatabaseTypeTomlData, detectExistingOperationForms, normalizeOperationFromToml, normalizeSubscriptionFromToml, SubscriptionAccessKeyConflictError, } from "../lib/toml-database-config.js";
 import { validateOperations, formatIssue, } from "../lib/toml-params-validator.js";
 import { getServerUrl, resolveAppId } from "../lib/config.js";
 import { resolveSyncDir, isAutoResolvedSyncDir, checkLegacySyncMigration, } from "../lib/sync-paths.js";
@@ -302,6 +302,12 @@ function serializeWorkflow(workflow, draft, configs) {
             perUserMaxRunning: workflow.perUserMaxRunning,
             perUserMaxQueued: workflow.perUserMaxQueued,
             dequeueOrder: workflow.dequeueOrder,
+            // Sync-callable flags (#807): always emit both so a fresh pull → push
+            // cycle round-trips them. The GET response normalizes these to booleans
+            // (requiresClientApply defaults true, syncCallable defaults false), so
+            // they're safe to coerce here.
+            requiresClientApply: workflow.requiresClientApply !== false,
+            syncCallable: workflow.syncCallable === true,
             // Schemas are at workflow level
             inputSchema: workflow.inputSchema ? JSON.stringify(workflow.inputSchema) : undefined,
             outputSchema: workflow.outputSchema ? JSON.stringify(workflow.outputSchema) : undefined,
@@ -421,6 +427,14 @@ export function parseDatabaseTypeToml(tomlData) {
     if (typeSection.autoPopulatedFields !== undefined) {
         typeConfig.autoPopulatedFields = typeSection.autoPopulatedFields;
     }
+    // timestamps (issue #748) — inline-table form
+    // (`timestamps = { create = "createdAt", update = "modifiedAt" }`) passes
+    // through unchanged; the server validates and normalizes. Either lifecycle
+    // key may be omitted, and an optional `models = [...]` filter rides on the
+    // same object.
+    if (typeSection.timestamps !== undefined) {
+        typeConfig.timestamps = typeSection.timestamps;
+    }
     if (tomlData.triggers) {
         typeConfig.triggers = tomlData.triggers;
     }
@@ -449,7 +463,12 @@ export function parseDatabaseTypeToml(tomlData) {
     //   - native: `[operations.definition]` table, `[[operations.params]]` rows
     // and returns the JS shape the server expects.
     const operations = (tomlData.operations || []).map((op) => normalizeOperationFromToml(op));
-    return { typeConfig, operations };
+    // Subscriptions (issue #803): `[[subscriptions]]` array-of-tables.
+    // `normalizeSubscriptionFromToml` maps `accessRule`→`access` (throwing on a
+    // conflicting both-present declaration) and forwards `select`/`emit`/`params`
+    // in the wire shape. Until #803 this block was dropped at parse time.
+    const subscriptions = (tomlData.subscriptions || []).map((sub) => normalizeSubscriptionFromToml(sub));
+    return { typeConfig, operations, subscriptions };
 }
 function parseRuleSetToml(tomlData) {
     const ruleSetSection = tomlData.ruleSet || {};
@@ -1074,10 +1093,20 @@ Directory Structure:
                 client.listGroupTypeConfigs(resolvedAppId).catch(() => []),
                 client.listCollectionTypeConfigs(resolvedAppId).catch(() => []),
             ]);
-            // Fetch operations for each database type
+            // Fetch operations + subscriptions for each database type. Issue #803:
+            // subscriptions are pulled symmetrically with operations so a
+            // pull → push cycle round-trips `[[subscriptions]]` blocks without a
+            // spurious diff. The list endpoint already excludes archived rows.
             const databaseTypesWithOps = await Promise.all((Array.isArray(databaseTypeConfigsResult) ? databaseTypeConfigsResult : []).map(async (typeConfig) => {
-                const ops = await client.listDatabaseTypeOperations(resolvedAppId, typeConfig.databaseType).catch(() => []);
-                return { typeConfig, operations: Array.isArray(ops) ? ops : [] };
+                const [ops, subs] = await Promise.all([
+                    client.listDatabaseTypeOperations(resolvedAppId, typeConfig.databaseType).catch(() => []),
+                    client.listDatabaseTypeSubscriptions(resolvedAppId, typeConfig.databaseType).catch(() => []),
+                ]);
+                return {
+                    typeConfig,
+                    operations: Array.isArray(ops) ? ops : [],
+                    subscriptions: Array.isArray(subs) ? subs : [],
+                };
             }));
             // Ensure directories exist
             ensureDir(configDir);
@@ -1269,7 +1298,7 @@ Directory Structure:
             }
             // Write database types
             const databaseTypeEntities = {};
-            for (const { typeConfig, operations } of databaseTypesWithOps) {
+            for (const { typeConfig, operations, subscriptions } of databaseTypesWithOps) {
                 const filename = `${typeConfig.databaseType}.toml`;
                 const filePath = join(configDir, "database-types", filename);
                 // Preserve the existing file's per-op form (issue #752): if the
@@ -1282,6 +1311,9 @@ Directory Structure:
                     hints,
                     defaultForm: "native",
                     logger: (msg) => info(`  ${msg}`),
+                    // Issue #803: emit `[[subscriptions]]` blocks so a pull → push
+                    // cycle round-trips subscriptions without a spurious diff.
+                    subscriptions,
                 }));
                 const opsEntities = {};
                 for (const op of operations) {
@@ -1289,10 +1321,22 @@ Directory Structure:
                         modifiedAt: op.modifiedAt || new Date().toISOString(),
                     };
                 }
+                // Track per-subscription modifiedAt in SyncState (issue #803),
+                // exactly like operations, so the next push skips unchanged
+                // subscriptions and detects removals.
+                const subsEntities = {};
+                for (const sub of subscriptions || []) {
+                    if (!sub?.subscriptionKey)
+                        continue;
+                    subsEntities[sub.subscriptionKey] = {
+                        modifiedAt: sub.modifiedAt || new Date().toISOString(),
+                    };
+                }
                 databaseTypeEntities[typeConfig.databaseType] = {
                     databaseType: typeConfig.databaseType,
                     modifiedAt: typeConfig.modifiedAt || new Date().toISOString(),
                     operations: Object.keys(opsEntities).length > 0 ? opsEntities : undefined,
+                    subscriptions: Object.keys(subsEntities).length > 0 ? subsEntities : undefined,
                     contentHash: computeFileHash(filePath),
                     hasSchema: typeof typeConfig.schema === "string" &&
                         typeConfig.schema.trim().length > 0,
@@ -2135,7 +2179,21 @@ Directory Structure:
                                 ? undefined
                                 : syncState?.entities?.workflows?.[key]?.modifiedAt;
                             try {
-                                // Update workflow metadata and schemas
+                                // Update workflow metadata and schemas.
+                                //
+                                // Sync-callable ordering (#807, codex review): the metadata
+                                // PATCH must NOT carry `syncCallable: true` here. The server
+                                // re-validates `syncCallable: true` against the workflow's
+                                // CURRENTLY-active server steps (`loadCurrentActiveSteps`),
+                                // not the steps being pushed in the same sync. So when a
+                                // single TOML edit both enables `syncCallable` and replaces
+                                // sync-incompatible active steps with compatible ones, a
+                                // combined metadata-first PATCH would be rejected against the
+                                // stale steps. Defer `syncCallable` to a second PATCH issued
+                                // AFTER the config/step update lands, so it validates against
+                                // the new (compatible) active steps. `requiresClientApply`
+                                // has no step-dependent validation, so it stays in the first
+                                // PATCH unchanged.
                                 const updated = await client.updateWorkflow(resolvedAppId, existingId, {
                                     name: workflow.name,
                                     description: workflow.description,
@@ -2144,9 +2202,19 @@ Directory Structure:
                                     perUserMaxRunning: workflow.perUserMaxRunning,
                                     perUserMaxQueued: workflow.perUserMaxQueued,
                                     dequeueOrder: workflow.dequeueOrder,
+                                    // Sync-callable flags (#807): pass through as-is. An absent
+                                    // TOML key is `undefined` (dropped by JSON.stringify, so the
+                                    // server's hasOwnProperty guard leaves the value untouched);
+                                    // an explicit `false` is preserved as a meaningful value.
+                                    requiresClientApply: workflow.requiresClientApply,
                                     inputSchema: workflow.inputSchema ? safeJsonParse(workflow.inputSchema) : null,
                                     outputSchema: workflow.outputSchema ? safeJsonParse(workflow.outputSchema) : null,
                                 }, expectedModifiedAt);
+                                // Track the latest workflow `modifiedAt` for the sync-state
+                                // write below. The first PATCH bumps it; the `syncCallable`
+                                // PATCH (if any) bumps it again. The config/step update does
+                                // NOT touch the workflow definition's `modifiedAt`.
+                                let latestModifiedAt = updated?.workflow?.modifiedAt;
                                 // Update active configuration steps (or draft for legacy).
                                 // Issue #687: name the slot we touched so the dev-loop
                                 // user can confirm before previewing.
@@ -2164,11 +2232,28 @@ Directory Structure:
                                     });
                                     updateSlotLabel = "draft (legacy)";
                                 }
+                                // Second PATCH: apply `syncCallable` now that the new steps
+                                // are active (#807). Only sent when the TOML actually carries
+                                // the key — an absent key stays `undefined` and we skip the
+                                // call entirely, preserving the no-clobber discipline. Chain
+                                // `expectedModifiedAt` off the first PATCH's returned value so
+                                // optimistic concurrency stays intact (the step update above
+                                // doesn't change the workflow definition's `modifiedAt`).
+                                if (workflow.syncCallable !== undefined) {
+                                    const syncCallableUpdated = await client.updateWorkflow(resolvedAppId, existingId, { syncCallable: workflow.syncCallable }, 
+                                    // Mirror the first PATCH's concurrency posture: `--force`
+                                    // skips the check (undefined), otherwise reuse the fresh
+                                    // `modifiedAt` from the first PATCH.
+                                    options.force ? undefined : latestModifiedAt);
+                                    if (syncCallableUpdated?.workflow?.modifiedAt) {
+                                        latestModifiedAt = syncCallableUpdated.workflow.modifiedAt;
+                                    }
+                                }
                                 info(`  Updated workflow: ${key} (${updateSlotLabel})`);
                                 // Update sync state with new modifiedAt. Store the *expanded*
                                 // content hash so future fragment-only edits are detected.
-                                if (syncState?.entities?.workflows?.[key] && updated?.workflow?.modifiedAt) {
-                                    syncState.entities.workflows[key].modifiedAt = updated.workflow.modifiedAt;
+                                if (syncState?.entities?.workflows?.[key] && latestModifiedAt) {
+                                    syncState.entities.workflows[key].modifiedAt = latestModifiedAt;
                                     syncState.entities.workflows[key].contentHash = computeExpandedContentHash(tomlData);
                                 }
                                 // Fetch full workflow to get config name→ID mappings
@@ -2211,6 +2296,12 @@ Directory Structure:
                                     perUserMaxRunning: workflow.perUserMaxRunning,
                                     perUserMaxQueued: workflow.perUserMaxQueued,
                                     dequeueOrder: workflow.dequeueOrder,
+                                    // Sync-callable flags (#807): pass through as-is. An absent
+                                    // TOML key is `undefined` (dropped by JSON.stringify), so the
+                                    // server applies its defaults (requiresClientApply=true,
+                                    // syncCallable=false); an explicit value is honored.
+                                    requiresClientApply: workflow.requiresClientApply,
+                                    syncCallable: workflow.syncCallable,
                                 });
                                 info(`  Created workflow: ${key}`);
                                 // Add new entity to sync state (including activeConfigId)
@@ -2273,7 +2364,25 @@ Directory Structure:
                     const filePath = join(dbTypesDir, file);
                     const rawToml = readFileSync(filePath, "utf-8");
                     const tomlData = parseTomlFile(filePath);
-                    const { typeConfig, operations } = parseDatabaseTypeToml(tomlData);
+                    let typeConfig;
+                    let operations;
+                    let subscriptions;
+                    try {
+                        ({ typeConfig, operations, subscriptions } =
+                            parseDatabaseTypeToml(tomlData));
+                    }
+                    catch (err) {
+                        // The only structured parse error today is the
+                        // `accessRule`/`access` conflict in a `[[subscriptions]]` block
+                        // (issue #803). Surface it with the file name and abort the
+                        // push rather than crashing with an opaque stack.
+                        if (err instanceof SubscriptionAccessKeyConflictError) {
+                            error(`  ${file}: ${err.message}`);
+                            error(`Aborting push: subscription \`access\`/\`accessRule\` conflict in ${file}.`);
+                            process.exit(1);
+                        }
+                        throw err;
+                    }
                     const dbType = typeConfig.databaseType || basename(file, ".toml");
                     // Resolve ruleSetName → ruleSetId if using key-based reference
                     resolveRuleSetReference(typeConfig, ruleSetNameToId, `database type ${dbType}`);
@@ -2325,6 +2434,9 @@ Directory Structure:
                                     updateData.autoPopulatedFields =
                                         typeConfig.autoPopulatedFields || null;
                                 }
+                                if ("timestamps" in typeConfig) {
+                                    updateData.timestamps = typeConfig.timestamps || null;
+                                }
                                 // Issue #666: forward `schema` when the local TOML declares
                                 // one (a set/update), OR when the server had one at last
                                 // sync and the local file no longer does (a deletion —
@@ -2411,6 +2523,7 @@ Directory Structure:
                                 "metadataAccess" in typeConfig ||
                                 "defaultAccess" in typeConfig ||
                                 "autoPopulatedFields" in typeConfig ||
+                                "timestamps" in typeConfig ||
                                 "schema" in typeConfig;
                             if (wouldUpdate) {
                                 changes.push({ type: "database-type", action: "update", key: dbType });
@@ -2434,6 +2547,8 @@ Directory Structure:
                                 createData.defaultAccess = typeConfig.defaultAccess;
                             if (typeConfig.autoPopulatedFields)
                                 createData.autoPopulatedFields = typeConfig.autoPopulatedFields;
+                            if (typeConfig.timestamps)
+                                createData.timestamps = typeConfig.timestamps;
                             // Issue #666 addendum A2: forward `schema` on POST so a
                             // single sync push can land both the type + its schema in
                             // one round-trip (instead of POST → PATCH). The op-edit gate
@@ -2584,6 +2699,115 @@ Directory Structure:
                             }
                         }
                     }
+                    // Process subscriptions for this type (issue #803). Mirrors the
+                    // operations reconcile above, keyed on `subscriptionKey`:
+                    // create new keys, PUT changed/existing keys, delete keys present
+                    // in SyncState but absent from the TOML. Server errors (409 key
+                    // collision, `#`-in-key 400, max-20 400, missing-required 400)
+                    // are surfaced cleanly via `wrapEntityError` — named, not a crash.
+                    const existingSubs = syncState?.entities?.databaseTypes?.[dbType]?.subscriptions || {};
+                    const tomlSubKeys = new Set(subscriptions.map((sub) => sub.subscriptionKey));
+                    for (const sub of subscriptions) {
+                        const subKey = sub.subscriptionKey;
+                        const existingSub = existingSubs[subKey];
+                        if (existingSub) {
+                            // Update existing subscription (PUT — idempotent).
+                            changes.push({ type: "subscription", action: "update", key: `${dbType}/${subKey}` });
+                            if (!options.dryRun) {
+                                try {
+                                    const updated = await client.updateDatabaseTypeSubscription(resolvedAppId, dbType, subKey, {
+                                        displayName: sub.displayName,
+                                        modelName: sub.modelName,
+                                        filter: sub.filter,
+                                        access: sub.access,
+                                        description: sub.description,
+                                        select: sub.select,
+                                        emit: sub.emit,
+                                        params: sub.params,
+                                        status: sub.status,
+                                    });
+                                    info(`  Updated subscription: ${dbType}/${subKey}`);
+                                    // Gate the SyncState write on a server-returned
+                                    // `modifiedAt`, mirroring the operations precedent above
+                                    // (don't fall back to a fresh local timestamp).
+                                    if (syncState?.entities?.databaseTypes?.[dbType]?.subscriptions?.[subKey] && updated?.modifiedAt) {
+                                        syncState.entities.databaseTypes[dbType].subscriptions[subKey].modifiedAt =
+                                            updated.modifiedAt;
+                                    }
+                                }
+                                catch (err) {
+                                    throw wrapEntityError(err, "update", "subscription", `${dbType}/${subKey}`);
+                                }
+                            }
+                        }
+                        else {
+                            // Create new subscription (POST).
+                            changes.push({ type: "subscription", action: "create", key: `${dbType}/${subKey}` });
+                            if (!options.dryRun) {
+                                try {
+                                    const created = await client.createDatabaseTypeSubscription(resolvedAppId, dbType, {
+                                        subscriptionKey: subKey,
+                                        displayName: sub.displayName,
+                                        modelName: sub.modelName,
+                                        filter: sub.filter,
+                                        access: sub.access,
+                                        description: sub.description,
+                                        select: sub.select,
+                                        emit: sub.emit,
+                                        params: sub.params,
+                                        status: sub.status,
+                                    });
+                                    info(`  Created subscription: ${dbType}/${subKey}`);
+                                    if (syncState?.entities?.databaseTypes?.[dbType]) {
+                                        if (!syncState.entities.databaseTypes[dbType].subscriptions) {
+                                            syncState.entities.databaseTypes[dbType].subscriptions = {};
+                                        }
+                                        syncState.entities.databaseTypes[dbType].subscriptions[subKey] = {
+                                            modifiedAt: created?.modifiedAt || new Date().toISOString(),
+                                        };
+                                    }
+                                }
+                                catch (err) {
+                                    throw wrapEntityError(err, "create", "subscription", `${dbType}/${subKey}`);
+                                }
+                            }
+                        }
+                    }
+                    // Delete subscriptions that were removed from TOML (hard-delete on
+                    // current `main`, PR #787 — frees the key for reuse). A 409 on
+                    // recreate is impossible here since we delete first; the pull-side
+                    // "active only" filter (controller `list` excludes archived)
+                    // agrees because hard-delete leaves no archived row behind.
+                    for (const existingSubKey of Object.keys(existingSubs)) {
+                        if (!tomlSubKeys.has(existingSubKey)) {
+                            changes.push({ type: "subscription", action: "delete", key: `${dbType}/${existingSubKey}` });
+                            if (!options.dryRun) {
+                                try {
+                                    await client.deleteDatabaseTypeSubscription(resolvedAppId, dbType, existingSubKey);
+                                    info(`  Deleted subscription: ${dbType}/${existingSubKey}`);
+                                    if (syncState?.entities?.databaseTypes?.[dbType]?.subscriptions) {
+                                        delete syncState.entities.databaseTypes[dbType].subscriptions[existingSubKey];
+                                    }
+                                }
+                                catch (err) {
+                                    throw wrapEntityError(err, "delete", "subscription", `${dbType}/${existingSubKey}`);
+                                }
+                            }
+                        }
+                    }
+                    // Refresh the file's content hash after processing operations +
+                    // subscriptions (issue #803, behavior g). The type-config update
+                    // block only refreshes the hash when a type-level field changed
+                    // (its `updateData` was non-empty); an operations- or
+                    // subscriptions-only edit left a stale hash, so the next push
+                    // would re-PUT every op/sub instead of skipping the unchanged
+                    // file. Recompute here so a truly-unchanged subsequent push skips
+                    // the whole file via `shouldPushFile`. `--force` bypasses that
+                    // skip, so forced re-pushes are unaffected.
+                    if (!options.dryRun && syncState?.entities?.databaseTypes?.[dbType]) {
+                        syncState.entities.databaseTypes[dbType].contentHash =
+                            computeFileHash(filePath);
+                    }
                 }
             }
             // Process group type configs
@@ -3343,12 +3567,30 @@ Directory Structure:
             const filePath = join(dbTypesDir, file);
             const rawBefore = readFileSync(filePath, "utf-8");
             const tomlData = parseTomlFile(filePath);
-            const { typeConfig, operations } = parseDatabaseTypeToml(tomlData);
+            // `accessRule`/`access` conflicts surface as a clear error here too,
+            // rather than silently dropping the field during a migrate rewrite.
+            let typeConfig;
+            let operations;
+            let subscriptions;
+            try {
+                ({ typeConfig, operations, subscriptions } =
+                    parseDatabaseTypeToml(tomlData));
+            }
+            catch (err) {
+                if (err instanceof SubscriptionAccessKeyConflictError) {
+                    error(`  ${file}: ${err.message}`);
+                    error(`Aborting migrate-toml: subscription \`access\`/\`accessRule\` conflict in ${file}.`);
+                    process.exit(1);
+                }
+                throw err;
+            }
             // Force-native: ignore existing hints so every TOMLable field
-            // ends up in nested-table form.
+            // ends up in nested-table form. Subscriptions are preserved on the
+            // rewrite (issue #803) so migrate-toml never drops them.
             const rewritten = serializeDatabaseType(typeConfig, operations, ruleSetIdToName, {
                 defaultForm: "native",
                 logger: (msg) => info(`  ${msg}`),
+                subscriptions,
             });
             if (rewritten === rawBefore) {
                 unchangedCount++;