streamby-core 0.42.0 → 0.43.1

package/dist/services/projectIntegration.d.ts

@@ -13,4 +13,18 @@ type ClassifyResult = {
     status: number;
 };
 export declare function classifyIntegrationId(id: string, auth: Auth, config: StreamByConfig, projectId: string): Promise<ClassifyResult>;
+declare const LIST_OF: {
+    readonly database: "dbConnections";
+    readonly storage: "storageConnections";
+    readonly service: "integrationConnections";
+};
+/**
+ * Connects an integration to a project: a built-in (the host's database or storage) or one of
+ * the user's own. A built-in it was disconnected from comes back with the id it had, so what
+ * was kept for it (its files, its records) is its own again. With `kind`, only one of that kind.
+ */
+export declare function connectIntegration(project: any, integrationId: string, auth: Auth, config: StreamByConfig, options?: {
+    kind?: keyof typeof LIST_OF;
+    description?: string;
+}): Promise<ClassifyResult>;
 export {};

package/dist/services/projectIntegration.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.classifyIntegrationId = classifyIntegrationId;
+exports.connectIntegration = connectIntegration;
 const crypto_1 = __importDefault(require("crypto"));
 const manager_1 = require("../models/manager");
 const builtinAccess_1 = require("../utils/builtinAccess");
@@ -71,3 +72,34 @@ async function classifyIntegrationId(id, auth, config, projectId) {
         },
     };
 }
+const LIST_OF = { database: 'dbConnections', storage: 'storageConnections', service: 'integrationConnections' };
+const NOUN_OF = { database: 'a database', storage: 'a storage', service: 'a service' };
+/**
+ * Connects an integration to a project: a built-in (the host's database or storage) or one of
+ * the user's own. A built-in it was disconnected from comes back with the id it had, so what
+ * was kept for it (its files, its records) is its own again. With `kind`, only one of that kind.
+ */
+async function connectIntegration(project, integrationId, auth, config, options = {}) {
+    const projectId = String(project._id ?? project.id);
+    const alreadyConnected = [...(project.dbConnections ?? []), ...(project.storageConnections ?? []), ...(project.integrationConnections ?? [])]
+        .some((c) => c.integrationId === integrationId);
+    if (alreadyConnected)
+        return { error: 'This integration is already connected to the project', status: 400 };
+    const result = await classifyIntegrationId(integrationId, auth, config, projectId);
+    if ('error' in result)
+        return result;
+    if (options.kind && result.kind !== options.kind)
+        return { error: `This integration isn't ${NOUN_OF[options.kind]}`, status: 400 };
+    const before = (project.disconnectedBuiltins ?? []).find((d) => d.integrationId === integrationId);
+    const connection = {
+        ...result.connection,
+        ...(before && result.connection.source === 'builtin' ? { id: before.connectionId } : {}),
+        ...(options.description ? { description: options.description } : {}),
+    };
+    const Project = (0, manager_1.getModel)('projects');
+    await Project.update({ _id: projectId }, { $push: { [LIST_OF[result.kind]]: connection } });
+    // By integrationId alone: a SQL project's $pull matches one key.
+    if (before)
+        await Project.update({ _id: projectId }, { $pull: { disconnectedBuiltins: { integrationId } } });
+    return { kind: result.kind, connection };
+}

package/dist/services/rights.d.ts

@@ -0,0 +1,44 @@
+export declare const LICENSE_TYPES: readonly ["CC BY", "CC BY-SA", "CC BY-ND", "CC BY-NC", "CC BY-NC-SA", "CC BY-NC-ND", "CC0", "Free Standard", "Standard", "Editorial"];
+export type LicenseType = typeof LICENSE_TYPES[number];
+export declare const RIGHTS_FIELDS: readonly ["sourceUrl", "creatorName", "creatorUrl", "licenseType", "comments"];
+type RightsField = typeof RIGHTS_FIELDS[number];
+export interface AssetRights {
+    assetId: string;
+    projectId: string;
+    sourceUrl: string | null;
+    creatorName: string | null;
+    creatorUrl: string | null;
+    licenseType: LicenseType | null;
+    comments: string | null;
+    updatedBy?: string;
+    updatedAt?: Date;
+}
+export type RightsValues = Partial<Pick<AssetRights, RightsField>>;
+/**
+ * The fields given of an asset's license metadata, checked: text (a link, an http(s) URL) and a
+ * license from the list. A field given empty or null is cleared.
+ */
+export declare function checkRights(body: any): {
+    values: RightsValues;
+} | {
+    error: string;
+};
+/** The license metadata given as query params (an upload's), checked; null if none given. */
+export declare function rightsFromQuery(query: Record<string, unknown>): {
+    values: RightsValues;
+} | {
+    error: string;
+} | null;
+export declare function getRights(projectId: string, assetId: string): Promise<AssetRights | null>;
+/** Several assets' license metadata, by asset id (those with none are missing). */
+export declare function getRightsOf(projectId: string, assetIds: string[]): Promise<Map<string, AssetRights>>;
+export declare function broadcastRights(project: any, data: {
+    action: 'saved' | 'deleted';
+    assetIds: string[];
+    rights?: AssetRights;
+}): void;
+/** Sets the fields given of an asset's license metadata (checked already), and tells the project. */
+export declare function saveRights(project: any, assetId: string, values: RightsValues, userId: string): Promise<AssetRights>;
+/** An asset's license metadata, gone with it (or, without an asset, a whole project's). */
+export declare function deleteRightsOf(projectId: string, assetId?: string): Promise<void>;
+export {};

package/dist/services/rights.js

@@ -0,0 +1,123 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RIGHTS_FIELDS = exports.LICENSE_TYPES = void 0;
+exports.checkRights = checkRights;
+exports.rightsFromQuery = rightsFromQuery;
+exports.getRights = getRights;
+exports.getRightsOf = getRightsOf;
+exports.broadcastRights = broadcastRights;
+exports.saveRights = saveRights;
+exports.deleteRightsOf = deleteRightsOf;
+const nosqlDb_1 = require("../utils/nosqlDb");
+const auth_1 = require("../utils/auth");
+const wsHub_1 = require("./wsHub");
+// An asset's license metadata: where it was taken from, who made it, and under what license.
+// Kept in the `asset_rights` collection, one record per asset (for all its versions), the
+// only place it lives: a credits sheet reads it from here, and writes to here.
+exports.LICENSE_TYPES = [
+    'CC BY', 'CC BY-SA', 'CC BY-ND', 'CC BY-NC', 'CC BY-NC-SA', 'CC BY-NC-ND', 'CC0', 'Free Standard', 'Standard', 'Editorial',
+];
+exports.RIGHTS_FIELDS = ['sourceUrl', 'creatorName', 'creatorUrl', 'licenseType', 'comments'];
+const URL_FIELDS = ['sourceUrl', 'creatorUrl'];
+const MAX_TEXT = 2000;
+const isLicense = (value) => exports.LICENSE_TYPES.includes(value);
+const isUrl = (value) => {
+    try {
+        return ['http:', 'https:'].includes(new URL(value).protocol);
+    }
+    catch {
+        return false;
+    }
+};
+/**
+ * The fields given of an asset's license metadata, checked: text (a link, an http(s) URL) and a
+ * license from the list. A field given empty or null is cleared.
+ */
+function checkRights(body) {
+    const values = {};
+    for (const field of exports.RIGHTS_FIELDS) {
+        const raw = body?.[field];
+        if (raw === undefined)
+            continue;
+        if (raw !== null && typeof raw !== 'string')
+            return { error: `${field} must be text` };
+        const value = typeof raw === 'string' ? raw.trim() : '';
+        if (!value) {
+            values[field] = null;
+            continue;
+        }
+        if (value.length > MAX_TEXT)
+            return { error: `${field} has at most ${MAX_TEXT} characters` };
+        if (field === 'licenseType') {
+            if (!isLicense(value))
+                return { error: `licenseType is one of: ${exports.LICENSE_TYPES.join(', ')}` };
+            values.licenseType = value;
+        }
+        else if (URL_FIELDS.includes(field) && !isUrl(value)) {
+            return { error: `${field} must be an http(s) URL` };
+        }
+        else {
+            values[field] = value;
+        }
+    }
+    return { values };
+}
+/** The license metadata given as query params (an upload's), checked; null if none given. */
+function rightsFromQuery(query) {
+    const given = Object.fromEntries(exports.RIGHTS_FIELDS.filter(f => typeof query[f] === 'string').map(f => [f, query[f]]));
+    return Object.keys(given).length ? checkRights(given) : null;
+}
+const collection = () => (0, nosqlDb_1.nosqlDb)()?.collection('asset_rights') ?? null;
+// A record as it's answered: only the fields it has now (a license off the list reads as none).
+const readRights = (record) => record && {
+    assetId: record.assetId,
+    projectId: record.projectId,
+    sourceUrl: record.sourceUrl ?? null,
+    creatorName: record.creatorName ?? null,
+    creatorUrl: record.creatorUrl ?? null,
+    licenseType: isLicense(record.licenseType) ? record.licenseType : null,
+    comments: record.comments ?? null,
+    ...(record.updatedBy ? { updatedBy: record.updatedBy } : {}),
+    ...(record.updatedAt ? { updatedAt: record.updatedAt } : {}),
+};
+async function getRights(projectId, assetId) {
+    const col = collection();
+    return col ? readRights(await col.findOne({ projectId, assetId })) : null;
+}
+/** Several assets' license metadata, by asset id (those with none are missing). */
+async function getRightsOf(projectId, assetIds) {
+    const col = collection();
+    if (!col || !assetIds.length)
+        return new Map();
+    const records = await col.find({ projectId, assetId: { $in: assetIds } }).toArray();
+    return new Map(records.map(r => [r.assetId, readRights(r)]));
+}
+// What changed in a project's assets' license metadata, to whoever is in it now: the credits
+// sheets they have open read their rows again.
+function broadcastRights(project, data) {
+    const projectId = String(project._id ?? project.id);
+    for (const member of project.members ?? []) {
+        if ((0, auth_1.activeMemberOf)(project, member.userId))
+            (0, wsHub_1.emitToUser)(member.userId, { type: 'assetRightsEvent', data: { projectId, ...data } });
+    }
+}
+/** Sets the fields given of an asset's license metadata (checked already), and tells the project. */
+async function saveRights(project, assetId, values, userId) {
+    const col = collection();
+    if (!col)
+        throw new Error('Database not available');
+    const projectId = String(project._id ?? project.id);
+    // The fields an earlier record had that this one doesn't (holder, territory...) go.
+    const $unset = { rightsHolder: '', licenseUrl: '', territory: '', expiresAt: '', usageRestrictions: '', notes: '' };
+    await col.updateOne({ projectId, assetId }, { $set: { assetId, projectId, ...values, updatedBy: userId, updatedAt: new Date() }, $unset }, { upsert: true });
+    const rights = (await getRights(projectId, assetId));
+    broadcastRights(project, { action: 'saved', assetIds: [assetId], rights });
+    return rights;
+}
+/** An asset's license metadata, gone with it (or, without an asset, a whole project's). */
+async function deleteRightsOf(projectId, assetId) {
+    const col = collection();
+    if (!col || !projectId)
+        return;
+    await col.deleteMany(assetId ? { projectId, assetId } : { projectId });
+}

package/dist/services/workflowResources.d.ts

@@ -0,0 +1,90 @@
+import { NodeSchema, StreamByConfig } from '../types';
+export type ResourceKind = 'db' | 'api' | 'storage' | 'service' | 'export' | 'pipeline';
+type Row = 'input' | 'export' | 'pipeline';
+interface NodeKind {
+    type: string;
+    field: string;
+    list: string;
+    row: Row;
+    subtitle: (resource: any) => string;
+}
+export declare const NODE_OF: Record<ResourceKind, NodeKind>;
+export declare const CONNECT_PREFIX = "__connect__:";
+declare const pendingOf: (value: unknown) => string | null;
+declare const emptySchema: () => NodeSchema;
+declare const schemaOf: (workflow: any) => NodeSchema;
+/** The resource a node is bound to, if it's one of a resource's nodes and bound already. */
+export declare function resourceOfNode(node: any): {
+    kind: ResourceKind;
+    id: string;
+} | null;
+/** The resources the workflow's nodes are bound to, by kind. */
+export declare function referencedIn(schema: NodeSchema): Record<ResourceKind, Set<string>>;
+/**
+ * Whether a node consumes its resource rather than being it: it holds a configuration of its own
+ * beside the resource's id (a data source reading one table, an Ingest importing one file, a
+ * service node set to an action). A workflow may have many beside the resource's own node, and
+ * they're never merged into it.
+ */
+export declare function consumes(node: any): boolean;
+/** Whether a node is its resource's own node (bound to it, and not one consuming it). */
+export declare const isOwnNode: (node: any, kind: ResourceKind, id: string) => boolean;
+/**
+ * The schema with the resource's own node, if it has none yet (as the canvas wires one): nodes
+ * consuming it (an Ingest importing one of its files) aren't it, and removing them all mustn't
+ * remove it unasked.
+ */
+export declare function withResourceNode(schema: NodeSchema, kind: ResourceKind, resource: any): NodeSchema;
+/** The schema without the resource's nodes, nor their edges. */
+export declare function withoutResourceNodes(schema: NodeSchema, kind: ResourceKind, id: string): NodeSchema;
+/** One node per resource: a second node bound to the same one merges into the first (its edges go to it). */
+export declare function oneNodePerResource(schema: NodeSchema): NodeSchema;
+/**
+ * A workflow made before it defined the project, brought to it once: the nodes once drawn from
+ * the project's lists bound by the id in their data (an API's had it only in its node id), the
+ * orchestrator there, a node for each resource it has without one, and one node per resource.
+ */
+export declare function migrateSchema(project: any, schema: NodeSchema): NodeSchema;
+/** Saves the project's workflow (its single one, made if it has none) with the changes given. */
+export declare function writeWorkflow(project: any, changes: Record<string, unknown>): Promise<any>;
+/** The project's workflow as it's answered: brought once to define the project (see migrateSchema). */
+export declare function readWorkflow(project: any): Promise<any>;
+/** A resource added from its section (or by an agent): its node, in the workflow. */
+export declare const addToWorkflow: (projectId: string, kind: ResourceKind, resource: any) => Promise<void>;
+/** A resource removed from its section: its nodes, gone from the workflow. */
+export declare const removeFromWorkflow: (projectId: string, kind: ResourceKind, id: string) => Promise<void>;
+/** Rewrites ProjectInfo.pipelines from the pipelines collection, sorted by order. */
+export declare function syncPipelineRefs(projectId: string): Promise<void>;
+/**
+ * Removes a resource from the project, only it (its node is the caller's: the workflow saved
+ * without it, or removeFromWorkflow). A built-in disconnected keeps its id, to come back as it was.
+ */
+export declare function removeResourceOf(project: any, kind: ResourceKind, id: string, config: StreamByConfig): Promise<void>;
+/** A pipeline made in the project (its refs kept in sync), as its section makes one. */
+export declare function createPipelineOf(projectId: string, name: string, userId: string): Promise<any>;
+/** The project's resource lists as they're answered after a change. */
+export declare function listsOf(projectId: string): Promise<{
+    dbConnections: (Omit<{
+        encryptedCredential?: string;
+    }, "encryptedCredential"> & {
+        hasCredential: boolean;
+    })[];
+    storageConnections: (Omit<{
+        encryptedCredential?: string;
+    }, "encryptedCredential"> & {
+        hasCredential: boolean;
+    })[];
+    integrationConnections: (Omit<{
+        encryptedCredential?: string;
+    }, "encryptedCredential"> & {
+        hasCredential: boolean;
+    })[];
+    connections: (Omit<{
+        encryptedCredential?: string;
+    }, "encryptedCredential"> & {
+        hasCredential: boolean;
+    })[];
+    exports: any[];
+    pipelines: any;
+}>;
+export { emptySchema, pendingOf, schemaOf };

package/dist/services/workflowResources.js

@@ -0,0 +1,288 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.schemaOf = exports.pendingOf = exports.emptySchema = exports.removeFromWorkflow = exports.addToWorkflow = exports.isOwnNode = exports.CONNECT_PREFIX = exports.NODE_OF = void 0;
+exports.resourceOfNode = resourceOfNode;
+exports.referencedIn = referencedIn;
+exports.consumes = consumes;
+exports.withResourceNode = withResourceNode;
+exports.withoutResourceNodes = withoutResourceNodes;
+exports.oneNodePerResource = oneNodePerResource;
+exports.migrateSchema = migrateSchema;
+exports.writeWorkflow = writeWorkflow;
+exports.readWorkflow = readWorkflow;
+exports.syncPipelineRefs = syncPipelineRefs;
+exports.removeResourceOf = removeResourceOf;
+exports.createPipelineOf = createPipelineOf;
+exports.listsOf = listsOf;
+const crypto_1 = __importDefault(require("crypto"));
+const mongodb_1 = require("mongodb");
+const manager_1 = require("../models/manager");
+const nosqlDb_1 = require("../utils/nosqlDb");
+const connection_1 = require("./connection");
+const export_1 = require("./export");
+const deliverables_1 = require("./deliverables");
+const sanitize_1 = require("../utils/sanitize");
+exports.NODE_OF = {
+    db: { type: 'dataSourceNode', field: 'connectionId', list: 'dbConnections', row: 'input', subtitle: r => r.dbType },
+    api: { type: 'connectionNode', field: 'connectionId', list: 'connections', row: 'input', subtitle: r => r.method ?? 'GET' },
+    storage: { type: 'ingestNode', field: 'storageConnectionId', list: 'storageConnections', row: 'input', subtitle: r => r.type },
+    service: { type: 'integrationConnectionNode', field: 'integrationConnectionId', list: 'integrationConnections', row: 'input', subtitle: r => r.provider },
+    export: { type: 'exportNode', field: 'exportId', list: 'exports', row: 'export', subtitle: r => r.method || 'GET' },
+    pipeline: { type: 'pipelineRefNode', field: 'pipelineId', list: 'pipelines', row: 'pipeline', subtitle: () => 'Pipeline' },
+};
+const KINDS = Object.keys(exports.NODE_OF);
+const KIND_OF_TYPE = new Map(KINDS.map(kind => [exports.NODE_OF[kind].type, kind]));
+// A node set to an integration the project doesn't have yet: connected when the workflow is saved.
+exports.CONNECT_PREFIX = '__connect__:';
+const pendingOf = (value) => (typeof value === 'string' && value.startsWith(exports.CONNECT_PREFIX) ? value.slice(exports.CONNECT_PREFIX.length) : null);
+exports.pendingOf = pendingOf;
+const ORCHESTRATOR = 'streamby';
+const EDGE_STYLE = { stroke: '#38b6ff', strokeWidth: 1.5 };
+const emptySchema = () => ({ nodes: [], edges: [] });
+exports.emptySchema = emptySchema;
+const schemaOf = (workflow) => ({
+    nodes: Array.isArray(workflow?.nodeSchema?.nodes) ? workflow.nodeSchema.nodes : [],
+    edges: Array.isArray(workflow?.nodeSchema?.edges) ? workflow.nodeSchema.edges : [],
+});
+exports.schemaOf = schemaOf;
+/** The resource a node is bound to, if it's one of a resource's nodes and bound already. */
+function resourceOfNode(node) {
+    const kind = KIND_OF_TYPE.get(node?.type);
+    const id = kind ? node?.data?.[exports.NODE_OF[kind].field] : null;
+    return kind && typeof id === 'string' && id && !pendingOf(id) ? { kind, id } : null;
+}
+/** The resources the workflow's nodes are bound to, by kind. */
+function referencedIn(schema) {
+    const refs = Object.fromEntries(KINDS.map(kind => [kind, new Set()]));
+    for (const node of schema.nodes) {
+        const ref = resourceOfNode(node);
+        if (ref)
+            refs[ref.kind].add(ref.id);
+    }
+    return refs;
+}
+// What a node's data holds only to be shown (its name, how it's drawn), not what it does.
+const PRESENTATION = new Set(['label', 'subtitle', 'icon', 'bgColor', 'iconColor', 'description']);
+/**
+ * Whether a node consumes its resource rather than being it: it holds a configuration of its own
+ * beside the resource's id (a data source reading one table, an Ingest importing one file, a
+ * service node set to an action). A workflow may have many beside the resource's own node, and
+ * they're never merged into it.
+ */
+function consumes(node) {
+    const kind = KIND_OF_TYPE.get(node?.type);
+    if (!kind)
+        return false;
+    return Object.entries(node?.data ?? {}).some(([key, value]) => key !== exports.NODE_OF[kind].field && !PRESENTATION.has(key) && value !== '' && value !== null && value !== undefined && !(typeof value === 'object' && !Object.keys(value).length));
+}
+/** Whether a node is its resource's own node (bound to it, and not one consuming it). */
+const isOwnNode = (node, kind, id) => {
+    const ref = resourceOfNode(node);
+    return ref?.kind === kind && ref.id === id && !consumes(node);
+};
+exports.isOwnNode = isOwnNode;
+// Where a new resource's node goes, as the canvas lays them out around the orchestrator: the
+// inputs above it, the exports below, the pipelines on its left.
+function placeFor(schema, row) {
+    const orchestrator = schema.nodes.find(n => n.id === ORCHESTRATOR)?.position ?? { x: 350, y: 20 };
+    const near = (a, b) => Math.abs(a - b) < 60;
+    if (row === 'pipeline') {
+        const x = orchestrator.x - 350;
+        const ys = schema.nodes.filter(n => n.position && near(n.position.x, x)).map(n => n.position.y);
+        return { x, y: ys.length ? Math.max(...ys) + 220 : orchestrator.y };
+    }
+    const y = row === 'input' ? orchestrator.y - 180 : orchestrator.y + 200;
+    const xs = schema.nodes.filter(n => n.position && near(n.position.y, y)).map(n => n.position.x);
+    return { x: xs.length ? Math.max(...xs) + 220 : orchestrator.x, y };
+}
+const orchestratorNode = () => ({ id: ORCHESTRATOR, type: 'orchestratorNode', position: { x: 350, y: 20 }, data: { label: 'StreamBy', subtitle: 'Orchestrator' } });
+/**
+ * The schema with the resource's own node, if it has none yet (as the canvas wires one): nodes
+ * consuming it (an Ingest importing one of its files) aren't it, and removing them all mustn't
+ * remove it unasked.
+ */
+function withResourceNode(schema, kind, resource) {
+    const id = String(resource.id);
+    if (schema.nodes.some(n => (0, exports.isOwnNode)(n, kind, id)))
+        return schema;
+    const def = exports.NODE_OF[kind];
+    const base = schema.nodes.some(n => n.id === ORCHESTRATOR) ? schema : { ...schema, nodes: [orchestratorNode(), ...schema.nodes] };
+    const nodeId = `${kind}-${id}-${crypto_1.default.randomUUID().slice(0, 8)}`;
+    const node = { id: nodeId, type: def.type, position: placeFor(base, def.row), data: { label: resource.name ?? id, subtitle: def.subtitle(resource), [def.field]: id } };
+    const edge = def.row === 'input'
+        ? { id: `e-${nodeId}`, source: nodeId, sourceHandle: 'out-stream', target: ORCHESTRATOR, targetHandle: 'in-top', animated: false, style: EDGE_STYLE }
+        : def.row === 'export'
+            ? { id: `e-${nodeId}`, source: ORCHESTRATOR, sourceHandle: 'out-bottom', target: nodeId, targetHandle: 'in-orchestrator-bottom', animated: true, style: EDGE_STYLE }
+            : { id: `e-${nodeId}`, source: ORCHESTRATOR, sourceHandle: 'out-pipeline', target: nodeId, targetHandle: 'in-orchestrator', animated: true, style: EDGE_STYLE };
+    return { nodes: [...base.nodes, node], edges: [...base.edges, edge] };
+}
+/** The schema without the resource's nodes, nor their edges. */
+function withoutResourceNodes(schema, kind, id) {
+    const gone = new Set(schema.nodes.filter(n => { const ref = resourceOfNode(n); return ref?.kind === kind && ref.id === id; }).map(n => n.id));
+    return { nodes: schema.nodes.filter(n => !gone.has(n.id)), edges: schema.edges.filter(e => !gone.has(e.source) && !gone.has(e.target)) };
+}
+/** One node per resource: a second node bound to the same one merges into the first (its edges go to it). */
+function oneNodePerResource(schema) {
+    const first = new Map();
+    const merged = new Map();
+    const nodes = schema.nodes.filter(node => {
+        const ref = resourceOfNode(node);
+        if (!ref || consumes(node))
+            return true;
+        const key = `${ref.kind}:${ref.id}`;
+        const kept = first.get(key);
+        if (!kept) {
+            first.set(key, node.id);
+            return true;
+        }
+        merged.set(node.id, kept);
+        return false;
+    });
+    const seen = new Set();
+    const edges = schema.edges
+        .map(e => ({ ...e, source: merged.get(e.source) ?? e.source, target: merged.get(e.target) ?? e.target }))
+        .filter(e => {
+        const key = `${e.source}|${e.sourceHandle ?? ''}|${e.target}|${e.targetHandle ?? ''}`;
+        if (e.source === e.target || seen.has(key))
+            return false;
+        seen.add(key);
+        return true;
+    });
+    return { nodes, edges };
+}
+const LEGACY_PREFIX = [['db-', 'db'], ['api-', 'api'], ['storage-', 'storage'], ['svc-', 'service'], ['export-', 'export'], ['pipeline-', 'pipeline']];
+/**
+ * A workflow made before it defined the project, brought to it once: the nodes once drawn from
+ * the project's lists bound by the id in their data (an API's had it only in its node id), the
+ * orchestrator there, a node for each resource it has without one, and one node per resource.
+ */
+function migrateSchema(project, schema) {
+    let nodes = schema.nodes.map(node => {
+        const legacy = LEGACY_PREFIX.find(([prefix, kind]) => typeof node?.id === 'string' && node.id.startsWith(prefix) && exports.NODE_OF[kind].type === node.type);
+        if (!legacy)
+            return node;
+        const def = exports.NODE_OF[legacy[1]];
+        return node.data?.[def.field] ? node : { ...node, data: { ...node.data, [def.field]: node.id.slice(legacy[0].length) } };
+    });
+    if (!nodes.some(n => n.id === ORCHESTRATOR))
+        nodes = [orchestratorNode(), ...nodes];
+    let migrated = { nodes, edges: schema.edges };
+    for (const kind of KINDS) {
+        for (const resource of (project[exports.NODE_OF[kind].list] ?? []))
+            migrated = withResourceNode(migrated, kind, resource);
+    }
+    return oneNodePerResource(migrated);
+}
+// ── Reading and writing a project's workflow ──
+/** Saves the project's workflow (its single one, made if it has none) with the changes given. */
+async function writeWorkflow(project, changes) {
+    const Project = (0, manager_1.getModel)('projects');
+    const projectId = String(project._id ?? project.id);
+    // Each save a later version than the one before, even within the same millisecond: a save is
+    // made on the version it was read at (baseUpdatedAt), so two can't share one.
+    const before = project.workflow?.updatedAt ? new Date(project.workflow.updatedAt).getTime() : 0;
+    const now = new Date(Math.max(Date.now(), before + 1));
+    const base = project.workflow ?? { id: crypto_1.default.randomUUID(), name: 'Workflow', description: '', status: 'draft', projectId, nodeSchema: null, createdAt: now };
+    const updated = { ...base, ...changes, id: base.id, projectId, updatedAt: now };
+    if (project.dbType === 'sql') {
+        await (0, manager_1.getModel)('projects', 'sql').update({ _id: projectId }, { workflow: updated });
+    }
+    else {
+        await Project.update({ _id: new mongodb_1.ObjectId(projectId) }, { $set: { workflow: updated } });
+    }
+    return updated;
+}
+/** The project's workflow as it's answered: brought once to define the project (see migrateSchema). */
+async function readWorkflow(project) {
+    if (project.workflow?.resourceNodesAt)
+        return project.workflow;
+    const nodeSchema = migrateSchema(project, schemaOf(project.workflow));
+    return writeWorkflow(project, { nodeSchema, resourceNodesAt: new Date() });
+}
+// A change to the workflow saved, made on it as it is now (read again: another one may have
+// been saved since the project was read).
+async function changeSchema(projectId, change) {
+    const project = await (0, manager_1.getModel)('projects').findOne({ _id: projectId });
+    if (!project)
+        return;
+    const current = project.workflow?.resourceNodesAt ? schemaOf(project.workflow) : migrateSchema(project, schemaOf(project.workflow));
+    await writeWorkflow(project, { nodeSchema: change(current), resourceNodesAt: project.workflow?.resourceNodesAt ?? new Date() });
+}
+/** A resource added from its section (or by an agent): its node, in the workflow. */
+const addToWorkflow = (projectId, kind, resource) => changeSchema(projectId, schema => withResourceNode(schema, kind, resource));
+exports.addToWorkflow = addToWorkflow;
+/** A resource removed from its section: its nodes, gone from the workflow. */
+const removeFromWorkflow = (projectId, kind, id) => changeSchema(projectId, schema => withoutResourceNodes(schema, kind, id));
+exports.removeFromWorkflow = removeFromWorkflow;
+// ── Removing a resource ──
+/** Rewrites ProjectInfo.pipelines from the pipelines collection, sorted by order. */
+async function syncPipelineRefs(projectId) {
+    const db = (0, nosqlDb_1.nosqlDb)();
+    if (!db)
+        return;
+    const all = await db.collection('pipelines').find({ projectId }).project({ _id: 0, id: 1, name: 1, order: 1 }).sort({ order: 1, createdAt: 1 }).toArray();
+    await (0, manager_1.getModel)('projects').update({ _id: projectId }, { pipelines: all.map((p) => ({ id: p.id, name: p.name, order: p.order })) });
+}
+/**
+ * Removes a resource from the project, only it (its node is the caller's: the workflow saved
+ * without it, or removeFromWorkflow). A built-in disconnected keeps its id, to come back as it was.
+ */
+async function removeResourceOf(project, kind, id, config) {
+    const Project = (0, manager_1.getModel)('projects');
+    const projectId = String(project._id ?? project.id);
+    if (kind === 'db' || kind === 'storage' || kind === 'service') {
+        const list = exports.NODE_OF[kind].list;
+        const conn = (project[list] ?? []).find((c) => c.id === id);
+        if (!conn)
+            return;
+        await Project.update({ _id: projectId }, { $pull: { [list]: { id } } });
+        if (conn.source === 'builtin' && conn.integrationId && kind !== 'service') {
+            await Project.update({ _id: projectId }, { $pull: { disconnectedBuiltins: { integrationId: conn.integrationId } } });
+            await Project.update({ _id: projectId }, { $push: { disconnectedBuiltins: { kind, integrationId: conn.integrationId, connectionId: conn.id } } });
+        }
+    }
+    else if (kind === 'api') {
+        if ((project.connections ?? []).some((c) => c.id === id))
+            await (0, connection_1.deleteConnection)(config, projectId, id);
+    }
+    else if (kind === 'export') {
+        if ((project.exports ?? []).some((e) => String(e.id) === id)) {
+            await (0, export_1.deleteExport)(config, projectId, id, project.dbType);
+            await (0, deliverables_1.deleteDeliverablesOf)(projectId, { exportId: id });
+        }
+    }
+    else {
+        await (0, nosqlDb_1.nosqlDb)()?.collection('pipelines').deleteOne({ id, projectId });
+        await syncPipelineRefs(projectId);
+    }
+}
+/** A pipeline made in the project (its refs kept in sync), as its section makes one. */
+async function createPipelineOf(projectId, name, userId) {
+    const db = (0, nosqlDb_1.nosqlDb)();
+    if (!db)
+        throw new Error('Database not available');
+    const project = await (0, manager_1.getModel)('projects').findOne({ _id: projectId });
+    const now = new Date();
+    const pipeline = {
+        id: crypto_1.default.randomUUID(), projectId, name, description: null, order: project?.pipelines?.length ?? 0,
+        nodeSchema: null, createdBy: userId, createdAt: now, updatedAt: now,
+    };
+    await db.collection('pipelines').insertOne({ ...pipeline });
+    await syncPipelineRefs(projectId);
+    return pipeline;
+}
+/** The project's resource lists as they're answered after a change. */
+async function listsOf(projectId) {
+    const project = await (0, manager_1.getModel)('projects').findOne({ _id: projectId });
+    return {
+        dbConnections: (0, sanitize_1.sanitizeConnections)(project?.dbConnections),
+        storageConnections: (0, sanitize_1.sanitizeConnections)(project?.storageConnections),
+        integrationConnections: (0, sanitize_1.sanitizeConnections)(project?.integrationConnections),
+        connections: (0, sanitize_1.sanitizeConnections)(project?.connections),
+        exports: (0, sanitize_1.exportSummaries)(project?.exports),
+        pipelines: project?.pipelines ?? [],
+    };
+}

package/dist/types/index.d.ts

@@ -82,6 +82,11 @@ export interface ProjectInfo {
     integrationConnections?: IntegrationConnection[];
     pipelines?: PipelineRef[];
     builtinBackfilledAt?: Date;
+    disconnectedBuiltins?: {
+        kind: 'db' | 'storage';
+        integrationId: string;
+        connectionId: string;
+    }[];
     workflow?: Workflow;
 }
 export type NodePersistence = 'database' | 'storage' | 'both' | 'none';
@@ -541,7 +546,7 @@ export interface DocumentAuthor {
     clientId?: string;
     clientName?: string;
 }
-export type DocumentKind = 'document' | 'folder';
+export type DocumentKind = 'document' | 'folder' | 'credits';
 /** A project's document as it is now (a folder only groups others: it has no content). */
 export interface ProjectDocument {
     id: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "streamby-core",
-  "version": "0.42.0",
+  "version": "0.43.1",
   "description": "StreamBy middleware framework for media storage management",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",