streamby-core 0.41.0 → 0.43.0

package/dist/services/workflowResources.d.ts

@@ -0,0 +1,90 @@
+import { NodeSchema, StreamByConfig } from '../types';
+export type ResourceKind = 'db' | 'api' | 'storage' | 'service' | 'export' | 'pipeline';
+type Row = 'input' | 'export' | 'pipeline';
+interface NodeKind {
+    type: string;
+    field: string;
+    list: string;
+    row: Row;
+    subtitle: (resource: any) => string;
+}
+export declare const NODE_OF: Record<ResourceKind, NodeKind>;
+export declare const CONNECT_PREFIX = "__connect__:";
+declare const pendingOf: (value: unknown) => string | null;
+declare const emptySchema: () => NodeSchema;
+declare const schemaOf: (workflow: any) => NodeSchema;
+/** The resource a node is bound to, if it's one of a resource's nodes and bound already. */
+export declare function resourceOfNode(node: any): {
+    kind: ResourceKind;
+    id: string;
+} | null;
+/** The resources the workflow's nodes are bound to, by kind. */
+export declare function referencedIn(schema: NodeSchema): Record<ResourceKind, Set<string>>;
+/**
+ * Whether a node consumes its resource rather than being it: it holds a configuration of its own
+ * beside the resource's id (a data source reading one table, an Ingest importing one file, a
+ * service node set to an action). A workflow may have many beside the resource's own node, and
+ * they're never merged into it.
+ */
+export declare function consumes(node: any): boolean;
+/** Whether a node is its resource's own node (bound to it, and not one consuming it). */
+export declare const isOwnNode: (node: any, kind: ResourceKind, id: string) => boolean;
+/**
+ * The schema with the resource's own node, if it has none yet (as the canvas wires one): nodes
+ * consuming it (an Ingest importing one of its files) aren't it, and removing them all mustn't
+ * remove it unasked.
+ */
+export declare function withResourceNode(schema: NodeSchema, kind: ResourceKind, resource: any): NodeSchema;
+/** The schema without the resource's nodes, nor their edges. */
+export declare function withoutResourceNodes(schema: NodeSchema, kind: ResourceKind, id: string): NodeSchema;
+/** One node per resource: a second node bound to the same one merges into the first (its edges go to it). */
+export declare function oneNodePerResource(schema: NodeSchema): NodeSchema;
+/**
+ * A workflow made before it defined the project, brought to it once: the nodes once drawn from
+ * the project's lists bound by the id in their data (an API's had it only in its node id), the
+ * orchestrator there, a node for each resource it has without one, and one node per resource.
+ */
+export declare function migrateSchema(project: any, schema: NodeSchema): NodeSchema;
+/** Saves the project's workflow (its single one, made if it has none) with the changes given. */
+export declare function writeWorkflow(project: any, changes: Record<string, unknown>): Promise<any>;
+/** The project's workflow as it's answered: brought once to define the project (see migrateSchema). */
+export declare function readWorkflow(project: any): Promise<any>;
+/** A resource added from its section (or by an agent): its node, in the workflow. */
+export declare const addToWorkflow: (projectId: string, kind: ResourceKind, resource: any) => Promise<void>;
+/** A resource removed from its section: its nodes, gone from the workflow. */
+export declare const removeFromWorkflow: (projectId: string, kind: ResourceKind, id: string) => Promise<void>;
+/** Rewrites ProjectInfo.pipelines from the pipelines collection, sorted by order. */
+export declare function syncPipelineRefs(projectId: string): Promise<void>;
+/**
+ * Removes a resource from the project, only it (its node is the caller's: the workflow saved
+ * without it, or removeFromWorkflow). A built-in disconnected keeps its id, to come back as it was.
+ */
+export declare function removeResourceOf(project: any, kind: ResourceKind, id: string, config: StreamByConfig): Promise<void>;
+/** A pipeline made in the project (its refs kept in sync), as its section makes one. */
+export declare function createPipelineOf(projectId: string, name: string, userId: string): Promise<any>;
+/** The project's resource lists as they're answered after a change. */
+export declare function listsOf(projectId: string): Promise<{
+    dbConnections: (Omit<{
+        encryptedCredential?: string;
+    }, "encryptedCredential"> & {
+        hasCredential: boolean;
+    })[];
+    storageConnections: (Omit<{
+        encryptedCredential?: string;
+    }, "encryptedCredential"> & {
+        hasCredential: boolean;
+    })[];
+    integrationConnections: (Omit<{
+        encryptedCredential?: string;
+    }, "encryptedCredential"> & {
+        hasCredential: boolean;
+    })[];
+    connections: (Omit<{
+        encryptedCredential?: string;
+    }, "encryptedCredential"> & {
+        hasCredential: boolean;
+    })[];
+    exports: any[];
+    pipelines: any;
+}>;
+export { emptySchema, pendingOf, schemaOf };

package/dist/services/workflowResources.js

@@ -0,0 +1,288 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.schemaOf = exports.pendingOf = exports.emptySchema = exports.removeFromWorkflow = exports.addToWorkflow = exports.isOwnNode = exports.CONNECT_PREFIX = exports.NODE_OF = void 0;
+exports.resourceOfNode = resourceOfNode;
+exports.referencedIn = referencedIn;
+exports.consumes = consumes;
+exports.withResourceNode = withResourceNode;
+exports.withoutResourceNodes = withoutResourceNodes;
+exports.oneNodePerResource = oneNodePerResource;
+exports.migrateSchema = migrateSchema;
+exports.writeWorkflow = writeWorkflow;
+exports.readWorkflow = readWorkflow;
+exports.syncPipelineRefs = syncPipelineRefs;
+exports.removeResourceOf = removeResourceOf;
+exports.createPipelineOf = createPipelineOf;
+exports.listsOf = listsOf;
+const crypto_1 = __importDefault(require("crypto"));
+const mongodb_1 = require("mongodb");
+const manager_1 = require("../models/manager");
+const nosqlDb_1 = require("../utils/nosqlDb");
+const connection_1 = require("./connection");
+const export_1 = require("./export");
+const deliverables_1 = require("./deliverables");
+const sanitize_1 = require("../utils/sanitize");
+exports.NODE_OF = {
+    db: { type: 'dataSourceNode', field: 'connectionId', list: 'dbConnections', row: 'input', subtitle: r => r.dbType },
+    api: { type: 'connectionNode', field: 'connectionId', list: 'connections', row: 'input', subtitle: r => r.method ?? 'GET' },
+    storage: { type: 'ingestNode', field: 'storageConnectionId', list: 'storageConnections', row: 'input', subtitle: r => r.type },
+    service: { type: 'integrationConnectionNode', field: 'integrationConnectionId', list: 'integrationConnections', row: 'input', subtitle: r => r.provider },
+    export: { type: 'exportNode', field: 'exportId', list: 'exports', row: 'export', subtitle: r => r.method || 'GET' },
+    pipeline: { type: 'pipelineRefNode', field: 'pipelineId', list: 'pipelines', row: 'pipeline', subtitle: () => 'Pipeline' },
+};
+const KINDS = Object.keys(exports.NODE_OF);
+const KIND_OF_TYPE = new Map(KINDS.map(kind => [exports.NODE_OF[kind].type, kind]));
+// A node set to an integration the project doesn't have yet: connected when the workflow is saved.
+exports.CONNECT_PREFIX = '__connect__:';
+const pendingOf = (value) => (typeof value === 'string' && value.startsWith(exports.CONNECT_PREFIX) ? value.slice(exports.CONNECT_PREFIX.length) : null);
+exports.pendingOf = pendingOf;
+const ORCHESTRATOR = 'streamby';
+const EDGE_STYLE = { stroke: '#38b6ff', strokeWidth: 1.5 };
+const emptySchema = () => ({ nodes: [], edges: [] });
+exports.emptySchema = emptySchema;
+const schemaOf = (workflow) => ({
+    nodes: Array.isArray(workflow?.nodeSchema?.nodes) ? workflow.nodeSchema.nodes : [],
+    edges: Array.isArray(workflow?.nodeSchema?.edges) ? workflow.nodeSchema.edges : [],
+});
+exports.schemaOf = schemaOf;
+/** The resource a node is bound to, if it's one of a resource's nodes and bound already. */
+function resourceOfNode(node) {
+    const kind = KIND_OF_TYPE.get(node?.type);
+    const id = kind ? node?.data?.[exports.NODE_OF[kind].field] : null;
+    return kind && typeof id === 'string' && id && !pendingOf(id) ? { kind, id } : null;
+}
+/** The resources the workflow's nodes are bound to, by kind. */
+function referencedIn(schema) {
+    const refs = Object.fromEntries(KINDS.map(kind => [kind, new Set()]));
+    for (const node of schema.nodes) {
+        const ref = resourceOfNode(node);
+        if (ref)
+            refs[ref.kind].add(ref.id);
+    }
+    return refs;
+}
+// What a node's data holds only to be shown (its name, how it's drawn), not what it does.
+const PRESENTATION = new Set(['label', 'subtitle', 'icon', 'bgColor', 'iconColor', 'description']);
+/**
+ * Whether a node consumes its resource rather than being it: it holds a configuration of its own
+ * beside the resource's id (a data source reading one table, an Ingest importing one file, a
+ * service node set to an action). A workflow may have many beside the resource's own node, and
+ * they're never merged into it.
+ */
+function consumes(node) {
+    const kind = KIND_OF_TYPE.get(node?.type);
+    if (!kind)
+        return false;
+    return Object.entries(node?.data ?? {}).some(([key, value]) => key !== exports.NODE_OF[kind].field && !PRESENTATION.has(key) && value !== '' && value !== null && value !== undefined && !(typeof value === 'object' && !Object.keys(value).length));
+}
+/** Whether a node is its resource's own node (bound to it, and not one consuming it). */
+const isOwnNode = (node, kind, id) => {
+    const ref = resourceOfNode(node);
+    return ref?.kind === kind && ref.id === id && !consumes(node);
+};
+exports.isOwnNode = isOwnNode;
+// Where a new resource's node goes, as the canvas lays them out around the orchestrator: the
+// inputs above it, the exports below, the pipelines on its left.
+function placeFor(schema, row) {
+    const orchestrator = schema.nodes.find(n => n.id === ORCHESTRATOR)?.position ?? { x: 350, y: 20 };
+    const near = (a, b) => Math.abs(a - b) < 60;
+    if (row === 'pipeline') {
+        const x = orchestrator.x - 350;
+        const ys = schema.nodes.filter(n => n.position && near(n.position.x, x)).map(n => n.position.y);
+        return { x, y: ys.length ? Math.max(...ys) + 220 : orchestrator.y };
+    }
+    const y = row === 'input' ? orchestrator.y - 180 : orchestrator.y + 200;
+    const xs = schema.nodes.filter(n => n.position && near(n.position.y, y)).map(n => n.position.x);
+    return { x: xs.length ? Math.max(...xs) + 220 : orchestrator.x, y };
+}
+const orchestratorNode = () => ({ id: ORCHESTRATOR, type: 'orchestratorNode', position: { x: 350, y: 20 }, data: { label: 'StreamBy', subtitle: 'Orchestrator' } });
+/**
+ * The schema with the resource's own node, if it has none yet (as the canvas wires one): nodes
+ * consuming it (an Ingest importing one of its files) aren't it, and removing them all mustn't
+ * remove it unasked.
+ */
+function withResourceNode(schema, kind, resource) {
+    const id = String(resource.id);
+    if (schema.nodes.some(n => (0, exports.isOwnNode)(n, kind, id)))
+        return schema;
+    const def = exports.NODE_OF[kind];
+    const base = schema.nodes.some(n => n.id === ORCHESTRATOR) ? schema : { ...schema, nodes: [orchestratorNode(), ...schema.nodes] };
+    const nodeId = `${kind}-${id}-${crypto_1.default.randomUUID().slice(0, 8)}`;
+    const node = { id: nodeId, type: def.type, position: placeFor(base, def.row), data: { label: resource.name ?? id, subtitle: def.subtitle(resource), [def.field]: id } };
+    const edge = def.row === 'input'
+        ? { id: `e-${nodeId}`, source: nodeId, sourceHandle: 'out-stream', target: ORCHESTRATOR, targetHandle: 'in-top', animated: false, style: EDGE_STYLE }
+        : def.row === 'export'
+            ? { id: `e-${nodeId}`, source: ORCHESTRATOR, sourceHandle: 'out-bottom', target: nodeId, targetHandle: 'in-orchestrator-bottom', animated: true, style: EDGE_STYLE }
+            : { id: `e-${nodeId}`, source: ORCHESTRATOR, sourceHandle: 'out-pipeline', target: nodeId, targetHandle: 'in-orchestrator', animated: true, style: EDGE_STYLE };
+    return { nodes: [...base.nodes, node], edges: [...base.edges, edge] };
+}
+/** The schema without the resource's nodes, nor their edges. */
+function withoutResourceNodes(schema, kind, id) {
+    const gone = new Set(schema.nodes.filter(n => { const ref = resourceOfNode(n); return ref?.kind === kind && ref.id === id; }).map(n => n.id));
+    return { nodes: schema.nodes.filter(n => !gone.has(n.id)), edges: schema.edges.filter(e => !gone.has(e.source) && !gone.has(e.target)) };
+}
+/** One node per resource: a second node bound to the same one merges into the first (its edges go to it). */
+function oneNodePerResource(schema) {
+    const first = new Map();
+    const merged = new Map();
+    const nodes = schema.nodes.filter(node => {
+        const ref = resourceOfNode(node);
+        if (!ref || consumes(node))
+            return true;
+        const key = `${ref.kind}:${ref.id}`;
+        const kept = first.get(key);
+        if (!kept) {
+            first.set(key, node.id);
+            return true;
+        }
+        merged.set(node.id, kept);
+        return false;
+    });
+    const seen = new Set();
+    const edges = schema.edges
+        .map(e => ({ ...e, source: merged.get(e.source) ?? e.source, target: merged.get(e.target) ?? e.target }))
+        .filter(e => {
+        const key = `${e.source}|${e.sourceHandle ?? ''}|${e.target}|${e.targetHandle ?? ''}`;
+        if (e.source === e.target || seen.has(key))
+            return false;
+        seen.add(key);
+        return true;
+    });
+    return { nodes, edges };
+}
+const LEGACY_PREFIX = [['db-', 'db'], ['api-', 'api'], ['storage-', 'storage'], ['svc-', 'service'], ['export-', 'export'], ['pipeline-', 'pipeline']];
+/**
+ * A workflow made before it defined the project, brought to it once: the nodes once drawn from
+ * the project's lists bound by the id in their data (an API's had it only in its node id), the
+ * orchestrator there, a node for each resource it has without one, and one node per resource.
+ */
+function migrateSchema(project, schema) {
+    let nodes = schema.nodes.map(node => {
+        const legacy = LEGACY_PREFIX.find(([prefix, kind]) => typeof node?.id === 'string' && node.id.startsWith(prefix) && exports.NODE_OF[kind].type === node.type);
+        if (!legacy)
+            return node;
+        const def = exports.NODE_OF[legacy[1]];
+        return node.data?.[def.field] ? node : { ...node, data: { ...node.data, [def.field]: node.id.slice(legacy[0].length) } };
+    });
+    if (!nodes.some(n => n.id === ORCHESTRATOR))
+        nodes = [orchestratorNode(), ...nodes];
+    let migrated = { nodes, edges: schema.edges };
+    for (const kind of KINDS) {
+        for (const resource of (project[exports.NODE_OF[kind].list] ?? []))
+            migrated = withResourceNode(migrated, kind, resource);
+    }
+    return oneNodePerResource(migrated);
+}
+// ── Reading and writing a project's workflow ──
+/** Saves the project's workflow (its single one, made if it has none) with the changes given. */
+async function writeWorkflow(project, changes) {
+    const Project = (0, manager_1.getModel)('projects');
+    const projectId = String(project._id ?? project.id);
+    // Each save a later version than the one before, even within the same millisecond: a save is
+    // made on the version it was read at (baseUpdatedAt), so two can't share one.
+    const before = project.workflow?.updatedAt ? new Date(project.workflow.updatedAt).getTime() : 0;
+    const now = new Date(Math.max(Date.now(), before + 1));
+    const base = project.workflow ?? { id: crypto_1.default.randomUUID(), name: 'Workflow', description: '', status: 'draft', projectId, nodeSchema: null, createdAt: now };
+    const updated = { ...base, ...changes, id: base.id, projectId, updatedAt: now };
+    if (project.dbType === 'sql') {
+        await (0, manager_1.getModel)('projects', 'sql').update({ _id: projectId }, { workflow: updated });
+    }
+    else {
+        await Project.update({ _id: new mongodb_1.ObjectId(projectId) }, { $set: { workflow: updated } });
+    }
+    return updated;
+}
+/** The project's workflow as it's answered: brought once to define the project (see migrateSchema). */
+async function readWorkflow(project) {
+    if (project.workflow?.resourceNodesAt)
+        return project.workflow;
+    const nodeSchema = migrateSchema(project, schemaOf(project.workflow));
+    return writeWorkflow(project, { nodeSchema, resourceNodesAt: new Date() });
+}
+// A change to the workflow saved, made on it as it is now (read again: another one may have
+// been saved since the project was read).
+async function changeSchema(projectId, change) {
+    const project = await (0, manager_1.getModel)('projects').findOne({ _id: projectId });
+    if (!project)
+        return;
+    const current = project.workflow?.resourceNodesAt ? schemaOf(project.workflow) : migrateSchema(project, schemaOf(project.workflow));
+    await writeWorkflow(project, { nodeSchema: change(current), resourceNodesAt: project.workflow?.resourceNodesAt ?? new Date() });
+}
+/** A resource added from its section (or by an agent): its node, in the workflow. */
+const addToWorkflow = (projectId, kind, resource) => changeSchema(projectId, schema => withResourceNode(schema, kind, resource));
+exports.addToWorkflow = addToWorkflow;
+/** A resource removed from its section: its nodes, gone from the workflow. */
+const removeFromWorkflow = (projectId, kind, id) => changeSchema(projectId, schema => withoutResourceNodes(schema, kind, id));
+exports.removeFromWorkflow = removeFromWorkflow;
+// ── Removing a resource ──
+/** Rewrites ProjectInfo.pipelines from the pipelines collection, sorted by order. */
+async function syncPipelineRefs(projectId) {
+    const db = (0, nosqlDb_1.nosqlDb)();
+    if (!db)
+        return;
+    const all = await db.collection('pipelines').find({ projectId }).project({ _id: 0, id: 1, name: 1, order: 1 }).sort({ order: 1, createdAt: 1 }).toArray();
+    await (0, manager_1.getModel)('projects').update({ _id: projectId }, { pipelines: all.map((p) => ({ id: p.id, name: p.name, order: p.order })) });
+}
+/**
+ * Removes a resource from the project, only it (its node is the caller's: the workflow saved
+ * without it, or removeFromWorkflow). A built-in disconnected keeps its id, to come back as it was.
+ */
+async function removeResourceOf(project, kind, id, config) {
+    const Project = (0, manager_1.getModel)('projects');
+    const projectId = String(project._id ?? project.id);
+    if (kind === 'db' || kind === 'storage' || kind === 'service') {
+        const list = exports.NODE_OF[kind].list;
+        const conn = (project[list] ?? []).find((c) => c.id === id);
+        if (!conn)
+            return;
+        await Project.update({ _id: projectId }, { $pull: { [list]: { id } } });
+        if (conn.source === 'builtin' && conn.integrationId && kind !== 'service') {
+            await Project.update({ _id: projectId }, { $pull: { disconnectedBuiltins: { integrationId: conn.integrationId } } });
+            await Project.update({ _id: projectId }, { $push: { disconnectedBuiltins: { kind, integrationId: conn.integrationId, connectionId: conn.id } } });
+        }
+    }
+    else if (kind === 'api') {
+        if ((project.connections ?? []).some((c) => c.id === id))
+            await (0, connection_1.deleteConnection)(config, projectId, id);
+    }
+    else if (kind === 'export') {
+        if ((project.exports ?? []).some((e) => String(e.id) === id)) {
+            await (0, export_1.deleteExport)(config, projectId, id, project.dbType);
+            await (0, deliverables_1.deleteDeliverablesOf)(projectId, { exportId: id });
+        }
+    }
+    else {
+        await (0, nosqlDb_1.nosqlDb)()?.collection('pipelines').deleteOne({ id, projectId });
+        await syncPipelineRefs(projectId);
+    }
+}
+/** A pipeline made in the project (its refs kept in sync), as its section makes one. */
+async function createPipelineOf(projectId, name, userId) {
+    const db = (0, nosqlDb_1.nosqlDb)();
+    if (!db)
+        throw new Error('Database not available');
+    const project = await (0, manager_1.getModel)('projects').findOne({ _id: projectId });
+    const now = new Date();
+    const pipeline = {
+        id: crypto_1.default.randomUUID(), projectId, name, description: null, order: project?.pipelines?.length ?? 0,
+        nodeSchema: null, createdBy: userId, createdAt: now, updatedAt: now,
+    };
+    await db.collection('pipelines').insertOne({ ...pipeline });
+    await syncPipelineRefs(projectId);
+    return pipeline;
+}
+/** The project's resource lists as they're answered after a change. */
+async function listsOf(projectId) {
+    const project = await (0, manager_1.getModel)('projects').findOne({ _id: projectId });
+    return {
+        dbConnections: (0, sanitize_1.sanitizeConnections)(project?.dbConnections),
+        storageConnections: (0, sanitize_1.sanitizeConnections)(project?.storageConnections),
+        integrationConnections: (0, sanitize_1.sanitizeConnections)(project?.integrationConnections),
+        connections: (0, sanitize_1.sanitizeConnections)(project?.connections),
+        exports: (0, sanitize_1.exportSummaries)(project?.exports),
+        pipelines: project?.pipelines ?? [],
+    };
+}

package/dist/types/index.d.ts

@@ -82,6 +82,11 @@ export interface ProjectInfo {
     integrationConnections?: IntegrationConnection[];
     pipelines?: PipelineRef[];
     builtinBackfilledAt?: Date;
+    disconnectedBuiltins?: {
+        kind: 'db' | 'storage';
+        integrationId: string;
+        connectionId: string;
+    }[];
     workflow?: Workflow;
 }
 export type NodePersistence = 'database' | 'storage' | 'both' | 'none';
@@ -529,3 +534,43 @@ export interface AssetVersionRecord {
     changeNote?: string;
     createdAt: Date;
 }
+/**
+ * Who saved a document's version: a user in the UI, an API token, or an app the user let act
+ * for them (OAuth, e.g. Claude in claude.ai).
+ */
+export interface DocumentAuthor {
+    userId: string;
+    username: string;
+    via: 'session' | 'apiToken' | 'oauth';
+    tokenId?: string;
+    clientId?: string;
+    clientName?: string;
+}
+export type DocumentKind = 'document' | 'folder' | 'credits';
+/** A project's document as it is now (a folder only groups others: it has no content). */
+export interface ProjectDocument {
+    id: string;
+    projectId: string;
+    kind: DocumentKind;
+    title: string;
+    parentId: string | null;
+    position: number;
+    content: string;
+    version: number;
+    exportable: boolean;
+    publishedVersion: number | null;
+    createdAt: Date;
+    createdBy: DocumentAuthor;
+    updatedAt: Date;
+    updatedBy: DocumentAuthor;
+}
+/** One save of a document: its title and content then, who saved it and when. */
+export interface DocumentVersion {
+    documentId: string;
+    projectId: string;
+    version: number;
+    title: string;
+    content: string;
+    author: DocumentAuthor;
+    createdAt: Date;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "streamby-core",
-  "version": "0.41.0",
+  "version": "0.43.0",
   "description": "StreamBy middleware framework for media storage management",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",