taskmeld 0.1.1

package/dist/src/artifacts/artifact-index.js

@@ -0,0 +1,262 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.rebuildArtifactIndex = exports.toStoredArtifactItem = exports.listIndexRecords = exports.readIndexRecords = exports.appendIndexRecord = exports.getIndexPath = void 0;
+const promises_1 = require("node:fs/promises");
+const node_fs_1 = require("node:fs");
+const node_readline_1 = require("node:readline");
+const node_path_1 = require("node:path");
+const INDEX_FILE_NAME = "index.jsonl";
+const getIndexPath = (artifactDir) => (0, node_path_1.resolve)(artifactDir, INDEX_FILE_NAME);
+exports.getIndexPath = getIndexPath;
+const encodeCursorV2 = (cursor) => `v2:${Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url")}`;
+/** 兼容解码 v2 和旧版 cursor。返回 null 表示无效。 */
+const decodeAnyCursor = (cursor) => {
+    if (cursor.startsWith("v2:")) {
+        try {
+            const parsed = JSON.parse(Buffer.from(cursor.slice(3), "base64url").toString("utf8"));
+            if (parsed.v === 2 && parsed.updatedAt && parsed.artifactId) {
+                return { updatedAt: parsed.updatedAt, artifactId: parsed.artifactId, indexUpdatedAt: parsed.indexUpdatedAt };
+            }
+            return null;
+        }
+        catch {
+            return null;
+        }
+    }
+    // 旧 v1 cursor 格式: base64url(updatedAt|artifactId)
+    try {
+        const raw = Buffer.from(cursor, "base64url").toString("utf8");
+        const idx = raw.lastIndexOf("|");
+        if (idx < 0)
+            return null;
+        return { updatedAt: raw.slice(0, idx), artifactId: raw.slice(idx + 1) };
+    }
+    catch {
+        return null;
+    }
+};
+const getIndexUpdatedAt = async (artifactDir) => {
+    try {
+        const s = await (0, promises_1.stat)((0, exports.getIndexPath)(artifactDir));
+        return s.mtime.toISOString();
+    }
+    catch {
+        return "";
+    }
+};
+/** 追加一条索引记录到 index.jsonl。追加失败不抛异常，返回失败信息供调用方观测。 */
+const appendIndexRecord = async (artifactDir, record) => {
+    const indexPath = (0, exports.getIndexPath)(artifactDir);
+    try {
+        await (0, promises_1.appendFile)(indexPath, `${JSON.stringify(record)}\n`, "utf8");
+        return { ok: true };
+    }
+    catch (error) {
+        return {
+            ok: false,
+            warning: {
+                code: "ARTIFACT_INDEX_APPEND_FAILED",
+                artifactDir,
+                indexPath,
+                relativePath: record.relativePath,
+                message: error instanceof Error ? error.message : String(error),
+            },
+        };
+    }
+};
+exports.appendIndexRecord = appendIndexRecord;
+/** 读取索引文件所有行，损坏行跳过。使用流式读取避免大索引一次性内存占用。 */
+const readIndexRecords = async (artifactDir) => {
+    const indexPath = (0, exports.getIndexPath)(artifactDir);
+    const records = [];
+    try {
+        const reader = (0, node_readline_1.createInterface)({
+            input: (0, node_fs_1.createReadStream)(indexPath, { encoding: "utf8" }),
+            crlfDelay: Infinity,
+        });
+        for await (const line of reader) {
+            if (!line.trim())
+                continue;
+            try {
+                records.push(JSON.parse(line));
+            }
+            catch {
+                // 损坏行跳过
+            }
+        }
+    }
+    catch {
+        return [];
+    }
+    return records;
+};
+exports.readIndexRecords = readIndexRecords;
+const dedupeLatestByArtifactId = (records) => {
+    const latest = new Map();
+    for (const record of records) {
+        const previous = latest.get(record.artifactId);
+        if (!previous || Date.parse(record.updatedAt) >= Date.parse(previous.updatedAt)) {
+            latest.set(record.artifactId, record);
+        }
+    }
+    return [...latest.values()];
+};
+/** 从索引读取、过滤、排序、分页。基于 readIndexRecords 之上提供查询能力。 */
+const listIndexRecords = async (artifactDir, filter) => {
+    const records = dedupeLatestByArtifactId(await (0, exports.readIndexRecords)(artifactDir));
+    const limitRaw = filter.limit ?? 100;
+    const limit = limitRaw <= 0 ? Number.MAX_SAFE_INTEGER
+        : Number.isFinite(limitRaw) ? Math.max(1, Math.min(5000, Math.trunc(limitRaw)))
+            : 100;
+    const statusSet = filter.statuses?.length ? new Set(filter.statuses) : null;
+    const kindSet = filter.kinds?.length ? new Set(filter.kinds) : null;
+    const nodeIdSet = filter.nodeIds?.length ? new Set(filter.nodeIds) : null;
+    const dateFrom = filter.dateFrom ?? null;
+    const dateTo = filter.dateTo ?? null;
+    const matched = [];
+    for (const record of records) {
+        if (filter.pipelineId && record.pipelineId !== filter.pipelineId)
+            continue;
+        if (statusSet && !statusSet.has(record.status))
+            continue;
+        if (kindSet && !kindSet.has(record.kind))
+            continue;
+        if (nodeIdSet && (!record.nodeId || !nodeIdSet.has(record.nodeId)))
+            continue;
+        if (filter.batchRunId && record.batchRunId !== filter.batchRunId)
+            continue;
+        if (filter.runId && record.runId !== filter.runId)
+            continue;
+        if (dateFrom && record.dateBucket < dateFrom)
+            continue;
+        if (dateTo && record.dateBucket > dateTo)
+            continue;
+        matched.push(record);
+    }
+    // 按 updatedAt DESC, artifactId ASC 排序
+    matched.sort((a, b) => {
+        const dateDiff = Date.parse(b.updatedAt) - Date.parse(a.updatedAt);
+        if (dateDiff !== 0)
+            return dateDiff;
+        return a.artifactId.localeCompare(b.artifactId);
+    });
+    const total = matched.length;
+    // cursor 分页：v2 cursor 含 indexUpdatedAt，索引重建后旧 cursor 自动从头开始
+    let startIndex = 0;
+    const currentIndexUpdatedAt = await getIndexUpdatedAt(artifactDir);
+    if (filter.cursor) {
+        const decoded = decodeAnyCursor(filter.cursor);
+        if (decoded) {
+            // 索引代际变化 → cursor 过期，从头开始
+            if (decoded.indexUpdatedAt && decoded.indexUpdatedAt !== currentIndexUpdatedAt) {
+                startIndex = 0;
+            }
+            else {
+                const cursorPos = matched.findIndex((rec) => rec.updatedAt === decoded.updatedAt && rec.artifactId === decoded.artifactId);
+                if (cursorPos >= 0)
+                    startIndex = cursorPos + 1;
+            }
+        }
+    }
+    const page = matched.slice(startIndex, startIndex + limit);
+    const nextCursor = page.length === limit && startIndex + limit < total
+        ? encodeCursorV2({ v: 2, updatedAt: page[page.length - 1].updatedAt, artifactId: page[page.length - 1].artifactId, indexUpdatedAt: currentIndexUpdatedAt })
+        : null;
+    return { items: page, nextCursor, total };
+};
+exports.listIndexRecords = listIndexRecords;
+/** 从 StoredArtifactIndexRecord 转换为 StoredArtifactItem。 */
+const toStoredArtifactItem = (record, pipelineId, pipelineTitle) => ({
+    pipelineId,
+    pipelineTitle,
+    status: record.status,
+    dateBucket: record.dateBucket,
+    runId: record.runId,
+    nodeId: record.nodeId,
+    relativePath: record.relativePath,
+    fileName: (0, node_path_1.basename)(record.relativePath),
+    sizeBytes: record.sizeBytes,
+    updatedAt: record.updatedAt,
+    artifactId: record.artifactId,
+});
+exports.toStoredArtifactItem = toStoredArtifactItem;
+/**
+ * 扫描一条流水线的产物目录，重建 index.jsonl。
+ */
+const rebuildArtifactIndex = async (definition, scan) => {
+    const warnings = [];
+    let indexed = 0;
+    let skipped = 0;
+    const indexPath = (0, exports.getIndexPath)(definition.artifactDir);
+    try {
+        const items = await scan(definition);
+        items.sort((a, b) => Date.parse(b.updatedAt) - Date.parse(a.updatedAt));
+        const tmpPath = `${indexPath}.tmp-${Date.now()}`;
+        const stream = [];
+        for (const item of items) {
+            const record = await enrichItemToIndexRecord(item, definition);
+            if (record) {
+                stream.push(JSON.stringify(record));
+                indexed += 1;
+            }
+            else {
+                skipped += 1;
+            }
+        }
+        await (0, promises_1.writeFile)(tmpPath, `${stream.join("\n")}\n`, "utf8");
+        await (0, promises_1.rename)(tmpPath, indexPath);
+    }
+    catch (error) {
+        warnings.push(`重建索引失败 (pipeline ${definition.id}): ${error instanceof Error ? error.message : String(error)}`);
+    }
+    return { indexed, skipped, warnings };
+};
+exports.rebuildArtifactIndex = rebuildArtifactIndex;
+/** 从 StoredArtifactItem + 读取文件内容 补充为完整的 IndexRecord。 */
+const enrichItemToIndexRecord = async (item, definition) => {
+    const filePath = (0, node_path_1.resolve)(definition.artifactDir, item.relativePath);
+    let parsed = null;
+    try {
+        const raw = await (0, promises_1.readFile)(filePath, "utf8");
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        // 无法读取时用路径启发式信息
+    }
+    const artifactObj = parsed && typeof parsed.artifact === "object" && parsed.artifact !== null
+        ? parsed.artifact
+        : null;
+    const kind = typeof parsed?.kind === "string"
+        ? parsed.kind
+        : parsed && "envelope" in parsed
+            ? "envelope"
+            : item.relativePath.includes("/envelopes/")
+                ? "envelope"
+                : item.fileName.includes("-adapter-output")
+                    ? "adapter"
+                    : item.fileName.includes("-group-output")
+                        ? "group"
+                        : "artifact";
+    return {
+        schemaVersion: 1,
+        artifactId: typeof parsed?.artifactId === "string" ? parsed.artifactId : `legacy:${item.pipelineId}:${item.relativePath}`,
+        pipelineId: item.pipelineId,
+        status: item.status,
+        kind,
+        dateBucket: item.dateBucket,
+        runId: item.runId,
+        batchRunId: null,
+        nodeId: item.nodeId ?? (typeof parsed?.nodeId === "string" ? parsed.nodeId : null),
+        groupId: typeof parsed?.groupId === "string" ? parsed.groupId : null,
+        itemKey: typeof parsed?.itemKey === "string" ? parsed.itemKey : null,
+        requestId: typeof parsed?.requestId === "string" ? parsed.requestId : null,
+        type: typeof artifactObj?.type === "string" ? artifactObj.type : "unknown",
+        artifactSchemaVersion: typeof artifactObj?.schemaVersion === "number" ? artifactObj.schemaVersion : 1,
+        name: typeof artifactObj?.name === "string" ? artifactObj.name : item.fileName,
+        relativePath: item.relativePath,
+        sizeBytes: item.sizeBytes,
+        hash: "",
+        createdAt: item.updatedAt,
+        updatedAt: item.updatedAt,
+    };
+};

package/dist/src/artifacts/artifact-rebuilder.js

@@ -0,0 +1,120 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.rebuildArtifactIndexIncremental = void 0;
+const promises_1 = require("node:fs/promises");
+const node_path_1 = require("node:path");
+const artifact_index_1 = require("./artifact-index");
+/**
+ * 增量重建产物索引 — 扫描磁盘目录，仅对缺失索引的文件补充记录。
+ * 已有索引记录的文件（按 relativePath 匹配）不会重复添加。
+ */
+const rebuildArtifactIndexIncremental = async (rootDir, pipelineId) => {
+    const warnings = [];
+    let indexed = 0;
+    let skipped = 0;
+    // 收集所有目标 pipeline 的 artifact 目录
+    const dirs = [];
+    // 扫描 rootDir 下所有 pipeline 目录
+    try {
+        const entries = await (0, promises_1.readdir)(rootDir, { withFileTypes: true });
+        for (const entry of entries) {
+            if (!entry.isDirectory())
+                continue;
+            const pipelineDir = (0, node_path_1.resolve)(rootDir, entry.name);
+            const artifactsPath = (0, node_path_1.resolve)(pipelineDir, "artifacts");
+            try {
+                const artStat = await (0, promises_1.stat)(artifactsPath);
+                if (artStat.isDirectory()) {
+                    if (!pipelineId || entry.name === pipelineId) {
+                        dirs.push({ pipelineId: entry.name, artifactDir: artifactsPath });
+                    }
+                }
+            }
+            catch {
+                // no artifacts dir
+            }
+        }
+    }
+    catch {
+        warnings.push(`无法扫描产物根目录: ${rootDir}`);
+    }
+    for (const dir of dirs) {
+        const existingRecords = await (0, artifact_index_1.readIndexRecords)(dir.artifactDir);
+        const indexedPaths = new Set(existingRecords.map((r) => r.relativePath));
+        // 递归扫描所有 artifact 文件
+        const scanDir = async (subPath) => {
+            let entries;
+            try {
+                entries = await (0, promises_1.readdir)(subPath, { withFileTypes: true });
+            }
+            catch {
+                return;
+            }
+            for (const entry of entries) {
+                const fullPath = (0, node_path_1.resolve)(subPath, entry.name);
+                if (entry.isDirectory()) {
+                    await scanDir(fullPath);
+                }
+                else if (entry.isFile() && entry.name.endsWith(".json") && !entry.name.startsWith(".tmp-")) {
+                    const relPath = fullPath.slice(dir.artifactDir.length + 1).replace(/\\/g, "/");
+                    if (indexedPaths.has(relPath)) {
+                        skipped += 1;
+                        continue;
+                    }
+                    try {
+                        const fileStat = await (0, promises_1.stat)(fullPath);
+                        // 构建基础索引记录
+                        const record = {
+                            schemaVersion: 1,
+                            artifactId: `rebuild:${dir.pipelineId}:${relPath}`,
+                            pipelineId: dir.pipelineId,
+                            status: "unknown",
+                            kind: relPath.includes("/envelopes/") ? "envelope" : "artifact",
+                            dateBucket: "",
+                            runId: null,
+                            batchRunId: null,
+                            nodeId: null,
+                            groupId: null,
+                            itemKey: null,
+                            requestId: null,
+                            type: "unknown",
+                            artifactSchemaVersion: 1,
+                            name: (0, node_path_1.basename)(fullPath),
+                            relativePath: relPath,
+                            sizeBytes: fileStat.size,
+                            hash: "",
+                            createdAt: fileStat.mtime.toISOString(),
+                            updatedAt: fileStat.mtime.toISOString(),
+                        };
+                        const result = await (0, artifact_index_1.appendIndexRecord)(dir.artifactDir, record);
+                        if (result.ok) {
+                            indexed += 1;
+                        }
+                        else {
+                            warnings.push(result.warning.message);
+                            skipped += 1;
+                        }
+                    }
+                    catch {
+                        skipped += 1;
+                    }
+                }
+            }
+        };
+        const statusDirs = ["success", "failed", "rejected"];
+        for (const statusDir of statusDirs) {
+            const statusPath = (0, node_path_1.resolve)(dir.artifactDir, statusDir);
+            try {
+                const s = await (0, promises_1.stat)(statusPath);
+                if (s.isDirectory()) {
+                    await scanDir(statusPath);
+                }
+            }
+            catch {
+                // dir doesn't exist
+            }
+        }
+    }
+    return { indexed, skipped, warnings };
+};
+exports.rebuildArtifactIndexIncremental = rebuildArtifactIndexIncremental;