pm-csv 2026.8.16 → 2026.8.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.md +37 -15
- package/dist/index.d.ts +133 -60
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +419 -463
- package/dist/index.js.map +1 -1
- package/manifest.json +3 -6
- package/package.json +23 -12
package/dist/index.js
CHANGED
|
@@ -2,6 +2,7 @@ import { readFileSync, writeFileSync, createReadStream } from "node:fs";
|
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
3
|
import { spawnSync } from "node:child_process";
|
|
4
4
|
import { createHash } from "node:crypto";
|
|
5
|
+
import { certifyCompleteListResult, inspectCompleteListResult } from "@unbrained/pm-cli/sdk";
|
|
5
6
|
// ---------------------------------------------------------------------------
|
|
6
7
|
// Error contract
|
|
7
8
|
// ---------------------------------------------------------------------------
|
|
@@ -20,7 +21,7 @@ const EXIT_CODE = {
|
|
|
20
21
|
/**
|
|
21
22
|
* Error carrying a semantic process exit code, thrown by every failure path
|
|
22
23
|
* the command runtime must turn into a clean nonzero exit (including the
|
|
23
|
-
* list
|
|
24
|
+
* complete-list refusals). Exported so regression tests can assert the
|
|
24
25
|
* typed failure instead of matching on message text alone.
|
|
25
26
|
*/
|
|
26
27
|
export class CommandError extends Error {
|
|
@@ -294,47 +295,25 @@ class StreamingCSVParser {
|
|
|
294
295
|
*/
|
|
295
296
|
function streamCSVFile(filePath, delimiter, encoding, onRow) {
|
|
296
297
|
const bufEnc = encoding === "utf-8" ? "utf8" : encoding;
|
|
297
|
-
return
|
|
298
|
+
return (async () => {
|
|
298
299
|
const stream = createReadStream(filePath, { encoding: bufEnc });
|
|
299
300
|
const parser = new StreamingCSVParser(delimiter, onRow);
|
|
300
|
-
let
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
reject(err);
|
|
308
|
-
};
|
|
309
|
-
stream.on("data", (chunk) => {
|
|
310
|
-
if (stopped)
|
|
311
|
-
return;
|
|
312
|
-
let text = typeof chunk === "string" ? chunk : chunk.toString(bufEnc);
|
|
313
|
-
if (!bomChecked) {
|
|
314
|
-
text = stripBOM(text);
|
|
315
|
-
bomChecked = true;
|
|
316
|
-
}
|
|
317
|
-
try {
|
|
301
|
+
let firstChunk = true;
|
|
302
|
+
try {
|
|
303
|
+
for await (const chunk of stream) {
|
|
304
|
+
// createReadStream was constructed with an encoding, so Node guarantees
|
|
305
|
+
// string chunks even though the broad async-iterator type retains Buffer.
|
|
306
|
+
const text = firstChunk ? stripBOM(chunk) : chunk;
|
|
307
|
+
firstChunk = false;
|
|
318
308
|
parser.push(text);
|
|
319
309
|
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
try {
|
|
328
|
-
parser.end();
|
|
329
|
-
}
|
|
330
|
-
catch (err) {
|
|
331
|
-
fail(err);
|
|
332
|
-
return;
|
|
333
|
-
}
|
|
334
|
-
resolve();
|
|
335
|
-
});
|
|
336
|
-
stream.on("error", fail);
|
|
337
|
-
});
|
|
310
|
+
parser.end();
|
|
311
|
+
}
|
|
312
|
+
catch (error) {
|
|
313
|
+
stream.destroy();
|
|
314
|
+
throw error;
|
|
315
|
+
}
|
|
316
|
+
})();
|
|
338
317
|
}
|
|
339
318
|
// ---------------------------------------------------------------------------
|
|
340
319
|
// Helpers
|
|
@@ -476,12 +455,102 @@ function assertPmOutputFit(result, label) {
|
|
|
476
455
|
return;
|
|
477
456
|
throw new CommandError(`${describePmNullStatus(result, label)}. The result is unusable, so it must never be treated as an empty or absent value.`);
|
|
478
457
|
}
|
|
458
|
+
/** Canonical argv shared by every whole-tracker subprocess. */
|
|
459
|
+
const COMPLETE_LIST_ARGS = [
|
|
460
|
+
"list",
|
|
461
|
+
"--all",
|
|
462
|
+
"--json",
|
|
463
|
+
"--include-body",
|
|
464
|
+
"--strict-read",
|
|
465
|
+
"--no-truncate",
|
|
466
|
+
"--output-budget",
|
|
467
|
+
"unbounded",
|
|
468
|
+
"--output-limit",
|
|
469
|
+
"unbounded",
|
|
470
|
+
"--output-include",
|
|
471
|
+
"full",
|
|
472
|
+
];
|
|
473
|
+
/** Build one canonical complete-list request for a tracker root. */
|
|
474
|
+
function completeListArgs(pmRoot) {
|
|
475
|
+
return ["--pm-path", pmRoot, ...COMPLETE_LIST_ARGS];
|
|
476
|
+
}
|
|
477
|
+
/** Narrow an unknown JSON value to a non-array record. */
|
|
478
|
+
function asRecord(value) {
|
|
479
|
+
return value !== null && typeof value === "object" && !Array.isArray(value)
|
|
480
|
+
? value
|
|
481
|
+
: undefined;
|
|
482
|
+
}
|
|
483
|
+
/** Render a receipt value while preserving a missing field as evidence. */
|
|
484
|
+
function describeReceiptValue(value) {
|
|
485
|
+
return value === undefined ? "<missing>" : JSON.stringify(value);
|
|
486
|
+
}
|
|
487
|
+
/** Collect the pm 2026.8.21 receipt gaps not yet rejected by the public SDK. */
|
|
488
|
+
function supplementalCompleteListFindings(record) {
|
|
489
|
+
const findings = [];
|
|
490
|
+
const completeness = asRecord(record.completeness);
|
|
491
|
+
for (const field of ["unreadable_item_count", "unreadable_directory_count"]) {
|
|
492
|
+
const value = completeness?.[field];
|
|
493
|
+
if (!Number.isSafeInteger(value) || value !== 0) {
|
|
494
|
+
findings.push(`completeness.${field}=${describeReceiptValue(value)}`);
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
const omission = asRecord(record.omission_receipt);
|
|
498
|
+
if (omission === undefined) {
|
|
499
|
+
findings.push("omission_receipt=<missing>");
|
|
500
|
+
}
|
|
501
|
+
else {
|
|
502
|
+
if (omission.has_omissions !== false) {
|
|
503
|
+
findings.push(`omission_receipt.has_omissions=${describeReceiptValue(omission.has_omissions)}`);
|
|
504
|
+
}
|
|
505
|
+
if (!Number.isSafeInteger(omission.omitted_field_group_count) || omission.omitted_field_group_count !== 0) {
|
|
506
|
+
findings.push(`omission_receipt.omitted_field_group_count=${describeReceiptValue(omission.omitted_field_group_count)}`);
|
|
507
|
+
}
|
|
508
|
+
if (!Array.isArray(omission.omitted_field_groups) || omission.omitted_field_groups.length !== 0) {
|
|
509
|
+
findings.push(`omission_receipt.omitted_field_groups=${describeReceiptValue(omission.omitted_field_groups)}`);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
const readOutput = asRecord(record.read_output);
|
|
513
|
+
if (readOutput === undefined) {
|
|
514
|
+
findings.push("read_output=<missing>");
|
|
515
|
+
}
|
|
516
|
+
else {
|
|
517
|
+
for (const [field, expected] of [
|
|
518
|
+
["contract_version", 1],
|
|
519
|
+
["command", "list"],
|
|
520
|
+
["within_budget", true],
|
|
521
|
+
["strings_compacted", false],
|
|
522
|
+
["rows_compacted", false],
|
|
523
|
+
["result_omitted", false],
|
|
524
|
+
]) {
|
|
525
|
+
if (readOutput[field] !== expected) {
|
|
526
|
+
findings.push(`read_output.${field}=${describeReceiptValue(readOutput[field])}`);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
const dimensions = readOutput.requested_dimensions;
|
|
530
|
+
if (!Array.isArray(dimensions)) {
|
|
531
|
+
findings.push("read_output.requested_dimensions=<missing>");
|
|
532
|
+
}
|
|
533
|
+
else {
|
|
534
|
+
for (const dimension of ["include", "amount", "cost"]) {
|
|
535
|
+
if (!dimensions.includes(dimension)) {
|
|
536
|
+
findings.push(`read_output.requested_dimensions missing ${dimension}`);
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
if (record.output_budget_truncation !== undefined)
|
|
542
|
+
findings.push("output_budget_truncation=<present>");
|
|
543
|
+
if (record.output_budget_exceeded !== undefined)
|
|
544
|
+
findings.push("output_budget_exceeded=<present>");
|
|
545
|
+
return findings;
|
|
546
|
+
}
|
|
479
547
|
/**
|
|
480
|
-
* Refuse an incomplete `pm list
|
|
548
|
+
* Refuse an incomplete canonical `pm list --all` envelope instead of consuming it.
|
|
481
549
|
*
|
|
482
550
|
* Reading `.items` without consulting the envelope's completeness receipt is
|
|
483
551
|
* how this package once shipped a 10-row CSV export from a 682-item workspace
|
|
484
|
-
* (pm 2026.8.14 defaulted list-all to a truncated answer;
|
|
552
|
+
* (pm 2026.8.14 defaulted the deprecated list-all alias to a truncated answer;
|
|
553
|
+
* verified on this
|
|
485
554
|
* host), saw none of the applied rows on an atomic-import resume scan, and
|
|
486
555
|
* missed upsert keys - each reporting success. Any one of four independent
|
|
487
556
|
* signals means the rows in `items` are NOT the whole workspace, so this
|
|
@@ -502,58 +571,30 @@ function assertPmOutputFit(result, label) {
|
|
|
502
571
|
* workspace, so refusing loudly is simpler and safer than a paging loop that
|
|
503
572
|
* could itself silently drop rows.
|
|
504
573
|
*
|
|
505
|
-
*
|
|
506
|
-
*
|
|
507
|
-
*
|
|
574
|
+
* Shared source, filter, pagination, projection, count, and identity rules come
|
|
575
|
+
* from the public SDK. A narrow package supplement rejects the missing or
|
|
576
|
+
* contradictory omission, read-output, and unreadable-counter evidence
|
|
577
|
+
* reproduced in pm-cli issue 1078. The returned rows are therefore safe for
|
|
578
|
+
* mutation planning, resume reconciliation, and CSV export.
|
|
579
|
+
*
|
|
580
|
+
* @param envelope - Parsed canonical complete-list JSON output.
|
|
581
|
+
* @returns Every certified full item row.
|
|
582
|
+
* @throws {@link CommandError} naming every tripped signal and the counts.
|
|
508
583
|
*/
|
|
509
584
|
export function assertListAllComplete(envelope) {
|
|
510
|
-
const
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
+ "(--status/--type) or raise the output budget, then retry.");
|
|
524
|
-
}
|
|
525
|
-
if (env.has_more === true) {
|
|
526
|
-
throw new CommandError(`Refusing incomplete pm list-all answer: has_more=true (${counts}). `
|
|
527
|
-
+ "Rows exist beyond the returned page; consuming the page as the whole "
|
|
528
|
-
+ "workspace would silently drop them.");
|
|
529
|
-
}
|
|
530
|
-
if (env.completeness?.status !== "complete") {
|
|
531
|
-
const status = env.completeness?.status === undefined ? "(missing)" : JSON.stringify(env.completeness.status);
|
|
532
|
-
const unreadable = `unreadable_item_count=${env.completeness?.unreadable_item_count ?? 0}`
|
|
533
|
-
+ `, unreadable_directory_count=${env.completeness?.unreadable_directory_count ?? 0}`;
|
|
534
|
-
throw new CommandError(`Refusing incomplete pm list-all answer: completeness.status=${status} `
|
|
535
|
-
+ `(${unreadable}; ${counts}). Some workspace items could not be read, so `
|
|
536
|
-
+ "the returned list is not the whole workspace.");
|
|
537
|
-
}
|
|
538
|
-
if (env.omission_receipt?.has_omissions === true) {
|
|
539
|
-
const groups = env.omission_receipt?.omitted_field_groups ?? [];
|
|
540
|
-
throw new CommandError(`Refusing incomplete pm list-all answer: omission_receipt.has_omissions=true `
|
|
541
|
-
+ `(omitted_field_groups: ${groups.length ? groups.join(", ") : "(none listed)"}; ${counts}). `
|
|
542
|
-
+ "Field groups were dropped from the projection, so the rows are present "
|
|
543
|
-
+ "but degraded; re-read without the field-omitting options.");
|
|
544
|
-
}
|
|
545
|
-
// Last, and independent of the four flags: the envelope's own arithmetic.
|
|
546
|
-
// `count` and `total` are read above only to describe the other refusals, but
|
|
547
|
-
// they are themselves a completeness claim — an envelope reporting count 10 of
|
|
548
|
-
// total 682 with every flag clear is internally inconsistent, and trusting the
|
|
549
|
-
// flags over the arithmetic would consume 10 rows as the whole workspace. This
|
|
550
|
-
// is the case where the receipt contradicts itself, and the safe reading of a
|
|
551
|
-
// self-contradictory receipt is "not complete".
|
|
552
|
-
if (typeof env.count === "number" && typeof env.total === "number" && env.count !== env.total) {
|
|
553
|
-
throw new CommandError(`Refusing incomplete pm list-all answer: ${counts} disagree while every `
|
|
554
|
-
+ "completeness flag is clear. The envelope contradicts itself, so it cannot "
|
|
555
|
-
+ "be treated as the whole workspace.");
|
|
556
|
-
}
|
|
585
|
+
const record = asRecord(envelope);
|
|
586
|
+
const sdkFindings = inspectCompleteListResult(envelope).findings
|
|
587
|
+
.map((finding) => `${finding.code}: ${finding.message}`);
|
|
588
|
+
const findings = record === undefined
|
|
589
|
+
? sdkFindings
|
|
590
|
+
: [...sdkFindings, ...supplementalCompleteListFindings(record)];
|
|
591
|
+
if (findings.length > 0 || record === undefined) {
|
|
592
|
+
const count = record && typeof record.count === "number" ? record.count : "unknown";
|
|
593
|
+
const total = record && typeof record.total === "number" ? record.total : "unknown";
|
|
594
|
+
throw new CommandError(`pm list --all complete-corpus answer was refused: ${findings.join("; ")}; count=${count} of total=${total}. `
|
|
595
|
+
+ "A partial workspace read would corrupt import reconciliation or CSV export; retry the canonical strict unbounded read.");
|
|
596
|
+
}
|
|
597
|
+
return certifyCompleteListResult(record).items;
|
|
557
598
|
}
|
|
558
599
|
/** Real {@link PmSpawn} over `child_process.spawnSync`. */
|
|
559
600
|
function spawnPm(args, options) {
|
|
@@ -591,6 +632,10 @@ function decodeKeyTagValue(value) {
|
|
|
591
632
|
return value;
|
|
592
633
|
}
|
|
593
634
|
}
|
|
635
|
+
/** Normalize JavaScript's open-ended thrown-value channel for operator errors. */
|
|
636
|
+
function errorMessage(error) {
|
|
637
|
+
return error instanceof Error ? error.message : String(error);
|
|
638
|
+
}
|
|
594
639
|
/**
|
|
595
640
|
* Parse a `--map csvHeader=field` spec (repeatable / comma-joined) into a
|
|
596
641
|
* lookup from a normalized CSV header name to the canonical pm field name.
|
|
@@ -843,6 +888,19 @@ function readCSVFile(filePath, delimiter, encoding = "utf-8", skipHeaders = fals
|
|
|
843
888
|
const dataRows = rows.slice(1);
|
|
844
889
|
return { headers, dataRows };
|
|
845
890
|
}
|
|
891
|
+
/** Create the canonical empty receipt shared by every CSV import engine. */
|
|
892
|
+
function createImportResult() {
|
|
893
|
+
return {
|
|
894
|
+
imported: 0,
|
|
895
|
+
updated: 0,
|
|
896
|
+
skipped: 0,
|
|
897
|
+
filtered: 0,
|
|
898
|
+
autoMappings: [],
|
|
899
|
+
errors: [],
|
|
900
|
+
previews: [],
|
|
901
|
+
fieldMapWarnings: [],
|
|
902
|
+
};
|
|
903
|
+
}
|
|
846
904
|
/**
|
|
847
905
|
* Build the canonical field accessor for a header row + data row.
|
|
848
906
|
*/
|
|
@@ -918,24 +976,24 @@ function rowMatchesFilter(row, filter) {
|
|
|
918
976
|
}
|
|
919
977
|
/**
|
|
920
978
|
* List existing items once and build a lookup from csv-key provenance value to
|
|
921
|
-
* item id, for idempotent upsert. The list
|
|
979
|
+
* item id, for idempotent upsert. The complete-list answer must be COMPLETE — a
|
|
922
980
|
* truncated answer silently missing keys would turn upserts into duplicate
|
|
923
981
|
* creates — so the envelope's receipt is checked before any key is read.
|
|
924
982
|
*/
|
|
925
983
|
function loadKeyIndex(pmRoot, spawn = spawnPm) {
|
|
926
984
|
const index = new Map();
|
|
927
|
-
const result = spawn(
|
|
985
|
+
const result = spawn(completeListArgs(pmRoot), { encoding: "utf-8", maxBuffer: pmListMaxBuffer() });
|
|
928
986
|
// Route a stdout overrun (status: null, empty stderr, ENOBUFS) through
|
|
929
987
|
// assertPmOutputFit as a hard, named error instead of the generic
|
|
930
|
-
// "pm list
|
|
988
|
+
// "pm list --all failed" the old `result.error` branch reported. An overrun is
|
|
931
989
|
// a size problem, not a pm failure: a too-large workspace would otherwise
|
|
932
990
|
// surface as a bare Node message and send the operator to pm's stderr instead
|
|
933
991
|
// of to PM_LIST_MAX_BUFFER, and — worse — a null status combined with this
|
|
934
992
|
// package's `status !== 0` guard would degrade into an empty key index,
|
|
935
993
|
// turning every upsert into a duplicate create.
|
|
936
|
-
assertPmOutputFit(result, "list
|
|
994
|
+
assertPmOutputFit(result, "pm list --all");
|
|
937
995
|
if (result.status !== 0) {
|
|
938
|
-
throw new CommandError(result.stderr?.trim() || "pm list
|
|
996
|
+
throw new CommandError(result.stderr?.trim() || "pm list --all failed");
|
|
939
997
|
}
|
|
940
998
|
let items = [];
|
|
941
999
|
let parsed;
|
|
@@ -948,12 +1006,11 @@ function loadKeyIndex(pmRoot, spawn = spawnPm) {
|
|
|
948
1006
|
// DUPLICATES for every row whose key it could not see. That is the same
|
|
949
1007
|
// silent-partial failure the completeness gate below exists to stop, so the
|
|
950
1008
|
// two paths must agree.
|
|
951
|
-
throw new CommandError("Could not parse `pm list
|
|
1009
|
+
throw new CommandError("Could not parse `pm list --all --json` output while building the upsert key index. "
|
|
952
1010
|
+ "Refusing rather than treating an unreadable response as an empty workspace, "
|
|
953
1011
|
+ "which would make --upsert create duplicates.");
|
|
954
1012
|
}
|
|
955
|
-
assertListAllComplete(parsed);
|
|
956
|
-
items = parsed.items ?? [];
|
|
1013
|
+
items = assertListAllComplete(parsed);
|
|
957
1014
|
for (const item of items) {
|
|
958
1015
|
for (const tag of item.tags ?? []) {
|
|
959
1016
|
if (tag.startsWith(KEY_TAG_PREFIX)) {
|
|
@@ -1042,11 +1099,37 @@ function computeFieldMapWarnings(rawHeaders, fieldMap) {
|
|
|
1042
1099
|
console.error(`Warning: ${w}`);
|
|
1043
1100
|
return warnings;
|
|
1044
1101
|
}
|
|
1102
|
+
/** Resolve, validate, and report one raw import header set. */
|
|
1103
|
+
function resolveImportHeaders(rawHeaders, opts) {
|
|
1104
|
+
const fieldMapWarnings = computeFieldMapWarnings(rawHeaders, opts.fieldMap);
|
|
1105
|
+
const mapResolution = resolveImportFieldMap(rawHeaders, opts.fieldMap, opts.autoMap ?? false);
|
|
1106
|
+
const headers = applyFieldMap(rawHeaders, mapResolution.fieldMap);
|
|
1107
|
+
assertImportHeaders(headers, opts);
|
|
1108
|
+
return { headers, autoMappings: mapResolution.autoMappings, fieldMapWarnings };
|
|
1109
|
+
}
|
|
1110
|
+
/** Read and resolve the shared in-memory prelude for normal and atomic imports. */
|
|
1111
|
+
function prepareInMemoryImport(filePath, opts) {
|
|
1112
|
+
const { headers: rawHeaders, dataRows } = readCSVFile(filePath, opts.delimiter, opts.encoding ?? "utf-8", opts.skipHeaders ?? false);
|
|
1113
|
+
return {
|
|
1114
|
+
rawHeaders,
|
|
1115
|
+
dataRows,
|
|
1116
|
+
result: createImportResult(),
|
|
1117
|
+
resolvedHeaders: rawHeaders.length > 0 ? resolveImportHeaders(rawHeaders, opts) : undefined,
|
|
1118
|
+
};
|
|
1119
|
+
}
|
|
1120
|
+
/** Process one complete in-memory row set through the shared mutation core. */
|
|
1121
|
+
function processImportRows(pmRoot, dataRows, headers, opts, result) {
|
|
1122
|
+
const keyIndex = opts.keyField ? loadKeyIndex(pmRoot) : new Map();
|
|
1123
|
+
for (let rowIndex = 0; rowIndex < dataRows.length; rowIndex++) {
|
|
1124
|
+
const lineNo = opts.skipHeaders ? rowIndex + 1 : rowIndex + 2;
|
|
1125
|
+
processImportRow(pmRoot, headers, dataRows[rowIndex], lineNo, opts, keyIndex, result);
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1045
1128
|
/**
|
|
1046
1129
|
* Import CSV rows into a pm tracker as items.
|
|
1047
1130
|
*
|
|
1048
1131
|
* Rejects rather than throws when `--atomic` is combined with `--stream`: the
|
|
1049
|
-
* two are genuinely incompatible, because
|
|
1132
|
+
* two are genuinely incompatible, because a bounded transaction plan needs the
|
|
1050
1133
|
* whole row set in hand and a stream is unbounded. Returning a rejected promise
|
|
1051
1134
|
* keeps the failure on the same channel as the rest of the async path.
|
|
1052
1135
|
*
|
|
@@ -1058,7 +1141,7 @@ function computeFieldMapWarnings(rawHeaders, fieldMap) {
|
|
|
1058
1141
|
*/
|
|
1059
1142
|
function importCSV(pmRoot, filePath, opts) {
|
|
1060
1143
|
if (opts.atomic && opts.stream) {
|
|
1061
|
-
return Promise.reject(new CommandError("--atomic cannot be combined with --stream: an unbounded stream cannot be committed as one
|
|
1144
|
+
return Promise.reject(new CommandError("--atomic cannot be combined with --stream: an unbounded stream cannot be committed as one bounded transaction.", EXIT_CODE.USAGE));
|
|
1062
1145
|
}
|
|
1063
1146
|
if (opts.atomic)
|
|
1064
1147
|
return importCSVAtomic(pmRoot, filePath, opts);
|
|
@@ -1071,30 +1154,14 @@ function importCSV(pmRoot, filePath, opts) {
|
|
|
1071
1154
|
* every data row. Used when `--stream` is not set (the default).
|
|
1072
1155
|
*/
|
|
1073
1156
|
function importCSVInMemory(pmRoot, filePath, opts) {
|
|
1074
|
-
const {
|
|
1075
|
-
|
|
1076
|
-
imported: 0,
|
|
1077
|
-
updated: 0,
|
|
1078
|
-
skipped: 0,
|
|
1079
|
-
filtered: 0,
|
|
1080
|
-
autoMappings: [],
|
|
1081
|
-
errors: [],
|
|
1082
|
-
previews: [],
|
|
1083
|
-
fieldMapWarnings: [],
|
|
1084
|
-
};
|
|
1085
|
-
if (rawHeaders.length === 0)
|
|
1157
|
+
const { dataRows, result, resolvedHeaders } = prepareInMemoryImport(filePath, opts);
|
|
1158
|
+
if (!resolvedHeaders)
|
|
1086
1159
|
return result;
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
result.autoMappings =
|
|
1091
|
-
|
|
1092
|
-
// Pre-load the dedup index only when upserting (one extra pm call, not per-row).
|
|
1093
|
-
const keyIndex = opts.keyField && !opts.dryRun ? loadKeyIndex(pmRoot) : new Map();
|
|
1094
|
-
for (let rowIndex = 0; rowIndex < dataRows.length; rowIndex++) {
|
|
1095
|
-
const lineNo = opts.skipHeaders ? rowIndex + 1 : rowIndex + 2;
|
|
1096
|
-
processImportRow(pmRoot, headers, dataRows[rowIndex], lineNo, opts, keyIndex, result);
|
|
1097
|
-
}
|
|
1160
|
+
// Pre-load the dedup index whenever upserting, including dry-run: a preview
|
|
1161
|
+
// that labels an existing key as a create is not a truthful preview.
|
|
1162
|
+
result.fieldMapWarnings = resolvedHeaders.fieldMapWarnings;
|
|
1163
|
+
result.autoMappings = resolvedHeaders.autoMappings;
|
|
1164
|
+
processImportRows(pmRoot, dataRows, resolvedHeaders.headers, opts, result);
|
|
1098
1165
|
return result;
|
|
1099
1166
|
}
|
|
1100
1167
|
// ---------------------------------------------------------------------------
|
|
@@ -1161,17 +1228,20 @@ export function atomicTransactionId(filePath, opts = {}) {
|
|
|
1161
1228
|
* whose CSV `status` is `closed`/`canceled` is legitimately imported as a
|
|
1162
1229
|
* closed item (upsertCreate/upsertUpdate apply the row's status, routing
|
|
1163
1230
|
* terminal transitions through `pm close --reason`), so it must
|
|
1164
|
-
* still be recognized on resume and never re-imported. Conversely a
|
|
1165
|
-
* create must NOT be recognized:
|
|
1166
|
-
* `
|
|
1167
|
-
*
|
|
1168
|
-
*
|
|
1169
|
-
*
|
|
1231
|
+
* still be recognized on resume and never re-imported. Conversely a
|
|
1232
|
+
* successfully compensated create must NOT be recognized:
|
|
1233
|
+
* `compensateCreate()` removes the markers only after closure succeeds or the
|
|
1234
|
+
* item is already confirmed closed, so a compensated
|
|
1235
|
+
* tombstone carries no marker and a post-rollback retry re-imports its row. If
|
|
1236
|
+
* closure fails, both markers remain and make the orphan discoverable for
|
|
1237
|
+
* reconciliation. (This is why matching by marker-presence is correct for both
|
|
1238
|
+
* a legitimately-closed applied row and a rolled-back one; see
|
|
1239
|
+
* compensateCreate.)
|
|
1170
1240
|
*/
|
|
1171
1241
|
function loadAppliedByTransaction(pmRoot, transactionId, spawn = spawnPm) {
|
|
1172
1242
|
const byRowIndex = new Map();
|
|
1173
1243
|
const rowMarkerPrefix = `${TX_ROW_TAG_PREFIX}${transactionId}${TX_ROW_TAG_SEPARATOR}`;
|
|
1174
|
-
const r = spawn(
|
|
1244
|
+
const r = spawn(completeListArgs(pmRoot), {
|
|
1175
1245
|
encoding: "utf-8",
|
|
1176
1246
|
maxBuffer: pmListMaxBuffer(),
|
|
1177
1247
|
});
|
|
@@ -1179,17 +1249,18 @@ function loadAppliedByTransaction(pmRoot, transactionId, spawn = spawnPm) {
|
|
|
1179
1249
|
// the previous guard (`r.error || r.status !== 0`) then returned an EMPTY map,
|
|
1180
1250
|
// so inspect() concluded "nothing was applied" and a resumed atomic import
|
|
1181
1251
|
// silently re-imported every already-applied row — wrong data, not a crash.
|
|
1182
|
-
// Route
|
|
1183
|
-
//
|
|
1184
|
-
//
|
|
1185
|
-
|
|
1252
|
+
// Route unusable null-status output through {@link assertPmOutputFit} as a
|
|
1253
|
+
// hard, named error. The following non-zero branch rejects ordinary command failures.
|
|
1254
|
+
// Parse failures and incomplete envelopes below also refuse the resume scan;
|
|
1255
|
+
// none of these unreadable states may become an empty applied-row map.
|
|
1256
|
+
assertPmOutputFit(r, "pm list --all");
|
|
1186
1257
|
// Fail closed on both failure shapes, for the same reason the completeness
|
|
1187
1258
|
// gate below refuses: an empty resume map says "nothing was applied", so a
|
|
1188
1259
|
// resume RE-IMPORTS rows that already landed. An unreadable answer is not an
|
|
1189
1260
|
// empty answer.
|
|
1190
1261
|
if (r.status !== 0) {
|
|
1191
1262
|
throw new CommandError(r.stderr?.trim()
|
|
1192
|
-
|| "pm list
|
|
1263
|
+
|| "pm list --all failed while scanning for applied rows. Refusing rather than "
|
|
1193
1264
|
+ "treating an unreadable response as an empty transaction, which would re-import applied rows.");
|
|
1194
1265
|
}
|
|
1195
1266
|
let items = [];
|
|
@@ -1198,15 +1269,14 @@ function loadAppliedByTransaction(pmRoot, transactionId, spawn = spawnPm) {
|
|
|
1198
1269
|
parsed = JSON.parse(r.stdout);
|
|
1199
1270
|
}
|
|
1200
1271
|
catch {
|
|
1201
|
-
throw new CommandError("Could not parse `pm list
|
|
1272
|
+
throw new CommandError("Could not parse `pm list --all --json` output while scanning for applied rows. "
|
|
1202
1273
|
+ "Refusing rather than treating an unreadable response as an empty transaction, "
|
|
1203
1274
|
+ "which would re-import rows that already landed.");
|
|
1204
1275
|
}
|
|
1205
1276
|
// An incomplete envelope must refuse here, NOT degrade to an empty map: a
|
|
1206
1277
|
// resume scan that misses applied rows silently re-imports them (wrong
|
|
1207
1278
|
// data, the exact failure this package exists to prevent).
|
|
1208
|
-
assertListAllComplete(parsed);
|
|
1209
|
-
items = parsed.items ?? [];
|
|
1279
|
+
items = assertListAllComplete(parsed);
|
|
1210
1280
|
for (const item of items) {
|
|
1211
1281
|
for (const tag of item.tags ?? []) {
|
|
1212
1282
|
if (!tag.startsWith(rowMarkerPrefix))
|
|
@@ -1236,8 +1306,8 @@ function loadAppliedByTransaction(pmRoot, transactionId, spawn = spawnPm) {
|
|
|
1236
1306
|
* non-zero exit (item truly absent, or a failed lookup such as the
|
|
1237
1307
|
* `fake-get-fail` regression) still returns `undefined` unchanged.
|
|
1238
1308
|
*/
|
|
1239
|
-
function itemStatus(pmRoot, id) {
|
|
1240
|
-
const r =
|
|
1309
|
+
function itemStatus(pmRoot, id, spawn = spawnPm) {
|
|
1310
|
+
const r = spawn(["--path", pmRoot, "get", id, "--json"], {
|
|
1241
1311
|
encoding: "utf-8",
|
|
1242
1312
|
maxBuffer: pmListMaxBuffer(),
|
|
1243
1313
|
});
|
|
@@ -1253,18 +1323,32 @@ function itemStatus(pmRoot, id) {
|
|
|
1253
1323
|
}
|
|
1254
1324
|
}
|
|
1255
1325
|
/**
|
|
1256
|
-
*
|
|
1257
|
-
*
|
|
1258
|
-
*
|
|
1326
|
+
* Read an item's status without letting a compensation-only lookup abort the
|
|
1327
|
+
* remaining best-effort sweep. `undefined` deliberately means either absent or
|
|
1328
|
+
* unverified; neither is positive evidence that an open item became closed.
|
|
1329
|
+
*/
|
|
1330
|
+
function compensationItemStatus(pmRoot, id) {
|
|
1331
|
+
try {
|
|
1332
|
+
return itemStatus(pmRoot, id);
|
|
1333
|
+
}
|
|
1334
|
+
catch {
|
|
1335
|
+
return undefined;
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
/**
|
|
1339
|
+
* Idempotently undo one row's create. First closes a still-open item via
|
|
1340
|
+
* `pm close` (NOT delete, to avoid the known history-resurrection issue), then
|
|
1341
|
+
* strips this transaction's ownership markers (`markers`) only after the item
|
|
1342
|
+
* is confirmed closed.
|
|
1259
1343
|
*
|
|
1260
|
-
* Stripping the markers
|
|
1261
|
-
* signal: a
|
|
1262
|
-
* retry re-imports it, while a *successfully* imported row (even
|
|
1263
|
-
* status is `closed`/`canceled`) keeps its marker and is resumed
|
|
1264
|
-
*
|
|
1265
|
-
*
|
|
1266
|
-
*
|
|
1267
|
-
*
|
|
1344
|
+
* Stripping the markers after closure makes marker-presence a correct
|
|
1345
|
+
* reconciliation signal: a compensated row no longer carries a marker, so a
|
|
1346
|
+
* post-rollback retry re-imports it, while a *successfully* imported row (even
|
|
1347
|
+
* one whose CSV status is `closed`/`canceled`) keeps its marker and is resumed
|
|
1348
|
+
* idempotently. If the close fails, the markers deliberately remain so the open
|
|
1349
|
+
* orphan can be found by transaction id and reconciled before retrying. The
|
|
1350
|
+
* strip also runs for an item that was already closed; every step tolerates a
|
|
1351
|
+
* missing item / absent tag, so repeated compensation is a safe no-op.
|
|
1268
1352
|
*
|
|
1269
1353
|
* The status lookup itself is also tolerated: if `itemStatus` throws (a stdout
|
|
1270
1354
|
* overrun or a signal kill on the `pm get`), the throw is caught and this row's
|
|
@@ -1275,38 +1359,35 @@ function itemStatus(pmRoot, id) {
|
|
|
1275
1359
|
* and leaving every later applied create un-compensated.
|
|
1276
1360
|
*/
|
|
1277
1361
|
function compensateCreate(pmRoot, id, markers = [], reason = "atomic csv import rolled back") {
|
|
1278
|
-
//
|
|
1279
|
-
//
|
|
1280
|
-
//
|
|
1281
|
-
|
|
1282
|
-
// CommandError from assertPmOutputFit; a bare catch is therefore equivalent to
|
|
1283
|
-
// catching CommandError and avoids an unreachable rethrow branch.)
|
|
1284
|
-
let status;
|
|
1285
|
-
try {
|
|
1286
|
-
status = itemStatus(pmRoot, id);
|
|
1287
|
-
}
|
|
1288
|
-
catch {
|
|
1289
|
-
return; // status lookup overran/killed: no-op for this row, sweep continues
|
|
1290
|
-
}
|
|
1362
|
+
// A failed status lookup is a no-op for this row and must never abort the
|
|
1363
|
+
// sweep over the remaining applied creates. The item remains marked for
|
|
1364
|
+
// reconciliation when it still exists.
|
|
1365
|
+
const status = compensationItemStatus(pmRoot, id);
|
|
1291
1366
|
if (status === undefined)
|
|
1292
|
-
return; //
|
|
1367
|
+
return; // absent or unverified; never invent closure
|
|
1368
|
+
if (status !== "closed") {
|
|
1369
|
+
// Compensation is an internal rollback, not a user closure: it must
|
|
1370
|
+
// reliably undo the create regardless of closure-validation governance, so
|
|
1371
|
+
// it bypasses `--validate-close` (off). `require_close_reason` still applies
|
|
1372
|
+
// and is satisfied by `reason`. Used by both the atomic transaction path
|
|
1373
|
+
// (rollback) and the non-atomic `upsertCreate` close-failure path.
|
|
1374
|
+
const closeResult = spawnSync("pm", ["--path", pmRoot, "close", id, "--reason", reason, "--validate-close", "off"], { encoding: "utf-8", maxBuffer: pmListMaxBuffer() });
|
|
1375
|
+
if (closeResult.status !== 0) {
|
|
1376
|
+
// A non-zero receipt is not proof either way: the close may have persisted
|
|
1377
|
+
// before its receipt failed. Re-read the item and clean markers only with
|
|
1378
|
+
// exact terminal evidence; otherwise retain identity for reconciliation.
|
|
1379
|
+
const postCloseStatus = compensationItemStatus(pmRoot, id);
|
|
1380
|
+
if (postCloseStatus !== "closed") {
|
|
1381
|
+
console.error(`atomic import: compensation close failed for ${id}: ${closeResult.stderr?.trim() || closeResult.stdout?.trim()}`);
|
|
1382
|
+
return;
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1293
1386
|
if (markers.length > 0) {
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
return; // terminal already; markers stripped above
|
|
1299
|
-
// Compensation is an internal rollback, not a user closure: it must reliably
|
|
1300
|
-
// undo the create regardless of closure-validation governance, so it bypasses
|
|
1301
|
-
// `--validate-close` (off). `require_close_reason` still applies and is
|
|
1302
|
-
// satisfied by `reason`. Without this a strict tracker would block the
|
|
1303
|
-
// rollback and leave the orphan open — the very leak compensation exists to
|
|
1304
|
-
// prevent. Used by both the atomic transaction path (rollback) and the
|
|
1305
|
-
// non-atomic `upsertCreate` close-failure path.
|
|
1306
|
-
const r = spawnSync("pm", ["--path", pmRoot, "close", id, "--reason", reason, "--validate-close", "off"], { encoding: "utf-8", maxBuffer: pmListMaxBuffer() });
|
|
1307
|
-
if (r.status !== 0 && r.status !== 4) {
|
|
1308
|
-
// Exit 4 is "already closed" / invalid state — treat as already compensated.
|
|
1309
|
-
console.error(`atomic import: compensation close failed for ${id}: ${r.stderr?.trim() || r.stdout?.trim()}`);
|
|
1387
|
+
const markerCleanup = spawnSync("pm", ["--path", pmRoot, "update", id, "--remove-tags", markers.join(",")], { encoding: "utf-8", maxBuffer: pmListMaxBuffer() });
|
|
1388
|
+
if (markerCleanup.status !== 0) {
|
|
1389
|
+
console.error(`atomic import: compensation marker cleanup failed for ${id}: ${markerCleanup.stderr?.trim() || markerCleanup.stdout?.trim()}`);
|
|
1390
|
+
}
|
|
1310
1391
|
}
|
|
1311
1392
|
}
|
|
1312
1393
|
/**
|
|
@@ -1314,40 +1395,28 @@ function compensateCreate(pmRoot, id, markers = [], reason = "atomic csv import
|
|
|
1314
1395
|
* {@link WorkspaceTransactionStep} committed under a single workspace
|
|
1315
1396
|
* writer-locked, crash-recoverable transaction. On success the same
|
|
1316
1397
|
* {@link ImportResult} as the non-atomic path is returned (counts derived from
|
|
1317
|
-
* the committed step results). On failure
|
|
1318
|
-
*
|
|
1319
|
-
*
|
|
1320
|
-
*
|
|
1398
|
+
* the committed step results). On failure the coordinator attempts reverse-
|
|
1399
|
+
* order compensation, but callers receive a conservative reconciliation
|
|
1400
|
+
* warning because create cleanup is best-effort and updates to pre-existing
|
|
1401
|
+
* items are intentionally not reverted. An interrupted run resumes from the
|
|
1402
|
+
* durable journal (inspect() skips already applied rows).
|
|
1321
1403
|
*/
|
|
1322
1404
|
async function importCSVAtomic(pmRoot, filePath, opts) {
|
|
1323
|
-
const {
|
|
1324
|
-
|
|
1325
|
-
imported: 0,
|
|
1326
|
-
updated: 0,
|
|
1327
|
-
skipped: 0,
|
|
1328
|
-
filtered: 0,
|
|
1329
|
-
autoMappings: [],
|
|
1330
|
-
errors: [],
|
|
1331
|
-
previews: [],
|
|
1332
|
-
fieldMapWarnings: [],
|
|
1333
|
-
};
|
|
1334
|
-
if (rawHeaders.length === 0)
|
|
1405
|
+
const { rawHeaders, dataRows, result, resolvedHeaders } = prepareInMemoryImport(filePath, opts);
|
|
1406
|
+
if (!resolvedHeaders)
|
|
1335
1407
|
return result;
|
|
1336
|
-
result.fieldMapWarnings =
|
|
1337
|
-
|
|
1338
|
-
const headers = applyFieldMap(rawHeaders, mapResolution.fieldMap);
|
|
1339
|
-
result.autoMappings = mapResolution.autoMappings;
|
|
1340
|
-
assertImportHeaders(headers, opts);
|
|
1408
|
+
result.fieldMapWarnings = resolvedHeaders.fieldMapWarnings;
|
|
1409
|
+
result.autoMappings = resolvedHeaders.autoMappings;
|
|
1341
1410
|
// A dry-run atomic import just previews like the non-atomic path (no
|
|
1342
1411
|
// transaction is committed, nothing is written).
|
|
1343
1412
|
if (opts.dryRun) {
|
|
1344
|
-
|
|
1345
|
-
for (let rowIndex = 0; rowIndex < dataRows.length; rowIndex++) {
|
|
1346
|
-
const lineNo = opts.skipHeaders ? rowIndex + 1 : rowIndex + 2;
|
|
1347
|
-
processImportRow(pmRoot, headers, dataRows[rowIndex], lineNo, opts, keyIndex, result);
|
|
1348
|
-
}
|
|
1413
|
+
processImportRows(pmRoot, dataRows, resolvedHeaders.headers, opts, result);
|
|
1349
1414
|
return result;
|
|
1350
1415
|
}
|
|
1416
|
+
const commitTransaction = opts.commitTransaction;
|
|
1417
|
+
if (!commitTransaction) {
|
|
1418
|
+
throw new CommandError("--atomic requires the host-injected commitWorkspaceTransaction SDK primitive. Upgrade the pm CLI and invoke pm-csv through the pm host.", EXIT_CODE.USAGE);
|
|
1419
|
+
}
|
|
1351
1420
|
// Fingerprint the exact parsed content (raw headers + all data rows) so that
|
|
1352
1421
|
// editing the file in place yields a fresh transaction id instead of reusing
|
|
1353
1422
|
// stale per-row markers from a previous import of different content.
|
|
@@ -1372,7 +1441,7 @@ async function importCSVAtomic(pmRoot, filePath, opts) {
|
|
|
1372
1441
|
const row = dataRows[rowIndex];
|
|
1373
1442
|
// Compute the field accessor once and reuse for both `parsed` and the
|
|
1374
1443
|
// --key value (previously rowFields() was called twice per row).
|
|
1375
|
-
const { get, parsed } = rowFields(headers, row);
|
|
1444
|
+
const { get, parsed } = rowFields(resolvedHeaders.headers, row);
|
|
1376
1445
|
if (!parsed.title) {
|
|
1377
1446
|
console.error(`Row ${lineNo}: skipping — 'title' is empty`);
|
|
1378
1447
|
result.skipped++;
|
|
@@ -1383,7 +1452,7 @@ async function importCSVAtomic(pmRoot, filePath, opts) {
|
|
|
1383
1452
|
result.filtered++;
|
|
1384
1453
|
continue;
|
|
1385
1454
|
}
|
|
1386
|
-
const keyValue = opts.keyField ?
|
|
1455
|
+
const keyValue = opts.keyField ? get(opts.keyField) : "";
|
|
1387
1456
|
const normalizedKey = keyValue ? normalizeKeyValue(keyValue) : "";
|
|
1388
1457
|
// In-batch duplicate-key guard: an earlier planned CREATE in this run
|
|
1389
1458
|
// already claimed this key. The first item does not exist yet at apply
|
|
@@ -1406,8 +1475,8 @@ async function importCSVAtomic(pmRoot, filePath, opts) {
|
|
|
1406
1475
|
// compensate() closes the created id.
|
|
1407
1476
|
// - update (--key match): apply() updates the pre-existing item and stamps
|
|
1408
1477
|
// both markers; compensate() is a no-op (an arbitrary update cannot be
|
|
1409
|
-
// safely reverted without capturing prior state
|
|
1410
|
-
//
|
|
1478
|
+
// safely reverted without capturing prior state; best-effort
|
|
1479
|
+
// compensation is therefore scoped to creates).
|
|
1411
1480
|
//
|
|
1412
1481
|
// Resume/compensation matching is per-row-precise via the per-row marker
|
|
1413
1482
|
// (csv-txrow:<transactionId>#<rowIndex>), so duplicate titles or duplicate
|
|
@@ -1443,8 +1512,6 @@ async function importCSVAtomic(pmRoot, filePath, opts) {
|
|
|
1443
1512
|
return row.existingId;
|
|
1444
1513
|
}
|
|
1445
1514
|
const newId = upsertCreate(pmRoot, row.parsed, opts.keyField ? row.keyValue : undefined, opts.source, ownershipTags);
|
|
1446
|
-
if (!newId)
|
|
1447
|
-
throw new Error(`pm create returned no id for row ${row.lineNo}`);
|
|
1448
1515
|
appliedId = newId;
|
|
1449
1516
|
return newId;
|
|
1450
1517
|
};
|
|
@@ -1464,17 +1531,17 @@ async function importCSVAtomic(pmRoot, filePath, opts) {
|
|
|
1464
1531
|
};
|
|
1465
1532
|
return { id: stepId, inspect, apply, prepareCompensation, compensate };
|
|
1466
1533
|
});
|
|
1467
|
-
const commitTransaction = opts.commitTransaction ??
|
|
1468
|
-
(await resolveCommitWorkspaceTransaction(pmRoot));
|
|
1469
1534
|
let committed;
|
|
1470
1535
|
try {
|
|
1471
1536
|
committed = await commitTransaction({ transactionId, author, steps });
|
|
1472
1537
|
}
|
|
1473
1538
|
catch (err) {
|
|
1474
|
-
const msg =
|
|
1475
|
-
//
|
|
1476
|
-
//
|
|
1477
|
-
|
|
1539
|
+
const msg = errorMessage(err);
|
|
1540
|
+
// The coordinator invokes compensations in reverse order, but this package
|
|
1541
|
+
// cannot prove a clean tracker: create cleanup deliberately tolerates an
|
|
1542
|
+
// unavailable status lookup or failed close so the remaining sweep can
|
|
1543
|
+
// continue, and update steps intentionally preserve pre-existing items.
|
|
1544
|
+
throw new CommandError(`Atomic CSV import failed. The transaction coordinator attempted reverse-order compensation, but pm-csv cannot claim a clean tracker: created items are closed on a best-effort basis, and pre-existing item updates are intentionally not reverted. Mutations from transaction ${transactionId} may remain; inspect and reconcile that transaction before retrying. Underlying error: ${msg}`, EXIT_CODE.GENERIC_FAILURE);
|
|
1478
1545
|
}
|
|
1479
1546
|
// Tally the ImportResult from the committed step results. Each committed
|
|
1480
1547
|
// result is the created/updated item id. Only rows that were NOT already
|
|
@@ -1496,47 +1563,6 @@ async function importCSVAtomic(pmRoot, filePath, opts) {
|
|
|
1496
1563
|
}
|
|
1497
1564
|
return result;
|
|
1498
1565
|
}
|
|
1499
|
-
/**
|
|
1500
|
-
* Cached SDK commit coordinator. Resolved once per process via a dynamic
|
|
1501
|
-
* `import("@unbrained/pm-cli/sdk")` so repeated --atomic calls don't re-import.
|
|
1502
|
-
* `null` means a previous attempt failed and the failure is not retried within
|
|
1503
|
-
* this process (each CLI invocation is a fresh process, so this is safe).
|
|
1504
|
-
*/
|
|
1505
|
-
let cachedCommitWorkspaceTransaction;
|
|
1506
|
-
/**
|
|
1507
|
-
* Dynamically resolve the SDK commit coordinator bound to a tracker root, for
|
|
1508
|
-
* the importer path (which has no host-injected `ctx.sdk`). Falls back to a
|
|
1509
|
-
* dynamic `import("@unbrained/pm-cli/sdk")` so a standalone-installed extension
|
|
1510
|
-
* still works when the SDK package is resolvable. The resolved function is
|
|
1511
|
-
* cached at module scope so repeated --atomic calls don't re-import. If the
|
|
1512
|
-
* import fails or `commitWorkspaceTransaction` is not exported, a clear,
|
|
1513
|
-
* actionable CommandError is thrown.
|
|
1514
|
-
*/
|
|
1515
|
-
async function resolveCommitWorkspaceTransaction(pmRoot) {
|
|
1516
|
-
if (cachedCommitWorkspaceTransaction === null) {
|
|
1517
|
-
throw new CommandError("--atomic requires @unbrained/pm-cli>=2026.7.19 with the commitWorkspaceTransaction SDK primitive, but it could not be resolved (a prior attempt in this process failed). Ensure @unbrained/pm-cli is installed and up to date.", EXIT_CODE.USAGE);
|
|
1518
|
-
}
|
|
1519
|
-
if (cachedCommitWorkspaceTransaction) {
|
|
1520
|
-
const cached = cachedCommitWorkspaceTransaction;
|
|
1521
|
-
return (opts) => cached({ pmRoot, ...opts });
|
|
1522
|
-
}
|
|
1523
|
-
let mod;
|
|
1524
|
-
try {
|
|
1525
|
-
mod = await import("@unbrained/pm-cli/sdk");
|
|
1526
|
-
}
|
|
1527
|
-
catch (err) {
|
|
1528
|
-
cachedCommitWorkspaceTransaction = null;
|
|
1529
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
1530
|
-
throw new CommandError(`--atomic requires @unbrained/pm-cli>=2026.7.19 with the commitWorkspaceTransaction SDK primitive, but the SDK could not be imported: ${msg}. Install or upgrade @unbrained/pm-cli.`, EXIT_CODE.USAGE);
|
|
1531
|
-
}
|
|
1532
|
-
const commit = mod.commitWorkspaceTransaction;
|
|
1533
|
-
if (typeof commit !== "function") {
|
|
1534
|
-
cachedCommitWorkspaceTransaction = null;
|
|
1535
|
-
throw new CommandError("--atomic requires @unbrained/pm-cli>=2026.7.19 with the commitWorkspaceTransaction SDK primitive, but the installed SDK does not export it as a function. Upgrade @unbrained/pm-cli to >=2026.7.19.", EXIT_CODE.USAGE);
|
|
1536
|
-
}
|
|
1537
|
-
cachedCommitWorkspaceTransaction = commit;
|
|
1538
|
-
return (opts) => commit({ pmRoot, ...opts });
|
|
1539
|
-
}
|
|
1540
1566
|
/**
|
|
1541
1567
|
* Streaming import: reads the file via a readable stream so large CSV files
|
|
1542
1568
|
* are never fully loaded into memory. The header row (or positional
|
|
@@ -1544,32 +1570,21 @@ async function resolveCommitWorkspaceTransaction(pmRoot) {
|
|
|
1544
1570
|
* first row, then every subsequent row is processed and upserted immediately.
|
|
1545
1571
|
*/
|
|
1546
1572
|
async function importCSVStreaming(pmRoot, filePath, opts) {
|
|
1547
|
-
const result =
|
|
1548
|
-
imported: 0,
|
|
1549
|
-
updated: 0,
|
|
1550
|
-
skipped: 0,
|
|
1551
|
-
filtered: 0,
|
|
1552
|
-
autoMappings: [],
|
|
1553
|
-
errors: [],
|
|
1554
|
-
previews: [],
|
|
1555
|
-
fieldMapWarnings: [],
|
|
1556
|
-
};
|
|
1573
|
+
const result = createImportResult();
|
|
1557
1574
|
let headers = [];
|
|
1558
1575
|
let headerResolved = opts.skipHeaders ?? false;
|
|
1559
1576
|
let logicalRowIndex = 0;
|
|
1560
1577
|
if (headerResolved) {
|
|
1561
1578
|
// --skip-headers: no header row in the file; use positional column order.
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
result.autoMappings = mapResolution.autoMappings;
|
|
1567
|
-
assertImportHeaders(headers, opts);
|
|
1579
|
+
const resolvedHeaders = resolveImportHeaders([...IMPORT_COLUMNS], opts);
|
|
1580
|
+
headers = resolvedHeaders.headers;
|
|
1581
|
+
result.fieldMapWarnings = resolvedHeaders.fieldMapWarnings;
|
|
1582
|
+
result.autoMappings = resolvedHeaders.autoMappings;
|
|
1568
1583
|
}
|
|
1569
1584
|
// Load only after the input header has been validated so malformed input
|
|
1570
1585
|
// reports its CSV error before any workspace lookup is attempted.
|
|
1571
1586
|
let keyIndex = new Map();
|
|
1572
|
-
if (headerResolved && opts.keyField
|
|
1587
|
+
if (headerResolved && opts.keyField)
|
|
1573
1588
|
keyIndex = loadKeyIndex(pmRoot);
|
|
1574
1589
|
await streamCSVFile(filePath, opts.delimiter, opts.encoding ?? "utf-8", (row) => {
|
|
1575
1590
|
logicalRowIndex++;
|
|
@@ -1577,14 +1592,12 @@ async function importCSVStreaming(pmRoot, filePath, opts) {
|
|
|
1577
1592
|
if (!row.some((f) => f.trim() !== ""))
|
|
1578
1593
|
return;
|
|
1579
1594
|
if (!headerResolved) {
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
result.autoMappings = mapResolution.autoMappings;
|
|
1585
|
-
assertImportHeaders(headers, opts);
|
|
1595
|
+
const resolvedHeaders = resolveImportHeaders(row.map((header) => header.trim().toLowerCase()), opts);
|
|
1596
|
+
headers = resolvedHeaders.headers;
|
|
1597
|
+
result.fieldMapWarnings = resolvedHeaders.fieldMapWarnings;
|
|
1598
|
+
result.autoMappings = resolvedHeaders.autoMappings;
|
|
1586
1599
|
headerResolved = true;
|
|
1587
|
-
if (opts.keyField
|
|
1600
|
+
if (opts.keyField)
|
|
1588
1601
|
keyIndex = loadKeyIndex(pmRoot);
|
|
1589
1602
|
return;
|
|
1590
1603
|
}
|
|
@@ -1610,6 +1623,18 @@ function appendRelationalArgs(args, p) {
|
|
|
1610
1623
|
if (p.blocked_by)
|
|
1611
1624
|
args.push("--blocked-by", p.blocked_by);
|
|
1612
1625
|
}
|
|
1626
|
+
/** Append item fields shared by create and update mutation commands. */
|
|
1627
|
+
function appendMutableItemArgs(args, p, preserveEmptyBody) {
|
|
1628
|
+
if (preserveEmptyBody ? p.body !== undefined : Boolean(p.body))
|
|
1629
|
+
args.push("--body", p.body ?? "");
|
|
1630
|
+
if (p.priority !== undefined)
|
|
1631
|
+
args.push("--priority", String(p.priority));
|
|
1632
|
+
if (p.type)
|
|
1633
|
+
args.push("--type", p.type);
|
|
1634
|
+
if (p.deadline)
|
|
1635
|
+
args.push("--deadline", p.deadline);
|
|
1636
|
+
appendRelationalArgs(args, p);
|
|
1637
|
+
}
|
|
1613
1638
|
/**
|
|
1614
1639
|
* Build the `pm close --reason` text for an imported row whose CSV status is
|
|
1615
1640
|
* terminal (`closed`/`canceled`). Since pm-cli 2026.8.3 the CLI enforces
|
|
@@ -1647,14 +1672,14 @@ function importCloseReason(status, source) {
|
|
|
1647
1672
|
* open item is left behind in this one unrecoverable case.
|
|
1648
1673
|
* 2. `pm close` failed after the item was persisted as open. The orphan is
|
|
1649
1674
|
* compensated (closed) via {@link compensateCreate} with `--validate-close
|
|
1650
|
-
* off`, so a
|
|
1675
|
+
* off`, so a verified cleanup leaves no open orphan and a retry re-imports
|
|
1676
|
+
* it. If
|
|
1651
1677
|
* compensation also fails, the thrown error carries the created id and
|
|
1652
1678
|
* states the item was left open, so the partial state is actionable and a
|
|
1653
1679
|
* retry can reconcile it.
|
|
1654
1680
|
*
|
|
1655
|
-
*
|
|
1656
|
-
*
|
|
1657
|
-
* `pm create` reported no id.
|
|
1681
|
+
* An id-less successful receipt is therefore a hard error for every status,
|
|
1682
|
+
* never a silent successful import.
|
|
1658
1683
|
*/
|
|
1659
1684
|
function upsertCreate(pmRoot, p, keyValue, source, extraTags) {
|
|
1660
1685
|
const tags = [...p.tags];
|
|
@@ -1674,15 +1699,7 @@ function upsertCreate(pmRoot, p, keyValue, source, extraTags) {
|
|
|
1674
1699
|
// after the create, never via `pm create --status closed`.
|
|
1675
1700
|
const createStatus = p.status === "closed" ? "open" : p.status;
|
|
1676
1701
|
const args = ["--path", pmRoot, "create", "--title", p.title, "--status", createStatus, "--json"];
|
|
1677
|
-
|
|
1678
|
-
args.push("--body", p.body);
|
|
1679
|
-
if (p.priority !== undefined)
|
|
1680
|
-
args.push("--priority", String(p.priority));
|
|
1681
|
-
if (p.type)
|
|
1682
|
-
args.push("--type", p.type);
|
|
1683
|
-
if (p.deadline)
|
|
1684
|
-
args.push("--deadline", p.deadline);
|
|
1685
|
-
appendRelationalArgs(args, p);
|
|
1702
|
+
appendMutableItemArgs(args, p, false);
|
|
1686
1703
|
if (tags.length > 0)
|
|
1687
1704
|
args.push("--tags", tags.join(","));
|
|
1688
1705
|
const r = spawnSync("pm", args, { encoding: "utf-8", maxBuffer: pmListMaxBuffer() });
|
|
@@ -1747,14 +1764,10 @@ function upsertCreate(pmRoot, p, keyValue, source, extraTags) {
|
|
|
1747
1764
|
}
|
|
1748
1765
|
if (r.status !== 0)
|
|
1749
1766
|
throw new Error(r.stderr?.trim() || "pm create failed");
|
|
1767
|
+
if (!id) {
|
|
1768
|
+
throw new Error(`pm create succeeded for row (title '${p.title}', source status '${p.status}') but returned no id, so the source status cannot be applied and the created item cannot be identified or safely reconciled; it may have been left as an orphan — reconcile manually`);
|
|
1769
|
+
}
|
|
1750
1770
|
if (p.status === "closed") {
|
|
1751
|
-
// (1) No id recovered: the close cannot be applied and the orphan cannot be
|
|
1752
|
-
// compensated without its id. Hard failure — never a silent success — so
|
|
1753
|
-
// the row is reported as failed, not imported. The created open orphan is
|
|
1754
|
-
// unrecoverable from here; the error names the title for manual reconcile.
|
|
1755
|
-
if (!id) {
|
|
1756
|
-
throw new Error(`pm create succeeded for closed row (title '${p.title}') but returned no id, so the terminal 'closed' status cannot be applied and the created open item cannot be compensated; it is left as an unrecoverable open orphan — reconcile manually`);
|
|
1757
|
-
}
|
|
1758
1771
|
const cr = spawnSync("pm", ["--path", pmRoot, "close", id, "--reason", importCloseReason("closed", source)], { encoding: "utf-8", maxBuffer: pmListMaxBuffer() });
|
|
1759
1772
|
// Deliberately NO assertPmOutputFit(cr, "close") here: a null close status
|
|
1760
1773
|
// (receipt overrun or a signal kill) must route INTO the compensation branch
|
|
@@ -1763,8 +1776,8 @@ function upsertCreate(pmRoot, p, keyValue, source, extraTags) {
|
|
|
1763
1776
|
// way, and the id is already known so nothing is stranded.
|
|
1764
1777
|
if (cr.status !== 0) {
|
|
1765
1778
|
// (2) The close failed but the item is already persisted as open.
|
|
1766
|
-
// Compensate (close) the orphan so a
|
|
1767
|
-
// retry without --key-field
|
|
1779
|
+
// Compensate (close) the orphan so a verified cleanup leaves no open item
|
|
1780
|
+
// that a retry without --key-field could duplicate.
|
|
1768
1781
|
// Compensation bypasses closure-validation governance (it is a rollback,
|
|
1769
1782
|
// not a user closure) so a strict tracker cannot block the cleanup.
|
|
1770
1783
|
compensateCreate(pmRoot, id, [], "csv import row failed; closing orphan created item");
|
|
@@ -1782,7 +1795,7 @@ function upsertCreate(pmRoot, p, keyValue, source, extraTags) {
|
|
|
1782
1795
|
: `also failed: the item was created and left OPEN (status '${compensatedStatus}')`;
|
|
1783
1796
|
throw new Error(`pm close failed for closed row (title '${p.title}', id ${id}) and compensation ${state} — retry or reconcile by id. ${closeErr}`);
|
|
1784
1797
|
}
|
|
1785
|
-
throw new Error(`pm close failed for closed row (title '${p.title}', id ${id}); the created open item was compensated (verified closed) so
|
|
1798
|
+
throw new Error(`pm close failed for closed row (title '${p.title}', id ${id}); the created open item was compensated (verified closed), so no open orphan remains. ${closeErr}`);
|
|
1786
1799
|
}
|
|
1787
1800
|
}
|
|
1788
1801
|
return id;
|
|
@@ -1808,15 +1821,7 @@ function upsertCreate(pmRoot, p, keyValue, source, extraTags) {
|
|
|
1808
1821
|
*/
|
|
1809
1822
|
function upsertUpdate(pmRoot, id, p, source, extraTags) {
|
|
1810
1823
|
const args = ["--path", pmRoot, "update", id, "--title", p.title];
|
|
1811
|
-
|
|
1812
|
-
args.push("--body", p.body);
|
|
1813
|
-
if (p.priority !== undefined)
|
|
1814
|
-
args.push("--priority", String(p.priority));
|
|
1815
|
-
if (p.type)
|
|
1816
|
-
args.push("--type", p.type);
|
|
1817
|
-
if (p.deadline)
|
|
1818
|
-
args.push("--deadline", p.deadline);
|
|
1819
|
-
appendRelationalArgs(args, p);
|
|
1824
|
+
appendMutableItemArgs(args, p, true);
|
|
1820
1825
|
// Preserve the csv-key tag (additive) and refresh the user tags.
|
|
1821
1826
|
const addTags = [...p.tags];
|
|
1822
1827
|
if (source)
|
|
@@ -2008,6 +2013,47 @@ function assertStrictImportReady(filePath, opts) {
|
|
|
2008
2013
|
}
|
|
2009
2014
|
return report;
|
|
2010
2015
|
}
|
|
2016
|
+
/** Parse one host option record into the canonical CSV import contract. */
|
|
2017
|
+
function parseCsvImportInvocation(rawOptions, dryRun) {
|
|
2018
|
+
return {
|
|
2019
|
+
strict: readBoolOption(rawOptions, "strict"),
|
|
2020
|
+
options: {
|
|
2021
|
+
delimiter: resolveDelimiter(rawOptions["delimiter"]),
|
|
2022
|
+
dryRun,
|
|
2023
|
+
fieldMap: parseFieldMap(rawOptions["map"]),
|
|
2024
|
+
autoMap: readBoolOption(rawOptions, "auto-map", "autoMap"),
|
|
2025
|
+
keyField: (rawOptions["key"] ?? "").trim().toLowerCase() || undefined,
|
|
2026
|
+
encoding: resolveEncoding(rawOptions["encoding"]),
|
|
2027
|
+
source: (rawOptions["source"] ?? "").trim() || undefined,
|
|
2028
|
+
filter: parseImportFilter(rawOptions["status"], rawOptions["type"], rawOptions["priority"]),
|
|
2029
|
+
skipHeaders: readBoolOption(rawOptions, "skip-headers", "skipHeaders"),
|
|
2030
|
+
stream: readBoolOption(rawOptions, "stream"),
|
|
2031
|
+
atomic: readBoolOption(rawOptions, "atomic"),
|
|
2032
|
+
},
|
|
2033
|
+
};
|
|
2034
|
+
}
|
|
2035
|
+
/** Validate strict input, then execute one fully parsed import invocation. */
|
|
2036
|
+
function executeCsvImport(pmRoot, filePath, invocation, commitTransaction, atomicAuthor) {
|
|
2037
|
+
if (invocation.strict) {
|
|
2038
|
+
assertStrictImportReady(filePath, {
|
|
2039
|
+
delimiter: invocation.options.delimiter,
|
|
2040
|
+
fieldMap: invocation.options.fieldMap,
|
|
2041
|
+
encoding: invocation.options.encoding,
|
|
2042
|
+
autoMap: invocation.options.autoMap,
|
|
2043
|
+
skipHeaders: invocation.options.skipHeaders,
|
|
2044
|
+
});
|
|
2045
|
+
}
|
|
2046
|
+
return importCSV(pmRoot, filePath, {
|
|
2047
|
+
...invocation.options,
|
|
2048
|
+
commitTransaction,
|
|
2049
|
+
atomicAuthor,
|
|
2050
|
+
});
|
|
2051
|
+
}
|
|
2052
|
+
/** Log an import's selected engine and execute it through the shared host path. */
|
|
2053
|
+
function executeLoggedCsvImport(label, pmRoot, filePath, invocation, commitTransaction, atomicAuthor) {
|
|
2054
|
+
console.error(`${label} ${filePath}${invocation.options.stream ? " (streaming)" : ""}${invocation.options.atomic ? " (atomic)" : ""}`);
|
|
2055
|
+
return executeCsvImport(pmRoot, filePath, invocation, commitTransaction, atomicAuthor);
|
|
2056
|
+
}
|
|
2011
2057
|
/**
|
|
2012
2058
|
* Discover custom item fields registered in the workspace runtime schema and
|
|
2013
2059
|
* return those that are NOT already covered by the built-in export columns
|
|
@@ -2122,18 +2168,19 @@ function resolveExportColumns(pmRoot, columnsSpec, discover) {
|
|
|
2122
2168
|
}
|
|
2123
2169
|
const columns = [...EXPORT_COLUMNS];
|
|
2124
2170
|
for (const f of discovered) {
|
|
2125
|
-
|
|
2126
|
-
|
|
2171
|
+
// Discovery excludes built-ins and de-duplicates by key, so every result is
|
|
2172
|
+
// additive by construction.
|
|
2173
|
+
columns.push(f.key);
|
|
2127
2174
|
}
|
|
2128
2175
|
return { columns, columnSource };
|
|
2129
2176
|
}
|
|
2130
2177
|
/**
|
|
2131
2178
|
* Read every item from a tracker and render the CSV export body.
|
|
2132
2179
|
*
|
|
2133
|
-
* Shells out to `pm list
|
|
2180
|
+
* Shells out to canonical `pm list --all --json --include-body` with an explicit
|
|
2134
2181
|
* `maxBuffer`, because a stdout overrun kills the child with a null status and
|
|
2135
2182
|
* an EMPTY stderr — which would otherwise surface as an unexplained
|
|
2136
|
-
* "pm list
|
|
2183
|
+
* "pm list --all failed" rather than as the size problem it is.
|
|
2137
2184
|
*
|
|
2138
2185
|
* Status and type filters are applied after the read, not pushed into the
|
|
2139
2186
|
* query. Each item's `csv_source` is then derived from its internal source tag
|
|
@@ -2155,27 +2202,26 @@ function resolveExportColumns(pmRoot, columnsSpec, discover) {
|
|
|
2155
2202
|
* naming the buffer overrun or the tripped completeness signal.
|
|
2156
2203
|
*/
|
|
2157
2204
|
function buildCsvExport(pmRoot, opts, spawn = spawnPm) {
|
|
2158
|
-
const result = spawn(
|
|
2205
|
+
const result = spawn(completeListArgs(pmRoot), { encoding: "utf-8", maxBuffer: pmListMaxBuffer() });
|
|
2159
2206
|
// A stdout overrun kills the child with status null and EMPTY stderr, so name
|
|
2160
|
-
// the real cause instead of reporting an unexplained "pm list
|
|
2207
|
+
// the real cause instead of reporting an unexplained "pm list --all failed".
|
|
2161
2208
|
if (result.error) {
|
|
2162
2209
|
const code = result.error.code;
|
|
2163
2210
|
throw new CommandError(code === "ENOBUFS"
|
|
2164
|
-
? `pm list
|
|
2165
|
-
: `pm list
|
|
2211
|
+
? `pm list --all output exceeded the ${pmListMaxBuffer()} byte read buffer; narrow the export (--status/--type) or raise the PM_LIST_MAX_BUFFER env var.`
|
|
2212
|
+
: `pm list --all failed: ${result.error.message}`);
|
|
2166
2213
|
}
|
|
2167
2214
|
if (result.status !== 0) {
|
|
2168
|
-
throw new CommandError(result.stderr || "pm list
|
|
2215
|
+
throw new CommandError(result.stderr || "pm list --all failed");
|
|
2169
2216
|
}
|
|
2170
2217
|
let parsed;
|
|
2171
2218
|
try {
|
|
2172
2219
|
parsed = JSON.parse(result.stdout);
|
|
2173
2220
|
}
|
|
2174
2221
|
catch (err) {
|
|
2175
|
-
throw new CommandError(`Could not parse \`pm list
|
|
2222
|
+
throw new CommandError(`Could not parse \`pm list --all --json\` output: ${err instanceof Error ? err.message : String(err)}`);
|
|
2176
2223
|
}
|
|
2177
|
-
assertListAllComplete(parsed);
|
|
2178
|
-
let items = parsed.items ?? [];
|
|
2224
|
+
let items = assertListAllComplete(parsed);
|
|
2179
2225
|
if (opts.statusFilter)
|
|
2180
2226
|
items = items.filter((i) => i.status === opts.statusFilter);
|
|
2181
2227
|
if (opts.typeFilter)
|
|
@@ -2215,42 +2261,39 @@ function buildCsvExport(pmRoot, opts, spawn = spawnPm) {
|
|
|
2215
2261
|
eol,
|
|
2216
2262
|
};
|
|
2217
2263
|
}
|
|
2218
|
-
/**
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2264
|
+
/** Parse export flags and render the tracker through the shared export core. */
|
|
2265
|
+
function prepareCsvExport(pmRoot, rawOptions) {
|
|
2266
|
+
const delimiter = resolveDelimiter(rawOptions["delimiter"]);
|
|
2267
|
+
const discover = readBoolOption(rawOptions, "all-fields", "allFields", "discover-fields", "discoverFields");
|
|
2268
|
+
const { columns, columnSource } = resolveExportColumns(pmRoot, rawOptions["columns"], discover);
|
|
2269
|
+
return {
|
|
2270
|
+
outputPath: rawOptions["output"],
|
|
2271
|
+
...buildCsvExport(pmRoot, {
|
|
2272
|
+
statusFilter: rawOptions["status"],
|
|
2273
|
+
typeFilter: rawOptions["type"],
|
|
2274
|
+
delimiter,
|
|
2275
|
+
columns,
|
|
2276
|
+
columnSource,
|
|
2277
|
+
noHeader: readNoHeaderOption(rawOptions),
|
|
2278
|
+
crlf: readBoolOption(rawOptions, "crlf"),
|
|
2279
|
+
excel: readBoolOption(rawOptions, "excel"),
|
|
2280
|
+
}),
|
|
2281
|
+
};
|
|
2282
|
+
}
|
|
2283
|
+
export default {
|
|
2228
2284
|
name: "pm-csv",
|
|
2229
|
-
version: "2026.8.
|
|
2285
|
+
version: "2026.8.29",
|
|
2230
2286
|
activate(api) {
|
|
2231
2287
|
// -----------------------------------------------------------------------
|
|
2232
2288
|
// Schema: register an optional `csv_source` provenance field so imported
|
|
2233
2289
|
// items can record where they came from (set via `pm csv import --source`).
|
|
2234
|
-
//
|
|
2235
|
-
//
|
|
2236
|
-
//
|
|
2237
|
-
//
|
|
2238
|
-
// NOTE: pm 2026.5.31 accepts the field into the schema registry but exposes
|
|
2239
|
-
// no `pm create --csv_source` setter for extension-registered scalar fields,
|
|
2240
|
-
// so the importer persists the provenance label as a `csv-source:` tag
|
|
2241
|
-
// (stripped from exports and surfaced back via the csv_source export column).
|
|
2290
|
+
// The declared pm host floor guarantees this SDK surface. Registration is
|
|
2291
|
+
// deliberately fail-closed: silently omitting provenance would make a
|
|
2292
|
+
// successfully activated package weaker than its manifest contract.
|
|
2242
2293
|
// -----------------------------------------------------------------------
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
{ name: "csv_source", type: "string", optional: true },
|
|
2247
|
-
]);
|
|
2248
|
-
}
|
|
2249
|
-
catch (err) {
|
|
2250
|
-
// Never let a schema-registration hiccup break command registration.
|
|
2251
|
-
console.error(`pm-csv: csv_source field not registered — ${err instanceof Error ? err.message : String(err)}`);
|
|
2252
|
-
}
|
|
2253
|
-
}
|
|
2294
|
+
api.registerItemFields([
|
|
2295
|
+
{ name: "csv_source", type: "string", optional: true },
|
|
2296
|
+
]);
|
|
2254
2297
|
// -----------------------------------------------------------------------
|
|
2255
2298
|
// Command: pm csv import <file>
|
|
2256
2299
|
// -----------------------------------------------------------------------
|
|
@@ -2281,7 +2324,7 @@ export default defineExtension({
|
|
|
2281
2324
|
"pm csv import items.csv --dry-run",
|
|
2282
2325
|
"pm csv import headerless.csv --skip-headers # no header row, positional columns",
|
|
2283
2326
|
"pm csv import big.csv --stream # stream large files without loading into memory",
|
|
2284
|
-
"pm csv import tasks.csv --atomic #
|
|
2327
|
+
"pm csv import tasks.csv --atomic # writer-locked, crash-recoverable import",
|
|
2285
2328
|
],
|
|
2286
2329
|
flags: [
|
|
2287
2330
|
{ long: "--delimiter", value_name: "char", description: "Field delimiter, or alias tab|comma|semicolon|pipe (default: ,)" },
|
|
@@ -2297,55 +2340,27 @@ export default defineExtension({
|
|
|
2297
2340
|
{ long: "--dry-run", description: "Preview without writing" },
|
|
2298
2341
|
{ long: "--skip-headers", description: "The CSV file has no header row; map columns positionally to the standard import order (title, type, status, priority, tags, deadline, body, parent, assignee, sprint, release, blocked_by)" },
|
|
2299
2342
|
{ long: "--stream", description: "Stream the file row-by-row instead of loading it fully into memory (recommended for large CSV files)" },
|
|
2300
|
-
{ long: "--atomic", description: "
|
|
2343
|
+
{ long: "--atomic", description: "Use one writer-locked, crash-recoverable transaction. On failure, attempt best-effort create compensation; pre-existing item updates are intentionally not reverted. Inspect and reconcile any marked orphan before retrying. Incompatible with --stream" },
|
|
2301
2344
|
],
|
|
2302
2345
|
async run(ctx) {
|
|
2303
2346
|
const filePath = ctx.args[0];
|
|
2304
2347
|
if (!filePath) {
|
|
2305
2348
|
throw new CommandError("Usage: pm csv import <file> [--delimiter <char>] [--map col=field] [--auto-map] [--key field] [--encoding enc] [--source label] [--status s] [--type t] [--priority n] [--dry-run]", EXIT_CODE.USAGE);
|
|
2306
2349
|
}
|
|
2307
|
-
const delimiter = resolveDelimiter(ctx.options["delimiter"]);
|
|
2308
2350
|
const dryRun = readBoolOption(ctx.options, "dry-run", "dryRun");
|
|
2309
|
-
const
|
|
2310
|
-
const autoMap = readBoolOption(ctx.options, "auto-map", "autoMap");
|
|
2311
|
-
const keyField = (ctx.options["key"] ?? "").trim().toLowerCase() || undefined;
|
|
2312
|
-
const encoding = resolveEncoding(ctx.options["encoding"]);
|
|
2313
|
-
const source = (ctx.options["source"] ?? "").trim() || undefined;
|
|
2314
|
-
const strict = readBoolOption(ctx.options, "strict");
|
|
2315
|
-
const skipHeaders = readBoolOption(ctx.options, "skip-headers", "skipHeaders");
|
|
2316
|
-
const stream = readBoolOption(ctx.options, "stream");
|
|
2317
|
-
const atomic = readBoolOption(ctx.options, "atomic");
|
|
2318
|
-
const filter = parseImportFilter(ctx.options["status"], ctx.options["type"], ctx.options["priority"]);
|
|
2351
|
+
const invocation = parseCsvImportInvocation(ctx.options, dryRun);
|
|
2319
2352
|
const absolutePath = resolve(filePath);
|
|
2320
|
-
console.error(`Reading CSV from: ${absolutePath}${stream ? " (streaming)" : ""}${atomic ? " (atomic)" : ""}`);
|
|
2321
2353
|
let res;
|
|
2322
2354
|
try {
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
dryRun,
|
|
2328
|
-
fieldMap,
|
|
2329
|
-
autoMap,
|
|
2330
|
-
keyField,
|
|
2331
|
-
encoding,
|
|
2332
|
-
source,
|
|
2333
|
-
filter,
|
|
2334
|
-
skipHeaders,
|
|
2335
|
-
stream,
|
|
2336
|
-
atomic,
|
|
2337
|
-
// Prefer the host-injected SDK coordinator (bound to the
|
|
2338
|
-
// tracker root) so a standalone-installed extension never relies
|
|
2339
|
-
// on package resolution; the atomic path falls back to a dynamic
|
|
2340
|
-
// import when ctx.sdk is absent.
|
|
2341
|
-
commitTransaction: ctx.sdk?.commitWorkspaceTransaction,
|
|
2342
|
-
atomicAuthor: ctx.global?.author ?? undefined,
|
|
2343
|
-
});
|
|
2355
|
+
res = await executeLoggedCsvImport("Reading CSV from:", ctx.pm_root, absolutePath, invocation,
|
|
2356
|
+
// Use the host-bound coordinator so a copied extension never
|
|
2357
|
+
// resolves a second CLI/SDK through its own module tree.
|
|
2358
|
+
ctx.sdk?.commitWorkspaceTransaction, ctx.global?.author ?? undefined);
|
|
2344
2359
|
}
|
|
2345
2360
|
catch (err) {
|
|
2346
2361
|
if (err instanceof CommandError)
|
|
2347
2362
|
throw err;
|
|
2348
|
-
const msg =
|
|
2363
|
+
const msg = errorMessage(err);
|
|
2349
2364
|
const exitCode = /ENOENT|no such file/i.test(msg) ? EXIT_CODE.NOT_FOUND : EXIT_CODE.GENERIC_FAILURE;
|
|
2350
2365
|
throw new CommandError(`Failed to import: ${msg}`, exitCode);
|
|
2351
2366
|
}
|
|
@@ -2423,9 +2438,7 @@ export default defineExtension({
|
|
|
2423
2438
|
report = validateCSV(absolutePath, { delimiter, fieldMap, autoMap, encoding, skipHeaders });
|
|
2424
2439
|
}
|
|
2425
2440
|
catch (err) {
|
|
2426
|
-
|
|
2427
|
-
throw err;
|
|
2428
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
2441
|
+
const msg = errorMessage(err);
|
|
2429
2442
|
const exitCode = /ENOENT|no such file/i.test(msg) ? EXIT_CODE.NOT_FOUND : EXIT_CODE.GENERIC_FAILURE;
|
|
2430
2443
|
throw new CommandError(`Failed to validate: ${msg}`, exitCode);
|
|
2431
2444
|
}
|
|
@@ -2488,24 +2501,8 @@ export default defineExtension({
|
|
|
2488
2501
|
{ long: "--excel", description: "Excel-friendly output: CRLF line endings + a UTF-8 BOM prefix" },
|
|
2489
2502
|
],
|
|
2490
2503
|
async run(ctx) {
|
|
2491
|
-
const delimiter = resolveDelimiter(ctx.options["delimiter"]);
|
|
2492
|
-
const outputPath = ctx.options["output"];
|
|
2493
|
-
const discover = readBoolOption(ctx.options, "all-fields", "allFields", "discover-fields", "discoverFields");
|
|
2494
|
-
const { columns, columnSource } = resolveExportColumns(ctx.pm_root, ctx.options["columns"], discover);
|
|
2495
|
-
const noHeader = readNoHeaderOption(ctx.options);
|
|
2496
|
-
const crlf = readBoolOption(ctx.options, "crlf");
|
|
2497
|
-
const excel = readBoolOption(ctx.options, "excel");
|
|
2498
2504
|
console.error("Fetching pm items…");
|
|
2499
|
-
const { csvText, count, eol } =
|
|
2500
|
-
statusFilter: ctx.options["status"],
|
|
2501
|
-
typeFilter: ctx.options["type"],
|
|
2502
|
-
delimiter,
|
|
2503
|
-
columns,
|
|
2504
|
-
columnSource,
|
|
2505
|
-
noHeader,
|
|
2506
|
-
crlf,
|
|
2507
|
-
excel,
|
|
2508
|
-
});
|
|
2505
|
+
const { csvText, count, eol, outputPath } = prepareCsvExport(ctx.pm_root, ctx.options);
|
|
2509
2506
|
if (count === 0) {
|
|
2510
2507
|
console.error("No items found.");
|
|
2511
2508
|
return { exported: 0 };
|
|
@@ -2528,23 +2525,7 @@ export default defineExtension({
|
|
|
2528
2525
|
// Mirrors the importer so CSV is a first-class import/export pair.
|
|
2529
2526
|
// -----------------------------------------------------------------------
|
|
2530
2527
|
api.registerExporter("csv-export", async (ctx) => {
|
|
2531
|
-
const
|
|
2532
|
-
const outputPath = ctx.options["output"];
|
|
2533
|
-
const discover = readBoolOption(ctx.options, "all-fields", "allFields", "discover-fields", "discoverFields");
|
|
2534
|
-
const { columns, columnSource } = resolveExportColumns(ctx.pm_root, ctx.options["columns"], discover);
|
|
2535
|
-
const noHeader = readNoHeaderOption(ctx.options);
|
|
2536
|
-
const crlf = readBoolOption(ctx.options, "crlf");
|
|
2537
|
-
const excel = readBoolOption(ctx.options, "excel");
|
|
2538
|
-
const { csvText, count, eol } = buildCsvExport(ctx.pm_root, {
|
|
2539
|
-
statusFilter: ctx.options["status"],
|
|
2540
|
-
typeFilter: ctx.options["type"],
|
|
2541
|
-
delimiter,
|
|
2542
|
-
columns,
|
|
2543
|
-
columnSource,
|
|
2544
|
-
noHeader,
|
|
2545
|
-
crlf,
|
|
2546
|
-
excel,
|
|
2547
|
-
});
|
|
2528
|
+
const { csvText, count, eol, outputPath } = prepareCsvExport(ctx.pm_root, ctx.options);
|
|
2548
2529
|
if (outputPath) {
|
|
2549
2530
|
const absolutePath = resolve(outputPath);
|
|
2550
2531
|
// Match the inter-record EOL on the trailing terminator (no lone LF in --crlf/--excel).
|
|
@@ -2564,40 +2545,14 @@ export default defineExtension({
|
|
|
2564
2545
|
console.error("csv-import: no 'file' provided in options — skipping.");
|
|
2565
2546
|
return;
|
|
2566
2547
|
}
|
|
2567
|
-
const
|
|
2568
|
-
const fieldMap = parseFieldMap(ctx.options["map"]);
|
|
2569
|
-
const autoMap = readBoolOption(ctx.options, "auto-map", "autoMap");
|
|
2570
|
-
const keyField = (ctx.options["key"] ?? "").trim().toLowerCase() || undefined;
|
|
2571
|
-
const encoding = resolveEncoding(ctx.options["encoding"]);
|
|
2572
|
-
const source = (ctx.options["source"] ?? "").trim() || undefined;
|
|
2573
|
-
const strict = readBoolOption(ctx.options, "strict");
|
|
2574
|
-
const skipHeaders = readBoolOption(ctx.options, "skip-headers", "skipHeaders");
|
|
2575
|
-
const stream = readBoolOption(ctx.options, "stream");
|
|
2576
|
-
const atomic = readBoolOption(ctx.options, "atomic");
|
|
2577
|
-
const filter = parseImportFilter(ctx.options["status"], ctx.options["type"], ctx.options["priority"]);
|
|
2548
|
+
const invocation = parseCsvImportInvocation(ctx.options, false);
|
|
2578
2549
|
const absolutePath = resolve(filePath);
|
|
2579
|
-
console.error(`csv-import: reading ${absolutePath}${stream ? " (streaming)" : ""}${atomic ? " (atomic)" : ""}`);
|
|
2580
2550
|
let res;
|
|
2581
2551
|
try {
|
|
2582
|
-
|
|
2583
|
-
assertStrictImportReady(absolutePath, { delimiter, fieldMap, encoding, autoMap, skipHeaders });
|
|
2584
|
-
res = await importCSV(ctx.pm_root, absolutePath, {
|
|
2585
|
-
delimiter,
|
|
2586
|
-
dryRun: false,
|
|
2587
|
-
fieldMap,
|
|
2588
|
-
autoMap,
|
|
2589
|
-
keyField,
|
|
2590
|
-
encoding,
|
|
2591
|
-
source,
|
|
2592
|
-
filter,
|
|
2593
|
-
skipHeaders,
|
|
2594
|
-
stream,
|
|
2595
|
-
atomic,
|
|
2596
|
-
atomicAuthor: ctx.global?.author ?? undefined,
|
|
2597
|
-
});
|
|
2552
|
+
res = await executeLoggedCsvImport("csv-import: reading", ctx.pm_root, absolutePath, invocation, ctx.sdk?.commitWorkspaceTransaction, ctx.global?.author ?? undefined);
|
|
2598
2553
|
}
|
|
2599
2554
|
catch (err) {
|
|
2600
|
-
const msg =
|
|
2555
|
+
const msg = errorMessage(err);
|
|
2601
2556
|
console.error(`csv-import: failed — ${msg}`);
|
|
2602
2557
|
return;
|
|
2603
2558
|
}
|
|
@@ -2608,11 +2563,11 @@ export default defineExtension({
|
|
|
2608
2563
|
console.error(`csv-import: done — imported ${res.imported}, updated ${res.updated}, skipped ${res.skipped}${filterNote}.`);
|
|
2609
2564
|
});
|
|
2610
2565
|
},
|
|
2611
|
-
}
|
|
2566
|
+
};
|
|
2612
2567
|
// ---------------------------------------------------------------------------
|
|
2613
2568
|
// Named exports — pure helpers exposed for unit testing (no side effects).
|
|
2614
2569
|
// ---------------------------------------------------------------------------
|
|
2615
|
-
export { parseCSV, serializeCSV, serializeField, stripBOM, resolveDelimiter, parseFieldMap, resolveImportFieldMap, applyFieldMap, normalizeStatus, parseTags, stringifyTags, encodeKeyTagValue, decodeKeyTagValue, normalizeKeyValue, selectExportColumns, resolveEncoding, validateParsedCSV, strictValidationIssues, parseImportFilter, rowMatchesFilter, discoverCustomFields, EXPORT_COLUMNS, IMPORT_COLUMNS, StreamingCSVParser, streamCSVFile, levenshtein, suggestClosest, validateFieldMapTargets, checkMapSourcesPresent, };
|
|
2570
|
+
export { parseCSV, serializeCSV, serializeField, stripBOM, resolveDelimiter, parseFieldMap, resolveImportFieldMap, applyFieldMap, normalizeStatus, parseTags, stringifyTags, encodeKeyTagValue, decodeKeyTagValue, errorMessage, normalizeKeyValue, selectExportColumns, resolveEncoding, validateParsedCSV, validateCSV, strictValidationIssues, parseImportFilter, rowMatchesFilter, discoverCustomFields, EXPORT_COLUMNS, IMPORT_COLUMNS, StreamingCSVParser, streamCSVFile, levenshtein, suggestClosest, validateFieldMapTargets, checkMapSourcesPresent, };
|
|
2616
2571
|
// ---------------------------------------------------------------------------
|
|
2617
2572
|
// Internal store helpers exposed for regression testing against oversized pm
|
|
2618
2573
|
// roots. Unlike the pure helpers above, these READ or WRITE the given pm root
|
|
@@ -2624,8 +2579,9 @@ export { parseCSV, serializeCSV, serializeField, stripBOM, resolveDelimiter, par
|
|
|
2624
2579
|
// with synthetic results.
|
|
2625
2580
|
// ---------------------------------------------------------------------------
|
|
2626
2581
|
export { loadAppliedByTransaction, itemStatus, compensateCreate };
|
|
2627
|
-
// list
|
|
2582
|
+
// Complete-list surface: the shared refusal gate plus the two internal
|
|
2628
2583
|
// readers whose silently-partial behavior the regression tests pin. Exported
|
|
2629
2584
|
// ONLY for those tests; production callers keep using the command paths.
|
|
2630
2585
|
export { loadKeyIndex, buildCsvExport };
|
|
2586
|
+
export { importCSV };
|
|
2631
2587
|
//# sourceMappingURL=index.js.map
|