pm-csv 2026.8.10 → 2026.8.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +37 -15
- package/dist/index.d.ts +244 -21
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +514 -413
- package/dist/index.js.map +1 -1
- package/manifest.json +3 -6
- package/package.json +22 -12
package/dist/index.js
CHANGED
|
@@ -2,6 +2,7 @@ import { readFileSync, writeFileSync, createReadStream } from "node:fs";
|
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
3
|
import { spawnSync } from "node:child_process";
|
|
4
4
|
import { createHash } from "node:crypto";
|
|
5
|
+
import { certifyCompleteListResult, inspectCompleteListResult } from "@unbrained/pm-cli/sdk";
|
|
5
6
|
// ---------------------------------------------------------------------------
|
|
6
7
|
// Error contract
|
|
7
8
|
// ---------------------------------------------------------------------------
|
|
@@ -17,7 +18,14 @@ const EXIT_CODE = {
|
|
|
17
18
|
USAGE: 2,
|
|
18
19
|
NOT_FOUND: 3,
|
|
19
20
|
};
|
|
20
|
-
|
|
21
|
+
/**
|
|
22
|
+
* Error carrying a semantic process exit code, thrown by every failure path
|
|
23
|
+
* the command runtime must turn into a clean nonzero exit (including the
|
|
24
|
+
* complete-list refusals). Exported so regression tests can assert the
|
|
25
|
+
* typed failure instead of matching on message text alone.
|
|
26
|
+
*/
|
|
27
|
+
export class CommandError extends Error {
|
|
28
|
+
/** Numeric exit code the runtime propagates to the shell. */
|
|
21
29
|
exitCode;
|
|
22
30
|
constructor(message, exitCode = EXIT_CODE.GENERIC_FAILURE) {
|
|
23
31
|
super(message);
|
|
@@ -287,47 +295,25 @@ class StreamingCSVParser {
|
|
|
287
295
|
*/
|
|
288
296
|
function streamCSVFile(filePath, delimiter, encoding, onRow) {
|
|
289
297
|
const bufEnc = encoding === "utf-8" ? "utf8" : encoding;
|
|
290
|
-
return
|
|
298
|
+
return (async () => {
|
|
291
299
|
const stream = createReadStream(filePath, { encoding: bufEnc });
|
|
292
300
|
const parser = new StreamingCSVParser(delimiter, onRow);
|
|
293
|
-
let
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
reject(err);
|
|
301
|
-
};
|
|
302
|
-
stream.on("data", (chunk) => {
|
|
303
|
-
if (stopped)
|
|
304
|
-
return;
|
|
305
|
-
let text = typeof chunk === "string" ? chunk : chunk.toString(bufEnc);
|
|
306
|
-
if (!bomChecked) {
|
|
307
|
-
text = stripBOM(text);
|
|
308
|
-
bomChecked = true;
|
|
309
|
-
}
|
|
310
|
-
try {
|
|
301
|
+
let firstChunk = true;
|
|
302
|
+
try {
|
|
303
|
+
for await (const chunk of stream) {
|
|
304
|
+
// createReadStream was constructed with an encoding, so Node guarantees
|
|
305
|
+
// string chunks even though the broad async-iterator type retains Buffer.
|
|
306
|
+
const text = firstChunk ? stripBOM(chunk) : chunk;
|
|
307
|
+
firstChunk = false;
|
|
311
308
|
parser.push(text);
|
|
312
309
|
}
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
try {
|
|
321
|
-
parser.end();
|
|
322
|
-
}
|
|
323
|
-
catch (err) {
|
|
324
|
-
fail(err);
|
|
325
|
-
return;
|
|
326
|
-
}
|
|
327
|
-
resolve();
|
|
328
|
-
});
|
|
329
|
-
stream.on("error", fail);
|
|
330
|
-
});
|
|
310
|
+
parser.end();
|
|
311
|
+
}
|
|
312
|
+
catch (error) {
|
|
313
|
+
stream.destroy();
|
|
314
|
+
throw error;
|
|
315
|
+
}
|
|
316
|
+
})();
|
|
331
317
|
}
|
|
332
318
|
// ---------------------------------------------------------------------------
|
|
333
319
|
// Helpers
|
|
@@ -469,6 +455,151 @@ function assertPmOutputFit(result, label) {
|
|
|
469
455
|
return;
|
|
470
456
|
throw new CommandError(`${describePmNullStatus(result, label)}. The result is unusable, so it must never be treated as an empty or absent value.`);
|
|
471
457
|
}
|
|
458
|
+
/** Canonical argv shared by every whole-tracker subprocess. */
|
|
459
|
+
const COMPLETE_LIST_ARGS = [
|
|
460
|
+
"list",
|
|
461
|
+
"--all",
|
|
462
|
+
"--json",
|
|
463
|
+
"--include-body",
|
|
464
|
+
"--strict-read",
|
|
465
|
+
"--no-truncate",
|
|
466
|
+
"--output-budget",
|
|
467
|
+
"unbounded",
|
|
468
|
+
"--output-limit",
|
|
469
|
+
"unbounded",
|
|
470
|
+
"--output-include",
|
|
471
|
+
"full",
|
|
472
|
+
];
|
|
473
|
+
/** Build one canonical complete-list request for a tracker root. */
|
|
474
|
+
function completeListArgs(pmRoot) {
|
|
475
|
+
return ["--pm-path", pmRoot, ...COMPLETE_LIST_ARGS];
|
|
476
|
+
}
|
|
477
|
+
/** Narrow an unknown JSON value to a non-array record. */
|
|
478
|
+
function asRecord(value) {
|
|
479
|
+
return value !== null && typeof value === "object" && !Array.isArray(value)
|
|
480
|
+
? value
|
|
481
|
+
: undefined;
|
|
482
|
+
}
|
|
483
|
+
/** Render a receipt value while preserving a missing field as evidence. */
|
|
484
|
+
function describeReceiptValue(value) {
|
|
485
|
+
return value === undefined ? "<missing>" : JSON.stringify(value);
|
|
486
|
+
}
|
|
487
|
+
/** Collect the pm 2026.8.21 receipt gaps not yet rejected by the public SDK. */
|
|
488
|
+
function supplementalCompleteListFindings(record) {
|
|
489
|
+
const findings = [];
|
|
490
|
+
const completeness = asRecord(record.completeness);
|
|
491
|
+
for (const field of ["unreadable_item_count", "unreadable_directory_count"]) {
|
|
492
|
+
const value = completeness?.[field];
|
|
493
|
+
if (!Number.isSafeInteger(value) || value !== 0) {
|
|
494
|
+
findings.push(`completeness.${field}=${describeReceiptValue(value)}`);
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
const omission = asRecord(record.omission_receipt);
|
|
498
|
+
if (omission === undefined) {
|
|
499
|
+
findings.push("omission_receipt=<missing>");
|
|
500
|
+
}
|
|
501
|
+
else {
|
|
502
|
+
if (omission.has_omissions !== false) {
|
|
503
|
+
findings.push(`omission_receipt.has_omissions=${describeReceiptValue(omission.has_omissions)}`);
|
|
504
|
+
}
|
|
505
|
+
if (!Number.isSafeInteger(omission.omitted_field_group_count) || omission.omitted_field_group_count !== 0) {
|
|
506
|
+
findings.push(`omission_receipt.omitted_field_group_count=${describeReceiptValue(omission.omitted_field_group_count)}`);
|
|
507
|
+
}
|
|
508
|
+
if (!Array.isArray(omission.omitted_field_groups) || omission.omitted_field_groups.length !== 0) {
|
|
509
|
+
findings.push(`omission_receipt.omitted_field_groups=${describeReceiptValue(omission.omitted_field_groups)}`);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
const readOutput = asRecord(record.read_output);
|
|
513
|
+
if (readOutput === undefined) {
|
|
514
|
+
findings.push("read_output=<missing>");
|
|
515
|
+
}
|
|
516
|
+
else {
|
|
517
|
+
for (const [field, expected] of [
|
|
518
|
+
["contract_version", 1],
|
|
519
|
+
["command", "list"],
|
|
520
|
+
["within_budget", true],
|
|
521
|
+
["strings_compacted", false],
|
|
522
|
+
["rows_compacted", false],
|
|
523
|
+
["result_omitted", false],
|
|
524
|
+
]) {
|
|
525
|
+
if (readOutput[field] !== expected) {
|
|
526
|
+
findings.push(`read_output.${field}=${describeReceiptValue(readOutput[field])}`);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
const dimensions = readOutput.requested_dimensions;
|
|
530
|
+
if (!Array.isArray(dimensions)) {
|
|
531
|
+
findings.push("read_output.requested_dimensions=<missing>");
|
|
532
|
+
}
|
|
533
|
+
else {
|
|
534
|
+
for (const dimension of ["include", "amount", "cost"]) {
|
|
535
|
+
if (!dimensions.includes(dimension)) {
|
|
536
|
+
findings.push(`read_output.requested_dimensions missing ${dimension}`);
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
if (record.output_budget_truncation !== undefined)
|
|
542
|
+
findings.push("output_budget_truncation=<present>");
|
|
543
|
+
if (record.output_budget_exceeded !== undefined)
|
|
544
|
+
findings.push("output_budget_exceeded=<present>");
|
|
545
|
+
return findings;
|
|
546
|
+
}
|
|
547
|
+
/**
|
|
548
|
+
* Refuse an incomplete canonical `pm list --all` envelope instead of consuming it.
|
|
549
|
+
*
|
|
550
|
+
* Reading `.items` without consulting the envelope's completeness receipt is
|
|
551
|
+
* how this package once shipped a 10-row CSV export from a 682-item workspace
|
|
552
|
+
* (pm 2026.8.14 defaulted the deprecated list-all alias to a truncated answer;
|
|
553
|
+
* verified on this
|
|
554
|
+
* host), saw none of the applied rows on an atomic-import resume scan, and
|
|
555
|
+
* missed upsert keys - each reporting success. Any one of four independent
|
|
556
|
+
* signals means the rows in `items` are NOT the whole workspace, so this
|
|
557
|
+
* throws (never returns a partial list, never logs and continues) naming the
|
|
558
|
+
* signal that tripped plus the `count`/`total` figures:
|
|
559
|
+
*
|
|
560
|
+
* - `truncated === true` - the row list was cut short (output budget, limit);
|
|
561
|
+
* - `has_more === true` - rows exist past the returned cursor boundary;
|
|
562
|
+
* - `completeness.status !== "complete"` - items/directories were unreadable
|
|
563
|
+
* ("partial") or the corpus was never scanned ("unchecked"); a missing
|
|
564
|
+
* receipt also trips this, because an unverifiable answer is not complete;
|
|
565
|
+
* - `omission_receipt.has_omissions === true` - field groups were dropped from
|
|
566
|
+
* the projection, so rows are present but degraded.
|
|
567
|
+
*
|
|
568
|
+
* Thrown errors are {@link CommandError} so the command runtime turns them
|
|
569
|
+
* into a clean nonzero exit. Paging is deliberately NOT attempted: every
|
|
570
|
+
* consumer here (CSV export, resume scan, key index) needs the WHOLE
|
|
571
|
+
* workspace, so refusing loudly is simpler and safer than a paging loop that
|
|
572
|
+
* could itself silently drop rows.
|
|
573
|
+
*
|
|
574
|
+
* Shared source, filter, pagination, projection, count, and identity rules come
|
|
575
|
+
* from the public SDK. A narrow package supplement rejects the missing or
|
|
576
|
+
* contradictory omission, read-output, and unreadable-counter evidence
|
|
577
|
+
* reproduced in pm-cli issue 1078. The returned rows are therefore safe for
|
|
578
|
+
* mutation planning, resume reconciliation, and CSV export.
|
|
579
|
+
*
|
|
580
|
+
* @param envelope - Parsed canonical complete-list JSON output.
|
|
581
|
+
* @returns Every certified full item row.
|
|
582
|
+
* @throws {@link CommandError} naming every tripped signal and the counts.
|
|
583
|
+
*/
|
|
584
|
+
export function assertListAllComplete(envelope) {
|
|
585
|
+
const record = asRecord(envelope);
|
|
586
|
+
const sdkFindings = inspectCompleteListResult(envelope).findings
|
|
587
|
+
.map((finding) => `${finding.code}: ${finding.message}`);
|
|
588
|
+
const findings = record === undefined
|
|
589
|
+
? sdkFindings
|
|
590
|
+
: [...sdkFindings, ...supplementalCompleteListFindings(record)];
|
|
591
|
+
if (findings.length > 0 || record === undefined) {
|
|
592
|
+
const count = record && typeof record.count === "number" ? record.count : "unknown";
|
|
593
|
+
const total = record && typeof record.total === "number" ? record.total : "unknown";
|
|
594
|
+
throw new CommandError(`pm list --all complete-corpus answer was refused: ${findings.join("; ")}; count=${count} of total=${total}. `
|
|
595
|
+
+ "A partial workspace read would corrupt import reconciliation or CSV export; retry the canonical strict unbounded read.");
|
|
596
|
+
}
|
|
597
|
+
return certifyCompleteListResult(record).items;
|
|
598
|
+
}
|
|
599
|
+
/** Real {@link PmSpawn} over `child_process.spawnSync`. */
|
|
600
|
+
function spawnPm(args, options) {
|
|
601
|
+
return spawnSync("pm", args, options);
|
|
602
|
+
}
|
|
472
603
|
/**
|
|
473
604
|
* Normalize a dedup key value for stable matching. pm lower-cases tags on
|
|
474
605
|
* storage, so a `csv-key:` tag written from "Fix Bug" comes back as
|
|
@@ -501,6 +632,10 @@ function decodeKeyTagValue(value) {
|
|
|
501
632
|
return value;
|
|
502
633
|
}
|
|
503
634
|
}
|
|
635
|
+
/** Normalize JavaScript's open-ended thrown-value channel for operator errors. */
|
|
636
|
+
function errorMessage(error) {
|
|
637
|
+
return error instanceof Error ? error.message : String(error);
|
|
638
|
+
}
|
|
504
639
|
/**
|
|
505
640
|
* Parse a `--map csvHeader=field` spec (repeatable / comma-joined) into a
|
|
506
641
|
* lookup from a normalized CSV header name to the canonical pm field name.
|
|
@@ -753,6 +888,19 @@ function readCSVFile(filePath, delimiter, encoding = "utf-8", skipHeaders = fals
|
|
|
753
888
|
const dataRows = rows.slice(1);
|
|
754
889
|
return { headers, dataRows };
|
|
755
890
|
}
|
|
891
|
+
/** Create the canonical empty receipt shared by every CSV import engine. */
|
|
892
|
+
function createImportResult() {
|
|
893
|
+
return {
|
|
894
|
+
imported: 0,
|
|
895
|
+
updated: 0,
|
|
896
|
+
skipped: 0,
|
|
897
|
+
filtered: 0,
|
|
898
|
+
autoMappings: [],
|
|
899
|
+
errors: [],
|
|
900
|
+
previews: [],
|
|
901
|
+
fieldMapWarnings: [],
|
|
902
|
+
};
|
|
903
|
+
}
|
|
756
904
|
/**
|
|
757
905
|
* Build the canonical field accessor for a header row + data row.
|
|
758
906
|
*/
|
|
@@ -828,23 +976,41 @@ function rowMatchesFilter(row, filter) {
|
|
|
828
976
|
}
|
|
829
977
|
/**
|
|
830
978
|
* List existing items once and build a lookup from csv-key provenance value to
|
|
831
|
-
* item id, for idempotent upsert.
|
|
979
|
+
* item id, for idempotent upsert. The complete-list answer must be COMPLETE — a
|
|
980
|
+
* truncated answer silently missing keys would turn upserts into duplicate
|
|
981
|
+
* creates — so the envelope's receipt is checked before any key is read.
|
|
832
982
|
*/
|
|
833
|
-
function loadKeyIndex(pmRoot) {
|
|
983
|
+
function loadKeyIndex(pmRoot, spawn = spawnPm) {
|
|
834
984
|
const index = new Map();
|
|
835
|
-
const result =
|
|
836
|
-
|
|
837
|
-
|
|
985
|
+
const result = spawn(completeListArgs(pmRoot), { encoding: "utf-8", maxBuffer: pmListMaxBuffer() });
|
|
986
|
+
// Route a stdout overrun (status: null, empty stderr, ENOBUFS) through
|
|
987
|
+
// assertPmOutputFit as a hard, named error instead of the generic
|
|
988
|
+
// "pm list --all failed" the old `result.error` branch reported. An overrun is
|
|
989
|
+
// a size problem, not a pm failure: a too-large workspace would otherwise
|
|
990
|
+
// surface as a bare Node message and send the operator to pm's stderr instead
|
|
991
|
+
// of to PM_LIST_MAX_BUFFER, and — worse — a null status combined with this
|
|
992
|
+
// package's `status !== 0` guard would degrade into an empty key index,
|
|
993
|
+
// turning every upsert into a duplicate create.
|
|
994
|
+
assertPmOutputFit(result, "pm list --all");
|
|
838
995
|
if (result.status !== 0) {
|
|
839
|
-
throw new CommandError(result.stderr?.trim() || "pm list
|
|
996
|
+
throw new CommandError(result.stderr?.trim() || "pm list --all failed");
|
|
840
997
|
}
|
|
841
998
|
let items = [];
|
|
999
|
+
let parsed;
|
|
842
1000
|
try {
|
|
843
|
-
|
|
1001
|
+
parsed = JSON.parse(result.stdout);
|
|
844
1002
|
}
|
|
845
1003
|
catch {
|
|
846
|
-
|
|
847
|
-
|
|
1004
|
+
// Fail closed. Returning an empty index here says "no keys exist", which is
|
|
1005
|
+
// indistinguishable from an empty workspace and makes --upsert CREATE
|
|
1006
|
+
// DUPLICATES for every row whose key it could not see. That is the same
|
|
1007
|
+
// silent-partial failure the completeness gate below exists to stop, so the
|
|
1008
|
+
// two paths must agree.
|
|
1009
|
+
throw new CommandError("Could not parse `pm list --all --json` output while building the upsert key index. "
|
|
1010
|
+
+ "Refusing rather than treating an unreadable response as an empty workspace, "
|
|
1011
|
+
+ "which would make --upsert create duplicates.");
|
|
1012
|
+
}
|
|
1013
|
+
items = assertListAllComplete(parsed);
|
|
848
1014
|
for (const item of items) {
|
|
849
1015
|
for (const tag of item.tags ?? []) {
|
|
850
1016
|
if (tag.startsWith(KEY_TAG_PREFIX)) {
|
|
@@ -933,11 +1099,37 @@ function computeFieldMapWarnings(rawHeaders, fieldMap) {
|
|
|
933
1099
|
console.error(`Warning: ${w}`);
|
|
934
1100
|
return warnings;
|
|
935
1101
|
}
|
|
1102
|
+
/** Resolve, validate, and report one raw import header set. */
|
|
1103
|
+
function resolveImportHeaders(rawHeaders, opts) {
|
|
1104
|
+
const fieldMapWarnings = computeFieldMapWarnings(rawHeaders, opts.fieldMap);
|
|
1105
|
+
const mapResolution = resolveImportFieldMap(rawHeaders, opts.fieldMap, opts.autoMap ?? false);
|
|
1106
|
+
const headers = applyFieldMap(rawHeaders, mapResolution.fieldMap);
|
|
1107
|
+
assertImportHeaders(headers, opts);
|
|
1108
|
+
return { headers, autoMappings: mapResolution.autoMappings, fieldMapWarnings };
|
|
1109
|
+
}
|
|
1110
|
+
/** Read and resolve the shared in-memory prelude for normal and atomic imports. */
|
|
1111
|
+
function prepareInMemoryImport(filePath, opts) {
|
|
1112
|
+
const { headers: rawHeaders, dataRows } = readCSVFile(filePath, opts.delimiter, opts.encoding ?? "utf-8", opts.skipHeaders ?? false);
|
|
1113
|
+
return {
|
|
1114
|
+
rawHeaders,
|
|
1115
|
+
dataRows,
|
|
1116
|
+
result: createImportResult(),
|
|
1117
|
+
resolvedHeaders: rawHeaders.length > 0 ? resolveImportHeaders(rawHeaders, opts) : undefined,
|
|
1118
|
+
};
|
|
1119
|
+
}
|
|
1120
|
+
/** Process one complete in-memory row set through the shared mutation core. */
|
|
1121
|
+
function processImportRows(pmRoot, dataRows, headers, opts, result) {
|
|
1122
|
+
const keyIndex = opts.keyField ? loadKeyIndex(pmRoot) : new Map();
|
|
1123
|
+
for (let rowIndex = 0; rowIndex < dataRows.length; rowIndex++) {
|
|
1124
|
+
const lineNo = opts.skipHeaders ? rowIndex + 1 : rowIndex + 2;
|
|
1125
|
+
processImportRow(pmRoot, headers, dataRows[rowIndex], lineNo, opts, keyIndex, result);
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
936
1128
|
/**
|
|
937
1129
|
* Import CSV rows into a pm tracker as items.
|
|
938
1130
|
*
|
|
939
1131
|
* Rejects rather than throws when `--atomic` is combined with `--stream`: the
|
|
940
|
-
* two are genuinely incompatible, because
|
|
1132
|
+
* two are genuinely incompatible, because a bounded transaction plan needs the
|
|
941
1133
|
* whole row set in hand and a stream is unbounded. Returning a rejected promise
|
|
942
1134
|
* keeps the failure on the same channel as the rest of the async path.
|
|
943
1135
|
*
|
|
@@ -949,7 +1141,7 @@ function computeFieldMapWarnings(rawHeaders, fieldMap) {
|
|
|
949
1141
|
*/
|
|
950
1142
|
function importCSV(pmRoot, filePath, opts) {
|
|
951
1143
|
if (opts.atomic && opts.stream) {
|
|
952
|
-
return Promise.reject(new CommandError("--atomic cannot be combined with --stream: an unbounded stream cannot be committed as one
|
|
1144
|
+
return Promise.reject(new CommandError("--atomic cannot be combined with --stream: an unbounded stream cannot be committed as one bounded transaction.", EXIT_CODE.USAGE));
|
|
953
1145
|
}
|
|
954
1146
|
if (opts.atomic)
|
|
955
1147
|
return importCSVAtomic(pmRoot, filePath, opts);
|
|
@@ -962,30 +1154,14 @@ function importCSV(pmRoot, filePath, opts) {
|
|
|
962
1154
|
* every data row. Used when `--stream` is not set (the default).
|
|
963
1155
|
*/
|
|
964
1156
|
function importCSVInMemory(pmRoot, filePath, opts) {
|
|
965
|
-
const {
|
|
966
|
-
|
|
967
|
-
imported: 0,
|
|
968
|
-
updated: 0,
|
|
969
|
-
skipped: 0,
|
|
970
|
-
filtered: 0,
|
|
971
|
-
autoMappings: [],
|
|
972
|
-
errors: [],
|
|
973
|
-
previews: [],
|
|
974
|
-
fieldMapWarnings: [],
|
|
975
|
-
};
|
|
976
|
-
if (rawHeaders.length === 0)
|
|
1157
|
+
const { dataRows, result, resolvedHeaders } = prepareInMemoryImport(filePath, opts);
|
|
1158
|
+
if (!resolvedHeaders)
|
|
977
1159
|
return result;
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
result.autoMappings =
|
|
982
|
-
|
|
983
|
-
// Pre-load the dedup index only when upserting (one extra pm call, not per-row).
|
|
984
|
-
const keyIndex = opts.keyField && !opts.dryRun ? loadKeyIndex(pmRoot) : new Map();
|
|
985
|
-
for (let rowIndex = 0; rowIndex < dataRows.length; rowIndex++) {
|
|
986
|
-
const lineNo = opts.skipHeaders ? rowIndex + 1 : rowIndex + 2;
|
|
987
|
-
processImportRow(pmRoot, headers, dataRows[rowIndex], lineNo, opts, keyIndex, result);
|
|
988
|
-
}
|
|
1160
|
+
// Pre-load the dedup index whenever upserting, including dry-run: a preview
|
|
1161
|
+
// that labels an existing key as a create is not a truthful preview.
|
|
1162
|
+
result.fieldMapWarnings = resolvedHeaders.fieldMapWarnings;
|
|
1163
|
+
result.autoMappings = resolvedHeaders.autoMappings;
|
|
1164
|
+
processImportRows(pmRoot, dataRows, resolvedHeaders.headers, opts, result);
|
|
989
1165
|
return result;
|
|
990
1166
|
}
|
|
991
1167
|
// ---------------------------------------------------------------------------
|
|
@@ -1052,17 +1228,20 @@ export function atomicTransactionId(filePath, opts = {}) {
|
|
|
1052
1228
|
* whose CSV `status` is `closed`/`canceled` is legitimately imported as a
|
|
1053
1229
|
* closed item (upsertCreate/upsertUpdate apply the row's status, routing
|
|
1054
1230
|
* terminal transitions through `pm close --reason`), so it must
|
|
1055
|
-
* still be recognized on resume and never re-imported. Conversely a
|
|
1056
|
-
* create must NOT be recognized:
|
|
1057
|
-
* `
|
|
1058
|
-
*
|
|
1059
|
-
*
|
|
1060
|
-
*
|
|
1231
|
+
* still be recognized on resume and never re-imported. Conversely a
|
|
1232
|
+
* successfully compensated create must NOT be recognized:
|
|
1233
|
+
* `compensateCreate()` removes the markers only after closure succeeds or the
|
|
1234
|
+
* item is already confirmed closed, so a compensated
|
|
1235
|
+
* tombstone carries no marker and a post-rollback retry re-imports its row. If
|
|
1236
|
+
* closure fails, both markers remain and make the orphan discoverable for
|
|
1237
|
+
* reconciliation. (This is why matching by marker-presence is correct for both
|
|
1238
|
+
* a legitimately-closed applied row and a rolled-back one; see
|
|
1239
|
+
* compensateCreate.)
|
|
1061
1240
|
*/
|
|
1062
|
-
function loadAppliedByTransaction(pmRoot, transactionId) {
|
|
1241
|
+
function loadAppliedByTransaction(pmRoot, transactionId, spawn = spawnPm) {
|
|
1063
1242
|
const byRowIndex = new Map();
|
|
1064
1243
|
const rowMarkerPrefix = `${TX_ROW_TAG_PREFIX}${transactionId}${TX_ROW_TAG_SEPARATOR}`;
|
|
1065
|
-
const r =
|
|
1244
|
+
const r = spawn(completeListArgs(pmRoot), {
|
|
1066
1245
|
encoding: "utf-8",
|
|
1067
1246
|
maxBuffer: pmListMaxBuffer(),
|
|
1068
1247
|
});
|
|
@@ -1070,19 +1249,34 @@ function loadAppliedByTransaction(pmRoot, transactionId) {
|
|
|
1070
1249
|
// the previous guard (`r.error || r.status !== 0`) then returned an EMPTY map,
|
|
1071
1250
|
// so inspect() concluded "nothing was applied" and a resumed atomic import
|
|
1072
1251
|
// silently re-imported every already-applied row — wrong data, not a crash.
|
|
1073
|
-
// Route
|
|
1074
|
-
//
|
|
1075
|
-
//
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1252
|
+
// Route unusable null-status output through {@link assertPmOutputFit} as a
|
|
1253
|
+
// hard, named error. The following non-zero branch rejects ordinary command failures.
|
|
1254
|
+
// Parse failures and incomplete envelopes below also refuse the resume scan;
|
|
1255
|
+
// none of these unreadable states may become an empty applied-row map.
|
|
1256
|
+
assertPmOutputFit(r, "pm list --all");
|
|
1257
|
+
// Fail closed on both failure shapes, for the same reason the completeness
|
|
1258
|
+
// gate below refuses: an empty resume map says "nothing was applied", so a
|
|
1259
|
+
// resume RE-IMPORTS rows that already landed. An unreadable answer is not an
|
|
1260
|
+
// empty answer.
|
|
1261
|
+
if (r.status !== 0) {
|
|
1262
|
+
throw new CommandError(r.stderr?.trim()
|
|
1263
|
+
|| "pm list --all failed while scanning for applied rows. Refusing rather than "
|
|
1264
|
+
+ "treating an unreadable response as an empty transaction, which would re-import applied rows.");
|
|
1265
|
+
}
|
|
1079
1266
|
let items = [];
|
|
1267
|
+
let parsed;
|
|
1080
1268
|
try {
|
|
1081
|
-
|
|
1269
|
+
parsed = JSON.parse(r.stdout);
|
|
1082
1270
|
}
|
|
1083
1271
|
catch {
|
|
1084
|
-
|
|
1085
|
-
|
|
1272
|
+
throw new CommandError("Could not parse `pm list --all --json` output while scanning for applied rows. "
|
|
1273
|
+
+ "Refusing rather than treating an unreadable response as an empty transaction, "
|
|
1274
|
+
+ "which would re-import rows that already landed.");
|
|
1275
|
+
}
|
|
1276
|
+
// An incomplete envelope must refuse here, NOT degrade to an empty map: a
|
|
1277
|
+
// resume scan that misses applied rows silently re-imports them (wrong
|
|
1278
|
+
// data, the exact failure this package exists to prevent).
|
|
1279
|
+
items = assertListAllComplete(parsed);
|
|
1086
1280
|
for (const item of items) {
|
|
1087
1281
|
for (const tag of item.tags ?? []) {
|
|
1088
1282
|
if (!tag.startsWith(rowMarkerPrefix))
|
|
@@ -1112,8 +1306,8 @@ function loadAppliedByTransaction(pmRoot, transactionId) {
|
|
|
1112
1306
|
* non-zero exit (item truly absent, or a failed lookup such as the
|
|
1113
1307
|
* `fake-get-fail` regression) still returns `undefined` unchanged.
|
|
1114
1308
|
*/
|
|
1115
|
-
function itemStatus(pmRoot, id) {
|
|
1116
|
-
const r =
|
|
1309
|
+
function itemStatus(pmRoot, id, spawn = spawnPm) {
|
|
1310
|
+
const r = spawn(["--path", pmRoot, "get", id, "--json"], {
|
|
1117
1311
|
encoding: "utf-8",
|
|
1118
1312
|
maxBuffer: pmListMaxBuffer(),
|
|
1119
1313
|
});
|
|
@@ -1129,18 +1323,32 @@ function itemStatus(pmRoot, id) {
|
|
|
1129
1323
|
}
|
|
1130
1324
|
}
|
|
1131
1325
|
/**
|
|
1132
|
-
*
|
|
1133
|
-
*
|
|
1134
|
-
*
|
|
1326
|
+
* Read an item's status without letting a compensation-only lookup abort the
|
|
1327
|
+
* remaining best-effort sweep. `undefined` deliberately means either absent or
|
|
1328
|
+
* unverified; neither is positive evidence that an open item became closed.
|
|
1329
|
+
*/
|
|
1330
|
+
function compensationItemStatus(pmRoot, id) {
|
|
1331
|
+
try {
|
|
1332
|
+
return itemStatus(pmRoot, id);
|
|
1333
|
+
}
|
|
1334
|
+
catch {
|
|
1335
|
+
return undefined;
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
/**
|
|
1339
|
+
* Idempotently undo one row's create. First closes a still-open item via
|
|
1340
|
+
* `pm close` (NOT delete, to avoid the known history-resurrection issue), then
|
|
1341
|
+
* strips this transaction's ownership markers (`markers`) only after the item
|
|
1342
|
+
* is confirmed closed.
|
|
1135
1343
|
*
|
|
1136
|
-
* Stripping the markers
|
|
1137
|
-
* signal: a
|
|
1138
|
-
* retry re-imports it, while a *successfully* imported row (even
|
|
1139
|
-
* status is `closed`/`canceled`) keeps its marker and is resumed
|
|
1140
|
-
*
|
|
1141
|
-
*
|
|
1142
|
-
*
|
|
1143
|
-
*
|
|
1344
|
+
* Stripping the markers after closure makes marker-presence a correct
|
|
1345
|
+
* reconciliation signal: a compensated row no longer carries a marker, so a
|
|
1346
|
+
* post-rollback retry re-imports it, while a *successfully* imported row (even
|
|
1347
|
+
* one whose CSV status is `closed`/`canceled`) keeps its marker and is resumed
|
|
1348
|
+
* idempotently. If the close fails, the markers deliberately remain so the open
|
|
1349
|
+
* orphan can be found by transaction id and reconciled before retrying. The
|
|
1350
|
+
* strip also runs for an item that was already closed; every step tolerates a
|
|
1351
|
+
* missing item / absent tag, so repeated compensation is a safe no-op.
|
|
1144
1352
|
*
|
|
1145
1353
|
* The status lookup itself is also tolerated: if `itemStatus` throws (a stdout
|
|
1146
1354
|
* overrun or a signal kill on the `pm get`), the throw is caught and this row's
|
|
@@ -1151,38 +1359,35 @@ function itemStatus(pmRoot, id) {
|
|
|
1151
1359
|
* and leaving every later applied create un-compensated.
|
|
1152
1360
|
*/
|
|
1153
1361
|
function compensateCreate(pmRoot, id, markers = [], reason = "atomic csv import rolled back") {
|
|
1154
|
-
//
|
|
1155
|
-
//
|
|
1156
|
-
//
|
|
1157
|
-
|
|
1158
|
-
// CommandError from assertPmOutputFit; a bare catch is therefore equivalent to
|
|
1159
|
-
// catching CommandError and avoids an unreachable rethrow branch.)
|
|
1160
|
-
let status;
|
|
1161
|
-
try {
|
|
1162
|
-
status = itemStatus(pmRoot, id);
|
|
1163
|
-
}
|
|
1164
|
-
catch {
|
|
1165
|
-
return; // status lookup overran/killed: no-op for this row, sweep continues
|
|
1166
|
-
}
|
|
1362
|
+
// A failed status lookup is a no-op for this row and must never abort the
|
|
1363
|
+
// sweep over the remaining applied creates. The item remains marked for
|
|
1364
|
+
// reconciliation when it still exists.
|
|
1365
|
+
const status = compensationItemStatus(pmRoot, id);
|
|
1167
1366
|
if (status === undefined)
|
|
1168
|
-
return; //
|
|
1367
|
+
return; // absent or unverified; never invent closure
|
|
1368
|
+
if (status !== "closed") {
|
|
1369
|
+
// Compensation is an internal rollback, not a user closure: it must
|
|
1370
|
+
// reliably undo the create regardless of closure-validation governance, so
|
|
1371
|
+
// it bypasses `--validate-close` (off). `require_close_reason` still applies
|
|
1372
|
+
// and is satisfied by `reason`. Used by both the atomic transaction path
|
|
1373
|
+
// (rollback) and the non-atomic `upsertCreate` close-failure path.
|
|
1374
|
+
const closeResult = spawnSync("pm", ["--path", pmRoot, "close", id, "--reason", reason, "--validate-close", "off"], { encoding: "utf-8", maxBuffer: pmListMaxBuffer() });
|
|
1375
|
+
if (closeResult.status !== 0) {
|
|
1376
|
+
// A non-zero receipt is not proof either way: the close may have persisted
|
|
1377
|
+
// before its receipt failed. Re-read the item and clean markers only with
|
|
1378
|
+
// exact terminal evidence; otherwise retain identity for reconciliation.
|
|
1379
|
+
const postCloseStatus = compensationItemStatus(pmRoot, id);
|
|
1380
|
+
if (postCloseStatus !== "closed") {
|
|
1381
|
+
console.error(`atomic import: compensation close failed for ${id}: ${closeResult.stderr?.trim() || closeResult.stdout?.trim()}`);
|
|
1382
|
+
return;
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1169
1386
|
if (markers.length > 0) {
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
return; // terminal already; markers stripped above
|
|
1175
|
-
// Compensation is an internal rollback, not a user closure: it must reliably
|
|
1176
|
-
// undo the create regardless of closure-validation governance, so it bypasses
|
|
1177
|
-
// `--validate-close` (off). `require_close_reason` still applies and is
|
|
1178
|
-
// satisfied by `reason`. Without this a strict tracker would block the
|
|
1179
|
-
// rollback and leave the orphan open — the very leak compensation exists to
|
|
1180
|
-
// prevent. Used by both the atomic transaction path (rollback) and the
|
|
1181
|
-
// non-atomic `upsertCreate` close-failure path.
|
|
1182
|
-
const r = spawnSync("pm", ["--path", pmRoot, "close", id, "--reason", reason, "--validate-close", "off"], { encoding: "utf-8", maxBuffer: pmListMaxBuffer() });
|
|
1183
|
-
if (r.status !== 0 && r.status !== 4) {
|
|
1184
|
-
// Exit 4 is "already closed" / invalid state — treat as already compensated.
|
|
1185
|
-
console.error(`atomic import: compensation close failed for ${id}: ${r.stderr?.trim() || r.stdout?.trim()}`);
|
|
1387
|
+
const markerCleanup = spawnSync("pm", ["--path", pmRoot, "update", id, "--remove-tags", markers.join(",")], { encoding: "utf-8", maxBuffer: pmListMaxBuffer() });
|
|
1388
|
+
if (markerCleanup.status !== 0) {
|
|
1389
|
+
console.error(`atomic import: compensation marker cleanup failed for ${id}: ${markerCleanup.stderr?.trim() || markerCleanup.stdout?.trim()}`);
|
|
1390
|
+
}
|
|
1186
1391
|
}
|
|
1187
1392
|
}
|
|
1188
1393
|
/**
|
|
@@ -1190,40 +1395,28 @@ function compensateCreate(pmRoot, id, markers = [], reason = "atomic csv import
|
|
|
1190
1395
|
* {@link WorkspaceTransactionStep} committed under a single workspace
|
|
1191
1396
|
* writer-locked, crash-recoverable transaction. On success the same
|
|
1192
1397
|
* {@link ImportResult} as the non-atomic path is returned (counts derived from
|
|
1193
|
-
* the committed step results). On failure
|
|
1194
|
-
*
|
|
1195
|
-
*
|
|
1196
|
-
*
|
|
1398
|
+
* the committed step results). On failure the coordinator attempts reverse-
|
|
1399
|
+
* order compensation, but callers receive a conservative reconciliation
|
|
1400
|
+
* warning because create cleanup is best-effort and updates to pre-existing
|
|
1401
|
+
* items are intentionally not reverted. An interrupted run resumes from the
|
|
1402
|
+
* durable journal (inspect() skips already applied rows).
|
|
1197
1403
|
*/
|
|
1198
1404
|
async function importCSVAtomic(pmRoot, filePath, opts) {
|
|
1199
|
-
const {
|
|
1200
|
-
|
|
1201
|
-
imported: 0,
|
|
1202
|
-
updated: 0,
|
|
1203
|
-
skipped: 0,
|
|
1204
|
-
filtered: 0,
|
|
1205
|
-
autoMappings: [],
|
|
1206
|
-
errors: [],
|
|
1207
|
-
previews: [],
|
|
1208
|
-
fieldMapWarnings: [],
|
|
1209
|
-
};
|
|
1210
|
-
if (rawHeaders.length === 0)
|
|
1405
|
+
const { rawHeaders, dataRows, result, resolvedHeaders } = prepareInMemoryImport(filePath, opts);
|
|
1406
|
+
if (!resolvedHeaders)
|
|
1211
1407
|
return result;
|
|
1212
|
-
result.fieldMapWarnings =
|
|
1213
|
-
|
|
1214
|
-
const headers = applyFieldMap(rawHeaders, mapResolution.fieldMap);
|
|
1215
|
-
result.autoMappings = mapResolution.autoMappings;
|
|
1216
|
-
assertImportHeaders(headers, opts);
|
|
1408
|
+
result.fieldMapWarnings = resolvedHeaders.fieldMapWarnings;
|
|
1409
|
+
result.autoMappings = resolvedHeaders.autoMappings;
|
|
1217
1410
|
// A dry-run atomic import just previews like the non-atomic path (no
|
|
1218
1411
|
// transaction is committed, nothing is written).
|
|
1219
1412
|
if (opts.dryRun) {
|
|
1220
|
-
|
|
1221
|
-
for (let rowIndex = 0; rowIndex < dataRows.length; rowIndex++) {
|
|
1222
|
-
const lineNo = opts.skipHeaders ? rowIndex + 1 : rowIndex + 2;
|
|
1223
|
-
processImportRow(pmRoot, headers, dataRows[rowIndex], lineNo, opts, keyIndex, result);
|
|
1224
|
-
}
|
|
1413
|
+
processImportRows(pmRoot, dataRows, resolvedHeaders.headers, opts, result);
|
|
1225
1414
|
return result;
|
|
1226
1415
|
}
|
|
1416
|
+
const commitTransaction = opts.commitTransaction;
|
|
1417
|
+
if (!commitTransaction) {
|
|
1418
|
+
throw new CommandError("--atomic requires the host-injected commitWorkspaceTransaction SDK primitive. Upgrade the pm CLI and invoke pm-csv through the pm host.", EXIT_CODE.USAGE);
|
|
1419
|
+
}
|
|
1227
1420
|
// Fingerprint the exact parsed content (raw headers + all data rows) so that
|
|
1228
1421
|
// editing the file in place yields a fresh transaction id instead of reusing
|
|
1229
1422
|
// stale per-row markers from a previous import of different content.
|
|
@@ -1248,7 +1441,7 @@ async function importCSVAtomic(pmRoot, filePath, opts) {
|
|
|
1248
1441
|
const row = dataRows[rowIndex];
|
|
1249
1442
|
// Compute the field accessor once and reuse for both `parsed` and the
|
|
1250
1443
|
// --key value (previously rowFields() was called twice per row).
|
|
1251
|
-
const { get, parsed } = rowFields(headers, row);
|
|
1444
|
+
const { get, parsed } = rowFields(resolvedHeaders.headers, row);
|
|
1252
1445
|
if (!parsed.title) {
|
|
1253
1446
|
console.error(`Row ${lineNo}: skipping — 'title' is empty`);
|
|
1254
1447
|
result.skipped++;
|
|
@@ -1259,7 +1452,7 @@ async function importCSVAtomic(pmRoot, filePath, opts) {
|
|
|
1259
1452
|
result.filtered++;
|
|
1260
1453
|
continue;
|
|
1261
1454
|
}
|
|
1262
|
-
const keyValue = opts.keyField ?
|
|
1455
|
+
const keyValue = opts.keyField ? get(opts.keyField) : "";
|
|
1263
1456
|
const normalizedKey = keyValue ? normalizeKeyValue(keyValue) : "";
|
|
1264
1457
|
// In-batch duplicate-key guard: an earlier planned CREATE in this run
|
|
1265
1458
|
// already claimed this key. The first item does not exist yet at apply
|
|
@@ -1282,8 +1475,8 @@ async function importCSVAtomic(pmRoot, filePath, opts) {
|
|
|
1282
1475
|
// compensate() closes the created id.
|
|
1283
1476
|
// - update (--key match): apply() updates the pre-existing item and stamps
|
|
1284
1477
|
// both markers; compensate() is a no-op (an arbitrary update cannot be
|
|
1285
|
-
// safely reverted without capturing prior state
|
|
1286
|
-
//
|
|
1478
|
+
// safely reverted without capturing prior state; best-effort
|
|
1479
|
+
// compensation is therefore scoped to creates).
|
|
1287
1480
|
//
|
|
1288
1481
|
// Resume/compensation matching is per-row-precise via the per-row marker
|
|
1289
1482
|
// (csv-txrow:<transactionId>#<rowIndex>), so duplicate titles or duplicate
|
|
@@ -1319,8 +1512,6 @@ async function importCSVAtomic(pmRoot, filePath, opts) {
|
|
|
1319
1512
|
return row.existingId;
|
|
1320
1513
|
}
|
|
1321
1514
|
const newId = upsertCreate(pmRoot, row.parsed, opts.keyField ? row.keyValue : undefined, opts.source, ownershipTags);
|
|
1322
|
-
if (!newId)
|
|
1323
|
-
throw new Error(`pm create returned no id for row ${row.lineNo}`);
|
|
1324
1515
|
appliedId = newId;
|
|
1325
1516
|
return newId;
|
|
1326
1517
|
};
|
|
@@ -1340,17 +1531,17 @@ async function importCSVAtomic(pmRoot, filePath, opts) {
|
|
|
1340
1531
|
};
|
|
1341
1532
|
return { id: stepId, inspect, apply, prepareCompensation, compensate };
|
|
1342
1533
|
});
|
|
1343
|
-
const commitTransaction = opts.commitTransaction ??
|
|
1344
|
-
(await resolveCommitWorkspaceTransaction(pmRoot));
|
|
1345
1534
|
let committed;
|
|
1346
1535
|
try {
|
|
1347
1536
|
committed = await commitTransaction({ transactionId, author, steps });
|
|
1348
1537
|
}
|
|
1349
1538
|
catch (err) {
|
|
1350
|
-
const msg =
|
|
1351
|
-
//
|
|
1352
|
-
//
|
|
1353
|
-
|
|
1539
|
+
const msg = errorMessage(err);
|
|
1540
|
+
// The coordinator invokes compensations in reverse order, but this package
|
|
1541
|
+
// cannot prove a clean tracker: create cleanup deliberately tolerates an
|
|
1542
|
+
// unavailable status lookup or failed close so the remaining sweep can
|
|
1543
|
+
// continue, and update steps intentionally preserve pre-existing items.
|
|
1544
|
+
throw new CommandError(`Atomic CSV import failed. The transaction coordinator attempted reverse-order compensation, but pm-csv cannot claim a clean tracker: created items are closed on a best-effort basis, and pre-existing item updates are intentionally not reverted. Mutations from transaction ${transactionId} may remain; inspect and reconcile that transaction before retrying. Underlying error: ${msg}`, EXIT_CODE.GENERIC_FAILURE);
|
|
1354
1545
|
}
|
|
1355
1546
|
// Tally the ImportResult from the committed step results. Each committed
|
|
1356
1547
|
// result is the created/updated item id. Only rows that were NOT already
|
|
@@ -1372,47 +1563,6 @@ async function importCSVAtomic(pmRoot, filePath, opts) {
|
|
|
1372
1563
|
}
|
|
1373
1564
|
return result;
|
|
1374
1565
|
}
|
|
1375
|
-
/**
|
|
1376
|
-
* Cached SDK commit coordinator. Resolved once per process via a dynamic
|
|
1377
|
-
* `import("@unbrained/pm-cli/sdk")` so repeated --atomic calls don't re-import.
|
|
1378
|
-
* `null` means a previous attempt failed and the failure is not retried within
|
|
1379
|
-
* this process (each CLI invocation is a fresh process, so this is safe).
|
|
1380
|
-
*/
|
|
1381
|
-
let cachedCommitWorkspaceTransaction;
|
|
1382
|
-
/**
|
|
1383
|
-
* Dynamically resolve the SDK commit coordinator bound to a tracker root, for
|
|
1384
|
-
* the importer path (which has no host-injected `ctx.sdk`). Falls back to a
|
|
1385
|
-
* dynamic `import("@unbrained/pm-cli/sdk")` so a standalone-installed extension
|
|
1386
|
-
* still works when the SDK package is resolvable. The resolved function is
|
|
1387
|
-
* cached at module scope so repeated --atomic calls don't re-import. If the
|
|
1388
|
-
* import fails or `commitWorkspaceTransaction` is not exported, a clear,
|
|
1389
|
-
* actionable CommandError is thrown.
|
|
1390
|
-
*/
|
|
1391
|
-
async function resolveCommitWorkspaceTransaction(pmRoot) {
|
|
1392
|
-
if (cachedCommitWorkspaceTransaction === null) {
|
|
1393
|
-
throw new CommandError("--atomic requires @unbrained/pm-cli>=2026.7.19 with the commitWorkspaceTransaction SDK primitive, but it could not be resolved (a prior attempt in this process failed). Ensure @unbrained/pm-cli is installed and up to date.", EXIT_CODE.USAGE);
|
|
1394
|
-
}
|
|
1395
|
-
if (cachedCommitWorkspaceTransaction) {
|
|
1396
|
-
const cached = cachedCommitWorkspaceTransaction;
|
|
1397
|
-
return (opts) => cached({ pmRoot, ...opts });
|
|
1398
|
-
}
|
|
1399
|
-
let mod;
|
|
1400
|
-
try {
|
|
1401
|
-
mod = await import("@unbrained/pm-cli/sdk");
|
|
1402
|
-
}
|
|
1403
|
-
catch (err) {
|
|
1404
|
-
cachedCommitWorkspaceTransaction = null;
|
|
1405
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
1406
|
-
throw new CommandError(`--atomic requires @unbrained/pm-cli>=2026.7.19 with the commitWorkspaceTransaction SDK primitive, but the SDK could not be imported: ${msg}. Install or upgrade @unbrained/pm-cli.`, EXIT_CODE.USAGE);
|
|
1407
|
-
}
|
|
1408
|
-
const commit = mod.commitWorkspaceTransaction;
|
|
1409
|
-
if (typeof commit !== "function") {
|
|
1410
|
-
cachedCommitWorkspaceTransaction = null;
|
|
1411
|
-
throw new CommandError("--atomic requires @unbrained/pm-cli>=2026.7.19 with the commitWorkspaceTransaction SDK primitive, but the installed SDK does not export it as a function. Upgrade @unbrained/pm-cli to >=2026.7.19.", EXIT_CODE.USAGE);
|
|
1412
|
-
}
|
|
1413
|
-
cachedCommitWorkspaceTransaction = commit;
|
|
1414
|
-
return (opts) => commit({ pmRoot, ...opts });
|
|
1415
|
-
}
|
|
1416
1566
|
/**
|
|
1417
1567
|
* Streaming import: reads the file via a readable stream so large CSV files
|
|
1418
1568
|
* are never fully loaded into memory. The header row (or positional
|
|
@@ -1420,32 +1570,21 @@ async function resolveCommitWorkspaceTransaction(pmRoot) {
|
|
|
1420
1570
|
* first row, then every subsequent row is processed and upserted immediately.
|
|
1421
1571
|
*/
|
|
1422
1572
|
async function importCSVStreaming(pmRoot, filePath, opts) {
|
|
1423
|
-
const result =
|
|
1424
|
-
imported: 0,
|
|
1425
|
-
updated: 0,
|
|
1426
|
-
skipped: 0,
|
|
1427
|
-
filtered: 0,
|
|
1428
|
-
autoMappings: [],
|
|
1429
|
-
errors: [],
|
|
1430
|
-
previews: [],
|
|
1431
|
-
fieldMapWarnings: [],
|
|
1432
|
-
};
|
|
1573
|
+
const result = createImportResult();
|
|
1433
1574
|
let headers = [];
|
|
1434
1575
|
let headerResolved = opts.skipHeaders ?? false;
|
|
1435
1576
|
let logicalRowIndex = 0;
|
|
1436
1577
|
if (headerResolved) {
|
|
1437
1578
|
// --skip-headers: no header row in the file; use positional column order.
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
result.autoMappings = mapResolution.autoMappings;
|
|
1443
|
-
assertImportHeaders(headers, opts);
|
|
1579
|
+
const resolvedHeaders = resolveImportHeaders([...IMPORT_COLUMNS], opts);
|
|
1580
|
+
headers = resolvedHeaders.headers;
|
|
1581
|
+
result.fieldMapWarnings = resolvedHeaders.fieldMapWarnings;
|
|
1582
|
+
result.autoMappings = resolvedHeaders.autoMappings;
|
|
1444
1583
|
}
|
|
1445
1584
|
// Load only after the input header has been validated so malformed input
|
|
1446
1585
|
// reports its CSV error before any workspace lookup is attempted.
|
|
1447
1586
|
let keyIndex = new Map();
|
|
1448
|
-
if (headerResolved && opts.keyField
|
|
1587
|
+
if (headerResolved && opts.keyField)
|
|
1449
1588
|
keyIndex = loadKeyIndex(pmRoot);
|
|
1450
1589
|
await streamCSVFile(filePath, opts.delimiter, opts.encoding ?? "utf-8", (row) => {
|
|
1451
1590
|
logicalRowIndex++;
|
|
@@ -1453,14 +1592,12 @@ async function importCSVStreaming(pmRoot, filePath, opts) {
|
|
|
1453
1592
|
if (!row.some((f) => f.trim() !== ""))
|
|
1454
1593
|
return;
|
|
1455
1594
|
if (!headerResolved) {
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
result.autoMappings = mapResolution.autoMappings;
|
|
1461
|
-
assertImportHeaders(headers, opts);
|
|
1595
|
+
const resolvedHeaders = resolveImportHeaders(row.map((header) => header.trim().toLowerCase()), opts);
|
|
1596
|
+
headers = resolvedHeaders.headers;
|
|
1597
|
+
result.fieldMapWarnings = resolvedHeaders.fieldMapWarnings;
|
|
1598
|
+
result.autoMappings = resolvedHeaders.autoMappings;
|
|
1462
1599
|
headerResolved = true;
|
|
1463
|
-
if (opts.keyField
|
|
1600
|
+
if (opts.keyField)
|
|
1464
1601
|
keyIndex = loadKeyIndex(pmRoot);
|
|
1465
1602
|
return;
|
|
1466
1603
|
}
|
|
@@ -1486,6 +1623,18 @@ function appendRelationalArgs(args, p) {
|
|
|
1486
1623
|
if (p.blocked_by)
|
|
1487
1624
|
args.push("--blocked-by", p.blocked_by);
|
|
1488
1625
|
}
|
|
1626
|
+
/** Append item fields shared by create and update mutation commands. */
|
|
1627
|
+
function appendMutableItemArgs(args, p, preserveEmptyBody) {
|
|
1628
|
+
if (preserveEmptyBody ? p.body !== undefined : Boolean(p.body))
|
|
1629
|
+
args.push("--body", p.body ?? "");
|
|
1630
|
+
if (p.priority !== undefined)
|
|
1631
|
+
args.push("--priority", String(p.priority));
|
|
1632
|
+
if (p.type)
|
|
1633
|
+
args.push("--type", p.type);
|
|
1634
|
+
if (p.deadline)
|
|
1635
|
+
args.push("--deadline", p.deadline);
|
|
1636
|
+
appendRelationalArgs(args, p);
|
|
1637
|
+
}
|
|
1489
1638
|
/**
|
|
1490
1639
|
* Build the `pm close --reason` text for an imported row whose CSV status is
|
|
1491
1640
|
* terminal (`closed`/`canceled`). Since pm-cli 2026.8.3 the CLI enforces
|
|
@@ -1523,14 +1672,14 @@ function importCloseReason(status, source) {
|
|
|
1523
1672
|
* open item is left behind in this one unrecoverable case.
|
|
1524
1673
|
* 2. `pm close` failed after the item was persisted as open. The orphan is
|
|
1525
1674
|
* compensated (closed) via {@link compensateCreate} with `--validate-close
|
|
1526
|
-
* off`, so a
|
|
1675
|
+
* off`, so a verified cleanup leaves no open orphan and a retry re-imports
|
|
1676
|
+
* it. If
|
|
1527
1677
|
* compensation also fails, the thrown error carries the created id and
|
|
1528
1678
|
* states the item was left open, so the partial state is actionable and a
|
|
1529
1679
|
* retry can reconcile it.
|
|
1530
1680
|
*
|
|
1531
|
-
*
|
|
1532
|
-
*
|
|
1533
|
-
* `pm create` reported no id.
|
|
1681
|
+
* An id-less successful receipt is therefore a hard error for every status,
|
|
1682
|
+
* never a silent successful import.
|
|
1534
1683
|
*/
|
|
1535
1684
|
function upsertCreate(pmRoot, p, keyValue, source, extraTags) {
|
|
1536
1685
|
const tags = [...p.tags];
|
|
@@ -1550,15 +1699,7 @@ function upsertCreate(pmRoot, p, keyValue, source, extraTags) {
|
|
|
1550
1699
|
// after the create, never via `pm create --status closed`.
|
|
1551
1700
|
const createStatus = p.status === "closed" ? "open" : p.status;
|
|
1552
1701
|
const args = ["--path", pmRoot, "create", "--title", p.title, "--status", createStatus, "--json"];
|
|
1553
|
-
|
|
1554
|
-
args.push("--body", p.body);
|
|
1555
|
-
if (p.priority !== undefined)
|
|
1556
|
-
args.push("--priority", String(p.priority));
|
|
1557
|
-
if (p.type)
|
|
1558
|
-
args.push("--type", p.type);
|
|
1559
|
-
if (p.deadline)
|
|
1560
|
-
args.push("--deadline", p.deadline);
|
|
1561
|
-
appendRelationalArgs(args, p);
|
|
1702
|
+
appendMutableItemArgs(args, p, false);
|
|
1562
1703
|
if (tags.length > 0)
|
|
1563
1704
|
args.push("--tags", tags.join(","));
|
|
1564
1705
|
const r = spawnSync("pm", args, { encoding: "utf-8", maxBuffer: pmListMaxBuffer() });
|
|
@@ -1623,14 +1764,10 @@ function upsertCreate(pmRoot, p, keyValue, source, extraTags) {
|
|
|
1623
1764
|
}
|
|
1624
1765
|
if (r.status !== 0)
|
|
1625
1766
|
throw new Error(r.stderr?.trim() || "pm create failed");
|
|
1767
|
+
if (!id) {
|
|
1768
|
+
throw new Error(`pm create succeeded for row (title '${p.title}', source status '${p.status}') but returned no id, so the source status cannot be applied and the created item cannot be identified or safely reconciled; it may have been left as an orphan — reconcile manually`);
|
|
1769
|
+
}
|
|
1626
1770
|
if (p.status === "closed") {
|
|
1627
|
-
// (1) No id recovered: the close cannot be applied and the orphan cannot be
|
|
1628
|
-
// compensated without its id. Hard failure — never a silent success — so
|
|
1629
|
-
// the row is reported as failed, not imported. The created open orphan is
|
|
1630
|
-
// unrecoverable from here; the error names the title for manual reconcile.
|
|
1631
|
-
if (!id) {
|
|
1632
|
-
throw new Error(`pm create succeeded for closed row (title '${p.title}') but returned no id, so the terminal 'closed' status cannot be applied and the created open item cannot be compensated; it is left as an unrecoverable open orphan — reconcile manually`);
|
|
1633
|
-
}
|
|
1634
1771
|
const cr = spawnSync("pm", ["--path", pmRoot, "close", id, "--reason", importCloseReason("closed", source)], { encoding: "utf-8", maxBuffer: pmListMaxBuffer() });
|
|
1635
1772
|
// Deliberately NO assertPmOutputFit(cr, "close") here: a null close status
|
|
1636
1773
|
// (receipt overrun or a signal kill) must route INTO the compensation branch
|
|
@@ -1639,8 +1776,8 @@ function upsertCreate(pmRoot, p, keyValue, source, extraTags) {
|
|
|
1639
1776
|
// way, and the id is already known so nothing is stranded.
|
|
1640
1777
|
if (cr.status !== 0) {
|
|
1641
1778
|
// (2) The close failed but the item is already persisted as open.
|
|
1642
|
-
// Compensate (close) the orphan so a
|
|
1643
|
-
// retry without --key-field
|
|
1779
|
+
// Compensate (close) the orphan so a verified cleanup leaves no open item
|
|
1780
|
+
// that a retry without --key-field could duplicate.
|
|
1644
1781
|
// Compensation bypasses closure-validation governance (it is a rollback,
|
|
1645
1782
|
// not a user closure) so a strict tracker cannot block the cleanup.
|
|
1646
1783
|
compensateCreate(pmRoot, id, [], "csv import row failed; closing orphan created item");
|
|
@@ -1658,7 +1795,7 @@ function upsertCreate(pmRoot, p, keyValue, source, extraTags) {
|
|
|
1658
1795
|
: `also failed: the item was created and left OPEN (status '${compensatedStatus}')`;
|
|
1659
1796
|
throw new Error(`pm close failed for closed row (title '${p.title}', id ${id}) and compensation ${state} — retry or reconcile by id. ${closeErr}`);
|
|
1660
1797
|
}
|
|
1661
|
-
throw new Error(`pm close failed for closed row (title '${p.title}', id ${id}); the created open item was compensated (verified closed) so
|
|
1798
|
+
throw new Error(`pm close failed for closed row (title '${p.title}', id ${id}); the created open item was compensated (verified closed), so no open orphan remains. ${closeErr}`);
|
|
1662
1799
|
}
|
|
1663
1800
|
}
|
|
1664
1801
|
return id;
|
|
@@ -1684,15 +1821,7 @@ function upsertCreate(pmRoot, p, keyValue, source, extraTags) {
|
|
|
1684
1821
|
*/
|
|
1685
1822
|
function upsertUpdate(pmRoot, id, p, source, extraTags) {
|
|
1686
1823
|
const args = ["--path", pmRoot, "update", id, "--title", p.title];
|
|
1687
|
-
|
|
1688
|
-
args.push("--body", p.body);
|
|
1689
|
-
if (p.priority !== undefined)
|
|
1690
|
-
args.push("--priority", String(p.priority));
|
|
1691
|
-
if (p.type)
|
|
1692
|
-
args.push("--type", p.type);
|
|
1693
|
-
if (p.deadline)
|
|
1694
|
-
args.push("--deadline", p.deadline);
|
|
1695
|
-
appendRelationalArgs(args, p);
|
|
1824
|
+
appendMutableItemArgs(args, p, true);
|
|
1696
1825
|
// Preserve the csv-key tag (additive) and refresh the user tags.
|
|
1697
1826
|
const addTags = [...p.tags];
|
|
1698
1827
|
if (source)
|
|
@@ -1884,6 +2013,47 @@ function assertStrictImportReady(filePath, opts) {
|
|
|
1884
2013
|
}
|
|
1885
2014
|
return report;
|
|
1886
2015
|
}
|
|
2016
|
+
/** Parse one host option record into the canonical CSV import contract. */
|
|
2017
|
+
function parseCsvImportInvocation(rawOptions, dryRun) {
|
|
2018
|
+
return {
|
|
2019
|
+
strict: readBoolOption(rawOptions, "strict"),
|
|
2020
|
+
options: {
|
|
2021
|
+
delimiter: resolveDelimiter(rawOptions["delimiter"]),
|
|
2022
|
+
dryRun,
|
|
2023
|
+
fieldMap: parseFieldMap(rawOptions["map"]),
|
|
2024
|
+
autoMap: readBoolOption(rawOptions, "auto-map", "autoMap"),
|
|
2025
|
+
keyField: (rawOptions["key"] ?? "").trim().toLowerCase() || undefined,
|
|
2026
|
+
encoding: resolveEncoding(rawOptions["encoding"]),
|
|
2027
|
+
source: (rawOptions["source"] ?? "").trim() || undefined,
|
|
2028
|
+
filter: parseImportFilter(rawOptions["status"], rawOptions["type"], rawOptions["priority"]),
|
|
2029
|
+
skipHeaders: readBoolOption(rawOptions, "skip-headers", "skipHeaders"),
|
|
2030
|
+
stream: readBoolOption(rawOptions, "stream"),
|
|
2031
|
+
atomic: readBoolOption(rawOptions, "atomic"),
|
|
2032
|
+
},
|
|
2033
|
+
};
|
|
2034
|
+
}
|
|
2035
|
+
/** Validate strict input, then execute one fully parsed import invocation. */
|
|
2036
|
+
function executeCsvImport(pmRoot, filePath, invocation, commitTransaction, atomicAuthor) {
|
|
2037
|
+
if (invocation.strict) {
|
|
2038
|
+
assertStrictImportReady(filePath, {
|
|
2039
|
+
delimiter: invocation.options.delimiter,
|
|
2040
|
+
fieldMap: invocation.options.fieldMap,
|
|
2041
|
+
encoding: invocation.options.encoding,
|
|
2042
|
+
autoMap: invocation.options.autoMap,
|
|
2043
|
+
skipHeaders: invocation.options.skipHeaders,
|
|
2044
|
+
});
|
|
2045
|
+
}
|
|
2046
|
+
return importCSV(pmRoot, filePath, {
|
|
2047
|
+
...invocation.options,
|
|
2048
|
+
commitTransaction,
|
|
2049
|
+
atomicAuthor,
|
|
2050
|
+
});
|
|
2051
|
+
}
|
|
2052
|
+
/** Log an import's selected engine and execute it through the shared host path. */
|
|
2053
|
+
function executeLoggedCsvImport(label, pmRoot, filePath, invocation, commitTransaction, atomicAuthor) {
|
|
2054
|
+
console.error(`${label} ${filePath}${invocation.options.stream ? " (streaming)" : ""}${invocation.options.atomic ? " (atomic)" : ""}`);
|
|
2055
|
+
return executeCsvImport(pmRoot, filePath, invocation, commitTransaction, atomicAuthor);
|
|
2056
|
+
}
|
|
1887
2057
|
/**
|
|
1888
2058
|
* Discover custom item fields registered in the workspace runtime schema and
|
|
1889
2059
|
* return those that are NOT already covered by the built-in export columns
|
|
@@ -1998,43 +2168,60 @@ function resolveExportColumns(pmRoot, columnsSpec, discover) {
|
|
|
1998
2168
|
}
|
|
1999
2169
|
const columns = [...EXPORT_COLUMNS];
|
|
2000
2170
|
for (const f of discovered) {
|
|
2001
|
-
|
|
2002
|
-
|
|
2171
|
+
// Discovery excludes built-ins and de-duplicates by key, so every result is
|
|
2172
|
+
// additive by construction.
|
|
2173
|
+
columns.push(f.key);
|
|
2003
2174
|
}
|
|
2004
2175
|
return { columns, columnSource };
|
|
2005
2176
|
}
|
|
2006
2177
|
/**
|
|
2007
2178
|
* Read every item from a tracker and render the CSV export body.
|
|
2008
2179
|
*
|
|
2009
|
-
* Shells out to `pm list
|
|
2180
|
+
* Shells out to canonical `pm list --all --json --include-body` with an explicit
|
|
2010
2181
|
* `maxBuffer`, because a stdout overrun kills the child with a null status and
|
|
2011
2182
|
* an EMPTY stderr — which would otherwise surface as an unexplained
|
|
2012
|
-
* "pm list
|
|
2183
|
+
* "pm list --all failed" rather than as the size problem it is.
|
|
2013
2184
|
*
|
|
2014
2185
|
* Status and type filters are applied after the read, not pushed into the
|
|
2015
2186
|
* query. Each item's `csv_source` is then derived from its internal source tag
|
|
2016
2187
|
* via {@link decodeKeyTagValue}, so a re-export carries the provenance of the
|
|
2017
2188
|
* file the row originally came from.
|
|
2018
2189
|
*
|
|
2190
|
+
* The envelope's completeness receipt is verified by
|
|
2191
|
+
* {@link assertListAllComplete} before any row is rendered: a truncated,
|
|
2192
|
+
* paged, partially-read or field-omitted answer refuses instead of exporting a
|
|
2193
|
+
* silent subset (the 2026.8.14 truncated-default regression shipped a 10-row
|
|
2194
|
+
* export of a 682-item workspace as success).
|
|
2195
|
+
*
|
|
2019
2196
|
* @param pmRoot - Tracker root to export.
|
|
2020
2197
|
* @param opts - Column selection, per-column property remap, and status/type filters.
|
|
2198
|
+
* @param spawn - Injectable shell-out seam (tests substitute a canned real
|
|
2199
|
+
* envelope); defaults to the real `spawnPm`.
|
|
2021
2200
|
* @returns The rendered CSV text, the exported row count, and the line ending used.
|
|
2022
|
-
* @throws CommandError when the pm read fails
|
|
2201
|
+
* @throws CommandError when the pm read fails or the answer is incomplete,
|
|
2202
|
+
* naming the buffer overrun or the tripped completeness signal.
|
|
2023
2203
|
*/
|
|
2024
|
-
function buildCsvExport(pmRoot, opts) {
|
|
2025
|
-
const result =
|
|
2204
|
+
function buildCsvExport(pmRoot, opts, spawn = spawnPm) {
|
|
2205
|
+
const result = spawn(completeListArgs(pmRoot), { encoding: "utf-8", maxBuffer: pmListMaxBuffer() });
|
|
2026
2206
|
// A stdout overrun kills the child with status null and EMPTY stderr, so name
|
|
2027
|
-
// the real cause instead of reporting an unexplained "pm list
|
|
2207
|
+
// the real cause instead of reporting an unexplained "pm list --all failed".
|
|
2028
2208
|
if (result.error) {
|
|
2029
2209
|
const code = result.error.code;
|
|
2030
2210
|
throw new CommandError(code === "ENOBUFS"
|
|
2031
|
-
? `pm list
|
|
2032
|
-
: `pm list
|
|
2211
|
+
? `pm list --all output exceeded the ${pmListMaxBuffer()} byte read buffer; narrow the export (--status/--type) or raise the PM_LIST_MAX_BUFFER env var.`
|
|
2212
|
+
: `pm list --all failed: ${result.error.message}`);
|
|
2033
2213
|
}
|
|
2034
2214
|
if (result.status !== 0) {
|
|
2035
|
-
throw new CommandError(result.stderr || "pm list
|
|
2215
|
+
throw new CommandError(result.stderr || "pm list --all failed");
|
|
2216
|
+
}
|
|
2217
|
+
let parsed;
|
|
2218
|
+
try {
|
|
2219
|
+
parsed = JSON.parse(result.stdout);
|
|
2036
2220
|
}
|
|
2037
|
-
|
|
2221
|
+
catch (err) {
|
|
2222
|
+
throw new CommandError(`Could not parse \`pm list --all --json\` output: ${err instanceof Error ? err.message : String(err)}`);
|
|
2223
|
+
}
|
|
2224
|
+
let items = assertListAllComplete(parsed);
|
|
2038
2225
|
if (opts.statusFilter)
|
|
2039
2226
|
items = items.filter((i) => i.status === opts.statusFilter);
|
|
2040
2227
|
if (opts.typeFilter)
|
|
@@ -2074,42 +2261,39 @@ function buildCsvExport(pmRoot, opts) {
|
|
|
2074
2261
|
eol,
|
|
2075
2262
|
};
|
|
2076
2263
|
}
|
|
2077
|
-
/**
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2264
|
+
/** Parse export flags and render the tracker through the shared export core. */
|
|
2265
|
+
function prepareCsvExport(pmRoot, rawOptions) {
|
|
2266
|
+
const delimiter = resolveDelimiter(rawOptions["delimiter"]);
|
|
2267
|
+
const discover = readBoolOption(rawOptions, "all-fields", "allFields", "discover-fields", "discoverFields");
|
|
2268
|
+
const { columns, columnSource } = resolveExportColumns(pmRoot, rawOptions["columns"], discover);
|
|
2269
|
+
return {
|
|
2270
|
+
outputPath: rawOptions["output"],
|
|
2271
|
+
...buildCsvExport(pmRoot, {
|
|
2272
|
+
statusFilter: rawOptions["status"],
|
|
2273
|
+
typeFilter: rawOptions["type"],
|
|
2274
|
+
delimiter,
|
|
2275
|
+
columns,
|
|
2276
|
+
columnSource,
|
|
2277
|
+
noHeader: readNoHeaderOption(rawOptions),
|
|
2278
|
+
crlf: readBoolOption(rawOptions, "crlf"),
|
|
2279
|
+
excel: readBoolOption(rawOptions, "excel"),
|
|
2280
|
+
}),
|
|
2281
|
+
};
|
|
2282
|
+
}
|
|
2283
|
+
export default {
|
|
2087
2284
|
name: "pm-csv",
|
|
2088
|
-
version: "2026.8.
|
|
2285
|
+
version: "2026.8.28",
|
|
2089
2286
|
activate(api) {
|
|
2090
2287
|
// -----------------------------------------------------------------------
|
|
2091
2288
|
// Schema: register an optional `csv_source` provenance field so imported
|
|
2092
2289
|
// items can record where they came from (set via `pm csv import --source`).
|
|
2093
|
-
//
|
|
2094
|
-
//
|
|
2095
|
-
//
|
|
2096
|
-
//
|
|
2097
|
-
// NOTE: pm 2026.5.31 accepts the field into the schema registry but exposes
|
|
2098
|
-
// no `pm create --csv_source` setter for extension-registered scalar fields,
|
|
2099
|
-
// so the importer persists the provenance label as a `csv-source:` tag
|
|
2100
|
-
// (stripped from exports and surfaced back via the csv_source export column).
|
|
2290
|
+
// The declared pm host floor guarantees this SDK surface. Registration is
|
|
2291
|
+
// deliberately fail-closed: silently omitting provenance would make a
|
|
2292
|
+
// successfully activated package weaker than its manifest contract.
|
|
2101
2293
|
// -----------------------------------------------------------------------
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
{ name: "csv_source", type: "string", optional: true },
|
|
2106
|
-
]);
|
|
2107
|
-
}
|
|
2108
|
-
catch (err) {
|
|
2109
|
-
// Never let a schema-registration hiccup break command registration.
|
|
2110
|
-
console.error(`pm-csv: csv_source field not registered — ${err instanceof Error ? err.message : String(err)}`);
|
|
2111
|
-
}
|
|
2112
|
-
}
|
|
2294
|
+
api.registerItemFields([
|
|
2295
|
+
{ name: "csv_source", type: "string", optional: true },
|
|
2296
|
+
]);
|
|
2113
2297
|
// -----------------------------------------------------------------------
|
|
2114
2298
|
// Command: pm csv import <file>
|
|
2115
2299
|
// -----------------------------------------------------------------------
|
|
@@ -2140,7 +2324,7 @@ export default defineExtension({
|
|
|
2140
2324
|
"pm csv import items.csv --dry-run",
|
|
2141
2325
|
"pm csv import headerless.csv --skip-headers # no header row, positional columns",
|
|
2142
2326
|
"pm csv import big.csv --stream # stream large files without loading into memory",
|
|
2143
|
-
"pm csv import tasks.csv --atomic #
|
|
2327
|
+
"pm csv import tasks.csv --atomic # writer-locked, crash-recoverable import",
|
|
2144
2328
|
],
|
|
2145
2329
|
flags: [
|
|
2146
2330
|
{ long: "--delimiter", value_name: "char", description: "Field delimiter, or alias tab|comma|semicolon|pipe (default: ,)" },
|
|
@@ -2156,55 +2340,27 @@ export default defineExtension({
|
|
|
2156
2340
|
{ long: "--dry-run", description: "Preview without writing" },
|
|
2157
2341
|
{ long: "--skip-headers", description: "The CSV file has no header row; map columns positionally to the standard import order (title, type, status, priority, tags, deadline, body, parent, assignee, sprint, release, blocked_by)" },
|
|
2158
2342
|
{ long: "--stream", description: "Stream the file row-by-row instead of loading it fully into memory (recommended for large CSV files)" },
|
|
2159
|
-
{ long: "--atomic", description: "
|
|
2343
|
+
{ long: "--atomic", description: "Use one writer-locked, crash-recoverable transaction. On failure, attempt best-effort create compensation; pre-existing item updates are intentionally not reverted. Inspect and reconcile any marked orphan before retrying. Incompatible with --stream" },
|
|
2160
2344
|
],
|
|
2161
2345
|
async run(ctx) {
|
|
2162
2346
|
const filePath = ctx.args[0];
|
|
2163
2347
|
if (!filePath) {
|
|
2164
2348
|
throw new CommandError("Usage: pm csv import <file> [--delimiter <char>] [--map col=field] [--auto-map] [--key field] [--encoding enc] [--source label] [--status s] [--type t] [--priority n] [--dry-run]", EXIT_CODE.USAGE);
|
|
2165
2349
|
}
|
|
2166
|
-
const delimiter = resolveDelimiter(ctx.options["delimiter"]);
|
|
2167
2350
|
const dryRun = readBoolOption(ctx.options, "dry-run", "dryRun");
|
|
2168
|
-
const
|
|
2169
|
-
const autoMap = readBoolOption(ctx.options, "auto-map", "autoMap");
|
|
2170
|
-
const keyField = (ctx.options["key"] ?? "").trim().toLowerCase() || undefined;
|
|
2171
|
-
const encoding = resolveEncoding(ctx.options["encoding"]);
|
|
2172
|
-
const source = (ctx.options["source"] ?? "").trim() || undefined;
|
|
2173
|
-
const strict = readBoolOption(ctx.options, "strict");
|
|
2174
|
-
const skipHeaders = readBoolOption(ctx.options, "skip-headers", "skipHeaders");
|
|
2175
|
-
const stream = readBoolOption(ctx.options, "stream");
|
|
2176
|
-
const atomic = readBoolOption(ctx.options, "atomic");
|
|
2177
|
-
const filter = parseImportFilter(ctx.options["status"], ctx.options["type"], ctx.options["priority"]);
|
|
2351
|
+
const invocation = parseCsvImportInvocation(ctx.options, dryRun);
|
|
2178
2352
|
const absolutePath = resolve(filePath);
|
|
2179
|
-
console.error(`Reading CSV from: ${absolutePath}${stream ? " (streaming)" : ""}${atomic ? " (atomic)" : ""}`);
|
|
2180
2353
|
let res;
|
|
2181
2354
|
try {
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
dryRun,
|
|
2187
|
-
fieldMap,
|
|
2188
|
-
autoMap,
|
|
2189
|
-
keyField,
|
|
2190
|
-
encoding,
|
|
2191
|
-
source,
|
|
2192
|
-
filter,
|
|
2193
|
-
skipHeaders,
|
|
2194
|
-
stream,
|
|
2195
|
-
atomic,
|
|
2196
|
-
// Prefer the host-injected SDK coordinator (bound to the
|
|
2197
|
-
// tracker root) so a standalone-installed extension never relies
|
|
2198
|
-
// on package resolution; the atomic path falls back to a dynamic
|
|
2199
|
-
// import when ctx.sdk is absent.
|
|
2200
|
-
commitTransaction: ctx.sdk?.commitWorkspaceTransaction,
|
|
2201
|
-
atomicAuthor: ctx.global?.author ?? undefined,
|
|
2202
|
-
});
|
|
2355
|
+
res = await executeLoggedCsvImport("Reading CSV from:", ctx.pm_root, absolutePath, invocation,
|
|
2356
|
+
// Use the host-bound coordinator so a copied extension never
|
|
2357
|
+
// resolves a second CLI/SDK through its own module tree.
|
|
2358
|
+
ctx.sdk?.commitWorkspaceTransaction, ctx.global?.author ?? undefined);
|
|
2203
2359
|
}
|
|
2204
2360
|
catch (err) {
|
|
2205
2361
|
if (err instanceof CommandError)
|
|
2206
2362
|
throw err;
|
|
2207
|
-
const msg =
|
|
2363
|
+
const msg = errorMessage(err);
|
|
2208
2364
|
const exitCode = /ENOENT|no such file/i.test(msg) ? EXIT_CODE.NOT_FOUND : EXIT_CODE.GENERIC_FAILURE;
|
|
2209
2365
|
throw new CommandError(`Failed to import: ${msg}`, exitCode);
|
|
2210
2366
|
}
|
|
@@ -2282,9 +2438,7 @@ export default defineExtension({
|
|
|
2282
2438
|
report = validateCSV(absolutePath, { delimiter, fieldMap, autoMap, encoding, skipHeaders });
|
|
2283
2439
|
}
|
|
2284
2440
|
catch (err) {
|
|
2285
|
-
|
|
2286
|
-
throw err;
|
|
2287
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
2441
|
+
const msg = errorMessage(err);
|
|
2288
2442
|
const exitCode = /ENOENT|no such file/i.test(msg) ? EXIT_CODE.NOT_FOUND : EXIT_CODE.GENERIC_FAILURE;
|
|
2289
2443
|
throw new CommandError(`Failed to validate: ${msg}`, exitCode);
|
|
2290
2444
|
}
|
|
@@ -2347,24 +2501,8 @@ export default defineExtension({
|
|
|
2347
2501
|
{ long: "--excel", description: "Excel-friendly output: CRLF line endings + a UTF-8 BOM prefix" },
|
|
2348
2502
|
],
|
|
2349
2503
|
async run(ctx) {
|
|
2350
|
-
const delimiter = resolveDelimiter(ctx.options["delimiter"]);
|
|
2351
|
-
const outputPath = ctx.options["output"];
|
|
2352
|
-
const discover = readBoolOption(ctx.options, "all-fields", "allFields", "discover-fields", "discoverFields");
|
|
2353
|
-
const { columns, columnSource } = resolveExportColumns(ctx.pm_root, ctx.options["columns"], discover);
|
|
2354
|
-
const noHeader = readNoHeaderOption(ctx.options);
|
|
2355
|
-
const crlf = readBoolOption(ctx.options, "crlf");
|
|
2356
|
-
const excel = readBoolOption(ctx.options, "excel");
|
|
2357
2504
|
console.error("Fetching pm items…");
|
|
2358
|
-
const { csvText, count, eol } =
|
|
2359
|
-
statusFilter: ctx.options["status"],
|
|
2360
|
-
typeFilter: ctx.options["type"],
|
|
2361
|
-
delimiter,
|
|
2362
|
-
columns,
|
|
2363
|
-
columnSource,
|
|
2364
|
-
noHeader,
|
|
2365
|
-
crlf,
|
|
2366
|
-
excel,
|
|
2367
|
-
});
|
|
2505
|
+
const { csvText, count, eol, outputPath } = prepareCsvExport(ctx.pm_root, ctx.options);
|
|
2368
2506
|
if (count === 0) {
|
|
2369
2507
|
console.error("No items found.");
|
|
2370
2508
|
return { exported: 0 };
|
|
@@ -2387,23 +2525,7 @@ export default defineExtension({
|
|
|
2387
2525
|
// Mirrors the importer so CSV is a first-class import/export pair.
|
|
2388
2526
|
// -----------------------------------------------------------------------
|
|
2389
2527
|
api.registerExporter("csv-export", async (ctx) => {
|
|
2390
|
-
const
|
|
2391
|
-
const outputPath = ctx.options["output"];
|
|
2392
|
-
const discover = readBoolOption(ctx.options, "all-fields", "allFields", "discover-fields", "discoverFields");
|
|
2393
|
-
const { columns, columnSource } = resolveExportColumns(ctx.pm_root, ctx.options["columns"], discover);
|
|
2394
|
-
const noHeader = readNoHeaderOption(ctx.options);
|
|
2395
|
-
const crlf = readBoolOption(ctx.options, "crlf");
|
|
2396
|
-
const excel = readBoolOption(ctx.options, "excel");
|
|
2397
|
-
const { csvText, count, eol } = buildCsvExport(ctx.pm_root, {
|
|
2398
|
-
statusFilter: ctx.options["status"],
|
|
2399
|
-
typeFilter: ctx.options["type"],
|
|
2400
|
-
delimiter,
|
|
2401
|
-
columns,
|
|
2402
|
-
columnSource,
|
|
2403
|
-
noHeader,
|
|
2404
|
-
crlf,
|
|
2405
|
-
excel,
|
|
2406
|
-
});
|
|
2528
|
+
const { csvText, count, eol, outputPath } = prepareCsvExport(ctx.pm_root, ctx.options);
|
|
2407
2529
|
if (outputPath) {
|
|
2408
2530
|
const absolutePath = resolve(outputPath);
|
|
2409
2531
|
// Match the inter-record EOL on the trailing terminator (no lone LF in --crlf/--excel).
|
|
@@ -2423,40 +2545,14 @@ export default defineExtension({
|
|
|
2423
2545
|
console.error("csv-import: no 'file' provided in options — skipping.");
|
|
2424
2546
|
return;
|
|
2425
2547
|
}
|
|
2426
|
-
const
|
|
2427
|
-
const fieldMap = parseFieldMap(ctx.options["map"]);
|
|
2428
|
-
const autoMap = readBoolOption(ctx.options, "auto-map", "autoMap");
|
|
2429
|
-
const keyField = (ctx.options["key"] ?? "").trim().toLowerCase() || undefined;
|
|
2430
|
-
const encoding = resolveEncoding(ctx.options["encoding"]);
|
|
2431
|
-
const source = (ctx.options["source"] ?? "").trim() || undefined;
|
|
2432
|
-
const strict = readBoolOption(ctx.options, "strict");
|
|
2433
|
-
const skipHeaders = readBoolOption(ctx.options, "skip-headers", "skipHeaders");
|
|
2434
|
-
const stream = readBoolOption(ctx.options, "stream");
|
|
2435
|
-
const atomic = readBoolOption(ctx.options, "atomic");
|
|
2436
|
-
const filter = parseImportFilter(ctx.options["status"], ctx.options["type"], ctx.options["priority"]);
|
|
2548
|
+
const invocation = parseCsvImportInvocation(ctx.options, false);
|
|
2437
2549
|
const absolutePath = resolve(filePath);
|
|
2438
|
-
console.error(`csv-import: reading ${absolutePath}${stream ? " (streaming)" : ""}${atomic ? " (atomic)" : ""}`);
|
|
2439
2550
|
let res;
|
|
2440
2551
|
try {
|
|
2441
|
-
|
|
2442
|
-
assertStrictImportReady(absolutePath, { delimiter, fieldMap, encoding, autoMap, skipHeaders });
|
|
2443
|
-
res = await importCSV(ctx.pm_root, absolutePath, {
|
|
2444
|
-
delimiter,
|
|
2445
|
-
dryRun: false,
|
|
2446
|
-
fieldMap,
|
|
2447
|
-
autoMap,
|
|
2448
|
-
keyField,
|
|
2449
|
-
encoding,
|
|
2450
|
-
source,
|
|
2451
|
-
filter,
|
|
2452
|
-
skipHeaders,
|
|
2453
|
-
stream,
|
|
2454
|
-
atomic,
|
|
2455
|
-
atomicAuthor: ctx.global?.author ?? undefined,
|
|
2456
|
-
});
|
|
2552
|
+
res = await executeLoggedCsvImport("csv-import: reading", ctx.pm_root, absolutePath, invocation, ctx.sdk?.commitWorkspaceTransaction, ctx.global?.author ?? undefined);
|
|
2457
2553
|
}
|
|
2458
2554
|
catch (err) {
|
|
2459
|
-
const msg =
|
|
2555
|
+
const msg = errorMessage(err);
|
|
2460
2556
|
console.error(`csv-import: failed — ${msg}`);
|
|
2461
2557
|
return;
|
|
2462
2558
|
}
|
|
@@ -2467,11 +2563,11 @@ export default defineExtension({
|
|
|
2467
2563
|
console.error(`csv-import: done — imported ${res.imported}, updated ${res.updated}, skipped ${res.skipped}${filterNote}.`);
|
|
2468
2564
|
});
|
|
2469
2565
|
},
|
|
2470
|
-
}
|
|
2566
|
+
};
|
|
2471
2567
|
// ---------------------------------------------------------------------------
|
|
2472
2568
|
// Named exports — pure helpers exposed for unit testing (no side effects).
|
|
2473
2569
|
// ---------------------------------------------------------------------------
|
|
2474
|
-
export { parseCSV, serializeCSV, serializeField, stripBOM, resolveDelimiter, parseFieldMap, resolveImportFieldMap, applyFieldMap, normalizeStatus, parseTags, stringifyTags, encodeKeyTagValue, decodeKeyTagValue, normalizeKeyValue, selectExportColumns, resolveEncoding, validateParsedCSV, strictValidationIssues, parseImportFilter, rowMatchesFilter, discoverCustomFields, EXPORT_COLUMNS, IMPORT_COLUMNS, StreamingCSVParser, streamCSVFile, levenshtein, suggestClosest, validateFieldMapTargets, checkMapSourcesPresent, };
|
|
2570
|
+
export { parseCSV, serializeCSV, serializeField, stripBOM, resolveDelimiter, parseFieldMap, resolveImportFieldMap, applyFieldMap, normalizeStatus, parseTags, stringifyTags, encodeKeyTagValue, decodeKeyTagValue, errorMessage, normalizeKeyValue, selectExportColumns, resolveEncoding, validateParsedCSV, validateCSV, strictValidationIssues, parseImportFilter, rowMatchesFilter, discoverCustomFields, EXPORT_COLUMNS, IMPORT_COLUMNS, StreamingCSVParser, streamCSVFile, levenshtein, suggestClosest, validateFieldMapTargets, checkMapSourcesPresent, };
|
|
2475
2571
|
// ---------------------------------------------------------------------------
|
|
2476
2572
|
// Internal store helpers exposed for regression testing against oversized pm
|
|
2477
2573
|
// roots. Unlike the pure helpers above, these READ or WRITE the given pm root
|
|
@@ -2483,4 +2579,9 @@ export { parseCSV, serializeCSV, serializeField, stripBOM, resolveDelimiter, par
|
|
|
2483
2579
|
// with synthetic results.
|
|
2484
2580
|
// ---------------------------------------------------------------------------
|
|
2485
2581
|
export { loadAppliedByTransaction, itemStatus, compensateCreate };
|
|
2582
|
+
// Complete-list surface: the shared refusal gate plus the two internal
|
|
2583
|
+
// readers whose silently-partial behavior the regression tests pin. Exported
|
|
2584
|
+
// ONLY for those tests; production callers keep using the command paths.
|
|
2585
|
+
export { loadKeyIndex, buildCsvExport };
|
|
2586
|
+
export { importCSV };
|
|
2486
2587
|
//# sourceMappingURL=index.js.map
|