preflight-mcp 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -7
- package/dist/bundle/github.js +34 -3
- package/dist/bundle/githubArchive.js +59 -7
- package/dist/bundle/service.js +283 -19
- package/dist/config.js +1 -0
- package/dist/context7/client.js +1 -1
- package/dist/evidence/dependencyGraph.js +312 -2
- package/dist/jobs/progressTracker.js +191 -0
- package/dist/server.js +310 -47
- package/package.json +1 -1
package/dist/bundle/service.js
CHANGED
|
@@ -14,6 +14,7 @@ import { ingestContext7Libraries } from './context7.js';
|
|
|
14
14
|
import { analyzeBundleStatic } from './analysis.js';
|
|
15
15
|
import { autoDetectTags, generateDisplayName, generateDescription } from './tagging.js';
|
|
16
16
|
import { bundleCreationLimiter } from '../core/concurrency-limiter.js';
|
|
17
|
+
import { getProgressTracker, calcPercent } from '../jobs/progressTracker.js';
|
|
17
18
|
const DEDUP_INDEX_FILE = '.preflight-dedup-index.json';
|
|
18
19
|
function sha256Hex(text) {
|
|
19
20
|
return crypto.createHash('sha256').update(text, 'utf8').digest('hex');
|
|
@@ -90,7 +91,7 @@ async function writeDedupIndex(storageDir, idx) {
|
|
|
90
91
|
throw err;
|
|
91
92
|
}
|
|
92
93
|
}
|
|
93
|
-
async function updateDedupIndexBestEffort(cfg, fingerprint, bundleId, bundleUpdatedAt) {
|
|
94
|
+
async function updateDedupIndexBestEffort(cfg, fingerprint, bundleId, bundleUpdatedAt, status = 'complete') {
|
|
94
95
|
for (const storageDir of cfg.storageDirs) {
|
|
95
96
|
try {
|
|
96
97
|
const parentAvailable = await isParentAvailable(storageDir);
|
|
@@ -98,7 +99,7 @@ async function updateDedupIndexBestEffort(cfg, fingerprint, bundleId, bundleUpda
|
|
|
98
99
|
continue;
|
|
99
100
|
await ensureDir(storageDir);
|
|
100
101
|
const idx = await readDedupIndex(storageDir);
|
|
101
|
-
idx.byFingerprint[fingerprint] = { bundleId, bundleUpdatedAt };
|
|
102
|
+
idx.byFingerprint[fingerprint] = { bundleId, bundleUpdatedAt, status };
|
|
102
103
|
idx.updatedAt = nowIso();
|
|
103
104
|
await writeDedupIndex(storageDir, idx);
|
|
104
105
|
}
|
|
@@ -107,6 +108,97 @@ async function updateDedupIndexBestEffort(cfg, fingerprint, bundleId, bundleUpda
|
|
|
107
108
|
}
|
|
108
109
|
}
|
|
109
110
|
}
|
|
111
|
+
/**
|
|
112
|
+
* Set in-progress lock for a fingerprint. Returns false if already locked (not timed out).
|
|
113
|
+
*/
|
|
114
|
+
async function setInProgressLock(cfg, fingerprint, taskId, repos) {
|
|
115
|
+
const now = nowIso();
|
|
116
|
+
const nowMs = Date.now();
|
|
117
|
+
for (const storageDir of cfg.storageDirs) {
|
|
118
|
+
try {
|
|
119
|
+
if (!(await isPathAvailable(storageDir)))
|
|
120
|
+
continue;
|
|
121
|
+
await ensureDir(storageDir);
|
|
122
|
+
const idx = await readDedupIndex(storageDir);
|
|
123
|
+
const existing = idx.byFingerprint[fingerprint];
|
|
124
|
+
// Check if there's an existing in-progress lock
|
|
125
|
+
if (existing?.status === 'in-progress' && existing.startedAt) {
|
|
126
|
+
const startedMs = new Date(existing.startedAt).getTime();
|
|
127
|
+
const elapsed = nowMs - startedMs;
|
|
128
|
+
// If lock hasn't timed out, return the existing entry
|
|
129
|
+
if (elapsed < cfg.inProgressLockTimeoutMs) {
|
|
130
|
+
return { locked: false, existingEntry: existing };
|
|
131
|
+
}
|
|
132
|
+
// Lock timed out - will be overwritten
|
|
133
|
+
logger.warn(`In-progress lock timed out for fingerprint ${fingerprint.slice(0, 8)}...`);
|
|
134
|
+
}
|
|
135
|
+
// Set new in-progress lock
|
|
136
|
+
idx.byFingerprint[fingerprint] = {
|
|
137
|
+
bundleId: '', // Will be set on completion
|
|
138
|
+
bundleUpdatedAt: now,
|
|
139
|
+
status: 'in-progress',
|
|
140
|
+
startedAt: now,
|
|
141
|
+
taskId,
|
|
142
|
+
repos,
|
|
143
|
+
};
|
|
144
|
+
idx.updatedAt = now;
|
|
145
|
+
await writeDedupIndex(storageDir, idx);
|
|
146
|
+
return { locked: true };
|
|
147
|
+
}
|
|
148
|
+
catch (err) {
|
|
149
|
+
logger.debug(`Failed to set in-progress lock in ${storageDir}`, err instanceof Error ? err : undefined);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
// If we couldn't write to any storage, assume we can proceed (best-effort)
|
|
153
|
+
return { locked: true };
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Clear in-progress lock (on failure or completion with status='complete').
|
|
157
|
+
*/
|
|
158
|
+
async function clearInProgressLock(cfg, fingerprint) {
|
|
159
|
+
for (const storageDir of cfg.storageDirs) {
|
|
160
|
+
try {
|
|
161
|
+
if (!(await isPathAvailable(storageDir)))
|
|
162
|
+
continue;
|
|
163
|
+
const idx = await readDedupIndex(storageDir);
|
|
164
|
+
const existing = idx.byFingerprint[fingerprint];
|
|
165
|
+
// Only clear if it's in-progress
|
|
166
|
+
if (existing?.status === 'in-progress') {
|
|
167
|
+
delete idx.byFingerprint[fingerprint];
|
|
168
|
+
idx.updatedAt = nowIso();
|
|
169
|
+
await writeDedupIndex(storageDir, idx);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
catch (err) {
|
|
173
|
+
logger.debug(`Failed to clear in-progress lock in ${storageDir}`, err instanceof Error ? err : undefined);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Check if a fingerprint has an in-progress lock (not timed out).
|
|
179
|
+
*/
|
|
180
|
+
export async function checkInProgressLock(cfg, fingerprint) {
|
|
181
|
+
const nowMs = Date.now();
|
|
182
|
+
for (const storageDir of cfg.storageDirs) {
|
|
183
|
+
try {
|
|
184
|
+
if (!(await isPathAvailable(storageDir)))
|
|
185
|
+
continue;
|
|
186
|
+
const idx = await readDedupIndex(storageDir);
|
|
187
|
+
const existing = idx.byFingerprint[fingerprint];
|
|
188
|
+
if (existing?.status === 'in-progress' && existing.startedAt) {
|
|
189
|
+
const startedMs = new Date(existing.startedAt).getTime();
|
|
190
|
+
const elapsed = nowMs - startedMs;
|
|
191
|
+
if (elapsed < cfg.inProgressLockTimeoutMs) {
|
|
192
|
+
return existing;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
// ignore
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
return null;
|
|
201
|
+
}
|
|
110
202
|
async function readBundleSummary(cfg, bundleId) {
|
|
111
203
|
const storageDir = (await findBundleStorageDir(cfg.storageDirs, bundleId)) ?? (await getEffectiveStorageDir(cfg));
|
|
112
204
|
const paths = getBundlePaths(storageDir, bundleId);
|
|
@@ -137,6 +229,9 @@ async function findExistingBundleByFingerprint(cfg, fingerprint) {
|
|
|
137
229
|
continue;
|
|
138
230
|
const idx = await readDedupIndex(storageDir);
|
|
139
231
|
const hit = idx.byFingerprint[fingerprint];
|
|
232
|
+
// Skip in-progress entries - they don't have a completed bundle yet
|
|
233
|
+
if (hit?.status === 'in-progress')
|
|
234
|
+
continue;
|
|
140
235
|
if (hit?.bundleId && (await bundleExistsMulti(cfg.storageDirs, hit.bundleId))) {
|
|
141
236
|
return hit.bundleId;
|
|
142
237
|
}
|
|
@@ -321,6 +416,33 @@ async function validateBundleCompleteness(bundleRoot) {
|
|
|
321
416
|
missingComponents,
|
|
322
417
|
};
|
|
323
418
|
}
|
|
419
|
+
/**
|
|
420
|
+
* Assert that a bundle is complete and ready for operations.
|
|
421
|
+
* Throws an error with helpful guidance if the bundle is incomplete.
|
|
422
|
+
* Should be called at the entry point of tools that require a complete bundle
|
|
423
|
+
* (e.g., dependency graph, trace links, search).
|
|
424
|
+
*/
|
|
425
|
+
export async function assertBundleComplete(cfg, bundleId) {
|
|
426
|
+
const storageDir = await findBundleStorageDir(cfg.storageDirs, bundleId);
|
|
427
|
+
if (!storageDir) {
|
|
428
|
+
throw new Error(`Bundle not found: ${bundleId}`);
|
|
429
|
+
}
|
|
430
|
+
const bundleRoot = getBundlePaths(storageDir, bundleId).rootDir;
|
|
431
|
+
const { isValid, missingComponents } = await validateBundleCompleteness(bundleRoot);
|
|
432
|
+
if (!isValid) {
|
|
433
|
+
const issues = missingComponents.join('\n - ');
|
|
434
|
+
throw new Error(`Bundle is incomplete and cannot be used for this operation.\n\n` +
|
|
435
|
+
`Bundle ID: ${bundleId}\n` +
|
|
436
|
+
`Missing components:\n - ${issues}\n\n` +
|
|
437
|
+
`This usually happens when:\n` +
|
|
438
|
+
`1. Bundle creation was interrupted (timeout, network error, etc.)\n` +
|
|
439
|
+
`2. Bundle download is still in progress\n\n` +
|
|
440
|
+
`Suggested actions:\n` +
|
|
441
|
+
`- Use preflight_update_bundle with force:true to re-download the repository\n` +
|
|
442
|
+
`- Or use preflight_delete_bundle and preflight_create_bundle to start fresh\n` +
|
|
443
|
+
`- Check preflight_get_task_status if creation might still be in progress`);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
324
446
|
/**
|
|
325
447
|
* Detect primary language from ingested files
|
|
326
448
|
*/
|
|
@@ -624,28 +746,67 @@ async function cloneAndIngestGitHubRepo(params) {
|
|
|
624
746
|
let repoRootForIngest = tmpCheckoutGit;
|
|
625
747
|
let headSha;
|
|
626
748
|
const notes = [];
|
|
749
|
+
const warnings = [];
|
|
627
750
|
let source = 'git';
|
|
628
751
|
let fetchedAt = nowIso();
|
|
629
752
|
let refUsed = params.ref;
|
|
630
753
|
try {
|
|
631
|
-
|
|
754
|
+
params.onProgress?.('cloning', 0, `Cloning ${repoId}...`);
|
|
755
|
+
await shallowClone(cloneUrl, tmpCheckoutGit, {
|
|
756
|
+
ref: params.ref,
|
|
757
|
+
timeoutMs: params.cfg.gitCloneTimeoutMs,
|
|
758
|
+
onProgress: (phase, percent, msg) => {
|
|
759
|
+
params.onProgress?.('cloning', percent, `${repoId}: ${msg}`);
|
|
760
|
+
},
|
|
761
|
+
});
|
|
632
762
|
headSha = await getLocalHeadSha(tmpCheckoutGit);
|
|
633
763
|
}
|
|
634
764
|
catch (err) {
|
|
635
765
|
// Fallback: GitHub archive download (zipball) + extract.
|
|
636
766
|
source = 'archive';
|
|
637
|
-
const
|
|
638
|
-
notes.push(`git clone failed; used GitHub archive fallback: ${
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
767
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
768
|
+
notes.push(`git clone failed; used GitHub archive fallback: ${errMsg}`);
|
|
769
|
+
// User-facing warning: communicate the network issue clearly
|
|
770
|
+
warnings.push(`⚠️ [${repoId}] Git clone failed (network issue), switched to ZIP download.\n` +
|
|
771
|
+
` Reason: ${errMsg.slice(0, 200)}${errMsg.length > 200 ? '...' : ''}`);
|
|
772
|
+
params.onProgress?.('downloading', 0, `Downloading ${repoId} archive...`);
|
|
773
|
+
// Track zip path for error message
|
|
774
|
+
const zipPath = path.join(tmpArchiveDir, `github-zipball-${params.owner}-${params.repo}-partial.zip`);
|
|
775
|
+
try {
|
|
776
|
+
const archive = await downloadAndExtractGitHubArchive({
|
|
777
|
+
cfg: params.cfg,
|
|
778
|
+
owner: params.owner,
|
|
779
|
+
repo: params.repo,
|
|
780
|
+
ref: params.ref,
|
|
781
|
+
destDir: tmpArchiveDir,
|
|
782
|
+
onProgress: (downloaded, total, msg) => {
|
|
783
|
+
const percent = total ? Math.round((downloaded / total) * 100) : 0;
|
|
784
|
+
params.onProgress?.('downloading', percent, `${repoId}: ${msg}`);
|
|
785
|
+
},
|
|
786
|
+
});
|
|
787
|
+
repoRootForIngest = archive.repoRoot;
|
|
788
|
+
fetchedAt = archive.fetchedAt;
|
|
789
|
+
refUsed = archive.refUsed;
|
|
790
|
+
// Success: ZIP download completed
|
|
791
|
+
warnings.push(`✅ [${repoId}] ZIP download completed successfully as fallback.`);
|
|
792
|
+
}
|
|
793
|
+
catch (zipErr) {
|
|
794
|
+
// ZIP download also failed - provide helpful error with temp path
|
|
795
|
+
const zipErrMsg = zipErr instanceof Error ? zipErr.message : String(zipErr);
|
|
796
|
+
// Check if partial file exists
|
|
797
|
+
const partialExists = await statOrNull(tmpArchiveDir);
|
|
798
|
+
const tempPathMsg = partialExists
|
|
799
|
+
? `\n Partial files may exist in: ${tmpArchiveDir}`
|
|
800
|
+
: '';
|
|
801
|
+
throw new Error(`Both git clone and ZIP download failed for ${repoId}.\n\n` +
|
|
802
|
+
`Git error: ${errMsg.slice(0, 150)}\n` +
|
|
803
|
+
`ZIP error: ${zipErrMsg.slice(0, 150)}${tempPathMsg}\n\n` +
|
|
804
|
+
`Suggestions:\n` +
|
|
805
|
+
`1. Check your network connection\n` +
|
|
806
|
+
`2. Verify the repository exists: https://github.com/${repoId}\n` +
|
|
807
|
+
`3. If you have the repo locally, use 'kind: local' with 'path: /your/local/path'\n` +
|
|
808
|
+
`4. If behind a proxy, configure GITHUB_TOKEN environment variable`);
|
|
809
|
+
}
|
|
649
810
|
}
|
|
650
811
|
const bundlePaths = getBundlePaths(params.storageDir, params.bundleId);
|
|
651
812
|
const rawDest = repoRawDir(bundlePaths, params.owner, params.repo);
|
|
@@ -679,7 +840,7 @@ async function cloneAndIngestGitHubRepo(params) {
|
|
|
679
840
|
});
|
|
680
841
|
await rmIfExists(tmpCheckoutGit);
|
|
681
842
|
await rmIfExists(tmpArchiveDir);
|
|
682
|
-
return { headSha, files: ingested.files, skipped: ingested.skipped, notes, source };
|
|
843
|
+
return { headSha, files: ingested.files, skipped: ingested.skipped, notes, warnings, source };
|
|
683
844
|
}
|
|
684
845
|
function groupFilesByRepoId(files) {
|
|
685
846
|
const byRepo = new Map();
|
|
@@ -721,6 +882,15 @@ export async function createBundle(cfg, input, options) {
|
|
|
721
882
|
}
|
|
722
883
|
async function createBundleInternal(cfg, input, options) {
|
|
723
884
|
const fingerprint = computeCreateInputFingerprint(input);
|
|
885
|
+
const repoIds = input.repos.map((r) => r.repo);
|
|
886
|
+
const onProgress = options?.onProgress;
|
|
887
|
+
const tracker = getProgressTracker();
|
|
888
|
+
// Helper to report progress
|
|
889
|
+
const reportProgress = (phase, progress, message, total) => {
|
|
890
|
+
if (onProgress) {
|
|
891
|
+
onProgress(phase, progress, message, total);
|
|
892
|
+
}
|
|
893
|
+
};
|
|
724
894
|
const ifExists = options?.ifExists ?? 'error';
|
|
725
895
|
if (ifExists !== 'createNew') {
|
|
726
896
|
const existing = await findExistingBundleByFingerprint(cfg, fingerprint);
|
|
@@ -735,6 +905,28 @@ async function createBundleInternal(cfg, input, options) {
|
|
|
735
905
|
throw new Error(`Bundle already exists for these inputs: ${existing}`);
|
|
736
906
|
}
|
|
737
907
|
}
|
|
908
|
+
// Start tracking this task
|
|
909
|
+
const taskId = tracker.startTask(fingerprint, repoIds);
|
|
910
|
+
reportProgress('starting', 0, `Starting bundle creation for ${repoIds.join(', ')}`);
|
|
911
|
+
// Try to acquire in-progress lock
|
|
912
|
+
const lockResult = await setInProgressLock(cfg, fingerprint, taskId, repoIds);
|
|
913
|
+
if (!lockResult.locked) {
|
|
914
|
+
// Another task is already creating this bundle
|
|
915
|
+
const entry = lockResult.existingEntry;
|
|
916
|
+
const elapsedSec = entry.startedAt
|
|
917
|
+
? Math.round((Date.now() - new Date(entry.startedAt).getTime()) / 1000)
|
|
918
|
+
: 0;
|
|
919
|
+
const msg = `Bundle creation already in progress (taskId: ${entry.taskId}, started ${elapsedSec}s ago). ` +
|
|
920
|
+
`Use preflight_get_task_status to check progress.`;
|
|
921
|
+
// Throw a special error that can be caught and handled
|
|
922
|
+
const err = new Error(msg);
|
|
923
|
+
err.code = 'BUNDLE_IN_PROGRESS';
|
|
924
|
+
err.taskId = entry.taskId;
|
|
925
|
+
err.fingerprint = fingerprint;
|
|
926
|
+
err.repos = entry.repos;
|
|
927
|
+
err.startedAt = entry.startedAt;
|
|
928
|
+
throw err;
|
|
929
|
+
}
|
|
738
930
|
const bundleId = crypto.randomUUID();
|
|
739
931
|
const createdAt = nowIso();
|
|
740
932
|
// Use effective storage dir (falls back if primary unavailable)
|
|
@@ -747,20 +939,36 @@ async function createBundleInternal(cfg, input, options) {
|
|
|
747
939
|
const finalPaths = getBundlePaths(effectiveStorageDir, bundleId);
|
|
748
940
|
const allIngestedFiles = [];
|
|
749
941
|
const reposSummary = [];
|
|
942
|
+
const allWarnings = [];
|
|
943
|
+
// Track temp checkout directory for cleanup
|
|
944
|
+
const tmpCheckoutsDir = path.join(cfg.tmpDir, 'checkouts', bundleId);
|
|
750
945
|
try {
|
|
751
946
|
// All operations happen in tmpPaths (temporary directory)
|
|
947
|
+
const totalRepos = input.repos.length;
|
|
948
|
+
let repoIndex = 0;
|
|
752
949
|
for (const repoInput of input.repos) {
|
|
950
|
+
repoIndex++;
|
|
951
|
+
const repoProgress = Math.round((repoIndex - 1) / totalRepos * 40); // 0-40% for repo fetching
|
|
753
952
|
if (repoInput.kind === 'github') {
|
|
754
953
|
const { owner, repo } = parseOwnerRepo(repoInput.repo);
|
|
755
|
-
|
|
954
|
+
reportProgress('cloning', repoProgress, `[${repoIndex}/${totalRepos}] Fetching ${owner}/${repo}...`);
|
|
955
|
+
tracker.updateProgress(taskId, 'cloning', repoProgress, `Fetching ${owner}/${repo}...`);
|
|
956
|
+
const { headSha, files, skipped, notes, warnings, source } = await cloneAndIngestGitHubRepo({
|
|
756
957
|
cfg,
|
|
757
958
|
bundleId,
|
|
758
959
|
storageDir: tmpBundlesDir,
|
|
759
960
|
owner,
|
|
760
961
|
repo,
|
|
761
962
|
ref: repoInput.ref,
|
|
963
|
+
onProgress: (phase, percent, msg) => {
|
|
964
|
+
// Map clone/download progress to overall progress (0-40% range per repo)
|
|
965
|
+
const overallProgress = repoProgress + Math.round(percent * 0.4 / totalRepos);
|
|
966
|
+
reportProgress(phase, overallProgress, `[${repoIndex}/${totalRepos}] ${msg}`);
|
|
967
|
+
tracker.updateProgress(taskId, phase, overallProgress, msg);
|
|
968
|
+
},
|
|
762
969
|
});
|
|
763
970
|
allIngestedFiles.push(...files);
|
|
971
|
+
allWarnings.push(...warnings);
|
|
764
972
|
reposSummary.push({
|
|
765
973
|
kind: 'github',
|
|
766
974
|
id: `${owner}/${repo}`,
|
|
@@ -772,6 +980,8 @@ async function createBundleInternal(cfg, input, options) {
|
|
|
772
980
|
else {
|
|
773
981
|
// Local repository
|
|
774
982
|
const { owner, repo } = parseOwnerRepo(repoInput.repo);
|
|
983
|
+
reportProgress('ingesting', repoProgress, `[${repoIndex}/${totalRepos}] Ingesting local ${owner}/${repo}...`);
|
|
984
|
+
tracker.updateProgress(taskId, 'ingesting', repoProgress, `Ingesting local ${owner}/${repo}...`);
|
|
775
985
|
const { files, skipped } = await ingestLocalRepo({
|
|
776
986
|
cfg,
|
|
777
987
|
bundleId,
|
|
@@ -801,6 +1011,8 @@ async function createBundleInternal(cfg, input, options) {
|
|
|
801
1011
|
librariesSummary = libIngest.libraries;
|
|
802
1012
|
}
|
|
803
1013
|
// Build index.
|
|
1014
|
+
reportProgress('indexing', 50, `Building search index (${allIngestedFiles.length} files)...`);
|
|
1015
|
+
tracker.updateProgress(taskId, 'indexing', 50, `Building search index (${allIngestedFiles.length} files)...`);
|
|
804
1016
|
await rebuildIndex(tmpPaths.searchDbPath, allIngestedFiles, {
|
|
805
1017
|
includeDocs: true,
|
|
806
1018
|
includeCode: true,
|
|
@@ -859,6 +1071,8 @@ async function createBundleInternal(cfg, input, options) {
|
|
|
859
1071
|
libraries: librariesSummary,
|
|
860
1072
|
});
|
|
861
1073
|
// Generate static facts (FACTS.json) FIRST. This is intentionally non-LLM and safe to keep inside bundles.
|
|
1074
|
+
reportProgress('analyzing', 70, 'Analyzing code structure...');
|
|
1075
|
+
tracker.updateProgress(taskId, 'analyzing', 70, 'Analyzing code structure...');
|
|
862
1076
|
await generateFactsBestEffort({
|
|
863
1077
|
bundleId,
|
|
864
1078
|
bundleRoot: tmpPaths.rootDir,
|
|
@@ -866,6 +1080,8 @@ async function createBundleInternal(cfg, input, options) {
|
|
|
866
1080
|
mode: cfg.analysisMode,
|
|
867
1081
|
});
|
|
868
1082
|
// Overview (S2: factual-only with evidence pointers) - generated AFTER FACTS.json
|
|
1083
|
+
reportProgress('generating', 80, 'Generating overview...');
|
|
1084
|
+
tracker.updateProgress(taskId, 'generating', 80, 'Generating overview...');
|
|
869
1085
|
const perRepoOverviews = reposSummary
|
|
870
1086
|
.filter((r) => r.kind === 'github' || r.kind === 'local')
|
|
871
1087
|
.map((r) => {
|
|
@@ -889,6 +1105,8 @@ async function createBundleInternal(cfg, input, options) {
|
|
|
889
1105
|
}
|
|
890
1106
|
// ATOMIC OPERATION: Move from temp to final location
|
|
891
1107
|
// This is atomic on most filesystems - bundle becomes visible only when complete
|
|
1108
|
+
reportProgress('finalizing', 90, 'Finalizing bundle...');
|
|
1109
|
+
tracker.updateProgress(taskId, 'finalizing', 90, 'Finalizing bundle...');
|
|
892
1110
|
logger.info(`Moving bundle ${bundleId} from temp to final location (atomic)`);
|
|
893
1111
|
await ensureDir(effectiveStorageDir);
|
|
894
1112
|
try {
|
|
@@ -915,13 +1133,17 @@ async function createBundleInternal(cfg, input, options) {
|
|
|
915
1133
|
await mirrorBundleToBackups(effectiveStorageDir, cfg.storageDirs, bundleId);
|
|
916
1134
|
}
|
|
917
1135
|
// Update de-duplication index (best-effort). This is intentionally after atomic move.
|
|
918
|
-
await updateDedupIndexBestEffort(cfg, fingerprint, bundleId, createdAt);
|
|
1136
|
+
await updateDedupIndexBestEffort(cfg, fingerprint, bundleId, createdAt, 'complete');
|
|
1137
|
+
// Mark task complete
|
|
1138
|
+
reportProgress('complete', 100, `Bundle created: ${bundleId}`);
|
|
1139
|
+
tracker.completeTask(taskId, bundleId);
|
|
919
1140
|
const summary = {
|
|
920
1141
|
bundleId,
|
|
921
1142
|
createdAt,
|
|
922
1143
|
updatedAt: createdAt,
|
|
923
1144
|
repos: reposSummary,
|
|
924
1145
|
libraries: librariesSummary,
|
|
1146
|
+
warnings: allWarnings.length > 0 ? allWarnings : undefined,
|
|
925
1147
|
};
|
|
926
1148
|
return summary;
|
|
927
1149
|
}
|
|
@@ -929,8 +1151,15 @@ async function createBundleInternal(cfg, input, options) {
|
|
|
929
1151
|
// Clean up temp directory on failure
|
|
930
1152
|
logger.error(`Bundle creation failed, cleaning up temp: ${bundleId}`, err instanceof Error ? err : undefined);
|
|
931
1153
|
await rmIfExists(tmpPaths.rootDir);
|
|
932
|
-
//
|
|
1154
|
+
// Clear in-progress lock on failure
|
|
1155
|
+
await clearInProgressLock(cfg, fingerprint);
|
|
1156
|
+
// Mark task failed
|
|
933
1157
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
1158
|
+
tracker.failTask(taskId, errorMsg);
|
|
1159
|
+
// Re-throw with enhanced message (unless it's already our BUNDLE_IN_PROGRESS error)
|
|
1160
|
+
if (err?.code === 'BUNDLE_IN_PROGRESS') {
|
|
1161
|
+
throw err;
|
|
1162
|
+
}
|
|
934
1163
|
throw new Error(`Failed to create bundle: ${errorMsg}`);
|
|
935
1164
|
}
|
|
936
1165
|
finally {
|
|
@@ -938,6 +1167,10 @@ async function createBundleInternal(cfg, input, options) {
|
|
|
938
1167
|
await rmIfExists(tmpPaths.rootDir).catch((err) => {
|
|
939
1168
|
logger.debug('Failed to cleanup temp bundle directory in finally block (non-critical)', err instanceof Error ? err : undefined);
|
|
940
1169
|
});
|
|
1170
|
+
// Clean up temp checkouts directory (git clones, zip extracts)
|
|
1171
|
+
await rmIfExists(tmpCheckoutsDir).catch((err) => {
|
|
1172
|
+
logger.debug('Failed to cleanup temp checkouts directory in finally block (non-critical)', err instanceof Error ? err : undefined);
|
|
1173
|
+
});
|
|
941
1174
|
}
|
|
942
1175
|
}
|
|
943
1176
|
/** Check if a bundle has upstream changes without applying updates. */
|
|
@@ -1100,6 +1333,14 @@ export async function repairBundle(cfg, bundleId, options) {
|
|
|
1100
1333
|
// Manifest is required for safe repairs (no fetching/re-ingest).
|
|
1101
1334
|
const manifest = await readManifest(paths.manifestPath);
|
|
1102
1335
|
const actionsTaken = [];
|
|
1336
|
+
const unfixableIssues = [];
|
|
1337
|
+
// Check for unfixable issues (require re-download, can't be repaired offline)
|
|
1338
|
+
const reposHasContent = before.missingComponents.every(c => !c.includes('repos/'));
|
|
1339
|
+
if (!reposHasContent) {
|
|
1340
|
+
unfixableIssues.push('repos/ directory is empty or missing - this requires re-downloading the repository. ' +
|
|
1341
|
+
'Use preflight_delete_bundle and preflight_create_bundle to start fresh, ' +
|
|
1342
|
+
'or use preflight_update_bundle with force:true to re-fetch.');
|
|
1343
|
+
}
|
|
1103
1344
|
// Determine what needs repair.
|
|
1104
1345
|
const stAgents = await statOrNull(paths.agentsPath);
|
|
1105
1346
|
const stStartHere = await statOrNull(paths.startHerePath);
|
|
@@ -1187,6 +1428,7 @@ export async function repairBundle(cfg, bundleId, options) {
|
|
|
1187
1428
|
mode,
|
|
1188
1429
|
repaired: actionsTaken.length > 0,
|
|
1189
1430
|
actionsTaken,
|
|
1431
|
+
unfixableIssues: unfixableIssues.length > 0 ? unfixableIssues : undefined,
|
|
1190
1432
|
before,
|
|
1191
1433
|
after,
|
|
1192
1434
|
updatedAt,
|
|
@@ -1198,15 +1440,27 @@ export async function updateBundle(cfg, bundleId, options) {
|
|
|
1198
1440
|
const paths = getBundlePaths(effectiveStorageDir, bundleId);
|
|
1199
1441
|
const manifest = await readManifest(paths.manifestPath);
|
|
1200
1442
|
const updatedAt = nowIso();
|
|
1443
|
+
const onProgress = options?.onProgress;
|
|
1444
|
+
// Report progress helper
|
|
1445
|
+
const reportProgress = (phase, progress, message, total) => {
|
|
1446
|
+
if (onProgress) {
|
|
1447
|
+
onProgress(phase, progress, message, total);
|
|
1448
|
+
}
|
|
1449
|
+
};
|
|
1450
|
+
reportProgress('starting', 0, `Updating bundle ${bundleId}...`);
|
|
1201
1451
|
let changed = false;
|
|
1202
1452
|
const allIngestedFiles = [];
|
|
1203
1453
|
const reposSummary = [];
|
|
1454
|
+
const totalRepos = manifest.inputs.repos.length;
|
|
1455
|
+
let repoIndex = 0;
|
|
1204
1456
|
// Rebuild everything obvious for now (simple + deterministic).
|
|
1205
1457
|
for (const repoInput of manifest.inputs.repos) {
|
|
1458
|
+
repoIndex++;
|
|
1206
1459
|
if (repoInput.kind === 'github') {
|
|
1207
1460
|
const { owner, repo } = parseOwnerRepo(repoInput.repo);
|
|
1208
1461
|
const repoId = `${owner}/${repo}`;
|
|
1209
1462
|
const cloneUrl = toCloneUrl({ owner, repo });
|
|
1463
|
+
reportProgress('cloning', calcPercent(repoIndex - 1, totalRepos), `Checking ${repoId}...`, totalRepos);
|
|
1210
1464
|
let remoteSha;
|
|
1211
1465
|
try {
|
|
1212
1466
|
remoteSha = await getRemoteHeadSha(cloneUrl);
|
|
@@ -1218,6 +1472,7 @@ export async function updateBundle(cfg, bundleId, options) {
|
|
|
1218
1472
|
if (remoteSha && prev?.headSha && remoteSha !== prev.headSha) {
|
|
1219
1473
|
changed = true;
|
|
1220
1474
|
}
|
|
1475
|
+
reportProgress('downloading', calcPercent(repoIndex - 1, totalRepos), `Fetching ${repoId}...`, totalRepos);
|
|
1221
1476
|
const { headSha, files, skipped, notes, source } = await cloneAndIngestGitHubRepo({
|
|
1222
1477
|
cfg,
|
|
1223
1478
|
bundleId,
|
|
@@ -1225,6 +1480,9 @@ export async function updateBundle(cfg, bundleId, options) {
|
|
|
1225
1480
|
owner,
|
|
1226
1481
|
repo,
|
|
1227
1482
|
ref: repoInput.ref,
|
|
1483
|
+
onProgress: (phase, progress, message) => {
|
|
1484
|
+
reportProgress(phase, progress, message);
|
|
1485
|
+
},
|
|
1228
1486
|
});
|
|
1229
1487
|
if (prev?.headSha && headSha && headSha !== prev.headSha) {
|
|
1230
1488
|
changed = true;
|
|
@@ -1257,6 +1515,7 @@ export async function updateBundle(cfg, bundleId, options) {
|
|
|
1257
1515
|
// Context7 libraries (best-effort).
|
|
1258
1516
|
let librariesSummary;
|
|
1259
1517
|
if (manifest.inputs.libraries?.length) {
|
|
1518
|
+
reportProgress('downloading', 80, 'Fetching Context7 libraries...');
|
|
1260
1519
|
await rmIfExists(paths.librariesDir);
|
|
1261
1520
|
await ensureDir(paths.librariesDir);
|
|
1262
1521
|
const libIngest = await ingestContext7Libraries({
|
|
@@ -1269,6 +1528,7 @@ export async function updateBundle(cfg, bundleId, options) {
|
|
|
1269
1528
|
librariesSummary = libIngest.libraries;
|
|
1270
1529
|
}
|
|
1271
1530
|
// Rebuild index.
|
|
1531
|
+
reportProgress('indexing', 85, `Rebuilding search index (${allIngestedFiles.length} files)...`);
|
|
1272
1532
|
await rebuildIndex(paths.searchDbPath, allIngestedFiles, {
|
|
1273
1533
|
includeDocs: manifest.index.includeDocs,
|
|
1274
1534
|
includeCode: manifest.index.includeCode,
|
|
@@ -1294,6 +1554,7 @@ export async function updateBundle(cfg, bundleId, options) {
|
|
|
1294
1554
|
};
|
|
1295
1555
|
await writeManifest(paths.manifestPath, newManifest);
|
|
1296
1556
|
// Regenerate guides + overview.
|
|
1557
|
+
reportProgress('generating', 90, 'Regenerating guides and overview...');
|
|
1297
1558
|
await writeAgentsMd(paths.agentsPath);
|
|
1298
1559
|
await writeStartHereMd({
|
|
1299
1560
|
targetPath: paths.startHerePath,
|
|
@@ -1316,6 +1577,7 @@ export async function updateBundle(cfg, bundleId, options) {
|
|
|
1316
1577
|
});
|
|
1317
1578
|
await writeOverviewFile(paths.overviewPath, overviewMd);
|
|
1318
1579
|
// Refresh static facts (FACTS.json) after update.
|
|
1580
|
+
reportProgress('analyzing', 95, 'Analyzing bundle...');
|
|
1319
1581
|
await generateFactsBestEffort({
|
|
1320
1582
|
bundleId,
|
|
1321
1583
|
bundleRoot: paths.rootDir,
|
|
@@ -1323,11 +1585,13 @@ export async function updateBundle(cfg, bundleId, options) {
|
|
|
1323
1585
|
mode: cfg.analysisMode,
|
|
1324
1586
|
});
|
|
1325
1587
|
// Mirror to backup storage directories (non-blocking on failures)
|
|
1588
|
+
reportProgress('finalizing', 98, 'Finalizing update...');
|
|
1326
1589
|
if (cfg.storageDirs.length > 1) {
|
|
1327
1590
|
await mirrorBundleToBackups(effectiveStorageDir, cfg.storageDirs, bundleId);
|
|
1328
1591
|
}
|
|
1329
1592
|
// Keep the de-duplication index fresh (best-effort).
|
|
1330
1593
|
await updateDedupIndexBestEffort(cfg, fingerprint, bundleId, updatedAt);
|
|
1594
|
+
reportProgress('complete', 100, `Bundle updated: ${bundleId}`);
|
|
1331
1595
|
const summary = {
|
|
1332
1596
|
bundleId,
|
|
1333
1597
|
createdAt: manifest.createdAt,
|
package/dist/config.js
CHANGED
|
@@ -88,5 +88,6 @@ export function getConfig() {
|
|
|
88
88
|
defaultMaxAgeHours: envNumber('PREFLIGHT_DEFAULT_MAX_AGE_HOURS', 24),
|
|
89
89
|
maxSearchLimit: envNumber('PREFLIGHT_MAX_SEARCH_LIMIT', 200),
|
|
90
90
|
defaultSearchLimit: envNumber('PREFLIGHT_DEFAULT_SEARCH_LIMIT', 30),
|
|
91
|
+
inProgressLockTimeoutMs: envNumber('PREFLIGHT_IN_PROGRESS_LOCK_TIMEOUT_MS', 30 * 60_000),
|
|
91
92
|
};
|
|
92
93
|
}
|
package/dist/context7/client.js
CHANGED
|
@@ -20,7 +20,7 @@ export async function connectContext7(cfg) {
|
|
|
20
20
|
maxRetries: 1,
|
|
21
21
|
},
|
|
22
22
|
});
|
|
23
|
-
const client = new Client({ name: 'preflight-context7', version: '0.1.
|
|
23
|
+
const client = new Client({ name: 'preflight-context7', version: '0.1.5' });
|
|
24
24
|
await client.connect(transport);
|
|
25
25
|
return {
|
|
26
26
|
client,
|