vesper-wizard 2.3.1 → 2.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -322
- package/package.json +34 -100
- package/vesper-mcp-config.json +6 -0
- package/{scripts/wizard.js → wizard.js} +1 -1
- package/LICENSE +0 -21
- package/build/cache/cdn.js +0 -34
- package/build/cache/service.js +0 -63
- package/build/cleaning/cleaner.js +0 -81
- package/build/cleaning/evaluator.js +0 -89
- package/build/cleaning/executor.js +0 -62
- package/build/cleaning/exporter.js +0 -87
- package/build/cleaning/planner.js +0 -127
- package/build/cleaning/rules.js +0 -57
- package/build/cleaning/types.js +0 -1
- package/build/cloud/adapters/local.js +0 -37
- package/build/cloud/adapters/s3.js +0 -24
- package/build/cloud/adapters/supabase.js +0 -49
- package/build/cloud/storage-manager.js +0 -26
- package/build/cloud/types.js +0 -1
- package/build/compliance/service.js +0 -73
- package/build/compliance/store.js +0 -80
- package/build/compliance/types.js +0 -1
- package/build/config/config-manager.js +0 -221
- package/build/config/secure-keys.js +0 -51
- package/build/config/user-config.js +0 -48
- package/build/data/processing-worker.js +0 -23
- package/build/data/streaming.js +0 -38
- package/build/data/worker-pool.js +0 -39
- package/build/export/exporter.js +0 -82
- package/build/export/packager.js +0 -100
- package/build/export/types.js +0 -1
- package/build/fusion/aligner.js +0 -56
- package/build/fusion/deduplicator.js +0 -69
- package/build/fusion/engine.js +0 -69
- package/build/fusion/harmonizer.js +0 -39
- package/build/fusion/orchestrator.js +0 -86
- package/build/fusion/types.js +0 -1
- package/build/gateway/unified-dataset-gateway.js +0 -410
- package/build/index.js +0 -3068
- package/build/ingestion/hf-downloader.js +0 -171
- package/build/ingestion/ingestor.js +0 -271
- package/build/ingestion/kaggle-downloader.js +0 -102
- package/build/install/install-service.js +0 -46
- package/build/jobs/manager.js +0 -136
- package/build/jobs/queue.js +0 -59
- package/build/jobs/types.js +0 -1
- package/build/lib/supabase.js +0 -3
- package/build/metadata/dataworld-source.js +0 -89
- package/build/metadata/domain.js +0 -147
- package/build/metadata/github-scraper.js +0 -47
- package/build/metadata/institutional-scrapers.js +0 -49
- package/build/metadata/kaggle-scraper.js +0 -182
- package/build/metadata/kaggle-source.js +0 -70
- package/build/metadata/license.js +0 -68
- package/build/metadata/monitoring-service.js +0 -107
- package/build/metadata/monitoring-store.js +0 -78
- package/build/metadata/monitoring-types.js +0 -1
- package/build/metadata/openml-source.js +0 -87
- package/build/metadata/quality.js +0 -48
- package/build/metadata/rate-limiter.js +0 -128
- package/build/metadata/scraper.js +0 -448
- package/build/metadata/store.js +0 -340
- package/build/metadata/types.js +0 -1
- package/build/metadata/uci-scraper.js +0 -49
- package/build/monitoring/observability.js +0 -76
- package/build/preparation/target-detector.js +0 -75
- package/build/python/__pycache__/config.cpython-312.pyc +0 -0
- package/build/python/asset_downloader_engine.py +0 -94
- package/build/python/cleaner.py +0 -226
- package/build/python/config.py +0 -263
- package/build/python/convert_engine.py +0 -92
- package/build/python/dataworld_engine.py +0 -208
- package/build/python/export_engine.py +0 -288
- package/build/python/framework_adapters.py +0 -100
- package/build/python/fusion_engine.py +0 -368
- package/build/python/github_adapter.py +0 -106
- package/build/python/hf_fallback.py +0 -298
- package/build/python/image_engine.py +0 -86
- package/build/python/kaggle_engine.py +0 -295
- package/build/python/media_engine.py +0 -133
- package/build/python/nasa_adapter.py +0 -82
- package/build/python/normalize_engine.py +0 -83
- package/build/python/openml_engine.py +0 -146
- package/build/python/quality_engine.py +0 -267
- package/build/python/row_count.py +0 -54
- package/build/python/splitter_engine.py +0 -283
- package/build/python/target_engine.py +0 -154
- package/build/python/test_framework_adapters.py +0 -61
- package/build/python/test_fusion_engine.py +0 -89
- package/build/python/uci_adapter.py +0 -94
- package/build/python/vesper/__init__.py +0 -1
- package/build/python/vesper/__pycache__/__init__.cpython-312.pyc +0 -0
- package/build/python/vesper/core/__init__.py +0 -1
- package/build/python/vesper/core/__pycache__/__init__.cpython-312.pyc +0 -0
- package/build/python/vesper/core/__pycache__/asset_downloader.cpython-312.pyc +0 -0
- package/build/python/vesper/core/__pycache__/download_recipe.cpython-312.pyc +0 -0
- package/build/python/vesper/core/asset_downloader.py +0 -679
- package/build/python/vesper/core/download_recipe.py +0 -104
- package/build/python/worldbank_adapter.py +0 -99
- package/build/quality/analyzer.js +0 -93
- package/build/quality/image-analyzer.js +0 -114
- package/build/quality/media-analyzer.js +0 -115
- package/build/quality/quality-orchestrator.js +0 -162
- package/build/quality/types.js +0 -1
- package/build/scripts/build-index.js +0 -54
- package/build/scripts/check-db.js +0 -73
- package/build/scripts/check-jobs.js +0 -24
- package/build/scripts/check-naruto.js +0 -17
- package/build/scripts/cleanup-kaggle.js +0 -41
- package/build/scripts/demo-full-pipeline.js +0 -62
- package/build/scripts/demo-ui.js +0 -58
- package/build/scripts/e2e-demo.js +0 -72
- package/build/scripts/massive-scrape.js +0 -103
- package/build/scripts/ops-dashboard.js +0 -33
- package/build/scripts/repro-bug.js +0 -37
- package/build/scripts/repro-export-bug.js +0 -56
- package/build/scripts/scrape-metadata.js +0 -100
- package/build/scripts/search-cli.js +0 -26
- package/build/scripts/test-bias.js +0 -45
- package/build/scripts/test-caching.js +0 -51
- package/build/scripts/test-cleaning.js +0 -76
- package/build/scripts/test-cloud-storage.js +0 -48
- package/build/scripts/test-compliance.js +0 -58
- package/build/scripts/test-conversion.js +0 -64
- package/build/scripts/test-custom-rules.js +0 -58
- package/build/scripts/test-db-opt.js +0 -63
- package/build/scripts/test-export-custom.js +0 -33
- package/build/scripts/test-exporter.js +0 -53
- package/build/scripts/test-fusion.js +0 -61
- package/build/scripts/test-github.js +0 -27
- package/build/scripts/test-group-split.js +0 -52
- package/build/scripts/test-hf-download.js +0 -29
- package/build/scripts/test-holdout-manager.js +0 -61
- package/build/scripts/test-hybrid-search.js +0 -41
- package/build/scripts/test-image-analysis.js +0 -50
- package/build/scripts/test-ingestion-infra.js +0 -39
- package/build/scripts/test-install.js +0 -40
- package/build/scripts/test-institutional.js +0 -26
- package/build/scripts/test-integrity.js +0 -41
- package/build/scripts/test-jit.js +0 -42
- package/build/scripts/test-job-queue.js +0 -62
- package/build/scripts/test-kaggle-download.js +0 -34
- package/build/scripts/test-large-data.js +0 -50
- package/build/scripts/test-mcp-v5.js +0 -74
- package/build/scripts/test-media-analysis.js +0 -61
- package/build/scripts/test-monitoring.js +0 -91
- package/build/scripts/test-observability.js +0 -106
- package/build/scripts/test-packager.js +0 -55
- package/build/scripts/test-pipeline.js +0 -50
- package/build/scripts/test-planning.js +0 -64
- package/build/scripts/test-privacy.js +0 -38
- package/build/scripts/test-production-sync.js +0 -36
- package/build/scripts/test-quality.js +0 -43
- package/build/scripts/test-robust-ingestion.js +0 -41
- package/build/scripts/test-schema.js +0 -45
- package/build/scripts/test-split-validation.js +0 -40
- package/build/scripts/test-splitter.js +0 -93
- package/build/scripts/test-target-detector.js +0 -29
- package/build/scripts/test-uci.js +0 -27
- package/build/scripts/test-unified-quality.js +0 -86
- package/build/scripts/test-write.js +0 -14
- package/build/scripts/verify-integration.js +0 -57
- package/build/scripts/verify-priority.js +0 -33
- package/build/search/embedder.js +0 -34
- package/build/search/engine.js +0 -190
- package/build/search/jit-orchestrator.js +0 -262
- package/build/search/query-intent.js +0 -509
- package/build/search/vector-store.js +0 -123
- package/build/splitting/splitter.js +0 -82
- package/build/splitting/types.js +0 -1
- package/build/tools/formatter.js +0 -251
- package/build/utils/downloader.js +0 -52
- package/build/utils/python-runtime.js +0 -130
- package/build/utils/selector.js +0 -69
- package/mcp-config-template.json +0 -18
- package/scripts/postinstall.cjs +0 -170
- package/scripts/preindex_registry.cjs +0 -157
- package/scripts/refresh-index.cjs +0 -87
- package/scripts/wizard.cjs +0 -601
- package/src/python/__pycache__/config.cpython-312.pyc +0 -0
- package/src/python/__pycache__/export_engine.cpython-312.pyc +0 -0
- package/src/python/__pycache__/framework_adapters.cpython-312.pyc +0 -0
- package/src/python/__pycache__/fusion_engine.cpython-312.pyc +0 -0
- package/src/python/__pycache__/kaggle_engine.cpython-312.pyc +0 -0
- package/src/python/asset_downloader_engine.py +0 -94
- package/src/python/cleaner.py +0 -226
- package/src/python/config.py +0 -263
- package/src/python/convert_engine.py +0 -92
- package/src/python/dataworld_engine.py +0 -208
- package/src/python/export_engine.py +0 -288
- package/src/python/framework_adapters.py +0 -100
- package/src/python/fusion_engine.py +0 -368
- package/src/python/github_adapter.py +0 -106
- package/src/python/hf_fallback.py +0 -298
- package/src/python/image_engine.py +0 -86
- package/src/python/kaggle_engine.py +0 -295
- package/src/python/media_engine.py +0 -133
- package/src/python/nasa_adapter.py +0 -82
- package/src/python/normalize_engine.py +0 -83
- package/src/python/openml_engine.py +0 -146
- package/src/python/quality_engine.py +0 -267
- package/src/python/requirements.txt +0 -12
- package/src/python/row_count.py +0 -54
- package/src/python/splitter_engine.py +0 -283
- package/src/python/target_engine.py +0 -154
- package/src/python/test_framework_adapters.py +0 -61
- package/src/python/test_fusion_engine.py +0 -89
- package/src/python/uci_adapter.py +0 -94
- package/src/python/vesper/__init__.py +0 -1
- package/src/python/vesper/core/__init__.py +0 -1
- package/src/python/vesper/core/asset_downloader.py +0 -679
- package/src/python/vesper/core/download_recipe.py +0 -104
- package/src/python/worldbank_adapter.py +0 -99
- package/wizard.cjs +0 -3
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
import { v4 as uuidv4 } from "uuid";
|
|
2
|
-
export class ComplianceService {
|
|
3
|
-
store;
|
|
4
|
-
constructor(store) {
|
|
5
|
-
this.store = store;
|
|
6
|
-
}
|
|
7
|
-
async verifyGDPR(dataset) {
|
|
8
|
-
const issues = [];
|
|
9
|
-
// 1. Check for PII metadata flag
|
|
10
|
-
if (dataset.has_personal_data) {
|
|
11
|
-
issues.push("Dataset explicitly flagged as containing personal data.");
|
|
12
|
-
}
|
|
13
|
-
// 2. Check quality warnings for PII
|
|
14
|
-
if (dataset.quality_warnings) {
|
|
15
|
-
const piiWarnings = dataset.quality_warnings.filter(w => w.toLowerCase().includes("pii") || w.toLowerCase().includes("personal"));
|
|
16
|
-
if (piiWarnings.length > 0) {
|
|
17
|
-
issues.push(...piiWarnings);
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
// 3. Check for consent record
|
|
21
|
-
const consent = this.store.getConsent(dataset.id);
|
|
22
|
-
if (!consent || !consent.consent_obtained) {
|
|
23
|
-
issues.push("No valid consent record found for this dataset.");
|
|
24
|
-
}
|
|
25
|
-
const result = {
|
|
26
|
-
dataset_id: dataset.id,
|
|
27
|
-
standard: "GDPR",
|
|
28
|
-
passed: issues.length === 0,
|
|
29
|
-
issues,
|
|
30
|
-
timestamp: new Date().toISOString()
|
|
31
|
-
};
|
|
32
|
-
this.store.saveCheck(result);
|
|
33
|
-
return result;
|
|
34
|
-
}
|
|
35
|
-
async verifyHIPAA(dataset) {
|
|
36
|
-
const issues = [];
|
|
37
|
-
// 1. De-identification check (simulated)
|
|
38
|
-
// If domain is medical but no de-identification flag exists
|
|
39
|
-
if (dataset.domain === "healthcare" || dataset.domain === "medical") {
|
|
40
|
-
if (dataset.has_personal_data) {
|
|
41
|
-
issues.push("Medical dataset contains personal data (not de-identified).");
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
const result = {
|
|
45
|
-
dataset_id: dataset.id,
|
|
46
|
-
standard: "HIPAA",
|
|
47
|
-
passed: issues.length === 0,
|
|
48
|
-
issues,
|
|
49
|
-
timestamp: new Date().toISOString()
|
|
50
|
-
};
|
|
51
|
-
this.store.saveCheck(result);
|
|
52
|
-
return result;
|
|
53
|
-
}
|
|
54
|
-
logOperation(userId, datasetId, operation, metadata = {}) {
|
|
55
|
-
const event = {
|
|
56
|
-
id: uuidv4(),
|
|
57
|
-
user_id: userId,
|
|
58
|
-
dataset_id: datasetId,
|
|
59
|
-
operation,
|
|
60
|
-
timestamp: new Date().toISOString(),
|
|
61
|
-
details: JSON.stringify(metadata)
|
|
62
|
-
};
|
|
63
|
-
this.store.saveAudit(event);
|
|
64
|
-
}
|
|
65
|
-
exportAuditLog(datasetId) {
|
|
66
|
-
const logs = this.store.getAuditLogs(datasetId);
|
|
67
|
-
if (logs.length === 0)
|
|
68
|
-
return "No audit logs found.";
|
|
69
|
-
const header = "ID,Timestamp,User,Operation,Dataset,Details\n";
|
|
70
|
-
const rows = logs.map(l => `${l.id},${l.timestamp},${l.user_id},${l.operation},${l.dataset_id},"${l.details.replace(/"/g, '""')}"`).join("\n");
|
|
71
|
-
return header + rows;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
export class ComplianceStore {
|
|
2
|
-
db;
|
|
3
|
-
constructor(db) {
|
|
4
|
-
this.db = db;
|
|
5
|
-
this.init();
|
|
6
|
-
}
|
|
7
|
-
init() {
|
|
8
|
-
this.db.exec(`
|
|
9
|
-
CREATE TABLE IF NOT EXISTS audit_logs (
|
|
10
|
-
id TEXT PRIMARY KEY,
|
|
11
|
-
user_id TEXT,
|
|
12
|
-
dataset_id TEXT,
|
|
13
|
-
operation TEXT,
|
|
14
|
-
timestamp TEXT,
|
|
15
|
-
details TEXT
|
|
16
|
-
);
|
|
17
|
-
|
|
18
|
-
CREATE TABLE IF NOT EXISTS compliance_checks (
|
|
19
|
-
dataset_id TEXT,
|
|
20
|
-
standard TEXT,
|
|
21
|
-
passed BOOLEAN,
|
|
22
|
-
issues TEXT, -- JSON array
|
|
23
|
-
timestamp TEXT,
|
|
24
|
-
PRIMARY KEY (dataset_id, standard)
|
|
25
|
-
);
|
|
26
|
-
|
|
27
|
-
CREATE TABLE IF NOT EXISTS consent_records (
|
|
28
|
-
dataset_id TEXT PRIMARY KEY,
|
|
29
|
-
consent_obtained BOOLEAN,
|
|
30
|
-
source TEXT,
|
|
31
|
-
last_verified TEXT
|
|
32
|
-
);
|
|
33
|
-
`);
|
|
34
|
-
}
|
|
35
|
-
saveAudit(event) {
|
|
36
|
-
const stmt = this.db.prepare(`
|
|
37
|
-
INSERT INTO audit_logs (id, user_id, dataset_id, operation, timestamp, details)
|
|
38
|
-
VALUES (?, ?, ?, ?, ?, ?)
|
|
39
|
-
`);
|
|
40
|
-
stmt.run(event.id, event.user_id, event.dataset_id, event.operation, event.timestamp, event.details);
|
|
41
|
-
}
|
|
42
|
-
getAuditLogs(datasetId) {
|
|
43
|
-
let query = "SELECT * FROM audit_logs";
|
|
44
|
-
const params = [];
|
|
45
|
-
if (datasetId) {
|
|
46
|
-
query += " WHERE dataset_id = ?";
|
|
47
|
-
params.push(datasetId);
|
|
48
|
-
}
|
|
49
|
-
query += " ORDER BY timestamp DESC";
|
|
50
|
-
return this.db.prepare(query).all(...params);
|
|
51
|
-
}
|
|
52
|
-
saveCheck(result) {
|
|
53
|
-
const stmt = this.db.prepare(`
|
|
54
|
-
INSERT INTO compliance_checks (dataset_id, standard, passed, issues, timestamp)
|
|
55
|
-
VALUES (?, ?, ?, ?, ?)
|
|
56
|
-
ON CONFLICT(dataset_id, standard) DO UPDATE SET
|
|
57
|
-
passed=excluded.passed,
|
|
58
|
-
issues=excluded.issues,
|
|
59
|
-
timestamp=excluded.timestamp
|
|
60
|
-
`);
|
|
61
|
-
stmt.run(result.dataset_id, result.standard, result.passed ? 1 : 0, JSON.stringify(result.issues), result.timestamp);
|
|
62
|
-
}
|
|
63
|
-
saveConsent(record) {
|
|
64
|
-
const stmt = this.db.prepare(`
|
|
65
|
-
INSERT INTO consent_records (dataset_id, consent_obtained, source, last_verified)
|
|
66
|
-
VALUES (?, ?, ?, ?)
|
|
67
|
-
ON CONFLICT(dataset_id) DO UPDATE SET
|
|
68
|
-
consent_obtained=excluded.consent_obtained,
|
|
69
|
-
source=excluded.source,
|
|
70
|
-
last_verified=excluded.last_verified
|
|
71
|
-
`);
|
|
72
|
-
stmt.run(record.dataset_id, record.consent_obtained ? 1 : 0, record.source, record.last_verified);
|
|
73
|
-
}
|
|
74
|
-
getConsent(datasetId) {
|
|
75
|
-
const row = this.db.prepare("SELECT * FROM consent_records WHERE dataset_id = ?").get(datasetId);
|
|
76
|
-
if (!row)
|
|
77
|
-
return null;
|
|
78
|
-
return { ...row, consent_obtained: Boolean(row.consent_obtained) };
|
|
79
|
-
}
|
|
80
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,221 +0,0 @@
|
|
|
1
|
-
import fs from "fs";
|
|
2
|
-
import path from "path";
|
|
3
|
-
import os from "os";
|
|
4
|
-
export class ConfigManager {
|
|
5
|
-
getHomeDir() {
|
|
6
|
-
return os.homedir();
|
|
7
|
-
}
|
|
8
|
-
/**
|
|
9
|
-
* All known coding agent configuration paths.
|
|
10
|
-
* Based on the `add-mcp` standard used by the MCP ecosystem.
|
|
11
|
-
*/
|
|
12
|
-
getAllAgents() {
|
|
13
|
-
const home = this.getHomeDir();
|
|
14
|
-
const isWin = process.platform === "win32";
|
|
15
|
-
const isMac = process.platform === "darwin";
|
|
16
|
-
const appData = process.env.APPDATA || path.join(home, "AppData", "Roaming");
|
|
17
|
-
const agents = [
|
|
18
|
-
// Claude Code: ~/.claude.json
|
|
19
|
-
{
|
|
20
|
-
name: "Claude Code",
|
|
21
|
-
path: path.join(home, ".claude.json"),
|
|
22
|
-
format: "json-mcpServers",
|
|
23
|
-
serverType: "stdio",
|
|
24
|
-
},
|
|
25
|
-
// Claude Desktop: %APPDATA%/Claude/claude_desktop_config.json
|
|
26
|
-
{
|
|
27
|
-
name: "Claude Desktop",
|
|
28
|
-
path: isWin
|
|
29
|
-
? path.join(appData, "Claude", "claude_desktop_config.json")
|
|
30
|
-
: isMac
|
|
31
|
-
? path.join(home, "Library", "Application Support", "Claude", "claude_desktop_config.json")
|
|
32
|
-
: path.join(home, ".config", "claude", "claude_desktop_config.json"),
|
|
33
|
-
format: "json-mcpServers",
|
|
34
|
-
serverType: "stdio",
|
|
35
|
-
},
|
|
36
|
-
// Cursor: ~/.cursor/mcp.json
|
|
37
|
-
{
|
|
38
|
-
name: "Cursor",
|
|
39
|
-
path: path.join(home, ".cursor", "mcp.json"),
|
|
40
|
-
format: "json-mcpServers",
|
|
41
|
-
serverType: "stdio",
|
|
42
|
-
},
|
|
43
|
-
// VS Code: %APPDATA%/Code/User/mcp.json (uses "servers" key, NOT "mcpServers")
|
|
44
|
-
{
|
|
45
|
-
name: "VS Code",
|
|
46
|
-
path: isWin
|
|
47
|
-
? path.join(appData, "Code", "User", "mcp.json")
|
|
48
|
-
: isMac
|
|
49
|
-
? path.join(home, "Library", "Application Support", "Code", "User", "mcp.json")
|
|
50
|
-
: path.join(home, ".config", "Code", "User", "mcp.json"),
|
|
51
|
-
format: "json-servers",
|
|
52
|
-
serverType: "stdio",
|
|
53
|
-
},
|
|
54
|
-
// Codex: ~/.codex/config.toml
|
|
55
|
-
{
|
|
56
|
-
name: "Codex",
|
|
57
|
-
path: path.join(home, ".codex", "config.toml"),
|
|
58
|
-
format: "toml",
|
|
59
|
-
serverType: "stdio",
|
|
60
|
-
},
|
|
61
|
-
// Antigravity / Gemini CLI: ~/.gemini/settings.json
|
|
62
|
-
{
|
|
63
|
-
name: "Antigravity",
|
|
64
|
-
path: path.join(home, ".gemini", "settings.json"),
|
|
65
|
-
format: "json-mcpServers",
|
|
66
|
-
serverType: "stdio",
|
|
67
|
-
},
|
|
68
|
-
];
|
|
69
|
-
return agents;
|
|
70
|
-
}
|
|
71
|
-
/**
|
|
72
|
-
* Detect which agents are installed on this machine.
|
|
73
|
-
*/
|
|
74
|
-
detectIDEs() {
|
|
75
|
-
const agents = this.getAllAgents();
|
|
76
|
-
const detected = [];
|
|
77
|
-
for (const agent of agents) {
|
|
78
|
-
const configDir = path.dirname(agent.path);
|
|
79
|
-
const configExists = fs.existsSync(agent.path);
|
|
80
|
-
const dirExists = fs.existsSync(configDir);
|
|
81
|
-
if (configExists || dirExists) {
|
|
82
|
-
detected.push(agent);
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
return detected;
|
|
86
|
-
}
|
|
87
|
-
/**
|
|
88
|
-
* Build the server config object for Vesper.
|
|
89
|
-
*/
|
|
90
|
-
getServerConfig() {
|
|
91
|
-
const isWin = process.platform === "win32";
|
|
92
|
-
return {
|
|
93
|
-
command: isWin ? "npx.cmd" : "npx",
|
|
94
|
-
args: ["-y", "-p", "@vespermcp/mcp-server@latest", "vespermcp"],
|
|
95
|
-
};
|
|
96
|
-
}
|
|
97
|
-
/**
|
|
98
|
-
* Install Vesper to a single agent configuration.
|
|
99
|
-
*/
|
|
100
|
-
async installTo(agent) {
|
|
101
|
-
try {
|
|
102
|
-
console.log(`[Vesper Setup] Installing to ${agent.name}...`);
|
|
103
|
-
if (agent.format === "toml") {
|
|
104
|
-
return this.installToToml(agent);
|
|
105
|
-
}
|
|
106
|
-
// JSON-based agents
|
|
107
|
-
let config = {};
|
|
108
|
-
if (fs.existsSync(agent.path)) {
|
|
109
|
-
try {
|
|
110
|
-
const content = fs.readFileSync(agent.path, "utf-8").trim();
|
|
111
|
-
config = content ? JSON.parse(content) : {};
|
|
112
|
-
}
|
|
113
|
-
catch (e) {
|
|
114
|
-
console.warn(`[Vesper Setup] Could not parse ${agent.path}, starting fresh`);
|
|
115
|
-
config = {};
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
else {
|
|
119
|
-
fs.mkdirSync(path.dirname(agent.path), { recursive: true });
|
|
120
|
-
}
|
|
121
|
-
const vesperConfig = this.getServerConfig();
|
|
122
|
-
const serverEntry = {
|
|
123
|
-
command: vesperConfig.command,
|
|
124
|
-
args: vesperConfig.args,
|
|
125
|
-
};
|
|
126
|
-
// Determine the correct key based on agent format
|
|
127
|
-
if (agent.format === "json-servers") {
|
|
128
|
-
// VS Code native MCP format: { "servers": { "vesper": { "type": "stdio", "command": ..., "args": ... } } }
|
|
129
|
-
if (!config.servers)
|
|
130
|
-
config.servers = {};
|
|
131
|
-
config.servers["vesper"] = {
|
|
132
|
-
type: "stdio",
|
|
133
|
-
...serverEntry,
|
|
134
|
-
};
|
|
135
|
-
console.log(` ✓ Added to "servers" key (VS Code native format)`);
|
|
136
|
-
}
|
|
137
|
-
else {
|
|
138
|
-
// Standard mcpServers format: { "mcpServers": { "vesper": { "command": ..., "args": ... } } }
|
|
139
|
-
if (!config.mcpServers)
|
|
140
|
-
config.mcpServers = {};
|
|
141
|
-
config.mcpServers["vesper"] = serverEntry;
|
|
142
|
-
console.log(` ✓ Added to "mcpServers" key`);
|
|
143
|
-
}
|
|
144
|
-
fs.writeFileSync(agent.path, JSON.stringify(config, null, 2), "utf8");
|
|
145
|
-
const size = fs.statSync(agent.path).size;
|
|
146
|
-
console.log(` ✓ ${agent.name}: ${agent.path} (${size} bytes)`);
|
|
147
|
-
return true;
|
|
148
|
-
}
|
|
149
|
-
catch (error) {
|
|
150
|
-
console.error(` ✗ Failed to install to ${agent.name}:`, error);
|
|
151
|
-
return false;
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
/**
|
|
155
|
-
* Install to TOML-based agents (Codex).
|
|
156
|
-
*/
|
|
157
|
-
installToToml(agent) {
|
|
158
|
-
try {
|
|
159
|
-
let content = "";
|
|
160
|
-
if (fs.existsSync(agent.path)) {
|
|
161
|
-
content = fs.readFileSync(agent.path, "utf-8");
|
|
162
|
-
}
|
|
163
|
-
else {
|
|
164
|
-
fs.mkdirSync(path.dirname(agent.path), { recursive: true });
|
|
165
|
-
}
|
|
166
|
-
// Check if vesper is already configured
|
|
167
|
-
if (content.includes("[mcp_servers.vesper]")) {
|
|
168
|
-
console.log(` ✓ ${agent.name}: Already configured`);
|
|
169
|
-
return true;
|
|
170
|
-
}
|
|
171
|
-
const vesperConfig = this.getServerConfig();
|
|
172
|
-
const isWin = process.platform === "win32";
|
|
173
|
-
const tomlEntry = `
|
|
174
|
-
[mcp_servers.vesper]
|
|
175
|
-
command = "${vesperConfig.command}"
|
|
176
|
-
args = [${vesperConfig.args.map(a => `"${a}"`).join(", ")}]
|
|
177
|
-
`;
|
|
178
|
-
content += tomlEntry;
|
|
179
|
-
fs.writeFileSync(agent.path, content, "utf8");
|
|
180
|
-
console.log(` ✓ ${agent.name}: ${agent.path}`);
|
|
181
|
-
return true;
|
|
182
|
-
}
|
|
183
|
-
catch (error) {
|
|
184
|
-
console.error(` ✗ Failed to install to ${agent.name}:`, error);
|
|
185
|
-
return false;
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
/**
|
|
189
|
-
* Install Vesper to ALL detected agents automatically.
|
|
190
|
-
*/
|
|
191
|
-
async installToAll() {
|
|
192
|
-
const agents = this.detectIDEs();
|
|
193
|
-
const success = [];
|
|
194
|
-
const failed = [];
|
|
195
|
-
console.log(`\n📦 Installing Vesper to ${agents.length} detected agents...\n`);
|
|
196
|
-
for (const agent of agents) {
|
|
197
|
-
const result = await this.installTo(agent);
|
|
198
|
-
if (result) {
|
|
199
|
-
success.push(agent.name);
|
|
200
|
-
}
|
|
201
|
-
else {
|
|
202
|
-
failed.push(agent.name);
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
console.log(`\n┌─────────────────────────────────────────────┐`);
|
|
206
|
-
console.log(`│ Installation Summary │`);
|
|
207
|
-
console.log(`├─────────────────────────────────────────────┤`);
|
|
208
|
-
if (success.length > 0) {
|
|
209
|
-
for (const name of success) {
|
|
210
|
-
console.log(`│ ✓ ${name.padEnd(40)}│`);
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
if (failed.length > 0) {
|
|
214
|
-
for (const name of failed) {
|
|
215
|
-
console.log(`│ ✗ ${name.padEnd(40)}│`);
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
console.log(`└─────────────────────────────────────────────┘\n`);
|
|
219
|
-
return { success, failed };
|
|
220
|
-
}
|
|
221
|
-
}
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
import { spawnSync } from "child_process";
|
|
2
|
-
import path from "path";
|
|
3
|
-
import fs from "fs";
|
|
4
|
-
export class SecureKeysManager {
|
|
5
|
-
pythonPath = "python";
|
|
6
|
-
scriptPath;
|
|
7
|
-
constructor(buildDir = process.cwd()) {
|
|
8
|
-
const homeDir = process.env.HOME || process.env.USERPROFILE || buildDir;
|
|
9
|
-
const dataRoot = path.join(homeDir, ".vesper");
|
|
10
|
-
const scriptPath0 = path.resolve(dataRoot, "python", "config.py");
|
|
11
|
-
const scriptPath1 = path.resolve(buildDir, "python", "config.py");
|
|
12
|
-
const scriptPath2 = path.resolve(buildDir, "..", "src", "python", "config.py");
|
|
13
|
-
if (fs.existsSync(scriptPath0))
|
|
14
|
-
this.scriptPath = scriptPath0;
|
|
15
|
-
else if (fs.existsSync(scriptPath1))
|
|
16
|
-
this.scriptPath = scriptPath1;
|
|
17
|
-
else if (fs.existsSync(scriptPath2))
|
|
18
|
-
this.scriptPath = scriptPath2;
|
|
19
|
-
else
|
|
20
|
-
this.scriptPath = scriptPath0;
|
|
21
|
-
if (process.platform === "win32")
|
|
22
|
-
this.pythonPath = "py";
|
|
23
|
-
}
|
|
24
|
-
getAll() {
|
|
25
|
-
const result = spawnSync(this.pythonPath, [this.scriptPath, "all"], { encoding: "utf-8" });
|
|
26
|
-
if (result.status !== 0)
|
|
27
|
-
return {};
|
|
28
|
-
try {
|
|
29
|
-
const parsed = JSON.parse(result.stdout || "{}");
|
|
30
|
-
if (!parsed.ok)
|
|
31
|
-
return {};
|
|
32
|
-
return (parsed.data || {});
|
|
33
|
-
}
|
|
34
|
-
catch {
|
|
35
|
-
return {};
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
set(name, value) {
|
|
39
|
-
const result = spawnSync(this.pythonPath, [this.scriptPath, "set", name, value], { encoding: "utf-8" });
|
|
40
|
-
if (result.status !== 0) {
|
|
41
|
-
return { ok: false, error: result.stderr || "Failed to store key" };
|
|
42
|
-
}
|
|
43
|
-
try {
|
|
44
|
-
const parsed = JSON.parse(result.stdout || "{}");
|
|
45
|
-
return { ok: !!parsed.ok, method: parsed.method, error: parsed.error };
|
|
46
|
-
}
|
|
47
|
-
catch {
|
|
48
|
-
return { ok: false, error: "Invalid response from config helper" };
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
}
|
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
import fs from "fs";
|
|
2
|
-
import path from "path";
|
|
3
|
-
import os from "os";
|
|
4
|
-
export class UserConfigStore {
|
|
5
|
-
configPath;
|
|
6
|
-
constructor(dataRoot) {
|
|
7
|
-
const homeDir = os.homedir() || process.env.HOME || process.env.USERPROFILE || process.cwd();
|
|
8
|
-
const root = dataRoot || path.join(homeDir, ".vesper");
|
|
9
|
-
if (!fs.existsSync(root)) {
|
|
10
|
-
fs.mkdirSync(root, { recursive: true });
|
|
11
|
-
}
|
|
12
|
-
this.configPath = path.join(root, "config.json");
|
|
13
|
-
}
|
|
14
|
-
getPath() {
|
|
15
|
-
return this.configPath;
|
|
16
|
-
}
|
|
17
|
-
load() {
|
|
18
|
-
if (!fs.existsSync(this.configPath))
|
|
19
|
-
return {};
|
|
20
|
-
try {
|
|
21
|
-
const raw = fs.readFileSync(this.configPath, "utf8");
|
|
22
|
-
const parsed = JSON.parse(raw);
|
|
23
|
-
return parsed || {};
|
|
24
|
-
}
|
|
25
|
-
catch {
|
|
26
|
-
return {};
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
save(config) {
|
|
30
|
-
const parent = path.dirname(this.configPath);
|
|
31
|
-
if (!fs.existsSync(parent))
|
|
32
|
-
fs.mkdirSync(parent, { recursive: true });
|
|
33
|
-
fs.writeFileSync(this.configPath, JSON.stringify(config, null, 2), "utf8");
|
|
34
|
-
}
|
|
35
|
-
setKaggleCredentials(username, key) {
|
|
36
|
-
const cfg = this.load();
|
|
37
|
-
cfg.kaggle = { username, key };
|
|
38
|
-
this.save(cfg);
|
|
39
|
-
}
|
|
40
|
-
getKaggleCredentials() {
|
|
41
|
-
const cfg = this.load();
|
|
42
|
-
return cfg.kaggle || {};
|
|
43
|
-
}
|
|
44
|
-
hasKaggleCredentials() {
|
|
45
|
-
const creds = this.getKaggleCredentials();
|
|
46
|
-
return !!(creds.username && creds.key);
|
|
47
|
-
}
|
|
48
|
-
}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import { parentPort } from "worker_threads";
|
|
2
|
-
if (parentPort) {
|
|
3
|
-
parentPort.on("message", (data) => {
|
|
4
|
-
const { records, task, params } = data;
|
|
5
|
-
let results = [];
|
|
6
|
-
switch (task) {
|
|
7
|
-
case "transform":
|
|
8
|
-
results = records.map(r => {
|
|
9
|
-
// Simulated transformation (e.g. alignment/harmonization logic)
|
|
10
|
-
const updated = { ...r, processed_at: new Date().toISOString() };
|
|
11
|
-
// Complex CPU bound work simulation
|
|
12
|
-
let sum = 0;
|
|
13
|
-
for (let i = 0; i < 1000; i++)
|
|
14
|
-
sum += i;
|
|
15
|
-
return updated;
|
|
16
|
-
});
|
|
17
|
-
break;
|
|
18
|
-
default:
|
|
19
|
-
results = records;
|
|
20
|
-
}
|
|
21
|
-
parentPort?.postMessage(results);
|
|
22
|
-
});
|
|
23
|
-
}
|
package/build/data/streaming.js
DELETED
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
export class StreamProcessor {
|
|
2
|
-
/**
|
|
3
|
-
* Processes a readable stream in chunks.
|
|
4
|
-
*/
|
|
5
|
-
static async processInChunks(stream, options, processor) {
|
|
6
|
-
let chunk = [];
|
|
7
|
-
const results = [];
|
|
8
|
-
let totalProcessed = 0;
|
|
9
|
-
for await (const record of stream) {
|
|
10
|
-
// Sampling logic
|
|
11
|
-
if (options.samplePercentage !== undefined) {
|
|
12
|
-
if (Math.random() * 100 > options.samplePercentage) {
|
|
13
|
-
continue;
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
chunk.push(record);
|
|
17
|
-
if (chunk.length >= options.chunkSize) {
|
|
18
|
-
const processedChunk = await processor(chunk);
|
|
19
|
-
results.push(...processedChunk);
|
|
20
|
-
totalProcessed += chunk.length;
|
|
21
|
-
chunk = [];
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
// Process remaining
|
|
25
|
-
if (chunk.length > 0) {
|
|
26
|
-
const processedChunk = await processor(chunk);
|
|
27
|
-
results.push(...processedChunk);
|
|
28
|
-
totalProcessed += chunk.length;
|
|
29
|
-
}
|
|
30
|
-
return results;
|
|
31
|
-
}
|
|
32
|
-
/**
|
|
33
|
-
* Creates a sampler stream that only emits a percentage of records.
|
|
34
|
-
*/
|
|
35
|
-
static createSampler(percentage) {
|
|
36
|
-
return () => Math.random() * 100 <= percentage;
|
|
37
|
-
}
|
|
38
|
-
}
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
import { Worker } from "worker_threads";
|
|
2
|
-
import path from "path";
|
|
3
|
-
import { fileURLToPath } from "url";
|
|
4
|
-
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
5
|
-
export class WorkerPool {
|
|
6
|
-
poolSize;
|
|
7
|
-
workers = [];
|
|
8
|
-
queue = [];
|
|
9
|
-
activeWorkers = 0;
|
|
10
|
-
constructor(poolSize = 4) {
|
|
11
|
-
this.poolSize = poolSize;
|
|
12
|
-
const workerPath = path.resolve(__dirname, "processing-worker.js");
|
|
13
|
-
// Note: In development with tsx, we might need to handle .ts vs .js
|
|
14
|
-
}
|
|
15
|
-
async process(records, task = "transform") {
|
|
16
|
-
return new Promise((resolve, reject) => {
|
|
17
|
-
const worker = new Worker(new URL("./processing-worker.ts", import.meta.url), {
|
|
18
|
-
execArgv: ["--import", "tsx"]
|
|
19
|
-
});
|
|
20
|
-
worker.postMessage({ records, task });
|
|
21
|
-
worker.on("message", (results) => {
|
|
22
|
-
worker.terminate();
|
|
23
|
-
resolve(results);
|
|
24
|
-
});
|
|
25
|
-
worker.on("error", reject);
|
|
26
|
-
});
|
|
27
|
-
}
|
|
28
|
-
/**
|
|
29
|
-
* Splits records into chunks and processes them in parallel across workers.
|
|
30
|
-
*/
|
|
31
|
-
async processParallel(records, chunkSize) {
|
|
32
|
-
const chunks = [];
|
|
33
|
-
for (let i = 0; i < records.length; i += chunkSize) {
|
|
34
|
-
chunks.push(records.slice(i, i + chunkSize));
|
|
35
|
-
}
|
|
36
|
-
const results = await Promise.all(chunks.map(chunk => this.process(chunk)));
|
|
37
|
-
return results.flat();
|
|
38
|
-
}
|
|
39
|
-
}
|
package/build/export/exporter.js
DELETED
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
import { spawn } from "child_process";
|
|
2
|
-
import path from "path";
|
|
3
|
-
import fs from "fs";
|
|
4
|
-
import { ensurePythonPackages, resolvePythonCommand } from "../utils/python-runtime.js";
|
|
5
|
-
export class DataExporter {
|
|
6
|
-
buildDir;
|
|
7
|
-
scriptPath;
|
|
8
|
-
constructor(buildDir = process.cwd()) {
|
|
9
|
-
this.buildDir = buildDir;
|
|
10
|
-
const homeDir = process.env.HOME || process.env.USERPROFILE || buildDir;
|
|
11
|
-
const dataRoot = path.join(homeDir, ".vesper");
|
|
12
|
-
const scriptPath0 = path.resolve(dataRoot, "python", "export_engine.py");
|
|
13
|
-
const scriptPath1 = path.resolve(buildDir, "python", "export_engine.py");
|
|
14
|
-
const scriptPath2 = path.resolve(buildDir, "..", "src", "python", "export_engine.py");
|
|
15
|
-
const scriptPath3 = path.resolve(buildDir, "..", "python", "export_engine.py");
|
|
16
|
-
if (fs.existsSync(scriptPath0)) {
|
|
17
|
-
this.scriptPath = scriptPath0;
|
|
18
|
-
}
|
|
19
|
-
else if (fs.existsSync(scriptPath1)) {
|
|
20
|
-
this.scriptPath = scriptPath1;
|
|
21
|
-
}
|
|
22
|
-
else if (fs.existsSync(scriptPath2)) {
|
|
23
|
-
this.scriptPath = scriptPath2;
|
|
24
|
-
}
|
|
25
|
-
else if (fs.existsSync(scriptPath3)) {
|
|
26
|
-
this.scriptPath = scriptPath3;
|
|
27
|
-
}
|
|
28
|
-
else {
|
|
29
|
-
this.scriptPath = scriptPath0;
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
/**
|
|
33
|
-
* Exports a dataset file to a specified format
|
|
34
|
-
*/
|
|
35
|
-
async export(inputFile, outputFile, format, options = {}) {
|
|
36
|
-
const pythonRequirements = [
|
|
37
|
-
{ module: "polars", packageName: "polars" },
|
|
38
|
-
];
|
|
39
|
-
if (format === "feather") {
|
|
40
|
-
pythonRequirements.push({ module: "pyarrow", packageName: "pyarrow" });
|
|
41
|
-
}
|
|
42
|
-
if (format === "tfrecord") {
|
|
43
|
-
pythonRequirements.push({ module: "tensorflow", packageName: "tensorflow" });
|
|
44
|
-
}
|
|
45
|
-
const pythonPath = await ensurePythonPackages(this.buildDir, pythonRequirements).catch(() => resolvePythonCommand(this.buildDir));
|
|
46
|
-
return new Promise((resolve, reject) => {
|
|
47
|
-
if (!fs.existsSync(inputFile)) {
|
|
48
|
-
reject(new Error(`Input file not found: ${inputFile}`));
|
|
49
|
-
return;
|
|
50
|
-
}
|
|
51
|
-
const args = [this.scriptPath, inputFile, outputFile, format, JSON.stringify(options)];
|
|
52
|
-
const childProcess = spawn(pythonPath, args, {
|
|
53
|
-
env: {
|
|
54
|
-
...process.env,
|
|
55
|
-
PYTHONIOENCODING: "utf-8",
|
|
56
|
-
},
|
|
57
|
-
});
|
|
58
|
-
let stdout = "";
|
|
59
|
-
let stderr = "";
|
|
60
|
-
childProcess.stdout.on("data", (data) => stdout += data.toString());
|
|
61
|
-
childProcess.stderr.on("data", (data) => stderr += data.toString());
|
|
62
|
-
childProcess.on("close", (code) => {
|
|
63
|
-
if (code !== 0) {
|
|
64
|
-
reject(new Error(`Export failed: ${stderr || stdout}`));
|
|
65
|
-
return;
|
|
66
|
-
}
|
|
67
|
-
try {
|
|
68
|
-
const result = JSON.parse(stdout);
|
|
69
|
-
if (result.error) {
|
|
70
|
-
reject(new Error(result.error));
|
|
71
|
-
}
|
|
72
|
-
else {
|
|
73
|
-
resolve(result);
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
catch (e) {
|
|
77
|
-
reject(new Error(`Failed to parse export output: ${stdout}`));
|
|
78
|
-
}
|
|
79
|
-
});
|
|
80
|
-
});
|
|
81
|
-
}
|
|
82
|
-
}
|