rediseed 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +165 -0
- package/dist/cli.d.mts +2 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +463 -0
- package/dist/cli.js.map +1 -0
- package/dist/cli.mjs +440 -0
- package/dist/cli.mjs.map +1 -0
- package/dist/index.d.mts +89 -0
- package/dist/index.d.ts +89 -0
- package/dist/index.js +366 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +321 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +72 -0
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/loaders/file.ts
|
|
4
|
+
import fs from "fs";
|
|
5
|
+
import { parse } from "csv-parse/sync";
|
|
6
|
+
function parseCSV(content, delimiter = ",") {
|
|
7
|
+
const records = parse(content, {
|
|
8
|
+
columns: true,
|
|
9
|
+
skip_empty_lines: true,
|
|
10
|
+
trim: true,
|
|
11
|
+
delimiter,
|
|
12
|
+
cast: false
|
|
13
|
+
});
|
|
14
|
+
return records;
|
|
15
|
+
}
|
|
16
|
+
function parseJSONOrJSONL(content) {
|
|
17
|
+
const trimmed = content.trim();
|
|
18
|
+
if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
|
|
19
|
+
const parsed = JSON.parse(trimmed);
|
|
20
|
+
if (!Array.isArray(parsed)) throw new Error("JSON root must be an array of objects.");
|
|
21
|
+
return parsed;
|
|
22
|
+
}
|
|
23
|
+
const lines = trimmed.split(/\r?\n/).map((l) => l.trim()).filter(Boolean);
|
|
24
|
+
const rows = [];
|
|
25
|
+
for (const line of lines) {
|
|
26
|
+
if (line.startsWith("{") && line.endsWith("}")) {
|
|
27
|
+
rows.push(JSON.parse(line));
|
|
28
|
+
} else {
|
|
29
|
+
rows.push({ value: line });
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return rows;
|
|
33
|
+
}
|
|
34
|
+
function parseLines(content) {
|
|
35
|
+
return content.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).map((l) => ({ value: l }));
|
|
36
|
+
}
|
|
37
|
+
function loadAndParseFile(filePath) {
|
|
38
|
+
if (!fs.existsSync(filePath)) {
|
|
39
|
+
throw new Error(`File not found: ${filePath}`);
|
|
40
|
+
}
|
|
41
|
+
const content = fs.readFileSync(filePath, "utf8");
|
|
42
|
+
const ext = filePath.split(".").pop()?.toLowerCase();
|
|
43
|
+
if (ext === "csv") {
|
|
44
|
+
return parseCSV(content);
|
|
45
|
+
} else if (ext === "tsv") {
|
|
46
|
+
return parseCSV(content, " ");
|
|
47
|
+
} else if (ext === "json" || ext === "jsonl" || ext === "ndjson") {
|
|
48
|
+
return parseJSONOrJSONL(content);
|
|
49
|
+
} else if (ext === "txt") {
|
|
50
|
+
return parseLines(content);
|
|
51
|
+
}
|
|
52
|
+
try {
|
|
53
|
+
return parseJSONOrJSONL(content);
|
|
54
|
+
} catch {
|
|
55
|
+
try {
|
|
56
|
+
return parseCSV(content);
|
|
57
|
+
} catch {
|
|
58
|
+
return parseLines(content);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// src/utils/template.ts
|
|
64
|
+
function renderTemplate(template, row) {
|
|
65
|
+
return template.replace(/\{\{\s*([a-zA-Z0-9_-]+)\s*\}\}/g, (_, key) => {
|
|
66
|
+
return row[key] !== void 0 ? String(row[key]) : "";
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
function extractRowValue(row, column, valueTemplate) {
|
|
70
|
+
if (valueTemplate) {
|
|
71
|
+
return renderTemplate(valueTemplate, row);
|
|
72
|
+
}
|
|
73
|
+
if (column && row[column] !== void 0) {
|
|
74
|
+
return String(row[column]);
|
|
75
|
+
}
|
|
76
|
+
const keys = Object.keys(row);
|
|
77
|
+
if (keys.length === 1) {
|
|
78
|
+
return String(row[keys[0]]);
|
|
79
|
+
}
|
|
80
|
+
return JSON.stringify(row);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// src/utils/k6.ts
|
|
84
|
+
function generateK6Snippet(keyName, structure) {
|
|
85
|
+
if (structure === "set") {
|
|
86
|
+
return `// ==========================================
|
|
87
|
+
// k6 Load Test - Redis Set Data Consumption
|
|
88
|
+
// Requirements: xk6-redis (k6 run --vus 50 --duration 1m script.js)
|
|
89
|
+
// ==========================================
|
|
90
|
+
import http from 'k6/http';
|
|
91
|
+
import { check, sleep } from 'k6';
|
|
92
|
+
import redis from 'k6/x/redis';
|
|
93
|
+
|
|
94
|
+
// Connect to Redis
|
|
95
|
+
const client = new redis.Client('redis://localhost:6379');
|
|
96
|
+
|
|
97
|
+
export const options = {
|
|
98
|
+
vus: 50,
|
|
99
|
+
duration: '1m',
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
export default function () {
|
|
103
|
+
// Option A: Pop unique token (guarantees NO collision between VUs)
|
|
104
|
+
const uniqueToken = client.spop('${keyName}');
|
|
105
|
+
|
|
106
|
+
// Option B: Random sample (if token is reusable across VUs)
|
|
107
|
+
// const uniqueToken = client.srandmember('${keyName}');
|
|
108
|
+
|
|
109
|
+
if (!uniqueToken) {
|
|
110
|
+
console.warn('Redis set "${keyName}" is exhausted!');
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// Execute authenticated HTTP request using seeded token
|
|
115
|
+
const res = http.get('https://api.example.com/v1/user/profile', {
|
|
116
|
+
headers: {
|
|
117
|
+
Authorization: \`Bearer \${uniqueToken}\`,
|
|
118
|
+
},
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
check(res, {
|
|
122
|
+
'status is 200': (r) => r.status === 200,
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
sleep(0.1);
|
|
126
|
+
}
|
|
127
|
+
`;
|
|
128
|
+
}
|
|
129
|
+
if (structure === "list") {
|
|
130
|
+
return `// ==========================================
|
|
131
|
+
// k6 Load Test - Redis Queue / List FIFO
|
|
132
|
+
// Requirements: xk6-redis
|
|
133
|
+
// ==========================================
|
|
134
|
+
import http from 'k6/http';
|
|
135
|
+
import { check } from 'k6';
|
|
136
|
+
import redis from 'k6/x/redis';
|
|
137
|
+
|
|
138
|
+
const client = new redis.Client('redis://localhost:6379');
|
|
139
|
+
|
|
140
|
+
export default function () {
|
|
141
|
+
// Pop from FIFO queue
|
|
142
|
+
const item = client.lpop('${keyName}');
|
|
143
|
+
|
|
144
|
+
if (!item) {
|
|
145
|
+
console.warn('Queue "${keyName}" is empty!');
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const payload = JSON.parse(item);
|
|
150
|
+
const res = http.post('https://api.example.com/v1/orders', JSON.stringify(payload), {
|
|
151
|
+
headers: { 'Content-Type': 'application/json' },
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
check(res, { 'status is 201': (r) => r.status === 201 });
|
|
155
|
+
}
|
|
156
|
+
`;
|
|
157
|
+
}
|
|
158
|
+
return `// ==========================================
|
|
159
|
+
// k6 Load Test - Redis Key-Value Fetch
|
|
160
|
+
// Requirements: xk6-redis
|
|
161
|
+
// ==========================================
|
|
162
|
+
import http from 'k6/http';
|
|
163
|
+
import { check } from 'k6';
|
|
164
|
+
import redis from 'k6/x/redis';
|
|
165
|
+
|
|
166
|
+
const client = new redis.Client('redis://localhost:6379');
|
|
167
|
+
|
|
168
|
+
export default function () {
|
|
169
|
+
const value = client.get('${keyName}');
|
|
170
|
+
const res = http.get(\`https://api.example.com/v1/item/\${value}\`);
|
|
171
|
+
check(res, { 'status is 200': (r) => r.status === 200 });
|
|
172
|
+
}
|
|
173
|
+
`;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// src/engine/client.ts
|
|
177
|
+
import { Redis } from "ioredis";
|
|
178
|
+
function createRedisClient(options = {}) {
|
|
179
|
+
const url = options.url || process.env.REDIS_URL || process.env.REDIS_URI;
|
|
180
|
+
if (url) {
|
|
181
|
+
return new Redis(url, {
|
|
182
|
+
lazyConnect: false,
|
|
183
|
+
maxRetriesPerRequest: 3,
|
|
184
|
+
enableReadyCheck: true
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
const redisConfig = {
|
|
188
|
+
host: options.host || process.env.REDIS_HOST || "127.0.0.1",
|
|
189
|
+
port: options.port || (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT, 10) : 6379),
|
|
190
|
+
password: options.password || process.env.REDIS_PASSWORD || void 0,
|
|
191
|
+
db: options.db !== void 0 ? options.db : process.env.REDIS_DB ? parseInt(process.env.REDIS_DB, 10) : 0,
|
|
192
|
+
maxRetriesPerRequest: 3,
|
|
193
|
+
lazyConnect: false
|
|
194
|
+
};
|
|
195
|
+
if (options.tls) {
|
|
196
|
+
redisConfig.tls = {};
|
|
197
|
+
}
|
|
198
|
+
return new Redis(redisConfig);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// src/engine/seeder.ts
|
|
202
|
+
async function executeSeed(redis, rows, options) {
|
|
203
|
+
const startTime = Date.now();
|
|
204
|
+
const batchSize = options.batchSize || 1e3;
|
|
205
|
+
let totalInserted = 0;
|
|
206
|
+
const uniqueKeys = /* @__PURE__ */ new Set();
|
|
207
|
+
if (options.flushFirst && options.keyName && !options.keyName.includes("{{")) {
|
|
208
|
+
await redis.del(options.keyName);
|
|
209
|
+
}
|
|
210
|
+
for (let i = 0; i < rows.length; i += batchSize) {
|
|
211
|
+
const batch = rows.slice(i, i + batchSize);
|
|
212
|
+
const pipeline = redis.pipeline();
|
|
213
|
+
for (const row of batch) {
|
|
214
|
+
const targetKey = options.keyName ? renderTemplate(options.keyName, row) : "seed:data";
|
|
215
|
+
uniqueKeys.add(targetKey);
|
|
216
|
+
const val = extractRowValue(row, options.column, options.valueTemplate);
|
|
217
|
+
switch (options.structure) {
|
|
218
|
+
case "set":
|
|
219
|
+
pipeline.sadd(targetKey, val);
|
|
220
|
+
break;
|
|
221
|
+
case "list":
|
|
222
|
+
pipeline.rpush(targetKey, val);
|
|
223
|
+
break;
|
|
224
|
+
case "string":
|
|
225
|
+
if (options.ttl) {
|
|
226
|
+
pipeline.set(targetKey, val, "EX", options.ttl);
|
|
227
|
+
} else {
|
|
228
|
+
pipeline.set(targetKey, val);
|
|
229
|
+
}
|
|
230
|
+
break;
|
|
231
|
+
case "hash":
|
|
232
|
+
const hashFields = {};
|
|
233
|
+
for (const [k, v] of Object.entries(row)) {
|
|
234
|
+
hashFields[k] = typeof v === "object" ? JSON.stringify(v) : String(v);
|
|
235
|
+
}
|
|
236
|
+
pipeline.hset(targetKey, hashFields);
|
|
237
|
+
if (options.ttl) {
|
|
238
|
+
pipeline.expire(targetKey, options.ttl);
|
|
239
|
+
}
|
|
240
|
+
break;
|
|
241
|
+
case "zset":
|
|
242
|
+
const score = options.scoreColumn && row[options.scoreColumn] !== void 0 ? parseFloat(row[options.scoreColumn]) || 0 : Date.now();
|
|
243
|
+
pipeline.zadd(targetKey, score, val);
|
|
244
|
+
break;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
await pipeline.exec();
|
|
248
|
+
totalInserted += batch.length;
|
|
249
|
+
}
|
|
250
|
+
const elapsedMs = Math.max(1, Date.now() - startTime);
|
|
251
|
+
const opsPerSec = Math.round(totalInserted / elapsedMs * 1e3);
|
|
252
|
+
return {
|
|
253
|
+
totalProcessed: rows.length,
|
|
254
|
+
totalInserted,
|
|
255
|
+
elapsedMs,
|
|
256
|
+
opsPerSec,
|
|
257
|
+
structure: options.structure,
|
|
258
|
+
keysAffected: uniqueKeys.size
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// src/engine/inspector.ts
|
|
263
|
+
async function inspectKey(redis, key, sampleCount = 5) {
|
|
264
|
+
const [type, ttl] = await Promise.all([
|
|
265
|
+
redis.type(key),
|
|
266
|
+
redis.ttl(key)
|
|
267
|
+
]);
|
|
268
|
+
let count = 0;
|
|
269
|
+
let samples = [];
|
|
270
|
+
switch (type) {
|
|
271
|
+
case "set":
|
|
272
|
+
count = await redis.scard(key);
|
|
273
|
+
samples = await redis.srandmember(key, sampleCount) || [];
|
|
274
|
+
break;
|
|
275
|
+
case "list":
|
|
276
|
+
count = await redis.llen(key);
|
|
277
|
+
samples = await redis.lrange(key, 0, sampleCount - 1) || [];
|
|
278
|
+
break;
|
|
279
|
+
case "hash":
|
|
280
|
+
count = await redis.hlen(key);
|
|
281
|
+
const hashData = await redis.hgetall(key);
|
|
282
|
+
samples = Object.entries(hashData).slice(0, sampleCount).map(([k, v]) => `${k}: ${v}`);
|
|
283
|
+
break;
|
|
284
|
+
case "zset":
|
|
285
|
+
count = await redis.zcard(key);
|
|
286
|
+
samples = await redis.zrange(key, 0, sampleCount - 1) || [];
|
|
287
|
+
break;
|
|
288
|
+
case "string":
|
|
289
|
+
count = 1;
|
|
290
|
+
const val = await redis.get(key);
|
|
291
|
+
samples = val ? [val] : [];
|
|
292
|
+
break;
|
|
293
|
+
}
|
|
294
|
+
let memoryBytes;
|
|
295
|
+
try {
|
|
296
|
+
const memory = await redis.memory("USAGE", key);
|
|
297
|
+
if (typeof memory === "number") memoryBytes = memory;
|
|
298
|
+
} catch {
|
|
299
|
+
}
|
|
300
|
+
return {
|
|
301
|
+
key,
|
|
302
|
+
type,
|
|
303
|
+
count,
|
|
304
|
+
memoryBytes,
|
|
305
|
+
ttl,
|
|
306
|
+
samples
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
export {
|
|
310
|
+
createRedisClient,
|
|
311
|
+
executeSeed,
|
|
312
|
+
extractRowValue,
|
|
313
|
+
generateK6Snippet,
|
|
314
|
+
inspectKey,
|
|
315
|
+
loadAndParseFile,
|
|
316
|
+
parseCSV,
|
|
317
|
+
parseJSONOrJSONL,
|
|
318
|
+
parseLines,
|
|
319
|
+
renderTemplate
|
|
320
|
+
};
|
|
321
|
+
//# sourceMappingURL=index.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/loaders/file.ts","../src/utils/template.ts","../src/utils/k6.ts","../src/engine/client.ts","../src/engine/seeder.ts","../src/engine/inspector.ts"],"sourcesContent":["import fs from \"fs\";\nimport { parse } from \"csv-parse/sync\";\nimport { ParsedRow } from \"../types.js\";\n\n/**\n * Parse a CSV string or file content into array of row objects.\n */\nexport function parseCSV(content: string, delimiter: string = \",\"): ParsedRow[] {\n const records = parse(content, {\n columns: true,\n skip_empty_lines: true,\n trim: true,\n delimiter,\n cast: false,\n });\n return records;\n}\n\n/**\n * Parse a JSON array, single JSON object, or JSONL string into array of row objects.\n */\nexport function parseJSONOrJSONL(content: string): ParsedRow[] {\n const trimmed = content.trim();\n if (trimmed.startsWith(\"[\") && trimmed.endsWith(\"]\")) {\n const parsed = JSON.parse(trimmed);\n if (!Array.isArray(parsed)) throw new Error(\"JSON root must be an array of objects.\");\n return parsed;\n }\n\n // Check if it is line-delimited JSON (JSONL)\n const lines = trimmed.split(/\\r?\\n/).map(l => l.trim()).filter(Boolean);\n const rows: ParsedRow[] = [];\n for (const line of lines) {\n if (line.startsWith(\"{\") && line.endsWith(\"}\")) {\n rows.push(JSON.parse(line));\n } else {\n // Plain text single column line\n rows.push({ value: line });\n }\n }\n\n return rows;\n}\n\n/**\n * Parse plain text lines into { value: line } rows.\n */\nexport function parseLines(content: string): ParsedRow[] {\n return content\n .split(/\\r?\\n/)\n .map(l => l.trim())\n .filter(Boolean)\n .map(l => ({ value: l }));\n}\n\n/**\n * Auto-detect file format from extension and contents.\n */\nexport function loadAndParseFile(filePath: string): ParsedRow[] {\n if (!fs.existsSync(filePath)) {\n throw new Error(`File not found: ${filePath}`);\n }\n\n const content = fs.readFileSync(filePath, \"utf8\");\n const ext = filePath.split(\".\").pop()?.toLowerCase();\n\n if (ext === \"csv\") {\n return parseCSV(content);\n } else if (ext === \"tsv\") {\n return parseCSV(content, \"\\t\");\n } else if (ext === \"json\" || ext === \"jsonl\" || ext === \"ndjson\") {\n return parseJSONOrJSONL(content);\n } else if (ext === \"txt\") {\n return parseLines(content);\n }\n\n // Fallback try JSON then CSV then lines\n try {\n return parseJSONOrJSONL(content);\n } catch {\n try {\n return parseCSV(content);\n } catch {\n return parseLines(content);\n }\n }\n}\n","import { ParsedRow } from \"../types.js\";\n\n/**\n * Render template string replacing `{{variable}}` with values from row.\n */\nexport function renderTemplate(template: string, row: ParsedRow): string {\n return template.replace(/\\{\\{\\s*([a-zA-Z0-9_-]+)\\s*\\}\\}/g, (_, key) => {\n return row[key] !== undefined ? String(row[key]) : \"\";\n });\n}\n\n/**\n * Extract item value from row based on column or template.\n */\nexport function extractRowValue(row: ParsedRow, column?: string, valueTemplate?: string): string {\n if (valueTemplate) {\n return renderTemplate(valueTemplate, row);\n }\n\n if (column && row[column] !== undefined) {\n return String(row[column]);\n }\n\n // If there's only 1 property in the row, use it\n const keys = Object.keys(row);\n if (keys.length === 1) {\n return String(row[keys[0]]);\n }\n\n // Otherwise return entire row as JSON string\n return JSON.stringify(row);\n}\n","import { RedisStructure } from \"../types.js\";\n\n/**\n * Generate ready-to-use k6 test script snippets using xk6-redis or HTTP mocks.\n */\nexport function generateK6Snippet(keyName: string, structure: RedisStructure): string {\n if (structure === \"set\") {\n return `// ==========================================\n// k6 Load Test - Redis Set Data Consumption\n// Requirements: xk6-redis (k6 run --vus 50 --duration 1m script.js)\n// ==========================================\nimport http from 'k6/http';\nimport { check, sleep } from 'k6';\nimport redis from 'k6/x/redis';\n\n// Connect to Redis\nconst client = new redis.Client('redis://localhost:6379');\n\nexport const options = {\n vus: 50,\n duration: '1m',\n};\n\nexport default function () {\n // Option A: Pop unique token (guarantees NO collision between VUs)\n const uniqueToken = client.spop('${keyName}');\n\n // Option B: Random sample (if token is reusable across VUs)\n // const uniqueToken = client.srandmember('${keyName}');\n\n if (!uniqueToken) {\n console.warn('Redis set \"${keyName}\" is exhausted!');\n return;\n }\n\n // Execute authenticated HTTP request using seeded token\n const res = http.get('https://api.example.com/v1/user/profile', {\n headers: {\n Authorization: \\`Bearer \\${uniqueToken}\\`,\n },\n });\n\n check(res, {\n 'status is 200': (r) => r.status === 200,\n });\n\n sleep(0.1);\n}\n`;\n }\n\n if (structure === \"list\") {\n return `// ==========================================\n// k6 Load Test - Redis Queue / List FIFO\n// Requirements: xk6-redis\n// ==========================================\nimport http from 'k6/http';\nimport { check } from 'k6';\nimport redis from 'k6/x/redis';\n\nconst client = new redis.Client('redis://localhost:6379');\n\nexport default function () {\n // Pop from FIFO queue\n const item = client.lpop('${keyName}');\n\n if (!item) {\n console.warn('Queue \"${keyName}\" is empty!');\n return;\n }\n\n const payload = JSON.parse(item);\n const res = http.post('https://api.example.com/v1/orders', JSON.stringify(payload), {\n headers: { 'Content-Type': 'application/json' },\n });\n\n check(res, { 'status is 201': (r) => r.status === 201 });\n}\n`;\n }\n\n // Default string / hash snippet\n return `// ==========================================\n// k6 Load Test - Redis Key-Value Fetch\n// Requirements: xk6-redis\n// ==========================================\nimport http from 'k6/http';\nimport { check } from 'k6';\nimport redis from 'k6/x/redis';\n\nconst client = new redis.Client('redis://localhost:6379');\n\nexport default function () {\n const value = client.get('${keyName}');\n const res = http.get(\\`https://api.example.com/v1/item/\\${value}\\`);\n check(res, { 'status is 200': (r) => r.status === 200 });\n}\n`;\n}\n","import { Redis, RedisOptions } from \"ioredis\";\nimport { RedisConnectionOptions } from \"../types.js\";\n\n/**\n * Create a reusable ioredis client instance.\n */\nexport function createRedisClient(options: RedisConnectionOptions = {}): Redis {\n const url = options.url || process.env.REDIS_URL || process.env.REDIS_URI;\n\n if (url) {\n return new Redis(url, {\n lazyConnect: false,\n maxRetriesPerRequest: 3,\n enableReadyCheck: true,\n });\n }\n\n const redisConfig: RedisOptions = {\n host: options.host || process.env.REDIS_HOST || \"127.0.0.1\",\n port: options.port || (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT, 10) : 6379),\n password: options.password || process.env.REDIS_PASSWORD || undefined,\n db: options.db !== undefined ? options.db : (process.env.REDIS_DB ? parseInt(process.env.REDIS_DB, 10) : 0),\n maxRetriesPerRequest: 3,\n lazyConnect: false,\n };\n\n if (options.tls) {\n redisConfig.tls = {};\n }\n\n return new Redis(redisConfig);\n}\n","import { Redis } from \"ioredis\";\nimport { SeedOptions, SeedResult, ParsedRow } from \"../types.js\";\nimport { renderTemplate, extractRowValue } from \"../utils/template.js\";\n\n/**\n * Execute high-speed batched seeding into Redis using pipelines.\n */\nexport async function executeSeed(\n redis: Redis,\n rows: ParsedRow[],\n options: SeedOptions\n): Promise<SeedResult> {\n const startTime = Date.now();\n const batchSize = options.batchSize || 1000;\n let totalInserted = 0;\n const uniqueKeys = new Set<string>();\n\n // If flushFirst is enabled and static keyName is provided, delete it first\n if (options.flushFirst && options.keyName && !options.keyName.includes(\"{{\")) {\n await redis.del(options.keyName);\n }\n\n // Process in batches using pipeline\n for (let i = 0; i < rows.length; i += batchSize) {\n const batch = rows.slice(i, i + batchSize);\n const pipeline = redis.pipeline();\n\n for (const row of batch) {\n const targetKey = options.keyName\n ? renderTemplate(options.keyName, row)\n : \"seed:data\";\n\n uniqueKeys.add(targetKey);\n const val = extractRowValue(row, options.column, options.valueTemplate);\n\n switch (options.structure) {\n case \"set\":\n pipeline.sadd(targetKey, val);\n break;\n\n case \"list\":\n pipeline.rpush(targetKey, val);\n break;\n\n case \"string\":\n if (options.ttl) {\n pipeline.set(targetKey, val, \"EX\", options.ttl);\n } else {\n pipeline.set(targetKey, val);\n }\n break;\n\n case \"hash\":\n // If row has multiple columns, store full hash fields\n const hashFields: Record<string, string> = {};\n for (const [k, v] of Object.entries(row)) {\n hashFields[k] = typeof v === \"object\" ? JSON.stringify(v) : String(v);\n }\n pipeline.hset(targetKey, hashFields);\n if (options.ttl) {\n pipeline.expire(targetKey, options.ttl);\n }\n break;\n\n case \"zset\":\n const score = options.scoreColumn && row[options.scoreColumn] !== undefined\n ? parseFloat(row[options.scoreColumn]) || 0\n : Date.now();\n pipeline.zadd(targetKey, score, val);\n break;\n }\n }\n\n await pipeline.exec();\n totalInserted += batch.length;\n }\n\n const elapsedMs = Math.max(1, Date.now() - startTime);\n const opsPerSec = Math.round((totalInserted / elapsedMs) * 1000);\n\n return {\n totalProcessed: rows.length,\n totalInserted,\n elapsedMs,\n opsPerSec,\n structure: options.structure,\n keysAffected: uniqueKeys.size,\n };\n}\n","import { Redis } from \"ioredis\";\n\nexport interface KeyInspection {\n key: string;\n type: string;\n count: number;\n memoryBytes?: number;\n ttl: number;\n samples: string[];\n}\n\n/**\n * Inspect a Redis key (count, type, memory, and sample items).\n */\nexport async function inspectKey(redis: Redis, key: string, sampleCount: number = 5): Promise<KeyInspection> {\n const [type, ttl] = await Promise.all([\n redis.type(key),\n redis.ttl(key),\n ]);\n\n let count = 0;\n let samples: string[] = [];\n\n switch (type) {\n case \"set\":\n count = await redis.scard(key);\n samples = await redis.srandmember(key, sampleCount) || [];\n break;\n\n case \"list\":\n count = await redis.llen(key);\n samples = await redis.lrange(key, 0, sampleCount - 1) || [];\n break;\n\n case \"hash\":\n count = await redis.hlen(key);\n const hashData = await redis.hgetall(key);\n samples = Object.entries(hashData).slice(0, sampleCount).map(([k, v]) => `${k}: ${v}`);\n break;\n\n case \"zset\":\n count = await redis.zcard(key);\n samples = await redis.zrange(key, 0, sampleCount - 1) || [];\n break;\n\n case \"string\":\n count = 1;\n const val = await redis.get(key);\n samples = val ? [val] : [];\n break;\n }\n\n let memoryBytes: number | undefined;\n try {\n const memory = await redis.memory(\"USAGE\", key) as number;\n if (typeof memory === \"number\") memoryBytes = memory;\n } catch {\n // MEMORY USAGE might not be supported on all Redis clusters/proxies\n }\n\n return {\n key,\n type,\n count,\n memoryBytes,\n ttl,\n samples,\n };\n}\n"],"mappings":";;;AAAA,OAAO,QAAQ;AACf,SAAS,aAAa;AAMf,SAAS,SAAS,SAAiB,YAAoB,KAAkB;AAC9E,QAAM,UAAU,MAAM,SAAS;AAAA,IAC7B,SAAS;AAAA,IACT,kBAAkB;AAAA,IAClB,MAAM;AAAA,IACN;AAAA,IACA,MAAM;AAAA,EACR,CAAC;AACD,SAAO;AACT;AAKO,SAAS,iBAAiB,SAA8B;AAC7D,QAAM,UAAU,QAAQ,KAAK;AAC7B,MAAI,QAAQ,WAAW,GAAG,KAAK,QAAQ,SAAS,GAAG,GAAG;AACpD,UAAM,SAAS,KAAK,MAAM,OAAO;AACjC,QAAI,CAAC,MAAM,QAAQ,MAAM,EAAG,OAAM,IAAI,MAAM,wCAAwC;AACpF,WAAO;AAAA,EACT;AAGA,QAAM,QAAQ,QAAQ,MAAM,OAAO,EAAE,IAAI,OAAK,EAAE,KAAK,CAAC,EAAE,OAAO,OAAO;AACtE,QAAM,OAAoB,CAAC;AAC3B,aAAW,QAAQ,OAAO;AACxB,QAAI,KAAK,WAAW,GAAG,KAAK,KAAK,SAAS,GAAG,GAAG;AAC9C,WAAK,KAAK,KAAK,MAAM,IAAI,CAAC;AAAA,IAC5B,OAAO;AAEL,WAAK,KAAK,EAAE,OAAO,KAAK,CAAC;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AACT;AAKO,SAAS,WAAW,SAA8B;AACvD,SAAO,QACJ,MAAM,OAAO,EACb,IAAI,OAAK,EAAE,KAAK,CAAC,EACjB,OAAO,OAAO,EACd,IAAI,QAAM,EAAE,OAAO,EAAE,EAAE;AAC5B;AAKO,SAAS,iBAAiB,UAA+B;AAC9D,MAAI,CAAC,GAAG,WAAW,QAAQ,GAAG;AAC5B,UAAM,IAAI,MAAM,mBAAmB,QAAQ,EAAE;AAAA,EAC/C;AAEA,QAAM,UAAU,GAAG,aAAa,UAAU,MAAM;AAChD,QAAM,MAAM,SAAS,MAAM,GAAG,EAAE,IAAI,GAAG,YAAY;AAEnD,MAAI,QAAQ,OAAO;AACjB,WAAO,SAAS,OAAO;AAAA,EACzB,WAAW,QAAQ,OAAO;AACxB,WAAO,SAAS,SAAS,GAAI;AAAA,EAC/B,WAAW,QAAQ,UAAU,QAAQ,WAAW,QAAQ,UAAU;AAChE,WAAO,iBAAiB,OAAO;AAAA,EACjC,WAAW,QAAQ,OAAO;AACxB,WAAO,WAAW,OAAO;AAAA,EAC3B;AAGA,MAAI;AACF,WAAO,iBAAiB,OAAO;AAAA,EACjC,QAAQ;AACN,QAAI;AACF,aAAO,SAAS,OAAO;AAAA,IACzB,QAAQ;AACN,aAAO,WAAW,OAAO;AAAA,IAC3B;AAAA,EACF;AACF;;;ACjFO,SAAS,eAAe,UAAkB,KAAwB;AACvE,SAAO,SAAS,QAAQ,mCAAmC,CAAC,GAAG,QAAQ;AACrE,WAAO,IAAI,GAAG,MAAM,SAAY,OAAO,IAAI,GAAG,CAAC,IAAI;AAAA,EACrD,CAAC;AACH;AAKO,SAAS,gBAAgB,KAAgB,QAAiB,eAAgC;AAC/F,MAAI,eAAe;AACjB,WAAO,eAAe,eAAe,GAAG;AAAA,EAC1C;AAEA,MAAI,UAAU,IAAI,MAAM,MAAM,QAAW;AACvC,WAAO,OAAO,IAAI,MAAM,CAAC;AAAA,EAC3B;AAGA,QAAM,OAAO,OAAO,KAAK,GAAG;AAC5B,MAAI,KAAK,WAAW,GAAG;AACrB,WAAO,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC;AAAA,EAC5B;AAGA,SAAO,KAAK,UAAU,GAAG;AAC3B;;;AC1BO,SAAS,kBAAkB,SAAiB,WAAmC;AACpF,MAAI,cAAc,OAAO;AACvB,WAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,qCAkB0B,OAAO;AAAA;AAAA;AAAA,+CAGG,OAAO;AAAA;AAAA;AAAA,+BAGvB,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAkBpC;AAEA,MAAI,cAAc,QAAQ;AACxB,WAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,8BAYmB,OAAO;AAAA;AAAA;AAAA,2BAGV,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYhC;AAGA,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,8BAWqB,OAAO;AAAA;AAAA;AAAA;AAAA;AAKrC;;;AClGA,SAAS,aAA2B;AAM7B,SAAS,kBAAkB,UAAkC,CAAC,GAAU;AAC7E,QAAM,MAAM,QAAQ,OAAO,QAAQ,IAAI,aAAa,QAAQ,IAAI;AAEhE,MAAI,KAAK;AACP,WAAO,IAAI,MAAM,KAAK;AAAA,MACpB,aAAa;AAAA,MACb,sBAAsB;AAAA,MACtB,kBAAkB;AAAA,IACpB,CAAC;AAAA,EACH;AAEA,QAAM,cAA4B;AAAA,IAChC,MAAM,QAAQ,QAAQ,QAAQ,IAAI,cAAc;AAAA,IAChD,MAAM,QAAQ,SAAS,QAAQ,IAAI,aAAa,SAAS,QAAQ,IAAI,YAAY,EAAE,IAAI;AAAA,IACvF,UAAU,QAAQ,YAAY,QAAQ,IAAI,kBAAkB;AAAA,IAC5D,IAAI,QAAQ,OAAO,SAAY,QAAQ,KAAM,QAAQ,IAAI,WAAW,SAAS,QAAQ,IAAI,UAAU,EAAE,IAAI;AAAA,IACzG,sBAAsB;AAAA,IACtB,aAAa;AAAA,EACf;AAEA,MAAI,QAAQ,KAAK;AACf,gBAAY,MAAM,CAAC;AAAA,EACrB;AAEA,SAAO,IAAI,MAAM,WAAW;AAC9B;;;ACxBA,eAAsB,YACpB,OACA,MACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,YAAY,QAAQ,aAAa;AACvC,MAAI,gBAAgB;AACpB,QAAM,aAAa,oBAAI,IAAY;AAGnC,MAAI,QAAQ,cAAc,QAAQ,WAAW,CAAC,QAAQ,QAAQ,SAAS,IAAI,GAAG;AAC5E,UAAM,MAAM,IAAI,QAAQ,OAAO;AAAA,EACjC;AAGA,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK,WAAW;AAC/C,UAAM,QAAQ,KAAK,MAAM,GAAG,IAAI,SAAS;AACzC,UAAM,WAAW,MAAM,SAAS;AAEhC,eAAW,OAAO,OAAO;AACvB,YAAM,YAAY,QAAQ,UACtB,eAAe,QAAQ,SAAS,GAAG,IACnC;AAEJ,iBAAW,IAAI,SAAS;AACxB,YAAM,MAAM,gBAAgB,KAAK,QAAQ,QAAQ,QAAQ,aAAa;AAEtE,cAAQ,QAAQ,WAAW;AAAA,QACzB,KAAK;AACH,mBAAS,KAAK,WAAW,GAAG;AAC5B;AAAA,QAEF,KAAK;AACH,mBAAS,MAAM,WAAW,GAAG;AAC7B;AAAA,QAEF,KAAK;AACH,cAAI,QAAQ,KAAK;AACf,qBAAS,IAAI,WAAW,KAAK,MAAM,QAAQ,GAAG;AAAA,UAChD,OAAO;AACL,qBAAS,IAAI,WAAW,GAAG;AAAA,UAC7B;AACA;AAAA,QAEF,KAAK;AAEH,gBAAM,aAAqC,CAAC;AAC5C,qBAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,GAAG,GAAG;AACxC,uBAAW,CAAC,IAAI,OAAO,MAAM,WAAW,KAAK,UAAU,CAAC,IAAI,OAAO,CAAC;AAAA,UACtE;AACA,mBAAS,KAAK,WAAW,UAAU;AACnC,cAAI,QAAQ,KAAK;AACf,qBAAS,OAAO,WAAW,QAAQ,GAAG;AAAA,UACxC;AACA;AAAA,QAEF,KAAK;AACH,gBAAM,QAAQ,QAAQ,eAAe,IAAI,QAAQ,WAAW,MAAM,SAC9D,WAAW,IAAI,QAAQ,WAAW,CAAC,KAAK,IACxC,KAAK,IAAI;AACb,mBAAS,KAAK,WAAW,OAAO,GAAG;AACnC;AAAA,MACJ;AAAA,IACF;AAEA,UAAM,SAAS,KAAK;AACpB,qBAAiB,MAAM;AAAA,EACzB;AAEA,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,IAAI,SAAS;AACpD,QAAM,YAAY,KAAK,MAAO,gBAAgB,YAAa,GAAI;AAE/D,SAAO;AAAA,IACL,gBAAgB,KAAK;AAAA,IACrB;AAAA,IACA;AAAA,IACA;AAAA,IACA,WAAW,QAAQ;AAAA,IACnB,cAAc,WAAW;AAAA,EAC3B;AACF;;;AC1EA,eAAsB,WAAW,OAAc,KAAa,cAAsB,GAA2B;AAC3G,QAAM,CAAC,MAAM,GAAG,IAAI,MAAM,QAAQ,IAAI;AAAA,IACpC,MAAM,KAAK,GAAG;AAAA,IACd,MAAM,IAAI,GAAG;AAAA,EACf,CAAC;AAED,MAAI,QAAQ;AACZ,MAAI,UAAoB,CAAC;AAEzB,UAAQ,MAAM;AAAA,IACZ,KAAK;AACH,cAAQ,MAAM,MAAM,MAAM,GAAG;AAC7B,gBAAU,MAAM,MAAM,YAAY,KAAK,WAAW,KAAK,CAAC;AACxD;AAAA,IAEF,KAAK;AACH,cAAQ,MAAM,MAAM,KAAK,GAAG;AAC5B,gBAAU,MAAM,MAAM,OAAO,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC;AAC1D;AAAA,IAEF,KAAK;AACH,cAAQ,MAAM,MAAM,KAAK,GAAG;AAC5B,YAAM,WAAW,MAAM,MAAM,QAAQ,GAAG;AACxC,gBAAU,OAAO,QAAQ,QAAQ,EAAE,MAAM,GAAG,WAAW,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,EAAE;AACrF;AAAA,IAEF,KAAK;AACH,cAAQ,MAAM,MAAM,MAAM,GAAG;AAC7B,gBAAU,MAAM,MAAM,OAAO,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC;AAC1D;AAAA,IAEF,KAAK;AACH,cAAQ;AACR,YAAM,MAAM,MAAM,MAAM,IAAI,GAAG;AAC/B,gBAAU,MAAM,CAAC,GAAG,IAAI,CAAC;AACzB;AAAA,EACJ;AAEA,MAAI;AACJ,MAAI;AACF,UAAM,SAAS,MAAM,MAAM,OAAO,SAAS,GAAG;AAC9C,QAAI,OAAO,WAAW,SAAU,eAAc;AAAA,EAChD,QAAQ;AAAA,EAER;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;","names":[]}
|
package/package.json
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "rediseed",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Blazing-fast Redis data seeder & loader from CSV, JSON, and line streams for k6 load testing, staging fixtures, and queue populating.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"redis",
|
|
7
|
+
"k6",
|
|
8
|
+
"load-testing",
|
|
9
|
+
"seeder",
|
|
10
|
+
"csv-to-redis",
|
|
11
|
+
"json-to-redis",
|
|
12
|
+
"xk6-redis",
|
|
13
|
+
"fixtures",
|
|
14
|
+
"performance-testing",
|
|
15
|
+
"cli"
|
|
16
|
+
],
|
|
17
|
+
"author": "authoritydmc",
|
|
18
|
+
"license": "MIT",
|
|
19
|
+
"repository": {
|
|
20
|
+
"type": "git",
|
|
21
|
+
"url": "git+https://github.com/authoritydmc/rediseed.git"
|
|
22
|
+
},
|
|
23
|
+
"bugs": {
|
|
24
|
+
"url": "https://github.com/authoritydmc/rediseed/issues"
|
|
25
|
+
},
|
|
26
|
+
"homepage": "https://github.com/authoritydmc/rediseed#readme",
|
|
27
|
+
"main": "./dist/index.js",
|
|
28
|
+
"module": "./dist/index.mjs",
|
|
29
|
+
"types": "./dist/index.d.ts",
|
|
30
|
+
"exports": {
|
|
31
|
+
".": {
|
|
32
|
+
"types": "./dist/index.d.ts",
|
|
33
|
+
"import": "./dist/index.mjs",
|
|
34
|
+
"require": "./dist/index.js"
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
"bin": {
|
|
38
|
+
"rediseed": "./dist/cli.js",
|
|
39
|
+
"seedx": "./dist/cli.js",
|
|
40
|
+
"redisx": "./dist/cli.js"
|
|
41
|
+
},
|
|
42
|
+
"files": [
|
|
43
|
+
"dist",
|
|
44
|
+
"README.md",
|
|
45
|
+
"LICENSE"
|
|
46
|
+
],
|
|
47
|
+
"scripts": {
|
|
48
|
+
"build": "tsup",
|
|
49
|
+
"dev": "tsup --watch",
|
|
50
|
+
"test": "tsx --test src/test/rediseed.test.ts",
|
|
51
|
+
"lint": "eslint src/ --ext .ts",
|
|
52
|
+
"prepublishOnly": "npm run build && npm test",
|
|
53
|
+
"release:patch": "npm version patch && git push origin main --follow-tags",
|
|
54
|
+
"release:minor": "npm version minor && git push origin main --follow-tags",
|
|
55
|
+
"release:major": "npm version major && git push origin main --follow-tags"
|
|
56
|
+
},
|
|
57
|
+
"dependencies": {
|
|
58
|
+
"commander": "^12.1.0",
|
|
59
|
+
"csv-parse": "^5.5.6",
|
|
60
|
+
"ioredis": "^5.4.1",
|
|
61
|
+
"picocolors": "^1.1.0"
|
|
62
|
+
},
|
|
63
|
+
"devDependencies": {
|
|
64
|
+
"@types/node": "^20.14.0",
|
|
65
|
+
"tsup": "^8.2.4",
|
|
66
|
+
"tsx": "^4.19.0",
|
|
67
|
+
"typescript": "^5.5.4"
|
|
68
|
+
},
|
|
69
|
+
"engines": {
|
|
70
|
+
"node": ">=18.0.0"
|
|
71
|
+
}
|
|
72
|
+
}
|