rediseed 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +165 -0
- package/dist/cli.d.mts +2 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +463 -0
- package/dist/cli.js.map +1 -0
- package/dist/cli.mjs +440 -0
- package/dist/cli.mjs.map +1 -0
- package/dist/index.d.mts +89 -0
- package/dist/index.d.ts +89 -0
- package/dist/index.js +366 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +321 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +72 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 authoritydmc
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# ⚡ rediseed (`seedx` / `redisx`)
|
|
2
|
+
|
|
3
|
+
> **Blazing-fast Redis data seeder & loader from CSV, JSON, and line streams.**
|
|
4
|
+
> Built for k6 performance testing, unique token popping (`SPOP`), queue fixture populating, and instant Redis inspection.
|
|
5
|
+
|
|
6
|
+
[](https://www.npmjs.com/package/rediseed)
|
|
7
|
+
[](https://opensource.org/licenses/MIT)
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
## 🚀 Key Features
|
|
12
|
+
|
|
13
|
+
- 🏎️ **High Throughput**: Pipelined batch insertion processing 100k+ records/sec.
|
|
14
|
+
- 🎯 **Load-Testing Ready (k6 / Locust / JMeter)**: Pre-seed test users, auth tokens, or session IDs into Redis Sets for collision-free `SPOP` or `SRANDMEMBER` consumption.
|
|
15
|
+
- 📁 **Universal Input Formats**: Auto-detects and loads **CSV**, **TSV**, **JSON arrays**, **JSONL / NDJSON**, and plaintext line streams.
|
|
16
|
+
- 🗄️ **All Redis Structures Supported**:
|
|
17
|
+
- **Sets (`SADD`)**: Collision-free unique user/token pools.
|
|
18
|
+
- **Lists (`RPUSH` / `LPUSH`)**: FIFO order queues & worker queues.
|
|
19
|
+
- **Hashes (`HSET`)**: Structured records indexed by primary key column.
|
|
20
|
+
- **Sorted Sets (`ZADD`)**: Prioritized or timestamp-ordered items.
|
|
21
|
+
- **String Keys (`SET`)**: Individual key-values with optional TTL expiration.
|
|
22
|
+
- 🧩 **Dynamic Key & Value Templating**: Define schema patterns like `--key "user:{{tenant_id}}:{{user_id}}"` or `--value "{{email}}:{{token}}"`.
|
|
23
|
+
- 🔍 **Inspect & Verify**: Check key data types, card/lengths, memory usage, and preview sample items in the terminal.
|
|
24
|
+
- 📝 **k6 Snippet Generator**: Instantly generate ready-to-run `xk6-redis` load testing scripts.
|
|
25
|
+
|
|
26
|
+
---
|
|
27
|
+
|
|
28
|
+
## 📦 Installation
|
|
29
|
+
|
|
30
|
+
Run instantly via `npx` or install globally:
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
# Run directly with npx
|
|
34
|
+
npx rediseed --help
|
|
35
|
+
npx seedx --help
|
|
36
|
+
npx redisx --help
|
|
37
|
+
|
|
38
|
+
# Or install globally
|
|
39
|
+
npm install -g rediseed
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## ⚡ Quick Usage Examples
|
|
45
|
+
|
|
46
|
+
### 1. 🎯 Load Test User Tokens into a Redis Set (for k6 `SPOP`)
|
|
47
|
+
|
|
48
|
+
Given a `users.csv` file:
|
|
49
|
+
```csv
|
|
50
|
+
id,username,token
|
|
51
|
+
1,alice,tok_live_948172
|
|
52
|
+
2,bob,tok_live_109283
|
|
53
|
+
3,charlie,tok_live_771920
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Load all tokens into a Redis set `k6:auth_tokens`:
|
|
57
|
+
```bash
|
|
58
|
+
# Using the shortcut set command
|
|
59
|
+
seedx set k6:auth_tokens users.csv --column token --flush
|
|
60
|
+
|
|
61
|
+
# Or with custom Redis URL
|
|
62
|
+
seedx set k6:auth_tokens users.csv --column token --url redis://:password@10.0.0.1:6379/0
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
### 2. 🧩 Seed Hash Records with Dynamic Key Templating
|
|
68
|
+
|
|
69
|
+
Seed each row as an individual Redis Hash keyed by user ID:
|
|
70
|
+
```bash
|
|
71
|
+
seedx seed users.csv --structure hash --key "app:users:{{id}}" --flush
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
---
|
|
75
|
+
|
|
76
|
+
### 3. 🔍 Inspect Seeded Redis Keys
|
|
77
|
+
|
|
78
|
+
Verify the count, data type, memory footprint, and preview sample items:
|
|
79
|
+
```bash
|
|
80
|
+
seedx inspect k6:auth_tokens
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Output:
|
|
84
|
+
```text
|
|
85
|
+
🔍 Redis Key Inspector
|
|
86
|
+
• Key: k6:auth_tokens
|
|
87
|
+
• Type: SET
|
|
88
|
+
• Total Items: 50,000
|
|
89
|
+
• TTL: No Expiration
|
|
90
|
+
• Memory Usage: 412.50 KB
|
|
91
|
+
|
|
92
|
+
Sample Items:
|
|
93
|
+
• tok_live_948172
|
|
94
|
+
• tok_live_109283
|
|
95
|
+
• tok_live_771920
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
---
|
|
99
|
+
|
|
100
|
+
### 4. 📝 Generate k6 Load Test Script
|
|
101
|
+
|
|
102
|
+
Generate a ready-to-run k6 test script consuming the seeded tokens via `xk6-redis`:
|
|
103
|
+
```bash
|
|
104
|
+
seedx k6 k6:auth_tokens --structure set
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Output:
|
|
108
|
+
```javascript
|
|
109
|
+
import http from 'k6/http';
|
|
110
|
+
import { check, sleep } from 'k6';
|
|
111
|
+
import redis from 'k6/x/redis';
|
|
112
|
+
|
|
113
|
+
const client = new redis.Client('redis://localhost:6379');
|
|
114
|
+
|
|
115
|
+
export const options = {
|
|
116
|
+
vus: 50,
|
|
117
|
+
duration: '1m',
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
export default function () {
|
|
121
|
+
// Pop unique token (guarantees NO collision between VUs)
|
|
122
|
+
const uniqueToken = client.spop('k6:auth_tokens');
|
|
123
|
+
|
|
124
|
+
if (!uniqueToken) {
|
|
125
|
+
console.warn('Tokens exhausted!');
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const res = http.get('https://api.example.com/v1/profile', {
|
|
130
|
+
headers: { Authorization: `Bearer ${uniqueToken}` },
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
check(res, { 'status is 200': (r) => r.status === 200 });
|
|
134
|
+
sleep(0.1);
|
|
135
|
+
}
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
---
|
|
139
|
+
|
|
140
|
+
## 🛠️ CLI Options Reference
|
|
141
|
+
|
|
142
|
+
```text
|
|
143
|
+
Usage: rediseed [options] [command]
|
|
144
|
+
|
|
145
|
+
Options:
|
|
146
|
+
-v, --version Output current version
|
|
147
|
+
-u, --url <url> Redis connection URL (redis:// or rediss://)
|
|
148
|
+
-h, --host <host> Redis host (default: "127.0.0.1")
|
|
149
|
+
-p, --port <port> Redis port (default: "6379")
|
|
150
|
+
-a, --password <password> Redis password
|
|
151
|
+
-n, --db <db> Redis database index (0-15)
|
|
152
|
+
--tls Enable TLS connection
|
|
153
|
+
|
|
154
|
+
Commands:
|
|
155
|
+
seed [options] <file> Seed CSV, JSON, or line file into Redis
|
|
156
|
+
set [options] <key> <file> Shortcut to load column into Redis Set
|
|
157
|
+
inspect [options] <key> Inspect type, item count, memory & samples
|
|
158
|
+
k6 [options] <key> Generate ready-to-run k6 test script
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
## 🛡️ License
|
|
164
|
+
|
|
165
|
+
MIT © [authoritydmc](https://github.com/authoritydmc)
|
package/dist/cli.d.mts
ADDED
package/dist/cli.d.ts
ADDED
package/dist/cli.js
ADDED
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
var __create = Object.create;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
6
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
7
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
8
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
9
|
+
var __copyProps = (to, from, except, desc) => {
|
|
10
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
11
|
+
for (let key of __getOwnPropNames(from))
|
|
12
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
13
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
14
|
+
}
|
|
15
|
+
return to;
|
|
16
|
+
};
|
|
17
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
18
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
19
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
20
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
21
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
22
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
23
|
+
mod
|
|
24
|
+
));
|
|
25
|
+
|
|
26
|
+
// src/cli.ts
|
|
27
|
+
var import_commander = require("commander");
|
|
28
|
+
var import_picocolors = __toESM(require("picocolors"));
|
|
29
|
+
|
|
30
|
+
// src/loaders/file.ts
|
|
31
|
+
var import_fs = __toESM(require("fs"));
|
|
32
|
+
var import_sync = require("csv-parse/sync");
|
|
33
|
+
function parseCSV(content, delimiter = ",") {
|
|
34
|
+
const records = (0, import_sync.parse)(content, {
|
|
35
|
+
columns: true,
|
|
36
|
+
skip_empty_lines: true,
|
|
37
|
+
trim: true,
|
|
38
|
+
delimiter,
|
|
39
|
+
cast: false
|
|
40
|
+
});
|
|
41
|
+
return records;
|
|
42
|
+
}
|
|
43
|
+
function parseJSONOrJSONL(content) {
|
|
44
|
+
const trimmed = content.trim();
|
|
45
|
+
if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
|
|
46
|
+
const parsed = JSON.parse(trimmed);
|
|
47
|
+
if (!Array.isArray(parsed)) throw new Error("JSON root must be an array of objects.");
|
|
48
|
+
return parsed;
|
|
49
|
+
}
|
|
50
|
+
const lines = trimmed.split(/\r?\n/).map((l) => l.trim()).filter(Boolean);
|
|
51
|
+
const rows = [];
|
|
52
|
+
for (const line of lines) {
|
|
53
|
+
if (line.startsWith("{") && line.endsWith("}")) {
|
|
54
|
+
rows.push(JSON.parse(line));
|
|
55
|
+
} else {
|
|
56
|
+
rows.push({ value: line });
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return rows;
|
|
60
|
+
}
|
|
61
|
+
function parseLines(content) {
|
|
62
|
+
return content.split(/\r?\n/).map((l) => l.trim()).filter(Boolean).map((l) => ({ value: l }));
|
|
63
|
+
}
|
|
64
|
+
function loadAndParseFile(filePath) {
|
|
65
|
+
if (!import_fs.default.existsSync(filePath)) {
|
|
66
|
+
throw new Error(`File not found: ${filePath}`);
|
|
67
|
+
}
|
|
68
|
+
const content = import_fs.default.readFileSync(filePath, "utf8");
|
|
69
|
+
const ext = filePath.split(".").pop()?.toLowerCase();
|
|
70
|
+
if (ext === "csv") {
|
|
71
|
+
return parseCSV(content);
|
|
72
|
+
} else if (ext === "tsv") {
|
|
73
|
+
return parseCSV(content, " ");
|
|
74
|
+
} else if (ext === "json" || ext === "jsonl" || ext === "ndjson") {
|
|
75
|
+
return parseJSONOrJSONL(content);
|
|
76
|
+
} else if (ext === "txt") {
|
|
77
|
+
return parseLines(content);
|
|
78
|
+
}
|
|
79
|
+
try {
|
|
80
|
+
return parseJSONOrJSONL(content);
|
|
81
|
+
} catch {
|
|
82
|
+
try {
|
|
83
|
+
return parseCSV(content);
|
|
84
|
+
} catch {
|
|
85
|
+
return parseLines(content);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// src/engine/client.ts
|
|
91
|
+
var import_ioredis = require("ioredis");
|
|
92
|
+
function createRedisClient(options = {}) {
|
|
93
|
+
const url = options.url || process.env.REDIS_URL || process.env.REDIS_URI;
|
|
94
|
+
if (url) {
|
|
95
|
+
return new import_ioredis.Redis(url, {
|
|
96
|
+
lazyConnect: false,
|
|
97
|
+
maxRetriesPerRequest: 3,
|
|
98
|
+
enableReadyCheck: true
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
const redisConfig = {
|
|
102
|
+
host: options.host || process.env.REDIS_HOST || "127.0.0.1",
|
|
103
|
+
port: options.port || (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT, 10) : 6379),
|
|
104
|
+
password: options.password || process.env.REDIS_PASSWORD || void 0,
|
|
105
|
+
db: options.db !== void 0 ? options.db : process.env.REDIS_DB ? parseInt(process.env.REDIS_DB, 10) : 0,
|
|
106
|
+
maxRetriesPerRequest: 3,
|
|
107
|
+
lazyConnect: false
|
|
108
|
+
};
|
|
109
|
+
if (options.tls) {
|
|
110
|
+
redisConfig.tls = {};
|
|
111
|
+
}
|
|
112
|
+
return new import_ioredis.Redis(redisConfig);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// src/utils/template.ts
|
|
116
|
+
function renderTemplate(template, row) {
|
|
117
|
+
return template.replace(/\{\{\s*([a-zA-Z0-9_-]+)\s*\}\}/g, (_, key) => {
|
|
118
|
+
return row[key] !== void 0 ? String(row[key]) : "";
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
function extractRowValue(row, column, valueTemplate) {
|
|
122
|
+
if (valueTemplate) {
|
|
123
|
+
return renderTemplate(valueTemplate, row);
|
|
124
|
+
}
|
|
125
|
+
if (column && row[column] !== void 0) {
|
|
126
|
+
return String(row[column]);
|
|
127
|
+
}
|
|
128
|
+
const keys = Object.keys(row);
|
|
129
|
+
if (keys.length === 1) {
|
|
130
|
+
return String(row[keys[0]]);
|
|
131
|
+
}
|
|
132
|
+
return JSON.stringify(row);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// src/engine/seeder.ts
|
|
136
|
+
async function executeSeed(redis, rows, options) {
|
|
137
|
+
const startTime = Date.now();
|
|
138
|
+
const batchSize = options.batchSize || 1e3;
|
|
139
|
+
let totalInserted = 0;
|
|
140
|
+
const uniqueKeys = /* @__PURE__ */ new Set();
|
|
141
|
+
if (options.flushFirst && options.keyName && !options.keyName.includes("{{")) {
|
|
142
|
+
await redis.del(options.keyName);
|
|
143
|
+
}
|
|
144
|
+
for (let i = 0; i < rows.length; i += batchSize) {
|
|
145
|
+
const batch = rows.slice(i, i + batchSize);
|
|
146
|
+
const pipeline = redis.pipeline();
|
|
147
|
+
for (const row of batch) {
|
|
148
|
+
const targetKey = options.keyName ? renderTemplate(options.keyName, row) : "seed:data";
|
|
149
|
+
uniqueKeys.add(targetKey);
|
|
150
|
+
const val = extractRowValue(row, options.column, options.valueTemplate);
|
|
151
|
+
switch (options.structure) {
|
|
152
|
+
case "set":
|
|
153
|
+
pipeline.sadd(targetKey, val);
|
|
154
|
+
break;
|
|
155
|
+
case "list":
|
|
156
|
+
pipeline.rpush(targetKey, val);
|
|
157
|
+
break;
|
|
158
|
+
case "string":
|
|
159
|
+
if (options.ttl) {
|
|
160
|
+
pipeline.set(targetKey, val, "EX", options.ttl);
|
|
161
|
+
} else {
|
|
162
|
+
pipeline.set(targetKey, val);
|
|
163
|
+
}
|
|
164
|
+
break;
|
|
165
|
+
case "hash":
|
|
166
|
+
const hashFields = {};
|
|
167
|
+
for (const [k, v] of Object.entries(row)) {
|
|
168
|
+
hashFields[k] = typeof v === "object" ? JSON.stringify(v) : String(v);
|
|
169
|
+
}
|
|
170
|
+
pipeline.hset(targetKey, hashFields);
|
|
171
|
+
if (options.ttl) {
|
|
172
|
+
pipeline.expire(targetKey, options.ttl);
|
|
173
|
+
}
|
|
174
|
+
break;
|
|
175
|
+
case "zset":
|
|
176
|
+
const score = options.scoreColumn && row[options.scoreColumn] !== void 0 ? parseFloat(row[options.scoreColumn]) || 0 : Date.now();
|
|
177
|
+
pipeline.zadd(targetKey, score, val);
|
|
178
|
+
break;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
await pipeline.exec();
|
|
182
|
+
totalInserted += batch.length;
|
|
183
|
+
}
|
|
184
|
+
const elapsedMs = Math.max(1, Date.now() - startTime);
|
|
185
|
+
const opsPerSec = Math.round(totalInserted / elapsedMs * 1e3);
|
|
186
|
+
return {
|
|
187
|
+
totalProcessed: rows.length,
|
|
188
|
+
totalInserted,
|
|
189
|
+
elapsedMs,
|
|
190
|
+
opsPerSec,
|
|
191
|
+
structure: options.structure,
|
|
192
|
+
keysAffected: uniqueKeys.size
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// src/engine/inspector.ts
|
|
197
|
+
async function inspectKey(redis, key, sampleCount = 5) {
|
|
198
|
+
const [type, ttl] = await Promise.all([
|
|
199
|
+
redis.type(key),
|
|
200
|
+
redis.ttl(key)
|
|
201
|
+
]);
|
|
202
|
+
let count = 0;
|
|
203
|
+
let samples = [];
|
|
204
|
+
switch (type) {
|
|
205
|
+
case "set":
|
|
206
|
+
count = await redis.scard(key);
|
|
207
|
+
samples = await redis.srandmember(key, sampleCount) || [];
|
|
208
|
+
break;
|
|
209
|
+
case "list":
|
|
210
|
+
count = await redis.llen(key);
|
|
211
|
+
samples = await redis.lrange(key, 0, sampleCount - 1) || [];
|
|
212
|
+
break;
|
|
213
|
+
case "hash":
|
|
214
|
+
count = await redis.hlen(key);
|
|
215
|
+
const hashData = await redis.hgetall(key);
|
|
216
|
+
samples = Object.entries(hashData).slice(0, sampleCount).map(([k, v]) => `${k}: ${v}`);
|
|
217
|
+
break;
|
|
218
|
+
case "zset":
|
|
219
|
+
count = await redis.zcard(key);
|
|
220
|
+
samples = await redis.zrange(key, 0, sampleCount - 1) || [];
|
|
221
|
+
break;
|
|
222
|
+
case "string":
|
|
223
|
+
count = 1;
|
|
224
|
+
const val = await redis.get(key);
|
|
225
|
+
samples = val ? [val] : [];
|
|
226
|
+
break;
|
|
227
|
+
}
|
|
228
|
+
let memoryBytes;
|
|
229
|
+
try {
|
|
230
|
+
const memory = await redis.memory("USAGE", key);
|
|
231
|
+
if (typeof memory === "number") memoryBytes = memory;
|
|
232
|
+
} catch {
|
|
233
|
+
}
|
|
234
|
+
return {
|
|
235
|
+
key,
|
|
236
|
+
type,
|
|
237
|
+
count,
|
|
238
|
+
memoryBytes,
|
|
239
|
+
ttl,
|
|
240
|
+
samples
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// src/utils/k6.ts
|
|
245
|
+
function generateK6Snippet(keyName, structure) {
|
|
246
|
+
if (structure === "set") {
|
|
247
|
+
return `// ==========================================
|
|
248
|
+
// k6 Load Test - Redis Set Data Consumption
|
|
249
|
+
// Requirements: xk6-redis (k6 run --vus 50 --duration 1m script.js)
|
|
250
|
+
// ==========================================
|
|
251
|
+
import http from 'k6/http';
|
|
252
|
+
import { check, sleep } from 'k6';
|
|
253
|
+
import redis from 'k6/x/redis';
|
|
254
|
+
|
|
255
|
+
// Connect to Redis
|
|
256
|
+
const client = new redis.Client('redis://localhost:6379');
|
|
257
|
+
|
|
258
|
+
export const options = {
|
|
259
|
+
vus: 50,
|
|
260
|
+
duration: '1m',
|
|
261
|
+
};
|
|
262
|
+
|
|
263
|
+
export default function () {
|
|
264
|
+
// Option A: Pop unique token (guarantees NO collision between VUs)
|
|
265
|
+
const uniqueToken = client.spop('${keyName}');
|
|
266
|
+
|
|
267
|
+
// Option B: Random sample (if token is reusable across VUs)
|
|
268
|
+
// const uniqueToken = client.srandmember('${keyName}');
|
|
269
|
+
|
|
270
|
+
if (!uniqueToken) {
|
|
271
|
+
console.warn('Redis set "${keyName}" is exhausted!');
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
// Execute authenticated HTTP request using seeded token
|
|
276
|
+
const res = http.get('https://api.example.com/v1/user/profile', {
|
|
277
|
+
headers: {
|
|
278
|
+
Authorization: \`Bearer \${uniqueToken}\`,
|
|
279
|
+
},
|
|
280
|
+
});
|
|
281
|
+
|
|
282
|
+
check(res, {
|
|
283
|
+
'status is 200': (r) => r.status === 200,
|
|
284
|
+
});
|
|
285
|
+
|
|
286
|
+
sleep(0.1);
|
|
287
|
+
}
|
|
288
|
+
`;
|
|
289
|
+
}
|
|
290
|
+
if (structure === "list") {
|
|
291
|
+
return `// ==========================================
|
|
292
|
+
// k6 Load Test - Redis Queue / List FIFO
|
|
293
|
+
// Requirements: xk6-redis
|
|
294
|
+
// ==========================================
|
|
295
|
+
import http from 'k6/http';
|
|
296
|
+
import { check } from 'k6';
|
|
297
|
+
import redis from 'k6/x/redis';
|
|
298
|
+
|
|
299
|
+
const client = new redis.Client('redis://localhost:6379');
|
|
300
|
+
|
|
301
|
+
export default function () {
|
|
302
|
+
// Pop from FIFO queue
|
|
303
|
+
const item = client.lpop('${keyName}');
|
|
304
|
+
|
|
305
|
+
if (!item) {
|
|
306
|
+
console.warn('Queue "${keyName}" is empty!');
|
|
307
|
+
return;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
const payload = JSON.parse(item);
|
|
311
|
+
const res = http.post('https://api.example.com/v1/orders', JSON.stringify(payload), {
|
|
312
|
+
headers: { 'Content-Type': 'application/json' },
|
|
313
|
+
});
|
|
314
|
+
|
|
315
|
+
check(res, { 'status is 201': (r) => r.status === 201 });
|
|
316
|
+
}
|
|
317
|
+
`;
|
|
318
|
+
}
|
|
319
|
+
return `// ==========================================
|
|
320
|
+
// k6 Load Test - Redis Key-Value Fetch
|
|
321
|
+
// Requirements: xk6-redis
|
|
322
|
+
// ==========================================
|
|
323
|
+
import http from 'k6/http';
|
|
324
|
+
import { check } from 'k6';
|
|
325
|
+
import redis from 'k6/x/redis';
|
|
326
|
+
|
|
327
|
+
const client = new redis.Client('redis://localhost:6379');
|
|
328
|
+
|
|
329
|
+
export default function () {
|
|
330
|
+
const value = client.get('${keyName}');
|
|
331
|
+
const res = http.get(\`https://api.example.com/v1/item/\${value}\`);
|
|
332
|
+
check(res, { 'status is 200': (r) => r.status === 200 });
|
|
333
|
+
}
|
|
334
|
+
`;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
// src/cli.ts
|
|
338
|
+
async function main() {
|
|
339
|
+
const program = new import_commander.Command();
|
|
340
|
+
program.name("rediseed").description(
|
|
341
|
+
"Blazing-fast Redis data seeder & loader from CSV, JSON, and line streams for k6 load testing & staging fixtures."
|
|
342
|
+
).version("1.0.0", "-v, --version", "Output current version");
|
|
343
|
+
program.option("-u, --url <url>", "Redis connection URL (e.g. redis://localhost:6379 or rediss:// for TLS)").option("-h, --host <host>", "Redis host", "127.0.0.1").option("-p, --port <port>", "Redis port", "6379").option("-a, --password <password>", "Redis auth password").option("-n, --db <db>", "Redis database index (0-15)", "0").option("--tls", "Enable TLS connection", false);
|
|
344
|
+
program.command("seed <file>").description("Load and seed data from CSV, JSON, or TXT file into Redis").option("-s, --structure <type>", "Redis structure: set, list, hash, string, zset", "set").option("-k, --key <name>", "Destination Redis key name or template (e.g. 'k6:user_ids' or 'user:{{id}}')", "seed:data").option("-c, --column <col>", "Extract specific column/property (for set, list, string)").option("--value <template>", "Value template for string/hash (e.g. '{{email}}:{{token}}')").option("--score-column <col>", "Column for zset score (defaults to timestamp)").option("-b, --batch-size <size>", "Pipeline batch size", "1000").option("--ttl <seconds>", "TTL expiration in seconds for keys").option("--flush", "Delete destination key before inserting", false).option("--dry-run", "Preview parsing and first 5 rows without writing to Redis", false).action(async (file, options) => {
|
|
345
|
+
try {
|
|
346
|
+
const globalOpts = program.opts();
|
|
347
|
+
console.log(import_picocolors.default.bold(`
|
|
348
|
+
\u{1F331} rediseed \u2014 Loading data from: ${import_picocolors.default.cyan(file)}`));
|
|
349
|
+
const rows = loadAndParseFile(file);
|
|
350
|
+
console.log(` \u2022 Parsed ${import_picocolors.default.green(rows.length.toLocaleString())} records.`);
|
|
351
|
+
if (options.dryRun) {
|
|
352
|
+
console.log(import_picocolors.default.yellow("\n[DRY RUN PREVIEW - First 5 items]"));
|
|
353
|
+
for (const row of rows.slice(0, 5)) {
|
|
354
|
+
console.log(" ", JSON.stringify(row));
|
|
355
|
+
}
|
|
356
|
+
console.log(import_picocolors.default.dim(`
|
|
357
|
+
Would insert ${rows.length} items into Redis structure '${options.structure}' with key '${options.key}'.
|
|
358
|
+
`));
|
|
359
|
+
return;
|
|
360
|
+
}
|
|
361
|
+
const redis = createRedisClient({
|
|
362
|
+
url: globalOpts.url,
|
|
363
|
+
host: globalOpts.host,
|
|
364
|
+
port: parseInt(globalOpts.port, 10),
|
|
365
|
+
password: globalOpts.password,
|
|
366
|
+
db: parseInt(globalOpts.db, 10),
|
|
367
|
+
tls: globalOpts.tls
|
|
368
|
+
});
|
|
369
|
+
const result = await executeSeed(redis, rows, {
|
|
370
|
+
structure: options.structure,
|
|
371
|
+
keyName: options.key,
|
|
372
|
+
column: options.column,
|
|
373
|
+
valueTemplate: options.value,
|
|
374
|
+
scoreColumn: options.scoreColumn,
|
|
375
|
+
batchSize: parseInt(options.batchSize, 10),
|
|
376
|
+
flushFirst: options.flush,
|
|
377
|
+
ttl: options.ttl ? parseInt(options.ttl, 10) : void 0
|
|
378
|
+
});
|
|
379
|
+
await redis.quit();
|
|
380
|
+
console.log(import_picocolors.default.green(`
|
|
381
|
+
\u2714 Seeding completed successfully!`));
|
|
382
|
+
console.log(` \u2022 Structure: ${import_picocolors.default.cyan(result.structure.toUpperCase())}`);
|
|
383
|
+
console.log(` \u2022 Target Key: ${import_picocolors.default.cyan(options.key)}`);
|
|
384
|
+
console.log(` \u2022 Total Items: ${import_picocolors.default.bold(result.totalInserted.toLocaleString())}`);
|
|
385
|
+
console.log(` \u2022 Speed: ${import_picocolors.default.bold(result.opsPerSec.toLocaleString())} ops/sec (${result.elapsedMs}ms)
|
|
386
|
+
`);
|
|
387
|
+
} catch (err) {
|
|
388
|
+
console.error(import_picocolors.default.red(`\u2716 Error: ${err.message || err}`));
|
|
389
|
+
process.exit(1);
|
|
390
|
+
}
|
|
391
|
+
});
|
|
392
|
+
program.command("set <keyName> <file>").description("Shortcut to load a file column directly into a Redis Set (ideal for k6 SPOP unique IDs)").option("-c, --column <col>", "Extract specific column name from CSV/JSON").option("--flush", "Delete set before inserting", false).action(async (keyName, file, options) => {
|
|
393
|
+
try {
|
|
394
|
+
const globalOpts = program.opts();
|
|
395
|
+
const rows = loadAndParseFile(file);
|
|
396
|
+
const redis = createRedisClient({
|
|
397
|
+
url: globalOpts.url,
|
|
398
|
+
host: globalOpts.host,
|
|
399
|
+
port: parseInt(globalOpts.port, 10),
|
|
400
|
+
password: globalOpts.password,
|
|
401
|
+
db: parseInt(globalOpts.db, 10),
|
|
402
|
+
tls: globalOpts.tls
|
|
403
|
+
});
|
|
404
|
+
const result = await executeSeed(redis, rows, {
|
|
405
|
+
structure: "set",
|
|
406
|
+
keyName,
|
|
407
|
+
column: options.column,
|
|
408
|
+
flushFirst: options.flush
|
|
409
|
+
});
|
|
410
|
+
await redis.quit();
|
|
411
|
+
console.log(import_picocolors.default.green(`
|
|
412
|
+
\u2714 Successfully loaded ${result.totalInserted.toLocaleString()} unique items into Redis Set: ${import_picocolors.default.cyan(keyName)} (${result.elapsedMs}ms)`));
|
|
413
|
+
} catch (err) {
|
|
414
|
+
console.error(import_picocolors.default.red(`\u2716 Error: ${err.message || err}`));
|
|
415
|
+
process.exit(1);
|
|
416
|
+
}
|
|
417
|
+
});
|
|
418
|
+
program.command("inspect <key>").description("Inspect item count, data type, memory usage, and samples from a Redis key").option("-s, --samples <count>", "Number of sample items to preview", "5").action(async (key, options) => {
|
|
419
|
+
try {
|
|
420
|
+
const globalOpts = program.opts();
|
|
421
|
+
const redis = createRedisClient({
|
|
422
|
+
url: globalOpts.url,
|
|
423
|
+
host: globalOpts.host,
|
|
424
|
+
port: parseInt(globalOpts.port, 10),
|
|
425
|
+
password: globalOpts.password,
|
|
426
|
+
db: parseInt(globalOpts.db, 10),
|
|
427
|
+
tls: globalOpts.tls
|
|
428
|
+
});
|
|
429
|
+
const info = await inspectKey(redis, key, parseInt(options.samples, 10));
|
|
430
|
+
await redis.quit();
|
|
431
|
+
console.log(`
|
|
432
|
+
${import_picocolors.default.bold("\u{1F50D} Redis Key Inspector")}`);
|
|
433
|
+
console.log(` \u2022 Key: ${import_picocolors.default.cyan(info.key)}`);
|
|
434
|
+
console.log(` \u2022 Type: ${import_picocolors.default.green(info.type.toUpperCase())}`);
|
|
435
|
+
console.log(` \u2022 Total Items: ${import_picocolors.default.bold(info.count.toLocaleString())}`);
|
|
436
|
+
console.log(` \u2022 TTL: ${info.ttl === -1 ? "No Expiration" : `${info.ttl}s`}`);
|
|
437
|
+
if (info.memoryBytes) {
|
|
438
|
+
console.log(` \u2022 Memory Usage: ${(info.memoryBytes / 1024).toFixed(2)} KB`);
|
|
439
|
+
}
|
|
440
|
+
console.log(`
|
|
441
|
+
${import_picocolors.default.bold("Sample Items:")}`);
|
|
442
|
+
for (const sample of info.samples) {
|
|
443
|
+
console.log(` \u2022 ${sample}`);
|
|
444
|
+
}
|
|
445
|
+
console.log("");
|
|
446
|
+
} catch (err) {
|
|
447
|
+
console.error(import_picocolors.default.red(`\u2716 Error: ${err.message || err}`));
|
|
448
|
+
process.exit(1);
|
|
449
|
+
}
|
|
450
|
+
});
|
|
451
|
+
program.command("k6 <key>").description("Generate ready-to-use k6 load test script snippet consuming this Redis key").option("-s, --structure <type>", "Redis structure (set, list, string)", "set").action((key, options) => {
|
|
452
|
+
const snippet = generateK6Snippet(key, options.structure);
|
|
453
|
+
console.log(`
|
|
454
|
+
${snippet}
|
|
455
|
+
`);
|
|
456
|
+
});
|
|
457
|
+
await program.parseAsync(process.argv);
|
|
458
|
+
}
|
|
459
|
+
main().catch((err) => {
|
|
460
|
+
console.error(err);
|
|
461
|
+
process.exit(1);
|
|
462
|
+
});
|
|
463
|
+
//# sourceMappingURL=cli.js.map
|