rehydra 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/anonymize.d.ts.map +1 -1
- package/dist/cli/commands/anonymize.js +163 -31
- package/dist/cli/commands/anonymize.js.map +1 -1
- package/dist/cli/utils/io.d.ts +9 -0
- package/dist/cli/utils/io.d.ts.map +1 -1
- package/dist/cli/utils/io.js +16 -0
- package/dist/cli/utils/io.js.map +1 -1
- package/dist/core/anonymizer.d.ts +14 -0
- package/dist/core/anonymizer.d.ts.map +1 -1
- package/dist/core/anonymizer.js +4 -0
- package/dist/core/anonymizer.js.map +1 -1
- package/dist/ner/ner-model.d.ts +18 -0
- package/dist/ner/ner-model.d.ts.map +1 -1
- package/dist/ner/ner-model.js +46 -8
- package/dist/ner/ner-model.js.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anonymize.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/anonymize.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"anonymize.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/anonymize.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAyFhD,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,MAAM,CAAC,CAcjB"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import { pipeline } from "node:stream/promises";
|
|
1
2
|
import { createAnonymizer, PIIType, mergePolicy, generateKey, uint8ArrayToBase64, ConfigKeyProvider, } from "../../index.js";
|
|
3
|
+
import { createAnonymizerStream, } from "../../streaming/index.js";
|
|
2
4
|
import { CLIError } from "../utils/errors.js";
|
|
3
|
-
import { readInput, writeOutput } from "../utils/io.js";
|
|
5
|
+
import { readInput, writeOutput, getInputStream, getOutputStream } from "../utils/io.js";
|
|
4
6
|
import { formatText, formatJson, formatNdjson, formatStats } from "../utils/format.js";
|
|
5
7
|
import { savePIIMapFile } from "../utils/pii-map-file.js";
|
|
6
8
|
function parseTypes(typesStr) {
|
|
@@ -39,46 +41,57 @@ function validateFormat(format) {
|
|
|
39
41
|
}
|
|
40
42
|
return format;
|
|
41
43
|
}
|
|
42
|
-
|
|
43
|
-
const nerMode = validateNerMode(options.ner);
|
|
44
|
-
const anonMode = validateMode(options.mode);
|
|
45
|
-
const format = validateFormat(options.format);
|
|
46
|
-
const input = await readInput(filePath);
|
|
47
|
-
// Set up encryption key
|
|
44
|
+
function setupKeyProvider(options) {
|
|
48
45
|
const envKey = process.env["REHYDRA_KEY"];
|
|
49
46
|
const flagKey = options.key;
|
|
50
47
|
const externalKey = flagKey ?? envKey;
|
|
51
|
-
let keyBase64;
|
|
52
|
-
let keyProvider;
|
|
53
48
|
if (externalKey !== undefined) {
|
|
54
|
-
keyProvider
|
|
55
|
-
// Don't store the key in the PII map file when user provides it
|
|
49
|
+
return { keyProvider: new ConfigKeyProvider(externalKey), keyBase64: undefined };
|
|
56
50
|
}
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
51
|
+
const keyBytes = generateKey();
|
|
52
|
+
const keyBase64 = uint8ArrayToBase64(keyBytes);
|
|
53
|
+
return { keyProvider: new ConfigKeyProvider(keyBase64), keyBase64 };
|
|
54
|
+
}
|
|
55
|
+
function buildNerConfig(nerMode, quiet) {
|
|
56
|
+
if (nerMode === "disabled")
|
|
57
|
+
return undefined;
|
|
58
|
+
return {
|
|
59
|
+
mode: nerMode,
|
|
60
|
+
autoDownload: true,
|
|
61
|
+
onStatus: quiet
|
|
62
|
+
? undefined
|
|
63
|
+
: (status) => {
|
|
64
|
+
process.stderr.write(`${status}\n`);
|
|
65
|
+
},
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
export async function anonymizeCommand(filePath, options) {
|
|
69
|
+
const nerMode = validateNerMode(options.ner);
|
|
70
|
+
const anonMode = validateMode(options.mode);
|
|
71
|
+
const format = validateFormat(options.format);
|
|
72
|
+
const { keyProvider, keyBase64 } = setupKeyProvider(options);
|
|
73
|
+
const policy = options.types !== undefined
|
|
74
|
+
? mergePolicy({ enabledTypes: parseTypes(options.types) })
|
|
75
|
+
: undefined;
|
|
76
|
+
// Use streaming for file inputs, batch for stdin
|
|
77
|
+
if (filePath !== undefined) {
|
|
78
|
+
return anonymizeFile(filePath, options, nerMode, anonMode, format, keyProvider, keyBase64, policy);
|
|
61
79
|
}
|
|
62
|
-
|
|
80
|
+
return anonymizeBatch(options, nerMode, anonMode, format, keyProvider, keyBase64, policy);
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Batch anonymization for stdin input.
|
|
84
|
+
*/
|
|
85
|
+
async function anonymizeBatch(options, nerMode, anonMode, format, keyProvider, keyBase64, policy) {
|
|
86
|
+
const input = await readInput();
|
|
63
87
|
const config = {
|
|
64
88
|
mode: anonMode,
|
|
65
89
|
keyProvider,
|
|
66
90
|
};
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
autoDownload: true,
|
|
71
|
-
onStatus: options.quiet
|
|
72
|
-
? undefined
|
|
73
|
-
: (status) => {
|
|
74
|
-
process.stderr.write(`${status}\n`);
|
|
75
|
-
},
|
|
76
|
-
};
|
|
91
|
+
const nerConfig = buildNerConfig(nerMode, options.quiet);
|
|
92
|
+
if (nerConfig !== undefined) {
|
|
93
|
+
config.ner = nerConfig;
|
|
77
94
|
}
|
|
78
|
-
// Build policy with type filtering
|
|
79
|
-
const policy = options.types !== undefined
|
|
80
|
-
? mergePolicy({ enabledTypes: parseTypes(options.types) })
|
|
81
|
-
: undefined;
|
|
82
95
|
const anonymizer = createAnonymizer(config);
|
|
83
96
|
await anonymizer.initialize();
|
|
84
97
|
try {
|
|
@@ -122,11 +135,130 @@ export async function anonymizeCommand(filePath, options) {
|
|
|
122
135
|
if (options.verbose) {
|
|
123
136
|
process.stderr.write(formatStats(result.stats) + "\n");
|
|
124
137
|
}
|
|
125
|
-
// Exit code 2 if no PII found
|
|
126
138
|
return result.stats.totalEntities > 0 ? 0 : 2;
|
|
127
139
|
}
|
|
128
140
|
finally {
|
|
129
141
|
await anonymizer.dispose();
|
|
130
142
|
}
|
|
131
143
|
}
|
|
144
|
+
/**
|
|
145
|
+
* Streaming anonymization for file inputs.
|
|
146
|
+
*/
|
|
147
|
+
async function anonymizeFile(filePath, options, nerMode, anonMode, format, keyProvider, keyBase64, policy) {
|
|
148
|
+
const countsByType = {};
|
|
149
|
+
const allEntities = [];
|
|
150
|
+
const textChunks = [];
|
|
151
|
+
let finishData;
|
|
152
|
+
const anonymizerConfig = {
|
|
153
|
+
mode: anonMode,
|
|
154
|
+
keyProvider,
|
|
155
|
+
};
|
|
156
|
+
const nerConfig = buildNerConfig(nerMode, options.quiet);
|
|
157
|
+
if (nerConfig !== undefined) {
|
|
158
|
+
anonymizerConfig.ner = nerConfig;
|
|
159
|
+
}
|
|
160
|
+
const streamConfig = {
|
|
161
|
+
anonymizer: anonymizerConfig,
|
|
162
|
+
policy: policy ?? undefined,
|
|
163
|
+
locale: options.locale,
|
|
164
|
+
keyProvider,
|
|
165
|
+
onChunk: (event) => {
|
|
166
|
+
for (const entity of event.entities) {
|
|
167
|
+
countsByType[entity.type] = (countsByType[entity.type] ?? 0) + 1;
|
|
168
|
+
allEntities.push({
|
|
169
|
+
type: entity.type,
|
|
170
|
+
id: entity.id,
|
|
171
|
+
confidence: entity.confidence,
|
|
172
|
+
source: entity.source,
|
|
173
|
+
...(entity.semantic !== undefined ? { semantic: entity.semantic } : {}),
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
},
|
|
177
|
+
onFinish: (event) => {
|
|
178
|
+
finishData = event;
|
|
179
|
+
},
|
|
180
|
+
};
|
|
181
|
+
const anonymizerStream = await createAnonymizerStream(streamConfig);
|
|
182
|
+
const inputStream = getInputStream(filePath);
|
|
183
|
+
const outputStream = getOutputStream(options.output);
|
|
184
|
+
if (format === "text") {
|
|
185
|
+
// Direct pipe: input → anonymizer → output
|
|
186
|
+
await pipeline(inputStream, anonymizerStream, outputStream);
|
|
187
|
+
// Add trailing newline if writing to file
|
|
188
|
+
if (options.output !== undefined) {
|
|
189
|
+
outputStream.write("\n");
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
// For json/ndjson: collect output text, format at end
|
|
194
|
+
anonymizerStream.on("data", (chunk) => {
|
|
195
|
+
textChunks.push(typeof chunk === "string" ? chunk : chunk.toString());
|
|
196
|
+
});
|
|
197
|
+
inputStream.pipe(anonymizerStream);
|
|
198
|
+
await new Promise((resolve, reject) => {
|
|
199
|
+
anonymizerStream.on("end", resolve);
|
|
200
|
+
anonymizerStream.on("error", reject);
|
|
201
|
+
inputStream.on("error", reject);
|
|
202
|
+
});
|
|
203
|
+
let output;
|
|
204
|
+
if (format === "json") {
|
|
205
|
+
output = JSON.stringify({
|
|
206
|
+
anonymizedText: textChunks.join(""),
|
|
207
|
+
entities: allEntities,
|
|
208
|
+
stats: {
|
|
209
|
+
totalEntities: finishData?.totalEntities ?? 0,
|
|
210
|
+
countsByType,
|
|
211
|
+
processingTimeMs: finishData?.totalProcessingTimeMs ?? 0,
|
|
212
|
+
},
|
|
213
|
+
}, null, 2);
|
|
214
|
+
}
|
|
215
|
+
else {
|
|
216
|
+
// ndjson: entity lines + summary
|
|
217
|
+
const lines = allEntities.map((e) => JSON.stringify(e));
|
|
218
|
+
lines.push(JSON.stringify({
|
|
219
|
+
_type: "summary",
|
|
220
|
+
anonymizedText: textChunks.join(""),
|
|
221
|
+
totalEntities: finishData?.totalEntities ?? 0,
|
|
222
|
+
processingTimeMs: finishData?.totalProcessingTimeMs ?? 0,
|
|
223
|
+
}));
|
|
224
|
+
output = lines.join("\n");
|
|
225
|
+
}
|
|
226
|
+
if (!output.endsWith("\n")) {
|
|
227
|
+
output += "\n";
|
|
228
|
+
}
|
|
229
|
+
await writeOutput(output, options.output);
|
|
230
|
+
}
|
|
231
|
+
// Close file output stream if writing to file (pipeline already handles this for text format)
|
|
232
|
+
if (format !== "text" && options.output !== undefined && outputStream !== process.stdout) {
|
|
233
|
+
await new Promise((resolve) => outputStream.end(resolve));
|
|
234
|
+
}
|
|
235
|
+
// Save PII map
|
|
236
|
+
if (anonMode === "pseudonymize" && finishData?.piiMap !== undefined) {
|
|
237
|
+
const piiMapFile = {
|
|
238
|
+
version: 1,
|
|
239
|
+
createdAt: new Date().toISOString(),
|
|
240
|
+
...(keyBase64 !== undefined ? { key: keyBase64 } : {}),
|
|
241
|
+
piiMap: finishData.piiMap,
|
|
242
|
+
stats: {
|
|
243
|
+
totalEntities: finishData.totalEntities,
|
|
244
|
+
countsByType,
|
|
245
|
+
},
|
|
246
|
+
};
|
|
247
|
+
await savePIIMapFile(options["pii-map"], piiMapFile);
|
|
248
|
+
if (!options.quiet) {
|
|
249
|
+
process.stderr.write(`PII map saved to ${options["pii-map"]}\n`);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
// Print stats to stderr if verbose
|
|
253
|
+
if (options.verbose && finishData !== undefined) {
|
|
254
|
+
process.stderr.write(formatStats({
|
|
255
|
+
totalEntities: finishData.totalEntities,
|
|
256
|
+
countsByType: countsByType,
|
|
257
|
+
processingTimeMs: finishData.totalProcessingTimeMs,
|
|
258
|
+
modelVersion: "",
|
|
259
|
+
policyVersion: "",
|
|
260
|
+
}) + "\n");
|
|
261
|
+
}
|
|
262
|
+
return (finishData?.totalEntities ?? 0) > 0 ? 0 : 2;
|
|
263
|
+
}
|
|
132
264
|
//# sourceMappingURL=anonymize.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anonymize.js","sourceRoot":"","sources":["../../../src/cli/commands/anonymize.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,
|
|
1
|
+
{"version":3,"file":"anonymize.js","sourceRoot":"","sources":["../../../src/cli/commands/anonymize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EACL,gBAAgB,EAIhB,OAAO,EACP,WAAW,EACX,WAAW,EACX,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,sBAAsB,GAIvB,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACzF,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,EAAE,cAAc,EAAmB,MAAM,0BAA0B,CAAC;AAE3E,SAAS,UAAU,CAAC,QAAgB;IAClC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAa,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,IAAI,GAAG,EAAW,CAAC;IAEjC,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE;YAAE,SAAS;QACvB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,QAAQ,CAChB,qBAAqB,GAAG,CAAC,IAAI,EAAE,kBAAkB,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC7E,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,CAAY,CAAC,CAAC;IAC1B,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,QAAQ,CAAC,4CAA4C,CAAC,CAAC;IACnE,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,KAAK,GAAG,CAAC,UAAU,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;IACpD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,QAAQ,CAChB,qBAAqB,IAAI,kBAAkB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC9D,CAAC;IACJ,CAAC;IACD,OAAO,IAAyB,CAAC;AACnC,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;QACpD,MAAM,IAAI,QAAQ,CAChB,iBAAiB,IAAI,wCAAwC,CAC9D,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,MAAc;IACpC,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QAClE,MAAM,IAAI,QAAQ,CAChB,mBAAmB,MAAM,qCAAqC,CAC/D,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAsB;IAI9C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;IAC5B,MAAM,WAAW,GAAG,OAAO,IAAI,MAAM,CAAC;IAEtC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,EAAE,WAAW,EAAE,IAAI,iBAAiB,CAAC,WAAW,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;IACnF,CAAC;IAED,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,EAAE,WAAW,EAAE,IAAI,iBAAiB,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,CAAC;AACtE,CAAC;AAED,SAAS,cAAc,CACrB,OAA0B,EAC1B,KAAc;IAEd,IAAI,OAAO,KAAK,UAAU;QAAE,OAAO,SAAS,CAAC;IAC7C,OAAO;QACL,IAAI,EAAE,OAAO;QACb,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,KAAK;YACb,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,CAAC,MAAc,EAAQ,EAAE;gBACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,CAAC;YACtC,CAAC;KACN,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,QAA4B,EAC5B,OAAsB;IAEtB,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7C,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,KAAK,SAAS;QACxC,CAAC,CAAC,WAAW,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1D,CAAC,CAAC,SAAS,CAAC;IAEd,iDAAiD;IACjD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IACrG,CAAC;IACD,OAAO,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;AAC5F,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAC3B,OAAsB,EACtB,OAA0B,EAC1B,QAAsC,EACtC,MAAkC,EAClC,WAA8B,EAC9B,SAA6B,EAC7B,MAAgD;IAEhD,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAC;IAEhC,MAAM,MAAM,GAAqB;QAC/B,IAAI,EAAE,QAAQ;QACd,WAAW;KACZ,CAAC;IAEF,MAAM,SAAS,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACzD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,GAAG,SAAS,CAAC;IACzB,CAAC;IAED,MAAM,UAAU,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,UAAU,CAAC,UAAU,EAAE,CAAC;IAE9B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAE5G,yCAAyC;QACzC,IAAI,QAAQ,KAAK,cAAc,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC/D,MAAM,UAAU,GAAe;gBAC7B,OAAO,EAAE,CAAC;gBACV,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtD,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,KAAK,EAAE;oBACL,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC,aAAa;oBACzC,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY;iBACxC;aACF,CAAC;YACF,MAAM,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,UAAU,CAAC,CAAC;YAErD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,oBAAoB,OAAO,CAAC,SAAS,CAAC,IAAI,CAC3C,CAAC;YACJ,CAAC;QACH,CAAC;QAED,gBAAgB;QAChB,IAAI,MAAc,CAAC;QACnB,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,MAAM;gBACT,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;gBAC5B,MAAM;YACR,KAAK,MAAM;gBACT,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;gBAC5B,MAAM;YACR,KAAK,QAAQ;gBACX,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC9B,MAAM;QACV,CAAC;QAED,0BAA0B;QAC1B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,IAAI,CAAC;QACjB,CAAC;QAED,MAAM,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAE1C,mCAAmC;QACnC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;QACzD,CAAC;QAED,OAAO,MAAM,CAAC,KAAK,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC;YAAS,CAAC;QACT,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,aAAa,CAC1B,QAAgB,EAChB,OAAsB,EACtB,OAA0B,EAC1B,QAAsC,EACtC,MAAkC,EAClC,WAA8B,EAC9B,SAA6B,EAC7B,MAAgD;IAEhD,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,MAAM,WAAW,GAA2F,EAAE,CAAC;IAC/G,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,IAAI,UAAyC,CAAC;IAE9C,MAAM,gBAAgB,GAAqB;QACzC,IAAI,EAAE,QAAQ;QACd,WAAW;KACZ,CAAC;IAEF,MAAM,SAAS,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACzD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,gBAAgB,CAAC,GAAG,GAAG,SAAS,CAAC;IACnC,CAAC;IAED,MAAM,YAAY,GAAiB;QACjC,UAAU,EAAE,gBAAgB;QAC5B,MAAM,EAAE,MAAM,IAAI,SAAS;QAC3B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,WAAW;QACX,OAAO,EAAE,CAAC,KAAuB,EAAE,EAAE;YACnC,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACpC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;gBACjE,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,EAAE,EAAE,MAAM,CAAC,EAAE;oBACb,UAAU,EAAE,MAAM,CAAC,UAAU;oBAC7B,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACxE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,QAAQ,EAAE,CAAC,KAAwB,EAAE,EAAE;YACrC,UAAU,GAAG,KAAK,CAAC;QACrB,CAAC;KACF,CAAC;IAEF,MAAM,gBAAgB,GAAG,MAAM,sBAAsB,CAAC,YAAY,CAAC,CAAC;IACpE,MAAM,WAAW,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAErD,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,2CAA2C;QAC3C,MAAM,QAAQ,CAAC,WAAW,EAAE,gBAAgB,EAAE,YAAY,CAAC,CAAC;QAC5D,0CAA0C;QAC1C,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACjC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;SAAM,CAAC;QACN,sDAAsD;QACtD,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAsB,EAAE,EAAE;YACrD,UAAU,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QACH,WAAW,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACnC,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACpC,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACrC,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,IAAI,MAAc,CAAC;QACnB,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACtB,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;gBACtB,cAAc,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnC,QAAQ,EAAE,WAAW;gBACrB,KAAK,EAAE;oBACL,aAAa,EAAE,UAAU,EAAE,aAAa,IAAI,CAAC;oBAC7C,YAAY;oBACZ,gBAAgB,EAAE,UAAU,EAAE,qBAAqB,IAAI,CAAC;iBACzD;aACF,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACd,CAAC;aAAM,CAAC;YACN,iCAAiC;YACjC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YACxD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,SAAS,CAAC;gBACb,KAAK,EAAE,SAAS;gBAChB,cAAc,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnC,aAAa,EAAE,UAAU,EAAE,aAAa,IAAI,CAAC;gBAC7C,gBAAgB,EAAE,UAAU,EAAE,qBAAqB,IAAI,CAAC;aACzD,CAAC,CACH,CAAC;YACF,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,IAAI,CAAC;QACjB,CAAC;QACD,MAAM,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5C,CAAC;IAED,8FAA8F;IAC9F,IAAI,MAAM,KAAK,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,YAAY,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;QACzF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IAClE,CAAC;IAED,eAAe;IACf,IAAI,QAAQ,KAAK,cAAc,IAAI,UAAU,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;QACpE,MAAM,UAAU,GAAe;YAC7B,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtD,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,KAAK,EAAE;gBACL,aAAa,EAAE,UAAU,CAAC,aAAa;gBACvC,YAAY;aACb;SACF,CAAC;QACF,MAAM,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,UAAU,CAAC,CAAC;QAErD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,oBAAoB,OAAO,CAAC,SAAS,CAAC,IAAI,CAC3C,CAAC;QACJ,CAAC;IACH,CAAC;IAED,mCAAmC;IACnC,IAAI,OAAO,CAAC,OAAO,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAChD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,WAAW,CAAC;YACV,aAAa,EAAE,UAAU,CAAC,aAAa;YACvC,YAAY,EAAE,YAAuC;YACrD,gBAAgB,EAAE,UAAU,CAAC,qBAAqB;YAClD,YAAY,EAAE,EAAE;YAChB,aAAa,EAAE,EAAE;SAClB,CAAC,GAAG,IAAI,CACV,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,UAAU,EAAE,aAAa,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACtD,CAAC"}
|
package/dist/cli/utils/io.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Readable, Writable } from "node:stream";
|
|
1
2
|
/**
|
|
2
3
|
* Read input from a file path or stdin.
|
|
3
4
|
* Throws CLIError if no file and stdin is a TTY (no piped data).
|
|
@@ -7,4 +8,12 @@ export declare function readInput(filePath?: string): Promise<string>;
|
|
|
7
8
|
* Write output to a file path or stdout.
|
|
8
9
|
*/
|
|
9
10
|
export declare function writeOutput(data: string, filePath?: string): Promise<void>;
|
|
11
|
+
/**
|
|
12
|
+
* Get a readable stream for a file path.
|
|
13
|
+
*/
|
|
14
|
+
export declare function getInputStream(filePath: string): Readable;
|
|
15
|
+
/**
|
|
16
|
+
* Get a writable stream for output (file or stdout).
|
|
17
|
+
*/
|
|
18
|
+
export declare function getOutputStream(filePath?: string): Writable;
|
|
10
19
|
//# sourceMappingURL=io.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"io.d.ts","sourceRoot":"","sources":["../../../src/cli/utils/io.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"io.d.ts","sourceRoot":"","sources":["../../../src/cli/utils/io.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAItD;;;GAGG;AACH,wBAAsB,SAAS,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAoBlE;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAMf;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAEzD;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,QAAQ,CAK3D"}
|
package/dist/cli/utils/io.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createReadStream, createWriteStream } from "node:fs";
|
|
1
2
|
import { readFile, writeFile } from "node:fs/promises";
|
|
2
3
|
import { text } from "node:stream/consumers";
|
|
3
4
|
import { CLIError } from "./errors.js";
|
|
@@ -34,4 +35,19 @@ export async function writeOutput(data, filePath) {
|
|
|
34
35
|
process.stdout.write(data);
|
|
35
36
|
}
|
|
36
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* Get a readable stream for a file path.
|
|
40
|
+
*/
|
|
41
|
+
export function getInputStream(filePath) {
|
|
42
|
+
return createReadStream(filePath, { encoding: "utf-8" });
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Get a writable stream for output (file or stdout).
|
|
46
|
+
*/
|
|
47
|
+
export function getOutputStream(filePath) {
|
|
48
|
+
if (filePath !== undefined) {
|
|
49
|
+
return createWriteStream(filePath, { encoding: "utf-8" });
|
|
50
|
+
}
|
|
51
|
+
return process.stdout;
|
|
52
|
+
}
|
|
37
53
|
//# sourceMappingURL=io.js.map
|
package/dist/cli/utils/io.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"io.js","sourceRoot":"","sources":["../../../src/cli/utils/io.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"io.js","sourceRoot":"","sources":["../../../src/cli/utils/io.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAEvD,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAiB;IAC/C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,GAAI,GAA6B,CAAC,IAAI,CAAC;YACjD,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACtB,MAAM,IAAI,QAAQ,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;YACpD,CAAC;YACD,MAAM,IAAI,QAAQ,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QACjC,MAAM,IAAI,QAAQ,CAChB,0DAA0D,CAC3D,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAY,EACZ,QAAiB;IAEjB,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,OAAO,gBAAgB,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,QAAiB;IAC/C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,iBAAiB,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,CAAC;AACxB,CAAC"}
|
|
@@ -73,6 +73,20 @@ export interface NERConfig {
|
|
|
73
73
|
* Only used when backend is 'inference-server'
|
|
74
74
|
*/
|
|
75
75
|
inferenceServerTimeout?: number;
|
|
76
|
+
/**
|
|
77
|
+
* Enable case-insensitive fallback for detecting lowercase names.
|
|
78
|
+
* Runs a second NER pass on title-cased text and merges new detections.
|
|
79
|
+
* Doubles NER inference time but catches names like "tom" that the
|
|
80
|
+
* case-sensitive model would otherwise miss.
|
|
81
|
+
* @default false
|
|
82
|
+
*/
|
|
83
|
+
caseFallback?: boolean;
|
|
84
|
+
/**
|
|
85
|
+
* Confidence penalty multiplier for case-fallback detections (0.0 - 1.0).
|
|
86
|
+
* Applied as: confidence * caseFallbackPenalty
|
|
87
|
+
* @default 0.85
|
|
88
|
+
*/
|
|
89
|
+
caseFallbackPenalty?: number;
|
|
76
90
|
}
|
|
77
91
|
/**
|
|
78
92
|
* Anonymizer configuration
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anonymizer.d.ts","sourceRoot":"","sources":["../../src/core/anonymizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EAEnB,iBAAiB,EAEjB,cAAc,EAEd,OAAO,EAER,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAEL,kBAAkB,EACnB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,KAAK,SAAS,EAKd,KAAK,iBAAiB,EACvB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,wBAAwB,EAC9B,MAAM,yBAAyB,CAAC;AAIjC,OAAO,EAGL,KAAK,SAAS,EACf,MAAM,uBAAuB,CAAC;AAY/B,OAAO,EAGL,KAAK,WAAW,EACjB,MAAM,oBAAoB,CAAC;AAI5B,YAAY,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAGvD,OAAO,KAAK,EACV,kBAAkB,EAClB,iBAAiB,EAClB,MAAM,qBAAqB,CAAC;AAG7B,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;AAoCtD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;;;;OAMG;IACH,IAAI,EAAE,YAAY,CAAC;IAEnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;OAEG;IACH,kBAAkB,CAAC,EAAE,wBAAwB,CAAC;IAE9C;;OAEG;IACH,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IAEpC;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IAE9C;;;;OAIG;IACH,cAAc,CAAC,EAAE,iBAAiB,CAAC;IAEnC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,OAAO,GAAG,kBAAkB,CAAC;IAEvC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;OAGG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"anonymizer.d.ts","sourceRoot":"","sources":["../../src/core/anonymizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EAEnB,iBAAiB,EAEjB,cAAc,EAEd,OAAO,EAER,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAEL,kBAAkB,EACnB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,KAAK,SAAS,EAKd,KAAK,iBAAiB,EACvB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,wBAAwB,EAC9B,MAAM,yBAAyB,CAAC;AAIjC,OAAO,EAGL,KAAK,SAAS,EACf,MAAM,uBAAuB,CAAC;AAY/B,OAAO,EAGL,KAAK,WAAW,EACjB,MAAM,oBAAoB,CAAC;AAI5B,YAAY,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAGvD,OAAO,KAAK,EACV,kBAAkB,EAClB,iBAAiB,EAClB,MAAM,qBAAqB,CAAC;AAG7B,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;AAoCtD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;;;;OAMG;IACH,IAAI,EAAE,YAAY,CAAC;IAEnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;OAEG;IACH,kBAAkB,CAAC,EAAE,wBAAwB,CAAC;IAE9C;;OAEG;IACH,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IAEpC;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IAE9C;;;;OAIG;IACH,cAAc,CAAC,EAAE,iBAAiB,CAAC;IAEnC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,OAAO,GAAG,kBAAkB,CAAC;IAEvC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;OAGG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yDAAyD;IACzD,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAE9B;;;;;OAKG;IACH,IAAI,CAAC,EAAE,iBAAiB,CAAC;IAEzB;;;OAGG;IACH,GAAG,CAAC,EAAE,SAAS,CAAC;IAEhB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAE1B,yEAAyE;IACzE,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,oDAAoD;IACpD,aAAa,CAAC,EAAE,mBAAmB,CAAC;IAEpC,2BAA2B;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,4BAA4B;IAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,kBAAkB,EAC3B,WAAW,EAAE,WAAW,KACrB,iBAAiB,CAAC;AAEvB;;;GAGG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,QAAQ,CAA0B;IAC1C,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,IAAI,CAAoB;IAChC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,kBAAkB,CAA4B;IACtD,OAAO,CAAC,aAAa,CAAsB;IAC3C,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAwB;gBAElC,MAAM,GAAE,gBAAqB,EAAE,cAAc,CAAC,EAAE,cAAc;IAyC1E;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA6HjC;;;;;;;OAOG;IACG,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,EACrC,cAAc,CAAC,EAAE,SAAS,GACzB,OAAO,CAAC,mBAAmB,CAAC;IA6H/B;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;OAEG;IACH,WAAW,IAAI,kBAAkB;IAIjC;;OAEG;IACH,WAAW,IAAI,SAAS,GAAG,IAAI;IAI/B;;OAEG;IACH,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB;CAmD9C;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,CAAC,EAAE,gBAAgB,EACzB,cAAc,CAAC,EAAE,cAAc,GAC9B,UAAU,CAEZ;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAS9B;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAQ9B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,GAAG;IAAE,IAAI,CAAC,EAAE,UAAU,GAAG,WAAW,CAAA;CAAE,EACxE,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAe9B"}
|
package/dist/core/anonymizer.js
CHANGED
|
@@ -139,6 +139,8 @@ export class Anonymizer {
|
|
|
139
139
|
vocabPath: this.nerConfig.vocabPath,
|
|
140
140
|
modelVersion: this.modelVersion,
|
|
141
141
|
sessionOptions: this.nerConfig.sessionOptions,
|
|
142
|
+
caseFallback: this.nerConfig.caseFallback,
|
|
143
|
+
caseFallbackPenalty: this.nerConfig.caseFallbackPenalty,
|
|
142
144
|
});
|
|
143
145
|
}
|
|
144
146
|
else {
|
|
@@ -164,6 +166,8 @@ export class Anonymizer {
|
|
|
164
166
|
labelMap,
|
|
165
167
|
modelVersion: this.modelVersion,
|
|
166
168
|
sessionOptions: this.nerConfig.sessionOptions,
|
|
169
|
+
caseFallback: this.nerConfig.caseFallback,
|
|
170
|
+
caseFallbackPenalty: this.nerConfig.caseFallbackPenalty,
|
|
167
171
|
});
|
|
168
172
|
}
|
|
169
173
|
// Load the NER model
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anonymizer.js","sourceRoot":"","sources":["../../src/core/anonymizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EASL,mBAAmB,GACpB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,qBAAqB,GAEtB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAEL,YAAY,EACZ,cAAc,EACd,6BAA6B,EAC7B,iBAAiB,GAElB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAEL,WAAW,GAEZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EACL,WAAW,EACX,mBAAmB,GAEpB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EACL,kBAAkB,EAClB,uBAAuB,EACvB,gBAAgB,GACjB,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EACL,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,aAAa,EACb,WAAW,GAEZ,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAc9C;;;GAGG;AACH,SAAS,mBAAmB,CAC1B,IAAyB,EACzB,OAAqC;IAErC,sCAAsC;IACtC,IAAI,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC;IACrD,IAAI,OAAO,CAAC,oBAAoB,KAAK,SAAS,EAAE,CAAC;QAC/C,oBAAoB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAC1D,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC;YAC7D,oBAAoB,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,OAAO;QACL,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;QACvD,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB;QACtE,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe;QAChE,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;QACvD,oBAAoB;QACpB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB;QACnE,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc;QAC7D,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB;QACnE,sBAAsB,EACpB,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,sBAAsB;QAC/D,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc;QAC7D,qBAAqB,EACnB,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAC,qBAAqB;KAC9D,CAAC;AACJ,CAAC;AAqID;;;GAGG;AACH,MAAM,OAAO,UAAU;IACb,QAAQ,CAAqB;IAC7B,QAAQ,GAAqB,IAAI,CAAC;IAClC,SAAS,CAAY;IACrB,cAAc,CAAiB;IAC/B,IAAI,CAAoB;IACxB,WAAW,CAAqB;IAChC,kBAAkB,CAA4B;IAC9C,aAAa,CAAsB;IACnC,YAAY,CAAS;IACrB,aAAa,CAAS;IACtB,WAAW,GAAG,KAAK,CAAC;IACpB,iBAAiB,GAAG,KAAK,CAAC;IAC1B,cAAc,CAAwB;IAE9C,YAAY,SAA2B,EAAE,EAAE,cAA+B;QACxE,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,qBAAqB,EAAE,CAAC;QAC3D,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,cAAc,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC;QAC9C,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,IAAI,IAAI,CAAC;QAC5D,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,mBAAmB,EAAE,CAAC;QACnE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,OAAO,CAAC;QACrD,IAAI,CAAC,cAAc,GAAG,cAAc,IAAI,IAAI,CAAC;QAE7C,2BAA2B;QAC3B,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;QACpD,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,OAAO,CAAC;QAEnD,uDAAuD;QACvD,IAAI,IAAI,CAAC,SAAS,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC5C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,CAAC;YACvE,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAC5C,IAAI,CAAC,SAAS,CAAC,UAAU,CAC1B,EAAE,CAAC;gBACF,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;oBAC5B,aAAa,CAAC,GAAG,CAAC,IAAe,EAAE,SAAS,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;YACD,IAAI,CAAC,aAAa,GAAG;gBACnB,GAAG,IAAI,CAAC,aAAa;gBACrB,oBAAoB,EAAE,aAAa;aACpC,CAAC;QACJ,CAAC;QAED,gCAAgC;QAChC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAE5D,+DAA+D;QAC/D,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;YAChC,IAAI,CAAC,aAAa,GAAG;gBACnB,GAAG,IAAI,CAAC,aAAa;gBACrB,qBAAqB,EAAE,IAAI;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,mDAAmD;QACnD,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACvC,IAAI,CAAC,QAAQ,GAAG,IAAI,YAAY,EAAE,CAAC;QACrC,CAAC;aAAM,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,KAAK,kBAAkB,EAAE,CAAC;YACzD,sDAAsD;YACtD,IACE,IAAI,CAAC,SAAS,CAAC,kBAAkB,KAAK,SAAS;gBAC/C,IAAI,CAAC,SAAS,CAAC,kBAAkB,KAAK,EAAE,EACxC,CAAC;gBACD,MAAM,IAAI,KAAK,CACb,2EAA2E;oBACzE,YAAY;oBACZ,wBAAwB;oBACxB,cAAc;oBACd,4BAA4B;oBAC5B,sCAAsC;oBACtC,sDAAsD;oBACtD,SAAS;oBACT,MAAM,CACT,CAAC;YACJ,CAAC;YAED,IAAI,CAAC,QAAQ,GAAG,6BAA6B,CAAC;gBAC5C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,kBAAkB;gBAC5C,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,sBAAsB;gBAC9C,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ;aAClC,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC5C,0BAA0B;YAC1B,IACE,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,SAAS;gBACtC,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,EAAE;gBAC/B,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,SAAS;gBACtC,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,EAAE,EAC/B,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;YACxE,CAAC;YAED,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;gBACnC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;gBACnC,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc;aAC9C,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,gEAAgE;YAChE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,MAAM,WAAW,CAC9D,IAAI,CAAC,SAAS,CAAC,IAAI,EACnB;gBACE,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,IAAI,IAAI;gBACjD,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,kBAAkB;gBAC7C,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ;aAClC,CACF,CAAC;YAEF,iBAAiB;YACjB,IAAI,QAAQ,GAAG,iBAAiB,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,kBAAkB,EAAE,CAAC;gBAC3C,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;gBACjE,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAa,CAAC;YACrD,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;YAED,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC;gBAC7B,SAAS;gBACT,SAAS;gBACT,QAAQ;gBACR,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc;aAC9C,CAAC,CAAC;QACL,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,sBAAsB,CAAC,CAAC;YAClD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC3B,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,mBAAmB,CAAC,CAAC;QACjD,CAAC;QAED,wCAAwC;QACxC,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;YAChC,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,YAAY,IAAI,IAAI,CAAC;YAE9D,qCAAqC;YACrC,MAAM,aAAa,GAAG,MAAM,uBAAuB,EAAE,CAAC;YACtD,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CACb,mEAAmE;wBACjE,mCAAmC;wBACnC,6EAA6E;wBAC7E,gCAAgC;wBAChC,sDAAsD,CACzD,CAAC;gBACJ,CAAC;gBAED,yBAAyB;gBACzB,MAAM,kBAAkB,CAAC;oBACvB,YAAY,EAAE,IAAI;oBAClB,UAAU,EAAE,IAAI,CAAC,cAAc,CAAC,kBAAkB;oBAClD,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;iBACvC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,CAAC,8BAA8B,CAAC,CAAC;YACjE,CAAC;YAED,iEAAiE;YACjE,MAAM,gBAAgB,EAAE,CAAC;YACzB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAChC,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,SAAS,CACb,IAAY,EACZ,MAAe,EACf,MAAqC,EACrC,cAA0B;QAE1B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,4DAA4D;QAC5D,6DAA6D;QAC7D,2DAA2D;QAC3D,MAAM,eAAe,GACnB,MAAM,KAAK,SAAS;YAClB,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC;YACjD,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC;QAEzB,6BAA6B;QAC7B,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAE1C,gCAAgC;QAChC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;QAE5E,wBAAwB;QACxB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAS,CAAC,OAAO,CAC5C,cAAc,EACd,eAAe,CAChB,CAAC;QACF,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC;QAEnC,qCAAqC;QACrC,MAAM,eAAe,GAAG,eAAe,CACrC,YAAY,EACZ,UAAU,EACV,eAAe,EACf,cAAc,CACf,CAAC;QAEF,iFAAiF;QACjF,kEAAkE;QAClE,MAAM,aAAa,GACjB,eAAe,CAAC,qBAAqB,KAAK,IAAI;YAC5C,CAAC,CAAC,uBAAuB,CAAC,eAAe,EAAE,cAAc,CAAC;YAC1D,CAAC,CAAC,eAAe,CAAC;QAEtB,8EAA8E;QAC9E,wFAAwF;QACxF,MAAM,qBAAqB,GAAG,eAAe,CAAC,qBAAqB;YACjE,CAAC,CAAC,sBAAsB,CAAC,aAAa,EAAE,cAAc,CAAC;YACvD,CAAC,CAAC,aAAa,CAAC;QAElB,yDAAyD;QACzD,8DAA8D;QAC9D,MAAM,eAAe,GAAG,eAAe,CAAC,qBAAqB;YAC3D,CAAC,CAAC,eAAe,CAAC,qBAAqB,EAAE;gBACrC,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,wBAAwB;aAC1F,CAAC;YACJ,CAAC,CAAC,qBAAqB,CAAC;QAE1B,yCAAyC;QACzC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,WAAW,CACtD,cAAc,EACd,eAAe,EACf,eAAe,EACf,cAAc,CACf,CAAC;QAEF,0BAA0B;QAC1B,MAAM,UAAU,GAAG,cAAc,CAC/B,cAAc,EACd,QAAQ,EACR,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,EACzB,eAAe,CAChB,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YACtB,mDAAmD;YACnD,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/C,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC,CAAC;YACJ,sCAAsC;YACtC,OAAO,CAAC,IAAI,CAAC,sBAAsB,EAAE,UAAU,CAAC,CAAC;QACnD,CAAC;QAED,sBAAsB;QACtB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,KAAK,GAAuB;YAChC,YAAY,EAAE,mBAAmB,CAAC,QAAQ,CAAC;YAC3C,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,gBAAgB,EAAE,OAAO,GAAG,SAAS;YACrC,cAAc,EAAE,UAAU,CAAC,cAAc;SAC1C,CAAC;QAEF,2DAA2D;QAC3D,MAAM,YAAY,GAAuC,QAAQ,CAAC,GAAG,CACnE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAC3C,CAAC;QAEF,sDAAsD;QACtD,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9B,gDAAgD;YAChD,OAAO;gBACL,cAAc;gBACd,QAAQ,EAAE,YAAY;gBACtB,KAAK;aACN,CAAC;QACJ,CAAC;QAED,iDAAiD;QACjD,MAAM,aAAa,GACjB,IAAI,CAAC,WAAW,KAAK,IAAI;YACvB,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;YACjC,CAAC,CAAC,WAAW,EAAE,CAAC;QAEpB,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAEnE,OAAO;YACL,cAAc;YACd,QAAQ,EAAE,YAAY;YACtB,MAAM,EAAE,eAAe;YACvB,KAAK;SACN,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAChC,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,OAAO,CAAC,SAAiB;QACvB,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,+DAA+D;gBAC7D,2EAA2E;gBAC3E,iCAAiC;gBACjC,iFAAiF;gBACjF,2CAA2C;gBAC3C,6BAA6B;gBAC7B,6DAA6D;gBAC7D,+CAA+C;gBAC/C,OAAO,CACV,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,KAAK,IAAI,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,+DAA+D;gBAC7D,mDAAmD;gBACnD,2CAA2C;gBAC3C,6DAA6D;gBAC7D,+CAA+C;gBAC/C,OAAO,CACV,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,wDAAwD;gBACtD,6EAA6E;gBAC7E,8CAA8C;gBAC9C,2CAA2C;gBAC3C,oCAAoC;gBACpC,+CAA+C;gBAC/C,OAAO,CACV,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CACb,4DAA4D,CAC7D,CAAC;QACJ,CAAC;QAED,OAAO,IAAI,CAAC,cAAc,CACxB,IAAI,EACJ,SAAS,EACT,IAAI,CAAC,kBAAkB,EACvB,IAAI,CAAC,WAAW,CACjB,CAAC;IACJ,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAyB,EACzB,cAA+B;IAE/B,OAAO,IAAI,UAAU,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;AAChD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,MAAe,EACf,MAAqC;IAErC,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;IACtC,MAAM,UAAU,CAAC,UAAU,EAAE,CAAC;IAE9B,IAAI,CAAC;QACH,OAAO,MAAM,UAAU,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1D,CAAC;YAAS,CAAC;QACT,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,MAAqC;IAErC,kCAAkC;IAClC,MAAM,eAAe,GAAiC;QACpD,GAAG,MAAM;QACT,eAAe,EAAE,IAAI,GAAG,EAAE,EAAE,wBAAwB;KACrD,CAAC;IAEF,OAAO,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,SAAwE,EACxE,MAAqC;IAErC,MAAM,UAAU,GAAG,gBAAgB,CAAC;QAClC,GAAG,EAAE;YACH,IAAI,EAAE,SAAS,CAAC,IAAI,IAAI,WAAW;YACnC,GAAG,SAAS;SACb;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,CAAC,UAAU,EAAE,CAAC;IAE9B,IAAI,CAAC;QACH,OAAO,MAAM,UAAU,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IAC7D,CAAC;YAAS,CAAC;QACT,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"anonymizer.js","sourceRoot":"","sources":["../../src/core/anonymizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EASL,mBAAmB,GACpB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,qBAAqB,GAEtB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAEL,YAAY,EACZ,cAAc,EACd,6BAA6B,EAC7B,iBAAiB,GAElB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAEL,WAAW,GAEZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EACL,WAAW,EACX,mBAAmB,GAEpB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EACL,kBAAkB,EAClB,uBAAuB,EACvB,gBAAgB,GACjB,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EACL,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,aAAa,EACb,WAAW,GAEZ,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAc9C;;;GAGG;AACH,SAAS,mBAAmB,CAC1B,IAAyB,EACzB,OAAqC;IAErC,sCAAsC;IACtC,IAAI,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC;IACrD,IAAI,OAAO,CAAC,oBAAoB,KAAK,SAAS,EAAE,CAAC;QAC/C,oBAAoB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAC1D,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC;YAC7D,oBAAoB,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,OAAO;QACL,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;QACvD,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB;QACtE,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe;QAChE,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;QACvD,oBAAoB;QACpB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB;QACnE,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc;QAC7D,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB;QACnE,sBAAsB,EACpB,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,sBAAsB;QAC/D,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc;QAC7D,qBAAqB,EACnB,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAC,qBAAqB;KAC9D,CAAC;AACJ,CAAC;AAqJD;;;GAGG;AACH,MAAM,OAAO,UAAU;IACb,QAAQ,CAAqB;IAC7B,QAAQ,GAAqB,IAAI,CAAC;IAClC,SAAS,CAAY;IACrB,cAAc,CAAiB;IAC/B,IAAI,CAAoB;IACxB,WAAW,CAAqB;IAChC,kBAAkB,CAA4B;IAC9C,aAAa,CAAsB;IACnC,YAAY,CAAS;IACrB,aAAa,CAAS;IACtB,WAAW,GAAG,KAAK,CAAC;IACpB,iBAAiB,GAAG,KAAK,CAAC;IAC1B,cAAc,CAAwB;IAE9C,YAAY,SAA2B,EAAE,EAAE,cAA+B;QACxE,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,qBAAqB,EAAE,CAAC;QAC3D,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,cAAc,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC;QAC9C,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,IAAI,IAAI,CAAC;QAC5D,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,mBAAmB,EAAE,CAAC;QACnE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,OAAO,CAAC;QACrD,IAAI,CAAC,cAAc,GAAG,cAAc,IAAI,IAAI,CAAC;QAE7C,2BAA2B;QAC3B,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;QACpD,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,OAAO,CAAC;QAEnD,uDAAuD;QACvD,IAAI,IAAI,CAAC,SAAS,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC5C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,CAAC;YACvE,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAC5C,IAAI,CAAC,SAAS,CAAC,UAAU,CAC1B,EAAE,CAAC;gBACF,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;oBAC5B,aAAa,CAAC,GAAG,CAAC,IAAe,EAAE,SAAS,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;YACD,IAAI,CAAC,aAAa,GAAG;gBACnB,GAAG,IAAI,CAAC,aAAa;gBACrB,oBAAoB,EAAE,aAAa;aACpC,CAAC;QACJ,CAAC;QAED,gCAAgC;QAChC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAE5D,+DAA+D;QAC/D,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;YAChC,IAAI,CAAC,aAAa,GAAG;gBACnB,GAAG,IAAI,CAAC,aAAa;gBACrB,qBAAqB,EAAE,IAAI;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,mDAAmD;QACnD,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACvC,IAAI,CAAC,QAAQ,GAAG,IAAI,YAAY,EAAE,CAAC;QACrC,CAAC;aAAM,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,KAAK,kBAAkB,EAAE,CAAC;YACzD,sDAAsD;YACtD,IACE,IAAI,CAAC,SAAS,CAAC,kBAAkB,KAAK,SAAS;gBAC/C,IAAI,CAAC,SAAS,CAAC,kBAAkB,KAAK,EAAE,EACxC,CAAC;gBACD,MAAM,IAAI,KAAK,CACb,2EAA2E;oBACzE,YAAY;oBACZ,wBAAwB;oBACxB,cAAc;oBACd,4BAA4B;oBAC5B,sCAAsC;oBACtC,sDAAsD;oBACtD,SAAS;oBACT,MAAM,CACT,CAAC;YACJ,CAAC;YAED,IAAI,CAAC,QAAQ,GAAG,6BAA6B,CAAC;gBAC5C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,kBAAkB;gBAC5C,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,sBAAsB;gBAC9C,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ;aAClC,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC5C,0BAA0B;YAC1B,IACE,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,SAAS;gBACtC,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,EAAE;gBAC/B,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,SAAS;gBACtC,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,EAAE,EAC/B,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;YACxE,CAAC;YAED,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;gBACnC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;gBACnC,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc;gBAC7C,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY;gBACzC,mBAAmB,EAAE,IAAI,CAAC,SAAS,CAAC,mBAAmB;aACxD,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,gEAAgE;YAChE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,MAAM,WAAW,CAC9D,IAAI,CAAC,SAAS,CAAC,IAAI,EACnB;gBACE,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,IAAI,IAAI;gBACjD,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,kBAAkB;gBAC7C,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ;aAClC,CACF,CAAC;YAEF,iBAAiB;YACjB,IAAI,QAAQ,GAAG,iBAAiB,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,kBAAkB,EAAE,CAAC;gBAC3C,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;gBACjE,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAa,CAAC;YACrD,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;YAED,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC;gBAC7B,SAAS;gBACT,SAAS;gBACT,QAAQ;gBACR,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc;gBAC7C,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY;gBACzC,mBAAmB,EAAE,IAAI,CAAC,SAAS,CAAC,mBAAmB;aACxD,CAAC,CAAC;QACL,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,sBAAsB,CAAC,CAAC;YAClD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC3B,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,mBAAmB,CAAC,CAAC;QACjD,CAAC;QAED,wCAAwC;QACxC,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;YAChC,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,YAAY,IAAI,IAAI,CAAC;YAE9D,qCAAqC;YACrC,MAAM,aAAa,GAAG,MAAM,uBAAuB,EAAE,CAAC;YACtD,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CACb,mEAAmE;wBACjE,mCAAmC;wBACnC,6EAA6E;wBAC7E,gCAAgC;wBAChC,sDAAsD,CACzD,CAAC;gBACJ,CAAC;gBAED,yBAAyB;gBACzB,MAAM,kBAAkB,CAAC;oBACvB,YAAY,EAAE,IAAI;oBAClB,UAAU,EAAE,IAAI,CAAC,cAAc,CAAC,kBAAkB;oBAClD,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;iBACvC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,CAAC,8BAA8B,CAAC,CAAC;YACjE,CAAC;YAED,iEAAiE;YACjE,MAAM,gBAAgB,EAAE,CAAC;YACzB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAChC,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,SAAS,CACb,IAAY,EACZ,MAAe,EACf,MAAqC,EACrC,cAA0B;QAE1B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,4DAA4D;QAC5D,6DAA6D;QAC7D,2DAA2D;QAC3D,MAAM,eAAe,GACnB,MAAM,KAAK,SAAS;YAClB,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC;YACjD,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC;QAEzB,6BAA6B;QAC7B,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAE1C,gCAAgC;QAChC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;QAE5E,wBAAwB;QACxB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAS,CAAC,OAAO,CAC5C,cAAc,EACd,eAAe,CAChB,CAAC;QACF,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC;QAEnC,qCAAqC;QACrC,MAAM,eAAe,GAAG,eAAe,CACrC,YAAY,EACZ,UAAU,EACV,eAAe,EACf,cAAc,CACf,CAAC;QAEF,iFAAiF;QACjF,kEAAkE;QAClE,MAAM,aAAa,GACjB,eAAe,CAAC,qBAAqB,KAAK,IAAI;YAC5C,CAAC,CAAC,uBAAuB,CAAC,eAAe,EAAE,cAAc,CAAC;YAC1D,CAAC,CAAC,eAAe,CAAC;QAEtB,8EAA8E;QAC9E,wFAAwF;QACxF,MAAM,qBAAqB,GAAG,eAAe,CAAC,qBAAqB;YACjE,CAAC,CAAC,sBAAsB,CAAC,aAAa,EAAE,cAAc,CAAC;YACvD,CAAC,CAAC,aAAa,CAAC;QAElB,yDAAyD;QACzD,8DAA8D;QAC9D,MAAM,eAAe,GAAG,eAAe,CAAC,qBAAqB;YAC3D,CAAC,CAAC,eAAe,CAAC,qBAAqB,EAAE;gBACrC,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,wBAAwB;aAC1F,CAAC;YACJ,CAAC,CAAC,qBAAqB,CAAC;QAE1B,yCAAyC;QACzC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,WAAW,CACtD,cAAc,EACd,eAAe,EACf,eAAe,EACf,cAAc,CACf,CAAC;QAEF,0BAA0B;QAC1B,MAAM,UAAU,GAAG,cAAc,CAC/B,cAAc,EACd,QAAQ,EACR,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,EACzB,eAAe,CAChB,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YACtB,mDAAmD;YACnD,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/C,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC,CAAC;YACJ,sCAAsC;YACtC,OAAO,CAAC,IAAI,CAAC,sBAAsB,EAAE,UAAU,CAAC,CAAC;QACnD,CAAC;QAED,sBAAsB;QACtB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,KAAK,GAAuB;YAChC,YAAY,EAAE,mBAAmB,CAAC,QAAQ,CAAC;YAC3C,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,gBAAgB,EAAE,OAAO,GAAG,SAAS;YACrC,cAAc,EAAE,UAAU,CAAC,cAAc;SAC1C,CAAC;QAEF,2DAA2D;QAC3D,MAAM,YAAY,GAAuC,QAAQ,CAAC,GAAG,CACnE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAC3C,CAAC;QAEF,sDAAsD;QACtD,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9B,gDAAgD;YAChD,OAAO;gBACL,cAAc;gBACd,QAAQ,EAAE,YAAY;gBACtB,KAAK;aACN,CAAC;QACJ,CAAC;QAED,iDAAiD;QACjD,MAAM,aAAa,GACjB,IAAI,CAAC,WAAW,KAAK,IAAI;YACvB,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;YACjC,CAAC,CAAC,WAAW,EAAE,CAAC;QAEpB,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAEnE,OAAO;YACL,cAAc;YACd,QAAQ,EAAE,YAAY;YACtB,MAAM,EAAE,eAAe;YACvB,KAAK;SACN,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAChC,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,OAAO,CAAC,SAAiB;QACvB,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,+DAA+D;gBAC7D,2EAA2E;gBAC3E,iCAAiC;gBACjC,iFAAiF;gBACjF,2CAA2C;gBAC3C,6BAA6B;gBAC7B,6DAA6D;gBAC7D,+CAA+C;gBAC/C,OAAO,CACV,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,KAAK,IAAI,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,+DAA+D;gBAC7D,mDAAmD;gBACnD,2CAA2C;gBAC3C,6DAA6D;gBAC7D,+CAA+C;gBAC/C,OAAO,CACV,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,wDAAwD;gBACtD,6EAA6E;gBAC7E,8CAA8C;gBAC9C,2CAA2C;gBAC3C,oCAAoC;gBACpC,+CAA+C;gBAC/C,OAAO,CACV,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CACb,4DAA4D,CAC7D,CAAC;QACJ,CAAC;QAED,OAAO,IAAI,CAAC,cAAc,CACxB,IAAI,EACJ,SAAS,EACT,IAAI,CAAC,kBAAkB,EACvB,IAAI,CAAC,WAAW,CACjB,CAAC;IACJ,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAyB,EACzB,cAA+B;IAE/B,OAAO,IAAI,UAAU,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;AAChD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,MAAe,EACf,MAAqC;IAErC,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;IACtC,MAAM,UAAU,CAAC,UAAU,EAAE,CAAC;IAE9B,IAAI,CAAC;QACH,OAAO,MAAM,UAAU,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1D,CAAC;YAAS,CAAC;QACT,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,MAAqC;IAErC,kCAAkC;IAClC,MAAM,eAAe,GAAiC;QACpD,GAAG,MAAM;QACT,eAAe,EAAE,IAAI,GAAG,EAAE,EAAE,wBAAwB;KACrD,CAAC;IAEF,OAAO,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,SAAwE,EACxE,MAAqC;IAErC,MAAM,UAAU,GAAG,gBAAgB,CAAC;QAClC,GAAG,EAAE;YACH,IAAI,EAAE,SAAS,CAAC,IAAI,IAAI,WAAW;YACnC,GAAG,SAAS;SACb;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,CAAC,UAAU,EAAE,CAAC;IAE9B,IAAI,CAAC;QACH,OAAO,MAAM,UAAU,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IAC7D,CAAC;YAAS,CAAC;QACT,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC"}
|
package/dist/ner/ner-model.d.ts
CHANGED
|
@@ -23,6 +23,18 @@ export interface NERModelConfig {
|
|
|
23
23
|
modelVersion: string;
|
|
24
24
|
/** ONNX session options for performance tuning */
|
|
25
25
|
sessionOptions?: OrtSessionOptions;
|
|
26
|
+
/**
|
|
27
|
+
* Enable case-insensitive fallback pass for detecting lowercase names.
|
|
28
|
+
* Runs a second NER pass on title-cased text and merges new detections.
|
|
29
|
+
* @default false
|
|
30
|
+
*/
|
|
31
|
+
caseFallback?: boolean;
|
|
32
|
+
/**
|
|
33
|
+
* Confidence penalty multiplier for case-fallback detections (0.0 - 1.0).
|
|
34
|
+
* Applied as: confidence * caseFallbackPenalty
|
|
35
|
+
* @default 0.85
|
|
36
|
+
*/
|
|
37
|
+
caseFallbackPenalty?: number;
|
|
26
38
|
}
|
|
27
39
|
/**
|
|
28
40
|
* NER prediction result for a single text
|
|
@@ -53,6 +65,12 @@ export declare class NERModel {
|
|
|
53
65
|
* Loads the model and tokenizer
|
|
54
66
|
*/
|
|
55
67
|
load(): Promise<void>;
|
|
68
|
+
/**
|
|
69
|
+
* Runs a single NER pass: tokenize, infer, decode BIO tags.
|
|
70
|
+
* @param inputText - Text to tokenize and feed to the model
|
|
71
|
+
* @param originalText - Original text used for extracting entity text (may differ in casing)
|
|
72
|
+
*/
|
|
73
|
+
private runNERPass;
|
|
56
74
|
/**
|
|
57
75
|
* Predicts entities in text
|
|
58
76
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ner-model.d.ts","sourceRoot":"","sources":["../../src/ner/ner-model.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAIL,KAAK,iBAAiB,EACvB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAcnE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,WAAW,EAAE,OAAO,CAAC;IACrB,iCAAiC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,cAAc,CAAC,EAAE,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"ner-model.d.ts","sourceRoot":"","sources":["../../src/ner/ner-model.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAIL,KAAK,iBAAiB,EACvB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAcnE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,WAAW,EAAE,OAAO,CAAC;IACrB,iCAAiC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,cAAc,CAAC,EAAE,iBAAiB,CAAC;IACnC;;;;OAIG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,4BAA4B;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB,UAU7B,CAAC;AA8BF;;GAEG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,GAAG,CAA2B;IACtC,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,SAAS,CAAmC;IACpD,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,QAAQ,CAAS;gBAEb,MAAM,EAAE,cAAc;IAIlC;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA0C3B;;;;OAIG;YACW,UAAU;IAwBxB;;OAEG;IACG,OAAO,CACX,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,mBAAmB,GAC3B,OAAO,CAAC,aAAa,CAAC;IAyDzB;;OAEG;YACW,YAAY;IAiE1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAqCrB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAexB;;OAEG;IACH,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED;;OAEG;IACH,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAOzB;AAoBD;;GAEG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACzE,QAAQ,CAcV;AAED;;;GAGG;AACH,qBAAa,YAAY;IACvB,QAAQ,CAAC,OAAO,gBAAgB;IAChC,QAAQ,CAAC,MAAM,QAAQ;IAEjB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,OAAO,CACL,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,mBAAmB,GAC5B,OAAO,CAAC,aAAa,CAAC;IAQzB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAIzB;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,YAAY,CAEjD;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAC5E,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B"}
|
package/dist/ner/ner-model.js
CHANGED
|
@@ -87,6 +87,22 @@ export class NERModel {
|
|
|
87
87
|
});
|
|
88
88
|
this.isLoaded = true;
|
|
89
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* Runs a single NER pass: tokenize, infer, decode BIO tags.
|
|
92
|
+
* @param inputText - Text to tokenize and feed to the model
|
|
93
|
+
* @param originalText - Original text used for extracting entity text (may differ in casing)
|
|
94
|
+
*/
|
|
95
|
+
async runNERPass(inputText, originalText, minConfidence) {
|
|
96
|
+
// Tokenize the input text (may be case-modified)
|
|
97
|
+
const tokenization = this.tokenizer.tokenize(inputText);
|
|
98
|
+
// Run inference
|
|
99
|
+
const { labels, confidences } = await this.runInference(tokenization);
|
|
100
|
+
// Decode BIO tags using original text for entity text extraction
|
|
101
|
+
// (offsets are identical since casing doesn't change string length)
|
|
102
|
+
const rawEntities = decodeBIOTags(tokenization.tokens, labels, confidences, originalText);
|
|
103
|
+
// Convert to SpanMatch format with confidence filtering
|
|
104
|
+
return convertToSpanMatches(rawEntities, minConfidence);
|
|
105
|
+
}
|
|
90
106
|
/**
|
|
91
107
|
* Predicts entities in text
|
|
92
108
|
*/
|
|
@@ -95,15 +111,28 @@ export class NERModel {
|
|
|
95
111
|
if (!this.isLoaded || this.session === null || this.tokenizer === null) {
|
|
96
112
|
throw new Error("Model not loaded. Call load() first.");
|
|
97
113
|
}
|
|
98
|
-
// Tokenize input
|
|
99
|
-
const tokenization = this.tokenizer.tokenize(text);
|
|
100
|
-
// Run inference
|
|
101
|
-
const { labels, confidences } = await this.runInference(tokenization);
|
|
102
|
-
// Decode BIO tags to entities
|
|
103
|
-
const rawEntities = decodeBIOTags(tokenization.tokens, labels, confidences, text);
|
|
104
|
-
// Convert to SpanMatch format with confidence filtering
|
|
105
114
|
const minConfidence = this.getMinConfidence(policy);
|
|
106
|
-
|
|
115
|
+
// Primary NER pass on original text
|
|
116
|
+
let spans = await this.runNERPass(text, text, minConfidence);
|
|
117
|
+
// Case fallback: run a second pass on title-cased text to catch lowercase names
|
|
118
|
+
const caseFallback = this.config.caseFallback ?? false;
|
|
119
|
+
if (caseFallback) {
|
|
120
|
+
const titleCased = titleCaseWords(text);
|
|
121
|
+
if (titleCased !== text) {
|
|
122
|
+
const penalty = this.config.caseFallbackPenalty ?? 0.85;
|
|
123
|
+
const fallbackSpans = await this.runNERPass(titleCased, text, minConfidence);
|
|
124
|
+
// Merge only non-overlapping fallback detections with a confidence penalty
|
|
125
|
+
for (const fallback of fallbackSpans) {
|
|
126
|
+
const overlaps = spans.some((primary) => primary.start < fallback.end && fallback.start < primary.end);
|
|
127
|
+
if (!overlaps) {
|
|
128
|
+
spans.push({
|
|
129
|
+
...fallback,
|
|
130
|
+
confidence: fallback.confidence * penalty,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
107
136
|
// Post-process spans
|
|
108
137
|
spans = cleanupSpanBoundaries(spans, text);
|
|
109
138
|
spans = mergeAdjacentSpans(spans, text);
|
|
@@ -224,6 +253,13 @@ export class NERModel {
|
|
|
224
253
|
return Promise.resolve();
|
|
225
254
|
}
|
|
226
255
|
}
|
|
256
|
+
/**
|
|
257
|
+
* Capitalizes the first letter of each word for case-insensitive NER fallback.
|
|
258
|
+
* Preserves string length so character offsets remain valid.
|
|
259
|
+
*/
|
|
260
|
+
function titleCaseWords(text) {
|
|
261
|
+
return text.replace(/\b[a-z]/g, (ch) => ch.toUpperCase());
|
|
262
|
+
}
|
|
227
263
|
/**
|
|
228
264
|
* Softmax function for probability calculation
|
|
229
265
|
*/
|
|
@@ -245,6 +281,8 @@ export function createNERModel(config) {
|
|
|
245
281
|
doLowerCase: config.doLowerCase ?? false, // XLM-RoBERTa is cased
|
|
246
282
|
modelVersion: config.modelVersion ?? "1.0.0",
|
|
247
283
|
sessionOptions: config.sessionOptions,
|
|
284
|
+
caseFallback: config.caseFallback,
|
|
285
|
+
caseFallbackPenalty: config.caseFallbackPenalty,
|
|
248
286
|
};
|
|
249
287
|
return new NERModel(fullConfig);
|
|
250
288
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ner-model.js","sourceRoot":"","sources":["../../src/ner/ner-model.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,WAAW,EACX,cAAc,GAGf,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,kBAAkB,EAClB,iBAAiB,GAElB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"ner-model.js","sourceRoot":"","sources":["../../src/ner/ner-model.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,WAAW,EACX,cAAc,GAGf,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,kBAAkB,EAClB,iBAAiB,GAElB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AA8CzD;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,GAAG;IACH,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,QAAQ;IACR,QAAQ;CACT,CAAC;AAEF;;;;GAIG;AACH,SAAS,mBAAmB,CAC1B,WAAkC,EAClC,aAAiC;IAEjC,MAAM,QAAQ,GAAsB;QAClC,sBAAsB,EAAE,KAAK;QAC7B,iBAAiB,EAAE,IAAI;QACvB,gBAAgB,EAAE,IAAI;QACtB,iBAAiB,EAAE,CAAC;QACpB,iBAAiB,EAAE,CAAC;KACrB,CAAC;IAEF,+CAA+C;IAC/C,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;QAC1B,wDAAwD;QACxD,QAAQ,CAAC,kBAAkB,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACnD,CAAC;IACD,wEAAwE;IAExE,+DAA+D;IAC/D,OAAO,EAAE,GAAG,QAAQ,EAAE,GAAG,aAAa,EAAE,CAAC;AAC3C,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,QAAQ;IACX,GAAG,GAAsB,IAAI,CAAC;IAC9B,OAAO,GAAY,IAAI,CAAC;IACxB,SAAS,GAA8B,IAAI,CAAC;IAC5C,MAAM,CAAiB;IACvB,QAAQ,GAAG,KAAK,CAAC;IAEzB,YAAY,MAAsB;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAE1B,gEAAgE;QAChE,IAAI,CAAC,GAAG,GAAG,MAAM,WAAW,EAAE,CAAC;QAE/B,mEAAmE;QACnE,MAAM,WAAW,GAAG,cAAc,EAAE,CAAC;QACrC,MAAM,cAAc,GAAG,mBAAmB,CACxC,WAAW,EACX,IAAI,CAAC,MAAM,CAAC,cAAc,CAC3B,CAAC;QAEF,kBAAkB;QAClB,oFAAoF;QACpF,4FAA4F;QAC5F,IAAI,SAAS,EAAE,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,MAAM,kBAAkB,EAAE,CAAC;YAC3C,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAChE,8CAA8C;YAC9C,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CACnD,SAAS,EACT,cAAc,CACf,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CACnD,IAAI,CAAC,MAAM,CAAC,SAAS,EACrB,cAAc,CACf,CAAC;QACJ,CAAC;QAED,0EAA0E;QAC1E,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,SAAS,GAAG,IAAI,kBAAkB,CAAC,KAAK,EAAE;YAC7C,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;YAChC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;SACrC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,UAAU,CACtB,SAAiB,EACjB,YAAoB,EACpB,aAAqB;QAErB,iDAAiD;QACjD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAEzD,gBAAgB;QAChB,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAEtE,iEAAiE;QACjE,oEAAoE;QACpE,MAAM,WAAW,GAAG,aAAa,CAC/B,YAAY,CAAC,MAAM,EACnB,MAAM,EACN,WAAW,EACX,YAAY,CACb,CAAC;QAEF,wDAAwD;QACxD,OAAO,oBAAoB,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CACX,IAAY,EACZ,MAA4B;QAE5B,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAEpD,oCAAoC;QACpC,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC;QAE7D,gFAAgF;QAChF,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,KAAK,CAAC;QACvD,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;gBACxB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC;gBACxD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC;gBAE7E,2EAA2E;gBAC3E,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;oBACrC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CACzB,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,GAAG,IAAI,QAAQ,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAC1E,CAAC;oBACF,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACd,KAAK,CAAC,IAAI,CAAC;4BACT,GAAG,QAAQ;4BACX,UAAU,EAAE,QAAQ,CAAC,UAAU,GAAG,OAAO;yBAC1C,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,KAAK,GAAG,qBAAqB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC3C,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAExC,oCAAoC;QACpC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,KAAK,GAAG,KAAK,CAAC,MAAM,CAClB,CAAC,IAAI,EAAE,EAAE,CACP,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;gBAClC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CACxC,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAElC,OAAO;YACL,KAAK;YACL,gBAAgB,EAAE,OAAO,GAAG,SAAS;YACrC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY;SACvC,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CACxB,YAAgC;QAEhC,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAMpB,CAAC;QAEF,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE/C,iBAAiB;QACjB,MAAM,cAAc,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CACxC,OAAO,EACP,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EACrD,CAAC,CAAC,EAAE,SAAS,CAAC,CACf,CAAC;QAEF,MAAM,mBAAmB,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAC7C,OAAO,EACP,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAC1D,CAAC,CAAC,EAAE,SAAS,CAAC,CACf,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAC5C,OAAO,EACP,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EACzD,CAAC,CAAC,EAAE,SAAS,CAAC,CACf,CAAC;QAEF,gBAAgB;QAChB,MAAM,KAAK,GAA4B;YACrC,SAAS,EAAE,cAAc;YACzB,cAAc,EAAE,mBAAmB;SACpC,CAAC;QAEF,uCAAuC;QACvC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACtC,IAAI,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC1C,KAAK,CAAC,gBAAgB,CAAC,GAAG,kBAAkB,CAAC;QAC/C,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEzC,oBAAoB;QACpB,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACnC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QAED,+CAA+C;QAC/C,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACK,aAAa,CACnB,MAA8B,EAC9B,SAAiB;QAEjB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE9C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,4BAA4B;YAC5B,MAAM,WAAW,GAAa,EAAE,CAAC;YACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YACjD,CAAC;YAED,gBAAgB;YAChB,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;YAEnC,aAAa;YACb,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC;oBAC9B,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;oBACxB,MAAM,GAAG,CAAC,CAAC;gBACb,CAAC;YACH,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;YACjD,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5B,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IACjC,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,MAA4B;QACnD,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,GAAG,CAAC;QAErC,yCAAyC;QACzC,IAAI,YAAY,GAAG,GAAG,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;YAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC;YAC/D,IAAI,SAAS,GAAG,YAAY,EAAE,CAAC;gBAC7B,YAAY,GAAG,SAAS,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,OAAO;QACL,+EAA+E;QAC/E,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;CACF;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;AAC5D,CAAC;AAED;;GAEG;AACH,SAAS,OAAO,CAAC,MAAgB;IAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACpD,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC5B,MAA0E;IAE1E,MAAM,UAAU,GAAmB;QACjC,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,iBAAiB;QAC9C,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,GAAG;QAClC,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,KAAK,EAAE,uBAAuB;QACjE,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,OAAO;QAC5C,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;KAChD,CAAC;IAEF,OAAO,IAAI,QAAQ,CAAC,UAAU,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,YAAY;IACd,OAAO,GAAG,YAAY,CAAC;IACvB,MAAM,GAAG,IAAI,CAAC;IAEvB,KAAK,CAAC,IAAI;QACR,QAAQ;IACV,CAAC;IAED,OAAO,CACL,KAAa,EACb,OAA6B;QAE7B,OAAO,OAAO,CAAC,OAAO,CAAC;YACrB,KAAK,EAAE,EAAE;YACT,gBAAgB,EAAE,CAAC;YACnB,YAAY,EAAE,IAAI,CAAC,OAAO;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,QAAQ;QACR,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,OAAO,IAAI,YAAY,EAAE,CAAC;AAC5B,CAAC"}
|