rehydra 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -3
- package/dist/browser.d.ts +54 -0
- package/dist/browser.d.ts.map +1 -0
- package/dist/browser.js +74 -0
- package/dist/browser.js.map +1 -0
- package/dist/core/anonymizer.d.ts +208 -0
- package/dist/core/anonymizer.d.ts.map +1 -0
- package/dist/core/anonymizer.js +411 -0
- package/dist/core/anonymizer.js.map +1 -0
- package/dist/core/index.d.ts +6 -0
- package/dist/core/index.d.ts.map +1 -0
- package/dist/core/index.js +6 -0
- package/dist/core/index.js.map +1 -0
- package/dist/index.d.ts +6 -166
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +13 -370
- package/dist/index.js.map +1 -1
- package/dist/storage/browser.d.ts +13 -0
- package/dist/storage/browser.d.ts.map +1 -0
- package/dist/storage/browser.js +13 -0
- package/dist/storage/browser.js.map +1 -0
- package/dist/storage/session.d.ts.map +1 -1
- package/dist/storage/session.js +19 -3
- package/dist/storage/session.js.map +1 -1
- package/dist/storage/sqlite.js +1 -1
- package/dist/storage/sqlite.js.map +1 -1
- package/dist/utils/storage.js +2 -2
- package/dist/utils/storage.js.map +1 -1
- package/package.json +33 -1
package/README.md
CHANGED
|
@@ -32,6 +32,8 @@ npm install rehydra
|
|
|
32
32
|
npm install rehydra onnxruntime-web
|
|
33
33
|
```
|
|
34
34
|
|
|
35
|
+
When using Vite, webpack, or other bundlers, the browser-safe entry point is automatically selected via [conditional exports](https://nodejs.org/api/packages.html#conditional-exports). This entry point excludes Node.js-specific modules like SQLite storage.
|
|
36
|
+
|
|
35
37
|
### Browser (without bundler)
|
|
36
38
|
|
|
37
39
|
```html
|
|
@@ -492,9 +494,11 @@ For applications that need to persist encrypted PII maps (e.g., chat application
|
|
|
492
494
|
| Provider | Environment | Persistence | Use Case |
|
|
493
495
|
|----------|-------------|-------------|----------|
|
|
494
496
|
| `InMemoryPIIStorageProvider` | All | None (lost on restart) | Development, testing |
|
|
495
|
-
| `SQLitePIIStorageProvider` | Node.js, Bun | File-based | Server-side applications |
|
|
497
|
+
| `SQLitePIIStorageProvider` | Node.js, Bun only* | File-based | Server-side applications |
|
|
496
498
|
| `IndexedDBPIIStorageProvider` | Browser | Browser storage | Client-side applications |
|
|
497
499
|
|
|
500
|
+
*\*Not available in browser builds. Use `IndexedDBPIIStorageProvider` for browser applications.*
|
|
501
|
+
|
|
498
502
|
### Important: Storage Only Works with Sessions
|
|
499
503
|
|
|
500
504
|
> **Note:** The `piiStorageProvider` is only used when you call `anonymizer.session()`.
|
|
@@ -624,12 +628,16 @@ await session.rehydrate(msg2.anonymizedText); // ✓
|
|
|
624
628
|
|
|
625
629
|
This ensures that follow-up messages referencing the same PII produce consistent placeholders, and rehydration works correctly across the entire conversation.
|
|
626
630
|
|
|
627
|
-
### SQLite Provider (Node.js + Bun)
|
|
631
|
+
### SQLite Provider (Node.js + Bun only)
|
|
632
|
+
|
|
633
|
+
The SQLite provider works on both Node.js and Bun with automatic runtime detection.
|
|
628
634
|
|
|
629
|
-
|
|
635
|
+
> **Note:** `SQLitePIIStorageProvider` is **not available in browser builds**. When bundling for browser with Vite/webpack, use `IndexedDBPIIStorageProvider` instead. The browser-safe build automatically excludes SQLite to avoid bundling Node.js dependencies.
|
|
630
636
|
|
|
631
637
|
```typescript
|
|
638
|
+
// Node.js / Bun only
|
|
632
639
|
import { SQLitePIIStorageProvider } from 'rehydra';
|
|
640
|
+
// Or explicitly: import { SQLitePIIStorageProvider } from 'rehydra/storage/sqlite';
|
|
633
641
|
|
|
634
642
|
// File-based database
|
|
635
643
|
const storage = new SQLitePIIStorageProvider('./data/pii-maps.db');
|
|
@@ -771,6 +779,52 @@ The library works seamlessly in browsers without any special configuration.
|
|
|
771
779
|
- **Offline support**: After initial download, everything works offline
|
|
772
780
|
- **Storage**: Uses IndexedDB and OPFS - data persists across sessions
|
|
773
781
|
|
|
782
|
+
### Bundler Support (Vite, webpack, esbuild)
|
|
783
|
+
|
|
784
|
+
The package uses [conditional exports](https://nodejs.org/api/packages.html#conditional-exports) to automatically provide a browser-safe build when bundling for the web. This means:
|
|
785
|
+
|
|
786
|
+
- **Automatic**: Vite, webpack, esbuild, and other modern bundlers will automatically use `dist/browser.js`
|
|
787
|
+
- **No Node.js modules**: The browser build excludes `SQLitePIIStorageProvider` and other Node.js-specific code
|
|
788
|
+
- **Tree-shakable**: Only the code you use is included in your bundle
|
|
789
|
+
|
|
790
|
+
```json
|
|
791
|
+
// package.json exports (simplified)
|
|
792
|
+
{
|
|
793
|
+
"exports": {
|
|
794
|
+
".": {
|
|
795
|
+
"browser": "./dist/browser.js",
|
|
796
|
+
"node": "./dist/index.js",
|
|
797
|
+
"default": "./dist/index.js"
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
```
|
|
802
|
+
|
|
803
|
+
**Explicit imports** (if needed):
|
|
804
|
+
|
|
805
|
+
```typescript
|
|
806
|
+
// Browser-only build (excludes SQLite, Node.js fs, etc.)
|
|
807
|
+
import { createAnonymizer } from 'rehydra/browser';
|
|
808
|
+
|
|
809
|
+
// Node.js build (includes everything)
|
|
810
|
+
import { createAnonymizer, SQLitePIIStorageProvider } from 'rehydra/node';
|
|
811
|
+
|
|
812
|
+
// SQLite storage only (Node.js only)
|
|
813
|
+
import { SQLitePIIStorageProvider } from 'rehydra/storage/sqlite';
|
|
814
|
+
```
|
|
815
|
+
|
|
816
|
+
**Browser build excludes:**
|
|
817
|
+
- `SQLitePIIStorageProvider` (use `IndexedDBPIIStorageProvider` instead)
|
|
818
|
+
- Node.js `fs`, `path`, `os` modules
|
|
819
|
+
|
|
820
|
+
**Browser build includes:**
|
|
821
|
+
- All recognizers (email, phone, IBAN, etc.)
|
|
822
|
+
- NER model support (with `onnxruntime-web`)
|
|
823
|
+
- Semantic enrichment
|
|
824
|
+
- `InMemoryPIIStorageProvider`
|
|
825
|
+
- `IndexedDBPIIStorageProvider`
|
|
826
|
+
- All crypto utilities
|
|
827
|
+
|
|
774
828
|
## Bun Support
|
|
775
829
|
|
|
776
830
|
This library works with [Bun](https://bun.sh). Since `onnxruntime-node` is a native Node.js addon, Bun uses `onnxruntime-web`:
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rehydra Module - Browser Entry Point
|
|
3
|
+
* Browser-safe entry point for on-device PII anonymization
|
|
4
|
+
*
|
|
5
|
+
* This module excludes Node.js-specific exports like SQLitePIIStorageProvider.
|
|
6
|
+
* For full functionality including SQLite, use the main entry point or import from 'rehydra/node'.
|
|
7
|
+
*/
|
|
8
|
+
export * from "./types/index.js";
|
|
9
|
+
export type { Recognizer } from "./recognizers/index.js";
|
|
10
|
+
export { RegexRecognizer, RecognizerRegistry, createDefaultRegistry, createRegistry, getGlobalRegistry, emailRecognizer, phoneRecognizer, ibanRecognizer, bicSwiftRecognizer, creditCardRecognizer, ipAddressRecognizer, urlRecognizer, createCustomIdRecognizer, createCaseIdRecognizer, createCustomerIdRecognizer, } from "./recognizers/index.js";
|
|
11
|
+
export { NERModel, NERModelStub, createNERModel, createNERModelStub, WordPieceTokenizer, loadVocabFromFile, parseVocab, loadRuntime, detectRuntime, getRuntimeType, type INERModel, type NERModelConfig, type NERPrediction, type NERModelMode, type DownloadProgressCallback, MODEL_REGISTRY, getModelCacheDir, isModelDownloaded, downloadModel, ensureModel, clearModelCache, listDownloadedModels, } from "./ner/index.js";
|
|
12
|
+
export { prenormalize, resolveEntities, tagEntities, validateOutput, generateTag, parseTag, rehydrate, enrichSemantics, inferGender, classifyLocation, getDatabaseStats, hasName, hasLocation, isSemanticDataAvailable, isSemanticDataDownloaded, getSemanticDataCacheDir, getDataDirectory, downloadSemanticData, ensureSemanticData, initializeSemanticData, loadSemanticData, clearSemanticData, clearSemanticDataCache, getSemanticDataInfo, SEMANTIC_DATA_FILES, extractTitle, extractTitlesFromSpans, mergeAdjacentTitleSpans, getTitlesForLanguage, getAllTitles, startsWithTitle, isOnlyTitle, type SemanticDataFileInfo, type EnricherConfig, type GenderResult, type LocationResult, type TitleExtractionResult, } from "./pipeline/index.js";
|
|
13
|
+
export type { KeyProvider } from "./crypto/index.js";
|
|
14
|
+
export { encryptPIIMap, decryptPIIMap, generateKey, deriveKey, generateSalt, InMemoryKeyProvider, ConfigKeyProvider, validateKey, secureCompare, uint8ArrayToBase64, base64ToUint8Array, } from "./crypto/index.js";
|
|
15
|
+
export { getStorageProvider, isNode, isBrowser, resetStorageProvider, setStorageProvider, type StorageProvider, } from "./utils/storage.js";
|
|
16
|
+
export { type PIIStorageProvider, type PIIMapMetadata, type StoredPIIMap, type ListOptions, type AnonymizerSession, InMemoryPIIStorageProvider, IndexedDBPIIStorageProvider, } from "./storage/browser.js";
|
|
17
|
+
export { join as pathJoin, dirname as pathDirname, basename as pathBasename, normalize as pathNormalize, extname as pathExtname, isAbsolute as pathIsAbsolute, } from "./utils/path.js";
|
|
18
|
+
import { Anonymizer as AnonymizerCore, anonymize as anonymizeCore, anonymizeRegexOnly as anonymizeRegexOnlyCore, anonymizeWithNER as anonymizeWithNERCore, type AnonymizerConfig, type NERConfig } from "./core/index.js";
|
|
19
|
+
export type { AnonymizerConfig, NERConfig };
|
|
20
|
+
/**
|
|
21
|
+
* Anonymizer instance
|
|
22
|
+
* Main class for performing PII anonymization
|
|
23
|
+
*/
|
|
24
|
+
export declare class Anonymizer extends AnonymizerCore {
|
|
25
|
+
constructor(config?: AnonymizerConfig);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Creates an anonymizer with the specified configuration
|
|
29
|
+
*
|
|
30
|
+
* @example
|
|
31
|
+
* ```typescript
|
|
32
|
+
* // Regex-only (no NER)
|
|
33
|
+
* const anonymizer = createAnonymizer();
|
|
34
|
+
*
|
|
35
|
+
* // With NER (auto-downloads model on first use)
|
|
36
|
+
* const anonymizer = createAnonymizer({
|
|
37
|
+
* ner: { mode: 'quantized' }
|
|
38
|
+
* });
|
|
39
|
+
*
|
|
40
|
+
* // With NER and progress callback
|
|
41
|
+
* const anonymizer = createAnonymizer({
|
|
42
|
+
* ner: {
|
|
43
|
+
* mode: 'standard',
|
|
44
|
+
* onStatus: (status) => console.log(status),
|
|
45
|
+
* onDownloadProgress: (p) => console.log(`${p.file}: ${p.percent}%`)
|
|
46
|
+
* }
|
|
47
|
+
* });
|
|
48
|
+
* ```
|
|
49
|
+
*/
|
|
50
|
+
export declare function createAnonymizer(config?: AnonymizerConfig): Anonymizer;
|
|
51
|
+
export declare const anonymize: typeof anonymizeCore;
|
|
52
|
+
export declare const anonymizeRegexOnly: typeof anonymizeRegexOnlyCore;
|
|
53
|
+
export declare const anonymizeWithNER: typeof anonymizeWithNERCore;
|
|
54
|
+
//# sourceMappingURL=browser.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,cAAc,kBAAkB,CAAC;AAGjC,YAAY,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,qBAAqB,EACrB,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,aAAa,EACb,wBAAwB,EACxB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,cAAc,EACd,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,wBAAwB,EAC7B,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,eAAe,EACf,oBAAoB,GACrB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,YAAY,EACZ,eAAe,EACf,WAAW,EACX,cAAc,EACd,WAAW,EACX,QAAQ,EACR,SAAS,EACT,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,OAAO,EACP,WAAW,EAEX,uBAAuB,EACvB,wBAAwB,EACxB,uBAAuB,EACvB,gBAAgB,EAChB,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,sBAAsB,EACtB,mBAAmB,EACnB,mBAAmB,EAEnB,YAAY,EACZ,sBAAsB,EACtB,uBAAuB,EACvB,oBAAoB,EACpB,YAAY,EACZ,eAAe,EACf,WAAW,EACX,KAAK,oBAAoB,EACzB,KAAK,cAAc,EACnB,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,KAAK,qBAAqB,GAC3B,MAAM,qBAAqB,CAAC;AAG7B,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EACL,aAAa,EACb,aAAa,EACb,WAAW,EACX,SAAS,EACT,YAAY,EACZ,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACX,aAAa,EACb,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EACL,kBAAkB,EAClB,MAAM,EACN,SAAS,EACT,oBAAoB,EACpB,kBAAkB,EAClB,KAAK,eAAe,GACrB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EACL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,iBAAiB,EACtB,0BAA0B,EAC1B,2BAA2B,GAG5B,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EACL,IAAI,IAAI,QAAQ,EAChB,OAAO,IAAI,WAAW,EACtB,QAAQ,IAAI,YAAY,EACxB,SAAS,IAAI,aAAa,EAC1B,OAAO,IAAI,WAAW,EACtB,UAAU,IAAI,cAAc,GAC7B,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,UAAU,IAAI,cAAc,EAC5B,SAAS,IAAI,aAAa,EAC1B,kBAAkB,IAAI,sBAAsB,EAC5C,gBAAgB,IAAI,oBAAoB,EACxC,KAAK,gBAAgB,EACrB,KAAK,SAAS,EACf,MAAM,iBAAiB,CAAC;AAMzB,YAAY,EAAE,gBAAgB,EAAE,SAAS,EAAE,CAAC;AAE5C;;;GAGG;AACH,qBAAa,UAAW,SAAQ,cAAc;gBAChC,MAAM,GAAE,gBAAqB;CAU1C;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,UAAU,CAEtE;AAGD,eAAO,MAAM,SAAS,sBAAgB,CAAC;AACvC,eAAO,MAAM,kBAAkB,+BAAyB,CAAC;AACzD,eAAO,MAAM,gBAAgB,6BAAuB,CAAC"}
|
package/dist/browser.js
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rehydra Module - Browser Entry Point
|
|
3
|
+
* Browser-safe entry point for on-device PII anonymization
|
|
4
|
+
*
|
|
5
|
+
* This module excludes Node.js-specific exports like SQLitePIIStorageProvider.
|
|
6
|
+
* For full functionality including SQLite, use the main entry point or import from 'rehydra/node'.
|
|
7
|
+
*/
|
|
8
|
+
// Re-export types
|
|
9
|
+
export * from "./types/index.js";
|
|
10
|
+
export { RegexRecognizer, RecognizerRegistry, createDefaultRegistry, createRegistry, getGlobalRegistry, emailRecognizer, phoneRecognizer, ibanRecognizer, bicSwiftRecognizer, creditCardRecognizer, ipAddressRecognizer, urlRecognizer, createCustomIdRecognizer, createCaseIdRecognizer, createCustomerIdRecognizer, } from "./recognizers/index.js";
|
|
11
|
+
// Re-export NER components
|
|
12
|
+
export { NERModel, NERModelStub, createNERModel, createNERModelStub, WordPieceTokenizer, loadVocabFromFile, parseVocab, loadRuntime, detectRuntime, getRuntimeType, MODEL_REGISTRY, getModelCacheDir, isModelDownloaded, downloadModel, ensureModel, clearModelCache, listDownloadedModels, } from "./ner/index.js";
|
|
13
|
+
// Re-export pipeline components
|
|
14
|
+
export { prenormalize, resolveEntities, tagEntities, validateOutput, generateTag, parseTag, rehydrate, enrichSemantics, inferGender, classifyLocation, getDatabaseStats, hasName, hasLocation,
|
|
15
|
+
// Semantic data loader exports
|
|
16
|
+
isSemanticDataAvailable, isSemanticDataDownloaded, getSemanticDataCacheDir, getDataDirectory, downloadSemanticData, ensureSemanticData, initializeSemanticData, loadSemanticData, clearSemanticData, clearSemanticDataCache, getSemanticDataInfo, SEMANTIC_DATA_FILES,
|
|
17
|
+
// Title extractor exports
|
|
18
|
+
extractTitle, extractTitlesFromSpans, mergeAdjacentTitleSpans, getTitlesForLanguage, getAllTitles, startsWithTitle, isOnlyTitle, } from "./pipeline/index.js";
|
|
19
|
+
export { encryptPIIMap, decryptPIIMap, generateKey, deriveKey, generateSalt, InMemoryKeyProvider, ConfigKeyProvider, validateKey, secureCompare, uint8ArrayToBase64, base64ToUint8Array, } from "./crypto/index.js";
|
|
20
|
+
// Re-export storage utilities (file system abstraction)
|
|
21
|
+
export { getStorageProvider, isNode, isBrowser, resetStorageProvider, setStorageProvider, } from "./utils/storage.js";
|
|
22
|
+
// Re-export PII storage providers (browser-safe only - no SQLite)
|
|
23
|
+
export { InMemoryPIIStorageProvider, IndexedDBPIIStorageProvider,
|
|
24
|
+
// Note: SQLitePIIStorageProvider is NOT exported in browser build
|
|
25
|
+
// Import from 'rehydra/storage/sqlite' if needed in Node.js
|
|
26
|
+
} from "./storage/browser.js";
|
|
27
|
+
// Re-export path utilities
|
|
28
|
+
export { join as pathJoin, dirname as pathDirname, basename as pathBasename, normalize as pathNormalize, extname as pathExtname, isAbsolute as pathIsAbsolute, } from "./utils/path.js";
|
|
29
|
+
// Import core anonymizer components
|
|
30
|
+
import { Anonymizer as AnonymizerCore, anonymize as anonymizeCore, anonymizeRegexOnly as anonymizeRegexOnlyCore, anonymizeWithNER as anonymizeWithNERCore, } from "./core/index.js";
|
|
31
|
+
// Import session implementation
|
|
32
|
+
import { AnonymizerSessionImpl } from "./storage/session.js";
|
|
33
|
+
/**
|
|
34
|
+
* Anonymizer instance
|
|
35
|
+
* Main class for performing PII anonymization
|
|
36
|
+
*/
|
|
37
|
+
export class Anonymizer extends AnonymizerCore {
|
|
38
|
+
constructor(config = {}) {
|
|
39
|
+
super(config, (anonymizer, sessionId, storage, keyProvider) => {
|
|
40
|
+
return new AnonymizerSessionImpl(anonymizer, sessionId, storage, keyProvider);
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Creates an anonymizer with the specified configuration
|
|
46
|
+
*
|
|
47
|
+
* @example
|
|
48
|
+
* ```typescript
|
|
49
|
+
* // Regex-only (no NER)
|
|
50
|
+
* const anonymizer = createAnonymizer();
|
|
51
|
+
*
|
|
52
|
+
* // With NER (auto-downloads model on first use)
|
|
53
|
+
* const anonymizer = createAnonymizer({
|
|
54
|
+
* ner: { mode: 'quantized' }
|
|
55
|
+
* });
|
|
56
|
+
*
|
|
57
|
+
* // With NER and progress callback
|
|
58
|
+
* const anonymizer = createAnonymizer({
|
|
59
|
+
* ner: {
|
|
60
|
+
* mode: 'standard',
|
|
61
|
+
* onStatus: (status) => console.log(status),
|
|
62
|
+
* onDownloadProgress: (p) => console.log(`${p.file}: ${p.percent}%`)
|
|
63
|
+
* }
|
|
64
|
+
* });
|
|
65
|
+
* ```
|
|
66
|
+
*/
|
|
67
|
+
export function createAnonymizer(config) {
|
|
68
|
+
return new Anonymizer(config);
|
|
69
|
+
}
|
|
70
|
+
// Re-export standalone functions from core
|
|
71
|
+
export const anonymize = anonymizeCore;
|
|
72
|
+
export const anonymizeRegexOnly = anonymizeRegexOnlyCore;
|
|
73
|
+
export const anonymizeWithNER = anonymizeWithNERCore;
|
|
74
|
+
//# sourceMappingURL=browser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"browser.js","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,kBAAkB;AAClB,cAAc,kBAAkB,CAAC;AAIjC,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,qBAAqB,EACrB,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,aAAa,EACb,wBAAwB,EACxB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,wBAAwB,CAAC;AAEhC,2BAA2B;AAC3B,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,cAAc,EACd,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EAMd,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,eAAe,EACf,oBAAoB,GACrB,MAAM,gBAAgB,CAAC;AAExB,gCAAgC;AAChC,OAAO,EACL,YAAY,EACZ,eAAe,EACf,WAAW,EACX,cAAc,EACd,WAAW,EACX,QAAQ,EACR,SAAS,EACT,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,OAAO,EACP,WAAW;AACX,+BAA+B;AAC/B,uBAAuB,EACvB,wBAAwB,EACxB,uBAAuB,EACvB,gBAAgB,EAChB,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,sBAAsB,EACtB,mBAAmB,EACnB,mBAAmB;AACnB,0BAA0B;AAC1B,YAAY,EACZ,sBAAsB,EACtB,uBAAuB,EACvB,oBAAoB,EACpB,YAAY,EACZ,eAAe,EACf,WAAW,GAMZ,MAAM,qBAAqB,CAAC;AAI7B,OAAO,EACL,aAAa,EACb,aAAa,EACb,WAAW,EACX,SAAS,EACT,YAAY,EACZ,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACX,aAAa,EACb,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,mBAAmB,CAAC;AAE3B,wDAAwD;AACxD,OAAO,EACL,kBAAkB,EAClB,MAAM,EACN,SAAS,EACT,oBAAoB,EACpB,kBAAkB,GAEnB,MAAM,oBAAoB,CAAC;AAE5B,kEAAkE;AAClE,OAAO,EAML,0BAA0B,EAC1B,2BAA2B;AAC3B,kEAAkE;AAClE,4DAA4D;EAC7D,MAAM,sBAAsB,CAAC;AAE9B,2BAA2B;AAC3B,OAAO,EACL,IAAI,IAAI,QAAQ,EAChB,OAAO,IAAI,WAAW,EACtB,QAAQ,IAAI,YAAY,EACxB,SAAS,IAAI,aAAa,EAC1B,OAAO,IAAI,WAAW,EACtB,UAAU,IAAI,cAAc,GAC7B,MAAM,iBAAiB,CAAC;AAEzB,oCAAoC;AACpC,OAAO,EACL,UAAU,IAAI,cAAc,EAC5B,SAAS,IAAI,aAAa,EAC1B,kBAAkB,IAAI,sBAAsB,EAC5C,gBAAgB,IAAI,oBAAoB,GAGzC,MAAM,iBAAiB,CAAC;AAEzB,gCAAgC;AAChC,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAK7D;;;GAGG;AACH,MAAM,OAAO,UAAW,SAAQ,cAAc;IAC5C,YAAY,SAA2B,EAAE;QACvC,KAAK,CAAC,MAAM,EAAE,CAAC,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE;YAC5D,OAAO,IAAI,qBAAqB,CAC9B,UAAU,EACV,SAAS,EACT,OAAO,EACP,WAAW,CACZ,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAyB;IACxD,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC;AAED,2CAA2C;AAC3C,MAAM,CAAC,MAAM,SAAS,GAAG,aAAa,CAAC;AACvC,MAAM,CAAC,MAAM,kBAAkB,GAAG,sBAAsB,CAAC;AACzD,MAAM,CAAC,MAAM,gBAAgB,GAAG,oBAAoB,CAAC"}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Core Anonymizer Module
|
|
3
|
+
* Shared implementation for both browser and Node.js entry points
|
|
4
|
+
*/
|
|
5
|
+
import { AnonymizationResult, AnonymizationPolicy, SemanticConfig, PIIType } from "../types/index.js";
|
|
6
|
+
import { RecognizerRegistry } from "../recognizers/index.js";
|
|
7
|
+
import { type INERModel } from "../ner/index.js";
|
|
8
|
+
import { type NERModelMode, type DownloadProgressCallback } from "../ner/model-manager.js";
|
|
9
|
+
import { type RawPIIMap } from "../pipeline/tagger.js";
|
|
10
|
+
import { type KeyProvider } from "../crypto/index.js";
|
|
11
|
+
export type { RawPIIMap } from "../pipeline/tagger.js";
|
|
12
|
+
import type { PIIStorageProvider, AnonymizerSession } from "../storage/types.js";
|
|
13
|
+
export type { PIIStorageProvider, AnonymizerSession };
|
|
14
|
+
/**
|
|
15
|
+
* NER configuration options
|
|
16
|
+
*/
|
|
17
|
+
export interface NERConfig {
|
|
18
|
+
/**
|
|
19
|
+
* NER model mode:
|
|
20
|
+
* - 'standard': Full-size multilingual model (~1.1 GB)
|
|
21
|
+
* - 'quantized': Smaller quantized model (~280 MB)
|
|
22
|
+
* - 'disabled': No NER, regex-only detection
|
|
23
|
+
* - 'custom': Use custom model paths
|
|
24
|
+
*/
|
|
25
|
+
mode: NERModelMode;
|
|
26
|
+
/**
|
|
27
|
+
* Custom model path (required when mode is 'custom')
|
|
28
|
+
*/
|
|
29
|
+
modelPath?: string;
|
|
30
|
+
/**
|
|
31
|
+
* Custom vocab path (required when mode is 'custom')
|
|
32
|
+
*/
|
|
33
|
+
vocabPath?: string;
|
|
34
|
+
/**
|
|
35
|
+
* Whether to auto-download model if not present
|
|
36
|
+
* @default true
|
|
37
|
+
*/
|
|
38
|
+
autoDownload?: boolean;
|
|
39
|
+
/**
|
|
40
|
+
* Callback for download progress
|
|
41
|
+
*/
|
|
42
|
+
onDownloadProgress?: DownloadProgressCallback;
|
|
43
|
+
/**
|
|
44
|
+
* Callback for status messages
|
|
45
|
+
*/
|
|
46
|
+
onStatus?: (status: string) => void;
|
|
47
|
+
/**
|
|
48
|
+
* Confidence thresholds per PII type (0.0 - 1.0)
|
|
49
|
+
* Overrides default thresholds for specified types
|
|
50
|
+
* @example { PERSON: 0.8, ORG: 0.7 }
|
|
51
|
+
*/
|
|
52
|
+
thresholds?: Partial<Record<PIIType, number>>;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Anonymizer configuration
|
|
56
|
+
*/
|
|
57
|
+
export interface AnonymizerConfig {
|
|
58
|
+
/** Recognizer registry (uses default if not provided) */
|
|
59
|
+
registry?: RecognizerRegistry;
|
|
60
|
+
/**
|
|
61
|
+
* NER configuration
|
|
62
|
+
* @default { mode: 'disabled' }
|
|
63
|
+
*/
|
|
64
|
+
ner?: NERConfig;
|
|
65
|
+
/**
|
|
66
|
+
* Semantic enrichment configuration
|
|
67
|
+
* Enables MT-friendly PII tags with gender/scope attributes
|
|
68
|
+
* @default { enabled: false }
|
|
69
|
+
*/
|
|
70
|
+
semantic?: SemanticConfig;
|
|
71
|
+
/** Key provider for encryption (generates random key if not provided) */
|
|
72
|
+
keyProvider?: KeyProvider;
|
|
73
|
+
/**
|
|
74
|
+
* PII storage provider for automatic session-based persistence
|
|
75
|
+
* When provided, enables the session() method for automatic PII map storage
|
|
76
|
+
*/
|
|
77
|
+
piiStorageProvider?: PIIStorageProvider;
|
|
78
|
+
/** Default policy (uses default if not provided) */
|
|
79
|
+
defaultPolicy?: AnonymizationPolicy;
|
|
80
|
+
/** Model version string */
|
|
81
|
+
modelVersion?: string;
|
|
82
|
+
/** Policy version string */
|
|
83
|
+
policyVersion?: string;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Session implementation factory type
|
|
87
|
+
*/
|
|
88
|
+
export type SessionFactory = (anonymizer: Anonymizer, sessionId: string, storage: PIIStorageProvider, keyProvider: KeyProvider) => AnonymizerSession;
|
|
89
|
+
/**
|
|
90
|
+
* Anonymizer instance
|
|
91
|
+
* Main class for performing PII anonymization
|
|
92
|
+
*/
|
|
93
|
+
export declare class Anonymizer {
|
|
94
|
+
private registry;
|
|
95
|
+
private nerModel;
|
|
96
|
+
private nerConfig;
|
|
97
|
+
private semanticConfig;
|
|
98
|
+
private keyProvider;
|
|
99
|
+
private piiStorageProvider;
|
|
100
|
+
private defaultPolicy;
|
|
101
|
+
private modelVersion;
|
|
102
|
+
private policyVersion;
|
|
103
|
+
private initialized;
|
|
104
|
+
private semanticDataReady;
|
|
105
|
+
private sessionFactory;
|
|
106
|
+
constructor(config?: AnonymizerConfig, sessionFactory?: SessionFactory);
|
|
107
|
+
/**
|
|
108
|
+
* Initializes the anonymizer
|
|
109
|
+
* Downloads NER model and semantic data if needed and loads them
|
|
110
|
+
*/
|
|
111
|
+
initialize(): Promise<void>;
|
|
112
|
+
/**
|
|
113
|
+
* Anonymizes text, replacing PII with placeholder tags
|
|
114
|
+
* @param text - Input text to anonymize
|
|
115
|
+
* @param locale - Optional locale hint (e.g., 'de-DE', 'en-US')
|
|
116
|
+
* @param policy - Optional policy override
|
|
117
|
+
* @param existingPiiMap - Optional existing PII map for session-level ID reuse
|
|
118
|
+
* @returns Anonymization result with anonymized text and encrypted PII map
|
|
119
|
+
*/
|
|
120
|
+
anonymize(text: string, locale?: string, policy?: Partial<AnonymizationPolicy>, existingPiiMap?: RawPIIMap): Promise<AnonymizationResult>;
|
|
121
|
+
/**
|
|
122
|
+
* Disposes of resources
|
|
123
|
+
*/
|
|
124
|
+
dispose(): Promise<void>;
|
|
125
|
+
/**
|
|
126
|
+
* Gets the recognizer registry
|
|
127
|
+
*/
|
|
128
|
+
getRegistry(): RecognizerRegistry;
|
|
129
|
+
/**
|
|
130
|
+
* Gets the NER model
|
|
131
|
+
*/
|
|
132
|
+
getNERModel(): INERModel | null;
|
|
133
|
+
/**
|
|
134
|
+
* Whether the anonymizer is initialized
|
|
135
|
+
*/
|
|
136
|
+
get isInitialized(): boolean;
|
|
137
|
+
/**
|
|
138
|
+
* Creates a session-bound interface for automatic PII map storage
|
|
139
|
+
*
|
|
140
|
+
* @param sessionId - Unique identifier for this session/conversation
|
|
141
|
+
* @returns AnonymizerSession that auto-saves on anonymize and auto-loads on rehydrate
|
|
142
|
+
* @throws Error if piiStorageProvider was not configured
|
|
143
|
+
*
|
|
144
|
+
* @example
|
|
145
|
+
* ```typescript
|
|
146
|
+
* const session = anonymizer.session('chat-123');
|
|
147
|
+
*
|
|
148
|
+
* // Anonymize - auto-saves to storage
|
|
149
|
+
* const result = await session.anonymize('Hello John Smith!');
|
|
150
|
+
*
|
|
151
|
+
* // Rehydrate - auto-loads and decrypts
|
|
152
|
+
* const original = await session.rehydrate(translatedText);
|
|
153
|
+
* ```
|
|
154
|
+
*/
|
|
155
|
+
session(sessionId: string): AnonymizerSession;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Creates an anonymizer with the specified configuration
|
|
159
|
+
*
|
|
160
|
+
* @example
|
|
161
|
+
* ```typescript
|
|
162
|
+
* // Regex-only (no NER)
|
|
163
|
+
* const anonymizer = createAnonymizer();
|
|
164
|
+
*
|
|
165
|
+
* // With NER (auto-downloads model on first use)
|
|
166
|
+
* const anonymizer = createAnonymizer({
|
|
167
|
+
* ner: { mode: 'quantized' }
|
|
168
|
+
* });
|
|
169
|
+
*
|
|
170
|
+
* // With NER and progress callback
|
|
171
|
+
* const anonymizer = createAnonymizer({
|
|
172
|
+
* ner: {
|
|
173
|
+
* mode: 'standard',
|
|
174
|
+
* onStatus: (status) => console.log(status),
|
|
175
|
+
* onDownloadProgress: (p) => console.log(`${p.file}: ${p.percent}%`)
|
|
176
|
+
* }
|
|
177
|
+
* });
|
|
178
|
+
* ```
|
|
179
|
+
*/
|
|
180
|
+
export declare function createAnonymizer(config?: AnonymizerConfig, sessionFactory?: SessionFactory): Anonymizer;
|
|
181
|
+
/**
|
|
182
|
+
* Convenience function for one-off anonymization
|
|
183
|
+
* Creates a temporary anonymizer with default settings (regex-only)
|
|
184
|
+
*/
|
|
185
|
+
export declare function anonymize(text: string, locale?: string, policy?: Partial<AnonymizationPolicy>): Promise<AnonymizationResult>;
|
|
186
|
+
/**
|
|
187
|
+
* Quick regex-only anonymization (no NER, faster)
|
|
188
|
+
*/
|
|
189
|
+
export declare function anonymizeRegexOnly(text: string, policy?: Partial<AnonymizationPolicy>): Promise<AnonymizationResult>;
|
|
190
|
+
/**
|
|
191
|
+
* Full anonymization with NER
|
|
192
|
+
* Auto-downloads the quantized model on first use
|
|
193
|
+
*
|
|
194
|
+
* @example
|
|
195
|
+
* ```typescript
|
|
196
|
+
* const result = await anonymizeWithNER(
|
|
197
|
+
* 'Contact John Smith at john@example.com',
|
|
198
|
+
* {
|
|
199
|
+
* mode: 'quantized',
|
|
200
|
+
* onStatus: console.log
|
|
201
|
+
* }
|
|
202
|
+
* );
|
|
203
|
+
* ```
|
|
204
|
+
*/
|
|
205
|
+
export declare function anonymizeWithNER(text: string, nerConfig: Omit<NERConfig, "mode"> & {
|
|
206
|
+
mode?: "standard" | "quantized";
|
|
207
|
+
}, policy?: Partial<AnonymizationPolicy>): Promise<AnonymizationResult>;
|
|
208
|
+
//# sourceMappingURL=anonymizer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"anonymizer.d.ts","sourceRoot":"","sources":["../../src/core/anonymizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EAGnB,cAAc,EAEd,OAAO,EAER,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAEL,kBAAkB,EACnB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,KAAK,SAAS,EAIf,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,wBAAwB,EAC9B,MAAM,yBAAyB,CAAC;AAIjC,OAAO,EAAoC,KAAK,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAYzF,OAAO,EAGL,KAAK,WAAW,EACjB,MAAM,oBAAoB,CAAC;AAI5B,YAAY,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAGvD,OAAO,KAAK,EACV,kBAAkB,EAClB,iBAAiB,EAClB,MAAM,qBAAqB,CAAC;AAG7B,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;AAoCtD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;;;;OAMG;IACH,IAAI,EAAE,YAAY,CAAC;IAEnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;OAEG;IACH,kBAAkB,CAAC,EAAE,wBAAwB,CAAC;IAE9C;;OAEG;IACH,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IAEpC;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yDAAyD;IACzD,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAE9B;;;OAGG;IACH,GAAG,CAAC,EAAE,SAAS,CAAC;IAEhB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAE1B,yEAAyE;IACzE,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,oDAAoD;IACpD,aAAa,CAAC,EAAE,mBAAmB,CAAC;IAEpC,2BAA2B;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,4BAA4B;IAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,kBAAkB,EAC3B,WAAW,EAAE,WAAW,KACrB,iBAAiB,CAAC;AAEvB;;;GAGG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,QAAQ,CAA0B;IAC1C,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,kBAAkB,CAA4B;IACtD,OAAO,CAAC,aAAa,CAAsB;IAC3C,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAwB;gBAElC,MAAM,GAAE,gBAAqB,EAAE,cAAc,CAAC,EAAE,cAAc;IAwC1E;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA6FjC;;;;;;;OAOG;IACG,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,EACrC,cAAc,CAAC,EAAE,SAAS,GACzB,OAAO,CAAC,mBAAmB,CAAC;IAmH/B;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;OAEG;IACH,WAAW,IAAI,kBAAkB;IAIjC;;OAEG;IACH,WAAW,IAAI,SAAS,GAAG,IAAI;IAI/B;;OAEG;IACH,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB;CAqC9C;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,CAAC,EAAE,gBAAgB,EAAE,cAAc,CAAC,EAAE,cAAc,GAAG,UAAU,CAEvG;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAS9B;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAQ9B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,GAAG;IAAE,IAAI,CAAC,EAAE,UAAU,GAAG,WAAW,CAAA;CAAE,EACxE,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,OAAO,CAAC,mBAAmB,CAAC,CAe9B"}
|