solve-engine 1.0.0-beta.2 → 1.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{Lexer-D9l4Yrk2.d.ts → Lexer-Cfm79Dak.d.cts} +18 -2
- package/dist/{Lexer-BJdhlnej.d.cts → Lexer-W9MBOP0V.d.ts} +18 -2
- package/dist/{PackageRegistry-DaH4aIKP.d.ts → PackageRegistry-CjDt-Jy_.d.ts} +60 -4
- package/dist/{PackageRegistry-D-Tv_7ca.d.cts → PackageRegistry-pHtKythi.d.cts} +60 -4
- package/dist/{Parselet-CK8cQu2a.d.cts → Parselet-Cu0bLSis.d.cts} +1 -1
- package/dist/{Parselet-ConOIdRO.d.ts → Parselet-DEdF9I7n.d.ts} +1 -1
- package/dist/{Token-0jpvUdPY.d.cts → Token-BzG5G4ja.d.cts} +44 -0
- package/dist/{Token-0jpvUdPY.d.ts → Token-BzG5G4ja.d.ts} +44 -0
- package/dist/TokenNormalizer-DGVa24Q-.d.cts +377 -0
- package/dist/TokenNormalizer-t_GotBxr.d.ts +377 -0
- package/dist/{chunk-HWSZJQCI.js → chunk-2CS6OMZK.js} +37 -15
- package/dist/chunk-2CS6OMZK.js.map +1 -0
- package/dist/{chunk-XJCMXB2E.cjs → chunk-2MV4HBKC.cjs} +187 -61
- package/dist/chunk-2MV4HBKC.cjs.map +1 -0
- package/dist/{chunk-J73SJHR3.cjs → chunk-2NPS5DQ3.cjs} +558 -341
- package/dist/chunk-2NPS5DQ3.cjs.map +1 -0
- package/dist/{chunk-QWY3VEZN.js → chunk-3MNKQZ77.js} +65 -12
- package/dist/chunk-3MNKQZ77.js.map +1 -0
- package/dist/{chunk-RZCWSXTA.cjs → chunk-43G3JT2E.cjs} +1491 -218
- package/dist/chunk-43G3JT2E.cjs.map +1 -0
- package/dist/{chunk-WQTTOGXC.cjs → chunk-4AVD7NZW.cjs} +4 -4
- package/dist/{chunk-WQTTOGXC.cjs.map → chunk-4AVD7NZW.cjs.map} +1 -1
- package/dist/{chunk-HTXVVJRA.cjs → chunk-4CVLFLOB.cjs} +112 -2
- package/dist/chunk-4CVLFLOB.cjs.map +1 -0
- package/dist/{chunk-5YEMOYSE.js → chunk-6GCKCWLB.js} +9 -3
- package/dist/chunk-6GCKCWLB.js.map +1 -0
- package/dist/{chunk-EBSPLUW4.cjs → chunk-AHCLWAM5.cjs} +28 -10
- package/dist/chunk-AHCLWAM5.cjs.map +1 -0
- package/dist/{chunk-KVILKGMS.js → chunk-AJA6LUI7.js} +36 -2
- package/dist/chunk-AJA6LUI7.js.map +1 -0
- package/dist/{chunk-EHAHVROS.cjs → chunk-CLVQBF5C.cjs} +5 -5
- package/dist/{chunk-EHAHVROS.cjs.map → chunk-CLVQBF5C.cjs.map} +1 -1
- package/dist/{chunk-34RRD7PC.js → chunk-CYFK5SY2.js} +111 -3
- package/dist/chunk-CYFK5SY2.js.map +1 -0
- package/dist/{chunk-M5LX5AOO.js → chunk-DDQVZGLO.js} +25 -12
- package/dist/chunk-DDQVZGLO.js.map +1 -0
- package/dist/{chunk-LR7YASZF.cjs → chunk-EAAHVJ4P.cjs} +3 -3
- package/dist/chunk-EAAHVJ4P.cjs.map +1 -0
- package/dist/{chunk-3PPFLFH4.js → chunk-FIFQPKBA.js} +1417 -144
- package/dist/chunk-FIFQPKBA.js.map +1 -0
- package/dist/{chunk-RIN643A3.js → chunk-GPPLSM2Z.js} +61 -2
- package/dist/chunk-GPPLSM2Z.js.map +1 -0
- package/dist/{chunk-4QADQTWS.js → chunk-KHJUSHFU.js} +231 -14
- package/dist/chunk-KHJUSHFU.js.map +1 -0
- package/dist/{chunk-NMD5VRN4.cjs → chunk-M4F66R4O.cjs} +74 -73
- package/dist/chunk-M4F66R4O.cjs.map +1 -0
- package/dist/{chunk-6NTVRDQV.cjs → chunk-M4Q47KHF.cjs} +126 -73
- package/dist/chunk-M4Q47KHF.cjs.map +1 -0
- package/dist/{chunk-LIPPNDBE.js → chunk-M5E34VG5.js} +3 -3
- package/dist/{chunk-LIPPNDBE.js.map → chunk-M5E34VG5.js.map} +1 -1
- package/dist/{chunk-53B6KDDJ.cjs → chunk-MBNQVDVC.cjs} +38 -3
- package/dist/{chunk-53B6KDDJ.cjs.map → chunk-MBNQVDVC.cjs.map} +1 -1
- package/dist/{chunk-SDGRK7EP.js → chunk-MEOHSQEH.js} +6 -5
- package/dist/chunk-MEOHSQEH.js.map +1 -0
- package/dist/{chunk-NMCRQP3Z.cjs → chunk-MTX2KVU7.cjs} +71 -70
- package/dist/chunk-MTX2KVU7.cjs.map +1 -0
- package/dist/{chunk-QNJ4ACRT.cjs → chunk-NJTXJ5AG.cjs} +37 -15
- package/dist/chunk-NJTXJ5AG.cjs.map +1 -0
- package/dist/{chunk-C4XZV6E7.cjs → chunk-NZFKROS7.cjs} +26 -20
- package/dist/chunk-NZFKROS7.cjs.map +1 -0
- package/dist/{chunk-64W6GLLZ.js → chunk-OFXOTECC.js} +24 -6
- package/dist/chunk-OFXOTECC.js.map +1 -0
- package/dist/{chunk-3YNVWKR2.cjs → chunk-OUHT5Z36.cjs} +28 -5
- package/dist/chunk-OUHT5Z36.cjs.map +1 -0
- package/dist/{chunk-EIGTWK5N.js → chunk-OWRHDUJD.js} +3 -3
- package/dist/chunk-OWRHDUJD.js.map +1 -0
- package/dist/{chunk-4MG4XKO2.js → chunk-PCPX42KL.js} +38 -3
- package/dist/{chunk-4MG4XKO2.js.map → chunk-PCPX42KL.js.map} +1 -1
- package/dist/{chunk-GW32KPCU.cjs → chunk-R24DI24X.cjs} +61 -2
- package/dist/chunk-R24DI24X.cjs.map +1 -0
- package/dist/{chunk-XVWCOTR6.js → chunk-SFQWJMKT.js} +7 -6
- package/dist/chunk-SFQWJMKT.js.map +1 -0
- package/dist/{chunk-6BKTCEUP.cjs → chunk-TY3TLZAW.cjs} +36 -2
- package/dist/chunk-TY3TLZAW.cjs.map +1 -0
- package/dist/{chunk-DM3LMRBC.js → chunk-UKPGSAZW.js} +187 -61
- package/dist/chunk-UKPGSAZW.js.map +1 -0
- package/dist/{chunk-NH2O2AUR.js → chunk-WC5FFSHB.js} +26 -4
- package/dist/chunk-WC5FFSHB.js.map +1 -0
- package/dist/{chunk-CLL7RUQV.cjs → chunk-WWOLFXTX.cjs} +40 -18
- package/dist/chunk-WWOLFXTX.cjs.map +1 -0
- package/dist/{chunk-JBSYC7BB.cjs → chunk-X2BWAZAU.cjs} +66 -53
- package/dist/chunk-X2BWAZAU.cjs.map +1 -0
- package/dist/{chunk-GOLDJNMZ.js → chunk-XBTEO4OB.js} +28 -5
- package/dist/chunk-XBTEO4OB.js.map +1 -0
- package/dist/{chunk-EPOXXJBK.js → chunk-YU2CUNFO.js} +3 -3
- package/dist/{chunk-EPOXXJBK.js.map → chunk-YU2CUNFO.js.map} +1 -1
- package/dist/constants.cjs +4 -4
- package/dist/constants.js +1 -1
- package/dist/engine.cjs +29 -30
- package/dist/engine.d.cts +6 -6
- package/dist/engine.d.ts +6 -6
- package/dist/engine.js +19 -20
- package/dist/format.cjs +7 -8
- package/dist/format.cjs.map +1 -1
- package/dist/format.js +2 -3
- package/dist/format.js.map +1 -1
- package/dist/index.cjs +28 -29
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +6 -6
- package/dist/index.d.ts +6 -6
- package/dist/index.js +20 -21
- package/dist/index.js.map +1 -1
- package/dist/language.cjs +74 -12
- package/dist/language.cjs.map +1 -1
- package/dist/language.d.cts +6 -6
- package/dist/language.d.ts +6 -6
- package/dist/language.js +68 -6
- package/dist/language.js.map +1 -1
- package/dist/lexer.cjs +19 -19
- package/dist/lexer.d.cts +3 -3
- package/dist/lexer.d.ts +3 -3
- package/dist/lexer.js +7 -7
- package/dist/normalizer.cjs +12 -12
- package/dist/normalizer.d.cts +4 -210
- package/dist/normalizer.d.ts +4 -210
- package/dist/normalizer.js +6 -6
- package/dist/packages.cjs +36 -37
- package/dist/packages.d.cts +5 -47
- package/dist/packages.d.ts +5 -47
- package/dist/packages.js +13 -14
- package/dist/parser.cjs +12 -12
- package/dist/parser.d.cts +3 -2
- package/dist/parser.d.ts +3 -2
- package/dist/parser.js +4 -4
- package/dist/resolvers.d.cts +1 -1
- package/dist/resolvers.d.ts +1 -1
- package/dist/uom.cjs +14 -14
- package/dist/uom.d.cts +12 -4
- package/dist/uom.d.ts +12 -4
- package/dist/uom.js +4 -4
- package/dist/vm.cjs +17 -17
- package/dist/vm.js +6 -6
- package/package.json +2 -1
- package/dist/NormalizerRule-BrVoVjmP.d.cts +0 -163
- package/dist/NormalizerRule-CEjf1FyD.d.ts +0 -163
- package/dist/chunk-34RRD7PC.js.map +0 -1
- package/dist/chunk-3PPFLFH4.js.map +0 -1
- package/dist/chunk-3YNVWKR2.cjs.map +0 -1
- package/dist/chunk-4QADQTWS.js.map +0 -1
- package/dist/chunk-5YEMOYSE.js.map +0 -1
- package/dist/chunk-64W6GLLZ.js.map +0 -1
- package/dist/chunk-6BKTCEUP.cjs.map +0 -1
- package/dist/chunk-6NTVRDQV.cjs.map +0 -1
- package/dist/chunk-C4XZV6E7.cjs.map +0 -1
- package/dist/chunk-CLL7RUQV.cjs.map +0 -1
- package/dist/chunk-DM3LMRBC.js.map +0 -1
- package/dist/chunk-EBSPLUW4.cjs.map +0 -1
- package/dist/chunk-EIGTWK5N.js.map +0 -1
- package/dist/chunk-GOLDJNMZ.js.map +0 -1
- package/dist/chunk-GW32KPCU.cjs.map +0 -1
- package/dist/chunk-HTXVVJRA.cjs.map +0 -1
- package/dist/chunk-HWSZJQCI.js.map +0 -1
- package/dist/chunk-J73SJHR3.cjs.map +0 -1
- package/dist/chunk-JBSYC7BB.cjs.map +0 -1
- package/dist/chunk-KVILKGMS.js.map +0 -1
- package/dist/chunk-LR7YASZF.cjs.map +0 -1
- package/dist/chunk-M5LX5AOO.js.map +0 -1
- package/dist/chunk-NH2O2AUR.js.map +0 -1
- package/dist/chunk-NMCRQP3Z.cjs.map +0 -1
- package/dist/chunk-NMD5VRN4.cjs.map +0 -1
- package/dist/chunk-OT6OJY7C.cjs +0 -114
- package/dist/chunk-OT6OJY7C.cjs.map +0 -1
- package/dist/chunk-QNJ4ACRT.cjs.map +0 -1
- package/dist/chunk-QWY3VEZN.js.map +0 -1
- package/dist/chunk-RFYD5TJE.js +0 -111
- package/dist/chunk-RFYD5TJE.js.map +0 -1
- package/dist/chunk-RIN643A3.js.map +0 -1
- package/dist/chunk-RZCWSXTA.cjs.map +0 -1
- package/dist/chunk-SDGRK7EP.js.map +0 -1
- package/dist/chunk-XJCMXB2E.cjs.map +0 -1
- package/dist/chunk-XVWCOTR6.js.map +0 -1
package/dist/lexer.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
export { Lexer, LexerState, buildTokenLookup, sharedLexer } from './chunk-
|
|
2
|
-
import './chunk-
|
|
3
|
-
export { ExpressionLexer, LexerToken } from './chunk-
|
|
4
|
-
export { TokenTypes, registerAllTokenTypes, registerTokenType, tokenTypeId, tokenTypeName } from './chunk-
|
|
5
|
-
import './chunk-
|
|
6
|
-
export { knownUnits } from './chunk-
|
|
7
|
-
import './chunk-
|
|
1
|
+
export { Lexer, LexerState, buildTokenLookup, sharedLexer } from './chunk-2CS6OMZK.js';
|
|
2
|
+
import './chunk-AJA6LUI7.js';
|
|
3
|
+
export { ExpressionLexer, LexerToken } from './chunk-SFQWJMKT.js';
|
|
4
|
+
export { TokenTypes, registerAllTokenTypes, registerTokenType, tokenTypeId, tokenTypeName } from './chunk-GPPLSM2Z.js';
|
|
5
|
+
import './chunk-XBTEO4OB.js';
|
|
6
|
+
export { knownUnits } from './chunk-M5E34VG5.js';
|
|
7
|
+
import './chunk-PCPX42KL.js';
|
|
8
8
|
import './chunk-6RDSQURE.js';
|
|
9
9
|
import './chunk-YLJ4XMA6.js';
|
|
10
10
|
//# sourceMappingURL=lexer.js.map
|
package/dist/normalizer.cjs
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
require('./chunk-NGBBNL4K.cjs');
|
|
4
|
-
var
|
|
5
|
-
require('./chunk-
|
|
6
|
-
require('./chunk-
|
|
7
|
-
require('./chunk-
|
|
8
|
-
require('./chunk-
|
|
9
|
-
require('./chunk-
|
|
4
|
+
var chunkAHCLWAM5_cjs = require('./chunk-AHCLWAM5.cjs');
|
|
5
|
+
require('./chunk-M4F66R4O.cjs');
|
|
6
|
+
require('./chunk-R24DI24X.cjs');
|
|
7
|
+
require('./chunk-OUHT5Z36.cjs');
|
|
8
|
+
require('./chunk-CLVQBF5C.cjs');
|
|
9
|
+
require('./chunk-MBNQVDVC.cjs');
|
|
10
10
|
require('./chunk-M4N5GA5Z.cjs');
|
|
11
11
|
require('./chunk-B7LXJ6LC.cjs');
|
|
12
12
|
|
|
@@ -14,27 +14,27 @@ require('./chunk-B7LXJ6LC.cjs');
|
|
|
14
14
|
|
|
15
15
|
Object.defineProperty(exports, "BUILTIN_PHRASES", {
|
|
16
16
|
enumerable: true,
|
|
17
|
-
get: function () { return
|
|
17
|
+
get: function () { return chunkAHCLWAM5_cjs.BUILTIN_PHRASES; }
|
|
18
18
|
});
|
|
19
19
|
Object.defineProperty(exports, "PhraseTrie", {
|
|
20
20
|
enumerable: true,
|
|
21
|
-
get: function () { return
|
|
21
|
+
get: function () { return chunkAHCLWAM5_cjs.PhraseTrie; }
|
|
22
22
|
});
|
|
23
23
|
Object.defineProperty(exports, "TokenNormalizer", {
|
|
24
24
|
enumerable: true,
|
|
25
|
-
get: function () { return
|
|
25
|
+
get: function () { return chunkAHCLWAM5_cjs.TokenNormalizer; }
|
|
26
26
|
});
|
|
27
27
|
Object.defineProperty(exports, "createBuiltinNormalizerRules", {
|
|
28
28
|
enumerable: true,
|
|
29
|
-
get: function () { return
|
|
29
|
+
get: function () { return chunkAHCLWAM5_cjs.createBuiltinNormalizerRules; }
|
|
30
30
|
});
|
|
31
31
|
Object.defineProperty(exports, "createFusedToken", {
|
|
32
32
|
enumerable: true,
|
|
33
|
-
get: function () { return
|
|
33
|
+
get: function () { return chunkAHCLWAM5_cjs.createFusedToken; }
|
|
34
34
|
});
|
|
35
35
|
Object.defineProperty(exports, "implicitMultiplyRule", {
|
|
36
36
|
enumerable: true,
|
|
37
|
-
get: function () { return
|
|
37
|
+
get: function () { return chunkAHCLWAM5_cjs.implicitMultiplyRule; }
|
|
38
38
|
});
|
|
39
39
|
//# sourceMappingURL=normalizer.cjs.map
|
|
40
40
|
//# sourceMappingURL=normalizer.cjs.map
|
package/dist/normalizer.d.cts
CHANGED
|
@@ -1,212 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* TokenNormalizer, post-lexer token normalization pass.
|
|
6
|
-
*
|
|
7
|
-
* ## Purpose
|
|
8
|
-
* Applies domain-specific {@link NormalizerRule | NormalizerRules} to the raw
|
|
9
|
-
* token stream produced by the {@link ExpressionLexer}. This keeps the lexer
|
|
10
|
-
* slim and focused on single-token production, while multi-token pattern
|
|
11
|
-
* matching (phrases, implicit operators, domain merges) lives here.
|
|
12
|
-
*
|
|
13
|
-
* ## What rules can do
|
|
14
|
-
* - **Phrase fusion**: Merge consecutive words into compound tokens
|
|
15
|
-
* (e.g., `IDENT + ... + IDENT` → `CARET`)
|
|
16
|
-
* - **Implicit operator insertion**: Insert missing operators between tokens
|
|
17
|
-
* (e.g., `NUMBER IDENT` → `NUMBER STAR IDENT`)
|
|
18
|
-
* - **Domain-specific transformations**: Coalesce item names, currency pairs,
|
|
19
|
-
* percentage syntax, etc.
|
|
20
|
-
*
|
|
21
|
-
* ## Architecture
|
|
22
|
-
* Providers register NormalizerRules alongside Parselets and OpCode handlers
|
|
23
|
-
* via {@link IEnginePackage.normalizerRules}. The normalizer applies them
|
|
24
|
-
* greedily left-to-right in multiple passes with safety limits.
|
|
25
|
-
*
|
|
26
|
-
* @module TokenNormalizer
|
|
27
|
-
*/
|
|
28
|
-
|
|
29
|
-
/**
|
|
30
|
-
* Configuration options for the normalization pass.
|
|
31
|
-
*
|
|
32
|
-
* These control safety limits and diagnostic callbacks. The defaults
|
|
33
|
-
* are chosen to be generous enough for any realistic expression while
|
|
34
|
-
* preventing runaway token expansion from recursive rules.
|
|
35
|
-
*/
|
|
36
|
-
interface NormalizerOptions {
|
|
37
|
-
/**
|
|
38
|
-
* Maximum number of full passes over the token stream before bailing out.
|
|
39
|
-
* Prevents infinite loops from recursive rule chains.
|
|
40
|
-
* @default 100
|
|
41
|
-
*/
|
|
42
|
-
maxPasses?: number;
|
|
43
|
-
/**
|
|
44
|
-
* Maximum number of tokens allowed after normalization.
|
|
45
|
-
* If exceeded, an Error is thrown rather than passing a bloated stream
|
|
46
|
-
* to the parser.
|
|
47
|
-
* @default 10000
|
|
48
|
-
*/
|
|
49
|
-
maxTokens?: number;
|
|
50
|
-
/**
|
|
51
|
-
* Callback invoked for each fusion event during normalization.
|
|
52
|
-
* Used by diagnostic mode to populate {@link NormalizerOutput.fusions}.
|
|
53
|
-
* When `undefined`, fusions are still tracked internally but no callbacks fire.
|
|
54
|
-
*/
|
|
55
|
-
onFusion?: (fusion: TokenFusion) => void;
|
|
56
|
-
}
|
|
57
|
-
/**
|
|
58
|
-
* Creates a new normalized token from fused source tokens.
|
|
59
|
-
*
|
|
60
|
-
* The fused token inherits position information (offset, line, column)
|
|
61
|
-
* from the first source token, which preserves source-map accuracy
|
|
62
|
-
* for error messages and diagnostic highlighting.
|
|
63
|
-
*
|
|
64
|
-
* @param type - The new token type (e.g., "CARET", "TIMES_BY")
|
|
65
|
-
* @param text - The combined text representation (e.g., "to the power of")
|
|
66
|
-
* @param sourceTokens - The original tokens being fused (at least 2)
|
|
67
|
-
* @returns A new {@link LexerToken} with the fused type and combined text
|
|
68
|
-
*/
|
|
69
|
-
declare function createFusedToken(type: string, text: string, sourceTokens: Token[]): Token;
|
|
70
|
-
/**
|
|
71
|
-
* Token normalizer: applies {@link NormalizerRule | NormalizerRules} to a token stream.
|
|
72
|
-
*
|
|
73
|
-
* ## Lifecycle
|
|
74
|
-
* 1. **Registration**: Rules are added via {@link register} and sorted by priority
|
|
75
|
-
* 2. **Normalization**: {@link normalize} applies rules greedily left-to-right
|
|
76
|
-
* 3. **Cleanup**: {@link clear} or {@link unregister} removes rules
|
|
77
|
-
*
|
|
78
|
-
* ## Normalization algorithm
|
|
79
|
-
* The normalizer uses a greedy left-to-right multi-pass algorithm:
|
|
80
|
-
* - At each token position, rules are tried in priority order (highest first)
|
|
81
|
-
* - When a rule matches, matched tokens are consumed and replaced
|
|
82
|
-
* - Processing continues from the replacement position
|
|
83
|
-
* - Multiple passes handle cascading matches (one rule's output triggers another)
|
|
84
|
-
* - Safety limits ({@link NormalizerOptions.maxPasses}) prevent infinite loops
|
|
85
|
-
*
|
|
86
|
-
* @example
|
|
87
|
-
* ```ts
|
|
88
|
-
* const normalizer = new TokenNormalizer();
|
|
89
|
-
* normalizer.register(phraseRule); // "to the power of" → CARET
|
|
90
|
-
* normalizer.register(implicitMultRule); // "2 x" → "2 * x"
|
|
91
|
-
* const normalized = normalizer.normalize(rawTokens);
|
|
92
|
-
* ```
|
|
93
|
-
*/
|
|
94
|
-
declare class TokenNormalizer {
|
|
95
|
-
/** Registered rules, unsorted, the source of truth. */
|
|
96
|
-
private rules;
|
|
97
|
-
/**
|
|
98
|
-
* Priority-sorted copy of {@link rules}, rebuilt lazily on the next
|
|
99
|
-
* {@link normalize} call after a mutation. Rules are registered once at
|
|
100
|
-
* engine/package-registration time and essentially never change during a
|
|
101
|
-
* session, but normalize() runs on every keystroke-driven evaluation, an
|
|
102
|
-
* earlier version re-sorted a fresh copy of `rules` on every single call,
|
|
103
|
-
* which meant every keystroke paid for an allocation + sort of a list that
|
|
104
|
-
* had usually not changed since the last one. `null` means "stale, rebuild
|
|
105
|
-
* on next use"; {@link register}/{@link unregister}/{@link clear} all
|
|
106
|
-
* invalidate it.
|
|
107
|
-
*/
|
|
108
|
-
private sortedRulesCache;
|
|
109
|
-
/**
|
|
110
|
-
* Phrase trie for single-pass multi-word phrase fusion.
|
|
111
|
-
* Tried at each token position BEFORE other rules, the trie walk
|
|
112
|
-
* is O(depth) vs O(R × W) for separate rule matching.
|
|
113
|
-
*/
|
|
114
|
-
private phraseTrie;
|
|
115
|
-
/** Merged options with defaults applied. */
|
|
116
|
-
private options;
|
|
117
|
-
/**
|
|
118
|
-
* @param options - Configuration overrides for safety limits and diagnostic callbacks
|
|
119
|
-
*/
|
|
120
|
-
constructor(options?: NormalizerOptions);
|
|
121
|
-
/**
|
|
122
|
-
* Register a normalization rule.
|
|
123
|
-
*
|
|
124
|
-
* Rules are sorted by priority (descending) on each {@link normalize} call.
|
|
125
|
-
* Multiple rules can share the same priority, they are tried in registration
|
|
126
|
-
* order when priorities are equal.
|
|
127
|
-
*
|
|
128
|
-
* @param rule - The rule to register
|
|
129
|
-
*/
|
|
130
|
-
register(rule: NormalizerRule): void;
|
|
131
|
-
/**
|
|
132
|
-
* Unregister a normalization rule by its {@link NormalizerRule.name | name}.
|
|
133
|
-
*
|
|
134
|
-
* If multiple rules share the same name, all are removed. This is safe to
|
|
135
|
-
* call with a name that doesn't match any rule, it simply has no effect.
|
|
136
|
-
*
|
|
137
|
-
* @param ruleName - The name of the rule to remove
|
|
138
|
-
*/
|
|
139
|
-
unregister(ruleName: string): void;
|
|
140
|
-
/**
|
|
141
|
-
* Remove all registered rules, resetting the normalizer to its initial state.
|
|
142
|
-
* Also clears the phrase trie.
|
|
143
|
-
*/
|
|
144
|
-
clear(): void;
|
|
145
|
-
/**
|
|
146
|
-
* Priority-sorted view of {@link rules} (descending priority; registration
|
|
147
|
-
* order preserved for ties, since {@link Array.prototype.sort} is stable).
|
|
148
|
-
* Cached until the next mutation. See {@link sortedRulesCache}.
|
|
149
|
-
*/
|
|
150
|
-
private getSortedRules;
|
|
151
|
-
/**
|
|
152
|
-
* Get the number of currently registered rules (excludes phrase trie entries).
|
|
153
|
-
*/
|
|
154
|
-
get ruleCount(): number;
|
|
155
|
-
/**
|
|
156
|
-
* Register a multi-word phrase for fusion into a single compound token.
|
|
157
|
-
*
|
|
158
|
-
* This is the preferred way to add phrase patterns. It inserts into the
|
|
159
|
-
* internal {@link PhraseTrie}, which collapses all phrase rules into a
|
|
160
|
-
* single O(depth) trie walk per position, no separate rule scanning.
|
|
161
|
-
*
|
|
162
|
-
* @param phrase - Multi-word phrase (e.g., "to the power of", "abyssal whip")
|
|
163
|
-
* @param tokenType - Target token type after fusion (e.g., "CARET", "ITEM")
|
|
164
|
-
*/
|
|
165
|
-
addPhrase(phrase: string, tokenType: string): void;
|
|
166
|
-
/**
|
|
167
|
-
* Check whether a word can start any registered phrase.
|
|
168
|
-
*
|
|
169
|
-
* Used by {@link implicitMultiplyRule} to suppress `*` insertion
|
|
170
|
-
* before phrase-starting identifiers (e.g., "2 power of 3" → `2 ^ 3`,
|
|
171
|
-
* not `2 * power of 3`). Delegates to {@link PhraseTrie.canStart}.
|
|
172
|
-
*/
|
|
173
|
-
/**
|
|
174
|
-
* Get all registered phrases and their target token types.
|
|
175
|
-
*
|
|
176
|
-
* Exposes the full phrase trie structure for diagnostic rendering
|
|
177
|
-
* in the playground's NormalizerTab. Returns ALL registered phrases,
|
|
178
|
-
* not just the ones that matched in the last evaluation.
|
|
179
|
-
*/
|
|
180
|
-
getPhrases(): Record<string, string>;
|
|
181
|
-
canStartPhrase(word: string): boolean;
|
|
182
|
-
/**
|
|
183
|
-
* Normalize a token stream by applying all registered rules.
|
|
184
|
-
*
|
|
185
|
-
* ## Algorithm
|
|
186
|
-
* Applies rules greedily left-to-right in multiple passes:
|
|
187
|
-
* 1. Sort rules by priority (descending)
|
|
188
|
-
* 2. Walk the token stream left to right
|
|
189
|
-
* 3. At each position, try rules in priority order
|
|
190
|
-
* 4. On match: consume matched tokens, insert replacements, restart from insert point
|
|
191
|
-
* 5. On no match: pass token through unchanged
|
|
192
|
-
* 6. Repeat until a full pass produces no changes, or maxPasses is reached
|
|
193
|
-
*
|
|
194
|
-
* ## Fusion tracking
|
|
195
|
-
* When a rule consumes more tokens than it produces, the normalizer calls
|
|
196
|
-
* `onFusion` with a {@link TokenFusion} record for diagnostic collection.
|
|
197
|
-
* This populates {@link NormalizerOutput.fusions} in the playground pipeline view.
|
|
198
|
-
*
|
|
199
|
-
* ## Safety
|
|
200
|
-
* If the normalized token count exceeds {@link NormalizerOptions.maxTokens},
|
|
201
|
-
* an Error is thrown to prevent memory exhaustion from runaway rule expansion.
|
|
202
|
-
*
|
|
203
|
-
* @param tokens - Raw tokens from the lexer
|
|
204
|
-
* @param onFusion - Optional fusion callback (overrides {@link NormalizerOptions.onFusion})
|
|
205
|
-
* @returns Normalized tokens ready for parsing
|
|
206
|
-
* @throws {Error} If the normalized token count exceeds maxTokens
|
|
207
|
-
*/
|
|
208
|
-
normalize(tokens: Token[], onFusion?: (fusion: TokenFusion) => void): Token[];
|
|
209
|
-
}
|
|
1
|
+
import { N as NormalizerMatch, a as NormalizerRule } from './TokenNormalizer-DGVa24Q-.cjs';
|
|
2
|
+
export { b as NormalizerOptions, T as TokenFusion, c as TokenNormalizer, d as createFusedToken } from './TokenNormalizer-DGVa24Q-.cjs';
|
|
3
|
+
import { T as Token } from './Token-BzG5G4ja.cjs';
|
|
210
4
|
|
|
211
5
|
/**
|
|
212
6
|
* PhraseTrie, optimized word-level trie for multi-word phrase fusion.
|
|
@@ -361,4 +155,4 @@ declare function createBuiltinNormalizerRules(): NormalizerRule[];
|
|
|
361
155
|
*/
|
|
362
156
|
declare const BUILTIN_PHRASES: Record<string, string>;
|
|
363
157
|
|
|
364
|
-
export { BUILTIN_PHRASES, NormalizerMatch,
|
|
158
|
+
export { BUILTIN_PHRASES, NormalizerMatch, NormalizerRule, PhraseTrie, createBuiltinNormalizerRules, implicitMultiplyRule };
|
package/dist/normalizer.d.ts
CHANGED
|
@@ -1,212 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* TokenNormalizer, post-lexer token normalization pass.
|
|
6
|
-
*
|
|
7
|
-
* ## Purpose
|
|
8
|
-
* Applies domain-specific {@link NormalizerRule | NormalizerRules} to the raw
|
|
9
|
-
* token stream produced by the {@link ExpressionLexer}. This keeps the lexer
|
|
10
|
-
* slim and focused on single-token production, while multi-token pattern
|
|
11
|
-
* matching (phrases, implicit operators, domain merges) lives here.
|
|
12
|
-
*
|
|
13
|
-
* ## What rules can do
|
|
14
|
-
* - **Phrase fusion**: Merge consecutive words into compound tokens
|
|
15
|
-
* (e.g., `IDENT + ... + IDENT` → `CARET`)
|
|
16
|
-
* - **Implicit operator insertion**: Insert missing operators between tokens
|
|
17
|
-
* (e.g., `NUMBER IDENT` → `NUMBER STAR IDENT`)
|
|
18
|
-
* - **Domain-specific transformations**: Coalesce item names, currency pairs,
|
|
19
|
-
* percentage syntax, etc.
|
|
20
|
-
*
|
|
21
|
-
* ## Architecture
|
|
22
|
-
* Providers register NormalizerRules alongside Parselets and OpCode handlers
|
|
23
|
-
* via {@link IEnginePackage.normalizerRules}. The normalizer applies them
|
|
24
|
-
* greedily left-to-right in multiple passes with safety limits.
|
|
25
|
-
*
|
|
26
|
-
* @module TokenNormalizer
|
|
27
|
-
*/
|
|
28
|
-
|
|
29
|
-
/**
|
|
30
|
-
* Configuration options for the normalization pass.
|
|
31
|
-
*
|
|
32
|
-
* These control safety limits and diagnostic callbacks. The defaults
|
|
33
|
-
* are chosen to be generous enough for any realistic expression while
|
|
34
|
-
* preventing runaway token expansion from recursive rules.
|
|
35
|
-
*/
|
|
36
|
-
interface NormalizerOptions {
|
|
37
|
-
/**
|
|
38
|
-
* Maximum number of full passes over the token stream before bailing out.
|
|
39
|
-
* Prevents infinite loops from recursive rule chains.
|
|
40
|
-
* @default 100
|
|
41
|
-
*/
|
|
42
|
-
maxPasses?: number;
|
|
43
|
-
/**
|
|
44
|
-
* Maximum number of tokens allowed after normalization.
|
|
45
|
-
* If exceeded, an Error is thrown rather than passing a bloated stream
|
|
46
|
-
* to the parser.
|
|
47
|
-
* @default 10000
|
|
48
|
-
*/
|
|
49
|
-
maxTokens?: number;
|
|
50
|
-
/**
|
|
51
|
-
* Callback invoked for each fusion event during normalization.
|
|
52
|
-
* Used by diagnostic mode to populate {@link NormalizerOutput.fusions}.
|
|
53
|
-
* When `undefined`, fusions are still tracked internally but no callbacks fire.
|
|
54
|
-
*/
|
|
55
|
-
onFusion?: (fusion: TokenFusion) => void;
|
|
56
|
-
}
|
|
57
|
-
/**
|
|
58
|
-
* Creates a new normalized token from fused source tokens.
|
|
59
|
-
*
|
|
60
|
-
* The fused token inherits position information (offset, line, column)
|
|
61
|
-
* from the first source token, which preserves source-map accuracy
|
|
62
|
-
* for error messages and diagnostic highlighting.
|
|
63
|
-
*
|
|
64
|
-
* @param type - The new token type (e.g., "CARET", "TIMES_BY")
|
|
65
|
-
* @param text - The combined text representation (e.g., "to the power of")
|
|
66
|
-
* @param sourceTokens - The original tokens being fused (at least 2)
|
|
67
|
-
* @returns A new {@link LexerToken} with the fused type and combined text
|
|
68
|
-
*/
|
|
69
|
-
declare function createFusedToken(type: string, text: string, sourceTokens: Token[]): Token;
|
|
70
|
-
/**
|
|
71
|
-
* Token normalizer: applies {@link NormalizerRule | NormalizerRules} to a token stream.
|
|
72
|
-
*
|
|
73
|
-
* ## Lifecycle
|
|
74
|
-
* 1. **Registration**: Rules are added via {@link register} and sorted by priority
|
|
75
|
-
* 2. **Normalization**: {@link normalize} applies rules greedily left-to-right
|
|
76
|
-
* 3. **Cleanup**: {@link clear} or {@link unregister} removes rules
|
|
77
|
-
*
|
|
78
|
-
* ## Normalization algorithm
|
|
79
|
-
* The normalizer uses a greedy left-to-right multi-pass algorithm:
|
|
80
|
-
* - At each token position, rules are tried in priority order (highest first)
|
|
81
|
-
* - When a rule matches, matched tokens are consumed and replaced
|
|
82
|
-
* - Processing continues from the replacement position
|
|
83
|
-
* - Multiple passes handle cascading matches (one rule's output triggers another)
|
|
84
|
-
* - Safety limits ({@link NormalizerOptions.maxPasses}) prevent infinite loops
|
|
85
|
-
*
|
|
86
|
-
* @example
|
|
87
|
-
* ```ts
|
|
88
|
-
* const normalizer = new TokenNormalizer();
|
|
89
|
-
* normalizer.register(phraseRule); // "to the power of" → CARET
|
|
90
|
-
* normalizer.register(implicitMultRule); // "2 x" → "2 * x"
|
|
91
|
-
* const normalized = normalizer.normalize(rawTokens);
|
|
92
|
-
* ```
|
|
93
|
-
*/
|
|
94
|
-
declare class TokenNormalizer {
|
|
95
|
-
/** Registered rules, unsorted, the source of truth. */
|
|
96
|
-
private rules;
|
|
97
|
-
/**
|
|
98
|
-
* Priority-sorted copy of {@link rules}, rebuilt lazily on the next
|
|
99
|
-
* {@link normalize} call after a mutation. Rules are registered once at
|
|
100
|
-
* engine/package-registration time and essentially never change during a
|
|
101
|
-
* session, but normalize() runs on every keystroke-driven evaluation, an
|
|
102
|
-
* earlier version re-sorted a fresh copy of `rules` on every single call,
|
|
103
|
-
* which meant every keystroke paid for an allocation + sort of a list that
|
|
104
|
-
* had usually not changed since the last one. `null` means "stale, rebuild
|
|
105
|
-
* on next use"; {@link register}/{@link unregister}/{@link clear} all
|
|
106
|
-
* invalidate it.
|
|
107
|
-
*/
|
|
108
|
-
private sortedRulesCache;
|
|
109
|
-
/**
|
|
110
|
-
* Phrase trie for single-pass multi-word phrase fusion.
|
|
111
|
-
* Tried at each token position BEFORE other rules, the trie walk
|
|
112
|
-
* is O(depth) vs O(R × W) for separate rule matching.
|
|
113
|
-
*/
|
|
114
|
-
private phraseTrie;
|
|
115
|
-
/** Merged options with defaults applied. */
|
|
116
|
-
private options;
|
|
117
|
-
/**
|
|
118
|
-
* @param options - Configuration overrides for safety limits and diagnostic callbacks
|
|
119
|
-
*/
|
|
120
|
-
constructor(options?: NormalizerOptions);
|
|
121
|
-
/**
|
|
122
|
-
* Register a normalization rule.
|
|
123
|
-
*
|
|
124
|
-
* Rules are sorted by priority (descending) on each {@link normalize} call.
|
|
125
|
-
* Multiple rules can share the same priority, they are tried in registration
|
|
126
|
-
* order when priorities are equal.
|
|
127
|
-
*
|
|
128
|
-
* @param rule - The rule to register
|
|
129
|
-
*/
|
|
130
|
-
register(rule: NormalizerRule): void;
|
|
131
|
-
/**
|
|
132
|
-
* Unregister a normalization rule by its {@link NormalizerRule.name | name}.
|
|
133
|
-
*
|
|
134
|
-
* If multiple rules share the same name, all are removed. This is safe to
|
|
135
|
-
* call with a name that doesn't match any rule, it simply has no effect.
|
|
136
|
-
*
|
|
137
|
-
* @param ruleName - The name of the rule to remove
|
|
138
|
-
*/
|
|
139
|
-
unregister(ruleName: string): void;
|
|
140
|
-
/**
|
|
141
|
-
* Remove all registered rules, resetting the normalizer to its initial state.
|
|
142
|
-
* Also clears the phrase trie.
|
|
143
|
-
*/
|
|
144
|
-
clear(): void;
|
|
145
|
-
/**
|
|
146
|
-
* Priority-sorted view of {@link rules} (descending priority; registration
|
|
147
|
-
* order preserved for ties, since {@link Array.prototype.sort} is stable).
|
|
148
|
-
* Cached until the next mutation. See {@link sortedRulesCache}.
|
|
149
|
-
*/
|
|
150
|
-
private getSortedRules;
|
|
151
|
-
/**
|
|
152
|
-
* Get the number of currently registered rules (excludes phrase trie entries).
|
|
153
|
-
*/
|
|
154
|
-
get ruleCount(): number;
|
|
155
|
-
/**
|
|
156
|
-
* Register a multi-word phrase for fusion into a single compound token.
|
|
157
|
-
*
|
|
158
|
-
* This is the preferred way to add phrase patterns. It inserts into the
|
|
159
|
-
* internal {@link PhraseTrie}, which collapses all phrase rules into a
|
|
160
|
-
* single O(depth) trie walk per position, no separate rule scanning.
|
|
161
|
-
*
|
|
162
|
-
* @param phrase - Multi-word phrase (e.g., "to the power of", "abyssal whip")
|
|
163
|
-
* @param tokenType - Target token type after fusion (e.g., "CARET", "ITEM")
|
|
164
|
-
*/
|
|
165
|
-
addPhrase(phrase: string, tokenType: string): void;
|
|
166
|
-
/**
|
|
167
|
-
* Check whether a word can start any registered phrase.
|
|
168
|
-
*
|
|
169
|
-
* Used by {@link implicitMultiplyRule} to suppress `*` insertion
|
|
170
|
-
* before phrase-starting identifiers (e.g., "2 power of 3" → `2 ^ 3`,
|
|
171
|
-
* not `2 * power of 3`). Delegates to {@link PhraseTrie.canStart}.
|
|
172
|
-
*/
|
|
173
|
-
/**
|
|
174
|
-
* Get all registered phrases and their target token types.
|
|
175
|
-
*
|
|
176
|
-
* Exposes the full phrase trie structure for diagnostic rendering
|
|
177
|
-
* in the playground's NormalizerTab. Returns ALL registered phrases,
|
|
178
|
-
* not just the ones that matched in the last evaluation.
|
|
179
|
-
*/
|
|
180
|
-
getPhrases(): Record<string, string>;
|
|
181
|
-
canStartPhrase(word: string): boolean;
|
|
182
|
-
/**
|
|
183
|
-
* Normalize a token stream by applying all registered rules.
|
|
184
|
-
*
|
|
185
|
-
* ## Algorithm
|
|
186
|
-
* Applies rules greedily left-to-right in multiple passes:
|
|
187
|
-
* 1. Sort rules by priority (descending)
|
|
188
|
-
* 2. Walk the token stream left to right
|
|
189
|
-
* 3. At each position, try rules in priority order
|
|
190
|
-
* 4. On match: consume matched tokens, insert replacements, restart from insert point
|
|
191
|
-
* 5. On no match: pass token through unchanged
|
|
192
|
-
* 6. Repeat until a full pass produces no changes, or maxPasses is reached
|
|
193
|
-
*
|
|
194
|
-
* ## Fusion tracking
|
|
195
|
-
* When a rule consumes more tokens than it produces, the normalizer calls
|
|
196
|
-
* `onFusion` with a {@link TokenFusion} record for diagnostic collection.
|
|
197
|
-
* This populates {@link NormalizerOutput.fusions} in the playground pipeline view.
|
|
198
|
-
*
|
|
199
|
-
* ## Safety
|
|
200
|
-
* If the normalized token count exceeds {@link NormalizerOptions.maxTokens},
|
|
201
|
-
* an Error is thrown to prevent memory exhaustion from runaway rule expansion.
|
|
202
|
-
*
|
|
203
|
-
* @param tokens - Raw tokens from the lexer
|
|
204
|
-
* @param onFusion - Optional fusion callback (overrides {@link NormalizerOptions.onFusion})
|
|
205
|
-
* @returns Normalized tokens ready for parsing
|
|
206
|
-
* @throws {Error} If the normalized token count exceeds maxTokens
|
|
207
|
-
*/
|
|
208
|
-
normalize(tokens: Token[], onFusion?: (fusion: TokenFusion) => void): Token[];
|
|
209
|
-
}
|
|
1
|
+
import { N as NormalizerMatch, a as NormalizerRule } from './TokenNormalizer-t_GotBxr.js';
|
|
2
|
+
export { b as NormalizerOptions, T as TokenFusion, c as TokenNormalizer, d as createFusedToken } from './TokenNormalizer-t_GotBxr.js';
|
|
3
|
+
import { T as Token } from './Token-BzG5G4ja.js';
|
|
210
4
|
|
|
211
5
|
/**
|
|
212
6
|
* PhraseTrie, optimized word-level trie for multi-word phrase fusion.
|
|
@@ -361,4 +155,4 @@ declare function createBuiltinNormalizerRules(): NormalizerRule[];
|
|
|
361
155
|
*/
|
|
362
156
|
declare const BUILTIN_PHRASES: Record<string, string>;
|
|
363
157
|
|
|
364
|
-
export { BUILTIN_PHRASES, NormalizerMatch,
|
|
158
|
+
export { BUILTIN_PHRASES, NormalizerMatch, NormalizerRule, PhraseTrie, createBuiltinNormalizerRules, implicitMultiplyRule };
|
package/dist/normalizer.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import './chunk-2RWAXT6O.js';
|
|
2
|
-
export { BUILTIN_PHRASES, PhraseTrie, TokenNormalizer, createBuiltinNormalizerRules, createFusedToken, implicitMultiplyRule } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import './chunk-
|
|
5
|
-
import './chunk-
|
|
6
|
-
import './chunk-
|
|
7
|
-
import './chunk-
|
|
2
|
+
export { BUILTIN_PHRASES, PhraseTrie, TokenNormalizer, createBuiltinNormalizerRules, createFusedToken, implicitMultiplyRule } from './chunk-OFXOTECC.js';
|
|
3
|
+
import './chunk-SFQWJMKT.js';
|
|
4
|
+
import './chunk-GPPLSM2Z.js';
|
|
5
|
+
import './chunk-XBTEO4OB.js';
|
|
6
|
+
import './chunk-M5E34VG5.js';
|
|
7
|
+
import './chunk-PCPX42KL.js';
|
|
8
8
|
import './chunk-6RDSQURE.js';
|
|
9
9
|
import './chunk-YLJ4XMA6.js';
|
|
10
10
|
//# sourceMappingURL=normalizer.js.map
|