tywrap 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -11
- package/SECURITY.md +39 -0
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +8 -0
- package/dist/config/index.js.map +1 -1
- package/dist/core/annotation-parser.d.ts +2 -1
- package/dist/core/annotation-parser.d.ts.map +1 -1
- package/dist/core/annotation-parser.js +6 -3
- package/dist/core/annotation-parser.js.map +1 -1
- package/dist/core/emit-call.d.ts.map +1 -1
- package/dist/core/emit-call.js +1 -1
- package/dist/core/emit-call.js.map +1 -1
- package/dist/core/generator.d.ts +23 -4
- package/dist/core/generator.d.ts.map +1 -1
- package/dist/core/generator.js +243 -170
- package/dist/core/generator.js.map +1 -1
- package/dist/core/mapper.d.ts +3 -2
- package/dist/core/mapper.d.ts.map +1 -1
- package/dist/core/mapper.js +5 -5
- package/dist/core/mapper.js.map +1 -1
- package/dist/dev.d.ts.map +1 -1
- package/dist/dev.js +1 -3
- package/dist/dev.js.map +1 -1
- package/dist/index.d.ts +7 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -4
- package/dist/index.js.map +1 -1
- package/dist/runtime/base-bridge.d.ts +3 -7
- package/dist/runtime/base-bridge.d.ts.map +1 -1
- package/dist/runtime/base-bridge.js +4 -17
- package/dist/runtime/base-bridge.js.map +1 -1
- package/dist/runtime/bounded-context.d.ts +3 -22
- package/dist/runtime/bounded-context.d.ts.map +1 -1
- package/dist/runtime/bounded-context.js +13 -53
- package/dist/runtime/bounded-context.js.map +1 -1
- package/dist/runtime/bridge-codec.d.ts +1 -1
- package/dist/runtime/bridge-codec.d.ts.map +1 -1
- package/dist/runtime/bridge-codec.js +78 -55
- package/dist/runtime/bridge-codec.js.map +1 -1
- package/dist/runtime/errors.d.ts +16 -0
- package/dist/runtime/errors.d.ts.map +1 -1
- package/dist/runtime/errors.js +17 -0
- package/dist/runtime/errors.js.map +1 -1
- package/dist/runtime/frame-codec.d.ts +111 -0
- package/dist/runtime/frame-codec.d.ts.map +1 -0
- package/dist/runtime/frame-codec.js +352 -0
- package/dist/runtime/frame-codec.js.map +1 -0
- package/dist/runtime/http-transport.d.ts +1 -1
- package/dist/runtime/http-transport.d.ts.map +1 -1
- package/dist/runtime/http-transport.js +1 -1
- package/dist/runtime/http-transport.js.map +1 -1
- package/dist/runtime/index.d.ts +2 -1
- package/dist/runtime/index.d.ts.map +1 -1
- package/dist/runtime/index.js +2 -1
- package/dist/runtime/index.js.map +1 -1
- package/dist/runtime/node.d.ts +7 -23
- package/dist/runtime/node.d.ts.map +1 -1
- package/dist/runtime/node.js +5 -78
- package/dist/runtime/node.js.map +1 -1
- package/dist/runtime/pooled-transport.d.ts +120 -59
- package/dist/runtime/pooled-transport.d.ts.map +1 -1
- package/dist/runtime/pooled-transport.js +345 -78
- package/dist/runtime/pooled-transport.js.map +1 -1
- package/dist/runtime/pyodide-bootstrap-core.generated.d.ts.map +1 -1
- package/dist/runtime/pyodide-bootstrap-core.generated.js +1 -1
- package/dist/runtime/pyodide-bootstrap-core.generated.js.map +1 -1
- package/dist/runtime/pyodide-transport.d.ts +1 -1
- package/dist/runtime/pyodide-transport.d.ts.map +1 -1
- package/dist/runtime/pyodide-transport.js +2 -3
- package/dist/runtime/pyodide-transport.js.map +1 -1
- package/dist/runtime/rpc-client.d.ts +7 -36
- package/dist/runtime/rpc-client.d.ts.map +1 -1
- package/dist/runtime/rpc-client.js +37 -72
- package/dist/runtime/rpc-client.js.map +1 -1
- package/dist/runtime/subprocess-transport.d.ts +134 -8
- package/dist/runtime/subprocess-transport.d.ts.map +1 -1
- package/dist/runtime/subprocess-transport.js +490 -65
- package/dist/runtime/subprocess-transport.js.map +1 -1
- package/dist/runtime/timed-out-request-tracker.d.ts +2 -1
- package/dist/runtime/timed-out-request-tracker.d.ts.map +1 -1
- package/dist/runtime/transport.d.ts +90 -18
- package/dist/runtime/transport.d.ts.map +1 -1
- package/dist/runtime/transport.js +21 -1
- package/dist/runtime/transport.js.map +1 -1
- package/dist/runtime/validators.d.ts +49 -0
- package/dist/runtime/validators.d.ts.map +1 -1
- package/dist/runtime/validators.js +152 -0
- package/dist/runtime/validators.js.map +1 -1
- package/dist/types/index.d.ts +14 -15
- package/dist/types/index.d.ts.map +1 -1
- package/dist/tywrap.d.ts +3 -2
- package/dist/tywrap.d.ts.map +1 -1
- package/dist/tywrap.js +140 -21
- package/dist/tywrap.js.map +1 -1
- package/dist/utils/cache.d.ts +3 -16
- package/dist/utils/cache.d.ts.map +1 -1
- package/dist/utils/codec.d.ts +1 -0
- package/dist/utils/codec.d.ts.map +1 -1
- package/dist/utils/codec.js +165 -17
- package/dist/utils/codec.js.map +1 -1
- package/dist/utils/ir-cache.d.ts +2 -1
- package/dist/utils/ir-cache.d.ts.map +1 -1
- package/dist/utils/runtime.d.ts +0 -29
- package/dist/utils/runtime.d.ts.map +1 -1
- package/dist/utils/runtime.js +16 -107
- package/dist/utils/runtime.js.map +1 -1
- package/dist/version.js +1 -1
- package/package.json +7 -6
- package/runtime/frame_codec.py +430 -0
- package/runtime/python_bridge.py +213 -54
- package/runtime/tywrap_bridge_core.py +217 -147
- package/src/config/index.ts +11 -0
- package/src/core/annotation-parser.ts +9 -5
- package/src/core/emit-call.ts +1 -7
- package/src/core/generator.ts +315 -205
- package/src/core/mapper.ts +8 -8
- package/src/dev.ts +1 -3
- package/src/index.ts +7 -3
- package/src/runtime/base-bridge.ts +5 -30
- package/src/runtime/bounded-context.ts +12 -67
- package/src/runtime/bridge-codec.ts +94 -65
- package/src/runtime/errors.ts +21 -0
- package/src/runtime/frame-codec.ts +469 -0
- package/src/runtime/http-transport.ts +6 -1
- package/src/runtime/index.ts +7 -6
- package/src/runtime/node.ts +17 -104
- package/src/runtime/pooled-transport.ts +424 -90
- package/src/runtime/pyodide-bootstrap-core.generated.ts +1 -1
- package/src/runtime/pyodide-transport.ts +7 -3
- package/src/runtime/rpc-client.ts +58 -93
- package/src/runtime/subprocess-transport.ts +585 -80
- package/src/runtime/timed-out-request-tracker.ts +1 -1
- package/src/runtime/transport.ts +112 -22
- package/src/runtime/validators.ts +204 -0
- package/src/types/index.ts +21 -35
- package/src/tywrap.ts +157 -30
- package/src/utils/cache.ts +3 -3
- package/src/utils/codec.ts +205 -16
- package/src/utils/ir-cache.ts +1 -1
- package/src/utils/runtime.ts +17 -128
- package/src/version.ts +1 -1
- package/dist/core/discovery.d.ts +0 -103
- package/dist/core/discovery.d.ts.map +0 -1
- package/dist/core/discovery.js +0 -380
- package/dist/core/discovery.js.map +0 -1
- package/dist/core/validation.d.ts +0 -102
- package/dist/core/validation.d.ts.map +0 -1
- package/dist/core/validation.js +0 -490
- package/dist/core/validation.js.map +0 -1
- package/dist/runtime/base.d.ts +0 -22
- package/dist/runtime/base.d.ts.map +0 -1
- package/dist/runtime/base.js +0 -23
- package/dist/runtime/base.js.map +0 -1
- package/dist/runtime/transport-pool.d.ts +0 -196
- package/dist/runtime/transport-pool.d.ts.map +0 -1
- package/dist/runtime/transport-pool.js +0 -418
- package/dist/runtime/transport-pool.js.map +0 -1
- package/runtime/__pycache__/_tywrap_member_fixtures.cpython-311.pyc +0 -0
- package/runtime/__pycache__/safe_codec.cpython-311.pyc +0 -0
- package/runtime/__pycache__/tywrap_bridge_core.cpython-311.pyc +0 -0
- package/runtime/safe_codec.py +0 -352
- package/src/core/discovery.ts +0 -477
- package/src/core/validation.ts +0 -729
- package/src/runtime/base.ts +0 -24
- package/src/runtime/transport-pool.ts +0 -538
package/src/tywrap.ts
CHANGED
|
@@ -16,13 +16,15 @@ import type {
|
|
|
16
16
|
Parameter,
|
|
17
17
|
PythonType,
|
|
18
18
|
PythonModuleConfig,
|
|
19
|
+
IrContract,
|
|
19
20
|
} from './types/index.js';
|
|
20
21
|
import { fsUtils, pathUtils, processUtils, isWindows } from './utils/runtime.js';
|
|
21
22
|
import { globalCache } from './utils/cache.js';
|
|
22
23
|
import { resolvePythonExecutable } from './utils/python.js';
|
|
23
24
|
import { computeIrCacheFilename } from './utils/ir-cache.js';
|
|
25
|
+
import { logger } from './utils/logger.js';
|
|
24
26
|
|
|
25
|
-
const TYWRAP_IR_VERSION = '0.
|
|
27
|
+
const TYWRAP_IR_VERSION = '0.4.0';
|
|
26
28
|
|
|
27
29
|
// Collect unknown typing constructs encountered during annotation parsing (per-generate run)
|
|
28
30
|
let unknownTypeNamesCollector: Map<string, number> = new Map();
|
|
@@ -34,7 +36,7 @@ function recordUnknown(name: string): void {
|
|
|
34
36
|
/**
|
|
35
37
|
* Main tywrap function
|
|
36
38
|
*/
|
|
37
|
-
|
|
39
|
+
interface TywrapInstance {
|
|
38
40
|
mapper: TypeMapper;
|
|
39
41
|
generator: CodeGenerator;
|
|
40
42
|
options: Partial<TywrapOptions>;
|
|
@@ -42,7 +44,9 @@ export interface TywrapInstance {
|
|
|
42
44
|
|
|
43
45
|
export async function tywrap(options: Partial<TywrapOptions> = {}): Promise<TywrapInstance> {
|
|
44
46
|
const mapper = new TypeMapper({ presets: options.types?.presets });
|
|
45
|
-
const generator = new CodeGenerator(mapper
|
|
47
|
+
const generator = new CodeGenerator(mapper, {
|
|
48
|
+
onTypeDegrade: typeName => recordUnknown(`unresolvable generated type ${typeName}`),
|
|
49
|
+
});
|
|
46
50
|
|
|
47
51
|
globalCache.setDebug(options.debug ?? false);
|
|
48
52
|
|
|
@@ -63,7 +67,7 @@ export interface GenerateRunOptions {
|
|
|
63
67
|
|
|
64
68
|
export interface GenerateFailure {
|
|
65
69
|
module: string;
|
|
66
|
-
code: 'ir-unavailable';
|
|
70
|
+
code: 'ir-unavailable' | 'ir-version-mismatch' | 'contract-invalid';
|
|
67
71
|
message: string;
|
|
68
72
|
}
|
|
69
73
|
|
|
@@ -83,6 +87,68 @@ function normalizeForComparison(text: string): string {
|
|
|
83
87
|
return text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
84
88
|
}
|
|
85
89
|
|
|
90
|
+
function stableJson(value: unknown): unknown {
|
|
91
|
+
if (Array.isArray(value)) {
|
|
92
|
+
return value.map(stableJson);
|
|
93
|
+
}
|
|
94
|
+
if (value !== null && typeof value === 'object') {
|
|
95
|
+
return Object.fromEntries(
|
|
96
|
+
Object.entries(value as Record<string, unknown>)
|
|
97
|
+
.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
|
|
98
|
+
.map(([key, child]) => [key, stableJson(child)])
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
return value;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function serializeContract(ir: IrContract): string {
|
|
105
|
+
const contract = { ...ir };
|
|
106
|
+
delete contract.metadata;
|
|
107
|
+
return `${JSON.stringify(stableJson(contract), null, 2)}\n`;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function validateIrVersion(
|
|
111
|
+
ir: unknown,
|
|
112
|
+
source: string
|
|
113
|
+
): { ir: IrContract | null; error?: string; code?: GenerateFailure['code'] } {
|
|
114
|
+
if (ir === null || typeof ir !== 'object' || Array.isArray(ir)) {
|
|
115
|
+
return { ir: null, code: 'contract-invalid', error: `${source} is not a JSON IR object.` };
|
|
116
|
+
}
|
|
117
|
+
const contract = ir as IrContract;
|
|
118
|
+
if (typeof contract.ir_version !== 'string') {
|
|
119
|
+
return {
|
|
120
|
+
ir: null,
|
|
121
|
+
code: 'contract-invalid',
|
|
122
|
+
error: `${source} is missing its string ir_version.`,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
if (contract.ir_version !== TYWRAP_IR_VERSION) {
|
|
126
|
+
return {
|
|
127
|
+
ir: null,
|
|
128
|
+
code: 'ir-version-mismatch',
|
|
129
|
+
error: `IR version mismatch: TypeScript expects ${TYWRAP_IR_VERSION}, but ${source} declares ${contract.ir_version}. Regenerate the contract with a matching tywrap_ir.`,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
if (typeof contract.module !== 'string' || contract.module.length === 0) {
|
|
133
|
+
return { ir: null, code: 'contract-invalid', error: `${source} is missing its module name.` };
|
|
134
|
+
}
|
|
135
|
+
const requiredArrays: ReadonlyArray<readonly [string, unknown]> = [
|
|
136
|
+
['functions', contract.functions],
|
|
137
|
+
['classes', contract.classes],
|
|
138
|
+
['type_aliases', contract.type_aliases],
|
|
139
|
+
];
|
|
140
|
+
for (const [field, value] of requiredArrays) {
|
|
141
|
+
if (!Array.isArray(value)) {
|
|
142
|
+
return {
|
|
143
|
+
ir: null,
|
|
144
|
+
code: 'contract-invalid',
|
|
145
|
+
error: `${source} is missing required array field ${field}.`,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return { ir: contract };
|
|
150
|
+
}
|
|
151
|
+
|
|
86
152
|
async function safeReadFile(path: string): Promise<string | null> {
|
|
87
153
|
try {
|
|
88
154
|
return await fsUtils.readFile(path);
|
|
@@ -106,13 +172,17 @@ async function fetchAndCacheIr(
|
|
|
106
172
|
cacheKey: string,
|
|
107
173
|
caching: boolean,
|
|
108
174
|
checkMode: boolean
|
|
109
|
-
): Promise<{ ir:
|
|
175
|
+
): Promise<{ ir: IrContract | null; error?: string; code?: GenerateFailure['code'] }> {
|
|
110
176
|
let ir: unknown | null = null;
|
|
111
177
|
let irError: string | undefined;
|
|
112
178
|
if (caching && fsUtils.isAvailable() && !checkMode) {
|
|
113
179
|
try {
|
|
114
180
|
const cached = await fsUtils.readFile(pathUtils.join(CACHE_DIR, cacheKey));
|
|
115
|
-
|
|
181
|
+
const cachedValidation = validateIrVersion(
|
|
182
|
+
JSON.parse(cached),
|
|
183
|
+
`Cached Python IR for ${moduleKey}`
|
|
184
|
+
);
|
|
185
|
+
ir = cachedValidation.ir;
|
|
116
186
|
} catch {
|
|
117
187
|
ir = null;
|
|
118
188
|
}
|
|
@@ -124,13 +194,60 @@ async function fetchAndCacheIr(
|
|
|
124
194
|
});
|
|
125
195
|
ir = fetchResult.ir;
|
|
126
196
|
irError = fetchResult.error;
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
197
|
+
}
|
|
198
|
+
if (!ir) {
|
|
199
|
+
return { ir: null, error: irError, code: 'ir-unavailable' };
|
|
200
|
+
}
|
|
201
|
+
const validated = validateIrVersion(ir, `Python IR for ${moduleKey}`);
|
|
202
|
+
if (!validated.ir) {
|
|
203
|
+
return {
|
|
204
|
+
ir: null,
|
|
205
|
+
error: irError ?? validated.error,
|
|
206
|
+
code: validated.code ?? 'ir-unavailable',
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
if (caching && fsUtils.isAvailable() && !checkMode) {
|
|
210
|
+
try {
|
|
211
|
+
await fsUtils.writeFile(pathUtils.join(CACHE_DIR, cacheKey), JSON.stringify(validated.ir));
|
|
212
|
+
} catch {}
|
|
213
|
+
}
|
|
214
|
+
return { ir: validated.ir };
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
async function readContractInput(
|
|
218
|
+
moduleKey: string,
|
|
219
|
+
contractInput: TywrapOptions['contractInput']
|
|
220
|
+
): Promise<{ ir: IrContract | null; error?: string; code?: GenerateFailure['code'] }> {
|
|
221
|
+
const inputPath = typeof contractInput === 'string' ? contractInput : contractInput?.[moduleKey];
|
|
222
|
+
if (!inputPath) {
|
|
223
|
+
return {
|
|
224
|
+
ir: null,
|
|
225
|
+
code: 'contract-invalid',
|
|
226
|
+
error: `No contractInput path configured for module ${moduleKey}.`,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
try {
|
|
230
|
+
const parsed = JSON.parse(await fsUtils.readFile(inputPath)) as unknown;
|
|
231
|
+
const validated = validateIrVersion(parsed, `Contract ${inputPath}`);
|
|
232
|
+
if (!validated.ir) {
|
|
233
|
+
return { ir: null, error: validated.error, code: validated.code };
|
|
131
234
|
}
|
|
235
|
+
if (validated.ir.module !== moduleKey) {
|
|
236
|
+
return {
|
|
237
|
+
ir: null,
|
|
238
|
+
code: 'contract-invalid',
|
|
239
|
+
error: `Contract ${inputPath} is for module ${validated.ir.module}, not ${moduleKey}.`,
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
return { ir: validated.ir };
|
|
243
|
+
} catch (error: unknown) {
|
|
244
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
245
|
+
return {
|
|
246
|
+
ir: null,
|
|
247
|
+
code: 'contract-invalid',
|
|
248
|
+
error: `Failed to read contract ${inputPath}: ${message}`,
|
|
249
|
+
};
|
|
132
250
|
}
|
|
133
|
-
return { ir, error: irError };
|
|
134
251
|
}
|
|
135
252
|
|
|
136
253
|
/**
|
|
@@ -320,25 +437,39 @@ export async function generate(
|
|
|
320
437
|
unknownTypeNamesCollector = new Map();
|
|
321
438
|
// Resolve module IR via the Python extractor (with optional cache)
|
|
322
439
|
const cacheKey = await computeCacheKey(moduleKey, resolvedOptions);
|
|
323
|
-
const
|
|
324
|
-
moduleKey,
|
|
325
|
-
resolvedOptions,
|
|
326
|
-
|
|
327
|
-
cacheKey,
|
|
328
|
-
caching,
|
|
329
|
-
checkMode
|
|
330
|
-
);
|
|
440
|
+
const irResult = resolvedOptions.contractInput
|
|
441
|
+
? await readContractInput(moduleKey, resolvedOptions.contractInput)
|
|
442
|
+
: await fetchAndCacheIr(moduleKey, resolvedOptions, pythonPath, cacheKey, caching, checkMode);
|
|
443
|
+
const { ir, error: irError } = irResult;
|
|
331
444
|
if (!ir) {
|
|
332
445
|
const message = `No IR produced for module ${moduleKey}${irError ? `: ${irError}` : ''}`;
|
|
333
446
|
warnings.push(message);
|
|
334
447
|
failures.push({
|
|
335
448
|
module: moduleKey,
|
|
336
|
-
code: 'ir-unavailable',
|
|
449
|
+
code: irResult.code ?? 'ir-unavailable',
|
|
337
450
|
message,
|
|
338
451
|
});
|
|
339
452
|
continue;
|
|
340
453
|
}
|
|
341
454
|
|
|
455
|
+
const irWarnings = Array.isArray((ir as Record<string, unknown>).warnings)
|
|
456
|
+
? ((ir as Record<string, unknown>).warnings as unknown[])
|
|
457
|
+
: [];
|
|
458
|
+
for (const warning of irWarnings) {
|
|
459
|
+
if (typeof warning !== 'string') {
|
|
460
|
+
continue;
|
|
461
|
+
}
|
|
462
|
+
// Only type-honesty degrades feed --fail-on-warn. Environment capability
|
|
463
|
+
// notices (e.g. typing.get_overloads missing on Python < 3.11) describe
|
|
464
|
+
// the analyzing interpreter, not the generated types, and must not fail
|
|
465
|
+
// an otherwise clean build.
|
|
466
|
+
if (warning.startsWith('Return annotation for ')) {
|
|
467
|
+
recordUnknown(`Python IR warning: ${warning}`);
|
|
468
|
+
} else {
|
|
469
|
+
logger.info(`Python IR notice: ${warning}`, { component: 'Generate' });
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
|
|
342
473
|
const moduleModel = transformIrToTsModel(ir);
|
|
343
474
|
|
|
344
475
|
// Apply module-level export filtering (functions/classes + excludes).
|
|
@@ -351,6 +482,10 @@ export async function generate(
|
|
|
351
482
|
const baseName = moduleModel.name || 'module';
|
|
352
483
|
const filesToEmit: Array<{ path: string; content: string }> = [
|
|
353
484
|
{ path: pathUtils.join(outputDir, `${baseName}.generated.ts`), content: gen.typescript },
|
|
485
|
+
{
|
|
486
|
+
path: pathUtils.join(outputDir, `${baseName}.contract.json`),
|
|
487
|
+
content: serializeContract(ir),
|
|
488
|
+
},
|
|
354
489
|
];
|
|
355
490
|
|
|
356
491
|
// Optional .d.ts emission (header-only declarations mirroring exports)
|
|
@@ -441,15 +576,7 @@ async function fetchPythonIr(
|
|
|
441
576
|
try {
|
|
442
577
|
const primary = await execAndParseIr(
|
|
443
578
|
pythonPath,
|
|
444
|
-
[
|
|
445
|
-
'-m',
|
|
446
|
-
'tywrap_ir',
|
|
447
|
-
'--module',
|
|
448
|
-
moduleName,
|
|
449
|
-
'--ir-version',
|
|
450
|
-
TYWRAP_IR_VERSION,
|
|
451
|
-
'--no-pretty',
|
|
452
|
-
],
|
|
579
|
+
['-m', 'tywrap_ir', '--module', moduleName, '--no-pretty'],
|
|
453
580
|
execOptions,
|
|
454
581
|
'tywrap_ir output'
|
|
455
582
|
);
|
|
@@ -484,7 +611,7 @@ async function fetchPythonIr(
|
|
|
484
611
|
|
|
485
612
|
const fallback = await execAndParseIr(
|
|
486
613
|
pythonPath,
|
|
487
|
-
[localMain, '--module', moduleName, '--
|
|
614
|
+
[localMain, '--module', moduleName, '--no-pretty'],
|
|
488
615
|
execOptions,
|
|
489
616
|
'tywrap_ir fallback output'
|
|
490
617
|
);
|
package/src/utils/cache.ts
CHANGED
|
@@ -12,7 +12,7 @@ import { getComponentLogger } from './logger.js';
|
|
|
12
12
|
|
|
13
13
|
const log = getComponentLogger('Cache');
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
interface CacheEntry<T = unknown> {
|
|
16
16
|
key: string;
|
|
17
17
|
data: T;
|
|
18
18
|
timestamp: number;
|
|
@@ -27,7 +27,7 @@ export interface CacheEntry<T = unknown> {
|
|
|
27
27
|
};
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
interface CacheStats {
|
|
31
31
|
totalEntries: number;
|
|
32
32
|
totalSize: number;
|
|
33
33
|
hitRate: number;
|
|
@@ -36,7 +36,7 @@ export interface CacheStats {
|
|
|
36
36
|
memoryUsage: number;
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
interface CacheConfig {
|
|
40
40
|
baseDir: string;
|
|
41
41
|
maxSize: number; // Maximum cache size in bytes
|
|
42
42
|
maxAge: number; // Maximum age in milliseconds
|
package/src/utils/codec.ts
CHANGED
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
* Sklearn estimators: { "__tywrap__": "sklearn.estimator", "encoding": "json", ... }
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
+
import { tagDecodedShape } from '../runtime/validators.js';
|
|
13
|
+
|
|
12
14
|
// Avoid hard dependency on apache-arrow types at compile time to keep install optional.
|
|
13
15
|
export type ArrowTable = { readonly numCols?: number; readonly numRows?: number } & Record<
|
|
14
16
|
string,
|
|
@@ -68,6 +70,7 @@ export type ValueEnvelope =
|
|
|
68
70
|
readonly b64?: string; // when encoding=arrow
|
|
69
71
|
readonly data?: unknown; // when encoding=json
|
|
70
72
|
readonly shape?: readonly number[];
|
|
73
|
+
readonly dtype?: string | null;
|
|
71
74
|
}
|
|
72
75
|
| {
|
|
73
76
|
readonly __tywrap__: 'scipy.sparse';
|
|
@@ -442,13 +445,16 @@ function decodeArrowOrJsonEnvelope<T>(
|
|
|
442
445
|
throw new Error(`Invalid ${typeTag} envelope: missing b64`);
|
|
443
446
|
}
|
|
444
447
|
const bytes = fromBase64(b64);
|
|
445
|
-
|
|
448
|
+
const decoded = decodeArrow(bytes);
|
|
449
|
+
return isPromiseLike(decoded)
|
|
450
|
+
? decoded.then(item => tagDecodedShape(item, { marker: typeTag as 'dataframe' | 'series' }))
|
|
451
|
+
: tagDecodedShape(decoded, { marker: typeTag as 'dataframe' | 'series' });
|
|
446
452
|
}
|
|
447
453
|
if (encoding === 'json') {
|
|
448
454
|
if (!('data' in value)) {
|
|
449
455
|
throw new Error(`Invalid ${typeTag} envelope: missing data`);
|
|
450
456
|
}
|
|
451
|
-
return value.data;
|
|
457
|
+
return tagDecodedShape(value.data, { marker: typeTag as 'dataframe' | 'series' });
|
|
452
458
|
}
|
|
453
459
|
throw new Error(`Invalid ${typeTag} envelope: unsupported encoding ${String(encoding)}`);
|
|
454
460
|
}
|
|
@@ -463,6 +469,8 @@ const decodeNdarrayEnvelope: EnvelopeHandler = (value, decodeArrow) => {
|
|
|
463
469
|
const encoding = value.encoding;
|
|
464
470
|
const shapeValue = value.shape;
|
|
465
471
|
const shape = isNumberArray(shapeValue) ? shapeValue : undefined;
|
|
472
|
+
const dtype = typeof value.dtype === 'string' ? value.dtype : undefined;
|
|
473
|
+
const metadata = { marker: 'ndarray' as const, dims: shape?.length, dtype };
|
|
466
474
|
|
|
467
475
|
if (encoding === 'arrow') {
|
|
468
476
|
const b64 = value.b64;
|
|
@@ -480,34 +488,59 @@ const decodeNdarrayEnvelope: EnvelopeHandler = (value, decodeArrow) => {
|
|
|
480
488
|
return decoded.then(data => {
|
|
481
489
|
const values = extractArrowValues(data);
|
|
482
490
|
if (!values) {
|
|
483
|
-
return data; // Fallback:
|
|
491
|
+
return tagDecodedShape(data, metadata); // Fallback: keep provenance on raw data.
|
|
484
492
|
}
|
|
485
493
|
// Reshape scalars and multi-dimensional arrays, but not 1D
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
494
|
+
return tagDecodedShape(
|
|
495
|
+
shape && shape.length !== 1 ? reshapeArray(values, shape) : values,
|
|
496
|
+
metadata
|
|
497
|
+
);
|
|
490
498
|
});
|
|
491
499
|
}
|
|
492
500
|
const values = extractArrowValues(decoded);
|
|
493
501
|
if (!values) {
|
|
494
|
-
return decoded; // Fallback:
|
|
502
|
+
return tagDecodedShape(decoded, metadata); // Fallback: keep provenance on raw data.
|
|
495
503
|
}
|
|
496
504
|
// Reshape scalars and multi-dimensional arrays, but not 1D
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
505
|
+
return tagDecodedShape(
|
|
506
|
+
shape && shape.length !== 1 ? reshapeArray(values, shape) : values,
|
|
507
|
+
metadata
|
|
508
|
+
);
|
|
501
509
|
}
|
|
502
510
|
if (encoding === 'json') {
|
|
503
511
|
if (!('data' in value)) {
|
|
504
512
|
throw new Error('Invalid ndarray envelope: missing data');
|
|
505
513
|
}
|
|
506
|
-
return value.data;
|
|
514
|
+
return tagDecodedShape(value.data, metadata);
|
|
507
515
|
}
|
|
508
516
|
throw new Error(`Invalid ndarray envelope: unsupported encoding ${String(encoding)}`);
|
|
509
517
|
};
|
|
510
518
|
|
|
519
|
+
/**
|
|
520
|
+
* Assert an array holds only integer indices within [0, bound). Used to re-validate
|
|
521
|
+
* scipy.sparse index arrays on the JS side so a corrupt/oversized index can never
|
|
522
|
+
* silently address out of the declared shape.
|
|
523
|
+
*
|
|
524
|
+
* Why: the JS decoder never reconstructs a Python object, but it IS the boundary a
|
|
525
|
+
* downstream consumer trusts — validating index ranges here turns a corrupt payload
|
|
526
|
+
* into a clear, early failure instead of a confusing downstream error.
|
|
527
|
+
*/
|
|
528
|
+
function assertIndexArrayInRange(arr: readonly unknown[], bound: number, label: string): void {
|
|
529
|
+
for (let i = 0; i < arr.length; i += 1) {
|
|
530
|
+
const idx = arr[i];
|
|
531
|
+
if (typeof idx !== 'number' || !Number.isInteger(idx)) {
|
|
532
|
+
throw new Error(
|
|
533
|
+
`Invalid scipy.sparse envelope: ${label}[${i}] must be an integer, got ${String(idx)}`
|
|
534
|
+
);
|
|
535
|
+
}
|
|
536
|
+
if (idx < 0 || idx >= bound) {
|
|
537
|
+
throw new Error(
|
|
538
|
+
`Invalid scipy.sparse envelope: ${label}[${i}]=${idx} is out of range [0, ${bound})`
|
|
539
|
+
);
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
|
|
511
544
|
const decodeScipySparseEnvelope: EnvelopeHandler = value => {
|
|
512
545
|
const encoding = value.encoding;
|
|
513
546
|
if (encoding !== 'json') {
|
|
@@ -522,10 +555,16 @@ const decodeScipySparseEnvelope: EnvelopeHandler = value => {
|
|
|
522
555
|
!Array.isArray(shape) ||
|
|
523
556
|
shape.length !== 2 ||
|
|
524
557
|
typeof shape[0] !== 'number' ||
|
|
525
|
-
|
|
558
|
+
!Number.isInteger(shape[0]) ||
|
|
559
|
+
shape[0] < 0 ||
|
|
560
|
+
typeof shape[1] !== 'number' ||
|
|
561
|
+
!Number.isInteger(shape[1]) ||
|
|
562
|
+
shape[1] < 0
|
|
526
563
|
) {
|
|
527
|
-
throw new Error('Invalid scipy.sparse envelope: shape must be a 2-item
|
|
564
|
+
throw new Error('Invalid scipy.sparse envelope: shape must be a 2-item non-negative integer[]');
|
|
528
565
|
}
|
|
566
|
+
const rows = shape[0];
|
|
567
|
+
const cols = shape[1];
|
|
529
568
|
const data = value.data;
|
|
530
569
|
if (!Array.isArray(data)) {
|
|
531
570
|
throw new Error('Invalid scipy.sparse envelope: data must be an array');
|
|
@@ -539,6 +578,16 @@ const decodeScipySparseEnvelope: EnvelopeHandler = value => {
|
|
|
539
578
|
if (!Array.isArray(row) || !Array.isArray(col)) {
|
|
540
579
|
throw new Error('Invalid scipy.sparse envelope: coo requires row and col arrays');
|
|
541
580
|
}
|
|
581
|
+
// COO: one (row, col, value) triple per stored entry, so all three arrays
|
|
582
|
+
// share a length; row/col index into [0, rows)/[0, cols) respectively.
|
|
583
|
+
if (row.length !== data.length || col.length !== data.length) {
|
|
584
|
+
throw new Error(
|
|
585
|
+
`Invalid scipy.sparse envelope: coo row/col/data lengths must match ` +
|
|
586
|
+
`(data=${data.length}, row=${row.length}, col=${col.length})`
|
|
587
|
+
);
|
|
588
|
+
}
|
|
589
|
+
assertIndexArrayInRange(row, rows, 'row');
|
|
590
|
+
assertIndexArrayInRange(col, cols, 'col');
|
|
542
591
|
return {
|
|
543
592
|
format,
|
|
544
593
|
shape,
|
|
@@ -554,6 +603,52 @@ const decodeScipySparseEnvelope: EnvelopeHandler = value => {
|
|
|
554
603
|
if (!Array.isArray(indices) || !Array.isArray(indptr)) {
|
|
555
604
|
throw new Error('Invalid scipy.sparse envelope: csr/csc requires indices and indptr arrays');
|
|
556
605
|
}
|
|
606
|
+
// CSR/CSC: one column-index (CSR) or row-index (CSC) per stored value, so
|
|
607
|
+
// indices and data share a length. indptr partitions indices into one segment
|
|
608
|
+
// per major axis (rows for CSR, cols for CSC), so indptr has majorAxis + 1
|
|
609
|
+
// entries; the inner indices address the minor axis.
|
|
610
|
+
if (indices.length !== data.length) {
|
|
611
|
+
throw new Error(
|
|
612
|
+
`Invalid scipy.sparse envelope: ${format} indices/data lengths must match ` +
|
|
613
|
+
`(data=${data.length}, indices=${indices.length})`
|
|
614
|
+
);
|
|
615
|
+
}
|
|
616
|
+
const majorAxis = format === 'csr' ? rows : cols;
|
|
617
|
+
const minorAxis = format === 'csr' ? cols : rows;
|
|
618
|
+
if (indptr.length !== majorAxis + 1) {
|
|
619
|
+
throw new Error(
|
|
620
|
+
`Invalid scipy.sparse envelope: ${format} indptr length must be ${majorAxis + 1} ` +
|
|
621
|
+
`(${format === 'csr' ? 'rows' : 'cols'}+1), got ${indptr.length}`
|
|
622
|
+
);
|
|
623
|
+
}
|
|
624
|
+
// indptr must be a valid CSR/CSC pointer array: integers that start at 0, end
|
|
625
|
+
// at data.length, and never decrease — otherwise a structurally impossible
|
|
626
|
+
// matrix (e.g. [0, 99, 1]) would slip through the length check above.
|
|
627
|
+
for (let i = 0; i < indptr.length; i += 1) {
|
|
628
|
+
const ptr = indptr[i];
|
|
629
|
+
if (typeof ptr !== 'number' || !Number.isInteger(ptr)) {
|
|
630
|
+
throw new Error(
|
|
631
|
+
`Invalid scipy.sparse envelope: indptr[${i}] must be an integer, got ${String(ptr)}`
|
|
632
|
+
);
|
|
633
|
+
}
|
|
634
|
+
if (ptr < 0 || ptr > data.length) {
|
|
635
|
+
throw new Error(
|
|
636
|
+
`Invalid scipy.sparse envelope: indptr[${i}]=${ptr} is out of range [0, ${data.length}]`
|
|
637
|
+
);
|
|
638
|
+
}
|
|
639
|
+
if (i > 0 && ptr < (indptr[i - 1] as number)) {
|
|
640
|
+
throw new Error('Invalid scipy.sparse envelope: indptr must be non-decreasing');
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
if (indptr[0] !== 0) {
|
|
644
|
+
throw new Error('Invalid scipy.sparse envelope: indptr must start at 0');
|
|
645
|
+
}
|
|
646
|
+
if (indptr[indptr.length - 1] !== data.length) {
|
|
647
|
+
throw new Error(
|
|
648
|
+
`Invalid scipy.sparse envelope: indptr must end at data.length (${data.length})`
|
|
649
|
+
);
|
|
650
|
+
}
|
|
651
|
+
assertIndexArrayInRange(indices, minorAxis, 'indices');
|
|
557
652
|
return {
|
|
558
653
|
format,
|
|
559
654
|
shape,
|
|
@@ -564,6 +659,11 @@ const decodeScipySparseEnvelope: EnvelopeHandler = value => {
|
|
|
564
659
|
} satisfies SparseMatrix;
|
|
565
660
|
};
|
|
566
661
|
|
|
662
|
+
/** Product of a shape's dimensions (the element count). [] (scalar) -> 1. */
|
|
663
|
+
function shapeProduct(shape: readonly number[]): number {
|
|
664
|
+
return shape.reduce((acc, dim) => acc * dim, 1);
|
|
665
|
+
}
|
|
666
|
+
|
|
567
667
|
const decodeTorchTensorEnvelope: EnvelopeHandler = <T>(
|
|
568
668
|
value: { [k: string]: unknown },
|
|
569
669
|
_decodeArrow: (bytes: Uint8Array) => MaybePromise<T>,
|
|
@@ -580,20 +680,96 @@ const decodeTorchTensorEnvelope: EnvelopeHandler = <T>(
|
|
|
580
680
|
if (!isObject(nested) || (nested as { __tywrap__?: unknown }).__tywrap__ !== 'ndarray') {
|
|
581
681
|
throw new Error('Invalid torch.tensor envelope: value must be an ndarray envelope');
|
|
582
682
|
}
|
|
583
|
-
const decoded = recurse(nested);
|
|
584
683
|
const shapeValue = value.shape;
|
|
585
684
|
const shape = isNumberArray(shapeValue) ? shapeValue : undefined;
|
|
685
|
+
// The tensor shape must be a non-negative-integer dimension list. A negative or
|
|
686
|
+
// non-integer dim is a corrupt envelope, not a valid tensor.
|
|
687
|
+
if (shape) {
|
|
688
|
+
for (let i = 0; i < shape.length; i += 1) {
|
|
689
|
+
const dim = shape[i] as number;
|
|
690
|
+
if (!Number.isInteger(dim) || dim < 0) {
|
|
691
|
+
throw new Error(
|
|
692
|
+
`Invalid torch.tensor envelope: shape[${i}]=${dim} must be a non-negative integer`
|
|
693
|
+
);
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
// Cross-check the tensor shape's element count against the nested ndarray's
|
|
698
|
+
// declared shape (metadata only — no decode needed). A mismatch means the two
|
|
699
|
+
// shapes disagree about how many elements the payload holds.
|
|
700
|
+
const nestedShapeValue = (nested as { shape?: unknown }).shape;
|
|
701
|
+
const nestedShape = isNumberArray(nestedShapeValue) ? nestedShapeValue : undefined;
|
|
702
|
+
if (shape && nestedShape && shapeProduct(shape) !== shapeProduct(nestedShape)) {
|
|
703
|
+
throw new Error(
|
|
704
|
+
`Invalid torch.tensor envelope: shape ${JSON.stringify(shape)} ` +
|
|
705
|
+
`(product ${shapeProduct(shape)}) disagrees with nested ndarray shape ` +
|
|
706
|
+
`${JSON.stringify(nestedShape)} (product ${shapeProduct(nestedShape)})`
|
|
707
|
+
);
|
|
708
|
+
}
|
|
586
709
|
const dtypeValue = value.dtype;
|
|
587
710
|
const dtype = typeof dtypeValue === 'string' ? dtypeValue : undefined;
|
|
588
711
|
const deviceValue = value.device;
|
|
712
|
+
if (deviceValue !== undefined && (typeof deviceValue !== 'string' || deviceValue.length === 0)) {
|
|
713
|
+
throw new Error(
|
|
714
|
+
'Invalid torch.tensor envelope: device must be a non-empty string when provided'
|
|
715
|
+
);
|
|
716
|
+
}
|
|
589
717
|
const device = typeof deviceValue === 'string' ? deviceValue : undefined;
|
|
590
718
|
|
|
719
|
+
const decoded = recurse(nested);
|
|
591
720
|
if (isPromiseLike(decoded)) {
|
|
592
721
|
return decoded.then(data => ({ data, shape, dtype, device })) as Promise<T | unknown>;
|
|
593
722
|
}
|
|
594
723
|
return { data: decoded, shape, dtype, device } satisfies TorchTensor;
|
|
595
724
|
};
|
|
596
725
|
|
|
726
|
+
/**
|
|
727
|
+
* Recursively assert a value is plain JSON (null | boolean | number | string |
|
|
728
|
+
* JSON array | plain object of JSON). Rejects functions, symbols, bigints, class
|
|
729
|
+
* instances, and any non-finite number — the things a metadata-only sklearn
|
|
730
|
+
* envelope must never carry. This validates; it never reconstructs.
|
|
731
|
+
*/
|
|
732
|
+
function assertPlainJson(value: unknown, path: string): void {
|
|
733
|
+
if (value === null) {
|
|
734
|
+
return;
|
|
735
|
+
}
|
|
736
|
+
const t = typeof value;
|
|
737
|
+
if (t === 'string' || t === 'boolean') {
|
|
738
|
+
return;
|
|
739
|
+
}
|
|
740
|
+
if (t === 'number') {
|
|
741
|
+
if (!Number.isFinite(value as number)) {
|
|
742
|
+
throw new Error(
|
|
743
|
+
`Invalid sklearn.estimator envelope: ${path} must be a finite JSON number, got ${String(value)}`
|
|
744
|
+
);
|
|
745
|
+
}
|
|
746
|
+
return;
|
|
747
|
+
}
|
|
748
|
+
if (Array.isArray(value)) {
|
|
749
|
+
value.forEach((item, i) => assertPlainJson(item, `${path}[${i}]`));
|
|
750
|
+
return;
|
|
751
|
+
}
|
|
752
|
+
if (t === 'object') {
|
|
753
|
+
// Reject exotic objects (class instances, Map/Set, etc.): a JSON object is a
|
|
754
|
+
// plain object whose prototype is Object.prototype or null.
|
|
755
|
+
const proto = Object.getPrototypeOf(value as object);
|
|
756
|
+
if (proto !== Object.prototype && proto !== null) {
|
|
757
|
+
throw new Error(
|
|
758
|
+
`Invalid sklearn.estimator envelope: ${path} must be a plain JSON object, ` +
|
|
759
|
+
`got ${(value as object).constructor?.name ?? 'object'}`
|
|
760
|
+
);
|
|
761
|
+
}
|
|
762
|
+
for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
|
|
763
|
+
assertPlainJson(v, `${path}.${k}`);
|
|
764
|
+
}
|
|
765
|
+
return;
|
|
766
|
+
}
|
|
767
|
+
// function | symbol | bigint | undefined
|
|
768
|
+
throw new Error(
|
|
769
|
+
`Invalid sklearn.estimator envelope: ${path} is not JSON-serializable (type ${t})`
|
|
770
|
+
);
|
|
771
|
+
}
|
|
772
|
+
|
|
597
773
|
const decodeSklearnEstimatorEnvelope: EnvelopeHandler = value => {
|
|
598
774
|
const encoding = value.encoding;
|
|
599
775
|
if (encoding !== 'json') {
|
|
@@ -607,6 +783,19 @@ const decodeSklearnEstimatorEnvelope: EnvelopeHandler = value => {
|
|
|
607
783
|
'Invalid sklearn.estimator envelope: expected className/module strings + params object'
|
|
608
784
|
);
|
|
609
785
|
}
|
|
786
|
+
// params must be a PLAIN JSON object end to end — metadata-only estimators never
|
|
787
|
+
// carry callables, class instances, or nested non-JSON values. Validate (do not
|
|
788
|
+
// reconstruct) so a corrupt envelope fails clearly instead of leaking a function
|
|
789
|
+
// or exotic object to a downstream consumer.
|
|
790
|
+
if (
|
|
791
|
+
Object.getPrototypeOf(params) !== Object.prototype &&
|
|
792
|
+
Object.getPrototypeOf(params) !== null
|
|
793
|
+
) {
|
|
794
|
+
throw new Error('Invalid sklearn.estimator envelope: params must be a plain JSON object');
|
|
795
|
+
}
|
|
796
|
+
for (const [k, v] of Object.entries(params)) {
|
|
797
|
+
assertPlainJson(v, `params.${k}`);
|
|
798
|
+
}
|
|
610
799
|
const versionValue = value.version;
|
|
611
800
|
if (versionValue !== undefined && typeof versionValue !== 'string') {
|
|
612
801
|
throw new Error('Invalid sklearn.estimator envelope: version must be a string when provided');
|