tywrap 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/generator.d.ts.map +1 -1
- package/dist/core/generator.js +40 -9
- package/dist/core/generator.js.map +1 -1
- package/dist/runtime/base-bridge.d.ts +57 -0
- package/dist/runtime/base-bridge.d.ts.map +1 -0
- package/dist/runtime/base-bridge.js +72 -0
- package/dist/runtime/base-bridge.js.map +1 -0
- package/dist/runtime/http-transport.d.ts +11 -1
- package/dist/runtime/http-transport.d.ts.map +1 -1
- package/dist/runtime/http-transport.js +19 -0
- package/dist/runtime/http-transport.js.map +1 -1
- package/dist/runtime/http.d.ts +5 -12
- package/dist/runtime/http.d.ts.map +1 -1
- package/dist/runtime/http.js +6 -29
- package/dist/runtime/http.js.map +1 -1
- package/dist/runtime/index.d.ts +1 -1
- package/dist/runtime/index.d.ts.map +1 -1
- package/dist/runtime/index.js.map +1 -1
- package/dist/runtime/node.d.ts +12 -19
- package/dist/runtime/node.d.ts.map +1 -1
- package/dist/runtime/node.js +14 -34
- package/dist/runtime/node.js.map +1 -1
- package/dist/runtime/pooled-transport.d.ts +21 -2
- package/dist/runtime/pooled-transport.d.ts.map +1 -1
- package/dist/runtime/pooled-transport.js +16 -0
- package/dist/runtime/pooled-transport.js.map +1 -1
- package/dist/runtime/pyodide-bootstrap-core.generated.d.ts.map +1 -1
- package/dist/runtime/pyodide-bootstrap-core.generated.js +1 -1
- package/dist/runtime/pyodide-bootstrap-core.generated.js.map +1 -1
- package/dist/runtime/pyodide-transport.d.ts +12 -1
- package/dist/runtime/pyodide-transport.d.ts.map +1 -1
- package/dist/runtime/pyodide-transport.js +20 -0
- package/dist/runtime/pyodide-transport.js.map +1 -1
- package/dist/runtime/pyodide.d.ts +5 -12
- package/dist/runtime/pyodide.d.ts.map +1 -1
- package/dist/runtime/pyodide.js +6 -29
- package/dist/runtime/pyodide.js.map +1 -1
- package/dist/runtime/rpc-client.d.ts +14 -1
- package/dist/runtime/rpc-client.d.ts.map +1 -1
- package/dist/runtime/rpc-client.js +15 -0
- package/dist/runtime/rpc-client.js.map +1 -1
- package/dist/runtime/subprocess-transport.d.ts +10 -1
- package/dist/runtime/subprocess-transport.d.ts.map +1 -1
- package/dist/runtime/subprocess-transport.js +18 -0
- package/dist/runtime/subprocess-transport.js.map +1 -1
- package/dist/runtime/transport.d.ts +59 -0
- package/dist/runtime/transport.d.ts.map +1 -1
- package/dist/runtime/transport.js +1 -0
- package/dist/runtime/transport.js.map +1 -1
- package/dist/types/index.d.ts +35 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/tywrap.d.ts.map +1 -1
- package/dist/tywrap.js +212 -149
- package/dist/tywrap.js.map +1 -1
- package/dist/utils/codec.d.ts +2 -0
- package/dist/utils/codec.d.ts.map +1 -1
- package/dist/utils/codec.js +53 -2
- package/dist/utils/codec.js.map +1 -1
- package/dist/version.js +1 -1
- package/package.json +7 -1
- package/runtime/__pycache__/_tywrap_member_fixtures.cpython-311.pyc +0 -0
- package/runtime/__pycache__/safe_codec.cpython-311.pyc +0 -0
- package/runtime/__pycache__/tywrap_bridge_core.cpython-311.pyc +0 -0
- package/runtime/tywrap_bridge_core.py +55 -3
- package/src/core/generator.ts +50 -11
- package/src/runtime/base-bridge.ts +106 -0
- package/src/runtime/http-transport.ts +21 -1
- package/src/runtime/http.ts +7 -51
- package/src/runtime/index.ts +1 -0
- package/src/runtime/node.ts +17 -52
- package/src/runtime/pooled-transport.ts +25 -2
- package/src/runtime/pyodide-bootstrap-core.generated.ts +1 -1
- package/src/runtime/pyodide-transport.ts +22 -0
- package/src/runtime/pyodide.ts +7 -52
- package/src/runtime/rpc-client.ts +17 -0
- package/src/runtime/subprocess-transport.ts +20 -1
- package/src/runtime/transport.ts +71 -0
- package/src/types/index.ts +37 -0
- package/src/tywrap.ts +274 -163
- package/src/utils/codec.ts +61 -4
- package/src/version.ts +1 -1
package/src/types/index.ts
CHANGED
|
@@ -13,6 +13,16 @@ export interface PythonModule {
|
|
|
13
13
|
exports: string[];
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
+
/**
|
|
17
|
+
* How a callable is bound on its owning class.
|
|
18
|
+
*
|
|
19
|
+
* Mirrors `IRFunction.method_kind` from the Python IR (tywrap_ir 0.3.0):
|
|
20
|
+
* `'instance'` (default, keeps `self`), `'class'` (keeps `cls`), or
|
|
21
|
+
* `'static'` (no implicit first parameter). Module-level functions retain the
|
|
22
|
+
* `'instance'` default; it is only meaningful for class members.
|
|
23
|
+
*/
|
|
24
|
+
export type PythonMethodKind = 'instance' | 'class' | 'static';
|
|
25
|
+
|
|
16
26
|
export interface PythonFunction {
|
|
17
27
|
name: string;
|
|
18
28
|
signature: FunctionSignature;
|
|
@@ -23,6 +33,28 @@ export interface PythonFunction {
|
|
|
23
33
|
typeParameters?: PythonGenericParameter[];
|
|
24
34
|
returnType: PythonType;
|
|
25
35
|
parameters: Parameter[];
|
|
36
|
+
/**
|
|
37
|
+
* Binding of this callable on its owning class. Defaults to `'instance'`.
|
|
38
|
+
* @see PythonMethodKind
|
|
39
|
+
*/
|
|
40
|
+
methodKind?: PythonMethodKind;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* A `@property` or `functools.cached_property` exposed on a class.
|
|
45
|
+
*
|
|
46
|
+
* Mirrors `IRAccessor` from the Python IR (tywrap_ir 0.3.0). Distinct from
|
|
47
|
+
* {@link PythonClass.properties} (which model TypedDict/NamedTuple/dataclass
|
|
48
|
+
* data shapes): accessors are computed attributes backed by a getter.
|
|
49
|
+
*/
|
|
50
|
+
export interface PythonAccessor {
|
|
51
|
+
name: string;
|
|
52
|
+
type: PythonType;
|
|
53
|
+
docstring?: string;
|
|
54
|
+
/** True when there is no setter. `undefined` when undeterminable. */
|
|
55
|
+
readOnly?: boolean;
|
|
56
|
+
/** True for `functools.cached_property`. */
|
|
57
|
+
isCached: boolean;
|
|
26
58
|
}
|
|
27
59
|
|
|
28
60
|
export interface PythonClass {
|
|
@@ -30,6 +62,11 @@ export interface PythonClass {
|
|
|
30
62
|
bases: string[];
|
|
31
63
|
methods: PythonFunction[];
|
|
32
64
|
properties: Property[];
|
|
65
|
+
/**
|
|
66
|
+
* `@property` / `functools.cached_property` accessors, emitted as TS getters.
|
|
67
|
+
* @see PythonAccessor
|
|
68
|
+
*/
|
|
69
|
+
accessors?: PythonAccessor[];
|
|
33
70
|
docstring?: string;
|
|
34
71
|
decorators: string[];
|
|
35
72
|
kind?: 'class' | 'protocol' | 'typed_dict' | 'namedtuple' | 'dataclass' | 'pydantic';
|
package/src/tywrap.ts
CHANGED
|
@@ -15,13 +15,14 @@ import type {
|
|
|
15
15
|
PythonTypeAlias,
|
|
16
16
|
Parameter,
|
|
17
17
|
PythonType,
|
|
18
|
+
PythonModuleConfig,
|
|
18
19
|
} from './types/index.js';
|
|
19
20
|
import { fsUtils, pathUtils, processUtils, isWindows } from './utils/runtime.js';
|
|
20
21
|
import { globalCache } from './utils/cache.js';
|
|
21
22
|
import { resolvePythonExecutable } from './utils/python.js';
|
|
22
23
|
import { computeIrCacheFilename } from './utils/ir-cache.js';
|
|
23
24
|
|
|
24
|
-
const TYWRAP_IR_VERSION = '0.
|
|
25
|
+
const TYWRAP_IR_VERSION = '0.3.0';
|
|
25
26
|
|
|
26
27
|
// Collect unknown typing constructs encountered during annotation parsing (per-generate run)
|
|
27
28
|
let unknownTypeNamesCollector: Map<string, number> = new Map();
|
|
@@ -90,6 +91,195 @@ async function safeReadFile(path: string): Promise<string | null> {
|
|
|
90
91
|
}
|
|
91
92
|
}
|
|
92
93
|
|
|
94
|
+
const CACHE_DIR = '.tywrap/cache';
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Resolve module IR via the Python extractor, consulting and (best-effort)
|
|
98
|
+
* populating the on-disk cache. Mirrors the original inline cache-then-fetch
|
|
99
|
+
* logic exactly: the cache is only read/written when caching is enabled, the
|
|
100
|
+
* filesystem is available, and we are not in check mode.
|
|
101
|
+
*/
|
|
102
|
+
async function fetchAndCacheIr(
|
|
103
|
+
moduleKey: string,
|
|
104
|
+
resolvedOptions: ReturnType<typeof createConfig>,
|
|
105
|
+
pythonPath: string,
|
|
106
|
+
cacheKey: string,
|
|
107
|
+
caching: boolean,
|
|
108
|
+
checkMode: boolean
|
|
109
|
+
): Promise<{ ir: unknown | null; error?: string }> {
|
|
110
|
+
let ir: unknown | null = null;
|
|
111
|
+
let irError: string | undefined;
|
|
112
|
+
if (caching && fsUtils.isAvailable() && !checkMode) {
|
|
113
|
+
try {
|
|
114
|
+
const cached = await fsUtils.readFile(pathUtils.join(CACHE_DIR, cacheKey));
|
|
115
|
+
ir = JSON.parse(cached);
|
|
116
|
+
} catch {
|
|
117
|
+
ir = null;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (!ir) {
|
|
121
|
+
const fetchResult = await fetchPythonIr(moduleKey, pythonPath, {
|
|
122
|
+
timeoutMs: resolvedOptions.runtime?.node?.timeout,
|
|
123
|
+
pythonImportPath: resolvedOptions.pythonImportPath,
|
|
124
|
+
});
|
|
125
|
+
ir = fetchResult.ir;
|
|
126
|
+
irError = fetchResult.error;
|
|
127
|
+
if (ir && caching && fsUtils.isAvailable() && !checkMode) {
|
|
128
|
+
try {
|
|
129
|
+
await fsUtils.writeFile(pathUtils.join(CACHE_DIR, cacheKey), JSON.stringify(ir));
|
|
130
|
+
} catch {}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return { ir, error: irError };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Apply module-level export filtering (functions/classes + excludes) in place.
|
|
138
|
+
* Pushes any filtering warnings onto the provided `warnings` array. Behavior is
|
|
139
|
+
* identical to the original inline block.
|
|
140
|
+
*/
|
|
141
|
+
function filterModuleExports(
|
|
142
|
+
moduleModel: TSPythonModule,
|
|
143
|
+
moduleConfig: PythonModuleConfig,
|
|
144
|
+
moduleKey: string,
|
|
145
|
+
warnings: string[]
|
|
146
|
+
): void {
|
|
147
|
+
const builtInDefaultExcludes = new Set([
|
|
148
|
+
'dataclass',
|
|
149
|
+
'property',
|
|
150
|
+
'staticmethod',
|
|
151
|
+
'classmethod',
|
|
152
|
+
'abstractmethod',
|
|
153
|
+
'cached_property',
|
|
154
|
+
]);
|
|
155
|
+
|
|
156
|
+
const excludeExact = new Set((moduleConfig.exclude ?? []).map(String));
|
|
157
|
+
const excludeRegexes: RegExp[] = [];
|
|
158
|
+
for (const pattern of moduleConfig.excludePatterns ?? []) {
|
|
159
|
+
try {
|
|
160
|
+
excludeRegexes.push(new RegExp(String(pattern)));
|
|
161
|
+
} catch (err: unknown) {
|
|
162
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
163
|
+
warnings.push(
|
|
164
|
+
`Module ${moduleKey}: invalid excludePatterns regex "${String(pattern)}": ${message}`
|
|
165
|
+
);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const shouldExclude = (name: string, applyBuiltInDefaults: boolean): boolean => {
|
|
170
|
+
if (excludeExact.has(name)) {
|
|
171
|
+
return true;
|
|
172
|
+
}
|
|
173
|
+
if (excludeRegexes.some(r => r.test(name))) {
|
|
174
|
+
return true;
|
|
175
|
+
}
|
|
176
|
+
if (applyBuiltInDefaults && builtInDefaultExcludes.has(name)) {
|
|
177
|
+
return true;
|
|
178
|
+
}
|
|
179
|
+
return false;
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
if (Array.isArray(moduleConfig.functions)) {
|
|
183
|
+
const allow = new Set(moduleConfig.functions.map(String));
|
|
184
|
+
for (const requested of allow) {
|
|
185
|
+
if (!moduleModel.functions.some(f => f.name === requested)) {
|
|
186
|
+
warnings.push(`Module ${moduleKey}: configured function "${requested}" not found in IR`);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
moduleModel.functions = moduleModel.functions.filter(
|
|
190
|
+
f => allow.has(f.name) && !shouldExclude(f.name, false)
|
|
191
|
+
);
|
|
192
|
+
} else {
|
|
193
|
+
moduleModel.functions = moduleModel.functions.filter(f => !shouldExclude(f.name, true));
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (Array.isArray(moduleConfig.classes)) {
|
|
197
|
+
const allow = new Set(moduleConfig.classes.map(String));
|
|
198
|
+
for (const requested of allow) {
|
|
199
|
+
if (!moduleModel.classes.some(c => c.name === requested)) {
|
|
200
|
+
warnings.push(`Module ${moduleKey}: configured class "${requested}" not found in IR`);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
moduleModel.classes = moduleModel.classes.filter(
|
|
204
|
+
c => allow.has(c.name) && !shouldExclude(c.name, false)
|
|
205
|
+
);
|
|
206
|
+
} else {
|
|
207
|
+
moduleModel.classes = moduleModel.classes.filter(c => !shouldExclude(c.name, true));
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
moduleModel.typeAliases = (moduleModel.typeAliases ?? []).filter(
|
|
211
|
+
alias => !shouldExclude(alias.name, false)
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Either compare the generated files against disk (check mode) or write them.
|
|
217
|
+
* Mutates `written`/`outOfDate` exactly as the original inline branches did.
|
|
218
|
+
*/
|
|
219
|
+
async function emitOrCompareFiles(
|
|
220
|
+
filesToEmit: Array<{ path: string; content: string }>,
|
|
221
|
+
checkMode: boolean,
|
|
222
|
+
written: string[],
|
|
223
|
+
outOfDate: string[]
|
|
224
|
+
): Promise<void> {
|
|
225
|
+
if (checkMode) {
|
|
226
|
+
for (const file of filesToEmit) {
|
|
227
|
+
const existing = await safeReadFile(file.path);
|
|
228
|
+
if (existing === null) {
|
|
229
|
+
outOfDate.push(file.path);
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
if (normalizeForComparison(existing) !== normalizeForComparison(file.content)) {
|
|
233
|
+
outOfDate.push(file.path);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
} else {
|
|
237
|
+
for (const file of filesToEmit) {
|
|
238
|
+
await fsUtils.writeFile(file.path, file.content);
|
|
239
|
+
written.push(file.path);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Emit the best-effort warning summary of unknown typing constructs collected
|
|
246
|
+
* for the current module, and (outside check mode) write the JSON report.
|
|
247
|
+
*/
|
|
248
|
+
async function reportUnknownTypes(
|
|
249
|
+
baseName: string,
|
|
250
|
+
unknownTypeNames: Map<string, number>,
|
|
251
|
+
checkMode: boolean,
|
|
252
|
+
warnings: string[]
|
|
253
|
+
): Promise<void> {
|
|
254
|
+
if (unknownTypeNames.size === 0) {
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
const entries = Array.from(unknownTypeNames.entries()).sort((a, b) => b[1] - a[1]);
|
|
258
|
+
const unkList = entries
|
|
259
|
+
.slice(0, 25)
|
|
260
|
+
.map(([n, c]) => `${n}:${c}`)
|
|
261
|
+
.join(', ');
|
|
262
|
+
warnings.push(
|
|
263
|
+
`Module ${baseName}: unknown typing constructs encountered: ${unkList}${entries.length > 25 ? '…' : ''}`
|
|
264
|
+
);
|
|
265
|
+
// Write JSON report
|
|
266
|
+
if (!checkMode) {
|
|
267
|
+
try {
|
|
268
|
+
const reportsDir = pathUtils.join('.tywrap', 'reports');
|
|
269
|
+
await fsUtils.writeFile(
|
|
270
|
+
pathUtils.join(reportsDir, `${baseName}.json`),
|
|
271
|
+
JSON.stringify({
|
|
272
|
+
module: baseName,
|
|
273
|
+
unknowns: Object.fromEntries(entries),
|
|
274
|
+
generatedAt: new Date().toISOString(),
|
|
275
|
+
})
|
|
276
|
+
);
|
|
277
|
+
} catch {
|
|
278
|
+
// ignore
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
93
283
|
/**
|
|
94
284
|
* Generate TypeScript wrappers for configured Python modules.
|
|
95
285
|
* Minimal MVP implementation: resolves module file, analyzes, generates TS, writes to output.dir
|
|
@@ -107,7 +297,6 @@ export async function generate(
|
|
|
107
297
|
const failures: GenerateFailure[] = [];
|
|
108
298
|
const outputDir = resolvedOptions.output.dir;
|
|
109
299
|
const caching = resolvedOptions.performance.caching;
|
|
110
|
-
const cacheDir = '.tywrap/cache';
|
|
111
300
|
const pythonPath = await resolvePythonExecutable({
|
|
112
301
|
pythonPath: resolvedOptions.runtime?.node?.pythonPath,
|
|
113
302
|
virtualEnv: resolvedOptions.runtime?.node?.virtualEnv,
|
|
@@ -131,29 +320,14 @@ export async function generate(
|
|
|
131
320
|
unknownTypeNamesCollector = new Map();
|
|
132
321
|
// Resolve module IR via the Python extractor (with optional cache)
|
|
133
322
|
const cacheKey = await computeCacheKey(moduleKey, resolvedOptions);
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
if (!ir) {
|
|
145
|
-
const fetchResult = await fetchPythonIr(moduleKey, pythonPath, {
|
|
146
|
-
timeoutMs: resolvedOptions.runtime?.node?.timeout,
|
|
147
|
-
pythonImportPath: resolvedOptions.pythonImportPath,
|
|
148
|
-
});
|
|
149
|
-
ir = fetchResult.ir;
|
|
150
|
-
irError = fetchResult.error;
|
|
151
|
-
if (ir && caching && fsUtils.isAvailable() && !checkMode) {
|
|
152
|
-
try {
|
|
153
|
-
await fsUtils.writeFile(pathUtils.join(cacheDir, cacheKey), JSON.stringify(ir));
|
|
154
|
-
} catch {}
|
|
155
|
-
}
|
|
156
|
-
}
|
|
323
|
+
const { ir, error: irError } = await fetchAndCacheIr(
|
|
324
|
+
moduleKey,
|
|
325
|
+
resolvedOptions,
|
|
326
|
+
pythonPath,
|
|
327
|
+
cacheKey,
|
|
328
|
+
caching,
|
|
329
|
+
checkMode
|
|
330
|
+
);
|
|
157
331
|
if (!ir) {
|
|
158
332
|
const message = `No IR produced for module ${moduleKey}${irError ? `: ${irError}` : ''}`;
|
|
159
333
|
warnings.push(message);
|
|
@@ -168,76 +342,7 @@ export async function generate(
|
|
|
168
342
|
const moduleModel = transformIrToTsModel(ir);
|
|
169
343
|
|
|
170
344
|
// Apply module-level export filtering (functions/classes + excludes).
|
|
171
|
-
|
|
172
|
-
const builtInDefaultExcludes = new Set([
|
|
173
|
-
'dataclass',
|
|
174
|
-
'property',
|
|
175
|
-
'staticmethod',
|
|
176
|
-
'classmethod',
|
|
177
|
-
'abstractmethod',
|
|
178
|
-
'cached_property',
|
|
179
|
-
]);
|
|
180
|
-
|
|
181
|
-
const excludeExact = new Set((moduleConfig.exclude ?? []).map(String));
|
|
182
|
-
const excludeRegexes: RegExp[] = [];
|
|
183
|
-
for (const pattern of moduleConfig.excludePatterns ?? []) {
|
|
184
|
-
try {
|
|
185
|
-
excludeRegexes.push(new RegExp(String(pattern)));
|
|
186
|
-
} catch (err: unknown) {
|
|
187
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
188
|
-
warnings.push(
|
|
189
|
-
`Module ${moduleKey}: invalid excludePatterns regex "${String(pattern)}": ${message}`
|
|
190
|
-
);
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
const shouldExclude = (name: string, applyBuiltInDefaults: boolean): boolean => {
|
|
195
|
-
if (excludeExact.has(name)) {
|
|
196
|
-
return true;
|
|
197
|
-
}
|
|
198
|
-
if (excludeRegexes.some(r => r.test(name))) {
|
|
199
|
-
return true;
|
|
200
|
-
}
|
|
201
|
-
if (applyBuiltInDefaults && builtInDefaultExcludes.has(name)) {
|
|
202
|
-
return true;
|
|
203
|
-
}
|
|
204
|
-
return false;
|
|
205
|
-
};
|
|
206
|
-
|
|
207
|
-
if (Array.isArray(moduleConfig.functions)) {
|
|
208
|
-
const allow = new Set(moduleConfig.functions.map(String));
|
|
209
|
-
for (const requested of allow) {
|
|
210
|
-
if (!moduleModel.functions.some(f => f.name === requested)) {
|
|
211
|
-
warnings.push(
|
|
212
|
-
`Module ${moduleKey}: configured function "${requested}" not found in IR`
|
|
213
|
-
);
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
moduleModel.functions = moduleModel.functions.filter(
|
|
217
|
-
f => allow.has(f.name) && !shouldExclude(f.name, false)
|
|
218
|
-
);
|
|
219
|
-
} else {
|
|
220
|
-
moduleModel.functions = moduleModel.functions.filter(f => !shouldExclude(f.name, true));
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
if (Array.isArray(moduleConfig.classes)) {
|
|
224
|
-
const allow = new Set(moduleConfig.classes.map(String));
|
|
225
|
-
for (const requested of allow) {
|
|
226
|
-
if (!moduleModel.classes.some(c => c.name === requested)) {
|
|
227
|
-
warnings.push(`Module ${moduleKey}: configured class "${requested}" not found in IR`);
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
moduleModel.classes = moduleModel.classes.filter(
|
|
231
|
-
c => allow.has(c.name) && !shouldExclude(c.name, false)
|
|
232
|
-
);
|
|
233
|
-
} else {
|
|
234
|
-
moduleModel.classes = moduleModel.classes.filter(c => !shouldExclude(c.name, true));
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
moduleModel.typeAliases = (moduleModel.typeAliases ?? []).filter(
|
|
238
|
-
alias => !shouldExclude(alias.name, false)
|
|
239
|
-
);
|
|
240
|
-
}
|
|
345
|
+
filterModuleExports(moduleModel, moduleConfig, moduleKey, warnings);
|
|
241
346
|
|
|
242
347
|
// Generate module code
|
|
243
348
|
const annotatedJSDoc = Boolean(resolvedOptions.output?.annotatedJSDoc);
|
|
@@ -264,51 +369,10 @@ export async function generate(
|
|
|
264
369
|
});
|
|
265
370
|
}
|
|
266
371
|
|
|
267
|
-
|
|
268
|
-
for (const file of filesToEmit) {
|
|
269
|
-
const existing = await safeReadFile(file.path);
|
|
270
|
-
if (existing === null) {
|
|
271
|
-
outOfDate.push(file.path);
|
|
272
|
-
continue;
|
|
273
|
-
}
|
|
274
|
-
if (normalizeForComparison(existing) !== normalizeForComparison(file.content)) {
|
|
275
|
-
outOfDate.push(file.path);
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
} else {
|
|
279
|
-
for (const file of filesToEmit) {
|
|
280
|
-
await fsUtils.writeFile(file.path, file.content);
|
|
281
|
-
written.push(file.path);
|
|
282
|
-
}
|
|
283
|
-
}
|
|
372
|
+
await emitOrCompareFiles(filesToEmit, checkMode, written, outOfDate);
|
|
284
373
|
|
|
285
374
|
// Emit warning summary of unknown typing constructs (best-effort)
|
|
286
|
-
|
|
287
|
-
const entries = Array.from(unknownTypeNamesCollector.entries()).sort((a, b) => b[1] - a[1]);
|
|
288
|
-
const unkList = entries
|
|
289
|
-
.slice(0, 25)
|
|
290
|
-
.map(([n, c]) => `${n}:${c}`)
|
|
291
|
-
.join(', ');
|
|
292
|
-
warnings.push(
|
|
293
|
-
`Module ${baseName}: unknown typing constructs encountered: ${unkList}${entries.length > 25 ? '…' : ''}`
|
|
294
|
-
);
|
|
295
|
-
// Write JSON report
|
|
296
|
-
if (!checkMode) {
|
|
297
|
-
try {
|
|
298
|
-
const reportsDir = pathUtils.join('.tywrap', 'reports');
|
|
299
|
-
await fsUtils.writeFile(
|
|
300
|
-
pathUtils.join(reportsDir, `${baseName}.json`),
|
|
301
|
-
JSON.stringify({
|
|
302
|
-
module: baseName,
|
|
303
|
-
unknowns: Object.fromEntries(entries),
|
|
304
|
-
generatedAt: new Date().toISOString(),
|
|
305
|
-
})
|
|
306
|
-
);
|
|
307
|
-
} catch {
|
|
308
|
-
// ignore
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
}
|
|
375
|
+
await reportUnknownTypes(baseName, unknownTypeNamesCollector, checkMode, warnings);
|
|
312
376
|
}
|
|
313
377
|
|
|
314
378
|
if (checkMode) {
|
|
@@ -318,6 +382,40 @@ export async function generate(
|
|
|
318
382
|
return { written, warnings, failures };
|
|
319
383
|
}
|
|
320
384
|
|
|
385
|
+
/**
|
|
386
|
+
* Run the IR extractor (`python <command...>`) and parse its JSON stdout.
|
|
387
|
+
*
|
|
388
|
+
* Returns the exec result alongside a parsed-IR outcome. On a zero exit code the
|
|
389
|
+
* stdout is JSON-parsed; on parse failure the supplied `parseErrorLabel` is used
|
|
390
|
+
* to build the error message. The caller is responsible for handling non-zero
|
|
391
|
+
* exit codes (e.g. the module-missing fallback), so a non-zero exit yields
|
|
392
|
+
* `ir: null` with no `error` set here.
|
|
393
|
+
*/
|
|
394
|
+
async function execAndParseIr(
|
|
395
|
+
pythonPath: string,
|
|
396
|
+
command: string[],
|
|
397
|
+
execOptions: { timeoutMs?: number; env?: Record<string, string> },
|
|
398
|
+
parseErrorLabel: 'tywrap_ir output' | 'tywrap_ir fallback output'
|
|
399
|
+
): Promise<{
|
|
400
|
+
result: { code: number; stdout: string; stderr: string };
|
|
401
|
+
ir: unknown | null;
|
|
402
|
+
error?: string;
|
|
403
|
+
}> {
|
|
404
|
+
const result = await processUtils.exec(pythonPath, command, execOptions);
|
|
405
|
+
if (result.code !== 0) {
|
|
406
|
+
return { result, ir: null };
|
|
407
|
+
}
|
|
408
|
+
try {
|
|
409
|
+
return { result, ir: JSON.parse(result.stdout) };
|
|
410
|
+
} catch {
|
|
411
|
+
return {
|
|
412
|
+
result,
|
|
413
|
+
ir: null,
|
|
414
|
+
error: `Failed to parse ${parseErrorLabel}. stderr: ${result.stderr.trim() || 'empty'}`,
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
321
419
|
/**
|
|
322
420
|
* Invoke the Python IR CLI to get JSON IR for a module.
|
|
323
421
|
*/
|
|
@@ -339,24 +437,27 @@ async function fetchPythonIr(
|
|
|
339
437
|
.filter(Boolean)
|
|
340
438
|
.join(delimiter);
|
|
341
439
|
const env = mergedPyPath ? { PYTHONPATH: mergedPyPath } : undefined;
|
|
440
|
+
const execOptions = { timeoutMs: options.timeoutMs, env };
|
|
342
441
|
try {
|
|
343
|
-
const
|
|
442
|
+
const primary = await execAndParseIr(
|
|
344
443
|
pythonPath,
|
|
345
|
-
[
|
|
346
|
-
|
|
444
|
+
[
|
|
445
|
+
'-m',
|
|
446
|
+
'tywrap_ir',
|
|
447
|
+
'--module',
|
|
448
|
+
moduleName,
|
|
449
|
+
'--ir-version',
|
|
450
|
+
TYWRAP_IR_VERSION,
|
|
451
|
+
'--no-pretty',
|
|
452
|
+
],
|
|
453
|
+
execOptions,
|
|
454
|
+
'tywrap_ir output'
|
|
347
455
|
);
|
|
348
|
-
if (result.code === 0) {
|
|
349
|
-
|
|
350
|
-
return { ir: JSON.parse(result.stdout) };
|
|
351
|
-
} catch {
|
|
352
|
-
return {
|
|
353
|
-
ir: null,
|
|
354
|
-
error: `Failed to parse tywrap_ir output. stderr: ${result.stderr.trim() || 'empty'}`,
|
|
355
|
-
};
|
|
356
|
-
}
|
|
456
|
+
if (primary.result.code === 0) {
|
|
457
|
+
return { ir: primary.ir, error: primary.error };
|
|
357
458
|
}
|
|
358
459
|
|
|
359
|
-
const stderrText = result.stderr.trim();
|
|
460
|
+
const stderrText = primary.result.stderr.trim();
|
|
360
461
|
const isTywrapIrMissing =
|
|
361
462
|
stderrText.includes('No module named') && stderrText.includes('tywrap_ir');
|
|
362
463
|
if (!isTywrapIrMissing) {
|
|
@@ -381,25 +482,19 @@ async function fetchPythonIr(
|
|
|
381
482
|
};
|
|
382
483
|
}
|
|
383
484
|
|
|
384
|
-
const fallback = await
|
|
485
|
+
const fallback = await execAndParseIr(
|
|
385
486
|
pythonPath,
|
|
386
487
|
[localMain, '--module', moduleName, '--ir-version', TYWRAP_IR_VERSION, '--no-pretty'],
|
|
387
|
-
|
|
488
|
+
execOptions,
|
|
489
|
+
'tywrap_ir fallback output'
|
|
388
490
|
);
|
|
389
|
-
if (fallback.code !== 0) {
|
|
390
|
-
return {
|
|
391
|
-
ir: null,
|
|
392
|
-
error: `tywrap_ir failed. stderr: ${fallback.stderr.trim() || stderrText || 'empty'}`,
|
|
393
|
-
};
|
|
394
|
-
}
|
|
395
|
-
try {
|
|
396
|
-
return { ir: JSON.parse(fallback.stdout) };
|
|
397
|
-
} catch {
|
|
491
|
+
if (fallback.result.code !== 0) {
|
|
398
492
|
return {
|
|
399
493
|
ir: null,
|
|
400
|
-
error: `
|
|
494
|
+
error: `tywrap_ir failed. stderr: ${fallback.result.stderr.trim() || stderrText || 'empty'}`,
|
|
401
495
|
};
|
|
402
496
|
}
|
|
497
|
+
return { ir: fallback.ir, error: fallback.error };
|
|
403
498
|
} catch (err) {
|
|
404
499
|
return { ir: null, error: err instanceof Error ? err.message : String(err) };
|
|
405
500
|
}
|
|
@@ -483,6 +578,9 @@ function transformIrToTsModel(ir: unknown): TSPythonModule {
|
|
|
483
578
|
keywordOnly: p.kind === 'KEYWORD_ONLY',
|
|
484
579
|
});
|
|
485
580
|
|
|
581
|
+
const mapMethodKind = (value: unknown): PythonFunction['methodKind'] =>
|
|
582
|
+
value === 'class' || value === 'static' ? value : 'instance';
|
|
583
|
+
|
|
486
584
|
const mapFunc = (
|
|
487
585
|
f: Record<string, unknown>,
|
|
488
586
|
inheritedTypeParameters: readonly PythonGenericParameter[] = []
|
|
@@ -512,6 +610,7 @@ function transformIrToTsModel(ir: unknown): TSPythonModule {
|
|
|
512
610
|
mapParam((v ?? {}) as Record<string, unknown>, annotationTypeParameters)
|
|
513
611
|
)
|
|
514
612
|
: [],
|
|
613
|
+
methodKind: mapMethodKind(f.method_kind),
|
|
515
614
|
};
|
|
516
615
|
};
|
|
517
616
|
|
|
@@ -538,6 +637,18 @@ function transformIrToTsModel(ir: unknown): TSPythonModule {
|
|
|
538
637
|
};
|
|
539
638
|
})
|
|
540
639
|
: [],
|
|
640
|
+
accessors: Array.isArray(c.accessors)
|
|
641
|
+
? (c.accessors as unknown[]).map(v => {
|
|
642
|
+
const a = (v ?? {}) as Record<string, unknown>;
|
|
643
|
+
return {
|
|
644
|
+
name: String(a.name ?? ''),
|
|
645
|
+
type: parseType(a.returns, classTypeParameters),
|
|
646
|
+
docstring: (a.docstring as string | undefined) ?? undefined,
|
|
647
|
+
readOnly: typeof a.read_only === 'boolean' ? a.read_only : undefined,
|
|
648
|
+
isCached: Boolean(a.is_cached),
|
|
649
|
+
};
|
|
650
|
+
})
|
|
651
|
+
: [],
|
|
541
652
|
docstring: (c.docstring as string | undefined) ?? undefined,
|
|
542
653
|
decorators: (c.typed_dict as boolean) ? ['__typed_dict__'] : [],
|
|
543
654
|
kind: (c.typed_dict as boolean)
|
package/src/utils/codec.ts
CHANGED
|
@@ -111,6 +111,24 @@ export type DecodedValue =
|
|
|
111
111
|
|
|
112
112
|
let arrowTableFrom: ((bytes: Uint8Array) => ArrowTable | Uint8Array) | undefined;
|
|
113
113
|
|
|
114
|
+
// Why: lazy auto-registration (on first Arrow decode) imports apache-arrow at most
|
|
115
|
+
// once per process. We cache the in-flight/settled attempt so concurrent decodes
|
|
116
|
+
// share a single dynamic import, and a missing module is not re-probed on every call.
|
|
117
|
+
let lazyRegistration: Promise<boolean> | undefined;
|
|
118
|
+
|
|
119
|
+
// Why: the lazy decode path imports apache-arrow through the default Node loader, which
|
|
120
|
+
// is hard to simulate-as-absent in a test env where the dependency IS installed. This
|
|
121
|
+
// internal seam lets the unit suite exercise the "apache-arrow missing" clear-failure
|
|
122
|
+
// branch deterministically. It is intentionally NOT re-exported from the package root
|
|
123
|
+
// (see test/api_surface.test.ts) and is reset by clearArrowDecoder().
|
|
124
|
+
let lazyArrowLoaderOverride: ArrowModuleLoader | undefined;
|
|
125
|
+
|
|
126
|
+
/** @internal Test-only: override the loader used by lazy auto-registration. */
|
|
127
|
+
export const _setLazyArrowLoaderForTesting = (loader: ArrowModuleLoader | undefined): void => {
|
|
128
|
+
lazyArrowLoaderOverride = loader;
|
|
129
|
+
lazyRegistration = undefined;
|
|
130
|
+
};
|
|
131
|
+
|
|
114
132
|
export function registerArrowDecoder(
|
|
115
133
|
decoder: (bytes: Uint8Array) => ArrowTable | Uint8Array
|
|
116
134
|
): void {
|
|
@@ -119,6 +137,10 @@ export function registerArrowDecoder(
|
|
|
119
137
|
|
|
120
138
|
export function clearArrowDecoder(): void {
|
|
121
139
|
arrowTableFrom = undefined;
|
|
140
|
+
// Why: reset the cached import attempt so tests (and reload helpers) can exercise
|
|
141
|
+
// the auto-registration path again from a clean slate.
|
|
142
|
+
lazyRegistration = undefined;
|
|
143
|
+
lazyArrowLoaderOverride = undefined;
|
|
122
144
|
}
|
|
123
145
|
|
|
124
146
|
/**
|
|
@@ -186,6 +208,12 @@ export async function autoRegisterArrowDecoder(
|
|
|
186
208
|
}
|
|
187
209
|
try {
|
|
188
210
|
const arrowModule = await loader();
|
|
211
|
+
// Another path may have registered a decoder while the import was in flight
|
|
212
|
+
// (e.g. an explicit registerArrowDecoder() during concurrent startup/reload).
|
|
213
|
+
// Don't clobber it — the explicit registration wins.
|
|
214
|
+
if (hasArrowDecoder()) {
|
|
215
|
+
return true;
|
|
216
|
+
}
|
|
189
217
|
registerArrowDecoderFromModule(arrowModule as { tableFromIPC?: unknown });
|
|
190
218
|
return true;
|
|
191
219
|
} catch {
|
|
@@ -214,17 +242,46 @@ function fromBase64(b64: string): Uint8Array {
|
|
|
214
242
|
throw new Error('Base64 decoding is not available in this runtime');
|
|
215
243
|
}
|
|
216
244
|
|
|
245
|
+
// Why: a single, actionable message for both decode paths so users always know the two
|
|
246
|
+
// supported remedies — install the optional dependency, or opt into the lossy JSON fallback.
|
|
247
|
+
const ARROW_MISSING_MESSAGE =
|
|
248
|
+
'Received an Arrow-encoded payload but no Arrow decoder is available. ' +
|
|
249
|
+
'Install the optional dependency with `npm install apache-arrow`, or set ' +
|
|
250
|
+
'TYWRAP_CODEC_FALLBACK=json on the Python side to receive JSON instead ' +
|
|
251
|
+
'(lossy for dtype/NA fidelity). tywrap never silently downgrades Arrow payloads.';
|
|
252
|
+
|
|
217
253
|
function requireArrowDecoder(): (bytes: Uint8Array) => ArrowTable | Uint8Array {
|
|
218
254
|
if (!arrowTableFrom) {
|
|
219
|
-
throw new Error(
|
|
220
|
-
|
|
221
|
-
|
|
255
|
+
throw new Error(ARROW_MISSING_MESSAGE);
|
|
256
|
+
}
|
|
257
|
+
return arrowTableFrom;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Ensure an Arrow decoder is registered, lazily importing apache-arrow on first use.
|
|
262
|
+
*
|
|
263
|
+
* Why: keep apache-arrow optional and zero-config. The first Arrow-encoded payload
|
|
264
|
+
* triggers a single best-effort dynamic import; if it succeeds the decoder is cached
|
|
265
|
+
* for the rest of the process. If apache-arrow is absent we throw a clear, actionable
|
|
266
|
+
* error rather than silently producing wrong data.
|
|
267
|
+
*/
|
|
268
|
+
async function ensureArrowDecoder(): Promise<(bytes: Uint8Array) => ArrowTable | Uint8Array> {
|
|
269
|
+
if (arrowTableFrom) {
|
|
270
|
+
return arrowTableFrom;
|
|
271
|
+
}
|
|
272
|
+
// Reuse a single import attempt across concurrent decodes.
|
|
273
|
+
lazyRegistration ??= autoRegisterArrowDecoder(
|
|
274
|
+
lazyArrowLoaderOverride ? { loader: lazyArrowLoaderOverride } : {}
|
|
275
|
+
);
|
|
276
|
+
await lazyRegistration;
|
|
277
|
+
if (!arrowTableFrom) {
|
|
278
|
+
throw new Error(ARROW_MISSING_MESSAGE);
|
|
222
279
|
}
|
|
223
280
|
return arrowTableFrom;
|
|
224
281
|
}
|
|
225
282
|
|
|
226
283
|
async function tryDecodeArrowTable(bytes: Uint8Array): Promise<ArrowTable | Uint8Array> {
|
|
227
|
-
const decoder =
|
|
284
|
+
const decoder = await ensureArrowDecoder();
|
|
228
285
|
try {
|
|
229
286
|
return decoder(bytes);
|
|
230
287
|
} catch (err) {
|
package/src/version.ts
CHANGED