pynect 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +478 -2
- package/SECURITY.md +34 -0
- package/binding.gyp +60 -0
- package/dist/cli/codegen.d.ts +3 -0
- package/dist/cli/codegen.d.ts.map +1 -0
- package/dist/cli/codegen.js +543 -0
- package/dist/cli/codegen.js.map +1 -0
- package/dist/cli/sync.d.ts +3 -0
- package/dist/cli/sync.d.ts.map +1 -0
- package/dist/cli/sync.js +48 -0
- package/dist/cli/sync.js.map +1 -0
- package/dist/config/project.d.ts +57 -0
- package/dist/config/project.d.ts.map +1 -0
- package/dist/config/project.js +176 -0
- package/dist/config/project.js.map +1 -0
- package/dist/core/IsolatedWorker.d.ts +39 -0
- package/dist/core/IsolatedWorker.d.ts.map +1 -0
- package/dist/core/IsolatedWorker.js +178 -0
- package/dist/core/IsolatedWorker.js.map +1 -0
- package/dist/core/PyCallable.d.ts +67 -0
- package/dist/core/PyCallable.d.ts.map +1 -0
- package/dist/core/PyCallable.js +104 -0
- package/dist/core/PyCallable.js.map +1 -0
- package/dist/core/PyContext.d.ts +76 -0
- package/dist/core/PyContext.d.ts.map +1 -0
- package/dist/core/PyContext.js +239 -0
- package/dist/core/PyContext.js.map +1 -0
- package/dist/core/PyIterator.d.ts +84 -0
- package/dist/core/PyIterator.d.ts.map +1 -0
- package/dist/core/PyIterator.js +243 -0
- package/dist/core/PyIterator.js.map +1 -0
- package/dist/core/PyModule.d.ts +53 -0
- package/dist/core/PyModule.d.ts.map +1 -0
- package/dist/core/PyModule.js +162 -0
- package/dist/core/PyModule.js.map +1 -0
- package/dist/core/PyProxy.d.ts +75 -0
- package/dist/core/PyProxy.d.ts.map +1 -0
- package/dist/core/PyProxy.js +563 -0
- package/dist/core/PyProxy.js.map +1 -0
- package/dist/core/PyRuntime.d.ts +112 -0
- package/dist/core/PyRuntime.d.ts.map +1 -0
- package/dist/core/PyRuntime.js +546 -0
- package/dist/core/PyRuntime.js.map +1 -0
- package/dist/env/PackageInstaller.d.ts +232 -0
- package/dist/env/PackageInstaller.d.ts.map +1 -0
- package/dist/env/PackageInstaller.js +628 -0
- package/dist/env/PackageInstaller.js.map +1 -0
- package/dist/env/ProjectEnvironment.d.ts +63 -0
- package/dist/env/ProjectEnvironment.d.ts.map +1 -0
- package/dist/env/ProjectEnvironment.js +379 -0
- package/dist/env/ProjectEnvironment.js.map +1 -0
- package/dist/env/PyfigAdapter.d.ts +32 -0
- package/dist/env/PyfigAdapter.d.ts.map +1 -0
- package/dist/env/PyfigAdapter.js +10 -0
- package/dist/env/PyfigAdapter.js.map +1 -0
- package/dist/env/PythonDetector.d.ts +48 -0
- package/dist/env/PythonDetector.d.ts.map +1 -0
- package/dist/env/PythonDetector.js +182 -0
- package/dist/env/PythonDetector.js.map +1 -0
- package/dist/env/PythonProvisioner.d.ts +16 -0
- package/dist/env/PythonProvisioner.d.ts.map +1 -0
- package/dist/env/PythonProvisioner.js +42 -0
- package/dist/env/PythonProvisioner.js.map +1 -0
- package/dist/env/VenvManager.d.ts +55 -0
- package/dist/env/VenvManager.d.ts.map +1 -0
- package/dist/env/VenvManager.js +153 -0
- package/dist/env/VenvManager.js.map +1 -0
- package/dist/env/executables.d.ts +3 -0
- package/dist/env/executables.d.ts.map +1 -0
- package/dist/env/executables.js +14 -0
- package/dist/env/executables.js.map +1 -0
- package/dist/index.d.ts +182 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +393 -0
- package/dist/index.js.map +1 -0
- package/dist/native/NativeAddonLoader.d.ts +11 -0
- package/dist/native/NativeAddonLoader.d.ts.map +1 -0
- package/dist/native/NativeAddonLoader.js +75 -0
- package/dist/native/NativeAddonLoader.js.map +1 -0
- package/dist/plugins/Plugin.interface.d.ts +41 -0
- package/dist/plugins/Plugin.interface.d.ts.map +1 -0
- package/dist/plugins/Plugin.interface.js +12 -0
- package/dist/plugins/Plugin.interface.js.map +1 -0
- package/dist/plugins/PluginManager.d.ts +26 -0
- package/dist/plugins/PluginManager.d.ts.map +1 -0
- package/dist/plugins/PluginManager.js +174 -0
- package/dist/plugins/PluginManager.js.map +1 -0
- package/dist/plugins/builtin/NumpyPlugin.d.ts +17 -0
- package/dist/plugins/builtin/NumpyPlugin.d.ts.map +1 -0
- package/dist/plugins/builtin/NumpyPlugin.js +41 -0
- package/dist/plugins/builtin/NumpyPlugin.js.map +1 -0
- package/dist/plugins/builtin/PandasPlugin.d.ts +19 -0
- package/dist/plugins/builtin/PandasPlugin.d.ts.map +1 -0
- package/dist/plugins/builtin/PandasPlugin.js +57 -0
- package/dist/plugins/builtin/PandasPlugin.js.map +1 -0
- package/dist/plugins/builtin/TorchPlugin.d.ts +23 -0
- package/dist/plugins/builtin/TorchPlugin.d.ts.map +1 -0
- package/dist/plugins/builtin/TorchPlugin.js +50 -0
- package/dist/plugins/builtin/TorchPlugin.js.map +1 -0
- package/dist/plugins/index.d.ts +7 -0
- package/dist/plugins/index.d.ts.map +1 -0
- package/dist/plugins/index.js +12 -0
- package/dist/plugins/index.js.map +1 -0
- package/dist/serialization/DataFrameBridge.d.ts +141 -0
- package/dist/serialization/DataFrameBridge.d.ts.map +1 -0
- package/dist/serialization/DataFrameBridge.js +355 -0
- package/dist/serialization/DataFrameBridge.js.map +1 -0
- package/dist/serialization/MsgPackSerializer.d.ts +45 -0
- package/dist/serialization/MsgPackSerializer.d.ts.map +1 -0
- package/dist/serialization/MsgPackSerializer.js +242 -0
- package/dist/serialization/MsgPackSerializer.js.map +1 -0
- package/dist/serialization/NumpyBridge.d.ts +96 -0
- package/dist/serialization/NumpyBridge.d.ts.map +1 -0
- package/dist/serialization/NumpyBridge.js +323 -0
- package/dist/serialization/NumpyBridge.js.map +1 -0
- package/dist/serialization/Serializer.d.ts +78 -0
- package/dist/serialization/Serializer.d.ts.map +1 -0
- package/dist/serialization/Serializer.js +281 -0
- package/dist/serialization/Serializer.js.map +1 -0
- package/dist/types/addon.d.ts +131 -0
- package/dist/types/addon.d.ts.map +1 -0
- package/dist/types/addon.js +32 -0
- package/dist/types/addon.js.map +1 -0
- package/dist/types/config.d.ts +233 -0
- package/dist/types/config.d.ts.map +1 -0
- package/dist/types/config.js +60 -0
- package/dist/types/config.js.map +1 -0
- package/dist/types/index.d.ts +10 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +12 -0
- package/dist/types/index.js.map +1 -0
- package/dist/types/python.d.ts +235 -0
- package/dist/types/python.d.ts.map +1 -0
- package/dist/types/python.js +7 -0
- package/dist/types/python.js.map +1 -0
- package/dist/utils/ErrorTranslator.d.ts +83 -0
- package/dist/utils/ErrorTranslator.d.ts.map +1 -0
- package/dist/utils/ErrorTranslator.js +210 -0
- package/dist/utils/ErrorTranslator.js.map +1 -0
- package/dist/utils/Locking.d.ts +3 -0
- package/dist/utils/Locking.d.ts.map +1 -0
- package/dist/utils/Locking.js +10 -0
- package/dist/utils/Locking.js.map +1 -0
- package/dist/utils/Logger.d.ts +27 -0
- package/dist/utils/Logger.d.ts.map +1 -0
- package/dist/utils/Logger.js +115 -0
- package/dist/utils/Logger.js.map +1 -0
- package/dist/utils/MemoryMonitor.d.ts +43 -0
- package/dist/utils/MemoryMonitor.d.ts.map +1 -0
- package/dist/utils/MemoryMonitor.js +143 -0
- package/dist/utils/MemoryMonitor.js.map +1 -0
- package/package.json +114 -4
- package/python/ast_inspector.py +248 -0
- package/python/error_handler.py +432 -0
- package/python/isolated_worker.py +94 -0
- package/python/pynect_dispatcher.py +382 -0
- package/python/pynect_runtime.py +574 -0
- package/python/serializer.py +373 -0
- package/python/type_inspector.py +315 -0
- package/scripts/build-native.js +50 -0
- package/scripts/cli.js +21 -0
- package/scripts/ensure-native.js +85 -0
- package/scripts/postinstall.js +40 -0
- package/scripts/pyfig.js +113 -0
- package/scripts/python-provisioner.js +23 -0
- package/scripts/smoke-python-boundary.js +103 -0
- package/src/native/NativeAddonLoader.ts +98 -0
- package/src/native/gil_guard.cpp +25 -0
- package/src/native/gil_guard.h +174 -0
- package/src/native/pynect_addon.cpp +942 -0
- package/src/native/python_bridge.cpp +904 -0
- package/src/native/python_bridge.h +263 -0
- package/src/native/thread_pool.cpp +375 -0
- package/src/native/thread_pool.h +185 -0
- package/src/native/type_converter.cpp +906 -0
- package/src/native/type_converter.h +271 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* pynect - Memory Monitor
|
|
4
|
+
* Tracks Python object references, heap usage, and triggers GC when needed.
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.MemoryMonitor = void 0;
|
|
8
|
+
const Logger_1 = require("./Logger");
|
|
9
|
+
const Locking_1 = require("./Locking");
|
|
10
|
+
class MemoryMonitor {
|
|
11
|
+
_logger;
|
|
12
|
+
_addon = null;
|
|
13
|
+
_snapshots = [];
|
|
14
|
+
_maxSnapshots = 100;
|
|
15
|
+
_monitorInterval = null;
|
|
16
|
+
_thresholds;
|
|
17
|
+
constructor(thresholds) {
|
|
18
|
+
this._logger = new Logger_1.Logger('MemoryMonitor');
|
|
19
|
+
this._thresholds = {
|
|
20
|
+
gcThreshold: this._parseBytes('1GB'),
|
|
21
|
+
objectWarningCount: 10000,
|
|
22
|
+
callbackWarningCount: 500,
|
|
23
|
+
...thresholds,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
attach(addon) {
|
|
27
|
+
this._addon = addon;
|
|
28
|
+
}
|
|
29
|
+
startMonitoring(intervalMs = 5000) {
|
|
30
|
+
if (this._monitorInterval) {
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
this._monitorInterval = setInterval(() => {
|
|
34
|
+
this._checkMemory().catch(err => {
|
|
35
|
+
this._logger.error('Memory check failed', err);
|
|
36
|
+
});
|
|
37
|
+
}, intervalMs);
|
|
38
|
+
// Don't keep event loop alive
|
|
39
|
+
if (this._monitorInterval.unref) {
|
|
40
|
+
this._monitorInterval.unref();
|
|
41
|
+
}
|
|
42
|
+
this._logger.debug(`Memory monitoring started (interval: ${intervalMs}ms)`);
|
|
43
|
+
}
|
|
44
|
+
stopMonitoring() {
|
|
45
|
+
if (this._monitorInterval) {
|
|
46
|
+
clearInterval(this._monitorInterval);
|
|
47
|
+
this._monitorInterval = null;
|
|
48
|
+
this._logger.debug('Memory monitoring stopped');
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
async takeSnapshot() {
|
|
52
|
+
const nodeHeap = process.memoryUsage();
|
|
53
|
+
let pythonStats = { pythonHeapBytes: 0, pythonHeapPeakBytes: 0, trackedObjects: 0, pendingCallbacks: 0 };
|
|
54
|
+
if (this._addon) {
|
|
55
|
+
const stats = this._addon.getMemoryStats();
|
|
56
|
+
pythonStats = {
|
|
57
|
+
pythonHeapBytes: stats.pythonHeapBytes,
|
|
58
|
+
pythonHeapPeakBytes: stats.pythonHeapPeakBytes,
|
|
59
|
+
trackedObjects: stats.trackedObjects,
|
|
60
|
+
pendingCallbacks: stats.pendingCallbacks,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
const snapshot = {
|
|
64
|
+
timestamp: Date.now(),
|
|
65
|
+
...pythonStats,
|
|
66
|
+
nodeHeapUsed: nodeHeap.heapUsed,
|
|
67
|
+
nodeHeapTotal: nodeHeap.heapTotal,
|
|
68
|
+
nodeExternal: nodeHeap.external,
|
|
69
|
+
};
|
|
70
|
+
this._snapshots.push(snapshot);
|
|
71
|
+
if (this._snapshots.length > this._maxSnapshots) {
|
|
72
|
+
this._snapshots.shift();
|
|
73
|
+
}
|
|
74
|
+
return snapshot;
|
|
75
|
+
}
|
|
76
|
+
getSnapshots() {
|
|
77
|
+
return [...this._snapshots];
|
|
78
|
+
}
|
|
79
|
+
getLatestSnapshot() {
|
|
80
|
+
return this._snapshots[this._snapshots.length - 1] ?? null;
|
|
81
|
+
}
|
|
82
|
+
async forceGC() {
|
|
83
|
+
await (0, Locking_1.withPynectLock)({ operation: 'python-gc' }, async () => {
|
|
84
|
+
if (!this._addon)
|
|
85
|
+
return;
|
|
86
|
+
this._logger.debug('Running Python garbage collection...');
|
|
87
|
+
await this._addon.collectGarbage();
|
|
88
|
+
this._logger.debug('Python GC completed');
|
|
89
|
+
}, async () => this._addon !== null, {
|
|
90
|
+
key: 'pynect-python-gc',
|
|
91
|
+
name: 'Pynect Python garbage collection',
|
|
92
|
+
namespace: 'pynect',
|
|
93
|
+
scope: 'memory',
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
async _checkMemory() {
|
|
97
|
+
const snapshot = await this.takeSnapshot();
|
|
98
|
+
// Check Python heap threshold
|
|
99
|
+
if (snapshot.pythonHeapBytes > this._thresholds.gcThreshold) {
|
|
100
|
+
this._logger.warn(`Python heap (${this._formatBytes(snapshot.pythonHeapBytes)}) exceeded GC threshold ` +
|
|
101
|
+
`(${this._formatBytes(this._thresholds.gcThreshold)}). Running GC...`);
|
|
102
|
+
await this.forceGC();
|
|
103
|
+
}
|
|
104
|
+
// Check tracked objects
|
|
105
|
+
if (snapshot.trackedObjects > this._thresholds.objectWarningCount) {
|
|
106
|
+
this._logger.warn(`High Python object count: ${snapshot.trackedObjects} tracked objects`);
|
|
107
|
+
}
|
|
108
|
+
// Check pending callbacks
|
|
109
|
+
if (snapshot.pendingCallbacks > this._thresholds.callbackWarningCount) {
|
|
110
|
+
this._logger.warn(`High pending callback count: ${snapshot.pendingCallbacks}`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
_parseBytes(str) {
|
|
114
|
+
const match = str.match(/^(\d+(?:\.\d+)?)\s*(B|KB|MB|GB|TB)?$/i);
|
|
115
|
+
if (!match) {
|
|
116
|
+
return 1_073_741_824;
|
|
117
|
+
} // 1GB default
|
|
118
|
+
const value = parseFloat(match[1]);
|
|
119
|
+
const unit = (match[2] ?? 'B').toUpperCase();
|
|
120
|
+
const units = {
|
|
121
|
+
B: 1,
|
|
122
|
+
KB: 1024,
|
|
123
|
+
MB: 1024 ** 2,
|
|
124
|
+
GB: 1024 ** 3,
|
|
125
|
+
TB: 1024 ** 4,
|
|
126
|
+
};
|
|
127
|
+
return Math.floor(value * (units[unit] ?? 1));
|
|
128
|
+
}
|
|
129
|
+
_formatBytes(bytes) {
|
|
130
|
+
if (bytes < 1024) {
|
|
131
|
+
return `${bytes}B`;
|
|
132
|
+
}
|
|
133
|
+
if (bytes < 1024 ** 2) {
|
|
134
|
+
return `${(bytes / 1024).toFixed(1)}KB`;
|
|
135
|
+
}
|
|
136
|
+
if (bytes < 1024 ** 3) {
|
|
137
|
+
return `${(bytes / 1024 ** 2).toFixed(1)}MB`;
|
|
138
|
+
}
|
|
139
|
+
return `${(bytes / 1024 ** 3).toFixed(2)}GB`;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
exports.MemoryMonitor = MemoryMonitor;
|
|
143
|
+
//# sourceMappingURL=MemoryMonitor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MemoryMonitor.js","sourceRoot":"","sources":["../../src/utils/MemoryMonitor.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,qCAAkC;AAElC,uCAA2C;AAsB3C,MAAa,aAAa;IACP,OAAO,CAAS;IACzB,MAAM,GAAuB,IAAI,CAAC;IAClC,UAAU,GAAqB,EAAE,CAAC;IAClC,aAAa,GAAG,GAAG,CAAC;IACpB,gBAAgB,GAA0B,IAAI,CAAC;IAC/C,WAAW,CAAmB;IAEtC,YAAY,UAAsC;QAChD,IAAI,CAAC,OAAO,GAAG,IAAI,eAAM,CAAC,eAAe,CAAC,CAAC;QAC3C,IAAI,CAAC,WAAW,GAAG;YACjB,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;YACpC,kBAAkB,EAAE,KAAK;YACzB,oBAAoB,EAAE,GAAG;YACzB,GAAG,UAAU;SACd,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,KAAkB;QACvB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,eAAe,CAAC,UAAU,GAAG,IAAI;QAC/B,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAAA,OAAO;QAAA,CAAC;QACpC,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC,GAAG,EAAE;YACvC,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;gBAC9B,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAC;YACjD,CAAC,CAAC,CAAC;QACL,CAAC,EAAE,UAAU,CAAC,CAAC;QAEf,8BAA8B;QAC9B,IAAI,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,wCAAwC,UAAU,KAAK,CAAC,CAAC;IAC9E,CAAC;IAED,cAAc;QACZ,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,aAAa,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACrC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QACvC,IAAI,WAAW,GAAG,EAAE,eAAe,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;QAEzG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YAC3C,WAAW,GAAG;gBACZ,eAAe,EAAE,KAAK,CAAC,eAAe;gBACtC,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;gBAC9C,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;aACzC,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAmB;YAC/B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,GAAG,WAAW;YACd,YAAY,EAAE,QAAQ,CAAC,QAAQ;YAC/B,aAAa,EAAE,QAAQ,CAAC,SAAS;YACjC,YAAY,EAAE,QAAQ,CAAC,QAAQ;SAChC,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/B,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAChD,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAC1B,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,YAAY;QACV,OAAO,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IAC9B,CAAC;IAED,iBAAiB;QACf,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,IAAA,wBAAc,EAClB,EAAE,SAAS,EAAE,WAAW,EAAE,EAC1B,KAAK,IAAI,EAAE;YACT,IAAI,CAAC,IAAI,CAAC,MAAM;gBAAE,OAAO;YACzB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAC3D,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAC5C,CAAC,EACD,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,EAChC;YACE,GAAG,EAAE,kBAAkB;YACvB,IAAI,EAAE,kCAAkC;YACxC,SAAS,EAAE,QAAQ;YACnB,KAAK,EAAE,QAAQ;SAChB,CACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE3C,8BAA8B;QAC9B,IAAI,QAAQ,CAAC,eAAe,GAAG,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;YAC5D,IAAI,CAAC,OAAO,CAAC,IAAI,CACf,gBAAgB,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,eAAe,CAAC,0BAA0B;gBACrF,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,kBAAkB,CACtE,CAAC;YACF,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACvB,CAAC;QAED,wBAAwB;QACxB,IAAI,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,kBAAkB,EAAE,CAAC;YAClE,IAAI,CAAC,OAAO,CAAC,IAAI,CACf,6BAA6B,QAAQ,CAAC,cAAc,kBAAkB,CACvE,CAAC;QACJ,CAAC;QAED,0BAA0B;QAC1B,IAAI,QAAQ,CAAC,gBAAgB,GAAG,IAAI,CAAC,WAAW,CAAC,oBAAoB,EAAE,CAAC;YACtE,IAAI,CAAC,OAAO,CAAC,IAAI,CACf,gCAAgC,QAAQ,CAAC,gBAAgB,EAAE,CAC5D,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,GAAW;QAC7B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;QACjE,IAAI,CAAC,KAAK,EAAE,CAAC;YAAA,OAAO,aAAa,CAAC;QAAA,CAAC,CAAC,cAAc;QAClD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;QACpC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QAC7C,MAAM,KAAK,GAA2B;YACpC,CAAC,EAAE,CAAC;YACJ,EAAE,EAAE,IAAI;YACR,EAAE,EAAE,IAAI,IAAI,CAAC;YACb,EAAE,EAAE,IAAI,IAAI,CAAC;YACb,EAAE,EAAE,IAAI,IAAI,CAAC;SACd,CAAC;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC;IAEO,YAAY,CAAC,KAAa;QAChC,IAAI,KAAK,GAAG,IAAI,EAAE,CAAC;YAAA,OAAO,GAAG,KAAK,GAAG,CAAC;QAAA,CAAC;QACvC,IAAI,KAAK,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC;YAAA,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QAAA,CAAC;QACjE,IAAI,KAAK,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC;YAAA,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QAAA,CAAC;QACtE,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/C,CAAC;CACF;AAvJD,sCAuJC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,116 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pynect",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"description": "
|
|
6
|
-
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"packageManager": "bun@1.4.4",
|
|
5
|
+
"description": "Typed, optionally Pydantic-validated Python calls from Node.js and Bun through embedded CPython.",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"import": "./dist/index.js",
|
|
12
|
+
"require": "./dist/index.js"
|
|
13
|
+
},
|
|
14
|
+
"./types": {
|
|
15
|
+
"types": "./dist/types/index.d.ts",
|
|
16
|
+
"import": "./dist/types/index.js",
|
|
17
|
+
"require": "./dist/types/index.js"
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
"bin": {
|
|
21
|
+
"pynect": "scripts/cli.js"
|
|
22
|
+
},
|
|
23
|
+
"files": [
|
|
24
|
+
"dist/",
|
|
25
|
+
"src/native/",
|
|
26
|
+
"python/*.py",
|
|
27
|
+
"scripts/build-native.js",
|
|
28
|
+
"scripts/cli.js",
|
|
29
|
+
"scripts/ensure-native.js",
|
|
30
|
+
"scripts/postinstall.js",
|
|
31
|
+
"scripts/pyfig.js",
|
|
32
|
+
"scripts/python-provisioner.js",
|
|
33
|
+
"scripts/smoke-python-boundary.js",
|
|
34
|
+
"binding.gyp",
|
|
35
|
+
"README.md",
|
|
36
|
+
"SECURITY.md",
|
|
37
|
+
"LICENSE"
|
|
38
|
+
],
|
|
39
|
+
"scripts": {
|
|
40
|
+
"build": "bun run build:ts && bun run build:native",
|
|
41
|
+
"build:ts": "bun scripts/build-typescript.js",
|
|
42
|
+
"build:native": "bun scripts/build-native.js",
|
|
43
|
+
"postinstall": "node scripts/postinstall.js",
|
|
44
|
+
"release": "bun scripts/release.ts",
|
|
45
|
+
"test": "bun test",
|
|
46
|
+
"test:python-boundary": "bun scripts/smoke-python-boundary.js",
|
|
47
|
+
"codegen": "bun run build:ts && bun scripts/cli.js codegen",
|
|
48
|
+
"typecheck": "tsc --noEmit -p tsconfig.json",
|
|
49
|
+
"typecheck:release": "tsc --noEmit -p tsconfig.release.json",
|
|
50
|
+
"verify": "bun run typecheck && bun run typecheck:release && bun test && bun run build && bun run test:python-boundary",
|
|
51
|
+
"prepack": "bun scripts/prepack.ts"
|
|
52
|
+
},
|
|
53
|
+
"keywords": [
|
|
54
|
+
"python",
|
|
55
|
+
"nodejs",
|
|
56
|
+
"bun",
|
|
57
|
+
"typescript",
|
|
58
|
+
"pydantic",
|
|
59
|
+
"cpython",
|
|
60
|
+
"native-addon",
|
|
61
|
+
"n-api"
|
|
62
|
+
],
|
|
63
|
+
"author": "Reggie Bowser and NodePyX contributors",
|
|
64
|
+
"license": "MIT",
|
|
65
|
+
"repository": {
|
|
66
|
+
"type": "git",
|
|
67
|
+
"url": "git+https://github.com/regbo/pynect.git"
|
|
68
|
+
},
|
|
69
|
+
"bugs": {
|
|
70
|
+
"url": "https://github.com/regbo/pynect/issues"
|
|
71
|
+
},
|
|
72
|
+
"homepage": "https://github.com/regbo/pynect#readme",
|
|
73
|
+
"upstream": "https://github.com/Brah-Timo/nodepyx",
|
|
74
|
+
"publishConfig": {
|
|
75
|
+
"access": "public",
|
|
76
|
+
"provenance": true,
|
|
77
|
+
"registry": "https://registry.npmjs.org"
|
|
78
|
+
},
|
|
79
|
+
"dependencies": {
|
|
80
|
+
"@msgpack/msgpack": "^3.1.2",
|
|
81
|
+
"commander": "^14.0.2",
|
|
82
|
+
"jsonc-parser": "^3.3.1",
|
|
83
|
+
"loqkit": "^0.0.3",
|
|
84
|
+
"node-addon-api": "^8.5.0",
|
|
85
|
+
"node-gyp": "^11.5.0",
|
|
86
|
+
"node-gyp-build": "^4.8.4",
|
|
87
|
+
"pacwich": "^0.7.3",
|
|
88
|
+
"pyfig": "^0.3.1",
|
|
89
|
+
"semver": "^7.8.5"
|
|
90
|
+
},
|
|
91
|
+
"devDependencies": {
|
|
92
|
+
"@types/bun": "^1.3.1",
|
|
93
|
+
"@types/node": "^24.10.0",
|
|
94
|
+
"@types/semver": "^7.7.1",
|
|
95
|
+
"tsx": "^4.20.6",
|
|
96
|
+
"typescript": "^5.9.3"
|
|
97
|
+
},
|
|
98
|
+
"engines": {
|
|
99
|
+
"node": ">=22.0.0",
|
|
100
|
+
"bun": ">=1.2.0"
|
|
101
|
+
},
|
|
102
|
+
"os": [
|
|
103
|
+
"linux",
|
|
104
|
+
"darwin",
|
|
105
|
+
"win32"
|
|
106
|
+
],
|
|
107
|
+
"cpu": [
|
|
108
|
+
"x64",
|
|
109
|
+
"arm64"
|
|
110
|
+
],
|
|
111
|
+
"binary": {
|
|
112
|
+
"module_name": "pynect_addon",
|
|
113
|
+
"module_path": "./prebuilds/{platform}-{arch}"
|
|
114
|
+
},
|
|
115
|
+
"gypfile": false
|
|
116
|
+
}
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
"""Static Python module inspection for safe TypeScript declaration generation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _annotation(node: ast.expr | None) -> str:
|
|
12
|
+
if node is None:
|
|
13
|
+
return "Any"
|
|
14
|
+
try:
|
|
15
|
+
return ast.unparse(node).replace("typing.", "")
|
|
16
|
+
except Exception:
|
|
17
|
+
return "Any"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _decorator_name(node: ast.expr) -> str:
|
|
21
|
+
if isinstance(node, ast.Call):
|
|
22
|
+
return _decorator_name(node.func)
|
|
23
|
+
if isinstance(node, ast.Attribute):
|
|
24
|
+
return f"{_decorator_name(node.value)}.{node.attr}".strip(".")
|
|
25
|
+
if isinstance(node, ast.Name):
|
|
26
|
+
return node.id
|
|
27
|
+
return ""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _decorators(node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef) -> set[str]:
|
|
31
|
+
return {_decorator_name(item) for item in node.decorator_list}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _has_decorator(names: set[str], leaf: str) -> bool:
|
|
35
|
+
return any(name == leaf or name.endswith(f".{leaf}") for name in names)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _default_text(node: ast.expr | None) -> str | None:
|
|
39
|
+
if node is None:
|
|
40
|
+
return None
|
|
41
|
+
try:
|
|
42
|
+
return ast.unparse(node)
|
|
43
|
+
except Exception:
|
|
44
|
+
return None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _parameters(node: ast.FunctionDef | ast.AsyncFunctionDef) -> list[dict[str, Any]]:
|
|
48
|
+
arguments = node.args
|
|
49
|
+
positional = [*arguments.posonlyargs, *arguments.args]
|
|
50
|
+
positional_defaults: list[ast.expr | None] = [None] * (
|
|
51
|
+
len(positional) - len(arguments.defaults)
|
|
52
|
+
) + list(arguments.defaults)
|
|
53
|
+
result: list[dict[str, Any]] = []
|
|
54
|
+
|
|
55
|
+
def add(argument: ast.arg, kind: str, default: ast.expr | None = None) -> None:
|
|
56
|
+
has_default = default is not None
|
|
57
|
+
result.append(
|
|
58
|
+
{
|
|
59
|
+
"name": argument.arg,
|
|
60
|
+
"type": _annotation(argument.annotation),
|
|
61
|
+
"annotation": _annotation(argument.annotation),
|
|
62
|
+
"optional": has_default or kind in {"var_positional", "var_keyword"},
|
|
63
|
+
"has_default": has_default,
|
|
64
|
+
"default": _default_text(default),
|
|
65
|
+
"kind": kind,
|
|
66
|
+
}
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
positional_only = len(arguments.posonlyargs)
|
|
70
|
+
for index, (argument, default) in enumerate(zip(positional, positional_defaults)):
|
|
71
|
+
kind = "positional_only" if index < positional_only else "positional_or_keyword"
|
|
72
|
+
add(argument, kind, default)
|
|
73
|
+
if arguments.vararg is not None:
|
|
74
|
+
add(arguments.vararg, "var_positional")
|
|
75
|
+
for argument, default in zip(arguments.kwonlyargs, arguments.kw_defaults):
|
|
76
|
+
add(argument, "keyword_only", default)
|
|
77
|
+
if arguments.kwarg is not None:
|
|
78
|
+
add(arguments.kwarg, "var_keyword")
|
|
79
|
+
return result
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _callable(
|
|
83
|
+
node: ast.FunctionDef | ast.AsyncFunctionDef,
|
|
84
|
+
*,
|
|
85
|
+
qualified_name: str | None = None,
|
|
86
|
+
) -> dict[str, Any]:
|
|
87
|
+
decorators = _decorators(node)
|
|
88
|
+
parameters = _parameters(node)
|
|
89
|
+
return {
|
|
90
|
+
"name": node.name,
|
|
91
|
+
"qual_name": qualified_name or node.name,
|
|
92
|
+
"docstring": ast.get_docstring(node) or "",
|
|
93
|
+
"parameters": parameters,
|
|
94
|
+
"return_type": _annotation(node.returns),
|
|
95
|
+
"is_method": qualified_name is not None,
|
|
96
|
+
"is_class_method": _has_decorator(decorators, "classmethod"),
|
|
97
|
+
"is_static_method": _has_decorator(decorators, "staticmethod"),
|
|
98
|
+
"is_coroutine": isinstance(node, ast.AsyncFunctionDef),
|
|
99
|
+
"is_generator": any(isinstance(item, (ast.Yield, ast.YieldFrom)) for item in ast.walk(node)),
|
|
100
|
+
"exported": _has_decorator(decorators, "export"),
|
|
101
|
+
"excluded": _has_decorator(decorators, "exclude"),
|
|
102
|
+
"fully_typed": node.returns is not None
|
|
103
|
+
and all(
|
|
104
|
+
parameter["type"] != "Any"
|
|
105
|
+
for parameter in parameters
|
|
106
|
+
if parameter["name"] not in {"self", "cls"}
|
|
107
|
+
),
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _class(node: ast.ClassDef) -> dict[str, Any]:
|
|
112
|
+
decorators = _decorators(node)
|
|
113
|
+
result: dict[str, Any] = {
|
|
114
|
+
"name": node.name,
|
|
115
|
+
"qual_name": node.name,
|
|
116
|
+
"docstring": ast.get_docstring(node) or "",
|
|
117
|
+
"bases": [_annotation(base) for base in node.bases],
|
|
118
|
+
"members": [],
|
|
119
|
+
"methods": [],
|
|
120
|
+
"properties": [],
|
|
121
|
+
"class_methods": [],
|
|
122
|
+
"static_methods": [],
|
|
123
|
+
"exported": _has_decorator(decorators, "export"),
|
|
124
|
+
"excluded": _has_decorator(decorators, "exclude"),
|
|
125
|
+
"fully_typed": False,
|
|
126
|
+
}
|
|
127
|
+
property_setters: set[str] = set()
|
|
128
|
+
for item in node.body:
|
|
129
|
+
if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name):
|
|
130
|
+
result["members"].append(
|
|
131
|
+
{
|
|
132
|
+
"name": item.target.id,
|
|
133
|
+
"type": _annotation(item.annotation),
|
|
134
|
+
"read_only": False,
|
|
135
|
+
"docstring": "",
|
|
136
|
+
"fully_typed": True,
|
|
137
|
+
"exported": False,
|
|
138
|
+
"excluded": False,
|
|
139
|
+
}
|
|
140
|
+
)
|
|
141
|
+
continue
|
|
142
|
+
if not isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
143
|
+
continue
|
|
144
|
+
names = _decorators(item)
|
|
145
|
+
qualified = f"{node.name}.{item.name}"
|
|
146
|
+
inspected = _callable(item, qualified_name=qualified)
|
|
147
|
+
if any(name.endswith(".setter") for name in names):
|
|
148
|
+
property_setters.add(item.name)
|
|
149
|
+
elif _has_decorator(names, "property"):
|
|
150
|
+
result["properties"].append(
|
|
151
|
+
{
|
|
152
|
+
"name": item.name,
|
|
153
|
+
"type": _annotation(item.returns),
|
|
154
|
+
"read_only": True,
|
|
155
|
+
"docstring": ast.get_docstring(item) or "",
|
|
156
|
+
"fully_typed": item.returns is not None,
|
|
157
|
+
"exported": inspected["exported"],
|
|
158
|
+
"excluded": inspected["excluded"],
|
|
159
|
+
}
|
|
160
|
+
)
|
|
161
|
+
elif inspected["is_static_method"]:
|
|
162
|
+
result["static_methods"].append(inspected)
|
|
163
|
+
elif inspected["is_class_method"]:
|
|
164
|
+
result["class_methods"].append(inspected)
|
|
165
|
+
else:
|
|
166
|
+
result["methods"].append(inspected)
|
|
167
|
+
for prop in result["properties"]:
|
|
168
|
+
if prop["name"] in property_setters:
|
|
169
|
+
prop["read_only"] = False
|
|
170
|
+
result["fully_typed"] = bool(result["members"]) or any(
|
|
171
|
+
method["fully_typed"]
|
|
172
|
+
for key in ("methods", "class_methods", "static_methods")
|
|
173
|
+
for method in result[key]
|
|
174
|
+
)
|
|
175
|
+
return result
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _literal_type(node: ast.expr | None) -> str:
|
|
179
|
+
if isinstance(node, ast.Constant):
|
|
180
|
+
value = node.value
|
|
181
|
+
if value is None:
|
|
182
|
+
return "None"
|
|
183
|
+
return type(value).__name__
|
|
184
|
+
if isinstance(node, (ast.List, ast.ListComp)):
|
|
185
|
+
return "list[Any]"
|
|
186
|
+
if isinstance(node, (ast.Dict, ast.DictComp)):
|
|
187
|
+
return "dict[str, Any]"
|
|
188
|
+
if isinstance(node, (ast.Tuple, ast.GeneratorExp)):
|
|
189
|
+
return "tuple[Any]"
|
|
190
|
+
if isinstance(node, (ast.Set, ast.SetComp)):
|
|
191
|
+
return "set[Any]"
|
|
192
|
+
return "Any"
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def inspect_file(file_path: str, module_name: str) -> dict[str, Any]:
|
|
196
|
+
path = Path(file_path).resolve()
|
|
197
|
+
source = path.read_text(encoding="utf-8-sig")
|
|
198
|
+
tree = ast.parse(source, filename=str(path), type_comments=True)
|
|
199
|
+
result: dict[str, Any] = {
|
|
200
|
+
"name": module_name,
|
|
201
|
+
"file": str(path),
|
|
202
|
+
"docstring": ast.get_docstring(tree) or "",
|
|
203
|
+
"version": None,
|
|
204
|
+
"functions": [],
|
|
205
|
+
"classes": [],
|
|
206
|
+
"constants": [],
|
|
207
|
+
"submodules": [],
|
|
208
|
+
"inspection": "ast",
|
|
209
|
+
}
|
|
210
|
+
for node in tree.body:
|
|
211
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
212
|
+
result["functions"].append(_callable(node))
|
|
213
|
+
elif isinstance(node, ast.ClassDef):
|
|
214
|
+
result["classes"].append(_class(node))
|
|
215
|
+
elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
|
|
216
|
+
result["constants"].append(
|
|
217
|
+
{
|
|
218
|
+
"name": node.target.id,
|
|
219
|
+
"type": _annotation(node.annotation),
|
|
220
|
+
"read_only": True,
|
|
221
|
+
"docstring": "",
|
|
222
|
+
"fully_typed": True,
|
|
223
|
+
"exported": False,
|
|
224
|
+
"excluded": False,
|
|
225
|
+
}
|
|
226
|
+
)
|
|
227
|
+
elif isinstance(node, ast.Assign):
|
|
228
|
+
for target in node.targets:
|
|
229
|
+
if not isinstance(target, ast.Name):
|
|
230
|
+
continue
|
|
231
|
+
if target.id == "__version__" and isinstance(node.value, ast.Constant):
|
|
232
|
+
result["version"] = str(node.value.value)
|
|
233
|
+
result["constants"].append(
|
|
234
|
+
{
|
|
235
|
+
"name": target.id,
|
|
236
|
+
"type": _literal_type(node.value),
|
|
237
|
+
"read_only": True,
|
|
238
|
+
"docstring": "",
|
|
239
|
+
"fully_typed": False,
|
|
240
|
+
"exported": False,
|
|
241
|
+
"excluded": False,
|
|
242
|
+
}
|
|
243
|
+
)
|
|
244
|
+
return result
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def inspect_file_json(file_path: str, module_name: str) -> str:
|
|
248
|
+
return json.dumps(inspect_file(file_path, module_name), default=str)
|