concurrent-c-node 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
cc_node/__init__.py
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
"""cc-node: JavaScript (and every npm package) from Python.
|
|
2
|
+
|
|
3
|
+
import cc_node
|
|
4
|
+
js = cc_node.create() # an Isolation Domain: one node child
|
|
5
|
+
_ = js.require('lodash') # resolved from YOUR cwd's node_modules
|
|
6
|
+
_.chunk([1, 2, 3, 4, 5], 2) # [[1, 2], [3, 4], [5]]
|
|
7
|
+
js.destroy()
|
|
8
|
+
|
|
9
|
+
The mirror of the cc-python bridge, same rules pointed the other way:
|
|
10
|
+
attribute access is property lookup (methods arrive bound), a call is a
|
|
11
|
+
call, plain data (finite numbers, strings, booleans, None, lists and
|
|
12
|
+
dicts of the same) crosses by value and everything else stays a live
|
|
13
|
+
handle owned by the domain. A thenable result is awaited in the child
|
|
14
|
+
before the reply, so async package APIs work with nothing extra. A
|
|
15
|
+
Python callable passed as an argument becomes a JS function; its
|
|
16
|
+
exceptions cross back as JS errors and vice versa, messages intact.
|
|
17
|
+
Handles never cross domains; every door after destroy() answers
|
|
18
|
+
articulately; destroy is idempotent and `with cc_node.create() as js:`
|
|
19
|
+
scopes it.
|
|
20
|
+
"""
|
|
21
|
+
import array
|
|
22
|
+
import atexit
|
|
23
|
+
import base64
|
|
24
|
+
import json
|
|
25
|
+
import math
|
|
26
|
+
import os
|
|
27
|
+
import subprocess
|
|
28
|
+
|
|
29
|
+
__all__ = ["create", "JsError", "JsHandle", "__version__"]
|
|
30
|
+
__version__ = "0.2.0"
|
|
31
|
+
|
|
32
|
+
# Typed buffers cross as typed arrays; big ones spill through shared
|
|
33
|
+
# memory (tmpfs where available) — one memcpy per side, receiver
|
|
34
|
+
# consumes-and-unlinks, sender sweeps after the reply.
|
|
35
|
+
_TA_TYPECODE = {"f64": "d", "f32": "f", "i32": "i", "i64": "q", "u8": "B"}
|
|
36
|
+
_TA_DTYPE = {"f64": "float64", "f32": "float32", "i32": "int32",
|
|
37
|
+
"i64": "int64", "u8": "uint8"}
|
|
38
|
+
_TA_BY_TYPECODE = {tc: k for k, tc in _TA_TYPECODE.items()}
|
|
39
|
+
_TA_BY_DTYPE = {dt: k for k, dt in _TA_DTYPE.items()}
|
|
40
|
+
_SHM_SPILL = 1 << 16
|
|
41
|
+
_shm_seq = [0]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _numpy():
|
|
45
|
+
try:
|
|
46
|
+
import numpy
|
|
47
|
+
return numpy
|
|
48
|
+
except Exception:
|
|
49
|
+
return None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _shm_dir():
|
|
53
|
+
d = os.environ.get("CC_NODE_SHM_DIR")
|
|
54
|
+
if d:
|
|
55
|
+
return d
|
|
56
|
+
return "/dev/shm" if os.path.isdir("/dev/shm") else None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _shm_write(raw):
|
|
60
|
+
_shm_seq[0] += 1
|
|
61
|
+
path = os.path.join(_shm_dir(), "ccnode-%d-%d" % (os.getpid(),
|
|
62
|
+
_shm_seq[0]))
|
|
63
|
+
with open(path, "wb") as f:
|
|
64
|
+
f.write(raw)
|
|
65
|
+
return path
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class JsError(RuntimeError):
|
|
69
|
+
"""A JavaScript error, message intact."""
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
_live = []
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _atexit():
|
|
76
|
+
for b in list(_live):
|
|
77
|
+
try:
|
|
78
|
+
b.destroy()
|
|
79
|
+
except Exception:
|
|
80
|
+
pass
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
atexit.register(_atexit)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class JsHandle:
|
|
87
|
+
"""A live JavaScript value owned by its domain."""
|
|
88
|
+
|
|
89
|
+
__slots__ = ("_d", "_h")
|
|
90
|
+
|
|
91
|
+
def __init__(self, domain, hid):
|
|
92
|
+
object.__setattr__(self, "_d", domain)
|
|
93
|
+
object.__setattr__(self, "_h", hid)
|
|
94
|
+
|
|
95
|
+
def __getattr__(self, name):
|
|
96
|
+
return self._d._req("get", h=self._h, name=name)
|
|
97
|
+
|
|
98
|
+
def __call__(self, *args):
|
|
99
|
+
return self._d._req("call", h=self._h,
|
|
100
|
+
args=[self._d._encode(a) for a in args])
|
|
101
|
+
|
|
102
|
+
def __str__(self):
|
|
103
|
+
return self._d._req("str", h=self._h)
|
|
104
|
+
|
|
105
|
+
def __repr__(self):
|
|
106
|
+
return "<JsHandle #%d%s>" % (self._h,
|
|
107
|
+
" (closed)" if self._d.closed else "")
|
|
108
|
+
|
|
109
|
+
def __del__(self):
|
|
110
|
+
try:
|
|
111
|
+
if not self._d.closed:
|
|
112
|
+
self._d._req("release", h=self._h)
|
|
113
|
+
except Exception:
|
|
114
|
+
pass
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class Bridge:
|
|
118
|
+
"""The domain: one spawned node child, one handle table, one wire."""
|
|
119
|
+
|
|
120
|
+
def __init__(self, node=None):
|
|
121
|
+
broker = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
|
122
|
+
"broker.cjs")
|
|
123
|
+
node = node or os.environ.get("CC_NODE_BIN", "node")
|
|
124
|
+
try:
|
|
125
|
+
self._p = subprocess.Popen(
|
|
126
|
+
[node, broker],
|
|
127
|
+
stdin=subprocess.PIPE,
|
|
128
|
+
stdout=subprocess.PIPE,
|
|
129
|
+
cwd=os.getcwd(),
|
|
130
|
+
)
|
|
131
|
+
except FileNotFoundError:
|
|
132
|
+
raise JsError(
|
|
133
|
+
"cc-node: no node executable (install Node, or set "
|
|
134
|
+
"CC_NODE_BIN)") from None
|
|
135
|
+
self.closed = False
|
|
136
|
+
self._nid = 1
|
|
137
|
+
self._cbs = {}
|
|
138
|
+
self._ncb = 1
|
|
139
|
+
self._shm_out = []
|
|
140
|
+
_live.append(self)
|
|
141
|
+
|
|
142
|
+
# ---- wire ----
|
|
143
|
+
|
|
144
|
+
def _send(self, obj):
|
|
145
|
+
line = json.dumps(obj, allow_nan=False) + "\n"
|
|
146
|
+
self._p.stdin.write(line.encode("utf-8"))
|
|
147
|
+
self._p.stdin.flush()
|
|
148
|
+
|
|
149
|
+
def _req(self, op, **kw):
|
|
150
|
+
if self.closed:
|
|
151
|
+
raise JsError("cc-node: bridge is closed")
|
|
152
|
+
rid = self._nid
|
|
153
|
+
self._nid += 1
|
|
154
|
+
kw["id"] = rid
|
|
155
|
+
kw["op"] = op
|
|
156
|
+
try:
|
|
157
|
+
self._send(kw)
|
|
158
|
+
except (BrokenPipeError, ValueError):
|
|
159
|
+
self.closed = True
|
|
160
|
+
raise JsError("cc-node: the node child exited") from None
|
|
161
|
+
try:
|
|
162
|
+
while True:
|
|
163
|
+
line = self._p.stdout.readline()
|
|
164
|
+
if not line:
|
|
165
|
+
self.closed = True
|
|
166
|
+
raise JsError("cc-node: the node child exited")
|
|
167
|
+
msg = json.loads(line)
|
|
168
|
+
if "cb" in msg:
|
|
169
|
+
self._serve_callback(msg)
|
|
170
|
+
continue
|
|
171
|
+
if msg.get("id") == rid:
|
|
172
|
+
if "e" in msg:
|
|
173
|
+
raise JsError(msg["e"])
|
|
174
|
+
return self._decode_result(msg)
|
|
175
|
+
raise JsError("cc-node: protocol violation (unexpected reply)")
|
|
176
|
+
finally:
|
|
177
|
+
# The child unlinks spill files as it decodes; this sweep only
|
|
178
|
+
# matters when it died first (ENOENT is the normal case).
|
|
179
|
+
for path in self._shm_out:
|
|
180
|
+
try:
|
|
181
|
+
os.unlink(path)
|
|
182
|
+
except OSError:
|
|
183
|
+
pass
|
|
184
|
+
del self._shm_out[:]
|
|
185
|
+
|
|
186
|
+
def _serve_callback(self, msg):
|
|
187
|
+
fn = self._cbs.get(msg["cb"])
|
|
188
|
+
try:
|
|
189
|
+
if fn is None:
|
|
190
|
+
raise JsError("cc-node: unknown callback")
|
|
191
|
+
args = [self._decode_result(a) for a in msg["args"]]
|
|
192
|
+
self._send({"cbr": msg["cbid"], "v": self._encode(fn(*args))})
|
|
193
|
+
except Exception as e: # crosses back as a JS error, message intact
|
|
194
|
+
self._send({"cbr": msg["cbid"], "e": str(e)})
|
|
195
|
+
|
|
196
|
+
# ---- values ----
|
|
197
|
+
|
|
198
|
+
def _encode(self, a):
|
|
199
|
+
if isinstance(a, JsHandle):
|
|
200
|
+
if a._d is not self:
|
|
201
|
+
raise JsError("cc-node: handle belongs to another bridge")
|
|
202
|
+
return {"$h": a._h}
|
|
203
|
+
if callable(a):
|
|
204
|
+
fid = self._ncb
|
|
205
|
+
self._ncb += 1
|
|
206
|
+
self._cbs[fid] = a
|
|
207
|
+
return {"$f": fid}
|
|
208
|
+
if a is None or isinstance(a, (bool, int, str)):
|
|
209
|
+
return a
|
|
210
|
+
if isinstance(a, float):
|
|
211
|
+
if math.isnan(a):
|
|
212
|
+
return {"$nf": "nan"}
|
|
213
|
+
if math.isinf(a):
|
|
214
|
+
return {"$nf": "inf" if a > 0 else "-inf"}
|
|
215
|
+
return a
|
|
216
|
+
if isinstance(a, (list, tuple)):
|
|
217
|
+
return [self._encode(x) for x in a]
|
|
218
|
+
if isinstance(a, dict):
|
|
219
|
+
out = {}
|
|
220
|
+
for k, v in a.items():
|
|
221
|
+
if not isinstance(k, str):
|
|
222
|
+
raise JsError("cc-node: dict keys must be strings")
|
|
223
|
+
if k.startswith("$"):
|
|
224
|
+
raise JsError("cc-node: dict keys starting with '$' are "
|
|
225
|
+
"reserved on the wire")
|
|
226
|
+
out[k] = self._encode(v)
|
|
227
|
+
return out
|
|
228
|
+
enc = self._encode_buffer(a)
|
|
229
|
+
if enc is not None:
|
|
230
|
+
return enc
|
|
231
|
+
raise JsError("cc-node: unsupported argument type: %r" % type(a))
|
|
232
|
+
|
|
233
|
+
def _encode_buffer(self, a):
|
|
234
|
+
# Typed buffers cross as typed arrays: bytes/array.array/1-D
|
|
235
|
+
# numpy — small inline as base64, big through the shared-memory
|
|
236
|
+
# spill (one memcpy per side; the receiver consumes-and-unlinks).
|
|
237
|
+
raw = None
|
|
238
|
+
kind = None
|
|
239
|
+
if isinstance(a, (bytes, bytearray, memoryview)):
|
|
240
|
+
raw, kind = bytes(a), "u8"
|
|
241
|
+
elif isinstance(a, array.array):
|
|
242
|
+
kind = _TA_BY_TYPECODE.get(a.typecode)
|
|
243
|
+
if kind is not None:
|
|
244
|
+
raw = a.tobytes()
|
|
245
|
+
else:
|
|
246
|
+
np = _numpy()
|
|
247
|
+
if np is not None and isinstance(a, np.ndarray) and a.ndim == 1:
|
|
248
|
+
kind = _TA_BY_DTYPE.get(str(a.dtype))
|
|
249
|
+
if kind is not None:
|
|
250
|
+
raw = np.ascontiguousarray(a).tobytes()
|
|
251
|
+
if raw is None or kind is None:
|
|
252
|
+
return None
|
|
253
|
+
if len(raw) > _SHM_SPILL and _shm_dir():
|
|
254
|
+
path = _shm_write(raw)
|
|
255
|
+
self._shm_out.append(path)
|
|
256
|
+
return {"$shm": path, "t": kind}
|
|
257
|
+
return {"$ta": kind, "b64": base64.b64encode(raw).decode("ascii")}
|
|
258
|
+
|
|
259
|
+
def _decode_buffer(self, msg):
|
|
260
|
+
if "shm" in msg:
|
|
261
|
+
path = msg["shm"]
|
|
262
|
+
try:
|
|
263
|
+
with open(path, "rb") as f:
|
|
264
|
+
raw = f.read()
|
|
265
|
+
finally:
|
|
266
|
+
try:
|
|
267
|
+
os.unlink(path)
|
|
268
|
+
except OSError:
|
|
269
|
+
pass
|
|
270
|
+
else:
|
|
271
|
+
raw = base64.b64decode(msg["b64"])
|
|
272
|
+
kind = msg.get("t") or msg.get("ta")
|
|
273
|
+
np = _numpy()
|
|
274
|
+
if np is not None:
|
|
275
|
+
return np.frombuffer(raw, dtype=_TA_DTYPE[kind]).copy()
|
|
276
|
+
a = array.array(_TA_TYPECODE[kind])
|
|
277
|
+
a.frombytes(raw)
|
|
278
|
+
return a
|
|
279
|
+
|
|
280
|
+
def _decode_result(self, msg):
|
|
281
|
+
if "u" in msg:
|
|
282
|
+
return None
|
|
283
|
+
if "h" in msg:
|
|
284
|
+
return JsHandle(self, msg["h"])
|
|
285
|
+
if "nf" in msg:
|
|
286
|
+
return {"nan": math.nan, "inf": math.inf,
|
|
287
|
+
"-inf": -math.inf}[msg["nf"]]
|
|
288
|
+
if "ta" in msg or "shm" in msg:
|
|
289
|
+
return self._decode_buffer(msg)
|
|
290
|
+
return msg.get("v")
|
|
291
|
+
|
|
292
|
+
# ---- surface ----
|
|
293
|
+
|
|
294
|
+
def require(self, name):
|
|
295
|
+
return self._req("require", name=name)
|
|
296
|
+
|
|
297
|
+
def import_module(self, name):
|
|
298
|
+
return self._req("import", name=name)
|
|
299
|
+
|
|
300
|
+
def eval(self, src):
|
|
301
|
+
return self._req("eval", src=src)
|
|
302
|
+
|
|
303
|
+
def release(self, handle):
|
|
304
|
+
if not isinstance(handle, JsHandle) or handle._d is not self:
|
|
305
|
+
raise JsError("cc-node: handle belongs to another bridge")
|
|
306
|
+
return self._req("release", h=handle._h)
|
|
307
|
+
|
|
308
|
+
def stats(self):
|
|
309
|
+
return self._req("stats")
|
|
310
|
+
|
|
311
|
+
def destroy(self):
|
|
312
|
+
if self.closed:
|
|
313
|
+
return
|
|
314
|
+
try:
|
|
315
|
+
self._req("close")
|
|
316
|
+
except JsError:
|
|
317
|
+
pass
|
|
318
|
+
self.closed = True
|
|
319
|
+
try:
|
|
320
|
+
self._p.stdin.close()
|
|
321
|
+
except Exception:
|
|
322
|
+
pass
|
|
323
|
+
try:
|
|
324
|
+
self._p.wait(timeout=5)
|
|
325
|
+
except Exception:
|
|
326
|
+
self._p.kill()
|
|
327
|
+
if self in _live:
|
|
328
|
+
_live.remove(self)
|
|
329
|
+
|
|
330
|
+
close = destroy
|
|
331
|
+
|
|
332
|
+
def __enter__(self):
|
|
333
|
+
return self
|
|
334
|
+
|
|
335
|
+
def __exit__(self, *exc):
|
|
336
|
+
self.destroy()
|
|
337
|
+
return False
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def create(node=None):
|
|
341
|
+
return Bridge(node=node)
|
cc_node/broker.cjs
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/* cc-node broker: the Node end of Python's JS bridge.
|
|
3
|
+
*
|
|
4
|
+
* Line-delimited JSON over stdio, strict request/response. Handles are
|
|
5
|
+
* integers into one table; results follow the bridge materialization
|
|
6
|
+
* rule — plain data (finite numbers, strings, booleans, null, arrays
|
|
7
|
+
* and plain objects of the same) crosses as a value, everything else
|
|
8
|
+
* stays a handle. A thenable result is awaited before the reply, so
|
|
9
|
+
* async package APIs need nothing special from the Python side. A
|
|
10
|
+
* Python callable crosses as {$f: id}; invoking it sends a nested `cb`
|
|
11
|
+
* request and BLOCKS on a synchronous read for the answer — legal
|
|
12
|
+
* because the protocol is strictly alternating, so nothing else can be
|
|
13
|
+
* in flight. stdin EOF is the host vanishing: exit. */
|
|
14
|
+
'use strict';
|
|
15
|
+
|
|
16
|
+
const fs = require('fs');
|
|
17
|
+
const { createRequire } = require('module');
|
|
18
|
+
|
|
19
|
+
/* Resolve packages from the HOST's cwd — `npm install lodash` next to
|
|
20
|
+
* your Python program is the point. */
|
|
21
|
+
const requireCwd = createRequire(process.cwd() + '/');
|
|
22
|
+
|
|
23
|
+
/* ---- one buffered reader over fd 0, sync and async ---- */
|
|
24
|
+
const rbuf = { data: Buffer.alloc(0) };
|
|
25
|
+
|
|
26
|
+
function takeLine() {
|
|
27
|
+
const i = rbuf.data.indexOf(10);
|
|
28
|
+
if (i < 0) return null;
|
|
29
|
+
const line = rbuf.data.subarray(0, i).toString('utf8');
|
|
30
|
+
rbuf.data = rbuf.data.subarray(i + 1);
|
|
31
|
+
return line;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function readLineSync() {
|
|
35
|
+
for (;;) {
|
|
36
|
+
const l = takeLine();
|
|
37
|
+
if (l !== null) return l;
|
|
38
|
+
const chunk = Buffer.alloc(65536);
|
|
39
|
+
let n = 0;
|
|
40
|
+
try {
|
|
41
|
+
n = fs.readSync(0, chunk, 0, chunk.length, null);
|
|
42
|
+
} catch (e) {
|
|
43
|
+
if (e.code === 'EAGAIN') continue;
|
|
44
|
+
if (e.code === 'EOF') return null;
|
|
45
|
+
throw e;
|
|
46
|
+
}
|
|
47
|
+
if (n === 0) return null;
|
|
48
|
+
rbuf.data = Buffer.concat([rbuf.data, chunk.subarray(0, n)]);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function readLineAsync() {
|
|
53
|
+
const l = takeLine();
|
|
54
|
+
if (l !== null) return Promise.resolve(l);
|
|
55
|
+
return new Promise((resolve, reject) => {
|
|
56
|
+
const chunk = Buffer.alloc(65536);
|
|
57
|
+
fs.read(0, chunk, 0, chunk.length, null, (err, n) => {
|
|
58
|
+
if (err) return err.code === 'EOF' ? resolve(null) : reject(err);
|
|
59
|
+
if (n === 0) return resolve(null);
|
|
60
|
+
rbuf.data = Buffer.concat([rbuf.data, chunk.subarray(0, n)]);
|
|
61
|
+
resolve(readLineAsync());
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function send(obj) {
|
|
67
|
+
fs.writeSync(1, JSON.stringify(obj) + '\n');
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/* ---- handles + materialization ---- */
|
|
71
|
+
const handles = new Map();
|
|
72
|
+
let nextH = 1;
|
|
73
|
+
const put = (v) => {
|
|
74
|
+
const id = nextH++;
|
|
75
|
+
handles.set(id, v);
|
|
76
|
+
return id;
|
|
77
|
+
};
|
|
78
|
+
const getH = (id) => {
|
|
79
|
+
if (!handles.has(id)) throw new Error('cc-node: unknown or released handle');
|
|
80
|
+
return handles.get(id);
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
/* Plain data crosses by value; non-finite numbers are NOT plain (JSON
|
|
84
|
+
* would silently null them — they go as tagged scalars or handles). */
|
|
85
|
+
function isPlain(v, depth) {
|
|
86
|
+
if (depth > 16) return false;
|
|
87
|
+
if (v === null) return true;
|
|
88
|
+
const t = typeof v;
|
|
89
|
+
if (t === 'number') return Number.isFinite(v);
|
|
90
|
+
if (t === 'string' || t === 'boolean') return true;
|
|
91
|
+
if (t !== 'object') return false;
|
|
92
|
+
const proto = Object.getPrototypeOf(v);
|
|
93
|
+
if (Array.isArray(v)) return v.every((x) => isPlain(x, depth + 1));
|
|
94
|
+
if (proto === Object.prototype || proto === null)
|
|
95
|
+
return Object.values(v).every((x) => isPlain(x, depth + 1));
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/* Typed buffers cross as tagged bytes: small inline as base64, big
|
|
100
|
+
* through the shared-memory spill (one memcpy per side; the receiver
|
|
101
|
+
* consumes-and-unlinks). Same discipline as cc-python's wire. */
|
|
102
|
+
const TA_KIND = new Map([
|
|
103
|
+
[Float64Array, 'f64'], [Float32Array, 'f32'],
|
|
104
|
+
[Int32Array, 'i32'], [BigInt64Array, 'i64'], [Uint8Array, 'u8'],
|
|
105
|
+
]);
|
|
106
|
+
const TA_CTOR = {
|
|
107
|
+
f64: Float64Array, f32: Float32Array,
|
|
108
|
+
i32: Int32Array, i64: BigInt64Array, u8: Uint8Array,
|
|
109
|
+
};
|
|
110
|
+
const SHM_SPILL = 1 << 16;
|
|
111
|
+
const SHM_DIR = process.env.CC_NODE_SHM_DIR ||
|
|
112
|
+
(fs.existsSync('/dev/shm') ? '/dev/shm'
|
|
113
|
+
: require('os').tmpdir());
|
|
114
|
+
let shmSeq = 0;
|
|
115
|
+
|
|
116
|
+
function encodeBuffer(kind, buf) {
|
|
117
|
+
if (buf.byteLength > SHM_SPILL) {
|
|
118
|
+
const p = require('path').join(
|
|
119
|
+
SHM_DIR, 'ccnode-' + process.pid + '-' + (++shmSeq));
|
|
120
|
+
fs.writeFileSync(p, buf);
|
|
121
|
+
return { shm: p, t: kind };
|
|
122
|
+
}
|
|
123
|
+
return { ta: kind, b64: buf.toString('base64') };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function encodeResult(v) {
|
|
127
|
+
if (v === undefined) return { u: 1 };
|
|
128
|
+
if (typeof v === 'number' && !Number.isFinite(v))
|
|
129
|
+
return { nf: Number.isNaN(v) ? 'nan' : v > 0 ? 'inf' : '-inf' };
|
|
130
|
+
if (v !== null && typeof v === 'object') {
|
|
131
|
+
const kind = TA_KIND.get(v.constructor);
|
|
132
|
+
if (kind)
|
|
133
|
+
return encodeBuffer(kind,
|
|
134
|
+
Buffer.from(v.buffer, v.byteOffset, v.byteLength));
|
|
135
|
+
if (Buffer.isBuffer(v)) return encodeBuffer('u8', v);
|
|
136
|
+
}
|
|
137
|
+
if (v === null || isPlain(v, 0)) return { v };
|
|
138
|
+
return { h: put(v) };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/* ---- Python callables ---- */
|
|
142
|
+
let nextCbId = 1;
|
|
143
|
+
|
|
144
|
+
function makeCallback(fid) {
|
|
145
|
+
const f = (...args) => {
|
|
146
|
+
const cbid = nextCbId++;
|
|
147
|
+
send({ cb: fid, cbid, args: args.map(encodeResult) });
|
|
148
|
+
const line = readLineSync();
|
|
149
|
+
if (line === null) throw new Error('cc-node: host went away');
|
|
150
|
+
const m = JSON.parse(line);
|
|
151
|
+
if (m.cbr !== cbid)
|
|
152
|
+
throw new Error('cc-node: protocol violation during callback');
|
|
153
|
+
if (m.e !== undefined) throw new Error(m.e);
|
|
154
|
+
return decodeVal(m.v);
|
|
155
|
+
};
|
|
156
|
+
return f;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function decodeVal(a) {
|
|
160
|
+
if (a && typeof a === 'object') {
|
|
161
|
+
if (a.$h !== undefined) return getH(a.$h);
|
|
162
|
+
if (a.$f !== undefined) return makeCallback(a.$f);
|
|
163
|
+
if (a.$nf !== undefined)
|
|
164
|
+
return a.$nf === 'nan' ? NaN : a.$nf === 'inf' ? Infinity : -Infinity;
|
|
165
|
+
if (a.$ta !== undefined || a.$shm !== undefined) {
|
|
166
|
+
let buf;
|
|
167
|
+
if (a.$shm !== undefined) {
|
|
168
|
+
buf = fs.readFileSync(a.$shm);
|
|
169
|
+
try { fs.unlinkSync(a.$shm); } catch (e) { /* consumed */ }
|
|
170
|
+
} else {
|
|
171
|
+
buf = Buffer.from(a.b64, 'base64');
|
|
172
|
+
}
|
|
173
|
+
const C = TA_CTOR[a.t !== undefined ? a.t : a.$ta];
|
|
174
|
+
return new C(buf.buffer, buf.byteOffset,
|
|
175
|
+
buf.byteLength / C.BYTES_PER_ELEMENT);
|
|
176
|
+
}
|
|
177
|
+
if (Array.isArray(a)) return a.map(decodeVal);
|
|
178
|
+
const o = {};
|
|
179
|
+
for (const k of Object.keys(a)) o[k] = decodeVal(a[k]);
|
|
180
|
+
return o;
|
|
181
|
+
}
|
|
182
|
+
return a;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/* ---- dispatch ---- */
|
|
186
|
+
async function main() {
|
|
187
|
+
for (;;) {
|
|
188
|
+
const line = await readLineAsync();
|
|
189
|
+
if (line === null) process.exit(0);
|
|
190
|
+
let req;
|
|
191
|
+
try {
|
|
192
|
+
req = JSON.parse(line);
|
|
193
|
+
} catch {
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
const id = req.id;
|
|
197
|
+
try {
|
|
198
|
+
let r;
|
|
199
|
+
switch (req.op) {
|
|
200
|
+
case 'require': r = requireCwd(req.name); break;
|
|
201
|
+
case 'import': r = await import(req.name); break;
|
|
202
|
+
case 'eval': r = (0, eval)(req.src); break;
|
|
203
|
+
case 'get': {
|
|
204
|
+
const o = getH(req.h);
|
|
205
|
+
const v = o[req.name];
|
|
206
|
+
r = typeof v === 'function' ? v.bind(o) : v;
|
|
207
|
+
break;
|
|
208
|
+
}
|
|
209
|
+
case 'call': {
|
|
210
|
+
const f = getH(req.h);
|
|
211
|
+
if (typeof f !== 'function')
|
|
212
|
+
throw new Error('cc-node: handle is not callable');
|
|
213
|
+
r = f(...(req.args || []).map(decodeVal));
|
|
214
|
+
break;
|
|
215
|
+
}
|
|
216
|
+
case 'str': r = String(getH(req.h)); break;
|
|
217
|
+
case 'release': handles.delete(req.h); r = handles.size; break;
|
|
218
|
+
case 'stats': r = handles.size; break;
|
|
219
|
+
case 'close': send({ id, v: true }); process.exit(0); break;
|
|
220
|
+
default: throw new Error('cc-node: unknown op ' + req.op);
|
|
221
|
+
}
|
|
222
|
+
if (r && typeof r.then === 'function') r = await r; /* async is free */
|
|
223
|
+
send({ id, ...encodeResult(r) });
|
|
224
|
+
} catch (e) {
|
|
225
|
+
send({ id, e: String(e && e.message !== undefined ? e.message : e) });
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
main();
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: concurrent-c-node
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: JavaScript and npm packages from Python over the Concurrent-C bridge: one spawned Node child per domain, host-controlled lifetime.
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Repository, https://github.com/sreekotay/concurrent-c
|
|
7
|
+
Project-URL: Documentation, https://github.com/sreekotay/concurrent-c/blob/main/pypi/cc-node/README.md
|
|
8
|
+
Keywords: javascript,node,npm,interop,concurrent-c
|
|
9
|
+
Requires-Python: >=3.8
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
|
|
12
|
+
# concurrent-c-node
|
|
13
|
+
|
|
14
|
+
JavaScript — and every npm package — from Python.
|
|
15
|
+
|
|
16
|
+
Part of [Concurrent-C](https://github.com/sreekotay/concurrent-c) — a
|
|
17
|
+
strict C11-superset preprocessor: `.ccs` lowers to plain C and compiles
|
|
18
|
+
with your host C compiler. (This bridge itself is pure Python.)
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
import cc_node
|
|
22
|
+
|
|
23
|
+
js = cc_node.create() # an Isolation Domain: one node child
|
|
24
|
+
_ = js.require('lodash') # resolved from YOUR cwd's node_modules
|
|
25
|
+
_.chunk([1, 2, 3, 4, 5], 2) # [[1, 2], [3, 4], [5]]
|
|
26
|
+
_.sortBy([{'n': 3}, {'n': 1}], 'n') # dicts cross as objects, and back
|
|
27
|
+
|
|
28
|
+
semver = js.require('semver')
|
|
29
|
+
semver.satisfies('1.2.3', '^1.0.0') # True
|
|
30
|
+
|
|
31
|
+
js.destroy() # or: with cc_node.create() as js: ...
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The bridge is **pure Python, stdlib only** — no compiled code, no
|
|
35
|
+
dependencies, nothing to build. The domain **is** a spawned `node`
|
|
36
|
+
child (~28ms to first call), so you get real Node: full stdlib, native
|
|
37
|
+
addons, whatever npm installs. Promise-based APIs look synchronous
|
|
38
|
+
from Python, and bulk data crosses through **shared memory** — an 8MB
|
|
39
|
+
array in **9ms** where the same values as a JSON list take 583ms.
|
|
40
|
+
|
|
41
|
+
```
|
|
42
|
+
pip install concurrent-c-node # needs node on PATH (or point at one)
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Import stays `import cc_node`. The mirror of
|
|
46
|
+
[`concurrent-c-python`](https://github.com/sreekotay/concurrent-c/tree/main/npm/cc-python)
|
|
47
|
+
— same domain model, same materialization rules, pointed the other way:
|
|
48
|
+
|
|
49
|
+
- **Values**: plain data (finite numbers, strings, booleans, `None`,
|
|
50
|
+
lists/dicts of the same) crosses by value; everything else is a live
|
|
51
|
+
handle owned by the domain — attribute access is property lookup
|
|
52
|
+
(methods arrive bound), calls are calls, `str()` is `String()`.
|
|
53
|
+
Non-finite floats cross tagged, never silently nulled.
|
|
54
|
+
- **The domain rules hold**: handles never cross bridges; `stats()` is
|
|
55
|
+
the handle ledger and `release()` drops one early; `destroy()` is
|
|
56
|
+
idempotent, every door answers `bridge is closed` after, and the
|
|
57
|
+
child dies with the bridge (and on host exit, via stdin EOF).
|
|
58
|
+
|
|
59
|
+
## Async is free
|
|
60
|
+
|
|
61
|
+
A thenable result is awaited **in the child** before the reply, so
|
|
62
|
+
promise-based package APIs need nothing special — no event loop on the
|
|
63
|
+
Python side, no `await`:
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
fetchish = js.eval('async (x) => { return { doubled: x * 2 } }')
|
|
67
|
+
fetchish(21) # {'doubled': 42} — just a call
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Whatever an npm package's API returns — value or promise — the call
|
|
71
|
+
site reads the same.
|
|
72
|
+
|
|
73
|
+
## Callbacks: Python functions as JS functions
|
|
74
|
+
|
|
75
|
+
A Python callable passed as an argument crosses as a JS function, and
|
|
76
|
+
may be called back any number of times — including from inside async
|
|
77
|
+
JS code:
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
mapped = js.eval('(f) => [1, 2, 3].map(f)')(lambda x, *rest: x * 10)
|
|
81
|
+
# [10, 20, 30] — JS conventions apply: map passes (value, index, array),
|
|
82
|
+
# so a lambda takes *rest. Exceptions cross both ways, messages intact.
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Nested callbacks compose (the wire alternates strictly), and a Python
|
|
86
|
+
exception inside one surfaces as the JS error at the call site — and
|
|
87
|
+
vice versa.
|
|
88
|
+
|
|
89
|
+
## Buffers: typed arrays, shared memory
|
|
90
|
+
|
|
91
|
+
`bytes`, `array.array`, and 1-D numpy arrays cross as
|
|
92
|
+
`Float64Array` / `Int32Array` / `Uint8Array` / … and come back as numpy
|
|
93
|
+
arrays (or `array.array` without numpy):
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
import array
|
|
97
|
+
total = js.eval('(a) => a.reduce((s, x) => s + x, 0)')
|
|
98
|
+
total(array.array('d', range(1_000_000))) # crosses via shared memory
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Small buffers inline; big ones spill through shared memory — one
|
|
102
|
+
memcpy per side, the receiver consumes the spill file, and the sender
|
|
103
|
+
sweeps it if the child died first. Nothing strays, and nothing is
|
|
104
|
+
silently truncated: an unsupported type is an articulate error.
|
|
105
|
+
|
|
106
|
+
## Choosing the node
|
|
107
|
+
|
|
108
|
+
Same ambient-first rule as the rest of the family: the domain runs
|
|
109
|
+
whatever `node` your project runs.
|
|
110
|
+
|
|
111
|
+
1. `create(node='/path/to/node')` from code — per-domain.
|
|
112
|
+
2. `CC_NODE_BIN` in the environment.
|
|
113
|
+
3. `node` on `PATH`.
|
|
114
|
+
|
|
115
|
+
And *which packages* it sees is the working directory's
|
|
116
|
+
`node_modules` — `require` resolves exactly as node itself would there.
|
|
117
|
+
Run Python in your project, get your project's packages: `npm install`
|
|
118
|
+
next to your program is the whole setup.
|
|
119
|
+
|
|
120
|
+
## Measured
|
|
121
|
+
|
|
122
|
+
From [`examples/bench_wire.py`](https://github.com/sreekotay/concurrent-c/blob/main/pypi/cc-node/examples/bench_wire.py)
|
|
123
|
+
on a 4-vCPU x86-64 box, node 22 / python 3.11 (dated baselines under
|
|
124
|
+
`perf/baselines/` in the repo):
|
|
125
|
+
|
|
126
|
+
| what | result |
|
|
127
|
+
|---|---|
|
|
128
|
+
| spawn a domain (node child, first eval) | 28ms |
|
|
129
|
+
| wire round trip (smallest call) | 116µs |
|
|
130
|
+
| Python-callback round trip (JS → Python → JS) | 238µs |
|
|
131
|
+
| 8MB `array('d')` argument, shm spill | **9.2ms** |
|
|
132
|
+
| the same 8MB as a JSON list | 583ms — the spill is **63x** |
|
|
133
|
+
|
|
134
|
+
The wire is strict request/response JSON over stdio with the
|
|
135
|
+
shared-memory spill for bulk data — the same discipline concurrent-c-python's
|
|
136
|
+
isolated domains speak, mirrored. True pinned zero-copy leases remain
|
|
137
|
+
future work.
|
|
138
|
+
|
|
139
|
+
A worked tour (builtin Node modules, chains, callbacks, thenables,
|
|
140
|
+
buffers — no npm install needed):
|
|
141
|
+
[`examples/use_node.py`](https://github.com/sreekotay/concurrent-c/blob/main/pypi/cc-node/examples/use_node.py).
|
|
142
|
+
|
|
143
|
+
And when the hot path is YOUR code rather than an npm package, skip the
|
|
144
|
+
wire entirely: a page of Concurrent-C (or C) exports as a native module
|
|
145
|
+
for Python and Node both — 40-90ns calls, stable-ABI artifacts. See
|
|
146
|
+
[Native modules for Node and Python](https://github.com/sreekotay/concurrent-c/blob/main/docs/js-py-modules.md).
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
cc_node/__init__.py,sha256=Jx0VzkeIoQs9oZUyP70Q5vQnpvo6ggYUu8_DY4XVk2o,10804
|
|
2
|
+
cc_node/broker.cjs,sha256=4hjGgUopSahKVc7MK56Im3heD2jmEmnwI6Pa7AOwbKo,7586
|
|
3
|
+
concurrent_c_node-0.2.0.dist-info/METADATA,sha256=xa3pKWGe0qtSeGRRibCmUncMPCtafJWiHuuQeqPDOj4,6010
|
|
4
|
+
concurrent_c_node-0.2.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
5
|
+
concurrent_c_node-0.2.0.dist-info/top_level.txt,sha256=dDOGyUKcGWZV9KTYNTTv4Cam8hu-_PMINVQs0rnMmus,8
|
|
6
|
+
concurrent_c_node-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
cc_node
|