tywrap 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -15
- package/SECURITY.md +5 -6
- package/dist/core/generator.d.ts.map +1 -1
- package/dist/core/generator.js +11 -0
- package/dist/core/generator.js.map +1 -1
- package/dist/runtime/bridge-codec.d.ts +2 -2
- package/dist/runtime/bridge-codec.d.ts.map +1 -1
- package/dist/runtime/bridge-codec.js +51 -7
- package/dist/runtime/bridge-codec.js.map +1 -1
- package/dist/runtime/pyodide-bootstrap-core.generated.d.ts.map +1 -1
- package/dist/runtime/pyodide-bootstrap-core.generated.js +1 -1
- package/dist/runtime/pyodide-bootstrap-core.generated.js.map +1 -1
- package/dist/runtime/validators.d.ts +3 -2
- package/dist/runtime/validators.d.ts.map +1 -1
- package/dist/runtime/validators.js +9 -7
- package/dist/runtime/validators.js.map +1 -1
- package/dist/utils/codec.d.ts +16 -0
- package/dist/utils/codec.d.ts.map +1 -1
- package/dist/utils/codec.js +493 -78
- package/dist/utils/codec.js.map +1 -1
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +1 -1
- package/runtime/tywrap_bridge_core.py +394 -49
- package/src/core/generator.ts +15 -0
- package/src/runtime/bridge-codec.ts +64 -9
- package/src/runtime/pyodide-bootstrap-core.generated.ts +1 -1
- package/src/runtime/validators.ts +17 -9
- package/src/utils/codec.ts +743 -114
- package/src/version.ts +1 -1
|
@@ -47,6 +47,7 @@ import importlib
|
|
|
47
47
|
import importlib.util
|
|
48
48
|
import json
|
|
49
49
|
import math
|
|
50
|
+
import re
|
|
50
51
|
import sys
|
|
51
52
|
import traceback
|
|
52
53
|
import uuid
|
|
@@ -57,6 +58,10 @@ from pathlib import Path, PurePath
|
|
|
57
58
|
PROTOCOL = 'tywrap/1'
|
|
58
59
|
PROTOCOL_VERSION = 1
|
|
59
60
|
CODEC_VERSION = 1
|
|
61
|
+
MAX_SERIALIZE_DEPTH = 900
|
|
62
|
+
MAX_SERIALIZE_NODES = 1_000_000
|
|
63
|
+
JS_SAFE_INTEGER_MAX = 2**53 - 1
|
|
64
|
+
_SERIALIZE_PATH_IDENTIFIER = re.compile(r'^[A-Za-z_$][\w$]*$', re.ASCII)
|
|
60
65
|
|
|
61
66
|
|
|
62
67
|
class ProtocolError(Exception):
|
|
@@ -376,7 +381,7 @@ def serialize_ndarray(obj, *, force_json_markers):
|
|
|
376
381
|
) from exc
|
|
377
382
|
try:
|
|
378
383
|
original_shape = list(obj.shape) if hasattr(obj, 'shape') else None
|
|
379
|
-
flat = obj.flatten() if hasattr(obj, 'ndim') and obj.ndim
|
|
384
|
+
flat = obj.flatten() if hasattr(obj, 'ndim') and obj.ndim != 1 else obj
|
|
380
385
|
arr = pa.array(flat)
|
|
381
386
|
table = pa.Table.from_arrays([arr], names=['value'])
|
|
382
387
|
sink = pa.BufferOutputStream()
|
|
@@ -397,7 +402,80 @@ def serialize_ndarray(obj, *, force_json_markers):
|
|
|
397
402
|
|
|
398
403
|
|
|
399
404
|
def serialize_ndarray_json(obj):
|
|
400
|
-
"""JSON fallback for ndarray
|
|
405
|
+
"""JSON fallback for ndarray values that JavaScript can represent safely."""
|
|
406
|
+
dtype = obj.dtype
|
|
407
|
+
dtype_label = str(dtype)
|
|
408
|
+
|
|
409
|
+
if not dtype.isnative:
|
|
410
|
+
raise RuntimeError(
|
|
411
|
+
f'JSON ndarray encoding does not support big-endian dtype={dtype_label}; '
|
|
412
|
+
"convert to native byte order with a.byteswap().view(a.dtype.newbyteorder('='))"
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
if dtype.kind in ('i', 'u'):
|
|
416
|
+
# JSON numbers become JavaScript Number values. Scan only on this explicit
|
|
417
|
+
# fallback path and reject before tolist() can silently round an integer.
|
|
418
|
+
if obj.size and (
|
|
419
|
+
(obj < -JS_SAFE_INTEGER_MAX).any() or (obj > JS_SAFE_INTEGER_MAX).any()
|
|
420
|
+
):
|
|
421
|
+
raise RuntimeError(
|
|
422
|
+
f'JSON ndarray encoding cannot safely represent dtype={dtype_label} values '
|
|
423
|
+
'outside the JavaScript safe integer range; use Arrow encoding or '
|
|
424
|
+
"cast/encode explicitly (e.g. .astype('float64') or str)"
|
|
425
|
+
)
|
|
426
|
+
elif dtype.kind == 'M':
|
|
427
|
+
raise RuntimeError(
|
|
428
|
+
f'JSON ndarray encoding does not support dtype={dtype_label}; use Arrow encoding '
|
|
429
|
+
"or convert explicitly (e.g. .astype('datetime64[ms]').astype(str) or int with "
|
|
430
|
+
'declared unit)'
|
|
431
|
+
)
|
|
432
|
+
elif dtype.kind == 'm':
|
|
433
|
+
raise RuntimeError(
|
|
434
|
+
f'JSON ndarray encoding does not support dtype={dtype_label}; use Arrow encoding '
|
|
435
|
+
"or convert explicitly (e.g. .astype('timedelta64[ms]').astype(str) or int with "
|
|
436
|
+
'declared unit)'
|
|
437
|
+
)
|
|
438
|
+
elif dtype.kind == 'V':
|
|
439
|
+
if dtype.fields is not None:
|
|
440
|
+
raise RuntimeError(
|
|
441
|
+
f'JSON ndarray encoding does not support structured dtype={dtype_label}; '
|
|
442
|
+
'encode each named field explicitly (e.g. as a plain JSON object)'
|
|
443
|
+
)
|
|
444
|
+
raise RuntimeError(
|
|
445
|
+
f'JSON ndarray encoding does not support void dtype={dtype_label}; convert the '
|
|
446
|
+
"raw bytes explicitly (e.g. .view('uint8'))"
|
|
447
|
+
)
|
|
448
|
+
elif dtype.kind == 'O':
|
|
449
|
+
raise RuntimeError(
|
|
450
|
+
f'JSON ndarray encoding does not support object dtype={dtype_label}; cast to a '
|
|
451
|
+
'concrete numeric dtype or encode elements explicitly as plain JSON'
|
|
452
|
+
)
|
|
453
|
+
elif dtype.kind == 'S':
|
|
454
|
+
raise RuntimeError(
|
|
455
|
+
f'JSON ndarray encoding does not support byte-string dtype={dtype_label}; '
|
|
456
|
+
'decode elements and return a plain JSON list explicitly'
|
|
457
|
+
)
|
|
458
|
+
elif dtype.kind == 'U':
|
|
459
|
+
raise RuntimeError(
|
|
460
|
+
f'JSON ndarray encoding does not support unicode dtype={dtype_label}; convert '
|
|
461
|
+
'explicitly to plain JSON strings with .tolist()'
|
|
462
|
+
)
|
|
463
|
+
elif dtype.kind == 'c':
|
|
464
|
+
raise RuntimeError(
|
|
465
|
+
f'JSON ndarray encoding does not support complex dtype={dtype_label}; encode '
|
|
466
|
+
'.real and .imag arrays explicitly'
|
|
467
|
+
)
|
|
468
|
+
elif dtype.kind == 'f' and dtype.itemsize > 8:
|
|
469
|
+
raise RuntimeError(
|
|
470
|
+
f'JSON ndarray encoding does not support float dtype={dtype_label} wider than '
|
|
471
|
+
"64 bits; cast explicitly (e.g. .astype('float64')) or use Arrow encoding"
|
|
472
|
+
)
|
|
473
|
+
elif dtype.kind not in ('b', 'f'):
|
|
474
|
+
raise RuntimeError(
|
|
475
|
+
f'JSON ndarray encoding does not support dtype={dtype_label}; cast to bool, '
|
|
476
|
+
'integer, or float dtype, or encode values explicitly as plain JSON'
|
|
477
|
+
)
|
|
478
|
+
|
|
401
479
|
try:
|
|
402
480
|
data = obj.tolist()
|
|
403
481
|
except Exception as exc:
|
|
@@ -408,6 +486,7 @@ def serialize_ndarray_json(obj):
|
|
|
408
486
|
'encoding': 'json',
|
|
409
487
|
'data': data,
|
|
410
488
|
'shape': getattr(obj, 'shape', None),
|
|
489
|
+
'dtype': dtype.name,
|
|
411
490
|
}
|
|
412
491
|
|
|
413
492
|
|
|
@@ -442,11 +521,42 @@ def serialize_dataframe(obj, *, force_json_markers):
|
|
|
442
521
|
|
|
443
522
|
|
|
444
523
|
def serialize_dataframe_json(obj):
|
|
445
|
-
"""JSON fallback for DataFrame
|
|
524
|
+
"""JSON fallback for DataFrame values that JavaScript can represent safely."""
|
|
525
|
+
import pandas as pd # type: ignore
|
|
526
|
+
|
|
527
|
+
_validate_pandas_json_index(obj.index, pd, 'DataFrame')
|
|
528
|
+
json_columns = [_json_object_key(column) for column in obj.columns]
|
|
529
|
+
supported_json_columns = [column for column in json_columns if column is not None]
|
|
530
|
+
if not obj.columns.is_unique or len(set(supported_json_columns)) != len(
|
|
531
|
+
supported_json_columns
|
|
532
|
+
):
|
|
533
|
+
raise RuntimeError(
|
|
534
|
+
'JSON pandas.DataFrame encoding requires column labels to remain unique after '
|
|
535
|
+
'JSON object-key coercion; rename columns or make them distinct before applying '
|
|
536
|
+
'.columns.astype(str)'
|
|
537
|
+
)
|
|
538
|
+
for column, dtype in obj.dtypes.items():
|
|
539
|
+
if isinstance(dtype, pd.CategoricalDtype):
|
|
540
|
+
raise RuntimeError(
|
|
541
|
+
f'JSON pandas.DataFrame encoding does not support categorical dtype in '
|
|
542
|
+
f'column {column!r}; use Arrow encoding or convert explicitly '
|
|
543
|
+
"(e.g. .astype(str))"
|
|
544
|
+
)
|
|
446
545
|
try:
|
|
447
|
-
data =
|
|
546
|
+
data = (
|
|
547
|
+
[{} for _ in range(len(obj.index))]
|
|
548
|
+
if len(obj.columns) == 0
|
|
549
|
+
else obj.to_dict(orient='records')
|
|
550
|
+
)
|
|
448
551
|
except Exception as exc:
|
|
449
552
|
raise RuntimeError('JSON fallback failed for pandas.DataFrame') from exc
|
|
553
|
+
for row_number, row in enumerate(data):
|
|
554
|
+
for column, value in row.items():
|
|
555
|
+
row[column] = _normalize_pandas_json_scalar(
|
|
556
|
+
value,
|
|
557
|
+
f'DataFrame cell at row {row_number}, column {column!r}',
|
|
558
|
+
pd,
|
|
559
|
+
)
|
|
450
560
|
return {
|
|
451
561
|
'__tywrap__': 'dataframe',
|
|
452
562
|
'codecVersion': CODEC_VERSION,
|
|
@@ -488,14 +598,23 @@ def serialize_series(obj, *, force_json_markers):
|
|
|
488
598
|
|
|
489
599
|
|
|
490
600
|
def serialize_series_json(obj):
|
|
491
|
-
"""JSON fallback for Series
|
|
601
|
+
"""JSON fallback for Series values that JavaScript can represent safely."""
|
|
602
|
+
import pandas as pd # type: ignore
|
|
603
|
+
|
|
604
|
+
_validate_pandas_json_index(obj.index, pd, 'Series')
|
|
605
|
+
if isinstance(obj.dtype, pd.CategoricalDtype):
|
|
606
|
+
raise RuntimeError(
|
|
607
|
+
'JSON pandas.Series encoding does not support categorical dtype; use Arrow '
|
|
608
|
+
"encoding or convert explicitly (e.g. .astype(str))"
|
|
609
|
+
)
|
|
492
610
|
try:
|
|
493
611
|
data = obj.to_list() # type: ignore
|
|
494
|
-
except Exception:
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
612
|
+
except Exception as exc:
|
|
613
|
+
raise RuntimeError('JSON fallback failed for pandas.Series') from exc
|
|
614
|
+
data = [
|
|
615
|
+
_normalize_pandas_json_scalar(value, f'Series value at position {position}', pd)
|
|
616
|
+
for position, value in enumerate(data)
|
|
617
|
+
]
|
|
499
618
|
return {
|
|
500
619
|
'__tywrap__': 'series',
|
|
501
620
|
'codecVersion': CODEC_VERSION,
|
|
@@ -505,6 +624,70 @@ def serialize_series_json(obj):
|
|
|
505
624
|
}
|
|
506
625
|
|
|
507
626
|
|
|
627
|
+
def _validate_pandas_json_index(index, pd, container_name):
|
|
628
|
+
"""Reject index metadata that records/list-oriented JSON would discard."""
|
|
629
|
+
if isinstance(index, pd.MultiIndex):
|
|
630
|
+
raise RuntimeError(
|
|
631
|
+
f'JSON pandas.{container_name} encoding does not support MultiIndex; use Arrow '
|
|
632
|
+
'encoding or flatten the index explicitly with .reset_index()'
|
|
633
|
+
)
|
|
634
|
+
if not (
|
|
635
|
+
isinstance(index, pd.RangeIndex)
|
|
636
|
+
and index.start == 0
|
|
637
|
+
and index.step == 1
|
|
638
|
+
and index.stop == len(index)
|
|
639
|
+
and index.name is None
|
|
640
|
+
):
|
|
641
|
+
raise RuntimeError(
|
|
642
|
+
f'JSON pandas.{container_name} encoding requires an unnamed RangeIndex starting '
|
|
643
|
+
'at 0 with step 1; use Arrow encoding or normalize explicitly with '
|
|
644
|
+
'.reset_index(drop=True)'
|
|
645
|
+
)
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
def _json_object_key(value):
|
|
649
|
+
"""Return json.dumps' object-key spelling, or None for unsupported keys."""
|
|
650
|
+
try:
|
|
651
|
+
encoded = json.dumps({value: None})
|
|
652
|
+
except (TypeError, ValueError):
|
|
653
|
+
return None
|
|
654
|
+
return next(iter(json.loads(encoded)))
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def _normalize_pandas_json_scalar(value, location, pd):
|
|
658
|
+
"""Normalize pandas nulls and reject values outside the plain JSON domain."""
|
|
659
|
+
np = sys.modules.get('numpy')
|
|
660
|
+
if np is not None and isinstance(value, np.generic):
|
|
661
|
+
value = value.item()
|
|
662
|
+
if value is None or value is pd.NA or value is pd.NaT:
|
|
663
|
+
return None
|
|
664
|
+
if type(value) is bool:
|
|
665
|
+
return value
|
|
666
|
+
if type(value) is int:
|
|
667
|
+
if value < -JS_SAFE_INTEGER_MAX or value > JS_SAFE_INTEGER_MAX:
|
|
668
|
+
raise RuntimeError(
|
|
669
|
+
f'JSON pandas encoding cannot safely represent {location} integer values '
|
|
670
|
+
'outside the JavaScript safe integer range; use Arrow encoding or '
|
|
671
|
+
"cast/encode explicitly (e.g. .astype('float64') or str)"
|
|
672
|
+
)
|
|
673
|
+
return value
|
|
674
|
+
if type(value) is float:
|
|
675
|
+
if not math.isfinite(value):
|
|
676
|
+
raise RuntimeError(
|
|
677
|
+
f'JSON pandas encoding cannot represent non-finite {location} float values '
|
|
678
|
+
'(NaN or Infinity); use .fillna(...) for intentional missing values or '
|
|
679
|
+
'Arrow encoding'
|
|
680
|
+
)
|
|
681
|
+
return value
|
|
682
|
+
if type(value) is str:
|
|
683
|
+
return value
|
|
684
|
+
raise RuntimeError(
|
|
685
|
+
f'JSON pandas encoding does not support {location} value of type '
|
|
686
|
+
f'{type(value).__name__}; use Arrow encoding or convert explicitly '
|
|
687
|
+
'(e.g. .astype(str))'
|
|
688
|
+
)
|
|
689
|
+
|
|
690
|
+
|
|
508
691
|
def serialize_sparse_matrix(obj):
|
|
509
692
|
"""
|
|
510
693
|
Serialize scipy sparse matrices into structured JSON envelopes (json-only;
|
|
@@ -575,12 +758,14 @@ def serialize_torch_tensor(obj, *, force_json_markers, torch_allow_copy=False):
|
|
|
575
758
|
Rejection order is significant: the categorical rejections (sparse / quantized
|
|
576
759
|
/ meta / complex) are checked BEFORE the device/contiguous opt-in branch so
|
|
577
760
|
they fail with a clear, specific message and are NOT bypassable by
|
|
578
|
-
TYWRAP_TORCH_ALLOW_COPY. The opt-in only governs the
|
|
579
|
-
|
|
761
|
+
TYWRAP_TORCH_ALLOW_COPY. The opt-in only governs the device transfer and
|
|
762
|
+
contiguous copy, never an unrepresentable layout/dtype.
|
|
580
763
|
"""
|
|
581
764
|
import torch # already importable: is_torch_tensor() gated the dispatch
|
|
582
765
|
|
|
583
766
|
tensor = obj.detach()
|
|
767
|
+
source_device = None
|
|
768
|
+
source_dtype = None
|
|
584
769
|
|
|
585
770
|
# Sparse tensors (COO/CSR/CSC/BSR/BSC -> any non-strided layout) have no dense
|
|
586
771
|
# numpy representation without a densify step, which is not the round-trip this
|
|
@@ -628,6 +813,7 @@ def serialize_torch_tensor(obj, *, force_json_markers, torch_allow_copy=False):
|
|
|
628
813
|
raise RuntimeError(
|
|
629
814
|
'Torch tensor is on a non-CPU device; set TYWRAP_TORCH_ALLOW_COPY=1 to allow CPU transfer'
|
|
630
815
|
)
|
|
816
|
+
source_device = str(tensor.device)
|
|
631
817
|
tensor = tensor.to('cpu')
|
|
632
818
|
if hasattr(tensor, 'is_contiguous') and not tensor.is_contiguous():
|
|
633
819
|
if not torch_allow_copy:
|
|
@@ -635,12 +821,15 @@ def serialize_torch_tensor(obj, *, force_json_markers, torch_allow_copy=False):
|
|
|
635
821
|
'Torch tensor is not contiguous; set TYWRAP_TORCH_ALLOW_COPY=1 to allow contiguous copy'
|
|
636
822
|
)
|
|
637
823
|
tensor = tensor.contiguous()
|
|
824
|
+
if tensor.dtype == torch.bfloat16:
|
|
825
|
+
source_dtype = str(tensor.dtype)
|
|
826
|
+
tensor = tensor.float()
|
|
638
827
|
try:
|
|
639
828
|
arr = tensor.numpy()
|
|
640
829
|
except Exception as exc:
|
|
641
830
|
raise RuntimeError('Failed to convert torch.Tensor to numpy') from exc
|
|
642
831
|
|
|
643
|
-
|
|
832
|
+
envelope = {
|
|
644
833
|
'__tywrap__': 'torch.tensor',
|
|
645
834
|
'codecVersion': CODEC_VERSION,
|
|
646
835
|
'encoding': 'ndarray',
|
|
@@ -649,6 +838,11 @@ def serialize_torch_tensor(obj, *, force_json_markers, torch_allow_copy=False):
|
|
|
649
838
|
'dtype': str(tensor.dtype),
|
|
650
839
|
'device': str(tensor.device),
|
|
651
840
|
}
|
|
841
|
+
if source_dtype is not None:
|
|
842
|
+
envelope['sourceDtype'] = source_dtype
|
|
843
|
+
if source_device is not None:
|
|
844
|
+
envelope['sourceDevice'] = source_device
|
|
845
|
+
return envelope
|
|
652
846
|
|
|
653
847
|
|
|
654
848
|
def serialize_sklearn_estimator(obj):
|
|
@@ -727,34 +921,48 @@ def serialize_stdlib(obj):
|
|
|
727
921
|
return None
|
|
728
922
|
|
|
729
923
|
|
|
730
|
-
|
|
731
|
-
"""
|
|
732
|
-
Top-level result serializer.
|
|
924
|
+
_NO_SCIENTIFIC = object()
|
|
733
925
|
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
926
|
+
|
|
927
|
+
def _check_serialize_depth(depth, path):
|
|
928
|
+
if depth > MAX_SERIALIZE_DEPTH:
|
|
929
|
+
raise RuntimeError(
|
|
930
|
+
f'Scientific envelope serialization maximum depth '
|
|
931
|
+
f'{MAX_SERIALIZE_DEPTH} exceeded at {path}'
|
|
932
|
+
)
|
|
933
|
+
|
|
934
|
+
|
|
935
|
+
def _check_serialize_nodes(nodes, path):
|
|
936
|
+
if nodes > MAX_SERIALIZE_NODES:
|
|
937
|
+
raise RuntimeError(
|
|
938
|
+
f'Scientific envelope serialization maximum visited nodes '
|
|
939
|
+
f'{MAX_SERIALIZE_NODES} exceeded at {path}'
|
|
940
|
+
)
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
def _serialize_scientific(obj, *, force_json_markers, torch_allow_copy, depth, path):
|
|
944
|
+
"""Serialize a supported scientific value, or return _NO_SCIENTIFIC."""
|
|
743
945
|
package = type(obj).__module__.split('.', 1)[0]
|
|
744
946
|
|
|
745
947
|
if package == 'numpy' and 'numpy' in sys.modules:
|
|
746
948
|
if is_numpy_array(obj):
|
|
949
|
+
_check_serialize_depth(depth, path)
|
|
747
950
|
return serialize_ndarray(obj, force_json_markers=force_json_markers)
|
|
748
951
|
elif package == 'pandas' and 'pandas' in sys.modules:
|
|
749
952
|
if is_pandas_dataframe(obj):
|
|
953
|
+
_check_serialize_depth(depth, path)
|
|
750
954
|
return serialize_dataframe(obj, force_json_markers=force_json_markers)
|
|
751
955
|
if is_pandas_series(obj):
|
|
956
|
+
_check_serialize_depth(depth, path)
|
|
752
957
|
return serialize_series(obj, force_json_markers=force_json_markers)
|
|
753
958
|
elif package == 'scipy' and 'scipy.sparse' in sys.modules:
|
|
754
959
|
if is_scipy_sparse(obj):
|
|
960
|
+
_check_serialize_depth(depth, path)
|
|
755
961
|
return serialize_sparse_matrix(obj)
|
|
756
962
|
elif package == 'torch' and 'torch' in sys.modules:
|
|
757
963
|
if is_torch_tensor(obj):
|
|
964
|
+
_check_serialize_depth(depth, path)
|
|
965
|
+
_check_serialize_depth(depth + 1, _serialize_path(path, 'value'))
|
|
758
966
|
return serialize_torch_tensor(
|
|
759
967
|
obj, force_json_markers=force_json_markers, torch_allow_copy=torch_allow_copy
|
|
760
968
|
)
|
|
@@ -763,18 +971,168 @@ def serialize(obj, *, force_json_markers, torch_allow_copy=False):
|
|
|
763
971
|
# BaseEstimator is sklearn's documented extension point, so user-defined
|
|
764
972
|
# estimators live outside the 'sklearn' package and must still get the
|
|
765
973
|
# estimator serializer (and its param-naming errors).
|
|
974
|
+
_check_serialize_depth(depth, path)
|
|
766
975
|
return serialize_sklearn_estimator(obj)
|
|
767
976
|
|
|
768
|
-
|
|
769
|
-
|
|
977
|
+
return _NO_SCIENTIFIC
|
|
978
|
+
|
|
979
|
+
|
|
980
|
+
def _serialize_path(base, key):
|
|
981
|
+
"""Build a decoder-compatible JSONPath-like result path."""
|
|
982
|
+
if isinstance(key, int):
|
|
983
|
+
return f'{base}[{key}]'
|
|
984
|
+
if _SERIALIZE_PATH_IDENTIFIER.fullmatch(key):
|
|
985
|
+
return f'{base}.{key}'
|
|
986
|
+
return f'{base}[{json.dumps(key, ensure_ascii=False)}]'
|
|
987
|
+
|
|
988
|
+
|
|
989
|
+
def _invalid_key_path(base, key):
|
|
990
|
+
"""Name a dict key that cannot be represented by JSON."""
|
|
991
|
+
return f'{base}[{key!r}]'
|
|
770
992
|
|
|
771
|
-
|
|
993
|
+
|
|
994
|
+
def _needs_serialize_visit(value):
|
|
995
|
+
"""Return whether value needs container or scientific traversal work."""
|
|
996
|
+
if type(value) in (type(None), bool, int, float, str):
|
|
997
|
+
return False
|
|
998
|
+
if type(value) in (dict, list, tuple):
|
|
999
|
+
return True
|
|
1000
|
+
package = type(value).__module__.split('.', 1)[0]
|
|
1001
|
+
if package in ('numpy', 'pandas', 'scipy', 'torch'):
|
|
1002
|
+
return True
|
|
1003
|
+
return 'sklearn.base' in sys.modules and is_sklearn_estimator(value)
|
|
1004
|
+
|
|
1005
|
+
|
|
1006
|
+
def _serialize_leaf(value):
|
|
1007
|
+
"""Apply non-container conversions without allocating a traversal frame."""
|
|
1008
|
+
if type(value) in (type(None), bool, int, float, str):
|
|
1009
|
+
return value
|
|
1010
|
+
pydantic_value = serialize_pydantic(value)
|
|
772
1011
|
if pydantic_value is not _NO_PYDANTIC:
|
|
773
1012
|
return pydantic_value
|
|
774
|
-
stdlib_value = serialize_stdlib(
|
|
1013
|
+
stdlib_value = serialize_stdlib(value)
|
|
775
1014
|
if stdlib_value is not None:
|
|
776
1015
|
return stdlib_value
|
|
777
|
-
return
|
|
1016
|
+
return value
|
|
1017
|
+
|
|
1018
|
+
|
|
1019
|
+
def serialize(obj, *, force_json_markers, torch_allow_copy=False):
|
|
1020
|
+
"""
|
|
1021
|
+
Top-level result serializer.
|
|
1022
|
+
|
|
1023
|
+
Scientific codecs are type-first and only inspect packages that the value can
|
|
1024
|
+
belong to. A value from an optional package implies that package is already in
|
|
1025
|
+
sys.modules, so these checks never cold-import the scientific stack. The
|
|
1026
|
+
package dispatch deliberately precedes the JSON-native fast path: e.g. a
|
|
1027
|
+
package-defined subclass of dict still receives its relevant codec check.
|
|
1028
|
+
Every other value is left untouched so the shared JSON encoder applies the
|
|
1029
|
+
exact same default conversion at the root and at any nested depth.
|
|
1030
|
+
"""
|
|
1031
|
+
root = [None]
|
|
1032
|
+
active_ids = set()
|
|
1033
|
+
stack = [('visit', obj, 0, 'result', root, 0)]
|
|
1034
|
+
visited_nodes = 0
|
|
1035
|
+
|
|
1036
|
+
# Repeated aliases have value semantics and are intentionally serialized twice.
|
|
1037
|
+
while stack:
|
|
1038
|
+
frame = stack.pop()
|
|
1039
|
+
action = frame[0]
|
|
1040
|
+
if action == 'dict':
|
|
1041
|
+
_, current, depth, path, parent, key, output, iterator = frame
|
|
1042
|
+
try:
|
|
1043
|
+
item_key, item = next(iterator)
|
|
1044
|
+
except StopIteration:
|
|
1045
|
+
active_ids.remove(id(current))
|
|
1046
|
+
parent[key] = output
|
|
1047
|
+
continue
|
|
1048
|
+
stack.append(frame)
|
|
1049
|
+
if not (isinstance(item_key, (str, int, float, bool)) or item_key is None):
|
|
1050
|
+
invalid_path = _invalid_key_path(path, item_key)
|
|
1051
|
+
raise TypeError(
|
|
1052
|
+
f'keys must be str, int, float, bool or None, not '
|
|
1053
|
+
f'{type(item_key).__name__} at {invalid_path}'
|
|
1054
|
+
)
|
|
1055
|
+
child_key = _json_object_key(item_key)
|
|
1056
|
+
child_path = _serialize_path(path, child_key)
|
|
1057
|
+
if _needs_serialize_visit(item):
|
|
1058
|
+
stack.append(('visit', item, depth + 1, child_path, output, item_key))
|
|
1059
|
+
else:
|
|
1060
|
+
output[item_key] = _serialize_leaf(item)
|
|
1061
|
+
continue
|
|
1062
|
+
if action == 'sequence':
|
|
1063
|
+
_, current, depth, path, parent, key, output, index = frame
|
|
1064
|
+
if index == len(output):
|
|
1065
|
+
active_ids.remove(id(current))
|
|
1066
|
+
parent[key] = output if type(current) is list else tuple(output)
|
|
1067
|
+
continue
|
|
1068
|
+
stack.append(('sequence', current, depth, path, parent, key, output, index + 1))
|
|
1069
|
+
item = current[index]
|
|
1070
|
+
if _needs_serialize_visit(item):
|
|
1071
|
+
stack.append(
|
|
1072
|
+
('visit', item, depth + 1, _serialize_path(path, index), output, index)
|
|
1073
|
+
)
|
|
1074
|
+
else:
|
|
1075
|
+
output[index] = _serialize_leaf(item)
|
|
1076
|
+
continue
|
|
1077
|
+
|
|
1078
|
+
_, current, depth, path, parent, key = frame
|
|
1079
|
+
try:
|
|
1080
|
+
scientific = _serialize_scientific(
|
|
1081
|
+
current,
|
|
1082
|
+
force_json_markers=force_json_markers,
|
|
1083
|
+
torch_allow_copy=torch_allow_copy,
|
|
1084
|
+
depth=depth,
|
|
1085
|
+
path=path,
|
|
1086
|
+
)
|
|
1087
|
+
except Exception as exc:
|
|
1088
|
+
if path == 'result':
|
|
1089
|
+
raise
|
|
1090
|
+
raise RuntimeError(f'Scientific value serialization failed at {path}: {exc}') from exc
|
|
1091
|
+
if scientific is not _NO_SCIENTIFIC:
|
|
1092
|
+
# Recognized envelopes are terminal containers to the JS decoder.
|
|
1093
|
+
visited_nodes += 1
|
|
1094
|
+
_check_serialize_nodes(visited_nodes, path)
|
|
1095
|
+
if scientific.get('__tywrap__') == 'torch.tensor':
|
|
1096
|
+
nested_path = _serialize_path(path, 'value')
|
|
1097
|
+
try:
|
|
1098
|
+
visited_nodes += 1
|
|
1099
|
+
_check_serialize_nodes(visited_nodes, nested_path)
|
|
1100
|
+
except Exception as exc:
|
|
1101
|
+
if path == 'result':
|
|
1102
|
+
raise
|
|
1103
|
+
raise RuntimeError(
|
|
1104
|
+
f'Scientific value serialization failed at {path}: {exc}'
|
|
1105
|
+
) from exc
|
|
1106
|
+
parent[key] = scientific
|
|
1107
|
+
continue
|
|
1108
|
+
|
|
1109
|
+
container_type = type(current)
|
|
1110
|
+
if container_type in (dict, list, tuple):
|
|
1111
|
+
_check_serialize_depth(depth, path)
|
|
1112
|
+
visited_nodes += 1
|
|
1113
|
+
_check_serialize_nodes(visited_nodes, path)
|
|
1114
|
+
current_id = id(current)
|
|
1115
|
+
if current_id in active_ids:
|
|
1116
|
+
raise RuntimeError(f'Circular reference detected at {path}')
|
|
1117
|
+
active_ids.add(current_id)
|
|
1118
|
+
|
|
1119
|
+
if container_type is dict:
|
|
1120
|
+
output = {}
|
|
1121
|
+
parent[key] = output
|
|
1122
|
+
stack.append(
|
|
1123
|
+
('dict', current, depth, path, parent, key, output, iter(current.items()))
|
|
1124
|
+
)
|
|
1125
|
+
continue
|
|
1126
|
+
|
|
1127
|
+
output = [None] * len(current)
|
|
1128
|
+
if container_type is list:
|
|
1129
|
+
parent[key] = output
|
|
1130
|
+
stack.append(('sequence', current, depth, path, parent, key, output, 0))
|
|
1131
|
+
continue
|
|
1132
|
+
|
|
1133
|
+
parent[key] = _serialize_leaf(current)
|
|
1134
|
+
|
|
1135
|
+
return root[0]
|
|
778
1136
|
|
|
779
1137
|
|
|
780
1138
|
# =============================================================================
|
|
@@ -839,20 +1197,9 @@ def make_default_encoder(*, allow_nan):
|
|
|
839
1197
|
if isinstance(obj, pd.Timedelta):
|
|
840
1198
|
return obj.total_seconds()
|
|
841
1199
|
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
return obj.isoformat()
|
|
846
|
-
if isinstance(obj, dt.time):
|
|
847
|
-
return obj.isoformat()
|
|
848
|
-
if isinstance(obj, dt.timedelta):
|
|
849
|
-
return obj.total_seconds()
|
|
850
|
-
if isinstance(obj, decimal.Decimal):
|
|
851
|
-
return str(obj)
|
|
852
|
-
if isinstance(obj, uuid.UUID):
|
|
853
|
-
return str(obj)
|
|
854
|
-
if isinstance(obj, (Path, PurePath)):
|
|
855
|
-
return str(obj)
|
|
1200
|
+
stdlib_value = serialize_stdlib(obj)
|
|
1201
|
+
if stdlib_value is not None:
|
|
1202
|
+
return stdlib_value
|
|
856
1203
|
|
|
857
1204
|
if isinstance(obj, (bytes, bytearray)):
|
|
858
1205
|
return {
|
|
@@ -896,12 +1243,10 @@ def encode_value(value, *, allow_nan):
|
|
|
896
1243
|
# ("...not JSON compliant: nan"), but 3.10/3.11 emit only the canonical
|
|
897
1244
|
# "Out of range float values are not JSON compliant". Match that phrase too
|
|
898
1245
|
# so the typed error message is stable across versions.
|
|
1246
|
+
nonfinite_token = re.search(r'(^|[^a-z])(nan|inf|infinity)([^a-z]|$)', error_msg)
|
|
899
1247
|
if (
|
|
900
|
-
'
|
|
901
|
-
|
|
902
|
-
or 'inf' in error_msg
|
|
903
|
-
or 'out of range float' in error_msg
|
|
904
|
-
):
|
|
1248
|
+
not allow_nan and 'out of range float values are not json compliant' in error_msg
|
|
1249
|
+
) or nonfinite_token:
|
|
905
1250
|
raise CodecError('Cannot serialize NaN - NaN/Infinity not allowed in JSON') from exc
|
|
906
1251
|
raise CodecError(f'JSON encoding failed: {exc}') from exc
|
|
907
1252
|
except TypeError as exc:
|
package/src/core/generator.ts
CHANGED
|
@@ -512,6 +512,21 @@ export class CodeGenerator {
|
|
|
512
512
|
if (leaf === 'ndarray' || leaf === 'NDArray' || full.includes('numpy')) {
|
|
513
513
|
return { kind: 'marker', marker: 'ndarray' };
|
|
514
514
|
}
|
|
515
|
+
if (
|
|
516
|
+
['csr_matrix', 'csc_matrix', 'coo_matrix', 'spmatrix'].includes(leaf) &&
|
|
517
|
+
(!current.module || current.module.startsWith('scipy'))
|
|
518
|
+
) {
|
|
519
|
+
return { kind: 'marker', marker: 'scipy.sparse' };
|
|
520
|
+
}
|
|
521
|
+
if (leaf === 'Tensor' && (!current.module || current.module.startsWith('torch'))) {
|
|
522
|
+
return { kind: 'marker', marker: 'torch.tensor' };
|
|
523
|
+
}
|
|
524
|
+
if (
|
|
525
|
+
leaf === 'BaseEstimator' &&
|
|
526
|
+
(!current.module || current.module.startsWith('sklearn'))
|
|
527
|
+
) {
|
|
528
|
+
return { kind: 'marker', marker: 'sklearn.estimator' };
|
|
529
|
+
}
|
|
515
530
|
return definitions.has(leaf) ? { kind: 'ref', name: leaf } : { kind: 'any' };
|
|
516
531
|
}
|
|
517
532
|
case 'typevar':
|