nbxtensor 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nbxtensor/THIRD_PARTY_NOTICES.md +287 -0
- nbxtensor/__init__.py +17 -0
- nbxtensor/backend/__init__.py +1 -0
- nbxtensor/backend/capabilities.py +207 -0
- nbxtensor/backend/cpu/__init__.py +1 -0
- nbxtensor/backend/cpu/blas.py +102 -0
- nbxtensor/backend/cpu/pool.py +204 -0
- nbxtensor/backend/cpu_backend.py +162 -0
- nbxtensor/backend/device_facts.py +105 -0
- nbxtensor/backend/device_files.py +129 -0
- nbxtensor/backend/device_names.py +18 -0
- nbxtensor/backend/devices/amd/cdna.yml +228 -0
- nbxtensor/backend/devices/amd/cdna2.yml +228 -0
- nbxtensor/backend/devices/amd/cdna3.yml +227 -0
- nbxtensor/backend/devices/apple/apple_m1.yml +266 -0
- nbxtensor/backend/devices/apple/apple_m1_max.yml +266 -0
- nbxtensor/backend/devices/apple/apple_m1_pro.yml +266 -0
- nbxtensor/backend/devices/apple/apple_m1_ultra.yml +266 -0
- nbxtensor/backend/devices/apple/apple_m2.yml +266 -0
- nbxtensor/backend/devices/apple/apple_m2_max.yml +266 -0
- nbxtensor/backend/devices/apple/apple_m2_pro.yml +266 -0
- nbxtensor/backend/devices/apple/apple_m2_ultra.yml +266 -0
- nbxtensor/backend/devices/apple/apple_m3.yml +266 -0
- nbxtensor/backend/devices/apple/apple_m3_max.yml +266 -0
- nbxtensor/backend/devices/apple/apple_m3_pro.yml +266 -0
- nbxtensor/backend/devices/apple/apple_m3_ultra.yml +266 -0
- nbxtensor/backend/devices/apple/apple_m4.yml +266 -0
- nbxtensor/backend/devices/apple/apple_m4_max.yml +266 -0
- nbxtensor/backend/devices/apple/apple_m4_pro.yml +308 -0
- nbxtensor/backend/devices/apple/apple_m5.yml +269 -0
- nbxtensor/backend/devices/apple/apple_m5_max.yml +269 -0
- nbxtensor/backend/devices/apple/apple_m5_pro.yml +269 -0
- nbxtensor/backend/devices/apple/apple_m5_ultra.yml +269 -0
- nbxtensor/backend/devices/apple/apple_m6.yml +269 -0
- nbxtensor/backend/devices/apple/apple_silicon.yml +263 -0
- nbxtensor/backend/devices/nvidia/ampere.yml +214 -0
- nbxtensor/backend/devices/nvidia/hopper.yml +213 -0
- nbxtensor/backend/devices/nvidia/volta.yml +411 -0
- nbxtensor/backend/faults.py +151 -0
- nbxtensor/backend/host_memory.py +185 -0
- nbxtensor/backend/launcher.py +2350 -0
- nbxtensor/backend/launcher_contract.py +429 -0
- nbxtensor/backend/metal_backend.py +661 -0
- nbxtensor/backend/metal_device.py +1274 -0
- nbxtensor/backend/process_memory.py +73 -0
- nbxtensor/backend/replay_graph.py +322 -0
- nbxtensor/backend/runtime.py +409 -0
- nbxtensor/backend/triton_ext_driver.py +735 -0
- nbxtensor/dtype/__init__.py +1 -0
- nbxtensor/dtype/convert.py +107 -0
- nbxtensor/dtype/dtypes.yml +70 -0
- nbxtensor/dtype/itemsize.py +166 -0
- nbxtensor/dtype/nbx_dtype.py +72 -0
- nbxtensor/dtype/operands.py +158 -0
- nbxtensor/dtype/outputs.py +64 -0
- nbxtensor/dtype/policy.py +155 -0
- nbxtensor/dtype/policy.yml +51 -0
- nbxtensor/dtype/promotion.py +112 -0
- nbxtensor/host.py +55 -0
- nbxtensor/io/__init__.py +1 -0
- nbxtensor/io/read_rate.py +197 -0
- nbxtensor/io/safetensors.py +314 -0
- nbxtensor/memory/__init__.py +9 -0
- nbxtensor/memory/allocator.py +1988 -0
- nbxtensor/memory/arena.py +75 -0
- nbxtensor/memory/device_memory.py +150 -0
- nbxtensor/memory/device_transfer.py +154 -0
- nbxtensor/memory/layout.py +43 -0
- nbxtensor/memory/memory_pool.py +191 -0
- nbxtensor/memory/metal.py +96 -0
- nbxtensor/memory/oom_advice.py +91 -0
- nbxtensor/memory/pointer_tables.py +323 -0
- nbxtensor/memory/pool.py +507 -0
- nbxtensor/memory/shadow.py +43 -0
- nbxtensor/memory/slab.py +273 -0
- nbxtensor/memory/trace_replay.py +122 -0
- nbxtensor/memory/traces/sana-video-2b-720p.16g.2026-11-02.tsv +13610 -0
- nbxtensor/memory/vmm.py +118 -0
- nbxtensor/ops/__init__.py +1 -0
- nbxtensor/ops/_autotune_policy.py +90 -0
- nbxtensor/ops/_common.py +32 -0
- nbxtensor/ops/_configs.py +1041 -0
- nbxtensor/ops/_cost.py +231 -0
- nbxtensor/ops/abs.py +17 -0
- nbxtensor/ops/adaptive_avg_pool2d.py +161 -0
- nbxtensor/ops/add.py +96 -0
- nbxtensor/ops/addcdiv.py +29 -0
- nbxtensor/ops/addcmul.py +29 -0
- nbxtensor/ops/addmv_op.py +64 -0
- nbxtensor/ops/addr.py +68 -0
- nbxtensor/ops/all_reduce.py +83 -0
- nbxtensor/ops/amax.py +40 -0
- nbxtensor/ops/any_reduce.py +83 -0
- nbxtensor/ops/arange_op.py +27 -0
- nbxtensor/ops/argmax.py +107 -0
- nbxtensor/ops/argmin.py +93 -0
- nbxtensor/ops/atan2.py +71 -0
- nbxtensor/ops/attention_wrappers.py +1648 -0
- nbxtensor/ops/autotune_bucket.py +137 -0
- nbxtensor/ops/autotune_cache.py +430 -0
- nbxtensor/ops/autotune_certified.py +1262 -0
- nbxtensor/ops/autotune_refusals.py +380 -0
- nbxtensor/ops/autotune_settings.py +58 -0
- nbxtensor/ops/avg_pool2d.py +73 -0
- nbxtensor/ops/baddbmm_op.py +132 -0
- nbxtensor/ops/batch_norm.py +163 -0
- nbxtensor/ops/binary_strided.py +196 -0
- nbxtensor/ops/bitwise_and.py +26 -0
- nbxtensor/ops/bitwise_left_shift.py +47 -0
- nbxtensor/ops/bitwise_not.py +24 -0
- nbxtensor/ops/bitwise_or.py +26 -0
- nbxtensor/ops/bitwise_right_shift.py +47 -0
- nbxtensor/ops/bucketize.py +37 -0
- nbxtensor/ops/cat_op.py +87 -0
- nbxtensor/ops/celu.py +20 -0
- nbxtensor/ops/clamp.py +24 -0
- nbxtensor/ops/clamp_min.py +22 -0
- nbxtensor/ops/conv2d.py +265 -0
- nbxtensor/ops/conv2d_m8n8k4.py +165 -0
- nbxtensor/ops/conv_depthwise2d.py +154 -0
- nbxtensor/ops/conv_transpose2d.py +148 -0
- nbxtensor/ops/conv_wrappers.py +1482 -0
- nbxtensor/ops/copy.py +18 -0
- nbxtensor/ops/copy_op.py +39 -0
- nbxtensor/ops/cos.py +17 -0
- nbxtensor/ops/count_nonzero.py +64 -0
- nbxtensor/ops/cross_entropy.py +67 -0
- nbxtensor/ops/cummax.py +109 -0
- nbxtensor/ops/cummin.py +105 -0
- nbxtensor/ops/cumsum.py +244 -0
- nbxtensor/ops/decode_attn_vec.py +162 -0
- nbxtensor/ops/depthwise_conv2d.py +132 -0
- nbxtensor/ops/dequant_gemv.py +190 -0
- nbxtensor/ops/dft.py +67 -0
- nbxtensor/ops/dispatch.py +1228 -0
- nbxtensor/ops/div.py +75 -0
- nbxtensor/ops/dot_op.py +67 -0
- nbxtensor/ops/dropout.py +23 -0
- nbxtensor/ops/dtype_convert.py +42 -0
- nbxtensor/ops/elementwise_launch.py +81 -0
- nbxtensor/ops/elementwise_wrappers.py +2315 -0
- nbxtensor/ops/elu.py +20 -0
- nbxtensor/ops/embedding.py +69 -0
- nbxtensor/ops/eq.py +32 -0
- nbxtensor/ops/erf.py +18 -0
- nbxtensor/ops/exp.py +17 -0
- nbxtensor/ops/exp2.py +18 -0
- nbxtensor/ops/fft_op.py +125 -0
- nbxtensor/ops/fill_op.py +20 -0
- nbxtensor/ops/flash_attention.py +307 -0
- nbxtensor/ops/flash_attention_m8n8k4.py +384 -0
- nbxtensor/ops/flash_decode.py +78 -0
- nbxtensor/ops/flip_op.py +60 -0
- nbxtensor/ops/floor.py +90 -0
- nbxtensor/ops/full_op.py +24 -0
- nbxtensor/ops/fused_moe.py +417 -0
- nbxtensor/ops/fused_moe_m8n8k4.py +99 -0
- nbxtensor/ops/fused_upsample_conv.py +413 -0
- nbxtensor/ops/gather_op.py +52 -0
- nbxtensor/ops/ge.py +32 -0
- nbxtensor/ops/gelu.py +31 -0
- nbxtensor/ops/gemv_vec.py +80 -0
- nbxtensor/ops/glu.py +43 -0
- nbxtensor/ops/grid_sampler.py +145 -0
- nbxtensor/ops/group_norm_tiled.py +67 -0
- nbxtensor/ops/groupnorm.py +123 -0
- nbxtensor/ops/gt.py +32 -0
- nbxtensor/ops/hardsigmoid.py +18 -0
- nbxtensor/ops/hardswish.py +19 -0
- nbxtensor/ops/im2col.py +57 -0
- nbxtensor/ops/index_add.py +95 -0
- nbxtensor/ops/index_put_op.py +92 -0
- nbxtensor/ops/index_select.py +121 -0
- nbxtensor/ops/index_wrappers.py +770 -0
- nbxtensor/ops/isfinite.py +20 -0
- nbxtensor/ops/isinf.py +20 -0
- nbxtensor/ops/isnan.py +18 -0
- nbxtensor/ops/launch_keys.py +1054 -0
- nbxtensor/ops/launch_observer.py +54 -0
- nbxtensor/ops/layernorm.py +71 -0
- nbxtensor/ops/le.py +35 -0
- nbxtensor/ops/leaky_relu.py +19 -0
- nbxtensor/ops/lerp.py +76 -0
- nbxtensor/ops/linalg_wrappers.py +1370 -0
- nbxtensor/ops/linspace_op.py +47 -0
- nbxtensor/ops/log.py +17 -0
- nbxtensor/ops/log_sigmoid.py +23 -0
- nbxtensor/ops/logical_and.py +26 -0
- nbxtensor/ops/logical_not.py +24 -0
- nbxtensor/ops/logical_or.py +26 -0
- nbxtensor/ops/logical_xor.py +37 -0
- nbxtensor/ops/loss_wrappers.py +282 -0
- nbxtensor/ops/lt.py +32 -0
- nbxtensor/ops/masked_fill.py +21 -0
- nbxtensor/ops/matmul.py +377 -0
- nbxtensor/ops/matmul_m8n8k4.py +235 -0
- nbxtensor/ops/max_pool2d.py +74 -0
- nbxtensor/ops/max_pool3d.py +93 -0
- nbxtensor/ops/max_reduce.py +91 -0
- nbxtensor/ops/maximum.py +18 -0
- nbxtensor/ops/mean.py +40 -0
- nbxtensor/ops/min_reduce.py +91 -0
- nbxtensor/ops/minimum.py +18 -0
- nbxtensor/ops/mish.py +22 -0
- nbxtensor/ops/mma_m8n8k4.py +152 -0
- nbxtensor/ops/moe_align.py +195 -0
- nbxtensor/ops/moe_decode_vec.py +374 -0
- nbxtensor/ops/moe_device_residency.py +141 -0
- nbxtensor/ops/moe_wrappers.py +345 -0
- nbxtensor/ops/mse_loss.py +79 -0
- nbxtensor/ops/mul.py +51 -0
- nbxtensor/ops/nan_to_num.py +23 -0
- nbxtensor/ops/ne.py +32 -0
- nbxtensor/ops/neg.py +17 -0
- nbxtensor/ops/nllloss.py +124 -0
- nbxtensor/ops/nonzero.py +33 -0
- nbxtensor/ops/nonzero_op.py +62 -0
- nbxtensor/ops/norm_wrappers.py +579 -0
- nbxtensor/ops/ones_op.py +20 -0
- nbxtensor/ops/oracles/__init__.py +6 -0
- nbxtensor/ops/oracles/conv2d_fp64.py +450 -0
- nbxtensor/ops/pad_op.py +159 -0
- nbxtensor/ops/paged_decode_attn.py +215 -0
- nbxtensor/ops/pixel_shuffle.py +144 -0
- nbxtensor/ops/pixel_unshuffle.py +72 -0
- nbxtensor/ops/pow.py +137 -0
- nbxtensor/ops/prod.py +81 -0
- nbxtensor/ops/rand_op.py +52 -0
- nbxtensor/ops/random_wrappers.py +366 -0
- nbxtensor/ops/reciprocal.py +17 -0
- nbxtensor/ops/reduction_wrappers.py +845 -0
- nbxtensor/ops/relu.py +18 -0
- nbxtensor/ops/remainder.py +46 -0
- nbxtensor/ops/repeat_interleave.py +48 -0
- nbxtensor/ops/repeat_op.py +132 -0
- nbxtensor/ops/residual_chain.py +374 -0
- nbxtensor/ops/rmsnorm.py +68 -0
- nbxtensor/ops/rope.py +142 -0
- nbxtensor/ops/rsqrt.py +17 -0
- nbxtensor/ops/scaled_softmax.py +96 -0
- nbxtensor/ops/scatter_op.py +139 -0
- nbxtensor/ops/screen_oracle.py +479 -0
- nbxtensor/ops/selu.py +22 -0
- nbxtensor/ops/shape_wrappers.py +1032 -0
- nbxtensor/ops/sigmoid.py +19 -0
- nbxtensor/ops/silu.py +27 -0
- nbxtensor/ops/sin.py +17 -0
- nbxtensor/ops/softmax.py +169 -0
- nbxtensor/ops/softplus.py +27 -0
- nbxtensor/ops/sort_op.py +304 -0
- nbxtensor/ops/spectral_wrappers.py +481 -0
- nbxtensor/ops/sqrt.py +17 -0
- nbxtensor/ops/stack_op.py +84 -0
- nbxtensor/ops/std.py +113 -0
- nbxtensor/ops/strided_copy.py +200 -0
- nbxtensor/ops/sub.py +35 -0
- nbxtensor/ops/sum.py +40 -0
- nbxtensor/ops/tan.py +19 -0
- nbxtensor/ops/tanh.py +21 -0
- nbxtensor/ops/threshold.py +19 -0
- nbxtensor/ops/tiling.yml +91 -0
- nbxtensor/ops/tiling_policy.py +43 -0
- nbxtensor/ops/tiling_sizes.py +370 -0
- nbxtensor/ops/topk.py +267 -0
- nbxtensor/ops/trace_op.py +64 -0
- nbxtensor/ops/tril.py +64 -0
- nbxtensor/ops/triu.py +62 -0
- nbxtensor/ops/unfold_op.py +37 -0
- nbxtensor/ops/unique_op.py +97 -0
- nbxtensor/ops/upsample_bilinear2d.py +106 -0
- nbxtensor/ops/upsample_nearest2d.py +52 -0
- nbxtensor/ops/upsample_nearest3d.py +61 -0
- nbxtensor/ops/var.py +136 -0
- nbxtensor/ops/vector_norm.py +216 -0
- nbxtensor/ops/weight_norm.py +104 -0
- nbxtensor/ops/where.py +39 -0
- nbxtensor/ops/wrappers.py +825 -0
- nbxtensor/ops/zeros_op.py +20 -0
- nbxtensor/quant/__init__.py +1 -0
- nbxtensor/quant/quantized_tensor.py +187 -0
- nbxtensor/random/__init__.py +1 -0
- nbxtensor/random/mt_randn.py +154 -0
- nbxtensor/random/rng_pin.py +59 -0
- nbxtensor/random/rng_stream.py +118 -0
- nbxtensor/random/seeded_draw.py +223 -0
- nbxtensor/shape/__init__.py +157 -0
- nbxtensor/tensor.py +1867 -0
- nbxtensor-0.0.1.dist-info/METADATA +379 -0
- nbxtensor-0.0.1.dist-info/RECORD +291 -0
- nbxtensor-0.0.1.dist-info/WHEEL +4 -0
- nbxtensor-0.0.1.dist-info/licenses/LICENSE +189 -0
|
@@ -0,0 +1,1228 @@
|
|
|
1
|
+
"""Triton Dispatch — ONE table for ALL ops.
|
|
2
|
+
|
|
3
|
+
Every op goes through this table: compute (GPU kernel) + metadata (CPU stride math).
|
|
4
|
+
If an op is not here → CRASH. No fallback, no second path.
|
|
5
|
+
|
|
6
|
+
Compute wrappers: wrappers.py. Metadata: NBXTensor methods.
|
|
7
|
+
Dependencies: wrappers.py, NBXTensor. Used exclusively in --triton mode.
|
|
8
|
+
"""
|
|
9
|
+
# The launch seam (R33, universal since 2026-09-05): every `kernel[grid](...)`
|
|
10
|
+
# of the house library goes through the NeuroBrix launcher (Triton compiles,
|
|
11
|
+
# the dispatch layer launches through a vendor driver of its own).
|
|
12
|
+
# NBX_LAUNCHER=triton keeps upstream's launcher for the differential arm.
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
from typing import Optional, Callable, Dict
|
|
16
|
+
import os
|
|
17
|
+
import triton
|
|
18
|
+
|
|
19
|
+
from nbxtensor.backend.runtime import set_device
|
|
20
|
+
from nbxtensor.shape import view_strides
|
|
21
|
+
|
|
22
|
+
# Diagnostic (default-off): log when the metadata expand/view runtime-resolution
|
|
23
|
+
# fixes actually fire. Used for R23 footprint screening (does a fix fire on a
|
|
24
|
+
# trace==runtime model like an LLM?). Zero cost when unset.
|
|
25
|
+
_META_RESOLVE_DEBUG = os.environ.get("NBX_DEBUG_META_RESOLVE")
|
|
26
|
+
def _meta_resolve_log(msg):
|
|
27
|
+
if _META_RESOLVE_DEBUG:
|
|
28
|
+
import sys
|
|
29
|
+
sys.stderr.write(f"[META_RESOLVE] {msg}\n")
|
|
30
|
+
|
|
31
|
+
_OP_MAP: Optional[Dict[str, Callable]] = None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _ew(n):
|
|
35
|
+
"""An element-wise launch's certified tile and warps (`ops/elementwise_launch.ew_meta`)."""
|
|
36
|
+
from nbxtensor.ops.elementwise_launch import ew_meta
|
|
37
|
+
return ew_meta(n)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _1d_grid_fixed(n):
|
|
41
|
+
"""Grid for element-wise kernels at the certified tile of the count's class."""
|
|
42
|
+
return (triton.cdiv(n, _ew(n)['BLOCK_SIZE']),)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _tensor_to_int(val):
|
|
46
|
+
"""Convert tensor-like to Python int if possible."""
|
|
47
|
+
if isinstance(val, (int, float)):
|
|
48
|
+
return int(val)
|
|
49
|
+
if hasattr(val, 'item'):
|
|
50
|
+
if hasattr(val, 'ndim') and val.ndim == 0:
|
|
51
|
+
return int(val.item())
|
|
52
|
+
if hasattr(val, 'numel') and val.numel() == 1:
|
|
53
|
+
return int(val.item())
|
|
54
|
+
return val
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _tensor_to_int_or_none(val):
|
|
58
|
+
if val is None:
|
|
59
|
+
return None
|
|
60
|
+
return _tensor_to_int(val)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# ============================================================================
|
|
64
|
+
# METADATA OPS — pure Python, NBXTensor methods, CPU stride math
|
|
65
|
+
# ============================================================================
|
|
66
|
+
|
|
67
|
+
def _report_invented_shape(x, shape):
|
|
68
|
+
"""Report-only twin of the instrument in `metadata_ops._reshape`, at MODE 2's site.
|
|
69
|
+
|
|
70
|
+
Step one of the owner's two-step decision (2026-09-24): know which containers a refusing
|
|
71
|
+
reshape would reject before writing the refusal. The first instrument sat in the compiled
|
|
72
|
+
op set, which a `--modes triton,triton-sequential` census never enters, so it could not
|
|
73
|
+
fire (a Kokoro-82M census with it set never created the file). `NBX_RESHAPE_REPORT=<file>`
|
|
74
|
+
appends one record per call whose target has no -1 and whose element count does not match
|
|
75
|
+
the input: an INVENTED shape. A -1 target is an inference and is not recorded. A 1-D
|
|
76
|
+
target is not exempt, because `NBXTensor.view` does not validate numel. The call has no
|
|
77
|
+
op uid here, so a record carries shapes, not a uid. Default off, one env lookup per call,
|
|
78
|
+
no behaviour change."""
|
|
79
|
+
rep = os.environ.get("NBX_RESHAPE_REPORT")
|
|
80
|
+
if not rep or -1 in shape:
|
|
81
|
+
return
|
|
82
|
+
numel = 1
|
|
83
|
+
for d in x.shape:
|
|
84
|
+
numel *= d
|
|
85
|
+
target = 1
|
|
86
|
+
for d in shape:
|
|
87
|
+
target *= d
|
|
88
|
+
if numel == target:
|
|
89
|
+
return
|
|
90
|
+
try:
|
|
91
|
+
import json
|
|
92
|
+
with open(rep, "a") as fh:
|
|
93
|
+
json.dump({
|
|
94
|
+
"model": os.environ.get("NBX_RESHAPE_REPORT_MODEL", ""),
|
|
95
|
+
"site": "dispatch._resolve_view_shape",
|
|
96
|
+
"op_uid": "",
|
|
97
|
+
"in_shape": [int(d) for d in x.shape],
|
|
98
|
+
"in_numel": int(numel),
|
|
99
|
+
"target": [int(d) for d in shape],
|
|
100
|
+
"target_numel": int(target),
|
|
101
|
+
"ratio": (numel / target) if target else None,
|
|
102
|
+
}, fh)
|
|
103
|
+
fh.write("\n")
|
|
104
|
+
except Exception:
|
|
105
|
+
pass # an instrument must never change the run it measures
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _resolve_view_shape(x, shape):
|
|
109
|
+
"""Reconcile a trace-baked view/reshape shape with the runtime numel.
|
|
110
|
+
|
|
111
|
+
NBXTensor.view does NOT validate numel — it re-strides blindly over the same
|
|
112
|
+
data_ptr, so a baked trace dim that differs at runtime becomes a silent
|
|
113
|
+
out-of-bounds view (not an error, unlike torch). The graph bakes trace-time
|
|
114
|
+
shapes; a variable-length model's runtime numel can differ (Kokoro predictor
|
|
115
|
+
view::9: input [1,640,14], baked shape [1,640,23]). Mirror of compiled_ops.py
|
|
116
|
+
view_or_reshape's -1 inference: when the baked product mismatches the input
|
|
117
|
+
numel, infer the single changed dim, preferring the axis whose inferred size
|
|
118
|
+
matches the input's actual dim there. Bit-identical when product == numel
|
|
119
|
+
(trace==runtime) → returns the shape unchanged. Returns a list.
|
|
120
|
+
"""
|
|
121
|
+
shape = list(shape)
|
|
122
|
+
_report_invented_shape(x, shape)
|
|
123
|
+
if -1 in shape or len(shape) < 2:
|
|
124
|
+
return shape # explicit infer dim, or scalar/1D — view handles it
|
|
125
|
+
numel = 1
|
|
126
|
+
for d in x.shape:
|
|
127
|
+
numel *= d
|
|
128
|
+
prod = 1
|
|
129
|
+
for d in shape:
|
|
130
|
+
prod *= d
|
|
131
|
+
if prod == numel:
|
|
132
|
+
return shape # exact (trace==runtime) — no change, byte-identical
|
|
133
|
+
# numel mismatch: collect positions whose -1 inference divides evenly
|
|
134
|
+
candidates = []
|
|
135
|
+
for i in range(len(shape)):
|
|
136
|
+
rest = 1
|
|
137
|
+
for j, d in enumerate(shape):
|
|
138
|
+
if j != i:
|
|
139
|
+
rest *= d
|
|
140
|
+
if rest > 0 and numel % rest == 0:
|
|
141
|
+
candidates.append((i, numel // rest))
|
|
142
|
+
if not candidates:
|
|
143
|
+
return shape # unresolvable (≥2 dims changed) — leave to caller
|
|
144
|
+
_orig = tuple(shape)
|
|
145
|
+
# prefer the axis whose inferred size matches the input's actual dim
|
|
146
|
+
for i, inferred in candidates:
|
|
147
|
+
if i < len(x.shape) and inferred == x.shape[i]:
|
|
148
|
+
shape[i] = inferred
|
|
149
|
+
_meta_resolve_log(f"view infer: in={tuple(x.shape)} baked={_orig} -> {tuple(shape)} (axis {i}, match)")
|
|
150
|
+
return shape
|
|
151
|
+
i, inferred = candidates[0]
|
|
152
|
+
shape[i] = inferred
|
|
153
|
+
_meta_resolve_log(f"view infer: in={tuple(x.shape)} baked={_orig} -> {tuple(shape)} (axis {i}, first)")
|
|
154
|
+
return shape
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _meta_view(x, shape):
|
|
158
|
+
"""The graph's aten.view: a VIEW whenever ATen's would be one (`nbxtensor.shape.view_strides`), a strided input included — the
|
|
159
|
+
graph was traced under ATen, where this op copies nothing. `NBXTensor.view` copies a non-contiguous input, and
|
|
160
|
+
mochi-1-preview's VAE held a 3 GB copy per permute -> view the graph never priced (8 GB rung, 2026-10-28)."""
|
|
161
|
+
if isinstance(shape, (list, tuple)):
|
|
162
|
+
shape = tuple(_resolve_view_shape(x, shape))
|
|
163
|
+
else:
|
|
164
|
+
shape = (shape,)
|
|
165
|
+
if not x.is_contiguous():
|
|
166
|
+
if -1 in shape:
|
|
167
|
+
known = 1
|
|
168
|
+
for s in shape:
|
|
169
|
+
if s != -1:
|
|
170
|
+
known *= s
|
|
171
|
+
shape = tuple(x.numel() // known if s == -1 else s for s in shape)
|
|
172
|
+
strides = view_strides(tuple(x.shape), tuple(x.stride()), shape)
|
|
173
|
+
if strides is not None:
|
|
174
|
+
return x.as_strided(shape, tuple(strides))
|
|
175
|
+
return x.view(*shape)
|
|
176
|
+
|
|
177
|
+
def _meta_reshape(x, shape):
|
|
178
|
+
if isinstance(shape, (list, tuple)):
|
|
179
|
+
return x.reshape(*_resolve_view_shape(x, shape))
|
|
180
|
+
return x.reshape(shape)
|
|
181
|
+
|
|
182
|
+
def _meta_flatten(x, start_dim=0, end_dim=-1):
|
|
183
|
+
return x.flatten(start_dim, end_dim)
|
|
184
|
+
|
|
185
|
+
def _meta_squeeze(x, dim=None):
|
|
186
|
+
return x.squeeze(dim) if dim is not None else x.squeeze()
|
|
187
|
+
|
|
188
|
+
def _meta_unsqueeze(x, dim):
|
|
189
|
+
return x.unsqueeze(_tensor_to_int(dim))
|
|
190
|
+
|
|
191
|
+
def _meta_permute(x, dims):
|
|
192
|
+
return x.permute(*dims) if isinstance(dims, (list, tuple)) else x.permute(dims)
|
|
193
|
+
|
|
194
|
+
def _meta_transpose(x, dim0, dim1):
|
|
195
|
+
return x.transpose(_tensor_to_int(dim0), _tensor_to_int(dim1))
|
|
196
|
+
|
|
197
|
+
def _meta_t(x):
|
|
198
|
+
return x.t()
|
|
199
|
+
|
|
200
|
+
def _meta_contiguous(x, *args, **kwargs):
|
|
201
|
+
return x.contiguous()
|
|
202
|
+
|
|
203
|
+
def _meta_expand(x, size):
|
|
204
|
+
# Multi-resolution fix — mirror of compiled_ops.py _make_expand. The graph
|
|
205
|
+
# bakes trace-time expand sizes; at runtime a variable-length model's actual
|
|
206
|
+
# input dim can differ (Kokoro predictor: d.T seq 23->14, alignment frames
|
|
207
|
+
# 34->62). expand can only broadcast from 1, so when the input's actual dim is
|
|
208
|
+
# non-1 and differs from the baked target, the actual dim is the only valid
|
|
209
|
+
# size — use it. Bit-identical for trace==runtime models (actual == target →
|
|
210
|
+
# no override); restores R30 parity with compiled.
|
|
211
|
+
if isinstance(size, (list, tuple)) and len(size) == len(x.shape):
|
|
212
|
+
new_size = list(size)
|
|
213
|
+
for i, (actual, target) in enumerate(zip(x.shape, size)):
|
|
214
|
+
if actual != 1 and actual != target and target != -1:
|
|
215
|
+
new_size[i] = actual
|
|
216
|
+
if _META_RESOLVE_DEBUG and new_size != list(size):
|
|
217
|
+
_meta_resolve_log(f"expand override: in={tuple(x.shape)} baked={tuple(size)} -> {tuple(new_size)}")
|
|
218
|
+
return x.expand(*new_size)
|
|
219
|
+
return x.expand(*size) if isinstance(size, (list, tuple)) else x.expand(size)
|
|
220
|
+
|
|
221
|
+
def _meta_expand_as(x, other):
|
|
222
|
+
return x.expand_as(other)
|
|
223
|
+
|
|
224
|
+
def _meta_narrow(x, dim, start, length):
|
|
225
|
+
return x.narrow(_tensor_to_int(dim), _tensor_to_int(start), _tensor_to_int(length))
|
|
226
|
+
|
|
227
|
+
def _meta_slice(x, dim, start=None, end=None, step=1):
|
|
228
|
+
dim = int(_force_int(dim))
|
|
229
|
+
start = int(_force_int(start)) if start is not None else 0
|
|
230
|
+
dim_size = x.size(dim) if callable(getattr(x, 'size', None)) else x.shape[dim]
|
|
231
|
+
end = int(_force_int(end)) if end is not None else dim_size
|
|
232
|
+
step = int(_force_int(step)) if step is not None else 1
|
|
233
|
+
if end > dim_size:
|
|
234
|
+
end = dim_size
|
|
235
|
+
if start < 0:
|
|
236
|
+
start = max(0, dim_size + start)
|
|
237
|
+
if end < 0:
|
|
238
|
+
end = max(0, dim_size + end)
|
|
239
|
+
length = max(0, end - start)
|
|
240
|
+
narrowed = x.narrow(dim, start, length)
|
|
241
|
+
if step == 1:
|
|
242
|
+
return narrowed
|
|
243
|
+
# Strided slice x[..., start:end:step]. narrow() only covers step==1; for a
|
|
244
|
+
# step>1 slice build a step-strided VIEW (size=ceil(length/step),
|
|
245
|
+
# stride[dim]*=step) and return it as a view — NOT materialised. A slice is
|
|
246
|
+
# a view in PyTorch/compiled (sequential_dispatcher maps aten::copy to
|
|
247
|
+
# inputs[0].copy_(inputs[1]) so an in-place write through the slice reaches
|
|
248
|
+
# the parent buffer). The interleaved-complex Wan RoPE relies on exactly
|
|
249
|
+
# that: `buf[...,0::2] = real; buf[...,1::2] = imag` lowers to step-2 slice
|
|
250
|
+
# + copy with the copy output orphaned — the real effect is mutation of the
|
|
251
|
+
# empty_like buffer that permute reads. Materialising the slice here made
|
|
252
|
+
# copy_ write into a throwaway contiguous tensor, leaving the buffer unwritten
|
|
253
|
+
# → zero self-attn Q/K → washed-out VACE/Wan2.2 (2026-06-28). The
|
|
254
|
+
# as_strided VIEW makes copy_ route through NBXTensor.copy_'s strided-dst
|
|
255
|
+
# branch (_strided_scatter) and scatter into the parent at the right stride.
|
|
256
|
+
# Read consumers of a step>1 slice are flat-indexed kernels that already
|
|
257
|
+
# self-guard with .contiguous() (the binary/elementwise _prepare_binary
|
|
258
|
+
# path — the rope-rotation muls — and the CLAUDE.md contiguous-guard at
|
|
259
|
+
# split/matmul wrappers). The as_strided encodes the correct strided shape
|
|
260
|
+
# (the historical "348 vs 174" bug was the MISSING as_strided, not the
|
|
261
|
+
# missing materialisation).
|
|
262
|
+
new_size = list(narrowed.shape)
|
|
263
|
+
new_stride = list(narrowed._strides)
|
|
264
|
+
new_size[dim] = (length + step - 1) // step
|
|
265
|
+
new_stride[dim] = narrowed._strides[dim] * step
|
|
266
|
+
return narrowed.as_strided(new_size, new_stride, narrowed._offset)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _force_int(val):
|
|
270
|
+
"""Force ANY value to Python int. Handles NBXTensor, scalars, etc."""
|
|
271
|
+
if isinstance(val, int):
|
|
272
|
+
return val
|
|
273
|
+
if isinstance(val, float):
|
|
274
|
+
return int(val)
|
|
275
|
+
if hasattr(val, 'item'):
|
|
276
|
+
return int(val.item())
|
|
277
|
+
return int(val)
|
|
278
|
+
|
|
279
|
+
def _meta_select(x, dim, index):
|
|
280
|
+
return x.select(_tensor_to_int(dim), _tensor_to_int(index))
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _meta_slice_scatter(x, src, dim=0, start=None, end=None, step=1):
|
|
284
|
+
"""aten.slice_scatter: a copy of `x` with `src` written where `x[..., start:end:step]` lies (along `dim`) —
|
|
285
|
+
the functional form of an in-place write into a slice view (a traced graph records it for, e.g., Swin's
|
|
286
|
+
shifted-window mask `img_mask[:, h, w, :] = cnt`). The slice is `_meta_slice`'s, the same bounds rules."""
|
|
287
|
+
out = x.clone()
|
|
288
|
+
_meta_slice(out, dim, start, end, step).copy_(src)
|
|
289
|
+
return out
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _meta_select_scatter(x, src, dim, index):
|
|
293
|
+
"""aten.select_scatter: a copy of `x` with `src` written at `index` along `dim` (the functional form of an
|
|
294
|
+
in-place write into a select view)."""
|
|
295
|
+
out = x.clone()
|
|
296
|
+
d, i = _tensor_to_int(dim), _tensor_to_int(index)
|
|
297
|
+
out.select(d, i if i >= 0 else out.shape[d] + i).copy_(src)
|
|
298
|
+
return out
|
|
299
|
+
|
|
300
|
+
def _meta_clone(x, *args, **kwargs):
|
|
301
|
+
return x.clone()
|
|
302
|
+
|
|
303
|
+
def _meta_detach(x):
|
|
304
|
+
return x.detach()
|
|
305
|
+
|
|
306
|
+
def _meta_alias(x):
|
|
307
|
+
return x
|
|
308
|
+
|
|
309
|
+
def _meta_fill(x, value):
|
|
310
|
+
# Functional aten::fill: output is a FRESH tensor shaped like x, all elements
|
|
311
|
+
# = value (x supplies only shape/dtype/device — its data is irrelevant, and it
|
|
312
|
+
# may be a non-contiguous view, e.g. a slice). Never mutate x in place: that
|
|
313
|
+
# would corrupt other consumers of the input and (on a strided view) write the
|
|
314
|
+
# wrong addresses under flat-indexed fill.
|
|
315
|
+
from nbxtensor.tensor import NBXTensor
|
|
316
|
+
if isinstance(value, NBXTensor):
|
|
317
|
+
value = value.item()
|
|
318
|
+
if value is None:
|
|
319
|
+
# Orphan scalar constant: an in-forward `mask[slice] = cnt` count baked
|
|
320
|
+
# as a `param::constant` with NO data in the container, consumed by
|
|
321
|
+
# aten::fill. The native compiled path materialises it as a 0-dim default
|
|
322
|
+
# (bind_weights `[]`, commit c0a1445) and the triton-sequential resolver
|
|
323
|
+
# mirrors that (graph_executor._resolve_sequential_arg) — but the triton
|
|
324
|
+
# COMPILED arena leaves the slot None, so the value reaches fill as None.
|
|
325
|
+
# Default to 0 here so both triton paths match the native 0-default (HAT
|
|
326
|
+
# OCAB counter; the value is a trace-time artefact, not runtime data).
|
|
327
|
+
value = 0
|
|
328
|
+
out = NBXTensor.empty(x._shape, x._dtype, f"cuda:{x._device_idx}")
|
|
329
|
+
return out.fill_(value)
|
|
330
|
+
|
|
331
|
+
def _meta_as_strided(x, size, stride, storage_offset=None):
|
|
332
|
+
return x.as_strided(size, stride, storage_offset)
|
|
333
|
+
|
|
334
|
+
def _meta_view_as(x, other):
|
|
335
|
+
return x.view_as(other)
|
|
336
|
+
|
|
337
|
+
def _meta_reshape_as(x, other):
|
|
338
|
+
return x.reshape_as(other)
|
|
339
|
+
|
|
340
|
+
def _meta_unfold(x, dimension, size, step):
|
|
341
|
+
return x.unfold(dimension, size, step)
|
|
342
|
+
|
|
343
|
+
def _meta_unbind(x, dim=0):
|
|
344
|
+
return x.unbind(dim)
|
|
345
|
+
|
|
346
|
+
def _chunk_narrow(x, chunks, dim=0):
|
|
347
|
+
"""torch.chunk semantics via narrow views.
|
|
348
|
+
|
|
349
|
+
chunk_size = ceil(n / chunks); ceil(n / chunk_size) chunks are
|
|
350
|
+
returned, the LAST carrying the remainder — exactly torch.chunk.
|
|
351
|
+
The former placeholder (`x.unbind(dim) if chunks == x.size(dim)
|
|
352
|
+
else (x,)`) silently returned the WHOLE tensor for any real
|
|
353
|
+
chunks > 1 request (same "simplified" landmine class as the dead
|
|
354
|
+
split_with_sizes → _meta_slice entry removed in P-TRITON-MLA).
|
|
355
|
+
"""
|
|
356
|
+
chunks = int(_force_int(chunks))
|
|
357
|
+
n = x.shape[dim]
|
|
358
|
+
if n == 0 or chunks <= 1:
|
|
359
|
+
return (x,)
|
|
360
|
+
size = -(-n // chunks) # ceil division
|
|
361
|
+
return tuple(x.narrow(dim, i * size, min(size, n - i * size))
|
|
362
|
+
for i in range(-(-n // size)))
|
|
363
|
+
|
|
364
|
+
def _meta_select_item(x):
|
|
365
|
+
return x.item()
|
|
366
|
+
|
|
367
|
+
def _meta_copy(x, src, *args, **kwargs):
|
|
368
|
+
return x.copy_(src)
|
|
369
|
+
|
|
370
|
+
def _meta_equal(x, y):
|
|
371
|
+
# Scalar comparison — returns bool
|
|
372
|
+
return x.numel() == y.numel() # simplified
|
|
373
|
+
|
|
374
|
+
def _meta_to(x, *args, **kwargs):
|
|
375
|
+
return x.to(*args, **kwargs)
|
|
376
|
+
|
|
377
|
+
def _meta_type_as(x, other):
|
|
378
|
+
return x.type_as(other)
|
|
379
|
+
|
|
380
|
+
def _meta_repeat(x, *repeats):
|
|
381
|
+
if len(repeats) == 1 and isinstance(repeats[0], (list, tuple)):
|
|
382
|
+
return x.repeat(*repeats[0])
|
|
383
|
+
return x.repeat(*repeats)
|
|
384
|
+
|
|
385
|
+
def _meta_movedim(x, src, dst):
|
|
386
|
+
return x.movedim(src, dst)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _meta_index(x, indices):
|
|
390
|
+
"""aten::index — advanced indexing with a list of optional index tensors.
|
|
391
|
+
|
|
392
|
+
Single index tensor on dim d: x[..., idx, ...] via index_select; output
|
|
393
|
+
shape = x.shape[:d] + idx.shape + x.shape[d+1:].
|
|
394
|
+
|
|
395
|
+
TWO+ index tensors = advanced ("diagonal"/joint) indexing: PyTorch selects
|
|
396
|
+
elements jointly, out[k] = x[i0[k], i1[k], ...], NOT the outer product.
|
|
397
|
+
The CLIP text pooler is exactly this: last_hidden_state[arange(B),
|
|
398
|
+
input_ids.argmax(-1)] picks the EOS token per batch → [B, hidden]. The old
|
|
399
|
+
loop returned after the FIRST index tensor, so it kept all 77 text tokens
|
|
400
|
+
([1,77,768]) instead of pooling to [1,768] — which then broadcast the 77
|
|
401
|
+
dim through the FLUX/Flex temb (silu::3 [1,77,3072] → addmm [1,77,18432] →
|
|
402
|
+
split(dim=1) yields 0 chunks → adaLN slice crash). Implement the joint
|
|
403
|
+
select via a flat row-major index over the consecutive indexed dims.
|
|
404
|
+
"""
|
|
405
|
+
from nbxtensor.ops import wrappers as w
|
|
406
|
+
from nbxtensor.ops import elementwise_wrappers as _nbx_elementwise_wrappers # the family the moved names live in (M2 split)
|
|
407
|
+
from nbxtensor.ops import index_wrappers as _nbx_index_wrappers # the family the moved names live in (M2 split)
|
|
408
|
+
if not isinstance(indices, (list, tuple)):
|
|
409
|
+
return _nbx_index_wrappers.index_select_wrapper(x, 0, indices)
|
|
410
|
+
|
|
411
|
+
present = [(d, i) for d, i in enumerate(indices) if i is not None]
|
|
412
|
+
if not present:
|
|
413
|
+
return x
|
|
414
|
+
|
|
415
|
+
if len(present) == 1:
|
|
416
|
+
# Single advanced index — unchanged behaviour (byte-identical path).
|
|
417
|
+
dim, idx = present[0]
|
|
418
|
+
_idx_dtn = str(getattr(idx, "dtype", "")).lower()
|
|
419
|
+
if (("bool" in _idx_dtn or "uint8" in _idx_dtn)
|
|
420
|
+
and dim == 0 and idx.ndim >= 1
|
|
421
|
+
and list(idx.shape) == list(x.shape[:idx.ndim])):
|
|
422
|
+
# Boolean-mask advanced indexing: x[mask] gathers the rows
|
|
423
|
+
# where the leading-dim-prefix mask is True —
|
|
424
|
+
# x.flatten(masked dims)[nonzero(mask.flatten())]. The
|
|
425
|
+
# DeepStack injection read (hidden[visual_pos_masks, :]).
|
|
426
|
+
# Deterministic compaction; integer masks keep the historic
|
|
427
|
+
# index_select route below untouched.
|
|
428
|
+
xc = x.contiguous()
|
|
429
|
+
flat_rows = 1
|
|
430
|
+
for _s in idx.shape:
|
|
431
|
+
flat_rows *= int(_s)
|
|
432
|
+
x_flat = xc.reshape([flat_rows] + list(x.shape[idx.ndim:]))
|
|
433
|
+
rows = _nbx_index_wrappers.nonzero_1d_wrapper(idx.reshape([flat_rows]))
|
|
434
|
+
if int(rows.shape[0]) == 0:
|
|
435
|
+
# All-False mask (the text-mode empty-visual stub path):
|
|
436
|
+
# a [0, tail...] gather — allocated directly, the select
|
|
437
|
+
# kernel has no zero-length grid.
|
|
438
|
+
from nbxtensor.tensor import NBXTensor
|
|
439
|
+
return NBXTensor.empty(
|
|
440
|
+
[0] + list(x.shape[idx.ndim:]),
|
|
441
|
+
dtype=x.dtype, device=x.device)
|
|
442
|
+
return _nbx_index_wrappers.index_select_wrapper(x_flat, 0, rows)
|
|
443
|
+
orig_idx_shape = idx.shape
|
|
444
|
+
flat_idx = idx.reshape(-1) if idx.ndim > 1 else idx
|
|
445
|
+
selected = _nbx_index_wrappers.index_select_wrapper(x, dim, flat_idx)
|
|
446
|
+
if idx.ndim > 1:
|
|
447
|
+
out_shape = list(orig_idx_shape) + list(x.shape[dim + 1:])
|
|
448
|
+
selected = selected.reshape(*out_shape)
|
|
449
|
+
return selected
|
|
450
|
+
|
|
451
|
+
# --- Advanced indexing with >=2 index tensors (joint select) ---
|
|
452
|
+
dims = [d for d, _ in present]
|
|
453
|
+
if dims != list(range(dims[0], dims[0] + len(dims))):
|
|
454
|
+
raise NotImplementedError(
|
|
455
|
+
f"[--triton] aten::index advanced indexing requires consecutive "
|
|
456
|
+
f"index dims; got {dims}")
|
|
457
|
+
first = dims[0]
|
|
458
|
+
idx_tensors = [i for _, i in present]
|
|
459
|
+
base_shape = list(idx_tensors[0].shape)
|
|
460
|
+
if any(list(t.shape) != base_shape for t in idx_tensors[1:]):
|
|
461
|
+
# PyTorch broadcasts differing index shapes together (right-aligned,
|
|
462
|
+
# size-1 dims stretch). First real consumer: the T5 attention-mask
|
|
463
|
+
# application traced with a pad-realistic stimulus (Allegro) —
|
|
464
|
+
# extended_mask indexing uses [(1,1,1,1), (1,1,1,512)]. Compute the
|
|
465
|
+
# common broadcast shape, expand every index (NBXTensor.expand is a
|
|
466
|
+
# zero-copy stride view), and fall through to the joint flat-select.
|
|
467
|
+
ndim = max(t.ndim for t in idx_tensors)
|
|
468
|
+
shapes = []
|
|
469
|
+
for t in idx_tensors:
|
|
470
|
+
s = [1] * (ndim - t.ndim) + list(t.shape)
|
|
471
|
+
shapes.append(s)
|
|
472
|
+
bshape = []
|
|
473
|
+
for ax in range(ndim):
|
|
474
|
+
sizes_ax = {s[ax] for s in shapes} - {1}
|
|
475
|
+
if len(sizes_ax) > 1:
|
|
476
|
+
raise NotImplementedError(
|
|
477
|
+
f"[--triton] aten::index index shapes not broadcastable: "
|
|
478
|
+
f"{[t.shape for t in idx_tensors]}")
|
|
479
|
+
bshape.append(sizes_ax.pop() if sizes_ax else 1)
|
|
480
|
+
# contiguous() after expand: the stride-0 view feeds reshape(-1)
|
|
481
|
+
# below (flat-indexed path) — the mandatory contiguous-guard
|
|
482
|
+
# (no-op for already-contiguous tensors via the early-return).
|
|
483
|
+
idx_tensors = [
|
|
484
|
+
t.reshape(*([1] * (ndim - t.ndim) + list(t.shape)))
|
|
485
|
+
.expand(*bshape).contiguous()
|
|
486
|
+
if list(t.shape) != bshape else t
|
|
487
|
+
for t in idx_tensors
|
|
488
|
+
]
|
|
489
|
+
base_shape = bshape
|
|
490
|
+
# Row-major flat index across the consecutive indexed dims:
|
|
491
|
+
# flat = ((i0)*n1 + i1)*n2 + i2 ... (ni = x.shape[indexed dim])
|
|
492
|
+
sizes = [x.shape[d] for d in dims]
|
|
493
|
+
# torch semantics: negative entries count from the end of THEIR dim —
|
|
494
|
+
# wrap each index before the row-major flatten (a raw negative would
|
|
495
|
+
# corrupt the joint offset; the single-dim gather wraps in-kernel).
|
|
496
|
+
flats = [_nbx_elementwise_wrappers.remainder_wrapper(t.reshape(-1), int(sz)) for t, sz in zip(idx_tensors, sizes)]
|
|
497
|
+
flat = flats[0]
|
|
498
|
+
for k in range(1, len(flats)):
|
|
499
|
+
flat = flat * sizes[k] + flats[k]
|
|
500
|
+
flat = flat.long()
|
|
501
|
+
block = 1
|
|
502
|
+
for s in sizes:
|
|
503
|
+
block *= s
|
|
504
|
+
collapsed = list(x.shape[:first]) + [block] + list(x.shape[first + len(dims):])
|
|
505
|
+
xc = x.reshape(*collapsed)
|
|
506
|
+
sel = _nbx_index_wrappers.index_select_wrapper(xc, first, flat)
|
|
507
|
+
out_shape = list(x.shape[:first]) + base_shape + list(x.shape[first + len(dims):])
|
|
508
|
+
return sel.reshape(*out_shape)
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def _meta_sdpa_efficient(*args, **kwargs):
|
|
512
|
+
"""aten::_scaled_dot_product_efficient_attention / _flash_attention.
|
|
513
|
+
|
|
514
|
+
These fused-backend SDPA variants have a DIFFERENT positional signature
|
|
515
|
+
than plain ``scaled_dot_product_attention``: an extra ``compute_log_sumexp``
|
|
516
|
+
bool sits at arg[4], shifting ``dropout_p``→[5], ``is_causal``→[6],
|
|
517
|
+
``scale``→[7]. The plain-SDPA wrapper expects
|
|
518
|
+
``(q, k, v, attn_mask, dropout_p, is_causal, scale)``, so a *direct*
|
|
519
|
+
positional call mis-reads ``is_causal`` (from ``dropout_p``) and ``scale``
|
|
520
|
+
(from ``is_causal``) — silently dropping the decoder's causal mask and
|
|
521
|
+
using scale=1.0. Invisible at seq_len=1 (single-element softmax), it
|
|
522
|
+
corrupts every seq_len>=2 forward (constant-token garbage on whisper).
|
|
523
|
+
|
|
524
|
+
Remap explicitly. Mirrors ``triton_sequential`` (sequential.py:198) so the
|
|
525
|
+
two triton modes stay symmetric (R30).
|
|
526
|
+
"""
|
|
527
|
+
from nbxtensor.ops import wrappers as w
|
|
528
|
+
from nbxtensor.ops import attention_wrappers as _nbx_attention_wrappers # the family the moved names live in (M2 split)
|
|
529
|
+
q, k, v = args[0], args[1], args[2]
|
|
530
|
+
attn_mask = args[3] if len(args) > 3 and args[3] is not None else None
|
|
531
|
+
dropout_p = args[5] if len(args) > 5 else 0.0
|
|
532
|
+
is_causal = args[6] if len(args) > 6 else False
|
|
533
|
+
scale = kwargs.get("scale", args[7] if len(args) > 7 else None)
|
|
534
|
+
if not isinstance(dropout_p, float):
|
|
535
|
+
dropout_p = float(dropout_p)
|
|
536
|
+
if not isinstance(is_causal, bool):
|
|
537
|
+
is_causal = bool(is_causal)
|
|
538
|
+
return _nbx_attention_wrappers.scaled_dot_product_attention_wrapper(
|
|
539
|
+
q, k, v, attn_mask=attn_mask, dropout_p=dropout_p,
|
|
540
|
+
is_causal=is_causal, scale=scale,
|
|
541
|
+
# The graph's recorded K layout, when the caller bound it onto this
|
|
542
|
+
# meta-op. Dropping it here would leave the wrapper reading the
|
|
543
|
+
# layout off a shape that cannot express it at seq_len == head_dim.
|
|
544
|
+
k_pre_transposed=kwargs.get("k_pre_transposed"))
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def _meta_weight_norm(v, g, dim=0, **kwargs):
|
|
548
|
+
"""aten::_weight_norm(v, g, dim) → w = v * g / ‖v‖, where ‖v‖ is the L2 norm
|
|
549
|
+
over ALL dims except `dim` (keepdim). Pure-Triton meta-op via the existing
|
|
550
|
+
mul/sum/sqrt/div wrappers (no dedicated kernel needed). Used by vocoders'
|
|
551
|
+
weight-normalized convs (chatterbox s3gen, HiFi-GAN-style).
|
|
552
|
+
"""
|
|
553
|
+
from nbxtensor.ops import wrappers as w
|
|
554
|
+
from nbxtensor.ops import elementwise_wrappers as _nbx_elementwise_wrappers # the family the moved names live in (M2 split)
|
|
555
|
+
from nbxtensor.ops import reduction_wrappers as _nbx_reduction_wrappers # the family the moved names live in (M2 split)
|
|
556
|
+
nd = v.ndim
|
|
557
|
+
d = dim % nd if nd else 0
|
|
558
|
+
vsq = _nbx_elementwise_wrappers.mul(v, v)
|
|
559
|
+
nsq = vsq
|
|
560
|
+
for rd in range(nd):
|
|
561
|
+
if rd == d:
|
|
562
|
+
continue
|
|
563
|
+
nsq = _nbx_reduction_wrappers.sum_wrapper(nsq, dim=rd, keepdim=True)
|
|
564
|
+
norm = _nbx_elementwise_wrappers.sqrt_wrapper(nsq)
|
|
565
|
+
# w = v * (g / norm); g and norm share the norm-shape (1s except dim d),
|
|
566
|
+
# broadcast against the full-shape v.
|
|
567
|
+
return _nbx_elementwise_wrappers.mul(v, _nbx_elementwise_wrappers.div(g, norm))
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
# ============================================================================
|
|
571
|
+
# CREATION OPS — ATen signatures, Triton fill kernel, NBXTensor allocation
|
|
572
|
+
#
|
|
573
|
+
# Reference: FlagGems ops/{ones,zeros,full,arange}.py
|
|
574
|
+
# Signatures match aten::ones, aten::zeros, etc.
|
|
575
|
+
# ============================================================================
|
|
576
|
+
|
|
577
|
+
def _extract_creation_args(args, kwargs):
|
|
578
|
+
"""Extract (size, dtype, device) from ATen creation op args.
|
|
579
|
+
|
|
580
|
+
ATen passes: (size_list, *, dtype=..., layout=..., device=..., pin_memory=...)
|
|
581
|
+
Compiled sequence may pass size as positional or keyword.
|
|
582
|
+
"""
|
|
583
|
+
size = args[0] if args else kwargs.get('size', ())
|
|
584
|
+
if isinstance(size, (int, float)):
|
|
585
|
+
size = (int(size),)
|
|
586
|
+
dtype = kwargs.get('dtype')
|
|
587
|
+
device = kwargs.get('device', 'cuda')
|
|
588
|
+
return size, dtype, device
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def _create_ones(*args, **kwargs):
|
|
592
|
+
"""aten::ones — fill with 1.0 via Triton fill kernel."""
|
|
593
|
+
from nbxtensor.tensor import NBXTensor
|
|
594
|
+
from nbxtensor.ops.fill_op import fill_kernel
|
|
595
|
+
size, dtype, device = _extract_creation_args(args, kwargs)
|
|
596
|
+
out = NBXTensor.empty(size, dtype=dtype, device=device or 'cuda')
|
|
597
|
+
n = out.numel()
|
|
598
|
+
if n > 0:
|
|
599
|
+
set_device(out)
|
|
600
|
+
fill_kernel[_1d_grid_fixed(n)](out, 1.0, n, **_ew(n))
|
|
601
|
+
return out
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
def _create_zeros(*args, **kwargs):
|
|
605
|
+
"""aten::zeros — cudaMemset zero."""
|
|
606
|
+
from nbxtensor.tensor import NBXTensor
|
|
607
|
+
size, dtype, device = _extract_creation_args(args, kwargs)
|
|
608
|
+
return NBXTensor.zeros(size, dtype=dtype, device=device or 'cuda')
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
def _create_empty(*args, **kwargs):
|
|
612
|
+
"""aten::empty — uninitialized allocation."""
|
|
613
|
+
from nbxtensor.tensor import NBXTensor
|
|
614
|
+
size, dtype, device = _extract_creation_args(args, kwargs)
|
|
615
|
+
return NBXTensor.empty(size, dtype=dtype, device=device or 'cuda')
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
def _create_full(*args, **kwargs):
|
|
619
|
+
"""aten::full — fill with arbitrary value via Triton fill kernel."""
|
|
620
|
+
from nbxtensor.tensor import NBXTensor
|
|
621
|
+
from nbxtensor.ops.fill_op import fill_kernel
|
|
622
|
+
size = args[0] if args else kwargs.get('size', ())
|
|
623
|
+
fill_value = args[1] if len(args) > 1 else kwargs.get('fill_value', 0)
|
|
624
|
+
dtype = kwargs.get('dtype')
|
|
625
|
+
device = kwargs.get('device', 'cuda')
|
|
626
|
+
out = NBXTensor.empty(size, dtype=dtype, device=device or 'cuda')
|
|
627
|
+
n = out.numel()
|
|
628
|
+
if n > 0:
|
|
629
|
+
set_device(out)
|
|
630
|
+
fill_kernel[_1d_grid_fixed(n)](out, float(fill_value), n, **_ew(n))
|
|
631
|
+
return out
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
def _create_scalar_tensor(value=0, **kwargs):
|
|
635
|
+
"""aten::scalar_tensor — single-element tensor."""
|
|
636
|
+
from nbxtensor.tensor import NBXTensor
|
|
637
|
+
from nbxtensor.ops.fill_op import fill_kernel
|
|
638
|
+
dtype = kwargs.get('dtype')
|
|
639
|
+
device = kwargs.get('device', 'cuda')
|
|
640
|
+
out = NBXTensor.empty((1,), dtype=dtype, device=device or 'cuda')
|
|
641
|
+
set_device(out)
|
|
642
|
+
fill_kernel[(1,)](out, float(value) if value is not None else 0.0, 1, **_ew(1))
|
|
643
|
+
return out
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
def _create_arange(start_or_end, end=None, step=1, *, dtype=None,
|
|
647
|
+
layout=None, device=None, pin_memory=None, **kwargs):
|
|
648
|
+
"""aten::arange — sequential values via Triton kernel.
|
|
649
|
+
|
|
650
|
+
Reference: FlagGems ops/arange.py — uses triton arange_func kernel.
|
|
651
|
+
"""
|
|
652
|
+
from nbxtensor.tensor import NBXTensor
|
|
653
|
+
from nbxtensor.ops.arange_op import arange_kernel
|
|
654
|
+
import math
|
|
655
|
+
if end is None:
|
|
656
|
+
start, end = 0, start_or_end
|
|
657
|
+
else:
|
|
658
|
+
start = start_or_end
|
|
659
|
+
start = float(start)
|
|
660
|
+
end = float(end)
|
|
661
|
+
step = float(step)
|
|
662
|
+
if step == 0:
|
|
663
|
+
raise ValueError("step must be non-zero")
|
|
664
|
+
size = max(0, math.ceil((end - start) / step))
|
|
665
|
+
if dtype is None:
|
|
666
|
+
from nbxtensor.dtype.nbx_dtype import NBXDtype
|
|
667
|
+
dtype = NBXDtype.int64 if isinstance(start_or_end, int) and (end is None or isinstance(end, int)) else NBXDtype.float32
|
|
668
|
+
out = NBXTensor.empty((size,), dtype=dtype, device=device or 'cuda')
|
|
669
|
+
if size > 0:
|
|
670
|
+
BLOCK_SIZE = 128
|
|
671
|
+
grid = (triton.cdiv(size, BLOCK_SIZE),)
|
|
672
|
+
set_device(out)
|
|
673
|
+
arange_kernel[grid](out, start, step, size, BLOCK_SIZE=BLOCK_SIZE)
|
|
674
|
+
return out
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def _create_zeros_like(x, *, dtype=None, layout=None, device=None,
|
|
678
|
+
pin_memory=None, **kwargs):
|
|
679
|
+
from nbxtensor.tensor import NBXTensor
|
|
680
|
+
return NBXTensor.zeros_like(x, dtype=dtype, device=device)
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def _create_ones_like(x, *, dtype=None, layout=None, device=None,
|
|
684
|
+
pin_memory=None, **kwargs):
|
|
685
|
+
from nbxtensor.tensor import NBXTensor
|
|
686
|
+
from nbxtensor.ops.fill_op import fill_kernel
|
|
687
|
+
out = NBXTensor.empty_like(x, dtype=dtype, device=device)
|
|
688
|
+
n = out.numel()
|
|
689
|
+
if n > 0:
|
|
690
|
+
set_device(out)
|
|
691
|
+
fill_kernel[_1d_grid_fixed(n)](out, 1.0, n, **_ew(n))
|
|
692
|
+
return out
|
|
693
|
+
|
|
694
|
+
|
|
695
|
+
def _create_empty_like(x, *, dtype=None, layout=None, device=None,
|
|
696
|
+
pin_memory=None, **kwargs):
|
|
697
|
+
from nbxtensor.tensor import NBXTensor
|
|
698
|
+
return NBXTensor.empty_like(x, dtype=dtype, device=device)
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def _create_full_like(x, fill_value, *, dtype=None, layout=None, device=None,
|
|
702
|
+
pin_memory=None, **kwargs):
|
|
703
|
+
from nbxtensor.tensor import NBXTensor
|
|
704
|
+
from nbxtensor.ops.fill_op import fill_kernel
|
|
705
|
+
out = NBXTensor.empty_like(x, dtype=dtype, device=device)
|
|
706
|
+
n = out.numel()
|
|
707
|
+
if n > 0:
|
|
708
|
+
set_device(out)
|
|
709
|
+
fill_kernel[_1d_grid_fixed(n)](out, float(fill_value), n, **_ew(n))
|
|
710
|
+
return out
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
def _create_linspace(start, end, steps, *, dtype=None, layout=None,
|
|
714
|
+
device=None, pin_memory=None, **kwargs):
|
|
715
|
+
"""aten::linspace — evenly spaced values via Triton kernel.
|
|
716
|
+
|
|
717
|
+
Reference: FlagGems ops/linspace.py — bidirectional kernel
|
|
718
|
+
(`kernels/ops/linspace_op.py`), forward from start for the first
|
|
719
|
+
half and backward from end for the second half so both endpoints
|
|
720
|
+
are exact (matches torch.linspace semantics).
|
|
721
|
+
|
|
722
|
+
steps == 1 degenerates to a single `start` value (torch returns
|
|
723
|
+
[start], not [end]); the bidirectional formula would yield `end`
|
|
724
|
+
for that lone element, so it is special-cased through fill_kernel.
|
|
725
|
+
"""
|
|
726
|
+
from nbxtensor.tensor import NBXTensor
|
|
727
|
+
from nbxtensor.dtype.nbx_dtype import NBXDtype
|
|
728
|
+
from nbxtensor.ops.linspace_op import linspace_kernel
|
|
729
|
+
from nbxtensor.ops.fill_op import fill_kernel
|
|
730
|
+
steps = int(steps)
|
|
731
|
+
start = float(start)
|
|
732
|
+
end = float(end)
|
|
733
|
+
out = NBXTensor.empty((steps,), dtype=dtype if dtype is not None else NBXDtype.float32,
|
|
734
|
+
device=device or 'cuda')
|
|
735
|
+
if steps <= 0:
|
|
736
|
+
return out
|
|
737
|
+
BLOCK_SIZE = 128
|
|
738
|
+
grid = (triton.cdiv(steps, BLOCK_SIZE),)
|
|
739
|
+
set_device(out)
|
|
740
|
+
if steps == 1:
|
|
741
|
+
fill_kernel[grid](out, start, 1, BLOCK_SIZE=BLOCK_SIZE)
|
|
742
|
+
return out
|
|
743
|
+
step_size = (end - start) / (steps - 1)
|
|
744
|
+
mid = steps // 2
|
|
745
|
+
linspace_kernel[grid](out, out.stride(0), start, mid, end,
|
|
746
|
+
step_size, steps, BLOCK_SIZE=BLOCK_SIZE)
|
|
747
|
+
return out
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
def _build_creation_ops():
|
|
751
|
+
"""Creation ops with proper ATen signatures."""
|
|
752
|
+
from nbxtensor.tensor import NBXTensor
|
|
753
|
+
|
|
754
|
+
return {
|
|
755
|
+
"zeros": _create_zeros,
|
|
756
|
+
"ones": _create_ones,
|
|
757
|
+
"empty": _create_empty,
|
|
758
|
+
"full": _create_full,
|
|
759
|
+
"scalar_tensor": _create_scalar_tensor,
|
|
760
|
+
"zeros_like": _create_zeros_like,
|
|
761
|
+
"ones_like": _create_ones_like,
|
|
762
|
+
"empty_like": _create_empty_like,
|
|
763
|
+
"full_like": _create_full_like,
|
|
764
|
+
"new_zeros": lambda x, size, **kw: x.new_zeros(size, **kw),
|
|
765
|
+
"new_ones": lambda x, size, **kw: x.new_ones(size, **kw),
|
|
766
|
+
"new_empty": lambda x, size, **kw: x.new_empty(size, **kw),
|
|
767
|
+
"arange": _create_arange,
|
|
768
|
+
"linspace": _create_linspace,
|
|
769
|
+
# Cat/Stack
|
|
770
|
+
"cat": lambda tensors, dim=0: NBXTensor.cat(tensors, dim),
|
|
771
|
+
"stack": lambda tensors, dim=0: NBXTensor.stack(tensors, dim),
|
|
772
|
+
"concat": lambda tensors, dim=0: NBXTensor.cat(tensors, dim),
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
# ============================================================================
|
|
777
|
+
# BUILD COMPLETE OP MAP
|
|
778
|
+
# ============================================================================
|
|
779
|
+
|
|
780
|
+
def _build_op_map() -> Dict[str, Callable]:
|
|
781
|
+
"""Build the COMPLETE aten_op → function mapping.
|
|
782
|
+
|
|
783
|
+
ONE table. ALL ops. Compute + Metadata + Creation.
|
|
784
|
+
No second path. No fallback. Missing = crash.
|
|
785
|
+
"""
|
|
786
|
+
from nbxtensor.ops import wrappers as w
|
|
787
|
+
from nbxtensor.ops import attention_wrappers as _nbx_attention_wrappers # the family the moved names live in (M2 split)
|
|
788
|
+
from nbxtensor.ops import conv_wrappers as _nbx_conv_wrappers # the family the moved names live in (M2 split)
|
|
789
|
+
from nbxtensor.ops import elementwise_wrappers as _nbx_elementwise_wrappers # the family the moved names live in (M2 split)
|
|
790
|
+
from nbxtensor.ops import index_wrappers as _nbx_index_wrappers # the family the moved names live in (M2 split)
|
|
791
|
+
from nbxtensor.ops import linalg_wrappers as _nbx_linalg_wrappers # the family the moved names live in (M2 split)
|
|
792
|
+
from nbxtensor.ops import loss_wrappers as _nbx_loss_wrappers # the family the moved names live in (M2 split)
|
|
793
|
+
from nbxtensor.ops import norm_wrappers as _nbx_norm_wrappers # the family the moved names live in (M2 split)
|
|
794
|
+
from nbxtensor.ops import random_wrappers as _nbx_random_wrappers # the family the moved names live in (M2 split)
|
|
795
|
+
from nbxtensor.ops import reduction_wrappers as _nbx_reduction_wrappers # the family the moved names live in (M2 split)
|
|
796
|
+
from nbxtensor.ops import shape_wrappers as _nbx_shape_wrappers # the family the moved names live in (M2 split)
|
|
797
|
+
from nbxtensor.ops import spectral_wrappers as _nbx_spectral_wrappers # the family the moved names live in (M2 split)
|
|
798
|
+
|
|
799
|
+
ops = {
|
|
800
|
+
# =================================================================
|
|
801
|
+
# COMPUTE OPS — Triton GPU kernels (wrappers.py)
|
|
802
|
+
# =================================================================
|
|
803
|
+
|
|
804
|
+
# --- Activations ---
|
|
805
|
+
"relu": _nbx_elementwise_wrappers.relu,
|
|
806
|
+
"silu": _nbx_elementwise_wrappers.silu,
|
|
807
|
+
"gelu": _nbx_elementwise_wrappers.gelu,
|
|
808
|
+
"sigmoid": _nbx_elementwise_wrappers.sigmoid_wrapper,
|
|
809
|
+
"tanh": _nbx_elementwise_wrappers.tanh_wrapper,
|
|
810
|
+
"hardsigmoid": _nbx_elementwise_wrappers.hardsigmoid,
|
|
811
|
+
"hardswish": _nbx_elementwise_wrappers.hardswish,
|
|
812
|
+
"leaky_relu": _nbx_elementwise_wrappers.leaky_relu,
|
|
813
|
+
"elu": _nbx_elementwise_wrappers.elu,
|
|
814
|
+
"mish": _nbx_elementwise_wrappers.mish,
|
|
815
|
+
"selu": _nbx_elementwise_wrappers.selu_wrapper,
|
|
816
|
+
|
|
817
|
+
# --- Recurrent (triton-pure assembly; self-manages dtype) ---
|
|
818
|
+
"lstm": _nbx_elementwise_wrappers.lstm_wrapper,
|
|
819
|
+
|
|
820
|
+
# --- Unary element-wise ---
|
|
821
|
+
"neg": _nbx_elementwise_wrappers.neg,
|
|
822
|
+
"exp": _nbx_elementwise_wrappers.exp,
|
|
823
|
+
"sin": _nbx_elementwise_wrappers.sin,
|
|
824
|
+
"cos": _nbx_elementwise_wrappers.cos,
|
|
825
|
+
"rsqrt": _nbx_elementwise_wrappers.rsqrt,
|
|
826
|
+
"sqrt": _nbx_elementwise_wrappers.sqrt_wrapper,
|
|
827
|
+
"abs": _nbx_elementwise_wrappers.abs_wrapper,
|
|
828
|
+
"log": _nbx_elementwise_wrappers.log_wrapper,
|
|
829
|
+
"reciprocal": _nbx_elementwise_wrappers.reciprocal,
|
|
830
|
+
"pow": _nbx_elementwise_wrappers.pow_wrapper,
|
|
831
|
+
"clamp": _nbx_elementwise_wrappers.clamp,
|
|
832
|
+
"clip": _nbx_elementwise_wrappers.clamp, # aten::clip is an alias of aten::clamp
|
|
833
|
+
"erf": _nbx_elementwise_wrappers.erf,
|
|
834
|
+
|
|
835
|
+
# --- Binary element-wise ---
|
|
836
|
+
"add": _nbx_elementwise_wrappers.add,
|
|
837
|
+
"mul": _nbx_elementwise_wrappers.mul,
|
|
838
|
+
"div": _nbx_elementwise_wrappers.div,
|
|
839
|
+
"floor_divide": _nbx_elementwise_wrappers.floor_divide_wrapper,
|
|
840
|
+
"sub": _nbx_elementwise_wrappers.sub,
|
|
841
|
+
"rsub": _nbx_elementwise_wrappers.rsub,
|
|
842
|
+
"where": _nbx_index_wrappers.where_wrapper,
|
|
843
|
+
"maximum": _nbx_elementwise_wrappers.maximum_wrapper,
|
|
844
|
+
"minimum": _nbx_elementwise_wrappers.minimum_wrapper,
|
|
845
|
+
"masked_fill": _nbx_index_wrappers.masked_fill,
|
|
846
|
+
"masked_scatter": _nbx_index_wrappers.masked_scatter_wrapper,
|
|
847
|
+
|
|
848
|
+
# --- Comparisons ---
|
|
849
|
+
"gt": _nbx_elementwise_wrappers.gt,
|
|
850
|
+
"ge": _nbx_elementwise_wrappers.ge,
|
|
851
|
+
"lt": _nbx_elementwise_wrappers.lt,
|
|
852
|
+
"eq": _nbx_elementwise_wrappers.eq,
|
|
853
|
+
"ne": _nbx_elementwise_wrappers.ne,
|
|
854
|
+
"le": _nbx_elementwise_wrappers.le,
|
|
855
|
+
|
|
856
|
+
# --- Normalization ---
|
|
857
|
+
"native_layer_norm": _nbx_norm_wrappers.native_layer_norm,
|
|
858
|
+
"layer_norm": _nbx_norm_wrappers.layer_norm_wrapper,
|
|
859
|
+
"rms_norm": _nbx_norm_wrappers.rms_norm,
|
|
860
|
+
"swiglu_fused": _nbx_elementwise_wrappers.swiglu_fused_wrapper,
|
|
861
|
+
"rope_fused": _nbx_elementwise_wrappers.rope_fused_wrapper,
|
|
862
|
+
"native_group_norm": _nbx_norm_wrappers.native_group_norm_wrapper,
|
|
863
|
+
"group_norm": _nbx_norm_wrappers.group_norm_wrapper,
|
|
864
|
+
"cudnn_batch_norm": _nbx_norm_wrappers.batch_norm_wrapper,
|
|
865
|
+
"native_batch_norm": _nbx_norm_wrappers.batch_norm_wrapper,
|
|
866
|
+
"batch_norm": _nbx_norm_wrappers.batch_norm_wrapper,
|
|
867
|
+
|
|
868
|
+
# --- Softmax ---
|
|
869
|
+
"softmax": _nbx_attention_wrappers.softmax,
|
|
870
|
+
"_softmax": _nbx_attention_wrappers.softmax,
|
|
871
|
+
"_safe_softmax": _nbx_attention_wrappers.softmax,
|
|
872
|
+
"log_softmax": _nbx_attention_wrappers.log_softmax,
|
|
873
|
+
"_log_softmax": _nbx_attention_wrappers.log_softmax,
|
|
874
|
+
|
|
875
|
+
# --- Matmul ---
|
|
876
|
+
"mm": _nbx_linalg_wrappers.mm,
|
|
877
|
+
"bmm": _nbx_linalg_wrappers.bmm,
|
|
878
|
+
"addmm": _nbx_linalg_wrappers.addmm,
|
|
879
|
+
"mm_epilogue": _nbx_linalg_wrappers.mm_epilogue_wrapper,
|
|
880
|
+
"addmm_epilogue": _nbx_linalg_wrappers.addmm_epilogue_wrapper,
|
|
881
|
+
"matmul": _nbx_linalg_wrappers.matmul_wrapper,
|
|
882
|
+
"linear": _nbx_linalg_wrappers.linear_wrapper,
|
|
883
|
+
"isin": _nbx_elementwise_wrappers.isin_wrapper,
|
|
884
|
+
"is_nonzero": _nbx_index_wrappers.is_nonzero_wrapper,
|
|
885
|
+
|
|
886
|
+
# --- Embedding ---
|
|
887
|
+
"embedding": _nbx_index_wrappers.embedding,
|
|
888
|
+
|
|
889
|
+
# --- Reductions ---
|
|
890
|
+
"mean": _nbx_reduction_wrappers.mean_wrapper,
|
|
891
|
+
"sum": _nbx_reduction_wrappers.sum_wrapper,
|
|
892
|
+
"norm": _nbx_norm_wrappers.norm_wrapper,
|
|
893
|
+
"amax": _nbx_reduction_wrappers.amax_wrapper,
|
|
894
|
+
"argmax": _nbx_reduction_wrappers.argmax_wrapper,
|
|
895
|
+
"argmin": _nbx_reduction_wrappers.argmin_wrapper,
|
|
896
|
+
"prod": _nbx_reduction_wrappers.prod_wrapper,
|
|
897
|
+
"min": _nbx_elementwise_wrappers.min_wrapper,
|
|
898
|
+
"max": _nbx_elementwise_wrappers.max_wrapper,
|
|
899
|
+
"std": _nbx_reduction_wrappers.std_wrapper,
|
|
900
|
+
"var": _nbx_reduction_wrappers.var_wrapper,
|
|
901
|
+
|
|
902
|
+
# --- GLU ---
|
|
903
|
+
"glu": _nbx_elementwise_wrappers.glu_wrapper,
|
|
904
|
+
|
|
905
|
+
# --- Spatial ---
|
|
906
|
+
"upsample_nearest1d": _nbx_shape_wrappers.upsample_nearest1d_wrapper,
|
|
907
|
+
"upsample_nearest2d": _nbx_shape_wrappers.upsample_nearest2d_wrapper,
|
|
908
|
+
# nearest-exact variants: bit-identical to nearest for integer ×2
|
|
909
|
+
# (every Wan VAE upsample); reuse the nearest wrappers.
|
|
910
|
+
"_upsample_nearest_exact1d": _nbx_shape_wrappers.upsample_nearest1d_wrapper,
|
|
911
|
+
"_upsample_nearest_exact2d": _nbx_shape_wrappers.upsample_nearest2d_wrapper,
|
|
912
|
+
"upsample_nearest3d": _nbx_shape_wrappers.upsample_nearest3d_wrapper,
|
|
913
|
+
"upsample_bilinear2d": _nbx_linalg_wrappers.upsample_bilinear2d_wrapper,
|
|
914
|
+
"upsample_linear1d": _nbx_linalg_wrappers.upsample_linear1d_wrapper,
|
|
915
|
+
"interpolate": _nbx_shape_wrappers.interpolate_wrapper,
|
|
916
|
+
"adaptive_avg_pool2d": _nbx_conv_wrappers.adaptive_avg_pool2d_wrapper,
|
|
917
|
+
"avg_pool2d": _nbx_conv_wrappers.avg_pool2d_wrapper,
|
|
918
|
+
"max_pool2d": _nbx_conv_wrappers.max_pool2d_wrapper,
|
|
919
|
+
# with_indices variants return (values, indices) tuples — the
|
|
920
|
+
# sequence binds them to ::out_0 / ::out_1.
|
|
921
|
+
"max_pool2d_with_indices": (
|
|
922
|
+
lambda x, kernel_size, stride=None, padding=0, dilation=1,
|
|
923
|
+
ceil_mode=False:
|
|
924
|
+
_nbx_conv_wrappers.max_pool2d_wrapper(x, kernel_size, stride, padding, dilation,
|
|
925
|
+
ceil_mode, return_indices=True)),
|
|
926
|
+
"max_pool3d": _nbx_conv_wrappers.max_pool3d_wrapper,
|
|
927
|
+
"max_pool3d_with_indices": _nbx_conv_wrappers.max_pool3d_with_indices_wrapper,
|
|
928
|
+
|
|
929
|
+
# --- Conv ---
|
|
930
|
+
"conv2d": _nbx_conv_wrappers.conv2d_wrapper,
|
|
931
|
+
"convolution": _nbx_conv_wrappers.conv2d_wrapper,
|
|
932
|
+
"conv1d": _nbx_conv_wrappers.conv1d_wrapper,
|
|
933
|
+
"conv_transpose1d": _nbx_conv_wrappers.conv_transpose1d_wrapper,
|
|
934
|
+
"depthwise_conv2d": _nbx_conv_wrappers.conv_depthwise2d_wrapper,
|
|
935
|
+
"conv_depthwise2d": _nbx_conv_wrappers.conv_depthwise2d_wrapper,
|
|
936
|
+
"conv_transpose2d": _nbx_conv_wrappers.conv_depthwise2d_wrapper,
|
|
937
|
+
|
|
938
|
+
# --- Indexing ---
|
|
939
|
+
"index_select": _nbx_index_wrappers.index_select_wrapper,
|
|
940
|
+
"gather": _nbx_index_wrappers.gather_wrapper,
|
|
941
|
+
"scatter": _nbx_index_wrappers.scatter_wrapper,
|
|
942
|
+
"scatter_add": _nbx_index_wrappers.scatter_add_wrapper,
|
|
943
|
+
"scatter_reduce": _nbx_reduction_wrappers.scatter_reduce_wrapper,
|
|
944
|
+
"index_add": _nbx_index_wrappers.index_add_wrapper,
|
|
945
|
+
"bincount": _nbx_elementwise_wrappers.bincount_wrapper,
|
|
946
|
+
|
|
947
|
+
# --- Triangular ---
|
|
948
|
+
"triu": _nbx_shape_wrappers.triu_wrapper,
|
|
949
|
+
"tril": _nbx_shape_wrappers.tril_wrapper,
|
|
950
|
+
|
|
951
|
+
# --- Cumulative ---
|
|
952
|
+
"cumsum": _nbx_reduction_wrappers.cumsum_wrapper,
|
|
953
|
+
|
|
954
|
+
# --- Dropout (inference = identity) ---
|
|
955
|
+
"dropout": _nbx_random_wrappers.dropout_wrapper,
|
|
956
|
+
"native_dropout": _nbx_random_wrappers.dropout_wrapper,
|
|
957
|
+
|
|
958
|
+
# --- Misc element-wise ---
|
|
959
|
+
"remainder": _nbx_elementwise_wrappers.remainder_wrapper,
|
|
960
|
+
"exp2": _nbx_elementwise_wrappers.exp2_wrapper,
|
|
961
|
+
"tan": _nbx_elementwise_wrappers.tan_wrapper,
|
|
962
|
+
"celu": _nbx_elementwise_wrappers.celu_wrapper,
|
|
963
|
+
"log_sigmoid": _nbx_elementwise_wrappers.log_sigmoid_wrapper,
|
|
964
|
+
"softplus": _nbx_elementwise_wrappers.softplus_wrapper,
|
|
965
|
+
"isfinite": _nbx_elementwise_wrappers.isfinite_wrapper,
|
|
966
|
+
"isinf": _nbx_elementwise_wrappers.isinf_wrapper,
|
|
967
|
+
"isnan": _nbx_elementwise_wrappers.isnan_wrapper,
|
|
968
|
+
"nan_to_num": _nbx_elementwise_wrappers.nan_to_num_wrapper,
|
|
969
|
+
"threshold": _nbx_elementwise_wrappers.threshold_wrapper,
|
|
970
|
+
"addcdiv": _nbx_elementwise_wrappers.addcdiv_wrapper,
|
|
971
|
+
"addcmul": _nbx_elementwise_wrappers.addcmul_wrapper,
|
|
972
|
+
"lerp": _nbx_elementwise_wrappers.lerp_wrapper,
|
|
973
|
+
|
|
974
|
+
# --- Logic/bitwise ---
|
|
975
|
+
"all": _nbx_reduction_wrappers.all_wrapper,
|
|
976
|
+
"any": _nbx_reduction_wrappers.any_wrapper,
|
|
977
|
+
"bitwise_and": _nbx_elementwise_wrappers.bitwise_and_wrapper,
|
|
978
|
+
"bitwise_or": _nbx_elementwise_wrappers.bitwise_or_wrapper,
|
|
979
|
+
"bitwise_not": _nbx_elementwise_wrappers.bitwise_not_wrapper,
|
|
980
|
+
"logical_and": _nbx_elementwise_wrappers.logical_and_wrapper,
|
|
981
|
+
"logical_or": _nbx_elementwise_wrappers.logical_or_wrapper,
|
|
982
|
+
"logical_not": _nbx_elementwise_wrappers.logical_not_wrapper,
|
|
983
|
+
"logical_xor": _nbx_elementwise_wrappers.logical_xor_wrapper,
|
|
984
|
+
"bitwise_left_shift": _nbx_elementwise_wrappers.bitwise_left_shift_wrapper,
|
|
985
|
+
"bitwise_right_shift": _nbx_elementwise_wrappers.bitwise_right_shift_wrapper,
|
|
986
|
+
|
|
987
|
+
# --- Linear algebra ---
|
|
988
|
+
"dot": _nbx_linalg_wrappers.dot_wrapper,
|
|
989
|
+
"mv": _nbx_linalg_wrappers.mv_wrapper,
|
|
990
|
+
"baddbmm": _nbx_linalg_wrappers.baddbmm_wrapper,
|
|
991
|
+
"_weight_norm": _meta_weight_norm,
|
|
992
|
+
"addmv": _nbx_linalg_wrappers.addmv_wrapper,
|
|
993
|
+
"linalg_vector_norm": _nbx_norm_wrappers.vector_norm_wrapper,
|
|
994
|
+
"addr": _nbx_elementwise_wrappers.addr_wrapper,
|
|
995
|
+
"trace": _nbx_elementwise_wrappers.trace_wrapper,
|
|
996
|
+
|
|
997
|
+
# --- Loss ---
|
|
998
|
+
"mse_loss": _nbx_loss_wrappers.mse_loss_wrapper,
|
|
999
|
+
"nll_loss": _nbx_loss_wrappers.nll_loss_wrapper,
|
|
1000
|
+
"cross_entropy_loss": _nbx_loss_wrappers.cross_entropy_wrapper,
|
|
1001
|
+
|
|
1002
|
+
# --- Sort/topk ---
|
|
1003
|
+
"sort": _nbx_elementwise_wrappers.sort_wrapper,
|
|
1004
|
+
"topk": _nbx_elementwise_wrappers.topk_wrapper,
|
|
1005
|
+
"flip": _nbx_shape_wrappers.flip_wrapper,
|
|
1006
|
+
"roll": _nbx_shape_wrappers.roll_wrapper,
|
|
1007
|
+
|
|
1008
|
+
"pixel_shuffle": _nbx_shape_wrappers.pixel_shuffle_wrapper,
|
|
1009
|
+
"pixel_unshuffle": _nbx_shape_wrappers.pixel_unshuffle_wrapper,
|
|
1010
|
+
|
|
1011
|
+
# --- Clamp ---
|
|
1012
|
+
"clamp_min": _nbx_elementwise_wrappers.clamp_min_wrapper,
|
|
1013
|
+
|
|
1014
|
+
# --- Weight norm ---
|
|
1015
|
+
"_weight_norm_interface": _nbx_norm_wrappers.weight_norm_interface_wrapper,
|
|
1016
|
+
"repeat_interleave": _nbx_shape_wrappers.repeat_interleave_wrapper,
|
|
1017
|
+
|
|
1018
|
+
# --- RNG ---
|
|
1019
|
+
"rand": _nbx_random_wrappers.rand_wrapper,
|
|
1020
|
+
"randn": _nbx_random_wrappers.randn_wrapper,
|
|
1021
|
+
"rand_like": _nbx_random_wrappers.rand_like_wrapper,
|
|
1022
|
+
"randn_like": _nbx_random_wrappers.randn_like_wrapper,
|
|
1023
|
+
"normal": _nbx_norm_wrappers.normal_wrapper,
|
|
1024
|
+
"uniform": _nbx_random_wrappers.uniform_wrapper,
|
|
1025
|
+
"bernoulli": _nbx_elementwise_wrappers.bernoulli_wrapper,
|
|
1026
|
+
"multinomial": _nbx_random_wrappers.multinomial_wrapper,
|
|
1027
|
+
|
|
1028
|
+
# --- Padding ---
|
|
1029
|
+
"constant_pad_nd": _nbx_shape_wrappers.constant_pad_nd_wrapper,
|
|
1030
|
+
"pad": _nbx_shape_wrappers.pad_wrapper,
|
|
1031
|
+
"reflection_pad1d": _nbx_shape_wrappers.reflection_pad1d_wrapper,
|
|
1032
|
+
"reflection_pad2d": _nbx_shape_wrappers.reflection_pad2d_wrapper,
|
|
1033
|
+
"reflection_pad3d": _nbx_shape_wrappers.reflection_pad3d_wrapper,
|
|
1034
|
+
"replication_pad1d": _nbx_shape_wrappers.replication_pad1d_wrapper,
|
|
1035
|
+
"replication_pad2d": _nbx_shape_wrappers.replication_pad2d_wrapper,
|
|
1036
|
+
"replication_pad3d": _nbx_shape_wrappers.replication_pad3d_wrapper,
|
|
1037
|
+
|
|
1038
|
+
# --- FFT ---
|
|
1039
|
+
"_fft_r2c": _nbx_spectral_wrappers.fft_r2c_wrapper,
|
|
1040
|
+
"_fft_c2r": _nbx_spectral_wrappers.fft_c2r_wrapper,
|
|
1041
|
+
"_fft_c2c": _nbx_spectral_wrappers.fft_c2c_wrapper,
|
|
1042
|
+
"fft_rfft": _nbx_spectral_wrappers.fft_rfft_wrapper,
|
|
1043
|
+
"fft_irfft": _nbx_spectral_wrappers.fft_irfft_wrapper,
|
|
1044
|
+
"stft": _nbx_spectral_wrappers.stft_wrapper,
|
|
1045
|
+
"istft": _nbx_spectral_wrappers.istft_wrapper,
|
|
1046
|
+
|
|
1047
|
+
# --- Complex ---
|
|
1048
|
+
"angle": _nbx_elementwise_wrappers.angle_wrapper,
|
|
1049
|
+
|
|
1050
|
+
# Attention — Dao-AILab Flash Attention v2 Triton kernel
|
|
1051
|
+
"scaled_dot_product_attention": _nbx_attention_wrappers.scaled_dot_product_attention_wrapper,
|
|
1052
|
+
"_scaled_dot_product_attention": _nbx_attention_wrappers.scaled_dot_product_attention_wrapper,
|
|
1053
|
+
# Fused-backend variants have a shifted positional signature
|
|
1054
|
+
# (extra compute_log_sumexp / return_debug_mask arg). Route through the
|
|
1055
|
+
# remap shim so is_causal / scale read from the right slots — mirrors
|
|
1056
|
+
# triton_sequential (sequential.py:198) for R30 symmetry.
|
|
1057
|
+
# CAVEAT: _meta_sdpa_efficient reads is_causal from arg[6], correct for
|
|
1058
|
+
# efficient + cudnn. The flash variant actually carries is_causal at
|
|
1059
|
+
# arg[4] (no attn_bias slot), so a CAUSAL flash op would lose its mask
|
|
1060
|
+
# here. No flash-backend model is in the current sweep; if one appears,
|
|
1061
|
+
# give flash a dedicated remap that reads arg[4] (cf. the KV-cache path
|
|
1062
|
+
# kv_cache.py::intercept_flash, which already does).
|
|
1063
|
+
"_scaled_dot_product_flash_attention": _meta_sdpa_efficient,
|
|
1064
|
+
"_scaled_dot_product_efficient_attention": _meta_sdpa_efficient,
|
|
1065
|
+
"_scaled_dot_product_cudnn_attention": _meta_sdpa_efficient,
|
|
1066
|
+
"_scaled_dot_product_flash_attention_for_cpu": _nbx_attention_wrappers.scaled_dot_product_attention_wrapper,
|
|
1067
|
+
"complex": _nbx_elementwise_wrappers.complex_wrapper,
|
|
1068
|
+
"fold": _nbx_elementwise_wrappers.fold_wrapper,
|
|
1069
|
+
"unfold_backward": _nbx_conv_wrappers.unfold_backward_wrapper,
|
|
1070
|
+
|
|
1071
|
+
# =================================================================
|
|
1072
|
+
# METADATA OPS — CPU stride math, NBXTensor methods
|
|
1073
|
+
# =================================================================
|
|
1074
|
+
|
|
1075
|
+
# View/reshape
|
|
1076
|
+
"view": _meta_view,
|
|
1077
|
+
"_unsafe_view": _meta_view,
|
|
1078
|
+
"reshape": _meta_reshape,
|
|
1079
|
+
"flatten": _meta_flatten,
|
|
1080
|
+
"squeeze": _meta_squeeze,
|
|
1081
|
+
"unsqueeze": _meta_unsqueeze,
|
|
1082
|
+
"permute": _meta_permute,
|
|
1083
|
+
"transpose": _meta_transpose,
|
|
1084
|
+
"t": _meta_t,
|
|
1085
|
+
"contiguous": _meta_contiguous,
|
|
1086
|
+
"expand": _meta_expand,
|
|
1087
|
+
"expand_as": _meta_expand_as,
|
|
1088
|
+
"repeat": _meta_repeat,
|
|
1089
|
+
"narrow": _meta_narrow,
|
|
1090
|
+
"slice": _meta_slice,
|
|
1091
|
+
"select": _meta_select,
|
|
1092
|
+
"slice_scatter": _meta_slice_scatter,
|
|
1093
|
+
"select_scatter": _meta_select_scatter,
|
|
1094
|
+
"unbind": _meta_unbind,
|
|
1095
|
+
# NOTE: split_with_sizes is bound ONCE, in the Split/chunk section
|
|
1096
|
+
# below (narrow-based multi-output tuple). A historical
|
|
1097
|
+
# `"split_with_sizes": _meta_slice # simplified` entry sat here and
|
|
1098
|
+
# was only harmless because the later duplicate key in this same
|
|
1099
|
+
# dict literal shadowed it — removed so a reorder can never
|
|
1100
|
+
# resurrect the wrong (single-output, wrong-signature) binding
|
|
1101
|
+
# (P-TRITON-MLA: the MLA q nope/rope split [128, 64] rides on the
|
|
1102
|
+
# narrow-based binding).
|
|
1103
|
+
"bucketize": _nbx_index_wrappers.bucketize_wrapper,
|
|
1104
|
+
"view_as": _meta_view_as,
|
|
1105
|
+
"reshape_as": _meta_reshape_as,
|
|
1106
|
+
"as_strided": _meta_as_strided,
|
|
1107
|
+
"unfold": _meta_unfold,
|
|
1108
|
+
"im2col": _nbx_conv_wrappers.im2col_wrapper,
|
|
1109
|
+
"movedim": _meta_movedim,
|
|
1110
|
+
|
|
1111
|
+
# Memory
|
|
1112
|
+
"clone": _meta_clone,
|
|
1113
|
+
"copy": _meta_copy,
|
|
1114
|
+
"copy_": _meta_copy,
|
|
1115
|
+
"detach": _meta_detach,
|
|
1116
|
+
"alias": _meta_alias,
|
|
1117
|
+
"lift_fresh": _meta_alias,
|
|
1118
|
+
|
|
1119
|
+
# Fill
|
|
1120
|
+
"fill": _meta_fill,
|
|
1121
|
+
"fill_": _meta_fill,
|
|
1122
|
+
|
|
1123
|
+
# Type conversion
|
|
1124
|
+
"to": _meta_to,
|
|
1125
|
+
"_to_copy": _meta_to,
|
|
1126
|
+
"type_as": _meta_type_as,
|
|
1127
|
+
"float": lambda x: x.float(),
|
|
1128
|
+
"half": lambda x: x.half(),
|
|
1129
|
+
"bfloat16": lambda x: x.bfloat16(),
|
|
1130
|
+
"int": lambda x: x.int(),
|
|
1131
|
+
"long": lambda x: x.long(),
|
|
1132
|
+
|
|
1133
|
+
# Math rounding ops — proper Triton kernels
|
|
1134
|
+
"floor": _nbx_elementwise_wrappers.floor_wrapper,
|
|
1135
|
+
"ceil": _nbx_elementwise_wrappers.ceil_wrapper,
|
|
1136
|
+
|
|
1137
|
+
# 2D grid sampler (VLM pos-embed adaptation: bicubic/bilinear)
|
|
1138
|
+
"grid_sampler_2d": _nbx_shape_wrappers.grid_sampler_2d_wrapper,
|
|
1139
|
+
"round": _nbx_elementwise_wrappers.round_wrapper,
|
|
1140
|
+
"trunc": _nbx_elementwise_wrappers.trunc_wrapper,
|
|
1141
|
+
|
|
1142
|
+
# Scalar
|
|
1143
|
+
"item": _meta_select_item,
|
|
1144
|
+
"_local_scalar_dense": _meta_select_item,
|
|
1145
|
+
"equal": _meta_equal,
|
|
1146
|
+
|
|
1147
|
+
# Complex views — real reinterprets (complex64 ↔ float[..,2]), see
|
|
1148
|
+
# NBXTensor.view_as_real/view_as_complex/.real/.imag.
|
|
1149
|
+
"view_as_real": lambda x: x.view_as_real(),
|
|
1150
|
+
"view_as_complex": lambda x: x.view_as_complex(),
|
|
1151
|
+
"real": lambda x: x.real,
|
|
1152
|
+
"imag": lambda x: x.imag,
|
|
1153
|
+
|
|
1154
|
+
# Queries (return Python scalars, not tensors)
|
|
1155
|
+
"size": lambda x, dim=None: x.size(dim) if dim is not None else x.size(),
|
|
1156
|
+
"dim": lambda x: x.dim(),
|
|
1157
|
+
"numel": lambda x: x.numel(),
|
|
1158
|
+
"stride": lambda x, dim=None: x.stride(dim) if dim is not None else x.stride(),
|
|
1159
|
+
"is_contiguous": lambda x: x.is_contiguous,
|
|
1160
|
+
|
|
1161
|
+
# Broadcast
|
|
1162
|
+
"broadcast_tensors": _nbx_shape_wrappers.broadcast_tensors_wrapper,
|
|
1163
|
+
|
|
1164
|
+
# Advanced indexing — aten::index(tensor, [idx0, idx1, ...])
|
|
1165
|
+
"index": _meta_index,
|
|
1166
|
+
"index_put": _nbx_index_wrappers.index_put_wrapper,
|
|
1167
|
+
"index_put_": _nbx_index_wrappers.index_put_wrapper,
|
|
1168
|
+
|
|
1169
|
+
# Split/chunk (use narrow internally). torch.split semantics:
|
|
1170
|
+
# ceil(shape/size) chunks, the LAST carries the remainder — the
|
|
1171
|
+
# floor-division form returned an EMPTY tuple whenever
|
|
1172
|
+
# split_size > shape[dim] (the audio stem's conv_chunksize 500
|
|
1173
|
+
# over 5 windows → None into the first convolution) and silently
|
|
1174
|
+
# DROPPED the tail chunk in the general case.
|
|
1175
|
+
"split": lambda x, size, dim=0: tuple(
|
|
1176
|
+
x.narrow(dim, i * size, min(size, x.shape[dim] - i * size))
|
|
1177
|
+
for i in range((x.shape[dim] + size - 1) // size)),
|
|
1178
|
+
"split_with_sizes": lambda x, sizes, dim=0: tuple(
|
|
1179
|
+
x.narrow(dim, sum(sizes[:i]), s) for i, s in enumerate(sizes)),
|
|
1180
|
+
"chunk": _chunk_narrow,
|
|
1181
|
+
"tensor_split": lambda x, sections, dim=0: (x,),
|
|
1182
|
+
|
|
1183
|
+
# Atleast
|
|
1184
|
+
"atleast_1d": lambda *t: t[0] if len(t) == 1 else t,
|
|
1185
|
+
"atleast_2d": lambda *t: t[0] if len(t) == 1 else t,
|
|
1186
|
+
|
|
1187
|
+
# Set/pack (identity)
|
|
1188
|
+
"set": lambda x, *a, **k: x,
|
|
1189
|
+
"_pack_padded_sequence": lambda inp, lengths, bf=False: inp,
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
# Add creation ops
|
|
1193
|
+
ops.update(_build_creation_ops())
|
|
1194
|
+
|
|
1195
|
+
return ops
|
|
1196
|
+
|
|
1197
|
+
|
|
1198
|
+
# ============================================================================
|
|
1199
|
+
# PUBLIC API — ONE function, ONE path
|
|
1200
|
+
# ============================================================================
|
|
1201
|
+
|
|
1202
|
+
def dispatch(op_type: str) -> Optional[Callable]:
|
|
1203
|
+
"""Look up handler for ANY aten op.
|
|
1204
|
+
|
|
1205
|
+
Returns function if found, None if not.
|
|
1206
|
+
In --triton mode, None = CRASH (compiled_ops.py enforces this).
|
|
1207
|
+
|
|
1208
|
+
ONE table. Compute + metadata + creation. No second path.
|
|
1209
|
+
"""
|
|
1210
|
+
global _OP_MAP
|
|
1211
|
+
if _OP_MAP is None:
|
|
1212
|
+
_OP_MAP = _build_op_map()
|
|
1213
|
+
|
|
1214
|
+
# The library's own op names, as given: a graph's aliases (aten::multiply for mul) are the application's to
|
|
1215
|
+
# resolve before it asks (NeuroBrix: neurobrix.kernels.classification.kernel_for).
|
|
1216
|
+
op_name = op_type.split("::")[-1] if "::" in op_type else op_type
|
|
1217
|
+
return _OP_MAP.get(op_name)
|
|
1218
|
+
|
|
1219
|
+
|
|
1220
|
+
def has_kernel(op_type: str) -> bool:
|
|
1221
|
+
return dispatch(op_type) is not None
|
|
1222
|
+
|
|
1223
|
+
|
|
1224
|
+
def list_kernels() -> list:
|
|
1225
|
+
global _OP_MAP
|
|
1226
|
+
if _OP_MAP is None:
|
|
1227
|
+
_OP_MAP = _build_op_map()
|
|
1228
|
+
return sorted(_OP_MAP.keys())
|