tensorc 0.12.0__tar.gz → 0.13.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tensorc-0.12.0/tensorc.egg-info → tensorc-0.13.0}/PKG-INFO +1 -1
- {tensorc-0.12.0 → tensorc-0.13.0}/pyproject.toml +1 -1
- tensorc-0.13.0/tensorc/__init__.py +29 -0
- {tensorc-0.12.0 → tensorc-0.13.0}/tensorc/_tensorc.c +157 -0
- {tensorc-0.12.0 → tensorc-0.13.0}/tensorc/core/include/tensorc/ndtensor.h +33 -0
- {tensorc-0.12.0 → tensorc-0.13.0}/tensorc/core/include/tensorc/tensorc.h +1 -1
- {tensorc-0.12.0 → tensorc-0.13.0}/tensorc/core/ndtensor.c +310 -1
- {tensorc-0.12.0 → tensorc-0.13.0/tensorc.egg-info}/PKG-INFO +1 -1
- tensorc-0.12.0/tensorc/__init__.py +0 -13
- {tensorc-0.12.0 → tensorc-0.13.0}/MANIFEST.in +0 -0
- {tensorc-0.12.0 → tensorc-0.13.0}/README.md +0 -0
- {tensorc-0.12.0 → tensorc-0.13.0}/setup.cfg +0 -0
- {tensorc-0.12.0 → tensorc-0.13.0}/setup.py +0 -0
- {tensorc-0.12.0 → tensorc-0.13.0}/sync_core.sh +0 -0
- {tensorc-0.12.0 → tensorc-0.13.0}/tensorc/core/minitf.c +0 -0
- {tensorc-0.12.0 → tensorc-0.13.0}/tensorc/core/minitf.h +0 -0
- {tensorc-0.12.0 → tensorc-0.13.0}/tensorc.egg-info/SOURCES.txt +0 -0
- {tensorc-0.12.0 → tensorc-0.13.0}/tensorc.egg-info/dependency_links.txt +0 -0
- {tensorc-0.12.0 → tensorc-0.13.0}/tensorc.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""TensorC: neural network framework written in C11, with autograd.
|
|
2
|
+
|
|
3
|
+
The numerical core is a C library; this package is a thin CPython
|
|
4
|
+
extension that wraps it. Tensors keep their parents alive automatically,
|
|
5
|
+
so ``backward()`` can be called from any scalar loss.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from ._tensorc import (
|
|
9
|
+
Adam,
|
|
10
|
+
Conv2DLayer,
|
|
11
|
+
DenseLayer,
|
|
12
|
+
LSTMLayer,
|
|
13
|
+
Tensor,
|
|
14
|
+
TransformerBlock,
|
|
15
|
+
sinusoidal_positions,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
__version__ = "0.13.0"
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"Tensor",
|
|
22
|
+
"Conv2DLayer",
|
|
23
|
+
"DenseLayer",
|
|
24
|
+
"LSTMLayer",
|
|
25
|
+
"TransformerBlock",
|
|
26
|
+
"Adam",
|
|
27
|
+
"sinusoidal_positions",
|
|
28
|
+
"__version__",
|
|
29
|
+
]
|
|
@@ -426,6 +426,49 @@ static PyObject *tensor_attention(TensorObject *self, PyObject *args,
|
|
|
426
426
|
tc_tensor_nd_attention(self->tensor, key, value, causal));
|
|
427
427
|
}
|
|
428
428
|
|
|
429
|
+
static PyObject *tensor_permute(TensorObject *self, PyObject *args) {
|
|
430
|
+
PyObject *axes_obj;
|
|
431
|
+
if (!PyArg_ParseTuple(args, "O", &axes_obj)) return NULL;
|
|
432
|
+
PyObject *seq = PySequence_Fast(axes_obj, "axes debe ser una secuencia");
|
|
433
|
+
if (!seq) return NULL;
|
|
434
|
+
const Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
|
|
435
|
+
size_t axes[8];
|
|
436
|
+
if (n < 1 || n > 8) {
|
|
437
|
+
Py_DECREF(seq);
|
|
438
|
+
PyErr_SetString(PyExc_ValueError, "axes debe tener entre 1 y 8 ejes");
|
|
439
|
+
return NULL;
|
|
440
|
+
}
|
|
441
|
+
for (Py_ssize_t i = 0; i < n; ++i) {
|
|
442
|
+
const long a = PyLong_AsLong(PySequence_Fast_GET_ITEM(seq, i));
|
|
443
|
+
if (a < 0 || PyErr_Occurred()) {
|
|
444
|
+
Py_DECREF(seq);
|
|
445
|
+
PyErr_SetString(PyExc_ValueError, "ejes inválidos en axes");
|
|
446
|
+
return NULL;
|
|
447
|
+
}
|
|
448
|
+
axes[i] = (size_t)a;
|
|
449
|
+
}
|
|
450
|
+
Py_DECREF(seq);
|
|
451
|
+
return (PyObject *)tensor_wrap(
|
|
452
|
+
tc_tensor_nd_permute(self->tensor, axes, (size_t)n));
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
static PyObject *tensor_multihead_attention(TensorObject *self, PyObject *args,
|
|
456
|
+
PyObject *kwargs) {
|
|
457
|
+
static char *names[] = {"key", "value", "heads", "causal", NULL};
|
|
458
|
+
PyObject *key_obj, *value_obj;
|
|
459
|
+
unsigned long heads;
|
|
460
|
+
int causal = 0;
|
|
461
|
+
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OOk|p", names, &key_obj,
|
|
462
|
+
&value_obj, &heads, &causal))
|
|
463
|
+
return NULL;
|
|
464
|
+
tc_tensor_nd *key = as_tensor(key_obj);
|
|
465
|
+
if (!key) return NULL;
|
|
466
|
+
tc_tensor_nd *value = as_tensor(value_obj);
|
|
467
|
+
if (!value) return NULL;
|
|
468
|
+
return (PyObject *)tensor_wrap(tc_tensor_nd_multihead_attention(
|
|
469
|
+
self->tensor, key, value, (size_t)heads, causal));
|
|
470
|
+
}
|
|
471
|
+
|
|
429
472
|
static PyMethodDef tensor_methods[] = {
|
|
430
473
|
{"zeros", (PyCFunction)tensor_zeros, METH_VARARGS | METH_KEYWORDS | METH_CLASS,
|
|
431
474
|
"Tensor.zeros(shape, requires_grad=False)"},
|
|
@@ -462,6 +505,11 @@ static PyMethodDef tensor_methods[] = {
|
|
|
462
505
|
{"attention", (PyCFunction)tensor_attention,
|
|
463
506
|
METH_VARARGS | METH_KEYWORDS,
|
|
464
507
|
"attention(key, value, causal=False) sobre [batch, seq, depth]."},
|
|
508
|
+
{"permute", (PyCFunction)tensor_permute, METH_VARARGS,
|
|
509
|
+
"permute(axes): out[i] = in[axes[i]] por dimensión."},
|
|
510
|
+
{"multihead_attention", (PyCFunction)tensor_multihead_attention,
|
|
511
|
+
METH_VARARGS | METH_KEYWORDS,
|
|
512
|
+
"multihead_attention(key, value, heads, causal=False) sobre [batch, seq, depth]."},
|
|
465
513
|
{NULL, NULL, 0, NULL}
|
|
466
514
|
};
|
|
467
515
|
|
|
@@ -573,7 +621,13 @@ static PyTypeObject LayerType = {
|
|
|
573
621
|
.tp_doc = "Trainable 2D convolution layer.",
|
|
574
622
|
};
|
|
575
623
|
|
|
624
|
+
/* Definida más abajo; declaración anticipada para la tabla de métodos. */
|
|
625
|
+
static PyObject *module_sinusoidal_positions(PyObject *self, PyObject *args);
|
|
626
|
+
|
|
576
627
|
static PyMethodDef module_methods[] = {
|
|
628
|
+
{"sinusoidal_positions", (PyCFunction)module_sinusoidal_positions,
|
|
629
|
+
METH_VARARGS,
|
|
630
|
+
"sinusoidal_positions(seq, depth) -> codificación posicional [seq, depth]."},
|
|
577
631
|
{NULL, NULL, 0, NULL}
|
|
578
632
|
};
|
|
579
633
|
|
|
@@ -743,6 +797,101 @@ static PyTypeObject LSTMLayerType = {
|
|
|
743
797
|
"(time-major, h0/c0 en cero).",
|
|
744
798
|
};
|
|
745
799
|
|
|
800
|
+
/* --- Capa Transformer (bloque pre-norm) --- */
|
|
801
|
+
typedef struct {
|
|
802
|
+
PyObject_HEAD
|
|
803
|
+
tc_transformer_block *block;
|
|
804
|
+
} TransformerBlockObject;
|
|
805
|
+
|
|
806
|
+
static PyTypeObject TransformerBlockType;
|
|
807
|
+
|
|
808
|
+
static void transformer_block_dealloc(TransformerBlockObject *self) {
|
|
809
|
+
tc_transformer_block_free(self->block);
|
|
810
|
+
PyObject_Del(self);
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
/* forward(x): x es [batch, seq, model_dim]; devuelve la misma forma. */
|
|
814
|
+
static PyObject *transformer_block_forward(TransformerBlockObject *self,
|
|
815
|
+
PyObject *args) {
|
|
816
|
+
PyObject *input_obj;
|
|
817
|
+
if (!PyArg_ParseTuple(args, "O", &input_obj)) return NULL;
|
|
818
|
+
tc_tensor_nd *input = as_tensor(input_obj);
|
|
819
|
+
if (!input) return NULL;
|
|
820
|
+
return (PyObject *)tensor_wrap(
|
|
821
|
+
tc_transformer_block_forward(self->block, input));
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
/* params: lista de los 8 tensores entrenables (para Adam). */
|
|
825
|
+
static PyObject *transformer_block_params(TransformerBlockObject *self,
|
|
826
|
+
void *closure) {
|
|
827
|
+
(void)closure;
|
|
828
|
+
const size_t count = tc_transformer_block_param_count(self->block);
|
|
829
|
+
PyObject *list = PyList_New((Py_ssize_t)count);
|
|
830
|
+
if (!list) return NULL;
|
|
831
|
+
for (size_t i = 0; i < count; ++i) {
|
|
832
|
+
tc_tensor_nd_retain(tc_transformer_block_param(self->block, i));
|
|
833
|
+
PyList_SET_ITEM(list, (Py_ssize_t)i,
|
|
834
|
+
(PyObject *)tensor_wrap(
|
|
835
|
+
tc_transformer_block_param(self->block, i)));
|
|
836
|
+
}
|
|
837
|
+
return list;
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
static PyObject *transformer_block_new(PyTypeObject *type, PyObject *args,
|
|
841
|
+
PyObject *kwargs) {
|
|
842
|
+
static char *names[] = {"model_dim", "ff_dim", "heads", "seed", NULL};
|
|
843
|
+
unsigned long model_dim, ff_dim, heads, seed = 1;
|
|
844
|
+
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "kkk|k", names, &model_dim,
|
|
845
|
+
&ff_dim, &heads, &seed))
|
|
846
|
+
return NULL;
|
|
847
|
+
TransformerBlockObject *obj =
|
|
848
|
+
PyObject_New(TransformerBlockObject, &TransformerBlockType);
|
|
849
|
+
if (!obj) return NULL;
|
|
850
|
+
obj->block = tc_transformer_block_create(model_dim, ff_dim, heads,
|
|
851
|
+
(unsigned int)seed);
|
|
852
|
+
if (!obj->block) {
|
|
853
|
+
PyObject_Del(obj);
|
|
854
|
+
PyErr_SetString(PyExc_RuntimeError,
|
|
855
|
+
"no se pudo crear el bloque transformer "
|
|
856
|
+
"(model_dim debe ser divisible entre heads)");
|
|
857
|
+
return NULL;
|
|
858
|
+
}
|
|
859
|
+
return (PyObject *)obj;
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
static PyMethodDef transformer_block_methods[] = {
|
|
863
|
+
{"forward", (PyCFunction)transformer_block_forward, METH_VARARGS,
|
|
864
|
+
"forward(x) con x [batch, seq, model_dim] -> misma forma"},
|
|
865
|
+
{NULL, NULL, 0, NULL}
|
|
866
|
+
};
|
|
867
|
+
|
|
868
|
+
static PyGetSetDef transformer_block_getset[] = {
|
|
869
|
+
{"params", (getter)transformer_block_params, NULL,
|
|
870
|
+
"Lista de los 8 tensores entrenables (para Adam).", NULL},
|
|
871
|
+
{NULL, NULL, NULL, NULL, NULL}
|
|
872
|
+
};
|
|
873
|
+
|
|
874
|
+
static PyTypeObject TransformerBlockType = {
|
|
875
|
+
PyVarObject_HEAD_INIT(NULL, 0)
|
|
876
|
+
.tp_name = "tensorc.TransformerBlock",
|
|
877
|
+
.tp_basicsize = sizeof(TransformerBlockObject),
|
|
878
|
+
.tp_dealloc = (destructor)transformer_block_dealloc,
|
|
879
|
+
.tp_flags = Py_TPFLAGS_DEFAULT,
|
|
880
|
+
.tp_methods = transformer_block_methods,
|
|
881
|
+
.tp_getset = transformer_block_getset,
|
|
882
|
+
.tp_new = transformer_block_new,
|
|
883
|
+
.tp_doc = "Bloque Transformer pre-norm (MHA + residual + FFN + residual).",
|
|
884
|
+
};
|
|
885
|
+
|
|
886
|
+
/* Codificación posicional sinusoidal [seq, depth]. */
|
|
887
|
+
static PyObject *module_sinusoidal_positions(PyObject *self, PyObject *args) {
|
|
888
|
+
(void)self;
|
|
889
|
+
unsigned long seq, depth;
|
|
890
|
+
if (!PyArg_ParseTuple(args, "kk", &seq, &depth)) return NULL;
|
|
891
|
+
return (PyObject *)tensor_wrap(
|
|
892
|
+
tc_tensor_nd_sinusoidal_positions((size_t)seq, (size_t)depth));
|
|
893
|
+
}
|
|
894
|
+
|
|
746
895
|
/* --- Optimizador Adam --- */
|
|
747
896
|
typedef struct {
|
|
748
897
|
PyObject_HEAD
|
|
@@ -835,6 +984,7 @@ PyMODINIT_FUNC PyInit__tensorc(void) {
|
|
|
835
984
|
if (PyType_Ready(&LayerType) < 0) return NULL;
|
|
836
985
|
if (PyType_Ready(&DenseLayerType) < 0) return NULL;
|
|
837
986
|
if (PyType_Ready(&LSTMLayerType) < 0) return NULL;
|
|
987
|
+
if (PyType_Ready(&TransformerBlockType) < 0) return NULL;
|
|
838
988
|
if (PyType_Ready(&OptimizerType) < 0) return NULL;
|
|
839
989
|
PyObject *module = PyModule_Create(&tensorc_module);
|
|
840
990
|
if (!module) return NULL;
|
|
@@ -864,6 +1014,13 @@ PyMODINIT_FUNC PyInit__tensorc(void) {
|
|
|
864
1014
|
Py_DECREF(module);
|
|
865
1015
|
return NULL;
|
|
866
1016
|
}
|
|
1017
|
+
Py_INCREF(&TransformerBlockType);
|
|
1018
|
+
if (PyModule_AddObject(module, "TransformerBlock",
|
|
1019
|
+
(PyObject *)&TransformerBlockType) < 0) {
|
|
1020
|
+
Py_DECREF(&TransformerBlockType);
|
|
1021
|
+
Py_DECREF(module);
|
|
1022
|
+
return NULL;
|
|
1023
|
+
}
|
|
867
1024
|
Py_INCREF(&OptimizerType);
|
|
868
1025
|
if (PyModule_AddObject(module, "Adam", (PyObject *)&OptimizerType) < 0) {
|
|
869
1026
|
Py_DECREF(&OptimizerType);
|
|
@@ -112,6 +112,20 @@ tc_tensor_nd *tc_tensor_nd_attention(const tc_tensor_nd *query,
|
|
|
112
112
|
const tc_tensor_nd *key,
|
|
113
113
|
const tc_tensor_nd *value,
|
|
114
114
|
int causal);
|
|
115
|
+
/* Permutación general de ejes: out[i] = in[axes[i]] por dimensión. */
|
|
116
|
+
tc_tensor_nd *tc_tensor_nd_permute(const tc_tensor_nd *input,
|
|
117
|
+
const size_t *axes, size_t ndim);
|
|
118
|
+
/*
|
|
119
|
+
* Atención multi-cabeza sobre [batch, seq, depth] con `heads` cabezas
|
|
120
|
+
* (depth debe ser divisible entre heads). Compone permute + reshape +
|
|
121
|
+
* atención single-head + concat, así el autograd fluye sin nodos nuevos.
|
|
122
|
+
*/
|
|
123
|
+
tc_tensor_nd *tc_tensor_nd_multihead_attention(const tc_tensor_nd *query,
|
|
124
|
+
const tc_tensor_nd *key,
|
|
125
|
+
const tc_tensor_nd *value,
|
|
126
|
+
size_t heads, int causal);
|
|
127
|
+
/* Codificación posicional sinusoidal [seq, depth] (hoja constante). */
|
|
128
|
+
tc_tensor_nd *tc_tensor_nd_sinusoidal_positions(size_t seq, size_t depth);
|
|
115
129
|
|
|
116
130
|
/* Runs reverse-mode autodiff from a scalar loss. Returns 1 on success. */
|
|
117
131
|
int tc_tensor_nd_backward(tc_tensor_nd *loss);
|
|
@@ -183,6 +197,25 @@ tc_tensor_nd *tc_lstm_layer_weight_ih(tc_lstm_layer *layer);
|
|
|
183
197
|
tc_tensor_nd *tc_lstm_layer_weight_hh(tc_lstm_layer *layer);
|
|
184
198
|
tc_tensor_nd *tc_lstm_layer_bias(tc_lstm_layer *layer);
|
|
185
199
|
|
|
200
|
+
/*
|
|
201
|
+
* Bloque Transformer pre-norm: atención multi-cabeza con residuales y
|
|
202
|
+
* FFN (ReLU) sobre [batch, seq, model_dim] -> misma forma. Pesos:
|
|
203
|
+
* W_q/W_k/W_v/W_o [model_dim, model_dim], W1 [model_dim, ff_dim],
|
|
204
|
+
* b1 [1, ff_dim], W2 [ff_dim, model_dim], b2 [1, model_dim], Xavier
|
|
205
|
+
* determinista. Los gradientes se acumulan en los 8 parámetros.
|
|
206
|
+
*/
|
|
207
|
+
typedef struct tc_transformer_block tc_transformer_block;
|
|
208
|
+
tc_transformer_block *tc_transformer_block_create(size_t model_dim,
|
|
209
|
+
size_t ff_dim,
|
|
210
|
+
size_t heads,
|
|
211
|
+
unsigned int seed);
|
|
212
|
+
void tc_transformer_block_free(tc_transformer_block *block);
|
|
213
|
+
tc_tensor_nd *tc_transformer_block_forward(tc_transformer_block *block,
|
|
214
|
+
const tc_tensor_nd *input);
|
|
215
|
+
size_t tc_transformer_block_param_count(const tc_transformer_block *block);
|
|
216
|
+
tc_tensor_nd *tc_transformer_block_param(tc_transformer_block *block,
|
|
217
|
+
size_t index);
|
|
218
|
+
|
|
186
219
|
#ifdef __cplusplus
|
|
187
220
|
}
|
|
188
221
|
#endif
|
|
@@ -29,7 +29,8 @@ typedef enum {
|
|
|
29
29
|
TC_ND_OP_CONCAT,
|
|
30
30
|
TC_ND_OP_LAYERNORM,
|
|
31
31
|
TC_ND_OP_TRANSPOSE,
|
|
32
|
-
TC_ND_OP_ATTENTION
|
|
32
|
+
TC_ND_OP_ATTENTION,
|
|
33
|
+
TC_ND_OP_PERMUTE
|
|
33
34
|
} tc_nd_op;
|
|
34
35
|
|
|
35
36
|
struct tc_tensor_nd {
|
|
@@ -50,6 +51,9 @@ struct tc_tensor_nd {
|
|
|
50
51
|
size_t slice_start;
|
|
51
52
|
/* Máscara causal de la atención (1 = no atender al futuro). */
|
|
52
53
|
int attn_causal;
|
|
54
|
+
/* Permutación de ejes para TC_ND_OP_PERMUTE (perm[i] = eje de la
|
|
55
|
+
* entrada que va a la posición i de la salida). */
|
|
56
|
+
size_t perm[8];
|
|
53
57
|
};
|
|
54
58
|
|
|
55
59
|
static int checked_product(const size_t *shape, size_t ndim, size_t *result) {
|
|
@@ -777,6 +781,40 @@ tc_tensor_nd *tc_tensor_nd_transpose(const tc_tensor_nd *input) {
|
|
|
777
781
|
return result;
|
|
778
782
|
}
|
|
779
783
|
|
|
784
|
+
/*
|
|
785
|
+
* Permutación general de ejes: la salida satisface
|
|
786
|
+
* out[i0, i1, ...] = in[i_{axes[0]}, i_{axes[1]}, ...]. Los ejes deben ser
|
|
787
|
+
* una permutación completa de 0..ndim-1 y ndim == rango de la entrada.
|
|
788
|
+
*/
|
|
789
|
+
tc_tensor_nd *tc_tensor_nd_permute(const tc_tensor_nd *input,
|
|
790
|
+
const size_t *axes, size_t ndim) {
|
|
791
|
+
if (!input || !axes || ndim == 0 || ndim != input->ndim || ndim > 8)
|
|
792
|
+
return NULL;
|
|
793
|
+
int seen[8] = {0};
|
|
794
|
+
size_t shape[8];
|
|
795
|
+
for (size_t i = 0; i < ndim; ++i) {
|
|
796
|
+
if (axes[i] >= ndim || seen[axes[i]]) return NULL;
|
|
797
|
+
seen[axes[i]] = 1;
|
|
798
|
+
shape[i] = input->shape[axes[i]];
|
|
799
|
+
}
|
|
800
|
+
tc_tensor_nd *result = unary_result(shape, ndim, TC_ND_OP_PERMUTE, input);
|
|
801
|
+
if (!result) return NULL;
|
|
802
|
+
for (size_t i = 0; i < ndim; ++i) result->perm[i] = axes[i];
|
|
803
|
+
/* Recorrido lineal de la entrada: cada índice lineal se descompone en
|
|
804
|
+
* índices multidimensionales y se reensambla en la posición de salida
|
|
805
|
+
* usando la permutación. */
|
|
806
|
+
size_t in_idx[8];
|
|
807
|
+
for (size_t lin = 0; lin < input->size; ++lin) {
|
|
808
|
+
for (size_t d = 0; d < ndim; ++d)
|
|
809
|
+
in_idx[d] = (lin / input->strides[d]) % input->shape[d];
|
|
810
|
+
size_t out_lin = 0;
|
|
811
|
+
for (size_t k = 0; k < ndim; ++k)
|
|
812
|
+
out_lin += in_idx[axes[k]] * result->strides[k];
|
|
813
|
+
result->data[out_lin] = input->data[lin];
|
|
814
|
+
}
|
|
815
|
+
return result;
|
|
816
|
+
}
|
|
817
|
+
|
|
780
818
|
/*
|
|
781
819
|
* Atención de producto punto escalado (single-head) sobre tensores
|
|
782
820
|
* [batch, seq, depth]: softmax((Q Kᵀ) / sqrt(depth) + máscara) V.
|
|
@@ -1240,6 +1278,21 @@ static void backward_node(tc_tensor_nd *tensor) {
|
|
|
1240
1278
|
add_grad(a, i * a->shape[1] + j,
|
|
1241
1279
|
tensor->grad[j * a->shape[0] + i]);
|
|
1242
1280
|
break;
|
|
1281
|
+
case TC_ND_OP_PERMUTE: {
|
|
1282
|
+
/* El forward es una biyección: cada índice lineal de la
|
|
1283
|
+
* entrada contribuye su gradiente desde la posición permutada
|
|
1284
|
+
* de la salida (mismo mapeo que en el forward). */
|
|
1285
|
+
size_t in_idx[8];
|
|
1286
|
+
for (size_t lin = 0; lin < a->size; ++lin) {
|
|
1287
|
+
for (size_t d = 0; d < a->ndim; ++d)
|
|
1288
|
+
in_idx[d] = (lin / a->strides[d]) % a->shape[d];
|
|
1289
|
+
size_t out_lin = 0;
|
|
1290
|
+
for (size_t k = 0; k < a->ndim; ++k)
|
|
1291
|
+
out_lin += in_idx[tensor->perm[k]] * tensor->strides[k];
|
|
1292
|
+
add_grad(a, lin, tensor->grad[out_lin]);
|
|
1293
|
+
}
|
|
1294
|
+
break;
|
|
1295
|
+
}
|
|
1243
1296
|
case TC_ND_OP_ATTENTION: {
|
|
1244
1297
|
/* Recomputa scores/P por bloque y aplica la regla
|
|
1245
1298
|
* dV=PᵀdO, dS=P⊙(dP−rowsum(dP⊙P))·scale, dQ=dS·K, dK=dSᵀ·Q. */
|
|
@@ -1792,3 +1845,259 @@ tc_tensor_nd *tc_lstm_layer_weight_hh(tc_lstm_layer *layer) {
|
|
|
1792
1845
|
tc_tensor_nd *tc_lstm_layer_bias(tc_lstm_layer *layer) {
|
|
1793
1846
|
return layer ? layer->bias : NULL;
|
|
1794
1847
|
}
|
|
1848
|
+
|
|
1849
|
+
/* --- Atención multi-cabeza y bloque Transformer --- */
|
|
1850
|
+
|
|
1851
|
+
/* [batch, seq, depth] -> [batch*heads, seq, depth/heads]. */
|
|
1852
|
+
static tc_tensor_nd *split_heads(const tc_tensor_nd *t, size_t heads) {
|
|
1853
|
+
const size_t batch = t->shape[0];
|
|
1854
|
+
const size_t seq = t->shape[1];
|
|
1855
|
+
const size_t depth = t->shape[2];
|
|
1856
|
+
const size_t head_depth = depth / heads;
|
|
1857
|
+
const size_t s1[] = {batch, seq, heads, head_depth};
|
|
1858
|
+
tc_tensor_nd *a = tc_tensor_nd_reshape_to(t, s1, 4);
|
|
1859
|
+
if (!a) return NULL;
|
|
1860
|
+
const size_t axes[] = {0, 2, 1, 3};
|
|
1861
|
+
tc_tensor_nd *b = tc_tensor_nd_permute(a, axes, 4);
|
|
1862
|
+
tc_tensor_nd_free(a);
|
|
1863
|
+
if (!b) return NULL;
|
|
1864
|
+
const size_t s2[] = {batch * heads, seq, head_depth};
|
|
1865
|
+
tc_tensor_nd *c = tc_tensor_nd_reshape_to(b, s2, 3);
|
|
1866
|
+
tc_tensor_nd_free(b);
|
|
1867
|
+
return c;
|
|
1868
|
+
}
|
|
1869
|
+
|
|
1870
|
+
/* [batch*heads, seq, depth/heads] -> [batch, seq, depth]. */
|
|
1871
|
+
static tc_tensor_nd *merge_heads(const tc_tensor_nd *t, size_t batch,
|
|
1872
|
+
size_t seq, size_t depth, size_t heads) {
|
|
1873
|
+
const size_t head_depth = depth / heads;
|
|
1874
|
+
const size_t s1[] = {batch, heads, seq, head_depth};
|
|
1875
|
+
tc_tensor_nd *a = tc_tensor_nd_reshape_to(t, s1, 4);
|
|
1876
|
+
if (!a) return NULL;
|
|
1877
|
+
const size_t axes[] = {0, 2, 1, 3};
|
|
1878
|
+
tc_tensor_nd *b = tc_tensor_nd_permute(a, axes, 4);
|
|
1879
|
+
tc_tensor_nd_free(a);
|
|
1880
|
+
if (!b) return NULL;
|
|
1881
|
+
const size_t s2[] = {batch, seq, depth};
|
|
1882
|
+
tc_tensor_nd *c = tc_tensor_nd_reshape_to(b, s2, 3);
|
|
1883
|
+
tc_tensor_nd_free(b);
|
|
1884
|
+
return c;
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1887
|
+
tc_tensor_nd *tc_tensor_nd_multihead_attention(const tc_tensor_nd *query,
|
|
1888
|
+
const tc_tensor_nd *key,
|
|
1889
|
+
const tc_tensor_nd *value,
|
|
1890
|
+
size_t heads, int causal) {
|
|
1891
|
+
if (!query || !key || !value || heads == 0 ||
|
|
1892
|
+
query->ndim != 3 || key->ndim != 3 || value->ndim != 3 ||
|
|
1893
|
+
query->shape[0] != key->shape[0] ||
|
|
1894
|
+
query->shape[0] != value->shape[0] ||
|
|
1895
|
+
query->shape[1] != key->shape[1] ||
|
|
1896
|
+
key->shape[1] != value->shape[1] ||
|
|
1897
|
+
query->shape[2] != key->shape[2] ||
|
|
1898
|
+
query->shape[2] % heads != 0 ||
|
|
1899
|
+
value->shape[2] % heads != 0)
|
|
1900
|
+
return NULL;
|
|
1901
|
+
const size_t batch = query->shape[0];
|
|
1902
|
+
const size_t seq = query->shape[1];
|
|
1903
|
+
const size_t depth = query->shape[2];
|
|
1904
|
+
|
|
1905
|
+
tc_tensor_nd *qh = split_heads(query, heads);
|
|
1906
|
+
tc_tensor_nd *kh = split_heads(key, heads);
|
|
1907
|
+
tc_tensor_nd *vh = split_heads(value, heads);
|
|
1908
|
+
if (!qh || !kh || !vh) {
|
|
1909
|
+
tc_tensor_nd_free(qh);
|
|
1910
|
+
tc_tensor_nd_free(kh);
|
|
1911
|
+
tc_tensor_nd_free(vh);
|
|
1912
|
+
return NULL;
|
|
1913
|
+
}
|
|
1914
|
+
tc_tensor_nd *attn = tc_tensor_nd_attention(qh, kh, vh, causal);
|
|
1915
|
+
tc_tensor_nd_free(qh);
|
|
1916
|
+
tc_tensor_nd_free(kh);
|
|
1917
|
+
tc_tensor_nd_free(vh);
|
|
1918
|
+
if (!attn) return NULL;
|
|
1919
|
+
tc_tensor_nd *merged = merge_heads(attn, batch, seq, depth, heads);
|
|
1920
|
+
tc_tensor_nd_free(attn);
|
|
1921
|
+
return merged;
|
|
1922
|
+
}
|
|
1923
|
+
|
|
1924
|
+
tc_tensor_nd *tc_tensor_nd_sinusoidal_positions(size_t seq, size_t depth) {
|
|
1925
|
+
if (seq == 0 || depth == 0) return NULL;
|
|
1926
|
+
const size_t shape[] = {seq, depth};
|
|
1927
|
+
tc_tensor_nd *pe = tc_tensor_nd_create(shape, 2, 0);
|
|
1928
|
+
if (!pe) return NULL;
|
|
1929
|
+
float *data = tc_tensor_nd_data(pe);
|
|
1930
|
+
for (size_t p = 0; p < seq; ++p)
|
|
1931
|
+
for (size_t i = 0; i < depth; ++i) {
|
|
1932
|
+
const double angle =
|
|
1933
|
+
(double)p / pow(10000.0, (double)(2 * (i / 2)) / (double)depth);
|
|
1934
|
+
data[p * depth + i] =
|
|
1935
|
+
(float)(i % 2 == 0 ? sin(angle) : cos(angle));
|
|
1936
|
+
}
|
|
1937
|
+
return pe;
|
|
1938
|
+
}
|
|
1939
|
+
|
|
1940
|
+
struct tc_transformer_block {
|
|
1941
|
+
size_t model_dim;
|
|
1942
|
+
size_t ff_dim;
|
|
1943
|
+
size_t heads;
|
|
1944
|
+
tc_tensor_nd *w_q;
|
|
1945
|
+
tc_tensor_nd *w_k;
|
|
1946
|
+
tc_tensor_nd *w_v;
|
|
1947
|
+
tc_tensor_nd *w_o;
|
|
1948
|
+
tc_tensor_nd *w1;
|
|
1949
|
+
tc_tensor_nd *b1;
|
|
1950
|
+
tc_tensor_nd *w2;
|
|
1951
|
+
tc_tensor_nd *b2;
|
|
1952
|
+
};
|
|
1953
|
+
|
|
1954
|
+
/* Inicializa `weight` con Xavier uniforme determinista a partir de `state`. */
|
|
1955
|
+
static void xavier_fill(tc_tensor_nd *weight, size_t fan_in, size_t fan_out,
|
|
1956
|
+
unsigned int *state) {
|
|
1957
|
+
const float limit =
|
|
1958
|
+
sqrtf(6.0f / ((float)fan_in + (float)fan_out));
|
|
1959
|
+
float *data = tc_tensor_nd_data(weight);
|
|
1960
|
+
const size_t count = tc_tensor_nd_numel(weight);
|
|
1961
|
+
for (size_t i = 0; i < count; ++i) {
|
|
1962
|
+
const float unit = (float)layer_rng_next(state) / 4294967296.0f;
|
|
1963
|
+
data[i] = (2.0f * unit - 1.0f) * limit;
|
|
1964
|
+
}
|
|
1965
|
+
}
|
|
1966
|
+
|
|
1967
|
+
static tc_tensor_nd *transformer_new_weight(size_t rows, size_t cols,
|
|
1968
|
+
unsigned int *state) {
|
|
1969
|
+
const size_t shape[] = {rows, cols};
|
|
1970
|
+
tc_tensor_nd *w = tc_tensor_nd_create(shape, 2, 1);
|
|
1971
|
+
if (w) xavier_fill(w, rows, cols, state);
|
|
1972
|
+
return w;
|
|
1973
|
+
}
|
|
1974
|
+
|
|
1975
|
+
static tc_tensor_nd *transformer_new_bias(size_t cols) {
|
|
1976
|
+
const size_t shape[] = {1, cols};
|
|
1977
|
+
return tc_tensor_nd_create(shape, 2, 1); /* datos ya en cero */
|
|
1978
|
+
}
|
|
1979
|
+
|
|
1980
|
+
tc_transformer_block *tc_transformer_block_create(size_t model_dim,
|
|
1981
|
+
size_t ff_dim,
|
|
1982
|
+
size_t heads,
|
|
1983
|
+
unsigned int seed) {
|
|
1984
|
+
if (model_dim == 0 || ff_dim == 0 || heads == 0 ||
|
|
1985
|
+
model_dim % heads != 0)
|
|
1986
|
+
return NULL;
|
|
1987
|
+
tc_transformer_block *block =
|
|
1988
|
+
(tc_transformer_block *)calloc(1, sizeof(*block));
|
|
1989
|
+
if (!block) return NULL;
|
|
1990
|
+
block->model_dim = model_dim;
|
|
1991
|
+
block->ff_dim = ff_dim;
|
|
1992
|
+
block->heads = heads;
|
|
1993
|
+
unsigned int state = seed ? seed : 1u;
|
|
1994
|
+
block->w_q = transformer_new_weight(model_dim, model_dim, &state);
|
|
1995
|
+
block->w_k = transformer_new_weight(model_dim, model_dim, &state);
|
|
1996
|
+
block->w_v = transformer_new_weight(model_dim, model_dim, &state);
|
|
1997
|
+
block->w_o = transformer_new_weight(model_dim, model_dim, &state);
|
|
1998
|
+
block->w1 = transformer_new_weight(model_dim, ff_dim, &state);
|
|
1999
|
+
block->b1 = transformer_new_bias(ff_dim);
|
|
2000
|
+
block->w2 = transformer_new_weight(ff_dim, model_dim, &state);
|
|
2001
|
+
block->b2 = transformer_new_bias(model_dim);
|
|
2002
|
+
if (!block->w_q || !block->w_k || !block->w_v || !block->w_o ||
|
|
2003
|
+
!block->w1 || !block->b1 || !block->w2 || !block->b2) {
|
|
2004
|
+
tc_transformer_block_free(block);
|
|
2005
|
+
return NULL;
|
|
2006
|
+
}
|
|
2007
|
+
return block;
|
|
2008
|
+
}
|
|
2009
|
+
|
|
2010
|
+
void tc_transformer_block_free(tc_transformer_block *block) {
|
|
2011
|
+
if (!block) return;
|
|
2012
|
+
tc_tensor_nd_free(block->w_q);
|
|
2013
|
+
tc_tensor_nd_free(block->w_k);
|
|
2014
|
+
tc_tensor_nd_free(block->w_v);
|
|
2015
|
+
tc_tensor_nd_free(block->w_o);
|
|
2016
|
+
tc_tensor_nd_free(block->w1);
|
|
2017
|
+
tc_tensor_nd_free(block->b1);
|
|
2018
|
+
tc_tensor_nd_free(block->w2);
|
|
2019
|
+
tc_tensor_nd_free(block->b2);
|
|
2020
|
+
free(block);
|
|
2021
|
+
}
|
|
2022
|
+
|
|
2023
|
+
size_t tc_transformer_block_param_count(const tc_transformer_block *block) {
|
|
2024
|
+
(void)block;
|
|
2025
|
+
return 8;
|
|
2026
|
+
}
|
|
2027
|
+
|
|
2028
|
+
tc_tensor_nd *tc_transformer_block_param(tc_transformer_block *block,
|
|
2029
|
+
size_t index) {
|
|
2030
|
+
if (!block || index >= 8) return NULL;
|
|
2031
|
+
tc_tensor_nd *params[] = {block->w_q, block->w_k, block->w_v, block->w_o,
|
|
2032
|
+
block->w1, block->b1, block->w2, block->b2};
|
|
2033
|
+
return params[index];
|
|
2034
|
+
}
|
|
2035
|
+
|
|
2036
|
+
/*
|
|
2037
|
+
* Pre-norm: x + MHA(LN(x)) y x + FFN(LN(x)). Las proyecciones y la FFN se
|
|
2038
|
+
* calculan en 2D [batch*seq, model_dim]; solo la atención vuelve a 3D.
|
|
2039
|
+
* Todas las operaciones son nodos de grafo: el backward es el estándar.
|
|
2040
|
+
*/
|
|
2041
|
+
tc_tensor_nd *tc_transformer_block_forward(tc_transformer_block *block,
|
|
2042
|
+
const tc_tensor_nd *input) {
|
|
2043
|
+
if (!block || !input || input->ndim != 3 ||
|
|
2044
|
+
input->shape[2] != block->model_dim)
|
|
2045
|
+
return NULL;
|
|
2046
|
+
const size_t batch = input->shape[0];
|
|
2047
|
+
const size_t seq = input->shape[1];
|
|
2048
|
+
const size_t dim = block->model_dim;
|
|
2049
|
+
const size_t flat = batch * seq;
|
|
2050
|
+
const size_t x2_shape[] = {flat, dim};
|
|
2051
|
+
const size_t x3_shape[] = {batch, seq, dim};
|
|
2052
|
+
|
|
2053
|
+
tc_tensor_nd *x2 = tc_tensor_nd_reshape_to(input, x2_shape, 2);
|
|
2054
|
+
if (!x2) return NULL;
|
|
2055
|
+
tc_tensor_nd *ln1 = tc_tensor_nd_layer_norm(x2);
|
|
2056
|
+
tc_tensor_nd *q2 = ln1 ? tc_tensor_nd_matmul(ln1, block->w_q) : NULL;
|
|
2057
|
+
tc_tensor_nd *k2 = ln1 ? tc_tensor_nd_matmul(ln1, block->w_k) : NULL;
|
|
2058
|
+
tc_tensor_nd *v2 = ln1 ? tc_tensor_nd_matmul(ln1, block->w_v) : NULL;
|
|
2059
|
+
tc_tensor_nd_free(ln1);
|
|
2060
|
+
tc_tensor_nd *q = q2 ? tc_tensor_nd_reshape_to(q2, x3_shape, 3) : NULL;
|
|
2061
|
+
tc_tensor_nd *k = k2 ? tc_tensor_nd_reshape_to(k2, x3_shape, 3) : NULL;
|
|
2062
|
+
tc_tensor_nd *v = v2 ? tc_tensor_nd_reshape_to(v2, x3_shape, 3) : NULL;
|
|
2063
|
+
tc_tensor_nd_free(q2);
|
|
2064
|
+
tc_tensor_nd_free(k2);
|
|
2065
|
+
tc_tensor_nd_free(v2);
|
|
2066
|
+
tc_tensor_nd *attn =
|
|
2067
|
+
(q && k && v)
|
|
2068
|
+
? tc_tensor_nd_multihead_attention(q, k, v, block->heads, 0)
|
|
2069
|
+
: NULL;
|
|
2070
|
+
tc_tensor_nd_free(q);
|
|
2071
|
+
tc_tensor_nd_free(k);
|
|
2072
|
+
tc_tensor_nd_free(v);
|
|
2073
|
+
tc_tensor_nd *attn2 =
|
|
2074
|
+
attn ? tc_tensor_nd_reshape_to(attn, x2_shape, 2) : NULL;
|
|
2075
|
+
tc_tensor_nd_free(attn);
|
|
2076
|
+
tc_tensor_nd *proj =
|
|
2077
|
+
attn2 ? tc_tensor_nd_matmul(attn2, block->w_o) : NULL;
|
|
2078
|
+
tc_tensor_nd_free(attn2);
|
|
2079
|
+
tc_tensor_nd *h1 = (x2 && proj) ? tc_tensor_nd_add(x2, proj) : NULL;
|
|
2080
|
+
tc_tensor_nd_free(proj);
|
|
2081
|
+
|
|
2082
|
+
tc_tensor_nd *ln2 = h1 ? tc_tensor_nd_layer_norm(h1) : NULL;
|
|
2083
|
+
tc_tensor_nd *ff1a = ln2 ? tc_tensor_nd_matmul(ln2, block->w1) : NULL;
|
|
2084
|
+
tc_tensor_nd_free(ln2);
|
|
2085
|
+
tc_tensor_nd *ff1 = (ff1a && block->b1)
|
|
2086
|
+
? tc_tensor_nd_add(ff1a, block->b1) : NULL;
|
|
2087
|
+
tc_tensor_nd_free(ff1a);
|
|
2088
|
+
tc_tensor_nd *ff2 = ff1 ? tc_tensor_nd_relu(ff1) : NULL;
|
|
2089
|
+
tc_tensor_nd_free(ff1);
|
|
2090
|
+
tc_tensor_nd *ff3a = ff2 ? tc_tensor_nd_matmul(ff2, block->w2) : NULL;
|
|
2091
|
+
tc_tensor_nd_free(ff2);
|
|
2092
|
+
tc_tensor_nd *ff3 = (ff3a && block->b2)
|
|
2093
|
+
? tc_tensor_nd_add(ff3a, block->b2) : NULL;
|
|
2094
|
+
tc_tensor_nd_free(ff3a);
|
|
2095
|
+
tc_tensor_nd *out2 = (h1 && ff3) ? tc_tensor_nd_add(h1, ff3) : NULL;
|
|
2096
|
+
tc_tensor_nd_free(ff3);
|
|
2097
|
+
tc_tensor_nd *out =
|
|
2098
|
+
out2 ? tc_tensor_nd_reshape_to(out2, x3_shape, 3) : NULL;
|
|
2099
|
+
tc_tensor_nd_free(out2);
|
|
2100
|
+
tc_tensor_nd_free(h1);
|
|
2101
|
+
tc_tensor_nd_free(x2);
|
|
2102
|
+
return out;
|
|
2103
|
+
}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
"""TensorC: neural network framework written in C11, with autograd.
|
|
2
|
-
|
|
3
|
-
The numerical core is a C library; this package is a thin CPython
|
|
4
|
-
extension that wraps it. Tensors keep their parents alive automatically,
|
|
5
|
-
so ``backward()`` can be called from any scalar loss.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
from ._tensorc import Adam, Conv2DLayer, DenseLayer, LSTMLayer, Tensor
|
|
9
|
-
|
|
10
|
-
__version__ = "0.12.0"
|
|
11
|
-
|
|
12
|
-
__all__ = ["Tensor", "Conv2DLayer", "DenseLayer", "LSTMLayer", "Adam",
|
|
13
|
-
"__version__"]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|