tensorc 0.12.0__tar.gz → 0.13.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tensorc
3
- Version: 0.12.0
3
+ Version: 0.13.0
4
4
  Summary: Neural network framework in C11 with autograd, callable from Python
5
5
  Author: Carlos Guedea
6
6
  License: Apache-2.0
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "tensorc"
7
- version = "0.12.0"
7
+ version = "0.13.0"
8
8
  description = "Neural network framework in C11 with autograd, callable from Python"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.9"
@@ -0,0 +1,29 @@
1
+ """TensorC: neural network framework written in C11, with autograd.
2
+
3
+ The numerical core is a C library; this package is a thin CPython
4
+ extension that wraps it. Tensors keep their parents alive automatically,
5
+ so ``backward()`` can be called from any scalar loss.
6
+ """
7
+
8
+ from ._tensorc import (
9
+ Adam,
10
+ Conv2DLayer,
11
+ DenseLayer,
12
+ LSTMLayer,
13
+ Tensor,
14
+ TransformerBlock,
15
+ sinusoidal_positions,
16
+ )
17
+
18
+ __version__ = "0.13.0"
19
+
20
+ __all__ = [
21
+ "Tensor",
22
+ "Conv2DLayer",
23
+ "DenseLayer",
24
+ "LSTMLayer",
25
+ "TransformerBlock",
26
+ "Adam",
27
+ "sinusoidal_positions",
28
+ "__version__",
29
+ ]
@@ -426,6 +426,49 @@ static PyObject *tensor_attention(TensorObject *self, PyObject *args,
426
426
  tc_tensor_nd_attention(self->tensor, key, value, causal));
427
427
  }
428
428
 
429
+ static PyObject *tensor_permute(TensorObject *self, PyObject *args) {
430
+ PyObject *axes_obj;
431
+ if (!PyArg_ParseTuple(args, "O", &axes_obj)) return NULL;
432
+ PyObject *seq = PySequence_Fast(axes_obj, "axes debe ser una secuencia");
433
+ if (!seq) return NULL;
434
+ const Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
435
+ size_t axes[8];
436
+ if (n < 1 || n > 8) {
437
+ Py_DECREF(seq);
438
+ PyErr_SetString(PyExc_ValueError, "axes debe tener entre 1 y 8 ejes");
439
+ return NULL;
440
+ }
441
+ for (Py_ssize_t i = 0; i < n; ++i) {
442
+ const long a = PyLong_AsLong(PySequence_Fast_GET_ITEM(seq, i));
443
+ if (a < 0 || PyErr_Occurred()) {
444
+ Py_DECREF(seq);
445
+ PyErr_SetString(PyExc_ValueError, "ejes inválidos en axes");
446
+ return NULL;
447
+ }
448
+ axes[i] = (size_t)a;
449
+ }
450
+ Py_DECREF(seq);
451
+ return (PyObject *)tensor_wrap(
452
+ tc_tensor_nd_permute(self->tensor, axes, (size_t)n));
453
+ }
454
+
455
+ static PyObject *tensor_multihead_attention(TensorObject *self, PyObject *args,
456
+ PyObject *kwargs) {
457
+ static char *names[] = {"key", "value", "heads", "causal", NULL};
458
+ PyObject *key_obj, *value_obj;
459
+ unsigned long heads;
460
+ int causal = 0;
461
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OOk|p", names, &key_obj,
462
+ &value_obj, &heads, &causal))
463
+ return NULL;
464
+ tc_tensor_nd *key = as_tensor(key_obj);
465
+ if (!key) return NULL;
466
+ tc_tensor_nd *value = as_tensor(value_obj);
467
+ if (!value) return NULL;
468
+ return (PyObject *)tensor_wrap(tc_tensor_nd_multihead_attention(
469
+ self->tensor, key, value, (size_t)heads, causal));
470
+ }
471
+
429
472
  static PyMethodDef tensor_methods[] = {
430
473
  {"zeros", (PyCFunction)tensor_zeros, METH_VARARGS | METH_KEYWORDS | METH_CLASS,
431
474
  "Tensor.zeros(shape, requires_grad=False)"},
@@ -462,6 +505,11 @@ static PyMethodDef tensor_methods[] = {
462
505
  {"attention", (PyCFunction)tensor_attention,
463
506
  METH_VARARGS | METH_KEYWORDS,
464
507
  "attention(key, value, causal=False) sobre [batch, seq, depth]."},
508
+ {"permute", (PyCFunction)tensor_permute, METH_VARARGS,
509
+ "permute(axes): out[i] = in[axes[i]] por dimensión."},
510
+ {"multihead_attention", (PyCFunction)tensor_multihead_attention,
511
+ METH_VARARGS | METH_KEYWORDS,
512
+ "multihead_attention(key, value, heads, causal=False) sobre [batch, seq, depth]."},
465
513
  {NULL, NULL, 0, NULL}
466
514
  };
467
515
 
@@ -573,7 +621,13 @@ static PyTypeObject LayerType = {
573
621
  .tp_doc = "Trainable 2D convolution layer.",
574
622
  };
575
623
 
624
+ /* Definida más abajo; declaración anticipada para la tabla de métodos. */
625
+ static PyObject *module_sinusoidal_positions(PyObject *self, PyObject *args);
626
+
576
627
  static PyMethodDef module_methods[] = {
628
+ {"sinusoidal_positions", (PyCFunction)module_sinusoidal_positions,
629
+ METH_VARARGS,
630
+ "sinusoidal_positions(seq, depth) -> codificación posicional [seq, depth]."},
577
631
  {NULL, NULL, 0, NULL}
578
632
  };
579
633
 
@@ -743,6 +797,101 @@ static PyTypeObject LSTMLayerType = {
743
797
  "(time-major, h0/c0 en cero).",
744
798
  };
745
799
 
800
+ /* --- Capa Transformer (bloque pre-norm) --- */
801
+ typedef struct {
802
+ PyObject_HEAD
803
+ tc_transformer_block *block;
804
+ } TransformerBlockObject;
805
+
806
+ static PyTypeObject TransformerBlockType;
807
+
808
+ static void transformer_block_dealloc(TransformerBlockObject *self) {
809
+ tc_transformer_block_free(self->block);
810
+ PyObject_Del(self);
811
+ }
812
+
813
+ /* forward(x): x es [batch, seq, model_dim]; devuelve la misma forma. */
814
+ static PyObject *transformer_block_forward(TransformerBlockObject *self,
815
+ PyObject *args) {
816
+ PyObject *input_obj;
817
+ if (!PyArg_ParseTuple(args, "O", &input_obj)) return NULL;
818
+ tc_tensor_nd *input = as_tensor(input_obj);
819
+ if (!input) return NULL;
820
+ return (PyObject *)tensor_wrap(
821
+ tc_transformer_block_forward(self->block, input));
822
+ }
823
+
824
+ /* params: lista de los 8 tensores entrenables (para Adam). */
825
+ static PyObject *transformer_block_params(TransformerBlockObject *self,
826
+ void *closure) {
827
+ (void)closure;
828
+ const size_t count = tc_transformer_block_param_count(self->block);
829
+ PyObject *list = PyList_New((Py_ssize_t)count);
830
+ if (!list) return NULL;
831
+ for (size_t i = 0; i < count; ++i) {
832
+ tc_tensor_nd_retain(tc_transformer_block_param(self->block, i));
833
+ PyList_SET_ITEM(list, (Py_ssize_t)i,
834
+ (PyObject *)tensor_wrap(
835
+ tc_transformer_block_param(self->block, i)));
836
+ }
837
+ return list;
838
+ }
839
+
840
+ static PyObject *transformer_block_new(PyTypeObject *type, PyObject *args,
841
+ PyObject *kwargs) {
842
+ static char *names[] = {"model_dim", "ff_dim", "heads", "seed", NULL};
843
+ unsigned long model_dim, ff_dim, heads, seed = 1;
844
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "kkk|k", names, &model_dim,
845
+ &ff_dim, &heads, &seed))
846
+ return NULL;
847
+ TransformerBlockObject *obj =
848
+ PyObject_New(TransformerBlockObject, &TransformerBlockType);
849
+ if (!obj) return NULL;
850
+ obj->block = tc_transformer_block_create(model_dim, ff_dim, heads,
851
+ (unsigned int)seed);
852
+ if (!obj->block) {
853
+ PyObject_Del(obj);
854
+ PyErr_SetString(PyExc_RuntimeError,
855
+ "no se pudo crear el bloque transformer "
856
+ "(model_dim debe ser divisible entre heads)");
857
+ return NULL;
858
+ }
859
+ return (PyObject *)obj;
860
+ }
861
+
862
+ static PyMethodDef transformer_block_methods[] = {
863
+ {"forward", (PyCFunction)transformer_block_forward, METH_VARARGS,
864
+ "forward(x) con x [batch, seq, model_dim] -> misma forma"},
865
+ {NULL, NULL, 0, NULL}
866
+ };
867
+
868
+ static PyGetSetDef transformer_block_getset[] = {
869
+ {"params", (getter)transformer_block_params, NULL,
870
+ "Lista de los 8 tensores entrenables (para Adam).", NULL},
871
+ {NULL, NULL, NULL, NULL, NULL}
872
+ };
873
+
874
+ static PyTypeObject TransformerBlockType = {
875
+ PyVarObject_HEAD_INIT(NULL, 0)
876
+ .tp_name = "tensorc.TransformerBlock",
877
+ .tp_basicsize = sizeof(TransformerBlockObject),
878
+ .tp_dealloc = (destructor)transformer_block_dealloc,
879
+ .tp_flags = Py_TPFLAGS_DEFAULT,
880
+ .tp_methods = transformer_block_methods,
881
+ .tp_getset = transformer_block_getset,
882
+ .tp_new = transformer_block_new,
883
+ .tp_doc = "Bloque Transformer pre-norm (MHA + residual + FFN + residual).",
884
+ };
885
+
886
+ /* Codificación posicional sinusoidal [seq, depth]. */
887
+ static PyObject *module_sinusoidal_positions(PyObject *self, PyObject *args) {
888
+ (void)self;
889
+ unsigned long seq, depth;
890
+ if (!PyArg_ParseTuple(args, "kk", &seq, &depth)) return NULL;
891
+ return (PyObject *)tensor_wrap(
892
+ tc_tensor_nd_sinusoidal_positions((size_t)seq, (size_t)depth));
893
+ }
894
+
746
895
  /* --- Optimizador Adam --- */
747
896
  typedef struct {
748
897
  PyObject_HEAD
@@ -835,6 +984,7 @@ PyMODINIT_FUNC PyInit__tensorc(void) {
835
984
  if (PyType_Ready(&LayerType) < 0) return NULL;
836
985
  if (PyType_Ready(&DenseLayerType) < 0) return NULL;
837
986
  if (PyType_Ready(&LSTMLayerType) < 0) return NULL;
987
+ if (PyType_Ready(&TransformerBlockType) < 0) return NULL;
838
988
  if (PyType_Ready(&OptimizerType) < 0) return NULL;
839
989
  PyObject *module = PyModule_Create(&tensorc_module);
840
990
  if (!module) return NULL;
@@ -864,6 +1014,13 @@ PyMODINIT_FUNC PyInit__tensorc(void) {
864
1014
  Py_DECREF(module);
865
1015
  return NULL;
866
1016
  }
1017
+ Py_INCREF(&TransformerBlockType);
1018
+ if (PyModule_AddObject(module, "TransformerBlock",
1019
+ (PyObject *)&TransformerBlockType) < 0) {
1020
+ Py_DECREF(&TransformerBlockType);
1021
+ Py_DECREF(module);
1022
+ return NULL;
1023
+ }
867
1024
  Py_INCREF(&OptimizerType);
868
1025
  if (PyModule_AddObject(module, "Adam", (PyObject *)&OptimizerType) < 0) {
869
1026
  Py_DECREF(&OptimizerType);
@@ -112,6 +112,20 @@ tc_tensor_nd *tc_tensor_nd_attention(const tc_tensor_nd *query,
112
112
  const tc_tensor_nd *key,
113
113
  const tc_tensor_nd *value,
114
114
  int causal);
115
+ /* Permutación general de ejes: out[i] = in[axes[i]] por dimensión. */
116
+ tc_tensor_nd *tc_tensor_nd_permute(const tc_tensor_nd *input,
117
+ const size_t *axes, size_t ndim);
118
+ /*
119
+ * Atención multi-cabeza sobre [batch, seq, depth] con `heads` cabezas
120
+ * (depth debe ser divisible entre heads). Compone permute + reshape +
121
+ * atención single-head + concat, así el autograd fluye sin nodos nuevos.
122
+ */
123
+ tc_tensor_nd *tc_tensor_nd_multihead_attention(const tc_tensor_nd *query,
124
+ const tc_tensor_nd *key,
125
+ const tc_tensor_nd *value,
126
+ size_t heads, int causal);
127
+ /* Codificación posicional sinusoidal [seq, depth] (hoja constante). */
128
+ tc_tensor_nd *tc_tensor_nd_sinusoidal_positions(size_t seq, size_t depth);
115
129
 
116
130
  /* Runs reverse-mode autodiff from a scalar loss. Returns 1 on success. */
117
131
  int tc_tensor_nd_backward(tc_tensor_nd *loss);
@@ -183,6 +197,25 @@ tc_tensor_nd *tc_lstm_layer_weight_ih(tc_lstm_layer *layer);
183
197
  tc_tensor_nd *tc_lstm_layer_weight_hh(tc_lstm_layer *layer);
184
198
  tc_tensor_nd *tc_lstm_layer_bias(tc_lstm_layer *layer);
185
199
 
200
+ /*
201
+ * Bloque Transformer pre-norm: atención multi-cabeza con residuales y
202
+ * FFN (ReLU) sobre [batch, seq, model_dim] -> misma forma. Pesos:
203
+ * W_q/W_k/W_v/W_o [model_dim, model_dim], W1 [model_dim, ff_dim],
204
+ * b1 [1, ff_dim], W2 [ff_dim, model_dim], b2 [1, model_dim], Xavier
205
+ * determinista. Los gradientes se acumulan en los 8 parámetros.
206
+ */
207
+ typedef struct tc_transformer_block tc_transformer_block;
208
+ tc_transformer_block *tc_transformer_block_create(size_t model_dim,
209
+ size_t ff_dim,
210
+ size_t heads,
211
+ unsigned int seed);
212
+ void tc_transformer_block_free(tc_transformer_block *block);
213
+ tc_tensor_nd *tc_transformer_block_forward(tc_transformer_block *block,
214
+ const tc_tensor_nd *input);
215
+ size_t tc_transformer_block_param_count(const tc_transformer_block *block);
216
+ tc_tensor_nd *tc_transformer_block_param(tc_transformer_block *block,
217
+ size_t index);
218
+
186
219
  #ifdef __cplusplus
187
220
  }
188
221
  #endif
@@ -9,7 +9,7 @@
9
9
  extern "C" {
10
10
  #endif
11
11
 
12
- #define TENSORC_VERSION "0.12.0"
12
+ #define TENSORC_VERSION "0.13.0"
13
13
 
14
14
  typedef MTTensor tc_tensor;
15
15
  typedef MTActivation tc_activation;
@@ -29,7 +29,8 @@ typedef enum {
29
29
  TC_ND_OP_CONCAT,
30
30
  TC_ND_OP_LAYERNORM,
31
31
  TC_ND_OP_TRANSPOSE,
32
- TC_ND_OP_ATTENTION
32
+ TC_ND_OP_ATTENTION,
33
+ TC_ND_OP_PERMUTE
33
34
  } tc_nd_op;
34
35
 
35
36
  struct tc_tensor_nd {
@@ -50,6 +51,9 @@ struct tc_tensor_nd {
50
51
  size_t slice_start;
51
52
  /* Máscara causal de la atención (1 = no atender al futuro). */
52
53
  int attn_causal;
54
+ /* Permutación de ejes para TC_ND_OP_PERMUTE (perm[i] = eje de la
55
+ * entrada que va a la posición i de la salida). */
56
+ size_t perm[8];
53
57
  };
54
58
 
55
59
  static int checked_product(const size_t *shape, size_t ndim, size_t *result) {
@@ -777,6 +781,40 @@ tc_tensor_nd *tc_tensor_nd_transpose(const tc_tensor_nd *input) {
777
781
  return result;
778
782
  }
779
783
 
784
+ /*
785
+ * Permutación general de ejes: la salida satisface
786
+ * out[i0, i1, ...] = in[i_{axes[0]}, i_{axes[1]}, ...]. Los ejes deben ser
787
+ * una permutación completa de 0..ndim-1 y ndim == rango de la entrada.
788
+ */
789
+ tc_tensor_nd *tc_tensor_nd_permute(const tc_tensor_nd *input,
790
+ const size_t *axes, size_t ndim) {
791
+ if (!input || !axes || ndim == 0 || ndim != input->ndim || ndim > 8)
792
+ return NULL;
793
+ int seen[8] = {0};
794
+ size_t shape[8];
795
+ for (size_t i = 0; i < ndim; ++i) {
796
+ if (axes[i] >= ndim || seen[axes[i]]) return NULL;
797
+ seen[axes[i]] = 1;
798
+ shape[i] = input->shape[axes[i]];
799
+ }
800
+ tc_tensor_nd *result = unary_result(shape, ndim, TC_ND_OP_PERMUTE, input);
801
+ if (!result) return NULL;
802
+ for (size_t i = 0; i < ndim; ++i) result->perm[i] = axes[i];
803
+ /* Recorrido lineal de la entrada: cada índice lineal se descompone en
804
+ * índices multidimensionales y se reensambla en la posición de salida
805
+ * usando la permutación. */
806
+ size_t in_idx[8];
807
+ for (size_t lin = 0; lin < input->size; ++lin) {
808
+ for (size_t d = 0; d < ndim; ++d)
809
+ in_idx[d] = (lin / input->strides[d]) % input->shape[d];
810
+ size_t out_lin = 0;
811
+ for (size_t k = 0; k < ndim; ++k)
812
+ out_lin += in_idx[axes[k]] * result->strides[k];
813
+ result->data[out_lin] = input->data[lin];
814
+ }
815
+ return result;
816
+ }
817
+
780
818
  /*
781
819
  * Atención de producto punto escalado (single-head) sobre tensores
782
820
  * [batch, seq, depth]: softmax((Q Kᵀ) / sqrt(depth) + máscara) V.
@@ -1240,6 +1278,21 @@ static void backward_node(tc_tensor_nd *tensor) {
1240
1278
  add_grad(a, i * a->shape[1] + j,
1241
1279
  tensor->grad[j * a->shape[0] + i]);
1242
1280
  break;
1281
+ case TC_ND_OP_PERMUTE: {
1282
+ /* El forward es una biyección: cada índice lineal de la
1283
+ * entrada contribuye su gradiente desde la posición permutada
1284
+ * de la salida (mismo mapeo que en el forward). */
1285
+ size_t in_idx[8];
1286
+ for (size_t lin = 0; lin < a->size; ++lin) {
1287
+ for (size_t d = 0; d < a->ndim; ++d)
1288
+ in_idx[d] = (lin / a->strides[d]) % a->shape[d];
1289
+ size_t out_lin = 0;
1290
+ for (size_t k = 0; k < a->ndim; ++k)
1291
+ out_lin += in_idx[tensor->perm[k]] * tensor->strides[k];
1292
+ add_grad(a, lin, tensor->grad[out_lin]);
1293
+ }
1294
+ break;
1295
+ }
1243
1296
  case TC_ND_OP_ATTENTION: {
1244
1297
  /* Recomputa scores/P por bloque y aplica la regla
1245
1298
  * dV=PᵀdO, dS=P⊙(dP−rowsum(dP⊙P))·scale, dQ=dS·K, dK=dSᵀ·Q. */
@@ -1792,3 +1845,259 @@ tc_tensor_nd *tc_lstm_layer_weight_hh(tc_lstm_layer *layer) {
1792
1845
  tc_tensor_nd *tc_lstm_layer_bias(tc_lstm_layer *layer) {
1793
1846
  return layer ? layer->bias : NULL;
1794
1847
  }
1848
+
1849
+ /* --- Atención multi-cabeza y bloque Transformer --- */
1850
+
1851
+ /* [batch, seq, depth] -> [batch*heads, seq, depth/heads]. */
1852
+ static tc_tensor_nd *split_heads(const tc_tensor_nd *t, size_t heads) {
1853
+ const size_t batch = t->shape[0];
1854
+ const size_t seq = t->shape[1];
1855
+ const size_t depth = t->shape[2];
1856
+ const size_t head_depth = depth / heads;
1857
+ const size_t s1[] = {batch, seq, heads, head_depth};
1858
+ tc_tensor_nd *a = tc_tensor_nd_reshape_to(t, s1, 4);
1859
+ if (!a) return NULL;
1860
+ const size_t axes[] = {0, 2, 1, 3};
1861
+ tc_tensor_nd *b = tc_tensor_nd_permute(a, axes, 4);
1862
+ tc_tensor_nd_free(a);
1863
+ if (!b) return NULL;
1864
+ const size_t s2[] = {batch * heads, seq, head_depth};
1865
+ tc_tensor_nd *c = tc_tensor_nd_reshape_to(b, s2, 3);
1866
+ tc_tensor_nd_free(b);
1867
+ return c;
1868
+ }
1869
+
1870
+ /* [batch*heads, seq, depth/heads] -> [batch, seq, depth]. */
1871
+ static tc_tensor_nd *merge_heads(const tc_tensor_nd *t, size_t batch,
1872
+ size_t seq, size_t depth, size_t heads) {
1873
+ const size_t head_depth = depth / heads;
1874
+ const size_t s1[] = {batch, heads, seq, head_depth};
1875
+ tc_tensor_nd *a = tc_tensor_nd_reshape_to(t, s1, 4);
1876
+ if (!a) return NULL;
1877
+ const size_t axes[] = {0, 2, 1, 3};
1878
+ tc_tensor_nd *b = tc_tensor_nd_permute(a, axes, 4);
1879
+ tc_tensor_nd_free(a);
1880
+ if (!b) return NULL;
1881
+ const size_t s2[] = {batch, seq, depth};
1882
+ tc_tensor_nd *c = tc_tensor_nd_reshape_to(b, s2, 3);
1883
+ tc_tensor_nd_free(b);
1884
+ return c;
1885
+ }
1886
+
1887
+ tc_tensor_nd *tc_tensor_nd_multihead_attention(const tc_tensor_nd *query,
1888
+ const tc_tensor_nd *key,
1889
+ const tc_tensor_nd *value,
1890
+ size_t heads, int causal) {
1891
+ if (!query || !key || !value || heads == 0 ||
1892
+ query->ndim != 3 || key->ndim != 3 || value->ndim != 3 ||
1893
+ query->shape[0] != key->shape[0] ||
1894
+ query->shape[0] != value->shape[0] ||
1895
+ query->shape[1] != key->shape[1] ||
1896
+ key->shape[1] != value->shape[1] ||
1897
+ query->shape[2] != key->shape[2] ||
1898
+ query->shape[2] % heads != 0 ||
1899
+ value->shape[2] % heads != 0)
1900
+ return NULL;
1901
+ const size_t batch = query->shape[0];
1902
+ const size_t seq = query->shape[1];
1903
+ const size_t depth = query->shape[2];
1904
+
1905
+ tc_tensor_nd *qh = split_heads(query, heads);
1906
+ tc_tensor_nd *kh = split_heads(key, heads);
1907
+ tc_tensor_nd *vh = split_heads(value, heads);
1908
+ if (!qh || !kh || !vh) {
1909
+ tc_tensor_nd_free(qh);
1910
+ tc_tensor_nd_free(kh);
1911
+ tc_tensor_nd_free(vh);
1912
+ return NULL;
1913
+ }
1914
+ tc_tensor_nd *attn = tc_tensor_nd_attention(qh, kh, vh, causal);
1915
+ tc_tensor_nd_free(qh);
1916
+ tc_tensor_nd_free(kh);
1917
+ tc_tensor_nd_free(vh);
1918
+ if (!attn) return NULL;
1919
+ tc_tensor_nd *merged = merge_heads(attn, batch, seq, depth, heads);
1920
+ tc_tensor_nd_free(attn);
1921
+ return merged;
1922
+ }
1923
+
1924
+ tc_tensor_nd *tc_tensor_nd_sinusoidal_positions(size_t seq, size_t depth) {
1925
+ if (seq == 0 || depth == 0) return NULL;
1926
+ const size_t shape[] = {seq, depth};
1927
+ tc_tensor_nd *pe = tc_tensor_nd_create(shape, 2, 0);
1928
+ if (!pe) return NULL;
1929
+ float *data = tc_tensor_nd_data(pe);
1930
+ for (size_t p = 0; p < seq; ++p)
1931
+ for (size_t i = 0; i < depth; ++i) {
1932
+ const double angle =
1933
+ (double)p / pow(10000.0, (double)(2 * (i / 2)) / (double)depth);
1934
+ data[p * depth + i] =
1935
+ (float)(i % 2 == 0 ? sin(angle) : cos(angle));
1936
+ }
1937
+ return pe;
1938
+ }
1939
+
1940
+ struct tc_transformer_block {
1941
+ size_t model_dim;
1942
+ size_t ff_dim;
1943
+ size_t heads;
1944
+ tc_tensor_nd *w_q;
1945
+ tc_tensor_nd *w_k;
1946
+ tc_tensor_nd *w_v;
1947
+ tc_tensor_nd *w_o;
1948
+ tc_tensor_nd *w1;
1949
+ tc_tensor_nd *b1;
1950
+ tc_tensor_nd *w2;
1951
+ tc_tensor_nd *b2;
1952
+ };
1953
+
1954
+ /* Inicializa `weight` con Xavier uniforme determinista a partir de `state`. */
1955
+ static void xavier_fill(tc_tensor_nd *weight, size_t fan_in, size_t fan_out,
1956
+ unsigned int *state) {
1957
+ const float limit =
1958
+ sqrtf(6.0f / ((float)fan_in + (float)fan_out));
1959
+ float *data = tc_tensor_nd_data(weight);
1960
+ const size_t count = tc_tensor_nd_numel(weight);
1961
+ for (size_t i = 0; i < count; ++i) {
1962
+ const float unit = (float)layer_rng_next(state) / 4294967296.0f;
1963
+ data[i] = (2.0f * unit - 1.0f) * limit;
1964
+ }
1965
+ }
1966
+
1967
+ static tc_tensor_nd *transformer_new_weight(size_t rows, size_t cols,
1968
+ unsigned int *state) {
1969
+ const size_t shape[] = {rows, cols};
1970
+ tc_tensor_nd *w = tc_tensor_nd_create(shape, 2, 1);
1971
+ if (w) xavier_fill(w, rows, cols, state);
1972
+ return w;
1973
+ }
1974
+
1975
+ static tc_tensor_nd *transformer_new_bias(size_t cols) {
1976
+ const size_t shape[] = {1, cols};
1977
+ return tc_tensor_nd_create(shape, 2, 1); /* datos ya en cero */
1978
+ }
1979
+
1980
+ tc_transformer_block *tc_transformer_block_create(size_t model_dim,
1981
+ size_t ff_dim,
1982
+ size_t heads,
1983
+ unsigned int seed) {
1984
+ if (model_dim == 0 || ff_dim == 0 || heads == 0 ||
1985
+ model_dim % heads != 0)
1986
+ return NULL;
1987
+ tc_transformer_block *block =
1988
+ (tc_transformer_block *)calloc(1, sizeof(*block));
1989
+ if (!block) return NULL;
1990
+ block->model_dim = model_dim;
1991
+ block->ff_dim = ff_dim;
1992
+ block->heads = heads;
1993
+ unsigned int state = seed ? seed : 1u;
1994
+ block->w_q = transformer_new_weight(model_dim, model_dim, &state);
1995
+ block->w_k = transformer_new_weight(model_dim, model_dim, &state);
1996
+ block->w_v = transformer_new_weight(model_dim, model_dim, &state);
1997
+ block->w_o = transformer_new_weight(model_dim, model_dim, &state);
1998
+ block->w1 = transformer_new_weight(model_dim, ff_dim, &state);
1999
+ block->b1 = transformer_new_bias(ff_dim);
2000
+ block->w2 = transformer_new_weight(ff_dim, model_dim, &state);
2001
+ block->b2 = transformer_new_bias(model_dim);
2002
+ if (!block->w_q || !block->w_k || !block->w_v || !block->w_o ||
2003
+ !block->w1 || !block->b1 || !block->w2 || !block->b2) {
2004
+ tc_transformer_block_free(block);
2005
+ return NULL;
2006
+ }
2007
+ return block;
2008
+ }
2009
+
2010
+ void tc_transformer_block_free(tc_transformer_block *block) {
2011
+ if (!block) return;
2012
+ tc_tensor_nd_free(block->w_q);
2013
+ tc_tensor_nd_free(block->w_k);
2014
+ tc_tensor_nd_free(block->w_v);
2015
+ tc_tensor_nd_free(block->w_o);
2016
+ tc_tensor_nd_free(block->w1);
2017
+ tc_tensor_nd_free(block->b1);
2018
+ tc_tensor_nd_free(block->w2);
2019
+ tc_tensor_nd_free(block->b2);
2020
+ free(block);
2021
+ }
2022
+
2023
+ size_t tc_transformer_block_param_count(const tc_transformer_block *block) {
2024
+ (void)block;
2025
+ return 8;
2026
+ }
2027
+
2028
+ tc_tensor_nd *tc_transformer_block_param(tc_transformer_block *block,
2029
+ size_t index) {
2030
+ if (!block || index >= 8) return NULL;
2031
+ tc_tensor_nd *params[] = {block->w_q, block->w_k, block->w_v, block->w_o,
2032
+ block->w1, block->b1, block->w2, block->b2};
2033
+ return params[index];
2034
+ }
2035
+
2036
+ /*
2037
+ * Pre-norm: x + MHA(LN(x)) y x + FFN(LN(x)). Las proyecciones y la FFN se
2038
+ * calculan en 2D [batch*seq, model_dim]; solo la atención vuelve a 3D.
2039
+ * Todas las operaciones son nodos de grafo: el backward es el estándar.
2040
+ */
2041
+ tc_tensor_nd *tc_transformer_block_forward(tc_transformer_block *block,
2042
+ const tc_tensor_nd *input) {
2043
+ if (!block || !input || input->ndim != 3 ||
2044
+ input->shape[2] != block->model_dim)
2045
+ return NULL;
2046
+ const size_t batch = input->shape[0];
2047
+ const size_t seq = input->shape[1];
2048
+ const size_t dim = block->model_dim;
2049
+ const size_t flat = batch * seq;
2050
+ const size_t x2_shape[] = {flat, dim};
2051
+ const size_t x3_shape[] = {batch, seq, dim};
2052
+
2053
+ tc_tensor_nd *x2 = tc_tensor_nd_reshape_to(input, x2_shape, 2);
2054
+ if (!x2) return NULL;
2055
+ tc_tensor_nd *ln1 = tc_tensor_nd_layer_norm(x2);
2056
+ tc_tensor_nd *q2 = ln1 ? tc_tensor_nd_matmul(ln1, block->w_q) : NULL;
2057
+ tc_tensor_nd *k2 = ln1 ? tc_tensor_nd_matmul(ln1, block->w_k) : NULL;
2058
+ tc_tensor_nd *v2 = ln1 ? tc_tensor_nd_matmul(ln1, block->w_v) : NULL;
2059
+ tc_tensor_nd_free(ln1);
2060
+ tc_tensor_nd *q = q2 ? tc_tensor_nd_reshape_to(q2, x3_shape, 3) : NULL;
2061
+ tc_tensor_nd *k = k2 ? tc_tensor_nd_reshape_to(k2, x3_shape, 3) : NULL;
2062
+ tc_tensor_nd *v = v2 ? tc_tensor_nd_reshape_to(v2, x3_shape, 3) : NULL;
2063
+ tc_tensor_nd_free(q2);
2064
+ tc_tensor_nd_free(k2);
2065
+ tc_tensor_nd_free(v2);
2066
+ tc_tensor_nd *attn =
2067
+ (q && k && v)
2068
+ ? tc_tensor_nd_multihead_attention(q, k, v, block->heads, 0)
2069
+ : NULL;
2070
+ tc_tensor_nd_free(q);
2071
+ tc_tensor_nd_free(k);
2072
+ tc_tensor_nd_free(v);
2073
+ tc_tensor_nd *attn2 =
2074
+ attn ? tc_tensor_nd_reshape_to(attn, x2_shape, 2) : NULL;
2075
+ tc_tensor_nd_free(attn);
2076
+ tc_tensor_nd *proj =
2077
+ attn2 ? tc_tensor_nd_matmul(attn2, block->w_o) : NULL;
2078
+ tc_tensor_nd_free(attn2);
2079
+ tc_tensor_nd *h1 = (x2 && proj) ? tc_tensor_nd_add(x2, proj) : NULL;
2080
+ tc_tensor_nd_free(proj);
2081
+
2082
+ tc_tensor_nd *ln2 = h1 ? tc_tensor_nd_layer_norm(h1) : NULL;
2083
+ tc_tensor_nd *ff1a = ln2 ? tc_tensor_nd_matmul(ln2, block->w1) : NULL;
2084
+ tc_tensor_nd_free(ln2);
2085
+ tc_tensor_nd *ff1 = (ff1a && block->b1)
2086
+ ? tc_tensor_nd_add(ff1a, block->b1) : NULL;
2087
+ tc_tensor_nd_free(ff1a);
2088
+ tc_tensor_nd *ff2 = ff1 ? tc_tensor_nd_relu(ff1) : NULL;
2089
+ tc_tensor_nd_free(ff1);
2090
+ tc_tensor_nd *ff3a = ff2 ? tc_tensor_nd_matmul(ff2, block->w2) : NULL;
2091
+ tc_tensor_nd_free(ff2);
2092
+ tc_tensor_nd *ff3 = (ff3a && block->b2)
2093
+ ? tc_tensor_nd_add(ff3a, block->b2) : NULL;
2094
+ tc_tensor_nd_free(ff3a);
2095
+ tc_tensor_nd *out2 = (h1 && ff3) ? tc_tensor_nd_add(h1, ff3) : NULL;
2096
+ tc_tensor_nd_free(ff3);
2097
+ tc_tensor_nd *out =
2098
+ out2 ? tc_tensor_nd_reshape_to(out2, x3_shape, 3) : NULL;
2099
+ tc_tensor_nd_free(out2);
2100
+ tc_tensor_nd_free(h1);
2101
+ tc_tensor_nd_free(x2);
2102
+ return out;
2103
+ }
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tensorc
3
- Version: 0.12.0
3
+ Version: 0.13.0
4
4
  Summary: Neural network framework in C11 with autograd, callable from Python
5
5
  Author: Carlos Guedea
6
6
  License: Apache-2.0
@@ -1,13 +0,0 @@
1
- """TensorC: neural network framework written in C11, with autograd.
2
-
3
- The numerical core is a C library; this package is a thin CPython
4
- extension that wraps it. Tensors keep their parents alive automatically,
5
- so ``backward()`` can be called from any scalar loss.
6
- """
7
-
8
- from ._tensorc import Adam, Conv2DLayer, DenseLayer, LSTMLayer, Tensor
9
-
10
- __version__ = "0.12.0"
11
-
12
- __all__ = ["Tensor", "Conv2DLayer", "DenseLayer", "LSTMLayer", "Adam",
13
- "__version__"]
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes