pytensorforge 0.1.0__py3-none-any.whl

pytensorforge-0.1.0.dist-info/METADATA

@@ -0,0 +1,103 @@
+Metadata-Version: 2.4
+Name: pytensorforge
+Version: 0.1.0
+Summary: Deep Learning Framework
+Author: Nnamdi Chidume
+Project-URL: Homepage, http://github.com/philipszdavido/pytensorforge
+Project-URL: Repository, http://github.com/philipszdavido/pytensorforge
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: numpy>=2.0
+Requires-Dist: pyyaml>=6.0
+
+# PyTensorForge
+
+PyTensorForge is a machine learning framework. It provides a modular core for tensor operations, neural network layers, activation functions, optimizers, loss metrics, and model definitions. It is designed to build and train enterprise-level models for computer vision, forecasting, LLM, NLP, etc.
+
+PyTensorForge is also designed for learning, experimentation, and rapid prototyping
+
+## Key Features
+
+- Core tensor and scalar abstractions for numerical computation
+- Layer-based neural network support with dense and sequential model building
+- Common activation functions such as ReLU, Sigmoid, Tanh, ELU, and SELU
+- Standard optimization algorithms with SGD and batch optimization
+- Loss functions including mean squared error, mean absolute error, and binary cross-entropy
+- Weight initialization utilities with Glorot, He, LeCun, and zeros initializers
+- Model implementations for regression and logistic classification
+
+## Project Structure
+
+- `src/core/` – Tensor, scalar, matrix, and vector primitives
+- `src/neural/` – Neural layer abstractions, parameters, and dense layers
+- `src/activations/` – Activation function implementations
+- `src/optimizer/` – Optimizer base classes and SGD implementation
+- `src/loss/` – Loss function abstractions and implementations
+- `src/initializers/` – Weight initializer utilities
+- `src/models/` – High-level model APIs for regression, classification, and sequential models
+- `src/scaling/` – Data scaling utilities
+- `src/models/gpt/` – decoder-only GPT Transformer (see `PHASE1_GPT_TRAINING.md`)
+- `src/data/`, `src/training/` – streaming corpora, token shards, resumable trainer (`PHASE1_GPT_TRAINING.md`, `PHASE2_DATA_PIPELINE.md`)
+- `src/tokenization/` – tokenizer abstraction, word-level and byte-level BPE (`TOKENIZER.md`)
+- `src/inference/` – KV-cache inference runtime, continuous batching, chat templates (`PHASE3_INFERENCE.md`)
+- `src/serving/` – OpenAI-compatible HTTP/SSE model server (`PHASE4_SERVING.md`)
+- RoPE, activation checkpointing, bf16/fp16 numerics, parallel tokenization (`PHASE5_TRAINING_EFFICIENCY.md`)
+- Context extension for RoPE models: extrapolate, linear, NTK (`PHASE6_CONTEXT_EXTENSION.md`)
+- Fine-tuning (`init_from`) and chat training with assistant-only loss masking (`PHASE7_FINETUNING.md`)
+- `web/` – TypeScript chat web client (prebuilt in `web/dist`)
+- `predict/` – Prediction utilities and example datasets
+- `test/` – Unit tests and example usage scripts
+
+## Installation
+
+1. Clone the repository:
+
+   ```bash
+   git clone https://github.com/philipsz-davido/PyTensorForge.git
+   cd PyTensorForge
+   ```
+
+2. Install dependencies:
+
+   ```bash
+   pip install -e .
+   ```
+
+   This installs the `pytensorforge` command (`train`, `resume`, `evaluate`, `inspect`,
+   `tokenize`, `prepare-dataset`, `generate`, `export`, `serve`). Runtime dependencies are
+   `numpy` and `pyyaml` only.
+
+> Note: This repository is intended for educational use and may require additional dependencies depending on the development environment.
+
+## Usage
+
+Import components from the package and build models using the provided modules. Example:
+
+```python
+from src.neural.Dense import Dense
+from src.neural.Layer import Layer
+from src.optimizers.SGD import SGD
+from src.loss.MSE import MSE
+```
+
+Refer to the `test/` directory for example scripts and usage patterns.
+
+Serving a trained model:
+
+```bash
+pytensorforge export checkpoints/latest --output exports/my-gpt --tokenizer tokenizer/ --chat-template ptf-chat
+pytensorforge serve exports/my-gpt --name my-gpt --api-key-file api_keys.txt
+```
+
+Then open `http://127.0.0.1:8000/` or point any OpenAI-compatible client at
+`http://127.0.0.1:8000/v1`. See `serve.example.yaml` and `PHASE4_SERVING.md`.
+
+## Contribution
+
+Contributions are welcome. Please open an issue or submit a pull request for feature requests, bug fixes, or documentation improvements.
+
+## License
+
+This project is provided under the terms of the repository license. Check the project root for license details.
+
+

pytensorforge-0.1.0.dist-info/RECORD

@@ -0,0 +1,146 @@
+cli.py,sha256=MBqjNkIJOE2xBT48qw85mCiV-aTa_jhVAxuzcteERvk,21982
+src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/config.py,sha256=136IZx99eEdTlroHrsqWtvz9qYDPPinHYDUvk6D2Xu0,3042
+src/activations/Activation.py,sha256=v1DZXlrP3oAHj_HxtlcPC4kxBAH2jC6qApFQCEZKEBE,79
+src/activations/ELU.py,sha256=H1nXwWoOB6YZ-ZoyfXccab8cAKUV66FNU9GCktqry2U,248
+src/activations/GELU.py,sha256=gbfZPS744b4t2vDsSmijH-99vpXnFpp-h6nk_ZYrq7g,126
+src/activations/ReLU.py,sha256=yeDrMxIIyxnKQWIwMrnh-Jzkn5TebzrLoCyFVD_Bpds,541
+src/activations/SELU.py,sha256=QHQDTRelmiiYEJAjjxlUZfaFK-tiScJz3TCxgpgoCTY,308
+src/activations/Sigmoid.py,sha256=knRXd320PfI7h2LdBOdutuoRQh80lpBO9cahLdreNtw,584
+src/activations/Softmax.py,sha256=Gktg_NG8bWBkBLKhrQseuY6CJ6Xt5sB5S1FGVoVUlBM,1714
+src/activations/Tanh.py,sha256=JZ8XgyBSfGpcprCfWrvEVZktbupjs_WzyTfg7mNiapk,551
+src/activations/__init__.py,sha256=JL4lFByOmW5VCRWfbK_67OhZrNydNFPq3mf4OwZjEuk,343
+src/core/Matrix.py,sha256=EANo7KDLzc1aQBrq7h9YL1bMCTiC3Hvyn8TIqv6hR2c,68
+src/core/Scalar.py,sha256=H0cl7ajxgwWnbVF1Y_vmfIA0_EoWQl9Y5jfDRoxfDTE,322
+src/core/Tensor.py,sha256=LDZoeSVJ18x-TdEK5XHW4D46jAd_YESM9rvFVrqzPSk,21085
+src/core/Vector.py,sha256=fGQH68H1rM4cNR5k_x7x8sDQDaer5NE1XeTxeAaZls8,636
+src/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/data/chat_dataset.py,sha256=EeNPqWnsuXFgkZ6mEsuEK_X8ANo9CgOp8f7e9oe8vmk,6326
+src/data/corpus.py,sha256=EjLk1CAm8GAKTfZV-GnMvdWVSDm3Jnj6dUh2qaX0aUk,3053
+src/data/document_stream.py,sha256=FZk9jhZjeIqcoXVcwPzb1iB-Lq5eW2GYeMYtfa7-xs4,4375
+src/data/parallel_encode.py,sha256=WWfzP33Ji49f1IpXZKKFzzn78BJGwwDpwWzhQQESdm0,2664
+src/data/prefetch.py,sha256=iLs1v15NbQmi--QBUHbKuRew5gnIm0GY5xHlImdOXsk,1606
+src/data/shard_builder.py,sha256=zQhiIkdMaklzxtGFaed2MHLQKCGvhcyuibvzCnUq2ug,3176
+src/data/shard_writer.py,sha256=y1skSHwbj6hMB657skDbUNKR4YafCl0AH_h8FY0NlVw,2076
+src/data/sharded_dataset.py,sha256=oCkS9PnDJmqRHN8uJO1t273ZI3ZUxwnSHg_KskZdhLs,3765
+src/data/streaming_dataset.py,sha256=A_1ZxuudRL8KqFwX8hpSEoLeKRBFlsOU-x5bQ7tEG3I,3858
+src/data/validation.py,sha256=pl2mgI_minPI5lVFjBdlkhdKb43bE-wg_iJRBJOQ_S8,7056
+src/inference/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/inference/chat_template.py,sha256=_siKaBCimD0R9Be2ujDdcEai6HrE-S2QqCEsaIYSK5k,13060
+src/inference/config.py,sha256=STgcXZOIqpY9bhpZ6jPp8nxch02A08X6x2VmrxCzQFM,1443
+src/inference/engine.py,sha256=zRYkTKL4P70JcKwKGGi7Bw88PTkKGS2UyeSgQI7xV3w,7682
+src/inference/export.py,sha256=9aJxQ6OeNGjff4wfs4Fa6Un4D8jlgrbHbDrBcse0LHg,4341
+src/inference/kv_cache.py,sha256=eJR-fL3IvdUTVELjcdm9bdMJ4zq5avkFQW885GNJqUU,1904
+src/inference/runtime.py,sha256=BOihJQPTym_D9f0KiyK1NhNvLLKbngqjXQul26m-hvk,5338
+src/inference/sampling.py,sha256=JWI5UOkTlfmE3ASZfAmdFAyaez3BvvT8oARkSv2ycqI,1212
+src/inference/scheduler.py,sha256=ArdrhwNEVc4M0whs0EOlyzKLAOWCR1Je505pqSn17vI,14407
+src/inference/text.py,sha256=iB7BNQRRrEUkgeBoeL55LrtntGKS8GSnlr1MAW76Ry4,1877
+src/initializers/Constant.py,sha256=_vIqP40_-CELGYlLRIETvO4peRH_XxPmB3M8A731u3Q,186
+src/initializers/GlorotNormal.py,sha256=fdcKXnQYRq2TGFhgomAS6XVjBLxeRe66vQJ2GYleMAE,265
+src/initializers/GlorotUniform.py,sha256=cgRvXpX6p6ay71JkY3S29FyAwDXQqQGDZ_-qELvNzcU,767
+src/initializers/HeNormal.py,sha256=JKN5qTMcVkh_MqznUkA_G1jbkFa5p6QE6TB-wtOLDAw,243
+src/initializers/HeUniform.py,sha256=QSqxROf_PizFhUps4jwaNOoVyEvBD2Zux7ZFck_1miI,320
+src/initializers/Initializer.py,sha256=-BNYQaksqWjlEiz5QJKTQ11JG5Wkya_hiop3iowA6PU,84
+src/initializers/LecunNormal.py,sha256=NtUKqMhqe7M-5IMJMQ_wl-UmiJGHxyrJ3ZB5zAEGbBo,247
+src/initializers/LecunUniform.py,sha256=eTqeRMoW-WazmlR5oa-EkzUTDJJgb9hwsjA8hj9p3uI,323
+src/initializers/Ones.py,sha256=_kViopeLVrYEw1R-XcwMeNd3CfvZH3SSUDnkeUFaDFg,111
+src/initializers/Orthogonal.py,sha256=24OYxuwJO_cj6jarSjlOldaOEVb7PM6aG0bYgqDKWWA,208
+src/initializers/RandomNormal.py,sha256=urxvesOUZ3GElpPpNnPUPP0oCK79qDVTF3Lb89oymZM,302
+src/initializers/RandomUniform.py,sha256=_90_kBzrn9HDmolktOFxRoscWkcHYU6XRjM1Gsu1OlQ,294
+src/initializers/Zeros.py,sha256=LdblqwgvijYw2Rd0vy7MNtYA_jb4Qc5q-dypLgBAVM0,180
+src/initializers/__init__.py,sha256=2SHBsYIFetXTol9pLfzTiYGFZVJQXiWR-EulzOOj64Y,476
+src/loss/CategoricalCrossEntropy.py,sha256=WSIELbAfaqM4M8jRRaeR-DXCEQrCCSTNkROLn_FLya8,205
+src/loss/CrossEntropyLoss.py,sha256=u7kVDxwhrXcvKrsxlDzz3RM8AooSuHQZKlZPy584boI,622
+src/loss/CrossEntropyWithLogitsLoss.py,sha256=cDT7TOieiCuoOWJHcp-RzBS5N8c_FOgt48H9I9kqubA,1152
+src/loss/Hinge.py,sha256=oKzCsfcYr6qPVLB1J94hX-MJbDwU3AZ1IRjSwqR1gAI,117
+src/loss/Huber.py,sha256=KoyTP8oqJtL1tVELG31StghWCwEj-lwlT_JmO1iDsy8,444
+src/loss/Loss.py,sha256=Hwm2MHDZF3dGHKx27Ycz6ajmflFg4O8e62I9oyqemJ8,155
+src/loss/MSE.py,sha256=Dj5M0RDpi8PKJ1KVczZ6ZS26tFDvIqLDbG4uuDscWHs,128
+src/loss/MSELoss.py,sha256=EorNihrllTCPIEAqTPBA0Y5-XhmwoBbFyObDHs6auk4,229
+src/loss/SparseCategoricalCrossEntropy.py,sha256=kDPk8fPfmVYmbhAfXhOuAyLbUTZrkLgonALyqrIv9DQ,284
+src/loss/__init__.py,sha256=mu7eWxqmcwScwuiUGQZ5izM4Kp__WtnMWkbyuGH7VD4,708
+src/loss/bce.py,sha256=fMHXG4rMhH9TK0Jp1XeqVJyDFvDIXGYHaD7A0J3p-YI,733
+src/loss/mae.py,sha256=lox0XXvK2rjviATXhXDNXcuj92_GRlePbIRpP3-jKL8,340
+src/math/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/math/clip.py,sha256=3qnqIJljFXvCpqms971UvP1fxsz-dc5zy8WhPCM8XDo,748
+src/math/exp.py,sha256=bfzxSyHDg1UhWQ88w_vMStEC-VIimbtYNWLDw267fmY,426
+src/math/log.py,sha256=0FfvD6j1M5EvAe9b8bkF37tgkMjvcbXZuOhMDOp5onM,416
+src/math/sigmoid.py,sha256=-45QD8qMbhZUfwPxjZmEekzuX91t3KC6d9UXIOMv4cY,69
+src/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/models/embedding/Embedding.py,sha256=0ntF7n2oXVg5hdz5i9I08bpTN6PJ7uTgulpTj-IPkro,1282
+src/models/embedding/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/models/gpt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/models/gpt/attention.py,sha256=zWyzu6TTcrmDfWSJCSpYSimlqtGxUDPVFshOKjBt7GQ,4194
+src/models/gpt/block.py,sha256=e2hlIC8X-8P06Xha628nGj0D6dH0tV8xiMeovbpvYCU,2305
+src/models/gpt/config.py,sha256=tJ36n8m8929yb_EvnFeKVoWT7kB5zsT5711KpvQejio,3430
+src/models/gpt/context.py,sha256=CcmJPDwxqcfWtKx_-L0T4GiaN9jvaodKbuXwpL4Js7Q,1526
+src/models/gpt/model.py,sha256=CLw6qTrhttnFafaYg1tKCPilegdLfvu2sO51TNe5Lm8,4788
+src/models/gpt/recompute.py,sha256=VIQ4BeviOuWF5lB6YHWu90PlxE7CVMBCP4neiSAFcnI,774
+src/models/gpt/rope.py,sha256=ey3DsJGw4ZU1zmkD1mtfwf20SiNDoz_Iuc593NyZEAY,2555
+src/models/regression/Linear.py,sha256=JDLXkvw7ozBTujY4C69p1RoCsKjEpv_GNI-oAMuUK-I,1293
+src/models/regression/Logistic.py,sha256=IKtpLDvUdJqyHeaiZiH-9ZG5doJ_F1fk8D2wYAa9FxY,910
+src/models/regression/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/models/seq/Sequential.py,sha256=ZyA7WzMPjU0wbK5-kRHN9BTD5od2l4ZTJum6ST9WfQQ,7218
+src/models/seq/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/models/svm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/models/tokenizer/BPETokenizer.py,sha256=fZUmTZ4r6ZR8c-OXTF7wsjzSh-TurbSA9_wS2EuJJyo,4081
+src/models/tokenizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/models/transformers/Dropout.py,sha256=1nCTQ9w2DHZ5zypxIT489aYi_0TTTC0ySczhRrBLvJ4,627
+src/models/transformers/LastToken.py,sha256=8eWkLho2JfJyII5K0MWk-Uy0sTVeyfCgnXAKt8SAf-0,159
+src/models/transformers/LayerNorm.py,sha256=6rkCNncEmPvcc6PIpSrZFawbts0PqnZXC8_KVUOEPPc,1410
+src/models/transformers/Linear.py,sha256=PVTVOXBAE2d5MnVi6qN6cSrY602sZV2YJhC-3RwY-4I,325
+src/models/transformers/MultiHeadAttention.py,sha256=Oe_HqyLwiiEgXl4ZxRBG6c-ej9UrB8W3RHYW3wc1_m4,2948
+src/models/transformers/TransformerBlock.py,sha256=0QJfKB8aXhNrkAnwSiiDmzrZhMaLIggh8kZqbJ-8uKI,2137
+src/models/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/neural/Dense.py,sha256=TY1U_XB5kTmLaf7eJ5wcwENfWpwLOCX2F8AAgu1mWBY,1635
+src/neural/LSTM.py,sha256=i56OsH65gHOLM63QU8UAE5LWdxJIgYdhWkKSaAOW8jY,3829
+src/neural/Layer.py,sha256=mq18zU7R1idQD7TdKAjVBp69f9rTfyF8JEsay_uX6kQ,1563
+src/neural/Parameter.py,sha256=InZ8a4xlOJVJN7TvywcnlZLeDri93uu8yotptGF1UjA,666
+src/neural/RNN.py,sha256=G6zZhwrdpxzDY7RdkS0YiEdtEt4910HH3C9N8ZE0FZ4,1777
+src/neural/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/ops/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/ops/stack.py,sha256=mst87o6DP-Q1OTiyhYVyPE0u2MKgtXWv3c8-rH2RhEM,782
+src/optimizers/Adagrad.py,sha256=56x-TkRGQ80J9NxRy9jTGxRnU9b9411vS6uO7pJAgzc,640
+src/optimizers/Adam.py,sha256=OtSzZaMn-sTS8HwIR34xDjWMCZKoh0Ov57f870c5NTk,2297
+src/optimizers/AdamW.py,sha256=v4NYJ6VQO9ezK8knMqmtAGp1be0rsD4s_7UwPV0p0-A,2112
+src/optimizers/Batch.py,sha256=OGq78eoTkA2DfliwcvF5Fgc6LxZw1NXqCv-l40tTf-8,238
+src/optimizers/Nesterov.py,sha256=cpVc1Gqn1c8RtDDCxXw7wxpiXDkYhhvP1jmqY5cCNBE,802
+src/optimizers/Optimizer.py,sha256=KVCFneLdNiHb5BD1J4zdVA105roQxHVKmeCWV60o6gQ,475
+src/optimizers/RMSProp.py,sha256=dU26xVf7F5-ENFJt2GfN6WM7s-ETYpThQC5zEgK0uOo,775
+src/optimizers/SGD.py,sha256=dwSmXPeBR40PUbaOIOy_naK7VbL2rLxHLWHrPuGF83w,760
+src/optimizers/SGDMomentum.py,sha256=b9MsuC1c1xwxU8dDD8lgA6e4Fjlm6USUIl_RL0cOXwk,594
+src/optimizers/__init__.py,sha256=-_Mh-5WwPqzPGtVhQXzn4viiWeAx51y6j49ooNBzHhw,159
+src/scaling/StandardScaler.py,sha256=kwXpvc_LNYvMYnm0x18VrEXeIWfTGkJ8KEcWBbdbIRM,335
+src/scaling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/serialization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/serialization/checkpoint.py,sha256=CGIh6rA5_tXobnPlDwP8-Nof6qwvklj-LDQox479vls,1021
+src/serialization/modelio.py,sha256=cvAtZOO3xK3I3w_ecvJtBz-F85lchbi2EIcwt5Y25xs,2298
+src/serving/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/serving/app.py,sha256=fk1qBNEGMm-L--RaPK0K8T_SBXMWYYJlr8KTYMzu2mg,27919
+src/serving/config.py,sha256=rBI6wmCfC7G1faZX4tM9UYiqm3XinyUev8RSDXleWu4,6781
+src/serving/errors.py,sha256=i6jAmyPGMowp6KsBml060DriQn8ktY1uFytF6ljfkGs,1800
+src/serving/http.py,sha256=GaJcC25GzxjxxWWEjaZ6r-APRwLfKfHTohZn3gOd-vc,18557
+src/serving/metrics.py,sha256=czKMn7IDiylOL9RMERmQ1tJDrwo2JKRcA8JYiDVw-LY,10891
+src/serving/model_server.py,sha256=PvAykqVOy6Kg_0A1YlMya2_CgGxuZeDpMvsJsVvaZO4,9195
+src/serving/protocol.py,sha256=E70JnNhCzUL9RIEDoAKtUVI9Nxyc5rYc4HxPtu9XKXU,12029
+src/serving/security.py,sha256=eI7B9hc5Kv6BQeztLl-T84ICne5CQirD0WSpCerC2QM,5417
+src/serving/server.py,sha256=B99pSShkHjKwTtyvHNW7OT2KKlzwC9P4-I-g71_plew,4059
+src/tokenization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/tokenization/base.py,sha256=yYRljT8vESIqzXhs-M3prS680_a2Ch3hI2eFW8lwHB8,1699
+src/tokenization/bpe.py,sha256=9Vjq4G-ApxMzeQCDCxCG7BYNOLIBp2fEEGJNOJQQ2H4,5435
+src/tokenization/bytebpe.py,sha256=kpsvTK0wAGt8Ml2NzN_HTmtxWg4_Grhh456PD9BzNs0,12713
+src/tokenization/registry.py,sha256=_qR0ILFeCtloEi1aAXtaMBvuAA7uecpXwqRZ_5Js4as,603
+src/training/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/training/checkpoint_manager.py,sha256=vO0fVe6hTnxC84erQSZDHZ0HJvsJlwcT6vH4PdkEA3A,2827
+src/training/experiment.py,sha256=FcG0el_2sYvL5tp2LBRXqLrusIg9KcZnBOYkeLTM0w8,1923
+src/training/losses.py,sha256=-JABRvkn-AWfTljGWHaXcYOHO3oxaV68hjrZqneIhbg,1346
+src/training/precision.py,sha256=A_7069FCGSlyj_J0Lr_tLCQquN1KNlYp-EnO0__Y_6c,4246
+src/training/profiler.py,sha256=xJIUr4RMHfGj8LutYIySv_0u6uLQk--nuYzLZrykveA,919
+src/training/scheduler.py,sha256=IHtqfeMXFGjriqa7hABFTj9krmeoFEatd92MhF8xEVY,1409
+src/training/trainer.py,sha256=240L0zXy4OFJA02ifuPAc-jSTklL66gOBT7Cb9LKBfc,21091
+pytensorforge-0.1.0.dist-info/METADATA,sha256=7EJC8VgNiGTj205NbaoodzBdsv823bbWmxqwMEr_C5c,4412
+pytensorforge-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+pytensorforge-0.1.0.dist-info/entry_points.txt,sha256=xu-_dU6lBAUNmGEUWjVdM2E9upSAmT95eveWyhqy0-k,43
+pytensorforge-0.1.0.dist-info/top_level.txt,sha256=cnbTmgy-9dPa7KK2pqKBQREjocvF6Y8MB_dF_HvtmP0,8
+pytensorforge-0.1.0.dist-info/RECORD,,

pytensorforge-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pytensorforge-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pytensorforge = cli:main

pytensorforge-0.1.0.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+cli
+src

src/activations/Activation.py

@@ -0,0 +1,4 @@
+
+class Activation:
+    def __call__(self, x):
+        raise NotImplementedError

src/activations/ELU.py

@@ -0,0 +1,11 @@
+import numpy as np
+
+from src.activations.Activation import Activation
+
+
+class ELU(Activation):
+    def __init__(self, alpha=1.0):
+        self.alpha = alpha
+
+    def __call__(self, x):
+        return np.where(x > 0, x, self.alpha * (np.exp(x) - 1))

src/activations/GELU.py

@@ -0,0 +1,6 @@
+from src.activations.Activation import Activation
+
+class GELU(Activation):
+
+    def __call__(self, x):
+        return x.gelu()

src/activations/ReLU.py

@@ -0,0 +1,27 @@
+import numpy as np
+
+from src.activations.Activation import Activation
+from src.core.Tensor import Tensor
+
+
+class ReLU(Activation):
+
+    @staticmethod
+    def forward(x):
+        out = Tensor(
+            np.maximum(0, x.data),
+            requires_grad=x.requires_grad,
+            parents=(x,),
+            op="ReLU",
+        )
+
+        def backward():
+            if x.requires_grad:
+                x.grad += out.grad * (x.data > 0)
+
+        out._backward = backward
+
+        return out
+
+    def __call__(self, x):
+        return x.relu()

src/activations/SELU.py

@@ -0,0 +1,14 @@
+import numpy as np
+
+from src.activations.Activation import Activation
+
+class SELU(Activation):
+    SCALE = 1.0507009873554805
+    ALPHA = 1.6732632423543772
+
+    def __call__(self, x):
+        return self.SCALE * np.where(
+            x > 0,
+            x,
+            self.ALPHA * (np.exp(x) - 1),
+        )

src/activations/Sigmoid.py

@@ -0,0 +1,27 @@
+from src.activations.Activation import Activation
+from src.core.Tensor import Tensor
+from src.math.sigmoid import sigmoid
+
+
+class Sigmoid(Activation):
+    @staticmethod
+    def forward(x):
+        sig = sigmoid(x.data)
+
+        out = Tensor(
+            sig,
+            requires_grad=x.requires_grad,
+            parents=(x,),
+            op="Sigmoid",
+        )
+
+        def _backward():
+            if x.requires_grad:
+                x.grad += out.grad * sig * (1 - sig)
+
+        out._backward = _backward
+
+        return out
+
+    def __call__(self, x):
+        return x.sigmoid()

src/activations/Softmax.py

@@ -0,0 +1,84 @@
+import numpy as np
+
+from src.core.Tensor import Tensor
+
+
+# class Softmax:
+#
+#     @staticmethod
+#     def forward(x):
+#
+#         shifted = x.data - np.max(x.data, axis=-1, keepdims=True)
+#
+#         exp = np.exp(shifted)
+#
+#         probs = exp / np.sum(exp, axis=-1, keepdims=True)
+#
+#         out = Tensor(
+#             probs,
+#             requires_grad=x.requires_grad,
+#             parents=(x,),
+#             op="Softmax",
+#         )
+#
+#         def _backward():
+#
+#             if not x.requires_grad:
+#                 return
+#
+#             grad = np.empty_like(probs)
+#
+#             for i in range(len(probs)):
+#                 p = probs[i].reshape(-1, 1)
+#
+#                 jacobian = np.diagflat(p) - p @ p.T
+#
+#                 grad[i] = jacobian @ out.grad[i]
+#
+#             x.grad += grad
+#
+#         out._backward = _backward
+#
+#         return out
+#
+#     def __call__(self, x):
+#         return self.forward(x)
+
+from src.activations.Activation import Activation
+
+
+class Softmax(Activation):
+
+    @staticmethod
+    def forward(x, axis=-1):
+
+        shifted = x.data - np.max(x.data, axis=axis, keepdims=True)
+
+        exp = np.exp(shifted)
+
+        probs = exp / np.sum(exp, axis=axis, keepdims=True)
+
+        out = Tensor(
+            probs,
+            requires_grad=x.requires_grad,
+            parents=(x,),
+            op="Softmax",
+        )
+
+        def _backward():
+
+            if not x.requires_grad:
+                return
+
+            grad = out.grad
+
+            dot = np.sum(grad * probs, axis=axis, keepdims=True)
+
+            x.grad += probs * (grad - dot)
+
+        out._backward = _backward
+
+        return out
+
+    def __call__(self, x):
+        return Softmax.forward(x)

src/activations/Tanh.py

@@ -0,0 +1,29 @@
+import numpy as np
+
+from src.activations.Activation import Activation
+from src.core.Tensor import Tensor
+
+
+class Tanh(Activation):
+
+    @staticmethod
+    def forward(x):
+        t = np.tanh(x.data)
+
+        out = Tensor(
+            t,
+            requires_grad=x.requires_grad,
+            parents=(x,),
+            op="Tanh",
+        )
+
+        def _backward():
+            if x.requires_grad:
+                x.grad += out.grad * (1 - t * t)
+
+        out._backward = _backward
+
+        return out
+
+    def __call__(self, x):
+        return x.tanh()

src/activations/__init__.py

@@ -0,0 +1,17 @@
+from .ELU import ELU
+from .GELU import GELU
+from .ReLU import ReLU
+from .SELU import SELU
+from .Sigmoid import Sigmoid
+from .Tanh import Tanh
+from .Softmax import Softmax
+
+activation_fns = {
+    "relu": ReLU(),
+    "tanh": Tanh(),
+    "sigmoid": Sigmoid(),
+    "elu": ELU(),
+    "selu": SELU(),
+    "softmax": Softmax(),
+    "gelu": GELU(),
+}

src/config.py

@@ -0,0 +1,120 @@
+from dataclasses import asdict, dataclass, field
+from typing import List, Optional
+
+import yaml
+
+
+@dataclass
+class ModelSpec:
+    vocab_size: int
+    context_length: int = 1024
+    d_model: int = 768
+    n_layers: int = 12
+    n_heads: int = 12
+    ff_dim: int = 3072
+    activation: str = "gelu"
+    dropout: float = 0.0
+    norm_eps: float = 1e-5
+    tie_weights: bool = True
+    position_encoding: str = "learned"
+    rope_theta: float = 10000.0
+    rope_scaling: str = "none"
+    rope_scaling_factor: float = 1.0
+    trained_context_length: Optional[int] = None
+
+
+@dataclass
+class DataSpec:
+    train: List[str] = field(default_factory=list)
+    validation: Optional[List[str]] = None
+    tokenizer: str = "tokenizer.json"
+    sequence_length: int = 1024
+    read_buffer_size: int = 1 << 20
+    insert_eos: bool = True
+    text_field: str = "text"
+    workers: int = 1
+    prefetch: int = 4
+    shuffle_files: bool = False
+    val_split_fraction: Optional[float] = None
+    format: str = "text"
+    chat_template: str = "plain"
+    chat_packing: str = "pack"
+    messages_field: str = "messages"
+
+
+@dataclass
+class TrainingSpec:
+    max_tokens: Optional[int] = None
+    max_epochs: Optional[int] = None
+    total_steps: Optional[int] = None
+    micro_batch_size: int = 8
+    gradient_accumulation_steps: int = 1
+    learning_rate: float = 3e-4
+    weight_decay: float = 0.01
+    warmup_steps: int = 0
+    decay: str = "cosine"
+    min_lr: float = 0.0
+    gradient_clip: float = 1.0
+    precision: str = "fp32"
+    loss_scaling: str = "auto"
+    initial_loss_scale: float = 65536.0
+    loss_scale_growth_interval: int = 2000
+    activation_checkpointing: bool = False
+    init_from: Optional[str] = None
+    seed: int = 1337
+
+
+@dataclass
+class CheckpointSpec:
+    directory: str = "checkpoints"
+    interval_steps: int = 500
+    keep_last: int = 3
+    keep_every: Optional[int] = None
+
+
+@dataclass
+class EvaluationSpec:
+    interval_steps: int = 500
+    interval_tokens: Optional[int] = None
+    max_batches: int = 50
+
+
+@dataclass
+class RuntimeSpec:
+    device: str = "cpu"
+    seed: int = 1337
+
+
+@dataclass
+class TrainConfig:
+    model: ModelSpec
+    data: DataSpec
+    training: TrainingSpec
+    checkpoint: CheckpointSpec
+    evaluation: EvaluationSpec
+    runtime: RuntimeSpec
+
+    def to_dict(self):
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, raw):
+        return cls(
+            model=ModelSpec(**raw.get("model", {})),
+            data=DataSpec(**raw.get("data", {})),
+            training=TrainingSpec(**raw.get("training", {})),
+            checkpoint=CheckpointSpec(**raw.get("checkpoint", {})),
+            evaluation=EvaluationSpec(**raw.get("evaluation", {})),
+            runtime=RuntimeSpec(**raw.get("runtime", {})),
+        )
+
+    @classmethod
+    def load(cls, path):
+        with open(path) as f:
+            raw = yaml.safe_load(f)
+
+        return cls.from_dict(raw)
+
+    def save(self, path):
+        with open(path, "w") as f:
+            yaml.safe_dump(self.to_dict(), f, sort_keys=False)

src/core/Matrix.py

@@ -0,0 +1,3 @@
+class Matrix:
+    def __init__(self, data):
+        self.data = data

src/core/Scalar.py

@@ -0,0 +1,18 @@
+class Scalar:
+    def __init__(self):
+        self.data = None
+
+    def log(self):
+        print(self.data)
+
+    def one(self):
+        self.data = 1
+        return self
+
+    def zero(self):
+        self.data = 0
+        return self
+
+    def add(self, other):
+        self.data = self.data + other.data
+        return self