initium-deeplearning 0.3.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/PKG-INFO +62 -22
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/README.md +61 -21
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/pyproject.toml +1 -1
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/__init__.py +1 -1
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/LNN_controller/cfc_backbone_parallel.py +92 -8
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/LNN_controller/cfc_controller.py +20 -1
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/MoE/moe_layer.py +38 -24
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/__init__.py +14 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/core_training.py +3872 -165
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/base_dataset.py +7 -1
- initium_deeplearning-0.5.0/src/initium/data/common/causal_prediction.py +75 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/dataset_registry.py +13 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/graph_traversal/graph_traversal.py +321 -1
- initium_deeplearning-0.5.0/src/initium/data/surprise_recall/__init__.py +5 -0
- initium_deeplearning-0.5.0/src/initium/data/surprise_recall/next_token_prediction.py +377 -0
- initium_deeplearning-0.5.0/src/initium/data/surprise_recall/surprise_recall.py +553 -0
- initium_deeplearning-0.5.0/src/initium/data/surprise_recall/validate.py +100 -0
- initium_deeplearning-0.5.0/src/initium/data/workspace_capability/README.md +299 -0
- initium_deeplearning-0.5.0/src/initium/data/workspace_capability/__init__.py +5 -0
- initium_deeplearning-0.5.0/src/initium/data/workspace_capability/curriculum.py +160 -0
- initium_deeplearning-0.5.0/src/initium/data/workspace_capability/dataset.py +2132 -0
- initium_deeplearning-0.5.0/src/initium/data/workspace_capability/episode.py +137 -0
- initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/__init__.py +1 -0
- initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/entity_state.py +303 -0
- initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/nth_farthest.py +91 -0
- initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/permutation.py +185 -0
- initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/relational_join.py +86 -0
- initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/retrieval_aggregation.py +103 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/diagnostics/segment_logging.py +68 -22
- initium_deeplearning-0.5.0/src/initium/diagnostics/split_graph_forward_trace.py +204 -0
- initium_deeplearning-0.5.0/src/initium/diagnostics/surprise_recall_logging.py +623 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/checkpoint_io.py +1 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/model_loader.py +36 -2
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/mamba_controller/mamba2_controller.py +11 -5
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/mamba_controller/mamba3_controller.py +9 -4
- initium_deeplearning-0.5.0/src/initium/mamba_controller/mamba_backbone_parallel.py +744 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/mamba_controller/mamba_controller.py +11 -11
- initium_deeplearning-0.5.0/src/initium/mamba_controller/numerical_stability.py +43 -0
- initium_deeplearning-0.5.0/src/initium/mamba_controller/sgr_backbone_only.py +141 -0
- initium_deeplearning-0.5.0/src/initium/mamba_controller/split_graph_dnc.py +1721 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/dnc_mechanisms.py +103 -0
- initium_deeplearning-0.5.0/src/initium/memory_manipulation/sequence_memories.py +513 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/stochastic_write_head_v2.py +103 -38
- initium_deeplearning-0.5.0/src/initium/workspace/README.md +159 -0
- initium_deeplearning-0.5.0/src/initium/workspace/broadcast.py +2288 -0
- initium_deeplearning-0.5.0/src/initium/workspace/kv_bridge.py +166 -0
- initium_deeplearning-0.5.0/src/initium/workspace/nth_farthest.py +43 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/README.md +9 -2
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/integration/test_split_graph_dnc.py +34 -0
- initium_deeplearning-0.5.0/tests/integration/test_workspace_capability_pipeline.py +396 -0
- initium_deeplearning-0.5.0/tests/unit/test_next_token_prediction.py +114 -0
- initium_deeplearning-0.5.0/tests/unit/test_numerical_stability.py +57 -0
- initium_deeplearning-0.5.0/tests/unit/test_sequence_memories.py +138 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_stochastic_write_head.py +1 -1
- initium_deeplearning-0.5.0/tests/unit/test_surprise_recall.py +68 -0
- initium_deeplearning-0.5.0/tests/unit/test_workspace_capability.py +645 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_workspace_reconsideration.py +4 -0
- initium_deeplearning-0.3.0/src/initium/mamba_controller/mamba_backbone_parallel.py +0 -332
- initium_deeplearning-0.3.0/src/initium/mamba_controller/split_graph_dnc.py +0 -802
- initium_deeplearning-0.3.0/src/initium/workspace/broadcast.py +0 -533
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/.gitignore +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/LICENSE +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/LNN_controller/chained_controller.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/LNN_controller/hybrid_controller.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/MoE/README.md +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/analysis/__init__.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/analysis/kl_attribution.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/config/controller_config.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/config/train_config.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/__init__.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/audio/__init__.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/audio/audio_dataset.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/common/__init__.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/common/chain_task.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/common/digit_codec.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/common/graph_io.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/common/real_data.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/graph_traversal/__init__.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/graph_traversal/order_probe.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/graph_traversal/relational_probe.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/multimodal/__init__.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/multimodal/multimodal_dataset.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/prepared_dataset.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/text/__init__.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/text/text_dataset.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/video/__init__.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/video/video_dataset.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/diagnostics/__init__.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/__init__.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/__init__.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/base_cache.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/cache_config.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/cache_registry.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/cached_engine.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/keys.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/prefix_cache.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/result_cache.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/state_utils.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/stores.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/capabilities.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/engine.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/inference_config.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/metrics.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/run_inference.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/run_logging.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/__init__.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/audio_task.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/base_task.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/graph_traversal_task.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/multimodal_task.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/task_registry.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/text_task.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/video_task.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/dynamic_memory_resize.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/dynamic_n_controller.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/link_matrix_ablation.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/nvrtc_compat.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/setup_wheels.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/workspace/__init__.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/workspace/bistability.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/checkpoint/test_checkpoint_roundtrip.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/checkpoint/test_resume_training.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/conftest.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/gradients/test_controller_cell_backward.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/gradients/test_kl_gradients.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/gradients/test_moe_aux_loss_gradients.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/integration/test_controller_matrix.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/integration/test_dynamic_n_end_to_end.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/integration/test_moe_integration.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/integration/test_resize_x_link_matrix.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/smoke/test_inference_smoke.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/smoke/test_tiny_training_run.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_bistable_workspace_gate.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_capabilities.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_checkpoint_io.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_dynamic_n_controller.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_link_matrix_ablation.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_model_loader.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_moe_layer.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_nvrtc_compat.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_optimized_dnc_memory.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_real_data_pipeline.py +0 -0
- {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_resize_memory.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: initium-deeplearning
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Differentiable Neural Computer research framework with pluggable LSTM/Mamba/CfC controllers, MoE, split-graph and stochastic write heads.
|
|
5
5
|
Project-URL: Homepage, https://github.com/samueleternity/initium
|
|
6
6
|
Project-URL: Issues, https://github.com/samueleternity/initium/issues
|
|
@@ -112,6 +112,46 @@ Dataset registry ──► task episodes / curriculum ──► batch and sequen
|
|
|
112
112
|
|
|
113
113
|
The training core delegates task construction and task-specific behavior to a dataset interface. The model combines a selected controller with DNC memory operations. During training, the loop computes task and optional auxiliary objectives, updates model parameters, periodically evaluates ID and OOD performance, advances the curriculum when its criteria are met, writes diagnostics, and saves resumable checkpoints. Inference reconstructs a model from a checkpoint, checks that the requested task and dimensions are supported, creates the task runner, and reports episode metrics. Optional inference caches can reuse eligible results or shared prefixes.
|
|
114
114
|
|
|
115
|
+
### Surprise-Gated Recall task-validity baseline
|
|
116
|
+
|
|
117
|
+
`--dataset-type surprise-recall` selects a synthetic next-token recall task with a seven-lesson curriculum. The generator records expected and observed tokens and exposes A1 marked events, A2 silent replacements drawn from the same episode pattern, A3 predictable alert-token decoys, A4 element skips/repeats that shift subsequent positions, A5 regime changes, and A6 benign channel jitter. Targets are masked to the answer span; the task-validity baseline adds no prediction decoder or error-to-Δ path. Its generator-level checks cover label consistency, A2/A3 content-detector behavior, a stateless random-guess reference, and the direct oracle-answer upper bound. These checks do not train the required no-memory or oracle-flag model controls. The command also reports generated sequence-token counts by lesson.
|
|
118
|
+
|
|
119
|
+
Run the gate from the repository root with `python -m initium.data.surprise_recall.validate` (in an editable/source checkout, set `PYTHONPATH=src` if needed). Start a training run with the requested split-graph architecture by adding `--dataset-type surprise-recall` to the usual command:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
The optional MoE and diagnostic flags remain independent architecture/training options. The task-validity baseline does not add prediction-error gating, DNC/workspace error salience, or a special inference task runner.
|
|
126
|
+
|
|
127
|
+
### Next-token decoder and prediction-signal evaluation
|
|
128
|
+
|
|
129
|
+
This experiment adds a dense next-token decoder over block 0's causal split-graph features, preserving the planned inter-layer signal source for the gating comparison. `--sgr-pred-loss-weight` enables its stream-only cross-entropy objective alongside the existing answer-masked recall objective; the decoder receives the representation at `t-1` and predicts token `t`. Query, answer, and padding positions are excluded. It adds no error injection, gating, MoE, or workspace path. Use a fresh run for each prediction-loss weight (the roadmap suggests 0.1, 0.3, and 1.0) and keep seed, step budget, and architecture matched to the task-validity baseline:
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --sgr-pred-loss-weight 0.3
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
`--sgr-pred-eval-episodes` controls the fixed, deterministic probe count per lesson (default 128). The signal evaluation writes a per-run SGR signal CSV alongside the general SGR logs. It records raw-surprisal and excess-surprise AUROC for each A type against clean normal positions, calibration (ECE, accuracy, and Brier score), and A5 recovery. A6's per-type AUROCs measure benign jitter against clean normal positions; the additional A6-versus-true-anomaly comparison is diagnostic. Treat the decoder and signal implementation as complete while its signal gate remains unevaluated on completed runs; this does not establish that prediction error improves recall.
|
|
136
|
+
|
|
137
|
+
When `--dataset-type surprise-recall` is active, the trainer also creates `run_<run-id>_sgr_train.csv`, `run_<run-id>_sgr_positions.csv`, and `run_<run-id>_sgr_eval.csv` in the log directory. The batch log records A1–A6 prevalence and task conditions; the position trace records each stream token against its expected pattern token, event label, alert flag, recall-target status, and benign-jitter status. The evaluation log records task and exact-episode accuracy, recalled-event accuracy by A type, event prevalence, alert-token and pattern-mismatch detector confusion, and detector responses at each A-type event (including A3 decoy flags and A6 benign-jitter false alarms). It includes periodic and final ID/OOD results plus paired ID memory-on/memory-off results.
|
|
138
|
+
|
|
139
|
+
#### Tracing split-graph forward instability
|
|
140
|
+
|
|
141
|
+
Use `--split-graph-numeric-trace-step STEP` to inspect one absolute optimizer step after resuming from a checkpoint saved before the suspected failure. Re-run with the same seed, architecture, task, and relevant training options. For every observed boundary, the trace prints dtype, shape, finite/non-finite counts, maximum finite absolute value, and (on failure) the first bad coordinate to the console and CSV. It checks before and after Mamba normalization, mixer, cast, and residual; it also covers the CfC adapter/cell/residual/state, DNC state/read, recurrent output, logits, and loss. Timesteps are included for recurrent boundaries. It writes `run_<run-id>_splitgraph_forward_trace.csv` in the configured log directory and stops at the first observed non-finite value. The completion line reports rows written and warns if no boundary was observed. For example, to investigate the reported step 11,635 failure:
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --resume <checkpoint-before-step-11635.pt> --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --total-steps 12000 --split-graph-numeric-trace-step 11635
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
The selected step synchronizes CUDA at every observed boundary, so this flag is intended for a narrow replay and adds no such checks to other steps. A clean selected step means no observed tensor was non-finite at those boundaries; use the largest `max_abs_finite` rows to identify where scale first grows. A failure reports the first *instrumented* bad boundary; it cannot identify an internal operation between two boundaries. MoE and workspace boundaries are included when those paths are active, but compare runs with identical options to preserve replay conditions.
|
|
148
|
+
|
|
149
|
+
For intermittent failures whose step shifts between runs, use `--split-graph-numeric-trace-on-error` instead of selecting a step. It performs no boundary tensor checks during healthy training. When the trainer's existing finite checks detect a non-finite recurrent output, task logit/loss, or SGR prediction loss, it replays that same batch once with detailed tracing enabled, prints each boundary to the console, and writes the same CSV. The original training failure is still raised before backward or an optimizer update; if the replay does not reproduce the non-finite value, the trace reports that result and preserves the original failure. This option requires `--split-graph`, is for training runs, and cannot be combined with `--split-graph-numeric-trace-step`.
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --resume <checkpoint.pt> --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --total-steps 30000 --split-graph-numeric-trace-on-error
|
|
153
|
+
```
|
|
154
|
+
|
|
115
155
|
## Architecture and major components
|
|
116
156
|
|
|
117
157
|
### Differentiable external memory
|
|
@@ -143,14 +183,14 @@ python -m initium.core_training 0.0 --split-graph --split-graph-variant mamba1 -
|
|
|
143
183
|
python -m initium.core_training 0.0 --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant mamba+cfc
|
|
144
184
|
```
|
|
145
185
|
|
|
146
|
-
###
|
|
186
|
+
### Workspace plumbing (MoE broadcast)
|
|
147
187
|
|
|
148
188
|
The optional `--workspace` path adds a persistent shared slot matrix to a split-graph model with MoE enabled. The final parallel-backbone MoE bank supplies the specialist candidates and its existing noisy top-k routes. At each sequence timestep, selected expert outputs compete for slot writes; a learned RMC-style gate blends proposed slot content with the previous matrix. Every expert in that bank then attends to the updated matrix, and the existing sparse MoE routes combine the context-enriched expert outputs. The workspace resets at the beginning of each episode and persists across its timesteps. The `--moe-backbone-residual-scale` setting scales the complete final-bank delta, including its broadcast contribution, just as it scales the ordinary MoE residuals in the selection-only backbone. The DNC memory uses the same fp32/autocast-disabled path with or without workspace. This is a shared-object broadcast implementation; it does not implement per-recipient translation.
|
|
149
189
|
|
|
150
|
-
Use the
|
|
190
|
+
Use the selected split-graph/CfC-combiner setup with MoE and the workspace enabled as follows:
|
|
151
191
|
|
|
152
192
|
```bash
|
|
153
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
193
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix workspace-coadapted --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --workspace --workspace-num-slots 5 --workspace-train-mode coadapted --total-steps 30000
|
|
154
194
|
```
|
|
155
195
|
|
|
156
196
|
`--workspace` requires both `--split-graph` and `--moe`. The default is 5 slots, based on Goyal et al.'s shared-workspace reference result; it is a starting configuration, not a claimed optimum for graph traversal. `--workspace-train-mode coadapted` trains the full model. Choose `frozen` to train only workspace parameters while freezing the backbone, MoE/combiner, DNC, stochastic write heads, and task output projection. `--workspace-no-selection` replaces competitive selected writes with random unweighted writes for the no-Selection control. `--workspace-reset-each-step` resets slots before each timestep to test persistence. `--workspace-num-slots 0` bypasses workspace writes and readback, providing a plumbing comparison against the existing MoE model. Run IDs encode these conditions to avoid log/checkpoint collisions.
|
|
@@ -159,21 +199,21 @@ For a frozen-workspace experiment, pass a compatible workspace-disabled baseline
|
|
|
159
199
|
|
|
160
200
|
The console prints a startup line that reports whether the workspace is active or bypassed, its capacity, write-selection mode, persistence setting, train mode, and active compute precision. Active workspace attention and slot-state updates run in fp32 outside CUDA autocast; zero-slot/closed-gate bypass retains the selection-only path. At the normal training log cadence, `workspace[...]` reports selected write mass and active-write fraction, the learned write-gate mean, slot-state RMS, readback-to-expert-output RMS ratio, and read-gate value. These values show whether the path is active and the relative scale of its write and broadcast signals; they do not by themselves establish a task benefit.
|
|
161
201
|
|
|
162
|
-
The shared object operates on the final backbone MoE bank; earlier backbone MoE layers retain their existing behavior. This is the
|
|
202
|
+
The shared object operates on the final backbone MoE bank; earlier backbone MoE layers retain their existing behavior. This is the workspace-plumbing target, not evidence that Broadcast improves task performance. The first short run should compare the zero-slot bypass against the same MoE configuration without `--workspace`; only after this plumbing gate should the frozen/co-adapted and no-Selection comparisons begin. Record seed, full command, commit, hardware/software environment, curriculum completion, ID/OOD metrics, convergence steps, and any failures in `D:/Projects/AI-Architecture-Research/07-Experiment/Broadcasting/Progress/Workspace-Plumbing-Implementation.md`. All current evidence is graph-traversal-specific; transfer across task families remains untested.
|
|
163
203
|
|
|
164
|
-
###
|
|
204
|
+
### Bistable write-gate experiment
|
|
165
205
|
|
|
166
|
-
The default `--workspace-gate-variant continuous` preserves the
|
|
206
|
+
The default `--workspace-gate-variant continuous` preserves the existing sigmoid update. The opt-in `bistable` variant treats each workspace slot as a competing write candidate. It combines the workspace's content-selection score and the MoE route confidence with independent Gaussian background noise, then iterates recurrent self-excitation and lateral inhibition before making a hard write/no-write decision. A straight-through estimator carries gradients through the pre-commit probabilities. This is an experimental adaptation of GNW ignition, not a mechanism implemented by the source neuroscience papers.
|
|
167
207
|
|
|
168
208
|
For the matched gate-only comparison, resume the same five-slot frozen-workspace checkpoint twice. The frozen control updates only the existing continuous `write_gate`; the bistable condition updates only the new recurrent gate parameters. The gate-variant warm start loads shared model/workspace weights, initializes only the new gate, and starts with a fresh optimizer. `--workspace-train-mode coadapted` remains available to train the full model from that same checkpoint as a separate condition. Bistable background noise uses an independent checkpointed RNG stream, so zero-noise and selection-strength ablations do not change the graph episode RNG stream.
|
|
169
209
|
|
|
170
210
|
```bash
|
|
171
|
-
python -m src.initium.core_training 0.0 --seed 0 --resume <step-8000-workspace-checkpoint.pt> --run-id-suffix
|
|
211
|
+
python -m src.initium.core_training 0.0 --seed 0 --resume <step-8000-workspace-checkpoint.pt> --run-id-suffix bistable-gate-continuous-control --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --workspace --workspace-num-slots 5 --workspace-train-mode frozen --workspace-train-scope gate --workspace-gate-variant continuous --total-steps 8100
|
|
172
212
|
```
|
|
173
213
|
|
|
174
214
|
Use the same command, checkpoint, seed, and step budget for the bistable condition, changing the suffix and adding `--workspace-gate-variant bistable`. Gate-only runs use a safety tripwire that aborts before backward/update if any instrumented segment exceeds `--workspace-stability-max-activation` (default `100000`). Segment diagnostics report activation/gradient RMS and maxima; bistable diagnostics additionally report the pre-commit probability histogram (10 bins), low/middle/high occupancy, hard commit frequency, writer frequencies, slot RMS, and readback scale during training and held-out ID/OOD evaluation. A zero-noise ablation uses `--workspace-bistable-noise-scale 0`; selection-strength sweeps use `--workspace-bistable-selection-scale` while holding `--eval-seed` fixed. The gate still samples noise during evaluation unless `--workspace-bistable-deterministic` is supplied.
|
|
175
215
|
|
|
176
|
-
The preregistered operational bimodality check is at least 10% of held-out candidate probabilities below 0.1, at least 10% above 0.9, and at most 20% in [0.1, 0.9], sustained across episodes. Also require finite, nonzero gate gradients, no repeated skipped optimizer updates, and no task or paired-memory-dependency collapse. Run the short stability/pilot budget first; do not start a long
|
|
216
|
+
The preregistered operational bimodality check is at least 10% of held-out candidate probabilities below 0.1, at least 10% above 0.9, and at most 20% in [0.1, 0.9], sustained across episodes. Also require finite, nonzero gate gradients, no repeated skipped optimizer updates, and no task or paired-memory-dependency collapse. Run the short stability/pilot budget first; do not start a long bistable write-gate run unless the histogram and stability gates pass. Results remain specific to graph traversal and the single available trained model seed.
|
|
177
217
|
|
|
178
218
|
After saving a pilot checkpoint, evaluate that same checkpoint and `--eval-seed` under each `--eval-workspace-mode checkpoint`, `reset-each-step`, and `bypass`. For noise and selection ablations, use the same `--eval-seed` and change only the corresponding bistable scale. The gate RNG is separately checkpointed, so these changes keep the episode stream fixed; use distinct run suffixes when recording each condition.
|
|
179
219
|
|
|
@@ -185,7 +225,7 @@ The SwitchMoE router projection and expert MLPs run in FP32 even when the surrou
|
|
|
185
225
|
|
|
186
226
|
Periodic memory-dependency checks replay the exact random episode batch used by the corresponding memory-on ID evaluation, then restore the RNG streams. This makes `memory on - memory off` a paired functional ablation rather than a difference between two independently sampled batches. The training loop also clips all trainable optimizer parameters, including the task output projection, stops before backward on non-finite forward outputs/losses, reports the first parameter with a non-finite unscaled gradient, logs skipped non-finite-gradient updates, and aborts after eight consecutive such steps instead of continuing a run that is no longer updating safely.
|
|
187
227
|
|
|
188
|
-
The supplied
|
|
228
|
+
The supplied selection-only baseline and workspace-run traces are pre-fix evidence. The workspace-run trace first reports a non-finite task loss at step 11,665, recurrent outputs become non-finite at step 11,666, and the process continues to step 14,106 with an AMP scale of zero before manual interruption. Memory-on/off checks in those logs used separate random episode batches, so their small positive or negative differences are not reliable paired evidence. See `D:/Projects/AI-Architecture-Research/07-Experiment/Broadcasting/Progress/Selection-Only-MoE-Failure-Analysis.md` for the causal breakdown and re-run gate. Re-run the selection-only baseline before the workspace experiment; keep the proposed 8-expert, top-1 configuration and matched seed/settings fixed while checking finite gradients, per-bank routing, and paired memory dependency.
|
|
189
229
|
|
|
190
230
|
Training console logs now show **one MoE-bank record per 100-step logging window**: route and router-importance fractions by expert, router entropy and maximum probability, input/expert/MoE-delta activation RMS and maximum magnitude, capacity-drop fraction, and accepted gate mass. The `moe_residual:` segment records the complete MoE block output after its skip connection, separately from `moe_delta_rms/max`. A separate `[segment-diag:train]` line reports each recognized module's call count, input/output RMS and max, non-finite fraction, boundary input/output gradient RMS and non-finite fraction, and hierarchical parameter-gradient norms. Boundary gradients are collected during training backward and during failure traces; evaluation remains forward-only. The collector recognizes Mamba and CfC backbones/blocks, Mamba/CfC controller wrappers/blocks/cells, MoE blocks, chained controllers, DNC memory, workspace, and task output; names include their module paths, so the same logger works across supported controller combinations.
|
|
191
231
|
|
|
@@ -197,40 +237,40 @@ To inspect DNC internals, add `--dnc-backward-trace` to a split-graph training c
|
|
|
197
237
|
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix dnc-backward-trace --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 1800 --dnc-backward-trace
|
|
198
238
|
```
|
|
199
239
|
|
|
200
|
-
Keep other flags exactly matched to the run being investigated. The fp32 DNC path removed the observed bad-key-gradient pattern in the 5,000-step matched validation capture. That run also acquired strong paired ID memory dependence by step 2,500; see `AI-Architecture-Research/07-Experiment/Broadcasting/Progress/
|
|
240
|
+
Keep other flags exactly matched to the run being investigated. The fp32 DNC path removed the observed bad-key-gradient pattern in the 5,000-step matched validation capture. That run also acquired strong paired ID memory dependence by step 2,500; see `D:/Projects/AI-Architecture-Research/07-Experiment/Broadcasting/Progress/Selection-Only-MoE-Failure-Analysis.md` for the complete result and remaining generalization limits. Repeat across seeds before treating the numerical and learning outcomes as robust.
|
|
201
241
|
|
|
202
242
|
Evaluation emits separate channel lines for ID, OOD, memory-off, combiner-stage ablations, and final ID/OOD checks. These summarize the evaluations independently from training windows. If a forward value or gradient becomes non-finite, the current segment window and MoE-bank window are printed immediately before training stops or skips the unsafe update. The summaries aggregate detached device-side scalars and synchronize at log/evaluation boundaries; they do not retain activation graphs.
|
|
203
243
|
|
|
204
244
|
To isolate where split-graph MoE changes the memory-learning transition, `--moe-placement` selects which component receives expert banks while keeping the model, seed, and router settings matched. `all` (the default) preserves the existing behavior; `backbone` enables MoE only in the parallel backbone; `combiner` enables MoE only in the sequential controller combiner. For the CfC combiner, the latter is its source-aware multi-source MoE bank. Component placement requires `--moe --split-graph`; combiner-only additionally requires controller combiner mode. The workspace requires backbone MoE and therefore cannot be combined with combiner-only placement.
|
|
205
245
|
|
|
206
|
-
Run these short, seed-matched diagnostics on the same source revision. Each is capped at 5,000 steps so the usual early memory-use transition can be compared before committing to the full
|
|
246
|
+
Run these short, seed-matched diagnostics on the same source revision. Each is capped at 5,000 steps so the usual early memory-use transition can be compared before committing to the full MoE training budget:
|
|
207
247
|
|
|
208
248
|
```bash
|
|
209
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
210
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
211
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
212
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
249
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-baseline-no-moe --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --total-steps 5000
|
|
250
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-placement-backbone --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
|
|
251
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-placement-combiner --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement combiner --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
|
|
252
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-placement-all --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
|
|
213
253
|
```
|
|
214
254
|
|
|
215
|
-
Compare the paired memory-on/off dependency, ID/OOD metrics, curriculum progress, per-component segment diagnostics, and per-bank routing/activation summaries. The placement runs distinguish backbone-MoE effects from the CfC multi-source path; the `all` condition checks their interaction. These are diagnostic smoke runs, not evidence of a completed
|
|
255
|
+
Compare the paired memory-on/off dependency, ID/OOD metrics, curriculum progress, per-component segment diagnostics, and per-bank routing/activation summaries. The placement runs distinguish backbone-MoE effects from the CfC multi-source path; the `all` condition checks their interaction. These are diagnostic smoke runs, not evidence of a completed MoE placement comparison.
|
|
216
256
|
|
|
217
257
|
For a backbone-only run, `--moe-backbone-block-isolation INDEX` installs an MoE bank after only the selected zero-based parallel-backbone block. The other backbone blocks still run their Mamba/CfC computation but have no MoE; the sequential combiner must also be excluded with `--moe-placement backbone`. The flag is range-checked against `--split-graph-num-blocks`, included in run IDs and checkpoint metadata, and cannot be combined with the workspace broadcast path. For a two-block Mamba backbone, run one isolated condition per block:
|
|
218
258
|
|
|
219
259
|
```bash
|
|
220
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
221
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
260
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-backbone-block-0 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 0 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
|
|
261
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-backbone-block-1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
|
|
222
262
|
```
|
|
223
263
|
|
|
224
264
|
`--moe-backbone-residual-scale SCALE` multiplies the active parallel-backbone MoE delta before its residual addition (`backbone_output + SCALE * moe_delta`). Its default is `1.0`, preserving prior behavior. The scale must be finite and non-negative, requires split-graph MoE with `--moe-placement backbone` or `all`, and is included in run IDs/checkpoint metadata. With `all`, this changes only the parallel-backbone MoE residuals; combiner MoE residuals remain at 1.0. It scales the task-loss path through the backbone MoE branch while leaving routing and the load-balancing auxiliary loss active. When workspace broadcast is enabled, the scale applies to the complete final-bank delta after expert readback, including both the selected expert outputs and their broadcast contribution. Workspace slot writes and router/load-balancing calculations remain active at their existing scales. Active workspace attention and slot updates run in fp32 outside CUDA autocast, and split-graph DNC memory math uses its fp32 path in both workspace and selection-only runs. In an isolated-block run, only that block has an MoE residual to scale. For the current block-1 follow-up, use a deliberately conservative 0.1 scale:
|
|
225
265
|
|
|
226
266
|
```bash
|
|
227
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
267
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-block-1-residual-scale-0p1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 1 --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
|
|
228
268
|
```
|
|
229
269
|
|
|
230
270
|
To test backbone and combiner MoE together after the backbone-only scale control, omit block isolation and select `--moe-placement all`. The residual scale still applies only to the backbone; the combiner MoE keeps its default residual scale of 1.0:
|
|
231
271
|
|
|
232
272
|
```bash
|
|
233
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
273
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-backbone-combiner-scale-0p1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
|
|
234
274
|
```
|
|
235
275
|
|
|
236
276
|
Each `moe-bank[backbone.moe_blocks.INDEX ...]` line reports router-selected expert load, router importance, entropy, maximum probability, and capacity-drop rate separately for valid graph timesteps and all-zero padded timesteps. The valid-token mask is derived from the graph input encoding and is diagnostic-only in this experiment: routing, shared capacity, balancing loss, and outputs still include padding, so the run measures the current behavior without applying a padding fix. The per-block isolation changes which banks are active but leaves the routing policy otherwise unchanged. Compare the two blocks' valid/padded distributions and memory-dependency curves against the non-isolated backbone-only trace before deciding whether to test mask-aware routing.
|
|
@@ -355,7 +395,7 @@ A separate study compared beta 0 and beta 0.001 in a Mamba, dynamic-memory, spli
|
|
|
355
395
|
### Evidence boundaries and open work
|
|
356
396
|
|
|
357
397
|
- Results above apply to the exact runs and evaluation procedures recorded; they do not establish broad performance across modalities, hardware, or model scales.
|
|
358
|
-
-
|
|
398
|
+
- The fixed-prior study's seed variation and the learned-prior study's four-seed selection limit aggregate claims. Report per-seed values and uncertainty alongside averages.
|
|
359
399
|
- The learned prior's apparent variance-dampening/mean-reversion effect needs more seeds and controls to separate it from failure-mode-specific repair.
|
|
360
400
|
- The alternate Mamba batch needs completion of unfinished seeds and late-curriculum evaluation before a gate-level conclusion.
|
|
361
401
|
- Mamba-2/Mamba-3 performance, MoE's useful scaling regime, memory-link trade-offs, dynamic memory growth, and real-data/multimodal behavior remain active investigation areas.
|
|
@@ -81,6 +81,46 @@ Dataset registry ──► task episodes / curriculum ──► batch and sequen
|
|
|
81
81
|
|
|
82
82
|
The training core delegates task construction and task-specific behavior to a dataset interface. The model combines a selected controller with DNC memory operations. During training, the loop computes task and optional auxiliary objectives, updates model parameters, periodically evaluates ID and OOD performance, advances the curriculum when its criteria are met, writes diagnostics, and saves resumable checkpoints. Inference reconstructs a model from a checkpoint, checks that the requested task and dimensions are supported, creates the task runner, and reports episode metrics. Optional inference caches can reuse eligible results or shared prefixes.
|
|
83
83
|
|
|
84
|
+
### Surprise-Gated Recall task-validity baseline
|
|
85
|
+
|
|
86
|
+
`--dataset-type surprise-recall` selects a synthetic next-token recall task with a seven-lesson curriculum. The generator records expected and observed tokens and exposes A1 marked events, A2 silent replacements drawn from the same episode pattern, A3 predictable alert-token decoys, A4 element skips/repeats that shift subsequent positions, A5 regime changes, and A6 benign channel jitter. Targets are masked to the answer span; the task-validity baseline adds no prediction decoder or error-to-Δ path. Its generator-level checks cover label consistency, A2/A3 content-detector behavior, a stateless random-guess reference, and the direct oracle-answer upper bound. These checks do not train the required no-memory or oracle-flag model controls. The command also reports generated sequence-token counts by lesson.
|
|
87
|
+
|
|
88
|
+
Run the gate from the repository root with `python -m initium.data.surprise_recall.validate` (in an editable/source checkout, set `PYTHONPATH=src` if needed). Start a training run with the requested split-graph architecture by adding `--dataset-type surprise-recall` to the usual command:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
The optional MoE and diagnostic flags remain independent architecture/training options. The task-validity baseline does not add prediction-error gating, DNC/workspace error salience, or a special inference task runner.
|
|
95
|
+
|
|
96
|
+
### Next-token decoder and prediction-signal evaluation
|
|
97
|
+
|
|
98
|
+
This experiment adds a dense next-token decoder over block 0's causal split-graph features, preserving the planned inter-layer signal source for the gating comparison. `--sgr-pred-loss-weight` enables its stream-only cross-entropy objective alongside the existing answer-masked recall objective; the decoder receives the representation at `t-1` and predicts token `t`. Query, answer, and padding positions are excluded. It adds no error injection, gating, MoE, or workspace path. Use a fresh run for each prediction-loss weight (the roadmap suggests 0.1, 0.3, and 1.0) and keep seed, step budget, and architecture matched to the task-validity baseline:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --sgr-pred-loss-weight 0.3
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
`--sgr-pred-eval-episodes` controls the fixed, deterministic probe count per lesson (default 128). The signal evaluation writes a per-run SGR signal CSV alongside the general SGR logs. It records raw-surprisal and excess-surprise AUROC for each A type against clean normal positions, calibration (ECE, accuracy, and Brier score), and A5 recovery. A6's per-type AUROCs measure benign jitter against clean normal positions; the additional A6-versus-true-anomaly comparison is diagnostic. Treat the decoder and signal implementation as complete while its signal gate remains unevaluated on completed runs; this does not establish that prediction error improves recall.
|
|
105
|
+
|
|
106
|
+
When `--dataset-type surprise-recall` is active, the trainer also creates `run_<run-id>_sgr_train.csv`, `run_<run-id>_sgr_positions.csv`, and `run_<run-id>_sgr_eval.csv` in the log directory. The batch log records A1–A6 prevalence and task conditions; the position trace records each stream token against its expected pattern token, event label, alert flag, recall-target status, and benign-jitter status. The evaluation log records task and exact-episode accuracy, recalled-event accuracy by A type, event prevalence, alert-token and pattern-mismatch detector confusion, and detector responses at each A-type event (including A3 decoy flags and A6 benign-jitter false alarms). It includes periodic and final ID/OOD results plus paired ID memory-on/memory-off results.
|
|
107
|
+
|
|
108
|
+
#### Tracing split-graph forward instability
|
|
109
|
+
|
|
110
|
+
Use `--split-graph-numeric-trace-step STEP` to inspect one absolute optimizer step after resuming from a checkpoint saved before the suspected failure. Re-run with the same seed, architecture, task, and relevant training options. For every observed boundary, the trace prints dtype, shape, finite/non-finite counts, maximum finite absolute value, and (on failure) the first bad coordinate to the console and CSV. It checks before and after Mamba normalization, mixer, cast, and residual; it also covers the CfC adapter/cell/residual/state, DNC state/read, recurrent output, logits, and loss. Timesteps are included for recurrent boundaries. It writes `run_<run-id>_splitgraph_forward_trace.csv` in the configured log directory and stops at the first observed non-finite value. The completion line reports rows written and warns if no boundary was observed. For example, to investigate the reported step 11,635 failure:
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --resume <checkpoint-before-step-11635.pt> --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --total-steps 12000 --split-graph-numeric-trace-step 11635
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The selected step synchronizes CUDA at every observed boundary, so this flag is intended for a narrow replay and adds no such checks to other steps. A clean selected step means no observed tensor was non-finite at those boundaries; use the largest `max_abs_finite` rows to identify where scale first grows. A failure reports the first *instrumented* bad boundary; it cannot identify an internal operation between two boundaries. MoE and workspace boundaries are included when those paths are active, but compare runs with identical options to preserve replay conditions.
|
|
117
|
+
|
|
118
|
+
For intermittent failures whose step shifts between runs, use `--split-graph-numeric-trace-on-error` instead of selecting a step. It performs no boundary tensor checks during healthy training. When the trainer's existing finite checks detect a non-finite recurrent output, task logit/loss, or SGR prediction loss, it replays that same batch once with detailed tracing enabled, prints each boundary to the console, and writes the same CSV. The original training failure is still raised before backward or an optimizer update; if the replay does not reproduce the non-finite value, the trace reports that result and preserves the original failure. This option requires `--split-graph`, is for training runs, and cannot be combined with `--split-graph-numeric-trace-step`.
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --resume <checkpoint.pt> --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --total-steps 30000 --split-graph-numeric-trace-on-error
|
|
122
|
+
```
|
|
123
|
+
|
|
84
124
|
## Architecture and major components
|
|
85
125
|
|
|
86
126
|
### Differentiable external memory
|
|
@@ -112,14 +152,14 @@ python -m initium.core_training 0.0 --split-graph --split-graph-variant mamba1 -
|
|
|
112
152
|
python -m initium.core_training 0.0 --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant mamba+cfc
|
|
113
153
|
```
|
|
114
154
|
|
|
115
|
-
###
|
|
155
|
+
### Workspace plumbing (MoE broadcast)
|
|
116
156
|
|
|
117
157
|
The optional `--workspace` path adds a persistent shared slot matrix to a split-graph model with MoE enabled. The final parallel-backbone MoE bank supplies the specialist candidates and its existing noisy top-k routes. At each sequence timestep, selected expert outputs compete for slot writes; a learned RMC-style gate blends proposed slot content with the previous matrix. Every expert in that bank then attends to the updated matrix, and the existing sparse MoE routes combine the context-enriched expert outputs. The workspace resets at the beginning of each episode and persists across its timesteps. The `--moe-backbone-residual-scale` setting scales the complete final-bank delta, including its broadcast contribution, just as it scales the ordinary MoE residuals in the selection-only backbone. The DNC memory uses the same fp32/autocast-disabled path with or without workspace. This is a shared-object broadcast implementation; it does not implement per-recipient translation.
|
|
118
158
|
|
|
119
|
-
Use the
|
|
159
|
+
Use the selected split-graph/CfC-combiner setup with MoE and the workspace enabled as follows:
|
|
120
160
|
|
|
121
161
|
```bash
|
|
122
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
162
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix workspace-coadapted --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --workspace --workspace-num-slots 5 --workspace-train-mode coadapted --total-steps 30000
|
|
123
163
|
```
|
|
124
164
|
|
|
125
165
|
`--workspace` requires both `--split-graph` and `--moe`. The default is 5 slots, based on Goyal et al.'s shared-workspace reference result; it is a starting configuration, not a claimed optimum for graph traversal. `--workspace-train-mode coadapted` trains the full model. Choose `frozen` to train only workspace parameters while freezing the backbone, MoE/combiner, DNC, stochastic write heads, and task output projection. `--workspace-no-selection` replaces competitive selected writes with random unweighted writes for the no-Selection control. `--workspace-reset-each-step` resets slots before each timestep to test persistence. `--workspace-num-slots 0` bypasses workspace writes and readback, providing a plumbing comparison against the existing MoE model. Run IDs encode these conditions to avoid log/checkpoint collisions.
|
|
@@ -128,21 +168,21 @@ For a frozen-workspace experiment, pass a compatible workspace-disabled baseline
|
|
|
128
168
|
|
|
129
169
|
The console prints a startup line that reports whether the workspace is active or bypassed, its capacity, write-selection mode, persistence setting, train mode, and active compute precision. Active workspace attention and slot-state updates run in fp32 outside CUDA autocast; zero-slot/closed-gate bypass retains the selection-only path. At the normal training log cadence, `workspace[...]` reports selected write mass and active-write fraction, the learned write-gate mean, slot-state RMS, readback-to-expert-output RMS ratio, and read-gate value. These values show whether the path is active and the relative scale of its write and broadcast signals; they do not by themselves establish a task benefit.
|
|
130
170
|
|
|
131
|
-
The shared object operates on the final backbone MoE bank; earlier backbone MoE layers retain their existing behavior. This is the
|
|
171
|
+
The shared object operates on the final backbone MoE bank; earlier backbone MoE layers retain their existing behavior. This is the workspace-plumbing target, not evidence that Broadcast improves task performance. The first short run should compare the zero-slot bypass against the same MoE configuration without `--workspace`; only after this plumbing gate should the frozen/co-adapted and no-Selection comparisons begin. Record seed, full command, commit, hardware/software environment, curriculum completion, ID/OOD metrics, convergence steps, and any failures in `D:/Projects/AI-Architecture-Research/07-Experiment/Broadcasting/Progress/Workspace-Plumbing-Implementation.md`. All current evidence is graph-traversal-specific; transfer across task families remains untested.
|
|
132
172
|
|
|
133
|
-
###
|
|
173
|
+
### Bistable write-gate experiment
|
|
134
174
|
|
|
135
|
-
The default `--workspace-gate-variant continuous` preserves the
|
|
175
|
+
The default `--workspace-gate-variant continuous` preserves the existing sigmoid update. The opt-in `bistable` variant treats each workspace slot as a competing write candidate. It combines the workspace's content-selection score and the MoE route confidence with independent Gaussian background noise, then iterates recurrent self-excitation and lateral inhibition before making a hard write/no-write decision. A straight-through estimator carries gradients through the pre-commit probabilities. This is an experimental adaptation of GNW ignition, not a mechanism implemented by the source neuroscience papers.
|
|
136
176
|
|
|
137
177
|
For the matched gate-only comparison, resume the same five-slot frozen-workspace checkpoint twice. The frozen control updates only the existing continuous `write_gate`; the bistable condition updates only the new recurrent gate parameters. The gate-variant warm start loads shared model/workspace weights, initializes only the new gate, and starts with a fresh optimizer. `--workspace-train-mode coadapted` remains available to train the full model from that same checkpoint as a separate condition. Bistable background noise uses an independent checkpointed RNG stream, so zero-noise and selection-strength ablations do not change the graph episode RNG stream.
|
|
138
178
|
|
|
139
179
|
```bash
|
|
140
|
-
python -m src.initium.core_training 0.0 --seed 0 --resume <step-8000-workspace-checkpoint.pt> --run-id-suffix
|
|
180
|
+
python -m src.initium.core_training 0.0 --seed 0 --resume <step-8000-workspace-checkpoint.pt> --run-id-suffix bistable-gate-continuous-control --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --workspace --workspace-num-slots 5 --workspace-train-mode frozen --workspace-train-scope gate --workspace-gate-variant continuous --total-steps 8100
|
|
141
181
|
```
|
|
142
182
|
|
|
143
183
|
Use the same command, checkpoint, seed, and step budget for the bistable condition, changing the suffix and adding `--workspace-gate-variant bistable`. Gate-only runs use a safety tripwire that aborts before backward/update if any instrumented segment exceeds `--workspace-stability-max-activation` (default `100000`). Segment diagnostics report activation/gradient RMS and maxima; bistable diagnostics additionally report the pre-commit probability histogram (10 bins), low/middle/high occupancy, hard commit frequency, writer frequencies, slot RMS, and readback scale during training and held-out ID/OOD evaluation. A zero-noise ablation uses `--workspace-bistable-noise-scale 0`; selection-strength sweeps use `--workspace-bistable-selection-scale` while holding `--eval-seed` fixed. The gate still samples noise during evaluation unless `--workspace-bistable-deterministic` is supplied.
|
|
144
184
|
|
|
145
|
-
The preregistered operational bimodality check is at least 10% of held-out candidate probabilities below 0.1, at least 10% above 0.9, and at most 20% in [0.1, 0.9], sustained across episodes. Also require finite, nonzero gate gradients, no repeated skipped optimizer updates, and no task or paired-memory-dependency collapse. Run the short stability/pilot budget first; do not start a long
|
|
185
|
+
The preregistered operational bimodality check is at least 10% of held-out candidate probabilities below 0.1, at least 10% above 0.9, and at most 20% in [0.1, 0.9], sustained across episodes. Also require finite, nonzero gate gradients, no repeated skipped optimizer updates, and no task or paired-memory-dependency collapse. Run the short stability/pilot budget first; do not start a long bistable write-gate run unless the histogram and stability gates pass. Results remain specific to graph traversal and the single available trained model seed.
|
|
146
186
|
|
|
147
187
|
After saving a pilot checkpoint, evaluate that same checkpoint and `--eval-seed` under each `--eval-workspace-mode checkpoint`, `reset-each-step`, and `bypass`. For noise and selection ablations, use the same `--eval-seed` and change only the corresponding bistable scale. The gate RNG is separately checkpointed, so these changes keep the episode stream fixed; use distinct run suffixes when recording each condition.
|
|
148
188
|
|
|
@@ -154,7 +194,7 @@ The SwitchMoE router projection and expert MLPs run in FP32 even when the surrou
|
|
|
154
194
|
|
|
155
195
|
Periodic memory-dependency checks replay the exact random episode batch used by the corresponding memory-on ID evaluation, then restore the RNG streams. This makes `memory on - memory off` a paired functional ablation rather than a difference between two independently sampled batches. The training loop also clips all trainable optimizer parameters, including the task output projection, stops before backward on non-finite forward outputs/losses, reports the first parameter with a non-finite unscaled gradient, logs skipped non-finite-gradient updates, and aborts after eight consecutive such steps instead of continuing a run that is no longer updating safely.
|
|
156
196
|
|
|
157
|
-
The supplied
|
|
197
|
+
The supplied selection-only baseline and workspace-run traces are pre-fix evidence. The workspace-run trace first reports a non-finite task loss at step 11,665, recurrent outputs become non-finite at step 11,666, and the process continues to step 14,106 with an AMP scale of zero before manual interruption. Memory-on/off checks in those logs used separate random episode batches, so their small positive or negative differences are not reliable paired evidence. See `D:/Projects/AI-Architecture-Research/07-Experiment/Broadcasting/Progress/Selection-Only-MoE-Failure-Analysis.md` for the causal breakdown and re-run gate. Re-run the selection-only baseline before the workspace experiment; keep the proposed 8-expert, top-1 configuration and matched seed/settings fixed while checking finite gradients, per-bank routing, and paired memory dependency.
|
|
158
198
|
|
|
159
199
|
Training console logs now show **one MoE-bank record per 100-step logging window**: route and router-importance fractions by expert, router entropy and maximum probability, input/expert/MoE-delta activation RMS and maximum magnitude, capacity-drop fraction, and accepted gate mass. The `moe_residual:` segment records the complete MoE block output after its skip connection, separately from `moe_delta_rms/max`. A separate `[segment-diag:train]` line reports each recognized module's call count, input/output RMS and max, non-finite fraction, boundary input/output gradient RMS and non-finite fraction, and hierarchical parameter-gradient norms. Boundary gradients are collected during training backward and during failure traces; evaluation remains forward-only. The collector recognizes Mamba and CfC backbones/blocks, Mamba/CfC controller wrappers/blocks/cells, MoE blocks, chained controllers, DNC memory, workspace, and task output; names include their module paths, so the same logger works across supported controller combinations.
|
|
160
200
|
|
|
@@ -166,40 +206,40 @@ To inspect DNC internals, add `--dnc-backward-trace` to a split-graph training c
|
|
|
166
206
|
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix dnc-backward-trace --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 1800 --dnc-backward-trace
|
|
167
207
|
```
|
|
168
208
|
|
|
169
|
-
Keep other flags exactly matched to the run being investigated. The fp32 DNC path removed the observed bad-key-gradient pattern in the 5,000-step matched validation capture. That run also acquired strong paired ID memory dependence by step 2,500; see `AI-Architecture-Research/07-Experiment/Broadcasting/Progress/
|
|
209
|
+
Keep other flags exactly matched to the run being investigated. The fp32 DNC path removed the observed bad-key-gradient pattern in the 5,000-step matched validation capture. That run also acquired strong paired ID memory dependence by step 2,500; see `D:/Projects/AI-Architecture-Research/07-Experiment/Broadcasting/Progress/Selection-Only-MoE-Failure-Analysis.md` for the complete result and remaining generalization limits. Repeat across seeds before treating the numerical and learning outcomes as robust.
|
|
170
210
|
|
|
171
211
|
Evaluation emits separate channel lines for ID, OOD, memory-off, combiner-stage ablations, and final ID/OOD checks. These summarize the evaluations independently from training windows. If a forward value or gradient becomes non-finite, the current segment window and MoE-bank window are printed immediately before training stops or skips the unsafe update. The summaries aggregate detached device-side scalars and synchronize at log/evaluation boundaries; they do not retain activation graphs.
|
|
172
212
|
|
|
173
213
|
To isolate where split-graph MoE changes the memory-learning transition, `--moe-placement` selects which component receives expert banks while keeping the model, seed, and router settings matched. `all` (the default) preserves the existing behavior; `backbone` enables MoE only in the parallel backbone; `combiner` enables MoE only in the sequential controller combiner. For the CfC combiner, the latter is its source-aware multi-source MoE bank. Component placement requires `--moe --split-graph`; combiner-only additionally requires controller combiner mode. The workspace requires backbone MoE and therefore cannot be combined with combiner-only placement.
|
|
174
214
|
|
|
175
|
-
Run these short, seed-matched diagnostics on the same source revision. Each is capped at 5,000 steps so the usual early memory-use transition can be compared before committing to the full
|
|
215
|
+
Run these short, seed-matched diagnostics on the same source revision. Each is capped at 5,000 steps so the usual early memory-use transition can be compared before committing to the full MoE training budget:
|
|
176
216
|
|
|
177
217
|
```bash
|
|
178
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
179
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
180
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
181
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
218
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-baseline-no-moe --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --total-steps 5000
|
|
219
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-placement-backbone --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
|
|
220
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-placement-combiner --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement combiner --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
|
|
221
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-placement-all --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
|
|
182
222
|
```
|
|
183
223
|
|
|
184
|
-
Compare the paired memory-on/off dependency, ID/OOD metrics, curriculum progress, per-component segment diagnostics, and per-bank routing/activation summaries. The placement runs distinguish backbone-MoE effects from the CfC multi-source path; the `all` condition checks their interaction. These are diagnostic smoke runs, not evidence of a completed
|
|
224
|
+
Compare the paired memory-on/off dependency, ID/OOD metrics, curriculum progress, per-component segment diagnostics, and per-bank routing/activation summaries. The placement runs distinguish backbone-MoE effects from the CfC multi-source path; the `all` condition checks their interaction. These are diagnostic smoke runs, not evidence of a completed MoE placement comparison.
|
|
185
225
|
|
|
186
226
|
For a backbone-only run, `--moe-backbone-block-isolation INDEX` installs an MoE bank after only the selected zero-based parallel-backbone block. The other backbone blocks still run their Mamba/CfC computation but have no MoE; the sequential combiner must also be excluded with `--moe-placement backbone`. The flag is range-checked against `--split-graph-num-blocks`, included in run IDs and checkpoint metadata, and cannot be combined with the workspace broadcast path. For a two-block Mamba backbone, run one isolated condition per block:
|
|
187
227
|
|
|
188
228
|
```bash
|
|
189
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
190
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
229
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-backbone-block-0 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 0 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
|
|
230
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-backbone-block-1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
|
|
191
231
|
```
|
|
192
232
|
|
|
193
233
|
`--moe-backbone-residual-scale SCALE` multiplies the active parallel-backbone MoE delta before its residual addition (`backbone_output + SCALE * moe_delta`). Its default is `1.0`, preserving prior behavior. The scale must be finite and non-negative, requires split-graph MoE with `--moe-placement backbone` or `all`, and is included in run IDs/checkpoint metadata. With `all`, this changes only the parallel-backbone MoE residuals; combiner MoE residuals remain at 1.0. It scales the task-loss path through the backbone MoE branch while leaving routing and the load-balancing auxiliary loss active. When workspace broadcast is enabled, the scale applies to the complete final-bank delta after expert readback, including both the selected expert outputs and their broadcast contribution. Workspace slot writes and router/load-balancing calculations remain active at their existing scales. Active workspace attention and slot updates run in fp32 outside CUDA autocast, and split-graph DNC memory math uses its fp32 path in both workspace and selection-only runs. In an isolated-block run, only that block has an MoE residual to scale. For the current block-1 follow-up, use a deliberately conservative 0.1 scale:
|
|
194
234
|
|
|
195
235
|
```bash
|
|
196
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
236
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-block-1-residual-scale-0p1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 1 --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
|
|
197
237
|
```
|
|
198
238
|
|
|
199
239
|
To test backbone and combiner MoE together after the backbone-only scale control, omit block isolation and select `--moe-placement all`. The residual scale still applies only to the backbone; the combiner MoE keeps its default residual scale of 1.0:
|
|
200
240
|
|
|
201
241
|
```bash
|
|
202
|
-
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix
|
|
242
|
+
python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-backbone-combiner-scale-0p1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
|
|
203
243
|
```
|
|
204
244
|
|
|
205
245
|
Each `moe-bank[backbone.moe_blocks.INDEX ...]` line reports router-selected expert load, router importance, entropy, maximum probability, and capacity-drop rate separately for valid graph timesteps and all-zero padded timesteps. The valid-token mask is derived from the graph input encoding and is diagnostic-only in this experiment: routing, shared capacity, balancing loss, and outputs still include padding, so the run measures the current behavior without applying a padding fix. The per-block isolation changes which banks are active but leaves the routing policy otherwise unchanged. Compare the two blocks' valid/padded distributions and memory-dependency curves against the non-isolated backbone-only trace before deciding whether to test mask-aware routing.
|
|
@@ -324,7 +364,7 @@ A separate study compared beta 0 and beta 0.001 in a Mamba, dynamic-memory, spli
|
|
|
324
364
|
### Evidence boundaries and open work
|
|
325
365
|
|
|
326
366
|
- Results above apply to the exact runs and evaluation procedures recorded; they do not establish broad performance across modalities, hardware, or model scales.
|
|
327
|
-
-
|
|
367
|
+
- The fixed-prior study's seed variation and the learned-prior study's four-seed selection limit aggregate claims. Report per-seed values and uncertainty alongside averages.
|
|
328
368
|
- The learned prior's apparent variance-dampening/mean-reversion effect needs more seeds and controls to separate it from failure-mode-specific repair.
|
|
329
369
|
- The alternate Mamba batch needs completion of unfinished seeds and late-curriculum evaluation before a gate-level conclusion.
|
|
330
370
|
- Mamba-2/Mamba-3 performance, MoE's useful scaling regime, memory-link trade-offs, dynamic memory growth, and real-data/multimodal behavior remain active investigation areas.
|
|
@@ -112,7 +112,7 @@ pretty = true
|
|
|
112
112
|
[[tool.mypy.overrides]]
|
|
113
113
|
module = [
|
|
114
114
|
"dnc", "dnc.*", "mamba_ssm", "mamba_ssm.*", "causal_conv1d", "causal_conv1d.*",
|
|
115
|
-
"ncps", "ncps.*", "tokenizers", "tokenizers.*", "cv2",
|
|
115
|
+
"ncps", "ncps.*", "einops", "tokenizers", "tokenizers.*", "cv2",
|
|
116
116
|
]
|
|
117
117
|
ignore_missing_imports = true
|
|
118
118
|
|
|
@@ -69,17 +69,53 @@ class _CfCParallelBlock(nn.Module):
|
|
|
69
69
|
if device is not None or dtype is not None:
|
|
70
70
|
self.cfc.to(device=device, dtype=dtype)
|
|
71
71
|
|
|
72
|
-
def
|
|
72
|
+
def _forward_with_timespans(self, x: torch.Tensor, timespans: torch.Tensor) -> torch.Tensor:
|
|
73
|
+
"""Run the ncps cell with batch-safe per-example elapsed times.
|
|
74
|
+
|
|
75
|
+
ncps' sequence wrapper squeezes each ``(B,)`` time slice before
|
|
76
|
+
passing it to CfCCell, which does not broadcast that shape against
|
|
77
|
+
``(B, H)`` for batches larger than one. Calling the same registered
|
|
78
|
+
cell directly with ``(B, 1)`` spans avoids that upstream shape bug
|
|
79
|
+
while preserving the package's recurrence and learned parameters.
|
|
80
|
+
"""
|
|
81
|
+
batch, sequence, _ = x.shape
|
|
82
|
+
h_state = x.new_zeros((batch, self.cfc.state_size), dtype=torch.float32)
|
|
83
|
+
c_state = torch.zeros_like(h_state) if self.cfc.use_mixed else None
|
|
84
|
+
outputs = []
|
|
85
|
+
for step in range(sequence):
|
|
86
|
+
step_input = x[:, step]
|
|
87
|
+
if self.cfc.use_mixed:
|
|
88
|
+
assert c_state is not None
|
|
89
|
+
h_state, c_state = self.cfc.lstm(step_input, (h_state, c_state))
|
|
90
|
+
step_output, h_state = self.cfc.rnn_cell(
|
|
91
|
+
step_input,
|
|
92
|
+
h_state,
|
|
93
|
+
timespans[:, step : step + 1],
|
|
94
|
+
)
|
|
95
|
+
outputs.append(self.cfc.fc(step_output))
|
|
96
|
+
return torch.stack(outputs, dim=1)
|
|
97
|
+
|
|
98
|
+
def forward(self, x, *, timespans: torch.Tensor | None = None): # x: (B, L, D)
|
|
73
99
|
if self.force_fp32:
|
|
74
100
|
with torch.autocast(device_type=x.device.type, enabled=False):
|
|
75
|
-
|
|
101
|
+
normalized = self.norm(x.float())
|
|
102
|
+
if timespans is None:
|
|
103
|
+
out, _ = self.cfc(normalized)
|
|
104
|
+
else:
|
|
105
|
+
out = self._forward_with_timespans(normalized, timespans)
|
|
76
106
|
else:
|
|
77
|
-
|
|
107
|
+
normalized = self.norm(x)
|
|
108
|
+
if timespans is None:
|
|
109
|
+
out, _ = self.cfc(normalized)
|
|
110
|
+
else:
|
|
111
|
+
out = self._forward_with_timespans(normalized, timespans)
|
|
78
112
|
out = out.to(x.dtype)
|
|
79
113
|
return x + out if self.residual else out
|
|
80
114
|
|
|
81
115
|
|
|
82
116
|
class CfCBackboneParallel(nn.Module):
|
|
117
|
+
supports_timespans = True
|
|
118
|
+
|
|
83
119
|
def __init__(
|
|
84
120
|
self,
|
|
85
121
|
in_dim,
|
|
@@ -164,20 +200,63 @@ class CfCBackboneParallel(nn.Module):
|
|
|
164
200
|
)
|
|
165
201
|
|
|
166
202
|
def forward(
|
|
167
|
-
self,
|
|
203
|
+
self,
|
|
204
|
+
x,
|
|
205
|
+
*,
|
|
206
|
+
token_mask: torch.Tensor | None = None,
|
|
207
|
+
timespans: torch.Tensor | None = None,
|
|
168
208
|
): # (B, L, in_dim) -> (B, L, d_model), called ONCE per training step
|
|
209
|
+
timespans = self._normalize_timespans(timespans, x)
|
|
169
210
|
h = self.in_adapter(x)
|
|
170
211
|
moe_blocks = self.moe_blocks
|
|
171
212
|
for i, block in enumerate(self.blocks):
|
|
172
|
-
h = block(h)
|
|
213
|
+
h = block(h, timespans=timespans)
|
|
173
214
|
if moe_blocks is not None and isinstance(moe_blocks[i], MoEBlock):
|
|
174
215
|
h = moe_blocks[i](h, token_mask=token_mask)
|
|
175
216
|
return h
|
|
176
217
|
|
|
177
|
-
|
|
218
|
+
@staticmethod
|
|
219
|
+
def _normalize_timespans(
|
|
220
|
+
timespans: torch.Tensor | None, x: torch.Tensor
|
|
221
|
+
) -> torch.Tensor | None:
|
|
222
|
+
if timespans is None:
|
|
223
|
+
return None
|
|
224
|
+
timespans = torch.as_tensor(timespans, device=x.device, dtype=torch.float32)
|
|
225
|
+
batch, sequence = x.shape[:2]
|
|
226
|
+
if timespans.ndim == 1 and timespans.shape[0] == sequence:
|
|
227
|
+
timespans = timespans.unsqueeze(0).expand(batch, -1)
|
|
228
|
+
elif timespans.ndim == 2 and timespans.shape == (1, sequence):
|
|
229
|
+
timespans = timespans.expand(batch, -1)
|
|
230
|
+
elif timespans.ndim != 2 or timespans.shape != (batch, sequence):
|
|
231
|
+
raise ValueError(
|
|
232
|
+
"CfC timespans must have shape (sequence,), (1, sequence), "
|
|
233
|
+
f"or (batch, sequence); got {tuple(timespans.shape)} for input {tuple(x.shape[:2])}"
|
|
234
|
+
)
|
|
235
|
+
return timespans
|
|
236
|
+
|
|
237
|
+
def forward_with_timespans(
|
|
238
|
+
self,
|
|
239
|
+
x: torch.Tensor,
|
|
240
|
+
timespans: torch.Tensor,
|
|
241
|
+
*,
|
|
242
|
+
token_mask: torch.Tensor | None = None,
|
|
243
|
+
) -> torch.Tensor:
|
|
244
|
+
"""Explicit elapsed-time entry point for composing mixed backbones."""
|
|
245
|
+
return self.forward(x, token_mask=token_mask, timespans=timespans)
|
|
246
|
+
|
|
247
|
+
def forward_with_specialists(
|
|
248
|
+
self,
|
|
249
|
+
x: torch.Tensor,
|
|
250
|
+
*,
|
|
251
|
+
token_mask: torch.Tensor | None = None,
|
|
252
|
+
timespans: torch.Tensor | None = None,
|
|
253
|
+
intermediate_observer=None,
|
|
254
|
+
selected_specialists_only: bool = True,
|
|
255
|
+
):
|
|
178
256
|
"""Return final MoE candidates and routes for shared-workspace use."""
|
|
179
257
|
if not self.moe_enabled or self.moe_blocks is None:
|
|
180
258
|
raise RuntimeError("workspace broadcast requires an enabled MoE expert pool")
|
|
259
|
+
timespans = self._normalize_timespans(timespans, x)
|
|
181
260
|
h = self.in_adapter(x)
|
|
182
261
|
final_index = len(self.blocks) - 1
|
|
183
262
|
if not isinstance(self.moe_blocks[final_index], MoEBlock):
|
|
@@ -185,11 +264,16 @@ class CfCBackboneParallel(nn.Module):
|
|
|
185
264
|
"workspace broadcast requires the final backbone block to have an active MoE bank"
|
|
186
265
|
)
|
|
187
266
|
for i, block in enumerate(self.blocks):
|
|
188
|
-
h = block(h)
|
|
267
|
+
h = block(h, timespans=timespans)
|
|
268
|
+
if i == 0 and intermediate_observer is not None:
|
|
269
|
+
intermediate_observer(h)
|
|
189
270
|
if i == final_index:
|
|
190
271
|
base = h
|
|
191
272
|
_, experts, indices, weights = self.moe_blocks[i](
|
|
192
|
-
h,
|
|
273
|
+
h,
|
|
274
|
+
return_details=not selected_specialists_only,
|
|
275
|
+
return_selected_details=selected_specialists_only,
|
|
276
|
+
token_mask=token_mask,
|
|
193
277
|
)
|
|
194
278
|
return base, experts, indices, weights
|
|
195
279
|
if isinstance(self.moe_blocks[i], MoEBlock):
|