initium-deeplearning 0.3.0__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/PKG-INFO +62 -22
  2. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/README.md +61 -21
  3. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/pyproject.toml +1 -1
  4. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/__init__.py +1 -1
  5. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/LNN_controller/cfc_backbone_parallel.py +92 -8
  6. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/LNN_controller/cfc_controller.py +20 -1
  7. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/MoE/moe_layer.py +38 -24
  8. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/__init__.py +14 -0
  9. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/core_training.py +3872 -165
  10. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/base_dataset.py +7 -1
  11. initium_deeplearning-0.5.0/src/initium/data/common/causal_prediction.py +75 -0
  12. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/dataset_registry.py +13 -0
  13. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/graph_traversal/graph_traversal.py +321 -1
  14. initium_deeplearning-0.5.0/src/initium/data/surprise_recall/__init__.py +5 -0
  15. initium_deeplearning-0.5.0/src/initium/data/surprise_recall/next_token_prediction.py +377 -0
  16. initium_deeplearning-0.5.0/src/initium/data/surprise_recall/surprise_recall.py +553 -0
  17. initium_deeplearning-0.5.0/src/initium/data/surprise_recall/validate.py +100 -0
  18. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/README.md +299 -0
  19. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/__init__.py +5 -0
  20. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/curriculum.py +160 -0
  21. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/dataset.py +2132 -0
  22. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/episode.py +137 -0
  23. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/__init__.py +1 -0
  24. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/entity_state.py +303 -0
  25. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/nth_farthest.py +91 -0
  26. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/permutation.py +185 -0
  27. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/relational_join.py +86 -0
  28. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/retrieval_aggregation.py +103 -0
  29. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/diagnostics/segment_logging.py +68 -22
  30. initium_deeplearning-0.5.0/src/initium/diagnostics/split_graph_forward_trace.py +204 -0
  31. initium_deeplearning-0.5.0/src/initium/diagnostics/surprise_recall_logging.py +623 -0
  32. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/checkpoint_io.py +1 -0
  33. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/model_loader.py +36 -2
  34. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/mamba_controller/mamba2_controller.py +11 -5
  35. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/mamba_controller/mamba3_controller.py +9 -4
  36. initium_deeplearning-0.5.0/src/initium/mamba_controller/mamba_backbone_parallel.py +744 -0
  37. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/mamba_controller/mamba_controller.py +11 -11
  38. initium_deeplearning-0.5.0/src/initium/mamba_controller/numerical_stability.py +43 -0
  39. initium_deeplearning-0.5.0/src/initium/mamba_controller/sgr_backbone_only.py +141 -0
  40. initium_deeplearning-0.5.0/src/initium/mamba_controller/split_graph_dnc.py +1721 -0
  41. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/dnc_mechanisms.py +103 -0
  42. initium_deeplearning-0.5.0/src/initium/memory_manipulation/sequence_memories.py +513 -0
  43. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/stochastic_write_head_v2.py +103 -38
  44. initium_deeplearning-0.5.0/src/initium/workspace/README.md +159 -0
  45. initium_deeplearning-0.5.0/src/initium/workspace/broadcast.py +2288 -0
  46. initium_deeplearning-0.5.0/src/initium/workspace/kv_bridge.py +166 -0
  47. initium_deeplearning-0.5.0/src/initium/workspace/nth_farthest.py +43 -0
  48. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/README.md +9 -2
  49. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/integration/test_split_graph_dnc.py +34 -0
  50. initium_deeplearning-0.5.0/tests/integration/test_workspace_capability_pipeline.py +396 -0
  51. initium_deeplearning-0.5.0/tests/unit/test_next_token_prediction.py +114 -0
  52. initium_deeplearning-0.5.0/tests/unit/test_numerical_stability.py +57 -0
  53. initium_deeplearning-0.5.0/tests/unit/test_sequence_memories.py +138 -0
  54. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_stochastic_write_head.py +1 -1
  55. initium_deeplearning-0.5.0/tests/unit/test_surprise_recall.py +68 -0
  56. initium_deeplearning-0.5.0/tests/unit/test_workspace_capability.py +645 -0
  57. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_workspace_reconsideration.py +4 -0
  58. initium_deeplearning-0.3.0/src/initium/mamba_controller/mamba_backbone_parallel.py +0 -332
  59. initium_deeplearning-0.3.0/src/initium/mamba_controller/split_graph_dnc.py +0 -802
  60. initium_deeplearning-0.3.0/src/initium/workspace/broadcast.py +0 -533
  61. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/.gitignore +0 -0
  62. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/LICENSE +0 -0
  63. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/LNN_controller/chained_controller.py +0 -0
  64. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/LNN_controller/hybrid_controller.py +0 -0
  65. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/MoE/README.md +0 -0
  66. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/analysis/__init__.py +0 -0
  67. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/analysis/kl_attribution.py +0 -0
  68. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/config/controller_config.py +0 -0
  69. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/config/train_config.py +0 -0
  70. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/__init__.py +0 -0
  71. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/audio/__init__.py +0 -0
  72. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/audio/audio_dataset.py +0 -0
  73. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/common/__init__.py +0 -0
  74. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/common/chain_task.py +0 -0
  75. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/common/digit_codec.py +0 -0
  76. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/common/graph_io.py +0 -0
  77. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/common/real_data.py +0 -0
  78. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/graph_traversal/__init__.py +0 -0
  79. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/graph_traversal/order_probe.py +0 -0
  80. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/graph_traversal/relational_probe.py +0 -0
  81. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/multimodal/__init__.py +0 -0
  82. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/multimodal/multimodal_dataset.py +0 -0
  83. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/prepared_dataset.py +0 -0
  84. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/text/__init__.py +0 -0
  85. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/text/text_dataset.py +0 -0
  86. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/video/__init__.py +0 -0
  87. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/data/video/video_dataset.py +0 -0
  88. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/diagnostics/__init__.py +0 -0
  89. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/__init__.py +0 -0
  90. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/__init__.py +0 -0
  91. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/base_cache.py +0 -0
  92. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/cache_config.py +0 -0
  93. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/cache_registry.py +0 -0
  94. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/cached_engine.py +0 -0
  95. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/keys.py +0 -0
  96. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/prefix_cache.py +0 -0
  97. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/result_cache.py +0 -0
  98. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/state_utils.py +0 -0
  99. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/stores.py +0 -0
  100. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/capabilities.py +0 -0
  101. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/engine.py +0 -0
  102. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/inference_config.py +0 -0
  103. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/metrics.py +0 -0
  104. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/run_inference.py +0 -0
  105. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/run_logging.py +0 -0
  106. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/__init__.py +0 -0
  107. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/audio_task.py +0 -0
  108. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/base_task.py +0 -0
  109. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/graph_traversal_task.py +0 -0
  110. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/multimodal_task.py +0 -0
  111. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/task_registry.py +0 -0
  112. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/text_task.py +0 -0
  113. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/video_task.py +0 -0
  114. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/dynamic_memory_resize.py +0 -0
  115. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/dynamic_n_controller.py +0 -0
  116. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/link_matrix_ablation.py +0 -0
  117. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/nvrtc_compat.py +0 -0
  118. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/setup_wheels.py +0 -0
  119. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/workspace/__init__.py +0 -0
  120. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/src/initium/workspace/bistability.py +0 -0
  121. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/checkpoint/test_checkpoint_roundtrip.py +0 -0
  122. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/checkpoint/test_resume_training.py +0 -0
  123. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/conftest.py +0 -0
  124. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/gradients/test_controller_cell_backward.py +0 -0
  125. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/gradients/test_kl_gradients.py +0 -0
  126. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/gradients/test_moe_aux_loss_gradients.py +0 -0
  127. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/integration/test_controller_matrix.py +0 -0
  128. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/integration/test_dynamic_n_end_to_end.py +0 -0
  129. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/integration/test_moe_integration.py +0 -0
  130. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/integration/test_resize_x_link_matrix.py +0 -0
  131. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/smoke/test_inference_smoke.py +0 -0
  132. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/smoke/test_tiny_training_run.py +0 -0
  133. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_bistable_workspace_gate.py +0 -0
  134. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_capabilities.py +0 -0
  135. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_checkpoint_io.py +0 -0
  136. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_dynamic_n_controller.py +0 -0
  137. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_link_matrix_ablation.py +0 -0
  138. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_model_loader.py +0 -0
  139. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_moe_layer.py +0 -0
  140. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_nvrtc_compat.py +0 -0
  141. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_optimized_dnc_memory.py +0 -0
  142. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_real_data_pipeline.py +0 -0
  143. {initium_deeplearning-0.3.0 → initium_deeplearning-0.5.0}/tests/unit/test_resize_memory.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: initium-deeplearning
3
- Version: 0.3.0
3
+ Version: 0.5.0
4
4
  Summary: Differentiable Neural Computer research framework with pluggable LSTM/Mamba/CfC controllers, MoE, split-graph and stochastic write heads.
5
5
  Project-URL: Homepage, https://github.com/samueleternity/initium
6
6
  Project-URL: Issues, https://github.com/samueleternity/initium/issues
@@ -112,6 +112,46 @@ Dataset registry ──► task episodes / curriculum ──► batch and sequen
112
112
 
113
113
  The training core delegates task construction and task-specific behavior to a dataset interface. The model combines a selected controller with DNC memory operations. During training, the loop computes task and optional auxiliary objectives, updates model parameters, periodically evaluates ID and OOD performance, advances the curriculum when its criteria are met, writes diagnostics, and saves resumable checkpoints. Inference reconstructs a model from a checkpoint, checks that the requested task and dimensions are supported, creates the task runner, and reports episode metrics. Optional inference caches can reuse eligible results or shared prefixes.
114
114
 
115
+ ### Surprise-Gated Recall task-validity baseline
116
+
117
+ `--dataset-type surprise-recall` selects a synthetic next-token recall task with a seven-lesson curriculum. The generator records expected and observed tokens and exposes A1 marked events, A2 silent replacements drawn from the same episode pattern, A3 predictable alert-token decoys, A4 element skips/repeats that shift subsequent positions, A5 regime changes, and A6 benign channel jitter. Targets are masked to the answer span; the task-validity baseline adds no prediction decoder or error-to-Δ path. Its generator-level checks cover label consistency, A2/A3 content-detector behavior, a stateless random-guess reference, and the direct oracle-answer upper bound. These checks do not train the required no-memory or oracle-flag model controls. The command also reports generated sequence-token counts by lesson.
118
+
119
+ Run the gate from the repository root with `python -m initium.data.surprise_recall.validate` (in an editable/source checkout, set `PYTHONPATH=src` if needed). Start a training run with the requested split-graph architecture by adding `--dataset-type surprise-recall` to the usual command:
120
+
121
+ ```bash
122
+ python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc
123
+ ```
124
+
125
+ The optional MoE and diagnostic flags remain independent architecture/training options. The task-validity baseline does not add prediction-error gating, DNC/workspace error salience, or a special inference task runner.
126
+
127
+ ### Next-token decoder and prediction-signal evaluation
128
+
129
+ This experiment adds a dense next-token decoder over block 0's causal split-graph features, preserving the planned inter-layer signal source for the gating comparison. `--sgr-pred-loss-weight` enables its stream-only cross-entropy objective alongside the existing answer-masked recall objective; the decoder receives the representation at `t-1` and predicts token `t`. Query, answer, and padding positions are excluded. It adds no error injection, gating, MoE, or workspace path. Use a fresh run for each prediction-loss weight (the roadmap suggests 0.1, 0.3, and 1.0) and keep seed, step budget, and architecture matched to the task-validity baseline:
130
+
131
+ ```bash
132
+ python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --sgr-pred-loss-weight 0.3
133
+ ```
134
+
135
+ `--sgr-pred-eval-episodes` controls the fixed, deterministic probe count per lesson (default 128). The signal evaluation writes a per-run SGR signal CSV alongside the general SGR logs. It records raw-surprisal and excess-surprise AUROC for each A type against clean normal positions, calibration (ECE, accuracy, and Brier score), and A5 recovery. A6's per-type AUROCs measure benign jitter against clean normal positions; the additional A6-versus-true-anomaly comparison is diagnostic. Treat the decoder and signal implementation as complete while its signal gate remains unevaluated on completed runs; this does not establish that prediction error improves recall.
136
+
137
+ When `--dataset-type surprise-recall` is active, the trainer also creates `run_<run-id>_sgr_train.csv`, `run_<run-id>_sgr_positions.csv`, and `run_<run-id>_sgr_eval.csv` in the log directory. The batch log records A1–A6 prevalence and task conditions; the position trace records each stream token against its expected pattern token, event label, alert flag, recall-target status, and benign-jitter status. The evaluation log records task and exact-episode accuracy, recalled-event accuracy by A type, event prevalence, alert-token and pattern-mismatch detector confusion, and detector responses at each A-type event (including A3 decoy flags and A6 benign-jitter false alarms). It includes periodic and final ID/OOD results plus paired ID memory-on/memory-off results.
138
+
139
+ #### Tracing split-graph forward instability
140
+
141
+ Use `--split-graph-numeric-trace-step STEP` to inspect one absolute optimizer step after resuming from a checkpoint saved before the suspected failure. Re-run with the same seed, architecture, task, and relevant training options. For every observed boundary, the trace prints dtype, shape, finite/non-finite counts, maximum finite absolute value, and (on failure) the first bad coordinate to the console and CSV. It checks before and after Mamba normalization, mixer, cast, and residual; it also covers the CfC adapter/cell/residual/state, DNC state/read, recurrent output, logits, and loss. Timesteps are included for recurrent boundaries. It writes `run_<run-id>_splitgraph_forward_trace.csv` in the configured log directory and stops at the first observed non-finite value. The completion line reports rows written and warns if no boundary was observed. For example, to investigate the reported step 11,635 failure:
142
+
143
+ ```bash
144
+ python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --resume <checkpoint-before-step-11635.pt> --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --total-steps 12000 --split-graph-numeric-trace-step 11635
145
+ ```
146
+
147
+ The selected step synchronizes CUDA at every observed boundary, so this flag is intended for a narrow replay and adds no such checks to other steps. A clean selected step means no observed tensor was non-finite at those boundaries; use the largest `max_abs_finite` rows to identify where scale first grows. A failure reports the first *instrumented* bad boundary; it cannot identify an internal operation between two boundaries. MoE and workspace boundaries are included when those paths are active, but compare runs with identical options to preserve replay conditions.
148
+
149
+ For intermittent failures whose step shifts between runs, use `--split-graph-numeric-trace-on-error` instead of selecting a step. It performs no boundary tensor checks during healthy training. When the trainer's existing finite checks detect a non-finite recurrent output, task logit/loss, or SGR prediction loss, it replays that same batch once with detailed tracing enabled, prints each boundary to the console, and writes the same CSV. The original training failure is still raised before backward or an optimizer update; if the replay does not reproduce the non-finite value, the trace reports that result and preserves the original failure. This option requires `--split-graph`, is for training runs, and cannot be combined with `--split-graph-numeric-trace-step`.
150
+
151
+ ```bash
152
+ python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --resume <checkpoint.pt> --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --total-steps 30000 --split-graph-numeric-trace-on-error
153
+ ```
154
+
115
155
  ## Architecture and major components
116
156
 
117
157
  ### Differentiable external memory
@@ -143,14 +183,14 @@ python -m initium.core_training 0.0 --split-graph --split-graph-variant mamba1 -
143
183
  python -m initium.core_training 0.0 --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant mamba+cfc
144
184
  ```
145
185
 
146
- ### Phase 0 shared workspace (MoE broadcast)
186
+ ### Workspace plumbing (MoE broadcast)
147
187
 
148
188
  The optional `--workspace` path adds a persistent shared slot matrix to a split-graph model with MoE enabled. The final parallel-backbone MoE bank supplies the specialist candidates and its existing noisy top-k routes. At each sequence timestep, selected expert outputs compete for slot writes; a learned RMC-style gate blends proposed slot content with the previous matrix. Every expert in that bank then attends to the updated matrix, and the existing sparse MoE routes combine the context-enriched expert outputs. The workspace resets at the beginning of each episode and persists across its timesteps. The `--moe-backbone-residual-scale` setting scales the complete final-bank delta, including its broadcast contribution, just as it scales the ordinary MoE residuals in the selection-only backbone. The DNC memory uses the same fp32/autocast-disabled path with or without workspace. This is a shared-object broadcast implementation; it does not implement per-recipient translation.
149
189
 
150
- Use the Phase 1 matched split-graph/CfC-combiner setup with MoE and the workspace enabled as follows:
190
+ Use the selected split-graph/CfC-combiner setup with MoE and the workspace enabled as follows:
151
191
 
152
192
  ```bash
153
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase2-workspace-coadapted --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --workspace --workspace-num-slots 5 --workspace-train-mode coadapted --total-steps 30000
193
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix workspace-coadapted --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --workspace --workspace-num-slots 5 --workspace-train-mode coadapted --total-steps 30000
154
194
  ```
155
195
 
156
196
  `--workspace` requires both `--split-graph` and `--moe`. The default is 5 slots, based on Goyal et al.'s shared-workspace reference result; it is a starting configuration, not a claimed optimum for graph traversal. `--workspace-train-mode coadapted` trains the full model. Choose `frozen` to train only workspace parameters while freezing the backbone, MoE/combiner, DNC, stochastic write heads, and task output projection. `--workspace-no-selection` replaces competitive selected writes with random unweighted writes for the no-Selection control. `--workspace-reset-each-step` resets slots before each timestep to test persistence. `--workspace-num-slots 0` bypasses workspace writes and readback, providing a plumbing comparison against the existing MoE model. Run IDs encode these conditions to avoid log/checkpoint collisions.
@@ -159,21 +199,21 @@ For a frozen-workspace experiment, pass a compatible workspace-disabled baseline
159
199
 
160
200
  The console prints a startup line that reports whether the workspace is active or bypassed, its capacity, write-selection mode, persistence setting, train mode, and active compute precision. Active workspace attention and slot-state updates run in fp32 outside CUDA autocast; zero-slot/closed-gate bypass retains the selection-only path. At the normal training log cadence, `workspace[...]` reports selected write mass and active-write fraction, the learned write-gate mean, slot-state RMS, readback-to-expert-output RMS ratio, and read-gate value. These values show whether the path is active and the relative scale of its write and broadcast signals; they do not by themselves establish a task benefit.
161
201
 
162
- The shared object operates on the final backbone MoE bank; earlier backbone MoE layers retain their existing behavior. This is the Phase 0 plumbing target, not evidence that Broadcast improves task performance. The first short run should compare the zero-slot bypass against the same MoE configuration without `--workspace`; only after this plumbing gate should the frozen/co-adapted and no-Selection comparisons begin. Record seed, full command, commit, hardware/software environment, curriculum completion, ID/OOD metrics, convergence steps, and any failures in `07-Experiment/Broadcasting/Progress/Phase-0-Implementation.md`. All current evidence is graph-traversal-specific; transfer across task families remains untested.
202
+ The shared object operates on the final backbone MoE bank; earlier backbone MoE layers retain their existing behavior. This is the workspace-plumbing target, not evidence that Broadcast improves task performance. The first short run should compare the zero-slot bypass against the same MoE configuration without `--workspace`; only after this plumbing gate should the frozen/co-adapted and no-Selection comparisons begin. Record seed, full command, commit, hardware/software environment, curriculum completion, ID/OOD metrics, convergence steps, and any failures in `D:/Projects/AI-Architecture-Research/07-Experiment/Broadcasting/Progress/Workspace-Plumbing-Implementation.md`. All current evidence is graph-traversal-specific; transfer across task families remains untested.
163
203
 
164
- ### Phase 4 experimental bistable write gate
204
+ ### Bistable write-gate experiment
165
205
 
166
- The default `--workspace-gate-variant continuous` preserves the Phase 0–3 sigmoid update. The opt-in `bistable` variant treats each workspace slot as a competing write candidate. It combines the workspace's content-selection score and the MoE route confidence with independent Gaussian background noise, then iterates recurrent self-excitation and lateral inhibition before making a hard write/no-write decision. A straight-through estimator carries gradients through the pre-commit probabilities. This is an experimental adaptation of GNW ignition, not a mechanism implemented by the source neuroscience papers.
206
+ The default `--workspace-gate-variant continuous` preserves the existing sigmoid update. The opt-in `bistable` variant treats each workspace slot as a competing write candidate. It combines the workspace's content-selection score and the MoE route confidence with independent Gaussian background noise, then iterates recurrent self-excitation and lateral inhibition before making a hard write/no-write decision. A straight-through estimator carries gradients through the pre-commit probabilities. This is an experimental adaptation of GNW ignition, not a mechanism implemented by the source neuroscience papers.
167
207
 
168
208
  For the matched gate-only comparison, resume the same five-slot frozen-workspace checkpoint twice. The frozen control updates only the existing continuous `write_gate`; the bistable condition updates only the new recurrent gate parameters. The gate-variant warm start loads shared model/workspace weights, initializes only the new gate, and starts with a fresh optimizer. `--workspace-train-mode coadapted` remains available to train the full model from that same checkpoint as a separate condition. Bistable background noise uses an independent checkpointed RNG stream, so zero-noise and selection-strength ablations do not change the graph episode RNG stream.
169
209
 
170
210
  ```bash
171
- python -m src.initium.core_training 0.0 --seed 0 --resume <step-8000-workspace-checkpoint.pt> --run-id-suffix phase4-continuous-gate-pilot --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --workspace --workspace-num-slots 5 --workspace-train-mode frozen --workspace-train-scope gate --workspace-gate-variant continuous --total-steps 8100
211
+ python -m src.initium.core_training 0.0 --seed 0 --resume <step-8000-workspace-checkpoint.pt> --run-id-suffix bistable-gate-continuous-control --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --workspace --workspace-num-slots 5 --workspace-train-mode frozen --workspace-train-scope gate --workspace-gate-variant continuous --total-steps 8100
172
212
  ```
173
213
 
174
214
  Use the same command, checkpoint, seed, and step budget for the bistable condition, changing the suffix and adding `--workspace-gate-variant bistable`. Gate-only runs use a safety tripwire that aborts before backward/update if any instrumented segment exceeds `--workspace-stability-max-activation` (default `100000`). Segment diagnostics report activation/gradient RMS and maxima; bistable diagnostics additionally report the pre-commit probability histogram (10 bins), low/middle/high occupancy, hard commit frequency, writer frequencies, slot RMS, and readback scale during training and held-out ID/OOD evaluation. A zero-noise ablation uses `--workspace-bistable-noise-scale 0`; selection-strength sweeps use `--workspace-bistable-selection-scale` while holding `--eval-seed` fixed. The gate still samples noise during evaluation unless `--workspace-bistable-deterministic` is supplied.
175
215
 
176
- The preregistered operational bimodality check is at least 10% of held-out candidate probabilities below 0.1, at least 10% above 0.9, and at most 20% in [0.1, 0.9], sustained across episodes. Also require finite, nonzero gate gradients, no repeated skipped optimizer updates, and no task or paired-memory-dependency collapse. Run the short stability/pilot budget first; do not start a long Phase 4 run unless the histogram and stability gates pass. Results remain specific to graph traversal and the single available trained model seed.
216
+ The preregistered operational bimodality check is at least 10% of held-out candidate probabilities below 0.1, at least 10% above 0.9, and at most 20% in [0.1, 0.9], sustained across episodes. Also require finite, nonzero gate gradients, no repeated skipped optimizer updates, and no task or paired-memory-dependency collapse. Run the short stability/pilot budget first; do not start a long bistable write-gate run unless the histogram and stability gates pass. Results remain specific to graph traversal and the single available trained model seed.
177
217
 
178
218
  After saving a pilot checkpoint, evaluate that same checkpoint and `--eval-seed` under each `--eval-workspace-mode checkpoint`, `reset-each-step`, and `bypass`. For noise and selection ablations, use the same `--eval-seed` and change only the corresponding bistable scale. The gate RNG is separately checkpointed, so these changes keep the episode stream fixed; use distinct run suffixes when recording each condition.
179
219
 
@@ -185,7 +225,7 @@ The SwitchMoE router projection and expert MLPs run in FP32 even when the surrou
185
225
 
186
226
  Periodic memory-dependency checks replay the exact random episode batch used by the corresponding memory-on ID evaluation, then restore the RNG streams. This makes `memory on - memory off` a paired functional ablation rather than a difference between two independently sampled batches. The training loop also clips all trainable optimizer parameters, including the task output projection, stops before backward on non-finite forward outputs/losses, reports the first parameter with a non-finite unscaled gradient, logs skipped non-finite-gradient updates, and aborts after eight consecutive such steps instead of continuing a run that is no longer updating safely.
187
227
 
188
- The supplied Phase 1/2 traces are pre-fix evidence. One Phase 2 trace first reports a non-finite task loss at step 11,665, recurrent outputs become non-finite at step 11,666, and the process continues to step 14,106 with an AMP scale of zero before manual interruption. Memory-on/off checks in those logs used separate random episode batches, so their small positive or negative differences are not reliable paired evidence. See `07-Experiment/Broadcasting/Progress/Phase-1-MoE-Failure-Analysis.md` for the causal breakdown and re-run gate. Re-run the Phase 1 selection-only condition before Phase 2; keep the proposed 8-expert, top-1 configuration and matched seed/settings fixed while checking finite gradients, per-bank routing, and paired memory dependency.
228
+ The supplied selection-only baseline and workspace-run traces are pre-fix evidence. The workspace-run trace first reports a non-finite task loss at step 11,665, recurrent outputs become non-finite at step 11,666, and the process continues to step 14,106 with an AMP scale of zero before manual interruption. Memory-on/off checks in those logs used separate random episode batches, so their small positive or negative differences are not reliable paired evidence. See `D:/Projects/AI-Architecture-Research/07-Experiment/Broadcasting/Progress/Selection-Only-MoE-Failure-Analysis.md` for the causal breakdown and re-run gate. Re-run the selection-only baseline before the workspace experiment; keep the proposed 8-expert, top-1 configuration and matched seed/settings fixed while checking finite gradients, per-bank routing, and paired memory dependency.
189
229
 
190
230
  Training console logs now show **one MoE-bank record per 100-step logging window**: route and router-importance fractions by expert, router entropy and maximum probability, input/expert/MoE-delta activation RMS and maximum magnitude, capacity-drop fraction, and accepted gate mass. The `moe_residual:` segment records the complete MoE block output after its skip connection, separately from `moe_delta_rms/max`. A separate `[segment-diag:train]` line reports each recognized module's call count, input/output RMS and max, non-finite fraction, boundary input/output gradient RMS and non-finite fraction, and hierarchical parameter-gradient norms. Boundary gradients are collected during training backward and during failure traces; evaluation remains forward-only. The collector recognizes Mamba and CfC backbones/blocks, Mamba/CfC controller wrappers/blocks/cells, MoE blocks, chained controllers, DNC memory, workspace, and task output; names include their module paths, so the same logger works across supported controller combinations.
191
231
 
@@ -197,40 +237,40 @@ To inspect DNC internals, add `--dnc-backward-trace` to a split-graph training c
197
237
  python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix dnc-backward-trace --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 1800 --dnc-backward-trace
198
238
  ```
199
239
 
200
- Keep other flags exactly matched to the run being investigated. The fp32 DNC path removed the observed bad-key-gradient pattern in the 5,000-step matched validation capture. That run also acquired strong paired ID memory dependence by step 2,500; see `AI-Architecture-Research/07-Experiment/Broadcasting/Progress/Phase-1-MoE-Failure-Analysis.md` for the complete result and remaining generalization limits. Repeat across seeds before treating the numerical and learning outcomes as robust.
240
+ Keep other flags exactly matched to the run being investigated. The fp32 DNC path removed the observed bad-key-gradient pattern in the 5,000-step matched validation capture. That run also acquired strong paired ID memory dependence by step 2,500; see `D:/Projects/AI-Architecture-Research/07-Experiment/Broadcasting/Progress/Selection-Only-MoE-Failure-Analysis.md` for the complete result and remaining generalization limits. Repeat across seeds before treating the numerical and learning outcomes as robust.
201
241
 
202
242
  Evaluation emits separate channel lines for ID, OOD, memory-off, combiner-stage ablations, and final ID/OOD checks. These summarize the evaluations independently from training windows. If a forward value or gradient becomes non-finite, the current segment window and MoE-bank window are printed immediately before training stops or skips the unsafe update. The summaries aggregate detached device-side scalars and synchronize at log/evaluation boundaries; they do not retain activation graphs.
203
243
 
204
244
  To isolate where split-graph MoE changes the memory-learning transition, `--moe-placement` selects which component receives expert banks while keeping the model, seed, and router settings matched. `all` (the default) preserves the existing behavior; `backbone` enables MoE only in the parallel backbone; `combiner` enables MoE only in the sequential controller combiner. For the CfC combiner, the latter is its source-aware multi-source MoE bank. Component placement requires `--moe --split-graph`; combiner-only additionally requires controller combiner mode. The workspace requires backbone MoE and therefore cannot be combined with combiner-only placement.
205
245
 
206
- Run these short, seed-matched diagnostics on the same source revision. Each is capped at 5,000 steps so the usual early memory-use transition can be compared before committing to the full Phase 1 horizon:
246
+ Run these short, seed-matched diagnostics on the same source revision. Each is capped at 5,000 steps so the usual early memory-use transition can be compared before committing to the full MoE training budget:
207
247
 
208
248
  ```bash
209
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-smoke-no-moe --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --total-steps 5000
210
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-smoke-moe-backbone --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
211
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-smoke-moe-combiner --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement combiner --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
212
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-smoke-moe-all --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
249
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-baseline-no-moe --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --total-steps 5000
250
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-placement-backbone --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
251
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-placement-combiner --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement combiner --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
252
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-placement-all --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
213
253
  ```
214
254
 
215
- Compare the paired memory-on/off dependency, ID/OOD metrics, curriculum progress, per-component segment diagnostics, and per-bank routing/activation summaries. The placement runs distinguish backbone-MoE effects from the CfC multi-source path; the `all` condition checks their interaction. These are diagnostic smoke runs, not evidence of a completed Phase 1 result.
255
+ Compare the paired memory-on/off dependency, ID/OOD metrics, curriculum progress, per-component segment diagnostics, and per-bank routing/activation summaries. The placement runs distinguish backbone-MoE effects from the CfC multi-source path; the `all` condition checks their interaction. These are diagnostic smoke runs, not evidence of a completed MoE placement comparison.
216
256
 
217
257
  For a backbone-only run, `--moe-backbone-block-isolation INDEX` installs an MoE bank after only the selected zero-based parallel-backbone block. The other backbone blocks still run their Mamba/CfC computation but have no MoE; the sequential combiner must also be excluded with `--moe-placement backbone`. The flag is range-checked against `--split-graph-num-blocks`, included in run IDs and checkpoint metadata, and cannot be combined with the workspace broadcast path. For a two-block Mamba backbone, run one isolated condition per block:
218
258
 
219
259
  ```bash
220
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-moe-backbone-block0 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 0 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
221
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-moe-backbone-block1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
260
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-backbone-block-0 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 0 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
261
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-backbone-block-1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
222
262
  ```
223
263
 
224
264
  `--moe-backbone-residual-scale SCALE` multiplies the active parallel-backbone MoE delta before its residual addition (`backbone_output + SCALE * moe_delta`). Its default is `1.0`, preserving prior behavior. The scale must be finite and non-negative, requires split-graph MoE with `--moe-placement backbone` or `all`, and is included in run IDs/checkpoint metadata. With `all`, this changes only the parallel-backbone MoE residuals; combiner MoE residuals remain at 1.0. It scales the task-loss path through the backbone MoE branch while leaving routing and the load-balancing auxiliary loss active. When workspace broadcast is enabled, the scale applies to the complete final-bank delta after expert readback, including both the selected expert outputs and their broadcast contribution. Workspace slot writes and router/load-balancing calculations remain active at their existing scales. Active workspace attention and slot updates run in fp32 outside CUDA autocast, and split-graph DNC memory math uses its fp32 path in both workspace and selection-only runs. In an isolated-block run, only that block has an MoE residual to scale. For the current block-1 follow-up, use a deliberately conservative 0.1 scale:
225
265
 
226
266
  ```bash
227
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-block1-residual-scale-0p1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 1 --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
267
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-block-1-residual-scale-0p1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 1 --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
228
268
  ```
229
269
 
230
270
  To test backbone and combiner MoE together after the backbone-only scale control, omit block isolation and select `--moe-placement all`. The residual scale still applies only to the backbone; the combiner MoE keeps its default residual scale of 1.0:
231
271
 
232
272
  ```bash
233
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-backbone-combiner-backbone-scale-0p1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
273
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-backbone-combiner-scale-0p1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
234
274
  ```
235
275
 
236
276
  Each `moe-bank[backbone.moe_blocks.INDEX ...]` line reports router-selected expert load, router importance, entropy, maximum probability, and capacity-drop rate separately for valid graph timesteps and all-zero padded timesteps. The valid-token mask is derived from the graph input encoding and is diagnostic-only in this experiment: routing, shared capacity, balancing loss, and outputs still include padding, so the run measures the current behavior without applying a padding fix. The per-block isolation changes which banks are active but leaves the routing policy otherwise unchanged. Compare the two blocks' valid/padded distributions and memory-dependency curves against the non-isolated backbone-only trace before deciding whether to test mask-aware routing.
@@ -355,7 +395,7 @@ A separate study compared beta 0 and beta 0.001 in a Mamba, dynamic-memory, spli
355
395
  ### Evidence boundaries and open work
356
396
 
357
397
  - Results above apply to the exact runs and evaluation procedures recorded; they do not establish broad performance across modalities, hardware, or model scales.
358
- - Phase 1's seed variation and Phase 2's four-seed selection limit aggregate claims. Report per-seed values and uncertainty alongside averages.
398
+ - The fixed-prior study's seed variation and the learned-prior study's four-seed selection limit aggregate claims. Report per-seed values and uncertainty alongside averages.
359
399
  - The learned prior's apparent variance-dampening/mean-reversion effect needs more seeds and controls to separate it from failure-mode-specific repair.
360
400
  - The alternate Mamba batch needs completion of unfinished seeds and late-curriculum evaluation before a gate-level conclusion.
361
401
  - Mamba-2/Mamba-3 performance, MoE's useful scaling regime, memory-link trade-offs, dynamic memory growth, and real-data/multimodal behavior remain active investigation areas.
@@ -81,6 +81,46 @@ Dataset registry ──► task episodes / curriculum ──► batch and sequen
81
81
 
82
82
  The training core delegates task construction and task-specific behavior to a dataset interface. The model combines a selected controller with DNC memory operations. During training, the loop computes task and optional auxiliary objectives, updates model parameters, periodically evaluates ID and OOD performance, advances the curriculum when its criteria are met, writes diagnostics, and saves resumable checkpoints. Inference reconstructs a model from a checkpoint, checks that the requested task and dimensions are supported, creates the task runner, and reports episode metrics. Optional inference caches can reuse eligible results or shared prefixes.
83
83
 
84
+ ### Surprise-Gated Recall task-validity baseline
85
+
86
+ `--dataset-type surprise-recall` selects a synthetic next-token recall task with a seven-lesson curriculum. The generator records expected and observed tokens and exposes A1 marked events, A2 silent replacements drawn from the same episode pattern, A3 predictable alert-token decoys, A4 element skips/repeats that shift subsequent positions, A5 regime changes, and A6 benign channel jitter. Targets are masked to the answer span; the task-validity baseline adds no prediction decoder or error-to-Δ path. Its generator-level checks cover label consistency, A2/A3 content-detector behavior, a stateless random-guess reference, and the direct oracle-answer upper bound. These checks do not train the required no-memory or oracle-flag model controls. The command also reports generated sequence-token counts by lesson.
87
+
88
+ Run the gate from the repository root with `python -m initium.data.surprise_recall.validate` (in an editable/source checkout, set `PYTHONPATH=src` if needed). Start a training run with the requested split-graph architecture by adding `--dataset-type surprise-recall` to the usual command:
89
+
90
+ ```bash
91
+ python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc
92
+ ```
93
+
94
+ The optional MoE and diagnostic flags remain independent architecture/training options. The task-validity baseline does not add prediction-error gating, DNC/workspace error salience, or a special inference task runner.
95
+
96
+ ### Next-token decoder and prediction-signal evaluation
97
+
98
+ This experiment adds a dense next-token decoder over block 0's causal split-graph features, preserving the planned inter-layer signal source for the gating comparison. `--sgr-pred-loss-weight` enables its stream-only cross-entropy objective alongside the existing answer-masked recall objective; the decoder receives the representation at `t-1` and predicts token `t`. Query, answer, and padding positions are excluded. It adds no error injection, gating, MoE, or workspace path. Use a fresh run for each prediction-loss weight (the roadmap suggests 0.1, 0.3, and 1.0) and keep seed, step budget, and architecture matched to the task-validity baseline:
99
+
100
+ ```bash
101
+ python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --sgr-pred-loss-weight 0.3
102
+ ```
103
+
104
+ `--sgr-pred-eval-episodes` controls the fixed, deterministic probe count per lesson (default 128). The signal evaluation writes a per-run SGR signal CSV alongside the general SGR logs. It records raw-surprisal and excess-surprise AUROC for each A type against clean normal positions, calibration (ECE, accuracy, and Brier score), and A5 recovery. A6's per-type AUROCs measure benign jitter against clean normal positions; the additional A6-versus-true-anomaly comparison is diagnostic. Treat the decoder and signal implementation as complete while its signal gate remains unevaluated on completed runs; this does not establish that prediction error improves recall.
105
+
106
+ When `--dataset-type surprise-recall` is active, the trainer also creates `run_<run-id>_sgr_train.csv`, `run_<run-id>_sgr_positions.csv`, and `run_<run-id>_sgr_eval.csv` in the log directory. The batch log records A1–A6 prevalence and task conditions; the position trace records each stream token against its expected pattern token, event label, alert flag, recall-target status, and benign-jitter status. The evaluation log records task and exact-episode accuracy, recalled-event accuracy by A type, event prevalence, alert-token and pattern-mismatch detector confusion, and detector responses at each A-type event (including A3 decoy flags and A6 benign-jitter false alarms). It includes periodic and final ID/OOD results plus paired ID memory-on/memory-off results.
107
+
108
+ #### Tracing split-graph forward instability
109
+
110
+ Use `--split-graph-numeric-trace-step STEP` to inspect one absolute optimizer step after resuming from a checkpoint saved before the suspected failure. Re-run with the same seed, architecture, task, and relevant training options. For every observed boundary, the trace prints dtype, shape, finite/non-finite counts, maximum finite absolute value, and (on failure) the first bad coordinate to the console and CSV. It checks before and after Mamba normalization, mixer, cast, and residual; it also covers the CfC adapter/cell/residual/state, DNC state/read, recurrent output, logits, and loss. Timesteps are included for recurrent boundaries. It writes `run_<run-id>_splitgraph_forward_trace.csv` in the configured log directory and stops at the first observed non-finite value. The completion line reports rows written and warns if no boundary was observed. For example, to investigate the reported step 11,635 failure:
111
+
112
+ ```bash
113
+ python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --resume <checkpoint-before-step-11635.pt> --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --total-steps 12000 --split-graph-numeric-trace-step 11635
114
+ ```
115
+
116
+ The selected step synchronizes CUDA at every observed boundary, so this flag is intended for a narrow replay and adds no such checks to other steps. A clean selected step means no observed tensor was non-finite at those boundaries; use the largest `max_abs_finite` rows to identify where scale first grows. A failure reports the first *instrumented* bad boundary; it cannot identify an internal operation between two boundaries. MoE and workspace boundaries are included when those paths are active, but compare runs with identical options to preserve replay conditions.
117
+
118
+ For intermittent failures whose step shifts between runs, use `--split-graph-numeric-trace-on-error` instead of selecting a step. It performs no boundary tensor checks during healthy training. When the trainer's existing finite checks detect a non-finite recurrent output, task logit/loss, or SGR prediction loss, it replays that same batch once with detailed tracing enabled, prints each boundary to the console, and writes the same CSV. The original training failure is still raised before backward or an optimizer update; if the replay does not reproduce the non-finite value, the trace reports that result and preserves the original failure. This option requires `--split-graph`, is for training runs, and cannot be combined with `--split-graph-numeric-trace-step`.
119
+
120
+ ```bash
121
+ python -m initium.core_training 0.0 --dataset-type surprise-recall --seed 0 --resume <checkpoint.pt> --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --total-steps 30000 --split-graph-numeric-trace-on-error
122
+ ```
123
+
84
124
  ## Architecture and major components
85
125
 
86
126
  ### Differentiable external memory
@@ -112,14 +152,14 @@ python -m initium.core_training 0.0 --split-graph --split-graph-variant mamba1 -
112
152
  python -m initium.core_training 0.0 --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant mamba+cfc
113
153
  ```
114
154
 
115
- ### Phase 0 shared workspace (MoE broadcast)
155
+ ### Workspace plumbing (MoE broadcast)
116
156
 
117
157
  The optional `--workspace` path adds a persistent shared slot matrix to a split-graph model with MoE enabled. The final parallel-backbone MoE bank supplies the specialist candidates and its existing noisy top-k routes. At each sequence timestep, selected expert outputs compete for slot writes; a learned RMC-style gate blends proposed slot content with the previous matrix. Every expert in that bank then attends to the updated matrix, and the existing sparse MoE routes combine the context-enriched expert outputs. The workspace resets at the beginning of each episode and persists across its timesteps. The `--moe-backbone-residual-scale` setting scales the complete final-bank delta, including its broadcast contribution, just as it scales the ordinary MoE residuals in the selection-only backbone. The DNC memory uses the same fp32/autocast-disabled path with or without workspace. This is a shared-object broadcast implementation; it does not implement per-recipient translation.
118
158
 
119
- Use the Phase 1 matched split-graph/CfC-combiner setup with MoE and the workspace enabled as follows:
159
+ Use the selected split-graph/CfC-combiner setup with MoE and the workspace enabled as follows:
120
160
 
121
161
  ```bash
122
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase2-workspace-coadapted --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --workspace --workspace-num-slots 5 --workspace-train-mode coadapted --total-steps 30000
162
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix workspace-coadapted --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --workspace --workspace-num-slots 5 --workspace-train-mode coadapted --total-steps 30000
123
163
  ```
124
164
 
125
165
  `--workspace` requires both `--split-graph` and `--moe`. The default is 5 slots, based on Goyal et al.'s shared-workspace reference result; it is a starting configuration, not a claimed optimum for graph traversal. `--workspace-train-mode coadapted` trains the full model. Choose `frozen` to train only workspace parameters while freezing the backbone, MoE/combiner, DNC, stochastic write heads, and task output projection. `--workspace-no-selection` replaces competitive selected writes with random unweighted writes for the no-Selection control. `--workspace-reset-each-step` resets slots before each timestep to test persistence. `--workspace-num-slots 0` bypasses workspace writes and readback, providing a plumbing comparison against the existing MoE model. Run IDs encode these conditions to avoid log/checkpoint collisions.
@@ -128,21 +168,21 @@ For a frozen-workspace experiment, pass a compatible workspace-disabled baseline
128
168
 
129
169
  The console prints a startup line that reports whether the workspace is active or bypassed, its capacity, write-selection mode, persistence setting, train mode, and active compute precision. Active workspace attention and slot-state updates run in fp32 outside CUDA autocast; zero-slot/closed-gate bypass retains the selection-only path. At the normal training log cadence, `workspace[...]` reports selected write mass and active-write fraction, the learned write-gate mean, slot-state RMS, readback-to-expert-output RMS ratio, and read-gate value. These values show whether the path is active and the relative scale of its write and broadcast signals; they do not by themselves establish a task benefit.
130
170
 
131
- The shared object operates on the final backbone MoE bank; earlier backbone MoE layers retain their existing behavior. This is the Phase 0 plumbing target, not evidence that Broadcast improves task performance. The first short run should compare the zero-slot bypass against the same MoE configuration without `--workspace`; only after this plumbing gate should the frozen/co-adapted and no-Selection comparisons begin. Record seed, full command, commit, hardware/software environment, curriculum completion, ID/OOD metrics, convergence steps, and any failures in `07-Experiment/Broadcasting/Progress/Phase-0-Implementation.md`. All current evidence is graph-traversal-specific; transfer across task families remains untested.
171
+ The shared object operates on the final backbone MoE bank; earlier backbone MoE layers retain their existing behavior. This is the workspace-plumbing target, not evidence that Broadcast improves task performance. The first short run should compare the zero-slot bypass against the same MoE configuration without `--workspace`; only after this plumbing gate should the frozen/co-adapted and no-Selection comparisons begin. Record seed, full command, commit, hardware/software environment, curriculum completion, ID/OOD metrics, convergence steps, and any failures in `D:/Projects/AI-Architecture-Research/07-Experiment/Broadcasting/Progress/Workspace-Plumbing-Implementation.md`. All current evidence is graph-traversal-specific; transfer across task families remains untested.
132
172
 
133
- ### Phase 4 experimental bistable write gate
173
+ ### Bistable write-gate experiment
134
174
 
135
- The default `--workspace-gate-variant continuous` preserves the Phase 0–3 sigmoid update. The opt-in `bistable` variant treats each workspace slot as a competing write candidate. It combines the workspace's content-selection score and the MoE route confidence with independent Gaussian background noise, then iterates recurrent self-excitation and lateral inhibition before making a hard write/no-write decision. A straight-through estimator carries gradients through the pre-commit probabilities. This is an experimental adaptation of GNW ignition, not a mechanism implemented by the source neuroscience papers.
175
+ The default `--workspace-gate-variant continuous` preserves the existing sigmoid update. The opt-in `bistable` variant treats each workspace slot as a competing write candidate. It combines the workspace's content-selection score and the MoE route confidence with independent Gaussian background noise, then iterates recurrent self-excitation and lateral inhibition before making a hard write/no-write decision. A straight-through estimator carries gradients through the pre-commit probabilities. This is an experimental adaptation of GNW ignition, not a mechanism implemented by the source neuroscience papers.
136
176
 
137
177
  For the matched gate-only comparison, resume the same five-slot frozen-workspace checkpoint twice. The frozen control updates only the existing continuous `write_gate`; the bistable condition updates only the new recurrent gate parameters. The gate-variant warm start loads shared model/workspace weights, initializes only the new gate, and starts with a fresh optimizer. `--workspace-train-mode coadapted` remains available to train the full model from that same checkpoint as a separate condition. Bistable background noise uses an independent checkpointed RNG stream, so zero-noise and selection-strength ablations do not change the graph episode RNG stream.
138
178
 
139
179
  ```bash
140
- python -m src.initium.core_training 0.0 --seed 0 --resume <step-8000-workspace-checkpoint.pt> --run-id-suffix phase4-continuous-gate-pilot --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --workspace --workspace-num-slots 5 --workspace-train-mode frozen --workspace-train-scope gate --workspace-gate-variant continuous --total-steps 8100
180
+ python -m src.initium.core_training 0.0 --seed 0 --resume <step-8000-workspace-checkpoint.pt> --run-id-suffix bistable-gate-continuous-control --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --workspace --workspace-num-slots 5 --workspace-train-mode frozen --workspace-train-scope gate --workspace-gate-variant continuous --total-steps 8100
141
181
  ```
142
182
 
143
183
  Use the same command, checkpoint, seed, and step budget for the bistable condition, changing the suffix and adding `--workspace-gate-variant bistable`. Gate-only runs use a safety tripwire that aborts before backward/update if any instrumented segment exceeds `--workspace-stability-max-activation` (default `100000`). Segment diagnostics report activation/gradient RMS and maxima; bistable diagnostics additionally report the pre-commit probability histogram (10 bins), low/middle/high occupancy, hard commit frequency, writer frequencies, slot RMS, and readback scale during training and held-out ID/OOD evaluation. A zero-noise ablation uses `--workspace-bistable-noise-scale 0`; selection-strength sweeps use `--workspace-bistable-selection-scale` while holding `--eval-seed` fixed. The gate still samples noise during evaluation unless `--workspace-bistable-deterministic` is supplied.
144
184
 
145
- The preregistered operational bimodality check is at least 10% of held-out candidate probabilities below 0.1, at least 10% above 0.9, and at most 20% in [0.1, 0.9], sustained across episodes. Also require finite, nonzero gate gradients, no repeated skipped optimizer updates, and no task or paired-memory-dependency collapse. Run the short stability/pilot budget first; do not start a long Phase 4 run unless the histogram and stability gates pass. Results remain specific to graph traversal and the single available trained model seed.
185
+ The preregistered operational bimodality check is at least 10% of held-out candidate probabilities below 0.1, at least 10% above 0.9, and at most 20% in [0.1, 0.9], sustained across episodes. Also require finite, nonzero gate gradients, no repeated skipped optimizer updates, and no task or paired-memory-dependency collapse. Run the short stability/pilot budget first; do not start a long bistable write-gate run unless the histogram and stability gates pass. Results remain specific to graph traversal and the single available trained model seed.
146
186
 
147
187
  After saving a pilot checkpoint, evaluate that same checkpoint and `--eval-seed` under each `--eval-workspace-mode checkpoint`, `reset-each-step`, and `bypass`. For noise and selection ablations, use the same `--eval-seed` and change only the corresponding bistable scale. The gate RNG is separately checkpointed, so these changes keep the episode stream fixed; use distinct run suffixes when recording each condition.
148
188
 
@@ -154,7 +194,7 @@ The SwitchMoE router projection and expert MLPs run in FP32 even when the surrou
154
194
 
155
195
  Periodic memory-dependency checks replay the exact random episode batch used by the corresponding memory-on ID evaluation, then restore the RNG streams. This makes `memory on - memory off` a paired functional ablation rather than a difference between two independently sampled batches. The training loop also clips all trainable optimizer parameters, including the task output projection, stops before backward on non-finite forward outputs/losses, reports the first parameter with a non-finite unscaled gradient, logs skipped non-finite-gradient updates, and aborts after eight consecutive such steps instead of continuing a run that is no longer updating safely.
156
196
 
157
- The supplied Phase 1/2 traces are pre-fix evidence. One Phase 2 trace first reports a non-finite task loss at step 11,665, recurrent outputs become non-finite at step 11,666, and the process continues to step 14,106 with an AMP scale of zero before manual interruption. Memory-on/off checks in those logs used separate random episode batches, so their small positive or negative differences are not reliable paired evidence. See `07-Experiment/Broadcasting/Progress/Phase-1-MoE-Failure-Analysis.md` for the causal breakdown and re-run gate. Re-run the Phase 1 selection-only condition before Phase 2; keep the proposed 8-expert, top-1 configuration and matched seed/settings fixed while checking finite gradients, per-bank routing, and paired memory dependency.
197
+ The supplied selection-only baseline and workspace-run traces are pre-fix evidence. The workspace-run trace first reports a non-finite task loss at step 11,665, recurrent outputs become non-finite at step 11,666, and the process continues to step 14,106 with an AMP scale of zero before manual interruption. Memory-on/off checks in those logs used separate random episode batches, so their small positive or negative differences are not reliable paired evidence. See `D:/Projects/AI-Architecture-Research/07-Experiment/Broadcasting/Progress/Selection-Only-MoE-Failure-Analysis.md` for the causal breakdown and re-run gate. Re-run the selection-only baseline before the workspace experiment; keep the proposed 8-expert, top-1 configuration and matched seed/settings fixed while checking finite gradients, per-bank routing, and paired memory dependency.
158
198
 
159
199
  Training console logs now show **one MoE-bank record per 100-step logging window**: route and router-importance fractions by expert, router entropy and maximum probability, input/expert/MoE-delta activation RMS and maximum magnitude, capacity-drop fraction, and accepted gate mass. The `moe_residual:` segment records the complete MoE block output after its skip connection, separately from `moe_delta_rms/max`. A separate `[segment-diag:train]` line reports each recognized module's call count, input/output RMS and max, non-finite fraction, boundary input/output gradient RMS and non-finite fraction, and hierarchical parameter-gradient norms. Boundary gradients are collected during training backward and during failure traces; evaluation remains forward-only. The collector recognizes Mamba and CfC backbones/blocks, Mamba/CfC controller wrappers/blocks/cells, MoE blocks, chained controllers, DNC memory, workspace, and task output; names include their module paths, so the same logger works across supported controller combinations.
160
200
 
@@ -166,40 +206,40 @@ To inspect DNC internals, add `--dnc-backward-trace` to a split-graph training c
166
206
  python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix dnc-backward-trace --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 1800 --dnc-backward-trace
167
207
  ```
168
208
 
169
- Keep other flags exactly matched to the run being investigated. The fp32 DNC path removed the observed bad-key-gradient pattern in the 5,000-step matched validation capture. That run also acquired strong paired ID memory dependence by step 2,500; see `AI-Architecture-Research/07-Experiment/Broadcasting/Progress/Phase-1-MoE-Failure-Analysis.md` for the complete result and remaining generalization limits. Repeat across seeds before treating the numerical and learning outcomes as robust.
209
+ Keep other flags exactly matched to the run being investigated. The fp32 DNC path removed the observed bad-key-gradient pattern in the 5,000-step matched validation capture. That run also acquired strong paired ID memory dependence by step 2,500; see `D:/Projects/AI-Architecture-Research/07-Experiment/Broadcasting/Progress/Selection-Only-MoE-Failure-Analysis.md` for the complete result and remaining generalization limits. Repeat across seeds before treating the numerical and learning outcomes as robust.
170
210
 
171
211
  Evaluation emits separate channel lines for ID, OOD, memory-off, combiner-stage ablations, and final ID/OOD checks. These summarize the evaluations independently from training windows. If a forward value or gradient becomes non-finite, the current segment window and MoE-bank window are printed immediately before training stops or skips the unsafe update. The summaries aggregate detached device-side scalars and synchronize at log/evaluation boundaries; they do not retain activation graphs.
172
212
 
173
213
  To isolate where split-graph MoE changes the memory-learning transition, `--moe-placement` selects which component receives expert banks while keeping the model, seed, and router settings matched. `all` (the default) preserves the existing behavior; `backbone` enables MoE only in the parallel backbone; `combiner` enables MoE only in the sequential controller combiner. For the CfC combiner, the latter is its source-aware multi-source MoE bank. Component placement requires `--moe --split-graph`; combiner-only additionally requires controller combiner mode. The workspace requires backbone MoE and therefore cannot be combined with combiner-only placement.
174
214
 
175
- Run these short, seed-matched diagnostics on the same source revision. Each is capped at 5,000 steps so the usual early memory-use transition can be compared before committing to the full Phase 1 horizon:
215
+ Run these short, seed-matched diagnostics on the same source revision. Each is capped at 5,000 steps so the usual early memory-use transition can be compared before committing to the full MoE training budget:
176
216
 
177
217
  ```bash
178
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-smoke-no-moe --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --total-steps 5000
179
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-smoke-moe-backbone --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
180
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-smoke-moe-combiner --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement combiner --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
181
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-smoke-moe-all --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
218
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-baseline-no-moe --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --total-steps 5000
219
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-placement-backbone --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
220
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-placement-combiner --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement combiner --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
221
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-placement-all --split-graph --split-graph-variant mamba1 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
182
222
  ```
183
223
 
184
- Compare the paired memory-on/off dependency, ID/OOD metrics, curriculum progress, per-component segment diagnostics, and per-bank routing/activation summaries. The placement runs distinguish backbone-MoE effects from the CfC multi-source path; the `all` condition checks their interaction. These are diagnostic smoke runs, not evidence of a completed Phase 1 result.
224
+ Compare the paired memory-on/off dependency, ID/OOD metrics, curriculum progress, per-component segment diagnostics, and per-bank routing/activation summaries. The placement runs distinguish backbone-MoE effects from the CfC multi-source path; the `all` condition checks their interaction. These are diagnostic smoke runs, not evidence of a completed MoE placement comparison.
185
225
 
186
226
  For a backbone-only run, `--moe-backbone-block-isolation INDEX` installs an MoE bank after only the selected zero-based parallel-backbone block. The other backbone blocks still run their Mamba/CfC computation but have no MoE; the sequential combiner must also be excluded with `--moe-placement backbone`. The flag is range-checked against `--split-graph-num-blocks`, included in run IDs and checkpoint metadata, and cannot be combined with the workspace broadcast path. For a two-block Mamba backbone, run one isolated condition per block:
187
227
 
188
228
  ```bash
189
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-moe-backbone-block0 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 0 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
190
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-moe-backbone-block1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
229
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-backbone-block-0 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 0 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
230
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-backbone-block-1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
191
231
  ```
192
232
 
193
233
  `--moe-backbone-residual-scale SCALE` multiplies the active parallel-backbone MoE delta before its residual addition (`backbone_output + SCALE * moe_delta`). Its default is `1.0`, preserving prior behavior. The scale must be finite and non-negative, requires split-graph MoE with `--moe-placement backbone` or `all`, and is included in run IDs/checkpoint metadata. With `all`, this changes only the parallel-backbone MoE residuals; combiner MoE residuals remain at 1.0. It scales the task-loss path through the backbone MoE branch while leaving routing and the load-balancing auxiliary loss active. When workspace broadcast is enabled, the scale applies to the complete final-bank delta after expert readback, including both the selected expert outputs and their broadcast contribution. Workspace slot writes and router/load-balancing calculations remain active at their existing scales. Active workspace attention and slot updates run in fp32 outside CUDA autocast, and split-graph DNC memory math uses its fp32 path in both workspace and selection-only runs. In an isolated-block run, only that block has an MoE residual to scale. For the current block-1 follow-up, use a deliberately conservative 0.1 scale:
194
234
 
195
235
  ```bash
196
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-block1-residual-scale-0p1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 1 --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
236
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-block-1-residual-scale-0p1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement backbone --moe-backbone-block-isolation 1 --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
197
237
  ```
198
238
 
199
239
  To test backbone and combiner MoE together after the backbone-only scale control, omit block isolation and select `--moe-placement all`. The residual scale still applies only to the backbone; the combiner MoE keeps its default residual scale of 1.0:
200
240
 
201
241
  ```bash
202
- python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix phase1-backbone-combiner-backbone-scale-0p1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
242
+ python -m src.initium.core_training 0.0 --seed 0 --run-id-suffix moe-backbone-combiner-scale-0p1 --split-graph --split-graph-variant mamba1 --split-graph-num-blocks 2 --split-graph-combiner-mode controller --split-graph-combiner-variant cfc --moe --moe-placement all --moe-backbone-residual-scale 0.1 --moe-num-experts 8 --moe-top-k 1 --moe-capacity-factor 1.5 --moe-load-balance-alpha 0.01 --total-steps 5000
203
243
  ```
204
244
 
205
245
  Each `moe-bank[backbone.moe_blocks.INDEX ...]` line reports router-selected expert load, router importance, entropy, maximum probability, and capacity-drop rate separately for valid graph timesteps and all-zero padded timesteps. The valid-token mask is derived from the graph input encoding and is diagnostic-only in this experiment: routing, shared capacity, balancing loss, and outputs still include padding, so the run measures the current behavior without applying a padding fix. The per-block isolation changes which banks are active but leaves the routing policy otherwise unchanged. Compare the two blocks' valid/padded distributions and memory-dependency curves against the non-isolated backbone-only trace before deciding whether to test mask-aware routing.
@@ -324,7 +364,7 @@ A separate study compared beta 0 and beta 0.001 in a Mamba, dynamic-memory, spli
324
364
  ### Evidence boundaries and open work
325
365
 
326
366
  - Results above apply to the exact runs and evaluation procedures recorded; they do not establish broad performance across modalities, hardware, or model scales.
327
- - Phase 1's seed variation and Phase 2's four-seed selection limit aggregate claims. Report per-seed values and uncertainty alongside averages.
367
+ - The fixed-prior study's seed variation and the learned-prior study's four-seed selection limit aggregate claims. Report per-seed values and uncertainty alongside averages.
328
368
  - The learned prior's apparent variance-dampening/mean-reversion effect needs more seeds and controls to separate it from failure-mode-specific repair.
329
369
  - The alternate Mamba batch needs completion of unfinished seeds and late-curriculum evaluation before a gate-level conclusion.
330
370
  - Mamba-2/Mamba-3 performance, MoE's useful scaling regime, memory-link trade-offs, dynamic memory growth, and real-data/multimodal behavior remain active investigation areas.
@@ -112,7 +112,7 @@ pretty = true
112
112
  [[tool.mypy.overrides]]
113
113
  module = [
114
114
  "dnc", "dnc.*", "mamba_ssm", "mamba_ssm.*", "causal_conv1d", "causal_conv1d.*",
115
- "ncps", "ncps.*", "tokenizers", "tokenizers.*", "cv2",
115
+ "ncps", "ncps.*", "einops", "tokenizers", "tokenizers.*", "cv2",
116
116
  ]
117
117
  ignore_missing_imports = true
118
118
 
@@ -1,3 +1,3 @@
1
1
  """initium - modular Differentiable Neural Computer research framework."""
2
2
 
3
- __version__ = "0.3.0"
3
+ __version__ = "0.5.0"
@@ -69,17 +69,53 @@ class _CfCParallelBlock(nn.Module):
69
69
  if device is not None or dtype is not None:
70
70
  self.cfc.to(device=device, dtype=dtype)
71
71
 
72
- def forward(self, x): # x: (B, L, D)
72
+ def _forward_with_timespans(self, x: torch.Tensor, timespans: torch.Tensor) -> torch.Tensor:
73
+ """Run the ncps cell with batch-safe per-example elapsed times.
74
+
75
+ ncps' sequence wrapper squeezes each ``(B,)`` time slice before
76
+ passing it to CfCCell, which does not broadcast that shape against
77
+ ``(B, H)`` for batches larger than one. Calling the same registered
78
+ cell directly with ``(B, 1)`` spans avoids that upstream shape bug
79
+ while preserving the package's recurrence and learned parameters.
80
+ """
81
+ batch, sequence, _ = x.shape
82
+ h_state = x.new_zeros((batch, self.cfc.state_size), dtype=torch.float32)
83
+ c_state = torch.zeros_like(h_state) if self.cfc.use_mixed else None
84
+ outputs = []
85
+ for step in range(sequence):
86
+ step_input = x[:, step]
87
+ if self.cfc.use_mixed:
88
+ assert c_state is not None
89
+ h_state, c_state = self.cfc.lstm(step_input, (h_state, c_state))
90
+ step_output, h_state = self.cfc.rnn_cell(
91
+ step_input,
92
+ h_state,
93
+ timespans[:, step : step + 1],
94
+ )
95
+ outputs.append(self.cfc.fc(step_output))
96
+ return torch.stack(outputs, dim=1)
97
+
98
+ def forward(self, x, *, timespans: torch.Tensor | None = None): # x: (B, L, D)
73
99
  if self.force_fp32:
74
100
  with torch.autocast(device_type=x.device.type, enabled=False):
75
- out, _ = self.cfc(self.norm(x.float()))
101
+ normalized = self.norm(x.float())
102
+ if timespans is None:
103
+ out, _ = self.cfc(normalized)
104
+ else:
105
+ out = self._forward_with_timespans(normalized, timespans)
76
106
  else:
77
- out, _ = self.cfc(self.norm(x))
107
+ normalized = self.norm(x)
108
+ if timespans is None:
109
+ out, _ = self.cfc(normalized)
110
+ else:
111
+ out = self._forward_with_timespans(normalized, timespans)
78
112
  out = out.to(x.dtype)
79
113
  return x + out if self.residual else out
80
114
 
81
115
 
82
116
  class CfCBackboneParallel(nn.Module):
117
+ supports_timespans = True
118
+
83
119
  def __init__(
84
120
  self,
85
121
  in_dim,
@@ -164,20 +200,63 @@ class CfCBackboneParallel(nn.Module):
164
200
  )
165
201
 
166
202
  def forward(
167
- self, x, *, token_mask: torch.Tensor | None = None
203
+ self,
204
+ x,
205
+ *,
206
+ token_mask: torch.Tensor | None = None,
207
+ timespans: torch.Tensor | None = None,
168
208
  ): # (B, L, in_dim) -> (B, L, d_model), called ONCE per training step
209
+ timespans = self._normalize_timespans(timespans, x)
169
210
  h = self.in_adapter(x)
170
211
  moe_blocks = self.moe_blocks
171
212
  for i, block in enumerate(self.blocks):
172
- h = block(h)
213
+ h = block(h, timespans=timespans)
173
214
  if moe_blocks is not None and isinstance(moe_blocks[i], MoEBlock):
174
215
  h = moe_blocks[i](h, token_mask=token_mask)
175
216
  return h
176
217
 
177
- def forward_with_specialists(self, x: torch.Tensor, *, token_mask: torch.Tensor | None = None):
218
+ @staticmethod
219
+ def _normalize_timespans(
220
+ timespans: torch.Tensor | None, x: torch.Tensor
221
+ ) -> torch.Tensor | None:
222
+ if timespans is None:
223
+ return None
224
+ timespans = torch.as_tensor(timespans, device=x.device, dtype=torch.float32)
225
+ batch, sequence = x.shape[:2]
226
+ if timespans.ndim == 1 and timespans.shape[0] == sequence:
227
+ timespans = timespans.unsqueeze(0).expand(batch, -1)
228
+ elif timespans.ndim == 2 and timespans.shape == (1, sequence):
229
+ timespans = timespans.expand(batch, -1)
230
+ elif timespans.ndim != 2 or timespans.shape != (batch, sequence):
231
+ raise ValueError(
232
+ "CfC timespans must have shape (sequence,), (1, sequence), "
233
+ f"or (batch, sequence); got {tuple(timespans.shape)} for input {tuple(x.shape[:2])}"
234
+ )
235
+ return timespans
236
+
237
+ def forward_with_timespans(
238
+ self,
239
+ x: torch.Tensor,
240
+ timespans: torch.Tensor,
241
+ *,
242
+ token_mask: torch.Tensor | None = None,
243
+ ) -> torch.Tensor:
244
+ """Explicit elapsed-time entry point for composing mixed backbones."""
245
+ return self.forward(x, token_mask=token_mask, timespans=timespans)
246
+
247
+ def forward_with_specialists(
248
+ self,
249
+ x: torch.Tensor,
250
+ *,
251
+ token_mask: torch.Tensor | None = None,
252
+ timespans: torch.Tensor | None = None,
253
+ intermediate_observer=None,
254
+ selected_specialists_only: bool = True,
255
+ ):
178
256
  """Return final MoE candidates and routes for shared-workspace use."""
179
257
  if not self.moe_enabled or self.moe_blocks is None:
180
258
  raise RuntimeError("workspace broadcast requires an enabled MoE expert pool")
259
+ timespans = self._normalize_timespans(timespans, x)
181
260
  h = self.in_adapter(x)
182
261
  final_index = len(self.blocks) - 1
183
262
  if not isinstance(self.moe_blocks[final_index], MoEBlock):
@@ -185,11 +264,16 @@ class CfCBackboneParallel(nn.Module):
185
264
  "workspace broadcast requires the final backbone block to have an active MoE bank"
186
265
  )
187
266
  for i, block in enumerate(self.blocks):
188
- h = block(h)
267
+ h = block(h, timespans=timespans)
268
+ if i == 0 and intermediate_observer is not None:
269
+ intermediate_observer(h)
189
270
  if i == final_index:
190
271
  base = h
191
272
  _, experts, indices, weights = self.moe_blocks[i](
192
- h, return_details=True, token_mask=token_mask
273
+ h,
274
+ return_details=not selected_specialists_only,
275
+ return_selected_details=selected_specialists_only,
276
+ token_mask=token_mask,
193
277
  )
194
278
  return base, experts, indices, weights
195
279
  if isinstance(self.moe_blocks[i], MoEBlock):