initium-deeplearning 0.4.0__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/PKG-INFO +1 -1
  2. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/__init__.py +1 -1
  3. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/LNN_controller/cfc_backbone_parallel.py +5 -1
  4. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/MoE/moe_layer.py +38 -24
  5. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/__init__.py +14 -0
  6. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/core_training.py +2661 -91
  7. initium_deeplearning-0.5.0/src/initium/data/common/causal_prediction.py +75 -0
  8. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/dataset_registry.py +8 -0
  9. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/graph_traversal/graph_traversal.py +320 -0
  10. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/README.md +299 -0
  11. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/__init__.py +5 -0
  12. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/curriculum.py +160 -0
  13. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/dataset.py +2132 -0
  14. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/episode.py +137 -0
  15. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/__init__.py +1 -0
  16. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/entity_state.py +303 -0
  17. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/nth_farthest.py +91 -0
  18. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/permutation.py +185 -0
  19. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/relational_join.py +86 -0
  20. initium_deeplearning-0.5.0/src/initium/data/workspace_capability/tracks/retrieval_aggregation.py +103 -0
  21. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/diagnostics/segment_logging.py +52 -16
  22. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/model_loader.py +28 -2
  23. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/mamba_controller/mamba_backbone_parallel.py +130 -10
  24. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/mamba_controller/split_graph_dnc.py +708 -26
  25. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/dnc_mechanisms.py +35 -0
  26. initium_deeplearning-0.5.0/src/initium/memory_manipulation/sequence_memories.py +513 -0
  27. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/stochastic_write_head_v2.py +103 -38
  28. initium_deeplearning-0.5.0/src/initium/workspace/README.md +159 -0
  29. initium_deeplearning-0.5.0/src/initium/workspace/broadcast.py +2288 -0
  30. initium_deeplearning-0.5.0/src/initium/workspace/kv_bridge.py +166 -0
  31. initium_deeplearning-0.5.0/src/initium/workspace/nth_farthest.py +43 -0
  32. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/integration/test_split_graph_dnc.py +34 -0
  33. initium_deeplearning-0.5.0/tests/integration/test_workspace_capability_pipeline.py +396 -0
  34. initium_deeplearning-0.5.0/tests/unit/test_sequence_memories.py +138 -0
  35. initium_deeplearning-0.5.0/tests/unit/test_workspace_capability.py +645 -0
  36. initium_deeplearning-0.4.0/src/initium/workspace/broadcast.py +0 -670
  37. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/.gitignore +0 -0
  38. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/LICENSE +0 -0
  39. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/README.md +0 -0
  40. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/pyproject.toml +0 -0
  41. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/LNN_controller/cfc_controller.py +0 -0
  42. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/LNN_controller/chained_controller.py +0 -0
  43. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/LNN_controller/hybrid_controller.py +0 -0
  44. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/MoE/README.md +0 -0
  45. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/analysis/__init__.py +0 -0
  46. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/analysis/kl_attribution.py +0 -0
  47. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/config/controller_config.py +0 -0
  48. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/config/train_config.py +0 -0
  49. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/__init__.py +0 -0
  50. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/audio/__init__.py +0 -0
  51. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/audio/audio_dataset.py +0 -0
  52. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/base_dataset.py +0 -0
  53. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/common/__init__.py +0 -0
  54. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/common/chain_task.py +0 -0
  55. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/common/digit_codec.py +0 -0
  56. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/common/graph_io.py +0 -0
  57. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/common/real_data.py +0 -0
  58. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/graph_traversal/__init__.py +0 -0
  59. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/graph_traversal/order_probe.py +0 -0
  60. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/graph_traversal/relational_probe.py +0 -0
  61. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/multimodal/__init__.py +0 -0
  62. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/multimodal/multimodal_dataset.py +0 -0
  63. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/prepared_dataset.py +0 -0
  64. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/surprise_recall/__init__.py +0 -0
  65. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/surprise_recall/next_token_prediction.py +0 -0
  66. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/surprise_recall/surprise_recall.py +0 -0
  67. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/surprise_recall/validate.py +0 -0
  68. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/text/__init__.py +0 -0
  69. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/text/text_dataset.py +0 -0
  70. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/video/__init__.py +0 -0
  71. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/data/video/video_dataset.py +0 -0
  72. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/diagnostics/__init__.py +0 -0
  73. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/diagnostics/split_graph_forward_trace.py +0 -0
  74. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/diagnostics/surprise_recall_logging.py +0 -0
  75. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/__init__.py +0 -0
  76. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/__init__.py +0 -0
  77. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/base_cache.py +0 -0
  78. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/cache_config.py +0 -0
  79. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/cache_registry.py +0 -0
  80. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/cached_engine.py +0 -0
  81. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/keys.py +0 -0
  82. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/prefix_cache.py +0 -0
  83. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/result_cache.py +0 -0
  84. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/state_utils.py +0 -0
  85. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/cache/stores.py +0 -0
  86. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/capabilities.py +0 -0
  87. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/checkpoint_io.py +0 -0
  88. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/engine.py +0 -0
  89. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/inference_config.py +0 -0
  90. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/metrics.py +0 -0
  91. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/run_inference.py +0 -0
  92. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/run_logging.py +0 -0
  93. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/__init__.py +0 -0
  94. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/audio_task.py +0 -0
  95. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/base_task.py +0 -0
  96. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/graph_traversal_task.py +0 -0
  97. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/multimodal_task.py +0 -0
  98. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/task_registry.py +0 -0
  99. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/text_task.py +0 -0
  100. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/inference/tasks/video_task.py +0 -0
  101. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/mamba_controller/mamba2_controller.py +0 -0
  102. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/mamba_controller/mamba3_controller.py +0 -0
  103. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/mamba_controller/mamba_controller.py +0 -0
  104. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/mamba_controller/numerical_stability.py +0 -0
  105. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/mamba_controller/sgr_backbone_only.py +0 -0
  106. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/dynamic_memory_resize.py +0 -0
  107. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/dynamic_n_controller.py +0 -0
  108. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/link_matrix_ablation.py +0 -0
  109. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/memory_manipulation/nvrtc_compat.py +0 -0
  110. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/setup_wheels.py +0 -0
  111. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/workspace/__init__.py +0 -0
  112. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/src/initium/workspace/bistability.py +0 -0
  113. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/README.md +0 -0
  114. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/checkpoint/test_checkpoint_roundtrip.py +0 -0
  115. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/checkpoint/test_resume_training.py +0 -0
  116. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/conftest.py +0 -0
  117. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/gradients/test_controller_cell_backward.py +0 -0
  118. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/gradients/test_kl_gradients.py +0 -0
  119. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/gradients/test_moe_aux_loss_gradients.py +0 -0
  120. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/integration/test_controller_matrix.py +0 -0
  121. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/integration/test_dynamic_n_end_to_end.py +0 -0
  122. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/integration/test_moe_integration.py +0 -0
  123. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/integration/test_resize_x_link_matrix.py +0 -0
  124. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/smoke/test_inference_smoke.py +0 -0
  125. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/smoke/test_tiny_training_run.py +0 -0
  126. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_bistable_workspace_gate.py +0 -0
  127. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_capabilities.py +0 -0
  128. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_checkpoint_io.py +0 -0
  129. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_dynamic_n_controller.py +0 -0
  130. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_link_matrix_ablation.py +0 -0
  131. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_model_loader.py +0 -0
  132. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_moe_layer.py +0 -0
  133. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_next_token_prediction.py +0 -0
  134. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_numerical_stability.py +0 -0
  135. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_nvrtc_compat.py +0 -0
  136. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_optimized_dnc_memory.py +0 -0
  137. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_real_data_pipeline.py +0 -0
  138. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_resize_memory.py +0 -0
  139. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_stochastic_write_head.py +0 -0
  140. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_surprise_recall.py +0 -0
  141. {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.0}/tests/unit/test_workspace_reconsideration.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: initium-deeplearning
3
- Version: 0.4.0
3
+ Version: 0.5.0
4
4
  Summary: Differentiable Neural Computer research framework with pluggable LSTM/Mamba/CfC controllers, MoE, split-graph and stochastic write heads.
5
5
  Project-URL: Homepage, https://github.com/samueleternity/initium
6
6
  Project-URL: Issues, https://github.com/samueleternity/initium/issues
@@ -1,3 +1,3 @@
1
1
  """initium - modular Differentiable Neural Computer research framework."""
2
2
 
3
- __version__ = "0.4.0"
3
+ __version__ = "0.5.0"
@@ -251,6 +251,7 @@ class CfCBackboneParallel(nn.Module):
251
251
  token_mask: torch.Tensor | None = None,
252
252
  timespans: torch.Tensor | None = None,
253
253
  intermediate_observer=None,
254
+ selected_specialists_only: bool = True,
254
255
  ):
255
256
  """Return final MoE candidates and routes for shared-workspace use."""
256
257
  if not self.moe_enabled or self.moe_blocks is None:
@@ -269,7 +270,10 @@ class CfCBackboneParallel(nn.Module):
269
270
  if i == final_index:
270
271
  base = h
271
272
  _, experts, indices, weights = self.moe_blocks[i](
272
- h, return_details=True, token_mask=token_mask
273
+ h,
274
+ return_details=not selected_specialists_only,
275
+ return_selected_details=selected_specialists_only,
276
+ token_mask=token_mask,
273
277
  )
274
278
  return base, experts, indices, weights
275
279
  if isinstance(self.moe_blocks[i], MoEBlock):
@@ -178,6 +178,7 @@ class SwitchMoE(nn.Module):
178
178
  x: torch.Tensor,
179
179
  *,
180
180
  return_details: bool = False,
181
+ return_selected_details: bool = False,
181
182
  token_mask: torch.Tensor | None = None,
182
183
  gradient_trace=None,
183
184
  ):
@@ -231,10 +232,10 @@ class SwitchMoE(nn.Module):
231
232
  # the full expert softmax (so the task objective can train the
232
233
  # router); top-k>1 normalizes among the selected experts.
233
234
  # Normal training dispatches selected, capacity-accepted routes into
234
- # bounded per-expert buffers. The optional all-expert result is
235
- # reserved for the workspace specialist path, which explicitly needs
236
- # every specialist candidate. This keeps top-k routing sparse in
237
- # compute and avoids expanding expert weights per route.
235
+ # bounded per-expert buffers. Workspace can request those selected
236
+ # candidates directly; only the explicit random-write control requests
237
+ # dense outputs from every expert. This keeps ordinary top-k workspace
238
+ # routing sparse in compute and avoids expanding expert weights per route.
238
239
  top_k = self.top_k
239
240
  topk_prob, topk_idx = probs.topk(top_k, dim=-1) # (T, k) each
240
241
  if top_k == 1:
@@ -349,18 +350,12 @@ class SwitchMoE(nn.Module):
349
350
  selected_outputs = trace("selected_outputs", selected_outputs)
350
351
  output32 = (selected_outputs * accepted_gates.unsqueeze(-1)).sum(dim=1)
351
352
  output32 = trace("routed_output", output32)
352
- accepted_outputs = (
353
- selected_outputs * keep.unsqueeze(-1).to(selected_outputs.dtype)
354
- if collect_diagnostics
355
- else None
356
- )
357
353
 
358
354
  expert_out_all = None
359
355
  if return_details:
360
- # Workspace broadcasting is the one caller that needs all
361
- # specialist candidates, including experts not selected by
362
- # the sparse route. Keep this expensive activation path
363
- # explicit, while still using each expert's shared weights.
356
+ # The random-write workspace control needs every specialist
357
+ # candidate, including experts not selected by sparse routing.
358
+ # Keep this expensive activation path explicit.
364
359
  all_expert_outputs: list[torch.Tensor] = []
365
360
  for raw_expert in self.experts:
366
361
  expert = cast(Expert, raw_expert)
@@ -388,9 +383,15 @@ class SwitchMoE(nn.Module):
388
383
  output = output32.to(output_dtype)
389
384
 
390
385
  if collect_diagnostics:
391
- assert accepted_outputs is not None
392
386
  with torch.no_grad():
393
- one_hot_k = F.one_hot(topk_idx, num_classes=self.num_experts).float()
387
+ accepted_selected = selected_outputs.detach()[keep]
388
+ total_route_values = max(1, keep.numel() * selected_outputs.shape[-1])
389
+ accepted_output_sq_mean = accepted_selected.square().sum() / total_route_values
390
+ accepted_output_absmax = (
391
+ accepted_selected.abs().amax()
392
+ if accepted_selected.numel()
393
+ else selected_outputs.new_zeros((), dtype=torch.float32)
394
+ )
394
395
  self._last_accepted = keep.detach()
395
396
  self._last_diag = {
396
397
  "cv_importance": (P_i.std() / P_i.mean().clamp(min=1e-8)).detach(),
@@ -405,8 +406,8 @@ class SwitchMoE(nn.Module):
405
406
  "topk_gate_mean": topk_prob.mean().detach(),
406
407
  "input_rms": flat32.square().mean().sqrt().detach(),
407
408
  "input_absmax": flat32.abs().amax().detach(),
408
- "expert_output_rms": accepted_outputs.square().mean().sqrt().detach(),
409
- "expert_output_absmax": accepted_outputs.abs().amax().detach(),
409
+ "expert_output_rms": accepted_output_sq_mean.sqrt().detach(),
410
+ "expert_output_absmax": accepted_output_absmax.detach(),
410
411
  "routed_output_rms": output32.square().mean().sqrt().detach(),
411
412
  "routed_output_absmax": output32.abs().amax().detach(),
412
413
  "capacity_drop_frac": (1.0 - keep.float().mean()).detach(),
@@ -423,9 +424,10 @@ class SwitchMoE(nn.Module):
423
424
  subset_weight = subset.to(probs.dtype)
424
425
  count = subset_weight.sum()
425
426
  denom = count.clamp_min(1.0)
426
- subset_load = (one_hot_k * subset_weight[:, None, None]).sum(dim=(0, 1)) / (
427
- denom * top_k
428
- )
427
+ selected_experts = topk_idx[subset].reshape(-1)
428
+ subset_load = torch.bincount(
429
+ selected_experts, minlength=self.num_experts
430
+ ).to(probs.dtype) / (denom * top_k)
429
431
  subset_importance = (probs * subset_weight[:, None]).sum(dim=0) / denom
430
432
  entropy = (per_token_entropy * subset_weight).sum() / denom
431
433
  max_prob = (per_token_max_prob * subset_weight).sum() / denom
@@ -451,10 +453,18 @@ class SwitchMoE(nn.Module):
451
453
  self._diag_calls += 1
452
454
 
453
455
  output = output.reshape(orig_shape)
454
- if return_details:
456
+ if return_details or return_selected_details:
457
+ route_shape = (*orig_shape[:-1], top_k)
458
+ selected_shape = (*orig_shape[:-1], top_k, d_model)
459
+ if return_selected_details:
460
+ return (
461
+ output,
462
+ selected_outputs.to(output_dtype).reshape(selected_shape),
463
+ topk_idx.reshape(route_shape),
464
+ accepted_gates.reshape(route_shape),
465
+ )
455
466
  assert expert_out_all is not None
456
467
  expert_shape = (*orig_shape[:-1], self.num_experts, d_model)
457
- route_shape = (*orig_shape[:-1], top_k)
458
468
  return (
459
469
  output,
460
470
  expert_out_all.to(output_dtype).reshape(expert_shape),
@@ -577,13 +587,17 @@ class MoEBlock(nn.Module):
577
587
  x: torch.Tensor,
578
588
  *,
579
589
  return_details: bool = False,
590
+ return_selected_details: bool = False,
580
591
  token_mask: torch.Tensor | None = None,
581
592
  ):
582
- if not return_details:
593
+ if not return_details and not return_selected_details:
583
594
  routed = self.moe(self.norm(x), token_mask=token_mask)
584
595
  return x + self.residual_scale * routed
585
596
  routed, experts, indices, weights = self.moe(
586
- self.norm(x), return_details=True, token_mask=token_mask
597
+ self.norm(x),
598
+ return_details=return_details,
599
+ return_selected_details=return_selected_details,
600
+ token_mask=token_mask,
587
601
  )
588
602
  return x + self.residual_scale * routed, experts, indices, weights
589
603
 
@@ -21,6 +21,10 @@ __all__ = [
21
21
  # DNC architectures
22
22
  "SplitGraphDNC",
23
23
  "MambaDNC",
24
+ # Sequence-parallel memory modules
25
+ "SequenceMemory",
26
+ "GatedDeltaMemory",
27
+ "CausalKVMemory",
24
28
  # Controller mechanisms and compositions
25
29
  "MambaControllerWrapper",
26
30
  "Mamba2ControllerWrapper",
@@ -43,6 +47,9 @@ __all__ = [
43
47
  _EXPORTS = {
44
48
  "SplitGraphDNC": "initium.mamba_controller.split_graph_dnc",
45
49
  "MambaDNC": "initium.mamba_controller.mamba_controller",
50
+ "SequenceMemory": "initium.memory_manipulation.sequence_memories",
51
+ "GatedDeltaMemory": "initium.memory_manipulation.sequence_memories",
52
+ "CausalKVMemory": "initium.memory_manipulation.sequence_memories",
46
53
  "MambaControllerWrapper": "initium.mamba_controller.mamba_controller",
47
54
  "Mamba2ControllerWrapper": "initium.mamba_controller.mamba2_controller",
48
55
  "Mamba3ControllerWrapper": "initium.mamba_controller.mamba3_controller",
@@ -83,6 +90,13 @@ if TYPE_CHECKING:
83
90
  from initium.mamba_controller.mamba_controller import MambaDNC as MambaDNC
84
91
  from initium.mamba_controller.split_graph_dnc import SplitGraphDNC as SplitGraphDNC
85
92
  from initium.memory_manipulation.dynamic_memory_resize import resize_memory as resize_memory
93
+ from initium.memory_manipulation.sequence_memories import (
94
+ CausalKVMemory as CausalKVMemory,
95
+ )
96
+ from initium.memory_manipulation.sequence_memories import (
97
+ GatedDeltaMemory as GatedDeltaMemory,
98
+ )
99
+ from initium.memory_manipulation.sequence_memories import SequenceMemory as SequenceMemory
86
100
  from initium.memory_manipulation.stochastic_write_head_v2 import (
87
101
  StochasticWriteHead as StochasticWriteHead,
88
102
  )