initium-deeplearning 0.4.0__tar.gz → 0.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/PKG-INFO +1 -1
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/pyproject.toml +1 -1
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/__init__.py +1 -1
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/LNN_controller/cfc_backbone_parallel.py +7 -1
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/MoE/moe_layer.py +38 -24
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/__init__.py +14 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/config/train_config.py +2 -2
- initium_deeplearning-0.5.1/src/initium/core_training.py +12274 -0
- initium_deeplearning-0.5.1/src/initium/data/common/causal_prediction.py +75 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/dataset_registry.py +8 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/graph_traversal/graph_traversal.py +320 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/README.md +629 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/__init__.py +5 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/curriculum.py +699 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/dataset.py +6324 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/episode.py +183 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/gnw_metrics.py +189 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/lang_dataset.py +1343 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/__init__.py +1 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/_dev_lexicon.py +110 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/_dev_semantics.py +111 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/_lang_grammar.py +511 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/_lang_text.py +155 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/_shared.py +64 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/attentional_blink.py +138 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/belief_tracking.py +130 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/bistable_perception.py +61 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/capacity_bottleneck.py +187 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/conditional_ops.py +209 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/developmental.py +274 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/developmental_lexical.py +161 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/developmental_production_semantic.py +117 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/developmental_semantic.py +110 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/entity_state.py +303 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/ignition.py +73 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/lang5_tracks.py +998 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/lang_tracks.py +645 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/masked_priming.py +96 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/mixed_skill.py +125 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/no_report_confidence.py +70 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/nth_farthest.py +91 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/permutation.py +198 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/production.py +175 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/psychological_refractory.py +66 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/relational_join.py +86 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/retrieval_aggregation.py +105 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/rule_switching.py +139 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/sustained_maintenance.py +48 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/trace_conditioning.py +58 -0
- initium_deeplearning-0.5.1/src/initium/data/workspace_capability/tracks/variable_binding.py +189 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/diagnostics/segment_logging.py +52 -16
- initium_deeplearning-0.5.1/src/initium/inference/lang_chat.py +533 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/model_loader.py +166 -3
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/mamba_controller/mamba_backbone_parallel.py +168 -17
- initium_deeplearning-0.5.1/src/initium/mamba_controller/split_graph_dnc.py +2397 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/memory_manipulation/dnc_mechanisms.py +35 -0
- initium_deeplearning-0.5.1/src/initium/memory_manipulation/sequence_memories.py +513 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/memory_manipulation/stochastic_write_head_v2.py +103 -38
- initium_deeplearning-0.5.1/src/initium/workspace/README.md +295 -0
- initium_deeplearning-0.5.1/src/initium/workspace/__init__.py +13 -0
- initium_deeplearning-0.5.1/src/initium/workspace/adaptive.py +229 -0
- initium_deeplearning-0.5.1/src/initium/workspace/adaptive_v2.py +230 -0
- initium_deeplearning-0.5.1/src/initium/workspace/adaptive_v3.py +1154 -0
- initium_deeplearning-0.5.1/src/initium/workspace/adaptive_v4.py +638 -0
- initium_deeplearning-0.5.1/src/initium/workspace/allocator.py +116 -0
- initium_deeplearning-0.5.1/src/initium/workspace/broadcast.py +3729 -0
- initium_deeplearning-0.5.1/src/initium/workspace/kv_bridge.py +166 -0
- initium_deeplearning-0.5.1/src/initium/workspace/nth_farthest.py +43 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/integration/test_split_graph_dnc.py +34 -0
- initium_deeplearning-0.5.1/tests/integration/test_workspace_capability_pipeline.py +396 -0
- initium_deeplearning-0.5.1/tests/unit/test_sequence_memories.py +138 -0
- initium_deeplearning-0.5.1/tests/unit/test_workspace_capability.py +674 -0
- initium_deeplearning-0.5.1/tests/unit/test_workspace_capability_gnw_tasks.py +260 -0
- initium_deeplearning-0.5.1/tests/unit/test_workspace_capability_new_tasks.py +526 -0
- initium_deeplearning-0.4.0/src/initium/core_training.py +0 -6088
- initium_deeplearning-0.4.0/src/initium/mamba_controller/split_graph_dnc.py +0 -1039
- initium_deeplearning-0.4.0/src/initium/workspace/__init__.py +0 -5
- initium_deeplearning-0.4.0/src/initium/workspace/broadcast.py +0 -670
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/.gitignore +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/LICENSE +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/README.md +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/LNN_controller/cfc_controller.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/LNN_controller/chained_controller.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/LNN_controller/hybrid_controller.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/MoE/README.md +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/analysis/__init__.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/analysis/kl_attribution.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/config/controller_config.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/__init__.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/audio/__init__.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/audio/audio_dataset.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/base_dataset.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/common/__init__.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/common/chain_task.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/common/digit_codec.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/common/graph_io.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/common/real_data.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/graph_traversal/__init__.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/graph_traversal/order_probe.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/graph_traversal/relational_probe.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/multimodal/__init__.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/multimodal/multimodal_dataset.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/prepared_dataset.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/surprise_recall/__init__.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/surprise_recall/next_token_prediction.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/surprise_recall/surprise_recall.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/surprise_recall/validate.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/text/__init__.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/text/text_dataset.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/video/__init__.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/data/video/video_dataset.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/diagnostics/__init__.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/diagnostics/split_graph_forward_trace.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/diagnostics/surprise_recall_logging.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/__init__.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/cache/__init__.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/cache/base_cache.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/cache/cache_config.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/cache/cache_registry.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/cache/cached_engine.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/cache/keys.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/cache/prefix_cache.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/cache/result_cache.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/cache/state_utils.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/cache/stores.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/capabilities.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/checkpoint_io.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/engine.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/inference_config.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/metrics.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/run_inference.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/run_logging.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/tasks/__init__.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/tasks/audio_task.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/tasks/base_task.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/tasks/graph_traversal_task.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/tasks/multimodal_task.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/tasks/task_registry.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/tasks/text_task.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/inference/tasks/video_task.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/mamba_controller/mamba2_controller.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/mamba_controller/mamba3_controller.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/mamba_controller/mamba_controller.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/mamba_controller/numerical_stability.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/mamba_controller/sgr_backbone_only.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/memory_manipulation/dynamic_memory_resize.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/memory_manipulation/dynamic_n_controller.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/memory_manipulation/link_matrix_ablation.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/memory_manipulation/nvrtc_compat.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/setup_wheels.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/workspace/bistability.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/README.md +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/checkpoint/test_checkpoint_roundtrip.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/checkpoint/test_resume_training.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/conftest.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/gradients/test_controller_cell_backward.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/gradients/test_kl_gradients.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/gradients/test_moe_aux_loss_gradients.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/integration/test_controller_matrix.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/integration/test_dynamic_n_end_to_end.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/integration/test_moe_integration.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/integration/test_resize_x_link_matrix.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/smoke/test_inference_smoke.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/smoke/test_tiny_training_run.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_bistable_workspace_gate.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_capabilities.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_checkpoint_io.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_dynamic_n_controller.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_link_matrix_ablation.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_model_loader.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_moe_layer.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_next_token_prediction.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_numerical_stability.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_nvrtc_compat.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_optimized_dnc_memory.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_real_data_pipeline.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_resize_memory.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_stochastic_write_head.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_surprise_recall.py +0 -0
- {initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/tests/unit/test_workspace_reconsideration.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: initium-deeplearning
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.1
|
|
4
4
|
Summary: Differentiable Neural Computer research framework with pluggable LSTM/Mamba/CfC controllers, MoE, split-graph and stochastic write heads.
|
|
5
5
|
Project-URL: Homepage, https://github.com/samueleternity/initium
|
|
6
6
|
Project-URL: Issues, https://github.com/samueleternity/initium/issues
|
|
@@ -112,7 +112,7 @@ pretty = true
|
|
|
112
112
|
[[tool.mypy.overrides]]
|
|
113
113
|
module = [
|
|
114
114
|
"dnc", "dnc.*", "mamba_ssm", "mamba_ssm.*", "causal_conv1d", "causal_conv1d.*",
|
|
115
|
-
"ncps", "ncps.*", "einops", "tokenizers", "tokenizers.*", "cv2",
|
|
115
|
+
"ncps", "ncps.*", "einops", "tokenizers", "tokenizers.*", "cv2", "sentencepiece",
|
|
116
116
|
]
|
|
117
117
|
ignore_missing_imports = true
|
|
118
118
|
|
|
@@ -251,6 +251,7 @@ class CfCBackboneParallel(nn.Module):
|
|
|
251
251
|
token_mask: torch.Tensor | None = None,
|
|
252
252
|
timespans: torch.Tensor | None = None,
|
|
253
253
|
intermediate_observer=None,
|
|
254
|
+
selected_specialists_only: bool = True,
|
|
254
255
|
):
|
|
255
256
|
"""Return final MoE candidates and routes for shared-workspace use."""
|
|
256
257
|
if not self.moe_enabled or self.moe_blocks is None:
|
|
@@ -269,7 +270,10 @@ class CfCBackboneParallel(nn.Module):
|
|
|
269
270
|
if i == final_index:
|
|
270
271
|
base = h
|
|
271
272
|
_, experts, indices, weights = self.moe_blocks[i](
|
|
272
|
-
h,
|
|
273
|
+
h,
|
|
274
|
+
return_details=not selected_specialists_only,
|
|
275
|
+
return_selected_details=selected_specialists_only,
|
|
276
|
+
token_mask=token_mask,
|
|
273
277
|
)
|
|
274
278
|
return base, experts, indices, weights
|
|
275
279
|
if isinstance(self.moe_blocks[i], MoEBlock):
|
|
@@ -295,6 +299,7 @@ def build_parallel_backbone(
|
|
|
295
299
|
moe_load_balance_alpha: float = 0.01,
|
|
296
300
|
moe_residual_scale: float = 1.0,
|
|
297
301
|
moe_block_isolation: int | None = None,
|
|
302
|
+
activation_checkpointing: bool = False,
|
|
298
303
|
device=None,
|
|
299
304
|
dtype=None,
|
|
300
305
|
):
|
|
@@ -348,6 +353,7 @@ def build_parallel_backbone(
|
|
|
348
353
|
moe_load_balance_alpha=moe_load_balance_alpha,
|
|
349
354
|
moe_residual_scale=moe_residual_scale,
|
|
350
355
|
moe_block_isolation=moe_block_isolation,
|
|
356
|
+
activation_checkpointing=activation_checkpointing,
|
|
351
357
|
)
|
|
352
358
|
)
|
|
353
359
|
cur = d_model
|
|
@@ -178,6 +178,7 @@ class SwitchMoE(nn.Module):
|
|
|
178
178
|
x: torch.Tensor,
|
|
179
179
|
*,
|
|
180
180
|
return_details: bool = False,
|
|
181
|
+
return_selected_details: bool = False,
|
|
181
182
|
token_mask: torch.Tensor | None = None,
|
|
182
183
|
gradient_trace=None,
|
|
183
184
|
):
|
|
@@ -231,10 +232,10 @@ class SwitchMoE(nn.Module):
|
|
|
231
232
|
# the full expert softmax (so the task objective can train the
|
|
232
233
|
# router); top-k>1 normalizes among the selected experts.
|
|
233
234
|
# Normal training dispatches selected, capacity-accepted routes into
|
|
234
|
-
# bounded per-expert buffers.
|
|
235
|
-
#
|
|
236
|
-
# every
|
|
237
|
-
# compute and avoids expanding expert weights per route.
|
|
235
|
+
# bounded per-expert buffers. Workspace can request those selected
|
|
236
|
+
# candidates directly; only the explicit random-write control requests
|
|
237
|
+
# dense outputs from every expert. This keeps ordinary top-k workspace
|
|
238
|
+
# routing sparse in compute and avoids expanding expert weights per route.
|
|
238
239
|
top_k = self.top_k
|
|
239
240
|
topk_prob, topk_idx = probs.topk(top_k, dim=-1) # (T, k) each
|
|
240
241
|
if top_k == 1:
|
|
@@ -349,18 +350,12 @@ class SwitchMoE(nn.Module):
|
|
|
349
350
|
selected_outputs = trace("selected_outputs", selected_outputs)
|
|
350
351
|
output32 = (selected_outputs * accepted_gates.unsqueeze(-1)).sum(dim=1)
|
|
351
352
|
output32 = trace("routed_output", output32)
|
|
352
|
-
accepted_outputs = (
|
|
353
|
-
selected_outputs * keep.unsqueeze(-1).to(selected_outputs.dtype)
|
|
354
|
-
if collect_diagnostics
|
|
355
|
-
else None
|
|
356
|
-
)
|
|
357
353
|
|
|
358
354
|
expert_out_all = None
|
|
359
355
|
if return_details:
|
|
360
|
-
#
|
|
361
|
-
#
|
|
362
|
-
#
|
|
363
|
-
# explicit, while still using each expert's shared weights.
|
|
356
|
+
# The random-write workspace control needs every specialist
|
|
357
|
+
# candidate, including experts not selected by sparse routing.
|
|
358
|
+
# Keep this expensive activation path explicit.
|
|
364
359
|
all_expert_outputs: list[torch.Tensor] = []
|
|
365
360
|
for raw_expert in self.experts:
|
|
366
361
|
expert = cast(Expert, raw_expert)
|
|
@@ -388,9 +383,15 @@ class SwitchMoE(nn.Module):
|
|
|
388
383
|
output = output32.to(output_dtype)
|
|
389
384
|
|
|
390
385
|
if collect_diagnostics:
|
|
391
|
-
assert accepted_outputs is not None
|
|
392
386
|
with torch.no_grad():
|
|
393
|
-
|
|
387
|
+
accepted_selected = selected_outputs.detach()[keep]
|
|
388
|
+
total_route_values = max(1, keep.numel() * selected_outputs.shape[-1])
|
|
389
|
+
accepted_output_sq_mean = accepted_selected.square().sum() / total_route_values
|
|
390
|
+
accepted_output_absmax = (
|
|
391
|
+
accepted_selected.abs().amax()
|
|
392
|
+
if accepted_selected.numel()
|
|
393
|
+
else selected_outputs.new_zeros((), dtype=torch.float32)
|
|
394
|
+
)
|
|
394
395
|
self._last_accepted = keep.detach()
|
|
395
396
|
self._last_diag = {
|
|
396
397
|
"cv_importance": (P_i.std() / P_i.mean().clamp(min=1e-8)).detach(),
|
|
@@ -405,8 +406,8 @@ class SwitchMoE(nn.Module):
|
|
|
405
406
|
"topk_gate_mean": topk_prob.mean().detach(),
|
|
406
407
|
"input_rms": flat32.square().mean().sqrt().detach(),
|
|
407
408
|
"input_absmax": flat32.abs().amax().detach(),
|
|
408
|
-
"expert_output_rms":
|
|
409
|
-
"expert_output_absmax":
|
|
409
|
+
"expert_output_rms": accepted_output_sq_mean.sqrt().detach(),
|
|
410
|
+
"expert_output_absmax": accepted_output_absmax.detach(),
|
|
410
411
|
"routed_output_rms": output32.square().mean().sqrt().detach(),
|
|
411
412
|
"routed_output_absmax": output32.abs().amax().detach(),
|
|
412
413
|
"capacity_drop_frac": (1.0 - keep.float().mean()).detach(),
|
|
@@ -423,9 +424,10 @@ class SwitchMoE(nn.Module):
|
|
|
423
424
|
subset_weight = subset.to(probs.dtype)
|
|
424
425
|
count = subset_weight.sum()
|
|
425
426
|
denom = count.clamp_min(1.0)
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
427
|
+
selected_experts = topk_idx[subset].reshape(-1)
|
|
428
|
+
subset_load = torch.bincount(
|
|
429
|
+
selected_experts, minlength=self.num_experts
|
|
430
|
+
).to(probs.dtype) / (denom * top_k)
|
|
429
431
|
subset_importance = (probs * subset_weight[:, None]).sum(dim=0) / denom
|
|
430
432
|
entropy = (per_token_entropy * subset_weight).sum() / denom
|
|
431
433
|
max_prob = (per_token_max_prob * subset_weight).sum() / denom
|
|
@@ -451,10 +453,18 @@ class SwitchMoE(nn.Module):
|
|
|
451
453
|
self._diag_calls += 1
|
|
452
454
|
|
|
453
455
|
output = output.reshape(orig_shape)
|
|
454
|
-
if return_details:
|
|
456
|
+
if return_details or return_selected_details:
|
|
457
|
+
route_shape = (*orig_shape[:-1], top_k)
|
|
458
|
+
selected_shape = (*orig_shape[:-1], top_k, d_model)
|
|
459
|
+
if return_selected_details:
|
|
460
|
+
return (
|
|
461
|
+
output,
|
|
462
|
+
selected_outputs.to(output_dtype).reshape(selected_shape),
|
|
463
|
+
topk_idx.reshape(route_shape),
|
|
464
|
+
accepted_gates.reshape(route_shape),
|
|
465
|
+
)
|
|
455
466
|
assert expert_out_all is not None
|
|
456
467
|
expert_shape = (*orig_shape[:-1], self.num_experts, d_model)
|
|
457
|
-
route_shape = (*orig_shape[:-1], top_k)
|
|
458
468
|
return (
|
|
459
469
|
output,
|
|
460
470
|
expert_out_all.to(output_dtype).reshape(expert_shape),
|
|
@@ -577,13 +587,17 @@ class MoEBlock(nn.Module):
|
|
|
577
587
|
x: torch.Tensor,
|
|
578
588
|
*,
|
|
579
589
|
return_details: bool = False,
|
|
590
|
+
return_selected_details: bool = False,
|
|
580
591
|
token_mask: torch.Tensor | None = None,
|
|
581
592
|
):
|
|
582
|
-
if not return_details:
|
|
593
|
+
if not return_details and not return_selected_details:
|
|
583
594
|
routed = self.moe(self.norm(x), token_mask=token_mask)
|
|
584
595
|
return x + self.residual_scale * routed
|
|
585
596
|
routed, experts, indices, weights = self.moe(
|
|
586
|
-
self.norm(x),
|
|
597
|
+
self.norm(x),
|
|
598
|
+
return_details=return_details,
|
|
599
|
+
return_selected_details=return_selected_details,
|
|
600
|
+
token_mask=token_mask,
|
|
587
601
|
)
|
|
588
602
|
return x + self.residual_scale * routed, experts, indices, weights
|
|
589
603
|
|
|
@@ -21,6 +21,10 @@ __all__ = [
|
|
|
21
21
|
# DNC architectures
|
|
22
22
|
"SplitGraphDNC",
|
|
23
23
|
"MambaDNC",
|
|
24
|
+
# Sequence-parallel memory modules
|
|
25
|
+
"SequenceMemory",
|
|
26
|
+
"GatedDeltaMemory",
|
|
27
|
+
"CausalKVMemory",
|
|
24
28
|
# Controller mechanisms and compositions
|
|
25
29
|
"MambaControllerWrapper",
|
|
26
30
|
"Mamba2ControllerWrapper",
|
|
@@ -43,6 +47,9 @@ __all__ = [
|
|
|
43
47
|
_EXPORTS = {
|
|
44
48
|
"SplitGraphDNC": "initium.mamba_controller.split_graph_dnc",
|
|
45
49
|
"MambaDNC": "initium.mamba_controller.mamba_controller",
|
|
50
|
+
"SequenceMemory": "initium.memory_manipulation.sequence_memories",
|
|
51
|
+
"GatedDeltaMemory": "initium.memory_manipulation.sequence_memories",
|
|
52
|
+
"CausalKVMemory": "initium.memory_manipulation.sequence_memories",
|
|
46
53
|
"MambaControllerWrapper": "initium.mamba_controller.mamba_controller",
|
|
47
54
|
"Mamba2ControllerWrapper": "initium.mamba_controller.mamba2_controller",
|
|
48
55
|
"Mamba3ControllerWrapper": "initium.mamba_controller.mamba3_controller",
|
|
@@ -83,6 +90,13 @@ if TYPE_CHECKING:
|
|
|
83
90
|
from initium.mamba_controller.mamba_controller import MambaDNC as MambaDNC
|
|
84
91
|
from initium.mamba_controller.split_graph_dnc import SplitGraphDNC as SplitGraphDNC
|
|
85
92
|
from initium.memory_manipulation.dynamic_memory_resize import resize_memory as resize_memory
|
|
93
|
+
from initium.memory_manipulation.sequence_memories import (
|
|
94
|
+
CausalKVMemory as CausalKVMemory,
|
|
95
|
+
)
|
|
96
|
+
from initium.memory_manipulation.sequence_memories import (
|
|
97
|
+
GatedDeltaMemory as GatedDeltaMemory,
|
|
98
|
+
)
|
|
99
|
+
from initium.memory_manipulation.sequence_memories import SequenceMemory as SequenceMemory
|
|
86
100
|
from initium.memory_manipulation.stochastic_write_head_v2 import (
|
|
87
101
|
StochasticWriteHead as StochasticWriteHead,
|
|
88
102
|
)
|
{initium_deeplearning-0.4.0 → initium_deeplearning-0.5.1}/src/initium/config/train_config.py
RENAMED
|
@@ -12,7 +12,7 @@ import math
|
|
|
12
12
|
BATCH_SIZE = 16 # bucketed, so padding waste stays low even >1
|
|
13
13
|
TOTAL_STEPS = 120000
|
|
14
14
|
LOG_EVERY = 100
|
|
15
|
-
EVAL_EVERY =
|
|
15
|
+
EVAL_EVERY = 3000 # curriculum-advance eval cadence
|
|
16
16
|
LR = 3e-4
|
|
17
17
|
LR_MIN = 3e-5
|
|
18
18
|
LR_DECAY_STEPS = TOTAL_STEPS # cosine anneal length. Derived from TOTAL_STEPS here on
|
|
@@ -26,7 +26,7 @@ AMP_INIT_SCALE = 128.0 # GradScaler start scale (default 65536 wasted ~14k step
|
|
|
26
26
|
|
|
27
27
|
# ---- Model size -------------------------------------------------------------
|
|
28
28
|
MODEL_HIDDEN_SIZE = 512
|
|
29
|
-
MODEL_NR_CELLS = 256
|
|
29
|
+
MODEL_NR_CELLS = 256
|
|
30
30
|
MODEL_CELL_SIZE = 192
|
|
31
31
|
MODEL_READ_HEADS = 8
|
|
32
32
|
|