megatron-core 0.16.0rc0.dev108138__tar.gz → 0.16.0rc0.dev109400__tar.gz

{megatron_core-0.16.0rc0.dev108138 → megatron_core-0.16.0rc0.dev109400}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megatron-core
-Version: 0.16.0rc0.dev108138
+Version: 0.16.0rc0.dev109400
 Summary: Megatron Core - a library for efficient and scalable training of transformer based models
 Author-email: NVIDIA <nemo-toolkit@nvidia.com>
 Maintainer-email: NVIDIA <nemo-toolkit@nvidia.com>
@@ -41,7 +41,7 @@ Requires-Dist: transformers; extra == "mlm"
 Provides-Extra: dev
 Requires-Dist: nvidia-modelopt[torch]; sys_platform != "darwin" and extra == "dev"
 Requires-Dist: transformer-engine[pytorch]<2.10.0,>=2.9.0a0; extra == "dev"
-Requires-Dist: nvidia-resiliency-ext; extra == "dev"
+Requires-Dist: nvidia-resiliency-ext<0.5.0,>=0.4.0a0; extra == "dev"
 Requires-Dist: tqdm; extra == "dev"
 Requires-Dist: einops~=0.8; extra == "dev"
 Requires-Dist: tensorstore!=0.1.46,!=0.1.72,~=0.1; extra == "dev"
@@ -59,20 +59,13 @@ Requires-Dist: wget; extra == "dev"
 Requires-Dist: onnxscript; extra == "dev"
 Provides-Extra: lts
 Requires-Dist: tqdm; extra == "lts"
-Requires-Dist: einops~=0.8; extra == "lts"
-Requires-Dist: tensorstore!=0.1.46,!=0.1.72,~=0.1; extra == "lts"
-Requires-Dist: nvtx~=0.2; extra == "lts"
-Requires-Dist: multi-storage-client~=0.27; extra == "lts"
-Requires-Dist: opentelemetry-api~=1.33.1; extra == "lts"
+Requires-Dist: einops; extra == "lts"
+Requires-Dist: tensorstore!=0.1.46,!=0.1.72; extra == "lts"
+Requires-Dist: nvtx; extra == "lts"
+Requires-Dist: transformers; extra == "lts"
+Requires-Dist: zarr; extra == "lts"
 Requires-Dist: setuptools<80.0.0; extra == "lts"
-Requires-Dist: mamba-ssm~=2.2; extra == "lts"
-Requires-Dist: causal-conv1d~=1.5; extra == "lts"
-Requires-Dist: nv-grouped-gemm~=1.1; extra == "lts"
-Requires-Dist: megatron-energon[av_decode]~=6.0; extra == "lts"
-Requires-Dist: av<16.0.0; extra == "lts"
-Requires-Dist: flashinfer-python; extra == "lts"
 Requires-Dist: wget; extra == "lts"
-Requires-Dist: onnxscript; extra == "lts"
 
 <div align="center">
 

{megatron_core-0.16.0rc0.dev108138 → megatron_core-0.16.0rc0.dev109400}/megatron/core/datasets/megatron_tokenizer.py

@@ -1,14 +1,11 @@
 # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
 import json
-import logging
 from abc import ABC, abstractmethod
 from collections import OrderedDict
 from typing import Any
 
 import numpy
 
-logger = logging.getLogger(__name__)
-
 
 class MegatronLegacyTokenizer(ABC):
     """Abstract class for tokenizer
@@ -23,12 +20,6 @@ class MegatronLegacyTokenizer(ABC):
     """
 
     def __init__(self, *tokenizer_paths: str, **tokenizer_options: Any):
-        # Deprecation warning
-        logger.warning(
-            "You’re using the legacy tokenizer system, which is deprecated "
-            "and will be removed in a future release. Please migrate to the new tokenizer system "
-            "(`megatron.core.tokenizers.MegatronTokenizer`)."
-        )
         self.unique_identifiers = OrderedDict()
         self.unique_identifiers["class"] = type(self).__name__
         self.unique_identifiers["tokenizer_path"] = list(tokenizer_paths)

{megatron_core-0.16.0rc0.dev108138 → megatron_core-0.16.0rc0.dev109400}/megatron/core/fp8_utils.py

@@ -10,12 +10,6 @@ from typing import List, Optional
 import torch
 
 from megatron.core.enums import Fp8Recipe
-from megatron.core.tensor_parallel import (
-    ColumnParallelLinear,
-    RowParallelLinear,
-    gather_from_sequence_parallel_region,
-    reduce_scatter_to_sequence_parallel_region,
-)
 from megatron.core.transformer.transformer_config import TransformerConfig
 from megatron.core.utils import get_te_version, is_te_min_version
 
@@ -118,27 +112,6 @@ def get_fp8_align_size(fp8_recipe: Fp8Recipe) -> int:
         return 16
 
 
-def is_column_parallel_linear(module):
-    """Returns whether the given module is a ColumnParallelLinear layer."""
-    if HAVE_TE and (
-        isinstance(module, TEColumnParallelLinear)
-        or isinstance(module, TELayerNormColumnParallelLinear)
-    ):
-        return True
-    elif isinstance(module, ColumnParallelLinear):
-        return True
-    return False
-
-
-def is_row_parallel_linear(module):
-    """Returns whether the given module is a RowParallelLinear layer."""
-    if HAVE_TE and isinstance(module, TERowParallelLinear):
-        return True
-    elif isinstance(module, RowParallelLinear):
-        return True
-    return False
-
-
 """
 The code below abstracts the functionalities needed for implementing "--fp8-param-gather" into
 several functions. It provides different implementations for each function based on different
@@ -614,18 +587,6 @@ if HAVE_TE:
             if not FP8GlobalStateManager.is_fp8_enabled():
                 return original_forward(input_tensor, *args, **kwargs)
 
-            # With sequence parallelism we need to all-gather before padding
-            # and reduce-scatter after unpadding
-            if is_sequence_parallel := getattr(module, "sequence_parallel", False):
-                if is_column_parallel_linear(module):
-                    input_tensor = gather_from_sequence_parallel_region(
-                        input_tensor, group=module.tp_group
-                    )
-
-                # Disable sequence parallelism on the module because we are handling the
-                # all-gather and reduce-scatter externally
-                module.sequence_parallel = False
-
             seq_len, batch_size, hidden_size = input_tensor.shape
             # Reshape to (S, B*H) to pad sequence dimension
             input_2d = input_tensor.reshape(seq_len, -1)
@@ -651,16 +612,6 @@ if HAVE_TE:
             unpadded_output_2d = _unpad_func(output_2d, [seq_len])
             unpadded_output = unpadded_output_2d.reshape(seq_len, batch_size, output_hidden_size)
 
-            if is_sequence_parallel:
-                # Reduce-scatter after unpadding
-                if is_row_parallel_linear(module):
-                    unpadded_output = reduce_scatter_to_sequence_parallel_region(
-                        unpadded_output, group=module.tp_group
-                    )
-
-                # Reset sequence parallelism flag on the module
-                module.sequence_parallel = True
-
             if other_outputs:
                 return (unpadded_output,) + other_outputs
             else:

{megatron_core-0.16.0rc0.dev108138 → megatron_core-0.16.0rc0.dev109400}/megatron/core/package_info.py

@@ -4,7 +4,7 @@
 MAJOR = 0
 MINOR = 16
 PATCH = 0
-PRE_RELEASE = 'rc0.dev108138'
+PRE_RELEASE = 'rc0.dev109400'
 
 # Use the following formatting: (major, minor, patch, pre-release)
 VERSION = (MAJOR, MINOR, PATCH, PRE_RELEASE)

{megatron_core-0.16.0rc0.dev108138 → megatron_core-0.16.0rc0.dev109400}/megatron_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megatron-core
-Version: 0.16.0rc0.dev108138
+Version: 0.16.0rc0.dev109400
 Summary: Megatron Core - a library for efficient and scalable training of transformer based models
 Author-email: NVIDIA <nemo-toolkit@nvidia.com>
 Maintainer-email: NVIDIA <nemo-toolkit@nvidia.com>
@@ -41,7 +41,7 @@ Requires-Dist: transformers; extra == "mlm"
 Provides-Extra: dev
 Requires-Dist: nvidia-modelopt[torch]; sys_platform != "darwin" and extra == "dev"
 Requires-Dist: transformer-engine[pytorch]<2.10.0,>=2.9.0a0; extra == "dev"
-Requires-Dist: nvidia-resiliency-ext; extra == "dev"
+Requires-Dist: nvidia-resiliency-ext<0.5.0,>=0.4.0a0; extra == "dev"
 Requires-Dist: tqdm; extra == "dev"
 Requires-Dist: einops~=0.8; extra == "dev"
 Requires-Dist: tensorstore!=0.1.46,!=0.1.72,~=0.1; extra == "dev"
@@ -59,20 +59,13 @@ Requires-Dist: wget; extra == "dev"
 Requires-Dist: onnxscript; extra == "dev"
 Provides-Extra: lts
 Requires-Dist: tqdm; extra == "lts"
-Requires-Dist: einops~=0.8; extra == "lts"
-Requires-Dist: tensorstore!=0.1.46,!=0.1.72,~=0.1; extra == "lts"
-Requires-Dist: nvtx~=0.2; extra == "lts"
-Requires-Dist: multi-storage-client~=0.27; extra == "lts"
-Requires-Dist: opentelemetry-api~=1.33.1; extra == "lts"
+Requires-Dist: einops; extra == "lts"
+Requires-Dist: tensorstore!=0.1.46,!=0.1.72; extra == "lts"
+Requires-Dist: nvtx; extra == "lts"
+Requires-Dist: transformers; extra == "lts"
+Requires-Dist: zarr; extra == "lts"
 Requires-Dist: setuptools<80.0.0; extra == "lts"
-Requires-Dist: mamba-ssm~=2.2; extra == "lts"
-Requires-Dist: causal-conv1d~=1.5; extra == "lts"
-Requires-Dist: nv-grouped-gemm~=1.1; extra == "lts"
-Requires-Dist: megatron-energon[av_decode]~=6.0; extra == "lts"
-Requires-Dist: av<16.0.0; extra == "lts"
-Requires-Dist: flashinfer-python; extra == "lts"
 Requires-Dist: wget; extra == "lts"
-Requires-Dist: onnxscript; extra == "lts"
 
 <div align="center">
 

{megatron_core-0.16.0rc0.dev108138 → megatron_core-0.16.0rc0.dev109400}/megatron_core.egg-info/requires.txt

@@ -4,7 +4,7 @@ packaging>=24.2
 
 [dev]
 transformer-engine[pytorch]<2.10.0,>=2.9.0a0
-nvidia-resiliency-ext
+nvidia-resiliency-ext<0.5.0,>=0.4.0a0
 tqdm
 einops~=0.8
 tensorstore!=0.1.46,!=0.1.72,~=0.1
@@ -26,20 +26,13 @@ nvidia-modelopt[torch]
 
 [lts]
 tqdm
-einops~=0.8
-tensorstore!=0.1.46,!=0.1.72,~=0.1
-nvtx~=0.2
-multi-storage-client~=0.27
-opentelemetry-api~=1.33.1
+einops
+tensorstore!=0.1.46,!=0.1.72
+nvtx
+transformers
+zarr
 setuptools<80.0.0
-mamba-ssm~=2.2
-causal-conv1d~=1.5
-nv-grouped-gemm~=1.1
-megatron-energon[av_decode]~=6.0
-av<16.0.0
-flashinfer-python
 wget
-onnxscript
 
 [mlm]
 flask-restful

{megatron_core-0.16.0rc0.dev108138 → megatron_core-0.16.0rc0.dev109400}/pyproject.toml

@@ -69,7 +69,7 @@ mlm = ["flask-restful", "sentencepiece", "tiktoken", "wandb", "transformers"]
 dev = [
     "nvidia-modelopt[torch]; sys_platform != 'darwin'",
     "transformer-engine[pytorch]>=2.9.0a0,<2.10.0",
-    "nvidia-resiliency-ext",
+    "nvidia-resiliency-ext>=0.4.0a0,<0.5.0",
     "tqdm",
     "einops~=0.8",
     "tensorstore~=0.1,!=0.1.46,!=0.1.72",
@@ -89,20 +89,13 @@ dev = [
 
 lts = [
     "tqdm",
-    "einops~=0.8",
-    "tensorstore~=0.1,!=0.1.46,!=0.1.72",
-    "nvtx~=0.2",
-    "multi-storage-client~=0.27",
-    "opentelemetry-api~=1.33.1",
+    "einops",
+    "tensorstore!=0.1.46,!=0.1.72",
+    "nvtx",
+    "transformers",
+    "zarr",
     "setuptools<80.0.0",
-    "mamba-ssm~=2.2",
-    "causal-conv1d~=1.5",
-    "nv-grouped-gemm~=1.1",
-    "megatron-energon[av_decode]~=6.0",
-    "av<16.0.0",                          # At the time, av 16.0.0 is not compatible with Python 3.12
-    "flashinfer-python",
     "wget",
-    "onnxscript",
 ]
 
 [dependency-groups]