presight-fairseq2 0.6.1__py3-none-any.whl

fairseq2/__init__.py

@@ -0,0 +1,50 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import fairseq2n  # Report any fairseq2n initialization error eagerly.
+
+import fairseq2.runtime.dependency
+from fairseq2.error import InvalidOperationError
+from fairseq2.runtime.dependency import DependencyContainer, DependencyResolver
+
+__version__ = "0.6.1"
+
+
+_in_call: bool = False
+
+
+def init_fairseq2(
+    *, extras: Callable[[DependencyContainer], None] | None = None
+) -> DependencyResolver:
+    from fairseq2.composition import _register_library
+
+    global _in_call
+
+    if fairseq2.runtime.dependency._resolver is not None:
+        raise InvalidOperationError("`init_fairseq2()` is already called.")
+
+    if _in_call:
+        raise InvalidOperationError("`init_fairseq2()` cannot be called recursively.")
+
+    _in_call = True
+
+    container = DependencyContainer()
+
+    try:
+        _register_library(container)
+
+        if extras is not None:
+            extras(container)
+    finally:
+        _in_call = False
+
+    fairseq2.runtime.dependency._resolver = container
+
+    return container

fairseq2/assets/__init__.py

@@ -0,0 +1,90 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import annotations
+
+from fairseq2.assets.card import AssetCard as AssetCard
+from fairseq2.assets.card import AssetCardError as AssetCardError
+from fairseq2.assets.card import AssetConfigLoader as AssetConfigLoader
+from fairseq2.assets.card import StandardAssetConfigLoader as StandardAssetConfigLoader
+from fairseq2.assets.dirs import AssetDirectoryAccessor as AssetDirectoryAccessor
+from fairseq2.assets.dirs import (
+    StandardAssetDirectoryAccessor as StandardAssetDirectoryAccessor,
+)
+from fairseq2.assets.download_manager import AssetDownloadError as AssetDownloadError
+from fairseq2.assets.download_manager import (
+    AssetDownloadManager as AssetDownloadManager,
+)
+from fairseq2.assets.download_manager import (
+    DelegatingAssetDownloadManager as DelegatingAssetDownloadManager,
+)
+from fairseq2.assets.download_manager import HuggingFaceHub as HuggingFaceHub
+from fairseq2.assets.download_manager import (
+    LocalAssetDownloadManager as LocalAssetDownloadManager,
+)
+from fairseq2.assets.download_manager import (
+    StandardAssetDownloadManager as StandardAssetDownloadManager,
+)
+from fairseq2.assets.metadata_provider import AssetMetadataError as AssetMetadataError
+from fairseq2.assets.metadata_provider import (
+    AssetMetadataFileLoader as AssetMetadataFileLoader,
+)
+from fairseq2.assets.metadata_provider import (
+    AssetMetadataProvider as AssetMetadataProvider,
+)
+from fairseq2.assets.metadata_provider import AssetMetadataSource as AssetMetadataSource
+from fairseq2.assets.metadata_provider import (
+    AssetSourceNotFoundError as AssetSourceNotFoundError,
+)
+from fairseq2.assets.metadata_provider import (
+    CachedAssetMetadataProvider as CachedAssetMetadataProvider,
+)
+from fairseq2.assets.metadata_provider import (
+    FileAssetMetadataLoader as FileAssetMetadataLoader,
+)
+from fairseq2.assets.metadata_provider import (
+    FileAssetMetadataSource as FileAssetMetadataSource,
+)
+from fairseq2.assets.metadata_provider import (
+    InMemoryAssetMetadataSource as InMemoryAssetMetadataSource,
+)
+from fairseq2.assets.metadata_provider import (
+    PackageAssetMetadataLoader as PackageAssetMetadataLoader,
+)
+from fairseq2.assets.metadata_provider import (
+    PackageAssetMetadataSource as PackageAssetMetadataSource,
+)
+from fairseq2.assets.metadata_provider import PackageFileLister as PackageFileLister
+from fairseq2.assets.metadata_provider import (
+    StandardFileAssetMetadataLoader as StandardFileAssetMetadataLoader,
+)
+from fairseq2.assets.metadata_provider import (
+    StandardPackageAssetMetadataLoader as StandardPackageAssetMetadataLoader,
+)
+from fairseq2.assets.metadata_provider import (
+    StandardPackageFileLister as StandardPackageFileLister,
+)
+from fairseq2.assets.metadata_provider import (
+    WellKnownAssetMetadataSource as WellKnownAssetMetadataSource,
+)
+from fairseq2.assets.metadata_provider import (
+    YamlAssetMetadataFileLoader as YamlAssetMetadataFileLoader,
+)
+from fairseq2.assets.metadata_provider import (
+    canonicalize_asset_name as canonicalize_asset_name,
+)
+from fairseq2.assets.metadata_provider import (
+    load_in_memory_asset_metadata as load_in_memory_asset_metadata,
+)
+from fairseq2.assets.metadata_provider import (
+    sanitize_base_asset_name as sanitize_base_asset_name,
+)
+from fairseq2.assets.store import AssetEnvironmentDetector as AssetEnvironmentDetector
+from fairseq2.assets.store import AssetEnvironmentResolver as AssetEnvironmentResolver
+from fairseq2.assets.store import AssetNotFoundError as AssetNotFoundError
+from fairseq2.assets.store import AssetStore as AssetStore
+from fairseq2.assets.store import StandardAssetStore as StandardAssetStore
+from fairseq2.assets.store import get_asset_store as get_asset_store

fairseq2/assets/__main__.py

@@ -0,0 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import annotations
+
+from fairseq2.assets.cli import _main
+
+_main()

fairseq2/assets/card.py

@@ -0,0 +1,205 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from pathlib import Path
+from typing import TypeVar, final
+
+from typing_extensions import override
+
+from fairseq2.utils.config import ConfigDirectiveError, ConfigMerger, ConfigProcessor
+from fairseq2.utils.structured import StructureError, ValueConverter
+from fairseq2.utils.uri import Uri
+
+T = TypeVar("T", bool, int, float, str)
+
+
+@final
+class AssetCard:
+    """Holds information about an asset."""
+
+    def __init__(
+        self, name: str, metadata: Mapping[str, object], base: AssetCard | None = None
+    ) -> None:
+        self._name = name
+        self._metadata = metadata
+        self._base = base
+
+    def field(self, name: str) -> AssetCardField:
+        field = self.maybe_get_field(name)
+        if field is None:
+            msg = f"{self._name} asset card does not have a field named {name}."
+
+            raise AssetCardError(self._name, msg)
+
+        return field
+
+    def maybe_get_field(self, name: str) -> AssetCardField | None:
+        card: AssetCard | None = self
+
+        while card is not None:
+            try:
+                value = card.metadata[name]
+            except KeyError:
+                pass
+            else:
+                return AssetCardField(name, self, value)
+
+            card = card.base
+
+        return None
+
+    def has_field(self, name: str) -> bool:
+        card: AssetCard | None = self
+
+        while card is not None:
+            if name in card.metadata:
+                return True
+
+            card = card.base
+
+        return False
+
+    def __repr__(self) -> str:
+        return f"{self._name}={self._metadata}"
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def metadata(self) -> Mapping[str, object]:
+        return self._metadata
+
+    @property
+    def base(self) -> AssetCard | None:
+        return self._base
+
+
+@final
+class AssetCardField:
+    def __init__(self, name: str, card: AssetCard, value: object) -> None:
+        self._name = name
+        self._card = card
+        self._value = value
+
+    @property
+    def value(self) -> object:
+        return self._value
+
+    def as_(self, kls: type[T]) -> T:
+        if not isinstance(self._value, kls):
+            msg = f"{self._name} field of the {self._card.name} asset card is expected to be of type `{kls}`, but is of type `{type(self._value)}` instead."
+
+            raise AssetCardError(self._card.name, msg)
+
+        return self._value
+
+    def as_uri(self) -> Uri:
+        value = self.as_(str)
+
+        uri = Uri.maybe_parse(value)
+        if uri is not None:
+            return uri
+
+        try:
+            path = Path(value)
+        except ValueError:
+            msg = f"{self._name} field of the {self._card.name} asset card cannot be parsed as a URI or a pathname."
+
+            raise AssetCardError(self._card.name, msg) from None
+
+        if not path.is_absolute():
+            base_path = self._card.metadata.get("__base_path__")
+            if not isinstance(base_path, Path):
+                msg = f"{self._name} field of the {self._card.name} asset card is a relative pathname and cannot be converted to a URI."
+
+                raise AssetCardError(self._card.name, msg)
+
+            path = base_path.joinpath(path)
+
+        return Uri.from_path(path)
+
+
+class AssetCardError(Exception):
+    def __init__(self, name: str, message: str) -> None:
+        super().__init__(message)
+
+        self.name = name
+
+
+class AssetConfigLoader(ABC):
+    @abstractmethod
+    def load(self, card: AssetCard, base_config: object, config_key: str) -> object: ...
+
+
+@final
+class StandardAssetConfigLoader(AssetConfigLoader):
+    def __init__(
+        self,
+        value_converter: ValueConverter,
+        config_merger: ConfigMerger,
+        config_processor: ConfigProcessor,
+    ) -> None:
+        self._value_converter = value_converter
+        self._config_merger = config_merger
+        self._config_processor = config_processor
+
+    @override
+    def load(self, card: AssetCard, base_config: object, config_key: str) -> object:
+        all_config_overrides = []
+
+        base_card: AssetCard | None = card
+
+        while base_card is not None:
+            config_overrides = base_card.metadata.get(config_key)
+            if config_overrides is not None:
+                all_config_overrides.append((base_card.name, config_overrides))
+
+            base_card = base_card.base
+
+        if not all_config_overrides:
+            return base_config
+
+        config_kls = type(base_config)
+
+        try:
+            unstructured_config = self._value_converter.unstructure(base_config)
+        except StructureError:
+            msg = f"{config_key} field of the {card.name} asset card cannot be parsed as of type `{config_kls}`."
+
+            raise AssetCardError(card.name, msg) from None
+
+        for name, config_overrides in reversed(all_config_overrides):
+            # TODO(balioglu): unescape _set_ and _del_ in config_overrides
+            try:
+                unstructured_config = self._config_merger.merge(
+                    unstructured_config, config_overrides
+                )
+            except (ValueError, TypeError) as ex:
+                msg = f"{config_key} field of the {name} asset card cannot be merged with the base configuration."
+
+                raise AssetCardError(name, msg) from ex
+
+            # TODO(balioglu): unescape config directives and run them.
+            try:
+                unstructured_config = self._config_processor.process(
+                    unstructured_config
+                )
+            except ConfigDirectiveError as ex:
+                msg = f"A directive in the {config_key} field of the {name} asset card cannot processed."
+
+                raise AssetCardError(name, msg) from ex
+
+        try:
+            return self._value_converter.structure(unstructured_config, config_kls)
+        except StructureError as ex:
+            msg = f"{config_key} field of the {card.name} asset card cannot be parsed as of type `{config_kls}`."
+
+            raise AssetCardError(card.name, msg) from ex

fairseq2/assets/cards/datasets/librilight.yaml

@@ -0,0 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+name: librilight_asr_10h
+dataset_family: wav2vec2_asr

fairseq2/assets/cards/datasets/librispeech.yaml

@@ -0,0 +1,20 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+name: librispeech_asr
+dataset_family: generic_asr
+tokenizer: "https://dl.fbaipublicfiles.com/fairseq/wav2vec/librispeech_asr.model"
+tokenizer_family: char_tokenizer
+
+---
+
+name: librispeech_asr_100h
+base: wav2vec2_asr
+
+---
+
+name: librispeech_960h
+dataset_family: wav2vec2_ssl

fairseq2/assets/cards/datasets/openeft.yaml

@@ -0,0 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+name: openeft
+dataset_family: generic_instruction

fairseq2/assets/cards/models/jepa.yaml

@@ -0,0 +1,39 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+name: jepa_vitl16
+model_family: jepa
+model_arch: large
+model_config_override:
+  encoder_config:
+    input_dims: [16, 224, 224]
+    patch_dims: [2, 16, 16]
+    uniform_power: true
+checkpoint: "https://dl.fbaipublicfiles.com/jepa/vitl16/vitl16.pth.tar"
+
+---
+
+name: jepa_vith16
+model_family: jepa
+model_arch: huge
+model_config_override:
+  encoder_config:
+    input_dims: [16, 224, 224]
+    patch_dims: [2, 16, 16]
+    uniform_power: true
+checkpoint: "https://dl.fbaipublicfiles.com/jepa/vith16/vith16.pth.tar"
+
+---
+
+name: jepa_vith16_384
+model_family: jepa
+model_arch: huge
+model_config_override:
+  encoder_config:
+    input_dims: [16, 384, 384]
+    patch_dims: [2, 16, 16]
+    uniform_power: true
+checkpoint: "https://dl.fbaipublicfiles.com/jepa/vith16-384/vith16-384.pth.tar"

fairseq2/assets/cards/models/llama.yaml

@@ -0,0 +1,210 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+name: llama
+model_family: llama
+checkpoint: "https://ai.meta.com/llama/;gated=true"
+tokenizer: "https://ai.meta.com/llama/;gated=true"
+tokenizer_family: llama
+
+---
+
+name: llama_7b
+base: llama
+model_arch: 7b
+
+---
+
+name: llama2
+model_family: llama
+checkpoint: "https://ai.meta.com/llama/;gated=true"
+tokenizer: "https://ai.meta.com/llama/;gated=true"
+tokenizer_family: llama
+
+---
+
+name: llama2_7b
+base: llama2
+model_arch: llama2_7b
+
+---
+
+name: llama2_7b_chat
+base: llama2
+model_arch: llama2_7b
+
+---
+
+name: llama2_13b
+base: llama2
+model_arch: llama2_13b
+
+---
+
+name: llama2_13b_chat
+base: llama2
+model_arch: llama2_13b
+
+---
+
+name: llama2_70b
+base: llama2
+model_arch: llama2_70b
+
+# legacy
+num_shards: 8
+
+---
+
+name: llama2_70b_chat
+base: llama2
+model_arch: llama2_70b
+
+# legacy
+num_shards: 8
+
+---
+
+name: llama3
+model_family: llama
+checkpoint: "https://ai.meta.com/llama/;gated=true"
+tokenizer: "https://ai.meta.com/llama/;gated=true"
+tokenizer_family: llama
+tokenizer_config_override:
+  impl: hg
+
+# legacy
+use_v2_tokenizer: true
+
+---
+
+name: llama3_instruct
+base: llama3
+tokenizer_config_override:
+  use_eot: true  # instruct tokenizer to use EOT instead of EOS
+
+---
+
+name: llama3_8b
+base: llama3
+model_arch: llama3_8b
+
+---
+
+name: llama3_8b_instruct
+base: llama3_instruct
+model_arch: llama3_8b
+
+---
+
+name: llama3_70b
+base: llama3
+model_arch: llama3_70b
+
+# legacy
+num_shards: 8
+
+---
+
+name: llama3_70b_instruct
+base: llama3_instruct
+model_arch: llama3_70b
+
+# legacy
+num_shards: 8
+
+---
+
+name: llama3_1_8b
+base: llama3
+model_arch: llama3_1_8b
+
+---
+
+name: llama3_1_8b_instruct
+base: llama3_instruct
+model_arch: llama3_1_8b
+
+---
+
+name: llama3_1_70b
+base: llama3
+model_arch: llama3_1_70b
+
+# legacy
+num_shards: 8
+
+---
+
+name: llama3_1_70b_instruct
+base: llama3_instruct
+model_arch: llama3_1_70b
+
+# legacy
+num_shards: 8
+
+---
+
+name: llama3_2_1b
+base: llama3
+model_arch: llama3_2_1b
+
+---
+
+name: llama3_2_1b_instruct
+base: llama3_instruct
+model_arch: llama3_2_1b
+
+---
+
+name: llama3_2_3b
+base: llama3
+model_arch: llama3_2_3b
+
+---
+
+name: llama3_2_3b_instruct
+base: llama3_instruct
+model_arch: llama3_2_3b
+
+---
+
+name: llama3_3_70b_instruct
+base: llama3_instruct
+model_arch: llama3_1_70b
+
+# legacy
+num_shards: 8
+
+---
+
+name: r1_distilled_llama3_1_8b_instruct
+model_family: llama
+model_arch: llama3_1_8b
+checkpoint: "hg://deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+tokenizer: "hg://deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
+tokenizer_family: llama
+
+---
+
+name: llama4
+model_family: llama4
+checkpoint: "https://ai.meta.com/llama/;gated=true"
+tokenizer: "https://ai.meta.com/llama/;gated=true"
+tokenizer_family: llama4
+
+---
+
+name: llama4_scout_17b_16e
+base: llama4
+model_arch: llama4_scout_17b_16e
+
+---
+
+name: llama4_scout_17b_16e_instruct
+base: llama4_scout_17b_16e
+tokenizer_config_override:
+  use_eot: true  # instruct tokenizer to use EOT instead of EOS

fairseq2/assets/cards/models/mistral.yaml

@@ -0,0 +1,21 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+name: mistral_7b
+model_family: mistral
+model_arch: 7b
+checkpoint: "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar;path=mistral-7B-v0.1%2Fconsolidated.00.pth"
+tokenizer: "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-v0.1.tar;path=mistral-7B-v0.1%2Ftokenizer.model"
+tokenizer_family: mistral
+
+---
+
+name: mistral_7b_instruct
+model_family: mistral
+model_arch: 7b
+checkpoint: "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-instruct-v0.1b.tar;path=Mistral-7B-instruct-v0.1%2Fconsolidated.00.pth"
+tokenizer: "https://files.mistral-7b-v0-1.mistral.ai/mistral-7B-instruct-v0.1b.tar;path=Mistral-7B-instruct-v0.1%2Ftokenizer.model"
+tokenizer_family: mistral