ctranslate2 4.8.4__cp315-cp315-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ctranslate2/__init__.py +97 -0
- ctranslate2/_ext.cp315-win_amd64.pyd +0 -0
- ctranslate2/converters/__init__.py +8 -0
- ctranslate2/converters/converter.py +109 -0
- ctranslate2/converters/drama3_codec.py +158 -0
- ctranslate2/converters/drama3_weights.py +160 -0
- ctranslate2/converters/eole_ct2.py +353 -0
- ctranslate2/converters/fairseq.py +362 -0
- ctranslate2/converters/marian.py +315 -0
- ctranslate2/converters/openai_gpt2.py +95 -0
- ctranslate2/converters/opennmt_py.py +376 -0
- ctranslate2/converters/opennmt_tf.py +455 -0
- ctranslate2/converters/opus_mt.py +44 -0
- ctranslate2/converters/transformers.py +4907 -0
- ctranslate2/converters/utils.py +127 -0
- ctranslate2/ctranslate2.dll +4 -0
- ctranslate2/cudnn64_9.dll +0 -0
- ctranslate2/extensions.py +589 -0
- ctranslate2/libiomp5md.dll +0 -0
- ctranslate2/logging.py +45 -0
- ctranslate2/models/__init__.py +22 -0
- ctranslate2/specs/__init__.py +19 -0
- ctranslate2/specs/attention_spec.py +107 -0
- ctranslate2/specs/common_spec.py +66 -0
- ctranslate2/specs/drama3_spec.py +155 -0
- ctranslate2/specs/model_spec.py +771 -0
- ctranslate2/specs/transformer_spec.py +812 -0
- ctranslate2/specs/wav2vec2_spec.py +72 -0
- ctranslate2/specs/wav2vec2bert_spec.py +97 -0
- ctranslate2/specs/whisper_spec.py +77 -0
- ctranslate2/version.py +3 -0
- ctranslate2-4.8.4.dist-info/METADATA +187 -0
- ctranslate2-4.8.4.dist-info/RECORD +36 -0
- ctranslate2-4.8.4.dist-info/WHEEL +5 -0
- ctranslate2-4.8.4.dist-info/entry_points.txt +8 -0
- ctranslate2-4.8.4.dist-info/top_level.txt +1 -0
ctranslate2/__init__.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
if sys.platform == "win32":
|
|
4
|
+
import ctypes
|
|
5
|
+
import glob
|
|
6
|
+
import os
|
|
7
|
+
|
|
8
|
+
from importlib.resources import files
|
|
9
|
+
|
|
10
|
+
module_name = sys.modules[__name__].__name__
|
|
11
|
+
package_dir = str(files(module_name))
|
|
12
|
+
|
|
13
|
+
# The libraries package name differs across ROCm releases.
|
|
14
|
+
site_dir = os.path.dirname(package_dir)
|
|
15
|
+
dll_dirs = [os.path.join(site_dir, "_rocm_sdk_core", "bin")]
|
|
16
|
+
dll_dirs += sorted(glob.glob(os.path.join(site_dir, "_rocm_sdk_libraries*", "bin")))
|
|
17
|
+
dll_dirs = [path for path in dll_dirs if os.path.isdir(path)]
|
|
18
|
+
# Fallback only: the search order between DLL directories is unspecified.
|
|
19
|
+
if not dll_dirs and os.environ.get("HIP_PATH"):
|
|
20
|
+
dll_dirs = [os.path.join(os.environ["HIP_PATH"], "bin")]
|
|
21
|
+
|
|
22
|
+
for path in [package_dir] + dll_dirs:
|
|
23
|
+
try:
|
|
24
|
+
os.add_dll_directory(path)
|
|
25
|
+
except OSError:
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
for library in glob.glob(os.path.join(package_dir, "*.dll")):
|
|
29
|
+
ctypes.CDLL(library)
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
from ctranslate2._ext import (
|
|
33
|
+
AsyncGenerationResult,
|
|
34
|
+
AsyncScoringResult,
|
|
35
|
+
AsyncTranslationResult,
|
|
36
|
+
DataType,
|
|
37
|
+
Device,
|
|
38
|
+
Encoder,
|
|
39
|
+
EncoderForwardOutput,
|
|
40
|
+
ExecutionStats,
|
|
41
|
+
GenerationResult,
|
|
42
|
+
GenerationStepResult,
|
|
43
|
+
Generator,
|
|
44
|
+
MpiInfo,
|
|
45
|
+
ScoringResult,
|
|
46
|
+
StorageView,
|
|
47
|
+
TranslationResult,
|
|
48
|
+
Translator,
|
|
49
|
+
contains_model,
|
|
50
|
+
get_cuda_device_count,
|
|
51
|
+
get_supported_compute_types,
|
|
52
|
+
set_random_seed,
|
|
53
|
+
)
|
|
54
|
+
from ctranslate2.extensions import register_extensions
|
|
55
|
+
from ctranslate2.logging import get_log_level, set_log_level
|
|
56
|
+
|
|
57
|
+
register_extensions()
|
|
58
|
+
del register_extensions
|
|
59
|
+
except ImportError as e:
|
|
60
|
+
# Allow using the Python package without the compiled extension.
|
|
61
|
+
if "No module named" in str(e):
|
|
62
|
+
pass
|
|
63
|
+
else:
|
|
64
|
+
raise
|
|
65
|
+
|
|
66
|
+
from ctranslate2 import models
|
|
67
|
+
from ctranslate2.version import __version__
|
|
68
|
+
|
|
69
|
+
# converters and specs import torch (and, for converters, transformers) at module level.
|
|
70
|
+
# Those dependencies are only needed to convert models, not to run inference, so import
|
|
71
|
+
# these submodules on first use to keep "import ctranslate2" free of them.
|
|
72
|
+
_LAZY_SUBMODULES = ("converters", "specs")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def __getattr__(name):
|
|
76
|
+
if name in _LAZY_SUBMODULES:
|
|
77
|
+
import importlib
|
|
78
|
+
|
|
79
|
+
module = importlib.import_module(f"{__name__}.{name}")
|
|
80
|
+
globals()[name] = module
|
|
81
|
+
return module
|
|
82
|
+
|
|
83
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def __dir__():
|
|
87
|
+
return sorted(set(globals()) | set(_LAZY_SUBMODULES))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# A wildcard import resolves ``__all__`` when it is defined and the module globals
|
|
91
|
+
# otherwise, so without this the lazy submodules would silently drop out of
|
|
92
|
+
# ``from ctranslate2 import *``. Deriving the list keeps the wildcard surface identical
|
|
93
|
+
# to what it was before they became lazy; a wildcard import asks for everything, so
|
|
94
|
+
# resolving them here is expected.
|
|
95
|
+
__all__ = sorted(
|
|
96
|
+
[name for name in globals() if not name.startswith("_")] + list(_LAZY_SUBMODULES)
|
|
97
|
+
)
|
|
Binary file
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
from ctranslate2.converters.converter import Converter
|
|
2
|
+
from ctranslate2.converters.fairseq import FairseqConverter
|
|
3
|
+
from ctranslate2.converters.marian import MarianConverter
|
|
4
|
+
from ctranslate2.converters.openai_gpt2 import OpenAIGPT2Converter
|
|
5
|
+
from ctranslate2.converters.opennmt_py import OpenNMTPyConverter
|
|
6
|
+
from ctranslate2.converters.opennmt_tf import OpenNMTTFConverter
|
|
7
|
+
from ctranslate2.converters.opus_mt import OpusMTConverter
|
|
8
|
+
from ctranslate2.converters.transformers import TransformersConverter
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
import argparse
|
|
3
|
+
import os
|
|
4
|
+
import shutil
|
|
5
|
+
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
from ctranslate2.specs.model_spec import ACCEPTED_MODEL_TYPES, ModelSpec
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Converter(abc.ABC):
|
|
12
|
+
"""Base class for model converters."""
|
|
13
|
+
|
|
14
|
+
@staticmethod
|
|
15
|
+
def declare_arguments(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
|
|
16
|
+
"""Adds common conversion options to the command line parser.
|
|
17
|
+
|
|
18
|
+
Arguments:
|
|
19
|
+
parser: Command line argument parser.
|
|
20
|
+
"""
|
|
21
|
+
parser.add_argument(
|
|
22
|
+
"--output_dir", required=True, help="Output model directory."
|
|
23
|
+
)
|
|
24
|
+
parser.add_argument(
|
|
25
|
+
"--vocab_mapping", default=None, help="Vocabulary mapping file (optional)."
|
|
26
|
+
)
|
|
27
|
+
parser.add_argument(
|
|
28
|
+
"--quantization",
|
|
29
|
+
default=None,
|
|
30
|
+
choices=ACCEPTED_MODEL_TYPES,
|
|
31
|
+
help="Weight quantization type.",
|
|
32
|
+
)
|
|
33
|
+
parser.add_argument(
|
|
34
|
+
"--force",
|
|
35
|
+
action="store_true",
|
|
36
|
+
help="Force conversion even if the output directory already exists.",
|
|
37
|
+
)
|
|
38
|
+
return parser
|
|
39
|
+
|
|
40
|
+
def convert_from_args(self, args: argparse.Namespace) -> str:
|
|
41
|
+
"""Helper function to call :meth:`ctranslate2.converters.Converter.convert`
|
|
42
|
+
with the parsed command line options.
|
|
43
|
+
|
|
44
|
+
Arguments:
|
|
45
|
+
args: Namespace containing parsed arguments.
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
Path to the output directory.
|
|
49
|
+
"""
|
|
50
|
+
return self.convert(
|
|
51
|
+
args.output_dir,
|
|
52
|
+
vmap=args.vocab_mapping,
|
|
53
|
+
quantization=args.quantization,
|
|
54
|
+
force=args.force,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
def convert(
|
|
58
|
+
self,
|
|
59
|
+
output_dir: str,
|
|
60
|
+
vmap: Optional[str] = None,
|
|
61
|
+
quantization: Optional[str] = None,
|
|
62
|
+
force: bool = False,
|
|
63
|
+
) -> str:
|
|
64
|
+
"""Converts the model to the CTranslate2 format.
|
|
65
|
+
|
|
66
|
+
Arguments:
|
|
67
|
+
output_dir: Output directory where the CTranslate2 model is saved.
|
|
68
|
+
vmap: Optional path to a vocabulary mapping file that will be included
|
|
69
|
+
in the converted model directory.
|
|
70
|
+
quantization: Weight quantization scheme (possible values are: int8, int8_float32,
|
|
71
|
+
int8_float16, int8_bfloat16, int16, float16, bfloat16, float32).
|
|
72
|
+
force: Override the output directory if it already exists.
|
|
73
|
+
|
|
74
|
+
Returns:
|
|
75
|
+
Path to the output directory.
|
|
76
|
+
|
|
77
|
+
Raises:
|
|
78
|
+
RuntimeError: If the output directory already exists and :obj:`force`
|
|
79
|
+
is not set.
|
|
80
|
+
NotImplementedError: If the converter cannot convert this model to the
|
|
81
|
+
CTranslate2 format.
|
|
82
|
+
"""
|
|
83
|
+
if os.path.exists(output_dir) and not force:
|
|
84
|
+
raise RuntimeError(
|
|
85
|
+
"output directory %s already exists, use --force to override"
|
|
86
|
+
% output_dir
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
model_spec = self._load()
|
|
90
|
+
if model_spec is None:
|
|
91
|
+
raise NotImplementedError(
|
|
92
|
+
"This model is not supported by CTranslate2 or this converter"
|
|
93
|
+
)
|
|
94
|
+
if vmap is not None:
|
|
95
|
+
model_spec.register_vocabulary_mapping(vmap)
|
|
96
|
+
|
|
97
|
+
model_spec.validate()
|
|
98
|
+
model_spec.optimize(quantization=quantization)
|
|
99
|
+
|
|
100
|
+
# Create model directory.
|
|
101
|
+
if os.path.exists(output_dir):
|
|
102
|
+
shutil.rmtree(output_dir)
|
|
103
|
+
os.makedirs(output_dir)
|
|
104
|
+
model_spec.save(output_dir)
|
|
105
|
+
return output_dir
|
|
106
|
+
|
|
107
|
+
@abc.abstractmethod
|
|
108
|
+
def _load(self):
|
|
109
|
+
raise NotImplementedError()
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""Load Drama3 codec weights for standard CTranslate2 conversion.
|
|
2
|
+
|
|
3
|
+
Fold weight normalization in row chunks and retain FLOAT32 arrays. PyTorch is
|
|
4
|
+
needed only to safely load the tensor-only checkpoint with memory mapping.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from ctranslate2.specs.drama3_spec import codec_parameter_shapes
|
|
10
|
+
|
|
11
|
+
FLOAT_TYPES = {
|
|
12
|
+
"float32",
|
|
13
|
+
"float16",
|
|
14
|
+
"float64",
|
|
15
|
+
"torch.float32",
|
|
16
|
+
"torch.float16",
|
|
17
|
+
"torch.float64",
|
|
18
|
+
"torch.bfloat16",
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def conversion_plan(schema):
|
|
23
|
+
"""Resolve direct/legacy/parametrized tensors using only names and shapes."""
|
|
24
|
+
keys = schema.get("keys", schema)
|
|
25
|
+
plan = []
|
|
26
|
+
for source, shape in codec_parameter_shapes().items():
|
|
27
|
+
candidates = []
|
|
28
|
+
if source in keys:
|
|
29
|
+
candidates.append((source, None))
|
|
30
|
+
if source.endswith(".weight"):
|
|
31
|
+
stem = source[: -len(".weight")]
|
|
32
|
+
for g, v in (
|
|
33
|
+
(
|
|
34
|
+
stem + ".parametrizations.weight.original0",
|
|
35
|
+
stem + ".parametrizations.weight.original1",
|
|
36
|
+
),
|
|
37
|
+
(stem + ".weight_g", stem + ".weight_v"),
|
|
38
|
+
):
|
|
39
|
+
if g in keys or v in keys:
|
|
40
|
+
if g not in keys or v not in keys:
|
|
41
|
+
raise ValueError(
|
|
42
|
+
"incomplete weight-normalization pair: " + source
|
|
43
|
+
)
|
|
44
|
+
candidates.append((v, g))
|
|
45
|
+
if len(candidates) != 1:
|
|
46
|
+
raise ValueError("missing or ambiguous official tensor: " + source)
|
|
47
|
+
value_key, magnitude_key = candidates[0]
|
|
48
|
+
if tuple(keys[value_key]["shape"]) != shape:
|
|
49
|
+
raise ValueError("unsupported codec weight shape: " + source)
|
|
50
|
+
selected = [value_key]
|
|
51
|
+
if magnitude_key:
|
|
52
|
+
if list(keys[magnitude_key]["shape"]) != [shape[0]] + [1] * (
|
|
53
|
+
len(shape) - 1
|
|
54
|
+
):
|
|
55
|
+
raise ValueError(
|
|
56
|
+
"weight-normalization magnitude must retain axis0: " + source
|
|
57
|
+
)
|
|
58
|
+
selected.append(magnitude_key)
|
|
59
|
+
if any(keys[key]["dtype"] not in FLOAT_TYPES for key in selected):
|
|
60
|
+
raise ValueError("non-floating codec tensor: " + source)
|
|
61
|
+
target = "codec/" + source.replace(".", "/")
|
|
62
|
+
plan.append(
|
|
63
|
+
{
|
|
64
|
+
"source_name": source,
|
|
65
|
+
"target": target,
|
|
66
|
+
"shape": list(keys[value_key]["shape"]),
|
|
67
|
+
"value_key": value_key,
|
|
68
|
+
"magnitude_key": magnitude_key,
|
|
69
|
+
}
|
|
70
|
+
)
|
|
71
|
+
return plan
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def numpy_float_chunk(value):
|
|
75
|
+
if isinstance(value, np.ndarray):
|
|
76
|
+
with np.errstate(over="ignore", invalid="ignore"):
|
|
77
|
+
result = np.asarray(value, dtype=np.float32, order="C")
|
|
78
|
+
else:
|
|
79
|
+
# Only the supplied slice is converted, never a whole checkpoint bank.
|
|
80
|
+
result = (
|
|
81
|
+
value.detach()
|
|
82
|
+
.to(device="cpu", dtype=__import__("torch").float32)
|
|
83
|
+
.contiguous()
|
|
84
|
+
.numpy()
|
|
85
|
+
)
|
|
86
|
+
if not np.isfinite(result).all():
|
|
87
|
+
raise ValueError("nonfinite codec weight or float32 conversion overflow")
|
|
88
|
+
return result
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def fold_axis0_chunk(direction, magnitude):
|
|
92
|
+
"""Independent axis-0 WN: each first-axis row has one magnitude."""
|
|
93
|
+
direction = numpy_float_chunk(direction)
|
|
94
|
+
magnitude = numpy_float_chunk(magnitude)
|
|
95
|
+
if direction.ndim < 2 or magnitude.shape != (direction.shape[0],) + (1,) * (
|
|
96
|
+
direction.ndim - 1
|
|
97
|
+
):
|
|
98
|
+
raise ValueError("incompatible axis0 weight-normalization shapes")
|
|
99
|
+
axes = tuple(range(1, direction.ndim))
|
|
100
|
+
# Float64 norm avoids squaring overflow; product is checked again in F32.
|
|
101
|
+
norm = np.sqrt(np.sum(direction.astype(np.float64) ** 2, axis=axes, keepdims=True))
|
|
102
|
+
if not np.isfinite(norm).all() or (norm == 0).any():
|
|
103
|
+
raise ValueError("zero or nonfinite weight-normalization norm")
|
|
104
|
+
folded = np.asarray(
|
|
105
|
+
direction * (magnitude.astype(np.float64) / norm), dtype=np.float32, order="C"
|
|
106
|
+
)
|
|
107
|
+
if not np.isfinite(folded).all():
|
|
108
|
+
raise ValueError("weight-normalization output overflows float32")
|
|
109
|
+
return folded
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def tensor_chunks(state, item, chunk_rows):
|
|
113
|
+
if type(chunk_rows) is not int or chunk_rows < 1:
|
|
114
|
+
raise ValueError("chunk_rows must be positive")
|
|
115
|
+
value = state[item["value_key"]]
|
|
116
|
+
magnitude = state[item["magnitude_key"]] if item["magnitude_key"] else None
|
|
117
|
+
for start in range(0, item["shape"][0], chunk_rows):
|
|
118
|
+
end = min(start + chunk_rows, item["shape"][0])
|
|
119
|
+
if magnitude is None:
|
|
120
|
+
yield numpy_float_chunk(value[start:end])
|
|
121
|
+
else:
|
|
122
|
+
yield fold_axis0_chunk(value[start:end], magnitude[start:end])
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def state_schema(state):
|
|
126
|
+
return {
|
|
127
|
+
"keys": {
|
|
128
|
+
name: {"shape": list(value.shape), "dtype": str(value.dtype)}
|
|
129
|
+
for name, value in state.items()
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def load_codec_tensors(state, plan=None, chunk_rows=32):
|
|
135
|
+
"""Load decoder weights as whole FLOAT32 arrays for standard ModelSpec."""
|
|
136
|
+
if type(chunk_rows) is not int or chunk_rows < 1:
|
|
137
|
+
raise ValueError("chunk_rows must be positive")
|
|
138
|
+
plan = conversion_plan(state_schema(state)) if plan is None else plan
|
|
139
|
+
tensors = {}
|
|
140
|
+
for item in plan:
|
|
141
|
+
value = np.empty(item["shape"], dtype=np.float32)
|
|
142
|
+
start = 0
|
|
143
|
+
for chunk in tensor_chunks(state, item, chunk_rows):
|
|
144
|
+
value[start : start + len(chunk)] = chunk
|
|
145
|
+
start += len(chunk)
|
|
146
|
+
tensors[item["target"]] = value
|
|
147
|
+
return tensors
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def load_checkpoint(path):
|
|
151
|
+
import torch
|
|
152
|
+
|
|
153
|
+
state = torch.load(path, map_location="cpu", weights_only=True, mmap=True)
|
|
154
|
+
if not isinstance(state, dict) or not all(
|
|
155
|
+
isinstance(k, str) and isinstance(v, torch.Tensor) for k, v in state.items()
|
|
156
|
+
):
|
|
157
|
+
raise ValueError("expected a flat tensor-only codec state dictionary")
|
|
158
|
+
return state
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""Bounded-memory safetensors matrix reads for an independent Drama 3 converter.
|
|
2
|
+
|
|
3
|
+
Rank-two matrices are supported.
|
|
4
|
+
No tensor-name conventions or automatic quantization-scale matching are assumed.
|
|
5
|
+
Files must remain immutable throughout reading. Safetensors validates and maps
|
|
6
|
+
shards; PyTorch converts individual FP8/BF16 chunks to float32.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
|
|
13
|
+
from safetensors import SafetensorError, safe_open
|
|
14
|
+
|
|
15
|
+
STORAGE_DTYPES = {
|
|
16
|
+
"F32": "<f4",
|
|
17
|
+
"F16": "<f2",
|
|
18
|
+
"BF16": "<u2",
|
|
19
|
+
"F8_E4M3FN": "u1",
|
|
20
|
+
"F8_E4M3": "u1",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _unique_object(pairs):
|
|
25
|
+
result = {}
|
|
26
|
+
for name, value in pairs:
|
|
27
|
+
if name in result:
|
|
28
|
+
raise ValueError(f"duplicate safetensors header key: {name}")
|
|
29
|
+
result[name] = value
|
|
30
|
+
return result
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _is_int(value):
|
|
34
|
+
return isinstance(value, int) and not isinstance(value, bool)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _float32(raw, dtype):
|
|
38
|
+
import torch
|
|
39
|
+
|
|
40
|
+
tensor = torch.from_numpy(raw)
|
|
41
|
+
if dtype == "BF16":
|
|
42
|
+
tensor = tensor.view(torch.bfloat16)
|
|
43
|
+
elif dtype.startswith("F8_"):
|
|
44
|
+
tensor = tensor.view(torch.float8_e4m3fn)
|
|
45
|
+
return tensor.to(dtype=torch.float32, copy=True).numpy()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class SafeTensorMatrixReader:
|
|
49
|
+
"""Inspect one immutable safetensors shard, mapping only requested tensors.
|
|
50
|
+
|
|
51
|
+
Conversion buffers scale with chunk_rows * matrix columns plus relevant
|
|
52
|
+
block-scale rows. The library maps the file without allocating a full dense
|
|
53
|
+
tensor buffer. Consumers must process/discard chunks rather than collect them
|
|
54
|
+
if bounded working memory is required.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
def __init__(self, path):
|
|
58
|
+
self.path = Path(path)
|
|
59
|
+
stat = self.path.stat()
|
|
60
|
+
self._identity = (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns)
|
|
61
|
+
try:
|
|
62
|
+
self._shard = safe_open(self.path, framework="pt", device="cpu")
|
|
63
|
+
self.tensors = {}
|
|
64
|
+
for name in self._shard.keys():
|
|
65
|
+
tensor = self._shard.get_slice(name)
|
|
66
|
+
dtype = tensor.get_dtype()
|
|
67
|
+
if dtype not in STORAGE_DTYPES:
|
|
68
|
+
raise ValueError(
|
|
69
|
+
f"unsupported safetensors dtype for {name}: {dtype}"
|
|
70
|
+
)
|
|
71
|
+
self.tensors[name] = {
|
|
72
|
+
"dtype": dtype,
|
|
73
|
+
"shape": tuple(tensor.get_shape()),
|
|
74
|
+
}
|
|
75
|
+
except SafetensorError as error:
|
|
76
|
+
raise ValueError(f"invalid safetensors shard: {error}") from error
|
|
77
|
+
|
|
78
|
+
def _map(self, name, index=None):
|
|
79
|
+
import torch
|
|
80
|
+
|
|
81
|
+
stat = self.path.stat()
|
|
82
|
+
if (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns) != self._identity:
|
|
83
|
+
raise ValueError("checkpoint shard changed after header inspection")
|
|
84
|
+
tensor = (
|
|
85
|
+
self._shard.get_tensor(name)
|
|
86
|
+
if index is None
|
|
87
|
+
else self._shard.get_slice(name)[index]
|
|
88
|
+
)
|
|
89
|
+
# Reinterpret unsupported NumPy dtypes without copying or decoding a bank.
|
|
90
|
+
if tensor.dtype == torch.bfloat16:
|
|
91
|
+
tensor = tensor.view(torch.uint16)
|
|
92
|
+
elif tensor.dtype == torch.float8_e4m3fn:
|
|
93
|
+
tensor = tensor.view(torch.uint8)
|
|
94
|
+
return tensor.numpy()
|
|
95
|
+
|
|
96
|
+
def iter_matrix_rows(
|
|
97
|
+
self,
|
|
98
|
+
name,
|
|
99
|
+
chunk_rows=128,
|
|
100
|
+
inverse_scale=None,
|
|
101
|
+
block_size=(128, 128),
|
|
102
|
+
scale_reader=None,
|
|
103
|
+
):
|
|
104
|
+
"""Yield bounded FP32 row chunks of a rank-two matrix.
|
|
105
|
+
|
|
106
|
+
scale_reader, when supplied, explicitly locates inverse_scale in another
|
|
107
|
+
safetensors shard; scale names and layouts are never inferred.
|
|
108
|
+
"""
|
|
109
|
+
if not _is_int(chunk_rows) or chunk_rows <= 0:
|
|
110
|
+
raise ValueError("chunk_rows must be positive")
|
|
111
|
+
if (
|
|
112
|
+
not isinstance(block_size, (tuple, list))
|
|
113
|
+
or len(block_size) != 2
|
|
114
|
+
or any(not _is_int(d) or d <= 0 for d in block_size)
|
|
115
|
+
):
|
|
116
|
+
raise ValueError("block_size must contain two positive integers")
|
|
117
|
+
tensor = self.tensors[name]
|
|
118
|
+
if len(tensor["shape"]) != 2:
|
|
119
|
+
raise ValueError("row conversion requires a rank-two matrix")
|
|
120
|
+
rows, cols = tensor["shape"]
|
|
121
|
+
if scale_reader is not None and not isinstance(
|
|
122
|
+
scale_reader, SafeTensorMatrixReader
|
|
123
|
+
):
|
|
124
|
+
raise ValueError("scale_reader must be a SafeTensorMatrixReader")
|
|
125
|
+
if scale_reader is not None and inverse_scale is None:
|
|
126
|
+
raise ValueError(
|
|
127
|
+
"scale_reader requires an explicit inverse_scale tensor name"
|
|
128
|
+
)
|
|
129
|
+
scale_source = self if scale_reader is None else scale_reader
|
|
130
|
+
br, bc = block_size
|
|
131
|
+
scale_meta = None
|
|
132
|
+
scales = None
|
|
133
|
+
if inverse_scale is not None:
|
|
134
|
+
if not tensor["dtype"].startswith("F8_"):
|
|
135
|
+
raise ValueError("inverse_scale is supported only for FP8 matrices")
|
|
136
|
+
scale_meta = scale_source.tensors[inverse_scale]
|
|
137
|
+
expected_shape = ((rows + br - 1) // br, (cols + bc - 1) // bc)
|
|
138
|
+
if scale_meta["shape"] != expected_shape:
|
|
139
|
+
raise ValueError("inverse_scale shape does not match block grid")
|
|
140
|
+
if scale_meta["dtype"] not in ("F32", "F16", "BF16"):
|
|
141
|
+
raise ValueError("inverse_scale must be F32/F16/BF16")
|
|
142
|
+
scales = scale_source._map(inverse_scale)
|
|
143
|
+
for start in range(0, rows, chunk_rows):
|
|
144
|
+
stop = min(start + chunk_rows, rows)
|
|
145
|
+
chunk = _float32(self._map(name, slice(start, stop)), tensor["dtype"])
|
|
146
|
+
if scales is not None:
|
|
147
|
+
first_block = start // br
|
|
148
|
+
scale_rows = _float32(
|
|
149
|
+
scales[first_block : (stop - 1) // br + 1], scale_meta["dtype"]
|
|
150
|
+
)
|
|
151
|
+
if not np.all(np.isfinite(scale_rows)) or np.any(scale_rows <= 0):
|
|
152
|
+
raise ValueError("inverse_scale values must be positive and finite")
|
|
153
|
+
# Avoid expanding an entire rows*columns scale matrix.
|
|
154
|
+
for row in range(stop - start):
|
|
155
|
+
sr = scale_rows[(start + row) // br - first_block]
|
|
156
|
+
for col_block in range(len(sr)):
|
|
157
|
+
chunk[
|
|
158
|
+
row, col_block * bc : min((col_block + 1) * bc, cols)
|
|
159
|
+
] *= sr[col_block]
|
|
160
|
+
yield chunk
|