ctranslate2 4.8.4__cp315-cp315-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. ctranslate2/__init__.py +97 -0
  2. ctranslate2/_ext.cp315-win_amd64.pyd +0 -0
  3. ctranslate2/converters/__init__.py +8 -0
  4. ctranslate2/converters/converter.py +109 -0
  5. ctranslate2/converters/drama3_codec.py +158 -0
  6. ctranslate2/converters/drama3_weights.py +160 -0
  7. ctranslate2/converters/eole_ct2.py +353 -0
  8. ctranslate2/converters/fairseq.py +362 -0
  9. ctranslate2/converters/marian.py +315 -0
  10. ctranslate2/converters/openai_gpt2.py +95 -0
  11. ctranslate2/converters/opennmt_py.py +376 -0
  12. ctranslate2/converters/opennmt_tf.py +455 -0
  13. ctranslate2/converters/opus_mt.py +44 -0
  14. ctranslate2/converters/transformers.py +4907 -0
  15. ctranslate2/converters/utils.py +127 -0
  16. ctranslate2/ctranslate2.dll +4 -0
  17. ctranslate2/cudnn64_9.dll +0 -0
  18. ctranslate2/extensions.py +589 -0
  19. ctranslate2/libiomp5md.dll +0 -0
  20. ctranslate2/logging.py +45 -0
  21. ctranslate2/models/__init__.py +22 -0
  22. ctranslate2/specs/__init__.py +19 -0
  23. ctranslate2/specs/attention_spec.py +107 -0
  24. ctranslate2/specs/common_spec.py +66 -0
  25. ctranslate2/specs/drama3_spec.py +155 -0
  26. ctranslate2/specs/model_spec.py +771 -0
  27. ctranslate2/specs/transformer_spec.py +812 -0
  28. ctranslate2/specs/wav2vec2_spec.py +72 -0
  29. ctranslate2/specs/wav2vec2bert_spec.py +97 -0
  30. ctranslate2/specs/whisper_spec.py +77 -0
  31. ctranslate2/version.py +3 -0
  32. ctranslate2-4.8.4.dist-info/METADATA +187 -0
  33. ctranslate2-4.8.4.dist-info/RECORD +36 -0
  34. ctranslate2-4.8.4.dist-info/WHEEL +5 -0
  35. ctranslate2-4.8.4.dist-info/entry_points.txt +8 -0
  36. ctranslate2-4.8.4.dist-info/top_level.txt +1 -0
@@ -0,0 +1,97 @@
1
+ import sys
2
+
3
+ if sys.platform == "win32":
4
+ import ctypes
5
+ import glob
6
+ import os
7
+
8
+ from importlib.resources import files
9
+
10
+ module_name = sys.modules[__name__].__name__
11
+ package_dir = str(files(module_name))
12
+
13
+ # The libraries package name differs across ROCm releases.
14
+ site_dir = os.path.dirname(package_dir)
15
+ dll_dirs = [os.path.join(site_dir, "_rocm_sdk_core", "bin")]
16
+ dll_dirs += sorted(glob.glob(os.path.join(site_dir, "_rocm_sdk_libraries*", "bin")))
17
+ dll_dirs = [path for path in dll_dirs if os.path.isdir(path)]
18
+ # Fallback only: the search order between DLL directories is unspecified.
19
+ if not dll_dirs and os.environ.get("HIP_PATH"):
20
+ dll_dirs = [os.path.join(os.environ["HIP_PATH"], "bin")]
21
+
22
+ for path in [package_dir] + dll_dirs:
23
+ try:
24
+ os.add_dll_directory(path)
25
+ except OSError:
26
+ pass
27
+
28
+ for library in glob.glob(os.path.join(package_dir, "*.dll")):
29
+ ctypes.CDLL(library)
30
+
31
+ try:
32
+ from ctranslate2._ext import (
33
+ AsyncGenerationResult,
34
+ AsyncScoringResult,
35
+ AsyncTranslationResult,
36
+ DataType,
37
+ Device,
38
+ Encoder,
39
+ EncoderForwardOutput,
40
+ ExecutionStats,
41
+ GenerationResult,
42
+ GenerationStepResult,
43
+ Generator,
44
+ MpiInfo,
45
+ ScoringResult,
46
+ StorageView,
47
+ TranslationResult,
48
+ Translator,
49
+ contains_model,
50
+ get_cuda_device_count,
51
+ get_supported_compute_types,
52
+ set_random_seed,
53
+ )
54
+ from ctranslate2.extensions import register_extensions
55
+ from ctranslate2.logging import get_log_level, set_log_level
56
+
57
+ register_extensions()
58
+ del register_extensions
59
+ except ImportError as e:
60
+ # Allow using the Python package without the compiled extension.
61
+ if "No module named" in str(e):
62
+ pass
63
+ else:
64
+ raise
65
+
66
+ from ctranslate2 import models
67
+ from ctranslate2.version import __version__
68
+
69
+ # converters and specs import torch (and, for converters, transformers) at module level.
70
+ # Those dependencies are only needed to convert models, not to run inference, so import
71
+ # these submodules on first use to keep "import ctranslate2" free of them.
72
+ _LAZY_SUBMODULES = ("converters", "specs")
73
+
74
+
75
+ def __getattr__(name):
76
+ if name in _LAZY_SUBMODULES:
77
+ import importlib
78
+
79
+ module = importlib.import_module(f"{__name__}.{name}")
80
+ globals()[name] = module
81
+ return module
82
+
83
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
84
+
85
+
86
+ def __dir__():
87
+ return sorted(set(globals()) | set(_LAZY_SUBMODULES))
88
+
89
+
90
+ # A wildcard import resolves ``__all__`` when it is defined and the module globals
91
+ # otherwise, so without this the lazy submodules would silently drop out of
92
+ # ``from ctranslate2 import *``. Deriving the list keeps the wildcard surface identical
93
+ # to what it was before they became lazy; a wildcard import asks for everything, so
94
+ # resolving them here is expected.
95
+ __all__ = sorted(
96
+ [name for name in globals() if not name.startswith("_")] + list(_LAZY_SUBMODULES)
97
+ )
Binary file
@@ -0,0 +1,8 @@
1
+ from ctranslate2.converters.converter import Converter
2
+ from ctranslate2.converters.fairseq import FairseqConverter
3
+ from ctranslate2.converters.marian import MarianConverter
4
+ from ctranslate2.converters.openai_gpt2 import OpenAIGPT2Converter
5
+ from ctranslate2.converters.opennmt_py import OpenNMTPyConverter
6
+ from ctranslate2.converters.opennmt_tf import OpenNMTTFConverter
7
+ from ctranslate2.converters.opus_mt import OpusMTConverter
8
+ from ctranslate2.converters.transformers import TransformersConverter
@@ -0,0 +1,109 @@
1
+ import abc
2
+ import argparse
3
+ import os
4
+ import shutil
5
+
6
+ from typing import Optional
7
+
8
+ from ctranslate2.specs.model_spec import ACCEPTED_MODEL_TYPES, ModelSpec
9
+
10
+
11
+ class Converter(abc.ABC):
12
+ """Base class for model converters."""
13
+
14
+ @staticmethod
15
+ def declare_arguments(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
16
+ """Adds common conversion options to the command line parser.
17
+
18
+ Arguments:
19
+ parser: Command line argument parser.
20
+ """
21
+ parser.add_argument(
22
+ "--output_dir", required=True, help="Output model directory."
23
+ )
24
+ parser.add_argument(
25
+ "--vocab_mapping", default=None, help="Vocabulary mapping file (optional)."
26
+ )
27
+ parser.add_argument(
28
+ "--quantization",
29
+ default=None,
30
+ choices=ACCEPTED_MODEL_TYPES,
31
+ help="Weight quantization type.",
32
+ )
33
+ parser.add_argument(
34
+ "--force",
35
+ action="store_true",
36
+ help="Force conversion even if the output directory already exists.",
37
+ )
38
+ return parser
39
+
40
+ def convert_from_args(self, args: argparse.Namespace) -> str:
41
+ """Helper function to call :meth:`ctranslate2.converters.Converter.convert`
42
+ with the parsed command line options.
43
+
44
+ Arguments:
45
+ args: Namespace containing parsed arguments.
46
+
47
+ Returns:
48
+ Path to the output directory.
49
+ """
50
+ return self.convert(
51
+ args.output_dir,
52
+ vmap=args.vocab_mapping,
53
+ quantization=args.quantization,
54
+ force=args.force,
55
+ )
56
+
57
+ def convert(
58
+ self,
59
+ output_dir: str,
60
+ vmap: Optional[str] = None,
61
+ quantization: Optional[str] = None,
62
+ force: bool = False,
63
+ ) -> str:
64
+ """Converts the model to the CTranslate2 format.
65
+
66
+ Arguments:
67
+ output_dir: Output directory where the CTranslate2 model is saved.
68
+ vmap: Optional path to a vocabulary mapping file that will be included
69
+ in the converted model directory.
70
+ quantization: Weight quantization scheme (possible values are: int8, int8_float32,
71
+ int8_float16, int8_bfloat16, int16, float16, bfloat16, float32).
72
+ force: Override the output directory if it already exists.
73
+
74
+ Returns:
75
+ Path to the output directory.
76
+
77
+ Raises:
78
+ RuntimeError: If the output directory already exists and :obj:`force`
79
+ is not set.
80
+ NotImplementedError: If the converter cannot convert this model to the
81
+ CTranslate2 format.
82
+ """
83
+ if os.path.exists(output_dir) and not force:
84
+ raise RuntimeError(
85
+ "output directory %s already exists, use --force to override"
86
+ % output_dir
87
+ )
88
+
89
+ model_spec = self._load()
90
+ if model_spec is None:
91
+ raise NotImplementedError(
92
+ "This model is not supported by CTranslate2 or this converter"
93
+ )
94
+ if vmap is not None:
95
+ model_spec.register_vocabulary_mapping(vmap)
96
+
97
+ model_spec.validate()
98
+ model_spec.optimize(quantization=quantization)
99
+
100
+ # Create model directory.
101
+ if os.path.exists(output_dir):
102
+ shutil.rmtree(output_dir)
103
+ os.makedirs(output_dir)
104
+ model_spec.save(output_dir)
105
+ return output_dir
106
+
107
+ @abc.abstractmethod
108
+ def _load(self):
109
+ raise NotImplementedError()
@@ -0,0 +1,158 @@
1
+ """Load Drama3 codec weights for standard CTranslate2 conversion.
2
+
3
+ Fold weight normalization in row chunks and retain FLOAT32 arrays. PyTorch is
4
+ needed only to safely load the tensor-only checkpoint with memory mapping.
5
+ """
6
+
7
+ import numpy as np
8
+
9
+ from ctranslate2.specs.drama3_spec import codec_parameter_shapes
10
+
11
+ FLOAT_TYPES = {
12
+ "float32",
13
+ "float16",
14
+ "float64",
15
+ "torch.float32",
16
+ "torch.float16",
17
+ "torch.float64",
18
+ "torch.bfloat16",
19
+ }
20
+
21
+
22
+ def conversion_plan(schema):
23
+ """Resolve direct/legacy/parametrized tensors using only names and shapes."""
24
+ keys = schema.get("keys", schema)
25
+ plan = []
26
+ for source, shape in codec_parameter_shapes().items():
27
+ candidates = []
28
+ if source in keys:
29
+ candidates.append((source, None))
30
+ if source.endswith(".weight"):
31
+ stem = source[: -len(".weight")]
32
+ for g, v in (
33
+ (
34
+ stem + ".parametrizations.weight.original0",
35
+ stem + ".parametrizations.weight.original1",
36
+ ),
37
+ (stem + ".weight_g", stem + ".weight_v"),
38
+ ):
39
+ if g in keys or v in keys:
40
+ if g not in keys or v not in keys:
41
+ raise ValueError(
42
+ "incomplete weight-normalization pair: " + source
43
+ )
44
+ candidates.append((v, g))
45
+ if len(candidates) != 1:
46
+ raise ValueError("missing or ambiguous official tensor: " + source)
47
+ value_key, magnitude_key = candidates[0]
48
+ if tuple(keys[value_key]["shape"]) != shape:
49
+ raise ValueError("unsupported codec weight shape: " + source)
50
+ selected = [value_key]
51
+ if magnitude_key:
52
+ if list(keys[magnitude_key]["shape"]) != [shape[0]] + [1] * (
53
+ len(shape) - 1
54
+ ):
55
+ raise ValueError(
56
+ "weight-normalization magnitude must retain axis0: " + source
57
+ )
58
+ selected.append(magnitude_key)
59
+ if any(keys[key]["dtype"] not in FLOAT_TYPES for key in selected):
60
+ raise ValueError("non-floating codec tensor: " + source)
61
+ target = "codec/" + source.replace(".", "/")
62
+ plan.append(
63
+ {
64
+ "source_name": source,
65
+ "target": target,
66
+ "shape": list(keys[value_key]["shape"]),
67
+ "value_key": value_key,
68
+ "magnitude_key": magnitude_key,
69
+ }
70
+ )
71
+ return plan
72
+
73
+
74
+ def numpy_float_chunk(value):
75
+ if isinstance(value, np.ndarray):
76
+ with np.errstate(over="ignore", invalid="ignore"):
77
+ result = np.asarray(value, dtype=np.float32, order="C")
78
+ else:
79
+ # Only the supplied slice is converted, never a whole checkpoint bank.
80
+ result = (
81
+ value.detach()
82
+ .to(device="cpu", dtype=__import__("torch").float32)
83
+ .contiguous()
84
+ .numpy()
85
+ )
86
+ if not np.isfinite(result).all():
87
+ raise ValueError("nonfinite codec weight or float32 conversion overflow")
88
+ return result
89
+
90
+
91
+ def fold_axis0_chunk(direction, magnitude):
92
+ """Independent axis-0 WN: each first-axis row has one magnitude."""
93
+ direction = numpy_float_chunk(direction)
94
+ magnitude = numpy_float_chunk(magnitude)
95
+ if direction.ndim < 2 or magnitude.shape != (direction.shape[0],) + (1,) * (
96
+ direction.ndim - 1
97
+ ):
98
+ raise ValueError("incompatible axis0 weight-normalization shapes")
99
+ axes = tuple(range(1, direction.ndim))
100
+ # Float64 norm avoids squaring overflow; product is checked again in F32.
101
+ norm = np.sqrt(np.sum(direction.astype(np.float64) ** 2, axis=axes, keepdims=True))
102
+ if not np.isfinite(norm).all() or (norm == 0).any():
103
+ raise ValueError("zero or nonfinite weight-normalization norm")
104
+ folded = np.asarray(
105
+ direction * (magnitude.astype(np.float64) / norm), dtype=np.float32, order="C"
106
+ )
107
+ if not np.isfinite(folded).all():
108
+ raise ValueError("weight-normalization output overflows float32")
109
+ return folded
110
+
111
+
112
+ def tensor_chunks(state, item, chunk_rows):
113
+ if type(chunk_rows) is not int or chunk_rows < 1:
114
+ raise ValueError("chunk_rows must be positive")
115
+ value = state[item["value_key"]]
116
+ magnitude = state[item["magnitude_key"]] if item["magnitude_key"] else None
117
+ for start in range(0, item["shape"][0], chunk_rows):
118
+ end = min(start + chunk_rows, item["shape"][0])
119
+ if magnitude is None:
120
+ yield numpy_float_chunk(value[start:end])
121
+ else:
122
+ yield fold_axis0_chunk(value[start:end], magnitude[start:end])
123
+
124
+
125
+ def state_schema(state):
126
+ return {
127
+ "keys": {
128
+ name: {"shape": list(value.shape), "dtype": str(value.dtype)}
129
+ for name, value in state.items()
130
+ }
131
+ }
132
+
133
+
134
+ def load_codec_tensors(state, plan=None, chunk_rows=32):
135
+ """Load decoder weights as whole FLOAT32 arrays for standard ModelSpec."""
136
+ if type(chunk_rows) is not int or chunk_rows < 1:
137
+ raise ValueError("chunk_rows must be positive")
138
+ plan = conversion_plan(state_schema(state)) if plan is None else plan
139
+ tensors = {}
140
+ for item in plan:
141
+ value = np.empty(item["shape"], dtype=np.float32)
142
+ start = 0
143
+ for chunk in tensor_chunks(state, item, chunk_rows):
144
+ value[start : start + len(chunk)] = chunk
145
+ start += len(chunk)
146
+ tensors[item["target"]] = value
147
+ return tensors
148
+
149
+
150
+ def load_checkpoint(path):
151
+ import torch
152
+
153
+ state = torch.load(path, map_location="cpu", weights_only=True, mmap=True)
154
+ if not isinstance(state, dict) or not all(
155
+ isinstance(k, str) and isinstance(v, torch.Tensor) for k, v in state.items()
156
+ ):
157
+ raise ValueError("expected a flat tensor-only codec state dictionary")
158
+ return state
@@ -0,0 +1,160 @@
1
+ """Bounded-memory safetensors matrix reads for an independent Drama 3 converter.
2
+
3
+ Rank-two matrices are supported.
4
+ No tensor-name conventions or automatic quantization-scale matching are assumed.
5
+ Files must remain immutable throughout reading. Safetensors validates and maps
6
+ shards; PyTorch converts individual FP8/BF16 chunks to float32.
7
+ """
8
+
9
+ from pathlib import Path
10
+
11
+ import numpy as np
12
+
13
+ from safetensors import SafetensorError, safe_open
14
+
15
+ STORAGE_DTYPES = {
16
+ "F32": "<f4",
17
+ "F16": "<f2",
18
+ "BF16": "<u2",
19
+ "F8_E4M3FN": "u1",
20
+ "F8_E4M3": "u1",
21
+ }
22
+
23
+
24
+ def _unique_object(pairs):
25
+ result = {}
26
+ for name, value in pairs:
27
+ if name in result:
28
+ raise ValueError(f"duplicate safetensors header key: {name}")
29
+ result[name] = value
30
+ return result
31
+
32
+
33
+ def _is_int(value):
34
+ return isinstance(value, int) and not isinstance(value, bool)
35
+
36
+
37
+ def _float32(raw, dtype):
38
+ import torch
39
+
40
+ tensor = torch.from_numpy(raw)
41
+ if dtype == "BF16":
42
+ tensor = tensor.view(torch.bfloat16)
43
+ elif dtype.startswith("F8_"):
44
+ tensor = tensor.view(torch.float8_e4m3fn)
45
+ return tensor.to(dtype=torch.float32, copy=True).numpy()
46
+
47
+
48
+ class SafeTensorMatrixReader:
49
+ """Inspect one immutable safetensors shard, mapping only requested tensors.
50
+
51
+ Conversion buffers scale with chunk_rows * matrix columns plus relevant
52
+ block-scale rows. The library maps the file without allocating a full dense
53
+ tensor buffer. Consumers must process/discard chunks rather than collect them
54
+ if bounded working memory is required.
55
+ """
56
+
57
+ def __init__(self, path):
58
+ self.path = Path(path)
59
+ stat = self.path.stat()
60
+ self._identity = (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns)
61
+ try:
62
+ self._shard = safe_open(self.path, framework="pt", device="cpu")
63
+ self.tensors = {}
64
+ for name in self._shard.keys():
65
+ tensor = self._shard.get_slice(name)
66
+ dtype = tensor.get_dtype()
67
+ if dtype not in STORAGE_DTYPES:
68
+ raise ValueError(
69
+ f"unsupported safetensors dtype for {name}: {dtype}"
70
+ )
71
+ self.tensors[name] = {
72
+ "dtype": dtype,
73
+ "shape": tuple(tensor.get_shape()),
74
+ }
75
+ except SafetensorError as error:
76
+ raise ValueError(f"invalid safetensors shard: {error}") from error
77
+
78
+ def _map(self, name, index=None):
79
+ import torch
80
+
81
+ stat = self.path.stat()
82
+ if (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns) != self._identity:
83
+ raise ValueError("checkpoint shard changed after header inspection")
84
+ tensor = (
85
+ self._shard.get_tensor(name)
86
+ if index is None
87
+ else self._shard.get_slice(name)[index]
88
+ )
89
+ # Reinterpret unsupported NumPy dtypes without copying or decoding a bank.
90
+ if tensor.dtype == torch.bfloat16:
91
+ tensor = tensor.view(torch.uint16)
92
+ elif tensor.dtype == torch.float8_e4m3fn:
93
+ tensor = tensor.view(torch.uint8)
94
+ return tensor.numpy()
95
+
96
+ def iter_matrix_rows(
97
+ self,
98
+ name,
99
+ chunk_rows=128,
100
+ inverse_scale=None,
101
+ block_size=(128, 128),
102
+ scale_reader=None,
103
+ ):
104
+ """Yield bounded FP32 row chunks of a rank-two matrix.
105
+
106
+ scale_reader, when supplied, explicitly locates inverse_scale in another
107
+ safetensors shard; scale names and layouts are never inferred.
108
+ """
109
+ if not _is_int(chunk_rows) or chunk_rows <= 0:
110
+ raise ValueError("chunk_rows must be positive")
111
+ if (
112
+ not isinstance(block_size, (tuple, list))
113
+ or len(block_size) != 2
114
+ or any(not _is_int(d) or d <= 0 for d in block_size)
115
+ ):
116
+ raise ValueError("block_size must contain two positive integers")
117
+ tensor = self.tensors[name]
118
+ if len(tensor["shape"]) != 2:
119
+ raise ValueError("row conversion requires a rank-two matrix")
120
+ rows, cols = tensor["shape"]
121
+ if scale_reader is not None and not isinstance(
122
+ scale_reader, SafeTensorMatrixReader
123
+ ):
124
+ raise ValueError("scale_reader must be a SafeTensorMatrixReader")
125
+ if scale_reader is not None and inverse_scale is None:
126
+ raise ValueError(
127
+ "scale_reader requires an explicit inverse_scale tensor name"
128
+ )
129
+ scale_source = self if scale_reader is None else scale_reader
130
+ br, bc = block_size
131
+ scale_meta = None
132
+ scales = None
133
+ if inverse_scale is not None:
134
+ if not tensor["dtype"].startswith("F8_"):
135
+ raise ValueError("inverse_scale is supported only for FP8 matrices")
136
+ scale_meta = scale_source.tensors[inverse_scale]
137
+ expected_shape = ((rows + br - 1) // br, (cols + bc - 1) // bc)
138
+ if scale_meta["shape"] != expected_shape:
139
+ raise ValueError("inverse_scale shape does not match block grid")
140
+ if scale_meta["dtype"] not in ("F32", "F16", "BF16"):
141
+ raise ValueError("inverse_scale must be F32/F16/BF16")
142
+ scales = scale_source._map(inverse_scale)
143
+ for start in range(0, rows, chunk_rows):
144
+ stop = min(start + chunk_rows, rows)
145
+ chunk = _float32(self._map(name, slice(start, stop)), tensor["dtype"])
146
+ if scales is not None:
147
+ first_block = start // br
148
+ scale_rows = _float32(
149
+ scales[first_block : (stop - 1) // br + 1], scale_meta["dtype"]
150
+ )
151
+ if not np.all(np.isfinite(scale_rows)) or np.any(scale_rows <= 0):
152
+ raise ValueError("inverse_scale values must be positive and finite")
153
+ # Avoid expanding an entire rows*columns scale matrix.
154
+ for row in range(stop - start):
155
+ sr = scale_rows[(start + row) // br - first_block]
156
+ for col_block in range(len(sr)):
157
+ chunk[
158
+ row, col_block * bc : min((col_block + 1) * bc, cols)
159
+ ] *= sr[col_block]
160
+ yield chunk