dgenerate-ultralytics-headless 8.3.237__py3-none-any.whl → 8.3.239__py3-none-any.whl

ultralytics/utils/files.py

@@ -28,15 +28,15 @@ class WorkingDirectory(contextlib.ContextDecorator):
 
     Examples:
         Using as a context manager:
-        >>> with WorkingDirectory('/path/to/new/dir'):
-        >>> # Perform operations in the new directory
-        >>>     pass
+        >>> with WorkingDirectory("/path/to/new/dir"):
+        ...     # Perform operations in the new directory
+        ...     pass
 
         Using as a decorator:
-        >>> @WorkingDirectory('/path/to/new/dir')
-        >>> def some_function():
-        >>> # Perform operations in the new directory
-        >>>     pass
+        >>> @WorkingDirectory("/path/to/new/dir")
+        ... def some_function():
+        ...     # Perform operations in the new directory
+        ...     pass
     """
 
     def __init__(self, new_dir: str | Path):
@@ -67,9 +67,9 @@ def spaces_in_path(path: str | Path):
         (Path | str): Temporary path with any spaces replaced by underscores.
 
     Examples:
-        >>> with spaces_in_path('/path/with spaces') as new_path:
-        >>> # Your code here
-        >>>     pass
+        >>> with spaces_in_path("/path/with spaces") as new_path:
+        ...     # Your code here
+        ...     pass
     """
     # If path has spaces, replace them with underscores
     if " " in str(path):
@@ -196,13 +196,14 @@ def update_models(model_names: tuple = ("yolo11n.pt",), source_dir: Path = Path(
     """
     from ultralytics import YOLO
     from ultralytics.nn.autobackend import default_class_names
+    from ultralytics.utils import LOGGER
 
     target_dir = source_dir / "updated_models"
     target_dir.mkdir(parents=True, exist_ok=True)  # Ensure target directory exists
 
     for model_name in model_names:
         model_path = source_dir / model_name
-        print(f"Loading model from {model_path}")
+        LOGGER.info(f"Loading model from {model_path}")
 
         # Load model
         model = YOLO(model_path)
@@ -214,5 +215,5 @@ def update_models(model_names: tuple = ("yolo11n.pt",), source_dir: Path = Path(
         save_path = target_dir / model_name
 
         # Save model using model.save()
-        print(f"Re-saving {model_name} model to {save_path}")
+        LOGGER.info(f"Re-saving {model_name} model to {save_path}")
         model.save(save_path)

ultralytics/utils/logger.py

@@ -267,6 +267,11 @@ class SystemLogger:
         self.net_start = psutil.net_io_counters()
         self.disk_start = psutil.disk_io_counters()
 
+        # For rate calculation
+        self._prev_net = self.net_start
+        self._prev_disk = self.disk_start
+        self._prev_time = time.time()
+
     def _init_nvidia(self):
         """Initialize NVIDIA GPU monitoring with pynvml."""
         try:
@@ -278,42 +283,49 @@ class SystemLogger:
         except Exception:
             return False
 
-    def get_metrics(self):
-        """Get current system metrics.
+    def get_metrics(self, rates=False):
+        """Get current system metrics including CPU, RAM, disk, network, and GPU usage.
 
         Collects comprehensive system metrics including CPU usage, RAM usage, disk I/O statistics, network I/O
-        statistics, and GPU metrics (if available). Example output:
+        statistics, and GPU metrics (if available).
 
+        Example output (rates=False, default):
         ```python
-        metrics = {
+        {
             "cpu": 45.2,
             "ram": 78.9,
             "disk": {"read_mb": 156.7, "write_mb": 89.3, "used_gb": 256.8},
             "network": {"recv_mb": 157.2, "sent_mb": 89.1},
             "gpus": {
-                0: {"usage": 95.6, "memory": 85.4, "temp": 72, "power": 285},
-                1: {"usage": 94.1, "memory": 82.7, "temp": 70, "power": 278},
+                "0": {"usage": 95.6, "memory": 85.4, "temp": 72, "power": 285},
+                "1": {"usage": 94.1, "memory": 82.7, "temp": 70, "power": 278},
             },
         }
         ```
 
-        - cpu (float): CPU usage percentage (0-100%)
-        - ram (float): RAM usage percentage (0-100%)
-        - disk (dict):
-            - read_mb (float): Cumulative disk read in MB since initialization
-            - write_mb (float): Cumulative disk write in MB since initialization
-            - used_gb (float): Total disk space used in GB
-        - network (dict):
-            - recv_mb (float): Cumulative network received in MB since initialization
-            - sent_mb (float): Cumulative network sent in MB since initialization
-        - gpus (dict): GPU metrics by device index (e.g., 0, 1) containing:
-            - usage (int): GPU utilization percentage (0-100%)
-            - memory (float): CUDA memory usage percentage (0-100%)
-            - temp (int): GPU temperature in degrees Celsius
-            - power (int): GPU power consumption in watts
+        Example output (rates=True):
+        ```python
+        {
+            "cpu": 45.2,
+            "ram": 78.9,
+            "disk": {"read_mbs": 12.5, "write_mbs": 8.3, "used_gb": 256.8},
+            "network": {"recv_mbs": 5.2, "sent_mbs": 1.1},
+            "gpus": {
+                "0": {"usage": 95.6, "memory": 85.4, "temp": 72, "power": 285},
+            },
+        }
+        ```
+
+        Args:
+            rates (bool): If True, return disk/network as MB/s rates instead of cumulative MB.
 
         Returns:
-            metrics (dict): System metrics containing 'cpu', 'ram', 'disk', 'network', 'gpus' with usage data.
+            (dict): Metrics dictionary with cpu, ram, disk, network, and gpus keys.
+
+        Examples:
+            >>> logger = SystemLogger()
+            >>> logger.get_metrics()["cpu"]  # CPU percentage
+            >>> logger.get_metrics(rates=True)["network"]["recv_mbs"]  # MB/s download rate
         """
         import psutil  # scoped as slow import
 
@@ -321,21 +333,44 @@ class SystemLogger:
         disk = psutil.disk_io_counters()
         memory = psutil.virtual_memory()
         disk_usage = shutil.disk_usage("/")
+        now = time.time()
 
         metrics = {
             "cpu": round(psutil.cpu_percent(), 3),
             "ram": round(memory.percent, 3),
-            "disk": {
+            "gpus": {},
+        }
+
+        # Calculate elapsed time since last call
+        elapsed = max(0.1, now - self._prev_time)  # Avoid division by zero
+
+        if rates:
+            # Calculate MB/s rates from delta since last call
+            metrics["disk"] = {
+                "read_mbs": round(max(0, (disk.read_bytes - self._prev_disk.read_bytes) / (1 << 20) / elapsed), 3),
+                "write_mbs": round(max(0, (disk.write_bytes - self._prev_disk.write_bytes) / (1 << 20) / elapsed), 3),
+                "used_gb": round(disk_usage.used / (1 << 30), 3),
+            }
+            metrics["network"] = {
+                "recv_mbs": round(max(0, (net.bytes_recv - self._prev_net.bytes_recv) / (1 << 20) / elapsed), 3),
+                "sent_mbs": round(max(0, (net.bytes_sent - self._prev_net.bytes_sent) / (1 << 20) / elapsed), 3),
+            }
+        else:
+            # Cumulative MB since initialization (original behavior)
+            metrics["disk"] = {
                 "read_mb": round((disk.read_bytes - self.disk_start.read_bytes) / (1 << 20), 3),
                 "write_mb": round((disk.write_bytes - self.disk_start.write_bytes) / (1 << 20), 3),
                 "used_gb": round(disk_usage.used / (1 << 30), 3),
-            },
-            "network": {
+            }
+            metrics["network"] = {
                 "recv_mb": round((net.bytes_recv - self.net_start.bytes_recv) / (1 << 20), 3),
                 "sent_mb": round((net.bytes_sent - self.net_start.bytes_sent) / (1 << 20), 3),
-            },
-            "gpus": {},
-        }
+            }
+
+        # Always update previous values for accurate rate calculation on next call
+        self._prev_net = net
+        self._prev_disk = disk
+        self._prev_time = now
 
         # Add GPU metrics (NVIDIA only)
         if self.nvidia_initialized:

ultralytics/utils/metrics.py

@@ -309,7 +309,7 @@ class ConfusionMatrix(DataExportMixin):
     Attributes:
         task (str): The type of task, either 'detect' or 'classify'.
         matrix (np.ndarray): The confusion matrix, with dimensions depending on the task.
-        nc (int): The number of category.
+        nc (int): The number of classes.
         names (list[str]): The names of the classes, used as labels on the plot.
         matches (dict): Contains the indices of ground truths and predictions categorized into TP, FP and FN.
     """

ultralytics/utils/ops.py

@@ -298,7 +298,7 @@ def xywh2ltwh(x):
         x (np.ndarray | torch.Tensor): Input bounding box coordinates in xywh format.
 
     Returns:
-        (np.ndarray | torch.Tensor): Bounding box coordinates in xyltwh format.
+        (np.ndarray | torch.Tensor): Bounding box coordinates in ltwh format.
     """
     y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)
     y[..., 0] = x[..., 0] - x[..., 2] / 2  # top left x
@@ -313,7 +313,7 @@ def xyxy2ltwh(x):
         x (np.ndarray | torch.Tensor): Input bounding box coordinates in xyxy format.
 
     Returns:
-        (np.ndarray | torch.Tensor): Bounding box coordinates in xyltwh format.
+        (np.ndarray | torch.Tensor): Bounding box coordinates in ltwh format.
     """
     y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)
     y[..., 2] = x[..., 2] - x[..., 0]  # width
@@ -325,7 +325,7 @@ def ltwh2xywh(x):
     """Convert bounding boxes from [x1, y1, w, h] to [x, y, w, h] where xy1=top-left, xy=center.
 
     Args:
-        x (torch.Tensor): Input bounding box coordinates.
+        x (np.ndarray | torch.Tensor): Input bounding box coordinates.
 
     Returns:
         (np.ndarray | torch.Tensor): Bounding box coordinates in xywh format.
@@ -398,8 +398,8 @@ def ltwh2xyxy(x):
         (np.ndarray | torch.Tensor): Bounding box coordinates in xyxy format.
     """
     y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)
-    y[..., 2] = x[..., 2] + x[..., 0]  # width
-    y[..., 3] = x[..., 3] + x[..., 1]  # height
+    y[..., 2] = x[..., 2] + x[..., 0]  # x2
+    y[..., 3] = x[..., 3] + x[..., 1]  # y2
     return y
 
 
@@ -655,7 +655,7 @@ def clean_str(s):
     Returns:
         (str): A string with special characters replaced by an underscore _.
     """
-    return re.sub(pattern="[|@#!¡·$€%&()=?¿^*;:,¨´><+]", repl="_", string=s)
+    return re.sub(pattern="[|@#!¡·$€%&()=?¿^*;:,¨`><+]", repl="_", string=s)
 
 
 def empty_like(x):

ultralytics/utils/patches.py

@@ -35,7 +35,7 @@ def imread(filename: str, flags: int = cv2.IMREAD_COLOR) -> np.ndarray | None:
     if filename.endswith((".tiff", ".tif")):
         success, frames = cv2.imdecodemulti(file_bytes, cv2.IMREAD_UNCHANGED)
         if success:
-            # Handle RGB images in tif/tiff format
+            # Handle multi-frame TIFFs and color images
             return frames[0] if len(frames) == 1 and frames[0].ndim == 3 else np.stack(frames, axis=2)
         return None
     else:
@@ -105,8 +105,8 @@ def torch_load(*args, **kwargs):
         (Any): The loaded PyTorch object.
 
     Notes:
-        For PyTorch versions 2.0 and above, this function automatically sets 'weights_only=False'
-        if the argument is not provided, to avoid deprecation warnings.
+        For PyTorch versions 1.13 and above, this function automatically sets `weights_only=False` if the argument is
+        not provided, to avoid deprecation warnings.
     """
     from ultralytics.utils.torch_utils import TORCH_1_13
 

ultralytics/utils/plotting.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import math
-import warnings
 from collections.abc import Callable
 from pathlib import Path
 from typing import Any
@@ -511,7 +510,7 @@ class Annotator:
             cv2.putText(self.im, text, xy, 0, self.sf, txt_color, thickness=self.tf, lineType=cv2.LINE_AA)
 
     def fromarray(self, im):
-        """Update self.im from a numpy array."""
+        """Update `self.im` from a NumPy array or PIL image."""
         self.im = im if isinstance(im, Image.Image) else Image.fromarray(im)
         self.draw = ImageDraw.Draw(self.im)
 
@@ -522,8 +521,8 @@ class Annotator:
 
     def show(self, title: str | None = None):
         """Show the annotated image."""
-        im = Image.fromarray(np.asarray(self.im)[..., ::-1])  # Convert numpy array to PIL Image with RGB to BGR
-        if IS_COLAB or IS_KAGGLE:  # can not use IS_JUPYTER as will run for all ipython environments
+        im = Image.fromarray(np.asarray(self.im)[..., ::-1])  # Convert BGR NumPy array to RGB PIL Image
+        if IS_COLAB or IS_KAGGLE:  # cannot use IS_JUPYTER as it runs for all IPython environments
             try:
                 display(im)  # noqa - display() function only available in ipython environments
             except ImportError as e:
@@ -536,11 +535,11 @@ class Annotator:
         cv2.imwrite(filename, np.asarray(self.im))
 
     @staticmethod
-    def get_bbox_dimension(bbox: tuple | None = None):
+    def get_bbox_dimension(bbox: tuple | list):
         """Calculate the dimensions and area of a bounding box.
 
         Args:
-            bbox (tuple): Bounding box coordinates in the format (x_min, y_min, x_max, y_max).
+            bbox (tuple | list): Bounding box coordinates in the format (x_min, y_min, x_max, y_max).
 
         Returns:
             width (float): Width of the bounding box.
@@ -575,10 +574,6 @@ def plot_labels(boxes, cls, names=(), save_dir=Path(""), on_plot=None):
     import polars
     from matplotlib.colors import LinearSegmentedColormap
 
-    # Filter matplotlib>=3.7.2 warning
-    warnings.filterwarnings("ignore", category=UserWarning, message="The figure layout has changed to tight")
-    warnings.filterwarnings("ignore", category=FutureWarning)
-
     # Plot dataset labels
     LOGGER.info(f"Plotting labels to {save_dir / 'labels.jpg'}... ")
     nc = int(cls.max() + 1)  # number of classes
@@ -600,8 +595,8 @@ def plot_labels(boxes, cls, names=(), save_dir=Path(""), on_plot=None):
         ax[0].set_xlabel("classes")
     boxes = np.column_stack([0.5 - boxes[:, 2:4] / 2, 0.5 + boxes[:, 2:4] / 2]) * 1000
     img = Image.fromarray(np.ones((1000, 1000, 3), dtype=np.uint8) * 255)
-    for cls, box in zip(cls[:500], boxes[:500]):
-        ImageDraw.Draw(img).rectangle(box.tolist(), width=1, outline=colors(cls))  # plot
+    for class_id, box in zip(cls[:500], boxes[:500]):
+        ImageDraw.Draw(img).rectangle(box.tolist(), width=1, outline=colors(class_id))  # plot
     ax[1].imshow(img)
     ax[1].axis("off")
 

ultralytics/utils/tuner.py

@@ -77,7 +77,7 @@ def run_ray_tune(
         "perspective": tune.uniform(0.0, 0.001),  # image perspective (+/- fraction), range 0-0.001
         "flipud": tune.uniform(0.0, 1.0),  # image flip up-down (probability)
         "fliplr": tune.uniform(0.0, 1.0),  # image flip left-right (probability)
-        "bgr": tune.uniform(0.0, 1.0),  # image channel BGR (probability)
+        "bgr": tune.uniform(0.0, 1.0),  # swap RGB↔BGR channels (probability)
         "mosaic": tune.uniform(0.0, 1.0),  # image mosaic (probability)
         "mixup": tune.uniform(0.0, 1.0),  # image mixup (probability)
         "cutmix": tune.uniform(0.0, 1.0),  # image cutmix (probability)

ultralytics/models/sam/sam3/tokenizer_ve.py

@@ -1,242 +0,0 @@
-# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
-
-# Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved
-
-"""
-Text Tokenizer.
-
-Copied and lightly adapted from VE repo, which in turn copied
-from open_clip and openAI CLIP.
-"""
-
-from __future__ import annotations
-
-import gzip
-import html
-import io
-import os
-import string
-from functools import lru_cache
-
-import ftfy
-import regex as re
-import torch
-from iopath.common.file_io import g_pathmgr
-
-
-@lru_cache
-def bytes_to_unicode():
-    """Returns list of utf-8 byte and a corresponding list of unicode strings. The reversible bpe codes work on unicode
-    strings. This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. When
-    you're at something like a 10B token dataset you end up needing around 5K for decent coverage. This is a
-    significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup tables between utf-8
-    bytes and unicode strings. And avoids mapping to whitespace/control characters the bpe code barfs on.
-    """
-    bs = list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
-    cs = bs[:]
-    n = 0
-    for b in range(2**8):
-        if b not in bs:
-            bs.append(b)
-            cs.append(2**8 + n)
-            n += 1
-    cs = [chr(n) for n in cs]
-    return dict(zip(bs, cs))
-
-
-def get_pairs(word):
-    """Return set of symbol pairs in a word. Word is represented as tuple of symbols (symbols being variable-length
-    strings).
-    """
-    pairs = set()
-    prev_char = word[0]
-    for char in word[1:]:
-        pairs.add((prev_char, char))
-        prev_char = char
-    return pairs
-
-
-def basic_clean(text):
-    """Basic text cleaning: fix unicode and unescape HTML entities."""
-    text = ftfy.fix_text(text)
-    text = html.unescape(html.unescape(text))
-    return text.strip()
-
-
-def whitespace_clean(text):
-    """Remove redundant whitespace."""
-    text = re.sub(r"\s+", " ", text)
-    text = text.strip()
-    return text
-
-
-def _clean_canonicalize(x):
-    """Clean text and canonicalize it."""
-    # basic, remove whitespace, remove punctuation, lower case
-    return canonicalize_text(basic_clean(x))
-
-
-def _clean_lower(x):
-    """Clean text and return lowercase."""
-    # basic, remove whitespace, lower case
-    return whitespace_clean(basic_clean(x)).lower()
-
-
-def _clean_whitespace(x):
-    """Clean text and remove redundant whitespace."""
-    # basic, remove whitespace
-    return whitespace_clean(basic_clean(x))
-
-
-def get_clean_fn(type: str):
-    """Get text cleaning function by name."""
-    if type == "canonicalize":
-        return _clean_canonicalize
-    elif type == "lower":
-        return _clean_lower
-    elif type == "whitespace":
-        return _clean_whitespace
-    else:
-        assert False, f"Invalid clean function ({type})."
-
-
-def canonicalize_text(text, *, keep_punctuation_exact_string=None):
-    """Returns canonicalized `text` (lowercase and punctuation removed). From:
-    https://github.com/google-research/big_vision/blob/53f18caf27a9419231bbf08d3388b07671616d3d/big_vision/evaluators/proj/image_text/prompt_engineering.py#L94.
-
-    Args:
-        text: string to be canonicalized.
-        keep_punctuation_exact_string: If provided, then this exact string kept. For example providing '{}' will keep
-            any occurrences of '{}' (but will still remove '{' and '}' that appear separately).
-    """
-    text = text.replace("_", " ")
-    if keep_punctuation_exact_string:
-        text = keep_punctuation_exact_string.join(
-            part.translate(str.maketrans("", "", string.punctuation))
-            for part in text.split(keep_punctuation_exact_string)
-        )
-    else:
-        text = text.translate(str.maketrans("", "", string.punctuation))
-    text = text.lower()
-    text = re.sub(r"\s+", " ", text)
-    return text.strip()
-
-
-class SimpleTokenizer:
-    """A simple tokenizer for text inputs."""
-
-    def __init__(
-        self,
-        bpe_path: str | os.PathLike,
-        additional_special_tokens: list[str] | None = None,
-        context_length: int = 77,
-        clean: str = "lower",
-    ):
-        """The tokenizer for text inputs."""
-        self.byte_encoder = bytes_to_unicode()
-        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
-        with g_pathmgr.open(bpe_path, "rb") as fh:
-            bpe_bytes = io.BytesIO(fh.read())
-            merges = gzip.open(bpe_bytes).read().decode("utf-8").split("\n")
-        # merges = gzip.open(bpe_path).read().decode("utf-8").split("\n")
-        merges = merges[1 : 49152 - 256 - 2 + 1]
-        merges = [tuple(merge.split()) for merge in merges]
-        vocab = list(bytes_to_unicode().values())
-        vocab = vocab + [v + "</w>" for v in vocab]
-        for merge in merges:
-            vocab.append("".join(merge))
-        special_tokens = ["<start_of_text>", "<end_of_text>"]
-        if additional_special_tokens:
-            special_tokens += additional_special_tokens
-        vocab.extend(special_tokens)
-        self.encoder = dict(zip(vocab, range(len(vocab))))
-        self.decoder = {v: k for k, v in self.encoder.items()}
-        self.bpe_ranks = dict(zip(merges, range(len(merges))))
-        self.cache = {t: t for t in special_tokens}
-        special = "|".join(special_tokens)
-        self.pat = re.compile(
-            special + r"""|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""",
-            re.IGNORECASE,
-        )
-        self.vocab_size = len(self.encoder)
-        self.all_special_ids = [self.encoder[t] for t in special_tokens]
-        self.sot_token_id = self.all_special_ids[0]
-        self.eot_token_id = self.all_special_ids[1]
-        self.context_length = context_length
-        self.clean_fn = get_clean_fn(clean)
-
-    def bpe(self, token):
-        """Byte Pair Encoding."""
-        if token in self.cache:
-            return self.cache[token]
-        word = (*tuple(token[:-1]), token[-1] + "</w>")
-        pairs = get_pairs(word)
-        if not pairs:
-            return token + "</w>"
-        while True:
-            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
-            if bigram not in self.bpe_ranks:
-                break
-            first, second = bigram
-            new_word = []
-            i = 0
-            while i < len(word):
-                try:
-                    j = word.index(first, i)
-                    new_word.extend(word[i:j])
-                    i = j
-                except Exception:
-                    new_word.extend(word[i:])
-                    break
-                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
-                    new_word.append(first + second)
-                    i += 2
-                else:
-                    new_word.append(word[i])
-                    i += 1
-            new_word = tuple(new_word)
-            word = new_word
-            if len(word) == 1:
-                break
-            else:
-                pairs = get_pairs(word)
-        word = " ".join(word)
-        self.cache[token] = word
-        return word
-
-    def encode(self, text):
-        """Encode text to a sequence of BPE tokens."""
-        bpe_tokens = []
-        text = self.clean_fn(text)
-        for token in re.findall(self.pat, text):
-            token = "".join(self.byte_encoder[b] for b in token.encode("utf-8"))
-            bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(" "))
-        return bpe_tokens
-
-    def decode(self, tokens):
-        """Decodes a sequence of tokens back into a text string."""
-        text = "".join([self.decoder[token] for token in tokens])
-        text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors="replace").replace("</w>", " ")
-        return text
-
-    def __call__(self, texts: str | list[str], context_length: int | None = None) -> torch.LongTensor:
-        """Returns the tokenized representation of given input string(s) Parameters. ---------- texts : Union[str,
-        list[str]] An input string or a list of input strings to tokenize context_length : int The context
-        length to use; all CLIP models use 77 as the context length.
-
-        Returns:
-            -------: A two-dimensional tensor containing the resulting tokens, shape = [number of input strings,
-                context_length]
-        """
-        if isinstance(texts, str):
-            texts = [texts]
-        context_length = context_length or self.context_length
-        assert context_length, "Please set a valid context length"
-        all_tokens = [[self.sot_token_id, *self.encode(text), self.eot_token_id] for text in texts]
-        result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
-        for i, tokens in enumerate(all_tokens):
-            if len(tokens) > context_length:
-                tokens = tokens[:context_length]  # Truncate
-                tokens[-1] = self.eot_token_id
-            result[i, : len(tokens)] = torch.tensor(tokens)
-        return result