autoencoders 0.1.0__py3-none-any.whl

autoencoders/__init__.py

@@ -0,0 +1,181 @@
+"""Top-level package for the autoencoders library."""
+
+from .models.aae.configuration_aae import AdversarialAutoencoderConfig
+from .configuration_utils import PretrainedConfig
+from .modeling_outputs import (
+    AdversarialAutoencoderOutput,
+    AutoencoderExport,
+    AutoencoderOutput,
+    BaseAutoencoderOutput,
+    ContractiveAutoencoderOutput,
+    DenoisingAutoencoderOutput,
+    DenoisingVariationalAutoencoderOutput,
+    FiniteScalarQuantizedAutoencoderOutput,
+    HierarchicalVariationalAutoencoderOutput,
+    KLSparseAutoencoderOutput,
+    QuantizedAutoencoderOutput,
+    SparseAutoencoderOutput,
+    TopKSparseAutoencoderOutput,
+    VariationalAutoencoderOutput,
+    WassersteinAutoencoderOutput,
+)
+from .models.ae.configuration_ae import AutoencoderConfig
+from .models.cae.configuration_cae import ContractiveAutoencoderConfig
+from .models.base.configuration_base import BaseAutoencoderConfig
+from .models.base.configuration_vae import BaseVariationalAutoencoderConfig
+from .models.base.configuration_vq import BaseVectorQuantizedAutoencoderConfig
+from .models.betavae.configuration_betavae import BetaVariationalAutoencoderConfig
+from .models.dae.configuration_dae import DenoisingAutoencoderConfig
+from .models.dvae.configuration_dvae import DenoisingVariationalAutoencoderConfig
+from .models.fsq.configuration_fsq import FiniteScalarQuantizedAutoencoderConfig
+from .models.hvae.configuration_hvae import HierarchicalVariationalAutoencoderConfig
+from .models.klsae.configuration_klsae import KLSparseAutoencoderConfig
+from .models.pqvae.configuration_pqvae import ProductQuantizedAutoencoderConfig
+from .models.rqvae.configuration_rqvae import ResidualQuantizedAutoencoderConfig
+from .models.sae.configuration_sae import SparseAutoencoderConfig
+from .models.topksae.configuration_topksae import TopKSparseAutoencoderConfig
+from .models.vae.configuration_vae import VariationalAutoencoderConfig
+from .models.wae.configuration_wae import WassersteinAutoencoderConfig
+from .models.vqvae.configuration_vqvae import VectorQuantizedAutoencoderConfig
+
+__all__ = [
+    "AdversarialAutoencoderConfig",
+    "AdversarialAutoencoderOutput",
+    "AutoencoderConfig",
+    "AutoencoderExport",
+    "AutoencoderOutput",
+    "BaseAutoencoderOutput",
+    "BaseAutoencoderConfig",
+    "BaseVariationalAutoencoderConfig",
+    "BaseVectorQuantizedAutoencoderConfig",
+    "BetaVariationalAutoencoderConfig",
+    "ContractiveAutoencoderConfig",
+    "ContractiveAutoencoderOutput",
+    "DenoisingAutoencoderConfig",
+    "DenoisingAutoencoderOutput",
+    "DenoisingVariationalAutoencoderConfig",
+    "DenoisingVariationalAutoencoderOutput",
+    "FiniteScalarQuantizedAutoencoderConfig",
+    "FiniteScalarQuantizedAutoencoderOutput",
+    "HierarchicalVariationalAutoencoderConfig",
+    "HierarchicalVariationalAutoencoderOutput",
+    "KLSparseAutoencoderConfig",
+    "KLSparseAutoencoderOutput",
+    "PretrainedConfig",
+    "ProductQuantizedAutoencoderConfig",
+    "QuantizedAutoencoderOutput",
+    "ResidualQuantizedAutoencoderConfig",
+    "SparseAutoencoderConfig",
+    "SparseAutoencoderOutput",
+    "TopKSparseAutoencoderConfig",
+    "TopKSparseAutoencoderOutput",
+    "VariationalAutoencoderConfig",
+    "VariationalAutoencoderOutput",
+    "WassersteinAutoencoderConfig",
+    "WassersteinAutoencoderOutput",
+    "VectorQuantizedAutoencoderConfig",
+]
+
+try:
+    from .models.aae.modeling_aae import AdversarialAutoencoderModel
+    from .modeling_utils import PreTrainedAutoencoderModel
+    from .data import (
+        AutoencoderDataset,
+        CachedDataset,
+        ConceptNetNumberbatchDataset,
+        DatasetLoaders,
+        DatasetSplits,
+        EmbeddingMatrix,
+        EmbeddingTensorDataset,
+        FastTextEnglishDataset,
+        GloVeDataset,
+        create_dataloaders,
+        load_dataset,
+        load_embedding_artifact,
+        load_text_embedding_matrix,
+        split_dataset,
+    )
+    from .models.ae.modeling_ae import AutoencoderModel
+    from .models.base.modeling_base import BaseAutoencoderModel
+    from .models.base.modeling_vae import BaseVariationalAutoencoderModel
+    from .models.base.modeling_vq import BaseVectorQuantizedAutoencoderModel
+    from .models.betavae.modeling_betavae import BetaVariationalAutoencoderModel
+    from .models.cae.modeling_cae import ContractiveAutoencoderModel
+    from .models.dae.modeling_dae import DenoisingAutoencoderModel
+    from .models.dvae.modeling_dvae import DenoisingVariationalAutoencoderModel
+    from .models.fsq.modeling_fsq import FiniteScalarQuantizedAutoencoderModel
+    from .models.hvae.modeling_hvae import HierarchicalVariationalAutoencoderModel
+    from .models.klsae.modeling_klsae import KLSparseAutoencoderModel
+    from .models.loading import load_model
+    from .models.pqvae.modeling_pqvae import ProductQuantizedAutoencoderModel
+    from .models.rqvae.modeling_rqvae import ResidualQuantizedAutoencoderModel
+    from .models.sae.modeling_sae import SparseAutoencoderModel
+    from .models.topksae.modeling_topksae import TopKSparseAutoencoderModel
+    from .models.vae.modeling_vae import VariationalAutoencoderModel
+    from .models.wae.modeling_wae import WassersteinAutoencoderModel
+    from .models.vqvae.modeling_vqvae import VectorQuantizedAutoencoderModel
+    from .training import (
+        AETrainer,
+        AdversarialAutoencoderTrainer,
+        AdversarialAutoencoderTrainingArguments,
+        TrainerDisplay,
+        TrainerDisplayConfig,
+        TrainingArguments,
+        VAETrainer,
+        VQTrainer,
+        resolve_device,
+        set_seed,
+    )
+except ModuleNotFoundError as exc:
+    if exc.name != "torch":
+        raise
+else:
+    __all__.extend(
+        [
+            "AETrainer",
+            "AdversarialAutoencoderModel",
+            "AdversarialAutoencoderTrainer",
+            "AdversarialAutoencoderTrainingArguments",
+            "AutoencoderModel",
+            "AutoencoderDataset",
+            "BaseAutoencoderModel",
+            "BaseVariationalAutoencoderModel",
+            "BaseVectorQuantizedAutoencoderModel",
+            "BetaVariationalAutoencoderModel",
+            "CachedDataset",
+            "ContractiveAutoencoderModel",
+            "ConceptNetNumberbatchDataset",
+            "DatasetLoaders",
+            "DatasetSplits",
+            "DenoisingAutoencoderModel",
+            "DenoisingVariationalAutoencoderModel",
+            "EmbeddingMatrix",
+            "EmbeddingTensorDataset",
+            "FastTextEnglishDataset",
+            "FiniteScalarQuantizedAutoencoderModel",
+            "GloVeDataset",
+            "HierarchicalVariationalAutoencoderModel",
+            "KLSparseAutoencoderModel",
+            "PreTrainedAutoencoderModel",
+            "ProductQuantizedAutoencoderModel",
+            "ResidualQuantizedAutoencoderModel",
+            "SparseAutoencoderModel",
+            "TopKSparseAutoencoderModel",
+            "TrainerDisplay",
+            "TrainerDisplayConfig",
+            "TrainingArguments",
+            "VAETrainer",
+            "VariationalAutoencoderModel",
+            "VQTrainer",
+            "VectorQuantizedAutoencoderModel",
+            "WassersteinAutoencoderModel",
+            "create_dataloaders",
+            "load_dataset",
+            "load_model",
+            "load_embedding_artifact",
+            "load_text_embedding_matrix",
+            "resolve_device",
+            "set_seed",
+            "split_dataset",
+        ]
+    )

autoencoders/configuration_utils.py

@@ -0,0 +1,49 @@
+"""Utilities for model configuration objects."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+class PretrainedConfig:
+    """A lightweight configuration base inspired by the transformers API."""
+
+    model_type = "config"
+
+    def __init__(self, **kwargs: Any) -> None:
+        self.return_dict = kwargs.pop("return_dict", True)
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = dict(self.__dict__)
+        payload["model_type"] = self.model_type
+        return payload
+
+    def to_json_string(self) -> str:
+        return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n"
+
+    def save_pretrained(self, save_directory: str | Path) -> Path:
+        save_path = Path(save_directory)
+        save_path.mkdir(parents=True, exist_ok=True)
+        config_path = save_path / "config.json"
+        config_path.write_text(self.to_json_string(), encoding="utf-8")
+        return config_path
+
+    @classmethod
+    def from_dict(cls, config_dict: dict[str, Any], **kwargs: Any) -> "PretrainedConfig":
+        payload = dict(config_dict)
+        payload.pop("model_type", None)
+        payload.update(kwargs)
+        return cls(**payload)
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: str | Path, **kwargs: Any) -> "PretrainedConfig":
+        config_path = Path(pretrained_model_name_or_path)
+        if config_path.is_dir():
+            config_path = config_path / "config.json"
+        config_dict = json.loads(config_path.read_text(encoding="utf-8"))
+        return cls.from_dict(config_dict, **kwargs)
+

autoencoders/data/__init__.py

@@ -0,0 +1,41 @@
+"""Utilities for working with real embedding matrices."""
+
+from .base import (
+    AutoencoderDataset,
+    CachedDataset,
+    DatasetLoaders,
+    DatasetSplits,
+    create_dataloaders,
+    default_cache_dir,
+    split_dataset,
+)
+from .embeddings import (
+    EmbeddingMatrix,
+    EmbeddingTensorDataset,
+    load_embedding_artifact,
+    load_text_embedding_matrix,
+    save_embedding_artifact,
+)
+from .fasttext import FastTextEnglishDataset
+from .glove import GloVeDataset
+from .loading import load_dataset
+from .numberbatch import ConceptNetNumberbatchDataset
+
+__all__ = [
+    "AutoencoderDataset",
+    "CachedDataset",
+    "DatasetLoaders",
+    "DatasetSplits",
+    "EmbeddingMatrix",
+    "EmbeddingTensorDataset",
+    "FastTextEnglishDataset",
+    "GloVeDataset",
+    "ConceptNetNumberbatchDataset",
+    "create_dataloaders",
+    "default_cache_dir",
+    "load_dataset",
+    "load_embedding_artifact",
+    "load_text_embedding_matrix",
+    "save_embedding_artifact",
+    "split_dataset",
+]

autoencoders/data/base.py

@@ -0,0 +1,276 @@
+"""Base dataset abstractions for autoencoder training and evaluation."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+import os
+from pathlib import Path
+import sys
+from typing import Callable
+import urllib.request
+
+import torch
+from torch.utils.data import DataLoader, Dataset, Subset
+
+
+def default_cache_dir() -> Path:
+    """Return the default cache directory for downloadable datasets."""
+
+    cache_dir = os.environ.get("AUTOENCODERS_CACHE")
+    if cache_dir:
+        return Path(cache_dir).expanduser()
+    return Path.home() / ".cache" / "autoencoders"
+
+
+def format_num_bytes(num_bytes: int) -> str:
+    """Format a byte count into a compact human-readable string."""
+
+    value = float(num_bytes)
+    units = ["B", "KB", "MB", "GB", "TB"]
+    for unit in units:
+        if value < 1024.0 or unit == units[-1]:
+            return f"{value:.1f}{unit}"
+        value /= 1024.0
+    return f"{num_bytes}B"
+
+
+class DownloadProgressBar:
+    """A small terminal progress bar for dataset downloads."""
+
+    def __init__(self, description: str, total_bytes: int | None, stream=None) -> None:
+        self.description = description
+        self.total_bytes = total_bytes
+        self.stream = sys.stderr if stream is None else stream
+        self.downloaded_bytes = 0
+        self._finished = False
+        self._render()
+
+    def update(self, chunk_size: int) -> None:
+        self.downloaded_bytes += chunk_size
+        self._render()
+
+    def close(self) -> None:
+        if self._finished:
+            return
+        self._finished = True
+        self._render(final=True)
+
+    def _render(self, final: bool = False) -> None:
+        if self.total_bytes is not None and self.total_bytes > 0:
+            ratio = min(self.downloaded_bytes / self.total_bytes, 1.0)
+            filled = int(ratio * 20)
+            bar = "=" * filled + "." * (20 - filled)
+            percent = int(ratio * 100)
+            message = (
+                f"\r{self.description} [{bar}] {percent:3d}% "
+                f"{format_num_bytes(self.downloaded_bytes)}/{format_num_bytes(self.total_bytes)}"
+            )
+        else:
+            message = f"\r{self.description} {format_num_bytes(self.downloaded_bytes)}"
+
+        if final:
+            message += "\n"
+
+        self.stream.write(message)
+        self.stream.flush()
+
+
+class AutoencoderDataset(Dataset[torch.Tensor]):
+    """Simple dataset contract for autoencoder-friendly tensor samples."""
+
+    split: str
+
+    def __init__(self, split: str = "train") -> None:
+        self.split = split
+
+
+class CachedDataset(ABC):
+    """Base class for datasets that download raw files and cache processed artifacts."""
+
+    dataset_name = "dataset"
+
+    def __init__(self, root: str | Path | None = None) -> None:
+        self.root = Path(root) if root is not None else default_cache_dir()
+        self.dataset_dir = self.root / self.dataset_name
+        self.raw_dir = self.dataset_dir / "raw"
+        self.external_dir = self.dataset_dir / "external"
+        self.processed_dir = self.dataset_dir / "processed"
+
+    def ensure_prepared(
+        self,
+        *,
+        download: bool = True,
+        force_download: bool = False,
+        force_prepare: bool = False,
+    ) -> Path:
+        """Ensure the processed artifact exists and return its directory."""
+
+        artifact_dir = self.artifact_dir
+        if self.is_prepared() and not force_prepare:
+            return artifact_dir
+
+        if download:
+            self.download(force=force_download)
+        elif not self.has_raw_data():
+            raise FileNotFoundError(
+                f"Raw data for {self.dataset_name!r} is missing under {self.raw_dir}."
+            )
+
+        self.prepare()
+        return artifact_dir
+
+    def is_prepared(self) -> bool:
+        """Return True when the processed artifact is complete and ready to use."""
+
+        return self.artifact_dir.exists()
+
+    def download_to_cache(
+        self,
+        *,
+        url: str,
+        destination: Path,
+        validator: Callable[[Path], bool] | None = None,
+        description: str | None = None,
+        force: bool = False,
+        chunk_size: int = 1024 * 1024,
+    ) -> Path:
+        """Download a file atomically with progress reporting and cache validation."""
+
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        temp_path = destination.with_name(f"{destination.name}.tmp")
+        self._cleanup_temp_file(temp_path)
+
+        if destination.exists() and not force:
+            if validator is None or validator(destination):
+                return destination
+            destination.unlink()
+        elif destination.exists():
+            destination.unlink()
+
+        try:
+            with urllib.request.urlopen(url) as response, temp_path.open("wb") as handle:
+                total_bytes = self._response_content_length(response)
+                progress = DownloadProgressBar(description or destination.name, total_bytes)
+                try:
+                    while True:
+                        chunk = response.read(chunk_size)
+                        if not chunk:
+                            break
+                        handle.write(chunk)
+                        progress.update(len(chunk))
+                finally:
+                    progress.close()
+        except Exception:
+            self._cleanup_temp_file(temp_path)
+            raise
+
+        if validator is not None and not validator(temp_path):
+            self._cleanup_temp_file(temp_path)
+            raise ValueError(f"Downloaded file {temp_path} failed validation.")
+
+        temp_path.replace(destination)
+        return destination
+
+    @staticmethod
+    def _cleanup_temp_file(path: Path) -> None:
+        if path.exists():
+            path.unlink()
+
+    @staticmethod
+    def _response_content_length(response) -> int | None:
+        length = response.headers.get("Content-Length")
+        if length is None:
+            return None
+        try:
+            return int(length)
+        except (TypeError, ValueError):
+            return None
+
+    @property
+    @abstractmethod
+    def artifact_dir(self) -> Path:
+        """Directory containing the processed dataset artifact."""
+
+    @abstractmethod
+    def has_raw_data(self) -> bool:
+        """Return True when enough raw files are present to prepare the dataset."""
+
+    @abstractmethod
+    def download(self, *, force: bool = False) -> None:
+        """Download the raw dataset files into the cache."""
+
+    @abstractmethod
+    def prepare(self) -> None:
+        """Convert raw files into a processed artifact."""
+
+
+@dataclass
+class DatasetSplits:
+    """A deterministic set of dataset splits."""
+
+    train: Dataset[torch.Tensor]
+    validation: Dataset[torch.Tensor]
+    test: Dataset[torch.Tensor]
+
+
+def split_dataset(
+    dataset: Dataset[torch.Tensor],
+    *,
+    validation_ratio: float = 0.1,
+    test_ratio: float = 0.1,
+    seed: int = 42,
+) -> DatasetSplits:
+    """Split a dataset into train, validation, and test subsets."""
+
+    if validation_ratio < 0 or test_ratio < 0:
+        raise ValueError("validation_ratio and test_ratio must be non-negative.")
+    if validation_ratio + test_ratio >= 1.0:
+        raise ValueError("validation_ratio + test_ratio must be less than 1.0.")
+
+    num_examples = len(dataset)
+    generator = torch.Generator().manual_seed(seed)
+    permutation = torch.randperm(num_examples, generator=generator).tolist()
+
+    num_validation = int(num_examples * validation_ratio)
+    num_test = int(num_examples * test_ratio)
+    num_train = num_examples - num_validation - num_test
+
+    train_indices = permutation[:num_train]
+    validation_indices = permutation[num_train : num_train + num_validation]
+    test_indices = permutation[num_train + num_validation :]
+
+    return DatasetSplits(
+        train=Subset(dataset, train_indices),
+        validation=Subset(dataset, validation_indices),
+        test=Subset(dataset, test_indices),
+    )
+
+
+@dataclass
+class DatasetLoaders:
+    """Convenience wrapper for train, validation, and test dataloaders."""
+
+    train: DataLoader[torch.Tensor]
+    validation: DataLoader[torch.Tensor]
+    test: DataLoader[torch.Tensor]
+
+
+def create_dataloaders(
+    splits: DatasetSplits,
+    *,
+    batch_size: int = 256,
+    num_workers: int = 0,
+) -> DatasetLoaders:
+    """Create dataloaders from deterministic dataset splits."""
+
+    return DatasetLoaders(
+        train=DataLoader(splits.train, batch_size=batch_size, shuffle=True, num_workers=num_workers),
+        validation=DataLoader(
+            splits.validation,
+            batch_size=batch_size,
+            shuffle=False,
+            num_workers=num_workers,
+        ),
+        test=DataLoader(splits.test, batch_size=batch_size, shuffle=False, num_workers=num_workers),
+    )

autoencoders/data/embeddings.py

@@ -0,0 +1,145 @@
+"""Helpers for loading and packaging embedding matrices."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+import torch
+
+from .base import AutoencoderDataset
+
+
+@dataclass
+class EmbeddingMatrix:
+    """A simple in-memory representation of a token embedding matrix."""
+
+    tokens: list[str]
+    matrix: torch.Tensor
+    token_to_index: dict[str, int]
+    source_path: str | None = None
+    name: str | None = None
+
+    @property
+    def num_embeddings(self) -> int:
+        return int(self.matrix.shape[0])
+
+    @property
+    def embedding_dim(self) -> int:
+        return int(self.matrix.shape[1])
+
+
+class EmbeddingTensorDataset(AutoencoderDataset):
+    """Dataset that exposes each embedding vector as one training sample."""
+
+    def __init__(self, embedding_matrix: EmbeddingMatrix, split: str = "train") -> None:
+        super().__init__(split=split)
+        self.embedding_matrix = embedding_matrix
+
+    def __len__(self) -> int:
+        return self.embedding_matrix.num_embeddings
+
+    def __getitem__(self, index: int) -> torch.Tensor:
+        return self.embedding_matrix.matrix[index]
+
+
+def load_text_embedding_matrix(
+    path: str | Path,
+    *,
+    max_vectors: int | None = None,
+    expected_dim: int | None = None,
+    skip_first_line: bool = False,
+    dtype: torch.dtype = torch.float32,
+) -> EmbeddingMatrix:
+    """Load a whitespace-separated embedding text file such as GloVe."""
+
+    file_path = Path(path)
+    tokens: list[str] = []
+    rows: list[list[float]] = []
+
+    with file_path.open("r", encoding="utf-8") as handle:
+        for line_number, line in enumerate(handle, start=1):
+            if skip_first_line and line_number == 1:
+                continue
+            stripped = line.strip()
+            if not stripped:
+                continue
+
+            parts = stripped.split()
+            if len(parts) < 2:
+                raise ValueError(f"Malformed embedding row at line {line_number}: {stripped!r}")
+
+            token = parts[0]
+            values = parts[1:]
+
+            if expected_dim is None:
+                expected_dim = len(values)
+            elif len(values) != expected_dim:
+                raise ValueError(
+                    f"Embedding dimension mismatch at line {line_number}: "
+                    f"expected {expected_dim}, got {len(values)}"
+                )
+
+            tokens.append(token)
+            rows.append([float(value) for value in values])
+
+            if max_vectors is not None and len(tokens) >= max_vectors:
+                break
+
+    if not rows:
+        raise ValueError(f"No embeddings were loaded from {file_path}.")
+
+    matrix = torch.tensor(rows, dtype=dtype)
+    token_to_index = {token: index for index, token in enumerate(tokens)}
+    return EmbeddingMatrix(
+        tokens=tokens,
+        matrix=matrix,
+        token_to_index=token_to_index,
+        source_path=str(file_path),
+        name=file_path.stem,
+    )
+
+
+def save_embedding_artifact(embedding_matrix: EmbeddingMatrix, output_dir: str | Path) -> Path:
+    """Save a processed embedding matrix in a compact torch-friendly format."""
+
+    save_dir = Path(output_dir)
+    save_dir.mkdir(parents=True, exist_ok=True)
+
+    tensor_path = save_dir / "embeddings.pt"
+    tokens_path = save_dir / "tokens.txt"
+    metadata_path = save_dir / "metadata.json"
+
+    torch.save(embedding_matrix.matrix, tensor_path)
+    tokens_path.write_text("\n".join(embedding_matrix.tokens) + "\n", encoding="utf-8")
+    metadata = {
+        "name": embedding_matrix.name,
+        "source_path": embedding_matrix.source_path,
+        "num_embeddings": embedding_matrix.num_embeddings,
+        "embedding_dim": embedding_matrix.embedding_dim,
+    }
+    metadata_path.write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return save_dir
+
+
+def load_embedding_artifact(path: str | Path) -> EmbeddingMatrix:
+    """Load a processed embedding artifact created by save_embedding_artifact()."""
+
+    load_dir = Path(path)
+    tensor_path = load_dir / "embeddings.pt"
+    tokens_path = load_dir / "tokens.txt"
+    metadata_path = load_dir / "metadata.json"
+
+    matrix = torch.load(tensor_path, map_location="cpu")
+    tokens = tokens_path.read_text(encoding="utf-8").splitlines()
+    metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+    token_to_index = {token: index for index, token in enumerate(tokens)}
+
+    return EmbeddingMatrix(
+        tokens=tokens,
+        matrix=matrix,
+        token_to_index=token_to_index,
+        source_path=metadata.get("source_path"),
+        name=metadata.get("name"),
+    )