ebm4subjects 0.5.3__tar.gz → 0.5.5__tar.gz

{ebm4subjects-0.5.3 → ebm4subjects-0.5.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ebm4subjects
-Version: 0.5.3
+Version: 0.5.5
 Summary: Embedding Based Matching for Automated Subject Indexing
 Author: Deutsche Nationalbibliothek
 Maintainer-email: Clemens Rietdorf <c.rietdorf@dnb.de>, Maximilian Kähler <m.kaehler@dnb.de>
@@ -14,12 +14,14 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.10
 Requires-Dist: duckdb>=1.3.0
 Requires-Dist: nltk~=3.9.1
+Requires-Dist: openai>=2.15.0
 Requires-Dist: polars>=1.30.0
 Requires-Dist: pyarrow>=21.0.0
 Requires-Dist: pyoxigraph>=0.4.11
 Requires-Dist: rdflib~=7.1.3
-Requires-Dist: sentence-transformers>=5.0.0
 Requires-Dist: xgboost>=3.0.2
+Provides-Extra: in-process
+Requires-Dist: sentence-transformers>=5.0.0; extra == 'in-process'
 Description-Content-Type: text/markdown
 
 # Embedding Based Matching for Automated Subject Indexing

{ebm4subjects-0.5.3 → ebm4subjects-0.5.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ebm4subjects"
-version = "0.5.3"
+version = "0.5.5"
 description = "Embedding Based Matching for Automated Subject Indexing"
 authors = [
     {name = "Deutsche Nationalbibliothek"},
@@ -29,13 +29,15 @@ requires-python = ">=3.10"
 dependencies = [
     "duckdb>=1.3.0",
     "nltk~=3.9.1",
+    "openai>=2.15.0",
     "polars>=1.30.0",
     "pyarrow>=21.0.0",
     "pyoxigraph>=0.4.11",
     "rdflib~=7.1.3",
-    "sentence-transformers>=5.0.0",
     "xgboost>=3.0.2",
 ]
+[project.optional-dependencies]
+in-process=["sentence-transformers>=5.0.0"]
 
 [build-system]
 requires = ["hatchling"]

{ebm4subjects-0.5.3 → ebm4subjects-0.5.5}/src/ebm4subjects/ebm_logging.py

@@ -39,17 +39,17 @@ class EbmLogger:
         else:
             self.logger.setLevel(logging.NOTSET)
 
-        # Create a file handler to log messages to a file
-        log_file_handler = logging.FileHandler(f"{log_path}/ebm.log")
-        log_file_handler.setFormatter(
-            logging.Formatter(
-                "%(asctime)s %(levelname)s: %(message)s",
-                "%Y-%m-%d %H:%M:%S",
+        # Create a file handler to log messages to a file 
+        if not self.logger.handlers:
+            log_file_handler = logging.FileHandler(f"{log_path}/ebm.log")
+            log_file_handler.setFormatter(
+                logging.Formatter(
+                    "%(asctime)s %(levelname)s: %(message)s",
+                    "%Y-%m-%d %H:%M:%S",
+                )
             )
-        )
 
-        # Add the file handler to the logger
-        self.logger.addHandler(log_file_handler)
+            self.logger.addHandler(log_file_handler)
 
     def get_logger(self) -> logging.Logger:
         """

{ebm4subjects-0.5.3 → ebm4subjects-0.5.5}/src/ebm4subjects/ebm_model.py

@@ -15,8 +15,9 @@ from ebm4subjects.duckdb_client import Duckdb_client
 from ebm4subjects.ebm_logging import EbmLogger, NullLogger, XGBLogging
 from ebm4subjects.embedding_generator import (
     EmbeddingGeneratorHuggingFaceTEI,
-    EmbeddingGeneratorOfflineInference,
     EmbeddingGeneratorMock,
+    EmbeddingGeneratorInProcess,
+    EmbeddingGeneratorOpenAI,
 )
 
 
@@ -43,12 +44,13 @@ class EbmModel:
         use_altLabels: bool = True,
         hnsw_index_params: dict | str | None = None,
         embedding_model_name: str | None = None,
-        embedding_model_type: str = "offline-inference",
+        embedding_model_deployment: str = "offline-inference",
         embedding_model_args: dict | str | None = None,
         encode_args_vocab: dict | str | None = None,
         encode_args_documents: dict | str | None = None,
         log_path: str | None = None,
         logger: logging.Logger | None = None,
+        logging_level: str = "info",
     ) -> None:
         """
         A class representing an Embedding-Based-Matching (EBM) model
@@ -99,7 +101,7 @@ class EbmModel:
 
         # Parameters for embedding generator
         self.generator = None
-        self.embedding_model_type = embedding_model_type
+        self.embedding_model_deployment = embedding_model_deployment
         self.embedding_model_name = embedding_model_name
         self.embedding_dimensions = int(embedding_dimensions)
         if isinstance(embedding_model_args, str) or not embedding_model_args:
@@ -138,7 +140,7 @@ class EbmModel:
         self.train_jobs = int(xgb_jobs)
 
         # Initiliaze logging
-        self.init_logger(log_path, logger)
+        self.init_logger(log_path, logger, logging_level)
 
         # Initialize EBM model
         self.model = None
@@ -179,20 +181,31 @@ class EbmModel:
             None
         """
         if self.generator is None:
-            if self.embedding_model_type == "offline-inference":
+            if self.embedding_model_deployment == "in-process":
                 self.logger.info("initializing offline-inference embedding generator")
-                self.generator = EmbeddingGeneratorOfflineInference(
+                self.generator = EmbeddingGeneratorInProcess(
                     model_name=self.embedding_model_name,
                     embedding_dimensions=self.embedding_dimensions,
+                    logger=self.logger,
                     **self.embedding_model_args,
                 )
-            elif self.embedding_model_type == "mock":
+            elif self.embedding_model_deployment == "mock":
                 self.logger.info("initializing mock embedding generator")
                 self.generator = EmbeddingGeneratorMock(self.embedding_dimensions)
-            elif self.embedding_model_type == "HuggingFaceTEI":
+            elif self.embedding_model_deployment == "HuggingFaceTEI":
                 self.logger.info("initializing API embedding generator")
                 self.generator = EmbeddingGeneratorHuggingFaceTEI(
+                    model_name=self.embedding_model_name,
+                    embedding_dimensions=self.embedding_dimensions,
+                    logger=self.logger,
+                    **self.embedding_model_args,
+                )
+            elif self.embedding_model_deployment == "OpenAI":
+                self.logger.info("initializing API embedding generator")
+                self.generator = EmbeddingGeneratorOpenAI(
+                    model_name=self.embedding_model_name,
                     embedding_dimensions=self.embedding_dimensions,
+                    logger=self.logger,
                     **self.embedding_model_args,
                 )
             else:
@@ -200,7 +213,10 @@ class EbmModel:
                 raise NotImplementedError
 
     def init_logger(
-        self, log_path: str | None = None, logger: logging.Logger | None = None
+        self,
+        log_path: str | None = None,
+        logger: logging.Logger | None = None,
+        logging_level: str = "info",
     ) -> None:
         """
         Initializes the logging for the EBM model.
@@ -209,7 +225,7 @@ class EbmModel:
             None
         """
         if log_path:
-            self.logger = EbmLogger(log_path, "info").get_logger()
+            self.logger = EbmLogger(log_path, logging_level).get_logger()
             self.xgb_logger = XGBLogging(self.logger, epoch_log_interval=1)
             self.xgb_callbacks = [self.xgb_logger]
         elif logger:
@@ -760,7 +776,4 @@ class EbmModel:
         Returns:
             EbmModel: The loaded EBM model instance.
         """
-        ebm_model = joblib.load(input_path)
-        ebm_model.init_logger()
-
-        return ebm_model
+        return joblib.load(input_path)

ebm4subjects-0.5.5/src/ebm4subjects/embedding_generator.py

@@ -0,0 +1,367 @@
+import logging
+import os
+
+import numpy as np
+import requests
+from openai import BadRequestError, NotFoundError, OpenAI
+from tqdm import tqdm
+
+
+class EmbeddingGenerator:
+    """
+    A base class for embedding generators.
+    """
+
+    def __init__(self) -> None:
+        """
+        Base method fot the initialization of an EmbeddingGenerator.
+        """
+        pass
+
+    def generate_embeddings(self, texts: list[str], **kwargs) -> np.ndarray:
+        """
+        Base method fot the creating embeddings with an EmbeddingGenerator.
+
+        Args:
+            texts (list[str]): A list of input texts.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            np.ndarray: A numpy array of shape (len(texts), embedding_dimensions)
+                containing the generated embeddings.
+        """
+        pass
+
+
+class EmbeddingGeneratorHuggingFaceTEI(EmbeddingGenerator):
+    """
+    A class for generating embeddings using the HuggingFaceTEI API.
+    """
+
+    def __init__(
+        self,
+        model_name: str,
+        embedding_dimensions: int,
+        logger: logging.Logger,
+        **kwargs,
+    ) -> None:
+        """
+        Initializes the HuggingFaceTEI API EmbeddingGenerator.
+
+        Sets the embedding dimensions, and initiliazes and
+        prepares a session with the API.
+
+        Args:
+            model_name (str): The name of the SentenceTransformer model.
+            embedding_dimensions (int): The dimensionality of the generated embeddings.
+            logger (Logger): A logger for the embedding generator.
+            **kwargs: Additional keyword arguments to pass to the model.
+        """
+
+        self.embedding_dimensions = embedding_dimensions
+        self.model_name = model_name
+        self.session = requests.Session()
+        self.api_address = kwargs.get("api_address")
+        self.headers = kwargs.get("headers", {"Content-Type": "application/json"})
+
+        self.logger = logger
+        self._test_api()
+
+    def _test_api(self):
+        """
+        Tests if the API is working with the given parameters
+        """
+        response = self.session.post(
+            self.api_address,
+            headers=self.headers,
+            json={"inputs": "This is a test request!", "truncate": True},
+        )
+        if response.status_code == 200:
+            self.logger.debug(
+                "API call successful. Everything seems to be working fine."
+            )
+        elif response.status_code == 404:
+            self.logger.error(
+                "API not found under given adress! Please check the corresponding parameter!"
+            )
+            raise RuntimeError(
+                "API not found under given adress! Please check the corresponding parameter!"
+            )
+        else:
+            self.logger.error(
+                "Request to API not possible! Please check the corresponding parameters!"
+            )
+            raise RuntimeError(
+                "Request to API not possible! Please check the corresponding parameters!"
+            )
+
+    def generate_embeddings(self, texts: list[str], **kwargs) -> np.ndarray:
+        """
+        Generates embeddings for a list of input texts using a model
+        via the HuggingFaceTEI API.
+
+        Args:
+            texts (list[str]): A list of input texts.
+            **kwargs: Additional keyword arguments to pass to the API.
+
+        Returns:
+            np.ndarray: A numpy array of shape (len(texts), embedding_dimensions)
+                containing the generated embeddings.
+        """
+        # prepare list for return
+        embeddings = []
+
+        # Check if the input list is empty
+        if not texts:
+            # If empty, return an empty numpy array with the correct shape
+            return np.empty((0, self.embedding_dimensions))
+
+        # Process in smaller batches to avoid memory overload
+        batch_size = min(32, len(texts))  # HuggingFaceTEI has a limit of 32 as default
+
+        for i in tqdm(range(0, len(texts), batch_size), desc="Processing batches"):
+            batch_texts = texts[i : i + batch_size]
+            # send a request to the HuggingFaceTEI API
+            data = {"inputs": batch_texts, "truncate": True}
+            response = self.session.post(
+                self.api_address, headers=self.headers, json=data
+            )
+
+            # add generated embeddings to return list if request was successfull
+            if response.status_code == 200:
+                embeddings.extend(response.json())
+            else:
+                self.logger.warn("Call to API NOT successful! Returning 0's.")
+                for _ in batch_texts:
+                    embeddings.append(
+                        [
+                            0
+                            for _ in range(
+                                min(
+                                    self.embedding_dimensions,
+                                    kwargs.get("truncate_prompt_tokens", float("inf")),
+                                ),
+                            )
+                        ]
+                    )
+
+        return np.array(embeddings)
+
+
+class EmbeddingGeneratorOpenAI(EmbeddingGenerator):
+    """
+    A class for generating embeddings using any OpenAI compatible API.
+    """
+
+    def __init__(
+        self,
+        model_name: str,
+        embedding_dimensions: int,
+        logger: logging.Logger,
+        **kwargs,
+    ) -> None:
+        """
+        Initializes the OpenAI API EmbeddingGenerator.
+
+        Sets the embedding dimensions, and initiliazes and
+        prepares a session with the API.
+
+        Args:
+            model_name (str): The name of the SentenceTransformer model.
+            embedding_dimensions (int): The dimensionality of the generated embeddings.
+            logger (Logger): A logger for the embedding generator.
+            **kwargs: Additional keyword arguments to pass to the model.
+        """
+
+        self.embedding_dimensions = embedding_dimensions
+        self.model_name = model_name
+
+        if not (api_key := os.environ.get("OPENAI_API_KEY")):
+            api_key = ""
+
+        self.client = OpenAI(api_key=api_key, base_url=kwargs.get("api_address"))
+
+        self.logger = logger
+        self._test_api()
+
+    def _test_api(self):
+        """
+        Tests if the API is working with the given parameters
+        """
+        try:
+            _ = self.client.embeddings.create(
+                input="This is a test request!",
+                model=self.model_name,
+                encoding_format="float",
+            )
+            self.logger.debug(
+                "API call successful. Everything seems to be working fine."
+            )
+        except NotFoundError:
+            self.logger.error(
+                "API not found under given adress! Please check the corresponding parameter!"
+            )
+            raise RuntimeError(
+                "API not found under given adress! Please check the corresponding parameter!"
+            )
+        except BadRequestError:
+            self.logger.error(
+                "Request to API not possible! Please check the corresponding parameters!"
+            )
+            raise RuntimeError(
+                "Request to API not possible! Please check the corresponding parameters!"
+            )
+
+    def generate_embeddings(self, texts: list[str], **kwargs) -> np.ndarray:
+        """
+        Generates embeddings for a list of input texts using a model
+        via an OpenAI compatible API.
+
+        Args:
+            texts (list[str]): A list of input texts.
+            **kwargs: Additional keyword arguments to pass to the API.
+
+        Returns:
+            np.ndarray: A numpy array of shape (len(texts), embedding_dimensions)
+                containing the generated embeddings.
+        """
+        # prepare list for return
+        embeddings = []
+
+        # Check if the input list is empty
+        if not texts:
+            # If empty, return an empty numpy array with the correct shape
+            return np.empty((0, self.embedding_dimensions))
+
+        # Process in smaller batches to avoid memory overload
+        batch_size = min(200, len(texts))
+        embeddings = []
+
+        for i in tqdm(range(0, len(texts), batch_size), desc="Processing batches"):
+            batch_texts = texts[i : i + batch_size]
+
+            # Try to get embeddings for the batch from the API
+            try:
+                embedding_response = self.client.embeddings.create(
+                    input=batch_texts,
+                    model=self.model_name,
+                    encoding_format="float",
+                    extra_body={**kwargs},
+                )
+
+                # Process all embeddings from the batch response
+                for i, _ in enumerate(batch_texts):
+                    embeddings.append(embedding_response.data[i].embedding)
+            except (NotFoundError, BadRequestError):
+                self.logger.warn("Call to API NOT successful! Returning 0's.")
+                for _ in batch_texts:
+                    embeddings.append([0 for _ in range(self.embedding_dimensions)])
+
+        return np.array(embeddings)
+
+
+class EmbeddingGeneratorInProcess(EmbeddingGenerator):
+    """
+    A class for generating embeddings using a given SentenceTransformer model
+    loaded in-process with SentenceTransformer.
+
+    Args:
+        model_name (str): The name of the SentenceTransformer model.
+        embedding_dimensions (int): The dimensionality of the generated embeddings.
+        logger (Logger): A logger for the embedding generator.
+        **kwargs: Additional keyword arguments to pass to the model.
+    """
+
+    def __init__(
+        self,
+        model_name: str,
+        embedding_dimensions: int,
+        logger: logging.Logger,
+        **kwargs,
+    ) -> None:
+        """
+        Initializes the EmbeddingGenerator in 'in-process' mode.
+
+        Sets the model name, embedding dimensions, and creates a
+        SentenceTransformer model instance.
+        """
+        from sentence_transformers import SentenceTransformer
+
+        self.model_name = model_name
+        self.embedding_dimensions = embedding_dimensions
+
+        # Create a SentenceTransformer model instance with the given
+        # model name and embedding dimensions
+        self.model = SentenceTransformer(
+            model_name, truncate_dim=embedding_dimensions, **kwargs
+        )
+        self.logger = logger
+        self.logger.debug(f"SentenceTransfomer model running on {self.model.device}")
+
+        # Disabel parallelism for tokenizer
+        # Needed because process might be already parallelized
+        # before embedding creation
+        os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+    def generate_embeddings(self, texts: list[str], **kwargs) -> np.ndarray:
+        """
+        Generates embeddings for a list of input texts using the
+        SentenceTransformer model.
+
+        Args:
+            texts (list[str]): A list of input texts.
+            **kwargs: Additional keyword arguments to pass to the
+                SentenceTransformer model.
+
+        Returns:
+            np.ndarray: A numpy array of shape (len(texts), embedding_dimensions)
+                containing the generated embeddings.
+        """
+        # Check if the input list is empty
+        if not texts:
+            # If empty, return an empty numpy array with the correct shape
+            return np.empty((0, self.embedding_dimensions))
+
+        # Generate embeddings using the SentenceTransformer model and return them
+        return self.model.encode(texts, **kwargs)
+
+
+class EmbeddingGeneratorMock(EmbeddingGenerator):
+    """
+    A mock class for generating fake embeddings. Used for testing.
+
+    Args:
+        embedding_dimensions (int): The dimensionality of the generated embeddings.
+        **kwargs: Additional keyword arguments to pass to the model.
+
+    Attributes:
+        embedding_dimensions (int): The dimensionality of the generated embeddings.
+    """
+
+    def __init__(self, embedding_dimensions: int, **kwargs) -> None:
+        """
+        Initializes the mock EmbeddingGenerator.
+
+        Sets the embedding dimensions.
+        """
+        self.embedding_dimensions = embedding_dimensions
+
+    def generate_embeddings(self, texts: list[str], **kwargs) -> np.ndarray:
+        """
+        Generates embeddings for a list of input texts.
+
+        Args:
+            texts (list[str]): A list of input texts.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            np.ndarray: A numpy array of shape (len(texts), embedding_dimensions)
+                containing the generated embeddings.
+        """
+        # Check if the input list is empty
+        if not texts:
+            # If empty, return an empty numpy array with the correct shape
+            return np.empty((0, self.embedding_dimensions))
+
+        # Generate mock embeddings return them
+        return np.ones((len(texts), 1024))

{ebm4subjects-0.5.3 → ebm4subjects-0.5.5}/tests/test_prepare_data.py

@@ -1,8 +1,10 @@
-import polars as pl
 from pathlib import Path
 
+import polars as pl
+
 from ebm4subjects.prepare_data import parse_vocab
 
+
 def test_parse_vocab_reads_ttl_and_returns_dataframe(tmp_path):
     # Copy the sample vocab.ttl to a temp location
     vocab_src = Path(__file__).parent / "data/vocab.ttl"

ebm4subjects-0.5.3/src/ebm4subjects/embedding_generator.py

@@ -1,202 +0,0 @@
-import os
-
-import numpy as np
-import requests
-from sentence_transformers import SentenceTransformer
-
-
-class EmbeddingGenerator:
-    """
-    A base class for embedding generators.
-    """
-
-    def __init__(self) -> None:
-        """
-        Base method fot the initialization of an EmbeddingGenerator.
-        """
-        pass
-
-    def generate_embeddings(self, texts: list[str], **kwargs) -> np.ndarray:
-        """
-        Base method fot the creating embeddings with an EmbeddingGenerator.
-
-        Args:
-            texts (list[str]): A list of input texts.
-            **kwargs: Additional keyword arguments.
-
-        Returns:
-            np.ndarray: A numpy array of shape (len(texts), embedding_dimensions)
-                containing the generated embeddings.
-        """
-        pass
-
-
-class EmbeddingGeneratorAPI(EmbeddingGenerator):
-    """
-    A base class for API embedding generators.
-
-    Attributes:
-        embedding_dimensions (int): The dimensionality of the generated embeddings.
-    """
-
-    def __init__(
-        self,
-        embedding_dimensions: int,
-        **kwargs,
-    ) -> None:
-        """
-        Initializes the API EmbeddingGenerator.
-
-        Sets the embedding dimensions, and initiliazes and
-        prepares a session with the API.
-        """
-
-        self.embedding_dimensions = embedding_dimensions
-
-        self.session = requests.Session()
-        self.api_address = kwargs.get("api_address")
-        self.headers = kwargs.get("headers", {"Content-Type": "application/json"})
-
-
-class EmbeddingGeneratorHuggingFaceTEI(EmbeddingGeneratorAPI):
-    """
-    A class for generating embeddings using the HuggingFaceTEI API.
-    """
-
-    def generate_embeddings(self, texts: list[str], **kwargs) -> np.ndarray:
-        """
-        Generates embeddings for a list of input texts using a model
-        via the HuggingFaceTEI API.
-
-        Args:
-            texts (list[str]): A list of input texts.
-            **kwargs: Additional keyword arguments to pass to the
-                SentenceTransformer model.
-
-        Returns:
-            np.ndarray: A numpy array of shape (len(texts), embedding_dimensions)
-                containing the generated embeddings.
-        """
-        # prepare list for return
-        embeddings = []
-
-        # Check if the input list is empty
-        if not texts:
-            # If empty, return an empty numpy array with the correct shape
-            return np.empty((0, self.embedding_dimensions))
-
-        # process each text
-        for text in texts:
-            # send a request to the HuggingFaceTEI API
-            data = {"inputs": text}
-            response = self.session.post(
-                self.api_address, headers=self.headers, json=data
-            )
-
-            # add generated embeddings to return list if request was successfull
-            if response.status_code == 200:
-                embeddings.append(response.json()[0])
-            else:
-                embeddings.append([0 for _ in range(self.embedding_dimensions)])
-
-        return np.array(embeddings)
-
-
-class EmbeddingGeneratorOfflineInference(EmbeddingGenerator):
-    """
-    A class for generating embeddings using a given SentenceTransformer model
-    loaded offline with SentenceTransformer.
-
-    Args:
-        model_name (str): The name of the SentenceTransformer model.
-        embedding_dimensions (int): The dimensionality of the generated embeddings.
-        **kwargs: Additional keyword arguments to pass to the model.
-
-    Attributes:
-        model_name (str): The name of the SentenceTransformer model.
-        embedding_dimensions (int): The dimensionality of the generated embeddings.
-    """
-
-    def __init__(self, model_name: str, embedding_dimensions: int, **kwargs) -> None:
-        """
-        Initializes the EmbeddingGenerator in offline inference mode.
-
-        Sets the model name, embedding dimensions, and creates a
-        SentenceTransformer model instance.
-        """
-        self.model_name = model_name
-        self.embedding_dimensions = embedding_dimensions
-
-        # Create a SentenceTransformer model instance with the given
-        # model name and embedding dimensions
-        self.model = SentenceTransformer(
-            model_name, truncate_dim=embedding_dimensions, **kwargs
-        )
-
-        # Disabel parallelism for tokenizer
-        # Needed because process might be already parallelized
-        # before embedding creation
-        os.environ["TOKENIZERS_PARALLELISM"] = "false"
-
-    def generate_embeddings(self, texts: list[str], **kwargs) -> np.ndarray:
-        """
-        Generates embeddings for a list of input texts using the
-        SentenceTransformer model.
-
-        Args:
-            texts (list[str]): A list of input texts.
-            **kwargs: Additional keyword arguments to pass to the
-                SentenceTransformer model.
-
-        Returns:
-            np.ndarray: A numpy array of shape (len(texts), embedding_dimensions)
-                containing the generated embeddings.
-        """
-        # Check if the input list is empty
-        if not texts:
-            # If empty, return an empty numpy array with the correct shape
-            return np.empty((0, self.embedding_dimensions))
-
-        # Generate embeddings using the SentenceTransformer model and return them
-        return self.model.encode(texts, **kwargs)
-
-
-class EmbeddingGeneratorMock(EmbeddingGenerator):
-    """
-    A mock class for generating fake embeddings. Used for testing.
-
-    Args:
-        embedding_dimensions (int): The dimensionality of the generated embeddings.
-        **kwargs: Additional keyword arguments to pass to the model.
-
-    Attributes:
-        embedding_dimensions (int): The dimensionality of the generated embeddings.
-    """
-
-    def __init__(self, embedding_dimensions: int, **kwargs) -> None:
-        """
-        Initializes the mock EmbeddingGenerator.
-
-        Sets the embedding dimensions.
-        """
-        self.embedding_dimensions = embedding_dimensions
-
-    def generate_embeddings(self, texts: list[str], **kwargs) -> np.ndarray:
-        """
-        Generates embeddings for a list of input texts.
-
-        Args:
-            texts (list[str]): A list of input texts.
-            **kwargs: Additional keyword arguments.
-
-        Returns:
-            np.ndarray: A numpy array of shape (len(texts), embedding_dimensions)
-                containing the generated embeddings.
-        """
-        # Check if the input list is empty
-        if not texts:
-            # If empty, return an empty numpy array with the correct shape
-            return np.empty((0, self.embedding_dimensions))
-
-        # Generate mock embeddings return them
-        return np.ones((len(texts), 1024))