dstklib 1.0.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

dstk/predict_models.py

@@ -1,189 +0,0 @@
-from gensim.models import Word2Vec
-import fasttext
-import numpy as np
-from sklearn.metrics.pairwise import cosine_similarity
-from pathlib import Path
-from .workflow_tools import requires, workflow, WorkflowManager
-from .matrix_base import MatrixRepresentation
-
-from .lib_types import ndarray, FastText
-
-# The workflow needs to be completely restructured
-
-STAGES = [
-    "start", # The path to the sentences file
-    "predict_model", # After a model has been generated,
-    "embeddings_operations" # After a metric distance has ben applied
-    "end" # After the model was saved or the model was turned into matrix representation
-]
-
-class PredictModels(WorkflowManager):
-    """
-    Provides a unified interface to work seamlessly with Gensim's Word2Vec and Facebook's FastText models.
-
-    This class simplifies the process of training, loading, and using word embeddings by integrating both popular algorithms under a single API.
-
-    :param path: The path to a file conatining a list of sentences or collocations from which to build word embeddings.
-    """
-
-    _start: str
-    _end: str | MatrixRepresentation
-
-    def __init__(self, path: str | None = None):
-        """
-        Initializes PredictModels with given attributes.
-        """
-
-        super().__init__()
-
-        # Stages
-
-        self._predict_model: Word2Vec | FastText
-        self._embeddings_operations: list[tuple[str,float]] | float
-        
-        self._set_workflow(input_arg=path)
-
-    @requires(stages=["start"])
-    @workflow(input_arg="path", input_process="_start", output_process="_predict_model", next_stage="predict_model")
-    def word2vec(self, *, path: str, **kwargs) -> Word2Vec:
-        """
-        Creates word embeddings using the Word2Vec algorithm.
-
-        :param path: The path to a file conatining a list of sentences or collocations from which to build word embeddings.
-        :param kwargs:  Additional keyword arguments to pass to gensim.models.Word2Vec. Common options include:
-            - vector_size: Size of the word embedding vectors.
-            - workers: Number of CPU cores to be used during the training process.
-            - sg: Training algorithm. 1 for skip-gram; 0 for CBOW (Continuous Bag of Words).
-            - window (int): Maximum distance between the current and predicted word.
-            - min_count (int): Ignores all words with total frequency lower than this.
-
-            For more information check: https://radimrehurek.com/gensim/models/word2vec.html
-        """
-
-        return Word2Vec(
-            corpus_file=path,
-            **kwargs
-        )
-
-    @requires(stages=["start"])
-    @workflow(input_arg="path", input_process="_start", output_process="_predict_model", next_stage="predict_model")
-    def fastText(self, *, path: str, **kwargs) -> FastText:
-        """
-        Creates word embeddings using the FastText algorithm.
-
-        :param path: The path to a file conatining a list of sentences or collocations from which to build word embeddings.
-        :param kwargs: Additional keyword arguments to pass to fasttext.train_unsupervised. 
-            Common options include:
-                - dim: Size of the word embedding vectors.
-                - model: Training algorithm: skipgram or cbow (Continuous Bag of Words)
-                - thread: Number of CPU cores to be used during the training process.
-
-                For more information check: https://fasttext.cc/docs/en/options.html
-        """
-
-        return fasttext.train_unsupervised(
-            path,
-            **kwargs
-        )
-
-    @requires(stages=["start"])
-    @workflow(input_arg="path", input_process="_start", output_process="_predict_model", next_stage="predict_model")
-    def load_model(self, *, path: str) -> Word2Vec | FastText:
-        """
-        Loads the trained embeddings in .model (Word2Vec) or .bin (FastText) format, depending on the algorithm used.
-
-        :param path: Path to the saved model file.
-        """
-
-        extension: str = Path(path).suffix.lower()
-
-        if extension == ".model":
-            return Word2Vec.load(path)
-        elif extension == ".bin":
-            return fasttext.load_model(path)
-        else:
-            raise ValueError(f"Model extension {extension} not recognized.")
-
-    @requires(stages=["predict_model", "embeddings_operations"])
-    @workflow(input_arg="model", input_process="_predict_model", output_process="_end", next_stage="end")
-    def save_model(self, *, model: Word2Vec | FastText, path: str) -> str:
-        """
-        Saves the trained embeddings in .model (Word2Vec) or .bin (FastText) format, depending on the algorithm used.
-
-        :param model: A trained Word2Vec or FastText model.
-        :param path: The path (without extension) where to save the model.
-        """
-        full_path: Path = Path(path)
-
-        if isinstance(model, Word2Vec):
-            model.save(str(full_path.with_suffix(".model")))
-        elif isinstance(model, FastText):
-            model.save_model(str(full_path.with_suffix(".bin")))
-        else:
-            raise NotImplementedError(f"Model identifier type {type(model.__name__)} not yet supported")
-        
-        return str(full_path.resolve())
-
-    @requires(stages=["predict_model", "embeddings_operations"], multiple_calls=True)
-    @workflow(input_arg="model", input_process="_predict_model", output_process="_embeddings_operations", next_stage="embeddings_operations")
-    def nearest_neighbors(self, *, model: Word2Vec | FastText, word: str, n_neighbors: int) -> list[tuple[str,float]]:
-        """
-        Returns the top N most semantically similar words to a given target word.
-
-        :param model: A trained Word2Vec or FastText model.
-        :param word: The target word to find neighbors for.
-        :param n_neighbors: Number of nearest neighbors to return.
-        """
-
-        if isinstance(model, Word2Vec):
-            return model.wv.most_similar(word, topn=n_neighbors)
-        elif isinstance(model, FastText):
-            result: list[tuple[float, str]] = model.get_nearest_neighbors(word, k=n_neighbors)
-            return [(word, score) for score, word in result]
-        else:
-            raise NotImplementedError(f"Model identifier type {type(model.__name__)} not yet supported")
-
-    @requires(stages=["predict_model", "embeddings_operations"], multiple_calls=True)
-    @workflow(input_arg="model", input_process="_predict_model", output_process="_embeddings_operations", next_stage="embeddings_operations")
-    def cos_similarity(self, *, model: Word2Vec | FastText, first_word: str, second_word: str) -> float:
-        """
-        Computes the cosine similarity between the embeddings of two words.
-
-        :param model: A trained Word2Vec or FastText model.
-        :param first_word: The first word in the pair.
-        :param second_word: The second word in the pair. 
-        """
-
-        if isinstance(model, Word2Vec):
-            return float(model.wv.similarity(first_word, second_word))
-        elif isinstance(model, FastText):
-            first_word_vector: ndarray = np.array(model[first_word]).reshape(1, -1)
-            second_word_vector: ndarray = np.array(model[second_word]).reshape(1, -1)
-
-            cos_sim: ndarray = cosine_similarity(first_word_vector, second_word_vector)
-
-            return float(cos_sim[0][0])
-        else:
-            raise NotImplementedError(f"Model identifier type {model.__name__} not yet supported")
-
-    @requires(stages=["predict_model"])
-    @workflow(input_arg="model", input_process="_predict_model", output_process="_end", next_stage="end")
-    def to_matrix(self, *, model: Word2Vec | FastText) -> MatrixRepresentation:
-        """
-        Returns a matrix represenation of the word embeddings and their associated labels.
-
-        :param model: A trained Word2Vec or FastText model.
-        """
-
-        word_vectors: ndarray
-        labels: list[str]
-
-        if isinstance(model, Word2Vec):
-            word_vectors = model.wv[model.wv.index_to_key]
-            labels = list(model.wv.index_to_key)
-        elif isinstance(model, FastText):
-            words: list[str] = model.words
-            word_vectors = np.array([model[word] for word in words])
-            labels = words
-
-        return MatrixRepresentation(matrix=word_vectors, rows=labels)

dstk/text_matrix_builder.py

@@ -1,87 +0,0 @@
-import pandas as pd
-from sklearn.feature_extraction.text import CountVectorizer
-import numpy as np
-from .workflow_tools import requires, workflow, WorkflowManager
-from .matrix_base import MatrixRepresentation, matrix_to_dataframe
-
-from .lib_types import csr_matrix, ndarray, DataFrame, csc_matrix, Matrix, Labels
-
-STAGES = [
-    "start", # Before any processing
-    "matrix_operations" # Matrix operations and transformations
-    "end" # Dataframe operations
-]
-
-class TextMatrixBuilder(WorkflowManager):
-    """
-    Creates a Document Term Matrix, a Co-ocurrence Matrix, and dataframes from them.
-
-    :param corpus: A list of sentences or collocations from which to build a matrix.
-    """
-
-    _start: list[str]
-    _end: DataFrame
-
-    def __init__(self, corpus: list[str] | None = None):
-        """
-        Initializes TextMatrixBuilder with given attributes.
-        """
-
-        super().__init__()
-
-        self._matrix_operations: MatrixRepresentation
-        
-        self._set_workflow(input_arg=corpus)
-
-    @requires(stages=["start"])
-    @workflow(input_arg="corpus", input_process="_start", output_process="_matrix_operations", next_stage="matrix_operations")
-    def create_dtm(self, *, corpus: list[str], **kwargs) -> MatrixRepresentation:
-        """
-        Creates Document Term Matrix (DTM).
-
-        :param corpus: A list of sentences or collocations from which to build a matrix.
-        :param kwargs: Additional keyword arguments to pass to sklearn's CountVectorizer.
-            Common options include:
-                - stop_words: If provided, a list of stopwords to remove from the corpus.
-                - ngram_range: A tuple (min_n, max_n) specifying the range of n-grams to consider.
-            For more information check: https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html
-        """
-
-        vectorizer: CountVectorizer = CountVectorizer(**kwargs)
-
-        dtm: csr_matrix = vectorizer.fit_transform(corpus)
-
-        return MatrixRepresentation(
-            matrix=dtm,
-            rows=np.array(corpus), 
-            columns=vectorizer.get_feature_names_out()
-        )
-
-    @requires(stages=["matrix_operations"])
-    @workflow(input_arg="matrix", input_attrs={"matrix": "matrix", "rows": "columns", "columns": "columns"},  input_process="_matrix_operations", output_process="_matrix_operations")
-    def create_co_occurrence_matrix(self, *, matrix: csr_matrix, rows: Labels = None, columns: Labels = None) -> csc_matrix:
-        """
-        Creates a Co-occurrence matrix.
-
-        :param matrix: A Document Term Matrix (DTM) from which o build a Co-occurrence matrix.
-        """
-
-        co_matrix = matrix.T @ matrix
-
-        return MatrixRepresentation(
-            matrix=co_matrix,
-            rows=rows,
-            columns=columns
-        )
-
-    @requires(stages=["matrix_operations"])
-    @workflow(input_arg="matrix", input_process="_matrix_operations", output_process="_end", next_stage="end")
-    def to_dataframe(self, *, matrix: MatrixRepresentation, **kwargs) -> DataFrame:
-        """
-        Creates a dataframe from a matrix representation.
-
-        :param matrix: A matrix representation from which to create a dataframe.
-        :param kwargs: Additional keyword arguments to pass to sklearn's pandas' DataFrame.
-        """
-        
-        return matrix_to_dataframe(matrix=matrix, **kwargs)