dstklib 1.0.0__py3-none-any.whl

dstk/text_processor.py

@@ -0,0 +1,450 @@
+from pathlib import Path
+import spacy
+from typing import Callable, cast, Any, TypeGuard
+from .workflow_tools import workflow, requires, WorkflowManager, accepts_generic
+
+from .lib_types.spacy_types import *
+from .lib_types.dstk_types import TokenIterator, POSIterator, SentenceIterator, TextIterator, Sentences, Sentence, POSTags
+
+STAGES = [
+    "start", # Before any processing
+    "model", # Text to nlp Doc
+    "token_manipulation", # Maniputation of spaCy tokens
+    "text_processing" # After the tokens have been transformed to text
+    "end" # End of the workflow. After this stage the user must necessarily call processed_text to continue with the analysis
+]
+
+UNITS = [
+    "sentences", # Processed the text by sentences
+    "words", # Processed the text by words
+]
+
+def is_pos_tags(tokens: Any) -> TypeGuard[POSTags]:
+    """
+    If tokens is of type POSTags, returns True. Else, returns False.
+
+    :param tokens: A list of tokens to check its type.
+    """
+
+    if not isinstance(tokens, list) or not tokens:
+        return False
+    return all(
+        isinstance(item, tuple) and
+        len(item) == 2 and 
+        (isinstance(item[0], Token) or 
+        isinstance(item[0], str)) and
+        isinstance(item[1], str)
+        for item in tokens
+    )
+
+def is_sentence(tokens: Any) -> TypeGuard[Sentences]:
+    """
+    If tokens is of type Sentences, returns True. Else, returns False.
+
+    :param tokens: A list of tokens to check its type.
+    """
+
+    return (
+        isinstance(tokens, list) and 
+        all(
+            isinstance(item, Span) or 
+            (
+                isinstance(item, list) and 
+                (
+                    all(isinstance(token, Token) for token in item) or 
+                    all(isinstance(token, str) for token in item)
+                )
+            ) or is_pos_tags(item)
+                for item in tokens
+            )
+    )
+
+def accepts_sentences(accepts: bool = True, custom_error_message: str = "", intercept: bool = True) -> Callable:
+    """
+    Decorator that allows a method to accept sentence-level input (i.e., a list of sentences).
+
+    This decorator uses a shared generic handler to determine whether the `tokens` argument contains sentence-level input.
+    If both `accepts` and `intercept` are True, it processes each sentence individually by invoking the decorated method once per sentence,
+    then aggregates the results into a single list.
+
+    If `accepts` is True but `intercept` is False, the method is called normally with the full input.
+    If the input is sentence-level but `accepts` is False, a ValueError is raised.
+
+    :param accepts: Whether the method should accept sentence-level inputs. Default is True.
+    :param custom_error_message: An optional message to include in raised errors.
+    :param intercept: If True, intercepts sentence input and splits it into per-sentence processing. If False, the method receives the input unchanged.
+
+    :raises ValueError: If sentence input is provided but the method does not accept it.
+
+    :return:  A wrapped method that optionally processes sentence input at the sentence level.
+    """
+
+    def intercept_sentence(self, input_value: Sentences, method: Callable, *args, **kwargs) -> Sentences:
+        sentences: Sentences = getattr(self, f"_{self._current_stage}") if self._flow else input_value
+        processed_sentences: Sentences = []
+
+        for sentence in sentences:
+            result: Sentence = cast(Sentence, method(self, *args, tokens=sentence, **kwargs))
+            processed_sentences.append(result)
+
+        filtered_sentences = cast(Sentences, [sentence for sentence in processed_sentences if sentence])
+
+        return filtered_sentences
+
+    return accepts_generic(
+        type_checker=is_sentence,
+        input_arg="tokens",
+        accepts=accepts,
+        intercept=intercept,
+        interceptor=intercept_sentence,
+        input_type=Sentences,
+        custom_error_message=custom_error_message
+    )
+
+def accepts_tags(accepts: bool = True, custom_error_message: str = "", intercept: bool = True) -> Callable:
+    """
+    Decorator that allows a method to accept POS-tagged input (i.e., list of (token, tag) tuples).
+
+    This decorator uses a shared generic handler to detect tagged input. If both `accepts` and `intercept` are True, it extracts the tokens (stripping the tags), passes them to the method, and reattaches the tags to the result.
+
+    If the result length differs from the original input, it attempts to realign the tags by matching token text.
+    If `intercept` is False, the method is called as-is with the original tagged input.
+    If tagged input is provided but `accepts` is False, a ValueError is raised.
+
+    :param accepts: Whether the method supports POS-tagged inputs. Default is True.
+    :param custom_error_message: Optional custom error message for input validation failure.
+    :param intercept: If True, extracts tokens, passes them to the method, and restores tags after processing.
+
+    :raises ValueError: If input is tagged but cannot be processed or restored properly.
+
+    :return: A wrapped method that conditionally handles tagged input transformation.
+    """
+
+    def intercept_tags(self, input_value: POSTags, method: Callable, *args, **kwargs) -> POSTags:
+        try: 
+            raw_word_tokens, raw_pos = zip(*input_value)
+            word_tokens = cast(list[Token | str], raw_word_tokens)
+            pos = cast(list[str], raw_pos)
+
+            result: list[Token | str] = cast(list[Token | str], method(self, *args, tokens=list(word_tokens), **kwargs))
+
+            if len(result) == len(input_value):
+                return list(zip(result, pos))
+            else:
+                original_pos_map = dict(zip([word_token.text.lower() if isinstance(word_token, Token) else word_token.lower() for word_token in word_tokens], pos))
+
+                result_with_pos: POSTags = [(word, original_pos_map[word.text.lower() if isinstance(word, Token) else word.lower()]) for word in result]
+
+                return result_with_pos if self._current_stage == "text_processing" else [(token, token.pos_) for token in cast(list[Token], result)]
+        except: 
+            raise ValueError(f"Method {method.__name__} does not accept a tagged text with the structre (Token, POS) as input.")
+            
+    return accepts_generic(
+        type_checker=is_pos_tags,
+        input_arg="tokens",
+        accepts=accepts,
+        intercept=intercept,
+        interceptor=intercept_tags,
+        input_type=POSTags,
+        custom_error_message=custom_error_message
+    )
+
+class TextProcessor(WorkflowManager):
+    """
+    Provides a set of methods for text processing, such as raw tokenizers (include punctuation and/or stopwords), tokenizers (remove both punctuation and stopwords, lemmatize), and others (vocabulary extractor, sentence extractor, pos_tagger, etc.)
+
+    :param text: The text to be processed. Defaults to None.
+    :param model: Either the name of installed spaCy language model (e.g., 'en_core_web_sm') or an already loaded spaCy model object. Defaults to None.
+    """
+
+    _start: Doc
+    _end: str
+
+    def __init__(self, text: str | None = None):
+        """
+        Initializes TextProcessor with given attributes.
+        """
+        super().__init__()
+
+        # Stages
+
+        self._model: Doc
+        self._token_manipulation: TokenIterator | SentenceIterator
+        self._text_processing: TextIterator
+
+        self._set_workflow(input_arg=text)
+    
+    @requires(stages=["start"])
+    @workflow(input_arg="text", input_process="_start", output_process="_model", next_stage="model")
+    def set_model(self, text: str, model: str | Language) -> Doc:
+        """
+        Takes a text and analyzes it using a language model. It returns a processed version of the text that includes helpful information like the words, their meanings, and how they relate to each other.
+
+        :param text: The text to be processed.
+        :param model: The name of the model to be used or its instance.
+        """
+
+        nlp: Language
+
+        if isinstance(model, str):
+            nlp = spacy.load(model)
+        else:
+            nlp = model
+
+        return nlp(text)
+
+    @requires(stages=["model"])
+    @workflow(input_arg="tokens", input_process="_model", output_process="_token_manipulation", next_stage="token_manipulation", set_unit="words")
+    def get_tokens(self, *, tokens: Doc) -> list[Token]:
+        """
+        Returns a list of spaCy tokens from a Doc object.
+
+        :param Doc: A spaCy Doc object. Defaults to None. 
+        """
+
+        return [token for token in tokens]
+        
+    @requires(stages=["model"])
+    @workflow(input_arg="tokens", input_process="_model", output_process="_token_manipulation", next_stage="token_manipulation", set_unit="sentences")
+    def get_sentences(self, *, tokens: Doc) -> list[Span]:
+        """
+        Returns a list containing sentences as strings or as spaCy Span objects.
+        
+        :param Doc: A spaCy Doc object. 
+        """
+
+        return list(tokens.sents)
+
+    @requires(stages=["token_manipulation"])
+    @workflow(input_arg="tokens", input_process="_token_manipulation", output_process="_token_manipulation")
+    @accepts_sentences()
+    @accepts_tags()
+    def remove_stop_words(self, *, tokens: TokenIterator, custom_stop_words: list[str] | None = None) -> list[Token]:
+        """
+        Filters tokens, returning only alphanumeric tokens that are not stop words.
+
+        :param tokens: A spaCy Doc or Span object or list of spaCy tokens. 
+        :param custom_stop_words: A list of custom stop words.
+
+        Supported inputs:
+
+        This method supports different token forms due to decorator-based preprocessing:
+            - tokens: TokenIterator
+            - sentences: Sentences
+            - taggged_tokens: POSTags
+        """
+
+        lower_stop_words: list[str]
+
+        if custom_stop_words:
+            lower_stop_words = [word.lower() for word in custom_stop_words] 
+
+        return [
+                token for token in tokens
+                if token.is_alpha and not token.is_stop and 
+                (custom_stop_words is None or token.text.lower() not in lower_stop_words)
+            ]
+
+    @requires(stages=["token_manipulation"])
+    @workflow(input_arg="tokens", input_process="_token_manipulation", output_process="_token_manipulation")
+    @accepts_sentences()
+    @accepts_tags()
+    def raw_tokenizer(self, *, tokens: TokenIterator) -> list[Token]:
+        """ 
+        Tokenizes a text including punctuation and stop words.
+
+        :param tokens: A spaCy Doc or Span object or list of spaCy tokens.
+
+        Supported inputs:
+
+        This method supports different token forms due to decorator-based preprocessing:
+            - tokens: TokenIterator
+            - sentences: Sentences
+            - taggged_tokens: POSTags
+        """
+
+        return [token for token in tokens]
+    
+    @requires(stages=["token_manipulation"])
+    @workflow(input_arg="tokens", input_process="_token_manipulation", output_process="_token_manipulation")
+    @accepts_sentences()
+    @accepts_tags()
+    def alphanumeric_raw_tokenizer(self, *, tokens: TokenIterator) -> list[Token]:
+        """
+        Tokenizes a text including only alphanumeric characters and stop words.
+
+        :param tokens: A spaCy Doc or Span object or list of spaCy tokens.
+
+        Supported inputs:
+        
+        This method supports different token forms due to decorator-based preprocessing:
+            - tokens: TokenIterator
+            - sentences: Sentences
+            - taggged_tokens: POSTags
+        """
+
+        return [
+            token 
+            for token in tokens
+            if token.text.isalpha()
+        ]
+
+    @requires(stages=["token_manipulation"])
+    @workflow(input_arg="tokens", input_process="_token_manipulation", output_process="_token_manipulation")
+    @accepts_sentences()
+    @accepts_tags()
+    def filter_by_pos(self, *, tokens: TokenIterator, pos: str) -> list[Token]:
+        """
+        Returns a list of spaCy tokens filtered by a spacific part-of-speech tag.
+
+        :param tokens: A spaCy Doc or Span object or list of spaCy tokens.
+        :param pos: The POS tag to filter by (e.g., 'NOUN', 'VERB', etc.). Case-sensitive.
+
+        Supported inputs:
+        
+        This method supports different token forms due to decorator-based preprocessing:
+            - tokens: TokenIterator
+            - sentences: Sentences
+            - taggged_tokens: POSTags
+        """
+    
+        return [token for token in tokens if token.pos_ == pos]
+
+    @requires(stages=["token_manipulation"])
+    @workflow(input_arg="tokens", input_process="_token_manipulation", output_process="_token_manipulation")
+    @accepts_sentences()
+    def pos_tagger(self, *, tokens: TokenIterator) -> list[tuple[Token, str]]:
+        """
+        Returns a list of (Token, POS) tuples, pairing each token with its part-of-speech tag.
+
+        :param tokens: A spaCy Doc or Span object or list of spaCy tokens.
+
+        Supported inputs:
+        
+        This method supports different token forms due to decorator-based preprocessing:
+            - tokens: TokenIterator
+            - sentences: Sentences
+            - taggged_tokens: POSTags
+        """
+
+        return [(token, token.pos_) for token in tokens]
+    
+    @requires(stages=["token_manipulation"])
+    @workflow(input_arg="tokens", input_process="_token_manipulation", output_process="_text_processing", next_stage="text_processing")
+    @accepts_sentences()
+    @accepts_tags()
+    def get_text(self, *, tokens: TokenIterator, lemmatize: bool = False) -> TextIterator:
+        """
+        Returns the text content from a list of spaCy tokens, Span objects or list of spaCy tokens.
+
+        :param tokens: A spaCy Doc or Span object or list of spaCy tokens.
+        :param lemmatize: If True, lemmatizes the words in the text. Defaults to False.
+
+        Supported inputs:
+        
+        This method supports different token forms due to decorator-based preprocessing:
+            - tokens: TokenIterator
+            - sentences: Sentences
+            - taggged_tokens: POSTags
+        """
+
+        return [token.lemma_.lower() if lemmatize else token.text for token in tokens]
+
+    @requires(stages=["text_processing"])
+    @workflow(input_arg="tokens", input_process="_text_processing", output_process="_text_processing")
+    @accepts_sentences()
+    @accepts_tags()
+    def to_lower(self, *, tokens: list[str]) -> list[str]:
+        """
+        Returns a list of lower cased words.
+
+        :param tokens: A list containing tokens as strings.
+
+        Supported inputs:
+        
+        This method supports different token forms due to decorator-based preprocessing:
+            - tokens: list[str]
+            - sentences: list[list[str]] | list[list[tuple[str, str]]]
+            - taggged_tokens: list[tuple[str, str]]
+        """
+        
+        return [string.lower() for string in tokens]
+
+    @requires(stages=["text_processing"])
+    @workflow(input_arg="tokens", input_process="_text_processing", output_process="_text_processing")
+    @accepts_sentences()
+    @accepts_tags(accepts=False)
+    def corpus_by_context_window(self, *, tokens: list[str], window_size: int) -> list[str]:
+        """
+        Splits the tokens into groups of window_size consecutive words and joins each group into a string.
+
+        :param tokens: A list containing tokens as strings.
+        :param window_size: size of the context window.
+
+        Supported inputs:
+        
+        This method supports different token forms due to decorator-based preprocessing:
+            - tokens: list[str]
+            - sentences: list[list[str]]
+        """
+
+        ngrams: list[list[str]] = [tokens[index:index + window_size] for index in range(len(tokens) - window_size + 1)]
+        return [' '.join(ngram) for ngram in ngrams]
+
+    @requires(stages=["text_processing"], unit="words")
+    @workflow(input_arg="tokens", input_process="_text_processing", output_process="_text_processing")
+    @accepts_tags(accepts=False)
+    def get_vocabulary(self, *, tokens: list[str]) -> list[str]:
+        """
+        Returns the vocabulary a text.
+
+        :param tokens: A list containing tokens as strings.
+
+        This method supports different token forms due to decorator-based preprocessing:
+            - tokens: list[str]
+            - sentences: list[list[str]]
+        """
+
+        return sorted(set(tokens))
+
+    @requires(stages=["text_processing"], unit="sentences")
+    @workflow(input_arg="tokens", input_process="_text_processing", output_process="_text_processing")
+    @accepts_sentences()
+    @accepts_tags(accepts=False)
+    def join(self, *, tokens: list[str]) -> str:
+        """
+        Joins a list of strings into a single string text.
+
+        :param tokens: A list containing tokens as strings.
+
+        This method supports different token forms due to decorator-based preprocessing:
+            - tokens: list[str]
+            - sentences: list[list[str]]
+        """
+
+        return " ".join(tokens)
+
+    @requires(stages=["text_processing"])
+    @workflow(input_arg="tokens", input_process="_text_processing", output_process="_end", next_stage="end")
+    @accepts_sentences(accepts=False, custom_error_message="You must first use join before saving them to a file.") 
+    @accepts_tags(accepts=True, intercept=False)
+    def save_to_file(self, *, tokens: list[str] | list[tuple[str, str]], path: str) -> str:
+        """
+        Saves a list of strings or (Token, POS) tuples in the specified path. If tokens is a list of strings, it saves each string in a new line. If it is a list of tuples, it saves each tuple in a new line as a pair or values separated by a comma, in a CSV format.
+
+        :param tokens: A list containing tokens as strings or a list of (Token, POS) tuples.
+        :param path: The path where to save the list of tokens.
+
+        This method supports different token forms due to decorator-based preprocessing:
+            - tokens: list[str]
+            - taggged_tokens: list[tuple[str, str]]
+        """
+
+        with open(path, "w") as file:
+            for token in tokens:
+                if type(token) == str:
+                    file.write(token + "\n")
+                else:
+                    file.write(token[0] + "," + token[1] + "\n")
+        
+        return str(Path(path).resolve())

dstk/weight_matrix.py

@@ -0,0 +1,71 @@
+import pandas as pd
+import numpy as np
+from sklearn.feature_extraction.text import TfidfTransformer
+from .workflow_tools import requires, workflow, WorkflowManager
+
+from .lib_types import DataFrame, ndarray, Series, csr_matrix
+
+STAGES = [
+    "start", # Before any processing
+    "end" # Weighted dataframe
+]
+
+class WeightMatrix(WorkflowManager):
+    """
+    Provides a set of methods to weight a Co-ocurrence matrix.
+
+    :param dataframe: A Co-occurrence matrix to be weighted.
+    """
+
+    _start: DataFrame
+    _end: DataFrame
+
+    def __init__(self, dataframe: DataFrame | None = None):
+        """
+        Initializes WeightMatrix with given attributes.
+        """
+
+        super().__init__()
+
+        self._set_workflow(input_arg=dataframe)
+
+    @requires(stages=["start"])
+    @workflow(input_arg="dataframe", input_process="_start", output_process="_end", next_stage="end")
+    def pmi(self, *, dataframe: DataFrame, positive: bool = False) -> DataFrame:
+        """
+        Weights a Co-occurrence matrix by PMI or PPMI.
+        
+        :param dataframe: A Co-occurrence matrix to be weighted.
+        :param positive: If True, weights the Co-ocurrence matrix by PPMI. If False, weighths it by PMI. Defaults to False.
+        """
+
+        df: DataFrame = dataframe
+
+        col_totals: Series = df.sum(axis=0)
+        total: float = col_totals.sum()
+        row_totals: Series = df.sum(axis=1)
+        expected: ndarray = np.outer(row_totals, col_totals) / total
+        df = df / expected
+        # Silence distracting warnings about log(0):
+        with np.errstate(divide='ignore'):
+            df = np.log(df)
+        df[np.isinf(df)] = 0.0  # log(0) = 0
+        if positive:
+            df[df < 0] = 0.0
+
+        return df
+
+    @requires(stages=["start"])
+    @workflow(input_arg="dataframe", input_process="_start", output_process="_end", next_stage="end")
+    def tf_idf(self, *, dataframe: DataFrame, **kwargs) -> DataFrame:
+        """
+        Weights a Co-occurrence matrix by Tf-idf.
+        
+        :param dataframe: A Co-occurrence matrix to be weighted.
+        """
+
+        transformer: TfidfTransformer = TfidfTransformer(**kwargs)
+        tf_idf_matrix: csr_matrix = transformer.fit_transform(dataframe)
+
+        return pd.DataFrame(tf_idf_matrix.toarray(), index=dataframe.index, columns=dataframe.columns)
+