dstklib 1.0.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

dstk/modules/geometric_distance.py

@@ -0,0 +1,114 @@
+"""
+This module provides functions to compute geometric distance and similarity measures  between word embeddings, enabling semantic comparison of words in vector space.
+
+Available metrics include:
+
+* Euclidean distance
+* Manhattan distance
+* Cosine similarity
+
+Additionally, it offers a method to find the nearest semantic neighbors of a given word based on specified distance or similarity metrics using scikit-learn's NearestNeighbors.
+
+All functions operate on word embeddings represented as Pandas DataFrames indexed by words, facilitating easy integration with common NLP and Computational Linguistic workflows.
+"""
+
+from sklearn.neighbors import NearestNeighbors
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+
+from ..lib_types import ndarray, Series, DataFrame, Neighbors, Neighbor
+
+def euclidean_distance(embeddings: DataFrame, first_word: str, second_word: str) -> float:
+    """
+    Computes the Euclidean distance between the embeddings of two words.
+
+    :param embeddings: A dataframe containing the word embeddings.
+    :type embeddings: DataFrame
+    :param first_word: The first word in the pair.
+    :type first_word: str
+    :param second_word: The second word in the pair.
+    :type second_word: str
+
+    :returns: The Euclidean distance between the first and second word.
+    :rtype: float
+    """
+
+    first_word_vector: Series = embeddings.loc[first_word]
+    second_word_vector: Series = embeddings.loc[second_word]
+
+    return float(np.linalg.norm(first_word_vector - second_word_vector))
+
+def manhattan_distance(embeddings: DataFrame, first_word: str, second_word: str) -> float:
+    """
+    Computes the Manhattan distance between the embeddings of two words.
+
+    :param embeddings: A dataframe containing the word embeddings.
+    :type embeddings: DataFrame
+    :param first_word: The first word in the pair.
+    :type first_word: str
+    :param second_word: The second word in the pair.
+    :type second_word: str
+
+    :returns: The Manhattan distance between the first and second word.
+    :rtype: float
+    """
+
+    first_word_vector: Series = embeddings.loc[first_word]
+    second_word_vector: Series = embeddings.loc[second_word]
+
+    return np.sum(np.abs(first_word_vector - second_word_vector))
+
+def cos_similarity(embeddings: DataFrame, first_word: str, second_word: str) -> float:
+    """
+    Computes the cosine similarity between the embeddings of two words.
+
+    :param embeddings: A dataframe containing the word embeddings.
+    :type embeddings: DataFrame
+    :param first_word: The first word in the pair.
+    :type first_word: str
+    :param second_word: The second word in the pair.
+    :type second_word: str
+
+    :returns: The cosine similarity between the first and second word.
+    :rtype: float
+    """
+
+    first_word_vector: ndarray = np.array(embeddings.loc[first_word]).reshape(1, -1)
+    second_word_vector: ndarray = np.array(embeddings.loc[second_word]).reshape(1, -1)
+
+    cos_sim: ndarray = cosine_similarity(first_word_vector, second_word_vector)
+
+    return cos_sim[0][0]
+
+def nearest_neighbors(embeddings: DataFrame, word: str, metric: str = "cosine", n_words: int = 5, **kwargs) -> Neighbors:
+    """
+    Returns the top N most semantically similar words to a given target word, based on the specified distance or similarity metric.
+
+    :param embeddings: A dataframe containing the word embeddings.
+    :type embeddings: DataFrame
+    :param word: The target word to find neighbors for.
+    :type word: str
+    :param metric: The distance or similarity metric to use (e.g., 'cosine', 'euclidean'). Defaults to 'cosine'.
+    :type metric: str
+    :param n_words: Number of nearest neighbors to return. Defaults to 5.
+    :type of n_words: int
+    :param kwargs: Additional keyword arguments to pass to sklearn's NearestNeighbors. For more information check: https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.NearestNeighbors.html
+
+    :returns: A list of `Neighbor` namedtuples, one for each word close to the target word.
+    :rtype: Neighbors
+    """
+
+    neighbors: NearestNeighbors = NearestNeighbors(n_neighbors=n_words, algorithm="auto", metric=metric, **kwargs)
+    neighbors.fit(embeddings.to_numpy())
+
+    word_vector: Series = embeddings.loc[word]
+
+    distances: ndarray
+    indices: ndarray
+    distances, indices = neighbors.kneighbors([word_vector], n_neighbors=n_words + 1)
+
+    neighbor_tuples = zip(indices[0], distances[0])
+
+    results: Neighbors = [Neighbor(embeddings.index[index], 1 - distance) for index, distance in neighbor_tuples if embeddings.index[index] != word]
+
+    return results

dstk/modules/ngrams.py

@@ -0,0 +1,156 @@
+"""
+This module provides utilities for extracting context-based collocates, bigrams, and n-grams from a list of words or tokens. It is designed to support both raw string tokens and spaCy `Token` objects, allowing for flexibility in preprocessing pipelines.
+
+The functions in this module focus on identifying co-occurrence patterns around a specific target word,  as well as extracting fixed-length n-grams from sequences of tokens. This is useful for tasks such as collocation analysis, feature engineering for machine learning models, and exploratory corpus analysis.
+
+Core functionalities include:
+
+* Extracting left and right context windows around a target word
+* Creating directed and undirected bigrams centered on a target
+* Generating fixed-length n-grams from a sequence of words
+* Counting the frequency of collocated words in context windows
+
+The module is compatible with both plain string tokens and spaCy Tokens.
+"""
+
+
+from collections import Counter
+import pandas as pd 
+
+from typing import Generator
+from ..lib_types import Words, WordCounts, DataFrame, CollocatesList, BigramList, DirectedCollocateList, Token, Bigram
+
+
+def _find_contexts(words: Words, target_word: str, window_size: tuple[int, int]) -> Generator[tuple[Words, Words], None, None]:
+    """
+    Yields left and right contexts for each occurrence of a target word in a list of words.
+
+    :param words: A list of words (strings or spaCy Tokens).
+    :type words: Words
+
+    :param target_word: The word to find within the list.
+    :type target_word: str
+
+    :param window_size: A tuple representing the number of words to include before and after the target word.
+    :type window_size: tuple[int, int]
+
+    :return: An Geberator of tuples, where each tuple contains the left and right context around a matched word.
+    :rtype: Generator[tuple[Words, Words], None, None]
+    """
+
+    for index, word in enumerate(words):
+        word_to_compare = word.text if isinstance(word, Token) else word
+        if word_to_compare == target_word:
+            start: int = max(0, index - window_size[0])
+            end: int = min(len(words), index + window_size[1] + 1)
+
+            left_context: Words = words[start:index]
+            right_context: Words = words[index + 1:end]
+
+            yield (left_context, right_context)
+
+def extract_collocates(words: Words, target_word: str, window_size: tuple[int, int]) -> CollocatesList:
+    """
+    Extracts the context words of the target word, returned as tuples whose lenght corresponds to the specified window_size.
+
+    :param words: A list of spaCy tokens or words represented as strings.
+    :type words: Words
+    :param target_word: The word to find within the list.
+    :type target_word: str
+    :param window_size: A tuple indicating how many words to capture to the left and right of the target.
+    :type window_size: tuple[int, int]
+
+    :returns: A list of collocates (left and right context words) of the target word.
+    :rtype: CollocatesList
+    """
+
+    return [tuple(left + right) for left, right in _find_contexts(words, target_word, window_size)]
+
+def extract_directed_bigrams(words: Words, target_word: str, window_size: tuple[int, int]) -> DirectedCollocateList:
+    """
+    Extracts directed bigrams (left and right context words) around a target word.
+
+    For each occurrence of `target_word` in the input `words`, this function collects two types of bigrams:
+    * Left bigrams: (context_word, ("L", target_word))
+    * Right bigrams: (context_word, ("R", target_word))
+
+    :param words: A list of spaCy tokens or words represented as strings.
+    :type words: Words
+
+    :param target_word: The word to search for in the list.
+    :type target_word: str
+
+    :param window_size: A tuple indicating how many words to capture to the left and right of the target.
+    :type window_size: tuple[int, int]
+
+    :return: A list of directed bigrams in the form `(word, ("L" | "R", target_word))`.
+    :rtype: DirectedCollocateList
+    """
+    bigrams: DirectedCollocateList = []
+
+    for left, right in _find_contexts(words, target_word, window_size):
+        bigrams.extend([(word, ("L", target_word)) for word in left])
+        bigrams.extend([(word, ("R", target_word)) for word in right])
+    
+    return bigrams
+    
+def extract_undirected_bigrams(words: Words, target_word: str, window_size: tuple[int, int]) -> BigramList:
+    """
+    Extracts undirected bigrams surrounding a target word.
+
+    For each occurrence of `target_word`, this function collects all context words within the specified window (both left and right), and forms a `Bigram` with:
+    
+    * `collocate`: the context word
+    * `target_word`: the target word
+
+    :param words: A list of spaCy tokens or words represented as strings.
+    :type words: Words
+
+    :param target_word: The word to search for in the list.
+    :type target_word: str
+
+    :param window_size: A tuple indicating how many words to capture to the left and right of the target.
+    :type window_size: tuple[int, int]
+
+    :return: A list of `Bigram` namedtuples, one for each context word around each target occurrence.
+    :rtype: BigramList
+    """
+    bigrams: BigramList = []
+
+    for left, right in _find_contexts(words, target_word, window_size):
+        bigrams.extend([Bigram(collocate=word, target_word=target_word) for word in left + right])
+
+    return bigrams
+
+def extract_ngrams(words: Words, window_size: int) -> CollocatesList:
+    """
+    Splits the tokens into groups of window_size consecutive words and joins each group into a string.
+
+    :param words: A list of spaCy tokens or words represented as strings.
+    :type words: Words
+    :param window_size: size of the square context window.
+    :type window_size: int
+
+    :return: A list of tuples, where each tuple contains `window_size` consecutive words from the input.
+    :rtype: CollocatesList
+    """
+
+    ngrams: CollocatesList = [tuple(words[index:index + window_size]) for index in range(len(words) - window_size + 1)]
+    return ngrams
+
+def count_collocates(collocates: CollocatesList) -> DataFrame:
+    """
+    Counts the frequency of words in a list of collocations and returns the result as a DataFrame.
+
+    :param collocates: A list of collocations, where each collocation is a tuple of words.
+    :type collocates: CollocatesList
+
+    :return: A DataFrame with two columns: "word" and "count", sorted by frequency.
+    :rtype: DataFrame
+    """
+
+    all_words: Words = [word.text if isinstance(word, Token) else word for collocation in collocates for word in collocation]
+    word_counts: WordCounts = Counter(all_words)
+    word_counts_df: DataFrame = pd.DataFrame(word_counts.items(), columns=["word", "count"])
+    
+    return word_counts_df

dstk/modules/predict_models.py

@@ -0,0 +1,109 @@
+"""
+This module provides utilities to train, save, and load word embedding models using neural networks models such as Word2Vec (gensim) and FastText (fasttext library).
+
+Functions include:
+
+* *word2vec:* Train Word2Vec embeddings from a corpus file.
+* *fastText:* Train FastText embeddings from a corpus file.
+* *load_model:* Load a saved model from disk (supports Word2Vec .model and FastText .bin formats).
+* *save_model:* Save a trained model to disk in the appropriate format.
+
+Each function supports passing additional keyword arguments to fine-tune training and loading.
+"""
+
+from gensim.models import Word2Vec
+import fasttext
+from pathlib import Path
+
+from ..lib_types import FastText, NeuralModels
+
+def word2vec(path: str, **kwargs) -> Word2Vec:
+    """
+    Creates word embeddings using the Word2Vec algorithm.
+
+    :param path: The path to a file conatining a list of sentences or collocations from which to build word embeddings.
+    :type path: str
+    :param kwargs:  Additional keyword arguments to pass to gensim.models.Word2Vec. Common options include:
+
+        * **vector_size:** Size of the word embedding vectors.
+        * **workers:** Number of CPU cores to be used during the training process.
+        * **sg:** Training algorithm. 1 for skip-gram; 0 for CBOW (Continuous Bag of Words).
+        * **window (int):** Maximum distance between the current and predicted word.
+        * **min_count (int):** Ignores all words with total frequency lower than this.
+
+    For more information check: https://radimrehurek.com/gensim/models/word2vec.html
+
+    :returns: An instance of gensim's Word2Vec.
+    :rtype: Word2Vec
+    """
+
+    return Word2Vec(
+        corpus_file=path,
+        **kwargs
+    )
+
+def fastText(path: str, **kwargs) -> FastText:
+    """
+    Creates word embeddings using the FastText algorithm.
+
+    :param path: The path to a file containing a list of sentences or collocations from which to build word embeddings.
+    :type path: str
+    :param kwargs: Additional keyword arguments to pass to fasttext.train_unsupervised. Common options include:
+
+        * **dim:** Size of the word embedding vectors.
+        * **model:** Training algorithm: skipgram or cbow (Continuous Bag of Words)
+        * **thread:** Number of CPU cores to be used during the training process.
+
+    For more information check: https://fasttext.cc/docs/en/options.html
+    
+    :returns: An instance of fasttext's FastText.
+    :rtype: FastText
+    """
+
+    return fasttext.train_unsupervised(
+        path,
+        **kwargs
+    )
+
+def load_model(path: str) -> NeuralModels:
+    """
+    Loads the trained embeddings in .model (Word2Vec) or .bin (FastText) format, depending on the algorithm used.
+
+    :param path: Path to the saved model file.
+    :type path: str
+
+    :returns: An instance of gensim's Word2Vec or fasttext's FastText.
+    :rtype: NeuralModels
+    """
+
+    extension: str = Path(path).suffix.lower()
+
+    if extension == ".model":
+        return Word2Vec.load(path)
+    elif extension == ".bin":
+        return fasttext.load_model(path)
+    else:
+        raise ValueError(f"Model extension {extension} not recognized.")
+
+def save_model(model: NeuralModels, path: str) -> str:
+    """
+    Saves the trained embeddings in .model (Word2Vec) or .bin (FastText) format, depending on the algorithm used.
+
+    :param model: A trained Word2Vec or FastText model.
+    :type model: NeuralModels
+    :param path: The path (without extension) where to save the model.
+    :type path: str
+
+    :returns: An instance of gensim's Word2Vec or fasttext's FastText.
+    :rtype: NeuralModels
+    """
+    full_path: Path = Path(path)
+
+    if isinstance(model, Word2Vec):
+        model.save(str(full_path.with_suffix(".model")))
+    elif isinstance(model, FastText):
+        model.save_model(str(full_path.with_suffix(".bin")))
+    else:
+        raise NotImplementedError(f"Model identifier type {type(model.__name__)} not yet supported")
+    
+    return str(full_path.resolve())

dstk/modules/text_matrix_builder.py

@@ -0,0 +1,55 @@
+"""
+This module provides functions to construct common matrix representations used in text analysis and natural language processing.
+
+Key features include:
+
+* Creating a Document-Term Matrix (DTM) from a corpus of text, leveraging sklearn's CountVectorizer with customizable parameters such as stop word removal and n-gram range.
+* Generating a Co-occurrence Matrix from a given Document-Term Matrix, capturing how frequently terms co-occur across documents.
+
+These matrices are foundational for many NLP and Computational Linguistics tasks, including topic modeling, word embedding training, and network analysis. The output is provided as Pandas DataFrames for ease of analysis and integration with data science workflows.
+"""
+
+from sklearn.feature_extraction.text import CountVectorizer
+import numpy as np
+import pandas as pd
+
+from ..lib_types import csr_matrix, DataFrame, ndarray
+
+def create_dtm(corpus: list[str], **kwargs) -> DataFrame:
+    """
+    Creates Document Term Matrix (DTM).
+
+    :param corpus: A list of sentences or collocations from which to build a matrix.
+    :type corpus: list[str]
+    :param kwargs: Additional keyword arguments to pass to sklearn's CountVectorizer. Common options include:
+
+        * **stop_words:** If provided, a list of stopwords to remove from the corpus.
+        * **ngram_range:** A tuple (min_n, max_n) specifying the range of n-grams to consider.
+    
+    For more information check: https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html
+
+    :return: A Document Term Matrix (DTM).
+    :rtype: DataFrame.
+    """
+
+    vectorizer: CountVectorizer = CountVectorizer(**kwargs)
+
+    dtm: csr_matrix = vectorizer.fit_transform(corpus)
+
+    return pd.DataFrame(dtm.toarray(), index=np.array(corpus), columns=vectorizer.get_feature_names_out())
+
+def create_co_occurrence_matrix(dtm: DataFrame) -> DataFrame:
+    """
+    Creates a Co-occurrence matrix from a Document Term Matrix (DTM).
+
+    :param dtm: A Document Term Matrix (DTM) from which to build a Co-occurrence matrix.
+    :type dtm: DataFrame
+
+    :return: A Co-occurrence matrix.
+    :rtype: DataFrame
+    """
+    matrix: ndarray = dtm.to_numpy()
+
+    co_matrix: ndarray = matrix.T @ matrix
+
+    return pd.DataFrame(co_matrix, index=dtm.columns, columns=dtm.columns)

dstk/modules/text_processor.py

@@ -0,0 +1,100 @@
+"""
+This module provides utility functions for processing tokenized or lemmatized text represented as lists of strings 
+or POS-tagged tuples. It supports common text normalization and transformation tasks, such as lowercasing, 
+vocabulary extraction, and joining tokens into a single string. Additionally, it includes functionality for saving 
+processed text or tagged data to a file in plain text or CSV format.
+
+Core functionalities include:
+
+* Converting spaCy tokens to strings (with optional lemmatization)
+* Lowercasing and vocabulary extraction
+* Joining word lists into full text strings
+* Saving word lists or (token, POS) pairs to disk in a consistent format
+
+This module is useful for preparing text data for further analysis, modeling, or storage.
+"""
+
+from pathlib import Path
+
+from ..lib_types.dstk_types import Words, POSTaggedWordList, Token, POSTaggedWord
+
+def tokens_to_text(tokens: Words[Token], lemmatize: bool = False) -> Words[str]:
+    """
+    Converts a list of spaCy Token objects to a list of words represented as strings.
+
+    :param tokens: A list of spaCy tokens. 
+    :type tokens: Words[Token]
+    :param lemmatize: Whether to return the lemmatized form of each token. Defaults to False.
+    :type lemmatize: bool
+
+    :return: A list words represented as strings.
+    :rtype: Words[str]
+    """
+
+    return [token.lemma_.lower() if lemmatize else token.text for token in tokens]
+
+def to_lower(words: Words[str]) -> Words[str]:
+    """
+    Returns a list of lower cased words.
+
+    :param words: A list words represented as strings.
+    :type words: Words[str]
+
+    :return: A list of words represented as strings.
+    :rtype: Words[str]
+    """
+    
+    return [word.lower() for word in words]
+
+def get_vocabulary(words: Words[str]) -> Words[str]:
+    """
+    Returns the vocabulary a text.
+
+    :param words: A list words represented as strings.
+    :type words: Words[str]
+
+    :return: A list of words represented as strings.
+    :rtype: Words[str]
+    """
+
+    return sorted(set(words))
+
+def join(words: Words[str]) -> str:
+    """
+    Joins a list of strings into a single string text.
+
+    :param words: A list words represented as strings.
+    :type words: Words[str]
+
+    :return:  A single string formed by concatenating the input words separated by spaces.
+    :rtype: Words[str]
+    """
+
+    return " ".join(words)
+    
+def save_to_file(words: Words[str] | POSTaggedWordList, path: str) -> str:
+    """
+    Saves a list of strings or (Token, POS) tuples in the specified path. If tokens is a list of strings, it saves each string in a new line. If it is a list of tuples, it saves each tuple in a new line as a pair or values separated by a comma, in a CSV format.
+
+    :param words: A list words represented as strings or a list of POSTaggedWord tuples.
+    :type words: Words[str] or POSTaggedWordList.
+    :param path: The path where to save the list of words.
+    :type path: str
+
+    :return: The path where the file was saved.
+    :rtype: str
+    """
+
+    with open(path, "w") as file:
+        for word in words:
+            if type(word) == str:
+                file.write(word + "\n")
+            elif isinstance(word, POSTaggedWord):
+                if isinstance(word[0], str):
+                    file.write(word[0] + "," + word[1] + "\n")
+                else:
+                    raise ValueError("You can only use save_to_file with a POSTaggedWordList if word is of type of str.")
+            else:
+                raise ValueError("You can only use save_to_file with Words[srt] | POSTaggedWordList")
+    
+    return str(Path(path).resolve())

dstk/modules/tokenizer.py

@@ -0,0 +1,139 @@
+"""
+This module provides utility functions for tokenizing texts using spaCy. 
+It offers tools to process raw text into structured linguistic data, extract tokens and sentences, filter words by specific criteria (e.g., stop words, alphanumeric characters, part-of-speech), and 
+generate POS-tagged outputs.
+
+Core functionalities include:
+
+* Segemtating a text by applying a spaCy language model to raw text
+* Extracting tokens and sentences from processed documents
+* Removing stop words and non-alphanumeric tokens
+* Filtering tokens by part-of-speech (POS) tags
+* Generating (token, POS) tuples for downstream NLP tasks
+
+The module is intended to provide tools for text segmentation and tagging.
+"""
+
+import spacy
+
+from ..lib_types.spacy_types import *
+from ..lib_types.dstk_types import Words, POSTaggedWordList, POSTaggedWord, WordSenteces
+
+def apply_model(text: str, model: str | Language) -> Doc:
+    """
+    Takes a text and analyzes it using a language model. It returns a processed version of the text that includes helpful information like the words, their meanings, and how they relate to each other.
+
+    :param text: The text to be processed.
+    :type text: str
+    :param model: The name of the model to be used or its instance.
+    :type model: str or Language
+
+    :return: A spaCy Doc object with linguistic annotations.
+    :rtype: Doc
+    """
+
+    nlp: Language
+
+    if isinstance(model, str):
+        nlp = spacy.load(model)
+    else:
+        nlp = model
+
+    return nlp(text)
+
+
+def get_tokens(document: Doc) -> Words[Token]:
+    """
+    Returns a list of spaCy tokens from a Doc object.
+
+    :param docuument: A spaCy Doc object.
+    :type document: Doc
+
+    :return: A list of spaCy tokens.
+    :rtype: Words[Token]
+    """
+
+    return [token for token in document]
+    
+
+def get_sentences(document: Doc) -> WordSenteces: # Check this one return type
+    """
+     Returns a list of sentences from a spaCy Doc, where each sentence is represented as a list of spaCy Token objects.
+    
+    :param document: A spaCy Doc object.
+    :type document: Doc
+
+    :return: A list of sentences, each sentence is a list of spaCy Tokens.
+    :rtype: WordSentences
+    """
+
+    return [[token for token in sentence] for sentence in document.sents]
+
+def remove_stop_words(tokens: Words[Token], custom_stop_words: list[str] | None = None) -> Words[Token]:
+    """
+    Filters tokens, returning only alphanumeric tokens that are not stop words.
+
+    :param tokens: A list of spaCy tokens. 
+    :type tokens: Words[Token]
+    :param custom_stop_words: If provided, a list of custom stop words. Defaults to None. 
+    :type custom_stop_words: list[str] or None
+
+    :return: A list of spaCy tokens.
+    :rtype: Words[Token]
+    """
+
+    lower_stop_words: list[str]
+
+    if custom_stop_words:
+        lower_stop_words = [word.lower() for word in custom_stop_words] 
+
+    return [
+            token for token in tokens
+            if token.is_alpha and not token.is_stop and 
+            (custom_stop_words is None or token.text.lower() not in lower_stop_words)
+        ]
+
+def alphanumeric_raw_tokenizer(tokens: Words[Token]) -> Words[Token]:
+    """
+    Tokenizes a text including only alphanumeric characters and stop words.
+
+    :param tokens: A list of spaCy tokens. 
+    :type tokens: Words[Token]
+
+    :return: A list of spaCy tokens.
+    :rtype: Words[Token] 
+    """
+
+    return [
+        token 
+        for token in tokens
+        if token.text.isalpha()
+    ]
+
+def filter_by_pos(tokens: Words[Token], pos: str) -> Words[Token]:
+    """
+    Returns a list of spaCy tokens filtered by a spacific part-of-speech tag.
+
+    :param tokens: A list of spaCy tokens.
+    :type tokens: Words[Token]
+    :param pos: The POS tag to filter by (e.g., 'NOUN', 'VERB', etc.). Case-sensitive.
+    :type pos: str
+
+    :return: A list of spaCy tokens.
+    :rtype: Words[Token] 
+    """
+
+    return [token for token in tokens if token.pos_ == pos]
+
+def pos_tagger(tokens: Words[Token]) -> POSTaggedWordList:
+    """
+    Returns a list of (Token, POS) tuples, pairing each token with its part-of-speech tag.
+
+    :param tokens: A list of spaCy tokens.
+    :type tokens: Words[Token]
+
+    :return: A list of POSTaggedWord tuples.
+    :rtype: POSTaggedWordList
+    """
+
+    return [POSTaggedWord(token, token.pos_) for token in tokens]