dstklib 1.0.2__py3-none-any.whl → 2.0.0__py3-none-any.whl

dstk/__init__.py

@@ -1,12 +1,10 @@
-from .count_models import *
-from .geometric_distance import *
-from .plot_embeddings import *
-from .predict_models import *
-from .collocations import *
-from .text_matrix_builder import *
-from .text_processor import *
-from .weight_matrix import *
-from .matrix_base import *
-from .workflow_tools import *
-from .pipeline_tools import *
-from .pipelines import *
+from .modules import *
+from .workflows import *
+from .models import *
+
+from .templates import *
+from .hooks import *
+from .adaptors import *
+
+from .lib_types import *
+

dstk/adaptors/__init__.py

@@ -0,0 +1,2 @@
+from .adaptors import *
+from .typeguards import *

dstk/adaptors/adaptors.py

@@ -0,0 +1,91 @@
+"""
+This module provides function decorators that adapt the input types of processing functions to improve flexibility and composability across workflows.
+
+Specifically, it includes:
+
+* `accepts_sentences_and_collocates`: Allows a function to seamlessly handle both individual token sequences and lists of such sequences (e.g., sentences or collocate groups).
+* `accepts_tags`: Allows functions designed for plain tokens to accept and return POS-tagged inputs (POSTaggedWord), preserving tag alignment.
+
+These adaptors make it easier to integrate diverse data types into a unified processing pipeline without requiring duplication of logic.
+"""
+
+from functools import wraps
+import inspect
+from inspect import BoundArguments
+from .typeguards import is_sentence, is_collocates, is_pos_tags
+
+from typing import TypeVar, Any, Callable, Sized, Iterable, cast
+from ..lib_types import Token, Sentences, POSTaggedWordList, POSTaggedWord
+
+T = TypeVar("T", bound=Sized)
+
+def accepts_sentences_and_collocates(method: Callable[..., T]) -> Callable[..., list[T] | T]:
+    """
+    Decorator that allows a function to accept either a single input (e.g., a list of tokens or collocates) or a list of such inputs (e.g., sentences or collocate groups). If a list of inputs is passed, the 
+    function is applied to each element in the list, and a list of results is returned.
+
+    If the input is not a list of sentences or collocates, the function is applied normally.
+
+    :param method: The function to wrap.
+    :type method: Callable[..., T]
+
+    :return: A wrapped function that handles both single and batched inputs.
+    :rtype: Callable[..., list[T] | T]
+    """
+
+    @wraps(method)
+    def wrapper(*args: Any, **kwargs: Any) -> list[T] | T:
+        signature = inspect.signature(method)
+        bound_args: BoundArguments = signature.bind(*args, **kwargs)
+        bound_args.apply_defaults()
+
+        tokens = next(iter(bound_args.arguments.values()), None)
+
+        if is_sentence(tokens) or is_collocates(tokens):
+            processed_sentences = [method(sentence, **kwargs) for sentence in tokens]
+
+            return [sentence for sentence in processed_sentences if sentence]
+        else: 
+            return method(tokens, **kwargs)
+    return wrapper
+
+
+def accepts_tags(method: Callable[..., T]) -> Callable[..., POSTaggedWordList | T]:
+    """
+    Decorator that allows a function designed to operate on plain tokens to also handle POS-tagged word inputs (i.e., sequences of POSTaggedWord). 
+
+    The function will automatically extract the token part, apply the method, and then re-attach the POS tags to the result. If the number of returned tokens does not match the original length, the POS tags are inferred from a lowercase mapping of the original words.
+
+    :param method: The function to wrap.
+    :type method: Callable[..., T]
+
+    :return: A wrapped function that processes POS-tagged input and returns a POSTaggedWordList.
+    :rtype: Callable[..., POSTaggedWordList | T]
+    """
+
+    @wraps(method)
+    def wrapper(*args: Any, **kwargs: Any) -> POSTaggedWordList | T:
+        signature = inspect.signature(method)
+        bound_args = signature.bind(*args, **kwargs)
+        bound_args.apply_defaults()
+
+        tokens = next(iter(bound_args.arguments.values()), None)
+
+        if is_pos_tags(tokens):
+            words, pos = zip(*tokens)
+
+            result = method(list(words), **kwargs) 
+
+            if len(result) == len(tokens):
+                return [POSTaggedWord(word, pos_tag) for word, pos_tag in zip(cast(Iterable, result), pos)]
+            else:
+                original_pos_map = dict(zip([word.text.lower() if isinstance(word, Token) else word.lower() for word in words], pos))
+
+                print(words, pos, original_pos_map)
+                result_with_pos = [POSTaggedWord(word, original_pos_map[word.text.lower() if isinstance(word, Token) else word.lower()]) for word in cast(Iterable, result)]
+
+                return result_with_pos
+        else:
+            return method(tokens, **kwargs)
+
+    return wrapper

dstk/adaptors/typeguards.py

@@ -0,0 +1,141 @@
+"""
+Provides a set of type guard functions to safely and explicitly check the types of various token and workflow-related objects.
+
+These functions help with runtime type checking and enable more precise type hinting and static analysis when working with linguistic data structures such as:
+
+* POS-tagged word lists
+* Collocates lists
+* Sentences (token or string sequences)
+* Workflow step definitions
+* Token-based collocates
+
+By using these type guards, code can branch safely based on the structure and types of input data, improving robustness and developer experience.
+
+Example:
+
+.. code-block:: python
+
+    if is_pos_tags(tokens):
+        # tokens is now narrowed to POSTaggedWordList type
+        process_pos_tags(tokens)
+"""
+
+from typing import Any, TypeGuard
+from ..lib_types import POSTaggedWordList, CollocatesList, Sentences, Token, Workflow, POSTaggedWord, Bigram, Collocates
+
+def is_pos_tags(tokens: Any) -> TypeGuard[POSTaggedWordList]:
+    """
+    Checks if the input is a list of POS-tagged words (POSTaggedWordList).
+
+    :param tokens: The object to check.
+    :type tokens: Any
+
+    :return: True if `tokens` is a non-empty list where all elements are instances of POSTaggedWord, otherwise False.
+    :rtype: bool
+    """
+
+    if not isinstance(tokens, list) or not tokens:
+        return False
+    return all(isinstance(item, POSTaggedWord) for item in tokens)
+
+def is_collocates(tokens: Any) -> TypeGuard[CollocatesList]:
+    """
+    Checks if the input is a list of collocate tuples, where each tuple contains strings or Token instances, cexcluding types like POSTaggedWord or Bigram.
+
+    :param tokens: The object to check.
+    :type tokens: Any
+
+    :return: True if `tokens` is a non-empty list of tuples of strings or Token instances (excluding POSTaggedWord and Bigram), otherwise False.
+    :rtype: bool
+    """
+
+    if not isinstance(tokens, list) or not tokens:
+        return False
+
+    return all(
+        isinstance(item, tuple) and
+        not isinstance(item, POSTaggedWord) and
+        not isinstance(item, Bigram) and
+        all(
+            isinstance(word, str) or
+            isinstance(word, Token)
+            for word in item
+        )
+        for item in tokens
+    )
+
+def is_sentence(tokens: Any) -> TypeGuard[Sentences]:
+    """
+    Checks if the input is a list of sentences, where each sentence is either:
+    
+    * A list of Token instances,
+    * A list of strings, or
+    * A list of POSTaggedWord instances.
+
+    :param tokens: The object to check.
+    :type tokens: Any
+
+    :return: True if `tokens` matches the described sentence structure, otherwise False.
+    :rtype: bool
+    """
+
+    if not isinstance(tokens, list) or not tokens:
+        return False
+
+    return (
+        isinstance(tokens, list) and 
+        all(
+            (
+                isinstance(item, list) and 
+                (
+                    all(isinstance(token, Token) for token in item) or 
+                    all(isinstance(token, str) for token in item)
+                )
+            ) or is_pos_tags(item)
+            for item in tokens
+        )
+    )
+
+def is_workflow(workflow: Any) -> TypeGuard[Workflow]:
+    """
+    Checks if the input is a workflow structure, i.e., a non-empty list of dictionaries where each dictionary maps string method names to argument dictionaries with string keys.
+
+    :param workflow: The object to check.
+    :type workflow: Any
+
+    :return: True if `workflow` matches the workflow structure, otherwise False.
+    :rtype: bool
+    """
+
+    if not isinstance(workflow, list) or not workflow:
+        return False
+
+    return all(
+        isinstance(method, dict) and
+        all(
+            isinstance(key, str) and
+            isinstance(value, dict) and
+            all(
+                isinstance(arg, str)
+                for arg in value.keys()
+            )
+            for key, value  in method.items()
+        )
+        for method in workflow
+    )
+
+def is_token_collocates(collocates: Collocates) -> TypeGuard[Collocates[Token]]:
+    """
+    Checks if the input collocates tuple consists exclusively of Token instances and excludes Bigram and POSTaggedWord types.
+
+    :param collocates: The collocates tuple to check.
+    :type collocates: Collocates
+
+    :return: True if all elements in `collocates` are Token instances and not Bigram or POSTaggedWord, otherwise False.
+    :rtype: bool
+    """
+
+    return isinstance(collocates, tuple) and not isinstance(collocates, Bigram) and not isinstance(collocates, POSTaggedWord) and all(
+        isinstance(token, Token)
+        for token in collocates
+    )

dstk/hooks/__init__.py

@@ -0,0 +1,2 @@
+from .type_conversion import *
+from .hook_tools import *

dstk/hooks/hook_tools.py

@@ -0,0 +1,89 @@
+"""
+Module providing the Hook class for wrapping callable methods with a strict single-argument interface.
+
+The Hook class allows encapsulating any callable that accepts exactly one argument, enforcing this constraint at runtime. It supports invocation, argument validation, and renaming of the hook instance.
+
+This module is useful for building extensible and modular modela where hooks act as customizable processing steps.
+"""
+
+from __future__ import annotations
+from typing import Any, Callable
+from inspect import signature, Signature, Parameter
+from copy import deepcopy
+from collections.abc import ValuesView
+
+class Hook:
+    """
+    Represents a callable hook that wraps a single-argument function.
+
+    A Hook encapsulates a method that must accept exactly one argument, enabling
+    modular processing steps or callbacks within workflows or pipelines.
+
+    :param name: A descriptive name for the hook.
+    :type name: str
+    :param method: A callable that takes exactly one argument and performs some operation
+    :type method: Callable[[Any], Any]
+
+    Usage:
+    
+    .. code-block:: python
+
+        CustomHook = Hook("example_hook", some_function)
+        result = CustomHook(data)  # Calls some_function(data)
+    """
+
+    def __init__(self, name: str, method: Callable[[Any], Any]):
+        """
+        Initializes WorkflowBuilder with given attributes.
+        """
+
+        self.name: str = name
+        self.method: Callable = method
+
+    def __call__(self, *args, **kwargs) -> Any:
+        """
+        Invokes the wrapped method with the provided arguments.
+
+        Ensures that the wrapped method accepts exactly one argument before calling it. Raises a ValueError if the method signature does not conform to this requirement.
+
+        :param args: Positional arguments to pass to the wrapped method.
+        :param kwargs: Keyword arguments to pass to the wrapped method.
+
+        :return: The result of calling the wrapped method with the given arguments.
+        :rtype: Any
+
+        :raises ValueError: If the wrapped method does not accept exactly one argument.
+        """
+        if not self._check_args():
+            raise ValueError("A hook must accept exactly one argument")
+
+        return self.method(*args, **kwargs)
+
+    def _check_args(self) -> bool:
+        """
+        Checks whether the wrapped method accepts exactly one argument (positional or keyword).
+
+        :return: True if the method accepts exactly one argument, False otherwise.
+        :rtype: bool
+        """
+        sig: Signature = signature(self.method)
+        params: ValuesView[Parameter] = sig.parameters.values()
+
+        normalized_params: list[Parameter] = [param for param in params if param.kind in (param.POSITIONAL_ONLY, param.POSITIONAL_OR_KEYWORD, param.KEYWORD_ONLY)]
+
+        return True if len(normalized_params) == 1 else False
+
+    def rename(self, new_name: str) -> Hook:
+        """
+        Creates a new Hook instance with the same method but a different name.
+
+        :param new_name: The new name for the hook.
+        :type new_name: str
+
+        :return: A new Hook instance with the updated name.
+        :rtype: Hook
+        """
+        new_instance: Hook = deepcopy(self)
+        new_instance.name = new_name
+
+        return new_instance

dstk/hooks/type_conversion.py

@@ -0,0 +1,40 @@
+"""
+This module provides hooks and utilities for converting between different data types used in the processing model. 
+
+Currently, it includes a hook to convert trained word embedding models (Word2Vec or FastText) into pandas DataFrames, enabling easier manipulation and analysis of embeddings.
+
+Additional conversion hooks can be added in the future to support other type transformations.
+"""
+
+import pandas as pd
+import numpy as np
+from .hook_tools import Hook
+
+from ..lib_types import Words, ndarray, DataFrame, Word2Vec, FastText, NeuralModels
+
+def model_to_dataframe(model: NeuralModels) -> DataFrame:
+    """
+    Converts a trained Word2Vec or FastText model into a DataFrame of word embeddings.
+
+    :param model: A trained Word2Vec or FastText model.
+    :type model: NeuralModels
+
+    :return: A DataFrame containing the word embeddings and their associated labels.
+    :rtype: DataFrame
+    """
+
+    word_vectors: ndarray
+    labels: list[str]
+
+    if isinstance(model, Word2Vec):
+        word_vectors = model.wv[model.wv.index_to_key]
+        labels = list(model.wv.index_to_key)
+    elif isinstance(model, FastText):
+        words: Words[str] = model.words
+        word_vectors = np.array([model[word] for word in words])
+        labels = words
+
+    return pd.DataFrame(word_vectors, index=labels)
+
+ModelToDataframe: Hook = Hook(name="ModelToDataframe", method=model_to_dataframe)
+

dstk/lib_types/__init__.py

@@ -2,8 +2,7 @@ from .spacy_types import *
 from .sklearn_types import *
 from .numpy_types import *
 from .pandas_types import *
-from .matplotlib_types import *
 from .fasttext_types import *
 from .gensim_types import *
-from .nltk_types import *
-from .dstk_types import *
+from .dstk_types import *
+from .plotly_types import *

dstk/lib_types/dstk_types.py

@@ -1,26 +1,198 @@
-from typing import TypeAlias, Callable, TypeVar, Any
-from .spacy_types import Doc, Span, Token
+from typing import TypeAlias, TypeVar, Any, TypedDict, NotRequired, NamedTuple, Generator
+from .spacy_types import Token
 from .sklearn_types import csc_matrix, csr_matrix
 from .numpy_types import ndarray, NDArray, str_
 from .pandas_types import Index
+from .fasttext_types import FastText
+from .gensim_types import Word2Vec
+from collections import Counter
 
-# TextProcessor
-TokenIterator: TypeAlias = Doc | Span | list[Token]
-POSIterator: TypeAlias = list[tuple[Token, str]]
-SentenceIterator: TypeAlias = list[Span] | list[list[Token]] | list[POSIterator]
-TextIterator: TypeAlias = list[str] | list[tuple[str, str]] | list[list[tuple[str, str]]]
+#: Numeric types accepted (integer or float).
+Number: TypeAlias = int | float
 
-Sentence: TypeAlias = Span | list[Token] | POSIterator | list[tuple[str, str]]
-Sentences: TypeAlias =  list[Sentence]
-POSTags: TypeAlias = list[tuple[Token | str, str]]
-Function = TypeVar("Function", bound=Callable[..., object])
+#: A generic type variable for words, bounded to str or spaCy Token.
+Word = TypeVar("Word", bound=str | Token)
 
-# TargetCollocations
-Collocate = tuple[str, tuple[str, str]] | tuple[str, str]
+#: A list of words (strings or spaCy Tokens).
+Words: TypeAlias = list[Word]
 
-# Matrices
+#: A tuple representing a group of collocates (words).
+Collocates: TypeAlias = tuple[Word, ...]
+
+#: A list of collocate tuples.
+CollocatesList = list[Collocates]
+
+class POSTaggedWord(NamedTuple):
+    """
+    Represents a word paired with its Part-Of-Speech (POS) tag.
+
+    :param word: The word, either as a string or spaCy Token.
+    :type word: str or Token
+    :param pos: The POS tag of the word.
+    :type pos: str
+    """
+
+    word: str | Token
+    pos: str
+
+#: A list of POS-tagged words.
+POSTaggedWordList: TypeAlias = list[POSTaggedWord]
+
+class Bigram(NamedTuple):
+    """
+    Represents a bigram collocation between two words.
+
+    :param collocate: The collocate word.
+    :type collocate: str or Token
+    :param target_word: The target word in the bigram.
+    :type target_word: str
+    """
+
+    collocate: str | Token
+    target_word: str
+
+#: A list of bigram tuples.
+BigramList: TypeAlias = list[Bigram]
+
+#: Directed collocates represented as a tuple of a word and a pair of directional tags.
+DirectedCollocates: TypeAlias = tuple[Word, tuple[str, str]]
+
+#: A list of directed collocates.
+DirectedCollocateList: TypeAlias = list[DirectedCollocates]
+
+#: Union type of all tagged word lists.
+TaggedWordsList: TypeAlias = CollocatesList | DirectedCollocateList | POSTaggedWordList | BigramList
+
+#: A list of sentences, where each sentence is a list of words.c
+WordSenteces: TypeAlias = list[Words]
+#: A list of tagged sentences, each containing tagged words.c
+TaggedSentences: TypeAlias = list[TaggedWordsList]
+
+#: Union type representing either plain or tagged sentences.
+Sentences: TypeAlias = WordSenteces | TaggedSentences
+
+
+#: A tuple representing a neighboring word and its association score.
+class Neighbor(NamedTuple):
+    word: str
+    score: float
+#: A list of neighboring words with scores.
+Neighbors: TypeAlias = list[Neighbor]
+
+#: A union of neural language model types.
+NeuralModels: TypeAlias = Word2Vec | FastText
+
+#: A counter mapping words (strings) to their frequency counts.
+WordCounts: TypeAlias = Counter[str]
+
+#: A union of matrix types from SciPy or NumPy.
 Matrix: TypeAlias = csr_matrix | csc_matrix | ndarray
+
+#: Labels used in pandas DataFrames, representing index or column labels.
+#:
+#: This can be a NumPy ndarray of strings, a pandas Index, a list of strings, or None.
 Labels: TypeAlias = NDArray[str_] | Index | list[str] | None
 
-#Workflow
-MethodSpec: TypeAlias = dict[str, dict[str, Any]]
+StepConfig = TypedDict(
+    "StepConfig",
+    {
+        "include": NotRequired[list[str] | str],
+        "exclude": NotRequired[dict[str, int]],
+        "repeat": bool,
+        "chaining": bool,
+        "step_name": str
+    },
+    total=True
+)
+"""
+Configuration for a processing step in a workflow.
+
+:param include: Methods to include, either a list of strings or a single string.
+:type include: list[str] or str, optional
+:param exclude: Methods to exclude, as a dictionary mapping strings to integers.
+:type exclude: dict[str, int], optional
+:param repeat: Whether the a method can be used more than once.
+:type repeat: bool
+:param chaining: Whether method cchaining is enabled.
+:type chaining: bool
+:param step_name: The name of the step.
+:type step_name: str
+"""
+
+WorkflowTemplate = TypedDict(
+    "WorkflowTemplate",
+    {
+        "steps": dict[int, StepConfig],
+        "base_type": str,
+        "triggers": dict[str, str]
+    }
+)
+"""
+Template for an entire workflow, consisting of steps, a base type and triggers.
+
+:param steps: Mapping from step numbers to step configurations.
+:type steps: dict[int, StepConfig]
+:param base_type: The base type of the workflow.
+:type base_type: str
+:param triggers: Mapping from method names to the data types they produce. When a method changes the current data type (the default return type),the corresponding trigger activates rules that enable or disable subsequent methods.
+:type triggers: dict[str, str]
+"""
+
+#: A workflow is a list of ordered steps, where each step is a dictionary
+#: mapping method names to their keyword arguments.
+Workflow: TypeAlias = list[dict[str, dict[str, Any]]]
+#: A stage workflow contains multiple workflows organized by module names.
+#: Each key is a module name (e.g., 'tokenizer', 'ngrams', 'text_processor'),
+#: and the value is the workflow steps for that module.
+StageWorkflow: TypeAlias = dict[str, Workflow]
+#: Mapping from stage names to their corresponding workflow templates.
+#:
+#: Each key is a stage name (a string identifying a module),
+#: and the value is a `WorkflowTemplate` describing the processing steps and triggers
+#: allowed in that stage.
+StageTemplate: TypeAlias = dict[str, WorkflowTemplate]
+#: Mapping from stage indices (integers) to sets of module names allowed in that stage.
+#:
+#: Each key is a stage number, and the value is a set of module names (strings) that
+#: are enabled or active during that stage of the stage workflow.
+StageModules: TypeAlias = dict[int, set[str]]
+
+ExcludedMethods = TypedDict(
+    "ExcludedMethods",
+    {
+        "exclude": list[str] | str
+    }
+)
+"""
+Specifies methods to exclude by name.
+
+:param exclude: A list of method names or a single method name to exclude.
+:type exclude: list[str] or str
+"""
+
+#: Template defining rules for excluding methods once a specific type is triggered.
+#:
+#: The outer dictionary keys are module names (e.g., 'tokenizer', 'text_processor'),
+#: and the values specify which methods should be excluded in that module.
+#:
+#: For example, when the data type changes to 'POSTaggedWordList', these rules
+#: prevent further usage of specific methods like 'pos_tagger' in the tokenizer module
+RulesTemplate: TypeAlias = dict[str, ExcludedMethods]
+
+class StepResult(NamedTuple):
+    """
+    Represents the result of executing a single workflow or model step.
+
+    :param name: The name of thec step.
+    :param result: The output produced by the step.
+    """
+
+    name: str
+    result: Any
+
+
+#: Generator that yields `StepResult` objects, each representing the name and result of a workflow step.
+StepGenerator: TypeAlias = Generator[StepResult, None, None]
+
+#: Generator that yields results of workflow steps without step metadata.
+ResultGenerator: TypeAlias = Generator[Any, None, None]

dstk/lib_types/plotly_types.py

@@ -0,0 +1 @@
+from plotly.graph_objects import Figure

dstk/method_index.py

@@ -0,0 +1,32 @@
+INDEX = {
+    "tokenizer": {
+        "apply_model": {
+            "input": "str",
+            "output": "Doc"
+        },
+        "get_tokens": {
+            "input": "Doc",
+            "output": "Words[Tokens]"
+        },
+        "get_sentences": {
+            "input": "Doc",
+            "output": "Check it"
+        },
+        "remove_stop_words": {
+            "input": "Words[Token]",
+            "output": "Words[Token]"
+        },
+        "alphanumeric_raw_tokenizer": {
+            "input": "Words[Token]",
+            "output": "Words[Token]"
+        },
+        "filter_by_pos": {
+            "input": "Words[Token]",
+            "output": "Words[Token]"
+        },
+        "pos_tagger": {
+            "input": "Words[Token]",
+            "output": "POSTaggedText"
+        },
+    }
+}

dstk/models/__init__.py

@@ -0,0 +1,2 @@
+from .model_tools import *
+from .models import *