dstklib 1.0.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

dstk/modules/weight_matrix.py

@@ -0,0 +1,65 @@
+"""
+This module provides functions to apply weighting schemes to co-occurrence matrices commonly used in natural language processing and text mining.
+
+Available weighting methods include:
+
+* Pointwise Mutual Information (PMI) and Positive PMI (PPMI), which measure the association strength between co-occurring terms by comparing observed co-occurrence frequencies to expected frequencies under independence.
+* Term Frequency-Inverse Document Frequency (Tf-idf), which reweights term importance based on frequency patterns, leveraging sklearn's TfidfTransformer.
+
+These weighting techniques help enhance the semantic relevance of co-occurrence matrices, improving downstream tasks such as word embedding, clustering, and semantic similarity analysis.
+
+All functions return weighted co-occurrence matrices as Pandas DataFrames for convenient further analysis.
+"""
+
+import pandas as pd
+import numpy as np
+from sklearn.feature_extraction.text import TfidfTransformer
+
+from ..lib_types import DataFrame, ndarray, Series, csr_matrix
+
+
+def pmi(co_matrix: DataFrame, positive: bool = False) -> DataFrame:
+    """
+    Weights a Co-occurrence matrix by PMI or PPMI.
+    
+    :param co_matrix: A Co-occurrence matrix to be weighted.
+    :type co_matrix: DataFrame
+    :param positive: If True, weights the Co-ocurrence matrix by PPMI. If False, weighths it by PMI. Defaults to False.
+    :type positive: bool
+
+    :returns: A Co-occurrence matrix weighted by PMI or PPMI.
+    :rtype: DataFrame
+    """
+
+    df: DataFrame = co_matrix
+
+    col_totals: Series = df.sum(axis=0)
+    total: float = col_totals.sum()
+    row_totals: Series = df.sum(axis=1)
+    expected: ndarray = np.outer(row_totals, col_totals) / total
+    df = df / expected
+    # Silence distracting warnings about log(0):
+    with np.errstate(divide='ignore'):
+        df = np.log(df)
+    df[np.isinf(df)] = 0.0  # log(0) = 0
+    if positive:
+        df[df < 0] = 0.0
+
+    return df
+
+def tf_idf(co_matrix: DataFrame, **kwargs) -> DataFrame:
+    """
+    Weights a Co-occurrence matrix by Tf-idf.
+    
+    :param co_matrix: A Co-occurrence matrix to be weighted.
+    :type co_matrix: DataFrame
+    :param kwargs: Additional keyword arguments to pass to sklearn's TfidfTransformer. For more information check: https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfTransformer.html
+        
+    :returns: A Co-occurrence matrix weighted by Tf-idf.
+    :rtype: DataFrame
+    """
+
+    transformer: TfidfTransformer = TfidfTransformer(**kwargs)
+    tf_idf_matrix: csr_matrix = transformer.fit_transform(co_matrix)
+
+    return pd.DataFrame(tf_idf_matrix.toarray(), index=co_matrix.index, columns=co_matrix.columns)

dstk/templates/__init__.py

@@ -0,0 +1,2 @@
+from .templates import *
+from .rules import *

dstk/templates/rules.py

@@ -0,0 +1,59 @@
+"""
+Defines type-based exclusion rules that constrain which methods can be applied to data at different stages in a workflow.
+
+Each rule maps a data type (e.g., `POSTaggedWordList`, `Sentences`, `str`, `Neighbors`) to a set of module-specific restrictions. These rules help ensure that operations are semantically valid and compatible with the current data representation, enabling type-aware validation and error handling during workflow execution.
+
+Structure:
+
+Each rule is a `RulesTemplate` (dict) where:
+
+* Keys are module names (e.g., "tokenizer", "text_processor").
+* Values define methods to exclude (either a list of method names or "*" for all).
+
+The `TypeRules` dictionary aggregates all individual rules and serves as a centralized configuration for type-based behavior enforcement.
+
+Use case:
+These rules are primarily used in the validation step of workflow builders to prevent method misuse based on data type.
+"""
+
+from ..lib_types import RulesTemplate
+
+POSTaggedWordListRules: RulesTemplate = {
+    "tokenizer": {
+        "exclude": ["pos_tagger"]
+    },
+    "text_processor": {
+        "exclude": ["join"]
+    },
+    "ngrams": {
+        "exclude": "*"
+    }
+}
+
+SentencesRules: RulesTemplate = {
+    "text_processor": {
+        "exclude": ["get_vocabulary", "save_to_file"]
+    },
+    "ngrams": {
+        "exclude": "*"
+    }
+}
+
+StringRules: RulesTemplate = {
+    "text_processor": {
+        "exclude": ["to_lower", "get_vocabulary", "join", "save_to_file"]
+    }
+}
+
+NeighborsRules: RulesTemplate = {
+    "geometric_distance": {
+        "exclude": "*"
+    }
+}
+
+TypeRules: dict[str, RulesTemplate] = {
+    "POSTaggedWordList": POSTaggedWordListRules,
+    "Sentences": SentencesRules,
+    "str": StringRules,
+    "Neighbors": NeighborsRules
+}

dstk/templates/templates.py

@@ -0,0 +1,231 @@
+"""
+Defines reusable templates that specify the structure and constraints of workflows and step-based pipelines.
+
+Each template outlines the allowed sequence of method steps for a given module, enforcing constraints such as:
+
+* Which methods are permitted or excluded at each step (`include` / `exclude`)
+* Whether methods can be used more than once (`repeat`)
+* Whether more than one method can be selected on each step (`chaining`)
+* How types are transformed via `triggers`
+
+Key Components:
+
+* **Workflow Templates** (`WorkflowTemplate`): Describe valid method sequences for individual modules (e.g., tokenization, text processing, dimensionality reduction).
+* **Stage Templates** (`StageTemplate`): Group related modules into stages to define multi-module workflows.
+* **Stage Modules** (`StageModules`): Define allowed module names for each stage in a stage-based workflow.
+
+These templates are used by `WorkflowBuilder` and `StageWorkflowBuilder` to validate workflows and enforce correct sequencing of operations.
+
+Examples of Defined Templates:
+
+* `TokenizerTemplate`: Defines the tokenization workflow including model selection, unit selection (sentences/tokens), and token processing.
+* `TextProcessorTemplate`: Defines generic text processing steps like  lowercasing, joining, etc.
+* `TextMatrixBuilderTemplate`: Specifies steps to create a document-term and co-occurrence matrix.
+* `PlotEmbeddingsTemplate`: Governs how word embeddings are plotted after clustering.
+
+The templates provide a flexible and declarative way to define what each step in a workflow is allowed to do based on processing intent and data type.
+"""
+
+from ..lib_types import WorkflowTemplate, StageTemplate, StageModules
+
+TokenizerTemplate: WorkflowTemplate = {
+    "steps": {
+        0: {
+            "include": ["apply_model"],
+            "repeat": False,
+            "chaining": False,
+            "step_name": "select_model"
+        },
+        1: {
+            "include": ["get_sentences", "get_tokens"],
+            "repeat": False,
+            "chaining": False,
+            "step_name": "select_processing_unit"
+        },
+        2: {
+            "include": ["remove_stop_words", "alphanumeric_raw_tokenizer"],
+            "repeat": False,
+            "chaining": False,
+            "step_name": "tokenization"
+        },
+        3: {
+            "include": "*",
+            "repeat": True,
+            "chaining": True,
+            "step_name": "token_processing"
+        }
+    },
+    "base_type": "Words", # is this necessary?
+    "triggers":  {
+        "pos_tagger": "POSTaggedWordList",
+        "get_sentences": "Sentences"
+    }
+}
+
+TextProcessorTemplate: WorkflowTemplate = {
+    "steps": {
+        0: {
+            "include": "*",
+            "repeat": False,
+            "chaining": True,
+            "step_name": "text_processing"
+        }
+    },
+    "base_type": "Words",
+    "triggers": {}
+}
+
+NgramsTemplate: WorkflowTemplate = {
+    "steps": {
+        0: {
+            "exclude": {"count_collocates": 1},
+            "repeat": False,
+            "chaining": False,
+            "step_name": "find_ngrams"
+        },
+        1: {
+            "include": ["count_collocates"],
+            "repeat": False,
+            "chaining": False,
+            "step_name": "count_collocates"
+        }
+    },
+    "base_type": "Words",
+    "triggers": {}
+}
+
+TextMatrixBuilderTemplate: WorkflowTemplate = {
+    "steps": {
+        0: {
+            "include": ["create_dtm"],
+            "repeat": False,
+            "chaining": True,
+            "step_name": "document_term_matrix"
+        },
+        1: {
+            "include": ["create_co_occurrence_matrix"],
+            "repeat": False,
+            "chaining": False,
+            "step_name": "co_occurrence_matrix"
+        }
+    },
+    "base_type": "DataFrame",
+    "triggers": {}
+}
+
+WeightMatrixTemplate: WorkflowTemplate = {
+    "steps": {
+        0: {
+            "include": "*",
+            "repeat": False,
+            "chaining": False,
+            "step_name": "weight_matrix"
+        }
+    },
+    "base_type": "DataFrame",
+    "triggers": {}
+}
+
+CountModelsTemplate: WorkflowTemplate = {
+    "steps": {
+        0: {
+            "include": ["scale_matrix"],
+            "repeat": False,
+            "chaining": False,
+            "step_name": "scale_matrix"
+        },
+        1: {
+            "include": "*",
+            "repeat": False,
+            "chaining": False,
+            "step_name": "dimensionality_reduction"
+        }
+    },
+    "base_type": "DataFrame",
+    "triggers": {}
+}
+
+GeometricDistanceTemplate: WorkflowTemplate = {
+    "steps": {
+        0: {
+            "include": "*",
+            "repeat": False,
+            "chaining": False,
+            "step_name": "geometric_distance"
+        }
+    },
+    "base_type": "float",
+    "triggers": {
+        "nearest_neighbors": "Neighbors"
+    }
+}
+
+PredictModelsTemplate: WorkflowTemplate = {
+    "steps": {
+        0: {
+            "exclude": {"save_model": 1},
+            "repeat": False,
+            "chaining": False,
+            "step_name": "select_model"
+        },
+        1: {
+            "include": ["save_model"],
+            "repeat": False,
+            "chaining": False,
+            "step_name": "save_model"
+        }
+    },
+    "base_type": "NeuralModels",
+    "triggers": {
+        "save_model": "str"
+    }
+}
+
+ClusteringTemplate: WorkflowTemplate = {
+    "steps": {
+        0: {
+            "include": "*",
+            "repeat": False,
+            "chaining": False,
+            "step_name": "clustering"
+        }
+    },
+    "base_type": "DataFrame",
+    "triggers": {}
+}
+
+PlotEmbeddingsTemplate: WorkflowTemplate = {
+    "steps": {
+        0: {
+            "include": "*",
+            "repeat": False,
+            "chaining": False,
+            "step_name": "embedings_plot"
+        }
+    },
+    "base_type": "Figure",
+    "triggers": {}
+}
+
+# Stage templates
+
+TextProcessingTemplates: StageTemplate = {
+    "tokenizer": TokenizerTemplate,
+    "text_processor": TextProcessorTemplate,
+    "ngrams": NgramsTemplate
+}
+
+TextProcessingStageModules: StageModules = {
+    0: {"tokenizer"},
+    1: {"text_processor", "ngrams"}
+}
+
+PlotEmbeddingsTemplates: StageTemplate = {
+    "data_visualization.clustering": ClusteringTemplate,
+    "data_visualization.embeddings": PlotEmbeddingsTemplate,
+}
+
+PlotEmbeddingsStageModules: StageModules = {
+    0: {"data_visualization.clustering"},
+    1: {"data_visualization.embeddings"}
+}

dstk/workflows/__init__.py

@@ -0,0 +1,2 @@
+from .workflow_tools import *
+from .stage_workflows import *

dstk/workflows/stage_workflows.py

@@ -0,0 +1,55 @@
+"""
+This module provides classes and factory functions to build and manage complex, multi-stage workflows composed of sequential method executions across different processing modules. It supports workflow validation against predefined templates, method chaining with type enforcement, and flexible execution  control, including partial or complete result retrieval.
+
+Key components include:
+
+* Factory functions like TextProcessing and PlotEmbeddings to easily instantiate common workflows with predefined templates and modules.
+
+Designed to facilitate modular, extensible, and maintainable workflow construction for tasks such as text processing and embedding visualization.
+"""
+
+from .workflow_tools import StageWorkflowBuilder
+from ..templates import TextProcessingTemplates, TextProcessingStageModules, PlotEmbeddingsTemplates, PlotEmbeddingsStageModules
+from ..lib_types import StageWorkflow
+
+def TextProcessing(name: str, workflows: StageWorkflow) -> StageWorkflowBuilder:
+    """
+    Creates a StageWorkflowBuilder configured for text processing workflows. The modules included are 'tokenizer' in the first stage and 'text_processor' or 'ngrams' in the second.
+
+    :param name: The name of the workflow instance.
+    :type name: str
+    :param workflows: A StageWorkflow dictionary defining the workflow steps per module/stage.
+    :type workflows: StageWorkflow
+
+    :return: An instance of StageWorkflowBuilder configured with text processing templates and modules.
+    :rtype: StageWorkflowBuilder
+    """
+    TextProcessingWorkflow = StageWorkflowBuilder(
+        templates=TextProcessingTemplates,
+        stage_modules=TextProcessingStageModules, 
+        name=name, 
+        workflows=workflows
+    )
+
+    return TextProcessingWorkflow
+
+def PlotEmbeddings(name: str, workflows: StageWorkflow) -> StageWorkflowBuilder:
+    """
+    Creates a StageWorkflowBuilder configured for word embedding plotting workflows. The modules included are 'data_visualization.clustering' in the first stage and 'data_visualization.embeddings' in the second.
+
+    :param name: The name of the workflow instance.
+    :type name: str
+    :param workflows: A StageWorkflow dictionary defining the workflow steps per module/stage.
+    :type workflows: StageWorkflow
+
+    :return: An instance of StageWorkflowBuilder configured with embedding plotting templates and modules.
+    :rtype: StageWorkflowBuilder
+    """
+    PlotEmbeddingsWorkflow = StageWorkflowBuilder(
+        templates=PlotEmbeddingsTemplates, 
+        stage_modules=PlotEmbeddingsStageModules, 
+        name=name, 
+        workflows=workflows
+    )
+
+    return PlotEmbeddingsWorkflow