speechmetryflow 0.2.2__py3-none-any.whl

speechmetryflow/speech_production/fluency.py

@@ -0,0 +1,42 @@
+from speechmetryflow.speech_production.assets import filled_pauses
+
+
+def count_silent_pauses(text, lang=None, pause_marker=None):
+    """
+    Counts the total number of occurrences of "<pause_marker>" in the text, according to the specified language.
+
+    Args:
+    text (str): The text to be analysed.
+    lang (str): The language of the text ('en' or 'fr').
+    pause_marker (str): Overwrite the specific language pause_marker
+
+    Returns:
+    int: The number of occurrences of silent pauses.
+    """
+    if pause_marker is None:
+        pause_markers = {"en": "[break]", "fr": "[pause]"}
+        pause_marker = pause_markers.get(lang)
+
+    if pause_marker is None:
+        return
+
+    return text.count(pause_marker)
+
+
+def count_filled_pauses(text, lang):
+    """
+    Counts the total number of occurrences of specified words in the text, according to language.
+
+    Args:
+    text (str): The text to be analysed.
+    language (str): The language of the text ('en' or 'fr').
+
+    Returns:
+    int: The number of occurrences of filled pauses.
+    """
+    pauses = filled_pauses.get(lang)
+
+    if pauses is None:
+        return
+
+    return sum(text.count(pause) for pause in pauses)

speechmetryflow/speech_production/fragment.py

@@ -0,0 +1,76 @@
+""""""
+
+import importlib_resources
+import json
+
+from speechmetryflow.speech_production.assets import target_words
+
+
+class Fragment:
+    def __init__(self, lang, task):
+        self.lang = lang
+        self.task = task
+        self.valid_words = self._load_valid_words()
+        self.target_words = target_words.get(lang)
+
+    def _load_valid_words(self):
+        if self.lang == "en":
+            from nltk.corpus import words
+
+            return set(words.words())
+
+        elif self.lang == "fr":
+            resources = importlib_resources.files(__name__) / "assets"
+            with (resources / "words_fr.json").open("r", encoding="utf-8") as file:
+                return set(json.load(file))
+
+        else:
+            print(
+                f'language "{self.lang}" not currently recognized for fragment counting'
+            )
+            return None
+
+    def counter(self, tokens, verbose=False):
+        """
+        Identifies and counts word fragments in a list of tokens
+
+        Args:
+        tokens (list): The list of tokens
+        verbose (bool): If true, displays fragments found.
+
+        Returns:
+        int: The number of word fragments
+        """
+        if self.valid_words is None:
+            return
+
+        # Fragments identification
+        fragments = [token for token in tokens if token.lower() not in self.valid_words]
+
+        if verbose:
+            print(fragments)
+
+        return len(fragments)
+
+    def context(self, tokens):
+        """
+        Identifies and counts word fragments in a list of tokens
+
+        Args:
+        tokens (list): The list of tokens
+        verbose (bool): If true, displays fragments found.
+
+        Returns:
+        int: The number of word fragments
+        """
+        if self.task != "cookie":
+            return
+
+        if self.target_words is None:
+            return
+
+        context_counter = 0
+        # Browse two consecutives tokens
+        for comb in zip(tokens, tokens[1:]):
+            context_counter += comb in self.target_words
+        return context_counter

speechmetryflow/syntactic/__init__.py

@@ -0,0 +1,119 @@
+from typing import Dict, Union
+import spacy
+
+
+from speechmetryflow.syntactic.dependency import Dependency
+from speechmetryflow.syntactic.sentences import (
+    Sentences,
+    nominal_sentences_stats,
+    clauses_per_sentence,
+)
+
+
+def count_verb_tenses(doc: spacy.tokens.Doc, lang: str) -> Dict[str, float | None]:
+    """
+    Count the number and proportion of verbs in different tenses within a document.
+
+    This function analyzes verb tenses for English texts based on part-of-speech
+    tags provided by SpaCy. For non-English languages, it returns `None` values
+    for all tense-related fields.
+
+    The following English verb tags are used:
+    - Present tense: VBP, VBZ, VBG
+    - Past tense: VBD, VBN
+
+    Parameters
+    ----------
+    doc : spacy.tokens.Doc
+        A SpaCy Doc object representing the processed text.
+    lang : str
+        The ISO language code of the document (e.g., 'en' for English).
+
+    Returns
+    -------
+    dict
+        A dictionary containing the following keys:
+        - "n_present_verb": int or None
+          Number of verbs in the present tense.
+        - "prop_present_verb": float or None
+          Proportion of present-tense verbs relative to document length.
+        - "n_past_verb": int or None
+          Number of verbs in the past tense.
+        - "prop_past_verb": float or None
+          Proportion of past-tense verbs relative to document length.
+    """
+    if lang != "en":
+        empty_results = {}
+        for tense in ["present", "past"]:
+            empty_results[f"n_{tense}_verb"] = None
+            empty_results[f"prop_{tense}_verb"] = None
+        return empty_results
+
+    verbs = [token for token in doc if token.pos_ == "VERB"]
+
+    tenses = {
+        "present": 0,
+        "past": 0,
+    }
+
+    for verb in verbs:
+        if verb.tag_ in [
+            "VBP",
+            "VBZ",
+            "VBG",
+        ]:  # Typical tags for the present tense in English
+            tenses["present"] += 1
+        elif verb.tag_ in ["VBD", "VBN"]:  # Typical tags for the past tense in English
+            tenses["past"] += 1
+        # Note: the future tense in English is often marked by auxiliary verbs and does not have a specific tag.
+
+    results = {}
+    for tense, count in tenses.items():
+        results[f"n_{tense}_verb"] = count
+        results[f"prop_{tense}_verb"] = count / len(doc)
+
+    return results
+
+
+def nouns_with_determiners_proportion(doc: spacy.tokens.Doc) -> Union[float, None]:
+    """
+    Compute the proportion of nouns in a document that have determiners.
+
+    The function identifies all tokens with the part-of-speech tag "NOUN"
+    and checks how many of them have at least one dependent token with
+    the dependency label "det" (determiner). It then returns the ratio
+    of such nouns to the total number of nouns in the document.
+
+    Parameters
+    ----------
+    doc : spacy.tokens.Doc
+        A SpaCy Doc object representing the processed text.
+
+    Returns
+    -------
+    float or None
+        The proportion of nouns that have determiners.
+        Returns None if the document contains no nouns.
+    """
+    nouns = [token for token in doc if token.pos_ == "NOUN"]
+    n_nouns = len(nouns)
+
+    determiners = 0
+    for noun in nouns:
+        if any(child.dep_ == "det" for child in noun.children):
+            determiners += 1
+
+    return determiners / n_nouns if n_nouns else None
+
+
+def metrics(doc, lang):
+    dependency = Dependency(doc)
+    metrics = dependency.metrics
+    metrics.update(Sentences(doc, lang).metrics)
+    metrics.update(nominal_sentences_stats(doc))
+    metrics.update(count_verb_tenses(doc, lang))
+    metrics["clauses_per_sentence"] = clauses_per_sentence(doc)
+    metrics["nouns_with_determiners_proportion"] = nouns_with_determiners_proportion(
+        doc
+    )
+    return metrics

speechmetryflow/syntactic/assets.py

@@ -0,0 +1,40 @@
+# Database Caractéristiques syntaxiques
+
+## Traductions des étiquettes de dépendance en français
+dep_mapping = {
+    "nsubj": "Sujet_nominal",
+    "csubj": "Sujets_Clausaux",
+    "ROOT": "Racine",
+    "det": "Determinant",
+    "amod": "Modificateur_adjectival",
+    "compound": "Compose",
+    "dobj": "Objet_direct",
+    "prep": "Preposition",
+    "pobj": "Objet_de_preposition",
+    "punct": "Ponctuation",
+    "expl": "Element_expletif",
+    "attr": "Attribut",
+    "acl": "Clause_adjectivale",
+    "advcl": "Clause_adverbiale",
+    "aux": "Auxiliaire",
+    "xcomp": "Complement_ouvert",
+    "ccomp": "Complements_Clausaux_Non_Controles",
+    "conj": "Conjonction",
+    "cc": "Conjonction_de_coordination",
+    "neg": "Negation",
+    "dative": "Datif",
+}
+
+translation_dict = {
+    "csubj": "Sujets_Clausaux",
+    "xcomp": "Complements_Clausaux_Controles",
+    "ccomp": "Complements_Clausaux_Non_Controles",
+    "advcl": "Modificateurs_Clauses_Adverbiaux",
+    "acl": "Modificateurs_Clauses_Adnominaux",
+}
+
+## Dictionnaire des conjonctions de coordination par langue
+coordinating_conjunctions = {
+    "en": {"and", "but", "for", "nor", "or", "yet", "so"},
+    "fr": {"et", "mais", "ou", "donc", "or", "ni", "car"},
+}

speechmetryflow/syntactic/dependency.py

@@ -0,0 +1,47 @@
+import numpy as np
+from collections import Counter
+
+from speechmetryflow.syntactic.assets import dep_mapping
+
+
+class Dependency:
+    def __init__(self, doc):
+        self.doc = doc
+        self.dependencies = [token.dep_ for token in doc]
+        self.counts = self._counts()
+        self.len_doc = len(doc)
+        self.distances = [abs(token.i - token.head.i) for token in self.doc]
+        self.lefts = [token.n_lefts for token in self.doc]
+        self.rights = [token.n_rights for token in self.doc]
+        self.metrics = {}
+        self._compute_metrics()
+
+    def _counts(self):
+        counts = Counter(dep_mapping.keys())
+        counts.subtract(dep_mapping.keys())
+        counts.update(self.dependencies)
+        return counts
+
+    def _compute_metrics(self):
+        self.counter()
+        self.proportions()
+
+        # average and maximum lengths of dependencies
+        self.metrics["dep_mean_distance"] = np.mean(self.distances)
+        self.metrics["dep_max_distance"] = max(self.distances)
+
+        # number of left-dependencies and right-dependencies for each word.
+        self.metrics["dep_total_n_lefts"] = sum(self.lefts)
+        self.metrics["dep_total_n_rights"] = sum(self.rights)
+        self.metrics["dep_mean_n_lefts"] = np.mean(self.lefts)
+        self.metrics["dep_mean_n_rights"] = np.mean(self.rights)
+
+    def counter(self):
+        for tag, count in self.counts.items():
+            self.metrics[f"dep_count_{tag}"] = count
+
+    def proportions(self):
+        for tag, count in self.counts.items():
+            self.metrics[f"dep_prop_{tag}"] = (
+                count / self.len_doc if self.len_doc > 0 else None
+            )

speechmetryflow/syntactic/sentences.py

@@ -0,0 +1,189 @@
+from typing import Iterable, Dict, Any, Union
+import spacy
+
+from speechmetryflow.syntactic.assets import coordinating_conjunctions
+
+
+class Sentences:
+    def __init__(self, doc, lang):
+        self.doc = doc
+        self.lang = lang
+        self.len_doc = len(doc)
+        self.metrics = {}
+        self._compute_metrics()
+
+    def _compute_metrics(self):
+        # average sentence length in the text
+        self.metrics["mean_length_sentence"] = self.len_doc / len(list(self.doc.sents))
+
+        # Count type of sentences
+        incomplete = 0
+        prepositional = 0
+        verbal = 0
+        conjunctions = 0
+        for sentence in self.doc.sents:
+            incomplete += is_incomplete(sentence)
+            prepositional += is_prepositional(sentence)
+            verbal += is_verbal(sentence)
+            conjunctions += is_conjunctions(sentence, self.lang)
+        self.metrics["n_incomplete_sentences"] = incomplete
+        self.metrics["n_prepositional_sentences"] = prepositional
+        self.metrics["n_verbal_sentences"] = verbal
+
+        if self.lang in coordinating_conjunctions.keys():
+            self.metrics["n_conjunctions_sentences"] = conjunctions
+            self.metrics["prop_conjunctions_sentences"] = conjunctions / self.len_doc
+        else:
+            self.metrics["n_conjunctions_sentences"] = None
+            self.metrics["prop_conjunctions_sentences"] = None
+
+
+def is_incomplete(sentence: Iterable[spacy.tokens.Token]) -> bool:
+    """
+    Determine whether a sentence is syntactically incomplete.
+
+    A sentence is considered incomplete if it lacks either a subject
+    (a token with dependency label 'nsubj' or 'csubj') or a main verb
+    (a token with part-of-speech tag 'VERB').
+
+    Parameters
+    ----------
+    sentence : Iterable[spacy.tokens.Token]
+        A sequence of SpaCy token objects representing the sentence.
+
+    Returns
+    -------
+    bool
+        True if the sentence is incomplete (missing subject or verb),
+        False otherwise.
+    """
+    sent_pos = [token.pos_ for token in sentence]
+    sent_dep = [token.dep_ for token in sentence]
+    has_subject = "nsubj" in sent_dep or "csubj" in sent_dep
+    has_verb = "VERB" in sent_pos
+    return not (has_verb and has_subject)
+
+
+def is_prepositional(sentence: Iterable[spacy.tokens.Token]) -> bool:
+    """
+    Determine whether a sentence is headed by or centered around
+    a prepositional structure.
+
+    The function checks if the sentence contains at least one preposition
+    (token with part-of-speech tag 'ADP') that governs an object
+    (dependency label 'pobj' or 'dobj'). This helps identify sentences
+    whose main syntactic relation is prepositional.
+
+    Parameters
+    ----------
+    sentence : Iterable[spacy.tokens.Token]
+        A sequence of spacy token objects representing the sentence.
+
+    Returns
+    -------
+    bool
+        True if the sentence contains a prepositional structure
+        (a preposition with an object), False otherwise.
+    """
+    for token in sentence:
+        if token.pos_ == "ADP":  # 'ADP' is the POS tag for prepositions in spacy
+            # Check if the preposition has an object as a dependent
+            return any(child.dep_ in ["pobj", "dobj"] for child in token.children)
+    return False
+
+
+def is_verbal(sentence: Iterable[spacy.tokens.Token]) -> bool:
+    """
+    Determine whether a sentence contains a verb.
+
+    The function checks if any token in the sentence has a part-of-speech
+    tag of 'VERB', indicating the presence of a verbal predicate or
+    verbal structure.
+
+    Parameters
+    ----------
+    sentence : Iterable[spacy.tokens.Token]
+        A sequence of SpaCy token objects representing the sentence.
+
+    Returns
+    -------
+    bool
+        True if the sentence contains at least one verb, False otherwise.
+    """
+    return any(token.pos_ == "VERB" for token in sentence)
+
+
+def is_conjunctions(sentence: Iterable[spacy.tokens.Token], lang: str) -> bool:
+    """
+    Determine whether a sentence contains any coordinating conjunctions.
+
+    Parameters
+    ----------
+    sentence : Iterable[spacy.tokens.Token]
+        A sequence of SpaCy token objects representing the sentence.
+    lang : str
+        The ISO language code of the sentence (e.g., 'en' for English, 'fr' for French).
+
+    Returns
+    -------
+    bool
+        True if the sentence contains at least one coordinating conjunction,
+        False otherwise.
+    """
+    conjunctions = coordinating_conjunctions.get(lang, set())
+    return any(token.lower_ in conjunctions for token in sentence)
+
+
+def nominal_sentences_stats(doc) -> Dict[str, Any]:
+    """
+    Compute statistics about nominal sentences (noun chunks).
+
+    Returns
+    -------
+        - "n_nominal_sentences": int
+        Number of nominal sentences (noun chunks)
+        - "mean_length_nominal_sentence": float
+        Average number of tokens per nominal sentence
+        - "prop_nominal_sentences": float
+        Proportion of nominal sentences relative to the document length
+    """
+    nominal_sentences = [chunk for chunk in doc.noun_chunks]
+    n_nominal_sentences = len(nominal_sentences)
+    length_nominal_sentences = sum(len(chunk) for chunk in nominal_sentences)
+    mean_length_nominal_sentence = (
+        length_nominal_sentences / n_nominal_sentences if n_nominal_sentences else None
+    )
+    return {
+        "n_nominal_sentences": n_nominal_sentences,
+        "mean_length_nominal_sentence": mean_length_nominal_sentence,
+        "prop_nominal_sentences": n_nominal_sentences / len(doc),
+    }
+
+
+def clauses_per_sentence(doc: spacy.tokens.Doc) -> Union[float, None]:
+    """
+    Compute the average number of clauses per sentence in a SpaCy document.
+
+    A clause is identified by the presence of specific dependency relations:
+    - 'csubj' (clausal subject)
+    - 'ccomp' (clausal complement)
+    - 'xcomp' (open clausal complement)
+
+    The function counts all tokens with these dependencies and divides by the
+    total number of sentences in the document.
+
+    Parameters
+    ----------
+    doc : spacy.tokens.Doc
+        A SpaCy Doc object representing the processed text.
+
+    Returns
+    -------
+    float or None
+        The mean number of clauses per sentence.
+        Returns None if the document contains no sentences.
+    """
+    # Count tokens whose dependency label indicates a clause
+    n_clauses = sum(1 for token in doc if token.dep_ in ["csubj", "ccomp", "xcomp"])
+    n_sents = len(list(doc.sents))
+    return n_clauses / n_sents if n_sents else None

speechmetryflow/ucsf_disfluency/__init__.py

@@ -0,0 +1,23 @@
+""""""
+
+from typing import Dict
+
+from collections import Counter
+
+
+def metrics(text: str) -> Dict:
+    disfluency_counter = Counter(text)
+    return {
+        "UCSF_disfluency_single_repetition": disfluency_counter["="],
+        "UCSF_disfluency_multiple_repetitions": disfluency_counter["@"],
+        "UCSF_disfluency_repeated_phrase": disfluency_counter["&"],
+        "UCSF_disfluency_restart_rephrase": disfluency_counter["#"],
+        "UCSF_disfluency_partial_word_false_start": disfluency_counter["%"],
+        "UCSF_disfluency_spoonerism": disfluency_counter["$"],
+    }
+
+
+def cleaning(text: str) -> str:
+    for char in "=@&#%$":
+        text = text.replace(char, "")
+    return text

speechmetryflow/utils.py

@@ -0,0 +1,53 @@
+""""""
+
+import re
+from nltk.stem import SnowballStemmer
+
+
+def general_cleaning(raw_text: str) -> str:
+    """
+    Cleans up text by performing several cleaning operations
+
+    Args:
+    raw_text (str): The text to be cleaned up
+
+    Returns:
+    str: Cleaned up text
+    """
+    # Removes content between square brackets ([...])
+    text = re.sub(r"\[.*?\]", "", raw_text)
+
+    # Removes punctuation, keeping letters, apostrophes, spaces, periods, and question marks
+    text = re.sub(r"[^\w\’ \.\?]", "", text)
+
+    # Removes periods following a space
+    text = re.sub(r"\s\.", "", text)
+
+    # Replaces two consecutive spaces with a single space
+    text = re.sub(r"\s\s+", " ", text)
+
+    return text
+
+
+def compute_stemming(tokens, lang):
+    """
+    Stemming with Snowball
+    https://snowballstem.org/
+
+    Args:
+        tokens list(str): text to compute.
+        lang (str): language of the text.
+
+    Returns:
+        list(str): stems from SnowballStemmer.
+    """
+    if lang == "en":
+        stemmer = SnowballStemmer("english")
+
+    elif lang == "fr":
+        stemmer = SnowballStemmer("french")
+
+    else:
+        return
+
+    return list(map(stemmer.stem, tokens))