speechmetryflow 0.2.2__py3-none-any.whl

speechmetryflow/lexical/database.py

@@ -0,0 +1,143 @@
+import pandas as pd
+import numpy as np
+import importlib_resources
+
+
+def load_database(tag, lang):
+    assets_folder = importlib_resources.files(__name__) / "assets"
+
+    if lang == "en":
+        tag2df = {
+            "concreteness": pd.read_csv(
+                assets_folder / "concreteness.tsv",
+                sep="\t",
+                usecols=["Word", "Conc.M"],
+                index_col="Word",
+            ).squeeze(),
+            "familiarity": pd.read_csv(
+                assets_folder / "familiarity_imageability.tsv",
+                sep="\t",
+                usecols=["Words", "FAM_M"],
+                index_col="Words",
+            ).squeeze(),
+            "imageability": pd.read_csv(
+                assets_folder / "familiarity_imageability.tsv",
+                sep="\t",
+                usecols=["Words", "IMAG_M"],
+                index_col="Words",
+            ).squeeze(),
+            "frequency": pd.read_csv(
+                assets_folder / "frequency.tsv",
+                sep="\t",
+                usecols=["Word", "SUBTLWF"],
+                index_col="Word",
+            ).squeeze(),
+            "valence": pd.read_csv(
+                assets_folder / "valence.tsv",
+                sep="\t",
+                usecols=["Word", "V.Mean.Sum"],
+                index_col="Word",
+            ).squeeze(),
+        }
+
+    else:
+        return
+
+    return tag2df.get(tag)
+
+
+def compute_database(tokens, lang, tag):
+    """
+    Compute the concreteness of words in a list from a concreteness database.
+
+    Args:
+        tokens (list): A list of words from which the average concreteness will be calculated.
+        lang (str): language of the text.
+
+    Returns:
+        numpy.array: concreteness of the words.
+    """
+    data = load_database(tag, lang)
+    if data is None:
+        return
+
+    return np.array(list(map(data.get, tokens))).astype(float)
+
+
+def compute_concreteness(tokens, lang):
+    """
+    Compute the concreteness of words in a list from a concreteness database.
+
+    Args:
+        tokens (list): A list of words from which the average concreteness will be calculated.
+        lang (str): language of the text.
+
+    Returns:
+        numpy.array: concreteness of the words.
+    """
+    return compute_database(tokens, lang, "concreteness")
+
+
+def compute_familiarity(tokens, lang):
+    """
+    Compute the familiarity of words in a list from a familiarity database.
+
+    Args:
+        tokens (list): A list of words from which the average familiarity will be calculated.
+        lang (str): language of the text.
+
+    Returns:
+        numpy.array: familiarity of the words.
+    """
+    return compute_database(tokens, lang, "familiarity")
+
+
+def compute_imageability(tokens, lang):
+    """
+    Compute the imageability of words in a list from a imageability database.
+
+    Args:
+        tokens (list): A list of words from which the average imageability will be calculated.
+        lang (str): language of the text.
+
+    Returns:
+        numpy.array: imageability of the words.
+    """
+    return compute_database(tokens, lang, "imageability")
+
+
+def compute_frequency(tokens, lang):
+    """
+    Compute the frequency of words in a list from a frequency database.
+
+    Args:
+        tokens (list): A list of words from which the average frequency will be calculated.
+        lang (str): language of the text.
+
+    Returns:
+        numpy.array: frequency of the words.
+    """
+    return compute_database(tokens, lang, "frequency")
+
+
+def compute_valence(tokens, lang):
+    """
+    Compute the valence of words in a list from a valence database.
+
+    Args:
+        tokens (list): A list of words from which the average valence will be calculated.
+        lang (str): language of the text.
+
+    Returns:
+        numpy.array: valence of the words.
+    """
+    return compute_database(tokens, lang, "valence")
+
+
+TAG2FUNC = {
+    "concreteness": compute_concreteness,
+    "familiarity": compute_familiarity,
+    "imageability": compute_imageability,
+    "frequency": compute_frequency,
+    "valence": compute_valence,
+}

speechmetryflow/lexical/part_of_speech.py

@@ -0,0 +1,172 @@
+import warnings
+import numpy as np
+from numpy.lib.stride_tricks import sliding_window_view
+from collections import Counter
+
+from speechmetryflow.lexical.assets import pos_mapping
+from speechmetryflow.lexical.database import TAG2FUNC
+
+
+class POS:
+    def __init__(self, doc, lang):
+        self.doc = doc
+        self.lang = lang
+        self.mapping = pos_mapping
+        self.pos = [token.pos_ for token in self.doc]
+        self.text = np.array([token.text for token in self.doc])
+        self.counts = self._counts()
+        self.len_text = len(doc)
+        self.metrics = {}
+        self._compute_metrics()
+
+    def _counts(self):
+        counts = Counter(pos_mapping.keys())
+        counts.subtract(pos_mapping.keys())
+        counts.update(self.pos)
+        return counts
+
+    def _compute_metrics(self):
+        self.counter()
+        self.proportions()
+        self.count_open_class_words()
+        self.count_closed_class_words()
+        self.count_verb_inflection()
+        self.count_verb_gerondif()
+        self.compute_ratios()
+        for window_shape in [10, 25, 40]:
+            self.compute_mattr(window_shape)
+        self.compute_database("concreteness")
+        self.compute_database("familiarity")
+        self.compute_database("imageability")
+        self.compute_database("frequency")
+        self.compute_database("valence")
+
+    def proportions(self):
+        for tag, count in self.counts.items():
+            self.metrics[f"pos_prop_{tag}"] = (
+                count / self.len_text if self.len_text > 0 else None
+            )
+
+    def counter(self):
+        for tag, count in self.counts.items():
+            self.metrics[f"pos_count_{tag}"] = count
+
+    def count_open_class_words(self):
+        open_classes = ["NOUN", "VERB", "ADJ", "ADV"]
+        num = sum(self.counts[pos] for pos in open_classes)
+        self.metrics["pos_count_open_words"] = num
+        self.metrics["pos_prop_open_words"] = (
+            num / self.len_text if self.len_text > 0 else None
+        )
+
+    def count_closed_class_words(self):
+        closed_classes = ["CONJ", "PRON", "DET", "ADP"]
+        num = sum(self.counts[pos] for pos in closed_classes)
+        self.metrics["pos_count_closed_words"] = num
+        self.metrics["pos_prop_closed_words"] = (
+            num / self.len_text if self.len_text > 0 else None
+        )
+
+    def count_verb_inflection(self):
+        """Verb inflection is the modification of a verb's form to express grammatical features such as tense."""
+        num = sum(
+            1 for token in self.doc if token.pos_ == "VERB" and token.tag_ != "VB"
+        )
+        self.metrics["pos_count_verb_inflection"] = num
+        self.metrics["pos_prop_verb_inflection"] = (
+            num / self.len_text if self.len_text > 0 else None
+        )
+
+    def count_verb_gerondif(self):
+        """
+        The gérondif is a French verb form expressing an action occurring in relation to another action.
+        Example: Il écoute de la musique en travaillant.
+        """
+        num = sum(1 for token in self.doc if token.tag_ != "VBG")
+        self.metrics["pos_count_verb_gerondif"] = num
+        self.metrics["pos_prop_verb_gerondif"] = (
+            num / self.len_text if self.len_text > 0 else None
+        )
+
+    def compute_ratios(self):
+        """
+        Computing the following ratios:
+        - PRON / (NOUN + PRON)
+        - NOUN / (NOUN + PRON)
+        - NOUN / (NOUN + VERB)
+        - VERB / (NOUN + VERB)
+        - VERB_INFLECTION / VERB
+        - VERB_GERONDIF / VERB
+        """
+        n_verb = self.counts["VERB"]
+        n_noun = self.counts["NOUN"]
+        n_pron = self.counts["PRON"]
+        n_inflection = self.metrics["pos_count_verb_inflection"]
+        n_gerondif = self.metrics["pos_count_verb_gerondif"]
+        # Calcul des ratios
+        self.metrics["PRON/(NOUN+PRON)"] = (
+            n_pron / (n_noun + n_pron) if (n_noun + n_pron) > 0 else None
+        )
+        self.metrics["NOUN/(NOUN+PRON)"] = (
+            n_noun / (n_noun + n_pron) if (n_noun + n_pron) > 0 else None
+        )
+        self.metrics["NOUN/(NOUN+VERB)"] = (
+            n_noun / (n_noun + n_verb) if (n_noun + n_verb) > 0 else None
+        )
+        self.metrics["VERB/(NOUN+VERB)"] = (
+            n_verb / (n_noun + n_verb) if (n_noun + n_verb) > 0 else None
+        )
+        self.metrics["VERB_INFLECTION/VERB"] = (
+            n_inflection / n_verb if n_verb > 0 else None
+        )
+        self.metrics["VERB_GERONDIF/VERB"] = n_gerondif / n_verb if n_verb > 0 else None
+
+    def compute_mattr(self, window_shape):
+        """
+        Compute the MATTR (Moving-Average Type-Token Ratio) using a window of specified size.
+
+        Args:
+            window_shape (int): The size of the text window to use.
+
+        Returns:
+            float: MATTR value.
+        """
+        try:
+            windows = sliding_window_view(self.pos, window_shape=window_shape)
+        except ValueError:
+            # if window_shape is larger than input array shape
+            return
+
+        # Compute the TTR (Type-Token Ratio) for each window
+        ttr_values = [len(set(window)) / window_shape for window in windows]
+
+        self.metrics[f"pos_mattr_{window_shape}"] = np.mean(ttr_values)
+
+    def compute_database(self, tag):
+        """"""
+        scores = TAG2FUNC[tag](self.text, self.lang)
+
+        if scores is None:
+            for pos_type in ["NOUN", "VERB", "ADJ"]:
+                self.metrics[f"{tag}_mean_{pos_type}"] = None
+            self.metrics[f"{tag}_mean_all_tokens"] = None
+            return
+
+        warnings.filterwarnings("error")
+        for pos_type in ["NOUN", "VERB", "ADJ"]:
+            pos_positions = np.nonzero(np.array(self.pos) == pos_type)[0]
+            sub_scores = scores[pos_positions]
+            try:
+                score = np.nanmean(sub_scores)
+            except RuntimeWarning:
+                score = 0.0
+            self.metrics[f"{tag}_mean_{pos_type}"] = score
+
+        # For all tokens
+        try:
+            score = np.nanmean(scores)
+        except RuntimeWarning:
+            score = 0.0
+        self.metrics[f"{tag}_mean_all_tokens"] = score
+
+        warnings.resetwarnings()

speechmetryflow/lexical/utils.py

@@ -0,0 +1,119 @@
+import math
+
+from speechmetryflow.lexical.assets import deictic_pronouns, indefinite_terms
+from speechmetryflow.speech_production import compute_counts
+
+
+def count_deictic_pronouns(tokens, lang):
+    """
+    Counts deictic pronouns in a text in English or French.
+
+    Args:
+    - tokens list[(str)]: The text to be analysed.
+    - lang (str): The language of the text ("fr", "en").
+
+    Returns:
+    - dict: A dictionary with the counts of spatial, personal, temporal pronouns and the total.
+    """
+    deictic_pronouns_data = deictic_pronouns.get(lang)
+
+    if deictic_pronouns_data is None:
+        return {
+            "n_deictic_pronouns_spatial": None,
+            "n_deictic_pronouns_personal": None,
+            "n_deictic_pronouns_temporal": None,
+            "n_deictic_pronouns_total": None,
+            "prop_deictic_pronouns_spatial": None,
+            "prop_deictic_pronouns_personal": None,
+            "prop_deictic_pronouns_temporal": None,
+            "prop_deictic_pronouns_total": None,
+        }
+
+    len_text = len(tokens)
+    n_spatial = sum(token in deictic_pronouns_data["spatial"] for token in tokens)
+    n_personal = sum(token in deictic_pronouns_data["personal"] for token in tokens)
+    n_temporal = sum(token in deictic_pronouns_data["temporal"] for token in tokens)
+    n_total = n_spatial + n_personal + n_temporal
+
+    return {
+        "n_deictic_pronouns_spatial": n_spatial,
+        "n_deictic_pronouns_personal": n_personal,
+        "n_deictic_pronouns_temporal": n_temporal,
+        "n_deictic_pronouns_total": n_total,
+        "prop_deictic_pronouns_spatial": n_spatial / len_text if len_text > 0 else None,
+        "prop_deictic_pronouns_personal": (
+            n_personal / len_text if len_text > 0 else None
+        ),
+        "prop_deictic_pronouns_temporal": (
+            n_temporal / len_text if len_text > 0 else None
+        ),
+        "prop_deictic_pronouns_total": n_total / len_text if len_text > 0 else None,
+    }
+
+
+def count_indefinite_terms(tokens, lang):
+    """
+    Counts the number of indefinite terms in a text based on the language.
+
+    Args:
+    - tokens list[(str)]: The text to be analysed.
+    - lang (str): The language of the text ("fr", "en").
+
+    Returns:
+        int: The number of indefinite terms found in the text.
+    """
+    indefinite_terms_data = indefinite_terms.get(lang)
+
+    if indefinite_terms_data is None:
+        return
+
+    len_text = len(tokens)
+    num = sum(token in indefinite_terms_data for token in tokens)
+    return {
+        "n_indefinite_terms": num,
+        "prop_indefinite_terms": num / len_text if len_text > 0 else None,
+    }
+
+
+def compute_honore_r_stat(stems):
+    """
+    Compute Honore's R statistic.
+    100*log(N)/(1-(n_unique/n_different))
+
+    Args:
+        stems list(str): list of stems to compute.
+
+    Returns:
+        float: value of Honore's R statistic.
+    """
+    if stems is None:
+        return
+
+    N, n_different, n_unique, _ = compute_counts(stems)
+    return (100 * math.log(N)) / (1 - (n_unique / n_different)) if N else None
+
+
+def compute_brunet_index(stems):
+    """
+    Compute the Brunet index
+    A measure of lexical diversity linking the length of the sample to the number of different words used in it.
+    Lower values of W correspond to richer texts.
+    Stemming is done on words and only the stems are considered.
+
+    Formula: W = N ^ (V ^ (-0.165))
+        W (float): Brunet's index.
+        N (int): total number of stems.
+        V (int): number of different stems.
+
+    Args:
+        stems list(str): list of stems to compute.
+
+    Returns:
+        float: value of Brunet index.
+    """
+    if stems is None:
+        return
+
+    N, n_different, _, _ = compute_counts(stems)
+
+    return math.pow(N, math.pow(n_different, -0.165)) if N else None

speechmetryflow/pragmatic/__init__.py

@@ -0,0 +1,60 @@
+import numpy as np
+
+from transformers import AutoTokenizer, AutoModelWithLMHead, pipeline
+
+from speechmetryflow.semantic.idea_density import compute_idea_density
+from speechmetryflow.pragmatic.database import compute_database
+
+
+def coherence_locale(doc):
+    similarities = []
+    for sent in doc.sents:
+        tokens = [token for token in sent]
+        (similarity,) = compute_idea_density(tokens, [len(tokens)]).values()
+        similarities.append(similarity)
+    return np.nanmean(similarities)
+
+
+def compute_emotion(text):
+    tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-emotion")
+    model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-emotion")
+    input_ids = tokenizer.encode(text + "</s>", return_tensors="pt")
+    output = model.generate(input_ids=input_ids, max_length=2)
+    dec = [tokenizer.decode(ids) for ids in output]
+    label = dec[0]
+    # Supprimer le token <pad> de la sortie
+    return label.replace("<pad>", "").strip()
+
+
+def compute_sentiment(text):
+    """
+    Analyses the sentiment of a text using the mrm8488/t5-base-finetuned-emotion model.
+    :param text: Text to be analysed.
+    :return: Sentiment label.
+    """
+    model_path = "cardiffnlp/twitter-xlm-roberta-base-sentiment"
+    # Creation of the sentiment analysis pipeline
+    sentiment_task = pipeline(
+        "sentiment-analysis",
+        model=model_path,
+        tokenizer=model_path,
+        max_length=512,
+        truncation=True,
+    )
+    # Sentiment analysis
+    result = sentiment_task(text)
+    # Sentiment label retrieval
+    label = result[0]["label"]
+    return label
+
+
+def metrics(doc, lang, n_tokens):
+    metrics = compute_database(doc, lang, n_tokens, "uncertainty_words")
+    metrics.update(compute_database(doc, lang, n_tokens, "formulaic_expressions"))
+    metrics.update(compute_database(doc, lang, n_tokens, "modal_expressions"))
+    metrics.update(compute_database(doc, lang, n_tokens, "filler_expressions"))
+    metrics.update(compute_database(doc, lang, n_tokens, "difficulty_words"))
+    metrics["coherence_locale"] = coherence_locale(doc)
+    metrics["emotion"] = compute_emotion(doc.text)
+    metrics["sentiment"] = compute_sentiment(doc.text)
+    return metrics

speechmetryflow/pragmatic/assets.py

@@ -0,0 +1,62 @@
+"""
+Pragmatic characteristics
+"""
+
+# Uncertainty words
+uncertainty_words = {
+    "en": {"think", "look", "like", "kind", "seem", "maybe", "can", "something"},
+    "fr": {
+        "penser",
+        "sembler",
+        "comme",
+        "genre",
+        "peut-être",
+        "peut",
+        "quelque",
+        "chose",
+    },
+}
+
+# Formulaic expressions
+formulaic_expressions = {
+    "en": {"well", "so", "i guess", "you know", "as it is", "as it were"},
+    "fr": {"bien", "alors", "je suppose", "tu sais", "comme il est", "comme il serait"},
+}
+
+# Modal expressions
+modal_expressions = {
+    "en": {
+        "i think",
+        "in my opinion",
+        "of course",
+        "naturally",
+        "unsure",
+        "likely",
+        "could be that",
+        "unfortunately",
+        "surely",
+    },
+    "fr": {
+        "je pense",
+        "à mon avis",
+        "bien sûr",
+        "naturellement",
+        "incertain",
+        "probablement",
+        "ça pourrait être que",
+        "malheureusement",
+        "sûrement",
+    },
+}
+
+# Filler expressions
+filler_expressions = {
+    "en": {"you know", "i mean"},
+    "fr": {"tu sais", "je veux dire"},
+}
+
+# Words indicating lexical access difficulties
+difficulty_words = {
+    "en": {"know", "remember", "unable"},
+    "fr": {"savoir", "se souvenir", "incapable"},
+}

speechmetryflow/pragmatic/database.py

@@ -0,0 +1,113 @@
+from speechmetryflow.pragmatic import assets
+
+
+def compute_database(doc, lang, n_tokens, tag):
+    """
+    Compute the concreteness of words in a list from a concreteness database.
+
+    Args:
+        tokens (list): A list of words from which the average concreteness will be calculated.
+        lang (str): language of the text.
+
+    Returns:
+        numpy.array: concreteness of the words.
+    """
+    tag2database = {
+        "uncertainty_words": assets.uncertainty_words,
+        "formulaic_expressions": assets.formulaic_expressions,
+        "modal_expressions": assets.modal_expressions,
+        "filler_expressions": assets.filler_expressions,
+        "difficulty_words": assets.difficulty_words,
+    }
+    data = tag2database[tag].get(lang)
+    if data is None:
+        return {
+            f"n_{tag}": None,
+            f"prop_{tag}": None,
+        }
+
+    text = doc.text.lower()
+    count = sum(text.count(expression) for expression in data)
+    return {
+        f"n_{tag}": count,
+        f"prop_{tag}": count / n_tokens if n_tokens else None,
+    }
+
+
+def compute_concreteness(tokens, lang):
+    """
+    Compute the concreteness of words in a list from a concreteness database.
+
+    Args:
+        tokens (list): A list of words from which the average concreteness will be calculated.
+        lang (str): language of the text.
+
+    Returns:
+        numpy.array: concreteness of the words.
+    """
+    return compute_database(tokens, lang, "concreteness")
+
+
+def compute_familiarity(tokens, lang):
+    """
+    Compute the familiarity of words in a list from a familiarity database.
+
+    Args:
+        tokens (list): A list of words from which the average familiarity will be calculated.
+        lang (str): language of the text.
+
+    Returns:
+        numpy.array: familiarity of the words.
+    """
+    return compute_database(tokens, lang, "familiarity")
+
+
+def compute_imageability(tokens, lang):
+    """
+    Compute the imageability of words in a list from a imageability database.
+
+    Args:
+        tokens (list): A list of words from which the average imageability will be calculated.
+        lang (str): language of the text.
+
+    Returns:
+        numpy.array: imageability of the words.
+    """
+    return compute_database(tokens, lang, "imageability")
+
+
+def compute_frequency(tokens, lang):
+    """
+    Compute the frequency of words in a list from a frequency database.
+
+    Args:
+        tokens (list): A list of words from which the average frequency will be calculated.
+        lang (str): language of the text.
+
+    Returns:
+        numpy.array: frequency of the words.
+    """
+    return compute_database(tokens, lang, "frequency")
+
+
+def compute_valence(tokens, lang):
+    """
+    Compute the valence of words in a list from a valence database.
+
+    Args:
+        tokens (list): A list of words from which the average valence will be calculated.
+        lang (str): language of the text.
+
+    Returns:
+        numpy.array: valence of the words.
+    """
+    return compute_database(tokens, lang, "valence")
+
+
+TAG2FUNC = {
+    "concreteness": compute_concreteness,
+    "familiarity": compute_familiarity,
+    "imageability": compute_imageability,
+    "frequency": compute_frequency,
+    "valence": compute_valence,
+}

speechmetryflow/semantic/__init__.py

@@ -0,0 +1,23 @@
+from speechmetryflow.semantic.icu import compute_icu
+from speechmetryflow.semantic.idea_density import compute_idea_density
+
+
+def metrics(text, tokens, lang, task, n_tokens):
+    metrics = compute_idea_density(tokens, [3, 10, 25, 40])
+
+    icu_data = compute_icu(text, lang, task)
+
+    # Total number of True ICU anf ICU efficacity
+    if icu_data is None:
+        n_icu_true = None
+        icu_efficacity = None
+
+    else:
+        metrics.update(icu_data)
+        n_icu_true = sum(icu_data.values())
+        icu_efficacity = n_tokens / n_icu_true if n_icu_true else None
+
+    metrics["n_icu_true"] = n_icu_true
+    metrics["icu_efficacity"] = icu_efficacity
+
+    return metrics