textdirectory 0.4.0__py3-none-any.whl

textdirectory/__init__.py

@@ -0,0 +1,11 @@
+"""Top-level package for textdirectory."""
+
+__author__ = 'Ingo Kleiber'
+__email__ = 'ingo@kleiber.me'
+__version__ = '0.4.0'
+
+from textdirectory import helpers, transformations
+from textdirectory.crudespellchecker import CrudeSpellChecker
+from textdirectory.textdirectory import TextDirectory
+
+__all__ = ['CrudeSpellChecker', 'TextDirectory', '__version__', 'helpers', 'transformations']

textdirectory/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for python -m textdirectory."""
+
+from textdirectory.cli import main
+
+if __name__ == '__main__':
+    main()

textdirectory/cli.py

@@ -0,0 +1,99 @@
+"""Console script for textdirectory."""
+
+import sys
+from typing import Any
+
+import click
+
+from textdirectory import helpers, textdirectory
+
+available_filters = helpers.get_available_filters()
+available_transformations = helpers.get_available_transformations()
+
+
+@click.command()
+@click.version_option(package_name='textdirectory')
+@click.option('--directory', help='The directory containing text files', type=str)
+@click.option('--output_file', help='The file to aggregate to', type=str)
+@click.option('--filetype', help='The file type to look for.', default='txt', type=str)
+@click.option('--encoding', help='The encoding of the files.', default='utf8', type=str)
+@click.option('--recursive', help='Recursion', type=bool)
+@click.option('--disable_tqdm', help='Disable progress bar', default=False, type=bool)
+@click.option('--filters', help=f'The filters you want to apply. Filters: {available_filters}', type=str)
+@click.option(
+    '--transformations',
+    help=f'The transformations you want to apply. Tranformations: {available_transformations}',
+    type=str,
+)
+def main(
+    directory: str | None,
+    output_file: str | None,
+    filetype: str,
+    encoding: str,
+    recursive: bool,
+    disable_tqdm: bool,
+    filters: str | None,
+    transformations: str | None,
+) -> int:
+    """Console script for textdirectory."""
+    if not directory:
+        click.echo('Welcome to TextDirectory!\nRun textdirectory --help for further information.')
+        click.echo(
+            'Example (Basic Aggregation): textdirectory --directory testdata --output_file aggregated.txt --filetype txt'
+        )
+        sys.exit()
+
+    if filters:
+        filters_list: list[list[Any]] = []
+        for filter in filters.split('/'):
+            name, *filter_args = filter.split(',')
+
+            # Coerce string arguments (e.g. '100') to the types the filter expects
+            filter_method = getattr(textdirectory.TextDirectory, name, None)
+            if filter_method is not None and name in available_filters:
+                filter_args = helpers.coerce_args_by_signature(filter_method, filter_args)
+
+            filters_list.append([name, *filter_args])
+
+    if transformations:
+        transformations_list: list[list[str]] = []
+        for transformation in transformations.split('/'):
+            transformations_list.append(transformation.split(','))
+
+    if disable_tqdm or not output_file:
+        disable_tqdm = True
+    else:
+        disable_tqdm = False
+
+    try:
+        td = textdirectory.TextDirectory(directory=directory, encoding=encoding, disable_tqdm=disable_tqdm)
+        td.load_files(recursive=recursive, filetype=filetype)
+    except NotADirectoryError:
+        click.echo('The directory could not be found.')
+        sys.exit(1)
+    except FileNotFoundError:
+        click.echo('There seem to be no files. Maybe you want to run with --recursive True.')
+        sys.exit(1)
+
+    try:
+        if filters and len(filters_list) > 0:
+            td.run_filters(filters_list)
+
+        if transformations and len(transformations_list) > 0:
+            for staged_transformation in transformations_list:
+                td.stage_transformation(staged_transformation)
+    except NameError as e:
+        click.echo(str(e))
+        sys.exit(1)
+
+    if output_file:
+        td.print_aggregation()
+        td.aggregate_to_file(output_file)
+    else:
+        click.echo(td.aggregate_to_memory())
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

textdirectory/crudespellchecker.py

@@ -0,0 +1,179 @@
+"""Spellchecker module."""
+
+import gzip
+import importlib.resources
+import pickle
+import re
+from collections import Counter
+from collections.abc import Iterable, Iterator
+from pathlib import Path
+from typing import Any
+
+
+class CrudeSpellChecker:
+    """A very simple and crude spellchecker based on Peter Norvig's design.
+    Simple Language Models:
+    crudesc_lm_en.gz.lm English based on COCA (sample), OANC (written), BNC
+    crudesc_lm_ame.lm American English based on COCA (sample) and OANC (written)
+    crudesc_lm_amehistorical.lm American English based on COHA (sample)
+    """
+
+    def __init__(self, caching: bool = True, language_model: str = 'crudesc_lm_en') -> None:
+        """
+        :param caching: caching of corrections
+        :type caching: bool
+        :param language_model: the name of the lm
+        :type language_model: str
+        """
+        self.caching = caching
+        self.cache: dict[str, str] = {}
+        self.language_model_name = language_model
+
+        model_resource = importlib.resources.files('textdirectory').joinpath(  # type: ignore[call-arg]
+            'data', 'language_models', f'{self.language_model_name}.gz.lm'
+        )
+        self.frequencies: Counter[str] = pickle.loads(gzip.decompress(model_resource.read_bytes()))
+
+    def p_word(self, word: str) -> float:
+        """
+        :param word: a word
+        :type word: str
+        """
+        return self.frequencies[word] / sum(self.frequencies.values())
+
+    def correction(self, word: str) -> str:
+        """
+        :param word: a word
+        :type word: str
+        :return: most probable spelling correction for word
+        """
+
+        # Preserve
+        word_isupper = word[0].isupper()
+        word = word.lower()
+
+        def reconstruct_case(word: str, word_isupper: bool) -> str:
+            """
+            :param word: the word
+            :type word: str
+            :param word_isupper: the initial capitalization
+            :type word_isupper: bool
+            :return: the word with its initial capitalization
+            """
+            if word_isupper:
+                return word.capitalize()
+            else:
+                return word
+
+        if word in self.cache:
+            return reconstruct_case(self.cache[word], word_isupper)
+        else:
+            correction = max(self.candidates(word), key=self.p_word)
+
+            if self.caching:
+                self.cache[word] = correction
+
+            return reconstruct_case(correction, word_isupper)
+
+    def candidates(self, word: str) -> set[str] | list[str]:
+        """
+        :param word: a word
+        :type word: str
+        :return: a list of candidates
+        """
+        return (
+            self.known([word])
+            or self.known(self.edit_distance_1(word))
+            or self.known(self.edit_distance_2(word))
+            or [word]
+        )
+
+    def known(self, words: Iterable[str]) -> set[str]:
+        """
+        :param word: a word
+        :type word: str
+        :return: a subset of words in the dictionary of frequencies
+        """
+        return {w for w in words if w in self.frequencies}
+
+    def edit_distance_1(self, word: str) -> set[str]:
+        """
+        :param word: a word
+        :type word: str
+        :return: all edits one edit away from the word
+        """
+        letters = 'abcdefghijklmnopqrstuvwxyz'
+        splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
+        deletes = [L + R[1:] for L, R in splits if R]
+        transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R) > 1]
+        replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
+        inserts = [L + c + R for L, R in splits for c in letters]
+
+        return set(deletes + transposes + replaces + inserts)
+
+    def edit_distance_2(self, word: str) -> Iterator[str]:
+        """
+        :param word: a word
+        :type word: str
+        :return: all edits two edits away from the word
+        """
+        return (e2 for e1 in self.edit_distance_1(word) for e2 in self.edit_distance_1(e1))
+
+    def correct_string(self, string: str, return_corrections: bool = False) -> Any:
+        """
+        :param string: the string to correct.
+        :type string: str
+        :param return_corrections: include the corrections in the result
+        :type return_corrections: bool
+        :return: the corrected string
+        """
+        corrections: list[tuple[str, str]] = []
+        corrected: list[str] = []
+        for word in string.split():
+            word_matches = re.findall(r'\w+', word)
+
+            # Tokens without any word characters (e.g. '---') are kept as-is
+            if not word_matches:
+                corrected.append(word)
+                continue
+
+            corrected_word = self.correction(word_matches[0])
+            corrected_word = re.sub(r'(.*?)(\w+)(.*?)', rf'\g<1>{corrected_word}\g<3>', word)
+            corrected.append(corrected_word)
+
+            if return_corrections and corrected_word != word:
+                corrections.append((word, corrected_word))
+
+        if return_corrections:
+            return (' '.join(corrected), corrections)
+        else:
+            return ' '.join(corrected)
+
+
+def generate_crudespellchecker_lm(corpus_directory: str | Path, model_name: str, strip_xml: bool = False) -> None:
+    """
+    :param corpus_directory: path the folder containing the files.
+    :type corpus_directory: str
+    :param model_name: th name of the model
+    :type model_name: str
+    :param strip_xml: stripping XML tags with bs4
+    :type strip_xml: bool
+    """
+    from bs4 import BeautifulSoup
+
+    frequencies: Counter[str] = Counter()
+    files = list(Path(corpus_directory).glob('*.txt'))
+
+    for file in files:
+        with open(file, encoding='utf-8', errors='ignore') as corpus_file:
+            if strip_xml:
+                soup = BeautifulSoup(corpus_file.read().lower(), 'lxml')
+                text = soup.get_text()
+            else:
+                text = corpus_file.read().lower()
+
+            file_frequency = Counter(re.findall(r'\b[^\d\W]+\b', text))
+        frequencies = frequencies + file_frequency
+
+    with gzip.open(model_name + '.gz.lm', 'wb') as pkl:
+        pickle.dump(frequencies, pkl)

textdirectory/data/stopwords/stopwords_en.txt

@@ -0,0 +1,96 @@
+# A fairly conservative stopwords list based on frequency with a strong focus on function words. Verbs have been excluded. (13.06.2019)
+a
+about
+above
+an
+and
+as
+at
+because
+been
+below
+between
+but
+by
+could
+for
+from
+he
+her
+hers
+herself
+him
+himself
+his
+how
+i
+if
+in
+into
+it
+its
+itself
+just
+mine
+more
+most
+my
+myself
+not
+of
+on
+or
+our
+ours
+ourselves
+out
+she
+should
+so
+some
+such
+the
+their
+theirs
+them
+themselves
+these
+they
+those
+through
+to
+us
+very
+we
+what
+when
+which
+while
+who
+would
+you
+your
+yours
+yourself
+yourselves
+this
+that
+whose
+both
+many
+much
+enough
+several
+yet
+neither
+although
+however
+before
+over
+across
+around
+within
+really
+quite
+somewhat
+rather

textdirectory/helpers.py

@@ -0,0 +1,273 @@
+"""Helpers module."""
+
+import copy
+import re
+from collections import Counter
+from collections.abc import Callable
+from typing import Any
+
+
+def tabulate_flat_list_of_dicts(list_of_dicts: list[dict[str, Any]], max_length: int = 25) -> str:
+    """
+    :param list_of_dicts: a list of dictionaries; each list is a row
+    :type list_of_dicts: list
+    :param max_length: the maximum length of a cell
+    :type max_length: int
+    :return: a table
+    :type return: str
+    """
+
+    # Create a copy of the list to prevent object mutation
+    list_of_dicts = copy.deepcopy(list_of_dicts)
+
+    if len(list_of_dicts) == 0:
+        return ''
+
+    # Enforce a maximum length
+    if max_length:
+        for row in list_of_dicts:
+            for key, value in row.items():
+                row[key] = str(value)[:max_length]
+
+    # Determine the width of the columns
+    longest_values: dict[str, int] = {}
+
+    for key in list_of_dicts[0].keys():
+        longest_values[key] = len(key)
+
+    for row in list_of_dicts:
+        for key, value in row.items():
+            value = str(value)
+            if key in longest_values:
+                if len(value) > longest_values[key]:
+                    longest_values[key] = len(value)
+            else:
+                longest_values[key] = len(value)
+
+    # Line / len(longest_values) = additonal characters for pipes
+    length = 0
+    for value in longest_values.values():
+        length += value
+
+    line = '\n|' + '-' * (length + len(longest_values) - 1) + '|'
+
+    # Header based on the first dictionary
+    table = line + '\n|'
+    for key in list_of_dicts[0].keys():
+        table += f'{key}'.ljust(longest_values[key]) + '|'
+
+    table += line
+
+    # Rows
+    for row in list_of_dicts:
+        table += '\n|'
+        for key, value in row.items():
+            # Remove linebreaks
+            value = value.replace('\n', '')
+            table += str(value).ljust(longest_values[key]) + '|'
+
+    table += line
+
+    return table
+
+
+def count_non_alphanum(string: str) -> int:
+    """
+    :param string: a string
+    :type string: str
+    :return: the number of non-alphanumeric characters in the string
+    :type return: int
+    """
+
+    non_alphanum = 0
+    for c in string:
+        if not c.isalpha():
+            non_alphanum += 1
+
+    return non_alphanum
+
+
+def chunk_text(string: str, chunk_size: int = 50000) -> list[str]:
+    """
+    :param string: a string
+    :type string: str
+    :param chunk_size: the max characters of one chunk
+    :type chunk_size: int
+    :return: a list of chunks
+    :type return: list
+    """
+
+    chunks = [string[i : i + chunk_size] for i in range(0, len(string), chunk_size)]
+    return chunks
+
+
+def simple_tokenizer(string: str, regular_expression: str = r'\w+') -> list[str]:
+    """
+    :param string: a string
+    :type string: str
+    :return: a list of tokens
+    :type return: list
+    """
+
+    if not regular_expression:
+        tokens = string.split(' ')
+    else:
+        tokens = re.findall(regular_expression, string)
+
+    return tokens
+
+
+def estimate_spacy_max_length(override: float | bool = False, tokenizer_only: bool = False) -> float:
+    """Returns a somewhat sensible suggestions for max_length."""
+    if override:
+        return override
+
+    try:
+        import psutil
+    except ImportError as e:
+        raise ImportError(
+            'estimate_spacy_max_length requires psutil, which is an optional dependency. '
+            "Install it with: pip install 'textdirectory[nlp]'"
+        ) from e
+
+    memory = psutil.virtual_memory()
+    gb_available = memory.available / 1024 / 1024 / 1024
+
+    # tagger, parser, ner 100,000 characters = 1 GB
+    estimated_max_length = gb_available * 100000
+
+    if tokenizer_only:
+        estimated_max_length = estimated_max_length * 3
+
+    return estimated_max_length
+
+
+def type_token_ratio(text: str) -> float:
+    """Returns a simple rounded type-token ratio of a text."""
+    tokens = simple_tokenizer(text)
+    no_types = len(Counter(tokens))
+    no_tokens = len(tokens)
+
+    if no_tokens == 0:
+        return 0.0
+
+    return round(no_types / no_tokens, 2)
+
+
+def get_human_from_docstring(doc: str) -> dict[str, str]:
+    """
+    :param doc: if True, also return the 'human name'
+    :type doc: string
+    :return: a dictionary of name_* keys/values from the docstring.
+    :type return: dict
+    """
+    doc = doc.replace('    ', '')
+    res = re.findall(r'human_(.*):(.*)', doc)
+
+    return {k: v.strip() for (k, v) in res}
+
+
+def coerce_args_by_signature(func: Callable[..., Any], args: list[Any]) -> list[Any]:
+    """
+    Coerce string arguments to the types suggested by a function's signature.
+
+    String values are converted to int, float, or bool when the corresponding
+    parameter's annotation or default value indicates such a type; all other
+    values are passed through unchanged.
+
+    :param func: the function or method the arguments are meant for
+    :type func: callable
+    :param args: the arguments to coerce
+    :type args: list
+    :return: a list of coerced arguments
+    :type return: list
+    """
+    import inspect
+
+    parameters = [p for p in inspect.signature(func).parameters.values() if p.name not in ('self', 'text')]
+
+    coerced: list[Any] = []
+    for arg, parameter in zip(args, parameters, strict=False):
+        target_type = None
+
+        if parameter.annotation in (int, float, bool):
+            target_type = parameter.annotation
+        elif parameter.default is not inspect.Parameter.empty and isinstance(parameter.default, (bool, int, float)):
+            target_type = type(parameter.default)
+
+        if not isinstance(arg, str) or target_type is None:
+            coerced.append(arg)
+        elif target_type is bool:
+            coerced.append(arg.lower() in ('1', 'true', 'yes'))
+        else:
+            try:
+                coerced.append(target_type(arg))
+            except ValueError:
+                coerced.append(arg)
+
+    # Surplus arguments (consumed by *args) are passed through untouched
+    coerced.extend(args[len(coerced) :])
+
+    return coerced
+
+
+def get_available_filters(get_human_name: bool = False) -> list[Any]:
+    """
+    :param get_human_name: if True, also return the 'human name'
+    :type get_human_name: bool
+    :return: a list of functions; if get_human_name a list of tuples
+    :type return: list
+    """
+
+    from textdirectory.textdirectory import TextDirectory
+
+    available_filters: list[Any] = [
+        filter
+        for filter in dir(TextDirectory)
+        if filter.startswith('filter_by_') and callable(getattr(TextDirectory, filter))
+    ]
+
+    if get_human_name:
+        available_filters_with_human: list[tuple[str, str]] = []
+        for f in available_filters:
+            doc = getattr(TextDirectory, f).__doc__
+            human = get_human_from_docstring(doc)
+            if 'name' in human:
+                available_filters_with_human.append((f, human['name']))
+            else:
+                available_filters_with_human.append((f, f))
+
+        available_filters = available_filters_with_human
+
+    return available_filters
+
+
+def get_available_transformations(get_human_name: bool = False) -> list[Any]:
+    """
+    :param get_human_name: if True, also return the 'human name'
+    :type string: bool
+    :return: a list of functions; if get_human_name a list of tuples
+    :type return: list
+    """
+
+    from textdirectory import transformations
+
+    available_transformations: list[Any] = [
+        transformation
+        for transformation in dir(transformations)
+        if transformation.startswith('transformation_') and callable(getattr(transformations, transformation))
+    ]
+
+    if get_human_name:
+        available_transformations_with_human: list[tuple[str, str]] = []
+        for t in available_transformations:
+            doc = getattr(transformations, t).__doc__
+            human = get_human_from_docstring(doc)
+            if 'name' in human:
+                available_transformations_with_human.append((t, human['name']))
+            else:
+                available_transformations_with_human.append((t, t))
+
+        available_transformations = available_transformations_with_human
+
+    return available_transformations