je-editor 0.0.57__py3-none-any.whl → 0.0.59__py3-none-any.whl

venv/Lib/site-packages/charset_normalizer/cd.py

@@ -1,391 +0,0 @@
-import importlib
-from codecs import IncrementalDecoder
-from collections import Counter
-from functools import lru_cache
-from typing import Counter as TypeCounter, Dict, List, Optional, Tuple
-
-from .assets import FREQUENCIES
-from .constant import KO_NAMES, LANGUAGE_SUPPORTED_COUNT, TOO_SMALL_SEQUENCE, ZH_NAMES
-from .md import is_suspiciously_successive_range
-from .models import CoherenceMatches
-from .utils import (
-    is_accentuated,
-    is_latin,
-    is_multi_byte_encoding,
-    is_unicode_range_secondary,
-    unicode_range,
-)
-
-
-def encoding_unicode_range(iana_name: str) -> List[str]:
-    """
-    Return associated unicode ranges in a single byte code page.
-    """
-    if is_multi_byte_encoding(iana_name):
-        raise IOError("Function not supported on multi-byte code page")
-
-    decoder = importlib.import_module(
-        "encodings.{}".format(iana_name)
-    ).IncrementalDecoder
-
-    p: IncrementalDecoder = decoder(errors="ignore")
-    seen_ranges: Dict[str, int] = {}
-    character_count: int = 0
-
-    for i in range(0x40, 0xFF):
-        chunk: str = p.decode(bytes([i]))
-
-        if chunk:
-            character_range: Optional[str] = unicode_range(chunk)
-
-            if character_range is None:
-                continue
-
-            if is_unicode_range_secondary(character_range) is False:
-                if character_range not in seen_ranges:
-                    seen_ranges[character_range] = 0
-                seen_ranges[character_range] += 1
-                character_count += 1
-
-    return sorted(
-        [
-            character_range
-            for character_range in seen_ranges
-            if seen_ranges[character_range] / character_count >= 0.15
-        ]
-    )
-
-
-def unicode_range_languages(primary_range: str) -> List[str]:
-    """
-    Return inferred languages used with a unicode range.
-    """
-    languages: List[str] = []
-
-    for language, characters in FREQUENCIES.items():
-        for character in characters:
-            if unicode_range(character) == primary_range:
-                languages.append(language)
-                break
-
-    return languages
-
-
-@lru_cache()
-def encoding_languages(iana_name: str) -> List[str]:
-    """
-    Single-byte encoding language association. Some code page are heavily linked to particular language(s).
-    This function does the correspondence.
-    """
-    unicode_ranges: List[str] = encoding_unicode_range(iana_name)
-    primary_range: Optional[str] = None
-
-    for specified_range in unicode_ranges:
-        if "Latin" not in specified_range:
-            primary_range = specified_range
-            break
-
-    if primary_range is None:
-        return ["Latin Based"]
-
-    return unicode_range_languages(primary_range)
-
-
-@lru_cache()
-def mb_encoding_languages(iana_name: str) -> List[str]:
-    """
-    Multi-byte encoding language association. Some code page are heavily linked to particular language(s).
-    This function does the correspondence.
-    """
-    if (
-        iana_name.startswith("shift_")
-        or iana_name.startswith("iso2022_jp")
-        or iana_name.startswith("euc_j")
-        or iana_name == "cp932"
-    ):
-        return ["Japanese"]
-    if iana_name.startswith("gb") or iana_name in ZH_NAMES:
-        return ["Chinese"]
-    if iana_name.startswith("iso2022_kr") or iana_name in KO_NAMES:
-        return ["Korean"]
-
-    return []
-
-
-@lru_cache(maxsize=LANGUAGE_SUPPORTED_COUNT)
-def get_target_features(language: str) -> Tuple[bool, bool]:
-    """
-    Determine main aspects from a supported language if it contains accents and if is pure Latin.
-    """
-    target_have_accents: bool = False
-    target_pure_latin: bool = True
-
-    for character in FREQUENCIES[language]:
-        if not target_have_accents and is_accentuated(character):
-            target_have_accents = True
-        if target_pure_latin and is_latin(character) is False:
-            target_pure_latin = False
-
-    return target_have_accents, target_pure_latin
-
-
-def alphabet_languages(
-    characters: List[str], ignore_non_latin: bool = False
-) -> List[str]:
-    """
-    Return associated languages associated to given characters.
-    """
-    languages: List[Tuple[str, float]] = []
-
-    source_have_accents = any(is_accentuated(character) for character in characters)
-
-    for language, language_characters in FREQUENCIES.items():
-
-        target_have_accents, target_pure_latin = get_target_features(language)
-
-        if ignore_non_latin and target_pure_latin is False:
-            continue
-
-        if target_have_accents is False and source_have_accents:
-            continue
-
-        character_count: int = len(language_characters)
-
-        character_match_count: int = len(
-            [c for c in language_characters if c in characters]
-        )
-
-        ratio: float = character_match_count / character_count
-
-        if ratio >= 0.2:
-            languages.append((language, ratio))
-
-    languages = sorted(languages, key=lambda x: x[1], reverse=True)
-
-    return [compatible_language[0] for compatible_language in languages]
-
-
-def characters_popularity_compare(
-    language: str, ordered_characters: List[str]
-) -> float:
-    """
-    Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language.
-    The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit).
-    Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.)
-    """
-    if language not in FREQUENCIES:
-        raise ValueError("{} not available".format(language))
-
-    character_approved_count: int = 0
-    FREQUENCIES_language_set = set(FREQUENCIES[language])
-
-    ordered_characters_count: int = len(ordered_characters)
-    target_language_characters_count: int = len(FREQUENCIES[language])
-
-    large_alphabet: bool = target_language_characters_count > 26
-
-    for character, character_rank in zip(
-        ordered_characters, range(0, ordered_characters_count)
-    ):
-        if character not in FREQUENCIES_language_set:
-            continue
-
-        character_rank_in_language: int = FREQUENCIES[language].index(character)
-        expected_projection_ratio: float = (
-            target_language_characters_count / ordered_characters_count
-        )
-        character_rank_projection: int = int(character_rank * expected_projection_ratio)
-
-        if (
-            large_alphabet is False
-            and abs(character_rank_projection - character_rank_in_language) > 4
-        ):
-            continue
-
-        if (
-            large_alphabet is True
-            and abs(character_rank_projection - character_rank_in_language)
-            < target_language_characters_count / 3
-        ):
-            character_approved_count += 1
-            continue
-
-        characters_before_source: List[str] = FREQUENCIES[language][
-            0:character_rank_in_language
-        ]
-        characters_after_source: List[str] = FREQUENCIES[language][
-            character_rank_in_language:
-        ]
-        characters_before: List[str] = ordered_characters[0:character_rank]
-        characters_after: List[str] = ordered_characters[character_rank:]
-
-        before_match_count: int = len(
-            set(characters_before) & set(characters_before_source)
-        )
-
-        after_match_count: int = len(
-            set(characters_after) & set(characters_after_source)
-        )
-
-        if len(characters_before_source) == 0 and before_match_count <= 4:
-            character_approved_count += 1
-            continue
-
-        if len(characters_after_source) == 0 and after_match_count <= 4:
-            character_approved_count += 1
-            continue
-
-        if (
-            before_match_count / len(characters_before_source) >= 0.4
-            or after_match_count / len(characters_after_source) >= 0.4
-        ):
-            character_approved_count += 1
-            continue
-
-    return character_approved_count / len(ordered_characters)
-
-
-def alpha_unicode_split(decoded_sequence: str) -> List[str]:
-    """
-    Given a decoded text sequence, return a list of str. Unicode range / alphabet separation.
-    Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list;
-    One containing the latin letters and the other hebrew.
-    """
-    layers: Dict[str, str] = {}
-
-    for character in decoded_sequence:
-        if character.isalpha() is False:
-            continue
-
-        character_range: Optional[str] = unicode_range(character)
-
-        if character_range is None:
-            continue
-
-        layer_target_range: Optional[str] = None
-
-        for discovered_range in layers:
-            if (
-                is_suspiciously_successive_range(discovered_range, character_range)
-                is False
-            ):
-                layer_target_range = discovered_range
-                break
-
-        if layer_target_range is None:
-            layer_target_range = character_range
-
-        if layer_target_range not in layers:
-            layers[layer_target_range] = character.lower()
-            continue
-
-        layers[layer_target_range] += character.lower()
-
-    return list(layers.values())
-
-
-def merge_coherence_ratios(results: List[CoherenceMatches]) -> CoherenceMatches:
-    """
-    This function merge results previously given by the function coherence_ratio.
-    The return type is the same as coherence_ratio.
-    """
-    per_language_ratios: Dict[str, List[float]] = {}
-    for result in results:
-        for sub_result in result:
-            language, ratio = sub_result
-            if language not in per_language_ratios:
-                per_language_ratios[language] = [ratio]
-                continue
-            per_language_ratios[language].append(ratio)
-
-    merge = [
-        (
-            language,
-            round(
-                sum(per_language_ratios[language]) / len(per_language_ratios[language]),
-                4,
-            ),
-        )
-        for language in per_language_ratios
-    ]
-
-    return sorted(merge, key=lambda x: x[1], reverse=True)
-
-
-def filter_alt_coherence_matches(results: CoherenceMatches) -> CoherenceMatches:
-    """
-    We shall NOT return "English—" in CoherenceMatches because it is an alternative
-    of "English". This function only keeps the best match and remove the em-dash in it.
-    """
-    index_results: Dict[str, List[float]] = dict()
-
-    for result in results:
-        language, ratio = result
-        no_em_name: str = language.replace("—", "")
-
-        if no_em_name not in index_results:
-            index_results[no_em_name] = []
-
-        index_results[no_em_name].append(ratio)
-
-    if any(len(index_results[e]) > 1 for e in index_results):
-        filtered_results: CoherenceMatches = []
-
-        for language in index_results:
-            filtered_results.append((language, max(index_results[language])))
-
-        return filtered_results
-
-    return results
-
-
-@lru_cache(maxsize=2048)
-def coherence_ratio(
-    decoded_sequence: str, threshold: float = 0.1, lg_inclusion: Optional[str] = None
-) -> CoherenceMatches:
-    """
-    Detect ANY language that can be identified in given sequence. The sequence will be analysed by layers.
-    A layer = Character extraction by alphabets/ranges.
-    """
-
-    results: List[Tuple[str, float]] = []
-    ignore_non_latin: bool = False
-
-    sufficient_match_count: int = 0
-
-    lg_inclusion_list = lg_inclusion.split(",") if lg_inclusion is not None else []
-    if "Latin Based" in lg_inclusion_list:
-        ignore_non_latin = True
-        lg_inclusion_list.remove("Latin Based")
-
-    for layer in alpha_unicode_split(decoded_sequence):
-        sequence_frequencies: TypeCounter[str] = Counter(layer)
-        most_common = sequence_frequencies.most_common()
-
-        character_count: int = sum(o for c, o in most_common)
-
-        if character_count <= TOO_SMALL_SEQUENCE:
-            continue
-
-        popular_character_ordered: List[str] = [c for c, o in most_common]
-
-        for language in lg_inclusion_list or alphabet_languages(
-            popular_character_ordered, ignore_non_latin
-        ):
-            ratio: float = characters_popularity_compare(
-                language, popular_character_ordered
-            )
-
-            if ratio < threshold:
-                continue
-            elif ratio >= 0.8:
-                sufficient_match_count += 1
-
-            results.append((language, round(ratio, 4)))
-
-            if sufficient_match_count >= 3:
-                break
-
-    return sorted(
-        filter_alt_coherence_matches(results), key=lambda x: x[1], reverse=True
-    )

venv/Lib/site-packages/charset_normalizer/cli/normalizer.py

@@ -1,298 +0,0 @@
-import argparse
-import sys
-from json import dumps
-from os.path import abspath, basename, dirname, join, realpath
-from platform import python_version
-from typing import List, Optional
-from unicodedata import unidata_version
-
-import charset_normalizer.md as md_module
-from charset_normalizer import from_fp
-from charset_normalizer.models import CliDetectionResult
-from charset_normalizer.version import __version__
-
-
-def query_yes_no(question: str, default: str = "yes") -> bool:
-    """Ask a yes/no question via input() and return their answer.
-
-    "question" is a string that is presented to the user.
-    "default" is the presumed answer if the user just hits <Enter>.
-        It must be "yes" (the default), "no" or None (meaning
-        an answer is required of the user).
-
-    The "answer" return value is True for "yes" or False for "no".
-
-    Credit goes to (c) https://stackoverflow.com/questions/3041986/apt-command-line-interface-like-yes-no-input
-    """
-    valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
-    if default is None:
-        prompt = " [y/n] "
-    elif default == "yes":
-        prompt = " [Y/n] "
-    elif default == "no":
-        prompt = " [y/N] "
-    else:
-        raise ValueError("invalid default answer: '%s'" % default)
-
-    while True:
-        sys.stdout.write(question + prompt)
-        choice = input().lower()
-        if default is not None and choice == "":
-            return valid[default]
-        elif choice in valid:
-            return valid[choice]
-        else:
-            sys.stdout.write("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n")
-
-
-def cli_detect(argv: Optional[List[str]] = None) -> int:
-    """
-    CLI assistant using ARGV and ArgumentParser
-    :param argv:
-    :return: 0 if everything is fine, anything else equal trouble
-    """
-    parser = argparse.ArgumentParser(
-        description="The Real First Universal Charset Detector. "
-        "Discover originating encoding used on text file. "
-        "Normalize text to unicode."
-    )
-
-    parser.add_argument(
-        "files", type=argparse.FileType("rb"), nargs="+", help="File(s) to be analysed"
-    )
-    parser.add_argument(
-        "-v",
-        "--verbose",
-        action="store_true",
-        default=False,
-        dest="verbose",
-        help="Display complementary information about file if any. "
-        "Stdout will contain logs about the detection process.",
-    )
-    parser.add_argument(
-        "-a",
-        "--with-alternative",
-        action="store_true",
-        default=False,
-        dest="alternatives",
-        help="Output complementary possibilities if any. Top-level JSON WILL be a list.",
-    )
-    parser.add_argument(
-        "-n",
-        "--normalize",
-        action="store_true",
-        default=False,
-        dest="normalize",
-        help="Permit to normalize input file. If not set, program does not write anything.",
-    )
-    parser.add_argument(
-        "-m",
-        "--minimal",
-        action="store_true",
-        default=False,
-        dest="minimal",
-        help="Only output the charset detected to STDOUT. Disabling JSON output.",
-    )
-    parser.add_argument(
-        "-r",
-        "--replace",
-        action="store_true",
-        default=False,
-        dest="replace",
-        help="Replace file when trying to normalize it instead of creating a new one.",
-    )
-    parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        dest="force",
-        help="Replace file without asking if you are sure, use this flag with caution.",
-    )
-    parser.add_argument(
-        "-t",
-        "--threshold",
-        action="store",
-        default=0.2,
-        type=float,
-        dest="threshold",
-        help="Define a custom maximum amount of chaos allowed in decoded content. 0. <= chaos <= 1.",
-    )
-    parser.add_argument(
-        "--version",
-        action="version",
-        version="Charset-Normalizer {} - Python {} - Unicode {} - SpeedUp {}".format(
-            __version__,
-            python_version(),
-            unidata_version,
-            "OFF" if md_module.__file__.lower().endswith(".py") else "ON",
-        ),
-        help="Show version information and exit.",
-    )
-
-    args = parser.parse_args(argv)
-
-    if args.replace is True and args.normalize is False:
-        print("Use --replace in addition of --normalize only.", file=sys.stderr)
-        return 1
-
-    if args.force is True and args.replace is False:
-        print("Use --force in addition of --replace only.", file=sys.stderr)
-        return 1
-
-    if args.threshold < 0.0 or args.threshold > 1.0:
-        print("--threshold VALUE should be between 0. AND 1.", file=sys.stderr)
-        return 1
-
-    x_ = []
-
-    for my_file in args.files:
-
-        matches = from_fp(my_file, threshold=args.threshold, explain=args.verbose)
-
-        best_guess = matches.best()
-
-        if best_guess is None:
-            print(
-                'Unable to identify originating encoding for "{}". {}'.format(
-                    my_file.name,
-                    "Maybe try increasing maximum amount of chaos."
-                    if args.threshold < 1.0
-                    else "",
-                ),
-                file=sys.stderr,
-            )
-            x_.append(
-                CliDetectionResult(
-                    abspath(my_file.name),
-                    None,
-                    [],
-                    [],
-                    "Unknown",
-                    [],
-                    False,
-                    1.0,
-                    0.0,
-                    None,
-                    True,
-                )
-            )
-        else:
-            x_.append(
-                CliDetectionResult(
-                    abspath(my_file.name),
-                    best_guess.encoding,
-                    best_guess.encoding_aliases,
-                    [
-                        cp
-                        for cp in best_guess.could_be_from_charset
-                        if cp != best_guess.encoding
-                    ],
-                    best_guess.language,
-                    best_guess.alphabets,
-                    best_guess.bom,
-                    best_guess.percent_chaos,
-                    best_guess.percent_coherence,
-                    None,
-                    True,
-                )
-            )
-
-            if len(matches) > 1 and args.alternatives:
-                for el in matches:
-                    if el != best_guess:
-                        x_.append(
-                            CliDetectionResult(
-                                abspath(my_file.name),
-                                el.encoding,
-                                el.encoding_aliases,
-                                [
-                                    cp
-                                    for cp in el.could_be_from_charset
-                                    if cp != el.encoding
-                                ],
-                                el.language,
-                                el.alphabets,
-                                el.bom,
-                                el.percent_chaos,
-                                el.percent_coherence,
-                                None,
-                                False,
-                            )
-                        )
-
-            if args.normalize is True:
-
-                if best_guess.encoding.startswith("utf") is True:
-                    print(
-                        '"{}" file does not need to be normalized, as it already came from unicode.'.format(
-                            my_file.name
-                        ),
-                        file=sys.stderr,
-                    )
-                    if my_file.closed is False:
-                        my_file.close()
-                    continue
-
-                dir_path = dirname(realpath(my_file.name))
-                file_name = basename(realpath(my_file.name))
-
-                o_: List[str] = file_name.split(".")
-
-                if args.replace is False:
-                    o_.insert(-1, best_guess.encoding)
-                    if my_file.closed is False:
-                        my_file.close()
-                elif (
-                    args.force is False
-                    and query_yes_no(
-                        'Are you sure to normalize "{}" by replacing it ?'.format(
-                            my_file.name
-                        ),
-                        "no",
-                    )
-                    is False
-                ):
-                    if my_file.closed is False:
-                        my_file.close()
-                    continue
-
-                try:
-                    x_[0].unicode_path = join(dir_path, ".".join(o_))
-
-                    with open(x_[0].unicode_path, "w", encoding="utf-8") as fp:
-                        fp.write(str(best_guess))
-                except IOError as e:
-                    print(str(e), file=sys.stderr)
-                    if my_file.closed is False:
-                        my_file.close()
-                    return 2
-
-        if my_file.closed is False:
-            my_file.close()
-
-    if args.minimal is False:
-        print(
-            dumps(
-                [el.__dict__ for el in x_] if len(x_) > 1 else x_[0].__dict__,
-                ensure_ascii=True,
-                indent=4,
-            )
-        )
-    else:
-        for my_file in args.files:
-            print(
-                ", ".join(
-                    [
-                        el.encoding or "undefined"
-                        for el in x_
-                        if el.path == abspath(my_file.name)
-                    ]
-                )
-            )
-
-    return 0
-
-
-if __name__ == "__main__":
-    cli_detect()