py2docfx 0.1.11.dev1830301__py3-none-any.whl → 0.1.11.dev1859874__py3-none-any.whl

py2docfx/venv/template/Lib/site-packages/pip/_vendor/chardet/sjisprober.py

@@ -1,105 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is mozilla.org code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Mark Pilgrim - port to Python
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-
-from typing import Union
-
-from .chardistribution import SJISDistributionAnalysis
-from .codingstatemachine import CodingStateMachine
-from .enums import MachineState, ProbingState
-from .jpcntx import SJISContextAnalysis
-from .mbcharsetprober import MultiByteCharSetProber
-from .mbcssm import SJIS_SM_MODEL
-
-
-class SJISProber(MultiByteCharSetProber):
-    def __init__(self) -> None:
-        super().__init__()
-        self.coding_sm = CodingStateMachine(SJIS_SM_MODEL)
-        self.distribution_analyzer = SJISDistributionAnalysis()
-        self.context_analyzer = SJISContextAnalysis()
-        self.reset()
-
-    def reset(self) -> None:
-        super().reset()
-        self.context_analyzer.reset()
-
-    @property
-    def charset_name(self) -> str:
-        return self.context_analyzer.charset_name
-
-    @property
-    def language(self) -> str:
-        return "Japanese"
-
-    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
-        assert self.coding_sm is not None
-        assert self.distribution_analyzer is not None
-
-        for i, byte in enumerate(byte_str):
-            coding_state = self.coding_sm.next_state(byte)
-            if coding_state == MachineState.ERROR:
-                self.logger.debug(
-                    "%s %s prober hit error at byte %s",
-                    self.charset_name,
-                    self.language,
-                    i,
-                )
-                self._state = ProbingState.NOT_ME
-                break
-            if coding_state == MachineState.ITS_ME:
-                self._state = ProbingState.FOUND_IT
-                break
-            if coding_state == MachineState.START:
-                char_len = self.coding_sm.get_current_charlen()
-                if i == 0:
-                    self._last_char[1] = byte
-                    self.context_analyzer.feed(
-                        self._last_char[2 - char_len :], char_len
-                    )
-                    self.distribution_analyzer.feed(self._last_char, char_len)
-                else:
-                    self.context_analyzer.feed(
-                        byte_str[i + 1 - char_len : i + 3 - char_len], char_len
-                    )
-                    self.distribution_analyzer.feed(byte_str[i - 1 : i + 1], char_len)
-
-        self._last_char[0] = byte_str[-1]
-
-        if self.state == ProbingState.DETECTING:
-            if self.context_analyzer.got_enough_data() and (
-                self.get_confidence() > self.SHORTCUT_THRESHOLD
-            ):
-                self._state = ProbingState.FOUND_IT
-
-        return self.state
-
-    def get_confidence(self) -> float:
-        assert self.distribution_analyzer is not None
-
-        context_conf = self.context_analyzer.get_confidence()
-        distrib_conf = self.distribution_analyzer.get_confidence()
-        return max(context_conf, distrib_conf)

py2docfx/venv/template/Lib/site-packages/pip/_vendor/chardet/universaldetector.py

@@ -1,362 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Universal charset detector code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 2001
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Mark Pilgrim - port to Python
-#   Shy Shalom - original C code
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-"""
-Module containing the UniversalDetector detector class, which is the primary
-class a user of ``chardet`` should use.
-
-:author: Mark Pilgrim (initial port to Python)
-:author: Shy Shalom (original C code)
-:author: Dan Blanchard (major refactoring for 3.0)
-:author: Ian Cordasco
-"""
-
-
-import codecs
-import logging
-import re
-from typing import List, Optional, Union
-
-from .charsetgroupprober import CharSetGroupProber
-from .charsetprober import CharSetProber
-from .enums import InputState, LanguageFilter, ProbingState
-from .escprober import EscCharSetProber
-from .latin1prober import Latin1Prober
-from .macromanprober import MacRomanProber
-from .mbcsgroupprober import MBCSGroupProber
-from .resultdict import ResultDict
-from .sbcsgroupprober import SBCSGroupProber
-from .utf1632prober import UTF1632Prober
-
-
-class UniversalDetector:
-    """
-    The ``UniversalDetector`` class underlies the ``chardet.detect`` function
-    and coordinates all of the different charset probers.
-
-    To get a ``dict`` containing an encoding and its confidence, you can simply
-    run:
-
-    .. code::
-
-            u = UniversalDetector()
-            u.feed(some_bytes)
-            u.close()
-            detected = u.result
-
-    """
-
-    MINIMUM_THRESHOLD = 0.20
-    HIGH_BYTE_DETECTOR = re.compile(b"[\x80-\xFF]")
-    ESC_DETECTOR = re.compile(b"(\033|~{)")
-    WIN_BYTE_DETECTOR = re.compile(b"[\x80-\x9F]")
-    ISO_WIN_MAP = {
-        "iso-8859-1": "Windows-1252",
-        "iso-8859-2": "Windows-1250",
-        "iso-8859-5": "Windows-1251",
-        "iso-8859-6": "Windows-1256",
-        "iso-8859-7": "Windows-1253",
-        "iso-8859-8": "Windows-1255",
-        "iso-8859-9": "Windows-1254",
-        "iso-8859-13": "Windows-1257",
-    }
-    # Based on https://encoding.spec.whatwg.org/#names-and-labels
-    # but altered to match Python names for encodings and remove mappings
-    # that break tests.
-    LEGACY_MAP = {
-        "ascii": "Windows-1252",
-        "iso-8859-1": "Windows-1252",
-        "tis-620": "ISO-8859-11",
-        "iso-8859-9": "Windows-1254",
-        "gb2312": "GB18030",
-        "euc-kr": "CP949",
-        "utf-16le": "UTF-16",
-    }
-
-    def __init__(
-        self,
-        lang_filter: LanguageFilter = LanguageFilter.ALL,
-        should_rename_legacy: bool = False,
-    ) -> None:
-        self._esc_charset_prober: Optional[EscCharSetProber] = None
-        self._utf1632_prober: Optional[UTF1632Prober] = None
-        self._charset_probers: List[CharSetProber] = []
-        self.result: ResultDict = {
-            "encoding": None,
-            "confidence": 0.0,
-            "language": None,
-        }
-        self.done = False
-        self._got_data = False
-        self._input_state = InputState.PURE_ASCII
-        self._last_char = b""
-        self.lang_filter = lang_filter
-        self.logger = logging.getLogger(__name__)
-        self._has_win_bytes = False
-        self.should_rename_legacy = should_rename_legacy
-        self.reset()
-
-    @property
-    def input_state(self) -> int:
-        return self._input_state
-
-    @property
-    def has_win_bytes(self) -> bool:
-        return self._has_win_bytes
-
-    @property
-    def charset_probers(self) -> List[CharSetProber]:
-        return self._charset_probers
-
-    def reset(self) -> None:
-        """
-        Reset the UniversalDetector and all of its probers back to their
-        initial states.  This is called by ``__init__``, so you only need to
-        call this directly in between analyses of different documents.
-        """
-        self.result = {"encoding": None, "confidence": 0.0, "language": None}
-        self.done = False
-        self._got_data = False
-        self._has_win_bytes = False
-        self._input_state = InputState.PURE_ASCII
-        self._last_char = b""
-        if self._esc_charset_prober:
-            self._esc_charset_prober.reset()
-        if self._utf1632_prober:
-            self._utf1632_prober.reset()
-        for prober in self._charset_probers:
-            prober.reset()
-
-    def feed(self, byte_str: Union[bytes, bytearray]) -> None:
-        """
-        Takes a chunk of a document and feeds it through all of the relevant
-        charset probers.
-
-        After calling ``feed``, you can check the value of the ``done``
-        attribute to see if you need to continue feeding the
-        ``UniversalDetector`` more data, or if it has made a prediction
-        (in the ``result`` attribute).
-
-        .. note::
-           You should always call ``close`` when you're done feeding in your
-           document if ``done`` is not already ``True``.
-        """
-        if self.done:
-            return
-
-        if not byte_str:
-            return
-
-        if not isinstance(byte_str, bytearray):
-            byte_str = bytearray(byte_str)
-
-        # First check for known BOMs, since these are guaranteed to be correct
-        if not self._got_data:
-            # If the data starts with BOM, we know it is UTF
-            if byte_str.startswith(codecs.BOM_UTF8):
-                # EF BB BF  UTF-8 with BOM
-                self.result = {
-                    "encoding": "UTF-8-SIG",
-                    "confidence": 1.0,
-                    "language": "",
-                }
-            elif byte_str.startswith((codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
-                # FF FE 00 00  UTF-32, little-endian BOM
-                # 00 00 FE FF  UTF-32, big-endian BOM
-                self.result = {"encoding": "UTF-32", "confidence": 1.0, "language": ""}
-            elif byte_str.startswith(b"\xFE\xFF\x00\x00"):
-                # FE FF 00 00  UCS-4, unusual octet order BOM (3412)
-                self.result = {
-                    # TODO: This encoding is not supported by Python. Should remove?
-                    "encoding": "X-ISO-10646-UCS-4-3412",
-                    "confidence": 1.0,
-                    "language": "",
-                }
-            elif byte_str.startswith(b"\x00\x00\xFF\xFE"):
-                # 00 00 FF FE  UCS-4, unusual octet order BOM (2143)
-                self.result = {
-                    # TODO: This encoding is not supported by Python. Should remove?
-                    "encoding": "X-ISO-10646-UCS-4-2143",
-                    "confidence": 1.0,
-                    "language": "",
-                }
-            elif byte_str.startswith((codecs.BOM_LE, codecs.BOM_BE)):
-                # FF FE  UTF-16, little endian BOM
-                # FE FF  UTF-16, big endian BOM
-                self.result = {"encoding": "UTF-16", "confidence": 1.0, "language": ""}
-
-            self._got_data = True
-            if self.result["encoding"] is not None:
-                self.done = True
-                return
-
-        # If none of those matched and we've only see ASCII so far, check
-        # for high bytes and escape sequences
-        if self._input_state == InputState.PURE_ASCII:
-            if self.HIGH_BYTE_DETECTOR.search(byte_str):
-                self._input_state = InputState.HIGH_BYTE
-            elif (
-                self._input_state == InputState.PURE_ASCII
-                and self.ESC_DETECTOR.search(self._last_char + byte_str)
-            ):
-                self._input_state = InputState.ESC_ASCII
-
-        self._last_char = byte_str[-1:]
-
-        # next we will look to see if it is appears to be either a UTF-16 or
-        # UTF-32 encoding
-        if not self._utf1632_prober:
-            self._utf1632_prober = UTF1632Prober()
-
-        if self._utf1632_prober.state == ProbingState.DETECTING:
-            if self._utf1632_prober.feed(byte_str) == ProbingState.FOUND_IT:
-                self.result = {
-                    "encoding": self._utf1632_prober.charset_name,
-                    "confidence": self._utf1632_prober.get_confidence(),
-                    "language": "",
-                }
-                self.done = True
-                return
-
-        # If we've seen escape sequences, use the EscCharSetProber, which
-        # uses a simple state machine to check for known escape sequences in
-        # HZ and ISO-2022 encodings, since those are the only encodings that
-        # use such sequences.
-        if self._input_state == InputState.ESC_ASCII:
-            if not self._esc_charset_prober:
-                self._esc_charset_prober = EscCharSetProber(self.lang_filter)
-            if self._esc_charset_prober.feed(byte_str) == ProbingState.FOUND_IT:
-                self.result = {
-                    "encoding": self._esc_charset_prober.charset_name,
-                    "confidence": self._esc_charset_prober.get_confidence(),
-                    "language": self._esc_charset_prober.language,
-                }
-                self.done = True
-        # If we've seen high bytes (i.e., those with values greater than 127),
-        # we need to do more complicated checks using all our multi-byte and
-        # single-byte probers that are left.  The single-byte probers
-        # use character bigram distributions to determine the encoding, whereas
-        # the multi-byte probers use a combination of character unigram and
-        # bigram distributions.
-        elif self._input_state == InputState.HIGH_BYTE:
-            if not self._charset_probers:
-                self._charset_probers = [MBCSGroupProber(self.lang_filter)]
-                # If we're checking non-CJK encodings, use single-byte prober
-                if self.lang_filter & LanguageFilter.NON_CJK:
-                    self._charset_probers.append(SBCSGroupProber())
-                self._charset_probers.append(Latin1Prober())
-                self._charset_probers.append(MacRomanProber())
-            for prober in self._charset_probers:
-                if prober.feed(byte_str) == ProbingState.FOUND_IT:
-                    self.result = {
-                        "encoding": prober.charset_name,
-                        "confidence": prober.get_confidence(),
-                        "language": prober.language,
-                    }
-                    self.done = True
-                    break
-            if self.WIN_BYTE_DETECTOR.search(byte_str):
-                self._has_win_bytes = True
-
-    def close(self) -> ResultDict:
-        """
-        Stop analyzing the current document and come up with a final
-        prediction.
-
-        :returns:  The ``result`` attribute, a ``dict`` with the keys
-                   `encoding`, `confidence`, and `language`.
-        """
-        # Don't bother with checks if we're already done
-        if self.done:
-            return self.result
-        self.done = True
-
-        if not self._got_data:
-            self.logger.debug("no data received!")
-
-        # Default to ASCII if it is all we've seen so far
-        elif self._input_state == InputState.PURE_ASCII:
-            self.result = {"encoding": "ascii", "confidence": 1.0, "language": ""}
-
-        # If we have seen non-ASCII, return the best that met MINIMUM_THRESHOLD
-        elif self._input_state == InputState.HIGH_BYTE:
-            prober_confidence = None
-            max_prober_confidence = 0.0
-            max_prober = None
-            for prober in self._charset_probers:
-                if not prober:
-                    continue
-                prober_confidence = prober.get_confidence()
-                if prober_confidence > max_prober_confidence:
-                    max_prober_confidence = prober_confidence
-                    max_prober = prober
-            if max_prober and (max_prober_confidence > self.MINIMUM_THRESHOLD):
-                charset_name = max_prober.charset_name
-                assert charset_name is not None
-                lower_charset_name = charset_name.lower()
-                confidence = max_prober.get_confidence()
-                # Use Windows encoding name instead of ISO-8859 if we saw any
-                # extra Windows-specific bytes
-                if lower_charset_name.startswith("iso-8859"):
-                    if self._has_win_bytes:
-                        charset_name = self.ISO_WIN_MAP.get(
-                            lower_charset_name, charset_name
-                        )
-                # Rename legacy encodings with superset encodings if asked
-                if self.should_rename_legacy:
-                    charset_name = self.LEGACY_MAP.get(
-                        (charset_name or "").lower(), charset_name
-                    )
-                self.result = {
-                    "encoding": charset_name,
-                    "confidence": confidence,
-                    "language": max_prober.language,
-                }
-
-        # Log all prober confidences if none met MINIMUM_THRESHOLD
-        if self.logger.getEffectiveLevel() <= logging.DEBUG:
-            if self.result["encoding"] is None:
-                self.logger.debug("no probers hit minimum threshold")
-                for group_prober in self._charset_probers:
-                    if not group_prober:
-                        continue
-                    if isinstance(group_prober, CharSetGroupProber):
-                        for prober in group_prober.probers:
-                            self.logger.debug(
-                                "%s %s confidence = %s",
-                                prober.charset_name,
-                                prober.language,
-                                prober.get_confidence(),
-                            )
-                    else:
-                        self.logger.debug(
-                            "%s %s confidence = %s",
-                            group_prober.charset_name,
-                            group_prober.language,
-                            group_prober.get_confidence(),
-                        )
-        return self.result

py2docfx/venv/template/Lib/site-packages/pip/_vendor/chardet/utf1632prober.py

@@ -1,225 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-#
-# Contributor(s):
-#   Jason Zavaglia
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-from typing import List, Union
-
-from .charsetprober import CharSetProber
-from .enums import ProbingState
-
-
-class UTF1632Prober(CharSetProber):
-    """
-    This class simply looks for occurrences of zero bytes, and infers
-    whether the file is UTF16 or UTF32 (low-endian or big-endian)
-    For instance, files looking like ( \0 \0 \0 [nonzero] )+
-    have a good probability to be UTF32BE.  Files looking like ( \0 [nonzero] )+
-    may be guessed to be UTF16BE, and inversely for little-endian varieties.
-    """
-
-    # how many logical characters to scan before feeling confident of prediction
-    MIN_CHARS_FOR_DETECTION = 20
-    # a fixed constant ratio of expected zeros or non-zeros in modulo-position.
-    EXPECTED_RATIO = 0.94
-
-    def __init__(self) -> None:
-        super().__init__()
-        self.position = 0
-        self.zeros_at_mod = [0] * 4
-        self.nonzeros_at_mod = [0] * 4
-        self._state = ProbingState.DETECTING
-        self.quad = [0, 0, 0, 0]
-        self.invalid_utf16be = False
-        self.invalid_utf16le = False
-        self.invalid_utf32be = False
-        self.invalid_utf32le = False
-        self.first_half_surrogate_pair_detected_16be = False
-        self.first_half_surrogate_pair_detected_16le = False
-        self.reset()
-
-    def reset(self) -> None:
-        super().reset()
-        self.position = 0
-        self.zeros_at_mod = [0] * 4
-        self.nonzeros_at_mod = [0] * 4
-        self._state = ProbingState.DETECTING
-        self.invalid_utf16be = False
-        self.invalid_utf16le = False
-        self.invalid_utf32be = False
-        self.invalid_utf32le = False
-        self.first_half_surrogate_pair_detected_16be = False
-        self.first_half_surrogate_pair_detected_16le = False
-        self.quad = [0, 0, 0, 0]
-
-    @property
-    def charset_name(self) -> str:
-        if self.is_likely_utf32be():
-            return "utf-32be"
-        if self.is_likely_utf32le():
-            return "utf-32le"
-        if self.is_likely_utf16be():
-            return "utf-16be"
-        if self.is_likely_utf16le():
-            return "utf-16le"
-        # default to something valid
-        return "utf-16"
-
-    @property
-    def language(self) -> str:
-        return ""
-
-    def approx_32bit_chars(self) -> float:
-        return max(1.0, self.position / 4.0)
-
-    def approx_16bit_chars(self) -> float:
-        return max(1.0, self.position / 2.0)
-
-    def is_likely_utf32be(self) -> bool:
-        approx_chars = self.approx_32bit_chars()
-        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
-            self.zeros_at_mod[0] / approx_chars > self.EXPECTED_RATIO
-            and self.zeros_at_mod[1] / approx_chars > self.EXPECTED_RATIO
-            and self.zeros_at_mod[2] / approx_chars > self.EXPECTED_RATIO
-            and self.nonzeros_at_mod[3] / approx_chars > self.EXPECTED_RATIO
-            and not self.invalid_utf32be
-        )
-
-    def is_likely_utf32le(self) -> bool:
-        approx_chars = self.approx_32bit_chars()
-        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
-            self.nonzeros_at_mod[0] / approx_chars > self.EXPECTED_RATIO
-            and self.zeros_at_mod[1] / approx_chars > self.EXPECTED_RATIO
-            and self.zeros_at_mod[2] / approx_chars > self.EXPECTED_RATIO
-            and self.zeros_at_mod[3] / approx_chars > self.EXPECTED_RATIO
-            and not self.invalid_utf32le
-        )
-
-    def is_likely_utf16be(self) -> bool:
-        approx_chars = self.approx_16bit_chars()
-        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
-            (self.nonzeros_at_mod[1] + self.nonzeros_at_mod[3]) / approx_chars
-            > self.EXPECTED_RATIO
-            and (self.zeros_at_mod[0] + self.zeros_at_mod[2]) / approx_chars
-            > self.EXPECTED_RATIO
-            and not self.invalid_utf16be
-        )
-
-    def is_likely_utf16le(self) -> bool:
-        approx_chars = self.approx_16bit_chars()
-        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
-            (self.nonzeros_at_mod[0] + self.nonzeros_at_mod[2]) / approx_chars
-            > self.EXPECTED_RATIO
-            and (self.zeros_at_mod[1] + self.zeros_at_mod[3]) / approx_chars
-            > self.EXPECTED_RATIO
-            and not self.invalid_utf16le
-        )
-
-    def validate_utf32_characters(self, quad: List[int]) -> None:
-        """
-        Validate if the quad of bytes is valid UTF-32.
-
-        UTF-32 is valid in the range 0x00000000 - 0x0010FFFF
-        excluding 0x0000D800 - 0x0000DFFF
-
-        https://en.wikipedia.org/wiki/UTF-32
-        """
-        if (
-            quad[0] != 0
-            or quad[1] > 0x10
-            or (quad[0] == 0 and quad[1] == 0 and 0xD8 <= quad[2] <= 0xDF)
-        ):
-            self.invalid_utf32be = True
-        if (
-            quad[3] != 0
-            or quad[2] > 0x10
-            or (quad[3] == 0 and quad[2] == 0 and 0xD8 <= quad[1] <= 0xDF)
-        ):
-            self.invalid_utf32le = True
-
-    def validate_utf16_characters(self, pair: List[int]) -> None:
-        """
-        Validate if the pair of bytes is  valid UTF-16.
-
-        UTF-16 is valid in the range 0x0000 - 0xFFFF excluding 0xD800 - 0xFFFF
-        with an exception for surrogate pairs, which must be in the range
-        0xD800-0xDBFF followed by 0xDC00-0xDFFF
-
-        https://en.wikipedia.org/wiki/UTF-16
-        """
-        if not self.first_half_surrogate_pair_detected_16be:
-            if 0xD8 <= pair[0] <= 0xDB:
-                self.first_half_surrogate_pair_detected_16be = True
-            elif 0xDC <= pair[0] <= 0xDF:
-                self.invalid_utf16be = True
-        else:
-            if 0xDC <= pair[0] <= 0xDF:
-                self.first_half_surrogate_pair_detected_16be = False
-            else:
-                self.invalid_utf16be = True
-
-        if not self.first_half_surrogate_pair_detected_16le:
-            if 0xD8 <= pair[1] <= 0xDB:
-                self.first_half_surrogate_pair_detected_16le = True
-            elif 0xDC <= pair[1] <= 0xDF:
-                self.invalid_utf16le = True
-        else:
-            if 0xDC <= pair[1] <= 0xDF:
-                self.first_half_surrogate_pair_detected_16le = False
-            else:
-                self.invalid_utf16le = True
-
-    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
-        for c in byte_str:
-            mod4 = self.position % 4
-            self.quad[mod4] = c
-            if mod4 == 3:
-                self.validate_utf32_characters(self.quad)
-                self.validate_utf16_characters(self.quad[0:2])
-                self.validate_utf16_characters(self.quad[2:4])
-            if c == 0:
-                self.zeros_at_mod[mod4] += 1
-            else:
-                self.nonzeros_at_mod[mod4] += 1
-            self.position += 1
-        return self.state
-
-    @property
-    def state(self) -> ProbingState:
-        if self._state in {ProbingState.NOT_ME, ProbingState.FOUND_IT}:
-            # terminal, decided states
-            return self._state
-        if self.get_confidence() > 0.80:
-            self._state = ProbingState.FOUND_IT
-        elif self.position > 4 * 1024:
-            # if we get to 4kb into the file, and we can't conclude it's UTF,
-            # let's give up
-            self._state = ProbingState.NOT_ME
-        return self._state
-
-    def get_confidence(self) -> float:
-        return (
-            0.85
-            if (
-                self.is_likely_utf16le()
-                or self.is_likely_utf16be()
-                or self.is_likely_utf32le()
-                or self.is_likely_utf32be()
-            )
-            else 0.00
-        )