bytesense 1.0.0__py3-none-any.whl

bytesense/streaming.py

@@ -0,0 +1,416 @@
+"""Bounded-memory streaming with explicit finalization and full decode validation."""
+
+from __future__ import annotations
+
+import codecs
+import tempfile
+from dataclasses import replace
+from typing import Any, Iterator, Optional
+
+from .api import (
+    _HIGH_BYTE,
+    _allowed,
+    _binary,
+    _bom_codec,
+    _filters,
+    _from_sample,
+    _result,
+    _transport_encoding,
+    from_bytes,
+)
+from .fingerprint import detect_null_pattern
+from .hints import hint_from_content, hint_from_http_headers
+from .models import DetectionResult
+
+
+class StreamDetector:
+    """Consume every fed byte; spool beyond ``memory_limit`` to a local temp file.
+
+    Preview scoring runs at exponentially increasing checkpoints, then stops
+    after the sample fills. ``is_stable`` describes a guess, never end-of-input.
+    ``finalize`` strictly validates the chosen codec over the complete stream.
+    Feeding after finalization raises instead of silently dropping data.
+    Use a context manager or ``close()`` to release an unfinished stream.
+    """
+
+    MIN_BYTES = 64
+    SATURATION = 8192
+    STABILITY_ROUNDS = 2
+
+    def __init__(
+        self,
+        threshold: float = 0.2,
+        language_threshold: float = 0.1,
+        auto_stop_confidence: float = 0.97,
+        *,
+        memory_limit: int = 1_048_576,
+        **kwargs: Any,
+    ) -> None:
+        if not isinstance(memory_limit, int) or isinstance(memory_limit, bool) or memory_limit < 64:
+            raise ValueError("memory_limit must be an integer of at least 64 bytes")
+        if not 0.0 <= auto_stop_confidence <= 1.0:
+            raise ValueError("auto_stop_confidence must be between 0 and 1")
+        self._options = dict(kwargs, threshold=threshold, language_threshold=language_threshold)
+        from_bytes(b"", **self._options)  # Validate configuration before accepting any data.
+        self._memory_limit = memory_limit
+        self._auto_stop_confidence = auto_stop_confidence
+        self._sample_size = kwargs.get("sample_size", 4096)
+        self._spool = tempfile.SpooledTemporaryFile(max_size=memory_limit, mode="w+b")
+        self._buf = bytearray()
+        self._late_sample = bytearray()
+        self._tail = b""
+        self._bytes_fed = 0
+        self._ascii = True
+        self._escape = False
+        self._next_probe = self.MIN_BYTES
+        self._result: Optional[DetectionResult] = None
+        self._finalized = False
+        self._stable_rounds = 0
+        self._prev_encoding: Optional[str] = None
+        self._declared_hint: Optional[str] = None
+
+    def feed(self, chunk: bytes | bytearray) -> None:
+        if self._finalized or self._spool.closed:
+            raise RuntimeError("stream is closed; call reset() before feeding more data")
+        if not isinstance(chunk, (bytes, bytearray)):
+            raise TypeError("stream chunks must be bytes or bytearray")
+        if not chunk:
+            return
+        if self._bytes_fed + len(chunk) > self._memory_limit:
+            self._spool.rollover()
+        self._spool.write(chunk)
+        self._bytes_fed += len(chunk)
+        ascii_chunk = chunk.isascii()
+        if self._late_sample and len(self._late_sample) < self._sample_size:
+            self._late_sample.extend(chunk[: self._sample_size - len(self._late_sample)])
+        # Preserve a continuous informative window, independent of chunk sizes.
+        if (self._ascii and not ascii_chunk) or (not self._escape and b"\x1b" in chunk):
+            signal = _HIGH_BYTE.search(chunk)
+            if (
+                signal is not None
+                and self._bytes_fed - len(chunk) + signal.start() > self._sample_size // 4
+            ):
+                start = signal.start() - 64
+                probe = (
+                    self._tail[start:] + bytes(chunk[: self._sample_size])
+                    if start < 0
+                    else bytes(chunk[start : start + self._sample_size])
+                )
+                self._late_sample = bytearray(probe[: self._sample_size])
+        self._tail = bytes(chunk[-64:]) if len(chunk) >= 64 else (self._tail + bytes(chunk))[-64:]
+        self._ascii = self._ascii and ascii_chunk
+        self._escape = self._escape or b"\x1b" in chunk
+        if len(self._buf) < self._sample_size:
+            self._buf.extend(chunk[: self._sample_size - len(self._buf)])
+        if not self._declared_hint and self._options.get("use_hints", True):
+            self._declared_hint = hint_from_content(bytes(self._buf))
+        if self._bytes_fed >= self._next_probe and self._next_probe <= self._sample_size:
+            self._run()
+            while self._next_probe <= self._bytes_fed:
+                self._next_probe *= 4
+
+    def _run(self) -> None:
+        probe = bytes(self._late_sample or self._buf)
+        r = from_bytes(probe, **self._options)
+        self._result = replace(r, byte_count=self._bytes_fed, complete=False, bytes_validated=0)
+        if r.encoding == self._prev_encoding:
+            self._stable_rounds += 1
+        else:
+            self._stable_rounds = 0
+        self._prev_encoding = r.encoding
+
+    def hint_from_headers(self, headers: dict[str, str]) -> None:
+        if self._finalized:
+            raise RuntimeError("cannot change hints on a finalized stream")
+        hint = hint_from_http_headers(headers)
+        if hint:
+            self._declared_hint = hint
+
+    def _validate(self, encoding: str) -> tuple[bool, bytes]:
+        self._spool.seek(0)
+        try:
+            decoder = codecs.getincrementaldecoder(encoding)(errors="strict")
+        except LookupError:
+            return False, b""
+        chunk = b""
+        try:
+            while True:
+                chunk = self._spool.read(65536)
+                if not chunk:
+                    break
+                decoder.decode(chunk, final=False)
+            decoder.decode(b"", final=True)
+            return True, b""
+        except UnicodeDecodeError as exc:
+            start = max(0, exc.start - 64)
+            return False, chunk[start : start + self._sample_size]
+        except UnicodeError:
+            return False, chunk[: self._sample_size]
+
+    def finalize(self) -> DetectionResult:
+        if self._finalized:
+            assert self._result is not None
+            return self._result
+        if self._spool.closed:
+            raise RuntimeError("stream is closed")
+        try:
+            self._result = self._finish()
+            self._finalized = True
+            return self._result
+        finally:
+            self._spool.close()
+
+    def _finish(self) -> DetectionResult:
+        prefix = bytes(self._buf)
+        n = self._bytes_fed
+        if n <= self._sample_size:
+            options = dict(self._options)
+            if self._declared_hint and not options.get("encoding_hint"):
+                options["encoding_hint"] = self._declared_hint
+            return from_bytes(prefix, **options)
+        include, exclude = _filters(
+            self._options.get("cp_isolation"), self._options.get("cp_exclusion")
+        )
+        minimum = self._options.get("min_confidence", 0.0)
+        if include == []:
+            return _result(None, n, "No encodings allowed.", examined=len(prefix))
+        bom = _bom_codec(prefix, include, exclude)
+        if bom:
+            valid, _ = self._validate(bom)
+            if valid:
+                return replace(
+                    _result(bom, n, "BOM; complete stream validated.", 1.0, examined=len(prefix)),
+                    bom_detected=True,
+                )
+            return _result(
+                None,
+                n,
+                "Invalid complete stream under its declared BOM.",
+                status="invalid",
+                examined=len(prefix),
+            )
+        if _binary(prefix):
+            return _result(
+                None,
+                n,
+                "Binary signature or excessive control bytes.",
+                status="binary",
+                examined=len(prefix),
+            )
+        transport = _transport_encoding(prefix) if self._ascii else None
+        if transport and _allowed(transport, include, exclude):
+            valid, _ = self._validate(transport)
+            if valid and minimum <= 0.85:
+                return _result(
+                    transport,
+                    n,
+                    "7-bit shift syntax; complete stream validated.",
+                    0.85,
+                    "ambiguous",
+                    len(prefix),
+                )
+        shape = detect_null_pattern(prefix)
+        if shape and _allowed(shape, include, exclude):
+            valid, _ = self._validate(shape)
+            if valid:
+                return (
+                    _result(
+                        shape,
+                        n,
+                        "Unicode byte-lane pattern; complete stream validated.",
+                        0.9,
+                        examined=len(prefix),
+                    )
+                    if minimum <= 0.9
+                    else _result(None, n, "Below minimum confidence.", examined=len(prefix))
+                )
+        if not shape and not self._escape:
+            if self._ascii and _allowed("ascii", include, exclude):
+                return _result("ascii", n, "All stream bytes are ASCII text.", 1.0, examined=n)
+            if _allowed("utf_8", include, exclude):
+                valid, _ = self._validate("utf_8")
+                if valid:
+                    r = _result("utf_8", n, "Complete stream validated as UTF-8.", 0.99, examined=n)
+                    if self._options.get("include_language"):
+                        from .coherence import detect_language
+
+                        decoded = codecs.getincrementaldecoder("utf_8")().decode(
+                            prefix, final=False
+                        )
+                        langs = detect_language(
+                            decoded, threshold=self._options["language_threshold"]
+                        )
+                        if langs:
+                            r = replace(r, language=langs[0][0], coherence=round(langs[0][1], 4))
+                    return (
+                        r
+                        if r.confidence >= minimum
+                        else _result(None, n, "Below minimum confidence.", examined=len(prefix))
+                    )
+        probe = bytes(self._late_sample) or prefix
+        options = {key: self._options[key] for key in ("threshold", "language_threshold")}
+        r = _from_sample(
+            probe, cp_isolation=include, cp_exclusion=exclude, enable_fallback=False, **options
+        )
+        candidates = ([r.encoding] if r.encoding else []) + [alt.encoding for alt in r.alternatives]
+        hint = self._options.get("encoding_hint") or self._declared_hint
+        if hint:
+            from .api import _codec
+
+            try:
+                hint = _codec(hint)
+            except (LookupError, TypeError):
+                hint = None
+            if hint and _allowed(hint, include, exclude):
+                candidates.insert(0, hint)
+        if self._options.get("enable_fallback", True) and include:
+            candidates.extend(include)
+        tried: set[str] = set()
+        examined = len(probe)
+        for enc in candidates:
+            if enc in tried or not _allowed(enc, include, exclude):
+                continue
+            tried.add(enc)
+            valid, failure = self._validate(enc)
+            if valid:
+                confidence = 0.95 if enc == hint else r.confidence if enc == r.encoding else 0.1
+                if confidence < minimum:
+                    continue
+                if self._options.get("include_language"):
+                    from .coherence import detect_language
+
+                    decoded = codecs.getincrementaldecoder(enc)().decode(probe, final=False)
+                    langs = detect_language(decoded, threshold=self._options["language_threshold"])
+                    if langs:
+                        r = replace(r, language=langs[0][0])
+                return replace(
+                    r,
+                    encoding=enc,
+                    confidence=confidence,
+                    byte_count=n,
+                    bytes_examined=min(n, examined),
+                    bytes_validated=n,
+                    complete=True,
+                    status="ambiguous" if r.alternatives else "matched",
+                    chaos=r.chaos if enc == r.encoding else 0.0,
+                    coherence=r.coherence if enc == r.encoding else 0.0,
+                    alternatives=[a for a in r.alternatives if a.encoding != enc],
+                    language=r.language
+                    if self._options.get("include_language") and enc == r.encoding
+                    else "",
+                    why=(r.why if enc == r.encoding else f"Selected validated alternative {enc}.")
+                    + " Complete stream validated."
+                    + (f" Hint: {hint}." if hint else ""),
+                )
+            if failure and len(tried) <= 4:
+                alternate = _from_sample(
+                    failure,
+                    cp_isolation=include,
+                    cp_exclusion=list(set(exclude) | tried),
+                    enable_fallback=False,
+                    **options,
+                )
+                examined += len(failure)
+                if alternate.encoding:
+                    candidates.append(alternate.encoding)
+        return _result(
+            None,
+            n,
+            "No permitted encoding validates the complete stream.",
+            examined=min(n, examined),
+        )
+
+    def close(self) -> None:
+        self._spool.close()
+
+    def reset(self) -> None:
+        self.close()
+        StreamDetector.__init__(
+            self,
+            auto_stop_confidence=self._auto_stop_confidence,
+            memory_limit=self._memory_limit,
+            **self._options,
+        )
+
+    def __enter__(self) -> StreamDetector:
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        self.close()
+
+    def snapshot(self) -> dict[str, object]:
+        return {
+            "bytes_fed": self.bytes_fed,
+            "buffered_bytes": len(self._buf),
+            "encoding": self.encoding,
+            "confidence": self.confidence,
+            "language": self.language,
+            "stable_rounds": self._stable_rounds,
+            "finalized": self._finalized,
+            "declared_hint": self._declared_hint,
+        }
+
+    @property
+    def result(self) -> Optional[DetectionResult]:
+        return self._result
+
+    @property
+    def encoding(self) -> Optional[str]:
+        return self._result.encoding if self._result is not None else None
+
+    @property
+    def confidence(self) -> float:
+        return self._result.confidence if self._result is not None else 0.0
+
+    @property
+    def language(self) -> str:
+        return self._result.language if self._result is not None else ""
+
+    @property
+    def bytes_fed(self) -> int:
+        return self._bytes_fed
+
+    @property
+    def is_stable(self) -> bool:
+        return self._finalized or (
+            self._stable_rounds >= self.STABILITY_ROUNDS
+            and self.confidence >= self._auto_stop_confidence
+        )
+
+
+def detect_stream(
+    chunks: Iterator[bytes],
+    *,
+    stop_confidence: float = 0.97,
+    max_bytes: Optional[int] = None,
+    early_stop: bool = False,
+    **kwargs: Any,
+) -> DetectionResult:
+    """Consume to EOF by default. Explicit budgets/early stopping mark incomplete results.
+
+    An oversized yielded chunk is sliced before analysis. Its unused remainder
+    has already been yielded by the source and cannot be restored by this API.
+    """
+    if max_bytes is not None and (
+        not isinstance(max_bytes, int) or isinstance(max_bytes, bool) or max_bytes <= 0
+    ):
+        raise ValueError("max_bytes must be a positive integer or None")
+    with StreamDetector(auto_stop_confidence=stop_confidence, **kwargs) as detector:
+        limited = False
+        for chunk in chunks:
+            if not isinstance(chunk, (bytes, bytearray)):
+                raise TypeError("stream chunks must be bytes or bytearray")
+            if max_bytes is not None:
+                chunk = chunk[: max_bytes - detector.bytes_fed]
+            detector.feed(chunk)
+            if (max_bytes is not None and detector.bytes_fed >= max_bytes) or (
+                early_stop and detector.is_stable
+            ):
+                limited = True
+                break
+        r = detector.finalize()
+        return (
+            replace(r, complete=False, why=r.why + " Input stopped before confirmed EOF.")
+            if limited
+            else r
+        )

bytesense/version.py

@@ -0,0 +1,4 @@
+from __future__ import annotations
+
+__version__: str = "1.0.0"
+VERSION: tuple[int, int, int] = (1, 0, 0)

bytesense-1.0.0.dist-info/METADATA

@@ -0,0 +1,160 @@
+Metadata-Version: 2.4
+Name: bytesense
+Version: 1.0.0
+Summary: Encoding detection with full-input validation, bounded-memory streams, and optional Rust acceleration.
+Author-email: Oğuzhan Kır <oguzhankir17@gmail.com>
+Maintainer-email: Oğuzhan Kır <oguzhankir17@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/oguzhankir/bytesense
+Project-URL: Repository, https://github.com/oguzhankir/bytesense
+Project-URL: Issues, https://github.com/oguzhankir/bytesense/issues
+Project-URL: Changelog, https://github.com/oguzhankir/bytesense/blob/main/CHANGELOG.md
+Project-URL: Documentation, https://oguzhankir.github.io/bytesense/
+Keywords: encoding,charset,detection,unicode,bytes,chardet,charset-normalizer
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Programming Language :: Rust
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Topic :: Utilities
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: fast
+Provides-Extra: docs
+Requires-Dist: mkdocs>=1.6; extra == "docs"
+Requires-Dist: mkdocs-material>=9.5; extra == "docs"
+Requires-Dist: pymdown-extensions>=10.0; extra == "docs"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0; extra == "dev"
+Requires-Dist: pytest-benchmark>=4.0; extra == "dev"
+Requires-Dist: mypy>=1.8; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Requires-Dist: build>=1.2; extra == "dev"
+Requires-Dist: hypothesis>=6.100; (platform_python_implementation != "PyPy" or python_version >= "3.11") and extra == "dev"
+Requires-Dist: hypothesis==6.100.0; (platform_python_implementation == "PyPy" and python_version < "3.11") and extra == "dev"
+Requires-Dist: setuptools-rust>=1.12; extra == "dev"
+Requires-Dist: charset-normalizer>=3.3; extra == "dev"
+Requires-Dist: chardet>=5.0; extra == "dev"
+Requires-Dist: requests>=2.31; extra == "dev"
+Requires-Dist: certifi>=2023.0; extra == "dev"
+Requires-Dist: twine>=5.0; extra == "dev"
+Dynamic: license-file
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/oguzhankir/bytesense/main/assets/bytesense_logo.svg" alt="bytesense" width="480" />
+</p>
+<p align="center"><strong>Detect the encoding. Validate every byte. Keep your data intact.</strong></p>
+<p align="center">
+  <a href="https://pypi.org/project/bytesense/"><img src="https://img.shields.io/pypi/v/bytesense.svg" alt="PyPI" /></a>
+  <a href="https://github.com/oguzhankir/bytesense/actions/workflows/ci.yml"><img src="https://github.com/oguzhankir/bytesense/actions/workflows/ci.yml/badge.svg" alt="CI" /></a>
+  <a href="https://github.com/oguzhankir/bytesense/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT license" /></a>
+</p>
+
+**bytesense** is a Python encoding detector for file imports, multilingual text pipelines and legacy data. It combines fast Unicode checks, compact character-pair statistics and optional Rust acceleration—with **zero runtime dependencies**.
+
+Every returned encoding strictly decodes the complete supplied input. Linguistic scoring uses a bounded sample; stream inputs spill to a temporary file after a configurable memory limit. Results tell you what was examined, what was validated, and whether the input ended.
+
+```python
+from bytesense import from_bytes
+
+data = "İstanbul'da çalışan mühendisler için doğru metin çözümleme. ".encode("cp1254")
+result = from_bytes(data)
+if result.encoding:
+    text = data.decode(result.encoding)  # strict decoding; no replacement characters
+    print(result.encoding, result.bytes_validated, result.complete)
+```
+
+## Install
+
+```bash
+pip install bytesense
+```
+
+Python **3.9+**. Platform wheels include Rust acceleration; the portable wheel works without a compiler. Source installations automatically use Rust when a toolchain is available. To explicitly choose a source build:
+
+```bash
+BYTESENSE_BUILD_RUST=0 pip install --no-binary=bytesense bytesense  # pure Python
+BYTESENSE_BUILD_RUST=1 pip install --no-binary=bytesense bytesense  # require Rust
+```
+
+The old `[fast]` extra is a compatibility alias. It does not install an accelerator separately. `BYTESENSE_PURE_PYTHON=1` selects the Python implementation at runtime.
+
+## Built for dependable imports
+
+- **Complete validation:** a valid prefix cannot hide an undecodable suffix. Unknown and binary inputs have explicit outcomes.
+- **Bounded streams:** feed small chunks or large ones; finalization accounts for every accepted byte. Preview stability never silently ends a stream.
+- **Useful controls:** codec aliases, inclusion/exclusion filters, encoding hints, optional language estimates and a minimum evidence score.
+- **Transparent results:** `why`, alternatives, sample/validation counts and completion status. Confidence is a heuristic score; no fabricated statistical intervals.
+- **Portable acceleration:** native character-pair scoring and byte histograms, with matching Python behavior and typed public APIs.
+- **Data repair tools:** opt-in mojibake repair and byte-preserving mixed-document segmentation, with strict decoding.
+
+Decodability alone cannot prove the original encoding. Several legacy codecs can decode identical bytes into different, plausible text. Use a known encoding or an explicit hint when you have one, and evaluate representative data before switching detectors.
+
+## Files, streams and existing integrations
+
+```python
+from bytesense import detect, detect_stream, from_path
+
+result = from_path("export.csv", include_language=True)
+print(result.to_dict())
+
+with open("archive.txt", "rb") as source:
+    result = detect_stream(iter(lambda: source.read(64 * 1024), b""))
+assert result.complete
+
+# Familiar dictionary interface for code that uses chardet.detect().
+metadata = detect(b"hello world")
+```
+
+`detect_stream()` consumes to EOF by default. `max_bytes=...` or `early_stop=True` is an explicit sampling choice and returns `complete=False` when it stops early. Streaming uses temporary disk space beyond `memory_limit` (1 MiB by default); close an unfinished `StreamDetector` or use it as a context manager.
+
+```bash
+bytesense report.csv
+bytesense --language --verbose report.csv
+cat report.csv | bytesense --minimal -
+```
+
+The CLI exits **0** when every input has an encoding, **1** for read failures or unknown/binary inputs, and **2** for invalid arguments. JSON goes to stdout; errors go to stderr.
+
+## Measured accuracy and performance
+
+The v1 engine replaces the previous collection of special-case ranking rules with a small, reproducibly generated statistical model. The evaluation separates development documents from held-out documents, grouping identical decoded text across encodings. Accuracy means **exact Unicode equality**, not just a compatible codec name.
+
+| Detector | Exact Unicode matches | Accuracy |
+|---|---:|---:|
+| **bytesense 1.0.0** | **1906 / 2078** | **91.72%** |
+| bytesense 0.1.2 | 883 / 2078 | 42.49% |
+| chardet 7.6.0 | 2060 / 2078 | 99.13% |
+| charset-normalizer 3.5.1 | 1679 / 2078 | 80.80% |
+| chardetng-py 0.3.5 | 776 / 2078 | 37.34% |
+
+See [benchmarks and methodology](https://oguzhankir.github.io/bytesense/benchmarks/) for measured comparisons with chardet, charset-normalizer and chardetng, including the cases where another detector wins. Timing reports distinguish default behavior from an additional full-decode validation step.
+
+## Upgrading from 0.x
+
+- Language reporting is opt-in: `include_language=True`.
+- `confidence_interval` remains available and is `None`; scores are not calibrated probabilities.
+- An empty `cp_isolation=[]` allows no codecs. Filters apply to every detection path.
+- BOM-bearing UTF-16/32 selects `utf_16`/`utf_32`, which consume the signature when decoding.
+- No unvalidated UTF-8 fallback; stream finalization and repair decoding reject silent data loss.
+- `StreamDetector.feed()` after finalization raises. `reset()` starts a new stream.
+- Python 3.8 is no longer supported. `steps` and `chunk_size` remain accepted compatibility arguments; use `sample_size` to control statistical work.
+
+[Quick start](https://oguzhankir.github.io/bytesense/quickstart/) · [API](https://oguzhankir.github.io/bytesense/api/) · [Changes](https://github.com/oguzhankir/bytesense/blob/main/CHANGELOG.md) · [Contributing](https://github.com/oguzhankir/bytesense/blob/main/CONTRIBUTING.md)
+
+Created by [Oğuzhan Kır](https://github.com/oguzhankir). MIT-licensed code. The aggregate model's data provenance and reproduction steps are documented with the benchmarks; upstream test documents remain copyrighted by their respective publishers.

bytesense-1.0.0.dist-info/RECORD

@@ -0,0 +1,28 @@
+bytesense/__init__.py,sha256=4G1g_QKN6tga0OfyGyZq8eGxSQikwBKykYvkOCOB2hs,1201
+bytesense/_rust.py,sha256=eA_IywAQI-wq478mT8LlSjMAH3vIz1Gh98_AIdUZPQc,1765
+bytesense/api.py,sha256=AtBzk8LbKH1O6YQGcVh3_r80GW_C4n_Y5vgs_atQKO4,17336
+bytesense/candidate.py,sha256=Pb6HxGettU0M8kA646jqIJVYYqIbSXX_RbuXZbn2-QA,6836
+bytesense/cli.py,sha256=D7tAocEeHjpFpDywqfUF0fLf-Ku-W5s0kfoTIj4dVJE,2603
+bytesense/coherence.py,sha256=GybGV0haTxRgnVZ3wr-IpHCvX5Htn9AEryGAlHEdxw4,2282
+bytesense/constant.py,sha256=d6Fb4ldzEo8sDgC9iOMw69bKc6XPX6H-fc4ThuH_JXM,22841
+bytesense/fingerprint.py,sha256=Se42g2-AHyGatrNxRKVQLMHqOmhw6TZqWIFEZB4qe7U,7703
+bytesense/heuristics.py,sha256=6HAqRQ7jBQ6ivnTB0Nsw0347gUvbRBrDaytJ67enOfs,5135
+bytesense/hints.py,sha256=dCuEcw3HlyikMYXyfaaNkBoyMEX4MFjrPf12sOHBzbU,3139
+bytesense/legacy.py,sha256=pTz4FbHFSso_2rbDbJozDirirzE_5pDPr5WKgx9Jk8I,1032
+bytesense/mess.py,sha256=z2Gi-UFy34fo0t1wGPqWxgGWnUu6KhlI5WqkRpizUHM,5219
+bytesense/models.py,sha256=5neGZYQHnd9MrUI2QbPbsS-hvu96TsB4VMD0BS4mZoU,3598
+bytesense/multi.py,sha256=q9QGxmI7StyeazafgCod4J319zObW2EgVpIwEY33Lm4,7370
+bytesense/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bytesense/repair.py,sha256=9Vr5byaqnC5TyqwEK731MIddxlOoUNpAK7zI5IPsTvw,9421
+bytesense/scoring.py,sha256=hudsXocAwT7H_Vp7RY3CaOhmYXbVZgfuWO1-TiuJSAw,3163
+bytesense/streaming.py,sha256=aBEooT6y3aGuURwZNhculQGSpiwOJGJdGgHHJG--oig,16644
+bytesense/version.py,sha256=FumfuTF6gd-I--l6_x0cb828NjsK2PsBpghXnklN-uI,105
+bytesense/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bytesense/data/fingerprints.py,sha256=LkcpUsBrobaCVMdYYlQ30fJu2dG-78-iOMRSQ0zxWwo,221882
+bytesense/data/language.json.gz,sha256=IhH1xsFnr-nOBg4oovwIZw_5t4OfOnr00ghdOg68Qgo,224991
+bytesense-1.0.0.dist-info/licenses/LICENSE,sha256=M8CBGXubxHI3vsjlww0lhntaHbNteNGCZDNbh2-6sJs,1070
+bytesense-1.0.0.dist-info/METADATA,sha256=FsRRoLnOwXKr_kdK-rjcrX9ly6zS_RvvGO6BI3l0VNM,9368
+bytesense-1.0.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+bytesense-1.0.0.dist-info/entry_points.txt,sha256=iAlLc4aTbSGKTaplaCRIOUS9OfpyVsem0Hxipn_dv-I,49
+bytesense-1.0.0.dist-info/top_level.txt,sha256=LjKoSh0s5BHlWtC22zfvIEa7sMD4pvJ8mtrAH8FRUhs,10
+bytesense-1.0.0.dist-info/RECORD,,

bytesense-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

bytesense-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+bytesense = bytesense.cli:main

bytesense-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Oğuzhan Kır
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bytesense-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+bytesense