chardet-rs 0.1.0__tar.gz

chardet_rs-0.1.0/Cargo.lock

@@ -0,0 +1,393 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "adler2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "block-buffer"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
+
+[[package]]
+name = "chardet-py"
+version = "0.1.0"
+dependencies = [
+ "chardet-rs",
+ "pyo3",
+]
+
+[[package]]
+name = "chardet-rs"
+version = "0.1.0"
+dependencies = [
+ "encoding_rs",
+ "flate2",
+ "once_cell",
+ "regex",
+ "sha2",
+]
+
+[[package]]
+name = "core_detect"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
+
+[[package]]
+name = "cpufeatures"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "crc32fast"
+version = "1.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crypto-common"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
+[[package]]
+name = "digest"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+]
+
+[[package]]
+name = "encoding_rs"
+version = "0.8.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679"
+dependencies = [
+ "cfg-if",
+ "core_detect",
+ "multiversion_no_op",
+ "rustversion",
+ "scopeguard",
+ "simdutf8",
+]
+
+[[package]]
+name = "flate2"
+version = "1.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+ "zlib-rs",
+]
+
+[[package]]
+name = "generic-array"
+version = "0.14.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "indoc"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.190"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "miniz_oxide"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
+dependencies = [
+ "adler2",
+ "simd-adler32",
+]
+
+[[package]]
+name = "multiversion_no_op"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "portable-atomic"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "regex"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+
+[[package]]
+name = "rustversion"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
+[[package]]
+name = "sha2"
+version = "0.10.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
+name = "simd-adler32"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
+
+[[package]]
+name = "simdutf8"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.12.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+
+[[package]]
+name = "typenum"
+version = "1.20.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
+
+[[package]]
+name = "unindent"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
+
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "zlib-rs"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"

chardet_rs-0.1.0/Cargo.toml

@@ -0,0 +1,19 @@
+[workspace]
+resolver = "2"
+members = ["chardet", "chardet-py"]
+
+[workspace.package]
+version = "0.1.0"
+edition = "2021"
+license = "0BSD"
+repository = "https://github.com/thanatosmoe/chardet-rs"
+homepage = "https://github.com/thanatosmoe/chardet-rs"
+documentation = "https://docs.rs/chardet-rs"
+authors = ["Thanatos"]
+keywords = ["encoding", "charset", "detection", "unicode", "chardet"]
+categories = ["text-processing", "encoding", "internationalization"]
+rust-version = "1.74"
+
+[profile.release]
+lto = true
+codegen-units = 1

chardet_rs-0.1.0/PKG-INFO

@@ -0,0 +1,258 @@
+Metadata-Version: 2.4
+Name: chardet-rs
+Version: 0.1.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Rust
+Classifier: Topic :: Text Processing :: Linguistic
+Summary: Universal character encoding detector — Rust port of chardet 7
+Author: Thanatos
+License-Expression: 0BSD
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/thanatosmoe/chardet-rs
+Project-URL: Repository, https://github.com/thanatosmoe/chardet-rs
+
+# chardet-rs
+
+[![CI](https://github.com/thanatosmoe/chardet-rs/actions/workflows/ci.yml/badge.svg)](https://github.com/thanatosmoe/chardet-rs/actions/workflows/ci.yml)
+[![License: 0BSD](https://img.shields.io/badge/License-0BSD-blue.svg)](LICENSE)
+
+A Rust rewrite of [chardet 7](https://github.com/chardet/chardet), the
+universal character encoding detector. Same detection results, native Rust
+core, optional Python bindings.
+
+This is a faithful port: it reuses chardet 7's trained bigram model artifacts
+and reproduces its 13-stage detection pipeline, aiming for byte-for-byte
+identical output. On the full official corpus
+([chardet/test-data](https://github.com/chardet/test-data), 3,196 files) it
+matches Python chardet 7 on **3,196/3,196** files — encoding, confidence
+(full `f64` precision), language, and MIME type.
+
+## Why
+
+- **Exact parity** — identical `encoding`, `confidence`, `language`, and
+  `mime_type` to chardet 7 on every corpus file.
+- **Reuses the reference models** — the trained `models.bin`, `idf.bin`,
+  `rowmax.bin`, and `confusion.bin` artifacts are parsed directly, so accuracy
+  is chardet 7's accuracy.
+- **Native Rust core** — a library crate plus a `chardetect` binary.
+- **Drop-in Python bindings** — a PyO3 extension packaged as the `chardet`
+  module, including the `chardet.enums`, `chardet.pipeline`, and
+  `chardet.equivalences` submodules.
+
+|  | chardet-rs | Python chardet 7 |
+| --- | --- | --- |
+| Corpus parity | **3,196 / 3,196 exact** | reference |
+| Encoding detector | Rust core | Python (+ mypyc/Cython) |
+| Python API | `detect`, `detect_all`, `UniversalDetector` | same |
+| CLI | `chardetect` | `chardetect` |
+| Supported encodings | 87 registry encodings | 87 |
+| License | 0BSD | 0BSD |
+
+## Layout
+
+```
+chardet/            Rust core crate (library + `chardetect` binary)
+  src/codecs/       CPython-compatible codec validity/decoding
+  src/models/       models.bin / rowmax.bin parsing and bigram scoring
+  src/pipeline/     the 13 detection stages + orchestrator
+  models/           bundled model artifacts (from chardet 7)
+chardet-py/         PyO3 bindings (drop-in `chardet` Python package)
+scripts_generate/   Python generators for the generated data tables
+scripts_compare/    parity harnesses against Python chardet 7
+```
+
+## Installation
+
+### Rust
+
+```console
+$ cargo add chardet-rs
+```
+
+The crate is published as `chardet-rs`; its library target is named `chardet`,
+so you still `use chardet::...`:
+
+```rust
+// Cargo.toml: chardet-rs = "0.1"
+use chardet::{detect, DetectOptions};
+```
+
+Or from a checkout:
+
+```console
+$ cargo build --release
+$ ./target/release/chardetect --version
+```
+
+### Python
+
+From PyPI (distribution `chardet-rs`, import module `chardet`):
+
+```console
+$ pip install chardet-rs
+```
+
+Or build from a checkout:
+
+```console
+$ python3 -m venv .venv && . .venv/bin/activate
+$ pip install maturin
+$ cd chardet-py && maturin build --release
+$ pip install ../target/wheels/chardet_rs-*.whl
+```
+
+> **Note:** the Python distribution is named `chardet-rs` because the name
+> `chardet` on PyPI belongs to the reference project. Installing it provides
+> the `chardet` import module and therefore conflicts with the reference
+> `chardet` package — install one or the other in a given environment.
+
+## Quick Start
+
+### Rust
+
+```rust
+let data = std::fs::read("unknown.txt")?;
+let result = chardet::detect(&data);
+println!("{:?}", result);
+
+// All ranked candidates
+let all = chardet::detect_all(&data, &chardet::DetectOptions::default())?;
+```
+
+### Python
+
+```python
+import chardet
+
+chardet.detect(b"Python is a great programming language.")
+# {'encoding': 'ascii', 'confidence': 1.0, 'language': 'en', 'mime_type': 'text/plain'}
+
+# All candidate encodings ranked by confidence
+for r in chardet.detect_all("Le café est une boisson très populaire.".encode("windows-1252")):
+    print(r["encoding"], round(r["confidence"], 2))
+```
+
+### Streaming detection
+
+```python
+from chardet import UniversalDetector
+
+detector = UniversalDetector()
+with open("unknown.txt", "rb") as f:
+    for line in f:
+        detector.feed(line)
+        if detector.done:
+            break
+print(detector.close())
+```
+
+### Encoding era filtering
+
+```python
+from chardet import detect_all
+from chardet.enums import EncodingEra
+
+data = "Москва является столицей Российской Федерации.".encode("windows-1251")
+for r in detect_all(data, encoding_era=EncodingEra.MODERN_WEB):
+    print(r["encoding"], round(r["confidence"], 2))
+```
+
+### CLI
+
+```console
+$ chardetect somefile.txt
+somefile.txt: utf-8 with confidence 0.99
+
+$ chardetect --minimal somefile.txt
+utf-8
+
+$ chardetect -l somefile.txt
+somefile.txt: utf-8 en (English) with confidence 0.99
+```
+
+## How It Works
+
+chardet-rs reproduces chardet 7's multi-stage pipeline, progressing from cheap
+deterministic checks to statistical analysis:
+
+1. BOM detection
+2. BOM-less UTF-16/UTF-32 null-byte patterns
+3. Escape sequences (ISO-2022, HZ-GB-2312, UTF-7)
+4. Magic-number binary detection (40+ formats)
+5. Binary content detection
+6. Markup charset declarations (HTML/XML/PEP 263, including EBCDIC)
+7. ASCII check
+8. UTF-8 structural validation
+9. Byte validity filtering
+10. CJK gating
+11. Structural probing
+12. Statistical bigram scoring against the trained models
+13. Post-processing rank corrections (confusion groups, niche-Latin
+    demotion, KOI8-T promotion, classic-Mac line endings, decode safety)
+
+Language detection (three tiers) and MIME type detection run alongside.
+
+## Parity Testing
+
+The port is validated against Python chardet 7:
+
+```console
+$ git clone --depth 1 https://github.com/chardet/test-data tests/data
+$ CHARDET_SRC=/path/to/chardet/src python3 scripts_compare/full_corpus.py tests/data
+mime match:     3196/3196 (100.00%)
+encoding match: 3196/3196 (100.00%)
+exact match:    3196/3196 (100.00%)
+```
+
+Harnesses live in `scripts_compare/`:
+
+- `parity.py` — hand-written multilingual samples.
+- `edge_parity.py` — BOMs, markup declarations, EBCDIC, niche-Latin/KOI8-T
+  arbitration, CR line endings, truncation at every length, magic numbers.
+- `fuzz.py` — randomized/structured byte sequences.
+- `full_corpus.py` — the full `chardet/test-data` corpus.
+- `api_parity.py` — `detect_all` / `UniversalDetector`.
+
+CI runs the Rust tests, builds the Python wheel, and compares against the
+reference `chardet` wheel from PyPI.
+
+## Regenerating data tables
+
+The generated Rust tables (single-byte codec maps, DBCS validity tables,
+Unicode categories, registry and CPython aliases) are produced from CPython so
+the Rust codecs match Python's behavior exactly:
+
+```console
+$ python3 scripts_generate/gen_registry.py     > chardet/src/registry_data.rs
+$ python3 scripts_generate/gen_codec_tables.py > chardet/src/codecs/sbcs_data.rs
+$ python3 scripts_generate/gen_dbcs_tables.py  > chardet/src/codecs/dbcs.bin
+$ python3 scripts_generate/gen_categories.py   > chardet/src/codecs/categories_data.rs
+```
+
+## Supported Encodings
+
+Full chardet 7 coverage, including Windows code pages, ISO-8859 family, Mac,
+KOI8, DOS, mainframe EBCDIC, CJK (CP932/CP949/GB18030/Big5/EUC-JP/EUC-KR/Johab
+and the ISO-2022/HZ escape encodings), and the Unicode family.
+
+## Project History
+
+chardet was created by [Mark Pilgrim](https://en.wikipedia.org/wiki/Mark_Pilgrim)
+in 2006 as a Python port of [Mozilla's universal charset detection
+library](https://www-archive.mozilla.org/projects/intl/chardet.html). After he
+deleted his online accounts in 2011, the project was continued by David Cramer,
+Erik Rose, Toshio Kuratomi, Ian Cordasco, and Dan Blanchard. In 2026 Dan
+Blanchard rewrote chardet as a ground-up 0BSD-licensed implementation, released
+as chardet 7.
+
+**chardet-rs** is an independent Rust rewrite of chardet 7. It reuses chardet
+7's trained model artifacts and mirrors its pipeline, so the detection results
+match, while the implementation is native Rust. All credit for the original
+algorithm, models, and training data belongs to the chardet authors listed
+above.
+
+## License
+
+0BSD — same as chardet 7. See [LICENSE](LICENSE).
+