classifyre-cli 0.4.93__tar.gz → 0.4.94__tar.gz

{classifyre_cli-0.4.93 → classifyre_cli-0.4.94}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 307 packages in 240ms
+Resolved 307 packages in 262ms
 Checked 51 packages in 1ms

{classifyre_cli-0.4.93 → classifyre_cli-0.4.94}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.93
+Version: 0.4.94
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.93 → classifyre_cli-0.4.94}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.93",
+  "version": "0.4.94",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.93 → classifyre_cli-0.4.94}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.93"
+version = "0.4.94"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

{classifyre_cli-0.4.93 → classifyre_cli-0.4.94}/src/utils/file_parser.py

@@ -732,6 +732,28 @@ def _refine_zip_mime(source: BinarySource) -> str:
     return "application/zip"
 
 
+def _null_bytes_look_interleaved(sample: bytes, unit: int) -> bool:
+    """
+    Whether the null bytes sit where a fixed-width encoding would put them.
+
+    This is the structural signal that separates real UTF-16/32 from binary
+    that merely happens to contain a null. Text in a fixed-width encoding pads
+    every character to `unit` bytes, so for any script whose code points fit in
+    the low byte the padding lands at the SAME offset within every unit, and it
+    is common: roughly half the bytes for UTF-16 Latin text, three quarters for
+    UTF-32. Binary with an incidental null shows neither property.
+    """
+    if len(sample) < unit * 2:
+        return False
+    nulls = sample.count(0)
+    # Well under what padding produces, so the nulls are incidental.
+    if nulls * unit < len(sample) * 0.5:
+        return False
+    offsets = {i % unit for i, byte in enumerate(sample) if byte == 0}
+    # Padding occupies fixed positions; scattered nulls are not padding.
+    return len(offsets) < unit
+
+
 def _is_null_byte_unicode_text(file_bytes: bytes) -> bool:
     """UTF-16/32 text interleaves null bytes with every character — still text."""
     # \xff\xfe prefixes both UTF-16 LE and UTF-32 LE BOMs.
@@ -752,6 +774,14 @@ def _is_null_byte_unicode_text(file_bytes: bytes) -> bool:
         decoded = sample.decode(encoding)
         if not decoded:
             return False
+        # Printability alone is not enough either, and this is where short
+        # binary payloads got through: chardet calls b"\x00\x01\x02\x03binary"
+        # utf-16-le with 0.95 confidence, it decodes to CJK code points, and
+        # every one of those is `isprintable()`. So the payload was reported as
+        # text/plain and the parser tried to read it as text. Require the nulls
+        # to sit where an encoding's padding would actually put them.
+        if not _null_bytes_look_interleaved(sample, unit):
+            return False
         printable = sum(1 for ch in decoded if ch.isprintable() or ch.isspace())
         return printable / len(decoded) >= 0.9
     except Exception:

{classifyre_cli-0.4.93 → classifyre_cli-0.4.94}/tests/utils/test_file_parser.py

@@ -80,6 +80,26 @@ class TestDetectMimeType:
         result = detect_mime_type(b"")
         assert result == "application/octet-stream"
 
+    def test_utf16_text_with_bom_is_text(self) -> None:
+        payload = "hello world this is plain utf16 text".encode("utf-16")
+        assert detect_mime_type(payload) == "text/plain"
+
+    def test_utf16_text_without_bom_is_text(self) -> None:
+        # No BOM, so this reaches the heuristic rather than the prefix check.
+        payload = "hello world this is plain utf16 text".encode("utf-16-le")
+        assert detect_mime_type(payload) == "text/plain"
+
+    def test_binary_is_not_mistaken_for_utf16(self) -> None:
+        # chardet calls this utf-16-le with 0.95 confidence and it decodes to
+        # CJK code points, every one of which is `isprintable()` — so a
+        # printability check alone accepted it as text. Real UTF-16 pads every
+        # character to a fixed width, putting its nulls at a consistent offset
+        # and in quantity; incidental nulls in binary do neither.
+        assert detect_mime_type(b"\x00\x01\x02\x03binary") == "application/octet-stream"
+        assert detect_mime_type(b"\x00\x01\x02\x03\x04\x05\x06\x07more binary") == (
+            "application/octet-stream"
+        )
+
 
 # ---------------------------------------------------------------------------
 # extract_text

{classifyre_cli-0.4.93 → classifyre_cli-0.4.94}/uv.lock

@@ -830,7 +830,7 @@ wheels = [
 
 [[package]]
 name = "classifyre-cli"
-version = "0.4.93"
+version = "0.4.94"
 source = { editable = "." }
 dependencies = [
     { name = "beautifulsoup4" },
@@ -1369,7 +1369,7 @@ youtube = [
 
 [[package]]
 name = "classifyre-schemas"
-version = "0.4.93"
+version = "0.4.94"
 source = { editable = "../../packages/schemas" }
 dependencies = [
     { name = "fastjsonschema" },
@@ -2520,7 +2520,7 @@ http2 = [
 
 [[package]]
 name = "huggingface-hub"
-version = "1.26.0"
+version = "1.26.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "click" },
@@ -2533,9 +2533,9 @@ dependencies = [
     { name = "tqdm" },
     { name = "typing-extensions" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/82/db/3582597f8be0d34bd6881365a26d390854f12893eabdd62dd36de9df5a47/huggingface_hub-1.26.0.tar.gz", hash = "sha256:c8cd4e2df1ba9402f77fce9b509ec1d52debb502551789473f34016acc14e361", size = 936665, upload-time = "2026-07-30T14:12:04.156Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/89/64/bfe23dab749cb2342e1e13b61c9e684ce46b4d189c7d433cd26f76f52baf/huggingface_hub-1.26.1.tar.gz", hash = "sha256:7c28860777594ac679233f571552d0e46df34ed4e4239e844190fad3ca05e4cd", size = 936700, upload-time = "2026-08-06T09:42:24.859Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/97/bb/63a644c75b545f3ff394b822e9bd1c4a9586489c618b77a4d8a44a33a23b/huggingface_hub-1.26.0-py3-none-any.whl", hash = "sha256:e8cca670caa5d8dfa7e45bf45e86b466698198cd8150c021bcdb4a86b9252364", size = 780357, upload-time = "2026-07-30T14:12:01.998Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/a7/b519cd01e57b685c5f3e9db02cb1bd7aaade35fb6554e0d36bee6d6aae28/huggingface_hub-1.26.1-py3-none-any.whl", hash = "sha256:d8676e4ec96c1e481a22a93232bd86d73bbac643fb767399aefaeaa503890fb0", size = 780754, upload-time = "2026-08-06T09:42:22.497Z" },
 ]
 
 [[package]]
@@ -2558,11 +2558,11 @@ wheels = [
 
 [[package]]
 name = "imap-tools"
-version = "1.14.0"
+version = "1.15.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/41/66/d52b6c7740aa81678404a9039668ab2b4887f46055290f232229a2d5b612/imap_tools-1.14.0.tar.gz", hash = "sha256:d730a35763c09ca02be45b93c8d5d638d9f098876c95841c7ce4d337ede403da", size = 48268, upload-time = "2026-07-24T06:23:43.804Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/59/4e/acf9a7d59932548bc749fe0f5ae852e6da00b640303ed72786f697944b69/imap_tools-1.15.0.tar.gz", hash = "sha256:a90f948549ab000d96caf17c90e63b114f8788ba4a0d3f9274a96c0e108c3805", size = 48497, upload-time = "2026-08-06T10:25:26.567Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/02/eb/8567f7542aaee43575002915a959e8891d2d9f3642d1bc5d32410d871bb5/imap_tools-1.14.0-py3-none-any.whl", hash = "sha256:eab3506e701eba44269bc04760b8b29a96c93b173553057180fe40a084d65426", size = 36410, upload-time = "2026-07-24T06:23:41.945Z" },
+    { url = "https://files.pythonhosted.org/packages/50/eb/a4ad6ec962af2806c3484adb1869c24a04d8ef8c2034f2dd2b0f48459b0a/imap_tools-1.15.0-py3-none-any.whl", hash = "sha256:7bb0003c8bf124b80968c7bef786e4577d8365e9afb6551dc15b13161b688bdc", size = 36512, upload-time = "2026-08-06T10:25:24.377Z" },
 ]
 
 [[package]]