classifyre-cli 0.4.93__tar.gz → 0.4.95__tar.gz

{classifyre_cli-0.4.93 → classifyre_cli-0.4.95}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 307 packages in 240ms
+Resolved 307 packages in 331ms
 Checked 51 packages in 1ms

{classifyre_cli-0.4.93 → classifyre_cli-0.4.95}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.93
+Version: 0.4.95
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.93 → classifyre_cli-0.4.95}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.93",
+  "version": "0.4.95",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.93 → classifyre_cli-0.4.95}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.93"
+version = "0.4.95"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

{classifyre_cli-0.4.93 → classifyre_cli-0.4.95}/src/utils/file_parser.py

@@ -732,6 +732,28 @@ def _refine_zip_mime(source: BinarySource) -> str:
     return "application/zip"
 
 
+def _null_bytes_look_interleaved(sample: bytes, unit: int) -> bool:
+    """
+    Whether the null bytes sit where a fixed-width encoding would put them.
+
+    This is the structural signal that separates real UTF-16/32 from binary
+    that merely happens to contain a null. Text in a fixed-width encoding pads
+    every character to `unit` bytes, so for any script whose code points fit in
+    the low byte the padding lands at the SAME offset within every unit, and it
+    is common: roughly half the bytes for UTF-16 Latin text, three quarters for
+    UTF-32. Binary with an incidental null shows neither property.
+    """
+    if len(sample) < unit * 2:
+        return False
+    nulls = sample.count(0)
+    # Well under what padding produces, so the nulls are incidental.
+    if nulls * unit < len(sample) * 0.5:
+        return False
+    offsets = {i % unit for i, byte in enumerate(sample) if byte == 0}
+    # Padding occupies fixed positions; scattered nulls are not padding.
+    return len(offsets) < unit
+
+
 def _is_null_byte_unicode_text(file_bytes: bytes) -> bool:
     """UTF-16/32 text interleaves null bytes with every character — still text."""
     # \xff\xfe prefixes both UTF-16 LE and UTF-32 LE BOMs.
@@ -752,6 +774,14 @@ def _is_null_byte_unicode_text(file_bytes: bytes) -> bool:
         decoded = sample.decode(encoding)
         if not decoded:
             return False
+        # Printability alone is not enough either, and this is where short
+        # binary payloads got through: chardet calls b"\x00\x01\x02\x03binary"
+        # utf-16-le with 0.95 confidence, it decodes to CJK code points, and
+        # every one of those is `isprintable()`. So the payload was reported as
+        # text/plain and the parser tried to read it as text. Require the nulls
+        # to sit where an encoding's padding would actually put them.
+        if not _null_bytes_look_interleaved(sample, unit):
+            return False
         printable = sum(1 for ch in decoded if ch.isprintable() or ch.isspace())
         return printable / len(decoded) >= 0.9
     except Exception:

{classifyre_cli-0.4.93 → classifyre_cli-0.4.95}/tests/utils/test_file_parser.py

@@ -80,6 +80,26 @@ class TestDetectMimeType:
         result = detect_mime_type(b"")
         assert result == "application/octet-stream"
 
+    def test_utf16_text_with_bom_is_text(self) -> None:
+        payload = "hello world this is plain utf16 text".encode("utf-16")
+        assert detect_mime_type(payload) == "text/plain"
+
+    def test_utf16_text_without_bom_is_text(self) -> None:
+        # No BOM, so this reaches the heuristic rather than the prefix check.
+        payload = "hello world this is plain utf16 text".encode("utf-16-le")
+        assert detect_mime_type(payload) == "text/plain"
+
+    def test_binary_is_not_mistaken_for_utf16(self) -> None:
+        # chardet calls this utf-16-le with 0.95 confidence and it decodes to
+        # CJK code points, every one of which is `isprintable()` — so a
+        # printability check alone accepted it as text. Real UTF-16 pads every
+        # character to a fixed width, putting its nulls at a consistent offset
+        # and in quantity; incidental nulls in binary do neither.
+        assert detect_mime_type(b"\x00\x01\x02\x03binary") == "application/octet-stream"
+        assert detect_mime_type(b"\x00\x01\x02\x03\x04\x05\x06\x07more binary") == (
+            "application/octet-stream"
+        )
+
 
 # ---------------------------------------------------------------------------
 # extract_text