makea-cli 0.2.28__tar.gz → 0.2.29__tar.gz

{makea_cli-0.2.28 → makea_cli-0.2.29}/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.2.29
+
+The file reader stays inside a web worker's memory (TECH-145).
+
+- It no longer raises Pillow's global decompression limit, and decodes no
+  image over 120 MP (the same bound as the backend); JPEGs decode at reduced
+  scale. A DXF sheet draws at most 40 pieces. Pictures shrink before they are
+  flattened, so a large PSD is not copied at full size several times.
+- `python -m makea_cli.file_reader <name>` reads the file on stdin and prints
+  the reading as JSON; `read_in_child()` runs it that way, so a long-lived
+  worker gets its memory back when the child exits.
+
 ## 0.2.28
 
 Read the files brands send that no model reads (TECH-143).

{makea_cli-0.2.28 → makea_cli-0.2.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: makea-cli
-Version: 0.2.28
+Version: 0.2.29
 Summary: Makea admin CLI (browser login + Makea admin HTTP API)
 Requires-Python: >=3.10
 Requires-Dist: click<9,>=8.1

makea_cli-0.2.29/makea_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.29"

{makea_cli-0.2.28 → makea_cli-0.2.29}/makea_cli/file_reader.py

@@ -33,6 +33,26 @@ _MAX_MEMBERS = 40
 _MAX_PICTURES = 8
 _PICTURE_SIDE = 1800
 
+# Never decode an image bigger than this. A decoded image costs w x h x 4
+# bytes whatever the file size, and this runs in web workers (previews) as
+# well as Celery: a 410 MB decode killed one and 502'd the site (TECH-145).
+# Same bound as garmentby_service/common/image_limits.MAX_DECODE_PIXELS.
+MAX_DECODE_PIXELS = 120_000_000
+_MAX_PIECES_DRAWN = 40
+
+
+def _open_bounded(data: bytes):
+    """Open an image and decode it only when it fits MAX_DECODE_PIXELS; JPEGs
+    are decoded at reduced scale. Raises ValueError when it is too big."""
+    Image = _pil()
+    im = Image.open(io.BytesIO(data))
+    if im.width * im.height > MAX_DECODE_PIXELS:
+        raise ValueError(f"{im.width} x {im.height} px is too large to draw here")
+    if im.format == "JPEG":
+        im.draft("RGB", (_PICTURE_SIDE, _PICTURE_SIDE))
+    im.load()
+    return im
+
 
 @dataclass
 class Picture:
@@ -109,20 +129,24 @@ def _pil():
             Image,
         )  # noqa: PLC0415 - optional at import time, required to draw
 
-        Image.MAX_IMAGE_PIXELS = 400_000_000
         return Image
     except ImportError:  # pragma: no cover - Pillow is a dependency
         return None
 
 
 def _png(im) -> bytes:
+    """Shrink first, then flatten: every conversion of the full-size image is
+    another full-size copy in memory."""
     Image = _pil()
+    if max(im.size) > _PICTURE_SIDE:
+        if im.mode in ("P", "PA", "1"):
+            im = im.convert("RGBA")  # palette images cannot be resampled
+        im.thumbnail((_PICTURE_SIDE, _PICTURE_SIDE))
     if im.mode not in ("RGB", "L"):
         rgba = im.convert("RGBA")
         bg = Image.new("RGB", rgba.size, "white")
         bg.paste(rgba, mask=rgba.split()[-1])
         im = bg
-    im.thumbnail((_PICTURE_SIDE, _PICTURE_SIDE))
     buf = io.BytesIO()
     im.save(buf, "PNG", optimize=True)
     return buf.getvalue()
@@ -499,8 +523,7 @@ def _read_eps(data: bytes, name: str, depth: int) -> Reading:
         Image = _pil()
         if tif_len and Image:
             try:
-                im = Image.open(io.BytesIO(_fix_preview_tiff(data[tif_off : tif_off + tif_len])))
-                im.load()
+                im = _open_bounded(_fix_preview_tiff(data[tif_off : tif_off + tif_len]))
                 pictures.append(Picture("preview", _png(im)))
             except Exception:  # noqa: BLE001 - no preview is still a reading
                 pass
@@ -574,8 +597,11 @@ def _read_psd(data: bytes, name: str, depth: int) -> Reading:
             layers = [layer[0] for layer in (getattr(im, "layers", None) or []) if layer[0]]
             if layers:
                 lines.append(f"{len(layers)} layer(s): " + "; ".join(layers[:30]) + ".")
-            im.load()
-            pictures.append(Picture("flattened image", _png(im)))
+            if width * height > MAX_DECODE_PIXELS:
+                lines.append("Too large to draw here.")
+            else:
+                im.load()
+                pictures.append(Picture("flattened image", _png(im)))
         except Exception:  # noqa: BLE001
             lines.append("The flattened image could not be decoded.")
     return Reading("psd", "PSD", " ".join(lines), pictures, readable=bool(pictures))
@@ -711,8 +737,7 @@ def _read_wilcom(data: bytes, name: str, depth: int) -> Reading:
         if Image and ole.exists(stream):
             raw = ole.openstream(stream).read()
             try:
-                im = Image.open(io.BytesIO(zlib.decompress(raw[4:])))
-                im.load()
+                im = _open_bounded(zlib.decompress(raw[4:]))
                 pictures.append(Picture("stitch-out preview", _png(im)))
                 break
             except Exception:  # noqa: BLE001
@@ -974,10 +999,12 @@ def _seam_allowance(b: _Block, cut: Tuple[float, float, float, float]) -> str:
 
 
 def _draw_dxf(items: List[Tuple[str, _Block]]) -> List[Picture]:
-    """One sheet with every piece of the base size, numbered."""
+    """One sheet with every piece of the base size, numbered (the first
+    _MAX_PIECES_DRAWN, so the sheet stays a bounded size)."""
     Image = _pil()
     if not Image or not items:
         return []
+    items = items[:_MAX_PIECES_DRAWN]
     from PIL import ImageDraw  # noqa: PLC0415
 
     shapes = []
@@ -1171,8 +1198,17 @@ def _read_image(data: bytes, name: str, depth: int) -> Reading:
     ext = os.path.splitext(name.lower())[1]
     im = Image.open(io.BytesIO(data))
     frames = getattr(im, "n_frames", 1)
-    im.load()
     text = f"{im.format or _format_label(ext)} image, {im.width} x {im.height} px"
+    if im.width * im.height > MAX_DECODE_PIXELS:
+        return Reading(
+            "image",
+            im.format or _format_label(ext),
+            text + ". Too large to draw here.",
+            readable=False,
+        )
+    if im.format == "JPEG":
+        im.draft("RGB", (_PICTURE_SIDE, _PICTURE_SIDE))
+    im.load()
     dpi = im.info.get("dpi")
     if dpi and dpi[0]:
         text += f", {dpi[0]:.0f} dpi ({im.width / dpi[0] * 25.4:.0f} x {im.height / dpi[1] * 25.4:.0f} mm)"
@@ -1234,3 +1270,78 @@ _READERS: List[Tuple[Callable[[bytes, str], bool], Callable[[bytes, str, int], R
     (_is_svg, _read_svg),
     (_is_image, _read_image),
 ]
+
+
+# ── Run in a child process ───────────────────────────────────────────────
+#
+# A long-lived worker never gives decoded-image memory back to the system, so
+# the backend reads through `python -m makea_cli.file_reader <name>` (file on
+# stdin, JSON on stdout): whatever the read costs is returned when the child
+# exits (TECH-145).
+
+
+def to_json(reading: Reading) -> dict:
+    import base64  # noqa: PLC0415
+
+    return {
+        "kind": reading.kind,
+        "format": reading.format,
+        "text": reading.text,
+        "readable": reading.readable,
+        "pictures": [
+            {"label": p.label, "png": base64.b64encode(p.png).decode("ascii")}
+            for p in reading.pictures
+        ],
+        "members": [[name, to_json(sub)] for name, sub in reading.members],
+        "raw": base64.b64encode(reading.raw).decode("ascii") if reading.raw is not None else None,
+    }
+
+
+def from_json(data: dict) -> Reading:
+    import base64  # noqa: PLC0415
+
+    return Reading(
+        kind=data["kind"],
+        format=data["format"],
+        text=data["text"],
+        readable=data["readable"],
+        pictures=[Picture(p["label"], base64.b64decode(p["png"])) for p in data["pictures"]],
+        members=[(name, from_json(sub)) for name, sub in data["members"]],
+        raw=base64.b64decode(data["raw"]) if data.get("raw") is not None else None,
+    )
+
+
+def read_in_child(data: bytes, name: str, *, timeout: float = 90.0) -> Reading:
+    """read_bytes in a child Python process; a failure or a timeout is a
+    reading that says so, never an exception."""
+    import json  # noqa: PLC0415
+    import subprocess  # noqa: PLC0415
+    import sys  # noqa: PLC0415
+
+    try:
+        done = subprocess.run(
+            [sys.executable, "-m", "makea_cli.file_reader", name],
+            input=data,
+            capture_output=True,
+            timeout=timeout,
+            check=True,
+        )
+        return from_json(json.loads(done.stdout))
+    except Exception as e:  # noqa: BLE001 - the file stays uploaded either way
+        detail = (
+            "it took too long" if isinstance(e, subprocess.TimeoutExpired) else type(e).__name__
+        )
+        ext = os.path.splitext(name.lower())[1]
+        return Reading(
+            "unknown",
+            _format_label(ext),
+            f"{_format_label(ext)} file; it could not be read here ({detail}).",
+            readable=False,
+        )
+
+
+if __name__ == "__main__":
+    import json
+    import sys
+
+    sys.stdout.write(json.dumps(to_json(read_bytes(sys.stdin.buffer.read(), sys.argv[1]))))

{makea_cli-0.2.28 → makea_cli-0.2.29}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "makea-cli"
-version = "0.2.28"
+version = "0.2.29"
 description = "Makea admin CLI (browser login + Makea admin HTTP API)"
 readme = "README.md"
 requires-python = ">=3.10"

{makea_cli-0.2.28 → makea_cli-0.2.29}/tests/test_file_reader.py

@@ -223,3 +223,51 @@ class IllustratorNativeTest(unittest.TestCase):
 
     def test_ai_with_real_pdf_content_stays_a_pdf(self):
         self.assertEqual(read_bytes(_blank_pdf(), "logo.ai").kind, "pdf")
+
+
+class DecodeBoundTest(unittest.TestCase):
+    """Nothing here decodes an image too big for a web worker (TECH-145)."""
+
+    def test_pillow_global_limit_is_left_alone(self):
+        before = Image.MAX_IMAGE_PIXELS
+        read_bytes(b"x", "a.dxf")
+        self.assertEqual(Image.MAX_IMAGE_PIXELS, before)
+
+    def test_an_image_over_the_bound_is_described_not_drawn(self):
+        from unittest import mock
+
+        from makea_cli import file_reader
+
+        buf = io.BytesIO()
+        Image.new("RGB", (400, 300), "white").save(buf, "TIFF")
+        with mock.patch.object(file_reader, "MAX_DECODE_PIXELS", 1000):
+            r = read_bytes(buf.getvalue(), "scan.tif")
+        self.assertEqual(r.pictures, [])
+        self.assertIn("400 x 300 px", r.text)
+        self.assertIn("Too large", r.text)
+
+
+class ChildProcessTest(unittest.TestCase):
+    def test_a_reading_comes_back_whole_from_the_child(self):
+        from makea_cli.file_reader import read_in_child
+
+        buf = io.BytesIO()
+        with zipfile.ZipFile(buf, "w") as z:
+            z.writestr("pattern.dxf", _dxf())
+            z.writestr("spec.docx", b"PK\x03\x04 docx")
+        r = read_in_child(buf.getvalue(), "pack.zip")
+        self.assertEqual(r.kind, "archive")
+        kinds = {n: m.kind for n, m in r.members}
+        self.assertEqual(kinds, {"pattern.dxf": "pattern", "spec.docx": "document"})
+        self.assertEqual(dict(r.members)["spec.docx"].raw, b"PK\x03\x04 docx")
+        self.assertTrue(dict(r.members)["pattern.dxf"].pictures[0].png.startswith(b"\x89PNG"))
+
+    def test_a_child_that_fails_is_a_reading(self):
+        from unittest import mock
+
+        from makea_cli import file_reader
+
+        with mock.patch("subprocess.run", side_effect=OSError("no python")):
+            r = file_reader.read_in_child(b"x", "a.dxf")
+        self.assertFalse(r.readable)
+        self.assertIn("could not be read", r.text)

{makea_cli-0.2.28 → makea_cli-0.2.29}/uv.lock

@@ -119,7 +119,7 @@ wheels = [
 
 [[package]]
 name = "makea-cli"
-version = "0.2.28"
+version = "0.2.29"
 source = { editable = "." }
 dependencies = [
     { name = "click" },

makea_cli-0.2.28/makea_cli/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.2.28"